Lines Matching +full:cdr +full:- +full:mode
1 /*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
29 * BBR - Congestion Based Congestion Control
159 /* Should the following be dynamic too -- loss wise */
193 * num-measures > min(0) and
227 /* thresholds for reduction on drain in sub-states/drain */
265 * means in non-recovery/retransmission scenarios
266 * cwnd will never be reached by the flight-size.
271 static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sack to use bbr */
292 * - Yuchung Cheng's RACK TCP (for which its named) that
295 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
297 * - Van Jacobson's et.al BBR.
319 * TCP output is also over-written with a new version since it
331 static int32_t bbr_reorder_fade = 60000000; /* 0 - never fade, def
332 * 60,000,000 - 60 seconds */
385 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
388 /* What is the max the 0 - bbr_cross_over MBPS TSO target
397 /* What is the min the 0 - bbr_cross-over MBPS TSO target can be */
401 /* Cross over point from algo-a to algo-b */
488 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts,
517 return(bbr->rc_bbr_substate); in bbr_state_val()
525 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), in get_min_cwnd()
526 bbr->r_ctl.rc_pace_max_segs); in get_min_cwnd()
539 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT; in bbr_get_persists_timer_val()
540 if (tp->t_srtt == 0) { in bbr_get_persists_timer_val()
544 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_get_persists_timer_val()
545 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT); in bbr_get_persists_timer_val()
547 TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]), in bbr_get_persists_timer_val()
566 if (bbr->rc_all_timers_stopped) { in bbr_timer_start()
570 if (bbr->rc_in_persist) { in bbr_timer_start()
574 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_timer_start()
576 ((tp->t_flags & TF_SACK_PERMIT) == 0) || in bbr_timer_start()
577 (tp->t_state < TCPS_ESTABLISHED)) { in bbr_timer_start()
580 if (SEQ_LT(tp->snd_una, tp->snd_max) || in bbr_timer_start()
581 sbavail(&tptosocket(tp)->so_snd)) { in bbr_timer_start()
585 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_timer_start()
587 idx = rsm->r_rtr_cnt - 1; in bbr_timer_start()
588 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time)) in bbr_timer_start()
589 tstmp_touse = rsm->r_tim_lastsent[idx]; in bbr_timer_start()
591 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time; in bbr_timer_start()
593 time_since_sent = cts - tstmp_touse; in bbr_timer_start()
595 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT; in bbr_timer_start()
596 if (tp->t_srtt == 0) in bbr_timer_start()
599 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) + in bbr_timer_start()
600 ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT); in bbr_timer_start()
601 if (tp->t_rxtshift) in bbr_timer_start()
602 tov *= tcp_backoff[tp->t_rxtshift]; in bbr_timer_start()
604 tov -= time_since_sent; in bbr_timer_start()
606 tov = bbr->r_ctl.rc_min_to; in bbr_timer_start()
608 (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC), in bbr_timer_start()
609 (bbr->rc_max_rto_sec * USECS_IN_SECOND)); in bbr_timer_start()
615 if (rsm->r_flags & BBR_ACKED) { in bbr_timer_start()
623 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_timer_start()
624 if ((tp->t_flags & TF_SENTFIN) && in bbr_timer_start()
625 ((tp->snd_max - tp->snd_una) == 1) && in bbr_timer_start()
626 (rsm->r_flags & BBR_HAS_FIN)) { in bbr_timer_start()
635 idx = rsm->r_rtr_cnt - 1; in bbr_timer_start()
636 exp = rsm->r_tim_lastsent[idx] + thresh; in bbr_timer_start()
638 to = exp - cts; in bbr_timer_start()
639 if (to < bbr->r_ctl.rc_min_to) { in bbr_timer_start()
640 to = bbr->r_ctl.rc_min_to; in bbr_timer_start()
643 to = bbr->r_ctl.rc_min_to; in bbr_timer_start()
647 if (bbr->rc_tlp_in_progress != 0) { in bbr_timer_start()
653 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext); in bbr_timer_start()
658 if (rsm->r_flags & BBR_HAS_FIN) { in bbr_timer_start()
664 idx = rsm->r_rtr_cnt - 1; in bbr_timer_start()
665 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time)) in bbr_timer_start()
666 tstmp_touse = rsm->r_tim_lastsent[idx]; in bbr_timer_start()
668 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time; in bbr_timer_start()
670 time_since_sent = cts - tstmp_touse; in bbr_timer_start()
675 to = thresh - time_since_sent; in bbr_timer_start()
677 to = bbr->r_ctl.rc_min_to; in bbr_timer_start()
678 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { in bbr_timer_start()
685 if ((bbr->rc_tlp_rtx_out == 1) && in bbr_timer_start()
686 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) { in bbr_timer_start()
691 bbr->rc_tlp_rtx_out = 0; in bbr_timer_start()
694 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) { in bbr_timer_start()
699 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; in bbr_timer_start()
700 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start; in bbr_timer_start()
705 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK; in bbr_timer_start()
708 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) { in bbr_timer_start()
716 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP; in bbr_timer_start()
725 return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options); in bbr_minseg()
744 if ((tp->t_state == TCPS_CLOSED) || in bbr_start_hpts_timer()
745 (tp->t_state == TCPS_LISTEN)) { in bbr_start_hpts_timer()
748 stopped = bbr->rc_tmr_stopped; in bbr_start_hpts_timer()
749 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { in bbr_start_hpts_timer()
750 left = bbr->r_ctl.rc_timer_exp - cts; in bbr_start_hpts_timer()
752 bbr->r_ctl.rc_hpts_flags = 0; in bbr_start_hpts_timer()
753 bbr->r_ctl.rc_timer_exp = 0; in bbr_start_hpts_timer()
754 prev_delay = bbr->r_ctl.rc_last_delay_val; in bbr_start_hpts_timer()
755 if (bbr->r_ctl.rc_last_delay_val && in bbr_start_hpts_timer()
762 slot = bbr->r_ctl.rc_last_delay_val; in bbr_start_hpts_timer()
763 if (TSTMP_GT(cts, bbr->rc_pacer_started)) { in bbr_start_hpts_timer()
765 delay_calc = cts - bbr->rc_pacer_started; in bbr_start_hpts_timer()
767 slot -= delay_calc; in bbr_start_hpts_timer()
771 if (bbr->r_agg_early_set) { in bbr_start_hpts_timer()
772 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2); in bbr_start_hpts_timer()
773 slot += bbr->r_ctl.rc_agg_early; in bbr_start_hpts_timer()
774 bbr->r_ctl.rc_agg_early = 0; in bbr_start_hpts_timer()
775 bbr->r_agg_early_set = 0; in bbr_start_hpts_timer()
778 if (bbr->r_ctl.rc_hptsi_agg_delay) { in bbr_start_hpts_timer()
779 if (slot > bbr->r_ctl.rc_hptsi_agg_delay) { in bbr_start_hpts_timer()
781 slot -= bbr->r_ctl.rc_hptsi_agg_delay; in bbr_start_hpts_timer()
782 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_start_hpts_timer()
785 bbr->r_ctl.rc_hptsi_agg_delay -= slot; in bbr_start_hpts_timer()
786 bbr->r_ctl.rc_last_delay_val = slot = 100; in bbr_start_hpts_timer()
789 bbr->r_ctl.rc_last_delay_val = slot; in bbr_start_hpts_timer()
791 if (tp->t_flags & TF_DELACK) { in bbr_start_hpts_timer()
792 if (bbr->rc_in_persist == 0) { in bbr_start_hpts_timer()
812 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; in bbr_start_hpts_timer()
818 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT; in bbr_start_hpts_timer()
822 * wheel, we resort to a keep-alive timer if its configured. in bbr_start_hpts_timer()
826 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_start_hpts_timer()
827 (tp->t_state <= TCPS_CLOSING)) { in bbr_start_hpts_timer()
830 * del-ack), we don't have segments being paced. So in bbr_start_hpts_timer()
833 if (TCPS_HAVEESTABLISHED(tp->t_state)) { in bbr_start_hpts_timer()
838 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP; in bbr_start_hpts_timer()
842 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) { in bbr_start_hpts_timer()
848 * keep-alive, delayed_ack we keep track of what was left in bbr_start_hpts_timer()
854 if (bbr->r_ctl.rc_incr_tmrs && slot && in bbr_start_hpts_timer()
855 (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) { in bbr_start_hpts_timer()
876 * Hack alert for now we can't time-out over 2147 seconds (a in bbr_start_hpts_timer()
881 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; in bbr_start_hpts_timer()
883 bbr->r_ctl.rc_timer_exp = 0; in bbr_start_hpts_timer()
885 (bbr->rc_use_google || in bbr_start_hpts_timer()
886 bbr->output_error_seen || in bbr_start_hpts_timer()
892 bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY; in bbr_start_hpts_timer()
893 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && in bbr_start_hpts_timer()
894 (bbr->rc_cwnd_limited == 0)) { in bbr_start_hpts_timer()
900 tp->t_flags2 |= TF2_DONT_SACK_QUEUE; in bbr_start_hpts_timer()
902 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE; in bbr_start_hpts_timer()
903 bbr->rc_pacer_started = cts; in bbr_start_hpts_timer()
907 bbr->rc_timer_first = 0; in bbr_start_hpts_timer()
908 bbr->bbr_timer_src = frm; in bbr_start_hpts_timer()
921 * on a keep-alive timer and a request comes in for in bbr_start_hpts_timer()
925 bbr->rc_pacer_started = cts; in bbr_start_hpts_timer()
926 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && in bbr_start_hpts_timer()
927 (bbr->rc_cwnd_limited == 0)) { in bbr_start_hpts_timer()
933 tp->t_flags2 |= (TF2_MBUF_QUEUE_READY | in bbr_start_hpts_timer()
937 tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY | in bbr_start_hpts_timer()
940 bbr->bbr_timer_src = frm; in bbr_start_hpts_timer()
943 bbr->rc_timer_first = 1; in bbr_start_hpts_timer()
945 bbr->rc_tmr_stopped = 0; in bbr_start_hpts_timer()
962 tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK; in bbr_timer_audit()
963 if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT)) in bbr_timer_audit()
965 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_timer_audit()
966 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) && in bbr_timer_audit()
971 inp = bbr->rc_inp; in bbr_timer_audit()
974 if (tp->t_flags & TF_DELACK) { in bbr_timer_audit()
981 } else if (sbavail(&inp->inp_socket->so_snd) && in bbr_timer_audit()
990 inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_timer_audit()
991 (tp->t_state <= TCPS_CLOSING)) && in bbr_timer_audit()
993 (tp->snd_max == tp->snd_una)) { in bbr_timer_audit()
998 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) { in bbr_timer_audit()
999 if ((tp->t_flags & TF_SENTFIN) && in bbr_timer_audit()
1000 ((tp->snd_max - tp->snd_una) == 1) && in bbr_timer_audit()
1001 (rsm->r_flags & BBR_HAS_FIN)) { in bbr_timer_audit()
1014 } else if (SEQ_GT(tp->snd_max, tp->snd_una) && in bbr_timer_audit()
1018 * Either a TLP or RXT is fine if no sack-passed is in place in bbr_timer_audit()
1031 if (SEQ_GT(tp->snd_max, tp->snd_una) && in bbr_timer_audit()
1050 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) { in bbr_timer_audit()
1054 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, in bbr_timer_audit()
1059 * timer. We don't bother with keep-alive, since when we in bbr_timer_audit()
1060 * jump through the output, it will start the keep-alive if in bbr_timer_audit()
1063 * We only need a delayed-ack added and or the hpts_timeout. in bbr_timer_audit()
1066 if (tp->t_flags & TF_DELACK) { in bbr_timer_audit()
1069 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; in bbr_timer_audit()
1073 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; in bbr_timer_audit()
1079 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; in bbr_timer_audit()
1101 * time-stamp taken in theory earlier return the difference. The in bbr_calc_time()
1106 return (cts - earlier_time); in bbr_calc_time()
1119 return (cts - earlier_time); in bbr_calc_time()
1129 if (error || req->newptr == NULL) in sysctl_bbr_clear_lost()
1154 printf("Clearing BBR out-size counters\n"); in sysctl_bbr_clear_lost()
1190 "How many mss's are outstanding during probe-rtt"); in bbr_init_sysctls()
1195 "If RTT has not shrank in this many micro-seconds enter probe-rtt"); in bbr_init_sysctls()
1200 "How many microseconds in probe-rtt"); in bbr_init_sysctls()
1215 "If we keep setting new low rtt's but delay going in probe-rtt can we force in??"); in bbr_init_sysctls()
1220 "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?"); in bbr_init_sysctls()
1225 "Can we dynamically adjust the probe-rtt limits and times?"); in bbr_init_sysctls()
1313 "For 0 -> 24Mbps what is floor number of segments for TSO"); in bbr_init_sysctls()
1318 "For 0 -> 24Mbps what is top number of segments for TSO"); in bbr_init_sysctls()
1400 "How many pkt-epoch's (0 is off) do we need before pacing is on?"); in bbr_init_sysctls()
1410 "Should google mode not use retransmission measurements for the b/w estimation?"); in bbr_init_sysctls()
1432 "Do we use a pkt-epoch for substate if 0 rttProp?"); in bbr_init_sysctls()
1437 "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?"); in bbr_init_sysctls()
1477 "How many packet-epochs does the b/w delivery rate last?"); in bbr_init_sysctls()
1482 "Does our sub-state drain invoke app limited if its long?"); in bbr_init_sysctls()
1487 "Should we set/recover cwnd for sub-state drain?"); in bbr_init_sysctls()
1492 "Should we set/recover cwnd for main-state drain?"); in bbr_init_sysctls()
1497 "Should we allow google probe-bw/drain to exit early at flight target?"); in bbr_init_sysctls()
1502 "Should we have losses exit gain of probebw in google mode??"); in bbr_init_sysctls()
1566 "What is the high-speed min cwnd (rttProp under 1ms)"); in bbr_init_sysctls()
1659 "Maximum RTO in seconds -- should be at least as large as min_rto"); in bbr_init_sysctls()
1716 "Do we call the policer detection code from a rack-timeout?"); in bbr_init_sysctls()
1721 "What packet epoch do we do false-positive detection at (0=no)?"); in bbr_init_sysctls()
1796 "Total number of enobufs for non-hardware paced flows"); in bbr_init_sysctls()
1860 l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate; in bbr_fill_in_logging_data()
1861 l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_fill_in_logging_data()
1862 l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop); in bbr_fill_in_logging_data()
1863 l->bw_inuse = bbr_get_bw(bbr); in bbr_fill_in_logging_data()
1864 l->inflight = ctf_flight_size(bbr->rc_tp, in bbr_fill_in_logging_data()
1865 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_fill_in_logging_data()
1866 l->applimited = bbr->r_ctl.r_app_limited_until; in bbr_fill_in_logging_data()
1867 l->delivered = bbr->r_ctl.rc_delivered; in bbr_fill_in_logging_data()
1868 l->timeStamp = cts; in bbr_fill_in_logging_data()
1869 l->lost = bbr->r_ctl.rc_lost; in bbr_fill_in_logging_data()
1870 l->bbr_state = bbr->rc_bbr_state; in bbr_fill_in_logging_data()
1871 l->bbr_substate = bbr_state_val(bbr); in bbr_fill_in_logging_data()
1872 l->epoch = bbr->r_ctl.rc_rtt_epoch; in bbr_fill_in_logging_data()
1873 l->lt_epoch = bbr->r_ctl.rc_lt_epoch; in bbr_fill_in_logging_data()
1874 l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain; in bbr_fill_in_logging_data()
1875 l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain; in bbr_fill_in_logging_data()
1876 l->inhpts = tcp_in_hpts(bbr->rc_tp); in bbr_fill_in_logging_data()
1877 l->use_lt_bw = bbr->rc_lt_use_bw; in bbr_fill_in_logging_data()
1878 l->pkts_out = bbr->r_ctl.rc_flight_at_input; in bbr_fill_in_logging_data()
1879 l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_fill_in_logging_data()
1885 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bw_reduce()
1888 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_bw_reduce()
1893 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate; in bbr_log_type_bw_reduce()
1895 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt; in bbr_log_type_bw_reduce()
1897 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bw_reduce()
1898 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bw_reduce()
1899 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bw_reduce()
1901 0, &log, false, &bbr->rc_tv); in bbr_log_type_bw_reduce()
1906 bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count) in bbr_log_type_rwnd_collapse() argument
1908 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_rwnd_collapse()
1911 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_rwnd_collapse()
1914 log.u_bbr.flex8 = mode; in bbr_log_type_rwnd_collapse()
1915 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_rwnd_collapse()
1916 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_rwnd_collapse()
1917 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_rwnd_collapse()
1919 0, &log, false, &bbr->rc_tv); in bbr_log_type_rwnd_collapse()
1927 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_just_return()
1932 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags; in bbr_log_type_just_return()
1933 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp; in bbr_log_type_just_return()
1935 log.u_bbr.flex5 = bbr->rc_in_persist; in bbr_log_type_just_return()
1936 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val; in bbr_log_type_just_return()
1938 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_type_just_return()
1941 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_just_return()
1942 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_just_return()
1943 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_just_return()
1945 tlen, &log, false, &bbr->rc_tv); in bbr_log_type_just_return()
1952 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_enter_rec()
1955 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_enter_rec()
1957 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent; in bbr_log_type_enter_rec()
1958 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start; in bbr_log_type_enter_rec()
1959 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_enter_rec()
1960 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_enter_rec()
1961 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_enter_rec()
1963 0, &log, false, &bbr->rc_tv); in bbr_log_type_enter_rec()
1979 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_msgsize_fail()
1980 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_msgsize_fail()
1982 0, &log, false, &bbr->rc_tv); in bbr_log_msgsize_fail()
1989 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_flowend()
1994 if (bbr->rc_inp->inp_socket) { in bbr_log_flowend()
1995 r = &bbr->rc_inp->inp_socket->so_rcv; in bbr_log_flowend()
1996 s = &bbr->rc_inp->inp_socket->so_snd; in bbr_log_flowend()
2001 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_flowend()
2012 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_pkt_epoch()
2018 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_log_pkt_epoch()
2019 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt; in bbr_log_pkt_epoch()
2020 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch; in bbr_log_pkt_epoch()
2021 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; in bbr_log_pkt_epoch()
2024 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count; in bbr_log_pkt_epoch()
2025 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_pkt_epoch()
2026 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_pkt_epoch()
2027 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_pkt_epoch()
2029 0, &log, false, &bbr->rc_tv); in bbr_log_pkt_epoch()
2036 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_time_epoch()
2040 log.u_bbr.flex1 = bbr->r_ctl.rc_lost; in bbr_log_time_epoch()
2041 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat; in bbr_log_time_epoch()
2042 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat; in bbr_log_time_epoch()
2044 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_time_epoch()
2045 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_time_epoch()
2046 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_time_epoch()
2048 0, &log, false, &bbr->rc_tv); in bbr_log_time_epoch()
2055 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_set_of_state_target()
2058 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_set_of_state_target()
2059 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state; in bbr_log_set_of_state_target()
2062 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs; in bbr_log_set_of_state_target()
2064 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs; in bbr_log_set_of_state_target()
2065 log.u_bbr.flex7 = bbr->rc_last_options; in bbr_log_set_of_state_target()
2067 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_set_of_state_target()
2068 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_set_of_state_target()
2069 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_set_of_state_target()
2071 0, &log, false, &bbr->rc_tv); in bbr_log_set_of_state_target()
2079 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_statechange()
2084 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; in bbr_log_type_statechange()
2085 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int; in bbr_log_type_statechange()
2090 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch; in bbr_log_type_statechange()
2091 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; in bbr_log_type_statechange()
2092 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000); in bbr_log_type_statechange()
2093 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra; in bbr_log_type_statechange()
2094 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state; in bbr_log_type_statechange()
2095 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_statechange()
2096 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_statechange()
2097 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_statechange()
2099 0, &log, false, &bbr->rc_tv); in bbr_log_type_statechange()
2107 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_rtt_shrinks()
2112 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; in bbr_log_rtt_shrinks()
2113 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt; in bbr_log_rtt_shrinks()
2116 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state; in bbr_log_rtt_shrinks()
2119 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_rtt_shrinks()
2120 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_rtt_shrinks()
2121 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_rtt_shrinks()
2123 0, &log, false, &bbr->rc_tv); in bbr_log_rtt_shrinks()
2130 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_exit_rec()
2133 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_exit_rec()
2134 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start; in bbr_log_type_exit_rec()
2135 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent; in bbr_log_type_exit_rec()
2136 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_type_exit_rec()
2137 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_exit_rec()
2138 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_exit_rec()
2139 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_exit_rec()
2141 0, &log, false, &bbr->rc_tv); in bbr_log_type_exit_rec()
2149 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_cwndupd()
2152 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_cwndupd()
2160 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_cwndupd()
2161 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_cwndupd()
2162 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_cwndupd()
2164 0, &log, false, &bbr->rc_tv); in bbr_log_type_cwndupd()
2175 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_rtt_sample()
2178 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_rtt_sample()
2180 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time; in bbr_log_rtt_sample()
2181 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay; in bbr_log_rtt_sample()
2182 log.u_bbr.flex4 = bbr->rc_tp->ts_offset; in bbr_log_rtt_sample()
2183 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_rtt_sample()
2184 log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_log_rtt_sample()
2187 log.u_bbr.flex8 = bbr->rc_ack_was_delayed; in bbr_log_rtt_sample()
2188 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_rtt_sample()
2189 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_rtt_sample()
2190 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_rtt_sample()
2192 0, &log, false, &bbr->rc_tv); in bbr_log_rtt_sample()
2199 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_pesist()
2206 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_pesist()
2207 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_pesist()
2208 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_pesist()
2210 0, &log, false, &bbr->rc_tv); in bbr_log_type_pesist()
2216 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_ack_clear()
2220 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age; in bbr_log_ack_clear()
2221 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; in bbr_log_ack_clear()
2222 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int; in bbr_log_ack_clear()
2223 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time; in bbr_log_ack_clear()
2224 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_ack_clear()
2225 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_ack_clear()
2226 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_ack_clear()
2227 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_ack_clear()
2229 0, &log, false, &bbr->rc_tv); in bbr_log_ack_clear()
2237 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_ack_event()
2243 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes; in bbr_log_ack_event()
2247 log.u_bbr.flex3 = m->m_flags; in bbr_log_ack_event()
2248 if (m->m_flags & M_TSTMP) { in bbr_log_ack_event()
2256 if (m->m_flags & M_TSTMP_LRO) { in bbr_log_ack_event()
2271 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state; in bbr_log_ack_event()
2272 log.u_bbr.flex7 = bbr->r_wanted_output; in bbr_log_ack_event()
2273 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_ack_event()
2274 TCP_LOG_EVENTP(bbr->rc_tp, th, in bbr_log_ack_event()
2275 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_ack_event()
2276 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_ack_event()
2278 tlen, &log, true, &bbr->rc_tv); in bbr_log_ack_event()
2285 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_doseg_done()
2291 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val; in bbr_log_doseg_done()
2292 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags; in bbr_log_doseg_done()
2293 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp; in bbr_log_doseg_done()
2294 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes; in bbr_log_doseg_done()
2295 log.u_bbr.flex7 = bbr->r_wanted_output; in bbr_log_doseg_done()
2296 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_doseg_done()
2297 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay; in bbr_log_doseg_done()
2298 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_doseg_done()
2299 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_doseg_done()
2300 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_doseg_done()
2302 0, &log, true, &bbr->rc_tv); in bbr_log_doseg_done()
2310 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_enobuf_jmp()
2318 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_enobuf_jmp()
2319 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_enobuf_jmp()
2320 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_enobuf_jmp()
2322 len, &log, true, &bbr->rc_tv); in bbr_log_enobuf_jmp()
2329 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_processing()
2335 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp; in bbr_log_to_processing()
2336 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_processing()
2338 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state; in bbr_log_to_processing()
2340 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_processing()
2341 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_processing()
2342 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_processing()
2344 0, &log, false, &bbr->rc_tv); in bbr_log_to_processing()
2351 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_event()
2356 log.u_bbr.flex1 = bbr->bbr_timer_src; in bbr_log_to_event()
2358 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_event()
2359 ar = (uintptr_t)(bbr->r_ctl.rc_resend); in bbr_log_to_event()
2363 ar = (uintptr_t)bbr->r_ctl.rc_resend; in bbr_log_to_event()
2366 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_to_event()
2368 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_event()
2369 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_event()
2370 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_event()
2372 0, &log, false, &bbr->rc_tv); in bbr_log_to_event()
2379 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_startup_event()
2387 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_startup_event()
2388 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; in bbr_log_startup_event()
2390 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_log_startup_event()
2391 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_startup_event()
2392 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_startup_event()
2393 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_startup_event()
2395 0, &log, false, &bbr->rc_tv); in bbr_log_startup_event()
2402 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_hpts_diag()
2406 log.u_bbr.flex1 = diag->p_nxt_slot; in bbr_log_hpts_diag()
2407 log.u_bbr.flex2 = diag->p_cur_slot; in bbr_log_hpts_diag()
2408 log.u_bbr.flex3 = diag->slot_req; in bbr_log_hpts_diag()
2409 log.u_bbr.flex4 = diag->inp_hptsslot; in bbr_log_hpts_diag()
2410 log.u_bbr.flex5 = diag->slot_remaining; in bbr_log_hpts_diag()
2411 log.u_bbr.flex6 = diag->need_new_to; in bbr_log_hpts_diag()
2412 log.u_bbr.flex7 = diag->p_hpts_active; in bbr_log_hpts_diag()
2413 log.u_bbr.flex8 = diag->p_on_min_sleep; in bbr_log_hpts_diag()
2415 log.u_bbr.epoch = diag->have_slept; in bbr_log_hpts_diag()
2416 log.u_bbr.lt_epoch = diag->yet_to_sleep; in bbr_log_hpts_diag()
2417 log.u_bbr.pkts_out = diag->co_ret; in bbr_log_hpts_diag()
2418 log.u_bbr.applimited = diag->hpts_sleep_time; in bbr_log_hpts_diag()
2419 log.u_bbr.delivered = diag->p_prev_slot; in bbr_log_hpts_diag()
2420 log.u_bbr.inflight = diag->p_runningslot; in bbr_log_hpts_diag()
2421 log.u_bbr.bw_inuse = diag->wheel_slot; in bbr_log_hpts_diag()
2422 log.u_bbr.rttProp = diag->wheel_cts; in bbr_log_hpts_diag()
2423 log.u_bbr.delRate = diag->maxslots; in bbr_log_hpts_diag()
2424 log.u_bbr.cur_del_rate = diag->p_curtick; in bbr_log_hpts_diag()
2426 log.u_bbr.cur_del_rate |= diag->p_lasttick; in bbr_log_hpts_diag()
2427 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_hpts_diag()
2428 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_hpts_diag()
2429 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_hpts_diag()
2431 0, &log, false, &bbr->rc_tv); in bbr_log_hpts_diag()
2436 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t s… in bbr_log_timer_var() argument
2439 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_timer_var()
2443 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar; in bbr_log_timer_var()
2448 log.u_bbr.flex6 = bbr->rc_tp->t_srtt; in bbr_log_timer_var()
2449 log.u_bbr.flex8 = mode; in bbr_log_timer_var()
2450 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_timer_var()
2451 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_timer_var()
2452 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_timer_var()
2454 0, &log, false, &bbr->rc_tv); in bbr_log_timer_var()
2462 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_pacing_delay_calc()
2477 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_pacing_delay_calc()
2478 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_pacing_delay_calc()
2479 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_pacing_delay_calc()
2481 len, &log, false, &bbr->rc_tv); in bbr_log_pacing_delay_calc()
2488 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_start()
2493 log.u_bbr.flex1 = bbr->bbr_timer_src; in bbr_log_to_start()
2495 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_start()
2497 log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot; in bbr_log_to_start()
2498 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_to_start()
2499 log.u_bbr.pkts_out = bbr->rc_tp->t_flags2; in bbr_log_to_start()
2501 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_start()
2502 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_start()
2503 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_start()
2505 0, &log, false, &bbr->rc_tv); in bbr_log_to_start()
2512 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_thresh_choice()
2518 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts; in bbr_log_thresh_choice()
2519 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]; in bbr_log_thresh_choice()
2520 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_thresh_choice()
2522 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift; in bbr_log_thresh_choice()
2524 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_thresh_choice()
2525 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_thresh_choice()
2526 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_thresh_choice()
2528 0, &log, false, &bbr->rc_tv); in bbr_log_thresh_choice()
2535 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_cancel()
2540 log.u_bbr.flex2 = bbr->bbr_timer_src; in bbr_log_to_cancel()
2541 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_cancel()
2542 log.u_bbr.flex4 = bbr->rc_in_persist; in bbr_log_to_cancel()
2543 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_to_cancel()
2544 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_to_cancel()
2546 log.u_bbr.pkts_out = bbr->rc_pacer_started; in bbr_log_to_cancel()
2547 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_cancel()
2548 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_cancel()
2549 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_cancel()
2551 0, &log, false, &bbr->rc_tv); in bbr_log_to_cancel()
2558 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_tstmp_validation()
2561 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_tstmp_validation()
2562 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio; in bbr_log_tstmp_validation()
2567 log.u_bbr.flex7 = bbr->rc_ts_clock_set; in bbr_log_tstmp_validation()
2568 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used; in bbr_log_tstmp_validation()
2569 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_tstmp_validation()
2570 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_tstmp_validation()
2571 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_tstmp_validation()
2573 0, &log, false, &bbr->rc_tv); in bbr_log_tstmp_validation()
2580 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_tsosize()
2587 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min; in bbr_log_type_tsosize()
2590 log.u_bbr.flex7 = bbr->rc_no_pacing; in bbr_log_type_tsosize()
2592 log.u_bbr.flex7 |= bbr->rc_past_init_win; in bbr_log_type_tsosize()
2594 log.u_bbr.flex8 = 0x80 | bbr->rc_use_google; in bbr_log_type_tsosize()
2596 log.u_bbr.flex8 = bbr->rc_use_google; in bbr_log_type_tsosize()
2597 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_tsosize()
2598 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_tsosize()
2599 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_tsosize()
2601 0, &log, false, &bbr->rc_tv); in bbr_log_type_tsosize()
2609 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_rsmclear()
2614 log.u_bbr.flex2 = rsm->r_start; in bbr_log_type_rsmclear()
2615 log.u_bbr.flex3 = rsm->r_end; in bbr_log_type_rsmclear()
2616 log.u_bbr.flex4 = rsm->r_delivered; in bbr_log_type_rsmclear()
2617 log.u_bbr.flex5 = rsm->r_rtr_cnt; in bbr_log_type_rsmclear()
2618 log.u_bbr.flex6 = rsm->r_dupack; in bbr_log_type_rsmclear()
2619 log.u_bbr.flex7 = rsm->r_tim_lastsent[0]; in bbr_log_type_rsmclear()
2620 log.u_bbr.flex8 = rsm->r_flags; in bbr_log_type_rsmclear()
2623 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_rsmclear()
2624 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_rsmclear()
2625 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_rsmclear()
2627 0, &log, false, &bbr->rc_tv); in bbr_log_type_rsmclear()
2638 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bbrupd()
2652 if (bbr->rc_ack_was_delayed) in bbr_log_type_bbrupd()
2653 log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay; in bbr_log_type_bbrupd()
2656 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bbrupd()
2657 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bbrupd()
2658 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bbrupd()
2660 flex2, &log, false, &bbr->rc_tv); in bbr_log_type_bbrupd()
2669 if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_ltbw()
2677 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost; in bbr_log_type_ltbw()
2678 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del; in bbr_log_type_ltbw()
2679 log.u_bbr.flex7 = bbr->rc_lt_is_sampling; in bbr_log_type_ltbw()
2681 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw; in bbr_log_type_ltbw()
2682 if (bbr->rc_lt_use_bw == 0) in bbr_log_type_ltbw()
2683 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch; in bbr_log_type_ltbw()
2685 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use; in bbr_log_type_ltbw()
2686 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_ltbw()
2687 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_ltbw()
2688 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_ltbw()
2690 0, &log, false, &bbr->rc_tv); in bbr_log_type_ltbw()
2697 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_progress_event()
2700 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_progress_event()
2703 log.u_bbr.flex3 = tp->t_maxunacktime; in bbr_log_progress_event()
2704 log.u_bbr.flex4 = tp->t_acktime; in bbr_log_progress_event()
2706 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_progress_event()
2707 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_progress_event()
2708 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_progress_event()
2710 0, &log, false, &bbr->rc_tv); in bbr_log_progress_event()
2719 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_type_log_hdwr_pacing()
2731 log.u_bbr.flex8 = bbr->skip_gain; in bbr_type_log_hdwr_pacing()
2733 log.u_bbr.flex8 |= bbr->gain_is_limited; in bbr_type_log_hdwr_pacing()
2735 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing; in bbr_type_log_hdwr_pacing()
2736 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg; in bbr_type_log_hdwr_pacing()
2737 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_type_log_hdwr_pacing()
2738 &bbr->rc_inp->inp_socket->so_rcv, in bbr_type_log_hdwr_pacing()
2739 &bbr->rc_inp->inp_socket->so_snd, in bbr_type_log_hdwr_pacing()
2741 0, &log, false, &bbr->rc_tv); in bbr_type_log_hdwr_pacing()
2748 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bbrsnd()
2756 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val; in bbr_log_type_bbrsnd()
2757 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay; in bbr_log_type_bbrsnd()
2758 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags); in bbr_log_type_bbrsnd()
2759 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_type_bbrsnd()
2760 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bbrsnd()
2761 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bbrsnd()
2762 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bbrsnd()
2764 len, &log, false, &bbr->rc_tv); in bbr_log_type_bbrsnd()
2771 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bbrrttprop()
2775 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered; in bbr_log_type_bbrrttprop()
2777 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt; in bbr_log_type_bbrrttprop()
2783 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bbrrttprop()
2784 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bbrrttprop()
2785 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bbrrttprop()
2787 0, &log, false, &bbr->rc_tv); in bbr_log_type_bbrrttprop()
2794 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_exit_gain()
2798 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state; in bbr_log_exit_gain()
2799 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_log_exit_gain()
2800 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch; in bbr_log_exit_gain()
2801 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs; in bbr_log_exit_gain()
2802 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs; in bbr_log_exit_gain()
2803 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight; in bbr_log_exit_gain()
2806 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_exit_gain()
2807 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_exit_gain()
2808 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_exit_gain()
2810 0, &log, false, &bbr->rc_tv); in bbr_log_exit_gain()
2817 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_settings_change()
2820 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_settings_change()
2821 /* R-HU */ in bbr_log_settings_change()
2829 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_settings_change()
2830 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_settings_change()
2831 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_settings_change()
2833 0, &log, false, &bbr->rc_tv); in bbr_log_settings_change()
2845 bw = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_get_full_bw()
2856 if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch) in bbr_set_pktepoch()
2857 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch; in bbr_set_pktepoch()
2860 del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del; in bbr_set_pktepoch()
2871 bbr->r_ctl.rc_pkt_epoch_loss_rate = (uint32_t)calclr; in bbr_set_pktepoch()
2872 if (IN_RECOVERY(bbr->rc_tp->t_flags)) in bbr_set_pktepoch()
2873 bbr->r_ctl.recovery_lr += (uint32_t)calclr; in bbr_set_pktepoch()
2874 bbr->r_ctl.rc_pkt_epoch++; in bbr_set_pktepoch()
2875 if (bbr->rc_no_pacing && in bbr_set_pktepoch()
2876 (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) { in bbr_set_pktepoch()
2877 bbr->rc_no_pacing = 0; in bbr_set_pktepoch()
2880 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time); in bbr_set_pktepoch()
2881 bbr->r_ctl.rc_pkt_epoch_time = cts; in bbr_set_pktepoch()
2884 bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered; in bbr_set_pktepoch()
2885 bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost; in bbr_set_pktepoch()
2894 bbr->r_ctl.rc_rtt_epoch++; in bbr_set_epoch()
2895 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start; in bbr_set_epoch()
2897 bbr->r_ctl.rc_rcv_epoch_start = cts; in bbr_set_epoch()
2903 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) { in bbr_isit_a_pkt_epoch()
2904 bbr->rc_is_pkt_epoch_now = 1; in bbr_isit_a_pkt_epoch()
2928 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in __bbr_get_bw()
2930 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop); in __bbr_get_bw()
2933 min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) * in __bbr_get_bw()
2936 if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) { in __bbr_get_bw()
2937 min_bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2940 } else if (bbr->rc_tp->t_srtt != 0) { in __bbr_get_bw()
2945 min_bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2950 if ((bbr->rc_past_init_win == 0) && in __bbr_get_bw()
2951 (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp))) in __bbr_get_bw()
2952 bbr->rc_past_init_win = 1; in __bbr_get_bw()
2953 if ((bbr->rc_use_google) && (bbr->r_ctl.r_measurement_count >= 1)) in __bbr_get_bw()
2956 ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) || in __bbr_get_bw()
2957 (bbr->rc_past_init_win == 0))) { in __bbr_get_bw()
2961 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop); in __bbr_get_bw()
2968 bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) * in __bbr_get_bw()
2971 if (bw < bbr->r_ctl.rc_initial_hptsi_bw) { in __bbr_get_bw()
2972 bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2976 bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2986 if (bbr->rc_lt_use_bw) in __bbr_get_bw()
2987 bw = bbr->r_ctl.rc_lt_bw; in __bbr_get_bw()
2988 else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0)) in __bbr_get_bw()
2989 bw = bbr->r_ctl.red_bw; in __bbr_get_bw()
2991 bw = get_filter_value(&bbr->r_ctl.rc_delrate); in __bbr_get_bw()
3016 bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_reset_lt_bw_interval()
3017 bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time; in bbr_reset_lt_bw_interval()
3018 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered; in bbr_reset_lt_bw_interval()
3019 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_reset_lt_bw_interval()
3025 bbr->rc_lt_is_sampling = 0; in bbr_reset_lt_bw_sampling()
3026 bbr->rc_lt_use_bw = 0; in bbr_reset_lt_bw_sampling()
3027 bbr->r_ctl.rc_lt_bw = 0; in bbr_reset_lt_bw_sampling()
3037 if (bbr->r_ctl.rc_lt_bw) { in bbr_lt_bw_samp_done()
3039 if (bbr->r_ctl.rc_lt_bw > bw) in bbr_lt_bw_samp_done()
3040 diff = bbr->r_ctl.rc_lt_bw - bw; in bbr_lt_bw_samp_done()
3042 diff = bw - bbr->r_ctl.rc_lt_bw; in bbr_lt_bw_samp_done()
3044 (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) { in bbr_lt_bw_samp_done()
3048 saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw; in bbr_lt_bw_samp_done()
3049 bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2; /* average of two */ in bbr_lt_bw_samp_done()
3050 bbr->rc_lt_use_bw = 1; in bbr_lt_bw_samp_done()
3051 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_lt_bw_samp_done()
3056 bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch; in bbr_lt_bw_samp_done()
3065 bbr->r_ctl.rc_lt_bw = bw; in bbr_lt_bw_samp_done()
3077 deduct = bbr->r_ctl.rc_level_state_extra / ran; in bbr_randomize_extra_state_time()
3078 bbr->r_ctl.rc_level_state_extra -= deduct; in bbr_randomize_extra_state_time()
3092 bbr->r_ctl.rc_exta_time_gd = 0; in bbr_pick_probebw_substate()
3093 bbr->rc_hit_state_1 = 0; in bbr_pick_probebw_substate()
3094 bbr->r_ctl.rc_level_state_extra = 0; in bbr_pick_probebw_substate()
3095 ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1)); in bbr_pick_probebw_substate()
3100 * we fully enter the state. Note that the (8 - 1 - ran) assures that in bbr_pick_probebw_substate()
3101 * we return 1 - 7, so we dont return 0 and end up starting in in bbr_pick_probebw_substate()
3104 ret_val = BBR_SUBSTATE_COUNT - 1 - ran; in bbr_pick_probebw_substate()
3106 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) in bbr_pick_probebw_substate()
3109 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_pick_probebw_substate()
3119 if (bbr->r_use_policer == 0) in bbr_lt_bw_sampling()
3121 if (bbr->rc_lt_use_bw) { in bbr_lt_bw_sampling()
3123 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use; in bbr_lt_bw_sampling()
3128 if (bbr->rc_filled_pipe) { in bbr_lt_bw_sampling()
3130 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_lt_bw_sampling()
3132 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_lt_bw_sampling()
3140 bbr->rc_bbr_state = BBR_STATE_STARTUP; in bbr_lt_bw_sampling()
3142 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_lt_bw_sampling()
3143 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_lt_bw_sampling()
3144 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; in bbr_lt_bw_sampling()
3145 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; in bbr_lt_bw_sampling()
3149 /* reason 0 is to stop using lt-bw */ in bbr_lt_bw_sampling()
3154 /* Not doing false-positive detection */ in bbr_lt_bw_sampling()
3160 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered; in bbr_lt_bw_sampling()
3161 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_lt_bw_sampling()
3164 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost; in bbr_lt_bw_sampling()
3165 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del; in bbr_lt_bw_sampling()
3180 * exhaust its tokens and estimate the steady-state rate allowed by in bbr_lt_bw_sampling()
3182 * over-estimate the bw. in bbr_lt_bw_sampling()
3184 if (bbr->rc_lt_is_sampling == 0) { in bbr_lt_bw_sampling()
3189 bbr->rc_lt_is_sampling = 1; in bbr_lt_bw_sampling()
3194 if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time)) in bbr_lt_bw_sampling()
3195 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time; in bbr_lt_bw_sampling()
3200 if (bbr->r_ctl.r_app_limited_until) { in bbr_lt_bw_sampling()
3201 /* Can not measure in app-limited state */ in bbr_lt_bw_sampling()
3207 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch; in bbr_lt_bw_sampling()
3214 /* 6 is not_enough time or no-loss */ in bbr_lt_bw_sampling()
3234 * tokens are exhausted under-estimates the policed rate. in bbr_lt_bw_sampling()
3237 /* 6 is not_enough time or no-loss */ in bbr_lt_bw_sampling()
3242 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost; in bbr_lt_bw_sampling()
3243 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del; in bbr_lt_bw_sampling()
3251 /* 6 is not_enough time or no-loss */ in bbr_lt_bw_sampling()
3280 bbr->r_ctl.rc_num_maps_alloced++; in bbr_alloc()
3283 if (bbr->r_ctl.rc_free_cnt) { in bbr_alloc()
3285 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); in bbr_alloc()
3286 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_alloc()
3287 bbr->r_ctl.rc_free_cnt--; in bbr_alloc()
3298 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { in bbr_alloc_full_limit()
3300 if (!bbr->alloc_limit_reported) { in bbr_alloc_full_limit()
3301 bbr->alloc_limit_reported = 1; in bbr_alloc_full_limit()
3318 bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) { in bbr_alloc_limit()
3320 if (!bbr->alloc_limit_reported) { in bbr_alloc_limit()
3321 bbr->alloc_limit_reported = 1; in bbr_alloc_limit()
3331 rsm->r_limit_type = limit_type; in bbr_alloc_limit()
3332 bbr->r_ctl.rc_num_split_allocs++; in bbr_alloc_limit()
3340 if (rsm->r_limit_type) { in bbr_free()
3342 bbr->r_ctl.rc_num_split_allocs--; in bbr_free()
3344 if (rsm->r_is_smallmap) in bbr_free()
3345 bbr->r_ctl.rc_num_small_maps_alloced--; in bbr_free()
3346 if (bbr->r_ctl.rc_tlp_send == rsm) in bbr_free()
3347 bbr->r_ctl.rc_tlp_send = NULL; in bbr_free()
3348 if (bbr->r_ctl.rc_resend == rsm) { in bbr_free()
3349 bbr->r_ctl.rc_resend = NULL; in bbr_free()
3351 if (bbr->r_ctl.rc_next == rsm) in bbr_free()
3352 bbr->r_ctl.rc_next = NULL; in bbr_free()
3353 if (bbr->r_ctl.rc_sacklast == rsm) in bbr_free()
3354 bbr->r_ctl.rc_sacklast = NULL; in bbr_free()
3355 if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) { in bbr_free()
3357 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_free()
3358 rsm->r_limit_type = 0; in bbr_free()
3359 bbr->r_ctl.rc_free_cnt++; in bbr_free()
3362 bbr->r_ctl.rc_num_maps_alloced--; in bbr_free()
3395 if (bbr->rc_init_win) { in bbr_initial_cwnd()
3396 i_cwnd = bbr->rc_init_win * tp->t_maxseg; in bbr_initial_cwnd()
3398 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg), in bbr_initial_cwnd()
3399 max(2 * tp->t_maxseg, 14600)); in bbr_initial_cwnd()
3401 i_cwnd = min(4 * tp->t_maxseg, in bbr_initial_cwnd()
3402 max(2 * tp->t_maxseg, 4380)); in bbr_initial_cwnd()
3405 if (tp->t_maxseg > 2190) in bbr_initial_cwnd()
3406 i_cwnd = 2 * tp->t_maxseg; in bbr_initial_cwnd()
3407 else if (tp->t_maxseg > 1095) in bbr_initial_cwnd()
3408 i_cwnd = 3 * tp->t_maxseg; in bbr_initial_cwnd()
3410 i_cwnd = 4 * tp->t_maxseg; in bbr_initial_cwnd()
3425 if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) || in bbr_get_raw_target_cwnd()
3428 return (bbr_initial_cwnd(bbr, bbr->rc_tp)); in bbr_get_raw_target_cwnd()
3438 cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT)); in bbr_get_raw_target_cwnd()
3448 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); in bbr_get_target_cwnd()
3453 * fq layer to trap packets in) quanta's per the I-D in bbr_get_target_cwnd()
3456 cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs); in bbr_get_target_cwnd()
3457 if (bbr->rc_use_google) { in bbr_get_target_cwnd()
3458 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && in bbr_get_target_cwnd()
3463 * is documented no-where except in the code. in bbr_get_target_cwnd()
3493 if (bbr->r_ctl.rc_inc_tcp_oh) { in bbr_get_header_oh()
3495 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr)); in bbr_get_header_oh()
3497 if (bbr->r_ctl.rc_inc_ip_oh) { in bbr_get_header_oh()
3500 if (bbr->r_is_v6) { in bbr_get_header_oh()
3511 if (bbr->r_ctl.rc_inc_enet_oh) { in bbr_get_header_oh()
3549 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in bbr_get_pacing_delay()
3550 num_segs = (len + maxseg - 1) / maxseg; in bbr_get_pacing_delay()
3551 if (bbr->rc_use_google == 0) { in bbr_get_pacing_delay()
3557 if (bbr->rc_use_google) { in bbr_get_pacing_delay()
3564 cbw = bw * (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount); in bbr_get_pacing_delay()
3579 (bbr->rc_use_google == 0) && in bbr_get_pacing_delay()
3604 if ((tp->t_flags & TF_GPUTINPROG) && in bbr_ack_received()
3605 SEQ_GEQ(th->th_ack, tp->gput_ack)) { in bbr_ack_received()
3615 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8; in bbr_ack_received()
3616 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000)); in bbr_ack_received()
3618 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT, in bbr_ack_received()
3620 if (tp->t_stats_gput_prev > 0) in bbr_ack_received()
3621 stats_voi_update_abs_s32(tp->t_stats, in bbr_ack_received()
3623 ((gput - tp->t_stats_gput_prev) * 100) / in bbr_ack_received()
3624 tp->t_stats_gput_prev); in bbr_ack_received()
3625 tp->t_flags &= ~TF_GPUTINPROG; in bbr_ack_received()
3626 tp->t_stats_gput_prev = cgput; in bbr_ack_received()
3629 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && in bbr_ack_received()
3630 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) { in bbr_ack_received()
3631 /* We don't change anything in probe-rtt */ in bbr_ack_received()
3634 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_ack_received()
3638 bytes_this_ack -= prev_acked; in bbr_ack_received()
3643 if ((bytes_this_ack < maxseg) && bbr->rc_use_google) in bbr_ack_received()
3649 cwnd = tp->snd_cwnd; in bbr_ack_received()
3650 bw = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_ack_received()
3654 (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain); in bbr_ack_received()
3656 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp); in bbr_ack_received()
3657 if (IN_RECOVERY(tp->t_flags) && in bbr_ack_received()
3658 (bbr->bbr_prev_in_rec == 0)) { in bbr_ack_received()
3663 bbr->pkt_conservation = 1; in bbr_ack_received()
3664 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime; in bbr_ack_received()
3666 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_ack_received()
3669 if (IN_RECOVERY(tp->t_flags)) { in bbr_ack_received()
3672 bbr->bbr_prev_in_rec = 1; in bbr_ack_received()
3674 cwnd -= losses; in bbr_ack_received()
3680 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_ack_received()
3683 if (bbr->pkt_conservation) { in bbr_ack_received()
3686 if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start)) in bbr_ack_received()
3687 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start; in bbr_ack_received()
3693 bbr->pkt_conservation = 0; in bbr_ack_received()
3699 tp->snd_cwnd = cwnd; in bbr_ack_received()
3701 prev_acked, 1, target_cwnd, th->th_ack, line); in bbr_ack_received()
3706 bbr->bbr_prev_in_rec = 0; in bbr_ack_received()
3707 if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) { in bbr_ack_received()
3708 bbr->r_ctl.restrict_growth--; in bbr_ack_received()
3712 if (bbr->rc_filled_pipe) { in bbr_ack_received()
3724 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing) in bbr_ack_received()
3732 (bbr->rc_past_init_win == 0)) { in bbr_ack_received()
3743 tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr)); in bbr_ack_received()
3744 …bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, li… in bbr_ack_received()
3752 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in tcp_bbr_partialack()
3755 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= in tcp_bbr_partialack()
3756 tp->snd_cwnd) { in tcp_bbr_partialack()
3757 bbr->r_wanted_output = 1; in tcp_bbr_partialack()
3768 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_post_recovery()
3772 EXIT_RECOVERY(tp->t_flags); in bbr_post_recovery()
3773 /* Lock in our b/w reduction for the specified number of pkt-epochs */ in bbr_post_recovery()
3774 bbr->r_recovery_bw = 0; in bbr_post_recovery()
3775 tp->snd_recover = tp->snd_una; in bbr_post_recovery()
3776 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); in bbr_post_recovery()
3777 bbr->pkt_conservation = 0; in bbr_post_recovery()
3778 if (bbr->rc_use_google == 0) { in bbr_post_recovery()
3780 * For non-google mode lets in bbr_post_recovery()
3786 bbr->bbr_prev_in_rec = 0; in bbr_post_recovery()
3789 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { in bbr_post_recovery()
3790 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent); in bbr_post_recovery()
3793 /* For probe-rtt case lets fix up its saved_cwnd */ in bbr_post_recovery()
3794 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) { in bbr_post_recovery()
3795 bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent; in bbr_post_recovery()
3800 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_post_recovery()
3801 if ((bbr->rc_use_google == 0) && in bbr_post_recovery()
3815 bbr->r_ctl.recovery_lr, 21, in bbr_post_recovery()
3817 bbr->r_ctl.rc_red_cwnd_pe, in bbr_post_recovery()
3821 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && in bbr_post_recovery()
3824 (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && in bbr_post_recovery()
3825 bbr->rc_hit_state_1 && in bbr_post_recovery()
3827 ((bbr->rc_bbr_state == BBR_STATE_DRAIN) && in bbr_post_recovery()
3833 cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_post_recovery()
3834 cwnd_p = &bbr->r_ctl.rc_saved_cwnd; in bbr_post_recovery()
3836 cwnd = tp->snd_cwnd; in bbr_post_recovery()
3837 cwnd_p = &tp->snd_cwnd; in bbr_post_recovery()
3839 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_post_recovery()
3841 if (bbr->r_ctl.rc_lost == 0) in bbr_post_recovery()
3843 else if (bbr->r_ctl.rc_delivered == 0) in bbr_post_recovery()
3846 lr2use = bbr->r_ctl.rc_lost * 1000; in bbr_post_recovery()
3847 lr2use /= bbr->r_ctl.rc_delivered; in bbr_post_recovery()
3849 lr2use += bbr->r_ctl.recovery_lr; in bbr_post_recovery()
3856 bbr->r_ctl.restrict_growth += acks_inflight; in bbr_post_recovery()
3862 newcwnd = roundup((cwnd - val), maxseg); in bbr_post_recovery()
3878 newcwnd = (get_min_cwnd(bbr) - acks_inflight); in bbr_post_recovery()
3890 if (tp->snd_cwnd > newcwnd) in bbr_post_recovery()
3891 tp->snd_cwnd = newcwnd; in bbr_post_recovery()
3895 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch; in bbr_post_recovery()
3898 bbr->r_ctl.recovery_lr = 0; in bbr_post_recovery()
3899 if (flight <= tp->snd_cwnd) { in bbr_post_recovery()
3900 bbr->r_wanted_output = 1; in bbr_post_recovery()
3902 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); in bbr_post_recovery()
3908 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_setup_red_bw()
3910 if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate) in bbr_setup_red_bw()
3911 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate; in bbr_setup_red_bw()
3912 if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2)) in bbr_setup_red_bw()
3913 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2; in bbr_setup_red_bw()
3924 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type); in bbr_cong_signal()
3926 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_cong_signal()
3929 if (!IN_RECOVERY(tp->t_flags)) { in bbr_cong_signal()
3930 tp->snd_recover = tp->snd_max; in bbr_cong_signal()
3932 bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_cong_signal()
3933 if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) { in bbr_cong_signal()
3939 bbr->r_ctl.rc_lt_epoch++; in bbr_cong_signal()
3941 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in bbr_cong_signal()
3948 bbr->r_ctl.rc_bbr_last_startup_epoch++; in bbr_cong_signal()
3950 bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd; in bbr_cong_signal()
3951 ENTER_RECOVERY(tp->t_flags); in bbr_cong_signal()
3952 bbr->rc_tlp_rtx_out = 0; in bbr_cong_signal()
3953 bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate; in bbr_cong_signal()
3954 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); in bbr_cong_signal()
3955 if (tcp_in_hpts(bbr->rc_tp) && in bbr_cong_signal()
3956 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) { in bbr_cong_signal()
3963 bbr->rc_timer_first = 1; in bbr_cong_signal()
3964 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_cong_signal()
3972 if ((bbr->rc_use_google == 0) && in bbr_cong_signal()
3973 (bbr->r_ctl.bbr_rttprobe_gain_val || in bbr_cong_signal()
3974 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) { in bbr_cong_signal()
3975 tp->snd_cwnd = ctf_flight_size(tp, in bbr_cong_signal()
3976 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_cong_signal()
3977 (tp->t_maxseg - bbr->rc_last_options); in bbr_cong_signal()
3978 if (tp->snd_cwnd < get_min_cwnd(bbr)) { in bbr_cong_signal()
3980 tp->snd_cwnd = get_min_cwnd(bbr); in bbr_cong_signal()
3984 bbr_log_type_enter_rec(bbr, rsm->r_start); in bbr_cong_signal()
3990 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime); in bbr_cong_signal()
3991 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { in bbr_cong_signal()
3992 tp->snd_cwnd = tp->snd_cwnd_prev; in bbr_cong_signal()
3993 tp->snd_ssthresh = tp->snd_ssthresh_prev; in bbr_cong_signal()
3994 tp->snd_recover = tp->snd_recover_prev; in bbr_cong_signal()
3995 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent); in bbr_cong_signal()
3998 tp->t_badrxtwin = 0; in bbr_cong_signal()
4006 * - There is no delayed ack timer in progress.
4007 * - Our last ack wasn't a 0-sized window. We never want to delay
4008 * the ack that opens up a 0-sized window.
4009 * - LRO wasn't used for this segment. We make sure by checking that the
4011 * - Delayed acks are enabled or this is a half-synchronized T/TCP
4013 * - The data being acked is less than a full segment (a stretch ack
4015 * - nsegs is 1 (if its more than that we received more than 1 ack).
4018 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
4019 ((tp->t_flags & TF_DELACK) == 0) && \
4020 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) && \
4021 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4036 * Walk the time-order transmitted list looking for an rsm that is in bbr_find_lowest_rsm()
4040 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) { in bbr_find_lowest_rsm()
4041 if (rsm->r_flags & BBR_ACKED) { in bbr_find_lowest_rsm()
4061 TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_find_high_nonack()
4062 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) { in bbr_find_high_nonack()
4083 * If reorder-fade is configured, then we track the last time we saw in bbr_calc_thresh_rack()
4084 * re-ordering occur. If we reach the point where enough time as in bbr_calc_thresh_rack()
4087 * Or if reorder-face is 0, then once we see reordering we consider in bbr_calc_thresh_rack()
4091 * In the end if lro is non-zero we add the extra time for in bbr_calc_thresh_rack()
4099 if (bbr->r_ctl.rc_reorder_ts) { in bbr_calc_thresh_rack()
4100 if (bbr->r_ctl.rc_reorder_fade) { in bbr_calc_thresh_rack()
4101 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) { in bbr_calc_thresh_rack()
4102 lro = cts - bbr->r_ctl.rc_reorder_ts; in bbr_calc_thresh_rack()
4114 if (lro > bbr->r_ctl.rc_reorder_fade) { in bbr_calc_thresh_rack()
4116 bbr->r_ctl.rc_reorder_ts = 0; in bbr_calc_thresh_rack()
4126 thresh = srtt + bbr->r_ctl.rc_pkt_delay; in bbr_calc_thresh_rack()
4129 if (bbr->r_ctl.rc_reorder_shift) in bbr_calc_thresh_rack()
4130 thresh += (srtt >> bbr->r_ctl.rc_reorder_shift); in bbr_calc_thresh_rack()
4137 if ((bbr->rc_tp)->t_srtt == 0) in bbr_calc_thresh_rack()
4140 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_calc_thresh_rack()
4145 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { in bbr_calc_thresh_rack()
4146 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND); in bbr_calc_thresh_rack()
4153 * Return to the caller the amount of time in mico-seconds
4167 if (bbr->rc_tlp_threshold) in bbr_calc_thresh_tlp()
4168 thresh = srtt + (srtt / bbr->rc_tlp_threshold); in bbr_calc_thresh_tlp()
4171 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_calc_thresh_tlp()
4173 len = rsm->r_end - rsm->r_start; in bbr_calc_thresh_tlp()
4180 * possible inter-packet delay (if any). in bbr_calc_thresh_tlp()
4185 idx = rsm->r_rtr_cnt - 1; in bbr_calc_thresh_tlp()
4186 nidx = prsm->r_rtr_cnt - 1; in bbr_calc_thresh_tlp()
4187 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) { in bbr_calc_thresh_tlp()
4189 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx]; in bbr_calc_thresh_tlp()
4194 * Possibly compensate for delayed-ack. in bbr_calc_thresh_tlp()
4203 if (tp->t_srtt == 0) in bbr_calc_thresh_tlp()
4206 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur); in bbr_calc_thresh_tlp()
4214 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { in bbr_calc_thresh_tlp()
4215 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND); in bbr_calc_thresh_tlp()
4233 f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop); in bbr_get_rtt()
4234 if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) { in bbr_get_rtt()
4236 if (bbr->rc_tp->t_srtt == 0) in bbr_get_rtt()
4239 f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_get_rtt()
4242 * delayed-ack min in bbr_get_rtt()
4248 /* Take the filter version or last measured pkt-rtt */ in bbr_get_rtt()
4252 if (bbr->r_ctl.rc_pkt_epoch_rtt) { in bbr_get_rtt()
4253 srtt = bbr->r_ctl.rc_pkt_epoch_rtt; in bbr_get_rtt()
4259 srtt = bbr->r_ctl.rc_last_rtt; in bbr_get_rtt()
4261 if (bbr->rc_ack_was_delayed) in bbr_get_rtt()
4262 srtt += bbr->r_ctl.rc_ack_hdwr_delay; in bbr_get_rtt()
4264 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_get_rtt()
4282 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) { in bbr_is_lost()
4304 if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) { in bbr_check_recovery_mode()
4308 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_check_recovery_mode()
4313 if (tp->t_flags & TF_SENTFIN) { in bbr_check_recovery_mode()
4317 if (rsm->r_flags & BBR_ACKED) { in bbr_check_recovery_mode()
4326 idx = rsm->r_rtr_cnt - 1; in bbr_check_recovery_mode()
4327 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) { in bbr_check_recovery_mode()
4333 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) || in bbr_check_recovery_mode()
4334 (rsm->r_flags & BBR_SACK_PASSED))) { in bbr_check_recovery_mode()
4335 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) { in bbr_check_recovery_mode()
4336 rsm->r_flags |= BBR_MARKED_LOST; in bbr_check_recovery_mode()
4337 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; in bbr_check_recovery_mode()
4338 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; in bbr_check_recovery_mode()
4342 if ((rsm->r_end - rsm->r_start) == 0) in bbr_check_recovery_mode()
4363 * retransmissions, if so we will enter fast-recovery. The output in bbr_timeout_rack()
4369 if (bbr->rc_all_timers_stopped) { in bbr_timeout_rack()
4372 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_timeout_rack()
4377 lost = bbr->r_ctl.rc_lost; in bbr_timeout_rack()
4378 if (bbr->r_state && (bbr->r_state != tp->t_state)) in bbr_timeout_rack()
4381 if (bbr->r_ctl.rc_resend == NULL) { in bbr_timeout_rack()
4383 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_timeout_rack()
4386 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); in bbr_timeout_rack()
4387 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK; in bbr_timeout_rack()
4396 nrsm->r_start = start; in bbr_clone_rsm()
4397 nrsm->r_end = rsm->r_end; in bbr_clone_rsm()
4398 nrsm->r_rtr_cnt = rsm->r_rtr_cnt; in bbr_clone_rsm()
4399 nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed; in bbr_clone_rsm()
4400 nrsm->r_flags = rsm->r_flags; in bbr_clone_rsm()
4402 nrsm->r_flags &= ~BBR_HAS_SYN; in bbr_clone_rsm()
4404 rsm->r_flags &= ~BBR_HAS_FIN; in bbr_clone_rsm()
4405 nrsm->r_dupack = rsm->r_dupack; in bbr_clone_rsm()
4406 nrsm->r_rtr_bytes = 0; in bbr_clone_rsm()
4407 nrsm->r_is_gain = rsm->r_is_gain; in bbr_clone_rsm()
4408 nrsm->r_is_drain = rsm->r_is_drain; in bbr_clone_rsm()
4409 nrsm->r_delivered = rsm->r_delivered; in bbr_clone_rsm()
4410 nrsm->r_ts_valid = rsm->r_ts_valid; in bbr_clone_rsm()
4411 nrsm->r_del_ack_ts = rsm->r_del_ack_ts; in bbr_clone_rsm()
4412 nrsm->r_del_time = rsm->r_del_time; in bbr_clone_rsm()
4413 nrsm->r_app_limited = rsm->r_app_limited; in bbr_clone_rsm()
4414 nrsm->r_first_sent_time = rsm->r_first_sent_time; in bbr_clone_rsm()
4415 nrsm->r_flight_at_send = rsm->r_flight_at_send; in bbr_clone_rsm()
4417 nrsm->r_bbr_state = rsm->r_bbr_state; in bbr_clone_rsm()
4418 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) { in bbr_clone_rsm()
4419 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx]; in bbr_clone_rsm()
4421 rsm->r_end = nrsm->r_start; in bbr_clone_rsm()
4422 idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); in bbr_clone_rsm()
4425 if ((rsm->r_is_smallmap == 0) && in bbr_clone_rsm()
4426 ((rsm->r_end - rsm->r_start) <= idx)) { in bbr_clone_rsm()
4427 bbr->r_ctl.rc_num_small_maps_alloced++; in bbr_clone_rsm()
4428 rsm->r_is_smallmap = 1; in bbr_clone_rsm()
4431 if ((nrsm->r_end - nrsm->r_start) <= idx) { in bbr_clone_rsm()
4432 bbr->r_ctl.rc_num_small_maps_alloced++; in bbr_clone_rsm()
4433 nrsm->r_is_smallmap = 1; in bbr_clone_rsm()
4456 if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) { in bbr_sack_mergable()
4458 if ((l_rsm->r_end == start) || in bbr_sack_mergable()
4459 (SEQ_LT(start, l_rsm->r_end) && in bbr_sack_mergable()
4460 SEQ_GT(end, l_rsm->r_end))) { in bbr_sack_mergable()
4462 * map blk |------| in bbr_sack_mergable()
4463 * sack blk |------| in bbr_sack_mergable()
4465 * map blk |------| in bbr_sack_mergable()
4466 * sack blk |------| in bbr_sack_mergable()
4471 if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) { in bbr_sack_mergable()
4473 if ((r_rsm->r_start == end) || in bbr_sack_mergable()
4474 (SEQ_LT(start, r_rsm->r_start) && in bbr_sack_mergable()
4475 SEQ_GT(end, r_rsm->r_start))) { in bbr_sack_mergable()
4477 * map blk |---------| in bbr_sack_mergable()
4478 * sack blk |----| in bbr_sack_mergable()
4480 * map blk |---------| in bbr_sack_mergable()
4481 * sack blk |-------| in bbr_sack_mergable()
4504 l_rsm->r_end = r_rsm->r_end; in bbr_merge_rsm()
4505 if (l_rsm->r_dupack < r_rsm->r_dupack) in bbr_merge_rsm()
4506 l_rsm->r_dupack = r_rsm->r_dupack; in bbr_merge_rsm()
4507 if (r_rsm->r_rtr_bytes) in bbr_merge_rsm()
4508 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes; in bbr_merge_rsm()
4509 if (r_rsm->r_in_tmap) { in bbr_merge_rsm()
4511 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext); in bbr_merge_rsm()
4513 if (r_rsm->r_app_limited) in bbr_merge_rsm()
4514 l_rsm->r_app_limited = r_rsm->r_app_limited; in bbr_merge_rsm()
4516 if (r_rsm->r_flags & BBR_HAS_FIN) in bbr_merge_rsm()
4517 l_rsm->r_flags |= BBR_HAS_FIN; in bbr_merge_rsm()
4518 if (r_rsm->r_flags & BBR_TLP) in bbr_merge_rsm()
4519 l_rsm->r_flags |= BBR_TLP; in bbr_merge_rsm()
4520 if (r_rsm->r_flags & BBR_RWND_COLLAPSED) in bbr_merge_rsm()
4521 l_rsm->r_flags |= BBR_RWND_COLLAPSED; in bbr_merge_rsm()
4522 if (r_rsm->r_flags & BBR_MARKED_LOST) { in bbr_merge_rsm()
4524 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start; in bbr_merge_rsm()
4526 TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next); in bbr_merge_rsm()
4527 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) { in bbr_merge_rsm()
4529 r_rsm->r_limit_type = l_rsm->r_limit_type; in bbr_merge_rsm()
4530 l_rsm->r_limit_type = 0; in bbr_merge_rsm()
4557 if (bbr->rc_all_timers_stopped) { in bbr_timeout_tlp()
4560 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_timeout_tlp()
4566 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_tlp()
4569 if (bbr->rc_in_persist) { in bbr_timeout_tlp()
4572 if (bbr->r_state && (bbr->r_state != tp->t_state)) in bbr_timeout_tlp()
4575 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_tlp()
4581 avail = sbavail(&so->so_snd); in bbr_timeout_tlp()
4583 if (out > tp->snd_wnd) { in bbr_timeout_tlp()
4590 amm = avail - out; in bbr_timeout_tlp()
4593 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) { in bbr_timeout_tlp()
4594 /* not enough to fill a MTU and no-delay is off */ in bbr_timeout_tlp()
4597 /* Set the send-new override */ in bbr_timeout_tlp()
4598 if ((out + amm) <= tp->snd_wnd) { in bbr_timeout_tlp()
4599 bbr->rc_tlp_new_data = 1; in bbr_timeout_tlp()
4603 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; in bbr_timeout_tlp()
4604 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max; in bbr_timeout_tlp()
4605 bbr->r_ctl.rc_tlp_send = NULL; in bbr_timeout_tlp()
4612 * Ok we need to arrange the last un-acked segment to be re-sent, or in bbr_timeout_tlp()
4613 * optionally the first un-acked segment. in bbr_timeout_tlp()
4616 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); in bbr_timeout_tlp()
4628 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_timeout_tlp()
4629 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) { in bbr_timeout_tlp()
4636 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_timeout_tlp()
4641 if ((rsm->r_end - rsm->r_start) > maxseg) { in bbr_timeout_tlp()
4652 * do the large send (BTLP :-) ). in bbr_timeout_tlp()
4656 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg)); in bbr_timeout_tlp()
4657 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_timeout_tlp()
4658 if (rsm->r_in_tmap) { in bbr_timeout_tlp()
4659 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_timeout_tlp()
4660 nrsm->r_in_tmap = 1; in bbr_timeout_tlp()
4662 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_timeout_tlp()
4666 bbr->r_ctl.rc_tlp_send = rsm; in bbr_timeout_tlp()
4667 bbr->rc_tlp_rtx_out = 1; in bbr_timeout_tlp()
4668 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) { in bbr_timeout_tlp()
4669 bbr->r_ctl.rc_tlp_seg_send_cnt++; in bbr_timeout_tlp()
4670 tp->t_rxtshift++; in bbr_timeout_tlp()
4672 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start; in bbr_timeout_tlp()
4673 bbr->r_ctl.rc_tlp_seg_send_cnt = 1; in bbr_timeout_tlp()
4676 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) { in bbr_timeout_tlp()
4682 bbr->rc_tlp_new_data = 0; in bbr_timeout_tlp()
4683 bbr->r_ctl.rc_tlp_send = NULL; in bbr_timeout_tlp()
4685 rsm->r_flags &= ~BBR_TLP; in bbr_timeout_tlp()
4689 rsm->r_flags |= BBR_TLP; in bbr_timeout_tlp()
4691 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) && in bbr_timeout_tlp()
4692 (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) { in bbr_timeout_tlp()
4700 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP; in bbr_timeout_tlp()
4715 if (bbr->rc_all_timers_stopped) { in bbr_timeout_delack()
4719 tp->t_flags &= ~TF_DELACK; in bbr_timeout_delack()
4720 tp->t_flags |= TF_ACKNOW; in bbr_timeout_delack()
4722 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK; in bbr_timeout_delack()
4727 * Here we send a KEEP-ALIVE like probe to the
4739 if (bbr->rc_all_timers_stopped) { in bbr_timeout_persist()
4742 if (bbr->rc_in_persist == 0) in bbr_timeout_persist()
4750 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT; in bbr_timeout_persist()
4757 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_persist()
4765 if (tp->t_rxtshift >= V_tcp_retries && in bbr_timeout_persist()
4766 (ticks - tp->t_rcvtime >= tcp_maxpersistidle || in bbr_timeout_persist()
4767 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) { in bbr_timeout_persist()
4770 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_persist()
4772 if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) && in bbr_timeout_persist()
4773 tp->snd_una == tp->snd_max) { in bbr_timeout_persist()
4782 if (tp->t_state > TCPS_CLOSE_WAIT && in bbr_timeout_persist()
4783 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) { in bbr_timeout_persist()
4786 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_persist()
4788 t_template = tcpip_maketemplate(bbr->rc_inp); in bbr_timeout_persist()
4790 tcp_respond(tp, t_template->tt_ipgen, in bbr_timeout_persist()
4791 &t_template->tt_t, (struct mbuf *)NULL, in bbr_timeout_persist()
4792 tp->rcv_nxt, tp->snd_una - 1, 0); in bbr_timeout_persist()
4794 if (tp->t_flags & TF_DELACK) in bbr_timeout_persist()
4795 tp->t_flags &= ~TF_DELACK; in bbr_timeout_persist()
4798 if (tp->t_rxtshift < V_tcp_retries) in bbr_timeout_persist()
4799 tp->t_rxtshift++; in bbr_timeout_persist()
4817 if (bbr->rc_all_timers_stopped) { in bbr_timeout_keepalive()
4820 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP; in bbr_timeout_keepalive()
4823 * Keep-alive timer went off; send something or drop connection if in bbr_timeout_keepalive()
4827 if (tp->t_state < TCPS_ESTABLISHED) in bbr_timeout_keepalive()
4829 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_timeout_keepalive()
4830 tp->t_state <= TCPS_CLOSING) { in bbr_timeout_keepalive()
4831 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp)) in bbr_timeout_keepalive()
4838 * number tp->snd_una-1 causes the transmitted zero-length in bbr_timeout_keepalive()
4846 tcp_respond(tp, t_template->tt_ipgen, in bbr_timeout_keepalive()
4847 &t_template->tt_t, (struct mbuf *)NULL, in bbr_timeout_keepalive()
4848 tp->rcv_nxt, tp->snd_una - 1, 0); in bbr_timeout_keepalive()
4857 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_keepalive()
4869 * un-acked. in bbr_remxt_tmr()
4875 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_remxt_tmr()
4876 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_remxt_tmr()
4877 lost = bbr->r_ctl.rc_lost; in bbr_remxt_tmr()
4878 if (bbr->r_state && (bbr->r_state != tp->t_state)) in bbr_remxt_tmr()
4881 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_remxt_tmr()
4882 if (rsm->r_flags & BBR_ACKED) { in bbr_remxt_tmr()
4885 rsm->r_dupack = 0; in bbr_remxt_tmr()
4886 if (rsm->r_in_tmap == 0) { in bbr_remxt_tmr()
4887 /* We must re-add it back to the tlist */ in bbr_remxt_tmr()
4889 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_remxt_tmr()
4891 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext); in bbr_remxt_tmr()
4893 rsm->r_in_tmap = 1; in bbr_remxt_tmr()
4895 old_flags = rsm->r_flags; in bbr_remxt_tmr()
4896 rsm->r_flags |= BBR_RXT_CLEARED; in bbr_remxt_tmr()
4897 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS); in bbr_remxt_tmr()
4900 if ((tp->t_state < TCPS_ESTABLISHED) && in bbr_remxt_tmr()
4901 (rsm->r_start == tp->snd_una)) { in bbr_remxt_tmr()
4909 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) { in bbr_remxt_tmr()
4910 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; in bbr_remxt_tmr()
4911 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; in bbr_remxt_tmr()
4918 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST; in bbr_remxt_tmr()
4919 rsm->r_flags &= ~BBR_WAS_SACKPASS; in bbr_remxt_tmr()
4928 rsm->r_flags |= BBR_MARKED_LOST; in bbr_remxt_tmr()
4929 rsm->r_flags &= ~BBR_WAS_SACKPASS; in bbr_remxt_tmr()
4930 rsm->r_flags &= ~BBR_SACK_PASSED; in bbr_remxt_tmr()
4935 bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_remxt_tmr()
4936 /* Clear the count (we just un-acked them) */ in bbr_remxt_tmr()
4938 bbr->rc_tlp_new_data = 0; in bbr_remxt_tmr()
4939 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; in bbr_remxt_tmr()
4941 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_remxt_tmr()
4942 bbr->r_agg_early_set = 0; in bbr_remxt_tmr()
4943 bbr->r_ctl.rc_agg_early = 0; in bbr_remxt_tmr()
4944 bbr->rc_tlp_rtx_out = 0; in bbr_remxt_tmr()
4945 bbr->r_ctl.rc_sacked = 0; in bbr_remxt_tmr()
4946 bbr->r_ctl.rc_sacklast = NULL; in bbr_remxt_tmr()
4947 bbr->r_timer_override = 1; in bbr_remxt_tmr()
4948 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); in bbr_remxt_tmr()
4952 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4963 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT; in bbr_timeout_rxt()
4964 if (bbr->rc_all_timers_stopped) { in bbr_timeout_rxt()
4967 if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_timeout_rxt()
4968 (tp->snd_una == tp->snd_max)) { in bbr_timeout_rxt()
4979 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_rxt()
4982 if ((bbr->r_ctl.rc_resend == NULL) || in bbr_timeout_rxt()
4983 ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) { in bbr_timeout_rxt()
4990 tp->t_rxtshift++; in bbr_timeout_rxt()
4992 if (tp->t_rxtshift > V_tcp_retries) { in bbr_timeout_rxt()
4993 tp->t_rxtshift = V_tcp_retries; in bbr_timeout_rxt()
4997 MPASS(tp->t_softerror >= 0); in bbr_timeout_rxt()
4998 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT; in bbr_timeout_rxt()
5001 if (tp->t_state == TCPS_SYN_SENT) { in bbr_timeout_rxt()
5006 tp->snd_cwnd = 1; in bbr_timeout_rxt()
5007 } else if (tp->t_rxtshift == 1) { in bbr_timeout_rxt()
5014 * End-to-End Network Path Properties" by Allman and Paxson in bbr_timeout_rxt()
5017 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_rxt()
5018 if (!IN_RECOVERY(tp->t_flags)) { in bbr_timeout_rxt()
5019 tp->snd_cwnd_prev = tp->snd_cwnd; in bbr_timeout_rxt()
5020 tp->snd_ssthresh_prev = tp->snd_ssthresh; in bbr_timeout_rxt()
5021 tp->snd_recover_prev = tp->snd_recover; in bbr_timeout_rxt()
5022 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); in bbr_timeout_rxt()
5023 tp->t_flags |= TF_PREVVALID; in bbr_timeout_rxt()
5025 tp->t_flags &= ~TF_PREVVALID; in bbr_timeout_rxt()
5027 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_rxt()
5029 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_rxt()
5030 tp->t_flags &= ~TF_PREVVALID; in bbr_timeout_rxt()
5033 if ((tp->t_state == TCPS_SYN_SENT) || in bbr_timeout_rxt()
5034 (tp->t_state == TCPS_SYN_RECEIVED)) in bbr_timeout_rxt()
5035 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift]; in bbr_timeout_rxt()
5037 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift]; in bbr_timeout_rxt()
5038 TCPT_RANGESET(tp->t_rxtcur, rexmt, in bbr_timeout_rxt()
5039 MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), in bbr_timeout_rxt()
5040 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000)); in bbr_timeout_rxt()
5049 isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false; in bbr_timeout_rxt()
5056 ((tp->t_state == TCPS_ESTABLISHED) || in bbr_timeout_rxt()
5057 (tp->t_state == TCPS_FIN_WAIT_1))) { in bbr_timeout_rxt()
5060 * 1448 -> 1188 -> 524) should be given 2 chances to recover in bbr_timeout_rxt()
5061 * before further clamping down. 'tp->t_rxtshift % 2 == 0' in bbr_timeout_rxt()
5064 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) == in bbr_timeout_rxt()
5066 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 && in bbr_timeout_rxt()
5067 tp->t_rxtshift % 2 == 0)) { in bbr_timeout_rxt()
5069 * Enter Path MTU Black-hole Detection mechanism: - in bbr_timeout_rxt()
5070 * Disable Path MTU Discovery (IP "DF" bit). - in bbr_timeout_rxt()
5074 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) { in bbr_timeout_rxt()
5079 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE; in bbr_timeout_rxt()
5081 tp->t_pmtud_saved_maxseg = tp->t_maxseg; in bbr_timeout_rxt()
5088 isipv6 = bbr->r_is_v6; in bbr_timeout_rxt()
5090 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) { in bbr_timeout_rxt()
5092 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss; in bbr_timeout_rxt()
5096 tp->t_maxseg = V_tcp_v6mssdflt; in bbr_timeout_rxt()
5101 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_timeout_rxt()
5109 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) { in bbr_timeout_rxt()
5111 tp->t_maxseg = V_tcp_pmtud_blackhole_mss; in bbr_timeout_rxt()
5115 tp->t_maxseg = V_tcp_mssdflt; in bbr_timeout_rxt()
5120 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_timeout_rxt()
5133 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) && in bbr_timeout_rxt()
5134 (tp->t_rxtshift >= 6)) { in bbr_timeout_rxt()
5135 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in bbr_timeout_rxt()
5136 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE; in bbr_timeout_rxt()
5137 tp->t_maxseg = tp->t_pmtud_saved_maxseg; in bbr_timeout_rxt()
5138 if (tp->t_maxseg < V_tcp_mssdflt) { in bbr_timeout_rxt()
5144 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT; in bbr_timeout_rxt()
5146 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT; in bbr_timeout_rxt()
5154 * third SYN to work-around some broken terminal servers (most of in bbr_timeout_rxt()
5157 * unknown-to-them TCP options. in bbr_timeout_rxt()
5159 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) && in bbr_timeout_rxt()
5160 (tp->t_rxtshift == 3)) in bbr_timeout_rxt()
5161 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT); in bbr_timeout_rxt()
5168 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) { in bbr_timeout_rxt()
5170 if (bbr->r_is_v6) in bbr_timeout_rxt()
5175 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT); in bbr_timeout_rxt()
5176 tp->t_srtt = 0; in bbr_timeout_rxt()
5178 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_timeout_rxt()
5179 tp->snd_recover = tp->snd_max; in bbr_timeout_rxt()
5180 tp->t_flags |= TF_ACKNOW; in bbr_timeout_rxt()
5181 tp->t_rtttime = 0; in bbr_timeout_rxt()
5190 int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK); in bbr_process_timers()
5195 if (tp->t_state == TCPS_LISTEN) { in bbr_process_timers()
5197 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) in bbr_process_timers()
5201 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_process_timers()
5204 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { in bbr_process_timers()
5205 ret = -1; in bbr_process_timers()
5210 ret = -2; in bbr_process_timers()
5218 left = bbr->r_ctl.rc_timer_exp - cts; in bbr_process_timers()
5219 ret = -3; in bbr_process_timers()
5224 bbr->rc_tmr_stopped = 0; in bbr_process_timers()
5225 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK; in bbr_process_timers()
5231 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5234 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5237 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5249 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { in bbr_timer_cancel()
5252 if (tcp_in_hpts(bbr->rc_tp) && in bbr_timer_cancel()
5253 (bbr->rc_timer_first == 1)) { in bbr_timer_cancel()
5260 tcp_hpts_remove(bbr->rc_tp); in bbr_timer_cancel()
5261 if (bbr->r_ctl.rc_last_delay_val) { in bbr_timer_cancel()
5265 if (TSTMP_GT(cts, bbr->rc_pacer_started)) in bbr_timer_cancel()
5266 time_since_send = cts - bbr->rc_pacer_started; in bbr_timer_cancel()
5269 if (bbr->r_ctl.rc_last_delay_val > time_since_send) { in bbr_timer_cancel()
5271 bbr->r_ctl.rc_last_delay_val -= time_since_send; in bbr_timer_cancel()
5273 bbr->r_ctl.rc_last_delay_val = 0; in bbr_timer_cancel()
5275 bbr->rc_pacer_started = cts; in bbr_timer_cancel()
5278 bbr->rc_timer_first = 0; in bbr_timer_cancel()
5280 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK; in bbr_timer_cancel()
5281 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK); in bbr_timer_cancel()
5290 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_stopall()
5291 bbr->rc_all_timers_stopped = 1; in bbr_stopall()
5303 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_get_earliest_send_outstanding()
5306 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]); in bbr_get_earliest_send_outstanding()
5315 rsm->r_rtr_cnt++; in bbr_update_rsm()
5316 rsm->r_dupack = 0; in bbr_update_rsm()
5317 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) { in bbr_update_rsm()
5318 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS; in bbr_update_rsm()
5319 rsm->r_flags |= BBR_OVERMAX; in bbr_update_rsm()
5321 if (rsm->r_flags & BBR_RWND_COLLAPSED) { in bbr_update_rsm()
5323 rsm->r_flags &= ~BBR_RWND_COLLAPSED; in bbr_update_rsm()
5325 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_update_rsm()
5327 rsm->r_flags &= ~BBR_MARKED_LOST; in bbr_update_rsm()
5328 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_update_rsm()
5330 if (rsm->r_flags & BBR_RXT_CLEARED) { in bbr_update_rsm()
5339 rsm->r_flags &= ~BBR_RXT_CLEARED; in bbr_update_rsm()
5341 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) { in bbr_update_rsm()
5342 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start); in bbr_update_rsm()
5343 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start); in bbr_update_rsm()
5345 idx = rsm->r_rtr_cnt - 1; in bbr_update_rsm()
5346 rsm->r_tim_lastsent[idx] = cts; in bbr_update_rsm()
5347 rsm->r_pacing_delay = pacing_time; in bbr_update_rsm()
5348 rsm->r_delivered = bbr->r_ctl.rc_delivered; in bbr_update_rsm()
5349 rsm->r_ts_valid = bbr->rc_ts_valid; in bbr_update_rsm()
5350 if (bbr->rc_ts_valid) in bbr_update_rsm()
5351 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts; in bbr_update_rsm()
5352 if (bbr->r_ctl.r_app_limited_until) in bbr_update_rsm()
5353 rsm->r_app_limited = 1; in bbr_update_rsm()
5355 rsm->r_app_limited = 0; in bbr_update_rsm()
5356 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) in bbr_update_rsm()
5357 rsm->r_bbr_state = bbr_state_val(bbr); in bbr_update_rsm()
5359 rsm->r_bbr_state = 8; in bbr_update_rsm()
5360 if (rsm->r_flags & BBR_ACKED) { in bbr_update_rsm()
5364 old_flags = rsm->r_flags; in bbr_update_rsm()
5365 rsm->r_flags &= ~BBR_ACKED; in bbr_update_rsm()
5367 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); in bbr_update_rsm()
5368 if (bbr->r_ctl.rc_sacked == 0) in bbr_update_rsm()
5369 bbr->r_ctl.rc_sacklast = NULL; in bbr_update_rsm()
5371 if (rsm->r_in_tmap) { in bbr_update_rsm()
5372 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_update_rsm()
5374 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_update_rsm()
5375 rsm->r_in_tmap = 1; in bbr_update_rsm()
5376 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_update_rsm()
5378 rsm->r_flags &= ~BBR_SACK_PASSED; in bbr_update_rsm()
5379 rsm->r_flags |= BBR_WAS_SACKPASS; in bbr_update_rsm()
5381 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); in bbr_update_rsm()
5382 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp, in bbr_update_rsm()
5383 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_update_rsm()
5384 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next); in bbr_update_rsm()
5385 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) { in bbr_update_rsm()
5386 rsm->r_is_gain = 1; in bbr_update_rsm()
5387 rsm->r_is_drain = 0; in bbr_update_rsm()
5388 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) { in bbr_update_rsm()
5389 rsm->r_is_drain = 1; in bbr_update_rsm()
5390 rsm->r_is_gain = 0; in bbr_update_rsm()
5392 rsm->r_is_drain = 0; in bbr_update_rsm()
5393 rsm->r_is_gain = 0; in bbr_update_rsm()
5395 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */ in bbr_update_rsm()
5409 * We (re-)transmitted starting at rsm->r_start for some length in bbr_update_entry()
5417 c_end = rsm->r_start + len; in bbr_update_entry()
5418 if (SEQ_GEQ(c_end, rsm->r_end)) { in bbr_update_entry()
5424 if (c_end == rsm->r_end) { in bbr_update_entry()
5431 act_len = rsm->r_end - rsm->r_start; in bbr_update_entry()
5432 *lenp = (len - act_len); in bbr_update_entry()
5433 return (rsm->r_end); in bbr_update_entry()
5454 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_update_entry()
5455 nrsm->r_dupack = 0; in bbr_update_entry()
5456 if (rsm->r_in_tmap) { in bbr_update_entry()
5457 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_update_entry()
5458 nrsm->r_in_tmap = 1; in bbr_update_entry()
5460 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_update_entry()
5485 if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) { in bbr_setup_less_of_rate()
5489 bbr->skip_gain = 1; in bbr_setup_less_of_rate()
5490 bbr->gain_is_limited = 0; in bbr_setup_less_of_rate()
5491 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate; in bbr_setup_less_of_rate()
5493 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts); in bbr_setup_less_of_rate()
5496 bbr->skip_gain = 0; in bbr_setup_less_of_rate()
5497 bbr->gain_is_limited = 1; in bbr_setup_less_of_rate()
5505 int error, rate = -1; in bbr_update_hardware_pacing_rate()
5507 if (bbr->r_ctl.crte == NULL) in bbr_update_hardware_pacing_rate()
5509 if ((bbr->rc_inp->inp_route.ro_nh == NULL) || in bbr_update_hardware_pacing_rate()
5510 (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) { in bbr_update_hardware_pacing_rate()
5512 /* Clear the way for a re-attempt */ in bbr_update_hardware_pacing_rate()
5513 bbr->bbr_attempt_hdwr_pace = 0; in bbr_update_hardware_pacing_rate()
5515 bbr->gain_is_limited = 0; in bbr_update_hardware_pacing_rate()
5516 bbr->skip_gain = 0; in bbr_update_hardware_pacing_rate()
5517 bbr->bbr_hdrw_pacing = 0; in bbr_update_hardware_pacing_rate()
5518 counter_u64_add(bbr_flows_whdwr_pacing, -1); in bbr_update_hardware_pacing_rate()
5524 nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte, in bbr_update_hardware_pacing_rate()
5525 bbr->rc_tp, in bbr_update_hardware_pacing_rate()
5526 bbr->rc_inp->inp_route.ro_nh->nh_ifp, in bbr_update_hardware_pacing_rate()
5533 if (nrte != bbr->r_ctl.crte) { in bbr_update_hardware_pacing_rate()
5534 bbr->r_ctl.crte = nrte; in bbr_update_hardware_pacing_rate()
5537 if (bbr->r_ctl.crte->rate < rate) { in bbr_update_hardware_pacing_rate()
5540 bbr->r_ctl.crte->rate, rate); in bbr_update_hardware_pacing_rate()
5543 bbr->gain_is_limited = 0; in bbr_update_hardware_pacing_rate()
5544 bbr->skip_gain = 0; in bbr_update_hardware_pacing_rate()
5549 bbr->gain_is_limited = 0; in bbr_update_hardware_pacing_rate()
5550 bbr->skip_gain = 0; in bbr_update_hardware_pacing_rate()
5551 bbr->bbr_hdrw_pacing = 0; in bbr_update_hardware_pacing_rate()
5554 bbr->r_ctl.crte->ptbl->rs_ifp, in bbr_update_hardware_pacing_rate()
5556 ((bbr->r_ctl.crte == NULL) ? 0 : bbr->r_ctl.crte->rate), in bbr_update_hardware_pacing_rate()
5574 if ((bbr->bbr_hdrw_pacing == 0) || in bbr_adjust_for_hw_pacing()
5575 (IN_RECOVERY(bbr->rc_tp->t_flags)) || in bbr_adjust_for_hw_pacing()
5576 (bbr->r_ctl.crte == NULL)) in bbr_adjust_for_hw_pacing()
5578 if (bbr->hw_pacing_set == 0) { in bbr_adjust_for_hw_pacing()
5586 rlp = bbr->r_ctl.crte; in bbr_adjust_for_hw_pacing()
5587 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) in bbr_adjust_for_hw_pacing()
5588 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in bbr_adjust_for_hw_pacing()
5590 maxseg = BBR_MIN_SEG - bbr->rc_last_options; in bbr_adjust_for_hw_pacing()
5598 bbr->r_ctl.rc_pace_max_segs, cts, 1); in bbr_adjust_for_hw_pacing()
5599 hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg; in bbr_adjust_for_hw_pacing()
5600 hdwr_delay *= rlp->time_between; in bbr_adjust_for_hw_pacing()
5602 delta = cur_delay - hdwr_delay; in bbr_adjust_for_hw_pacing()
5606 (bbr->r_ctl.rc_pace_max_segs / maxseg), in bbr_adjust_for_hw_pacing()
5609 (delta < (max(rlp->time_between, in bbr_adjust_for_hw_pacing()
5610 bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) { in bbr_adjust_for_hw_pacing()
5624 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between), in bbr_adjust_for_hw_pacing()
5625 (bbr->r_ctl.rc_pace_max_segs/maxseg)); in bbr_adjust_for_hw_pacing()
5628 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) { in bbr_adjust_for_hw_pacing()
5634 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg; in bbr_adjust_for_hw_pacing()
5648 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust; in bbr_adjust_for_hw_pacing()
5650 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) { in bbr_adjust_for_hw_pacing()
5656 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg; in bbr_adjust_for_hw_pacing()
5670 seg_sz = bbr->r_ctl.rc_pace_max_segs; in bbr_adjust_for_hw_pacing()
5672 if (seg_sz > bbr->r_ctl.rc_pace_max_segs) in bbr_adjust_for_hw_pacing()
5675 new_tso = bbr->r_ctl.rc_pace_max_segs; in bbr_adjust_for_hw_pacing()
5676 if (new_tso >= (PACE_MAX_IP_BYTES-maxseg)) in bbr_adjust_for_hw_pacing()
5677 new_tso = PACE_MAX_IP_BYTES - maxseg; in bbr_adjust_for_hw_pacing()
5679 if (new_tso != bbr->r_ctl.rc_pace_max_segs) { in bbr_adjust_for_hw_pacing()
5680 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0); in bbr_adjust_for_hw_pacing()
5681 bbr->r_ctl.rc_pace_max_segs = new_tso; in bbr_adjust_for_hw_pacing()
5708 * a full iwnd just like new-reno/cubic. in tcp_bbr_tso_size_check()
5713 * if ( bw <= per-tcb-cross-over) in tcp_bbr_tso_size_check()
5715 * can send in goal-time seconds. in tcp_bbr_tso_size_check()
5721 * if (tso > per-tcb-max) in tcp_bbr_tso_size_check()
5722 * tso = per-tcb-max in tcp_bbr_tso_size_check()
5724 * tso = max-tso (64k/mss) in tcp_bbr_tso_size_check()
5726 * goal_tso = bw / per-tcb-divsor in tcp_bbr_tso_size_check()
5727 * seg = (goal_tso + mss-1)/mss in tcp_bbr_tso_size_check()
5730 * if (tso < per-tcb-floor) in tcp_bbr_tso_size_check()
5731 * tso = per-tcb-floor in tcp_bbr_tso_size_check()
5732 * if (tso > per-tcb-utter_max) in tcp_bbr_tso_size_check()
5733 * tso = per-tcb-utter_max in tcp_bbr_tso_size_check()
5735 * Note the default per-tcb-divisor is 1000 (same as google). in tcp_bbr_tso_size_check()
5739 * cross-over = 23,168,000 bps in tcp_bbr_tso_size_check()
5740 * goal-time = 18000 in tcp_bbr_tso_size_check()
5741 * per-tcb-max = 2 in tcp_bbr_tso_size_check()
5742 * per-tcb-divisor = 1000 in tcp_bbr_tso_size_check()
5743 * per-tcb-floor = 1 in tcp_bbr_tso_size_check()
5752 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) { in tcp_bbr_tso_size_check()
5753 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in tcp_bbr_tso_size_check()
5755 maxseg = BBR_MIN_SEG - bbr->rc_last_options; in tcp_bbr_tso_size_check()
5757 old_tso = bbr->r_ctl.rc_pace_max_segs; in tcp_bbr_tso_size_check()
5758 if (bbr->rc_past_init_win == 0) { in tcp_bbr_tso_size_check()
5764 if (bbr->rc_use_google) in tcp_bbr_tso_size_check()
5765 bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2); in tcp_bbr_tso_size_check()
5766 else if (bbr->bbr_init_win_cheat) in tcp_bbr_tso_size_check()
5767 bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp); in tcp_bbr_tso_size_check()
5769 bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in tcp_bbr_tso_size_check()
5770 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg) in tcp_bbr_tso_size_check()
5771 bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg; in tcp_bbr_tso_size_check()
5772 if (bbr->r_ctl.rc_pace_max_segs == 0) { in tcp_bbr_tso_size_check()
5773 bbr->r_ctl.rc_pace_max_segs = maxseg; in tcp_bbr_tso_size_check()
5775 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0); in tcp_bbr_tso_size_check()
5785 if (IN_RECOVERY(bbr->rc_tp->t_flags) && in tcp_bbr_tso_size_check()
5786 (bbr->rc_use_google == 0)) { in tcp_bbr_tso_size_check()
5789 } else if (bbr->rc_use_google) { in tcp_bbr_tso_size_check()
5793 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) { in tcp_bbr_tso_size_check()
5794 bw *= bbr->r_ctl.rc_bbr_hptsi_gain; in tcp_bbr_tso_size_check()
5808 } else if (bbr->rc_no_pacing) { in tcp_bbr_tso_size_check()
5810 } else if (bw <= bbr->r_ctl.bbr_cross_over) { in tcp_bbr_tso_size_check()
5817 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw); in tcp_bbr_tso_size_check()
5820 if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max) in tcp_bbr_tso_size_check()
5821 new_tso = bbr->r_ctl.bbr_hptsi_segments_max; in tcp_bbr_tso_size_check()
5854 bw /= bbr->r_ctl.bbr_hptsi_per_second; in tcp_bbr_tso_size_check()
5861 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg)) in tcp_bbr_tso_size_check()
5862 new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg); in tcp_bbr_tso_size_check()
5864 …if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floo… in tcp_bbr_tso_size_check()
5865 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor; in tcp_bbr_tso_size_check()
5869 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) { in tcp_bbr_tso_size_check()
5870 new_tso = bbr->r_ctl.bbr_utter_max * maxseg; in tcp_bbr_tso_size_check()
5875 bbr->r_ctl.rc_pace_max_segs = new_tso; in tcp_bbr_tso_size_check()
5910 * We don't log errors -- we could but snd_max does not in bbr_log_output()
5923 snd_una = tp->snd_una; in bbr_log_output()
5931 if ((th_flags & TH_SYN) && (tp->iss == seq_out)) in bbr_log_output()
5937 /* Are sending an old segment to induce an ack (keep-alive)? */ in bbr_log_output()
5946 len = end - seq_out; in bbr_log_output()
5948 snd_max = tp->snd_max; in bbr_log_output()
5953 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1); in bbr_log_output()
5961 rsm->r_flags = 0; in bbr_log_output()
5963 rsm->r_flags |= BBR_HAS_SYN; in bbr_log_output()
5965 rsm->r_flags |= BBR_HAS_FIN; in bbr_log_output()
5966 rsm->r_tim_lastsent[0] = cts; in bbr_log_output()
5967 rsm->r_rtr_cnt = 1; in bbr_log_output()
5968 rsm->r_rtr_bytes = 0; in bbr_log_output()
5969 rsm->r_start = seq_out; in bbr_log_output()
5970 rsm->r_end = rsm->r_start + len; in bbr_log_output()
5971 rsm->r_dupack = 0; in bbr_log_output()
5972 rsm->r_delivered = bbr->r_ctl.rc_delivered; in bbr_log_output()
5973 rsm->r_pacing_delay = pacing_time; in bbr_log_output()
5974 rsm->r_ts_valid = bbr->rc_ts_valid; in bbr_log_output()
5975 if (bbr->rc_ts_valid) in bbr_log_output()
5976 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts; in bbr_log_output()
5977 rsm->r_del_time = bbr->r_ctl.rc_del_time; in bbr_log_output()
5978 if (bbr->r_ctl.r_app_limited_until) in bbr_log_output()
5979 rsm->r_app_limited = 1; in bbr_log_output()
5981 rsm->r_app_limited = 0; in bbr_log_output()
5982 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); in bbr_log_output()
5983 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp, in bbr_log_output()
5984 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_log_output()
5989 rsm->r_flight_at_send += len; in bbr_log_output()
5990 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_log_output()
5991 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_log_output()
5992 rsm->r_in_tmap = 1; in bbr_log_output()
5993 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) in bbr_log_output()
5994 rsm->r_bbr_state = bbr_state_val(bbr); in bbr_log_output()
5996 rsm->r_bbr_state = 8; in bbr_log_output()
5997 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) { in bbr_log_output()
5998 rsm->r_is_gain = 1; in bbr_log_output()
5999 rsm->r_is_drain = 0; in bbr_log_output()
6000 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) { in bbr_log_output()
6001 rsm->r_is_drain = 1; in bbr_log_output()
6002 rsm->r_is_gain = 0; in bbr_log_output()
6004 rsm->r_is_drain = 0; in bbr_log_output()
6005 rsm->r_is_gain = 0; in bbr_log_output()
6013 if (hintrsm && (hintrsm->r_start == seq_out)) { in bbr_log_output()
6016 } else if (bbr->r_ctl.rc_next) { in bbr_log_output()
6018 rsm = bbr->r_ctl.rc_next; in bbr_log_output()
6023 if ((rsm) && (rsm->r_start == seq_out)) { in bbr_log_output()
6036 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_log_output()
6037 if (rsm->r_start == seq_out) { in bbr_log_output()
6039 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next); in bbr_log_output()
6046 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) { in bbr_log_output()
6062 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_log_output()
6063 if (rsm->r_in_tmap) { in bbr_log_output()
6064 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_log_output()
6065 nrsm->r_in_tmap = 1; in bbr_log_output()
6067 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_log_output()
6078 if (seq_out == tp->snd_max) { in bbr_log_output()
6080 } else if (SEQ_LT(seq_out, tp->snd_max)) { in bbr_log_output()
6082 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n", in bbr_log_output()
6083 seq_out, len, tp->snd_una, tp->snd_max); in bbr_log_output()
6085 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_log_output()
6087 rsm, rsm->r_start, rsm->r_end); in bbr_log_output()
6096 * Hmm beyond sndmax? (only if we are using the new rtt-pack in bbr_log_output()
6100 seq_out, len, tp->snd_max, tp); in bbr_log_output()
6109 * Collapse timeout back the cum-ack moved. in bbr_collapse_rtt()
6111 tp->t_rxtshift = 0; in bbr_collapse_rtt()
6112 tp->t_softerror = 0; in bbr_collapse_rtt()
6118 bbr->rtt_valid = 1; in tcp_bbr_xmit_timer()
6119 bbr->r_ctl.cur_rtt = rtt_usecs; in tcp_bbr_xmit_timer()
6120 bbr->r_ctl.ts_in = tsin; in tcp_bbr_xmit_timer()
6122 bbr->r_ctl.cur_rtt_send_time = rsm_send_time; in tcp_bbr_xmit_timer()
6130 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp). in bbr_make_timestamp_determination()
6131 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts). in bbr_make_timestamp_determination()
6132 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts) in bbr_make_timestamp_determination()
6133 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time) in bbr_make_timestamp_determination()
6137 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts in bbr_make_timestamp_determination()
6141 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp in bbr_make_timestamp_determination()
6148 * out time-slices of the actual b/w. This would mean we could not use in bbr_make_timestamp_determination()
6152 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio. in bbr_make_timestamp_determination()
6166 delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts; in bbr_make_timestamp_determination()
6176 peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp; in bbr_make_timestamp_determination()
6195 * then we saw then in micro-seconds. in bbr_make_timestamp_determination()
6198 /* well it looks like the peer is a micro-second clock. */ in bbr_make_timestamp_determination()
6199 bbr->rc_ts_clock_set = 1; in bbr_make_timestamp_determination()
6200 bbr->r_ctl.bbr_peer_tsratio = 1; in bbr_make_timestamp_determination()
6202 bbr->rc_ts_cant_be_used = 1; in bbr_make_timestamp_determination()
6203 bbr->rc_ts_clock_set = 1; in bbr_make_timestamp_determination()
6208 bbr->rc_ts_clock_set = 1; in bbr_make_timestamp_determination()
6213 bbr->r_ctl.bbr_peer_tsratio = 1; in bbr_make_timestamp_determination()
6220 bbr->r_ctl.bbr_peer_tsratio = 10; in bbr_make_timestamp_determination()
6227 bbr->r_ctl.bbr_peer_tsratio = 100; in bbr_make_timestamp_determination()
6234 bbr->r_ctl.bbr_peer_tsratio = 1000; in bbr_make_timestamp_determination()
6241 bbr->r_ctl.bbr_peer_tsratio = 10000; in bbr_make_timestamp_determination()
6245 bbr->rc_ts_cant_be_used = 1; in bbr_make_timestamp_determination()
6246 bbr->r_ctl.bbr_peer_tsratio = 0; in bbr_make_timestamp_determination()
6251 * Collect new round-trip time estimate
6261 if (bbr->rtt_valid == 0) in tcp_bbr_xmit_timer_commit()
6265 rtt = bbr->r_ctl.cur_rtt; in tcp_bbr_xmit_timer_commit()
6266 tsin = bbr->r_ctl.ts_in; in tcp_bbr_xmit_timer_commit()
6267 if (bbr->rc_prtt_set_ts) { in tcp_bbr_xmit_timer_commit()
6276 * and when they enter probe-rtt they update the in tcp_bbr_xmit_timer_commit()
6281 bbr->rc_prtt_set_ts = 0; in tcp_bbr_xmit_timer_commit()
6282 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop); in tcp_bbr_xmit_timer_commit()
6284 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts); in tcp_bbr_xmit_timer_commit()
6286 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in tcp_bbr_xmit_timer_commit()
6289 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt)); in tcp_bbr_xmit_timer_commit()
6291 if (bbr->rc_ack_was_delayed) in tcp_bbr_xmit_timer_commit()
6292 rtt += bbr->r_ctl.rc_ack_hdwr_delay; in tcp_bbr_xmit_timer_commit()
6294 if (rtt < bbr->r_ctl.rc_lowest_rtt) in tcp_bbr_xmit_timer_commit()
6295 bbr->r_ctl.rc_lowest_rtt = rtt; in tcp_bbr_xmit_timer_commit()
6297 if (bbr->r_init_rtt) { in tcp_bbr_xmit_timer_commit()
6299 * The initial rtt is not-trusted, nuke it and lets get in tcp_bbr_xmit_timer_commit()
6302 bbr->r_init_rtt = 0; in tcp_bbr_xmit_timer_commit()
6303 tp->t_srtt = 0; in tcp_bbr_xmit_timer_commit()
6305 if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) { in tcp_bbr_xmit_timer_commit()
6310 * series of cum-ack's to determine in tcp_bbr_xmit_timer_commit()
6313 if (bbr->rc_ack_is_cumack) { in tcp_bbr_xmit_timer_commit()
6314 if (bbr->rc_ts_data_set) { in tcp_bbr_xmit_timer_commit()
6318 bbr->rc_ts_data_set = 1; in tcp_bbr_xmit_timer_commit()
6319 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts; in tcp_bbr_xmit_timer_commit()
6320 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time; in tcp_bbr_xmit_timer_commit()
6327 bbr->rc_ts_data_set = 0; in tcp_bbr_xmit_timer_commit()
6331 rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1))); in tcp_bbr_xmit_timer_commit()
6334 if (tp->t_srtt != 0) { in tcp_bbr_xmit_timer_commit()
6343 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT) in tcp_bbr_xmit_timer_commit()
6344 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT)); in tcp_bbr_xmit_timer_commit()
6346 tp->t_srtt += delta; in tcp_bbr_xmit_timer_commit()
6347 if (tp->t_srtt <= 0) in tcp_bbr_xmit_timer_commit()
6348 tp->t_srtt = 1; in tcp_bbr_xmit_timer_commit()
6358 * wired-in beta. in tcp_bbr_xmit_timer_commit()
6361 delta = -delta; in tcp_bbr_xmit_timer_commit()
6362 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT); in tcp_bbr_xmit_timer_commit()
6363 tp->t_rttvar += delta; in tcp_bbr_xmit_timer_commit()
6364 if (tp->t_rttvar <= 0) in tcp_bbr_xmit_timer_commit()
6365 tp->t_rttvar = 1; in tcp_bbr_xmit_timer_commit()
6368 * No rtt measurement yet - use the unsmoothed rtt. Set the in tcp_bbr_xmit_timer_commit()
6372 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT; in tcp_bbr_xmit_timer_commit()
6373 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1); in tcp_bbr_xmit_timer_commit()
6376 if (tp->t_rttupdated < UCHAR_MAX) in tcp_bbr_xmit_timer_commit()
6377 tp->t_rttupdated++; in tcp_bbr_xmit_timer_commit()
6379 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks)); in tcp_bbr_xmit_timer_commit()
6385 * tick of rounding and 1 extra tick because of +-1/2 tick in tcp_bbr_xmit_timer_commit()
6391 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp), in tcp_bbr_xmit_timer_commit()
6392 max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2), in tcp_bbr_xmit_timer_commit()
6393 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000)); in tcp_bbr_xmit_timer_commit()
6402 tp->t_softerror = 0; in tcp_bbr_xmit_timer_commit()
6403 rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); in tcp_bbr_xmit_timer_commit()
6404 if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt) in tcp_bbr_xmit_timer_commit()
6405 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt; in tcp_bbr_xmit_timer_commit()
6411 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_set_reduced_rtt()
6413 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && in bbr_set_reduced_rtt()
6414 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { in bbr_set_reduced_rtt()
6416 * We should enter probe-rtt its been too long in bbr_set_reduced_rtt()
6429 if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) { in tcp_bbr_commit_bw()
6435 if (bbr->r_ctl.r_measurement_count < 0xffffffff) in tcp_bbr_commit_bw()
6436 bbr->r_ctl.r_measurement_count++; in tcp_bbr_commit_bw()
6437 orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate); in tcp_bbr_commit_bw()
6438 apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch); in tcp_bbr_commit_bw()
6440 (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate), in tcp_bbr_commit_bw()
6443 (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) { in tcp_bbr_commit_bw()
6444 if (bbr->bbr_hdrw_pacing) { in tcp_bbr_commit_bw()
6453 if (bbr->r_recovery_bw) { in tcp_bbr_commit_bw()
6457 } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate)) in tcp_bbr_commit_bw()
6464 if (bbr->rc_in_persist == 0) { in bbr_nf_measurement()
6470 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time)) in bbr_nf_measurement()
6471 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time); in bbr_nf_measurement()
6477 * calculate the cdr. in bbr_nf_measurement()
6479 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered); in bbr_nf_measurement()
6489 * can see in the trace viewer if it gets over-ridden. in bbr_nf_measurement()
6491 if (rsm->r_ts_valid && in bbr_nf_measurement()
6492 bbr->rc_ts_valid && in bbr_nf_measurement()
6493 bbr->rc_ts_clock_set && in bbr_nf_measurement()
6494 (bbr->rc_ts_cant_be_used == 0) && in bbr_nf_measurement()
6495 bbr->rc_use_ts_limit) { in bbr_nf_measurement()
6496 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1); in bbr_nf_measurement()
6497 ts_diff *= bbr->r_ctl.bbr_peer_tsratio; in bbr_nf_measurement()
6503 bbr->r_ctl.last_inbound_ts, in bbr_nf_measurement()
6504 rsm->r_del_ack_ts, 0, in bbr_nf_measurement()
6514 if ((bbr->ts_can_raise) && in bbr_nf_measurement()
6534 if (rsm->r_first_sent_time && in bbr_nf_measurement()
6535 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) { in bbr_nf_measurement()
6548 sbw = (uint64_t)(rsm->r_flight_at_send); in bbr_nf_measurement()
6550 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time; in bbr_nf_measurement()
6551 sti += rsm->r_pacing_delay; in bbr_nf_measurement()
6559 rsm->r_first_sent_time, 0, (sbw >> 32), in bbr_nf_measurement()
6565 bbr->r_ctl.rc_bbr_cur_del_rate = bw; in bbr_nf_measurement()
6566 if ((rsm->r_app_limited == 0) || in bbr_nf_measurement()
6567 (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) { in bbr_nf_measurement()
6570 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time); in bbr_nf_measurement()
6578 if (bbr->rc_in_persist == 0) { in bbr_google_measurement()
6585 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time)) in bbr_google_measurement()
6586 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time); in bbr_google_measurement()
6592 * calculate the cdr. in bbr_google_measurement()
6594 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered); in bbr_google_measurement()
6598 if (tim < bbr->r_ctl.rc_lowest_rtt) { in bbr_google_measurement()
6600 tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0); in bbr_google_measurement()
6606 * can see in the trace viewer if it gets over-ridden. in bbr_google_measurement()
6608 bbr->r_ctl.rc_bbr_cur_del_rate = bw; in bbr_google_measurement()
6610 if (rsm->r_first_sent_time && in bbr_google_measurement()
6611 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) { in bbr_google_measurement()
6624 sbw = (uint64_t)(rsm->r_flight_at_send); in bbr_google_measurement()
6626 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time; in bbr_google_measurement()
6627 sti += rsm->r_pacing_delay; in bbr_google_measurement()
6635 rsm->r_first_sent_time, 0, (sbw >> 32), in bbr_google_measurement()
6640 (sti < bbr->r_ctl.rc_lowest_rtt)) { in bbr_google_measurement()
6642 (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0); in bbr_google_measurement()
6647 bbr->r_ctl.rc_bbr_cur_del_rate = bw; in bbr_google_measurement()
6649 ((rsm->r_app_limited == 0) || in bbr_google_measurement()
6650 (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) { in bbr_google_measurement()
6653 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time); in bbr_google_measurement()
6665 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start); in bbr_update_bbr_info()
6666 bbr->r_ctl.rc_del_time = cts; in bbr_update_bbr_info()
6674 if ((bbr->rc_use_google == 0) && in bbr_update_bbr_info()
6687 if ((bbr_no_retran && bbr->rc_use_google) && in bbr_update_bbr_info()
6691 * We only do measurements in google mode in bbr_update_bbr_info()
6697 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin); in bbr_update_bbr_info()
6699 bbr->rc_ack_is_cumack = 1; in bbr_update_bbr_info()
6701 bbr->rc_ack_is_cumack = 0; in bbr_update_bbr_info()
6708 * going into probe-rtt (we were seeing cases where that in bbr_update_bbr_info()
6717 bbr_log_type_bbrrttprop(bbr, rtt, (rsm ? rsm->r_end : 0), uts, cts, in bbr_update_bbr_info()
6718 match, rsm->r_start, rsm->r_flags); in bbr_update_bbr_info()
6719 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_update_bbr_info()
6722 * The RTT-prop moved, reset the target (may be a in bbr_update_bbr_info()
6726 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) in bbr_update_bbr_info()
6733 if ((bbr->rc_use_google == 0) && in bbr_update_bbr_info()
6742 if (bbr->r_ctl.r_app_limited_until && in bbr_update_bbr_info()
6743 (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) { in bbr_update_bbr_info()
6744 /* We are no longer app-limited */ in bbr_update_bbr_info()
6745 bbr->r_ctl.r_app_limited_until = 0; in bbr_update_bbr_info()
6747 if (bbr->rc_use_google) { in bbr_update_bbr_info()
6764 msec = cts - (MS_IN_USEC * sec); in bbr_ts_convert()
6779 if ((rsm->r_flags & BBR_ACKED) || in bbr_update_rtt()
6780 (rsm->r_flags & BBR_WAS_RENEGED) || in bbr_update_rtt()
6781 (rsm->r_flags & BBR_RXT_CLEARED)) { in bbr_update_rtt()
6785 if (rsm->r_rtt_not_allowed) { in bbr_update_rtt()
6789 if (rsm->r_rtr_cnt == 1) { in bbr_update_rtt()
6793 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0])) in bbr_update_rtt()
6794 t = cts - rsm->r_tim_lastsent[0]; in bbr_update_rtt()
6799 bbr->r_ctl.rc_last_rtt = t; in bbr_update_rtt()
6800 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, in bbr_update_rtt()
6801 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to); in bbr_update_rtt()
6806 (bbr->rc_use_google == 1) && in bbr_update_rtt()
6808 (to->to_flags & TOF_TS) && in bbr_update_rtt()
6809 (to->to_tsecr != 0)) { in bbr_update_rtt()
6810 t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr; in bbr_update_rtt()
6814 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, in bbr_update_rtt()
6816 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)], in bbr_update_rtt()
6820 uts = bbr_ts_convert(to->to_tsecr); in bbr_update_rtt()
6821 if ((to->to_flags & TOF_TS) && in bbr_update_rtt()
6822 (to->to_tsecr != 0) && in bbr_update_rtt()
6824 ((rsm->r_flags & BBR_OVERMAX) == 0)) { in bbr_update_rtt()
6832 for (i = 0; i < rsm->r_rtr_cnt; i++) { in bbr_update_rtt()
6833 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) && in bbr_update_rtt()
6834 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) { in bbr_update_rtt()
6835 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6836 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6841 bbr->r_ctl.rc_last_rtt = t; in bbr_update_rtt()
6842 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING, in bbr_update_rtt()
6843 rsm->r_tim_lastsent[i], ack_type, to); in bbr_update_rtt()
6844 if ((i + 1) < rsm->r_rtr_cnt) { in bbr_update_rtt()
6847 } else if (rsm->r_flags & BBR_TLP) { in bbr_update_rtt()
6848 bbr->rc_tlp_rtx_out = 0; in bbr_update_rtt()
6858 * time-stamp since its not there or the time the peer last in bbr_update_rtt()
6859 * received a segment that moved forward its cum-ack point. in bbr_update_rtt()
6865 i = rsm->r_rtr_cnt - 1; in bbr_update_rtt()
6866 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6867 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6870 if (t < bbr->r_ctl.rc_lowest_rtt) { in bbr_update_rtt()
6876 * the rack-draft. in bbr_update_rtt()
6882 if ((rsm->r_flags & BBR_OVERMAX) == 0) { in bbr_update_rtt()
6884 if (rsm->r_rtr_cnt == 1) in bbr_update_rtt()
6885 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags); in bbr_update_rtt()
6887 i = rsm->r_rtr_cnt - 2; in bbr_update_rtt()
6888 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6889 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6892 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET, in bbr_update_rtt()
6893 rsm->r_tim_lastsent[i], ack_type, to); in bbr_update_rtt()
6901 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, in bbr_update_rtt()
6909 if (rsm->r_flags & BBR_TLP) in bbr_update_rtt()
6910 bbr->rc_tlp_rtx_out = 0; in bbr_update_rtt()
6911 if ((rsm->r_flags & BBR_OVERMAX) == 0) in bbr_update_rtt()
6912 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, in bbr_update_rtt()
6915 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, in bbr_update_rtt()
6932 TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap, in bbr_log_sack_passed()
6938 if (nrsm->r_flags & BBR_ACKED) { in bbr_log_sack_passed()
6942 if (nrsm->r_flags & BBR_SACK_PASSED) { in bbr_log_sack_passed()
6951 nrsm->r_flags |= BBR_SACK_PASSED; in bbr_log_sack_passed()
6952 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) && in bbr_log_sack_passed()
6953 bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) { in bbr_log_sack_passed()
6954 bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start; in bbr_log_sack_passed()
6955 bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start; in bbr_log_sack_passed()
6956 nrsm->r_flags |= BBR_MARKED_LOST; in bbr_log_sack_passed()
6958 nrsm->r_flags &= ~BBR_WAS_SACKPASS; in bbr_log_sack_passed()
6976 start = sack->start; in bbr_proc_sack_blk()
6977 end = sack->end; in bbr_proc_sack_blk()
6983 if (rsm && SEQ_LT(start, rsm->r_start)) { in bbr_proc_sack_blk()
6985 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_proc_sack_blk()
6986 if (SEQ_GEQ(start, rsm->r_start) && in bbr_proc_sack_blk()
6987 SEQ_LT(start, rsm->r_end)) { in bbr_proc_sack_blk()
6998 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_proc_sack_blk()
6999 if (SEQ_GEQ(start, rsm->r_start) && in bbr_proc_sack_blk()
7000 SEQ_LT(start, rsm->r_end)) { in bbr_proc_sack_blk()
7012 if (tp->t_flags & TF_SENTFIN) { in bbr_proc_sack_blk()
7017 nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); in bbr_proc_sack_blk()
7018 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) { in bbr_proc_sack_blk()
7022 nrsm->r_end++; in bbr_proc_sack_blk()
7041 if (rsm->r_start != start) { in bbr_proc_sack_blk()
7055 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk); in bbr_proc_sack_blk()
7059 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_proc_sack_blk()
7060 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7061 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_proc_sack_blk()
7062 nrsm->r_in_tmap = 1; in bbr_proc_sack_blk()
7064 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_proc_sack_blk()
7067 if (SEQ_GEQ(end, rsm->r_end)) { in bbr_proc_sack_blk()
7072 if ((rsm->r_flags & BBR_ACKED) == 0) { in bbr_proc_sack_blk()
7074 changed += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7075 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7077 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7078 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7081 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_proc_sack_blk()
7083 bbr->r_ctl.rc_reorder_ts = cts; in bbr_proc_sack_blk()
7084 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7085 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7086 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_proc_sack_blk()
7088 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_proc_sack_blk()
7091 rsm->r_flags |= BBR_ACKED; in bbr_proc_sack_blk()
7092 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST); in bbr_proc_sack_blk()
7093 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7094 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_proc_sack_blk()
7095 rsm->r_in_tmap = 0; in bbr_proc_sack_blk()
7099 if (end == rsm->r_end) { in bbr_proc_sack_blk()
7100 /* This block only - done */ in bbr_proc_sack_blk()
7104 start = rsm->r_end; in bbr_proc_sack_blk()
7110 if (rsm->r_flags & BBR_ACKED) { in bbr_proc_sack_blk()
7125 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk); in bbr_proc_sack_blk()
7131 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_proc_sack_blk()
7132 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_proc_sack_blk()
7133 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7134 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_proc_sack_blk()
7135 nrsm->r_in_tmap = 1; in bbr_proc_sack_blk()
7137 nrsm->r_dupack = 0; in bbr_proc_sack_blk()
7140 changed += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7141 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7144 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7145 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7147 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_proc_sack_blk()
7149 bbr->r_ctl.rc_reorder_ts = cts; in bbr_proc_sack_blk()
7150 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7151 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7152 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_proc_sack_blk()
7154 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_proc_sack_blk()
7157 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST); in bbr_proc_sack_blk()
7158 rsm->r_flags |= BBR_ACKED; in bbr_proc_sack_blk()
7159 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7160 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_proc_sack_blk()
7161 rsm->r_in_tmap = 0; in bbr_proc_sack_blk()
7164 if (rsm && (rsm->r_flags & BBR_ACKED)) { in bbr_proc_sack_blk()
7172 (nrsm->r_flags & BBR_ACKED)) { in bbr_proc_sack_blk()
7179 (nrsm->r_flags & BBR_ACKED)) { in bbr_proc_sack_blk()
7198 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next); in bbr_proc_sack_blk()
7200 bbr->r_ctl.rc_sacklast = NULL; in bbr_proc_sack_blk()
7212 while (rsm && (rsm->r_flags & BBR_ACKED)) { in bbr_peer_reneges()
7215 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); in bbr_peer_reneges()
7217 if (rsm->r_in_tmap) { in bbr_peer_reneges()
7219 bbr, rsm, rsm->r_flags); in bbr_peer_reneges()
7222 oflags = rsm->r_flags; in bbr_peer_reneges()
7223 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_peer_reneges()
7224 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_peer_reneges()
7225 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_peer_reneges()
7226 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_peer_reneges()
7228 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_peer_reneges()
7230 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST); in bbr_peer_reneges()
7231 rsm->r_flags |= BBR_WAS_RENEGED; in bbr_peer_reneges()
7232 rsm->r_flags |= BBR_RXT_CLEARED; in bbr_peer_reneges()
7233 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__); in bbr_peer_reneges()
7236 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_peer_reneges()
7239 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext); in bbr_peer_reneges()
7242 tmap->r_in_tmap = 1; in bbr_peer_reneges()
7257 sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack); in bbr_peer_reneges()
7267 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_log_syn()
7268 cts = bbr->r_ctl.rc_rcvtime; in bbr_log_syn()
7269 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_syn()
7270 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) { in bbr_log_syn()
7271 if ((rsm->r_end - rsm->r_start) <= 1) { in bbr_log_syn()
7273 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; in bbr_log_syn()
7274 rsm->r_rtr_bytes = 0; in bbr_log_syn()
7275 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_log_syn()
7276 if (rsm->r_in_tmap) { in bbr_log_syn()
7277 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_log_syn()
7278 rsm->r_in_tmap = 0; in bbr_log_syn()
7280 if (bbr->r_ctl.rc_next == rsm) { in bbr_log_syn()
7282 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_syn()
7289 rsm->r_flags &= ~BBR_HAS_SYN; in bbr_log_syn()
7290 rsm->r_start++; in bbr_log_syn()
7318 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_log_ack()
7319 cts = bbr->r_ctl.rc_rcvtime; in bbr_log_ack()
7321 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7323 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_log_ack()
7324 p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg); in bbr_log_ack()
7325 th_ack = th->th_ack; in bbr_log_ack()
7326 if (SEQ_GT(th_ack, tp->snd_una)) { in bbr_log_ack()
7327 acked = th_ack - tp->snd_una; in bbr_log_ack()
7329 bbr->rc_tp->t_acktime = ticks; in bbr_log_ack()
7332 if (SEQ_LEQ(th_ack, tp->snd_una)) { in bbr_log_ack()
7336 if (rsm && SEQ_GT(th_ack, rsm->r_start)) { in bbr_log_ack()
7337 changed = th_ack - rsm->r_start; in bbr_log_ack()
7338 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) { in bbr_log_ack()
7344 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) { in bbr_log_ack()
7351 ts = bbr_ts_convert(to->to_tsecr); in bbr_log_ack()
7352 now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv)); in bbr_log_ack()
7353 rtt = now - ts; in bbr_log_ack()
7357 tp->iss, 0, cts, in bbr_log_ack()
7358 BBR_RTT_BY_TIMESTAMP, tp->iss, 0); in bbr_log_ack()
7359 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_log_ack()
7361 bbr->r_wanted_output = 1; in bbr_log_ack()
7374 bbr->r_wanted_output = 1; in bbr_log_ack()
7377 if (tp->t_flags & TF_SENTFIN) { in bbr_log_ack()
7384 th, tp->t_state, bbr, in bbr_log_ack()
7385 tp->snd_una, tp->snd_max, changed); in bbr_log_ack()
7390 if (SEQ_LT(th_ack, rsm->r_start)) { in bbr_log_ack()
7394 rsm->r_start, in bbr_log_ack()
7395 th_ack, tp->t_state, in bbr_log_ack()
7396 bbr->r_state, bbr); in bbr_log_ack()
7397 panic("th-ack is bad bbr:%p tp:%p", bbr, tp); in bbr_log_ack()
7400 } else if (th_ack == rsm->r_start) { in bbr_log_ack()
7409 rsm->r_dupack = 0; in bbr_log_ack()
7411 if (SEQ_GEQ(th_ack, rsm->r_end)) { in bbr_log_ack()
7415 if (rsm->r_flags & BBR_ACKED) { in bbr_log_ack()
7417 * It was acked on the scoreboard -- remove it from in bbr_log_ack()
7420 p_acked += (rsm->r_end - rsm->r_start); in bbr_log_ack()
7421 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); in bbr_log_ack()
7422 if (bbr->r_ctl.rc_sacked == 0) in bbr_log_ack()
7423 bbr->r_ctl.rc_sacklast = NULL; in bbr_log_ack()
7426 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_log_ack()
7427 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_log_ack()
7429 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_log_ack()
7436 bbr->r_ctl.rc_reorder_ts = cts; in bbr_log_ack()
7437 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_log_ack()
7438 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_log_ack()
7439 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_log_ack()
7441 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_log_ack()
7444 rsm->r_flags &= ~BBR_MARKED_LOST; in bbr_log_ack()
7446 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; in bbr_log_ack()
7447 rsm->r_rtr_bytes = 0; in bbr_log_ack()
7448 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_log_ack()
7449 if (rsm->r_in_tmap) { in bbr_log_ack()
7450 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_log_ack()
7451 rsm->r_in_tmap = 0; in bbr_log_ack()
7453 if (bbr->r_ctl.rc_next == rsm) { in bbr_log_ack()
7455 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7459 left = th_ack - rsm->r_end; in bbr_log_ack()
7463 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7468 if (rsm->r_flags & BBR_ACKED) { in bbr_log_ack()
7470 * It was acked on the scoreboard -- remove it from total in bbr_log_ack()
7471 * for the part being cum-acked. in bbr_log_ack()
7473 p_acked += (rsm->r_end - rsm->r_start); in bbr_log_ack()
7474 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start); in bbr_log_ack()
7475 if (bbr->r_ctl.rc_sacked == 0) in bbr_log_ack()
7476 bbr->r_ctl.rc_sacklast = NULL; in bbr_log_ack()
7487 if ((rsm->r_flags & BBR_MARKED_LOST) && in bbr_log_ack()
7488 ((rsm->r_flags & BBR_ACKED) == 0)) { in bbr_log_ack()
7494 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start; in bbr_log_ack()
7497 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; in bbr_log_ack()
7498 rsm->r_rtr_bytes = 0; in bbr_log_ack()
7500 rsm->r_start = th_ack; in bbr_log_ack()
7503 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7504 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) { in bbr_log_ack()
7510 * us snd_una up to (rsm->r_end). We need to undo the acked in bbr_log_ack()
7514 * rsm->r_start in case we get an old ack where th_ack is in bbr_log_ack()
7517 bbr_peer_reneges(bbr, rsm, th->th_ack); in bbr_log_ack()
7519 if ((to->to_flags & TOF_SACK) == 0) { in bbr_log_ack()
7523 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); in bbr_log_ack()
7525 last_seq = rsm->r_end; in bbr_log_ack()
7527 last_seq = tp->snd_max; in bbr_log_ack()
7530 if (SEQ_GT(th_ack, tp->snd_una)) in bbr_log_ack()
7533 ack_point = tp->snd_una; in bbr_log_ack()
7534 for (i = 0; i < to->to_nsacks; i++) { in bbr_log_ack()
7535 bcopy((to->to_sacks + i * TCPOLEN_SACK), in bbr_log_ack()
7541 SEQ_LT(sack.start, tp->snd_max) && in bbr_log_ack()
7543 SEQ_LEQ(sack.end, tp->snd_max)) { in bbr_log_ack()
7544 if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) && in bbr_log_ack()
7546 ((sack.end - sack.start) < (p_maxseg / 8))) { in bbr_log_ack()
7559 * Its a D-SACK block. in bbr_log_ack()
7570 new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks, in bbr_log_ack()
7571 num_sack_blks, th->th_ack); in bbr_log_ack()
7572 ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks); in bbr_log_ack()
7574 BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb)); in bbr_log_ack()
7612 * Now collapse out the dup-sack and in bbr_log_ack()
7620 num_sack_blks--; in bbr_log_ack()
7627 rsm = bbr->r_ctl.rc_sacklast; in bbr_log_ack()
7631 bbr->r_wanted_output = 1; in bbr_log_ack()
7638 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) { in bbr_log_ack()
7649 bbr->r_wanted_output = 1; in bbr_log_ack()
7654 if (bbr->r_ctl.rc_resend == NULL) { in bbr_log_ack()
7655 bbr->r_ctl.rc_resend = rsm; in bbr_log_ack()
7659 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) { in bbr_log_ack()
7661 * See if we need to rack-retransmit anything if so set it in bbr_log_ack()
7665 if (bbr->r_ctl.rc_resend == NULL) { in bbr_log_ack()
7666 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_log_ack()
7671 * ack-received code to augment what was changed between th_ack <-> in bbr_log_ack()
7682 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_strike_dupack()
7683 if (rsm && (rsm->r_dupack < 0xff)) { in bbr_strike_dupack()
7684 rsm->r_dupack++; in bbr_strike_dupack()
7685 if (rsm->r_dupack >= DUP_ACK_THRESHOLD) in bbr_strike_dupack()
7686 bbr->r_wanted_output = 1; in bbr_strike_dupack()
7693 * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7711 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_process_ack()
7712 lost = bbr->r_ctl.rc_lost; in bbr_process_ack()
7713 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_process_ack()
7714 if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) { in bbr_process_ack()
7716 tp->t_flags2 |= TF2_NO_ISS_CHECK; in bbr_process_ack()
7722 if (tp->t_flags2 & TF2_NO_ISS_CHECK) { in bbr_process_ack()
7724 seq_min = tp->snd_una - tp->max_sndwnd; in bbr_process_ack()
7727 if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) { in bbr_process_ack()
7729 seq_min = tp->iss + 1; in bbr_process_ack()
7736 seq_min = tp->snd_una - tp->max_sndwnd; in bbr_process_ack()
7740 if (SEQ_LT(th->th_ack, seq_min)) { in bbr_process_ack()
7747 bbr->r_wanted_output = 1; in bbr_process_ack()
7751 if (SEQ_GT(th->th_ack, tp->snd_max)) { in bbr_process_ack()
7753 bbr->r_wanted_output = 1; in bbr_process_ack()
7756 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { in bbr_process_ack()
7758 if (bbr->rc_in_persist) in bbr_process_ack()
7759 tp->t_rxtshift = 0; in bbr_process_ack()
7760 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd)) in bbr_process_ack()
7764 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost)); in bbr_process_ack()
7765 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { in bbr_process_ack()
7770 if (th->th_ack == tp->snd_una) { in bbr_process_ack()
7772 if (bbr->r_state == TCPS_SYN_SENT) { in bbr_process_ack()
7775 * the SYN-ACK is processed in syn_sent in bbr_process_ack()
7792 if (tp->t_flags & TF_NEEDSYN) { in bbr_process_ack()
7794 * T/TCP: Connection was half-synchronized, and our SYN has in bbr_process_ack()
7796 * to non-starred state, increment snd_una for ACK of SYN, in bbr_process_ack()
7799 tp->t_flags &= ~TF_NEEDSYN; in bbr_process_ack()
7800 tp->snd_una++; in bbr_process_ack()
7802 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == in bbr_process_ack()
7804 tp->rcv_scale = tp->request_r_scale; in bbr_process_ack()
7820 if (tp->t_flags & TF_PREVVALID) { in bbr_process_ack()
7821 tp->t_flags &= ~TF_PREVVALID; in bbr_process_ack()
7822 if (tp->t_rxtshift == 1 && in bbr_process_ack()
7823 (int)(ticks - tp->t_badrxtwin) < 0) in bbr_process_ack()
7827 acked_amount = min(acked, (int)sbavail(&so->so_snd)); in bbr_process_ack()
7828 tp->snd_wnd -= acked_amount; in bbr_process_ack()
7829 mfree = sbcut_locked(&so->so_snd, acked_amount); in bbr_process_ack()
7833 if (SEQ_GT(th->th_ack, tp->snd_una)) { in bbr_process_ack()
7836 tp->snd_una = th->th_ack; in bbr_process_ack()
7837 …bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - los… in bbr_process_ack()
7838 if (IN_RECOVERY(tp->t_flags)) { in bbr_process_ack()
7839 if (SEQ_LT(th->th_ack, tp->snd_recover) && in bbr_process_ack()
7840 (SEQ_LT(th->th_ack, tp->snd_max))) { in bbr_process_ack()
7846 if (SEQ_GT(tp->snd_una, tp->snd_recover)) { in bbr_process_ack()
7847 tp->snd_recover = tp->snd_una; in bbr_process_ack()
7849 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_process_ack()
7850 tp->snd_nxt = tp->snd_max; in bbr_process_ack()
7852 if (tp->snd_una == tp->snd_max) { in bbr_process_ack()
7856 if (sbavail(&so->so_snd) == 0) in bbr_process_ack()
7857 bbr->rc_tp->t_acktime = 0; in bbr_process_ack()
7858 if ((sbused(&so->so_snd) == 0) && in bbr_process_ack()
7859 (tp->t_flags & TF_SENTFIN)) { in bbr_process_ack()
7862 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_ack()
7863 if (bbr->rc_in_persist == 0) { in bbr_process_ack()
7864 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime; in bbr_process_ack()
7866 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_process_ack()
7867 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime); in bbr_process_ack()
7873 if ((tp->t_state >= TCPS_FIN_WAIT_1) && in bbr_process_ack()
7874 (sbavail(&so->so_snd) == 0) && in bbr_process_ack()
7875 (tp->t_flags2 & TF2_DROP_AF_DATA)) { in bbr_process_ack()
7882 /* tcp_close will kill the inp pre-log the Reset */ in bbr_process_ack()
7890 bbr->r_wanted_output = 1; in bbr_process_ack()
7900 if (bbr->rc_in_persist == 0) { in bbr_enter_persist()
7902 bbr->r_ctl.rc_last_delay_val = 0; in bbr_enter_persist()
7903 tp->t_rxtshift = 0; in bbr_enter_persist()
7904 bbr->rc_in_persist = 1; in bbr_enter_persist()
7905 bbr->r_ctl.rc_went_idle_time = cts; in bbr_enter_persist()
7909 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_enter_persist()
7912 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_enter_persist()
7913 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_enter_persist()
7919 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_enter_persist()
7922 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_enter_persist()
7935 if (bbr->rc_use_idle_restart) { in bbr_restart_after_idle()
7936 bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT; in bbr_restart_after_idle()
7943 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_restart_after_idle()
7944 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_restart_after_idle()
7947 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; in bbr_restart_after_idle()
7948 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; in bbr_restart_after_idle()
7950 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_restart_after_idle()
7961 if (bbr->rc_in_persist == 0) in bbr_exit_persist()
7963 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time); in bbr_exit_persist()
7964 bbr->rc_in_persist = 0; in bbr_exit_persist()
7965 bbr->rc_hit_state_1 = 0; in bbr_exit_persist()
7966 bbr->r_ctl.rc_del_time = cts; in bbr_exit_persist()
7972 if (tcp_in_hpts(bbr->rc_tp)) { in bbr_exit_persist()
7973 tcp_hpts_remove(bbr->rc_tp); in bbr_exit_persist()
7974 bbr->rc_timer_first = 0; in bbr_exit_persist()
7975 bbr->r_ctl.rc_hpts_flags = 0; in bbr_exit_persist()
7976 bbr->r_ctl.rc_last_delay_val = 0; in bbr_exit_persist()
7977 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_exit_persist()
7978 bbr->r_agg_early_set = 0; in bbr_exit_persist()
7979 bbr->r_ctl.rc_agg_early = 0; in bbr_exit_persist()
7989 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_exit_persist()
7991 tp->t_rxtshift = 0; in bbr_exit_persist()
8002 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_exit_persist()
8003 /* Time un-freezes for the state */ in bbr_exit_persist()
8004 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_exit_persist()
8005 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) || in bbr_exit_persist()
8006 (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) { in bbr_exit_persist()
8008 * If we are going back to probe-bw in bbr_exit_persist()
8041 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in bbr_collapsed_window()
8042 max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd; in bbr_collapsed_window()
8044 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_collapsed_window()
8046 if (rsm->r_flags & BBR_RWND_COLLAPSED) in bbr_collapsed_window()
8047 rsm->r_flags &= ~BBR_RWND_COLLAPSED; in bbr_collapsed_window()
8048 if (SEQ_GEQ(max_seq, rsm->r_start) && in bbr_collapsed_window()
8049 SEQ_GEQ(rsm->r_end, max_seq)) { in bbr_collapsed_window()
8054 bbr->rc_has_collapsed = 0; in bbr_collapsed_window()
8067 if ((max_seq != rsm->r_start) && in bbr_collapsed_window()
8068 (max_seq != rsm->r_end)){ in bbr_collapsed_window()
8072 res1 = max_seq - rsm->r_start; in bbr_collapsed_window()
8073 res2 = rsm->r_end - max_seq; in bbr_collapsed_window()
8078 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) { in bbr_collapsed_window()
8084 if (max_seq == rsm->r_start) { in bbr_collapsed_window()
8087 } else if (max_seq == rsm->r_end) { in bbr_collapsed_window()
8094 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) { in bbr_collapsed_window()
8105 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_collapsed_window()
8106 if (rsm->r_in_tmap) { in bbr_collapsed_window()
8107 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_collapsed_window()
8108 nrsm->r_in_tmap = 1; in bbr_collapsed_window()
8121 TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) { in bbr_collapsed_window()
8122 nrsm->r_flags |= BBR_RWND_COLLAPSED; in bbr_collapsed_window()
8124 bbr->rc_has_collapsed = 1; in bbr_collapsed_window()
8135 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_un_collapse_window()
8136 if (rsm->r_flags & BBR_RWND_COLLAPSED) { in bbr_un_collapse_window()
8138 rsm->r_flags &= ~BBR_RWND_COLLAPSED; in bbr_un_collapse_window()
8144 (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared); in bbr_un_collapse_window()
8145 bbr->rc_has_collapsed = 0; in bbr_un_collapse_window()
8166 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_process_data()
8168 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_process_data()
8170 (SEQ_LT(tp->snd_wl1, th->th_seq) || in bbr_process_data()
8171 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || in bbr_process_data()
8172 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { in bbr_process_data()
8175 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) in bbr_process_data()
8177 tp->snd_wnd = tiwin; in bbr_process_data()
8178 tp->snd_wl1 = th->th_seq; in bbr_process_data()
8179 tp->snd_wl2 = th->th_ack; in bbr_process_data()
8180 if (tp->snd_wnd > tp->max_sndwnd) in bbr_process_data()
8181 tp->max_sndwnd = tp->snd_wnd; in bbr_process_data()
8182 bbr->r_wanted_output = 1; in bbr_process_data()
8184 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) { in bbr_process_data()
8185 tp->snd_wnd = tiwin; in bbr_process_data()
8186 tp->snd_wl1 = th->th_seq; in bbr_process_data()
8187 tp->snd_wl2 = th->th_ack; in bbr_process_data()
8190 if (tp->snd_wnd < ctf_outstanding(tp)) in bbr_process_data()
8193 else if (bbr->rc_has_collapsed) in bbr_process_data()
8196 if ((bbr->rc_in_persist != 0) && in bbr_process_data()
8197 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2), in bbr_process_data()
8200 * Make the rate persist at end of persist mode if idle long in bbr_process_data()
8203 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_process_data()
8206 bbr->r_wanted_output = 1; in bbr_process_data()
8209 if ((bbr->rc_in_persist == 0) && in bbr_process_data()
8210 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_process_data()
8211 TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_process_data()
8212 (tp->snd_max == tp->snd_una) && in bbr_process_data()
8213 sbavail(&so->so_snd) && in bbr_process_data()
8214 (sbavail(&so->so_snd) > tp->snd_wnd)) { in bbr_process_data()
8216 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_process_data()
8218 if (tp->t_flags2 & TF2_DROP_AF_DATA) { in bbr_process_data()
8228 tp->rcv_up = tp->rcv_nxt; in bbr_process_data()
8233 * This process logically involves adjusting tp->rcv_wnd as data is in bbr_process_data()
8238 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && in bbr_process_data()
8239 (tp->t_flags & TF_FASTOPEN)); in bbr_process_data()
8241 TCPS_HAVERCVDFIN(tp->t_state) == 0) { in bbr_process_data()
8242 tcp_seq save_start = th->th_seq; in bbr_process_data()
8243 tcp_seq save_rnxt = tp->rcv_nxt; in bbr_process_data()
8258 if (th->th_seq == tp->rcv_nxt && in bbr_process_data()
8260 (TCPS_HAVEESTABLISHED(tp->t_state) || in bbr_process_data()
8265 if (so->so_rcv.sb_shlim) { in bbr_process_data()
8268 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, in bbr_process_data()
8278 bbr->bbr_segs_rcvd += max(1, nsegs); in bbr_process_data()
8279 tp->t_flags |= TF_DELACK; in bbr_process_data()
8280 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_data()
8282 bbr->r_wanted_output = 1; in bbr_process_data()
8283 tp->t_flags |= TF_ACKNOW; in bbr_process_data()
8285 tp->rcv_nxt += tlen; in bbr_process_data()
8287 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && in bbr_process_data()
8288 (tp->t_fbyte_in == 0)) { in bbr_process_data()
8289 tp->t_fbyte_in = ticks; in bbr_process_data()
8290 if (tp->t_fbyte_in == 0) in bbr_process_data()
8291 tp->t_fbyte_in = 1; in bbr_process_data()
8292 if (tp->t_fbyte_out && tp->t_fbyte_in) in bbr_process_data()
8293 tp->t_flags2 |= TF2_FBYTES_COMPLETE; in bbr_process_data()
8299 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) in bbr_process_data()
8305 sbappendstream_locked(&so->so_rcv, m, 0); in bbr_process_data()
8309 if (so->so_rcv.sb_shlim && appended != mcnt) in bbr_process_data()
8310 counter_fo_release(so->so_rcv.sb_shlim, in bbr_process_data()
8311 mcnt - appended); in bbr_process_data()
8324 tp->t_flags |= TF_ACKNOW; in bbr_process_data()
8325 if (tp->t_flags & TF_WAKESOR) { in bbr_process_data()
8326 tp->t_flags &= ~TF_WAKESOR; in bbr_process_data()
8331 if ((tp->t_flags & TF_SACK_PERMIT) && in bbr_process_data()
8333 TCPS_HAVEESTABLISHED(tp->t_state)) { in bbr_process_data()
8341 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) { in bbr_process_data()
8342 if ((tp->rcv_numsacks >= 1) && in bbr_process_data()
8343 (tp->sackblks[0].end == save_start)) { in bbr_process_data()
8349 tp->sackblks[0].start, in bbr_process_data()
8350 tp->sackblks[0].end); in bbr_process_data()
8374 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { in bbr_process_data()
8378 * If connection is half-synchronized (ie NEEDSYN in bbr_process_data()
8384 if (tp->t_flags & TF_NEEDSYN) { in bbr_process_data()
8385 tp->t_flags |= TF_DELACK; in bbr_process_data()
8387 __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_data()
8389 tp->t_flags |= TF_ACKNOW; in bbr_process_data()
8391 tp->rcv_nxt++; in bbr_process_data()
8393 switch (tp->t_state) { in bbr_process_data()
8399 tp->t_starttime = ticks; in bbr_process_data()
8415 * starting the time-wait timer, turning off the in bbr_process_data()
8419 bbr->rc_timer_first = 1; in bbr_process_data()
8421 __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_data()
8429 if ((tp->t_flags & TF_ACKNOW) || in bbr_process_data()
8430 (sbavail(&so->so_snd) > ctf_outstanding(tp))) { in bbr_process_data()
8431 bbr->r_wanted_output = 1; in bbr_process_data()
8438 * have broken out the fast-data path also just like
8439 * the fast-ack. Return 1 if we processed the packet
8440 * return 0 if you need to take the "slow-path".
8455 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_fastnewdata()
8462 if (bbr->r_ctl.rc_resend != NULL) { in bbr_do_fastnewdata()
8465 if (tiwin && tiwin != tp->snd_wnd) { in bbr_do_fastnewdata()
8468 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) { in bbr_do_fastnewdata()
8471 if (__predict_false((to->to_flags & TOF_TS) && in bbr_do_fastnewdata()
8472 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) { in bbr_do_fastnewdata()
8475 if (__predict_false((th->th_ack != tp->snd_una))) { in bbr_do_fastnewdata()
8478 if (__predict_false(tlen > sbspace(&so->so_rcv))) { in bbr_do_fastnewdata()
8481 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_fastnewdata()
8482 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { in bbr_do_fastnewdata()
8483 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_fastnewdata()
8484 tp->ts_recent = to->to_tsval; in bbr_do_fastnewdata()
8487 * This is a pure, in-sequence data packet with nothing on the in bbr_do_fastnewdata()
8490 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_do_fastnewdata()
8493 if (so->so_rcv.sb_shlim) { in bbr_do_fastnewdata()
8496 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, in bbr_do_fastnewdata()
8505 if (tp->rcv_numsacks) in bbr_do_fastnewdata()
8508 tp->rcv_nxt += tlen; in bbr_do_fastnewdata()
8510 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && in bbr_do_fastnewdata()
8511 (tp->t_fbyte_in == 0)) { in bbr_do_fastnewdata()
8512 tp->t_fbyte_in = ticks; in bbr_do_fastnewdata()
8513 if (tp->t_fbyte_in == 0) in bbr_do_fastnewdata()
8514 tp->t_fbyte_in = 1; in bbr_do_fastnewdata()
8515 if (tp->t_fbyte_out && tp->t_fbyte_in) in bbr_do_fastnewdata()
8516 tp->t_flags2 |= TF2_FBYTES_COMPLETE; in bbr_do_fastnewdata()
8521 tp->snd_wl1 = th->th_seq; in bbr_do_fastnewdata()
8525 tp->rcv_up = tp->rcv_nxt; in bbr_do_fastnewdata()
8532 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_fastnewdata()
8541 so->so_rcv.sb_flags &= ~SB_AUTOSIZE; in bbr_do_fastnewdata()
8547 sbappendstream_locked(&so->so_rcv, m, 0); in bbr_do_fastnewdata()
8553 if (so->so_rcv.sb_shlim && mcnt != appended) in bbr_do_fastnewdata()
8554 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended); in bbr_do_fastnewdata()
8557 bbr->bbr_segs_rcvd += max(1, nsegs); in bbr_do_fastnewdata()
8558 tp->t_flags |= TF_DELACK; in bbr_do_fastnewdata()
8559 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_do_fastnewdata()
8561 bbr->r_wanted_output = 1; in bbr_do_fastnewdata()
8562 tp->t_flags |= TF_ACKNOW; in bbr_do_fastnewdata()
8570 * in sequence to remain in the fast-path. We also add
8574 * slow-path. If we return 1, then all is well and
8588 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { in bbr_fastack()
8592 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_fastack()
8600 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) { in bbr_fastack()
8604 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) { in bbr_fastack()
8608 if (__predict_false(IN_RECOVERY(tp->t_flags))) { in bbr_fastack()
8612 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_fastack()
8613 if (__predict_false(bbr->r_ctl.rc_resend != NULL)) { in bbr_fastack()
8617 if (__predict_false(bbr->rc_in_persist != 0)) { in bbr_fastack()
8618 /* In persist mode */ in bbr_fastack()
8621 if (bbr->r_ctl.rc_sacked) { in bbr_fastack()
8625 /* Ok if we reach here, we can process a fast-ack */ in bbr_fastack()
8626 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_fastack()
8633 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0); in bbr_fastack()
8635 if (tiwin != tp->snd_wnd) { in bbr_fastack()
8636 tp->snd_wnd = tiwin; in bbr_fastack()
8637 tp->snd_wl1 = th->th_seq; in bbr_fastack()
8638 if (tp->snd_wnd > tp->max_sndwnd) in bbr_fastack()
8639 tp->max_sndwnd = tp->snd_wnd; in bbr_fastack()
8642 if ((bbr->rc_in_persist != 0) && in bbr_fastack()
8643 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2), in bbr_fastack()
8645 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_fastack()
8646 bbr->r_wanted_output = 1; in bbr_fastack()
8649 if ((bbr->rc_in_persist == 0) && in bbr_fastack()
8650 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_fastack()
8651 TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_fastack()
8652 (tp->snd_max == tp->snd_una) && in bbr_fastack()
8653 sbavail(&so->so_snd) && in bbr_fastack()
8654 (sbavail(&so->so_snd) > tp->snd_wnd)) { in bbr_fastack()
8656 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_fastack()
8663 if ((to->to_flags & TOF_TS) != 0 && in bbr_fastack()
8664 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { in bbr_fastack()
8665 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime; in bbr_fastack()
8666 tp->ts_recent = to->to_tsval; in bbr_fastack()
8676 if (tp->t_flags & TF_PREVVALID) { in bbr_fastack()
8677 tp->t_flags &= ~TF_PREVVALID; in bbr_fastack()
8678 if (tp->t_rxtshift == 1 && in bbr_fastack()
8679 (int)(ticks - tp->t_badrxtwin) < 0) in bbr_fastack()
8698 sbdrop(&so->so_snd, acked); in bbr_fastack()
8700 if (SEQ_GT(th->th_ack, tp->snd_una)) in bbr_fastack()
8702 tp->snd_una = th->th_ack; in bbr_fastack()
8703 if (tp->snd_wnd < ctf_outstanding(tp)) in bbr_fastack()
8706 else if (bbr->rc_has_collapsed) in bbr_fastack()
8709 if (SEQ_GT(tp->snd_una, tp->snd_recover)) { in bbr_fastack()
8710 tp->snd_recover = tp->snd_una; in bbr_fastack()
8716 tp->snd_wl2 = th->th_ack; in bbr_fastack()
8720 * otherwise restart timer using current (possibly backed-off) in bbr_fastack()
8727 if (tp->snd_una == tp->snd_max) { in bbr_fastack()
8730 if (sbavail(&so->so_snd) == 0) in bbr_fastack()
8731 bbr->rc_tp->t_acktime = 0; in bbr_fastack()
8732 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_fastack()
8733 if (bbr->rc_in_persist == 0) { in bbr_fastack()
8734 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime; in bbr_fastack()
8736 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_fastack()
8737 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime); in bbr_fastack()
8743 bbr->r_wanted_output = 1; in bbr_fastack()
8745 if (sbavail(&so->so_snd)) { in bbr_fastack()
8746 bbr->r_wanted_output = 1; in bbr_fastack()
8768 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_syn_sent()
8774 * this is an acceptable SYN segment initialize tp->rcv_nxt and in bbr_do_syn_sent()
8775 * tp->irs if seg contains ack then advance tp->snd_una. BRR does in bbr_do_syn_sent()
8782 (SEQ_LEQ(th->th_ack, tp->iss) || in bbr_do_syn_sent()
8783 SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_do_syn_sent()
8803 tp->irs = th->th_seq; in bbr_do_syn_sent()
8814 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == in bbr_do_syn_sent()
8816 tp->rcv_scale = tp->request_r_scale; in bbr_do_syn_sent()
8818 tp->rcv_adv += min(tp->rcv_wnd, in bbr_do_syn_sent()
8819 TCP_MAXWIN << tp->rcv_scale); in bbr_do_syn_sent()
8824 if ((tp->t_flags & TF_FASTOPEN) && in bbr_do_syn_sent()
8825 (tp->snd_una != tp->snd_max)) { in bbr_do_syn_sent()
8826 tp->snd_nxt = th->th_ack; in bbr_do_syn_sent()
8834 bbr->bbr_segs_rcvd += 1; in bbr_do_syn_sent()
8835 tp->t_flags |= TF_DELACK; in bbr_do_syn_sent()
8836 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_do_syn_sent()
8838 bbr->r_wanted_output = 1; in bbr_do_syn_sent()
8839 tp->t_flags |= TF_ACKNOW; in bbr_do_syn_sent()
8841 if (SEQ_GT(th->th_ack, tp->iss)) { in bbr_do_syn_sent()
8848 if (SEQ_GT(th->th_ack, tp->snd_una)) { in bbr_do_syn_sent()
8854 * ack-processing since the in bbr_do_syn_sent()
8855 * data stream in our send-map in bbr_do_syn_sent()
8861 tp->snd_una++; in bbr_do_syn_sent()
8865 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1 in bbr_do_syn_sent()
8867 tp->t_starttime = ticks; in bbr_do_syn_sent()
8868 if (tp->t_flags & TF_NEEDFIN) { in bbr_do_syn_sent()
8870 tp->t_flags &= ~TF_NEEDFIN; in bbr_do_syn_sent()
8880 * Received initial SYN in SYN-SENT[*] state => simultaneous in bbr_do_syn_sent()
8883 * half-synchronized. Otherwise, do 3-way handshake: in bbr_do_syn_sent()
8884 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If in bbr_do_syn_sent()
8887 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN); in bbr_do_syn_sent()
8891 * Advance th->th_seq to correspond to first data byte. If data, in bbr_do_syn_sent()
8894 th->th_seq++; in bbr_do_syn_sent()
8895 if (tlen > tp->rcv_wnd) { in bbr_do_syn_sent()
8896 todrop = tlen - tp->rcv_wnd; in bbr_do_syn_sent()
8897 m_adj(m, -todrop); in bbr_do_syn_sent()
8898 tlen = tp->rcv_wnd; in bbr_do_syn_sent()
8903 tp->snd_wl1 = th->th_seq - 1; in bbr_do_syn_sent()
8904 tp->rcv_up = th->th_seq; in bbr_do_syn_sent()
8912 if ((to->to_flags & TOF_TS) != 0) { in bbr_do_syn_sent()
8915 t = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_syn_sent()
8916 if (TSTMP_GEQ(t, to->to_tsecr)) { in bbr_do_syn_sent()
8917 rtt = t - to->to_tsecr; in bbr_do_syn_sent()
8923 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, in bbr_do_syn_sent()
8924 rtt, bbr->r_ctl.rc_rcvtime); in bbr_do_syn_sent()
8930 if (tp->t_state == TCPS_FIN_WAIT_1) { in bbr_do_syn_sent()
8947 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_syn_sent()
8978 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_syn_recv()
8981 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_syn_recv()
8984 (SEQ_LEQ(th->th_ack, tp->snd_una) || in bbr_do_syn_recv()
8985 SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_do_syn_recv()
8990 if (tp->t_flags & TF_FASTOPEN) { in bbr_do_syn_recv()
9002 /* non-initial SYN is ignored */ in bbr_do_syn_recv()
9003 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) || in bbr_do_syn_recv()
9004 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) || in bbr_do_syn_recv()
9005 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) { in bbr_do_syn_recv()
9018 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_syn_recv()
9019 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_syn_recv()
9024 * In the SYN-RECEIVED state, validate that the packet belongs to in bbr_do_syn_recv()
9030 if (SEQ_LT(th->th_seq, tp->irs)) { in bbr_do_syn_recv()
9052 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_syn_recv()
9053 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_syn_recv()
9054 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_syn_recv()
9056 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_syn_recv()
9057 tp->ts_recent = to->to_tsval; in bbr_do_syn_recv()
9059 tp->snd_wnd = tiwin; in bbr_do_syn_recv()
9061 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_syn_recv()
9062 * is on (half-synchronized state), then queue data for later in bbr_do_syn_recv()
9066 if (tp->t_flags & TF_FASTOPEN) { in bbr_do_syn_recv()
9073 if (tp->t_flags & TF_SONOTCONN) { in bbr_do_syn_recv()
9074 tp->t_flags &= ~TF_SONOTCONN; in bbr_do_syn_recv()
9078 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == in bbr_do_syn_recv()
9080 tp->rcv_scale = tp->request_r_scale; in bbr_do_syn_recv()
9086 if ((to->to_flags & TOF_TS) != 0) { in bbr_do_syn_recv()
9089 t = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_syn_recv()
9090 if (TSTMP_GEQ(t, to->to_tsecr)) { in bbr_do_syn_recv()
9091 rtt = t - to->to_tsecr; in bbr_do_syn_recv()
9097 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime); in bbr_do_syn_recv()
9103 if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) { in bbr_do_syn_recv()
9104 tcp_fastopen_decrement_counter(tp->t_tfo_pending); in bbr_do_syn_recv()
9105 tp->t_tfo_pending = NULL; in bbr_do_syn_recv()
9108 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* -> in bbr_do_syn_recv()
9109 * FIN-WAIT-1 in bbr_do_syn_recv()
9111 tp->t_starttime = ticks; in bbr_do_syn_recv()
9112 if (tp->t_flags & TF_NEEDFIN) { in bbr_do_syn_recv()
9114 tp->t_flags &= ~TF_NEEDFIN; in bbr_do_syn_recv()
9125 if (!(tp->t_flags & TF_FASTOPEN)) in bbr_do_syn_recv()
9133 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN)) in bbr_do_syn_recv()
9134 tp->snd_una++; in bbr_do_syn_recv()
9142 if (tp->t_flags & TF_WAKESOR) { in bbr_do_syn_recv()
9143 tp->t_flags &= ~TF_WAKESOR; in bbr_do_syn_recv()
9148 tp->snd_wl1 = th->th_seq - 1; in bbr_do_syn_recv()
9152 if (tp->t_state == TCPS_FIN_WAIT_1) { in bbr_do_syn_recv()
9169 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_syn_recv()
9200 * uni-directional data xfer. If the packet has no control flags, in bbr_do_established()
9201 * is in-sequence, the window didn't change and we're not in bbr_do_established()
9205 * waiting for space. If the length is non-zero and the ack didn't in bbr_do_established()
9206 * move, we're the receiver side. If we're getting packets in-order in bbr_do_established()
9209 * hidden state-flags are also off. Since we check for in bbr_do_established()
9212 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_established()
9213 if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) { in bbr_do_established()
9219 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd); in bbr_do_established()
9221 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) && in bbr_do_established()
9224 __predict_true(th->th_seq == tp->rcv_nxt)) { in bbr_do_established()
9240 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_established()
9254 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_established()
9255 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_established()
9276 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_established()
9277 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_established()
9278 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_established()
9280 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_established()
9281 tp->ts_recent = to->to_tsval; in bbr_do_established()
9284 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_established()
9285 * is on (half-synchronized state), then queue data for later in bbr_do_established()
9289 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_established()
9292 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_established()
9294 bbr->r_wanted_output = 1; in bbr_do_established()
9307 if (sbavail(&so->so_snd)) { in bbr_do_established()
9334 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_close_wait()
9337 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_close_wait()
9351 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_close_wait()
9352 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_close_wait()
9373 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_close_wait()
9374 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_close_wait()
9375 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_close_wait()
9377 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_close_wait()
9378 tp->ts_recent = to->to_tsval; in bbr_do_close_wait()
9381 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_close_wait()
9382 * is on (half-synchronized state), then queue data for later in bbr_do_close_wait()
9386 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_close_wait()
9389 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_close_wait()
9391 bbr->r_wanted_output = 1; in bbr_do_close_wait()
9404 if (sbavail(&so->so_snd)) { in bbr_do_close_wait()
9420 if (bbr->rc_allow_data_af_clo == 0) { in bbr_check_data_after_close()
9423 /* tcp_close will kill the inp pre-log the Reset */ in bbr_check_data_after_close()
9430 if (sbavail(&so->so_snd) == 0) in bbr_check_data_after_close()
9433 tp->rcv_nxt = th->th_seq + *tlen; in bbr_check_data_after_close()
9434 tp->t_flags2 |= TF2_DROP_AF_DATA; in bbr_check_data_after_close()
9435 bbr->r_wanted_output = 1; in bbr_check_data_after_close()
9456 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_fin_wait_1()
9459 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_fin_wait_1()
9473 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_fin_wait_1()
9474 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_fin_wait_1()
9487 if ((tp->t_flags & TF_CLOSED) && tlen && in bbr_do_fin_wait_1()
9504 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_fin_wait_1()
9505 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_fin_wait_1()
9506 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_fin_wait_1()
9508 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_fin_wait_1()
9509 tp->ts_recent = to->to_tsval; in bbr_do_fin_wait_1()
9512 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_fin_wait_1()
9513 * is on (half-synchronized state), then queue data for later in bbr_do_fin_wait_1()
9517 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_fin_wait_1()
9520 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_fin_wait_1()
9522 bbr->r_wanted_output = 1; in bbr_do_fin_wait_1()
9545 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_fin_wait_1()
9554 if (sbavail(&so->so_snd)) { in bbr_do_fin_wait_1()
9581 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_closing()
9584 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_closing()
9598 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_closing()
9599 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_closing()
9620 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_closing()
9621 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_closing()
9622 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_closing()
9624 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_closing()
9625 tp->ts_recent = to->to_tsval; in bbr_do_closing()
9628 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_closing()
9629 * is on (half-synchronized state), then queue data for later in bbr_do_closing()
9633 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_closing()
9636 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_closing()
9638 bbr->r_wanted_output = 1; in bbr_do_closing()
9656 if (sbavail(&so->so_snd)) { in bbr_do_closing()
9683 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_lastack()
9686 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_lastack()
9700 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_lastack()
9701 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_lastack()
9722 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_lastack()
9723 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_lastack()
9724 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_lastack()
9726 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_lastack()
9727 tp->ts_recent = to->to_tsval; in bbr_do_lastack()
9730 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_lastack()
9731 * is on (half-synchronized state), then queue data for later in bbr_do_lastack()
9735 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_lastack()
9738 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_lastack()
9740 bbr->r_wanted_output = 1; in bbr_do_lastack()
9758 if (sbavail(&so->so_snd)) { in bbr_do_lastack()
9785 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_fin_wait_2()
9787 /* Reset receive buffer auto scaling when not in bulk receive mode. */ in bbr_do_fin_wait_2()
9789 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_fin_wait_2()
9804 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_fin_wait_2()
9805 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_fin_wait_2()
9819 if ((tp->t_flags & TF_CLOSED) && tlen && in bbr_do_fin_wait_2()
9836 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_fin_wait_2()
9837 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_fin_wait_2()
9838 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_fin_wait_2()
9840 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_fin_wait_2()
9841 tp->ts_recent = to->to_tsval; in bbr_do_fin_wait_2()
9844 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_fin_wait_2()
9845 * is on (half-synchronized state), then queue data for later in bbr_do_fin_wait_2()
9849 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_fin_wait_2()
9852 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_fin_wait_2()
9854 bbr->r_wanted_output = 1; in bbr_do_fin_wait_2()
9867 if (sbavail(&so->so_snd)) { in bbr_do_fin_wait_2()
9886 bbr->rc_in_persist = 1; in bbr_stop_all_timers()
9888 if (tcp_in_hpts(bbr->rc_tp)) { in bbr_stop_all_timers()
9889 tcp_hpts_remove(bbr->rc_tp); in bbr_stop_all_timers()
9896 bbr->rc_use_google = 1; in bbr_google_mode_on()
9897 bbr->rc_no_pacing = 0; in bbr_google_mode_on()
9898 bbr->r_ctl.bbr_google_discount = bbr_google_discount; in bbr_google_mode_on()
9899 bbr->r_use_policer = bbr_policer_detection_enabled; in bbr_google_mode_on()
9900 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10); in bbr_google_mode_on()
9901 bbr->bbr_use_rack_cheat = 0; in bbr_google_mode_on()
9902 bbr->r_ctl.rc_incr_tmrs = 0; in bbr_google_mode_on()
9903 bbr->r_ctl.rc_inc_tcp_oh = 0; in bbr_google_mode_on()
9904 bbr->r_ctl.rc_inc_ip_oh = 0; in bbr_google_mode_on()
9905 bbr->r_ctl.rc_inc_enet_oh = 0; in bbr_google_mode_on()
9906 reset_time(&bbr->r_ctl.rc_delrate, in bbr_google_mode_on()
9908 reset_time_small(&bbr->r_ctl.rc_rttprop, in bbr_google_mode_on()
9910 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv)); in bbr_google_mode_on()
9916 bbr->rc_use_google = 0; in bbr_google_mode_off()
9917 bbr->r_ctl.bbr_google_discount = 0; in bbr_google_mode_off()
9918 bbr->no_pacing_until = bbr_no_pacing_until; in bbr_google_mode_off()
9919 bbr->r_use_policer = 0; in bbr_google_mode_off()
9920 if (bbr->no_pacing_until) in bbr_google_mode_off()
9921 bbr->rc_no_pacing = 1; in bbr_google_mode_off()
9923 bbr->rc_no_pacing = 0; in bbr_google_mode_off()
9925 bbr->bbr_use_rack_cheat = 1; in bbr_google_mode_off()
9927 bbr->bbr_use_rack_cheat = 0; in bbr_google_mode_off()
9929 bbr->r_ctl.rc_incr_tmrs = 1; in bbr_google_mode_off()
9931 bbr->r_ctl.rc_incr_tmrs = 0; in bbr_google_mode_off()
9933 bbr->r_ctl.rc_inc_tcp_oh = 1; in bbr_google_mode_off()
9935 bbr->r_ctl.rc_inc_tcp_oh = 0; in bbr_google_mode_off()
9937 bbr->r_ctl.rc_inc_ip_oh = 1; in bbr_google_mode_off()
9939 bbr->r_ctl.rc_inc_ip_oh = 0; in bbr_google_mode_off()
9941 bbr->r_ctl.rc_inc_enet_oh = 1; in bbr_google_mode_off()
9943 bbr->r_ctl.rc_inc_enet_oh = 0; in bbr_google_mode_off()
9944 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; in bbr_google_mode_off()
9945 reset_time(&bbr->r_ctl.rc_delrate, in bbr_google_mode_off()
9947 reset_time_small(&bbr->r_ctl.rc_rttprop, in bbr_google_mode_off()
9949 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv)); in bbr_google_mode_off()
9952 * Return 0 on success, non-zero on failure
9975 bbr->rtt_valid = 0; in bbr_init()
9976 tp->t_flags2 |= TF2_CANNOT_DO_ECN; in bbr_init()
9977 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ; in bbr_init()
9979 tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY; in bbr_init()
9980 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE; in bbr_init()
9981 tp->t_flags2 &= ~TF2_MBUF_ACKCMP; in bbr_init()
9982 tp->t_flags2 &= ~TF2_MBUF_L_ACKS; in bbr_init()
9984 TAILQ_INIT(&bbr->r_ctl.rc_map); in bbr_init()
9985 TAILQ_INIT(&bbr->r_ctl.rc_free); in bbr_init()
9986 TAILQ_INIT(&bbr->r_ctl.rc_tmap); in bbr_init()
9987 bbr->rc_tp = tp; in bbr_init()
9988 bbr->rc_inp = inp; in bbr_init()
9989 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_init()
9990 tp->t_acktime = 0; in bbr_init()
9991 bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close; in bbr_init()
9992 bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade; in bbr_init()
9993 bbr->rc_tlp_threshold = bbr_tlp_thresh; in bbr_init()
9994 bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh; in bbr_init()
9995 bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay; in bbr_init()
9996 bbr->r_ctl.rc_min_to = bbr_min_to; in bbr_init()
9997 bbr->rc_bbr_state = BBR_STATE_STARTUP; in bbr_init()
9998 bbr->r_ctl.bbr_lost_at_state = 0; in bbr_init()
9999 bbr->r_ctl.rc_lost_at_startup = 0; in bbr_init()
10000 bbr->rc_all_timers_stopped = 0; in bbr_init()
10001 bbr->r_ctl.rc_bbr_lastbtlbw = 0; in bbr_init()
10002 bbr->r_ctl.rc_pkt_epoch_del = 0; in bbr_init()
10003 bbr->r_ctl.rc_pkt_epoch = 0; in bbr_init()
10004 bbr->r_ctl.rc_lowest_rtt = 0xffffffff; in bbr_init()
10005 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain; in bbr_init()
10006 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain; in bbr_init()
10007 bbr->r_ctl.rc_went_idle_time = cts; in bbr_init()
10008 bbr->rc_pacer_started = cts; in bbr_init()
10009 bbr->r_ctl.rc_pkt_epoch_time = cts; in bbr_init()
10010 bbr->r_ctl.rc_rcvtime = cts; in bbr_init()
10011 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_init()
10012 bbr->r_ctl.rc_del_time = cts; in bbr_init()
10013 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_init()
10014 bbr->r_ctl.last_in_probertt = cts; in bbr_init()
10015 bbr->skip_gain = 0; in bbr_init()
10016 bbr->gain_is_limited = 0; in bbr_init()
10017 bbr->no_pacing_until = bbr_no_pacing_until; in bbr_init()
10018 if (bbr->no_pacing_until) in bbr_init()
10019 bbr->rc_no_pacing = 1; in bbr_init()
10021 bbr->rc_no_pacing = 0; in bbr_init()
10022 bbr->rc_use_google = 1; in bbr_init()
10023 bbr->r_ctl.bbr_google_discount = bbr_google_discount; in bbr_init()
10024 bbr->r_use_policer = bbr_policer_detection_enabled; in bbr_init()
10026 bbr->rc_use_google = 0; in bbr_init()
10027 bbr->r_ctl.bbr_google_discount = 0; in bbr_init()
10028 bbr->r_use_policer = 0; in bbr_init()
10031 bbr->rc_use_ts_limit = 1; in bbr_init()
10033 bbr->rc_use_ts_limit = 0; in bbr_init()
10035 bbr->ts_can_raise = 1; in bbr_init()
10037 bbr->ts_can_raise = 0; in bbr_init()
10039 tp->t_delayed_ack = 2; in bbr_init()
10041 tp->t_delayed_ack = 0; in bbr_init()
10043 tp->t_delayed_ack = V_tcp_delack_enabled; in bbr_init()
10045 tp->t_delayed_ack = 2; in bbr_init()
10046 if (bbr->rc_use_google == 0) in bbr_init()
10047 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; in bbr_init()
10049 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10); in bbr_init()
10050 bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms; in bbr_init()
10051 bbr->rc_max_rto_sec = bbr_rto_max_sec; in bbr_init()
10052 bbr->rc_init_win = bbr_def_init_win; in bbr_init()
10053 if (tp->t_flags & TF_REQ_TSTMP) in bbr_init()
10054 bbr->rc_last_options = TCP_TS_OVERHEAD; in bbr_init()
10055 bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options; in bbr_init()
10056 bbr->r_ctl.rc_high_rwnd = tp->snd_wnd; in bbr_init()
10057 bbr->r_init_rtt = 1; in bbr_init()
10061 bbr->bbr_hdw_pace_ena = 1; in bbr_init()
10063 bbr->bbr_hdw_pace_ena = 0; in bbr_init()
10065 bbr->bbr_init_win_cheat = 1; in bbr_init()
10067 bbr->bbr_init_win_cheat = 0; in bbr_init()
10068 bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max; in bbr_init()
10069 bbr->r_ctl.rc_drain_pg = bbr_drain_gain; in bbr_init()
10070 bbr->r_ctl.rc_startup_pg = bbr_high_gain; in bbr_init()
10071 bbr->rc_loss_exit = bbr_exit_startup_at_loss; in bbr_init()
10072 bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain; in bbr_init()
10073 bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second; in bbr_init()
10074 bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar; in bbr_init()
10075 bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max; in bbr_init()
10076 bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor; in bbr_init()
10077 bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min; in bbr_init()
10078 bbr->r_ctl.bbr_cross_over = bbr_cross_over; in bbr_init()
10079 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_init()
10080 if (bbr->rc_use_google) { in bbr_init()
10081 setup_time_filter(&bbr->r_ctl.rc_delrate, in bbr_init()
10084 setup_time_filter_small(&bbr->r_ctl.rc_rttprop, in bbr_init()
10087 setup_time_filter(&bbr->r_ctl.rc_delrate, in bbr_init()
10090 setup_time_filter_small(&bbr->r_ctl.rc_rttprop, in bbr_init()
10095 bbr->rc_use_idle_restart = 1; in bbr_init()
10097 bbr->rc_use_idle_restart = 0; in bbr_init()
10098 bbr->r_ctl.rc_bbr_cur_del_rate = 0; in bbr_init()
10099 bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps; in bbr_init()
10101 bbr->rc_resends_use_tso = 1; in bbr_init()
10102 if (tp->snd_una != tp->snd_max) { in bbr_init()
10112 rsm->r_rtt_not_allowed = 1; in bbr_init()
10113 rsm->r_tim_lastsent[0] = cts; in bbr_init()
10114 rsm->r_rtr_cnt = 1; in bbr_init()
10115 rsm->r_rtr_bytes = 0; in bbr_init()
10116 rsm->r_start = tp->snd_una; in bbr_init()
10117 rsm->r_end = tp->snd_max; in bbr_init()
10118 rsm->r_dupack = 0; in bbr_init()
10119 rsm->r_delivered = bbr->r_ctl.rc_delivered; in bbr_init()
10120 rsm->r_ts_valid = 0; in bbr_init()
10121 rsm->r_del_ack_ts = tp->ts_recent; in bbr_init()
10122 rsm->r_del_time = cts; in bbr_init()
10123 if (bbr->r_ctl.r_app_limited_until) in bbr_init()
10124 rsm->r_app_limited = 1; in bbr_init()
10126 rsm->r_app_limited = 0; in bbr_init()
10127 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_init()
10128 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_init()
10129 rsm->r_in_tmap = 1; in bbr_init()
10130 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) in bbr_init()
10131 rsm->r_bbr_state = bbr_state_val(bbr); in bbr_init()
10133 rsm->r_bbr_state = 8; in bbr_init()
10135 if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0)) in bbr_init()
10136 bbr->bbr_use_rack_cheat = 1; in bbr_init()
10137 if (bbr_incr_timers && (bbr->rc_use_google == 0)) in bbr_init()
10138 bbr->r_ctl.rc_incr_tmrs = 1; in bbr_init()
10139 if (bbr_include_tcp_oh && (bbr->rc_use_google == 0)) in bbr_init()
10140 bbr->r_ctl.rc_inc_tcp_oh = 1; in bbr_init()
10141 if (bbr_include_ip_oh && (bbr->rc_use_google == 0)) in bbr_init()
10142 bbr->r_ctl.rc_inc_ip_oh = 1; in bbr_init()
10143 if (bbr_include_enet_oh && (bbr->rc_use_google == 0)) in bbr_init()
10144 bbr->r_ctl.rc_inc_enet_oh = 1; in bbr_init()
10147 if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_init()
10148 (tp->t_srtt)) { in bbr_init()
10151 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_init()
10152 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_init()
10170 TCPT_RANGESET(tp->t_rxtcur, in bbr_init()
10171 ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1, in bbr_init()
10172 tp->t_rttmin, TCPTV_REXMTMAX); in bbr_init()
10179 * non-zero if we can't handle the connection. A EAGAIN
10187 if ((tp->t_state == TCPS_CLOSED) || in bbr_handoff_ok()
10188 (tp->t_state == TCPS_LISTEN)) { in bbr_handoff_ok()
10192 if ((tp->t_state == TCPS_SYN_SENT) || in bbr_handoff_ok()
10193 (tp->t_state == TCPS_SYN_RECEIVED)) { in bbr_handoff_ok()
10200 if (tp->t_flags & TF_SENTFIN) in bbr_handoff_ok()
10202 if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) { in bbr_handoff_ok()
10215 if (tp->t_fb_ptr) { in bbr_fini()
10220 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_fini()
10221 if (bbr->r_ctl.crte) in bbr_fini()
10222 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp); in bbr_fini()
10224 bbr->rc_tp = NULL; in bbr_fini()
10225 if (bbr->bbr_hdrw_pacing) in bbr_fini()
10226 counter_u64_add(bbr_flows_whdwr_pacing, -1); in bbr_fini()
10228 counter_u64_add(bbr_flows_nohdwr_pacing, -1); in bbr_fini()
10229 if (bbr->r_ctl.crte != NULL) { in bbr_fini()
10230 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp); in bbr_fini()
10231 bbr->r_ctl.crte = NULL; in bbr_fini()
10233 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_fini()
10235 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_fini()
10237 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_fini()
10239 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); in bbr_fini()
10241 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_fini()
10243 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); in bbr_fini()
10245 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd; in bbr_fini()
10246 if (calc > (bbr->r_ctl.rc_init_rwnd / 10)) in bbr_fini()
10250 bbr->r_ctl.rc_free_cnt = 0; in bbr_fini()
10251 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr); in bbr_fini()
10252 tp->t_fb_ptr = NULL; in bbr_fini()
10255 tp->snd_nxt = tp->snd_max; in bbr_fini()
10261 switch (tp->t_state) { in bbr_set_state()
10263 bbr->r_state = TCPS_SYN_SENT; in bbr_set_state()
10264 bbr->r_substate = bbr_do_syn_sent; in bbr_set_state()
10267 bbr->r_state = TCPS_SYN_RECEIVED; in bbr_set_state()
10268 bbr->r_substate = bbr_do_syn_recv; in bbr_set_state()
10271 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd); in bbr_set_state()
10272 bbr->r_state = TCPS_ESTABLISHED; in bbr_set_state()
10273 bbr->r_substate = bbr_do_established; in bbr_set_state()
10276 bbr->r_state = TCPS_CLOSE_WAIT; in bbr_set_state()
10277 bbr->r_substate = bbr_do_close_wait; in bbr_set_state()
10280 bbr->r_state = TCPS_FIN_WAIT_1; in bbr_set_state()
10281 bbr->r_substate = bbr_do_fin_wait_1; in bbr_set_state()
10284 bbr->r_state = TCPS_CLOSING; in bbr_set_state()
10285 bbr->r_substate = bbr_do_closing; in bbr_set_state()
10288 bbr->r_state = TCPS_LAST_ACK; in bbr_set_state()
10289 bbr->r_substate = bbr_do_lastack; in bbr_set_state()
10292 bbr->r_state = TCPS_FIN_WAIT_2; in bbr_set_state()
10293 bbr->r_substate = bbr_do_fin_wait_2; in bbr_set_state()
10314 /* Save the lowest srtt we saw in our end of the sub-state */ in bbr_substate_change()
10315 bbr->rc_hit_state_1 = 0; in bbr_substate_change()
10316 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff) in bbr_substate_change()
10317 bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state; in bbr_substate_change()
10319 bbr->rc_bbr_substate++; in bbr_substate_change()
10320 if (bbr->rc_bbr_substate >= BBR_SUBSTATE_COUNT) { in bbr_substate_change()
10321 /* Cycle back to first state-> gain */ in bbr_substate_change()
10322 bbr->rc_bbr_substate = 0; in bbr_substate_change()
10329 if (bbr->skip_gain) { in bbr_substate_change()
10335 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1]; in bbr_substate_change()
10336 } else if (bbr->gain_is_limited && in bbr_substate_change()
10337 bbr->bbr_hdrw_pacing && in bbr_substate_change()
10338 bbr->r_ctl.crte) { in bbr_substate_change()
10348 rate = bbr->r_ctl.crte->rate; in bbr_substate_change()
10354 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc; in bbr_substate_change()
10356 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN]; in bbr_substate_change()
10359 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN]; in bbr_substate_change()
10360 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) { in bbr_substate_change()
10361 bbr->r_ctl.rc_bbr_state_atflight = cts; in bbr_substate_change()
10363 bbr->r_ctl.rc_bbr_state_atflight = 0; in bbr_substate_change()
10365 bbr->rc_hit_state_1 = 1; in bbr_substate_change()
10366 bbr->r_ctl.rc_exta_time_gd = 0; in bbr_substate_change()
10367 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp, in bbr_substate_change()
10368 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_substate_change()
10370 bbr->r_ctl.rc_bbr_state_atflight = 0; in bbr_substate_change()
10372 bbr->r_ctl.rc_bbr_state_atflight = cts; in bbr_substate_change()
10373 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN]; in bbr_substate_change()
10375 /* All other cycles hit here 2-7 */ in bbr_substate_change()
10376 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) { in bbr_substate_change()
10378 (bbr->rc_use_google == 0) && in bbr_substate_change()
10379 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { in bbr_substate_change()
10380 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_substate_change()
10383 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP)) in bbr_substate_change()
10384 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) - in bbr_substate_change()
10387 bbr->r_ctl.rc_exta_time_gd = 0; in bbr_substate_change()
10388 if (bbr->r_ctl.rc_exta_time_gd) { in bbr_substate_change()
10389 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd; in bbr_substate_change()
10391 bbr->r_ctl.rc_level_state_extra /= 7; in bbr_substate_change()
10392 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) { in bbr_substate_change()
10398 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); in bbr_substate_change()
10399 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)]; in bbr_substate_change()
10401 if (bbr->rc_use_google) { in bbr_substate_change()
10402 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); in bbr_substate_change()
10404 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_substate_change()
10405 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain; in bbr_substate_change()
10409 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_substate_change()
10412 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_substate_change()
10413 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_substate_change()
10416 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_substate_change()
10419 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff; in bbr_substate_change()
10422 (bbr->rc_use_google == 0) && in bbr_substate_change()
10425 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_substate_change()
10426 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_substate_change()
10429 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + in bbr_substate_change()
10430 ctf_flight_size(bbr->rc_tp, in bbr_substate_change()
10431 (bbr->r_ctl.rc_sacked + in bbr_substate_change()
10432 bbr->r_ctl.rc_lost_bytes))); in bbr_substate_change()
10436 if (bbr->rc_lt_use_bw) { in bbr_substate_change()
10437 /* In policed mode we clamp pacing_gain to BBR_UNIT */ in bbr_substate_change()
10438 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_substate_change()
10441 if (bbr->rc_use_google) in bbr_substate_change()
10443 bbr->r_ctl.gain_epoch = cts; in bbr_substate_change()
10444 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_substate_change()
10445 bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch; in bbr_substate_change()
10453 (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) { in bbr_set_probebw_google_gains()
10457 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_set_probebw_google_gains()
10460 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) { in bbr_set_probebw_google_gains()
10473 * we stay in GAIN (gain-to-target). in bbr_set_probebw_google_gains()
10477 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) { in bbr_set_probebw_google_gains()
10491 if (bbr->rc_use_google) { in bbr_set_probebw_gains()
10508 if (bbr->r_ctl.rc_bbr_state_atflight == 0) { in bbr_set_probebw_gains()
10510 flight = ctf_flight_size(bbr->rc_tp, in bbr_set_probebw_gains()
10511 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_set_probebw_gains()
10512 if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) { in bbr_set_probebw_gains()
10514 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) { in bbr_set_probebw_gains()
10515 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_set_probebw_gains()
10520 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight); in bbr_set_probebw_gains()
10523 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) && in bbr_set_probebw_gains()
10524 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) || in bbr_set_probebw_gains()
10525 (flight >= bbr->r_ctl.flightsize_at_drain))) { in bbr_set_probebw_gains()
10533 bbr->r_ctl.flightsize_at_drain = flight; in bbr_set_probebw_gains()
10538 bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul; in bbr_set_probebw_gains()
10539 bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div; in bbr_set_probebw_gains()
10542 bbr->r_ctl.rc_bbr_hptsi_gain *= 4; in bbr_set_probebw_gains()
10543 bbr->r_ctl.rc_bbr_hptsi_gain /= 5; in bbr_set_probebw_gains()
10545 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) { in bbr_set_probebw_gains()
10547 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1); in bbr_set_probebw_gains()
10554 bbr->r_ctl.gain_epoch = cts; in bbr_set_probebw_gains()
10556 if (flight <= bbr->r_ctl.rc_target_at_state) { in bbr_set_probebw_gains()
10558 (bbr->rc_use_google == 0) && in bbr_set_probebw_gains()
10559 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { in bbr_set_probebw_gains()
10560 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_set_probebw_gains()
10563 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10568 if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) { in bbr_set_probebw_gains()
10569 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10572 if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) || in bbr_set_probebw_gains()
10573 ((ctf_outstanding(bbr->rc_tp) + bbr->rc_tp->t_maxseg - 1) >= in bbr_set_probebw_gains()
10574 bbr->rc_tp->snd_wnd)) { in bbr_set_probebw_gains()
10575 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10591 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) in bbr_set_probebw_gains()
10593 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) { in bbr_set_probebw_gains()
10594 /* Less than a full time-period has passed */ in bbr_set_probebw_gains()
10597 if (bbr->r_ctl.rc_level_state_extra && in bbr_set_probebw_gains()
10599 ((cts - bbr->r_ctl.rc_bbr_state_time) < in bbr_set_probebw_gains()
10600 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) { in bbr_set_probebw_gains()
10601 /* Less than a full time-period + extra has passed */ in bbr_set_probebw_gains()
10605 bbr->r_ctl.rc_level_state_extra && in bbr_set_probebw_gains()
10607 ((cts - bbr->r_ctl.rc_bbr_state_time) < in bbr_set_probebw_gains()
10608 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) { in bbr_set_probebw_gains()
10609 /* Less than a full time-period + extra has passed */ in bbr_set_probebw_gains()
10620 if (bbr->rc_use_google) { in bbr_get_a_state_target()
10624 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), in bbr_get_a_state_target()
10625 bbr->r_ctl.rc_pace_max_segs); in bbr_get_a_state_target()
10641 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && in bbr_set_state_target()
10642 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) { in bbr_set_state_target()
10643 /* Special case using old probe-rtt method */ in bbr_set_state_target()
10644 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_set_state_target()
10647 /* Non-probe-rtt case and reduced probe-rtt */ in bbr_set_state_target()
10648 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && in bbr_set_state_target()
10649 (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) { in bbr_set_state_target()
10651 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain); in bbr_set_state_target()
10653 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) { in bbr_set_state_target()
10663 * for non-google mode and default (non-configured). in bbr_set_state_target()
10666 if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN]) { in bbr_set_state_target()
10670 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain); in bbr_set_state_target()
10676 bbr->r_ctl.rc_target_at_state = tar; in bbr_set_state_target()
10685 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_enter_probe_rtt()
10686 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp, in bbr_enter_probe_rtt()
10687 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_enter_probe_rtt()
10688 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain in bbr_enter_probe_rtt()
10689 + bbr->r_ctl.rc_delivered); in bbr_enter_probe_rtt()
10691 if (bbr->rc_use_google || bbr_probertt_sets_rtt) in bbr_enter_probe_rtt()
10692 bbr->rc_prtt_set_ts = 1; in bbr_enter_probe_rtt()
10693 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_enter_probe_rtt()
10694 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_enter_probe_rtt()
10695 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_enter_probe_rtt()
10698 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_enter_probe_rtt()
10699 bbr->r_ctl.last_in_probertt = cts; in bbr_enter_probe_rtt()
10700 bbr->r_ctl.rc_probertt_srttchktim = cts; in bbr_enter_probe_rtt()
10701 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_enter_probe_rtt()
10702 bbr->rc_bbr_state = BBR_STATE_PROBE_RTT; in bbr_enter_probe_rtt()
10706 bbr->rc_hit_state_1 && in bbr_enter_probe_rtt()
10707 (bbr->rc_use_google == 0) && in bbr_enter_probe_rtt()
10709 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd) in bbr_enter_probe_rtt()
10710 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_enter_probe_rtt()
10712 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_enter_probe_rtt()
10714 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_enter_probe_rtt()
10715 if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){ in bbr_enter_probe_rtt()
10716 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN) */ in bbr_enter_probe_rtt()
10717 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_enter_probe_rtt()
10719 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_enter_probe_rtt()
10720 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_enter_probe_rtt()
10721 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6); in bbr_enter_probe_rtt()
10722 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd; in bbr_enter_probe_rtt()
10729 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val; in bbr_enter_probe_rtt()
10730 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_enter_probe_rtt()
10733 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_enter_probe_rtt()
10734 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_enter_probe_rtt()
10738 if (ctf_flight_size(bbr->rc_tp, in bbr_enter_probe_rtt()
10739 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= in bbr_enter_probe_rtt()
10740 bbr->r_ctl.rc_target_at_state) { in bbr_enter_probe_rtt()
10742 bbr->r_ctl.rc_bbr_enters_probertt = cts; in bbr_enter_probe_rtt()
10745 bbr->r_ctl.rc_bbr_enters_probertt = 0; in bbr_enter_probe_rtt()
10747 bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch; in bbr_enter_probe_rtt()
10757 * Sanity check on probe-rtt intervals. in bbr_check_probe_rtt_limits()
10759 * against new-reno flows with huge buffers in bbr_check_probe_rtt_limits()
10760 * our rtt-prop interval could come to dominate in bbr_check_probe_rtt_limits()
10765 (bbr->rc_use_google == 0)) { in bbr_check_probe_rtt_limits()
10769 /* Are we to small and go into probe-rtt to often? */ in bbr_check_probe_rtt_limits()
10775 newval = cur_rttp + (fval - bbr_rtt_probe_limit); in bbr_check_probe_rtt_limits()
10784 if (cur_rttp > bbr->r_ctl.rc_probertt_int) { in bbr_check_probe_rtt_limits()
10785 bbr->r_ctl.rc_probertt_int = cur_rttp; in bbr_check_probe_rtt_limits()
10786 reset_time_small(&bbr->r_ctl.rc_rttprop, newval); in bbr_check_probe_rtt_limits()
10793 if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) { in bbr_check_probe_rtt_limits()
10799 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; in bbr_check_probe_rtt_limits()
10800 reset_time_small(&bbr->r_ctl.rc_rttprop, in bbr_check_probe_rtt_limits()
10809 if (cur_rttp < bbr->r_ctl.rc_probertt_int) { in bbr_check_probe_rtt_limits()
10811 bbr->r_ctl.rc_probertt_int = cur_rttp; in bbr_check_probe_rtt_limits()
10812 reset_time_small(&bbr->r_ctl.rc_rttprop, newval); in bbr_check_probe_rtt_limits()
10826 /* Exit probe-rtt */ in bbr_exit_probe_rtt()
10828 if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) { in bbr_exit_probe_rtt()
10829 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_exit_probe_rtt()
10833 bbr->rc_hit_state_1 = 0; in bbr_exit_probe_rtt()
10834 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_exit_probe_rtt()
10835 bbr->r_ctl.last_in_probertt = cts; in bbr_exit_probe_rtt()
10837 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_exit_probe_rtt()
10838 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, in bbr_exit_probe_rtt()
10839 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_exit_probe_rtt()
10840 bbr->r_ctl.rc_delivered); in bbr_exit_probe_rtt()
10841 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_exit_probe_rtt()
10844 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_exit_probe_rtt()
10845 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_exit_probe_rtt()
10847 if (bbr->rc_filled_pipe) { in bbr_exit_probe_rtt()
10849 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_exit_probe_rtt()
10850 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_exit_probe_rtt()
10851 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain; in bbr_exit_probe_rtt()
10856 bbr->rc_bbr_state = BBR_STATE_STARTUP; in bbr_exit_probe_rtt()
10857 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_exit_probe_rtt()
10861 * the number of pe's we were in probe-rtt in bbr_exit_probe_rtt()
10865 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt); in bbr_exit_probe_rtt()
10866 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_exit_probe_rtt()
10868 if (bbr->r_ctl.rc_lost && in bbr_exit_probe_rtt()
10870 (bbr->rc_use_google == 0)) in bbr_exit_probe_rtt()
10871 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower; in bbr_exit_probe_rtt()
10873 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; in bbr_exit_probe_rtt()
10874 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; in bbr_exit_probe_rtt()
10878 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_exit_probe_rtt()
10879 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0); in bbr_exit_probe_rtt()
10887 if ((bbr->rc_past_init_win == 1) && in bbr_should_enter_probe_rtt()
10888 (bbr->rc_in_persist == 0) && in bbr_should_enter_probe_rtt()
10889 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) { in bbr_should_enter_probe_rtt()
10893 (bbr->rc_in_persist == 0) && in bbr_should_enter_probe_rtt()
10894 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && in bbr_should_enter_probe_rtt()
10895 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { in bbr_should_enter_probe_rtt()
10907 * Need to be on a pkt-epoch to continue. in bbr_google_startup()
10912 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * in bbr_google_startup()
10913 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_google_startup()
10915 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_google_startup()
10916 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_google_startup()
10917 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3); in bbr_google_startup()
10918 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw; in bbr_google_startup()
10920 if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) in bbr_google_startup()
10922 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_google_startup()
10923 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8); in bbr_google_startup()
10935 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) && in bbr_state_startup()
10936 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_state_startup()
10949 if (bbr->rc_use_google) in bbr_state_startup()
10952 if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) && in bbr_state_startup()
10955 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower; in bbr_state_startup()
10959 * Need to be on a pkt-epoch to continue. in bbr_state_startup()
10969 if (bbr->r_ctl.rc_pkt_epoch_rtt && in bbr_state_startup()
10970 bbr->r_ctl.startup_last_srtt && in bbr_state_startup()
10971 (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) { in bbr_state_startup()
10972 delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt; in bbr_state_startup()
10973 rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt; in bbr_state_startup()
10976 if ((bbr->r_ctl.startup_last_srtt == 0) || in bbr_state_startup()
10977 (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt)) in bbr_state_startup()
10979 bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt; in bbr_state_startup()
10981 if ((bbr->r_ctl.rc_lost == 0) && in bbr_state_startup()
10988 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch) in bbr_state_startup()
10989 bbr->r_ctl.rc_bbr_last_startup_epoch++; in bbr_state_startup()
10991 delta, bbr->r_ctl.startup_last_srtt, 10); in bbr_state_startup()
10995 if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) && in bbr_state_startup()
10996 (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) && in bbr_state_startup()
10997 (!IN_RECOVERY(bbr->rc_tp->t_flags))) { in bbr_state_startup()
11002 * the number of non-gain we have already accumulated. in bbr_state_startup()
11004 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch) in bbr_state_startup()
11005 bbr->r_ctl.rc_bbr_last_startup_epoch++; in bbr_state_startup()
11006 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11007 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9); in bbr_state_startup()
11011 if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost) in bbr_state_startup()
11012 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11013 bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count; in bbr_state_startup()
11015 if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower) in bbr_state_startup()
11016 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * in bbr_state_startup()
11017 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_state_startup()
11019 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * in bbr_state_startup()
11020 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_state_startup()
11022 if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw) in bbr_state_startup()
11023 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw; in bbr_state_startup()
11025 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_state_startup()
11027 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11028 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11029 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3); in bbr_state_startup()
11031 if ((bbr->rc_loss_exit && in bbr_state_startup()
11032 (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) && in bbr_state_startup()
11033 (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) && in bbr_state_startup()
11034 ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) { in bbr_state_startup()
11042 if ((ctf_flight_size(bbr->rc_tp, in bbr_state_startup()
11043 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_state_startup()
11044 (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) { in bbr_state_startup()
11046 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11047 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4); in bbr_state_startup()
11050 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11051 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11052 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5); in bbr_state_startup()
11055 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11056 if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) || in bbr_state_startup()
11062 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11063 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8); in bbr_state_startup()
11074 if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) && in bbr_state_change()
11075 (bbr->rc_bbr_state != BBR_STATE_DRAIN) && in bbr_state_change()
11076 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) && in bbr_state_change()
11077 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) && in bbr_state_change()
11078 (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) { in bbr_state_change()
11080 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state); in bbr_state_change()
11083 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in bbr_state_change()
11088 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_change()
11089 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6); in bbr_state_change()
11090 bbr->rc_filled_pipe = 1; in bbr_state_change()
11091 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_state_change()
11092 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_state_change()
11093 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_state_change()
11094 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_state_change()
11097 if (bbr->rc_no_pacing) in bbr_state_change()
11098 bbr->rc_no_pacing = 0; in bbr_state_change()
11099 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_state_change()
11100 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg; in bbr_state_change()
11101 bbr->rc_bbr_state = BBR_STATE_DRAIN; in bbr_state_change()
11103 if ((bbr->rc_use_google == 0) && in bbr_state_change()
11105 /* Here we don't have to worry about probe-rtt */ in bbr_state_change()
11106 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_state_change()
11107 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_state_change()
11110 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain; in bbr_state_change()
11112 if (ctf_flight_size(bbr->rc_tp, in bbr_state_change()
11113 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= in bbr_state_change()
11114 bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11119 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_state_change()
11121 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_state_change()
11125 } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) { in bbr_state_change()
11129 tp = bbr->rc_tp; in bbr_state_change()
11131 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_state_change()
11132 if (inflight >= bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11134 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_state_change()
11135 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_state_change()
11136 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_state_change()
11142 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff; in bbr_state_change()
11143 bbr->rc_bbr_substate = BBR_SUB_LEVEL6; in bbr_state_change()
11146 } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) { in bbr_state_change()
11147 /* Has in-flight reached the bdp (or less)? */ in bbr_state_change()
11151 tp = bbr->rc_tp; in bbr_state_change()
11153 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_state_change()
11154 if ((bbr->rc_use_google == 0) && in bbr_state_change()
11156 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_state_change()
11158 * Here we don't have to worry about probe-rtt in bbr_state_change()
11159 * re-slam it, but keep it slammed down. in bbr_state_change()
11161 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_state_change()
11164 if (inflight <= bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11166 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_state_change()
11167 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_state_change()
11168 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_state_change()
11171 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_state_change()
11172 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_state_change()
11174 if ((bbr->rc_use_google == 0) && in bbr_state_change()
11176 (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { in bbr_state_change()
11178 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_state_change()
11181 /* Setup probe-rtt has being done now RRS-HERE */ in bbr_state_change()
11182 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_state_change()
11183 bbr->r_ctl.last_in_probertt = cts; in bbr_state_change()
11185 /* Randomly pick a sub-state */ in bbr_state_change()
11186 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_state_change()
11190 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) { in bbr_state_change()
11193 flight = ctf_flight_size(bbr->rc_tp, in bbr_state_change()
11194 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_state_change()
11195 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered); in bbr_state_change()
11196 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) && in bbr_state_change()
11197 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_state_change()
11201 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_state_change()
11204 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_state_change()
11205 /* Re-slam it */ in bbr_state_change()
11206 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_state_change()
11209 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) { in bbr_state_change()
11211 if (flight <= bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11213 bbr->r_ctl.rc_bbr_enters_probertt = cts; in bbr_state_change()
11215 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) in bbr_state_change()
11216 bbr->r_ctl.rc_bbr_enters_probertt = 1; in bbr_state_change()
11217 if (bbr->rc_use_google == 0) { in bbr_state_change()
11222 if (bbr->r_ctl.bbr_rttprobe_gain_val) in bbr_state_change()
11223 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val; in bbr_state_change()
11225 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_state_change()
11228 if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) && in bbr_state_change()
11229 (bbr->rc_use_google == 0) && in bbr_state_change()
11230 bbr->r_ctl.bbr_rttprobe_gain_val && in bbr_state_change()
11231 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) || in bbr_state_change()
11232 (flight >= bbr->r_ctl.flightsize_at_drain))) { in bbr_state_change()
11243 bbr->r_ctl.flightsize_at_drain = flight; in bbr_state_change()
11244 bbr->r_ctl.rc_probertt_srttchktim = cts; in bbr_state_change()
11245 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1); in bbr_state_change()
11246 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) { in bbr_state_change()
11248 bbr->r_ctl.rc_bbr_hptsi_gain -= red; in bbr_state_change()
11250 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) { in bbr_state_change()
11252 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1); in bbr_state_change()
11256 bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1); in bbr_state_change()
11260 if (bbr->r_ctl.rc_bbr_enters_probertt && in bbr_state_change()
11261 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) && in bbr_state_change()
11262 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) { in bbr_state_change()
11264 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts); in bbr_state_change()
11266 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_state_change()
11267 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) && in bbr_state_change()
11268 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_state_change()
11290 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) { in bbr_check_bbr_for_state()
11295 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses); in bbr_check_bbr_for_state()
11317 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_do_segment_nounlock()
11318 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_segment_nounlock()
11324 * If this is either a state-changing packet or current state isn't in bbr_do_segment_nounlock()
11331 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", in bbr_do_segment_nounlock()
11333 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", in bbr_do_segment_nounlock()
11336 tp->t_rcvtime = ticks; in bbr_do_segment_nounlock()
11338 * Unscale the window into a 32-bit value. For the SYN_SENT state in bbr_do_segment_nounlock()
11341 tiwin = th->th_win << tp->snd_scale; in bbr_do_segment_nounlock()
11343 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin); in bbr_do_segment_nounlock()
11346 if (m->m_flags & M_TSTMP) { in bbr_do_segment_nounlock()
11351 bbr->rc_tv.tv_sec = ts.tv_sec; in bbr_do_segment_nounlock()
11352 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000; in bbr_do_segment_nounlock()
11353 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv); in bbr_do_segment_nounlock()
11354 } else if (m->m_flags & M_TSTMP_LRO) { in bbr_do_segment_nounlock()
11359 bbr->rc_tv.tv_sec = ts.tv_sec; in bbr_do_segment_nounlock()
11360 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000; in bbr_do_segment_nounlock()
11361 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv); in bbr_do_segment_nounlock()
11366 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv); in bbr_do_segment_nounlock()
11372 (th->th_off << 2) - sizeof(struct tcphdr), in bbr_do_segment_nounlock()
11374 if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) { in bbr_do_segment_nounlock()
11389 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) && in bbr_do_segment_nounlock()
11401 to.to_tsecr -= tp->ts_offset; in bbr_do_segment_nounlock()
11402 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv))) in bbr_do_segment_nounlock()
11409 if (bbr->r_state == 0) { in bbr_do_segment_nounlock()
11417 if (bbr->rc_inp == NULL) { in bbr_do_segment_nounlock()
11418 bbr->rc_inp = inp; in bbr_do_segment_nounlock()
11424 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { in bbr_do_segment_nounlock()
11426 (tp->t_flags & TF_REQ_SCALE)) { in bbr_do_segment_nounlock()
11427 tp->t_flags |= TF_RCVD_SCALE; in bbr_do_segment_nounlock()
11428 tp->snd_scale = to.to_wscale; in bbr_do_segment_nounlock()
11430 tp->t_flags &= ~TF_REQ_SCALE; in bbr_do_segment_nounlock()
11435 tp->snd_wnd = th->th_win; in bbr_do_segment_nounlock()
11437 (tp->t_flags & TF_REQ_TSTMP)) { in bbr_do_segment_nounlock()
11438 tp->t_flags |= TF_RCVD_TSTMP; in bbr_do_segment_nounlock()
11439 tp->ts_recent = to.to_tsval; in bbr_do_segment_nounlock()
11440 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_segment_nounlock()
11442 tp->t_flags &= ~TF_REQ_TSTMP; in bbr_do_segment_nounlock()
11445 if ((tp->t_flags & TF_SACK_PERMIT) && in bbr_do_segment_nounlock()
11447 tp->t_flags &= ~TF_SACK_PERMIT; in bbr_do_segment_nounlock()
11448 if (tp->t_flags & TF_FASTOPEN) { in bbr_do_segment_nounlock()
11455 if ((inp->inp_vflag & INP_IPV6) != 0) in bbr_do_segment_nounlock()
11471 if ((tp->t_flags & TF_SACK_PERMIT) == 0) { in bbr_do_segment_nounlock()
11474 (*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen, in bbr_do_segment_nounlock()
11479 bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0; in bbr_do_segment_nounlock()
11481 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack); in bbr_do_segment_nounlock()
11492 * always. All other times (timers etc) we must have a rack-state in bbr_do_segment_nounlock()
11497 if (bbr->r_state != tp->t_state) in bbr_do_segment_nounlock()
11500 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL) in bbr_do_segment_nounlock()
11502 prev_state = bbr->r_state; in bbr_do_segment_nounlock()
11503 bbr->rc_ack_was_delayed = 0; in bbr_do_segment_nounlock()
11504 lost = bbr->r_ctl.rc_lost; in bbr_do_segment_nounlock()
11505 bbr->rc_is_pkt_epoch_now = 0; in bbr_do_segment_nounlock()
11506 if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) { in bbr_do_segment_nounlock()
11510 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts; in bbr_do_segment_nounlock()
11511 bbr->rc_ack_was_delayed = 1; in bbr_do_segment_nounlock()
11512 if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay, in bbr_do_segment_nounlock()
11513 bbr->r_ctl.highest_hdwr_delay)) in bbr_do_segment_nounlock()
11514 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay; in bbr_do_segment_nounlock()
11516 bbr->r_ctl.rc_ack_hdwr_delay = 0; in bbr_do_segment_nounlock()
11517 bbr->rc_ack_was_delayed = 0; in bbr_do_segment_nounlock()
11520 bbr->r_ctl.rc_ack_hdwr_delay = 0; in bbr_do_segment_nounlock()
11521 bbr->rc_ack_was_delayed = 0; in bbr_do_segment_nounlock()
11530 * If a segment with the ACK-bit set arrives in the SYN-SENT state in bbr_do_segment_nounlock()
11533 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && in bbr_do_segment_nounlock()
11534 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_do_segment_nounlock()
11539 if (tiwin > bbr->r_ctl.rc_high_rwnd) in bbr_do_segment_nounlock()
11540 bbr->r_ctl.rc_high_rwnd = tiwin; in bbr_do_segment_nounlock()
11541 bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp, in bbr_do_segment_nounlock()
11542 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_do_segment_nounlock()
11543 bbr->rtt_valid = 0; in bbr_do_segment_nounlock()
11545 bbr->rc_ts_valid = 1; in bbr_do_segment_nounlock()
11546 bbr->r_ctl.last_inbound_ts = to.to_tsval; in bbr_do_segment_nounlock()
11548 bbr->rc_ts_valid = 0; in bbr_do_segment_nounlock()
11549 bbr->r_ctl.last_inbound_ts = 0; in bbr_do_segment_nounlock()
11551 retval = (*bbr->r_substate) (m, th, so, in bbr_do_segment_nounlock()
11565 if (bbr->rc_is_pkt_epoch_now) in bbr_do_segment_nounlock()
11567 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost)); in bbr_do_segment_nounlock()
11569 if ((bbr->r_wanted_output != 0) || in bbr_do_segment_nounlock()
11570 (tp->t_flags & TF_ACKNOW)) { in bbr_do_segment_nounlock()
11572 bbr->rc_output_starts_timer = 0; in bbr_do_segment_nounlock()
11580 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) && in bbr_do_segment_nounlock()
11581 (SEQ_GT(tp->snd_max, tp->snd_una) || in bbr_do_segment_nounlock()
11582 (tp->t_flags & TF_DELACK) || in bbr_do_segment_nounlock()
11583 ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_do_segment_nounlock()
11584 (tp->t_state <= TCPS_CLOSING)))) { in bbr_do_segment_nounlock()
11589 if ((tp->snd_max == tp->snd_una) && in bbr_do_segment_nounlock()
11590 ((tp->t_flags & TF_DELACK) == 0) && in bbr_do_segment_nounlock()
11592 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { in bbr_do_segment_nounlock()
11601 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && in bbr_do_segment_nounlock()
11602 (TSTMP_GT(lcts, bbr->rc_pacer_started))) { in bbr_do_segment_nounlock()
11605 del = lcts - bbr->rc_pacer_started; in bbr_do_segment_nounlock()
11606 if (bbr->r_ctl.rc_last_delay_val > del) { in bbr_do_segment_nounlock()
11608 bbr->r_ctl.rc_last_delay_val -= del; in bbr_do_segment_nounlock()
11609 bbr->rc_pacer_started = lcts; in bbr_do_segment_nounlock()
11612 bbr->r_ctl.rc_last_delay_val = 0; in bbr_do_segment_nounlock()
11619 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val, in bbr_do_segment_nounlock()
11622 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) { in bbr_do_segment_nounlock()
11624 bbr_timer_audit(tp, bbr, lcts, &so->so_snd); in bbr_do_segment_nounlock()
11627 if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS)) in bbr_do_segment_nounlock()
11628 tp->t_flags2 &= ~TF2_HPTS_CALLS; in bbr_do_segment_nounlock()
11630 if (bbr->r_state != tp->t_state) in bbr_do_segment_nounlock()
11635 bbr->r_wanted_output = 0; in bbr_do_segment_nounlock()
11648 if (!STAILQ_EMPTY(&tp->t_inqueue)) { in bbr_do_segment()
11654 if (m->m_flags & M_TSTMP_LRO) { in bbr_do_segment()
11678 if (ctf_outstanding(tp) >= tp->snd_wnd) { in bbr_what_can_we_send()
11679 /* We never want to go over our peers rcv-window */ in bbr_what_can_we_send()
11684 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_what_can_we_send()
11689 * >= tp->snd_wnd). in bbr_what_can_we_send()
11693 len = sendwin - flight; in bbr_what_can_we_send()
11694 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) { in bbr_what_can_we_send()
11696 len = tp->snd_wnd - ctf_outstanding(tp); in bbr_what_can_we_send()
11703 len = avail - sb_offset; in bbr_what_can_we_send()
11716 if (rsm->r_flags & BBR_TLP) { in bbr_do_send_accounting()
11725 tp->t_sndrexmitpack++; in bbr_do_send_accounting()
11729 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB, in bbr_do_send_accounting()
11734 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is in bbr_do_send_accounting()
11735 * sub-state in bbr_do_send_accounting()
11737 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len); in bbr_do_send_accounting()
11738 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) { in bbr_do_send_accounting()
11740 counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len); in bbr_do_send_accounting()
11743 * Log our probe state 3, and log also 5-13 to show in bbr_do_send_accounting()
11744 * us the recovery sub-state for the send. This in bbr_do_send_accounting()
11765 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB, in bbr_do_send_accounting()
11774 if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) { in bbr_cwnd_limiting()
11785 if (tp->snd_cwnd > target) in bbr_cwnd_limiting()
11786 tp->snd_cwnd = target; in bbr_cwnd_limiting()
11796 * account that we are limited by TCP_MAXWIN << tp->rcv_scale. in bbr_window_update_needed()
11802 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) { in bbr_window_update_needed()
11803 oldwin = (tp->rcv_adv - tp->rcv_nxt); in bbr_window_update_needed()
11805 adv -= oldwin; in bbr_window_update_needed()
11818 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale) in bbr_window_update_needed()
11822 (adv >= (so->so_rcv.sb_hiwat / 4) || in bbr_window_update_needed()
11823 recwin <= (so->so_rcv.sb_hiwat / 8) || in bbr_window_update_needed()
11824 so->so_rcv.sb_hiwat <= 8 * maxseg)) { in bbr_window_update_needed()
11827 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat) in bbr_window_update_needed()
11835 * if the TCB was on the hpts. A non-zero return
11893 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_output_wtime()
11895 memcpy(&bbr->rc_tv, tv, sizeof(struct timeval)); in bbr_output_wtime()
11896 cts = tcp_tv_to_usectick(&bbr->rc_tv); in bbr_output_wtime()
11897 inp = bbr->rc_inp; in bbr_output_wtime()
11898 hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS); in bbr_output_wtime()
11899 tp->t_flags2 &= ~TF2_HPTS_CALLS; in bbr_output_wtime()
11900 so = inp->inp_socket; in bbr_output_wtime()
11901 sb = &so->so_snd; in bbr_output_wtime()
11902 if (tp->t_nic_ktls_xmit) in bbr_output_wtime()
11907 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_output_wtime()
11912 pace_max_segs = bbr->r_ctl.rc_pace_max_segs; in bbr_output_wtime()
11916 if (tp->t_flags & TF_TOE) in bbr_output_wtime()
11921 if (bbr->r_state) { in bbr_output_wtime()
11923 isipv6 = bbr->r_is_v6; in bbr_output_wtime()
11925 isipv6 = (inp->inp_vflag & INP_IPV6) != 0; in bbr_output_wtime()
11928 if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && in bbr_output_wtime()
11934 if ((tp->t_flags & TF_ACKNOW) == 0) { in bbr_output_wtime()
11939 * whatever timer is running (KEEP/DEL-ACK?) and in bbr_output_wtime()
11944 recwin = lmin(lmax(sbspace(&so->so_rcv), 0), in bbr_output_wtime()
11945 (long)TCP_MAXWIN << tp->rcv_scale); in bbr_output_wtime()
11947 ((tcp_outflags[tp->t_state] & TH_RST) == 0) && in bbr_output_wtime()
11948 ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <= in bbr_output_wtime()
11949 (tp->snd_max - tp->snd_una))) { in bbr_output_wtime()
11953 * let the timer-run off. in bbr_output_wtime()
11961 if (bbr->r_ctl.rc_last_delay_val) { in bbr_output_wtime()
11963 if (SEQ_GT(cts, bbr->rc_pacer_started)) in bbr_output_wtime()
11964 delay_calc = cts - bbr->rc_pacer_started; in bbr_output_wtime()
11965 if (delay_calc >= bbr->r_ctl.rc_last_delay_val) in bbr_output_wtime()
11966 delay_calc -= bbr->r_ctl.rc_last_delay_val; in bbr_output_wtime()
11971 if ((bbr->r_timer_override) || in bbr_output_wtime()
11972 (tp->t_state < TCPS_ESTABLISHED)) { in bbr_output_wtime()
11977 if ((bbr->r_ctl.rc_last_delay_val) && in bbr_output_wtime()
11978 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && in bbr_output_wtime()
11987 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
11989 } else if (tp->t_state == TCPS_CLOSED) { in bbr_output_wtime()
11990 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12002 bbr->rc_cwnd_limited = 0; in bbr_output_wtime()
12003 if (bbr->r_ctl.rc_last_delay_val) { in bbr_output_wtime()
12005 if (SEQ_GT(cts, bbr->rc_pacer_started)) in bbr_output_wtime()
12006 delay_calc = cts - bbr->rc_pacer_started; in bbr_output_wtime()
12009 if (delay_calc >= bbr->r_ctl.rc_last_delay_val) in bbr_output_wtime()
12011 delay_calc -= bbr->r_ctl.rc_last_delay_val; in bbr_output_wtime()
12019 bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc); in bbr_output_wtime()
12020 bbr->r_agg_early_set = 1; in bbr_output_wtime()
12021 if (bbr->r_ctl.rc_hptsi_agg_delay) { in bbr_output_wtime()
12022 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) { in bbr_output_wtime()
12024 bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early; in bbr_output_wtime()
12025 bbr->r_agg_early_set = 0; in bbr_output_wtime()
12026 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
12028 bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay; in bbr_output_wtime()
12029 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_output_wtime()
12032 merged_val = bbr->rc_pacer_started; in bbr_output_wtime()
12034 merged_val |= bbr->r_ctl.rc_last_delay_val; in bbr_output_wtime()
12036 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val, in bbr_output_wtime()
12037 bbr->r_agg_early_set, 3); in bbr_output_wtime()
12038 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12044 if (bbr->r_agg_early_set) in bbr_output_wtime()
12045 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
12046 bbr->r_agg_early_set = 0; in bbr_output_wtime()
12055 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay) in bbr_output_wtime()
12056 bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff; in bbr_output_wtime()
12058 bbr->r_ctl.rc_hptsi_agg_delay += delay_calc; in bbr_output_wtime()
12060 sendwin = min(tp->snd_wnd, tp->snd_cwnd); in bbr_output_wtime()
12061 if ((tp->snd_una == tp->snd_max) && in bbr_output_wtime()
12062 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) && in bbr_output_wtime()
12067 * suite of states or a fast-ramp up. in bbr_output_wtime()
12070 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time)); in bbr_output_wtime()
12078 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { in bbr_output_wtime()
12091 bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY; in bbr_output_wtime()
12093 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { in bbr_output_wtime()
12094 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12096 bbr->r_timer_override = 0; in bbr_output_wtime()
12097 bbr->r_wanted_output = 0; in bbr_output_wtime()
12102 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12103 ((tp->t_state == TCPS_SYN_RECEIVED) || in bbr_output_wtime()
12104 (tp->t_state == TCPS_SYN_SENT)) && in bbr_output_wtime()
12105 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */ in bbr_output_wtime()
12106 (tp->t_rxtshift == 0)) { /* not a retransmit */ in bbr_output_wtime()
12115 if (bbr->rc_use_google == 0) in bbr_output_wtime()
12134 sendwin = min(tp->snd_wnd, tp->snd_cwnd); in bbr_output_wtime()
12135 sb_offset = tp->snd_max - tp->snd_una; in bbr_output_wtime()
12136 flags = tcp_outflags[tp->t_state]; in bbr_output_wtime()
12145 while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) { in bbr_output_wtime()
12151 tot_len = tp->t_maxseg; in bbr_output_wtime()
12157 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_output_wtime()
12158 bbr->r_ctl.rc_free_cnt++; in bbr_output_wtime()
12162 if (bbr->r_ctl.rc_resend == NULL) { in bbr_output_wtime()
12164 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_output_wtime()
12165 if (bbr->r_ctl.rc_resend) { in bbr_output_wtime()
12169 bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend); in bbr_output_wtime()
12172 if (bbr->r_ctl.rc_resend) { in bbr_output_wtime()
12173 rsm = bbr->r_ctl.rc_resend; in bbr_output_wtime()
12177 /* Remove any TLP flags its a RACK or T-O */ in bbr_output_wtime()
12178 rsm->r_flags &= ~BBR_TLP; in bbr_output_wtime()
12179 bbr->r_ctl.rc_resend = NULL; in bbr_output_wtime()
12180 if (SEQ_LT(rsm->r_start, tp->snd_una)) { in bbr_output_wtime()
12183 tp, bbr, rsm, rsm->r_start, tp->snd_una); in bbr_output_wtime()
12191 if (rsm->r_flags & BBR_HAS_SYN) { in bbr_output_wtime()
12196 rsm->r_start++; in bbr_output_wtime()
12197 if (rsm->r_start == rsm->r_end) { in bbr_output_wtime()
12202 rsm->r_flags &= ~BBR_HAS_SYN; in bbr_output_wtime()
12203 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12212 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12213 if ((bbr->rc_resends_use_tso == 0) && in bbr_output_wtime()
12218 sb_offset = rsm->r_start - tp->snd_una; in bbr_output_wtime()
12230 } else if (bbr->r_ctl.rc_tlp_send) { in bbr_output_wtime()
12235 rsm = bbr->r_ctl.rc_tlp_send; in bbr_output_wtime()
12236 bbr->r_ctl.rc_tlp_send = NULL; in bbr_output_wtime()
12238 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12239 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg)) in bbr_output_wtime()
12242 if (SEQ_GT(tp->snd_una, rsm->r_start)) { in bbr_output_wtime()
12245 tp, bbr, tp->snd_una, rsm, rsm->r_start); in bbr_output_wtime()
12252 sb_offset = rsm->r_start - tp->snd_una; in bbr_output_wtime()
12261 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { in bbr_output_wtime()
12263 if (!bbr->alloc_limit_reported) { in bbr_output_wtime()
12264 bbr->alloc_limit_reported = 1; in bbr_output_wtime()
12270 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) { in bbr_output_wtime()
12279 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL)) in bbr_output_wtime()
12281 if (tp->t_flags & TF_NEEDSYN) in bbr_output_wtime()
12284 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) { in bbr_output_wtime()
12286 len--; in bbr_output_wtime()
12302 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext); in bbr_output_wtime()
12313 * fast-retransmit because TCP will reset snd_nxt to snd_max after in bbr_output_wtime()
12314 * the fast-retransmit. in bbr_output_wtime()
12316 * In the normal retransmit-FIN-only case, however, snd_nxt will be in bbr_output_wtime()
12327 if (SEQ_GT(tp->snd_max, tp->snd_una)) in bbr_output_wtime()
12328 sb_offset = tp->snd_max - tp->snd_una; in bbr_output_wtime()
12331 if (bbr->rc_tlp_new_data) { in bbr_output_wtime()
12338 if (tlplen > (uint32_t)(avail - sb_offset)) { in bbr_output_wtime()
12339 tlplen = (uint32_t)(avail - sb_offset); in bbr_output_wtime()
12341 if (tlplen > tp->snd_wnd) { in bbr_output_wtime()
12342 len = tp->snd_wnd; in bbr_output_wtime()
12346 bbr->rc_tlp_new_data = 0; in bbr_output_wtime()
12350 (bbr->rc_in_persist == 0) && in bbr_output_wtime()
12352 ((avail - sb_offset) >= p_maxseg)) { in bbr_output_wtime()
12364 if (bbr->rc_in_persist) { in bbr_output_wtime()
12371 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_output_wtime()
12373 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12374 if (rsm->r_flags & BBR_HAS_FIN) in bbr_output_wtime()
12375 len--; in bbr_output_wtime()
12376 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg)) in bbr_output_wtime()
12389 sb_offset = rsm->r_start - tp->snd_una; in bbr_output_wtime()
12410 * SYN-SENT state and if segment contains data and if we don't know in bbr_output_wtime()
12414 SEQ_GT(tp->snd_max, tp->snd_una)) { in bbr_output_wtime()
12415 if (tp->t_state != TCPS_SYN_RECEIVED) in bbr_output_wtime()
12421 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12422 (tp->t_state == TCPS_SYN_RECEIVED)) in bbr_output_wtime()
12424 sb_offset--, len++; in bbr_output_wtime()
12432 len--; in bbr_output_wtime()
12439 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) { in bbr_output_wtime()
12446 * - When retransmitting SYN|ACK on a passively-created socket in bbr_output_wtime()
12447 * - When retransmitting SYN on an actively created socket in bbr_output_wtime()
12448 * - When sending a zero-length cookie (cookie request) on an in bbr_output_wtime()
12450 * - When the socket is in the CLOSED state (RST is being sent) in bbr_output_wtime()
12452 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12453 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) || in bbr_output_wtime()
12454 ((tp->t_state == TCPS_SYN_SENT) && in bbr_output_wtime()
12455 (tp->t_tfo_client_cookie_len == 0)) || in bbr_output_wtime()
12461 /* Without fast-open there should never be data sent on a SYN */ in bbr_output_wtime()
12462 if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN)) in bbr_output_wtime()
12476 * 0-byte window. This makes sure the persist timer is set in bbr_output_wtime()
12481 if ((tp->snd_wnd == 0) && in bbr_output_wtime()
12482 (TCPS_HAVEESTABLISHED(tp->t_state)) && in bbr_output_wtime()
12483 (tp->snd_una == tp->snd_max) && in bbr_output_wtime()
12493 (len < bbr->r_ctl.rc_pace_max_segs)) { in bbr_output_wtime()
12499 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_output_wtime()
12500 (TCPS_HAVEESTABLISHED(tp->t_state)) && in bbr_output_wtime()
12501 (len < (int)(sbavail(sb) - sb_offset))) { in bbr_output_wtime()
12510 if (tp->snd_max == tp->snd_una) in bbr_output_wtime()
12512 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) && in bbr_output_wtime()
12513 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12514 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) && in bbr_output_wtime()
12515 (len < (int)(sbavail(sb) - sb_offset)) && in bbr_output_wtime()
12527 bbr->rc_cwnd_limited = 1; in bbr_output_wtime()
12529 } else if (((tp->snd_wnd - ctf_outstanding(tp)) < in bbr_output_wtime()
12530 min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_output_wtime()
12531 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12532 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) && in bbr_output_wtime()
12533 (len < (int)(sbavail(sb) - sb_offset)) && in bbr_output_wtime()
12534 (TCPS_HAVEESTABLISHED(tp->t_state))) { in bbr_output_wtime()
12552 if (bbr->rc_in_persist && in bbr_output_wtime()
12555 (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) { in bbr_output_wtime()
12563 sbleft = sbavail(sb) - sb_offset; in bbr_output_wtime()
12566 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) { in bbr_output_wtime()
12576 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP in bbr_output_wtime()
12593 if (inp->inp_options) in bbr_output_wtime()
12594 ipoptlen = inp->inp_options->m_len - in bbr_output_wtime()
12600 * Pre-calculate here as we save another lookup into the darknesses in bbr_output_wtime()
12618 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && in bbr_output_wtime()
12620 (tp->t_port == 0) && in bbr_output_wtime()
12621 ((tp->t_flags & TF_SIGNATURE) == 0) && in bbr_output_wtime()
12625 recwin = lmin(lmax(sbspace(&so->so_rcv), 0), in bbr_output_wtime()
12626 (long)TCP_MAXWIN << tp->rcv_scale); in bbr_output_wtime()
12629 * conditions when len is non-zero: in bbr_output_wtime()
12631 * - We have a full segment (or more with TSO) - This is the last in bbr_output_wtime()
12633 * NODELAY - we've timed out (e.g. persist timer) - we have more in bbr_output_wtime()
12635 * limited the window size) - we need to retransmit in bbr_output_wtime()
12650 if (((tp->t_flags & TF_MORETOCOME) == 0) && /* normal case */ in bbr_output_wtime()
12651 ((tp->t_flags & TF_NODELAY) || in bbr_output_wtime()
12652 ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) && in bbr_output_wtime()
12653 (tp->t_flags & TF_NOPUSH) == 0) { in bbr_output_wtime()
12656 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */ in bbr_output_wtime()
12659 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) { in bbr_output_wtime()
12687 * pending (it will get piggy-backed on it) or the remote side in bbr_output_wtime()
12688 * already has done a half-close and won't send more data. Skip in bbr_output_wtime()
12689 * this if the connection is in T/TCP half-open state. in bbr_output_wtime()
12691 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) && in bbr_output_wtime()
12692 !(tp->t_flags & TF_DELACK) && in bbr_output_wtime()
12693 !TCPS_HAVERCVDFIN(tp->t_state)) { in bbr_output_wtime()
12700 * is also a catch-all for the retransmit timer timeout case. in bbr_output_wtime()
12702 if (tp->t_flags & TF_ACKNOW) { in bbr_output_wtime()
12709 if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) { in bbr_output_wtime()
12717 ((tp->t_flags & TF_SENTFIN) == 0)) { in bbr_output_wtime()
12727 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); in bbr_output_wtime()
12728 if (bbr->rc_no_pacing) in bbr_output_wtime()
12731 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >= in bbr_output_wtime()
12732 tp->snd_wnd) { in bbr_output_wtime()
12736 if ((bbr->rc_in_persist == 0) && in bbr_output_wtime()
12737 TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_output_wtime()
12738 (tp->snd_max == tp->snd_una) && in bbr_output_wtime()
12739 sbavail(&so->so_snd)) { in bbr_output_wtime()
12741 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_output_wtime()
12747 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12748 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) { in bbr_output_wtime()
12751 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12752 bbr->r_ctl.rc_lost_bytes))); in bbr_output_wtime()
12753 bbr->rc_cwnd_limited = 1; in bbr_output_wtime()
12759 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_output_wtime()
12760 bbr->r_agg_early_set = 0; in bbr_output_wtime()
12761 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
12762 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12763 } else if (bbr->rc_use_google == 0) in bbr_output_wtime()
12771 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12772 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered); in bbr_output_wtime()
12777 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12778 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
12781 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_output_wtime()
12783 tp->snd_nxt = tp->snd_max; in bbr_output_wtime()
12796 bbr->rc_tlp_in_progress = 0; in bbr_output_wtime()
12797 bbr->rc_tlp_rtx_out = 0; in bbr_output_wtime()
12802 bbr->rc_tlp_in_progress = 1; in bbr_output_wtime()
12808 * This is sub-optimal. We only send a stand alone in bbr_output_wtime()
12813 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) { in bbr_output_wtime()
12826 (((rsm->r_flags & BBR_HAS_FIN) == 0) && in bbr_output_wtime()
12829 len--; in bbr_output_wtime()
12834 if ((tp->snd_una == tp->snd_max) && in bbr_output_wtime()
12835 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_output_wtime()
12845 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT; in bbr_output_wtime()
12847 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT; in bbr_output_wtime()
12866 * established connection segments. Options for SYN-ACK segments in bbr_output_wtime()
12871 if ((tp->t_flags & TF_NOOPT) == 0) { in bbr_output_wtime()
12874 to.to_mss = tcp_mssopt(&inp->inp_inc); in bbr_output_wtime()
12875 if (tp->t_port) in bbr_output_wtime()
12876 to.to_mss -= V_tcp_udp_tunneling_overhead; in bbr_output_wtime()
12885 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12886 (tp->t_rxtshift == 0)) { in bbr_output_wtime()
12887 if (tp->t_state == TCPS_SYN_RECEIVED) { in bbr_output_wtime()
12890 (u_int8_t *)&tp->t_tfo_cookie.server; in bbr_output_wtime()
12893 } else if (tp->t_state == TCPS_SYN_SENT) { in bbr_output_wtime()
12895 tp->t_tfo_client_cookie_len; in bbr_output_wtime()
12897 tp->t_tfo_cookie.client; in bbr_output_wtime()
12904 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) { in bbr_output_wtime()
12905 to.to_wscale = tp->request_r_scale; in bbr_output_wtime()
12909 if ((tp->t_flags & TF_RCVD_TSTMP) || in bbr_output_wtime()
12910 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) { in bbr_output_wtime()
12911 to.to_tsval = tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset; in bbr_output_wtime()
12912 to.to_tsecr = tp->ts_recent; in bbr_output_wtime()
12917 if (tp->rfbuf_ts == 0 && in bbr_output_wtime()
12918 (so->so_rcv.sb_flags & SB_AUTOSIZE)) in bbr_output_wtime()
12919 tp->rfbuf_ts = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_output_wtime()
12923 else if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_output_wtime()
12924 tp->rcv_numsacks > 0) { in bbr_output_wtime()
12926 to.to_nsacks = tp->rcv_numsacks; in bbr_output_wtime()
12927 to.to_sacks = (u_char *)tp->sackblks; in bbr_output_wtime()
12930 /* TCP-MD5 (RFC2385). */ in bbr_output_wtime()
12931 if (tp->t_flags & TF_SIGNATURE) in bbr_output_wtime()
12941 if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie && in bbr_output_wtime()
12945 if (tp->t_port) { in bbr_output_wtime()
12958 if (inp->inp_options) in bbr_output_wtime()
12959 ipoptlen = inp->inp_options->m_len - in bbr_output_wtime()
12967 if (bbr->rc_last_options != local_options) { in bbr_output_wtime()
12972 bbr->rc_last_options = local_options; in bbr_output_wtime()
12974 maxseg = tp->t_maxseg - (ipoptlen + optlen); in bbr_output_wtime()
12990 if_hw_tsomax = tp->t_tsomax; in bbr_output_wtime()
12991 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; in bbr_output_wtime()
12992 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; in bbr_output_wtime()
13002 max_len = (if_hw_tsomax - hdrlen - in bbr_output_wtime()
13017 len -= moff; in bbr_output_wtime()
13030 if (optlen + ipoptlen >= tp->t_maxseg) { in bbr_output_wtime()
13068 if (SEQ_LT(rsm->r_start, tp->snd_una)) { in bbr_output_wtime()
13070 rsm, tp, bbr, rsm->r_start, tp->snd_una); in bbr_output_wtime()
13114 m->m_data += max_linkhdr; in bbr_output_wtime()
13115 m->m_len = hdrlen; in bbr_output_wtime()
13140 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_output_wtime()
13147 if (rsm->r_start != tp->snd_una) { in bbr_output_wtime()
13155 return (-EFAULT); /* tcp_drop() */ in bbr_output_wtime()
13157 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
13165 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) { in bbr_output_wtime()
13170 m->m_len += len; in bbr_output_wtime()
13184 tp->snd_una, rsm->r_flags, rsm->r_start, in bbr_output_wtime()
13190 tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una); in bbr_output_wtime()
13194 m->m_next = tcp_m_copym( in bbr_output_wtime()
13212 if (m->m_next == NULL) { in bbr_output_wtime()
13232 foo = foo->m_next; in bbr_output_wtime()
13252 if (tp->t_flags & TF_ACKNOW) in bbr_output_wtime()
13274 m->m_data += max_linkhdr; in bbr_output_wtime()
13275 m->m_len = hdrlen; in bbr_output_wtime()
13278 m->m_pkthdr.rcvif = (struct ifnet *)0; in bbr_output_wtime()
13285 if (tp->t_port) { in bbr_output_wtime()
13287 udp->uh_sport = htons(V_tcp_udp_tunneling_port); in bbr_output_wtime()
13288 udp->uh_dport = tp->t_port; in bbr_output_wtime()
13289 ulen = hdrlen + len - sizeof(struct ip6_hdr); in bbr_output_wtime()
13290 udp->uh_ulen = htons(ulen); in bbr_output_wtime()
13295 tcpip_fillheaders(inp, tp->t_port, ip6, th); in bbr_output_wtime()
13300 if (tp->t_port) { in bbr_output_wtime()
13302 udp->uh_sport = htons(V_tcp_udp_tunneling_port); in bbr_output_wtime()
13303 udp->uh_dport = tp->t_port; in bbr_output_wtime()
13304 ulen = hdrlen + len - sizeof(struct ip); in bbr_output_wtime()
13305 udp->uh_ulen = htons(ulen); in bbr_output_wtime()
13310 tcpip_fillheaders(inp, tp->t_port, ip, th); in bbr_output_wtime()
13326 th->th_seq = htonl(tp->snd_max); in bbr_output_wtime()
13327 bbr_seq = tp->snd_max; in bbr_output_wtime()
13330 th->th_seq = htonl(tp->iss); in bbr_output_wtime()
13331 bbr_seq = tp->iss; in bbr_output_wtime()
13333 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) { in bbr_output_wtime()
13338 th->th_seq = (htonl(tp->snd_max - 1)); in bbr_output_wtime()
13339 bbr_seq = (tp->snd_max - 1); in bbr_output_wtime()
13342 th->th_seq = htonl(tp->snd_max); in bbr_output_wtime()
13343 bbr_seq = tp->snd_max; in bbr_output_wtime()
13355 * might be better to send (tp->snd_una - 1) which in bbr_output_wtime()
13358 if (tp->t_flags & TF_SENTFIN) { in bbr_output_wtime()
13359 th->th_seq = htonl(tp->snd_max - 1); in bbr_output_wtime()
13360 bbr_seq = (tp->snd_max - 1); in bbr_output_wtime()
13362 th->th_seq = htonl(tp->snd_max); in bbr_output_wtime()
13363 bbr_seq = tp->snd_max; in bbr_output_wtime()
13368 th->th_seq = htonl(rsm->r_start); in bbr_output_wtime()
13369 bbr_seq = rsm->r_start; in bbr_output_wtime()
13371 th->th_ack = htonl(tp->rcv_nxt); in bbr_output_wtime()
13374 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; in bbr_output_wtime()
13381 if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) && in bbr_output_wtime()
13384 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && in bbr_output_wtime()
13385 recwin < (tp->rcv_adv - tp->rcv_nxt)) in bbr_output_wtime()
13386 recwin = (tp->rcv_adv - tp->rcv_nxt); in bbr_output_wtime()
13387 if (recwin > TCP_MAXWIN << tp->rcv_scale) in bbr_output_wtime()
13388 recwin = TCP_MAXWIN << tp->rcv_scale; in bbr_output_wtime()
13396 th->th_win = htons((u_short) in bbr_output_wtime()
13397 (min(sbspace(&so->so_rcv), TCP_MAXWIN))); in bbr_output_wtime()
13400 recwin = roundup2(recwin, 1 << tp->rcv_scale); in bbr_output_wtime()
13401 th->th_win = htons((u_short)(recwin >> tp->rcv_scale)); in bbr_output_wtime()
13404 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0 in bbr_output_wtime()
13411 if (th->th_win == 0) { in bbr_output_wtime()
13412 tp->t_sndzerowin++; in bbr_output_wtime()
13413 tp->t_flags |= TF_RXWIN0SENT; in bbr_output_wtime()
13415 tp->t_flags &= ~TF_RXWIN0SENT; in bbr_output_wtime()
13420 tp->snd_up = tp->snd_una; in bbr_output_wtime()
13425 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ in bbr_output_wtime()
13435 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) { in bbr_output_wtime()
13451 if (tp->t_port) { in bbr_output_wtime()
13452 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; in bbr_output_wtime()
13453 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); in bbr_output_wtime()
13454 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); in bbr_output_wtime()
13455 th->th_sum = htons(0); in bbr_output_wtime()
13458 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; in bbr_output_wtime()
13459 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); in bbr_output_wtime()
13460 th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) + in bbr_output_wtime()
13470 if (tp->t_port) { in bbr_output_wtime()
13471 m->m_pkthdr.csum_flags = CSUM_UDP; in bbr_output_wtime()
13472 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); in bbr_output_wtime()
13473 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, in bbr_output_wtime()
13474 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); in bbr_output_wtime()
13475 th->th_sum = htons(0); in bbr_output_wtime()
13478 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP; in bbr_output_wtime()
13479 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); in bbr_output_wtime()
13480 th->th_sum = in_pseudo(ip->ip_src.s_addr, in bbr_output_wtime()
13481 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + in bbr_output_wtime()
13485 KASSERT(ip->ip_v == IPVERSION, in bbr_output_wtime()
13486 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); in bbr_output_wtime()
13498 m->m_pkthdr.csum_flags |= CSUM_TSO; in bbr_output_wtime()
13500 m->m_pkthdr.tso_segsz = maxseg; in bbr_output_wtime()
13517 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay; in bbr_output_wtime()
13518 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3); in bbr_output_wtime()
13521 log.u_bbr.flex5 = bbr->rc_past_init_win; in bbr_output_wtime()
13523 log.u_bbr.flex5 |= bbr->rc_no_pacing; in bbr_output_wtime()
13525 log.u_bbr.flex5 |= tp->t_maxseg; in bbr_output_wtime()
13526 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs; in bbr_output_wtime()
13527 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr); in bbr_output_wtime()
13529 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg; in bbr_output_wtime()
13538 lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, in bbr_output_wtime()
13549 * m->m_pkthdr.len should have been set before cksum calcuration, in bbr_output_wtime()
13560 ip6->ip6_hlim = in6_selecthlim(inp, NULL); in bbr_output_wtime()
13567 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); in bbr_output_wtime()
13570 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13572 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13574 if (tp->t_state == TCPS_SYN_SENT) in bbr_output_wtime()
13579 error = ip6_output(m, inp->in6p_outputopts, in bbr_output_wtime()
13580 &inp->inp_route6, in bbr_output_wtime()
13584 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL) in bbr_output_wtime()
13585 mtu = inp->inp_route6.ro_nh->nh_mtu; in bbr_output_wtime()
13593 ip->ip_len = htons(m->m_pkthdr.len); in bbr_output_wtime()
13596 ip->ip_ttl = in6_selecthlim(inp, NULL); in bbr_output_wtime()
13608 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { in bbr_output_wtime()
13609 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13610 if (tp->t_port == 0 || len < V_tcp_minmss) { in bbr_output_wtime()
13611 ip->ip_off |= htons(IP_DF); in bbr_output_wtime()
13614 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13617 if (tp->t_state == TCPS_SYN_SENT) in bbr_output_wtime()
13622 error = ip_output(m, inp->inp_options, &inp->inp_route, in bbr_output_wtime()
13625 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL) in bbr_output_wtime()
13626 mtu = inp->inp_route.ro_nh->nh_mtu; in bbr_output_wtime()
13630 lgb->tlb_errno = error; in bbr_output_wtime()
13641 if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_output_wtime()
13642 (tp->t_flags & TF_SACK_PERMIT) && in bbr_output_wtime()
13643 tp->rcv_numsacks > 0) in bbr_output_wtime()
13646 bbr->output_error_seen = 0; in bbr_output_wtime()
13647 bbr->oerror_cnt = 0; in bbr_output_wtime()
13648 bbr->bbr_segs_rcvd = 0; in bbr_output_wtime()
13654 if (tp->snd_una == tp->snd_max) { in bbr_output_wtime()
13661 bbr->r_ctl.rc_del_time = cts; in bbr_output_wtime()
13666 counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1); in bbr_output_wtime()
13671 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options); in bbr_output_wtime()
13673 idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1); in bbr_output_wtime()
13687 if (tp->snd_una == tp->snd_max) in bbr_output_wtime()
13688 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_output_wtime()
13700 if (bbr->rc_in_persist == 0) { in bbr_output_wtime()
13708 if (tp->snd_una == tp->snd_max && in bbr_output_wtime()
13715 bbr->rc_tp->t_acktime = ticks; in bbr_output_wtime()
13723 tp->snd_max = tp->iss + 1; in bbr_output_wtime()
13725 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) { in bbr_output_wtime()
13726 tp->snd_max++; in bbr_output_wtime()
13727 tp->t_flags |= TF_SENTFIN; in bbr_output_wtime()
13731 tp->snd_max += len; in bbr_output_wtime()
13744 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) { in bbr_output_wtime()
13746 tp->t_flags |= TF_SENTFIN; in bbr_output_wtime()
13748 if (xlen && (tp->snd_una == tp->snd_max)) { in bbr_output_wtime()
13754 bbr->rc_tp->t_acktime = ticks; in bbr_output_wtime()
13757 tp->snd_max += xlen; in bbr_output_wtime()
13764 * case of ENOBUFS we will fall out and become ack-clocked. in bbr_output_wtime()
13772 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_output_wtime()
13773 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
13774 bbr->r_agg_early_set = 0; in bbr_output_wtime()
13775 bbr->output_error_seen = 1; in bbr_output_wtime()
13776 if (bbr->oerror_cnt < 0xf) in bbr_output_wtime()
13777 bbr->oerror_cnt++; in bbr_output_wtime()
13778 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) { in bbr_output_wtime()
13780 return (-ENETDOWN); in bbr_output_wtime()
13787 * slam him below a T-O (1MSS). in bbr_output_wtime()
13789 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { in bbr_output_wtime()
13790 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
13791 bbr->r_ctl.rc_lost_bytes)) - maxseg; in bbr_output_wtime()
13792 if (tp->snd_cwnd < maxseg) in bbr_output_wtime()
13793 tp->snd_cwnd = maxseg; in bbr_output_wtime()
13795 slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt; in bbr_output_wtime()
13797 if (bbr->bbr_hdrw_pacing) in bbr_output_wtime()
13822 old_maxseg = tp->t_maxseg; in bbr_output_wtime()
13826 tcp_mss_update(tp, -1, mtu, NULL, NULL); in bbr_output_wtime()
13827 if (old_maxseg <= tp->t_maxseg) { in bbr_output_wtime()
13829 tp->t_maxseg = old_maxseg - 40; in bbr_output_wtime()
13830 if (tp->t_maxseg < V_tcp_mssdflt) { in bbr_output_wtime()
13836 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT; in bbr_output_wtime()
13838 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT; in bbr_output_wtime()
13846 if ((tot_len + len) && (len >= tp->t_maxseg)) { in bbr_output_wtime()
13848 bbr->r_ctl.rc_bbr_hptsi_gain, in bbr_output_wtime()
13851 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt; in bbr_output_wtime()
13853 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt; in bbr_output_wtime()
13854 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
13861 tp->t_softerror = error; in bbr_output_wtime()
13867 if (TCPS_HAVERCVDSYN(tp->t_state)) { in bbr_output_wtime()
13868 tp->t_softerror = error; in bbr_output_wtime()
13873 slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt; in bbr_output_wtime()
13874 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
13879 } else if (((tp->t_flags & TF_GPUTINPROG) == 0) && in bbr_output_wtime()
13882 (bbr->rc_in_persist == 0)) { in bbr_output_wtime()
13883 tp->gput_seq = bbr_seq; in bbr_output_wtime()
13884 tp->gput_ack = bbr_seq + in bbr_output_wtime()
13885 min(sbavail(&so->so_snd) - sb_offset, sendwin); in bbr_output_wtime()
13886 tp->gput_ts = cts; in bbr_output_wtime()
13887 tp->t_flags |= TF_GPUTINPROG; in bbr_output_wtime()
13891 if ((bbr->bbr_hdw_pace_ena) && in bbr_output_wtime()
13892 (bbr->bbr_attempt_hdwr_pace == 0) && in bbr_output_wtime()
13893 (bbr->rc_past_init_win) && in bbr_output_wtime()
13894 (bbr->rc_bbr_state != BBR_STATE_STARTUP) && in bbr_output_wtime()
13895 (get_filter_value(&bbr->r_ctl.rc_delrate)) && in bbr_output_wtime()
13896 (inp->inp_route.ro_nh && in bbr_output_wtime()
13897 inp->inp_route.ro_nh->nh_ifp)) { in bbr_output_wtime()
13909 bbr->bbr_attempt_hdwr_pace = 1; in bbr_output_wtime()
13910 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp, in bbr_output_wtime()
13911 inp->inp_route.ro_nh->nh_ifp, in bbr_output_wtime()
13915 if (bbr->r_ctl.crte) { in bbr_output_wtime()
13917 bbr->r_ctl.crte->ptbl->rs_ifp, in bbr_output_wtime()
13919 bbr->r_ctl.crte->rate, in bbr_output_wtime()
13922 counter_u64_add(bbr_flows_nohdwr_pacing, -1); in bbr_output_wtime()
13924 bbr->bbr_hdrw_pacing = 1; in bbr_output_wtime()
13926 if (bbr->r_ctl.crte->rate < rate_wanted) { in bbr_output_wtime()
13929 bbr->r_ctl.crte->rate, rate_wanted); in bbr_output_wtime()
13932 bbr->gain_is_limited = 0; in bbr_output_wtime()
13933 bbr->skip_gain = 0; in bbr_output_wtime()
13938 inp->inp_route.ro_nh->nh_ifp, in bbr_output_wtime()
13945 if (bbr->bbr_hdrw_pacing) { in bbr_output_wtime()
13952 if (inp->inp_snd_tag == NULL) { in bbr_output_wtime()
13954 bbr->bbr_hdrw_pacing = 0; in bbr_output_wtime()
13955 } else if ((inp->inp_route.ro_nh == NULL) || in bbr_output_wtime()
13956 (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) { in bbr_output_wtime()
13960 * and setup to re-attempt next go in bbr_output_wtime()
13963 bbr->bbr_hdrw_pacing = 0; in bbr_output_wtime()
13964 bbr->bbr_attempt_hdwr_pace = 0; in bbr_output_wtime()
13965 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp); in bbr_output_wtime()
13974 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) in bbr_output_wtime()
13975 tp->rcv_adv = tp->rcv_nxt + recwin; in bbr_output_wtime()
13977 tp->last_ack_sent = tp->rcv_nxt; in bbr_output_wtime()
13979 (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) && in bbr_output_wtime()
13985 (IN_RECOVERY(tp->t_flags) == 0) && in bbr_output_wtime()
13986 (bbr->rc_in_persist == 0) && in bbr_output_wtime()
13987 (tot_len < bbr->r_ctl.rc_pace_max_segs)) { in bbr_output_wtime()
13989 * For non-tso we need to goto again until we have sent out in bbr_output_wtime()
13993 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_output_wtime()
13995 tp->snd_nxt = tp->snd_max; in bbr_output_wtime()
14003 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); in bbr_output_wtime()
14013 * Calculate/Re-Calculate the hptsi slot in usecs based on in bbr_output_wtime()
14016 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); in bbr_output_wtime()
14017 if (bbr->rc_no_pacing) in bbr_output_wtime()
14020 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); in bbr_output_wtime()
14022 if (bbr->rc_use_google == 0) in bbr_output_wtime()
14024 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
14025 bbr->r_ctl.rc_lost_bytes))); in bbr_output_wtime()
14026 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
14027 if (bbr->bbr_use_rack_cheat && in bbr_output_wtime()
14029 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) { in bbr_output_wtime()
14034 if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) { in bbr_output_wtime()
14040 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++; in bbr_output_wtime()
14041 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) { in bbr_output_wtime()
14042 bbr->hw_pacing_set = 1; in bbr_output_wtime()
14047 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_output_wtime()
14049 tp->snd_nxt = tp->snd_max; in bbr_output_wtime()
14084 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_mtu_chg()
14085 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_mtu_chg()
14086 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_mtu_chg()
14087 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_mtu_chg()
14089 if (rsm->r_flags & BBR_ACKED) in bbr_mtu_chg()
14091 if ((rsm->r_end - rsm->r_start) > maxseg) { in bbr_mtu_chg()
14093 * We mark sack-passed on all the previous large in bbr_mtu_chg()
14096 rsm->r_flags |= BBR_SACK_PASSED; in bbr_mtu_chg()
14097 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) && in bbr_mtu_chg()
14098 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) { in bbr_mtu_chg()
14099 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; in bbr_mtu_chg()
14100 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; in bbr_mtu_chg()
14101 rsm->r_flags |= BBR_MARKED_LOST; in bbr_mtu_chg()
14108 bbr->r_ctl.rc_resend = frsm; in bbr_mtu_chg()
14140 tp->t_flags2 |= TF2_CANNOT_DO_ECN; in bbr_switch_failed()
14141 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ; in bbr_switch_failed()
14143 if (tp->t_in_hpts > IHPTS_NONE) { in bbr_switch_failed()
14146 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_switch_failed()
14148 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { in bbr_switch_failed()
14149 if (TSTMP_GT(bbr->rc_pacer_started, cts)) { in bbr_switch_failed()
14150 toval = bbr->rc_pacer_started - cts; in bbr_switch_failed()
14155 } else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { in bbr_switch_failed()
14156 if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { in bbr_switch_failed()
14157 toval = bbr->r_ctl.rc_timer_exp - cts; in bbr_switch_failed()
14189 * socket option arguments. When it re-acquires the lock after the copy, it
14201 switch (sopt->sopt_level) { in bbr_set_sockopt()
14207 switch (sopt->sopt_name) { in bbr_set_sockopt()
14256 if (inp->inp_flags & INP_DROPPED) { in bbr_set_sockopt()
14260 if (tp->t_fb != &__tcp_bbr) { in bbr_set_sockopt()
14264 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_set_sockopt()
14265 switch (sopt->sopt_name) { in bbr_set_sockopt()
14268 bbr->r_ctl.bbr_hptsi_per_second = optval; in bbr_set_sockopt()
14272 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval; in bbr_set_sockopt()
14276 bbr->r_ctl.bbr_hptsi_segments_max = optval; in bbr_set_sockopt()
14280 bbr->r_ctl.bbr_hptsi_bytes_min = optval; in bbr_set_sockopt()
14284 bbr->r_ctl.bbr_cross_over = optval; in bbr_set_sockopt()
14288 if (optval && (bbr->rc_use_google == 0)) { in bbr_set_sockopt()
14289 /* Turn on the google mode */ in bbr_set_sockopt()
14296 bbr->r_ctl.bbr_google_discount = optval; in bbr_set_sockopt()
14298 } else if ((optval == 0) && (bbr->rc_use_google == 1)) { in bbr_set_sockopt()
14299 /* Turn off the google mode */ in bbr_set_sockopt()
14306 bbr->rc_use_ts_limit = 1; in bbr_set_sockopt()
14308 bbr->rc_use_ts_limit = 0; in bbr_set_sockopt()
14318 bbr->rc_init_win = optval; in bbr_set_sockopt()
14320 if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd)) in bbr_set_sockopt()
14321 tp->snd_cwnd = twin; in bbr_set_sockopt()
14330 bbr->r_ctl.rc_startup_pg = optval; in bbr_set_sockopt()
14331 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in bbr_set_sockopt()
14332 bbr->r_ctl.rc_bbr_hptsi_gain = optval; in bbr_set_sockopt()
14340 bbr->r_ctl.rc_drain_pg = optval; in bbr_set_sockopt()
14347 reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND)); in bbr_set_sockopt()
14354 bbr->r_ctl.bbr_rttprobe_gain_val = optval; in bbr_set_sockopt()
14361 bbr->r_ctl.rc_probertt_int = optval; in bbr_set_sockopt()
14368 bbr->no_pacing_until = 0; in bbr_set_sockopt()
14369 bbr->rc_no_pacing = 0; in bbr_set_sockopt()
14371 bbr->no_pacing_until = optval; in bbr_set_sockopt()
14372 if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) && in bbr_set_sockopt()
14373 (bbr->rc_bbr_state == BBR_STATE_STARTUP)){ in bbr_set_sockopt()
14375 bbr->rc_no_pacing = 1; in bbr_set_sockopt()
14382 bbr->rc_loss_exit = optval; in bbr_set_sockopt()
14389 bbr->r_ctl.rc_min_rto_ms = optval; in bbr_set_sockopt()
14393 bbr->rc_max_rto_sec = optval; in bbr_set_sockopt()
14396 /* Minimum time between rack t-o's in ms */ in bbr_set_sockopt()
14398 bbr->r_ctl.rc_min_to = optval; in bbr_set_sockopt()
14404 bbr->r_ctl.rc_reorder_shift = optval; in bbr_set_sockopt()
14411 bbr->r_ctl.rc_reorder_fade = optval; in bbr_set_sockopt()
14417 bbr->rc_tlp_threshold = optval; in bbr_set_sockopt()
14423 if (bbr->rc_use_google) { in bbr_set_sockopt()
14429 bbr->bbr_use_rack_cheat = 1; in bbr_set_sockopt()
14431 bbr->bbr_use_rack_cheat = 0; in bbr_set_sockopt()
14436 bbr->r_ctl.bbr_hptsi_segments_floor = optval; in bbr_set_sockopt()
14443 bbr->r_ctl.bbr_utter_max = optval; in bbr_set_sockopt()
14451 bbr->rc_use_idle_restart = 1; in bbr_set_sockopt()
14453 bbr->rc_use_idle_restart = 0; in bbr_set_sockopt()
14458 bbr->bbr_init_win_cheat = 1; in bbr_set_sockopt()
14459 if (bbr->rc_past_init_win == 0) { in bbr_set_sockopt()
14461 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_set_sockopt()
14465 bbr->bbr_init_win_cheat = 0; in bbr_set_sockopt()
14470 bbr->bbr_hdw_pace_ena = 1; in bbr_set_sockopt()
14471 bbr->bbr_attempt_hdwr_pace = 0; in bbr_set_sockopt()
14473 bbr->bbr_hdw_pace_ena = 0; in bbr_set_sockopt()
14475 if (bbr->r_ctl.crte != NULL) { in bbr_set_sockopt()
14476 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp); in bbr_set_sockopt()
14477 bbr->r_ctl.crte = NULL; in bbr_set_sockopt()
14487 tp->t_delayed_ack = 0; in bbr_set_sockopt()
14489 tp->t_delayed_ack = 2; in bbr_set_sockopt()
14491 tp->t_delayed_ack = optval; in bbr_set_sockopt()
14492 if (tp->t_flags & TF_DELACK) { in bbr_set_sockopt()
14493 tp->t_flags &= ~TF_DELACK; in bbr_set_sockopt()
14494 tp->t_flags |= TF_ACKNOW; in bbr_set_sockopt()
14503 /* RACK added ms i.e. rack-rtt + reord + N */ in bbr_set_sockopt()
14505 bbr->r_ctl.rc_pkt_delay = optval; in bbr_set_sockopt()
14511 bbr->rc_resends_use_tso = 1; in bbr_set_sockopt()
14513 bbr->rc_resends_use_tso = 0; in bbr_set_sockopt()
14518 bbr->rc_allow_data_af_clo = 1; in bbr_set_sockopt()
14520 bbr->rc_allow_data_af_clo = 0; in bbr_set_sockopt()
14524 if (bbr->rc_use_google == 0) in bbr_set_sockopt()
14527 bbr->r_use_policer = 1; in bbr_set_sockopt()
14529 bbr->r_use_policer = 0; in bbr_set_sockopt()
14535 bbr->ts_can_raise = 1; in bbr_set_sockopt()
14537 bbr->ts_can_raise = 0; in bbr_set_sockopt()
14541 if (bbr->rc_use_google) { in bbr_set_sockopt()
14545 bbr->r_ctl.rc_incr_tmrs = 1; in bbr_set_sockopt()
14547 bbr->r_ctl.rc_incr_tmrs = 0; in bbr_set_sockopt()
14552 if (bbr->rc_use_google) { in bbr_set_sockopt()
14562 bbr->r_ctl.rc_inc_tcp_oh = 1; in bbr_set_sockopt()
14564 bbr->r_ctl.rc_inc_tcp_oh = 0; in bbr_set_sockopt()
14566 bbr->r_ctl.rc_inc_ip_oh = 1; in bbr_set_sockopt()
14568 bbr->r_ctl.rc_inc_ip_oh = 0; in bbr_set_sockopt()
14570 bbr->r_ctl.rc_inc_enet_oh = 1; in bbr_set_sockopt()
14572 bbr->r_ctl.rc_inc_enet_oh = 0; in bbr_set_sockopt()
14579 tcp_log_socket_option(tp, sopt->sopt_name, optval, error); in bbr_set_sockopt()
14585 * return 0 on success, error-num on failure
14594 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_get_sockopt()
14605 switch (sopt->sopt_name) { in bbr_get_sockopt()
14607 optval = bbr->r_ctl.bbr_hptsi_per_second; in bbr_get_sockopt()
14610 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar; in bbr_get_sockopt()
14613 optval = bbr->r_ctl.bbr_hptsi_segments_max; in bbr_get_sockopt()
14616 optval = bbr->no_pacing_until; in bbr_get_sockopt()
14619 optval = bbr->r_ctl.bbr_hptsi_bytes_min; in bbr_get_sockopt()
14622 optval = bbr->r_ctl.bbr_cross_over; in bbr_get_sockopt()
14625 optval = bbr->rc_use_google; in bbr_get_sockopt()
14628 optval = bbr->rc_use_ts_limit; in bbr_get_sockopt()
14631 optval = bbr->rc_init_win; in bbr_get_sockopt()
14634 optval = bbr->r_ctl.rc_startup_pg; in bbr_get_sockopt()
14637 optval = bbr->r_ctl.rc_drain_pg; in bbr_get_sockopt()
14640 optval = bbr->r_ctl.rc_probertt_int; in bbr_get_sockopt()
14643 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND); in bbr_get_sockopt()
14646 optval = bbr->r_ctl.bbr_rttprobe_gain_val; in bbr_get_sockopt()
14649 optval = bbr->rc_loss_exit; in bbr_get_sockopt()
14655 optval = bbr->r_ctl.rc_min_rto_ms; in bbr_get_sockopt()
14658 optval = bbr->rc_max_rto_sec; in bbr_get_sockopt()
14662 optval = bbr->r_ctl.rc_pace_max_segs; in bbr_get_sockopt()
14665 /* Minimum time between rack t-o's in ms */ in bbr_get_sockopt()
14666 optval = bbr->r_ctl.rc_min_to; in bbr_get_sockopt()
14670 optval = bbr->r_ctl.rc_reorder_shift; in bbr_get_sockopt()
14674 optval = bbr->r_ctl.rc_reorder_fade; in bbr_get_sockopt()
14678 optval = bbr->bbr_use_rack_cheat; in bbr_get_sockopt()
14681 optval = bbr->r_ctl.bbr_hptsi_segments_floor; in bbr_get_sockopt()
14684 optval = bbr->r_ctl.bbr_utter_max; in bbr_get_sockopt()
14688 optval = bbr->bbr_init_win_cheat; in bbr_get_sockopt()
14691 optval = bbr->rc_use_idle_restart; in bbr_get_sockopt()
14695 optval = bbr->rc_tlp_threshold; in bbr_get_sockopt()
14698 /* RACK added ms i.e. rack-rtt + reord + N */ in bbr_get_sockopt()
14699 optval = bbr->r_ctl.rc_pkt_delay; in bbr_get_sockopt()
14702 optval = bbr->rc_resends_use_tso; in bbr_get_sockopt()
14705 optval = bbr->rc_allow_data_af_clo; in bbr_get_sockopt()
14708 optval = tp->t_delayed_ack; in bbr_get_sockopt()
14711 optval = bbr->bbr_hdw_pace_ena; in bbr_get_sockopt()
14714 optval = bbr->r_use_policer; in bbr_get_sockopt()
14717 optval = bbr->ts_can_raise; in bbr_get_sockopt()
14720 optval = bbr->r_ctl.rc_incr_tmrs; in bbr_get_sockopt()
14724 if (bbr->r_ctl.rc_inc_tcp_oh) in bbr_get_sockopt()
14726 if (bbr->r_ctl.rc_inc_ip_oh) in bbr_get_sockopt()
14728 if (bbr->r_ctl.rc_inc_enet_oh) in bbr_get_sockopt()
14741 * return 0 on success, error-num on failure
14746 if (sopt->sopt_dir == SOPT_SET) { in bbr_ctloutput()
14748 } else if (sopt->sopt_dir == SOPT_GET) { in bbr_ctloutput()
14751 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir); in bbr_ctloutput()