Lines Matching +full:gain +full:- +full:scaling +full:- +full:p
1 /*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
29 * BBR - Congestion Based Congestion Control
159 /* Should the following be dynamic too -- loss wise */
173 * How much gain do we need to see to
178 static int32_t bbr_low_start_exit = 25; /* When we are in reduced gain */
193 * num-measures > min(0) and
221 * is used for both cwnd and hptsi gain's.
227 /* thresholds for reduction on drain in sub-states/drain */
251 * hptsi gain and thus drainging the packets
258 * The cwnd_gain is the default cwnd gain applied when
265 * means in non-recovery/retransmission scenarios
266 * cwnd will never be reached by the flight-size.
271 static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sack to use bbr */
292 * - Yuchung Cheng's RACK TCP (for which its named) that
295 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
297 * - Van Jacobson's et.al BBR.
319 * TCP output is also over-written with a new version since it
331 static int32_t bbr_reorder_fade = 60000000; /* 0 - never fade, def
332 * 60,000,000 - 60 seconds */
385 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
388 /* What is the max the 0 - bbr_cross_over MBPS TSO target
397 /* What is the min the 0 - bbr_cross-over MBPS TSO target can be */
401 /* Cross over point from algo-a to algo-b */
438 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain,
441 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain);
450 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain);
509 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
517 return(bbr->rc_bbr_substate); in bbr_state_val()
525 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), in get_min_cwnd()
526 bbr->r_ctl.rc_pace_max_segs); in get_min_cwnd()
539 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT; in bbr_get_persists_timer_val()
540 if (tp->t_srtt == 0) { in bbr_get_persists_timer_val()
544 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_get_persists_timer_val()
545 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT); in bbr_get_persists_timer_val()
547 TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]), in bbr_get_persists_timer_val()
566 if (bbr->rc_all_timers_stopped) { in bbr_timer_start()
570 if (bbr->rc_in_persist) { in bbr_timer_start()
574 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_timer_start()
576 ((tp->t_flags & TF_SACK_PERMIT) == 0) || in bbr_timer_start()
577 (tp->t_state < TCPS_ESTABLISHED)) { in bbr_timer_start()
580 if (SEQ_LT(tp->snd_una, tp->snd_max) || in bbr_timer_start()
581 sbavail(&tptosocket(tp)->so_snd)) { in bbr_timer_start()
585 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_timer_start()
587 idx = rsm->r_rtr_cnt - 1; in bbr_timer_start()
588 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time)) in bbr_timer_start()
589 tstmp_touse = rsm->r_tim_lastsent[idx]; in bbr_timer_start()
591 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time; in bbr_timer_start()
593 time_since_sent = cts - tstmp_touse; in bbr_timer_start()
595 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT; in bbr_timer_start()
596 if (tp->t_srtt == 0) in bbr_timer_start()
599 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) + in bbr_timer_start()
600 ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT); in bbr_timer_start()
601 if (tp->t_rxtshift) in bbr_timer_start()
602 tov *= tcp_backoff[tp->t_rxtshift]; in bbr_timer_start()
604 tov -= time_since_sent; in bbr_timer_start()
606 tov = bbr->r_ctl.rc_min_to; in bbr_timer_start()
608 (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC), in bbr_timer_start()
609 (bbr->rc_max_rto_sec * USECS_IN_SECOND)); in bbr_timer_start()
615 if (rsm->r_flags & BBR_ACKED) { in bbr_timer_start()
623 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_timer_start()
624 if ((tp->t_flags & TF_SENTFIN) && in bbr_timer_start()
625 ((tp->snd_max - tp->snd_una) == 1) && in bbr_timer_start()
626 (rsm->r_flags & BBR_HAS_FIN)) { in bbr_timer_start()
635 idx = rsm->r_rtr_cnt - 1; in bbr_timer_start()
636 exp = rsm->r_tim_lastsent[idx] + thresh; in bbr_timer_start()
638 to = exp - cts; in bbr_timer_start()
639 if (to < bbr->r_ctl.rc_min_to) { in bbr_timer_start()
640 to = bbr->r_ctl.rc_min_to; in bbr_timer_start()
643 to = bbr->r_ctl.rc_min_to; in bbr_timer_start()
647 if (bbr->rc_tlp_in_progress != 0) { in bbr_timer_start()
653 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext); in bbr_timer_start()
658 if (rsm->r_flags & BBR_HAS_FIN) { in bbr_timer_start()
664 idx = rsm->r_rtr_cnt - 1; in bbr_timer_start()
665 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time)) in bbr_timer_start()
666 tstmp_touse = rsm->r_tim_lastsent[idx]; in bbr_timer_start()
668 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time; in bbr_timer_start()
670 time_since_sent = cts - tstmp_touse; in bbr_timer_start()
675 to = thresh - time_since_sent; in bbr_timer_start()
677 to = bbr->r_ctl.rc_min_to; in bbr_timer_start()
678 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { in bbr_timer_start()
685 if ((bbr->rc_tlp_rtx_out == 1) && in bbr_timer_start()
686 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) { in bbr_timer_start()
691 bbr->rc_tlp_rtx_out = 0; in bbr_timer_start()
694 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) { in bbr_timer_start()
699 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; in bbr_timer_start()
700 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start; in bbr_timer_start()
705 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK; in bbr_timer_start()
708 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) { in bbr_timer_start()
716 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP; in bbr_timer_start()
725 return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options); in bbr_minseg()
744 if ((tp->t_state == TCPS_CLOSED) || in bbr_start_hpts_timer()
745 (tp->t_state == TCPS_LISTEN)) { in bbr_start_hpts_timer()
748 stopped = bbr->rc_tmr_stopped; in bbr_start_hpts_timer()
749 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { in bbr_start_hpts_timer()
750 left = bbr->r_ctl.rc_timer_exp - cts; in bbr_start_hpts_timer()
752 bbr->r_ctl.rc_hpts_flags = 0; in bbr_start_hpts_timer()
753 bbr->r_ctl.rc_timer_exp = 0; in bbr_start_hpts_timer()
754 prev_delay = bbr->r_ctl.rc_last_delay_val; in bbr_start_hpts_timer()
755 if (bbr->r_ctl.rc_last_delay_val && in bbr_start_hpts_timer()
762 slot = bbr->r_ctl.rc_last_delay_val; in bbr_start_hpts_timer()
763 if (TSTMP_GT(cts, bbr->rc_pacer_started)) { in bbr_start_hpts_timer()
765 delay_calc = cts - bbr->rc_pacer_started; in bbr_start_hpts_timer()
767 slot -= delay_calc; in bbr_start_hpts_timer()
771 if (bbr->r_agg_early_set) { in bbr_start_hpts_timer()
772 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2); in bbr_start_hpts_timer()
773 slot += bbr->r_ctl.rc_agg_early; in bbr_start_hpts_timer()
774 bbr->r_ctl.rc_agg_early = 0; in bbr_start_hpts_timer()
775 bbr->r_agg_early_set = 0; in bbr_start_hpts_timer()
778 if (bbr->r_ctl.rc_hptsi_agg_delay) { in bbr_start_hpts_timer()
779 if (slot > bbr->r_ctl.rc_hptsi_agg_delay) { in bbr_start_hpts_timer()
781 slot -= bbr->r_ctl.rc_hptsi_agg_delay; in bbr_start_hpts_timer()
782 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_start_hpts_timer()
785 bbr->r_ctl.rc_hptsi_agg_delay -= slot; in bbr_start_hpts_timer()
786 bbr->r_ctl.rc_last_delay_val = slot = 100; in bbr_start_hpts_timer()
789 bbr->r_ctl.rc_last_delay_val = slot; in bbr_start_hpts_timer()
791 if (tp->t_flags & TF_DELACK) { in bbr_start_hpts_timer()
792 if (bbr->rc_in_persist == 0) { in bbr_start_hpts_timer()
812 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; in bbr_start_hpts_timer()
818 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT; in bbr_start_hpts_timer()
822 * wheel, we resort to a keep-alive timer if its configured. in bbr_start_hpts_timer()
826 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_start_hpts_timer()
827 (tp->t_state <= TCPS_CLOSING)) { in bbr_start_hpts_timer()
830 * del-ack), we don't have segments being paced. So in bbr_start_hpts_timer()
833 if (TCPS_HAVEESTABLISHED(tp->t_state)) { in bbr_start_hpts_timer()
838 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP; in bbr_start_hpts_timer()
842 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) { in bbr_start_hpts_timer()
848 * keep-alive, delayed_ack we keep track of what was left in bbr_start_hpts_timer()
854 if (bbr->r_ctl.rc_incr_tmrs && slot && in bbr_start_hpts_timer()
855 (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) { in bbr_start_hpts_timer()
876 * Hack alert for now we can't time-out over 2147 seconds (a in bbr_start_hpts_timer()
881 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; in bbr_start_hpts_timer()
883 bbr->r_ctl.rc_timer_exp = 0; in bbr_start_hpts_timer()
885 (bbr->rc_use_google || in bbr_start_hpts_timer()
886 bbr->output_error_seen || in bbr_start_hpts_timer()
892 bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY; in bbr_start_hpts_timer()
893 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && in bbr_start_hpts_timer()
894 (bbr->rc_cwnd_limited == 0)) { in bbr_start_hpts_timer()
900 tp->t_flags2 |= TF2_DONT_SACK_QUEUE; in bbr_start_hpts_timer()
902 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE; in bbr_start_hpts_timer()
903 bbr->rc_pacer_started = cts; in bbr_start_hpts_timer()
907 bbr->rc_timer_first = 0; in bbr_start_hpts_timer()
908 bbr->bbr_timer_src = frm; in bbr_start_hpts_timer()
921 * on a keep-alive timer and a request comes in for in bbr_start_hpts_timer()
925 bbr->rc_pacer_started = cts; in bbr_start_hpts_timer()
926 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && in bbr_start_hpts_timer()
927 (bbr->rc_cwnd_limited == 0)) { in bbr_start_hpts_timer()
933 tp->t_flags2 |= (TF2_MBUF_QUEUE_READY | in bbr_start_hpts_timer()
937 tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY | in bbr_start_hpts_timer()
940 bbr->bbr_timer_src = frm; in bbr_start_hpts_timer()
943 bbr->rc_timer_first = 1; in bbr_start_hpts_timer()
945 bbr->rc_tmr_stopped = 0; in bbr_start_hpts_timer()
962 tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK; in bbr_timer_audit()
963 if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT)) in bbr_timer_audit()
965 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_timer_audit()
966 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) && in bbr_timer_audit()
971 inp = bbr->rc_inp; in bbr_timer_audit()
974 if (tp->t_flags & TF_DELACK) { in bbr_timer_audit()
982 inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_timer_audit()
983 (tp->t_state <= TCPS_CLOSING)) && in bbr_timer_audit()
985 (tp->snd_max == tp->snd_una)) { in bbr_timer_audit()
990 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) { in bbr_timer_audit()
991 if ((tp->t_flags & TF_SENTFIN) && in bbr_timer_audit()
992 ((tp->snd_max - tp->snd_una) == 1) && in bbr_timer_audit()
993 (rsm->r_flags & BBR_HAS_FIN)) { in bbr_timer_audit()
1006 } else if (SEQ_GT(tp->snd_max, tp->snd_una) && in bbr_timer_audit()
1010 * Either a TLP or RXT is fine if no sack-passed is in place in bbr_timer_audit()
1023 if (SEQ_GT(tp->snd_max, tp->snd_una) && in bbr_timer_audit()
1042 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) { in bbr_timer_audit()
1046 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, in bbr_timer_audit()
1051 * timer. We don't bother with keep-alive, since when we in bbr_timer_audit()
1052 * jump through the output, it will start the keep-alive if in bbr_timer_audit()
1055 * We only need a delayed-ack added and or the hpts_timeout. in bbr_timer_audit()
1058 if (tp->t_flags & TF_DELACK) { in bbr_timer_audit()
1061 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; in bbr_timer_audit()
1065 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; in bbr_timer_audit()
1071 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; in bbr_timer_audit()
1093 * time-stamp taken in theory earlier return the difference. The in bbr_calc_time()
1098 return (cts - earlier_time); in bbr_calc_time()
1111 return (cts - earlier_time); in bbr_calc_time()
1121 if (error || req->newptr == NULL) in sysctl_bbr_clear_lost()
1146 printf("Clearing BBR out-size counters\n"); in sysctl_bbr_clear_lost()
1175 OID_AUTO, "gain", CTLFLAG_RW, in bbr_init_sysctls()
1177 "What is the filter gain drop in probe_rtt (0=disable)?"); in bbr_init_sysctls()
1182 "How many mss's are outstanding during probe-rtt"); in bbr_init_sysctls()
1187 "If RTT has not shrank in this many micro-seconds enter probe-rtt"); in bbr_init_sysctls()
1192 "How many microseconds in probe-rtt"); in bbr_init_sysctls()
1207 "If we keep setting new low rtt's but delay going in probe-rtt can we force in??"); in bbr_init_sysctls()
1212 "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?"); in bbr_init_sysctls()
1217 "Can we dynamically adjust the probe-rtt limits and times?"); in bbr_init_sysctls()
1305 "For 0 -> 24Mbps what is floor number of segments for TSO"); in bbr_init_sysctls()
1310 "For 0 -> 24Mbps what is top number of segments for TSO"); in bbr_init_sysctls()
1392 "How many pkt-epoch's (0 is off) do we need before pacing is on?"); in bbr_init_sysctls()
1424 "Do we use a pkt-epoch for substate if 0 rttProp?"); in bbr_init_sysctls()
1429 "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?"); in bbr_init_sysctls()
1444 "Does probe bw gain to target??"); in bbr_init_sysctls()
1449 "Does probe bw gain get the extra time too?"); in bbr_init_sysctls()
1469 "How many packet-epochs does the b/w delivery rate last?"); in bbr_init_sysctls()
1474 "Does our sub-state drain invoke app limited if its long?"); in bbr_init_sysctls()
1479 "Should we set/recover cwnd for sub-state drain?"); in bbr_init_sysctls()
1484 "Should we set/recover cwnd for main-state drain?"); in bbr_init_sysctls()
1489 "Should we allow google probe-bw/drain to exit early at flight target?"); in bbr_init_sysctls()
1494 "Should we have losses exit gain of probebw in google mode??"); in bbr_init_sysctls()
1516 "Should we use a lower hptsi gain if we see loss in startup?"); in bbr_init_sysctls()
1519 OID_AUTO, "gain", CTLFLAG_RW, in bbr_init_sysctls()
1521 "What gain percent do we need to see to stay in startup??"); in bbr_init_sysctls()
1526 "What gain percent do we need to see to stay in the lower gain startup??"); in bbr_init_sysctls()
1558 "What is the high-speed min cwnd (rttProp under 1ms)"); in bbr_init_sysctls()
1651 "Maximum RTO in seconds -- should be at least as large as min_rto"); in bbr_init_sysctls()
1708 "Do we call the policer detection code from a rack-timeout?"); in bbr_init_sysctls()
1713 "What packet epoch do we do false-positive detection at (0=no)?"); in bbr_init_sysctls()
1788 "Total number of enobufs for non-hardware paced flows"); in bbr_init_sysctls()
1852 l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate; in bbr_fill_in_logging_data()
1853 l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_fill_in_logging_data()
1854 l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop); in bbr_fill_in_logging_data()
1855 l->bw_inuse = bbr_get_bw(bbr); in bbr_fill_in_logging_data()
1856 l->inflight = ctf_flight_size(bbr->rc_tp, in bbr_fill_in_logging_data()
1857 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_fill_in_logging_data()
1858 l->applimited = bbr->r_ctl.r_app_limited_until; in bbr_fill_in_logging_data()
1859 l->delivered = bbr->r_ctl.rc_delivered; in bbr_fill_in_logging_data()
1860 l->timeStamp = cts; in bbr_fill_in_logging_data()
1861 l->lost = bbr->r_ctl.rc_lost; in bbr_fill_in_logging_data()
1862 l->bbr_state = bbr->rc_bbr_state; in bbr_fill_in_logging_data()
1863 l->bbr_substate = bbr_state_val(bbr); in bbr_fill_in_logging_data()
1864 l->epoch = bbr->r_ctl.rc_rtt_epoch; in bbr_fill_in_logging_data()
1865 l->lt_epoch = bbr->r_ctl.rc_lt_epoch; in bbr_fill_in_logging_data()
1866 l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain; in bbr_fill_in_logging_data()
1867 l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain; in bbr_fill_in_logging_data()
1868 l->inhpts = tcp_in_hpts(bbr->rc_tp); in bbr_fill_in_logging_data()
1869 l->use_lt_bw = bbr->rc_lt_use_bw; in bbr_fill_in_logging_data()
1870 l->pkts_out = bbr->r_ctl.rc_flight_at_input; in bbr_fill_in_logging_data()
1871 l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_fill_in_logging_data()
1877 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bw_reduce()
1880 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_bw_reduce()
1885 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate; in bbr_log_type_bw_reduce()
1887 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt; in bbr_log_type_bw_reduce()
1889 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bw_reduce()
1890 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bw_reduce()
1891 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bw_reduce()
1893 0, &log, false, &bbr->rc_tv); in bbr_log_type_bw_reduce()
1900 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_rwnd_collapse()
1903 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_rwnd_collapse()
1907 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_rwnd_collapse()
1908 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_rwnd_collapse()
1909 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_rwnd_collapse()
1911 0, &log, false, &bbr->rc_tv); in bbr_log_type_rwnd_collapse()
1919 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_just_return()
1924 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags; in bbr_log_type_just_return()
1925 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp; in bbr_log_type_just_return()
1927 log.u_bbr.flex5 = bbr->rc_in_persist; in bbr_log_type_just_return()
1928 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val; in bbr_log_type_just_return()
1930 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_type_just_return()
1933 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_just_return()
1934 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_just_return()
1935 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_just_return()
1937 tlen, &log, false, &bbr->rc_tv); in bbr_log_type_just_return()
1944 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_enter_rec()
1947 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_enter_rec()
1949 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent; in bbr_log_type_enter_rec()
1950 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start; in bbr_log_type_enter_rec()
1951 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_enter_rec()
1952 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_enter_rec()
1953 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_enter_rec()
1955 0, &log, false, &bbr->rc_tv); in bbr_log_type_enter_rec()
1971 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_msgsize_fail()
1972 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_msgsize_fail()
1974 0, &log, false, &bbr->rc_tv); in bbr_log_msgsize_fail()
1981 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_flowend()
1986 if (bbr->rc_inp->inp_socket) { in bbr_log_flowend()
1987 r = &bbr->rc_inp->inp_socket->so_rcv; in bbr_log_flowend()
1988 s = &bbr->rc_inp->inp_socket->so_snd; in bbr_log_flowend()
1993 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_flowend()
2004 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_pkt_epoch()
2010 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_log_pkt_epoch()
2011 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt; in bbr_log_pkt_epoch()
2012 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch; in bbr_log_pkt_epoch()
2013 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; in bbr_log_pkt_epoch()
2016 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count; in bbr_log_pkt_epoch()
2017 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_pkt_epoch()
2018 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_pkt_epoch()
2019 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_pkt_epoch()
2021 0, &log, false, &bbr->rc_tv); in bbr_log_pkt_epoch()
2028 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_time_epoch()
2032 log.u_bbr.flex1 = bbr->r_ctl.rc_lost; in bbr_log_time_epoch()
2033 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat; in bbr_log_time_epoch()
2034 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat; in bbr_log_time_epoch()
2036 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_time_epoch()
2037 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_time_epoch()
2038 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_time_epoch()
2040 0, &log, false, &bbr->rc_tv); in bbr_log_time_epoch()
2047 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_set_of_state_target()
2050 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_set_of_state_target()
2051 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state; in bbr_log_set_of_state_target()
2054 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs; in bbr_log_set_of_state_target()
2056 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs; in bbr_log_set_of_state_target()
2057 log.u_bbr.flex7 = bbr->rc_last_options; in bbr_log_set_of_state_target()
2059 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_set_of_state_target()
2060 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_set_of_state_target()
2061 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_set_of_state_target()
2063 0, &log, false, &bbr->rc_tv); in bbr_log_set_of_state_target()
2071 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_statechange()
2076 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; in bbr_log_type_statechange()
2077 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int; in bbr_log_type_statechange()
2082 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch; in bbr_log_type_statechange()
2083 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; in bbr_log_type_statechange()
2084 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000); in bbr_log_type_statechange()
2085 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra; in bbr_log_type_statechange()
2086 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state; in bbr_log_type_statechange()
2087 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_statechange()
2088 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_statechange()
2089 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_statechange()
2091 0, &log, false, &bbr->rc_tv); in bbr_log_type_statechange()
2099 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_rtt_shrinks()
2104 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; in bbr_log_rtt_shrinks()
2105 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt; in bbr_log_rtt_shrinks()
2108 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state; in bbr_log_rtt_shrinks()
2111 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_rtt_shrinks()
2112 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_rtt_shrinks()
2113 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_rtt_shrinks()
2115 0, &log, false, &bbr->rc_tv); in bbr_log_rtt_shrinks()
2122 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_exit_rec()
2125 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_exit_rec()
2126 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start; in bbr_log_type_exit_rec()
2127 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent; in bbr_log_type_exit_rec()
2128 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_type_exit_rec()
2129 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_exit_rec()
2130 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_exit_rec()
2131 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_exit_rec()
2133 0, &log, false, &bbr->rc_tv); in bbr_log_type_exit_rec()
2141 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_cwndupd()
2144 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_type_cwndupd()
2152 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_cwndupd()
2153 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_cwndupd()
2154 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_cwndupd()
2156 0, &log, false, &bbr->rc_tv); in bbr_log_type_cwndupd()
2167 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_rtt_sample()
2170 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_rtt_sample()
2172 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time; in bbr_log_rtt_sample()
2173 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay; in bbr_log_rtt_sample()
2174 log.u_bbr.flex4 = bbr->rc_tp->ts_offset; in bbr_log_rtt_sample()
2175 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_rtt_sample()
2176 log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_log_rtt_sample()
2179 log.u_bbr.flex8 = bbr->rc_ack_was_delayed; in bbr_log_rtt_sample()
2180 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_rtt_sample()
2181 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_rtt_sample()
2182 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_rtt_sample()
2184 0, &log, false, &bbr->rc_tv); in bbr_log_rtt_sample()
2191 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_pesist()
2198 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_pesist()
2199 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_pesist()
2200 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_pesist()
2202 0, &log, false, &bbr->rc_tv); in bbr_log_type_pesist()
2208 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_ack_clear()
2212 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age; in bbr_log_ack_clear()
2213 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; in bbr_log_ack_clear()
2214 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int; in bbr_log_ack_clear()
2215 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time; in bbr_log_ack_clear()
2216 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_ack_clear()
2217 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_ack_clear()
2218 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_ack_clear()
2219 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_ack_clear()
2221 0, &log, false, &bbr->rc_tv); in bbr_log_ack_clear()
2229 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_ack_event()
2235 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes; in bbr_log_ack_event()
2239 log.u_bbr.flex3 = m->m_flags; in bbr_log_ack_event()
2240 if (m->m_flags & M_TSTMP) { in bbr_log_ack_event()
2248 if (m->m_flags & M_TSTMP_LRO) { in bbr_log_ack_event()
2263 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state; in bbr_log_ack_event()
2264 log.u_bbr.flex7 = bbr->r_wanted_output; in bbr_log_ack_event()
2265 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_ack_event()
2266 TCP_LOG_EVENTP(bbr->rc_tp, th, in bbr_log_ack_event()
2267 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_ack_event()
2268 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_ack_event()
2270 tlen, &log, true, &bbr->rc_tv); in bbr_log_ack_event()
2277 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_doseg_done()
2283 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val; in bbr_log_doseg_done()
2284 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags; in bbr_log_doseg_done()
2285 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp; in bbr_log_doseg_done()
2286 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes; in bbr_log_doseg_done()
2287 log.u_bbr.flex7 = bbr->r_wanted_output; in bbr_log_doseg_done()
2288 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_doseg_done()
2289 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay; in bbr_log_doseg_done()
2290 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_doseg_done()
2291 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_doseg_done()
2292 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_doseg_done()
2294 0, &log, true, &bbr->rc_tv); in bbr_log_doseg_done()
2302 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_enobuf_jmp()
2310 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_enobuf_jmp()
2311 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_enobuf_jmp()
2312 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_enobuf_jmp()
2314 len, &log, true, &bbr->rc_tv); in bbr_log_enobuf_jmp()
2321 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_processing()
2327 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp; in bbr_log_to_processing()
2328 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_processing()
2330 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state; in bbr_log_to_processing()
2332 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_processing()
2333 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_processing()
2334 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_processing()
2336 0, &log, false, &bbr->rc_tv); in bbr_log_to_processing()
2343 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_event()
2348 log.u_bbr.flex1 = bbr->bbr_timer_src; in bbr_log_to_event()
2350 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_event()
2351 ar = (uintptr_t)(bbr->r_ctl.rc_resend); in bbr_log_to_event()
2355 ar = (uintptr_t)bbr->r_ctl.rc_resend; in bbr_log_to_event()
2358 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_to_event()
2360 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_event()
2361 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_event()
2362 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_event()
2364 0, &log, false, &bbr->rc_tv); in bbr_log_to_event()
2371 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_startup_event()
2379 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_startup_event()
2380 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; in bbr_log_startup_event()
2382 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_log_startup_event()
2383 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_startup_event()
2384 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_startup_event()
2385 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_startup_event()
2387 0, &log, false, &bbr->rc_tv); in bbr_log_startup_event()
2394 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_hpts_diag()
2398 log.u_bbr.flex1 = diag->p_nxt_slot; in bbr_log_hpts_diag()
2399 log.u_bbr.flex2 = diag->p_cur_slot; in bbr_log_hpts_diag()
2400 log.u_bbr.flex3 = diag->slot_req; in bbr_log_hpts_diag()
2401 log.u_bbr.flex4 = diag->inp_hptsslot; in bbr_log_hpts_diag()
2402 log.u_bbr.flex5 = diag->slot_remaining; in bbr_log_hpts_diag()
2403 log.u_bbr.flex6 = diag->need_new_to; in bbr_log_hpts_diag()
2404 log.u_bbr.flex7 = diag->p_hpts_active; in bbr_log_hpts_diag()
2405 log.u_bbr.flex8 = diag->p_on_min_sleep; in bbr_log_hpts_diag()
2407 log.u_bbr.epoch = diag->have_slept; in bbr_log_hpts_diag()
2408 log.u_bbr.lt_epoch = diag->yet_to_sleep; in bbr_log_hpts_diag()
2409 log.u_bbr.pkts_out = diag->co_ret; in bbr_log_hpts_diag()
2410 log.u_bbr.applimited = diag->hpts_sleep_time; in bbr_log_hpts_diag()
2411 log.u_bbr.delivered = diag->p_prev_slot; in bbr_log_hpts_diag()
2412 log.u_bbr.inflight = diag->p_runningslot; in bbr_log_hpts_diag()
2413 log.u_bbr.bw_inuse = diag->wheel_slot; in bbr_log_hpts_diag()
2414 log.u_bbr.rttProp = diag->wheel_cts; in bbr_log_hpts_diag()
2415 log.u_bbr.delRate = diag->maxslots; in bbr_log_hpts_diag()
2416 log.u_bbr.cur_del_rate = diag->p_curtick; in bbr_log_hpts_diag()
2418 log.u_bbr.cur_del_rate |= diag->p_lasttick; in bbr_log_hpts_diag()
2419 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_hpts_diag()
2420 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_hpts_diag()
2421 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_hpts_diag()
2423 0, &log, false, &bbr->rc_tv); in bbr_log_hpts_diag()
2431 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_timer_var()
2435 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar; in bbr_log_timer_var()
2440 log.u_bbr.flex6 = bbr->rc_tp->t_srtt; in bbr_log_timer_var()
2442 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_timer_var()
2443 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_timer_var()
2444 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_timer_var()
2446 0, &log, false, &bbr->rc_tv); in bbr_log_timer_var()
2451 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len, in bbr_log_pacing_delay_calc() argument
2454 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_pacing_delay_calc()
2467 log.u_bbr.flex7 = gain; in bbr_log_pacing_delay_calc()
2469 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_pacing_delay_calc()
2470 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_pacing_delay_calc()
2471 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_pacing_delay_calc()
2473 len, &log, false, &bbr->rc_tv); in bbr_log_pacing_delay_calc()
2480 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_start()
2485 log.u_bbr.flex1 = bbr->bbr_timer_src; in bbr_log_to_start()
2487 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_start()
2489 log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot; in bbr_log_to_start()
2490 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_to_start()
2491 log.u_bbr.pkts_out = bbr->rc_tp->t_flags2; in bbr_log_to_start()
2493 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_start()
2494 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_start()
2495 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_start()
2497 0, &log, false, &bbr->rc_tv); in bbr_log_to_start()
2504 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_thresh_choice()
2510 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts; in bbr_log_thresh_choice()
2511 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]; in bbr_log_thresh_choice()
2512 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_thresh_choice()
2514 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift; in bbr_log_thresh_choice()
2516 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_thresh_choice()
2517 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_thresh_choice()
2518 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_thresh_choice()
2520 0, &log, false, &bbr->rc_tv); in bbr_log_thresh_choice()
2527 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_to_cancel()
2532 log.u_bbr.flex2 = bbr->bbr_timer_src; in bbr_log_to_cancel()
2533 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; in bbr_log_to_cancel()
2534 log.u_bbr.flex4 = bbr->rc_in_persist; in bbr_log_to_cancel()
2535 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; in bbr_log_to_cancel()
2536 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_log_to_cancel()
2538 log.u_bbr.pkts_out = bbr->rc_pacer_started; in bbr_log_to_cancel()
2539 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_to_cancel()
2540 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_to_cancel()
2541 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_to_cancel()
2543 0, &log, false, &bbr->rc_tv); in bbr_log_to_cancel()
2550 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_tstmp_validation()
2553 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_tstmp_validation()
2554 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio; in bbr_log_tstmp_validation()
2559 log.u_bbr.flex7 = bbr->rc_ts_clock_set; in bbr_log_tstmp_validation()
2560 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used; in bbr_log_tstmp_validation()
2561 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_tstmp_validation()
2562 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_tstmp_validation()
2563 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_tstmp_validation()
2565 0, &log, false, &bbr->rc_tv); in bbr_log_tstmp_validation()
2572 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_tsosize()
2579 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min; in bbr_log_type_tsosize()
2582 log.u_bbr.flex7 = bbr->rc_no_pacing; in bbr_log_type_tsosize()
2584 log.u_bbr.flex7 |= bbr->rc_past_init_win; in bbr_log_type_tsosize()
2586 log.u_bbr.flex8 = 0x80 | bbr->rc_use_google; in bbr_log_type_tsosize()
2588 log.u_bbr.flex8 = bbr->rc_use_google; in bbr_log_type_tsosize()
2589 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_tsosize()
2590 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_tsosize()
2591 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_tsosize()
2593 0, &log, false, &bbr->rc_tv); in bbr_log_type_tsosize()
2601 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_rsmclear()
2606 log.u_bbr.flex2 = rsm->r_start; in bbr_log_type_rsmclear()
2607 log.u_bbr.flex3 = rsm->r_end; in bbr_log_type_rsmclear()
2608 log.u_bbr.flex4 = rsm->r_delivered; in bbr_log_type_rsmclear()
2609 log.u_bbr.flex5 = rsm->r_rtr_cnt; in bbr_log_type_rsmclear()
2610 log.u_bbr.flex6 = rsm->r_dupack; in bbr_log_type_rsmclear()
2611 log.u_bbr.flex7 = rsm->r_tim_lastsent[0]; in bbr_log_type_rsmclear()
2612 log.u_bbr.flex8 = rsm->r_flags; in bbr_log_type_rsmclear()
2615 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_rsmclear()
2616 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_rsmclear()
2617 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_rsmclear()
2619 0, &log, false, &bbr->rc_tv); in bbr_log_type_rsmclear()
2630 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bbrupd()
2644 if (bbr->rc_ack_was_delayed) in bbr_log_type_bbrupd()
2645 log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay; in bbr_log_type_bbrupd()
2648 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bbrupd()
2649 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bbrupd()
2650 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bbrupd()
2652 flex2, &log, false, &bbr->rc_tv); in bbr_log_type_bbrupd()
2661 if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_ltbw()
2669 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost; in bbr_log_type_ltbw()
2670 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del; in bbr_log_type_ltbw()
2671 log.u_bbr.flex7 = bbr->rc_lt_is_sampling; in bbr_log_type_ltbw()
2673 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw; in bbr_log_type_ltbw()
2674 if (bbr->rc_lt_use_bw == 0) in bbr_log_type_ltbw()
2675 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch; in bbr_log_type_ltbw()
2677 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use; in bbr_log_type_ltbw()
2678 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_ltbw()
2679 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_ltbw()
2680 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_ltbw()
2682 0, &log, false, &bbr->rc_tv); in bbr_log_type_ltbw()
2689 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_progress_event()
2692 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_progress_event()
2695 log.u_bbr.flex3 = tp->t_maxunacktime; in bbr_log_progress_event()
2696 log.u_bbr.flex4 = tp->t_acktime; in bbr_log_progress_event()
2698 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_progress_event()
2699 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_progress_event()
2700 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_progress_event()
2702 0, &log, false, &bbr->rc_tv); in bbr_log_progress_event()
2711 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_type_log_hdwr_pacing()
2723 log.u_bbr.flex8 = bbr->skip_gain; in bbr_type_log_hdwr_pacing()
2725 log.u_bbr.flex8 |= bbr->gain_is_limited; in bbr_type_log_hdwr_pacing()
2727 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing; in bbr_type_log_hdwr_pacing()
2728 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg; in bbr_type_log_hdwr_pacing()
2729 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_type_log_hdwr_pacing()
2730 &bbr->rc_inp->inp_socket->so_rcv, in bbr_type_log_hdwr_pacing()
2731 &bbr->rc_inp->inp_socket->so_snd, in bbr_type_log_hdwr_pacing()
2733 0, &log, false, &bbr->rc_tv); in bbr_type_log_hdwr_pacing()
2740 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bbrsnd()
2748 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val; in bbr_log_type_bbrsnd()
2749 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay; in bbr_log_type_bbrsnd()
2750 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags); in bbr_log_type_bbrsnd()
2751 log.u_bbr.flex8 = bbr->rc_in_persist; in bbr_log_type_bbrsnd()
2752 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bbrsnd()
2753 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bbrsnd()
2754 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bbrsnd()
2756 len, &log, false, &bbr->rc_tv); in bbr_log_type_bbrsnd()
2763 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_type_bbrrttprop()
2767 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered; in bbr_log_type_bbrrttprop()
2769 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt; in bbr_log_type_bbrrttprop()
2775 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_type_bbrrttprop()
2776 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_type_bbrrttprop()
2777 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_type_bbrrttprop()
2779 0, &log, false, &bbr->rc_tv); in bbr_log_type_bbrrttprop()
2786 if (tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_exit_gain()
2790 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state; in bbr_log_exit_gain()
2791 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_log_exit_gain()
2792 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch; in bbr_log_exit_gain()
2793 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs; in bbr_log_exit_gain()
2794 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs; in bbr_log_exit_gain()
2795 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight; in bbr_log_exit_gain()
2798 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_exit_gain()
2799 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_exit_gain()
2800 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_exit_gain()
2802 0, &log, false, &bbr->rc_tv); in bbr_log_exit_gain()
2809 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) { in bbr_log_settings_change()
2812 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); in bbr_log_settings_change()
2813 /* R-HU */ in bbr_log_settings_change()
2821 TCP_LOG_EVENTP(bbr->rc_tp, NULL, in bbr_log_settings_change()
2822 &bbr->rc_inp->inp_socket->so_rcv, in bbr_log_settings_change()
2823 &bbr->rc_inp->inp_socket->so_snd, in bbr_log_settings_change()
2825 0, &log, false, &bbr->rc_tv); in bbr_log_settings_change()
2837 bw = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_get_full_bw()
2848 if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch) in bbr_set_pktepoch()
2849 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch; in bbr_set_pktepoch()
2852 del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del; in bbr_set_pktepoch()
2863 bbr->r_ctl.rc_pkt_epoch_loss_rate = (uint32_t)calclr; in bbr_set_pktepoch()
2864 if (IN_RECOVERY(bbr->rc_tp->t_flags)) in bbr_set_pktepoch()
2865 bbr->r_ctl.recovery_lr += (uint32_t)calclr; in bbr_set_pktepoch()
2866 bbr->r_ctl.rc_pkt_epoch++; in bbr_set_pktepoch()
2867 if (bbr->rc_no_pacing && in bbr_set_pktepoch()
2868 (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) { in bbr_set_pktepoch()
2869 bbr->rc_no_pacing = 0; in bbr_set_pktepoch()
2872 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time); in bbr_set_pktepoch()
2873 bbr->r_ctl.rc_pkt_epoch_time = cts; in bbr_set_pktepoch()
2876 bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered; in bbr_set_pktepoch()
2877 bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost; in bbr_set_pktepoch()
2886 bbr->r_ctl.rc_rtt_epoch++; in bbr_set_epoch()
2887 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start; in bbr_set_epoch()
2889 bbr->r_ctl.rc_rcv_epoch_start = cts; in bbr_set_epoch()
2895 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) { in bbr_isit_a_pkt_epoch()
2896 bbr->rc_is_pkt_epoch_now = 1; in bbr_isit_a_pkt_epoch()
2920 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in __bbr_get_bw()
2922 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop); in __bbr_get_bw()
2925 min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) * in __bbr_get_bw()
2928 if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) { in __bbr_get_bw()
2929 min_bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2932 } else if (bbr->rc_tp->t_srtt != 0) { in __bbr_get_bw()
2937 min_bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2942 if ((bbr->rc_past_init_win == 0) && in __bbr_get_bw()
2943 (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp))) in __bbr_get_bw()
2944 bbr->rc_past_init_win = 1; in __bbr_get_bw()
2945 if ((bbr->rc_use_google) && (bbr->r_ctl.r_measurement_count >= 1)) in __bbr_get_bw()
2948 ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) || in __bbr_get_bw()
2949 (bbr->rc_past_init_win == 0))) { in __bbr_get_bw()
2953 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop); in __bbr_get_bw()
2960 bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) * in __bbr_get_bw()
2963 if (bw < bbr->r_ctl.rc_initial_hptsi_bw) { in __bbr_get_bw()
2964 bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2968 bw = bbr->r_ctl.rc_initial_hptsi_bw; in __bbr_get_bw()
2978 if (bbr->rc_lt_use_bw) in __bbr_get_bw()
2979 bw = bbr->r_ctl.rc_lt_bw; in __bbr_get_bw()
2980 else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0)) in __bbr_get_bw()
2981 bw = bbr->r_ctl.red_bw; in __bbr_get_bw()
2983 bw = get_filter_value(&bbr->r_ctl.rc_delrate); in __bbr_get_bw()
3005 bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_reset_lt_bw_interval()
3006 bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time; in bbr_reset_lt_bw_interval()
3007 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered; in bbr_reset_lt_bw_interval()
3008 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_reset_lt_bw_interval()
3014 bbr->rc_lt_is_sampling = 0; in bbr_reset_lt_bw_sampling()
3015 bbr->rc_lt_use_bw = 0; in bbr_reset_lt_bw_sampling()
3016 bbr->r_ctl.rc_lt_bw = 0; in bbr_reset_lt_bw_sampling()
3026 if (bbr->r_ctl.rc_lt_bw) { in bbr_lt_bw_samp_done()
3028 if (bbr->r_ctl.rc_lt_bw > bw) in bbr_lt_bw_samp_done()
3029 diff = bbr->r_ctl.rc_lt_bw - bw; in bbr_lt_bw_samp_done()
3031 diff = bw - bbr->r_ctl.rc_lt_bw; in bbr_lt_bw_samp_done()
3033 (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) { in bbr_lt_bw_samp_done()
3037 saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw; in bbr_lt_bw_samp_done()
3038 bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2; /* average of two */ in bbr_lt_bw_samp_done()
3039 bbr->rc_lt_use_bw = 1; in bbr_lt_bw_samp_done()
3040 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_lt_bw_samp_done()
3045 bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch; in bbr_lt_bw_samp_done()
3054 bbr->r_ctl.rc_lt_bw = bw; in bbr_lt_bw_samp_done()
3066 deduct = bbr->r_ctl.rc_level_state_extra / ran; in bbr_randomize_extra_state_time()
3067 bbr->r_ctl.rc_level_state_extra -= deduct; in bbr_randomize_extra_state_time()
3081 bbr->r_ctl.rc_exta_time_gd = 0; in bbr_pick_probebw_substate()
3082 bbr->rc_hit_state_1 = 0; in bbr_pick_probebw_substate()
3083 bbr->r_ctl.rc_level_state_extra = 0; in bbr_pick_probebw_substate()
3084 ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1)); in bbr_pick_probebw_substate()
3089 * we fully enter the state. Note that the (8 - 1 - ran) assures that in bbr_pick_probebw_substate()
3090 * we return 1 - 7, so we dont return 0 and end up starting in in bbr_pick_probebw_substate()
3093 ret_val = BBR_SUBSTATE_COUNT - 1 - ran; in bbr_pick_probebw_substate()
3095 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) in bbr_pick_probebw_substate()
3098 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_pick_probebw_substate()
3108 if (bbr->r_use_policer == 0) in bbr_lt_bw_sampling()
3110 if (bbr->rc_lt_use_bw) { in bbr_lt_bw_sampling()
3112 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use; in bbr_lt_bw_sampling()
3117 if (bbr->rc_filled_pipe) { in bbr_lt_bw_sampling()
3119 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_lt_bw_sampling()
3121 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_lt_bw_sampling()
3129 bbr->rc_bbr_state = BBR_STATE_STARTUP; in bbr_lt_bw_sampling()
3131 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_lt_bw_sampling()
3132 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_lt_bw_sampling()
3133 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; in bbr_lt_bw_sampling()
3134 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; in bbr_lt_bw_sampling()
3138 /* reason 0 is to stop using lt-bw */ in bbr_lt_bw_sampling()
3143 /* Not doing false-positive detection */ in bbr_lt_bw_sampling()
3149 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered; in bbr_lt_bw_sampling()
3150 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_lt_bw_sampling()
3153 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost; in bbr_lt_bw_sampling()
3154 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del; in bbr_lt_bw_sampling()
3169 * exhaust its tokens and estimate the steady-state rate allowed by in bbr_lt_bw_sampling()
3171 * over-estimate the bw. in bbr_lt_bw_sampling()
3173 if (bbr->rc_lt_is_sampling == 0) { in bbr_lt_bw_sampling()
3178 bbr->rc_lt_is_sampling = 1; in bbr_lt_bw_sampling()
3183 if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time)) in bbr_lt_bw_sampling()
3184 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time; in bbr_lt_bw_sampling()
3189 if (bbr->r_ctl.r_app_limited_until) { in bbr_lt_bw_sampling()
3190 /* Can not measure in app-limited state */ in bbr_lt_bw_sampling()
3196 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch; in bbr_lt_bw_sampling()
3203 /* 6 is not_enough time or no-loss */ in bbr_lt_bw_sampling()
3223 * tokens are exhausted under-estimates the policed rate. in bbr_lt_bw_sampling()
3226 /* 6 is not_enough time or no-loss */ in bbr_lt_bw_sampling()
3231 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost; in bbr_lt_bw_sampling()
3232 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del; in bbr_lt_bw_sampling()
3240 /* 6 is not_enough time or no-loss */ in bbr_lt_bw_sampling()
3269 bbr->r_ctl.rc_num_maps_alloced++; in bbr_alloc()
3272 if (bbr->r_ctl.rc_free_cnt) { in bbr_alloc()
3274 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); in bbr_alloc()
3275 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_alloc()
3276 bbr->r_ctl.rc_free_cnt--; in bbr_alloc()
3287 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { in bbr_alloc_full_limit()
3289 if (!bbr->alloc_limit_reported) { in bbr_alloc_full_limit()
3290 bbr->alloc_limit_reported = 1; in bbr_alloc_full_limit()
3307 bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) { in bbr_alloc_limit()
3309 if (!bbr->alloc_limit_reported) { in bbr_alloc_limit()
3310 bbr->alloc_limit_reported = 1; in bbr_alloc_limit()
3320 rsm->r_limit_type = limit_type; in bbr_alloc_limit()
3321 bbr->r_ctl.rc_num_split_allocs++; in bbr_alloc_limit()
3329 if (rsm->r_limit_type) { in bbr_free()
3331 bbr->r_ctl.rc_num_split_allocs--; in bbr_free()
3333 if (rsm->r_is_smallmap) in bbr_free()
3334 bbr->r_ctl.rc_num_small_maps_alloced--; in bbr_free()
3335 if (bbr->r_ctl.rc_tlp_send == rsm) in bbr_free()
3336 bbr->r_ctl.rc_tlp_send = NULL; in bbr_free()
3337 if (bbr->r_ctl.rc_resend == rsm) { in bbr_free()
3338 bbr->r_ctl.rc_resend = NULL; in bbr_free()
3340 if (bbr->r_ctl.rc_next == rsm) in bbr_free()
3341 bbr->r_ctl.rc_next = NULL; in bbr_free()
3342 if (bbr->r_ctl.rc_sacklast == rsm) in bbr_free()
3343 bbr->r_ctl.rc_sacklast = NULL; in bbr_free()
3344 if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) { in bbr_free()
3346 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_free()
3347 rsm->r_limit_type = 0; in bbr_free()
3348 bbr->r_ctl.rc_free_cnt++; in bbr_free()
3351 bbr->r_ctl.rc_num_maps_alloced--; in bbr_free()
3363 * returned value per RTT to match that rate. Gain will normally in bbr_get_bw_delay_prod()
3384 if (bbr->rc_init_win) { in bbr_initial_cwnd()
3385 i_cwnd = bbr->rc_init_win * tp->t_maxseg; in bbr_initial_cwnd()
3387 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg), in bbr_initial_cwnd()
3388 max(2 * tp->t_maxseg, 14600)); in bbr_initial_cwnd()
3390 i_cwnd = min(4 * tp->t_maxseg, in bbr_initial_cwnd()
3391 max(2 * tp->t_maxseg, 4380)); in bbr_initial_cwnd()
3394 if (tp->t_maxseg > 2190) in bbr_initial_cwnd()
3395 i_cwnd = 2 * tp->t_maxseg; in bbr_initial_cwnd()
3396 else if (tp->t_maxseg > 1095) in bbr_initial_cwnd()
3397 i_cwnd = 3 * tp->t_maxseg; in bbr_initial_cwnd()
3399 i_cwnd = 4 * tp->t_maxseg; in bbr_initial_cwnd()
3405 * Given a specified gain, return the target
3406 * cwnd based on that gain.
3409 bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw) in bbr_get_raw_target_cwnd() argument
3414 if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) || in bbr_get_raw_target_cwnd()
3417 return (bbr_initial_cwnd(bbr, bbr->rc_tp)); in bbr_get_raw_target_cwnd()
3426 /* Now apply the gain */ in bbr_get_raw_target_cwnd()
3427 cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT)); in bbr_get_raw_target_cwnd()
3433 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain) in bbr_get_target_cwnd() argument
3437 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); in bbr_get_target_cwnd()
3438 /* Get the base cwnd with gain rounded to a mss */ in bbr_get_target_cwnd()
3439 cwnd = roundup(bbr_get_raw_target_cwnd(bbr, bw, gain), mss); in bbr_get_target_cwnd()
3442 * fq layer to trap packets in) quanta's per the I-D in bbr_get_target_cwnd()
3445 cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs); in bbr_get_target_cwnd()
3446 if (bbr->rc_use_google) { in bbr_get_target_cwnd()
3447 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && in bbr_get_target_cwnd()
3451 * an extra 2 x mss in gain cycle which in bbr_get_target_cwnd()
3452 * is documented no-where except in the code. in bbr_get_target_cwnd()
3469 bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain) in bbr_gain_adjust() argument
3471 if (gain < 1) in bbr_gain_adjust()
3472 gain = 1; in bbr_gain_adjust()
3473 return (gain); in bbr_gain_adjust()
3482 if (bbr->r_ctl.rc_inc_tcp_oh) { in bbr_get_header_oh()
3484 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr)); in bbr_get_header_oh()
3486 if (bbr->r_ctl.rc_inc_ip_oh) { in bbr_get_header_oh()
3489 if (bbr->r_is_v6) { in bbr_get_header_oh()
3500 if (bbr->r_ctl.rc_inc_enet_oh) { in bbr_get_header_oh()
3508 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw) in bbr_get_pacing_length() argument
3514 gain = bbr_gain_adjust(bbr, gain); in bbr_get_pacing_length()
3517 res = (tim * bw * gain) / divor; in bbr_get_pacing_length()
3524 * Given a gain and a length return the delay in useconds that
3526 * on the connection (based on the gain factor).
3529 bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog) in bbr_get_pacing_delay() argument
3538 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in bbr_get_pacing_delay()
3539 num_segs = (len + maxseg - 1) / maxseg; in bbr_get_pacing_delay()
3540 if (bbr->rc_use_google == 0) { in bbr_get_pacing_delay()
3544 gain = bbr_gain_adjust(bbr, gain); in bbr_get_pacing_delay()
3546 if (bbr->rc_use_google) { in bbr_get_pacing_delay()
3553 cbw = bw * (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount); in bbr_get_pacing_delay()
3562 res = lentim / ((uint64_t)gain * bw); in bbr_get_pacing_delay()
3568 (bbr->rc_use_google == 0) && in bbr_get_pacing_delay()
3578 bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1); in bbr_get_pacing_delay()
3593 if ((tp->t_flags & TF_GPUTINPROG) && in bbr_ack_received()
3594 SEQ_GEQ(th->th_ack, tp->gput_ack)) { in bbr_ack_received()
3604 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8; in bbr_ack_received()
3605 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000)); in bbr_ack_received()
3607 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT, in bbr_ack_received()
3609 if (tp->t_stats_gput_prev > 0) in bbr_ack_received()
3610 stats_voi_update_abs_s32(tp->t_stats, in bbr_ack_received()
3612 ((gput - tp->t_stats_gput_prev) * 100) / in bbr_ack_received()
3613 tp->t_stats_gput_prev); in bbr_ack_received()
3614 tp->t_flags &= ~TF_GPUTINPROG; in bbr_ack_received()
3615 tp->t_stats_gput_prev = cgput; in bbr_ack_received()
3618 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && in bbr_ack_received()
3619 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) { in bbr_ack_received()
3620 /* We don't change anything in probe-rtt */ in bbr_ack_received()
3623 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_ack_received()
3627 bytes_this_ack -= prev_acked; in bbr_ack_received()
3632 if ((bytes_this_ack < maxseg) && bbr->rc_use_google) in bbr_ack_received()
3638 cwnd = tp->snd_cwnd; in bbr_ack_received()
3639 bw = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_ack_received()
3643 (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain); in bbr_ack_received()
3645 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp); in bbr_ack_received()
3646 if (IN_RECOVERY(tp->t_flags) && in bbr_ack_received()
3647 (bbr->bbr_prev_in_rec == 0)) { in bbr_ack_received()
3652 bbr->pkt_conservation = 1; in bbr_ack_received()
3653 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime; in bbr_ack_received()
3655 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_ack_received()
3658 if (IN_RECOVERY(tp->t_flags)) { in bbr_ack_received()
3661 bbr->bbr_prev_in_rec = 1; in bbr_ack_received()
3663 cwnd -= losses; in bbr_ack_received()
3669 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_ack_received()
3672 if (bbr->pkt_conservation) { in bbr_ack_received()
3675 if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start)) in bbr_ack_received()
3676 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start; in bbr_ack_received()
3682 bbr->pkt_conservation = 0; in bbr_ack_received()
3688 tp->snd_cwnd = cwnd; in bbr_ack_received()
3690 prev_acked, 1, target_cwnd, th->th_ack, line); in bbr_ack_received()
3695 bbr->bbr_prev_in_rec = 0; in bbr_ack_received()
3696 if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) { in bbr_ack_received()
3697 bbr->r_ctl.restrict_growth--; in bbr_ack_received()
3701 if (bbr->rc_filled_pipe) { in bbr_ack_received()
3713 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing) in bbr_ack_received()
3721 (bbr->rc_past_init_win == 0)) { in bbr_ack_received()
3726 * Method 4 means we are at target so no gain in in bbr_ack_received()
3732 tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr)); in bbr_ack_received()
3733 …bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, li… in bbr_ack_received()
3741 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in tcp_bbr_partialack()
3744 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= in tcp_bbr_partialack()
3745 tp->snd_cwnd) { in tcp_bbr_partialack()
3746 bbr->r_wanted_output = 1; in tcp_bbr_partialack()
3757 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_post_recovery()
3761 EXIT_RECOVERY(tp->t_flags); in bbr_post_recovery()
3762 /* Lock in our b/w reduction for the specified number of pkt-epochs */ in bbr_post_recovery()
3763 bbr->r_recovery_bw = 0; in bbr_post_recovery()
3764 tp->snd_recover = tp->snd_una; in bbr_post_recovery()
3765 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); in bbr_post_recovery()
3766 bbr->pkt_conservation = 0; in bbr_post_recovery()
3767 if (bbr->rc_use_google == 0) { in bbr_post_recovery()
3769 * For non-google mode lets in bbr_post_recovery()
3775 bbr->bbr_prev_in_rec = 0; in bbr_post_recovery()
3778 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { in bbr_post_recovery()
3779 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent); in bbr_post_recovery()
3782 /* For probe-rtt case lets fix up its saved_cwnd */ in bbr_post_recovery()
3783 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) { in bbr_post_recovery()
3784 bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent; in bbr_post_recovery()
3789 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_post_recovery()
3790 if ((bbr->rc_use_google == 0) && in bbr_post_recovery()
3804 bbr->r_ctl.recovery_lr, 21, in bbr_post_recovery()
3806 bbr->r_ctl.rc_red_cwnd_pe, in bbr_post_recovery()
3810 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && in bbr_post_recovery()
3813 (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && in bbr_post_recovery()
3814 bbr->rc_hit_state_1 && in bbr_post_recovery()
3816 ((bbr->rc_bbr_state == BBR_STATE_DRAIN) && in bbr_post_recovery()
3822 cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_post_recovery()
3823 cwnd_p = &bbr->r_ctl.rc_saved_cwnd; in bbr_post_recovery()
3825 cwnd = tp->snd_cwnd; in bbr_post_recovery()
3826 cwnd_p = &tp->snd_cwnd; in bbr_post_recovery()
3828 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_post_recovery()
3830 if (bbr->r_ctl.rc_lost == 0) in bbr_post_recovery()
3832 else if (bbr->r_ctl.rc_delivered == 0) in bbr_post_recovery()
3835 lr2use = (uint64_t)bbr->r_ctl.rc_lost * (uint64_t)1000; in bbr_post_recovery()
3836 lr2use /= bbr->r_ctl.rc_delivered; in bbr_post_recovery()
3838 lr2use += bbr->r_ctl.recovery_lr; in bbr_post_recovery()
3845 bbr->r_ctl.restrict_growth += acks_inflight; in bbr_post_recovery()
3851 newcwnd = roundup((cwnd - val), maxseg); in bbr_post_recovery()
3867 newcwnd = (get_min_cwnd(bbr) - acks_inflight); in bbr_post_recovery()
3879 if (tp->snd_cwnd > newcwnd) in bbr_post_recovery()
3880 tp->snd_cwnd = newcwnd; in bbr_post_recovery()
3884 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch; in bbr_post_recovery()
3887 bbr->r_ctl.recovery_lr = 0; in bbr_post_recovery()
3888 if (flight <= tp->snd_cwnd) { in bbr_post_recovery()
3889 bbr->r_wanted_output = 1; in bbr_post_recovery()
3891 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); in bbr_post_recovery()
3897 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_setup_red_bw()
3899 if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate) in bbr_setup_red_bw()
3900 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate; in bbr_setup_red_bw()
3901 if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2)) in bbr_setup_red_bw()
3902 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2; in bbr_setup_red_bw()
3913 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type); in bbr_cong_signal()
3915 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_cong_signal()
3918 if (!IN_RECOVERY(tp->t_flags)) { in bbr_cong_signal()
3919 tp->snd_recover = tp->snd_max; in bbr_cong_signal()
3921 bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_cong_signal()
3922 if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) { in bbr_cong_signal()
3928 bbr->r_ctl.rc_lt_epoch++; in bbr_cong_signal()
3930 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in bbr_cong_signal()
3937 bbr->r_ctl.rc_bbr_last_startup_epoch++; in bbr_cong_signal()
3939 bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd; in bbr_cong_signal()
3940 ENTER_RECOVERY(tp->t_flags); in bbr_cong_signal()
3941 bbr->rc_tlp_rtx_out = 0; in bbr_cong_signal()
3942 bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate; in bbr_cong_signal()
3943 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); in bbr_cong_signal()
3944 if (tcp_in_hpts(bbr->rc_tp) && in bbr_cong_signal()
3945 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) { in bbr_cong_signal()
3948 * any timers. This may mean we gain an agg in bbr_cong_signal()
3952 bbr->rc_timer_first = 1; in bbr_cong_signal()
3953 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_cong_signal()
3957 * delivery rate with no gain. We get the bdp in bbr_cong_signal()
3961 if ((bbr->rc_use_google == 0) && in bbr_cong_signal()
3962 (bbr->r_ctl.bbr_rttprobe_gain_val || in bbr_cong_signal()
3963 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) { in bbr_cong_signal()
3964 tp->snd_cwnd = ctf_flight_size(tp, in bbr_cong_signal()
3965 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_cong_signal()
3966 (tp->t_maxseg - bbr->rc_last_options); in bbr_cong_signal()
3967 if (tp->snd_cwnd < get_min_cwnd(bbr)) { in bbr_cong_signal()
3969 tp->snd_cwnd = get_min_cwnd(bbr); in bbr_cong_signal()
3973 bbr_log_type_enter_rec(bbr, rsm->r_start); in bbr_cong_signal()
3979 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime); in bbr_cong_signal()
3980 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { in bbr_cong_signal()
3981 tp->snd_cwnd = tp->snd_cwnd_prev; in bbr_cong_signal()
3982 tp->snd_ssthresh = tp->snd_ssthresh_prev; in bbr_cong_signal()
3983 tp->snd_recover = tp->snd_recover_prev; in bbr_cong_signal()
3984 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent); in bbr_cong_signal()
3987 tp->t_badrxtwin = 0; in bbr_cong_signal()
3995 * - There is no delayed ack timer in progress.
3996 * - Our last ack wasn't a 0-sized window. We never want to delay
3997 * the ack that opens up a 0-sized window.
3998 * - LRO wasn't used for this segment. We make sure by checking that the
4000 * - Delayed acks are enabled or this is a half-synchronized T/TCP
4002 * - The data being acked is less than a full segment (a stretch ack
4004 * - nsegs is 1 (if its more than that we received more than 1 ack).
4007 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
4008 ((tp->t_flags & TF_DELACK) == 0) && \
4009 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) && \
4010 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4025 * Walk the time-order transmitted list looking for an rsm that is in bbr_find_lowest_rsm()
4029 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) { in bbr_find_lowest_rsm()
4030 if (rsm->r_flags & BBR_ACKED) { in bbr_find_lowest_rsm()
4050 TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_find_high_nonack()
4051 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) { in bbr_find_high_nonack()
4072 * If reorder-fade is configured, then we track the last time we saw in bbr_calc_thresh_rack()
4073 * re-ordering occur. If we reach the point where enough time as in bbr_calc_thresh_rack()
4076 * Or if reorder-face is 0, then once we see reordering we consider in bbr_calc_thresh_rack()
4080 * In the end if lro is non-zero we add the extra time for in bbr_calc_thresh_rack()
4088 if (bbr->r_ctl.rc_reorder_ts) { in bbr_calc_thresh_rack()
4089 if (bbr->r_ctl.rc_reorder_fade) { in bbr_calc_thresh_rack()
4090 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) { in bbr_calc_thresh_rack()
4091 lro = cts - bbr->r_ctl.rc_reorder_ts; in bbr_calc_thresh_rack()
4103 if (lro > bbr->r_ctl.rc_reorder_fade) { in bbr_calc_thresh_rack()
4105 bbr->r_ctl.rc_reorder_ts = 0; in bbr_calc_thresh_rack()
4115 thresh = srtt + bbr->r_ctl.rc_pkt_delay; in bbr_calc_thresh_rack()
4118 if (bbr->r_ctl.rc_reorder_shift) in bbr_calc_thresh_rack()
4119 thresh += (srtt >> bbr->r_ctl.rc_reorder_shift); in bbr_calc_thresh_rack()
4126 if ((bbr->rc_tp)->t_srtt == 0) in bbr_calc_thresh_rack()
4129 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); in bbr_calc_thresh_rack()
4134 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { in bbr_calc_thresh_rack()
4135 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND); in bbr_calc_thresh_rack()
4142 * Return to the caller the amount of time in mico-seconds
4156 if (bbr->rc_tlp_threshold) in bbr_calc_thresh_tlp()
4157 thresh = srtt + (srtt / bbr->rc_tlp_threshold); in bbr_calc_thresh_tlp()
4160 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_calc_thresh_tlp()
4162 len = rsm->r_end - rsm->r_start; in bbr_calc_thresh_tlp()
4169 * possible inter-packet delay (if any). in bbr_calc_thresh_tlp()
4174 idx = rsm->r_rtr_cnt - 1; in bbr_calc_thresh_tlp()
4175 nidx = prsm->r_rtr_cnt - 1; in bbr_calc_thresh_tlp()
4176 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) { in bbr_calc_thresh_tlp()
4178 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx]; in bbr_calc_thresh_tlp()
4183 * Possibly compensate for delayed-ack. in bbr_calc_thresh_tlp()
4192 if (tp->t_srtt == 0) in bbr_calc_thresh_tlp()
4195 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur); in bbr_calc_thresh_tlp()
4203 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { in bbr_calc_thresh_tlp()
4204 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND); in bbr_calc_thresh_tlp()
4222 f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop); in bbr_get_rtt()
4223 if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) { in bbr_get_rtt()
4225 if (bbr->rc_tp->t_srtt == 0) in bbr_get_rtt()
4228 f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_get_rtt()
4231 * delayed-ack min in bbr_get_rtt()
4237 /* Take the filter version or last measured pkt-rtt */ in bbr_get_rtt()
4241 if (bbr->r_ctl.rc_pkt_epoch_rtt) { in bbr_get_rtt()
4242 srtt = bbr->r_ctl.rc_pkt_epoch_rtt; in bbr_get_rtt()
4248 srtt = bbr->r_ctl.rc_last_rtt; in bbr_get_rtt()
4250 if (bbr->rc_ack_was_delayed) in bbr_get_rtt()
4251 srtt += bbr->r_ctl.rc_ack_hdwr_delay; in bbr_get_rtt()
4253 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_get_rtt()
4271 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) { in bbr_is_lost()
4293 if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) { in bbr_check_recovery_mode()
4297 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_check_recovery_mode()
4302 if (tp->t_flags & TF_SENTFIN) { in bbr_check_recovery_mode()
4306 if (rsm->r_flags & BBR_ACKED) { in bbr_check_recovery_mode()
4315 idx = rsm->r_rtr_cnt - 1; in bbr_check_recovery_mode()
4316 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) { in bbr_check_recovery_mode()
4322 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) || in bbr_check_recovery_mode()
4323 (rsm->r_flags & BBR_SACK_PASSED))) { in bbr_check_recovery_mode()
4324 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) { in bbr_check_recovery_mode()
4325 rsm->r_flags |= BBR_MARKED_LOST; in bbr_check_recovery_mode()
4326 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; in bbr_check_recovery_mode()
4327 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; in bbr_check_recovery_mode()
4331 if ((rsm->r_end - rsm->r_start) == 0) in bbr_check_recovery_mode()
4332 panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm); in bbr_check_recovery_mode()
4352 * retransmissions, if so we will enter fast-recovery. The output in bbr_timeout_rack()
4358 if (bbr->rc_all_timers_stopped) { in bbr_timeout_rack()
4361 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_timeout_rack()
4366 lost = bbr->r_ctl.rc_lost; in bbr_timeout_rack()
4367 if (bbr->r_state && (bbr->r_state != tp->t_state)) in bbr_timeout_rack()
4370 if (bbr->r_ctl.rc_resend == NULL) { in bbr_timeout_rack()
4372 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_timeout_rack()
4375 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); in bbr_timeout_rack()
4376 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK; in bbr_timeout_rack()
4385 nrsm->r_start = start; in bbr_clone_rsm()
4386 nrsm->r_end = rsm->r_end; in bbr_clone_rsm()
4387 nrsm->r_rtr_cnt = rsm->r_rtr_cnt; in bbr_clone_rsm()
4388 nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed; in bbr_clone_rsm()
4389 nrsm->r_flags = rsm->r_flags; in bbr_clone_rsm()
4391 nrsm->r_flags &= ~BBR_HAS_SYN; in bbr_clone_rsm()
4393 rsm->r_flags &= ~BBR_HAS_FIN; in bbr_clone_rsm()
4394 nrsm->r_dupack = rsm->r_dupack; in bbr_clone_rsm()
4395 nrsm->r_rtr_bytes = 0; in bbr_clone_rsm()
4396 nrsm->r_is_gain = rsm->r_is_gain; in bbr_clone_rsm()
4397 nrsm->r_is_drain = rsm->r_is_drain; in bbr_clone_rsm()
4398 nrsm->r_delivered = rsm->r_delivered; in bbr_clone_rsm()
4399 nrsm->r_ts_valid = rsm->r_ts_valid; in bbr_clone_rsm()
4400 nrsm->r_del_ack_ts = rsm->r_del_ack_ts; in bbr_clone_rsm()
4401 nrsm->r_del_time = rsm->r_del_time; in bbr_clone_rsm()
4402 nrsm->r_app_limited = rsm->r_app_limited; in bbr_clone_rsm()
4403 nrsm->r_first_sent_time = rsm->r_first_sent_time; in bbr_clone_rsm()
4404 nrsm->r_flight_at_send = rsm->r_flight_at_send; in bbr_clone_rsm()
4406 nrsm->r_bbr_state = rsm->r_bbr_state; in bbr_clone_rsm()
4407 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) { in bbr_clone_rsm()
4408 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx]; in bbr_clone_rsm()
4410 rsm->r_end = nrsm->r_start; in bbr_clone_rsm()
4411 idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); in bbr_clone_rsm()
4414 if ((rsm->r_is_smallmap == 0) && in bbr_clone_rsm()
4415 ((rsm->r_end - rsm->r_start) <= idx)) { in bbr_clone_rsm()
4416 bbr->r_ctl.rc_num_small_maps_alloced++; in bbr_clone_rsm()
4417 rsm->r_is_smallmap = 1; in bbr_clone_rsm()
4420 if ((nrsm->r_end - nrsm->r_start) <= idx) { in bbr_clone_rsm()
4421 bbr->r_ctl.rc_num_small_maps_alloced++; in bbr_clone_rsm()
4422 nrsm->r_is_smallmap = 1; in bbr_clone_rsm()
4445 if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) { in bbr_sack_mergable()
4447 if ((l_rsm->r_end == start) || in bbr_sack_mergable()
4448 (SEQ_LT(start, l_rsm->r_end) && in bbr_sack_mergable()
4449 SEQ_GT(end, l_rsm->r_end))) { in bbr_sack_mergable()
4451 * map blk |------| in bbr_sack_mergable()
4452 * sack blk |------| in bbr_sack_mergable()
4454 * map blk |------| in bbr_sack_mergable()
4455 * sack blk |------| in bbr_sack_mergable()
4460 if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) { in bbr_sack_mergable()
4462 if ((r_rsm->r_start == end) || in bbr_sack_mergable()
4463 (SEQ_LT(start, r_rsm->r_start) && in bbr_sack_mergable()
4464 SEQ_GT(end, r_rsm->r_start))) { in bbr_sack_mergable()
4466 * map blk |---------| in bbr_sack_mergable()
4467 * sack blk |----| in bbr_sack_mergable()
4469 * map blk |---------| in bbr_sack_mergable()
4470 * sack blk |-------| in bbr_sack_mergable()
4493 l_rsm->r_end = r_rsm->r_end; in bbr_merge_rsm()
4494 if (l_rsm->r_dupack < r_rsm->r_dupack) in bbr_merge_rsm()
4495 l_rsm->r_dupack = r_rsm->r_dupack; in bbr_merge_rsm()
4496 if (r_rsm->r_rtr_bytes) in bbr_merge_rsm()
4497 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes; in bbr_merge_rsm()
4498 if (r_rsm->r_in_tmap) { in bbr_merge_rsm()
4500 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext); in bbr_merge_rsm()
4502 if (r_rsm->r_app_limited) in bbr_merge_rsm()
4503 l_rsm->r_app_limited = r_rsm->r_app_limited; in bbr_merge_rsm()
4505 if (r_rsm->r_flags & BBR_HAS_FIN) in bbr_merge_rsm()
4506 l_rsm->r_flags |= BBR_HAS_FIN; in bbr_merge_rsm()
4507 if (r_rsm->r_flags & BBR_TLP) in bbr_merge_rsm()
4508 l_rsm->r_flags |= BBR_TLP; in bbr_merge_rsm()
4509 if (r_rsm->r_flags & BBR_RWND_COLLAPSED) in bbr_merge_rsm()
4510 l_rsm->r_flags |= BBR_RWND_COLLAPSED; in bbr_merge_rsm()
4511 if (r_rsm->r_flags & BBR_MARKED_LOST) { in bbr_merge_rsm()
4513 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start; in bbr_merge_rsm()
4515 TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next); in bbr_merge_rsm()
4516 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) { in bbr_merge_rsm()
4518 r_rsm->r_limit_type = l_rsm->r_limit_type; in bbr_merge_rsm()
4519 l_rsm->r_limit_type = 0; in bbr_merge_rsm()
4546 if (bbr->rc_all_timers_stopped) { in bbr_timeout_tlp()
4549 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_timeout_tlp()
4555 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_tlp()
4558 if (bbr->rc_in_persist) { in bbr_timeout_tlp()
4561 if (bbr->r_state && (bbr->r_state != tp->t_state)) in bbr_timeout_tlp()
4564 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_tlp()
4570 avail = sbavail(&so->so_snd); in bbr_timeout_tlp()
4572 if (out > tp->snd_wnd) { in bbr_timeout_tlp()
4579 amm = avail - out; in bbr_timeout_tlp()
4582 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) { in bbr_timeout_tlp()
4583 /* not enough to fill a MTU and no-delay is off */ in bbr_timeout_tlp()
4586 /* Set the send-new override */ in bbr_timeout_tlp()
4587 if ((out + amm) <= tp->snd_wnd) { in bbr_timeout_tlp()
4588 bbr->rc_tlp_new_data = 1; in bbr_timeout_tlp()
4592 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; in bbr_timeout_tlp()
4593 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max; in bbr_timeout_tlp()
4594 bbr->r_ctl.rc_tlp_send = NULL; in bbr_timeout_tlp()
4601 * Ok we need to arrange the last un-acked segment to be re-sent, or in bbr_timeout_tlp()
4602 * optionally the first un-acked segment. in bbr_timeout_tlp()
4605 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); in bbr_timeout_tlp()
4606 if (rsm && (rsm->r_flags & (BBR_ACKED | BBR_HAS_FIN))) { in bbr_timeout_tlp()
4617 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_timeout_tlp()
4618 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) { in bbr_timeout_tlp()
4625 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_timeout_tlp()
4630 if ((rsm->r_end - rsm->r_start) > maxseg) { in bbr_timeout_tlp()
4641 * do the large send (BTLP :-) ). in bbr_timeout_tlp()
4645 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg)); in bbr_timeout_tlp()
4646 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_timeout_tlp()
4647 if (rsm->r_in_tmap) { in bbr_timeout_tlp()
4648 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_timeout_tlp()
4649 nrsm->r_in_tmap = 1; in bbr_timeout_tlp()
4651 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_timeout_tlp()
4655 bbr->r_ctl.rc_tlp_send = rsm; in bbr_timeout_tlp()
4656 bbr->rc_tlp_rtx_out = 1; in bbr_timeout_tlp()
4657 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) { in bbr_timeout_tlp()
4658 bbr->r_ctl.rc_tlp_seg_send_cnt++; in bbr_timeout_tlp()
4659 tp->t_rxtshift++; in bbr_timeout_tlp()
4661 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start; in bbr_timeout_tlp()
4662 bbr->r_ctl.rc_tlp_seg_send_cnt = 1; in bbr_timeout_tlp()
4665 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) { in bbr_timeout_tlp()
4671 bbr->rc_tlp_new_data = 0; in bbr_timeout_tlp()
4672 bbr->r_ctl.rc_tlp_send = NULL; in bbr_timeout_tlp()
4674 rsm->r_flags &= ~BBR_TLP; in bbr_timeout_tlp()
4678 rsm->r_flags |= BBR_TLP; in bbr_timeout_tlp()
4680 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) && in bbr_timeout_tlp()
4681 (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) { in bbr_timeout_tlp()
4689 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP; in bbr_timeout_tlp()
4704 if (bbr->rc_all_timers_stopped) { in bbr_timeout_delack()
4708 tp->t_flags &= ~TF_DELACK; in bbr_timeout_delack()
4709 tp->t_flags |= TF_ACKNOW; in bbr_timeout_delack()
4711 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK; in bbr_timeout_delack()
4716 * Here we send a KEEP-ALIVE like probe to the
4728 if (bbr->rc_all_timers_stopped) { in bbr_timeout_persist()
4731 if (bbr->rc_in_persist == 0) in bbr_timeout_persist()
4739 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT; in bbr_timeout_persist()
4746 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_persist()
4754 if (tp->t_rxtshift >= V_tcp_retries && in bbr_timeout_persist()
4755 (ticks - tp->t_rcvtime >= tcp_maxpersistidle || in bbr_timeout_persist()
4756 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) { in bbr_timeout_persist()
4759 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_persist()
4761 if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) && in bbr_timeout_persist()
4762 tp->snd_una == tp->snd_max) { in bbr_timeout_persist()
4771 if (tp->t_state > TCPS_CLOSE_WAIT && in bbr_timeout_persist()
4772 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) { in bbr_timeout_persist()
4775 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_persist()
4777 t_template = tcpip_maketemplate(bbr->rc_inp); in bbr_timeout_persist()
4779 tcp_respond(tp, t_template->tt_ipgen, in bbr_timeout_persist()
4780 &t_template->tt_t, (struct mbuf *)NULL, in bbr_timeout_persist()
4781 tp->rcv_nxt, tp->snd_una - 1, 0); in bbr_timeout_persist()
4783 if (tp->t_flags & TF_DELACK) in bbr_timeout_persist()
4784 tp->t_flags &= ~TF_DELACK; in bbr_timeout_persist()
4787 if (tp->t_rxtshift < V_tcp_retries) in bbr_timeout_persist()
4788 tp->t_rxtshift++; in bbr_timeout_persist()
4806 if (bbr->rc_all_timers_stopped) { in bbr_timeout_keepalive()
4809 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP; in bbr_timeout_keepalive()
4812 * Keep-alive timer went off; send something or drop connection if in bbr_timeout_keepalive()
4816 if (tp->t_state < TCPS_ESTABLISHED) in bbr_timeout_keepalive()
4818 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_timeout_keepalive()
4819 tp->t_state <= TCPS_CLOSING) { in bbr_timeout_keepalive()
4820 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp)) in bbr_timeout_keepalive()
4827 * number tp->snd_una-1 causes the transmitted zero-length in bbr_timeout_keepalive()
4835 tcp_respond(tp, t_template->tt_ipgen, in bbr_timeout_keepalive()
4836 &t_template->tt_t, (struct mbuf *)NULL, in bbr_timeout_keepalive()
4837 tp->rcv_nxt, tp->snd_una - 1, 0); in bbr_timeout_keepalive()
4846 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_keepalive()
4858 * un-acked. in bbr_remxt_tmr()
4864 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_remxt_tmr()
4865 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_remxt_tmr()
4866 lost = bbr->r_ctl.rc_lost; in bbr_remxt_tmr()
4867 if (bbr->r_state && (bbr->r_state != tp->t_state)) in bbr_remxt_tmr()
4870 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_remxt_tmr()
4871 if (rsm->r_flags & BBR_ACKED) { in bbr_remxt_tmr()
4874 rsm->r_dupack = 0; in bbr_remxt_tmr()
4875 if (rsm->r_in_tmap == 0) { in bbr_remxt_tmr()
4876 /* We must re-add it back to the tlist */ in bbr_remxt_tmr()
4878 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_remxt_tmr()
4880 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext); in bbr_remxt_tmr()
4882 rsm->r_in_tmap = 1; in bbr_remxt_tmr()
4884 old_flags = rsm->r_flags; in bbr_remxt_tmr()
4885 rsm->r_flags |= BBR_RXT_CLEARED; in bbr_remxt_tmr()
4886 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS); in bbr_remxt_tmr()
4889 if ((tp->t_state < TCPS_ESTABLISHED) && in bbr_remxt_tmr()
4890 (rsm->r_start == tp->snd_una)) { in bbr_remxt_tmr()
4898 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) { in bbr_remxt_tmr()
4899 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; in bbr_remxt_tmr()
4900 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; in bbr_remxt_tmr()
4907 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST; in bbr_remxt_tmr()
4908 rsm->r_flags &= ~BBR_WAS_SACKPASS; in bbr_remxt_tmr()
4917 rsm->r_flags |= BBR_MARKED_LOST; in bbr_remxt_tmr()
4918 rsm->r_flags &= ~BBR_WAS_SACKPASS; in bbr_remxt_tmr()
4919 rsm->r_flags &= ~BBR_SACK_PASSED; in bbr_remxt_tmr()
4924 bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_remxt_tmr()
4925 /* Clear the count (we just un-acked them) */ in bbr_remxt_tmr()
4927 bbr->rc_tlp_new_data = 0; in bbr_remxt_tmr()
4928 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; in bbr_remxt_tmr()
4930 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_remxt_tmr()
4931 bbr->r_agg_early_set = 0; in bbr_remxt_tmr()
4932 bbr->r_ctl.rc_agg_early = 0; in bbr_remxt_tmr()
4933 bbr->rc_tlp_rtx_out = 0; in bbr_remxt_tmr()
4934 bbr->r_ctl.rc_sacked = 0; in bbr_remxt_tmr()
4935 bbr->r_ctl.rc_sacklast = NULL; in bbr_remxt_tmr()
4936 bbr->r_timer_override = 1; in bbr_remxt_tmr()
4937 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); in bbr_remxt_tmr()
4941 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4952 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT; in bbr_timeout_rxt()
4953 if (bbr->rc_all_timers_stopped) { in bbr_timeout_rxt()
4956 if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_timeout_rxt()
4957 (tp->snd_una == tp->snd_max)) { in bbr_timeout_rxt()
4968 return (-ETIMEDOUT); /* tcp_drop() */ in bbr_timeout_rxt()
4971 if ((bbr->r_ctl.rc_resend == NULL) || in bbr_timeout_rxt()
4972 ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) { in bbr_timeout_rxt()
4979 tp->t_rxtshift++; in bbr_timeout_rxt()
4981 if (tp->t_rxtshift > V_tcp_retries) { in bbr_timeout_rxt()
4982 tp->t_rxtshift = V_tcp_retries; in bbr_timeout_rxt()
4986 MPASS(tp->t_softerror >= 0); in bbr_timeout_rxt()
4987 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT; in bbr_timeout_rxt()
4990 if (tp->t_state == TCPS_SYN_SENT) { in bbr_timeout_rxt()
4995 tp->snd_cwnd = 1; in bbr_timeout_rxt()
4996 } else if (tp->t_rxtshift == 1) { in bbr_timeout_rxt()
5003 * End-to-End Network Path Properties" by Allman and Paxson in bbr_timeout_rxt()
5006 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_rxt()
5007 if (!IN_RECOVERY(tp->t_flags)) { in bbr_timeout_rxt()
5008 tp->snd_cwnd_prev = tp->snd_cwnd; in bbr_timeout_rxt()
5009 tp->snd_ssthresh_prev = tp->snd_ssthresh; in bbr_timeout_rxt()
5010 tp->snd_recover_prev = tp->snd_recover; in bbr_timeout_rxt()
5011 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); in bbr_timeout_rxt()
5012 tp->t_flags |= TF_PREVVALID; in bbr_timeout_rxt()
5014 tp->t_flags &= ~TF_PREVVALID; in bbr_timeout_rxt()
5016 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_rxt()
5018 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; in bbr_timeout_rxt()
5019 tp->t_flags &= ~TF_PREVVALID; in bbr_timeout_rxt()
5022 if ((tp->t_state == TCPS_SYN_SENT) || in bbr_timeout_rxt()
5023 (tp->t_state == TCPS_SYN_RECEIVED)) in bbr_timeout_rxt()
5024 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift]; in bbr_timeout_rxt()
5026 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift]; in bbr_timeout_rxt()
5027 TCPT_RANGESET(tp->t_rxtcur, rexmt, in bbr_timeout_rxt()
5028 MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), in bbr_timeout_rxt()
5029 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000)); in bbr_timeout_rxt()
5038 isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false; in bbr_timeout_rxt()
5045 ((tp->t_state == TCPS_ESTABLISHED) || in bbr_timeout_rxt()
5046 (tp->t_state == TCPS_FIN_WAIT_1))) { in bbr_timeout_rxt()
5049 * 1448 -> 1188 -> 524) should be given 2 chances to recover in bbr_timeout_rxt()
5050 * before further clamping down. 'tp->t_rxtshift % 2 == 0' in bbr_timeout_rxt()
5053 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) == in bbr_timeout_rxt()
5055 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 && in bbr_timeout_rxt()
5056 tp->t_rxtshift % 2 == 0)) { in bbr_timeout_rxt()
5058 * Enter Path MTU Black-hole Detection mechanism: - in bbr_timeout_rxt()
5059 * Disable Path MTU Discovery (IP "DF" bit). - in bbr_timeout_rxt()
5063 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) { in bbr_timeout_rxt()
5068 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE; in bbr_timeout_rxt()
5070 tp->t_pmtud_saved_maxseg = tp->t_maxseg; in bbr_timeout_rxt()
5077 isipv6 = bbr->r_is_v6; in bbr_timeout_rxt()
5079 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) { in bbr_timeout_rxt()
5081 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss; in bbr_timeout_rxt()
5085 tp->t_maxseg = V_tcp_v6mssdflt; in bbr_timeout_rxt()
5090 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_timeout_rxt()
5098 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) { in bbr_timeout_rxt()
5100 tp->t_maxseg = V_tcp_pmtud_blackhole_mss; in bbr_timeout_rxt()
5104 tp->t_maxseg = V_tcp_mssdflt; in bbr_timeout_rxt()
5109 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_timeout_rxt()
5122 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) && in bbr_timeout_rxt()
5123 (tp->t_rxtshift >= 6)) { in bbr_timeout_rxt()
5124 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in bbr_timeout_rxt()
5125 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE; in bbr_timeout_rxt()
5126 tp->t_maxseg = tp->t_pmtud_saved_maxseg; in bbr_timeout_rxt()
5127 if (tp->t_maxseg < V_tcp_mssdflt) { in bbr_timeout_rxt()
5133 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT; in bbr_timeout_rxt()
5135 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT; in bbr_timeout_rxt()
5143 * third SYN to work-around some broken terminal servers (most of in bbr_timeout_rxt()
5146 * unknown-to-them TCP options. in bbr_timeout_rxt()
5148 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) && in bbr_timeout_rxt()
5149 (tp->t_rxtshift == 3)) in bbr_timeout_rxt()
5150 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT); in bbr_timeout_rxt()
5157 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) { in bbr_timeout_rxt()
5159 if (bbr->r_is_v6) in bbr_timeout_rxt()
5164 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT); in bbr_timeout_rxt()
5165 tp->t_srtt = 0; in bbr_timeout_rxt()
5167 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_timeout_rxt()
5168 tp->snd_recover = tp->snd_max; in bbr_timeout_rxt()
5169 tp->t_flags |= TF_ACKNOW; in bbr_timeout_rxt()
5170 tp->t_rtttime = 0; in bbr_timeout_rxt()
5179 int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK); in bbr_process_timers()
5184 if (tp->t_state == TCPS_LISTEN) { in bbr_process_timers()
5186 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) in bbr_process_timers()
5190 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_process_timers()
5193 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { in bbr_process_timers()
5194 ret = -1; in bbr_process_timers()
5199 ret = -2; in bbr_process_timers()
5207 left = bbr->r_ctl.rc_timer_exp - cts; in bbr_process_timers()
5208 ret = -3; in bbr_process_timers()
5213 bbr->rc_tmr_stopped = 0; in bbr_process_timers()
5214 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK; in bbr_process_timers()
5220 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5223 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5226 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5238 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { in bbr_timer_cancel()
5241 if (tcp_in_hpts(bbr->rc_tp) && in bbr_timer_cancel()
5242 (bbr->rc_timer_first == 1)) { in bbr_timer_cancel()
5249 tcp_hpts_remove(bbr->rc_tp); in bbr_timer_cancel()
5250 if (bbr->r_ctl.rc_last_delay_val) { in bbr_timer_cancel()
5254 if (TSTMP_GT(cts, bbr->rc_pacer_started)) in bbr_timer_cancel()
5255 time_since_send = cts - bbr->rc_pacer_started; in bbr_timer_cancel()
5258 if (bbr->r_ctl.rc_last_delay_val > time_since_send) { in bbr_timer_cancel()
5260 bbr->r_ctl.rc_last_delay_val -= time_since_send; in bbr_timer_cancel()
5262 bbr->r_ctl.rc_last_delay_val = 0; in bbr_timer_cancel()
5264 bbr->rc_pacer_started = cts; in bbr_timer_cancel()
5267 bbr->rc_timer_first = 0; in bbr_timer_cancel()
5269 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK; in bbr_timer_cancel()
5270 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK); in bbr_timer_cancel()
5279 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_stopall()
5280 bbr->rc_all_timers_stopped = 1; in bbr_stopall()
5292 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_get_earliest_send_outstanding()
5295 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]); in bbr_get_earliest_send_outstanding()
5304 rsm->r_rtr_cnt++; in bbr_update_rsm()
5305 rsm->r_dupack = 0; in bbr_update_rsm()
5306 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) { in bbr_update_rsm()
5307 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS; in bbr_update_rsm()
5308 rsm->r_flags |= BBR_OVERMAX; in bbr_update_rsm()
5310 if (rsm->r_flags & BBR_RWND_COLLAPSED) { in bbr_update_rsm()
5312 rsm->r_flags &= ~BBR_RWND_COLLAPSED; in bbr_update_rsm()
5314 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_update_rsm()
5316 rsm->r_flags &= ~BBR_MARKED_LOST; in bbr_update_rsm()
5317 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_update_rsm()
5319 if (rsm->r_flags & BBR_RXT_CLEARED) { in bbr_update_rsm()
5328 rsm->r_flags &= ~BBR_RXT_CLEARED; in bbr_update_rsm()
5330 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) { in bbr_update_rsm()
5331 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start); in bbr_update_rsm()
5332 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start); in bbr_update_rsm()
5334 idx = rsm->r_rtr_cnt - 1; in bbr_update_rsm()
5335 rsm->r_tim_lastsent[idx] = cts; in bbr_update_rsm()
5336 rsm->r_pacing_delay = pacing_time; in bbr_update_rsm()
5337 rsm->r_delivered = bbr->r_ctl.rc_delivered; in bbr_update_rsm()
5338 rsm->r_ts_valid = bbr->rc_ts_valid; in bbr_update_rsm()
5339 if (bbr->rc_ts_valid) in bbr_update_rsm()
5340 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts; in bbr_update_rsm()
5341 if (bbr->r_ctl.r_app_limited_until) in bbr_update_rsm()
5342 rsm->r_app_limited = 1; in bbr_update_rsm()
5344 rsm->r_app_limited = 0; in bbr_update_rsm()
5345 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) in bbr_update_rsm()
5346 rsm->r_bbr_state = bbr_state_val(bbr); in bbr_update_rsm()
5348 rsm->r_bbr_state = 8; in bbr_update_rsm()
5349 if (rsm->r_flags & BBR_ACKED) { in bbr_update_rsm()
5353 old_flags = rsm->r_flags; in bbr_update_rsm()
5354 rsm->r_flags &= ~BBR_ACKED; in bbr_update_rsm()
5356 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); in bbr_update_rsm()
5357 if (bbr->r_ctl.rc_sacked == 0) in bbr_update_rsm()
5358 bbr->r_ctl.rc_sacklast = NULL; in bbr_update_rsm()
5360 if (rsm->r_in_tmap) { in bbr_update_rsm()
5361 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_update_rsm()
5363 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_update_rsm()
5364 rsm->r_in_tmap = 1; in bbr_update_rsm()
5365 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_update_rsm()
5367 rsm->r_flags &= ~BBR_SACK_PASSED; in bbr_update_rsm()
5368 rsm->r_flags |= BBR_WAS_SACKPASS; in bbr_update_rsm()
5370 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); in bbr_update_rsm()
5371 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp, in bbr_update_rsm()
5372 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_update_rsm()
5373 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next); in bbr_update_rsm()
5374 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) { in bbr_update_rsm()
5375 rsm->r_is_gain = 1; in bbr_update_rsm()
5376 rsm->r_is_drain = 0; in bbr_update_rsm()
5377 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) { in bbr_update_rsm()
5378 rsm->r_is_drain = 1; in bbr_update_rsm()
5379 rsm->r_is_gain = 0; in bbr_update_rsm()
5381 rsm->r_is_drain = 0; in bbr_update_rsm()
5382 rsm->r_is_gain = 0; in bbr_update_rsm()
5384 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */ in bbr_update_rsm()
5398 * We (re-)transmitted starting at rsm->r_start for some length in bbr_update_entry()
5406 c_end = rsm->r_start + len; in bbr_update_entry()
5407 if (SEQ_GEQ(c_end, rsm->r_end)) { in bbr_update_entry()
5413 if (c_end == rsm->r_end) { in bbr_update_entry()
5420 act_len = rsm->r_end - rsm->r_start; in bbr_update_entry()
5421 *lenp = (len - act_len); in bbr_update_entry()
5422 return (rsm->r_end); in bbr_update_entry()
5443 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_update_entry()
5444 nrsm->r_dupack = 0; in bbr_update_entry()
5445 if (rsm->r_in_tmap) { in bbr_update_entry()
5446 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_update_entry()
5447 nrsm->r_in_tmap = 1; in bbr_update_entry()
5449 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_update_entry()
5474 if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) { in bbr_setup_less_of_rate()
5478 bbr->skip_gain = 1; in bbr_setup_less_of_rate()
5479 bbr->gain_is_limited = 0; in bbr_setup_less_of_rate()
5480 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate; in bbr_setup_less_of_rate()
5482 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts); in bbr_setup_less_of_rate()
5484 /* We can use a lower gain */ in bbr_setup_less_of_rate()
5485 bbr->skip_gain = 0; in bbr_setup_less_of_rate()
5486 bbr->gain_is_limited = 1; in bbr_setup_less_of_rate()
5494 int error, rate = -1; in bbr_update_hardware_pacing_rate()
5496 if (bbr->r_ctl.crte == NULL) in bbr_update_hardware_pacing_rate()
5498 if ((bbr->rc_inp->inp_route.ro_nh == NULL) || in bbr_update_hardware_pacing_rate()
5499 (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) { in bbr_update_hardware_pacing_rate()
5501 /* Clear the way for a re-attempt */ in bbr_update_hardware_pacing_rate()
5502 bbr->bbr_attempt_hdwr_pace = 0; in bbr_update_hardware_pacing_rate()
5504 bbr->gain_is_limited = 0; in bbr_update_hardware_pacing_rate()
5505 bbr->skip_gain = 0; in bbr_update_hardware_pacing_rate()
5506 bbr->bbr_hdrw_pacing = 0; in bbr_update_hardware_pacing_rate()
5507 counter_u64_add(bbr_flows_whdwr_pacing, -1); in bbr_update_hardware_pacing_rate()
5513 nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte, in bbr_update_hardware_pacing_rate()
5514 bbr->rc_tp, in bbr_update_hardware_pacing_rate()
5515 bbr->rc_inp->inp_route.ro_nh->nh_ifp, in bbr_update_hardware_pacing_rate()
5522 if (nrte != bbr->r_ctl.crte) { in bbr_update_hardware_pacing_rate()
5523 bbr->r_ctl.crte = nrte; in bbr_update_hardware_pacing_rate()
5526 if (bbr->r_ctl.crte->rate < rate) { in bbr_update_hardware_pacing_rate()
5529 bbr->r_ctl.crte->rate, rate); in bbr_update_hardware_pacing_rate()
5532 bbr->gain_is_limited = 0; in bbr_update_hardware_pacing_rate()
5533 bbr->skip_gain = 0; in bbr_update_hardware_pacing_rate()
5538 bbr->gain_is_limited = 0; in bbr_update_hardware_pacing_rate()
5539 bbr->skip_gain = 0; in bbr_update_hardware_pacing_rate()
5540 bbr->bbr_hdrw_pacing = 0; in bbr_update_hardware_pacing_rate()
5543 bbr->r_ctl.crte->ptbl->rs_ifp, in bbr_update_hardware_pacing_rate()
5545 bbr->r_ctl.crte->rate, in bbr_update_hardware_pacing_rate()
5563 if ((bbr->bbr_hdrw_pacing == 0) || in bbr_adjust_for_hw_pacing()
5564 (IN_RECOVERY(bbr->rc_tp->t_flags)) || in bbr_adjust_for_hw_pacing()
5565 (bbr->r_ctl.crte == NULL)) in bbr_adjust_for_hw_pacing()
5567 if (bbr->hw_pacing_set == 0) { in bbr_adjust_for_hw_pacing()
5575 rlp = bbr->r_ctl.crte; in bbr_adjust_for_hw_pacing()
5576 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) in bbr_adjust_for_hw_pacing()
5577 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in bbr_adjust_for_hw_pacing()
5579 maxseg = BBR_MIN_SEG - bbr->rc_last_options; in bbr_adjust_for_hw_pacing()
5587 bbr->r_ctl.rc_pace_max_segs, cts, 1); in bbr_adjust_for_hw_pacing()
5588 hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg; in bbr_adjust_for_hw_pacing()
5589 hdwr_delay *= rlp->time_between; in bbr_adjust_for_hw_pacing()
5591 delta = cur_delay - hdwr_delay; in bbr_adjust_for_hw_pacing()
5595 (bbr->r_ctl.rc_pace_max_segs / maxseg), in bbr_adjust_for_hw_pacing()
5598 (delta < (max(rlp->time_between, in bbr_adjust_for_hw_pacing()
5599 bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) { in bbr_adjust_for_hw_pacing()
5613 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between), in bbr_adjust_for_hw_pacing()
5614 (bbr->r_ctl.rc_pace_max_segs/maxseg)); in bbr_adjust_for_hw_pacing()
5617 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) { in bbr_adjust_for_hw_pacing()
5623 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg; in bbr_adjust_for_hw_pacing()
5637 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust; in bbr_adjust_for_hw_pacing()
5639 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) { in bbr_adjust_for_hw_pacing()
5645 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg; in bbr_adjust_for_hw_pacing()
5656 * just gain a bit of spacing from the in bbr_adjust_for_hw_pacing()
5659 seg_sz = bbr->r_ctl.rc_pace_max_segs; in bbr_adjust_for_hw_pacing()
5661 if (seg_sz > bbr->r_ctl.rc_pace_max_segs) in bbr_adjust_for_hw_pacing()
5664 new_tso = bbr->r_ctl.rc_pace_max_segs; in bbr_adjust_for_hw_pacing()
5665 if (new_tso >= (PACE_MAX_IP_BYTES-maxseg)) in bbr_adjust_for_hw_pacing()
5666 new_tso = PACE_MAX_IP_BYTES - maxseg; in bbr_adjust_for_hw_pacing()
5668 if (new_tso != bbr->r_ctl.rc_pace_max_segs) { in bbr_adjust_for_hw_pacing()
5669 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0); in bbr_adjust_for_hw_pacing()
5670 bbr->r_ctl.rc_pace_max_segs = new_tso; in bbr_adjust_for_hw_pacing()
5697 * a full iwnd just like new-reno/cubic. in tcp_bbr_tso_size_check()
5702 * if ( bw <= per-tcb-cross-over) in tcp_bbr_tso_size_check()
5704 * can send in goal-time seconds. in tcp_bbr_tso_size_check()
5710 * if (tso > per-tcb-max) in tcp_bbr_tso_size_check()
5711 * tso = per-tcb-max in tcp_bbr_tso_size_check()
5713 * tso = max-tso (64k/mss) in tcp_bbr_tso_size_check()
5715 * goal_tso = bw / per-tcb-divsor in tcp_bbr_tso_size_check()
5716 * seg = (goal_tso + mss-1)/mss in tcp_bbr_tso_size_check()
5719 * if (tso < per-tcb-floor) in tcp_bbr_tso_size_check()
5720 * tso = per-tcb-floor in tcp_bbr_tso_size_check()
5721 * if (tso > per-tcb-utter_max) in tcp_bbr_tso_size_check()
5722 * tso = per-tcb-utter_max in tcp_bbr_tso_size_check()
5724 * Note the default per-tcb-divisor is 1000 (same as google). in tcp_bbr_tso_size_check()
5728 * cross-over = 23,168,000 bps in tcp_bbr_tso_size_check()
5729 * goal-time = 18000 in tcp_bbr_tso_size_check()
5730 * per-tcb-max = 2 in tcp_bbr_tso_size_check()
5731 * per-tcb-divisor = 1000 in tcp_bbr_tso_size_check()
5732 * per-tcb-floor = 1 in tcp_bbr_tso_size_check()
5741 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) { in tcp_bbr_tso_size_check()
5742 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in tcp_bbr_tso_size_check()
5744 maxseg = BBR_MIN_SEG - bbr->rc_last_options; in tcp_bbr_tso_size_check()
5746 old_tso = bbr->r_ctl.rc_pace_max_segs; in tcp_bbr_tso_size_check()
5747 if (bbr->rc_past_init_win == 0) { in tcp_bbr_tso_size_check()
5753 if (bbr->rc_use_google) in tcp_bbr_tso_size_check()
5754 bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2); in tcp_bbr_tso_size_check()
5755 else if (bbr->bbr_init_win_cheat) in tcp_bbr_tso_size_check()
5756 bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp); in tcp_bbr_tso_size_check()
5758 bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in tcp_bbr_tso_size_check()
5759 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg) in tcp_bbr_tso_size_check()
5760 bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg; in tcp_bbr_tso_size_check()
5761 if (bbr->r_ctl.rc_pace_max_segs == 0) { in tcp_bbr_tso_size_check()
5762 bbr->r_ctl.rc_pace_max_segs = maxseg; in tcp_bbr_tso_size_check()
5764 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0); in tcp_bbr_tso_size_check()
5774 if (IN_RECOVERY(bbr->rc_tp->t_flags) && in tcp_bbr_tso_size_check()
5775 (bbr->rc_use_google == 0)) { in tcp_bbr_tso_size_check()
5778 } else if (bbr->rc_use_google) { in tcp_bbr_tso_size_check()
5781 /* Google considers the gain too */ in tcp_bbr_tso_size_check()
5782 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) { in tcp_bbr_tso_size_check()
5783 bw *= bbr->r_ctl.rc_bbr_hptsi_gain; in tcp_bbr_tso_size_check()
5797 } else if (bbr->rc_no_pacing) { in tcp_bbr_tso_size_check()
5799 } else if (bw <= bbr->r_ctl.bbr_cross_over) { in tcp_bbr_tso_size_check()
5806 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw); in tcp_bbr_tso_size_check()
5809 if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max) in tcp_bbr_tso_size_check()
5810 new_tso = bbr->r_ctl.bbr_hptsi_segments_max; in tcp_bbr_tso_size_check()
5843 bw /= bbr->r_ctl.bbr_hptsi_per_second; in tcp_bbr_tso_size_check()
5850 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg)) in tcp_bbr_tso_size_check()
5851 new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg); in tcp_bbr_tso_size_check()
5853 …if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floo… in tcp_bbr_tso_size_check()
5854 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor; in tcp_bbr_tso_size_check()
5858 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) { in tcp_bbr_tso_size_check()
5859 new_tso = bbr->r_ctl.bbr_utter_max * maxseg; in tcp_bbr_tso_size_check()
5864 bbr->r_ctl.rc_pace_max_segs = new_tso; in tcp_bbr_tso_size_check()
5899 * We don't log errors -- we could but snd_max does not in bbr_log_output()
5912 snd_una = tp->snd_una; in bbr_log_output()
5920 if ((th_flags & TH_SYN) && (tp->iss == seq_out)) in bbr_log_output()
5926 /* Are sending an old segment to induce an ack (keep-alive)? */ in bbr_log_output()
5935 len = end - seq_out; in bbr_log_output()
5937 snd_max = tp->snd_max; in bbr_log_output()
5942 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1); in bbr_log_output()
5950 rsm->r_flags = 0; in bbr_log_output()
5952 rsm->r_flags |= BBR_HAS_SYN; in bbr_log_output()
5954 rsm->r_flags |= BBR_HAS_FIN; in bbr_log_output()
5955 rsm->r_tim_lastsent[0] = cts; in bbr_log_output()
5956 rsm->r_rtr_cnt = 1; in bbr_log_output()
5957 rsm->r_rtr_bytes = 0; in bbr_log_output()
5958 rsm->r_start = seq_out; in bbr_log_output()
5959 rsm->r_end = rsm->r_start + len; in bbr_log_output()
5960 rsm->r_dupack = 0; in bbr_log_output()
5961 rsm->r_delivered = bbr->r_ctl.rc_delivered; in bbr_log_output()
5962 rsm->r_pacing_delay = pacing_time; in bbr_log_output()
5963 rsm->r_ts_valid = bbr->rc_ts_valid; in bbr_log_output()
5964 if (bbr->rc_ts_valid) in bbr_log_output()
5965 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts; in bbr_log_output()
5966 rsm->r_del_time = bbr->r_ctl.rc_del_time; in bbr_log_output()
5967 if (bbr->r_ctl.r_app_limited_until) in bbr_log_output()
5968 rsm->r_app_limited = 1; in bbr_log_output()
5970 rsm->r_app_limited = 0; in bbr_log_output()
5971 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); in bbr_log_output()
5972 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp, in bbr_log_output()
5973 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_log_output()
5978 rsm->r_flight_at_send += len; in bbr_log_output()
5979 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_log_output()
5980 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_log_output()
5981 rsm->r_in_tmap = 1; in bbr_log_output()
5982 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) in bbr_log_output()
5983 rsm->r_bbr_state = bbr_state_val(bbr); in bbr_log_output()
5985 rsm->r_bbr_state = 8; in bbr_log_output()
5986 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) { in bbr_log_output()
5987 rsm->r_is_gain = 1; in bbr_log_output()
5988 rsm->r_is_drain = 0; in bbr_log_output()
5989 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) { in bbr_log_output()
5990 rsm->r_is_drain = 1; in bbr_log_output()
5991 rsm->r_is_gain = 0; in bbr_log_output()
5993 rsm->r_is_drain = 0; in bbr_log_output()
5994 rsm->r_is_gain = 0; in bbr_log_output()
6002 if (hintrsm && (hintrsm->r_start == seq_out)) { in bbr_log_output()
6005 } else if (bbr->r_ctl.rc_next) { in bbr_log_output()
6007 rsm = bbr->r_ctl.rc_next; in bbr_log_output()
6012 if ((rsm) && (rsm->r_start == seq_out)) { in bbr_log_output()
6025 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_log_output()
6026 if (rsm->r_start == seq_out) { in bbr_log_output()
6028 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next); in bbr_log_output()
6035 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) { in bbr_log_output()
6051 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_log_output()
6052 if (rsm->r_in_tmap) { in bbr_log_output()
6053 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_log_output()
6054 nrsm->r_in_tmap = 1; in bbr_log_output()
6056 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_log_output()
6067 if (seq_out == tp->snd_max) { in bbr_log_output()
6069 } else if (SEQ_LT(seq_out, tp->snd_max)) { in bbr_log_output()
6071 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n", in bbr_log_output()
6072 seq_out, len, tp->snd_una, tp->snd_max); in bbr_log_output()
6074 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_log_output()
6075 printf("rsm:%p start:%u end:%u\n", in bbr_log_output()
6076 rsm, rsm->r_start, rsm->r_end); in bbr_log_output()
6079 panic("seq_out not found rack:%p tp:%p", in bbr_log_output()
6085 * Hmm beyond sndmax? (only if we are using the new rtt-pack in bbr_log_output()
6088 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p", in bbr_log_output()
6089 seq_out, len, tp->snd_max, tp); in bbr_log_output()
6098 * Collapse timeout back the cum-ack moved. in bbr_collapse_rtt()
6100 tp->t_rxtshift = 0; in bbr_collapse_rtt()
6101 tp->t_softerror = 0; in bbr_collapse_rtt()
6107 bbr->rtt_valid = 1; in tcp_bbr_xmit_timer()
6108 bbr->r_ctl.cur_rtt = rtt_usecs; in tcp_bbr_xmit_timer()
6109 bbr->r_ctl.ts_in = tsin; in tcp_bbr_xmit_timer()
6111 bbr->r_ctl.cur_rtt_send_time = rsm_send_time; in tcp_bbr_xmit_timer()
6119 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp). in bbr_make_timestamp_determination()
6120 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts). in bbr_make_timestamp_determination()
6121 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts) in bbr_make_timestamp_determination()
6122 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time) in bbr_make_timestamp_determination()
6126 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts in bbr_make_timestamp_determination()
6130 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp in bbr_make_timestamp_determination()
6137 * out time-slices of the actual b/w. This would mean we could not use in bbr_make_timestamp_determination()
6141 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio. in bbr_make_timestamp_determination()
6155 delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts; in bbr_make_timestamp_determination()
6165 peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp; in bbr_make_timestamp_determination()
6184 * then we saw then in micro-seconds. in bbr_make_timestamp_determination()
6187 /* well it looks like the peer is a micro-second clock. */ in bbr_make_timestamp_determination()
6188 bbr->rc_ts_clock_set = 1; in bbr_make_timestamp_determination()
6189 bbr->r_ctl.bbr_peer_tsratio = 1; in bbr_make_timestamp_determination()
6191 bbr->rc_ts_cant_be_used = 1; in bbr_make_timestamp_determination()
6192 bbr->rc_ts_clock_set = 1; in bbr_make_timestamp_determination()
6197 bbr->rc_ts_clock_set = 1; in bbr_make_timestamp_determination()
6202 bbr->r_ctl.bbr_peer_tsratio = 1; in bbr_make_timestamp_determination()
6209 bbr->r_ctl.bbr_peer_tsratio = 10; in bbr_make_timestamp_determination()
6216 bbr->r_ctl.bbr_peer_tsratio = 100; in bbr_make_timestamp_determination()
6223 bbr->r_ctl.bbr_peer_tsratio = 1000; in bbr_make_timestamp_determination()
6230 bbr->r_ctl.bbr_peer_tsratio = 10000; in bbr_make_timestamp_determination()
6234 bbr->rc_ts_cant_be_used = 1; in bbr_make_timestamp_determination()
6235 bbr->r_ctl.bbr_peer_tsratio = 0; in bbr_make_timestamp_determination()
6240 * Collect new round-trip time estimate
6250 if (bbr->rtt_valid == 0) in tcp_bbr_xmit_timer_commit()
6254 rtt = bbr->r_ctl.cur_rtt; in tcp_bbr_xmit_timer_commit()
6255 tsin = bbr->r_ctl.ts_in; in tcp_bbr_xmit_timer_commit()
6256 if (bbr->rc_prtt_set_ts) { in tcp_bbr_xmit_timer_commit()
6265 * and when they enter probe-rtt they update the in tcp_bbr_xmit_timer_commit()
6270 bbr->rc_prtt_set_ts = 0; in tcp_bbr_xmit_timer_commit()
6271 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop); in tcp_bbr_xmit_timer_commit()
6273 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts); in tcp_bbr_xmit_timer_commit()
6275 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in tcp_bbr_xmit_timer_commit()
6278 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt)); in tcp_bbr_xmit_timer_commit()
6280 if (bbr->rc_ack_was_delayed) in tcp_bbr_xmit_timer_commit()
6281 rtt += bbr->r_ctl.rc_ack_hdwr_delay; in tcp_bbr_xmit_timer_commit()
6283 if (rtt < bbr->r_ctl.rc_lowest_rtt) in tcp_bbr_xmit_timer_commit()
6284 bbr->r_ctl.rc_lowest_rtt = rtt; in tcp_bbr_xmit_timer_commit()
6286 if (bbr->r_init_rtt) { in tcp_bbr_xmit_timer_commit()
6288 * The initial rtt is not-trusted, nuke it and lets get in tcp_bbr_xmit_timer_commit()
6291 bbr->r_init_rtt = 0; in tcp_bbr_xmit_timer_commit()
6292 tp->t_srtt = 0; in tcp_bbr_xmit_timer_commit()
6294 if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) { in tcp_bbr_xmit_timer_commit()
6299 * series of cum-ack's to determine in tcp_bbr_xmit_timer_commit()
6302 if (bbr->rc_ack_is_cumack) { in tcp_bbr_xmit_timer_commit()
6303 if (bbr->rc_ts_data_set) { in tcp_bbr_xmit_timer_commit()
6307 bbr->rc_ts_data_set = 1; in tcp_bbr_xmit_timer_commit()
6308 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts; in tcp_bbr_xmit_timer_commit()
6309 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time; in tcp_bbr_xmit_timer_commit()
6316 bbr->rc_ts_data_set = 0; in tcp_bbr_xmit_timer_commit()
6320 rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1))); in tcp_bbr_xmit_timer_commit()
6321 if (tp->t_srtt != 0) { in tcp_bbr_xmit_timer_commit()
6330 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT) in tcp_bbr_xmit_timer_commit()
6331 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT)); in tcp_bbr_xmit_timer_commit()
6333 tp->t_srtt += delta; in tcp_bbr_xmit_timer_commit()
6334 if (tp->t_srtt <= 0) in tcp_bbr_xmit_timer_commit()
6335 tp->t_srtt = 1; in tcp_bbr_xmit_timer_commit()
6345 * wired-in beta. in tcp_bbr_xmit_timer_commit()
6348 delta = -delta; in tcp_bbr_xmit_timer_commit()
6349 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT); in tcp_bbr_xmit_timer_commit()
6350 tp->t_rttvar += delta; in tcp_bbr_xmit_timer_commit()
6351 if (tp->t_rttvar <= 0) in tcp_bbr_xmit_timer_commit()
6352 tp->t_rttvar = 1; in tcp_bbr_xmit_timer_commit()
6355 * No rtt measurement yet - use the unsmoothed rtt. Set the in tcp_bbr_xmit_timer_commit()
6359 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT; in tcp_bbr_xmit_timer_commit()
6360 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1); in tcp_bbr_xmit_timer_commit()
6363 if (tp->t_rttupdated < UCHAR_MAX) in tcp_bbr_xmit_timer_commit()
6364 tp->t_rttupdated++; in tcp_bbr_xmit_timer_commit()
6366 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks)); in tcp_bbr_xmit_timer_commit()
6372 * tick of rounding and 1 extra tick because of +-1/2 tick in tcp_bbr_xmit_timer_commit()
6378 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp), in tcp_bbr_xmit_timer_commit()
6379 max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2), in tcp_bbr_xmit_timer_commit()
6380 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000)); in tcp_bbr_xmit_timer_commit()
6389 tp->t_softerror = 0; in tcp_bbr_xmit_timer_commit()
6390 rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); in tcp_bbr_xmit_timer_commit()
6391 if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt) in tcp_bbr_xmit_timer_commit()
6392 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt; in tcp_bbr_xmit_timer_commit()
6398 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_set_reduced_rtt()
6400 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && in bbr_set_reduced_rtt()
6401 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { in bbr_set_reduced_rtt()
6403 * We should enter probe-rtt its been too long in bbr_set_reduced_rtt()
6416 if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) { in tcp_bbr_commit_bw()
6422 if (bbr->r_ctl.r_measurement_count < 0xffffffff) in tcp_bbr_commit_bw()
6423 bbr->r_ctl.r_measurement_count++; in tcp_bbr_commit_bw()
6424 orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate); in tcp_bbr_commit_bw()
6425 apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch); in tcp_bbr_commit_bw()
6427 (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate), in tcp_bbr_commit_bw()
6430 (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) { in tcp_bbr_commit_bw()
6431 if (bbr->bbr_hdrw_pacing) { in tcp_bbr_commit_bw()
6440 if (bbr->r_recovery_bw) { in tcp_bbr_commit_bw()
6444 } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate)) in tcp_bbr_commit_bw()
6451 if (bbr->rc_in_persist == 0) { in bbr_nf_measurement()
6457 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time)) in bbr_nf_measurement()
6458 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time); in bbr_nf_measurement()
6466 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered); in bbr_nf_measurement()
6476 * can see in the trace viewer if it gets over-ridden. in bbr_nf_measurement()
6478 if (rsm->r_ts_valid && in bbr_nf_measurement()
6479 bbr->rc_ts_valid && in bbr_nf_measurement()
6480 bbr->rc_ts_clock_set && in bbr_nf_measurement()
6481 (bbr->rc_ts_cant_be_used == 0) && in bbr_nf_measurement()
6482 bbr->rc_use_ts_limit) { in bbr_nf_measurement()
6483 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1); in bbr_nf_measurement()
6484 ts_diff *= bbr->r_ctl.bbr_peer_tsratio; in bbr_nf_measurement()
6490 bbr->r_ctl.last_inbound_ts, in bbr_nf_measurement()
6491 rsm->r_del_ack_ts, 0, in bbr_nf_measurement()
6501 if ((bbr->ts_can_raise) && in bbr_nf_measurement()
6521 if (rsm->r_first_sent_time && in bbr_nf_measurement()
6522 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) { in bbr_nf_measurement()
6535 sbw = (uint64_t)(rsm->r_flight_at_send); in bbr_nf_measurement()
6537 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time; in bbr_nf_measurement()
6538 sti += rsm->r_pacing_delay; in bbr_nf_measurement()
6546 rsm->r_first_sent_time, 0, (sbw >> 32), in bbr_nf_measurement()
6552 bbr->r_ctl.rc_bbr_cur_del_rate = bw; in bbr_nf_measurement()
6553 if ((rsm->r_app_limited == 0) || in bbr_nf_measurement()
6554 (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) { in bbr_nf_measurement()
6557 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time); in bbr_nf_measurement()
6565 if (bbr->rc_in_persist == 0) { in bbr_google_measurement()
6572 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time)) in bbr_google_measurement()
6573 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time); in bbr_google_measurement()
6581 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered); in bbr_google_measurement()
6585 if (tim < bbr->r_ctl.rc_lowest_rtt) { in bbr_google_measurement()
6587 tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0); in bbr_google_measurement()
6593 * can see in the trace viewer if it gets over-ridden. in bbr_google_measurement()
6595 bbr->r_ctl.rc_bbr_cur_del_rate = bw; in bbr_google_measurement()
6597 if (rsm->r_first_sent_time && in bbr_google_measurement()
6598 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) { in bbr_google_measurement()
6611 sbw = (uint64_t)(rsm->r_flight_at_send); in bbr_google_measurement()
6613 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time; in bbr_google_measurement()
6614 sti += rsm->r_pacing_delay; in bbr_google_measurement()
6622 rsm->r_first_sent_time, 0, (sbw >> 32), in bbr_google_measurement()
6627 (sti < bbr->r_ctl.rc_lowest_rtt)) { in bbr_google_measurement()
6629 (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0); in bbr_google_measurement()
6634 bbr->r_ctl.rc_bbr_cur_del_rate = bw; in bbr_google_measurement()
6636 ((rsm->r_app_limited == 0) || in bbr_google_measurement()
6637 (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) { in bbr_google_measurement()
6640 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time); in bbr_google_measurement()
6652 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start); in bbr_update_bbr_info()
6653 bbr->r_ctl.rc_del_time = cts; in bbr_update_bbr_info()
6661 if ((bbr->rc_use_google == 0) && in bbr_update_bbr_info()
6674 if ((bbr_no_retran && bbr->rc_use_google) && in bbr_update_bbr_info()
6684 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin); in bbr_update_bbr_info()
6686 bbr->rc_ack_is_cumack = 1; in bbr_update_bbr_info()
6688 bbr->rc_ack_is_cumack = 0; in bbr_update_bbr_info()
6695 * going into probe-rtt (we were seeing cases where that in bbr_update_bbr_info()
6704 bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts, in bbr_update_bbr_info()
6705 match, rsm->r_start, rsm->r_flags); in bbr_update_bbr_info()
6706 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_update_bbr_info()
6709 * The RTT-prop moved, reset the target (may be a in bbr_update_bbr_info()
6713 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) in bbr_update_bbr_info()
6720 if ((bbr->rc_use_google == 0) && in bbr_update_bbr_info()
6729 if (bbr->r_ctl.r_app_limited_until && in bbr_update_bbr_info()
6730 (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) { in bbr_update_bbr_info()
6731 /* We are no longer app-limited */ in bbr_update_bbr_info()
6732 bbr->r_ctl.r_app_limited_until = 0; in bbr_update_bbr_info()
6734 if (bbr->rc_use_google) { in bbr_update_bbr_info()
6751 msec = cts - (MS_IN_USEC * sec); in bbr_ts_convert()
6766 if ((rsm->r_flags & BBR_ACKED) || in bbr_update_rtt()
6767 (rsm->r_flags & BBR_WAS_RENEGED) || in bbr_update_rtt()
6768 (rsm->r_flags & BBR_RXT_CLEARED)) { in bbr_update_rtt()
6772 if (rsm->r_rtt_not_allowed) { in bbr_update_rtt()
6776 if (rsm->r_rtr_cnt == 1) { in bbr_update_rtt()
6780 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0])) in bbr_update_rtt()
6781 t = cts - rsm->r_tim_lastsent[0]; in bbr_update_rtt()
6784 bbr->r_ctl.rc_last_rtt = t; in bbr_update_rtt()
6785 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, in bbr_update_rtt()
6786 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to); in bbr_update_rtt()
6791 (bbr->rc_use_google == 1) && in bbr_update_rtt()
6793 (to->to_flags & TOF_TS) && in bbr_update_rtt()
6794 (to->to_tsecr != 0)) { in bbr_update_rtt()
6795 t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr; in bbr_update_rtt()
6799 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, in bbr_update_rtt()
6801 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)], in bbr_update_rtt()
6805 uts = bbr_ts_convert(to->to_tsecr); in bbr_update_rtt()
6806 if ((to->to_flags & TOF_TS) && in bbr_update_rtt()
6807 (to->to_tsecr != 0) && in bbr_update_rtt()
6809 ((rsm->r_flags & BBR_OVERMAX) == 0)) { in bbr_update_rtt()
6817 for (i = 0; i < rsm->r_rtr_cnt; i++) { in bbr_update_rtt()
6818 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) && in bbr_update_rtt()
6819 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) { in bbr_update_rtt()
6820 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6821 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6824 bbr->r_ctl.rc_last_rtt = t; in bbr_update_rtt()
6825 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING, in bbr_update_rtt()
6826 rsm->r_tim_lastsent[i], ack_type, to); in bbr_update_rtt()
6827 if ((i + 1) < rsm->r_rtr_cnt) { in bbr_update_rtt()
6830 } else if (rsm->r_flags & BBR_TLP) { in bbr_update_rtt()
6831 bbr->rc_tlp_rtx_out = 0; in bbr_update_rtt()
6841 * time-stamp since its not there or the time the peer last in bbr_update_rtt()
6842 * received a segment that moved forward its cum-ack point. in bbr_update_rtt()
6848 i = rsm->r_rtr_cnt - 1; in bbr_update_rtt()
6849 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6850 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6853 if (t < bbr->r_ctl.rc_lowest_rtt) { in bbr_update_rtt()
6859 * the rack-draft. in bbr_update_rtt()
6865 if ((rsm->r_flags & BBR_OVERMAX) == 0) { in bbr_update_rtt()
6867 if (rsm->r_rtr_cnt == 1) in bbr_update_rtt()
6868 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags); in bbr_update_rtt()
6870 i = rsm->r_rtr_cnt - 2; in bbr_update_rtt()
6871 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6872 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6875 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET, in bbr_update_rtt()
6876 rsm->r_tim_lastsent[i], ack_type, to); in bbr_update_rtt()
6884 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, in bbr_update_rtt()
6892 if (rsm->r_flags & BBR_TLP) in bbr_update_rtt()
6893 bbr->rc_tlp_rtx_out = 0; in bbr_update_rtt()
6894 if ((rsm->r_flags & BBR_OVERMAX) == 0) in bbr_update_rtt()
6895 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, in bbr_update_rtt()
6898 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, in bbr_update_rtt()
6915 TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap, in bbr_log_sack_passed()
6921 if (nrsm->r_flags & BBR_ACKED) { in bbr_log_sack_passed()
6925 if (nrsm->r_flags & BBR_SACK_PASSED) { in bbr_log_sack_passed()
6934 nrsm->r_flags |= BBR_SACK_PASSED; in bbr_log_sack_passed()
6935 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) && in bbr_log_sack_passed()
6936 bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) { in bbr_log_sack_passed()
6937 bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start; in bbr_log_sack_passed()
6938 bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start; in bbr_log_sack_passed()
6939 nrsm->r_flags |= BBR_MARKED_LOST; in bbr_log_sack_passed()
6941 nrsm->r_flags &= ~BBR_WAS_SACKPASS; in bbr_log_sack_passed()
6959 start = sack->start; in bbr_proc_sack_blk()
6960 end = sack->end; in bbr_proc_sack_blk()
6966 if (rsm && SEQ_LT(start, rsm->r_start)) { in bbr_proc_sack_blk()
6968 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_proc_sack_blk()
6969 if (SEQ_GEQ(start, rsm->r_start) && in bbr_proc_sack_blk()
6970 SEQ_LT(start, rsm->r_end)) { in bbr_proc_sack_blk()
6981 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_proc_sack_blk()
6982 if (SEQ_GEQ(start, rsm->r_start) && in bbr_proc_sack_blk()
6983 SEQ_LT(start, rsm->r_end)) { in bbr_proc_sack_blk()
6995 if (tp->t_flags & TF_SENTFIN) { in bbr_proc_sack_blk()
7000 nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); in bbr_proc_sack_blk()
7001 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) { in bbr_proc_sack_blk()
7005 nrsm->r_end++; in bbr_proc_sack_blk()
7012 panic("tp:%p bbr:%p sack:%p to:%p prsm:%p", in bbr_proc_sack_blk()
7024 if (rsm->r_start != start) { in bbr_proc_sack_blk()
7038 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk); in bbr_proc_sack_blk()
7042 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_proc_sack_blk()
7043 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7044 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_proc_sack_blk()
7045 nrsm->r_in_tmap = 1; in bbr_proc_sack_blk()
7047 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_proc_sack_blk()
7050 if (SEQ_GEQ(end, rsm->r_end)) { in bbr_proc_sack_blk()
7055 if ((rsm->r_flags & BBR_ACKED) == 0) { in bbr_proc_sack_blk()
7057 changed += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7058 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7060 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7061 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7064 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_proc_sack_blk()
7066 bbr->r_ctl.rc_reorder_ts = cts; in bbr_proc_sack_blk()
7067 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7068 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7069 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_proc_sack_blk()
7071 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_proc_sack_blk()
7074 rsm->r_flags |= BBR_ACKED; in bbr_proc_sack_blk()
7075 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST); in bbr_proc_sack_blk()
7076 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7077 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_proc_sack_blk()
7078 rsm->r_in_tmap = 0; in bbr_proc_sack_blk()
7082 if (end == rsm->r_end) { in bbr_proc_sack_blk()
7083 /* This block only - done */ in bbr_proc_sack_blk()
7087 start = rsm->r_end; in bbr_proc_sack_blk()
7093 if (rsm->r_flags & BBR_ACKED) { in bbr_proc_sack_blk()
7108 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk); in bbr_proc_sack_blk()
7114 rsm->r_flags &= (~BBR_HAS_FIN); in bbr_proc_sack_blk()
7115 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_proc_sack_blk()
7116 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7117 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_proc_sack_blk()
7118 nrsm->r_in_tmap = 1; in bbr_proc_sack_blk()
7120 nrsm->r_dupack = 0; in bbr_proc_sack_blk()
7123 changed += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7124 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); in bbr_proc_sack_blk()
7127 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7128 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7130 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_proc_sack_blk()
7132 bbr->r_ctl.rc_reorder_ts = cts; in bbr_proc_sack_blk()
7133 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_proc_sack_blk()
7134 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_proc_sack_blk()
7135 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_proc_sack_blk()
7137 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_proc_sack_blk()
7140 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST); in bbr_proc_sack_blk()
7141 rsm->r_flags |= BBR_ACKED; in bbr_proc_sack_blk()
7142 if (rsm->r_in_tmap) { in bbr_proc_sack_blk()
7143 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_proc_sack_blk()
7144 rsm->r_in_tmap = 0; in bbr_proc_sack_blk()
7147 if (rsm && (rsm->r_flags & BBR_ACKED)) { in bbr_proc_sack_blk()
7155 (nrsm->r_flags & BBR_ACKED)) { in bbr_proc_sack_blk()
7162 (nrsm->r_flags & BBR_ACKED)) { in bbr_proc_sack_blk()
7181 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next); in bbr_proc_sack_blk()
7183 bbr->r_ctl.rc_sacklast = NULL; in bbr_proc_sack_blk()
7195 while (rsm && (rsm->r_flags & BBR_ACKED)) { in bbr_peer_reneges()
7198 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); in bbr_peer_reneges()
7200 if (rsm->r_in_tmap) { in bbr_peer_reneges()
7201 panic("bbr:%p rsm:%p flags:0x%x in tmap?", in bbr_peer_reneges()
7202 bbr, rsm, rsm->r_flags); in bbr_peer_reneges()
7205 oflags = rsm->r_flags; in bbr_peer_reneges()
7206 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_peer_reneges()
7207 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_peer_reneges()
7208 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_peer_reneges()
7209 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_peer_reneges()
7211 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_peer_reneges()
7213 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST); in bbr_peer_reneges()
7214 rsm->r_flags |= BBR_WAS_RENEGED; in bbr_peer_reneges()
7215 rsm->r_flags |= BBR_RXT_CLEARED; in bbr_peer_reneges()
7216 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__); in bbr_peer_reneges()
7219 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_peer_reneges()
7222 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext); in bbr_peer_reneges()
7225 tmap->r_in_tmap = 1; in bbr_peer_reneges()
7240 sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack); in bbr_peer_reneges()
7250 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_log_syn()
7251 cts = bbr->r_ctl.rc_rcvtime; in bbr_log_syn()
7252 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_syn()
7253 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) { in bbr_log_syn()
7254 if ((rsm->r_end - rsm->r_start) <= 1) { in bbr_log_syn()
7256 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; in bbr_log_syn()
7257 rsm->r_rtr_bytes = 0; in bbr_log_syn()
7258 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_log_syn()
7259 if (rsm->r_in_tmap) { in bbr_log_syn()
7260 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_log_syn()
7261 rsm->r_in_tmap = 0; in bbr_log_syn()
7263 if (bbr->r_ctl.rc_next == rsm) { in bbr_log_syn()
7265 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_syn()
7272 rsm->r_flags &= ~BBR_HAS_SYN; in bbr_log_syn()
7273 rsm->r_start++; in bbr_log_syn()
7301 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_log_ack()
7302 cts = bbr->r_ctl.rc_rcvtime; in bbr_log_ack()
7304 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7306 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_log_ack()
7307 p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg); in bbr_log_ack()
7308 th_ack = th->th_ack; in bbr_log_ack()
7309 if (SEQ_GT(th_ack, tp->snd_una)) { in bbr_log_ack()
7311 bbr->rc_tp->t_acktime = ticks; in bbr_log_ack()
7313 if (SEQ_LEQ(th_ack, tp->snd_una)) { in bbr_log_ack()
7317 if (rsm && SEQ_GT(th_ack, rsm->r_start)) { in bbr_log_ack()
7318 changed = th_ack - rsm->r_start; in bbr_log_ack()
7319 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) { in bbr_log_ack()
7325 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) { in bbr_log_ack()
7332 ts = bbr_ts_convert(to->to_tsecr); in bbr_log_ack()
7333 now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv)); in bbr_log_ack()
7334 rtt = now - ts; in bbr_log_ack()
7338 tp->iss, 0, cts, in bbr_log_ack()
7339 BBR_RTT_BY_TIMESTAMP, tp->iss, 0); in bbr_log_ack()
7340 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_log_ack()
7342 bbr->r_wanted_output = 1; in bbr_log_ack()
7355 bbr->r_wanted_output = 1; in bbr_log_ack()
7358 if (tp->t_flags & TF_SENTFIN) { in bbr_log_ack()
7363 panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n", in bbr_log_ack()
7365 th, tp->t_state, bbr, in bbr_log_ack()
7366 tp->snd_una, tp->snd_max, changed); in bbr_log_ack()
7371 if (SEQ_LT(th_ack, rsm->r_start)) { in bbr_log_ack()
7374 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d bbr:%p\n", in bbr_log_ack()
7375 rsm->r_start, in bbr_log_ack()
7376 th_ack, tp->t_state, in bbr_log_ack()
7377 bbr->r_state, bbr); in bbr_log_ack()
7378 panic("th-ack is bad bbr:%p tp:%p", bbr, tp); in bbr_log_ack()
7381 } else if (th_ack == rsm->r_start) { in bbr_log_ack()
7390 rsm->r_dupack = 0; in bbr_log_ack()
7392 if (SEQ_GEQ(th_ack, rsm->r_end)) { in bbr_log_ack()
7396 if (rsm->r_flags & BBR_ACKED) { in bbr_log_ack()
7398 * It was acked on the scoreboard -- remove it from in bbr_log_ack()
7401 p_acked += (rsm->r_end - rsm->r_start); in bbr_log_ack()
7402 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); in bbr_log_ack()
7403 if (bbr->r_ctl.rc_sacked == 0) in bbr_log_ack()
7404 bbr->r_ctl.rc_sacklast = NULL; in bbr_log_ack()
7407 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_log_ack()
7408 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; in bbr_log_ack()
7410 if (rsm->r_flags & BBR_SACK_PASSED) { in bbr_log_ack()
7417 bbr->r_ctl.rc_reorder_ts = cts; in bbr_log_ack()
7418 if (rsm->r_flags & BBR_MARKED_LOST) { in bbr_log_ack()
7419 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; in bbr_log_ack()
7420 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) in bbr_log_ack()
7422 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; in bbr_log_ack()
7425 rsm->r_flags &= ~BBR_MARKED_LOST; in bbr_log_ack()
7427 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; in bbr_log_ack()
7428 rsm->r_rtr_bytes = 0; in bbr_log_ack()
7429 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_log_ack()
7430 if (rsm->r_in_tmap) { in bbr_log_ack()
7431 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_log_ack()
7432 rsm->r_in_tmap = 0; in bbr_log_ack()
7434 if (bbr->r_ctl.rc_next == rsm) { in bbr_log_ack()
7436 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7440 left = th_ack - rsm->r_end; in bbr_log_ack()
7444 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7449 if (rsm->r_flags & BBR_ACKED) { in bbr_log_ack()
7451 * It was acked on the scoreboard -- remove it from total in bbr_log_ack()
7452 * for the part being cum-acked. in bbr_log_ack()
7454 p_acked += (rsm->r_end - rsm->r_start); in bbr_log_ack()
7455 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start); in bbr_log_ack()
7456 if (bbr->r_ctl.rc_sacked == 0) in bbr_log_ack()
7457 bbr->r_ctl.rc_sacklast = NULL; in bbr_log_ack()
7468 if ((rsm->r_flags & BBR_MARKED_LOST) && in bbr_log_ack()
7469 ((rsm->r_flags & BBR_ACKED) == 0)) { in bbr_log_ack()
7475 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start; in bbr_log_ack()
7478 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; in bbr_log_ack()
7479 rsm->r_rtr_bytes = 0; in bbr_log_ack()
7481 rsm->r_start = th_ack; in bbr_log_ack()
7484 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_log_ack()
7485 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) { in bbr_log_ack()
7491 * us snd_una up to (rsm->r_end). We need to undo the acked in bbr_log_ack()
7495 * rsm->r_start in case we get an old ack where th_ack is in bbr_log_ack()
7498 bbr_peer_reneges(bbr, rsm, th->th_ack); in bbr_log_ack()
7500 if ((to->to_flags & TOF_SACK) == 0) { in bbr_log_ack()
7504 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); in bbr_log_ack()
7506 last_seq = rsm->r_end; in bbr_log_ack()
7508 last_seq = tp->snd_max; in bbr_log_ack()
7511 if (SEQ_GT(th_ack, tp->snd_una)) in bbr_log_ack()
7514 ack_point = tp->snd_una; in bbr_log_ack()
7515 for (i = 0; i < to->to_nsacks; i++) { in bbr_log_ack()
7516 bcopy((to->to_sacks + i * TCPOLEN_SACK), in bbr_log_ack()
7522 SEQ_LT(sack.start, tp->snd_max) && in bbr_log_ack()
7524 SEQ_LEQ(sack.end, tp->snd_max)) { in bbr_log_ack()
7525 if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) && in bbr_log_ack()
7527 ((sack.end - sack.start) < (p_maxseg / 8))) { in bbr_log_ack()
7540 * Its a D-SACK block. in bbr_log_ack()
7551 new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks, in bbr_log_ack()
7552 num_sack_blks, th->th_ack); in bbr_log_ack()
7553 ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks); in bbr_log_ack()
7555 BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb)); in bbr_log_ack()
7593 * Now collapse out the dup-sack and in bbr_log_ack()
7601 num_sack_blks--; in bbr_log_ack()
7608 rsm = bbr->r_ctl.rc_sacklast; in bbr_log_ack()
7612 bbr->r_wanted_output = 1; in bbr_log_ack()
7619 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) { in bbr_log_ack()
7630 bbr->r_wanted_output = 1; in bbr_log_ack()
7635 if (bbr->r_ctl.rc_resend == NULL) { in bbr_log_ack()
7636 bbr->r_ctl.rc_resend = rsm; in bbr_log_ack()
7640 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) { in bbr_log_ack()
7642 * See if we need to rack-retransmit anything if so set it in bbr_log_ack()
7646 if (bbr->r_ctl.rc_resend == NULL) { in bbr_log_ack()
7647 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_log_ack()
7652 * ack-received code to augment what was changed between th_ack <-> in bbr_log_ack()
7663 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); in bbr_strike_dupack()
7664 if (rsm && (rsm->r_dupack < 0xff)) { in bbr_strike_dupack()
7665 rsm->r_dupack++; in bbr_strike_dupack()
7666 if (rsm->r_dupack >= DUP_ACK_THRESHOLD) in bbr_strike_dupack()
7667 bbr->r_wanted_output = 1; in bbr_strike_dupack()
7674 * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7692 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_process_ack()
7693 lost = bbr->r_ctl.rc_lost; in bbr_process_ack()
7694 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_process_ack()
7695 if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) { in bbr_process_ack()
7697 tp->t_flags2 |= TF2_NO_ISS_CHECK; in bbr_process_ack()
7703 if (tp->t_flags2 & TF2_NO_ISS_CHECK) { in bbr_process_ack()
7705 seq_min = tp->snd_una - tp->max_sndwnd; in bbr_process_ack()
7708 if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) { in bbr_process_ack()
7710 seq_min = tp->iss + 1; in bbr_process_ack()
7717 seq_min = tp->snd_una - tp->max_sndwnd; in bbr_process_ack()
7721 if (SEQ_LT(th->th_ack, seq_min)) { in bbr_process_ack()
7728 bbr->r_wanted_output = 1; in bbr_process_ack()
7732 if (SEQ_GT(th->th_ack, tp->snd_max)) { in bbr_process_ack()
7734 bbr->r_wanted_output = 1; in bbr_process_ack()
7737 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { in bbr_process_ack()
7739 if (bbr->rc_in_persist) in bbr_process_ack()
7740 tp->t_rxtshift = 0; in bbr_process_ack()
7741 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd)) in bbr_process_ack()
7745 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost)); in bbr_process_ack()
7746 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { in bbr_process_ack()
7751 if (th->th_ack == tp->snd_una) { in bbr_process_ack()
7753 if (bbr->r_state == TCPS_SYN_SENT) { in bbr_process_ack()
7756 * the SYN-ACK is processed in syn_sent in bbr_process_ack()
7773 if (tp->t_flags & TF_NEEDSYN) { in bbr_process_ack()
7775 * T/TCP: Connection was half-synchronized, and our SYN has in bbr_process_ack()
7777 * to non-starred state, increment snd_una for ACK of SYN, in bbr_process_ack()
7778 * and check if we can do window scaling. in bbr_process_ack()
7780 tp->t_flags &= ~TF_NEEDSYN; in bbr_process_ack()
7781 tp->snd_una++; in bbr_process_ack()
7782 /* Do window scaling? */ in bbr_process_ack()
7783 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == in bbr_process_ack()
7785 tp->rcv_scale = tp->request_r_scale; in bbr_process_ack()
7801 if (tp->t_flags & TF_PREVVALID) { in bbr_process_ack()
7802 tp->t_flags &= ~TF_PREVVALID; in bbr_process_ack()
7803 if (tp->t_rxtshift == 1 && in bbr_process_ack()
7804 (int)(ticks - tp->t_badrxtwin) < 0) in bbr_process_ack()
7808 acked_amount = min(acked, (int)sbavail(&so->so_snd)); in bbr_process_ack()
7809 tp->snd_wnd -= acked_amount; in bbr_process_ack()
7810 mfree = sbcut_locked(&so->so_snd, acked_amount); in bbr_process_ack()
7814 if (SEQ_GT(th->th_ack, tp->snd_una)) { in bbr_process_ack()
7817 tp->snd_una = th->th_ack; in bbr_process_ack()
7818 …bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - los… in bbr_process_ack()
7819 if (IN_RECOVERY(tp->t_flags)) { in bbr_process_ack()
7820 if (SEQ_LT(th->th_ack, tp->snd_recover) && in bbr_process_ack()
7821 (SEQ_LT(th->th_ack, tp->snd_max))) { in bbr_process_ack()
7827 if (SEQ_GT(tp->snd_una, tp->snd_recover)) { in bbr_process_ack()
7828 tp->snd_recover = tp->snd_una; in bbr_process_ack()
7830 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_process_ack()
7831 tp->snd_nxt = tp->snd_max; in bbr_process_ack()
7833 if (tp->snd_una == tp->snd_max) { in bbr_process_ack()
7837 if (sbavail(&so->so_snd) == 0) in bbr_process_ack()
7838 bbr->rc_tp->t_acktime = 0; in bbr_process_ack()
7839 if ((sbused(&so->so_snd) == 0) && in bbr_process_ack()
7840 (tp->t_flags & TF_SENTFIN)) { in bbr_process_ack()
7843 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_ack()
7844 if (bbr->rc_in_persist == 0) { in bbr_process_ack()
7845 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime; in bbr_process_ack()
7847 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_process_ack()
7848 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime); in bbr_process_ack()
7854 if ((tp->t_state >= TCPS_FIN_WAIT_1) && in bbr_process_ack()
7855 (sbavail(&so->so_snd) == 0) && in bbr_process_ack()
7856 (tp->t_flags2 & TF2_DROP_AF_DATA)) { in bbr_process_ack()
7863 /* tcp_close will kill the inp pre-log the Reset */ in bbr_process_ack()
7871 bbr->r_wanted_output = 1; in bbr_process_ack()
7881 if (bbr->rc_in_persist == 0) { in bbr_enter_persist()
7883 bbr->r_ctl.rc_last_delay_val = 0; in bbr_enter_persist()
7884 tp->t_rxtshift = 0; in bbr_enter_persist()
7885 bbr->rc_in_persist = 1; in bbr_enter_persist()
7886 bbr->r_ctl.rc_went_idle_time = cts; in bbr_enter_persist()
7890 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_enter_persist()
7893 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_enter_persist()
7894 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_enter_persist()
7900 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_enter_persist()
7903 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_enter_persist()
7916 if (bbr->rc_use_idle_restart) { in bbr_restart_after_idle()
7917 bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT; in bbr_restart_after_idle()
7924 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_restart_after_idle()
7925 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_restart_after_idle()
7928 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; in bbr_restart_after_idle()
7929 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; in bbr_restart_after_idle()
7931 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_restart_after_idle()
7942 if (bbr->rc_in_persist == 0) in bbr_exit_persist()
7944 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time); in bbr_exit_persist()
7945 bbr->rc_in_persist = 0; in bbr_exit_persist()
7946 bbr->rc_hit_state_1 = 0; in bbr_exit_persist()
7947 bbr->r_ctl.rc_del_time = cts; in bbr_exit_persist()
7953 if (tcp_in_hpts(bbr->rc_tp)) { in bbr_exit_persist()
7954 tcp_hpts_remove(bbr->rc_tp); in bbr_exit_persist()
7955 bbr->rc_timer_first = 0; in bbr_exit_persist()
7956 bbr->r_ctl.rc_hpts_flags = 0; in bbr_exit_persist()
7957 bbr->r_ctl.rc_last_delay_val = 0; in bbr_exit_persist()
7958 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_exit_persist()
7959 bbr->r_agg_early_set = 0; in bbr_exit_persist()
7960 bbr->r_ctl.rc_agg_early = 0; in bbr_exit_persist()
7970 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_exit_persist()
7972 tp->t_rxtshift = 0; in bbr_exit_persist()
7983 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_exit_persist()
7984 /* Time un-freezes for the state */ in bbr_exit_persist()
7985 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_exit_persist()
7986 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) || in bbr_exit_persist()
7987 (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) { in bbr_exit_persist()
7989 * If we are going back to probe-bw in bbr_exit_persist()
8022 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; in bbr_collapsed_window()
8023 max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd; in bbr_collapsed_window()
8025 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_collapsed_window()
8027 if (rsm->r_flags & BBR_RWND_COLLAPSED) in bbr_collapsed_window()
8028 rsm->r_flags &= ~BBR_RWND_COLLAPSED; in bbr_collapsed_window()
8029 if (SEQ_GEQ(max_seq, rsm->r_start) && in bbr_collapsed_window()
8030 SEQ_GEQ(rsm->r_end, max_seq)) { in bbr_collapsed_window()
8035 bbr->rc_has_collapsed = 0; in bbr_collapsed_window()
8048 if ((max_seq != rsm->r_start) && in bbr_collapsed_window()
8049 (max_seq != rsm->r_end)){ in bbr_collapsed_window()
8053 res1 = max_seq - rsm->r_start; in bbr_collapsed_window()
8054 res2 = rsm->r_end - max_seq; in bbr_collapsed_window()
8059 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) { in bbr_collapsed_window()
8065 if (max_seq == rsm->r_start) { in bbr_collapsed_window()
8068 } else if (max_seq == rsm->r_end) { in bbr_collapsed_window()
8075 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) { in bbr_collapsed_window()
8086 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); in bbr_collapsed_window()
8087 if (rsm->r_in_tmap) { in bbr_collapsed_window()
8088 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); in bbr_collapsed_window()
8089 nrsm->r_in_tmap = 1; in bbr_collapsed_window()
8102 TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) { in bbr_collapsed_window()
8103 nrsm->r_flags |= BBR_RWND_COLLAPSED; in bbr_collapsed_window()
8105 bbr->rc_has_collapsed = 1; in bbr_collapsed_window()
8116 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { in bbr_un_collapse_window()
8117 if (rsm->r_flags & BBR_RWND_COLLAPSED) { in bbr_un_collapse_window()
8119 rsm->r_flags &= ~BBR_RWND_COLLAPSED; in bbr_un_collapse_window()
8125 (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared); in bbr_un_collapse_window()
8126 bbr->rc_has_collapsed = 0; in bbr_un_collapse_window()
8147 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_process_data()
8149 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_process_data()
8151 (SEQ_LT(tp->snd_wl1, th->th_seq) || in bbr_process_data()
8152 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || in bbr_process_data()
8153 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { in bbr_process_data()
8156 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) in bbr_process_data()
8158 tp->snd_wnd = tiwin; in bbr_process_data()
8159 tp->snd_wl1 = th->th_seq; in bbr_process_data()
8160 tp->snd_wl2 = th->th_ack; in bbr_process_data()
8161 if (tp->snd_wnd > tp->max_sndwnd) in bbr_process_data()
8162 tp->max_sndwnd = tp->snd_wnd; in bbr_process_data()
8163 bbr->r_wanted_output = 1; in bbr_process_data()
8165 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) { in bbr_process_data()
8166 tp->snd_wnd = tiwin; in bbr_process_data()
8167 tp->snd_wl1 = th->th_seq; in bbr_process_data()
8168 tp->snd_wl2 = th->th_ack; in bbr_process_data()
8171 if (tp->snd_wnd < ctf_outstanding(tp)) in bbr_process_data()
8174 else if (bbr->rc_has_collapsed) in bbr_process_data()
8177 if ((bbr->rc_in_persist != 0) && in bbr_process_data()
8178 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2), in bbr_process_data()
8184 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_process_data()
8187 bbr->r_wanted_output = 1; in bbr_process_data()
8190 if ((bbr->rc_in_persist == 0) && in bbr_process_data()
8191 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_process_data()
8192 TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_process_data()
8193 (tp->snd_max == tp->snd_una) && in bbr_process_data()
8194 sbavail(&so->so_snd) && in bbr_process_data()
8195 (sbavail(&so->so_snd) > tp->snd_wnd)) { in bbr_process_data()
8197 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_process_data()
8199 if (tp->t_flags2 & TF2_DROP_AF_DATA) { in bbr_process_data()
8209 tp->rcv_up = tp->rcv_nxt; in bbr_process_data()
8214 * This process logically involves adjusting tp->rcv_wnd as data is in bbr_process_data()
8219 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && in bbr_process_data()
8220 (tp->t_flags & TF_FASTOPEN)); in bbr_process_data()
8222 TCPS_HAVERCVDFIN(tp->t_state) == 0) { in bbr_process_data()
8223 tcp_seq save_start = th->th_seq; in bbr_process_data()
8224 tcp_seq save_rnxt = tp->rcv_nxt; in bbr_process_data()
8239 if (th->th_seq == tp->rcv_nxt && in bbr_process_data()
8241 (TCPS_HAVEESTABLISHED(tp->t_state) || in bbr_process_data()
8246 if (so->so_rcv.sb_shlim) { in bbr_process_data()
8249 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, in bbr_process_data()
8259 bbr->bbr_segs_rcvd += max(1, nsegs); in bbr_process_data()
8260 tp->t_flags |= TF_DELACK; in bbr_process_data()
8261 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_data()
8263 bbr->r_wanted_output = 1; in bbr_process_data()
8264 tp->t_flags |= TF_ACKNOW; in bbr_process_data()
8266 tp->rcv_nxt += tlen; in bbr_process_data()
8268 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && in bbr_process_data()
8269 (tp->t_fbyte_in == 0)) { in bbr_process_data()
8270 tp->t_fbyte_in = ticks; in bbr_process_data()
8271 if (tp->t_fbyte_in == 0) in bbr_process_data()
8272 tp->t_fbyte_in = 1; in bbr_process_data()
8273 if (tp->t_fbyte_out && tp->t_fbyte_in) in bbr_process_data()
8274 tp->t_flags2 |= TF2_FBYTES_COMPLETE; in bbr_process_data()
8280 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) in bbr_process_data()
8286 sbappendstream_locked(&so->so_rcv, m, 0); in bbr_process_data()
8290 if (so->so_rcv.sb_shlim && appended != mcnt) in bbr_process_data()
8291 counter_fo_release(so->so_rcv.sb_shlim, in bbr_process_data()
8292 mcnt - appended); in bbr_process_data()
8305 tp->t_flags |= TF_ACKNOW; in bbr_process_data()
8306 if (tp->t_flags & TF_WAKESOR) { in bbr_process_data()
8307 tp->t_flags &= ~TF_WAKESOR; in bbr_process_data()
8312 if ((tp->t_flags & TF_SACK_PERMIT) && in bbr_process_data()
8314 TCPS_HAVEESTABLISHED(tp->t_state)) { in bbr_process_data()
8322 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) { in bbr_process_data()
8323 if ((tp->rcv_numsacks >= 1) && in bbr_process_data()
8324 (tp->sackblks[0].end == save_start)) { in bbr_process_data()
8330 tp->sackblks[0].start, in bbr_process_data()
8331 tp->sackblks[0].end); in bbr_process_data()
8355 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { in bbr_process_data()
8359 * If connection is half-synchronized (ie NEEDSYN in bbr_process_data()
8365 if (tp->t_flags & TF_NEEDSYN) { in bbr_process_data()
8366 tp->t_flags |= TF_DELACK; in bbr_process_data()
8368 __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_data()
8370 tp->t_flags |= TF_ACKNOW; in bbr_process_data()
8372 tp->rcv_nxt++; in bbr_process_data()
8374 switch (tp->t_state) { in bbr_process_data()
8380 tp->t_starttime = ticks; in bbr_process_data()
8396 * starting the time-wait timer, turning off the in bbr_process_data()
8400 bbr->rc_timer_first = 1; in bbr_process_data()
8402 __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_process_data()
8410 if ((tp->t_flags & TF_ACKNOW) || in bbr_process_data()
8411 (sbavail(&so->so_snd) > ctf_outstanding(tp))) { in bbr_process_data()
8412 bbr->r_wanted_output = 1; in bbr_process_data()
8419 * have broken out the fast-data path also just like
8420 * the fast-ack. Return 1 if we processed the packet
8421 * return 0 if you need to take the "slow-path".
8429 int32_t newsize = 0; /* automatic sockbuf scaling */ in bbr_do_fastnewdata()
8436 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_fastnewdata()
8443 if (bbr->r_ctl.rc_resend != NULL) { in bbr_do_fastnewdata()
8446 if (tiwin && tiwin != tp->snd_wnd) { in bbr_do_fastnewdata()
8449 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) { in bbr_do_fastnewdata()
8452 if (__predict_false((to->to_flags & TOF_TS) && in bbr_do_fastnewdata()
8453 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) { in bbr_do_fastnewdata()
8456 if (__predict_false((th->th_ack != tp->snd_una))) { in bbr_do_fastnewdata()
8459 if (__predict_false(tlen > sbspace(&so->so_rcv))) { in bbr_do_fastnewdata()
8462 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_fastnewdata()
8463 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { in bbr_do_fastnewdata()
8464 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_fastnewdata()
8465 tp->ts_recent = to->to_tsval; in bbr_do_fastnewdata()
8468 * This is a pure, in-sequence data packet with nothing on the in bbr_do_fastnewdata()
8471 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_do_fastnewdata()
8474 if (so->so_rcv.sb_shlim) { in bbr_do_fastnewdata()
8477 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, in bbr_do_fastnewdata()
8486 if (tp->rcv_numsacks) in bbr_do_fastnewdata()
8489 tp->rcv_nxt += tlen; in bbr_do_fastnewdata()
8491 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && in bbr_do_fastnewdata()
8492 (tp->t_fbyte_in == 0)) { in bbr_do_fastnewdata()
8493 tp->t_fbyte_in = ticks; in bbr_do_fastnewdata()
8494 if (tp->t_fbyte_in == 0) in bbr_do_fastnewdata()
8495 tp->t_fbyte_in = 1; in bbr_do_fastnewdata()
8496 if (tp->t_fbyte_out && tp->t_fbyte_in) in bbr_do_fastnewdata()
8497 tp->t_flags2 |= TF2_FBYTES_COMPLETE; in bbr_do_fastnewdata()
8502 tp->snd_wl1 = th->th_seq; in bbr_do_fastnewdata()
8506 tp->rcv_up = tp->rcv_nxt; in bbr_do_fastnewdata()
8513 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_fastnewdata()
8522 so->so_rcv.sb_flags &= ~SB_AUTOSIZE; in bbr_do_fastnewdata()
8528 sbappendstream_locked(&so->so_rcv, m, 0); in bbr_do_fastnewdata()
8534 if (so->so_rcv.sb_shlim && mcnt != appended) in bbr_do_fastnewdata()
8535 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended); in bbr_do_fastnewdata()
8538 bbr->bbr_segs_rcvd += max(1, nsegs); in bbr_do_fastnewdata()
8539 tp->t_flags |= TF_DELACK; in bbr_do_fastnewdata()
8540 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_do_fastnewdata()
8542 bbr->r_wanted_output = 1; in bbr_do_fastnewdata()
8543 tp->t_flags |= TF_ACKNOW; in bbr_do_fastnewdata()
8551 * in sequence to remain in the fast-path. We also add
8555 * slow-path. If we return 1, then all is well and
8569 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { in bbr_fastack()
8573 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_fastack()
8581 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) { in bbr_fastack()
8585 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) { in bbr_fastack()
8589 if (__predict_false(IN_RECOVERY(tp->t_flags))) { in bbr_fastack()
8593 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_fastack()
8594 if (__predict_false(bbr->r_ctl.rc_resend != NULL)) { in bbr_fastack()
8598 if (__predict_false(bbr->rc_in_persist != 0)) { in bbr_fastack()
8602 if (bbr->r_ctl.rc_sacked) { in bbr_fastack()
8606 /* Ok if we reach here, we can process a fast-ack */ in bbr_fastack()
8607 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_fastack()
8614 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0); in bbr_fastack()
8616 if (tiwin != tp->snd_wnd) { in bbr_fastack()
8617 tp->snd_wnd = tiwin; in bbr_fastack()
8618 tp->snd_wl1 = th->th_seq; in bbr_fastack()
8619 if (tp->snd_wnd > tp->max_sndwnd) in bbr_fastack()
8620 tp->max_sndwnd = tp->snd_wnd; in bbr_fastack()
8623 if ((bbr->rc_in_persist != 0) && in bbr_fastack()
8624 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2), in bbr_fastack()
8626 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_fastack()
8627 bbr->r_wanted_output = 1; in bbr_fastack()
8630 if ((bbr->rc_in_persist == 0) && in bbr_fastack()
8631 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_fastack()
8632 TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_fastack()
8633 (tp->snd_max == tp->snd_una) && in bbr_fastack()
8634 sbavail(&so->so_snd) && in bbr_fastack()
8635 (sbavail(&so->so_snd) > tp->snd_wnd)) { in bbr_fastack()
8637 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_fastack()
8644 if ((to->to_flags & TOF_TS) != 0 && in bbr_fastack()
8645 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { in bbr_fastack()
8646 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime; in bbr_fastack()
8647 tp->ts_recent = to->to_tsval; in bbr_fastack()
8657 if (tp->t_flags & TF_PREVVALID) { in bbr_fastack()
8658 tp->t_flags &= ~TF_PREVVALID; in bbr_fastack()
8659 if (tp->t_rxtshift == 1 && in bbr_fastack()
8660 (int)(ticks - tp->t_badrxtwin) < 0) in bbr_fastack()
8679 sbdrop(&so->so_snd, acked); in bbr_fastack()
8681 if (SEQ_GT(th->th_ack, tp->snd_una)) in bbr_fastack()
8683 tp->snd_una = th->th_ack; in bbr_fastack()
8684 if (tp->snd_wnd < ctf_outstanding(tp)) in bbr_fastack()
8687 else if (bbr->rc_has_collapsed) in bbr_fastack()
8690 if (SEQ_GT(tp->snd_una, tp->snd_recover)) { in bbr_fastack()
8691 tp->snd_recover = tp->snd_una; in bbr_fastack()
8697 tp->snd_wl2 = th->th_ack; in bbr_fastack()
8701 * otherwise restart timer using current (possibly backed-off) in bbr_fastack()
8708 if (tp->snd_una == tp->snd_max) { in bbr_fastack()
8711 if (sbavail(&so->so_snd) == 0) in bbr_fastack()
8712 bbr->rc_tp->t_acktime = 0; in bbr_fastack()
8713 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_fastack()
8714 if (bbr->rc_in_persist == 0) { in bbr_fastack()
8715 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime; in bbr_fastack()
8717 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_fastack()
8718 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime); in bbr_fastack()
8724 bbr->r_wanted_output = 1; in bbr_fastack()
8726 if (sbavail(&so->so_snd)) { in bbr_fastack()
8727 bbr->r_wanted_output = 1; in bbr_fastack()
8749 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_syn_sent()
8755 * this is an acceptable SYN segment initialize tp->rcv_nxt and in bbr_do_syn_sent()
8756 * tp->irs if seg contains ack then advance tp->snd_una. BRR does in bbr_do_syn_sent()
8763 (SEQ_LEQ(th->th_ack, tp->iss) || in bbr_do_syn_sent()
8764 SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_do_syn_sent()
8784 tp->irs = th->th_seq; in bbr_do_syn_sent()
8794 /* Do window scaling on this connection? */ in bbr_do_syn_sent()
8795 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == in bbr_do_syn_sent()
8797 tp->rcv_scale = tp->request_r_scale; in bbr_do_syn_sent()
8799 tp->rcv_adv += min(tp->rcv_wnd, in bbr_do_syn_sent()
8800 TCP_MAXWIN << tp->rcv_scale); in bbr_do_syn_sent()
8805 if ((tp->t_flags & TF_FASTOPEN) && in bbr_do_syn_sent()
8806 (tp->snd_una != tp->snd_max)) { in bbr_do_syn_sent()
8807 tp->snd_nxt = th->th_ack; in bbr_do_syn_sent()
8815 bbr->bbr_segs_rcvd += 1; in bbr_do_syn_sent()
8816 tp->t_flags |= TF_DELACK; in bbr_do_syn_sent()
8817 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); in bbr_do_syn_sent()
8819 bbr->r_wanted_output = 1; in bbr_do_syn_sent()
8820 tp->t_flags |= TF_ACKNOW; in bbr_do_syn_sent()
8822 if (SEQ_GT(th->th_ack, tp->iss)) { in bbr_do_syn_sent()
8829 if (SEQ_GT(th->th_ack, tp->snd_una)) { in bbr_do_syn_sent()
8835 * ack-processing since the in bbr_do_syn_sent()
8836 * data stream in our send-map in bbr_do_syn_sent()
8842 tp->snd_una++; in bbr_do_syn_sent()
8846 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1 in bbr_do_syn_sent()
8848 tp->t_starttime = ticks; in bbr_do_syn_sent()
8849 if (tp->t_flags & TF_NEEDFIN) { in bbr_do_syn_sent()
8851 tp->t_flags &= ~TF_NEEDFIN; in bbr_do_syn_sent()
8861 * Received initial SYN in SYN-SENT[*] state => simultaneous in bbr_do_syn_sent()
8864 * half-synchronized. Otherwise, do 3-way handshake: in bbr_do_syn_sent()
8865 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If in bbr_do_syn_sent()
8868 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN); in bbr_do_syn_sent()
8872 * Advance th->th_seq to correspond to first data byte. If data, in bbr_do_syn_sent()
8875 th->th_seq++; in bbr_do_syn_sent()
8876 if (tlen > tp->rcv_wnd) { in bbr_do_syn_sent()
8877 todrop = tlen - tp->rcv_wnd; in bbr_do_syn_sent()
8878 m_adj(m, -todrop); in bbr_do_syn_sent()
8879 tlen = tp->rcv_wnd; in bbr_do_syn_sent()
8884 tp->snd_wl1 = th->th_seq - 1; in bbr_do_syn_sent()
8885 tp->rcv_up = th->th_seq; in bbr_do_syn_sent()
8893 if ((to->to_flags & TOF_TS) != 0) { in bbr_do_syn_sent()
8896 t = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_syn_sent()
8897 if (TSTMP_GEQ(t, to->to_tsecr)) { in bbr_do_syn_sent()
8898 rtt = t - to->to_tsecr; in bbr_do_syn_sent()
8904 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, in bbr_do_syn_sent()
8905 rtt, bbr->r_ctl.rc_rcvtime); in bbr_do_syn_sent()
8911 if (tp->t_state == TCPS_FIN_WAIT_1) { in bbr_do_syn_sent()
8928 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_syn_sent()
8959 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_syn_recv()
8962 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_syn_recv()
8965 (SEQ_LEQ(th->th_ack, tp->snd_una) || in bbr_do_syn_recv()
8966 SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_do_syn_recv()
8971 if (tp->t_flags & TF_FASTOPEN) { in bbr_do_syn_recv()
8983 /* non-initial SYN is ignored */ in bbr_do_syn_recv()
8984 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) || in bbr_do_syn_recv()
8985 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) || in bbr_do_syn_recv()
8986 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) { in bbr_do_syn_recv()
8999 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_syn_recv()
9000 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_syn_recv()
9005 * In the SYN-RECEIVED state, validate that the packet belongs to in bbr_do_syn_recv()
9011 if (SEQ_LT(th->th_seq, tp->irs)) { in bbr_do_syn_recv()
9030 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_syn_recv()
9033 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_syn_recv()
9034 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_syn_recv()
9035 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_syn_recv()
9037 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_syn_recv()
9038 tp->ts_recent = to->to_tsval; in bbr_do_syn_recv()
9040 tp->snd_wnd = tiwin; in bbr_do_syn_recv()
9042 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_syn_recv()
9043 * is on (half-synchronized state), then queue data for later in bbr_do_syn_recv()
9047 if (tp->t_flags & TF_FASTOPEN) { in bbr_do_syn_recv()
9054 if (tp->t_flags & TF_SONOTCONN) { in bbr_do_syn_recv()
9055 tp->t_flags &= ~TF_SONOTCONN; in bbr_do_syn_recv()
9058 /* Do window scaling? */ in bbr_do_syn_recv()
9059 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == in bbr_do_syn_recv()
9061 tp->rcv_scale = tp->request_r_scale; in bbr_do_syn_recv()
9067 if ((to->to_flags & TOF_TS) != 0) { in bbr_do_syn_recv()
9070 t = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_syn_recv()
9071 if (TSTMP_GEQ(t, to->to_tsecr)) { in bbr_do_syn_recv()
9072 rtt = t - to->to_tsecr; in bbr_do_syn_recv()
9078 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime); in bbr_do_syn_recv()
9084 if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) { in bbr_do_syn_recv()
9085 tcp_fastopen_decrement_counter(tp->t_tfo_pending); in bbr_do_syn_recv()
9086 tp->t_tfo_pending = NULL; in bbr_do_syn_recv()
9089 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* -> in bbr_do_syn_recv()
9090 * FIN-WAIT-1 in bbr_do_syn_recv()
9092 tp->t_starttime = ticks; in bbr_do_syn_recv()
9093 if (tp->t_flags & TF_NEEDFIN) { in bbr_do_syn_recv()
9095 tp->t_flags &= ~TF_NEEDFIN; in bbr_do_syn_recv()
9106 if (!(tp->t_flags & TF_FASTOPEN)) in bbr_do_syn_recv()
9114 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN)) in bbr_do_syn_recv()
9115 tp->snd_una++; in bbr_do_syn_recv()
9123 if (tp->t_flags & TF_WAKESOR) { in bbr_do_syn_recv()
9124 tp->t_flags &= ~TF_WAKESOR; in bbr_do_syn_recv()
9129 tp->snd_wl1 = th->th_seq - 1; in bbr_do_syn_recv()
9133 if (tp->t_state == TCPS_FIN_WAIT_1) { in bbr_do_syn_recv()
9150 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_syn_recv()
9181 * uni-directional data xfer. If the packet has no control flags, in bbr_do_established()
9182 * is in-sequence, the window didn't change and we're not in bbr_do_established()
9186 * waiting for space. If the length is non-zero and the ack didn't in bbr_do_established()
9187 * move, we're the receiver side. If we're getting packets in-order in bbr_do_established()
9190 * hidden state-flags are also off. Since we check for in bbr_do_established()
9193 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_established()
9194 if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) { in bbr_do_established()
9200 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd); in bbr_do_established()
9202 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) && in bbr_do_established()
9205 __predict_true(th->th_seq == tp->rcv_nxt)) { in bbr_do_established()
9221 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_established()
9235 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_established()
9236 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_established()
9254 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_established()
9257 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_established()
9258 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_established()
9259 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_established()
9261 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_established()
9262 tp->ts_recent = to->to_tsval; in bbr_do_established()
9265 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_established()
9266 * is on (half-synchronized state), then queue data for later in bbr_do_established()
9270 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_established()
9273 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_established()
9275 bbr->r_wanted_output = 1; in bbr_do_established()
9288 if (sbavail(&so->so_snd)) { in bbr_do_established()
9315 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_close_wait()
9318 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_close_wait()
9332 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_close_wait()
9333 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_close_wait()
9351 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_close_wait()
9354 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_close_wait()
9355 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_close_wait()
9356 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_close_wait()
9358 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_close_wait()
9359 tp->ts_recent = to->to_tsval; in bbr_do_close_wait()
9362 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_close_wait()
9363 * is on (half-synchronized state), then queue data for later in bbr_do_close_wait()
9367 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_close_wait()
9370 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_close_wait()
9372 bbr->r_wanted_output = 1; in bbr_do_close_wait()
9385 if (sbavail(&so->so_snd)) { in bbr_do_close_wait()
9401 if (bbr->rc_allow_data_af_clo == 0) { in bbr_check_data_after_close()
9404 /* tcp_close will kill the inp pre-log the Reset */ in bbr_check_data_after_close()
9411 if (sbavail(&so->so_snd) == 0) in bbr_check_data_after_close()
9414 tp->rcv_nxt = th->th_seq + *tlen; in bbr_check_data_after_close()
9415 tp->t_flags2 |= TF2_DROP_AF_DATA; in bbr_check_data_after_close()
9416 bbr->r_wanted_output = 1; in bbr_check_data_after_close()
9437 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_fin_wait_1()
9440 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_fin_wait_1()
9454 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_fin_wait_1()
9455 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_fin_wait_1()
9468 if ((tp->t_flags & TF_CLOSED) && tlen && in bbr_do_fin_wait_1()
9482 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_fin_wait_1()
9485 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_fin_wait_1()
9486 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_fin_wait_1()
9487 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_fin_wait_1()
9489 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_fin_wait_1()
9490 tp->ts_recent = to->to_tsval; in bbr_do_fin_wait_1()
9493 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_fin_wait_1()
9494 * is on (half-synchronized state), then queue data for later in bbr_do_fin_wait_1()
9498 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_fin_wait_1()
9501 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_fin_wait_1()
9503 bbr->r_wanted_output = 1; in bbr_do_fin_wait_1()
9526 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { in bbr_do_fin_wait_1()
9535 if (sbavail(&so->so_snd)) { in bbr_do_fin_wait_1()
9562 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_closing()
9565 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_closing()
9579 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_closing()
9580 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_closing()
9598 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_closing()
9601 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_closing()
9602 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_closing()
9603 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_closing()
9605 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_closing()
9606 tp->ts_recent = to->to_tsval; in bbr_do_closing()
9609 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_closing()
9610 * is on (half-synchronized state), then queue data for later in bbr_do_closing()
9614 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_closing()
9617 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_closing()
9619 bbr->r_wanted_output = 1; in bbr_do_closing()
9637 if (sbavail(&so->so_snd)) { in bbr_do_closing()
9664 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_lastack()
9667 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_lastack()
9681 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_lastack()
9682 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_lastack()
9700 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_lastack()
9703 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_lastack()
9704 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_lastack()
9705 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_lastack()
9707 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_lastack()
9708 tp->ts_recent = to->to_tsval; in bbr_do_lastack()
9711 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_lastack()
9712 * is on (half-synchronized state), then queue data for later in bbr_do_lastack()
9716 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_lastack()
9719 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_lastack()
9721 bbr->r_wanted_output = 1; in bbr_do_lastack()
9739 if (sbavail(&so->so_snd)) { in bbr_do_lastack()
9766 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_fin_wait_2()
9768 /* Reset receive buffer auto scaling when not in bulk receive mode. */ in bbr_do_fin_wait_2()
9770 (tp->t_fin_is_rst && (thflags & TH_FIN))) in bbr_do_fin_wait_2()
9785 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && in bbr_do_fin_wait_2()
9786 TSTMP_LT(to->to_tsval, tp->ts_recent)) { in bbr_do_fin_wait_2()
9800 if ((tp->t_flags & TF_CLOSED) && tlen && in bbr_do_fin_wait_2()
9814 * p.869. In such cases, we can still calculate the RTT correctly in bbr_do_fin_wait_2()
9817 if ((to->to_flags & TOF_TS) != 0 && in bbr_do_fin_wait_2()
9818 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && in bbr_do_fin_wait_2()
9819 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + in bbr_do_fin_wait_2()
9821 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_fin_wait_2()
9822 tp->ts_recent = to->to_tsval; in bbr_do_fin_wait_2()
9825 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag in bbr_do_fin_wait_2()
9826 * is on (half-synchronized state), then queue data for later in bbr_do_fin_wait_2()
9830 if (tp->t_flags & TF_NEEDSYN) { in bbr_do_fin_wait_2()
9833 } else if (tp->t_flags & TF_ACKNOW) { in bbr_do_fin_wait_2()
9835 bbr->r_wanted_output = 1; in bbr_do_fin_wait_2()
9848 if (sbavail(&so->so_snd)) { in bbr_do_fin_wait_2()
9867 bbr->rc_in_persist = 1; in bbr_stop_all_timers()
9869 if (tcp_in_hpts(bbr->rc_tp)) { in bbr_stop_all_timers()
9870 tcp_hpts_remove(bbr->rc_tp); in bbr_stop_all_timers()
9877 bbr->rc_use_google = 1; in bbr_google_mode_on()
9878 bbr->rc_no_pacing = 0; in bbr_google_mode_on()
9879 bbr->r_ctl.bbr_google_discount = bbr_google_discount; in bbr_google_mode_on()
9880 bbr->r_use_policer = bbr_policer_detection_enabled; in bbr_google_mode_on()
9881 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10); in bbr_google_mode_on()
9882 bbr->bbr_use_rack_cheat = 0; in bbr_google_mode_on()
9883 bbr->r_ctl.rc_incr_tmrs = 0; in bbr_google_mode_on()
9884 bbr->r_ctl.rc_inc_tcp_oh = 0; in bbr_google_mode_on()
9885 bbr->r_ctl.rc_inc_ip_oh = 0; in bbr_google_mode_on()
9886 bbr->r_ctl.rc_inc_enet_oh = 0; in bbr_google_mode_on()
9887 reset_time(&bbr->r_ctl.rc_delrate, in bbr_google_mode_on()
9889 reset_time_small(&bbr->r_ctl.rc_rttprop, in bbr_google_mode_on()
9891 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv)); in bbr_google_mode_on()
9897 bbr->rc_use_google = 0; in bbr_google_mode_off()
9898 bbr->r_ctl.bbr_google_discount = 0; in bbr_google_mode_off()
9899 bbr->no_pacing_until = bbr_no_pacing_until; in bbr_google_mode_off()
9900 bbr->r_use_policer = 0; in bbr_google_mode_off()
9901 if (bbr->no_pacing_until) in bbr_google_mode_off()
9902 bbr->rc_no_pacing = 1; in bbr_google_mode_off()
9904 bbr->rc_no_pacing = 0; in bbr_google_mode_off()
9906 bbr->bbr_use_rack_cheat = 1; in bbr_google_mode_off()
9908 bbr->bbr_use_rack_cheat = 0; in bbr_google_mode_off()
9910 bbr->r_ctl.rc_incr_tmrs = 1; in bbr_google_mode_off()
9912 bbr->r_ctl.rc_incr_tmrs = 0; in bbr_google_mode_off()
9914 bbr->r_ctl.rc_inc_tcp_oh = 1; in bbr_google_mode_off()
9916 bbr->r_ctl.rc_inc_tcp_oh = 0; in bbr_google_mode_off()
9918 bbr->r_ctl.rc_inc_ip_oh = 1; in bbr_google_mode_off()
9920 bbr->r_ctl.rc_inc_ip_oh = 0; in bbr_google_mode_off()
9922 bbr->r_ctl.rc_inc_enet_oh = 1; in bbr_google_mode_off()
9924 bbr->r_ctl.rc_inc_enet_oh = 0; in bbr_google_mode_off()
9925 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; in bbr_google_mode_off()
9926 reset_time(&bbr->r_ctl.rc_delrate, in bbr_google_mode_off()
9928 reset_time_small(&bbr->r_ctl.rc_rttprop, in bbr_google_mode_off()
9930 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv)); in bbr_google_mode_off()
9933 * Return 0 on success, non-zero on failure
9956 bbr->rtt_valid = 0; in bbr_init()
9957 tp->t_flags2 |= TF2_CANNOT_DO_ECN; in bbr_init()
9958 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ; in bbr_init()
9960 tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY; in bbr_init()
9961 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE; in bbr_init()
9962 tp->t_flags2 &= ~TF2_MBUF_ACKCMP; in bbr_init()
9963 tp->t_flags2 &= ~TF2_MBUF_L_ACKS; in bbr_init()
9965 TAILQ_INIT(&bbr->r_ctl.rc_map); in bbr_init()
9966 TAILQ_INIT(&bbr->r_ctl.rc_free); in bbr_init()
9967 TAILQ_INIT(&bbr->r_ctl.rc_tmap); in bbr_init()
9968 bbr->rc_tp = tp; in bbr_init()
9969 bbr->rc_inp = inp; in bbr_init()
9970 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_init()
9971 tp->t_acktime = 0; in bbr_init()
9972 bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close; in bbr_init()
9973 bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade; in bbr_init()
9974 bbr->rc_tlp_threshold = bbr_tlp_thresh; in bbr_init()
9975 bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh; in bbr_init()
9976 bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay; in bbr_init()
9977 bbr->r_ctl.rc_min_to = bbr_min_to; in bbr_init()
9978 bbr->rc_bbr_state = BBR_STATE_STARTUP; in bbr_init()
9979 bbr->r_ctl.bbr_lost_at_state = 0; in bbr_init()
9980 bbr->r_ctl.rc_lost_at_startup = 0; in bbr_init()
9981 bbr->rc_all_timers_stopped = 0; in bbr_init()
9982 bbr->r_ctl.rc_bbr_lastbtlbw = 0; in bbr_init()
9983 bbr->r_ctl.rc_pkt_epoch_del = 0; in bbr_init()
9984 bbr->r_ctl.rc_pkt_epoch = 0; in bbr_init()
9985 bbr->r_ctl.rc_lowest_rtt = 0xffffffff; in bbr_init()
9986 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain; in bbr_init()
9987 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain; in bbr_init()
9988 bbr->r_ctl.rc_went_idle_time = cts; in bbr_init()
9989 bbr->rc_pacer_started = cts; in bbr_init()
9990 bbr->r_ctl.rc_pkt_epoch_time = cts; in bbr_init()
9991 bbr->r_ctl.rc_rcvtime = cts; in bbr_init()
9992 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_init()
9993 bbr->r_ctl.rc_del_time = cts; in bbr_init()
9994 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_init()
9995 bbr->r_ctl.last_in_probertt = cts; in bbr_init()
9996 bbr->skip_gain = 0; in bbr_init()
9997 bbr->gain_is_limited = 0; in bbr_init()
9998 bbr->no_pacing_until = bbr_no_pacing_until; in bbr_init()
9999 if (bbr->no_pacing_until) in bbr_init()
10000 bbr->rc_no_pacing = 1; in bbr_init()
10002 bbr->rc_no_pacing = 0; in bbr_init()
10003 bbr->rc_use_google = 1; in bbr_init()
10004 bbr->r_ctl.bbr_google_discount = bbr_google_discount; in bbr_init()
10005 bbr->r_use_policer = bbr_policer_detection_enabled; in bbr_init()
10007 bbr->rc_use_google = 0; in bbr_init()
10008 bbr->r_ctl.bbr_google_discount = 0; in bbr_init()
10009 bbr->r_use_policer = 0; in bbr_init()
10012 bbr->rc_use_ts_limit = 1; in bbr_init()
10014 bbr->rc_use_ts_limit = 0; in bbr_init()
10016 bbr->ts_can_raise = 1; in bbr_init()
10018 bbr->ts_can_raise = 0; in bbr_init()
10020 tp->t_delayed_ack = 2; in bbr_init()
10022 tp->t_delayed_ack = 0; in bbr_init()
10024 tp->t_delayed_ack = V_tcp_delack_enabled; in bbr_init()
10026 tp->t_delayed_ack = 2; in bbr_init()
10027 if (bbr->rc_use_google == 0) in bbr_init()
10028 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; in bbr_init()
10030 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10); in bbr_init()
10031 bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms; in bbr_init()
10032 bbr->rc_max_rto_sec = bbr_rto_max_sec; in bbr_init()
10033 bbr->rc_init_win = bbr_def_init_win; in bbr_init()
10034 if (tp->t_flags & TF_REQ_TSTMP) in bbr_init()
10035 bbr->rc_last_options = TCP_TS_OVERHEAD; in bbr_init()
10036 bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options; in bbr_init()
10037 bbr->r_ctl.rc_high_rwnd = tp->snd_wnd; in bbr_init()
10038 bbr->r_init_rtt = 1; in bbr_init()
10042 bbr->bbr_hdw_pace_ena = 1; in bbr_init()
10044 bbr->bbr_hdw_pace_ena = 0; in bbr_init()
10046 bbr->bbr_init_win_cheat = 1; in bbr_init()
10048 bbr->bbr_init_win_cheat = 0; in bbr_init()
10049 bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max; in bbr_init()
10050 bbr->r_ctl.rc_drain_pg = bbr_drain_gain; in bbr_init()
10051 bbr->r_ctl.rc_startup_pg = bbr_high_gain; in bbr_init()
10052 bbr->rc_loss_exit = bbr_exit_startup_at_loss; in bbr_init()
10053 bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain; in bbr_init()
10054 bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second; in bbr_init()
10055 bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar; in bbr_init()
10056 bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max; in bbr_init()
10057 bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor; in bbr_init()
10058 bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min; in bbr_init()
10059 bbr->r_ctl.bbr_cross_over = bbr_cross_over; in bbr_init()
10060 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_init()
10061 if (bbr->rc_use_google) { in bbr_init()
10062 setup_time_filter(&bbr->r_ctl.rc_delrate, in bbr_init()
10065 setup_time_filter_small(&bbr->r_ctl.rc_rttprop, in bbr_init()
10068 setup_time_filter(&bbr->r_ctl.rc_delrate, in bbr_init()
10071 setup_time_filter_small(&bbr->r_ctl.rc_rttprop, in bbr_init()
10076 bbr->rc_use_idle_restart = 1; in bbr_init()
10078 bbr->rc_use_idle_restart = 0; in bbr_init()
10079 bbr->r_ctl.rc_bbr_cur_del_rate = 0; in bbr_init()
10080 bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps; in bbr_init()
10082 bbr->rc_resends_use_tso = 1; in bbr_init()
10083 if (tp->snd_una != tp->snd_max) { in bbr_init()
10093 rsm->r_rtt_not_allowed = 1; in bbr_init()
10094 rsm->r_tim_lastsent[0] = cts; in bbr_init()
10095 rsm->r_rtr_cnt = 1; in bbr_init()
10096 rsm->r_rtr_bytes = 0; in bbr_init()
10097 rsm->r_start = tp->snd_una; in bbr_init()
10098 rsm->r_end = tp->snd_max; in bbr_init()
10099 rsm->r_dupack = 0; in bbr_init()
10100 rsm->r_delivered = bbr->r_ctl.rc_delivered; in bbr_init()
10101 rsm->r_ts_valid = 0; in bbr_init()
10102 rsm->r_del_ack_ts = tp->ts_recent; in bbr_init()
10103 rsm->r_del_time = cts; in bbr_init()
10104 if (bbr->r_ctl.r_app_limited_until) in bbr_init()
10105 rsm->r_app_limited = 1; in bbr_init()
10107 rsm->r_app_limited = 0; in bbr_init()
10108 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_init()
10109 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); in bbr_init()
10110 rsm->r_in_tmap = 1; in bbr_init()
10111 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) in bbr_init()
10112 rsm->r_bbr_state = bbr_state_val(bbr); in bbr_init()
10114 rsm->r_bbr_state = 8; in bbr_init()
10116 if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0)) in bbr_init()
10117 bbr->bbr_use_rack_cheat = 1; in bbr_init()
10118 if (bbr_incr_timers && (bbr->rc_use_google == 0)) in bbr_init()
10119 bbr->r_ctl.rc_incr_tmrs = 1; in bbr_init()
10120 if (bbr_include_tcp_oh && (bbr->rc_use_google == 0)) in bbr_init()
10121 bbr->r_ctl.rc_inc_tcp_oh = 1; in bbr_init()
10122 if (bbr_include_ip_oh && (bbr->rc_use_google == 0)) in bbr_init()
10123 bbr->r_ctl.rc_inc_ip_oh = 1; in bbr_init()
10124 if (bbr_include_enet_oh && (bbr->rc_use_google == 0)) in bbr_init()
10125 bbr->r_ctl.rc_inc_enet_oh = 1; in bbr_init()
10128 if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_init()
10129 (tp->t_srtt)) { in bbr_init()
10132 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT); in bbr_init()
10133 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_init()
10151 TCPT_RANGESET(tp->t_rxtcur, in bbr_init()
10152 ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1, in bbr_init()
10153 tp->t_rttmin, TCPTV_REXMTMAX); in bbr_init()
10160 * non-zero if we can't handle the connection. A EAGAIN
10168 if ((tp->t_state == TCPS_CLOSED) || in bbr_handoff_ok()
10169 (tp->t_state == TCPS_LISTEN)) { in bbr_handoff_ok()
10173 if ((tp->t_state == TCPS_SYN_SENT) || in bbr_handoff_ok()
10174 (tp->t_state == TCPS_SYN_RECEIVED)) { in bbr_handoff_ok()
10181 if (tp->t_flags & TF_SENTFIN) in bbr_handoff_ok()
10183 if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) { in bbr_handoff_ok()
10196 if (tp->t_fb_ptr) { in bbr_fini()
10201 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_fini()
10202 if (bbr->r_ctl.crte) in bbr_fini()
10203 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp); in bbr_fini()
10205 bbr->rc_tp = NULL; in bbr_fini()
10206 if (bbr->bbr_hdrw_pacing) in bbr_fini()
10207 counter_u64_add(bbr_flows_whdwr_pacing, -1); in bbr_fini()
10209 counter_u64_add(bbr_flows_nohdwr_pacing, -1); in bbr_fini()
10210 if (bbr->r_ctl.crte != NULL) { in bbr_fini()
10211 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp); in bbr_fini()
10212 bbr->r_ctl.crte = NULL; in bbr_fini()
10214 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_fini()
10216 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); in bbr_fini()
10218 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_fini()
10220 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); in bbr_fini()
10222 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_fini()
10224 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); in bbr_fini()
10226 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd; in bbr_fini()
10227 if (calc > (bbr->r_ctl.rc_init_rwnd / 10)) in bbr_fini()
10231 bbr->r_ctl.rc_free_cnt = 0; in bbr_fini()
10232 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr); in bbr_fini()
10233 tp->t_fb_ptr = NULL; in bbr_fini()
10236 tp->snd_nxt = tp->snd_max; in bbr_fini()
10242 switch (tp->t_state) { in bbr_set_state()
10244 bbr->r_state = TCPS_SYN_SENT; in bbr_set_state()
10245 bbr->r_substate = bbr_do_syn_sent; in bbr_set_state()
10248 bbr->r_state = TCPS_SYN_RECEIVED; in bbr_set_state()
10249 bbr->r_substate = bbr_do_syn_recv; in bbr_set_state()
10252 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd); in bbr_set_state()
10253 bbr->r_state = TCPS_ESTABLISHED; in bbr_set_state()
10254 bbr->r_substate = bbr_do_established; in bbr_set_state()
10257 bbr->r_state = TCPS_CLOSE_WAIT; in bbr_set_state()
10258 bbr->r_substate = bbr_do_close_wait; in bbr_set_state()
10261 bbr->r_state = TCPS_FIN_WAIT_1; in bbr_set_state()
10262 bbr->r_substate = bbr_do_fin_wait_1; in bbr_set_state()
10265 bbr->r_state = TCPS_CLOSING; in bbr_set_state()
10266 bbr->r_substate = bbr_do_closing; in bbr_set_state()
10269 bbr->r_state = TCPS_LAST_ACK; in bbr_set_state()
10270 bbr->r_substate = bbr_do_lastack; in bbr_set_state()
10273 bbr->r_state = TCPS_FIN_WAIT_2; in bbr_set_state()
10274 bbr->r_substate = bbr_do_fin_wait_2; in bbr_set_state()
10295 /* Save the lowest srtt we saw in our end of the sub-state */ in bbr_substate_change()
10296 bbr->rc_hit_state_1 = 0; in bbr_substate_change()
10297 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff) in bbr_substate_change()
10298 bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state; in bbr_substate_change()
10300 bbr->rc_bbr_substate++; in bbr_substate_change()
10303 * We enter the gain(5/4) cycle (possibly less if in bbr_substate_change()
10306 if (bbr->skip_gain) { in bbr_substate_change()
10310 * do level i.e. no gain. in bbr_substate_change()
10312 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1]; in bbr_substate_change()
10313 } else if (bbr->gain_is_limited && in bbr_substate_change()
10314 bbr->bbr_hdrw_pacing && in bbr_substate_change()
10315 bbr->r_ctl.crte) { in bbr_substate_change()
10317 * We can't gain above the hardware pacing in bbr_substate_change()
10318 * rate which is less than our rate + the gain in bbr_substate_change()
10319 * calculate the gain needed to reach the hardware in bbr_substate_change()
10325 rate = bbr->r_ctl.crte->rate; in bbr_substate_change()
10331 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc; in bbr_substate_change()
10333 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN]; in bbr_substate_change()
10336 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN]; in bbr_substate_change()
10337 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) { in bbr_substate_change()
10338 bbr->r_ctl.rc_bbr_state_atflight = cts; in bbr_substate_change()
10340 bbr->r_ctl.rc_bbr_state_atflight = 0; in bbr_substate_change()
10342 bbr->rc_hit_state_1 = 1; in bbr_substate_change()
10343 bbr->r_ctl.rc_exta_time_gd = 0; in bbr_substate_change()
10344 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp, in bbr_substate_change()
10345 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_substate_change()
10347 bbr->r_ctl.rc_bbr_state_atflight = 0; in bbr_substate_change()
10349 bbr->r_ctl.rc_bbr_state_atflight = cts; in bbr_substate_change()
10350 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN]; in bbr_substate_change()
10352 /* All other cycles hit here 2-7 */ in bbr_substate_change()
10353 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) { in bbr_substate_change()
10355 (bbr->rc_use_google == 0) && in bbr_substate_change()
10356 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { in bbr_substate_change()
10357 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_substate_change()
10360 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP)) in bbr_substate_change()
10361 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) - in bbr_substate_change()
10364 bbr->r_ctl.rc_exta_time_gd = 0; in bbr_substate_change()
10365 if (bbr->r_ctl.rc_exta_time_gd) { in bbr_substate_change()
10366 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd; in bbr_substate_change()
10368 bbr->r_ctl.rc_level_state_extra /= 7; in bbr_substate_change()
10369 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) { in bbr_substate_change()
10375 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); in bbr_substate_change()
10376 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)]; in bbr_substate_change()
10378 if (bbr->rc_use_google) { in bbr_substate_change()
10379 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); in bbr_substate_change()
10381 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_substate_change()
10382 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain; in bbr_substate_change()
10386 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_substate_change()
10389 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_substate_change()
10390 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_substate_change()
10393 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_substate_change()
10396 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff; in bbr_substate_change()
10399 (bbr->rc_use_google == 0) && in bbr_substate_change()
10402 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_substate_change()
10403 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_substate_change()
10406 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + in bbr_substate_change()
10407 ctf_flight_size(bbr->rc_tp, in bbr_substate_change()
10408 (bbr->r_ctl.rc_sacked + in bbr_substate_change()
10409 bbr->r_ctl.rc_lost_bytes))); in bbr_substate_change()
10413 if (bbr->rc_lt_use_bw) { in bbr_substate_change()
10415 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_substate_change()
10418 if (bbr->rc_use_google) in bbr_substate_change()
10420 bbr->r_ctl.gain_epoch = cts; in bbr_substate_change()
10421 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_substate_change()
10422 bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch; in bbr_substate_change()
10430 (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) { in bbr_set_probebw_google_gains()
10434 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_set_probebw_google_gains()
10437 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) { in bbr_set_probebw_google_gains()
10447 * the gain cycle extra rules apply: in bbr_set_probebw_google_gains()
10450 * we stay in GAIN (gain-to-target). in bbr_set_probebw_google_gains()
10454 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) { in bbr_set_probebw_google_gains()
10459 /* For gain we must reach our target, all others last 1 rttProp */ in bbr_set_probebw_google_gains()
10468 if (bbr->rc_use_google) { in bbr_set_probebw_gains()
10485 if (bbr->r_ctl.rc_bbr_state_atflight == 0) { in bbr_set_probebw_gains()
10487 flight = ctf_flight_size(bbr->rc_tp, in bbr_set_probebw_gains()
10488 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_set_probebw_gains()
10489 if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) { in bbr_set_probebw_gains()
10491 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) { in bbr_set_probebw_gains()
10492 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_set_probebw_gains()
10497 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight); in bbr_set_probebw_gains()
10500 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) && in bbr_set_probebw_gains()
10501 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) || in bbr_set_probebw_gains()
10502 (flight >= bbr->r_ctl.flightsize_at_drain))) { in bbr_set_probebw_gains()
10505 * the gain. We need to drain harder in bbr_set_probebw_gains()
10507 * the gain drop is capped at DRAIN states in bbr_set_probebw_gains()
10510 bbr->r_ctl.flightsize_at_drain = flight; in bbr_set_probebw_gains()
10515 bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul; in bbr_set_probebw_gains()
10516 bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div; in bbr_set_probebw_gains()
10519 bbr->r_ctl.rc_bbr_hptsi_gain *= 4; in bbr_set_probebw_gains()
10520 bbr->r_ctl.rc_bbr_hptsi_gain /= 5; in bbr_set_probebw_gains()
10522 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) { in bbr_set_probebw_gains()
10523 /* Reduce our gain again to the bottom */ in bbr_set_probebw_gains()
10524 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1); in bbr_set_probebw_gains()
10531 bbr->r_ctl.gain_epoch = cts; in bbr_set_probebw_gains()
10533 if (flight <= bbr->r_ctl.rc_target_at_state) { in bbr_set_probebw_gains()
10535 (bbr->rc_use_google == 0) && in bbr_set_probebw_gains()
10536 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { in bbr_set_probebw_gains()
10537 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_set_probebw_gains()
10540 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10544 /* Its a gain */ in bbr_set_probebw_gains()
10545 if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) { in bbr_set_probebw_gains()
10546 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10549 if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) || in bbr_set_probebw_gains()
10550 ((ctf_outstanding(bbr->rc_tp) + bbr->rc_tp->t_maxseg - 1) >= in bbr_set_probebw_gains()
10551 bbr->rc_tp->snd_wnd)) { in bbr_set_probebw_gains()
10552 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10568 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) in bbr_set_probebw_gains()
10570 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) { in bbr_set_probebw_gains()
10571 /* Less than a full time-period has passed */ in bbr_set_probebw_gains()
10574 if (bbr->r_ctl.rc_level_state_extra && in bbr_set_probebw_gains()
10576 ((cts - bbr->r_ctl.rc_bbr_state_time) < in bbr_set_probebw_gains()
10577 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) { in bbr_set_probebw_gains()
10578 /* Less than a full time-period + extra has passed */ in bbr_set_probebw_gains()
10582 bbr->r_ctl.rc_level_state_extra && in bbr_set_probebw_gains()
10584 ((cts - bbr->r_ctl.rc_bbr_state_time) < in bbr_set_probebw_gains()
10585 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) { in bbr_set_probebw_gains()
10586 /* Less than a full time-period + extra has passed */ in bbr_set_probebw_gains()
10593 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain) in bbr_get_a_state_target() argument
10597 if (bbr->rc_use_google) { in bbr_get_a_state_target()
10599 tar = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), gain); in bbr_get_a_state_target()
10601 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), in bbr_get_a_state_target()
10602 bbr->r_ctl.rc_pace_max_segs); in bbr_get_a_state_target()
10603 /* Get the base cwnd with gain rounded to a mss */ in bbr_get_a_state_target()
10605 gain), mss); in bbr_get_a_state_target()
10618 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && in bbr_set_state_target()
10619 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) { in bbr_set_state_target()
10620 /* Special case using old probe-rtt method */ in bbr_set_state_target()
10621 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_set_state_target()
10624 /* Non-probe-rtt case and reduced probe-rtt */ in bbr_set_state_target()
10625 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && in bbr_set_state_target()
10626 (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) { in bbr_set_state_target()
10627 /* For gain cycle we use the hptsi gain */ in bbr_set_state_target()
10628 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain); in bbr_set_state_target()
10630 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) { in bbr_set_state_target()
10639 * Or we set a target based on the pacing gain in bbr_set_state_target()
10640 * for non-google mode and default (non-configured). in bbr_set_state_target()
10643 if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN]) { in bbr_set_state_target()
10647 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain); in bbr_set_state_target()
10653 bbr->r_ctl.rc_target_at_state = tar; in bbr_set_state_target()
10662 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_enter_probe_rtt()
10663 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp, in bbr_enter_probe_rtt()
10664 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_enter_probe_rtt()
10665 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain in bbr_enter_probe_rtt()
10666 + bbr->r_ctl.rc_delivered); in bbr_enter_probe_rtt()
10668 if (bbr->rc_use_google || bbr_probertt_sets_rtt) in bbr_enter_probe_rtt()
10669 bbr->rc_prtt_set_ts = 1; in bbr_enter_probe_rtt()
10670 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_enter_probe_rtt()
10671 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_enter_probe_rtt()
10672 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_enter_probe_rtt()
10675 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_enter_probe_rtt()
10676 bbr->r_ctl.last_in_probertt = cts; in bbr_enter_probe_rtt()
10677 bbr->r_ctl.rc_probertt_srttchktim = cts; in bbr_enter_probe_rtt()
10678 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_enter_probe_rtt()
10679 bbr->rc_bbr_state = BBR_STATE_PROBE_RTT; in bbr_enter_probe_rtt()
10683 bbr->rc_hit_state_1 && in bbr_enter_probe_rtt()
10684 (bbr->rc_use_google == 0) && in bbr_enter_probe_rtt()
10686 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd) in bbr_enter_probe_rtt()
10687 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_enter_probe_rtt()
10689 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_enter_probe_rtt()
10691 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_enter_probe_rtt()
10692 if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){ in bbr_enter_probe_rtt()
10693 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN) */ in bbr_enter_probe_rtt()
10694 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_enter_probe_rtt()
10696 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_enter_probe_rtt()
10697 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_enter_probe_rtt()
10698 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6); in bbr_enter_probe_rtt()
10699 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd; in bbr_enter_probe_rtt()
10702 * We bring it down slowly by using a hptsi gain that is in bbr_enter_probe_rtt()
10706 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val; in bbr_enter_probe_rtt()
10707 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_enter_probe_rtt()
10710 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_enter_probe_rtt()
10711 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_enter_probe_rtt()
10715 if (ctf_flight_size(bbr->rc_tp, in bbr_enter_probe_rtt()
10716 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= in bbr_enter_probe_rtt()
10717 bbr->r_ctl.rc_target_at_state) { in bbr_enter_probe_rtt()
10719 bbr->r_ctl.rc_bbr_enters_probertt = cts; in bbr_enter_probe_rtt()
10722 bbr->r_ctl.rc_bbr_enters_probertt = 0; in bbr_enter_probe_rtt()
10724 bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch; in bbr_enter_probe_rtt()
10734 * Sanity check on probe-rtt intervals. in bbr_check_probe_rtt_limits()
10736 * against new-reno flows with huge buffers in bbr_check_probe_rtt_limits()
10737 * our rtt-prop interval could come to dominate in bbr_check_probe_rtt_limits()
10742 (bbr->rc_use_google == 0)) { in bbr_check_probe_rtt_limits()
10746 /* Are we to small and go into probe-rtt to often? */ in bbr_check_probe_rtt_limits()
10752 newval = cur_rttp + (fval - bbr_rtt_probe_limit); in bbr_check_probe_rtt_limits()
10761 if (cur_rttp > bbr->r_ctl.rc_probertt_int) { in bbr_check_probe_rtt_limits()
10762 bbr->r_ctl.rc_probertt_int = cur_rttp; in bbr_check_probe_rtt_limits()
10763 reset_time_small(&bbr->r_ctl.rc_rttprop, newval); in bbr_check_probe_rtt_limits()
10770 if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) { in bbr_check_probe_rtt_limits()
10776 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; in bbr_check_probe_rtt_limits()
10777 reset_time_small(&bbr->r_ctl.rc_rttprop, in bbr_check_probe_rtt_limits()
10786 if (cur_rttp < bbr->r_ctl.rc_probertt_int) { in bbr_check_probe_rtt_limits()
10788 bbr->r_ctl.rc_probertt_int = cur_rttp; in bbr_check_probe_rtt_limits()
10789 reset_time_small(&bbr->r_ctl.rc_rttprop, newval); in bbr_check_probe_rtt_limits()
10803 /* Exit probe-rtt */ in bbr_exit_probe_rtt()
10805 if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) { in bbr_exit_probe_rtt()
10806 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_exit_probe_rtt()
10810 bbr->rc_hit_state_1 = 0; in bbr_exit_probe_rtt()
10811 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_exit_probe_rtt()
10812 bbr->r_ctl.last_in_probertt = cts; in bbr_exit_probe_rtt()
10814 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_exit_probe_rtt()
10815 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, in bbr_exit_probe_rtt()
10816 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_exit_probe_rtt()
10817 bbr->r_ctl.rc_delivered); in bbr_exit_probe_rtt()
10818 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_exit_probe_rtt()
10821 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_exit_probe_rtt()
10822 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_exit_probe_rtt()
10824 if (bbr->rc_filled_pipe) { in bbr_exit_probe_rtt()
10826 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_exit_probe_rtt()
10827 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_exit_probe_rtt()
10828 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain; in bbr_exit_probe_rtt()
10833 bbr->rc_bbr_state = BBR_STATE_STARTUP; in bbr_exit_probe_rtt()
10834 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_exit_probe_rtt()
10838 * the number of pe's we were in probe-rtt in bbr_exit_probe_rtt()
10842 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt); in bbr_exit_probe_rtt()
10843 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_exit_probe_rtt()
10845 if (bbr->r_ctl.rc_lost && in bbr_exit_probe_rtt()
10847 (bbr->rc_use_google == 0)) in bbr_exit_probe_rtt()
10848 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower; in bbr_exit_probe_rtt()
10850 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; in bbr_exit_probe_rtt()
10851 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; in bbr_exit_probe_rtt()
10855 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_exit_probe_rtt()
10856 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0); in bbr_exit_probe_rtt()
10864 if ((bbr->rc_past_init_win == 1) && in bbr_should_enter_probe_rtt()
10865 (bbr->rc_in_persist == 0) && in bbr_should_enter_probe_rtt()
10866 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) { in bbr_should_enter_probe_rtt()
10870 (bbr->rc_in_persist == 0) && in bbr_should_enter_probe_rtt()
10871 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && in bbr_should_enter_probe_rtt()
10872 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { in bbr_should_enter_probe_rtt()
10881 uint64_t btlbw, gain; in bbr_google_startup() local
10884 * Need to be on a pkt-epoch to continue. in bbr_google_startup()
10889 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * in bbr_google_startup()
10890 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_google_startup()
10891 if (btlbw >= gain) { in bbr_google_startup()
10892 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_google_startup()
10893 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_google_startup()
10894 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3); in bbr_google_startup()
10895 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw; in bbr_google_startup()
10897 if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) in bbr_google_startup()
10899 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_google_startup()
10900 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8); in bbr_google_startup()
10908 uint64_t btlbw, gain; in bbr_state_startup() local
10912 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) && in bbr_state_startup()
10913 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_state_startup()
10926 if (bbr->rc_use_google) in bbr_state_startup()
10929 if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) && in bbr_state_startup()
10931 /* Drop to a lower gain 1.5 x since we saw loss */ in bbr_state_startup()
10932 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower; in bbr_state_startup()
10936 * Need to be on a pkt-epoch to continue. in bbr_state_startup()
10944 * gain in rtt over a set threshold? in bbr_state_startup()
10946 if (bbr->r_ctl.rc_pkt_epoch_rtt && in bbr_state_startup()
10947 bbr->r_ctl.startup_last_srtt && in bbr_state_startup()
10948 (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) { in bbr_state_startup()
10949 delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt; in bbr_state_startup()
10950 rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt; in bbr_state_startup()
10953 if ((bbr->r_ctl.startup_last_srtt == 0) || in bbr_state_startup()
10954 (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt)) in bbr_state_startup()
10956 bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt; in bbr_state_startup()
10958 if ((bbr->r_ctl.rc_lost == 0) && in bbr_state_startup()
10962 * our gain threhold for in bbr_state_startup()
10965 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch) in bbr_state_startup()
10966 bbr->r_ctl.rc_bbr_last_startup_epoch++; in bbr_state_startup()
10968 delta, bbr->r_ctl.startup_last_srtt, 10); in bbr_state_startup()
10972 if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) && in bbr_state_startup()
10973 (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) && in bbr_state_startup()
10974 (!IN_RECOVERY(bbr->rc_tp->t_flags))) { in bbr_state_startup()
10979 * the number of non-gain we have already accumulated. in bbr_state_startup()
10981 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch) in bbr_state_startup()
10982 bbr->r_ctl.rc_bbr_last_startup_epoch++; in bbr_state_startup()
10983 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
10984 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9); in bbr_state_startup()
10988 if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost) in bbr_state_startup()
10989 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
10990 bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count; in bbr_state_startup()
10992 if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower) in bbr_state_startup()
10993 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * in bbr_state_startup()
10994 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_state_startup()
10996 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * in bbr_state_startup()
10997 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; in bbr_state_startup()
10999 if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw) in bbr_state_startup()
11000 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw; in bbr_state_startup()
11001 if (btlbw >= gain) { in bbr_state_startup()
11002 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch; in bbr_state_startup()
11004 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11005 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11006 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3); in bbr_state_startup()
11008 if ((bbr->rc_loss_exit && in bbr_state_startup()
11009 (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) && in bbr_state_startup()
11010 (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) && in bbr_state_startup()
11011 ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) { in bbr_state_startup()
11013 * If we had no gain, we had loss and that loss was above in bbr_state_startup()
11019 if ((ctf_flight_size(bbr->rc_tp, in bbr_state_startup()
11020 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + in bbr_state_startup()
11021 (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) { in bbr_state_startup()
11023 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11024 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4); in bbr_state_startup()
11027 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11028 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11029 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5); in bbr_state_startup()
11032 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; in bbr_state_startup()
11033 if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) || in bbr_state_startup()
11039 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11040 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8); in bbr_state_startup()
11051 if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) && in bbr_state_change()
11052 (bbr->rc_bbr_state != BBR_STATE_DRAIN) && in bbr_state_change()
11053 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) && in bbr_state_change()
11054 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) && in bbr_state_change()
11055 (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) { in bbr_state_change()
11057 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state); in bbr_state_change()
11060 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in bbr_state_change()
11065 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_change()
11066 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6); in bbr_state_change()
11067 bbr->rc_filled_pipe = 1; in bbr_state_change()
11068 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_state_change()
11069 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_state_change()
11070 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_state_change()
11071 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_state_change()
11074 if (bbr->rc_no_pacing) in bbr_state_change()
11075 bbr->rc_no_pacing = 0; in bbr_state_change()
11076 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_state_change()
11077 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg; in bbr_state_change()
11078 bbr->rc_bbr_state = BBR_STATE_DRAIN; in bbr_state_change()
11080 if ((bbr->rc_use_google == 0) && in bbr_state_change()
11082 /* Here we don't have to worry about probe-rtt */ in bbr_state_change()
11083 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; in bbr_state_change()
11084 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_state_change()
11087 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain; in bbr_state_change()
11089 if (ctf_flight_size(bbr->rc_tp, in bbr_state_change()
11090 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= in bbr_state_change()
11091 bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11096 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_state_change()
11098 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_state_change()
11102 } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) { in bbr_state_change()
11106 tp = bbr->rc_tp; in bbr_state_change()
11108 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_state_change()
11109 if (inflight >= bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11111 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_state_change()
11112 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_state_change()
11113 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; in bbr_state_change()
11119 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff; in bbr_state_change()
11120 bbr->rc_bbr_substate = BBR_SUB_LEVEL6; in bbr_state_change()
11123 } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) { in bbr_state_change()
11124 /* Has in-flight reached the bdp (or less)? */ in bbr_state_change()
11128 tp = bbr->rc_tp; in bbr_state_change()
11130 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_state_change()
11131 if ((bbr->rc_use_google == 0) && in bbr_state_change()
11133 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_state_change()
11135 * Here we don't have to worry about probe-rtt in bbr_state_change()
11136 * re-slam it, but keep it slammed down. in bbr_state_change()
11138 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_state_change()
11141 if (inflight <= bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11143 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; in bbr_state_change()
11144 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; in bbr_state_change()
11145 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_state_change()
11148 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_state_change()
11149 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); in bbr_state_change()
11151 if ((bbr->rc_use_google == 0) && in bbr_state_change()
11153 (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { in bbr_state_change()
11155 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; in bbr_state_change()
11158 /* Setup probe-rtt has being done now RRS-HERE */ in bbr_state_change()
11159 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_state_change()
11160 bbr->r_ctl.last_in_probertt = cts; in bbr_state_change()
11162 /* Randomly pick a sub-state */ in bbr_state_change()
11163 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_state_change()
11167 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) { in bbr_state_change()
11170 flight = ctf_flight_size(bbr->rc_tp, in bbr_state_change()
11171 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_state_change()
11172 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered); in bbr_state_change()
11173 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) && in bbr_state_change()
11174 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_state_change()
11178 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); in bbr_state_change()
11181 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { in bbr_state_change()
11182 /* Re-slam it */ in bbr_state_change()
11183 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; in bbr_state_change()
11186 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) { in bbr_state_change()
11188 if (flight <= bbr->r_ctl.rc_target_at_state) { in bbr_state_change()
11190 bbr->r_ctl.rc_bbr_enters_probertt = cts; in bbr_state_change()
11192 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) in bbr_state_change()
11193 bbr->r_ctl.rc_bbr_enters_probertt = 1; in bbr_state_change()
11194 if (bbr->rc_use_google == 0) { in bbr_state_change()
11199 if (bbr->r_ctl.bbr_rttprobe_gain_val) in bbr_state_change()
11200 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val; in bbr_state_change()
11202 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; in bbr_state_change()
11205 if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) && in bbr_state_change()
11206 (bbr->rc_use_google == 0) && in bbr_state_change()
11207 bbr->r_ctl.bbr_rttprobe_gain_val && in bbr_state_change()
11208 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) || in bbr_state_change()
11209 (flight >= bbr->r_ctl.flightsize_at_drain))) { in bbr_state_change()
11212 * gain an srtt and have still not made it in bbr_state_change()
11214 * Lets reduce the gain by xx% in bbr_state_change()
11220 bbr->r_ctl.flightsize_at_drain = flight; in bbr_state_change()
11221 bbr->r_ctl.rc_probertt_srttchktim = cts; in bbr_state_change()
11222 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1); in bbr_state_change()
11223 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) { in bbr_state_change()
11224 /* Reduce our gain again */ in bbr_state_change()
11225 bbr->r_ctl.rc_bbr_hptsi_gain -= red; in bbr_state_change()
11227 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) { in bbr_state_change()
11229 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1); in bbr_state_change()
11233 bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1); in bbr_state_change()
11237 if (bbr->r_ctl.rc_bbr_enters_probertt && in bbr_state_change()
11238 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) && in bbr_state_change()
11239 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) { in bbr_state_change()
11241 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts); in bbr_state_change()
11243 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { in bbr_state_change()
11244 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) && in bbr_state_change()
11245 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_state_change()
11267 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) { in bbr_check_bbr_for_state()
11272 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses); in bbr_check_bbr_for_state()
11294 nsegs = max(1, m->m_pkthdr.lro_nsegs); in bbr_do_segment_nounlock()
11295 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_do_segment_nounlock()
11301 * If this is either a state-changing packet or current state isn't in bbr_do_segment_nounlock()
11308 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", in bbr_do_segment_nounlock()
11310 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", in bbr_do_segment_nounlock()
11313 tp->t_rcvtime = ticks; in bbr_do_segment_nounlock()
11315 * Unscale the window into a 32-bit value. For the SYN_SENT state in bbr_do_segment_nounlock()
11318 tiwin = th->th_win << tp->snd_scale; in bbr_do_segment_nounlock()
11320 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin); in bbr_do_segment_nounlock()
11323 if (m->m_flags & M_TSTMP) { in bbr_do_segment_nounlock()
11328 bbr->rc_tv.tv_sec = ts.tv_sec; in bbr_do_segment_nounlock()
11329 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000; in bbr_do_segment_nounlock()
11330 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv); in bbr_do_segment_nounlock()
11331 } else if (m->m_flags & M_TSTMP_LRO) { in bbr_do_segment_nounlock()
11336 bbr->rc_tv.tv_sec = ts.tv_sec; in bbr_do_segment_nounlock()
11337 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000; in bbr_do_segment_nounlock()
11338 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv); in bbr_do_segment_nounlock()
11343 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv); in bbr_do_segment_nounlock()
11349 (th->th_off << 2) - sizeof(struct tcphdr), in bbr_do_segment_nounlock()
11351 if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) { in bbr_do_segment_nounlock()
11366 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) && in bbr_do_segment_nounlock()
11378 to.to_tsecr -= tp->ts_offset; in bbr_do_segment_nounlock()
11379 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv))) in bbr_do_segment_nounlock()
11386 if (bbr->r_state == 0) { in bbr_do_segment_nounlock()
11394 if (bbr->rc_inp == NULL) { in bbr_do_segment_nounlock()
11395 bbr->rc_inp = inp; in bbr_do_segment_nounlock()
11401 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { in bbr_do_segment_nounlock()
11403 (tp->t_flags & TF_REQ_SCALE)) { in bbr_do_segment_nounlock()
11404 tp->t_flags |= TF_RCVD_SCALE; in bbr_do_segment_nounlock()
11405 tp->snd_scale = to.to_wscale; in bbr_do_segment_nounlock()
11407 tp->t_flags &= ~TF_REQ_SCALE; in bbr_do_segment_nounlock()
11412 tp->snd_wnd = th->th_win; in bbr_do_segment_nounlock()
11414 (tp->t_flags & TF_REQ_TSTMP)) { in bbr_do_segment_nounlock()
11415 tp->t_flags |= TF_RCVD_TSTMP; in bbr_do_segment_nounlock()
11416 tp->ts_recent = to.to_tsval; in bbr_do_segment_nounlock()
11417 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_do_segment_nounlock()
11419 tp->t_flags &= ~TF_REQ_TSTMP; in bbr_do_segment_nounlock()
11422 if ((tp->t_flags & TF_SACK_PERMIT) && in bbr_do_segment_nounlock()
11424 tp->t_flags &= ~TF_SACK_PERMIT; in bbr_do_segment_nounlock()
11425 if (tp->t_flags & TF_FASTOPEN) { in bbr_do_segment_nounlock()
11432 if ((inp->inp_vflag & INP_IPV6) != 0) in bbr_do_segment_nounlock()
11448 if ((tp->t_flags & TF_SACK_PERMIT) == 0) { in bbr_do_segment_nounlock()
11451 (*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen, in bbr_do_segment_nounlock()
11456 bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0; in bbr_do_segment_nounlock()
11458 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack); in bbr_do_segment_nounlock()
11469 * always. All other times (timers etc) we must have a rack-state in bbr_do_segment_nounlock()
11474 if (bbr->r_state != tp->t_state) in bbr_do_segment_nounlock()
11477 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL) in bbr_do_segment_nounlock()
11479 prev_state = bbr->r_state; in bbr_do_segment_nounlock()
11480 bbr->rc_ack_was_delayed = 0; in bbr_do_segment_nounlock()
11481 lost = bbr->r_ctl.rc_lost; in bbr_do_segment_nounlock()
11482 bbr->rc_is_pkt_epoch_now = 0; in bbr_do_segment_nounlock()
11483 if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) { in bbr_do_segment_nounlock()
11487 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts; in bbr_do_segment_nounlock()
11488 bbr->rc_ack_was_delayed = 1; in bbr_do_segment_nounlock()
11489 if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay, in bbr_do_segment_nounlock()
11490 bbr->r_ctl.highest_hdwr_delay)) in bbr_do_segment_nounlock()
11491 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay; in bbr_do_segment_nounlock()
11493 bbr->r_ctl.rc_ack_hdwr_delay = 0; in bbr_do_segment_nounlock()
11494 bbr->rc_ack_was_delayed = 0; in bbr_do_segment_nounlock()
11497 bbr->r_ctl.rc_ack_hdwr_delay = 0; in bbr_do_segment_nounlock()
11498 bbr->rc_ack_was_delayed = 0; in bbr_do_segment_nounlock()
11507 * If a segment with the ACK-bit set arrives in the SYN-SENT state in bbr_do_segment_nounlock()
11510 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && in bbr_do_segment_nounlock()
11511 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { in bbr_do_segment_nounlock()
11516 if (tiwin > bbr->r_ctl.rc_high_rwnd) in bbr_do_segment_nounlock()
11517 bbr->r_ctl.rc_high_rwnd = tiwin; in bbr_do_segment_nounlock()
11518 bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp, in bbr_do_segment_nounlock()
11519 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_do_segment_nounlock()
11520 bbr->rtt_valid = 0; in bbr_do_segment_nounlock()
11522 bbr->rc_ts_valid = 1; in bbr_do_segment_nounlock()
11523 bbr->r_ctl.last_inbound_ts = to.to_tsval; in bbr_do_segment_nounlock()
11525 bbr->rc_ts_valid = 0; in bbr_do_segment_nounlock()
11526 bbr->r_ctl.last_inbound_ts = 0; in bbr_do_segment_nounlock()
11528 retval = (*bbr->r_substate) (m, th, so, in bbr_do_segment_nounlock()
11542 if (bbr->rc_is_pkt_epoch_now) in bbr_do_segment_nounlock()
11544 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost)); in bbr_do_segment_nounlock()
11546 if ((bbr->r_wanted_output != 0) || in bbr_do_segment_nounlock()
11547 (tp->t_flags & TF_ACKNOW)) { in bbr_do_segment_nounlock()
11549 bbr->rc_output_starts_timer = 0; in bbr_do_segment_nounlock()
11557 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) && in bbr_do_segment_nounlock()
11558 (SEQ_GT(tp->snd_max, tp->snd_una) || in bbr_do_segment_nounlock()
11559 (tp->t_flags & TF_DELACK) || in bbr_do_segment_nounlock()
11560 ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) && in bbr_do_segment_nounlock()
11561 (tp->t_state <= TCPS_CLOSING)))) { in bbr_do_segment_nounlock()
11566 if ((tp->snd_max == tp->snd_una) && in bbr_do_segment_nounlock()
11567 ((tp->t_flags & TF_DELACK) == 0) && in bbr_do_segment_nounlock()
11569 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { in bbr_do_segment_nounlock()
11578 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && in bbr_do_segment_nounlock()
11579 (TSTMP_GT(lcts, bbr->rc_pacer_started))) { in bbr_do_segment_nounlock()
11582 del = lcts - bbr->rc_pacer_started; in bbr_do_segment_nounlock()
11583 if (bbr->r_ctl.rc_last_delay_val > del) { in bbr_do_segment_nounlock()
11585 bbr->r_ctl.rc_last_delay_val -= del; in bbr_do_segment_nounlock()
11586 bbr->rc_pacer_started = lcts; in bbr_do_segment_nounlock()
11589 bbr->r_ctl.rc_last_delay_val = 0; in bbr_do_segment_nounlock()
11596 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val, in bbr_do_segment_nounlock()
11599 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) { in bbr_do_segment_nounlock()
11601 bbr_timer_audit(tp, bbr, lcts, &so->so_snd); in bbr_do_segment_nounlock()
11604 if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS)) in bbr_do_segment_nounlock()
11605 tp->t_flags2 &= ~TF2_HPTS_CALLS; in bbr_do_segment_nounlock()
11607 if (bbr->r_state != tp->t_state) in bbr_do_segment_nounlock()
11612 bbr->r_wanted_output = 0; in bbr_do_segment_nounlock()
11625 if (!STAILQ_EMPTY(&tp->t_inqueue)) { in bbr_do_segment()
11631 if (m->m_flags & M_TSTMP_LRO) { in bbr_do_segment()
11655 if (ctf_outstanding(tp) >= tp->snd_wnd) { in bbr_what_can_we_send()
11656 /* We never want to go over our peers rcv-window */ in bbr_what_can_we_send()
11661 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); in bbr_what_can_we_send()
11666 * >= tp->snd_wnd). in bbr_what_can_we_send()
11670 len = sendwin - flight; in bbr_what_can_we_send()
11671 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) { in bbr_what_can_we_send()
11673 len = tp->snd_wnd - ctf_outstanding(tp); in bbr_what_can_we_send()
11680 len = avail - sb_offset; in bbr_what_can_we_send()
11693 if (rsm->r_flags & BBR_TLP) { in bbr_do_send_accounting()
11702 tp->t_sndrexmitpack++; in bbr_do_send_accounting()
11706 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB, in bbr_do_send_accounting()
11711 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is in bbr_do_send_accounting()
11712 * sub-state in bbr_do_send_accounting()
11714 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len); in bbr_do_send_accounting()
11715 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) { in bbr_do_send_accounting()
11717 counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len); in bbr_do_send_accounting()
11720 * Log our probe state 3, and log also 5-13 to show in bbr_do_send_accounting()
11721 * us the recovery sub-state for the send. This in bbr_do_send_accounting()
11742 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB, in bbr_do_send_accounting()
11751 if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) { in bbr_cwnd_limiting()
11762 if (tp->snd_cwnd > target) in bbr_cwnd_limiting()
11763 tp->snd_cwnd = target; in bbr_cwnd_limiting()
11773 * account that we are limited by TCP_MAXWIN << tp->rcv_scale. in bbr_window_update_needed()
11779 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) { in bbr_window_update_needed()
11780 oldwin = (tp->rcv_adv - tp->rcv_nxt); in bbr_window_update_needed()
11782 adv -= oldwin; in bbr_window_update_needed()
11795 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale) in bbr_window_update_needed()
11799 (adv >= (so->so_rcv.sb_hiwat / 4) || in bbr_window_update_needed()
11800 recwin <= (so->so_rcv.sb_hiwat / 8) || in bbr_window_update_needed()
11801 so->so_rcv.sb_hiwat <= 8 * maxseg)) { in bbr_window_update_needed()
11804 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat) in bbr_window_update_needed()
11812 * if the TCB was on the hpts. A non-zero return
11870 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_output_wtime()
11872 memcpy(&bbr->rc_tv, tv, sizeof(struct timeval)); in bbr_output_wtime()
11873 cts = tcp_tv_to_usectick(&bbr->rc_tv); in bbr_output_wtime()
11874 inp = bbr->rc_inp; in bbr_output_wtime()
11875 hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS); in bbr_output_wtime()
11876 tp->t_flags2 &= ~TF2_HPTS_CALLS; in bbr_output_wtime()
11877 so = inp->inp_socket; in bbr_output_wtime()
11878 sb = &so->so_snd; in bbr_output_wtime()
11879 if (tp->t_nic_ktls_xmit) in bbr_output_wtime()
11884 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_output_wtime()
11889 pace_max_segs = bbr->r_ctl.rc_pace_max_segs; in bbr_output_wtime()
11893 if (tp->t_flags & TF_TOE) in bbr_output_wtime()
11898 if (bbr->r_state) { in bbr_output_wtime()
11900 isipv6 = bbr->r_is_v6; in bbr_output_wtime()
11902 isipv6 = (inp->inp_vflag & INP_IPV6) != 0; in bbr_output_wtime()
11905 if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && in bbr_output_wtime()
11911 if ((tp->t_flags & TF_ACKNOW) == 0) { in bbr_output_wtime()
11916 * whatever timer is running (KEEP/DEL-ACK?) and in bbr_output_wtime()
11921 recwin = lmin(lmax(sbspace(&so->so_rcv), 0), in bbr_output_wtime()
11922 (long)TCP_MAXWIN << tp->rcv_scale); in bbr_output_wtime()
11924 ((tcp_outflags[tp->t_state] & TH_RST) == 0) && in bbr_output_wtime()
11925 ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <= in bbr_output_wtime()
11926 (tp->snd_max - tp->snd_una))) { in bbr_output_wtime()
11930 * let the timer-run off. in bbr_output_wtime()
11938 if (bbr->r_ctl.rc_last_delay_val) { in bbr_output_wtime()
11940 if (SEQ_GT(cts, bbr->rc_pacer_started)) in bbr_output_wtime()
11941 delay_calc = cts - bbr->rc_pacer_started; in bbr_output_wtime()
11942 if (delay_calc >= bbr->r_ctl.rc_last_delay_val) in bbr_output_wtime()
11943 delay_calc -= bbr->r_ctl.rc_last_delay_val; in bbr_output_wtime()
11948 if ((bbr->r_timer_override) || in bbr_output_wtime()
11949 (tp->t_state < TCPS_ESTABLISHED)) { in bbr_output_wtime()
11954 if ((bbr->r_ctl.rc_last_delay_val) && in bbr_output_wtime()
11955 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && in bbr_output_wtime()
11964 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
11966 } else if (tp->t_state == TCPS_CLOSED) { in bbr_output_wtime()
11967 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
11979 bbr->rc_cwnd_limited = 0; in bbr_output_wtime()
11980 if (bbr->r_ctl.rc_last_delay_val) { in bbr_output_wtime()
11982 if (SEQ_GT(cts, bbr->rc_pacer_started)) in bbr_output_wtime()
11983 delay_calc = cts - bbr->rc_pacer_started; in bbr_output_wtime()
11986 if (delay_calc >= bbr->r_ctl.rc_last_delay_val) in bbr_output_wtime()
11988 delay_calc -= bbr->r_ctl.rc_last_delay_val; in bbr_output_wtime()
11996 bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc); in bbr_output_wtime()
11997 bbr->r_agg_early_set = 1; in bbr_output_wtime()
11998 if (bbr->r_ctl.rc_hptsi_agg_delay) { in bbr_output_wtime()
11999 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) { in bbr_output_wtime()
12001 bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early; in bbr_output_wtime()
12002 bbr->r_agg_early_set = 0; in bbr_output_wtime()
12003 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
12005 bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay; in bbr_output_wtime()
12006 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_output_wtime()
12009 merged_val = bbr->rc_pacer_started; in bbr_output_wtime()
12011 merged_val |= bbr->r_ctl.rc_last_delay_val; in bbr_output_wtime()
12013 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val, in bbr_output_wtime()
12014 bbr->r_agg_early_set, 3); in bbr_output_wtime()
12015 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12021 if (bbr->r_agg_early_set) in bbr_output_wtime()
12022 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
12023 bbr->r_agg_early_set = 0; in bbr_output_wtime()
12032 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay) in bbr_output_wtime()
12033 bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff; in bbr_output_wtime()
12035 bbr->r_ctl.rc_hptsi_agg_delay += delay_calc; in bbr_output_wtime()
12037 sendwin = min(tp->snd_wnd, tp->snd_cwnd); in bbr_output_wtime()
12038 if ((tp->snd_una == tp->snd_max) && in bbr_output_wtime()
12039 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) && in bbr_output_wtime()
12044 * suite of states or a fast-ramp up. in bbr_output_wtime()
12047 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time)); in bbr_output_wtime()
12055 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { in bbr_output_wtime()
12068 bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY; in bbr_output_wtime()
12070 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { in bbr_output_wtime()
12071 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12073 bbr->r_timer_override = 0; in bbr_output_wtime()
12074 bbr->r_wanted_output = 0; in bbr_output_wtime()
12079 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12080 ((tp->t_state == TCPS_SYN_RECEIVED) || in bbr_output_wtime()
12081 (tp->t_state == TCPS_SYN_SENT)) && in bbr_output_wtime()
12082 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */ in bbr_output_wtime()
12083 (tp->t_rxtshift == 0)) { /* not a retransmit */ in bbr_output_wtime()
12092 if (bbr->rc_use_google == 0) in bbr_output_wtime()
12111 sendwin = min(tp->snd_wnd, tp->snd_cwnd); in bbr_output_wtime()
12112 sb_offset = tp->snd_max - tp->snd_una; in bbr_output_wtime()
12113 flags = tcp_outflags[tp->t_state]; in bbr_output_wtime()
12122 while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) { in bbr_output_wtime()
12128 tot_len = tp->t_maxseg; in bbr_output_wtime()
12134 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next); in bbr_output_wtime()
12135 bbr->r_ctl.rc_free_cnt++; in bbr_output_wtime()
12139 if (bbr->r_ctl.rc_resend == NULL) { in bbr_output_wtime()
12141 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_output_wtime()
12142 if (bbr->r_ctl.rc_resend) { in bbr_output_wtime()
12146 bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend); in bbr_output_wtime()
12149 if (bbr->r_ctl.rc_resend) { in bbr_output_wtime()
12150 rsm = bbr->r_ctl.rc_resend; in bbr_output_wtime()
12154 /* Remove any TLP flags its a RACK or T-O */ in bbr_output_wtime()
12155 rsm->r_flags &= ~BBR_TLP; in bbr_output_wtime()
12156 bbr->r_ctl.rc_resend = NULL; in bbr_output_wtime()
12157 if (SEQ_LT(rsm->r_start, tp->snd_una)) { in bbr_output_wtime()
12159 panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n", in bbr_output_wtime()
12160 tp, bbr, rsm, rsm->r_start, tp->snd_una); in bbr_output_wtime()
12168 if (rsm->r_flags & BBR_HAS_SYN) { in bbr_output_wtime()
12173 rsm->r_start++; in bbr_output_wtime()
12174 if (rsm->r_start == rsm->r_end) { in bbr_output_wtime()
12179 rsm->r_flags &= ~BBR_HAS_SYN; in bbr_output_wtime()
12180 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12189 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12190 if ((bbr->rc_resends_use_tso == 0) && in bbr_output_wtime()
12195 sb_offset = rsm->r_start - tp->snd_una; in bbr_output_wtime()
12207 } else if (bbr->r_ctl.rc_tlp_send) { in bbr_output_wtime()
12212 rsm = bbr->r_ctl.rc_tlp_send; in bbr_output_wtime()
12213 bbr->r_ctl.rc_tlp_send = NULL; in bbr_output_wtime()
12215 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12216 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg)) in bbr_output_wtime()
12219 if (SEQ_GT(tp->snd_una, rsm->r_start)) { in bbr_output_wtime()
12221 panic("tp:%p bbc:%p snd_una:%u rsm:%p r_start:%u", in bbr_output_wtime()
12222 tp, bbr, tp->snd_una, rsm, rsm->r_start); in bbr_output_wtime()
12229 sb_offset = rsm->r_start - tp->snd_una; in bbr_output_wtime()
12238 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { in bbr_output_wtime()
12240 if (!bbr->alloc_limit_reported) { in bbr_output_wtime()
12241 bbr->alloc_limit_reported = 1; in bbr_output_wtime()
12247 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) { in bbr_output_wtime()
12248 panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u", in bbr_output_wtime()
12256 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL)) in bbr_output_wtime()
12258 if (tp->t_flags & TF_NEEDSYN) in bbr_output_wtime()
12261 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) { in bbr_output_wtime()
12263 len--; in bbr_output_wtime()
12279 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext); in bbr_output_wtime()
12290 * fast-retransmit because TCP will reset snd_nxt to snd_max after in bbr_output_wtime()
12291 * the fast-retransmit. in bbr_output_wtime()
12293 * In the normal retransmit-FIN-only case, however, snd_nxt will be in bbr_output_wtime()
12304 if (SEQ_GT(tp->snd_max, tp->snd_una)) in bbr_output_wtime()
12305 sb_offset = tp->snd_max - tp->snd_una; in bbr_output_wtime()
12308 if (bbr->rc_tlp_new_data) { in bbr_output_wtime()
12315 if (tlplen > (uint32_t)(avail - sb_offset)) { in bbr_output_wtime()
12316 tlplen = (uint32_t)(avail - sb_offset); in bbr_output_wtime()
12318 if (tlplen > tp->snd_wnd) { in bbr_output_wtime()
12319 len = tp->snd_wnd; in bbr_output_wtime()
12323 bbr->rc_tlp_new_data = 0; in bbr_output_wtime()
12327 (bbr->rc_in_persist == 0) && in bbr_output_wtime()
12329 ((avail - sb_offset) >= p_maxseg)) { in bbr_output_wtime()
12341 if (bbr->rc_in_persist) { in bbr_output_wtime()
12348 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_output_wtime()
12350 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
12351 if (rsm->r_flags & BBR_HAS_FIN) in bbr_output_wtime()
12352 len--; in bbr_output_wtime()
12353 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg)) in bbr_output_wtime()
12366 sb_offset = rsm->r_start - tp->snd_una; in bbr_output_wtime()
12387 * SYN-SENT state and if segment contains data and if we don't know in bbr_output_wtime()
12391 SEQ_GT(tp->snd_max, tp->snd_una)) { in bbr_output_wtime()
12392 if (tp->t_state != TCPS_SYN_RECEIVED) in bbr_output_wtime()
12398 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12399 (tp->t_state == TCPS_SYN_RECEIVED)) in bbr_output_wtime()
12401 sb_offset--, len++; in bbr_output_wtime()
12409 len--; in bbr_output_wtime()
12416 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) { in bbr_output_wtime()
12423 * - When retransmitting SYN|ACK on a passively-created socket in bbr_output_wtime()
12424 * - When retransmitting SYN on an actively created socket in bbr_output_wtime()
12425 * - When sending a zero-length cookie (cookie request) on an in bbr_output_wtime()
12427 * - When the socket is in the CLOSED state (RST is being sent) in bbr_output_wtime()
12429 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12430 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) || in bbr_output_wtime()
12431 ((tp->t_state == TCPS_SYN_SENT) && in bbr_output_wtime()
12432 (tp->t_tfo_client_cookie_len == 0)) || in bbr_output_wtime()
12438 /* Without fast-open there should never be data sent on a SYN */ in bbr_output_wtime()
12439 if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN)) in bbr_output_wtime()
12453 * 0-byte window. This makes sure the persist timer is set in bbr_output_wtime()
12458 if ((tp->snd_wnd == 0) && in bbr_output_wtime()
12459 (TCPS_HAVEESTABLISHED(tp->t_state)) && in bbr_output_wtime()
12460 (tp->snd_una == tp->snd_max) && in bbr_output_wtime()
12470 (len < bbr->r_ctl.rc_pace_max_segs)) { in bbr_output_wtime()
12476 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_output_wtime()
12477 (TCPS_HAVEESTABLISHED(tp->t_state)) && in bbr_output_wtime()
12478 (len < (int)(sbavail(sb) - sb_offset))) { in bbr_output_wtime()
12487 if (tp->snd_max == tp->snd_una) in bbr_output_wtime()
12489 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) && in bbr_output_wtime()
12490 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12491 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) && in bbr_output_wtime()
12492 (len < (int)(sbavail(sb) - sb_offset)) && in bbr_output_wtime()
12504 bbr->rc_cwnd_limited = 1; in bbr_output_wtime()
12506 } else if (((tp->snd_wnd - ctf_outstanding(tp)) < in bbr_output_wtime()
12507 min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && in bbr_output_wtime()
12508 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12509 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) && in bbr_output_wtime()
12510 (len < (int)(sbavail(sb) - sb_offset)) && in bbr_output_wtime()
12511 (TCPS_HAVEESTABLISHED(tp->t_state))) { in bbr_output_wtime()
12529 if (bbr->rc_in_persist && in bbr_output_wtime()
12532 (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) { in bbr_output_wtime()
12540 sbleft = sbavail(sb) - sb_offset; in bbr_output_wtime()
12543 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) { in bbr_output_wtime()
12553 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP in bbr_output_wtime()
12570 if (inp->inp_options) in bbr_output_wtime()
12571 ipoptlen = inp->inp_options->m_len - in bbr_output_wtime()
12577 * Pre-calculate here as we save another lookup into the darknesses in bbr_output_wtime()
12595 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && in bbr_output_wtime()
12597 (tp->t_port == 0) && in bbr_output_wtime()
12598 ((tp->t_flags & TF_SIGNATURE) == 0) && in bbr_output_wtime()
12602 recwin = lmin(lmax(sbspace(&so->so_rcv), 0), in bbr_output_wtime()
12603 (long)TCP_MAXWIN << tp->rcv_scale); in bbr_output_wtime()
12606 * conditions when len is non-zero: in bbr_output_wtime()
12608 * - We have a full segment (or more with TSO) - This is the last in bbr_output_wtime()
12610 * NODELAY - we've timed out (e.g. persist timer) - we have more in bbr_output_wtime()
12612 * limited the window size) - we need to retransmit in bbr_output_wtime()
12627 if (((tp->t_flags & TF_MORETOCOME) == 0) && /* normal case */ in bbr_output_wtime()
12628 ((tp->t_flags & TF_NODELAY) || in bbr_output_wtime()
12629 ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) && in bbr_output_wtime()
12630 (tp->t_flags & TF_NOPUSH) == 0) { in bbr_output_wtime()
12633 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */ in bbr_output_wtime()
12636 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) { in bbr_output_wtime()
12664 * pending (it will get piggy-backed on it) or the remote side in bbr_output_wtime()
12665 * already has done a half-close and won't send more data. Skip in bbr_output_wtime()
12666 * this if the connection is in T/TCP half-open state. in bbr_output_wtime()
12668 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) && in bbr_output_wtime()
12669 !(tp->t_flags & TF_DELACK) && in bbr_output_wtime()
12670 !TCPS_HAVERCVDFIN(tp->t_state)) { in bbr_output_wtime()
12677 * is also a catch-all for the retransmit timer timeout case. in bbr_output_wtime()
12679 if (tp->t_flags & TF_ACKNOW) { in bbr_output_wtime()
12686 if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) { in bbr_output_wtime()
12694 ((tp->t_flags & TF_SENTFIN) == 0)) { in bbr_output_wtime()
12704 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); in bbr_output_wtime()
12705 if (bbr->rc_no_pacing) in bbr_output_wtime()
12708 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >= in bbr_output_wtime()
12709 tp->snd_wnd) { in bbr_output_wtime()
12713 if ((bbr->rc_in_persist == 0) && in bbr_output_wtime()
12714 TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_output_wtime()
12715 (tp->snd_max == tp->snd_una) && in bbr_output_wtime()
12716 sbavail(&so->so_snd)) { in bbr_output_wtime()
12718 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); in bbr_output_wtime()
12724 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12725 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) { in bbr_output_wtime()
12728 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12729 bbr->r_ctl.rc_lost_bytes))); in bbr_output_wtime()
12730 bbr->rc_cwnd_limited = 1; in bbr_output_wtime()
12736 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_output_wtime()
12737 bbr->r_agg_early_set = 0; in bbr_output_wtime()
12738 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
12739 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12740 } else if (bbr->rc_use_google == 0) in bbr_output_wtime()
12748 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
12749 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered); in bbr_output_wtime()
12754 bbr->r_ctl.rc_last_delay_val = 0; in bbr_output_wtime()
12755 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
12758 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_output_wtime()
12760 tp->snd_nxt = tp->snd_max; in bbr_output_wtime()
12773 bbr->rc_tlp_in_progress = 0; in bbr_output_wtime()
12774 bbr->rc_tlp_rtx_out = 0; in bbr_output_wtime()
12779 bbr->rc_tlp_in_progress = 1; in bbr_output_wtime()
12785 * This is sub-optimal. We only send a stand alone in bbr_output_wtime()
12790 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) { in bbr_output_wtime()
12803 (((rsm->r_flags & BBR_HAS_FIN) == 0) && in bbr_output_wtime()
12806 len--; in bbr_output_wtime()
12811 if ((tp->snd_una == tp->snd_max) && in bbr_output_wtime()
12812 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_output_wtime()
12822 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT; in bbr_output_wtime()
12824 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT; in bbr_output_wtime()
12843 * established connection segments. Options for SYN-ACK segments in bbr_output_wtime()
12848 if ((tp->t_flags & TF_NOOPT) == 0) { in bbr_output_wtime()
12851 to.to_mss = tcp_mssopt(&inp->inp_inc); in bbr_output_wtime()
12852 if (tp->t_port) in bbr_output_wtime()
12853 to.to_mss -= V_tcp_udp_tunneling_overhead; in bbr_output_wtime()
12862 if ((tp->t_flags & TF_FASTOPEN) && in bbr_output_wtime()
12863 (tp->t_rxtshift == 0)) { in bbr_output_wtime()
12864 if (tp->t_state == TCPS_SYN_RECEIVED) { in bbr_output_wtime()
12867 (u_int8_t *)&tp->t_tfo_cookie.server; in bbr_output_wtime()
12870 } else if (tp->t_state == TCPS_SYN_SENT) { in bbr_output_wtime()
12872 tp->t_tfo_client_cookie_len; in bbr_output_wtime()
12874 tp->t_tfo_cookie.client; in bbr_output_wtime()
12880 /* Window scaling. */ in bbr_output_wtime()
12881 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) { in bbr_output_wtime()
12882 to.to_wscale = tp->request_r_scale; in bbr_output_wtime()
12886 if ((tp->t_flags & TF_RCVD_TSTMP) || in bbr_output_wtime()
12887 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) { in bbr_output_wtime()
12888 to.to_tsval = tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset; in bbr_output_wtime()
12889 to.to_tsecr = tp->ts_recent; in bbr_output_wtime()
12894 if (tp->rfbuf_ts == 0 && in bbr_output_wtime()
12895 (so->so_rcv.sb_flags & SB_AUTOSIZE)) in bbr_output_wtime()
12896 tp->rfbuf_ts = tcp_tv_to_mssectick(&bbr->rc_tv); in bbr_output_wtime()
12900 else if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_output_wtime()
12901 tp->rcv_numsacks > 0) { in bbr_output_wtime()
12903 to.to_nsacks = tp->rcv_numsacks; in bbr_output_wtime()
12904 to.to_sacks = (u_char *)tp->sackblks; in bbr_output_wtime()
12907 /* TCP-MD5 (RFC2385). */ in bbr_output_wtime()
12908 if (tp->t_flags & TF_SIGNATURE) in bbr_output_wtime()
12918 if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie && in bbr_output_wtime()
12922 if (tp->t_port) { in bbr_output_wtime()
12935 if (inp->inp_options) in bbr_output_wtime()
12936 ipoptlen = inp->inp_options->m_len - in bbr_output_wtime()
12944 if (bbr->rc_last_options != local_options) { in bbr_output_wtime()
12949 bbr->rc_last_options = local_options; in bbr_output_wtime()
12951 maxseg = tp->t_maxseg - (ipoptlen + optlen); in bbr_output_wtime()
12967 if_hw_tsomax = tp->t_tsomax; in bbr_output_wtime()
12968 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; in bbr_output_wtime()
12969 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; in bbr_output_wtime()
12979 max_len = (if_hw_tsomax - hdrlen - in bbr_output_wtime()
12994 len -= moff; in bbr_output_wtime()
13007 if (optlen + ipoptlen >= tp->t_maxseg) { in bbr_output_wtime()
13045 if (SEQ_LT(rsm->r_start, tp->snd_una)) { in bbr_output_wtime()
13046 panic("RSM:%p TP:%p bbr:%p start:%u is < snd_una:%u", in bbr_output_wtime()
13047 rsm, tp, bbr, rsm->r_start, tp->snd_una); in bbr_output_wtime()
13091 m->m_data += max_linkhdr; in bbr_output_wtime()
13092 m->m_len = hdrlen; in bbr_output_wtime()
13100 panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u", in bbr_output_wtime()
13117 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); in bbr_output_wtime()
13124 if (rsm->r_start != tp->snd_una) { in bbr_output_wtime()
13132 return (-EFAULT); /* tcp_drop() */ in bbr_output_wtime()
13134 len = rsm->r_end - rsm->r_start; in bbr_output_wtime()
13142 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) { in bbr_output_wtime()
13147 m->m_len += len; in bbr_output_wtime()
13158 …panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d… in bbr_output_wtime()
13161 tp->snd_una, rsm->r_flags, rsm->r_start, in bbr_output_wtime()
13166 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u sb_offset:%u snd_una:%u", in bbr_output_wtime()
13167 tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una); in bbr_output_wtime()
13171 m->m_next = tcp_m_copym( in bbr_output_wtime()
13189 if (m->m_next == NULL) { in bbr_output_wtime()
13199 panic("tp:%p tso on, but len:%d < maxseg:%d", in bbr_output_wtime()
13209 foo = foo->m_next; in bbr_output_wtime()
13229 if (tp->t_flags & TF_ACKNOW) in bbr_output_wtime()
13251 m->m_data += max_linkhdr; in bbr_output_wtime()
13252 m->m_len = hdrlen; in bbr_output_wtime()
13255 m->m_pkthdr.rcvif = (struct ifnet *)0; in bbr_output_wtime()
13262 if (tp->t_port) { in bbr_output_wtime()
13264 udp->uh_sport = htons(V_tcp_udp_tunneling_port); in bbr_output_wtime()
13265 udp->uh_dport = tp->t_port; in bbr_output_wtime()
13266 ulen = hdrlen + len - sizeof(struct ip6_hdr); in bbr_output_wtime()
13267 udp->uh_ulen = htons(ulen); in bbr_output_wtime()
13272 tcpip_fillheaders(inp, tp->t_port, ip6, th); in bbr_output_wtime()
13277 if (tp->t_port) { in bbr_output_wtime()
13279 udp->uh_sport = htons(V_tcp_udp_tunneling_port); in bbr_output_wtime()
13280 udp->uh_dport = tp->t_port; in bbr_output_wtime()
13281 ulen = hdrlen + len - sizeof(struct ip); in bbr_output_wtime()
13282 udp->uh_ulen = htons(ulen); in bbr_output_wtime()
13287 tcpip_fillheaders(inp, tp->t_port, ip, th); in bbr_output_wtime()
13303 th->th_seq = htonl(tp->snd_max); in bbr_output_wtime()
13304 bbr_seq = tp->snd_max; in bbr_output_wtime()
13307 th->th_seq = htonl(tp->iss); in bbr_output_wtime()
13308 bbr_seq = tp->iss; in bbr_output_wtime()
13310 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) { in bbr_output_wtime()
13315 th->th_seq = (htonl(tp->snd_max - 1)); in bbr_output_wtime()
13316 bbr_seq = (tp->snd_max - 1); in bbr_output_wtime()
13319 th->th_seq = htonl(tp->snd_max); in bbr_output_wtime()
13320 bbr_seq = tp->snd_max; in bbr_output_wtime()
13332 * might be better to send (tp->snd_una - 1) which in bbr_output_wtime()
13335 if (tp->t_flags & TF_SENTFIN) { in bbr_output_wtime()
13336 th->th_seq = htonl(tp->snd_max - 1); in bbr_output_wtime()
13337 bbr_seq = (tp->snd_max - 1); in bbr_output_wtime()
13339 th->th_seq = htonl(tp->snd_max); in bbr_output_wtime()
13340 bbr_seq = tp->snd_max; in bbr_output_wtime()
13345 th->th_seq = htonl(rsm->r_start); in bbr_output_wtime()
13346 bbr_seq = rsm->r_start; in bbr_output_wtime()
13348 th->th_ack = htonl(tp->rcv_nxt); in bbr_output_wtime()
13351 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; in bbr_output_wtime()
13358 if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) && in bbr_output_wtime()
13361 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && in bbr_output_wtime()
13362 recwin < (tp->rcv_adv - tp->rcv_nxt)) in bbr_output_wtime()
13363 recwin = (tp->rcv_adv - tp->rcv_nxt); in bbr_output_wtime()
13364 if (recwin > TCP_MAXWIN << tp->rcv_scale) in bbr_output_wtime()
13365 recwin = TCP_MAXWIN << tp->rcv_scale; in bbr_output_wtime()
13373 th->th_win = htons((u_short) in bbr_output_wtime()
13374 (min(sbspace(&so->so_rcv), TCP_MAXWIN))); in bbr_output_wtime()
13376 /* Avoid shrinking window with window scaling. */ in bbr_output_wtime()
13377 recwin = roundup2(recwin, 1 << tp->rcv_scale); in bbr_output_wtime()
13378 th->th_win = htons((u_short)(recwin >> tp->rcv_scale)); in bbr_output_wtime()
13381 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0 in bbr_output_wtime()
13388 if (th->th_win == 0) { in bbr_output_wtime()
13389 tp->t_sndzerowin++; in bbr_output_wtime()
13390 tp->t_flags |= TF_RXWIN0SENT; in bbr_output_wtime()
13392 tp->t_flags &= ~TF_RXWIN0SENT; in bbr_output_wtime()
13397 tp->snd_up = tp->snd_una; in bbr_output_wtime()
13402 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ in bbr_output_wtime()
13412 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) { in bbr_output_wtime()
13428 if (tp->t_port) { in bbr_output_wtime()
13429 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; in bbr_output_wtime()
13430 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); in bbr_output_wtime()
13431 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); in bbr_output_wtime()
13432 th->th_sum = htons(0); in bbr_output_wtime()
13435 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; in bbr_output_wtime()
13436 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); in bbr_output_wtime()
13437 th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) + in bbr_output_wtime()
13447 if (tp->t_port) { in bbr_output_wtime()
13448 m->m_pkthdr.csum_flags = CSUM_UDP; in bbr_output_wtime()
13449 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); in bbr_output_wtime()
13450 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, in bbr_output_wtime()
13451 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); in bbr_output_wtime()
13452 th->th_sum = htons(0); in bbr_output_wtime()
13455 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP; in bbr_output_wtime()
13456 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); in bbr_output_wtime()
13457 th->th_sum = in_pseudo(ip->ip_src.s_addr, in bbr_output_wtime()
13458 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + in bbr_output_wtime()
13462 KASSERT(ip->ip_v == IPVERSION, in bbr_output_wtime()
13463 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); in bbr_output_wtime()
13475 m->m_pkthdr.csum_flags |= CSUM_TSO; in bbr_output_wtime()
13477 m->m_pkthdr.tso_segsz = maxseg; in bbr_output_wtime()
13494 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay; in bbr_output_wtime()
13495 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3); in bbr_output_wtime()
13498 log.u_bbr.flex5 = bbr->rc_past_init_win; in bbr_output_wtime()
13500 log.u_bbr.flex5 |= bbr->rc_no_pacing; in bbr_output_wtime()
13502 log.u_bbr.flex5 |= tp->t_maxseg; in bbr_output_wtime()
13503 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs; in bbr_output_wtime()
13504 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr); in bbr_output_wtime()
13506 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg; in bbr_output_wtime()
13515 lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, in bbr_output_wtime()
13526 * m->m_pkthdr.len should have been set before cksum calcuration, in bbr_output_wtime()
13537 ip6->ip6_hlim = in6_selecthlim(inp, NULL); in bbr_output_wtime()
13544 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); in bbr_output_wtime()
13547 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13549 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13551 if (tp->t_state == TCPS_SYN_SENT) in bbr_output_wtime()
13556 error = ip6_output(m, inp->in6p_outputopts, in bbr_output_wtime()
13557 &inp->inp_route6, in bbr_output_wtime()
13561 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL) in bbr_output_wtime()
13562 mtu = inp->inp_route6.ro_nh->nh_mtu; in bbr_output_wtime()
13570 ip->ip_len = htons(m->m_pkthdr.len); in bbr_output_wtime()
13573 ip->ip_ttl = in6_selecthlim(inp, NULL); in bbr_output_wtime()
13585 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { in bbr_output_wtime()
13586 tp->t_flags2 |= TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13587 if (tp->t_port == 0 || len < V_tcp_minmss) { in bbr_output_wtime()
13588 ip->ip_off |= htons(IP_DF); in bbr_output_wtime()
13591 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; in bbr_output_wtime()
13594 if (tp->t_state == TCPS_SYN_SENT) in bbr_output_wtime()
13599 error = ip_output(m, inp->inp_options, &inp->inp_route, in bbr_output_wtime()
13602 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL) in bbr_output_wtime()
13603 mtu = inp->inp_route.ro_nh->nh_mtu; in bbr_output_wtime()
13607 lgb->tlb_errno = error; in bbr_output_wtime()
13618 if (TCPS_HAVEESTABLISHED(tp->t_state) && in bbr_output_wtime()
13619 (tp->t_flags & TF_SACK_PERMIT) && in bbr_output_wtime()
13620 tp->rcv_numsacks > 0) in bbr_output_wtime()
13623 bbr->output_error_seen = 0; in bbr_output_wtime()
13624 bbr->oerror_cnt = 0; in bbr_output_wtime()
13625 bbr->bbr_segs_rcvd = 0; in bbr_output_wtime()
13631 if (tp->snd_una == tp->snd_max) { in bbr_output_wtime()
13638 bbr->r_ctl.rc_del_time = cts; in bbr_output_wtime()
13643 counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1); in bbr_output_wtime()
13648 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options); in bbr_output_wtime()
13650 idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1); in bbr_output_wtime()
13664 if (tp->snd_una == tp->snd_max) in bbr_output_wtime()
13665 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_output_wtime()
13677 if (bbr->rc_in_persist == 0) { in bbr_output_wtime()
13685 if (tp->snd_una == tp->snd_max && in bbr_output_wtime()
13692 bbr->rc_tp->t_acktime = ticks; in bbr_output_wtime()
13700 tp->snd_max = tp->iss + 1; in bbr_output_wtime()
13702 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) { in bbr_output_wtime()
13703 tp->snd_max++; in bbr_output_wtime()
13704 tp->t_flags |= TF_SENTFIN; in bbr_output_wtime()
13708 tp->snd_max += len; in bbr_output_wtime()
13721 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) { in bbr_output_wtime()
13723 tp->t_flags |= TF_SENTFIN; in bbr_output_wtime()
13725 if (xlen && (tp->snd_una == tp->snd_max)) { in bbr_output_wtime()
13731 bbr->rc_tp->t_acktime = ticks; in bbr_output_wtime()
13734 tp->snd_max += xlen; in bbr_output_wtime()
13741 * case of ENOBUFS we will fall out and become ack-clocked. in bbr_output_wtime()
13749 bbr->r_ctl.rc_hptsi_agg_delay = 0; in bbr_output_wtime()
13750 bbr->r_ctl.rc_agg_early = 0; in bbr_output_wtime()
13751 bbr->r_agg_early_set = 0; in bbr_output_wtime()
13752 bbr->output_error_seen = 1; in bbr_output_wtime()
13753 if (bbr->oerror_cnt < 0xf) in bbr_output_wtime()
13754 bbr->oerror_cnt++; in bbr_output_wtime()
13755 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) { in bbr_output_wtime()
13757 return (-ENETDOWN); in bbr_output_wtime()
13764 * slam him below a T-O (1MSS). in bbr_output_wtime()
13766 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { in bbr_output_wtime()
13767 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
13768 bbr->r_ctl.rc_lost_bytes)) - maxseg; in bbr_output_wtime()
13769 if (tp->snd_cwnd < maxseg) in bbr_output_wtime()
13770 tp->snd_cwnd = maxseg; in bbr_output_wtime()
13772 slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt; in bbr_output_wtime()
13774 if (bbr->bbr_hdrw_pacing) in bbr_output_wtime()
13799 old_maxseg = tp->t_maxseg; in bbr_output_wtime()
13803 tcp_mss_update(tp, -1, mtu, NULL, NULL); in bbr_output_wtime()
13804 if (old_maxseg <= tp->t_maxseg) { in bbr_output_wtime()
13806 tp->t_maxseg = old_maxseg - 40; in bbr_output_wtime()
13807 if (tp->t_maxseg < V_tcp_mssdflt) { in bbr_output_wtime()
13813 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT; in bbr_output_wtime()
13815 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT; in bbr_output_wtime()
13823 if ((tot_len + len) && (len >= tp->t_maxseg)) { in bbr_output_wtime()
13825 bbr->r_ctl.rc_bbr_hptsi_gain, in bbr_output_wtime()
13828 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt; in bbr_output_wtime()
13830 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt; in bbr_output_wtime()
13831 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
13838 tp->t_softerror = error; in bbr_output_wtime()
13844 if (TCPS_HAVERCVDSYN(tp->t_state)) { in bbr_output_wtime()
13845 tp->t_softerror = error; in bbr_output_wtime()
13850 slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt; in bbr_output_wtime()
13851 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
13856 } else if (((tp->t_flags & TF_GPUTINPROG) == 0) && in bbr_output_wtime()
13859 (bbr->rc_in_persist == 0)) { in bbr_output_wtime()
13860 tp->gput_seq = bbr_seq; in bbr_output_wtime()
13861 tp->gput_ack = bbr_seq + in bbr_output_wtime()
13862 min(sbavail(&so->so_snd) - sb_offset, sendwin); in bbr_output_wtime()
13863 tp->gput_ts = cts; in bbr_output_wtime()
13864 tp->t_flags |= TF_GPUTINPROG; in bbr_output_wtime()
13868 if ((bbr->bbr_hdw_pace_ena) && in bbr_output_wtime()
13869 (bbr->bbr_attempt_hdwr_pace == 0) && in bbr_output_wtime()
13870 (bbr->rc_past_init_win) && in bbr_output_wtime()
13871 (bbr->rc_bbr_state != BBR_STATE_STARTUP) && in bbr_output_wtime()
13872 (get_filter_value(&bbr->r_ctl.rc_delrate)) && in bbr_output_wtime()
13873 (inp->inp_route.ro_nh && in bbr_output_wtime()
13874 inp->inp_route.ro_nh->nh_ifp)) { in bbr_output_wtime()
13886 bbr->bbr_attempt_hdwr_pace = 1; in bbr_output_wtime()
13887 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp, in bbr_output_wtime()
13888 inp->inp_route.ro_nh->nh_ifp, in bbr_output_wtime()
13892 if (bbr->r_ctl.crte) { in bbr_output_wtime()
13894 bbr->r_ctl.crte->ptbl->rs_ifp, in bbr_output_wtime()
13896 bbr->r_ctl.crte->rate, in bbr_output_wtime()
13899 counter_u64_add(bbr_flows_nohdwr_pacing, -1); in bbr_output_wtime()
13901 bbr->bbr_hdrw_pacing = 1; in bbr_output_wtime()
13902 /* Now what is our gain status? */ in bbr_output_wtime()
13903 if (bbr->r_ctl.crte->rate < rate_wanted) { in bbr_output_wtime()
13906 bbr->r_ctl.crte->rate, rate_wanted); in bbr_output_wtime()
13909 bbr->gain_is_limited = 0; in bbr_output_wtime()
13910 bbr->skip_gain = 0; in bbr_output_wtime()
13915 inp->inp_route.ro_nh->nh_ifp, in bbr_output_wtime()
13922 if (bbr->bbr_hdrw_pacing) { in bbr_output_wtime()
13929 if (inp->inp_snd_tag == NULL) { in bbr_output_wtime()
13931 bbr->bbr_hdrw_pacing = 0; in bbr_output_wtime()
13932 } else if ((inp->inp_route.ro_nh == NULL) || in bbr_output_wtime()
13933 (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) { in bbr_output_wtime()
13937 * and setup to re-attempt next go in bbr_output_wtime()
13940 bbr->bbr_hdrw_pacing = 0; in bbr_output_wtime()
13941 bbr->bbr_attempt_hdwr_pace = 0; in bbr_output_wtime()
13942 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp); in bbr_output_wtime()
13951 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) in bbr_output_wtime()
13952 tp->rcv_adv = tp->rcv_nxt + recwin; in bbr_output_wtime()
13954 tp->last_ack_sent = tp->rcv_nxt; in bbr_output_wtime()
13956 (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) && in bbr_output_wtime()
13962 (IN_RECOVERY(tp->t_flags) == 0) && in bbr_output_wtime()
13963 (bbr->rc_in_persist == 0) && in bbr_output_wtime()
13964 (tot_len < bbr->r_ctl.rc_pace_max_segs)) { in bbr_output_wtime()
13966 * For non-tso we need to goto again until we have sent out in bbr_output_wtime()
13970 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_output_wtime()
13972 tp->snd_nxt = tp->snd_max; in bbr_output_wtime()
13980 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); in bbr_output_wtime()
13990 * Calculate/Re-Calculate the hptsi slot in usecs based on in bbr_output_wtime()
13993 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); in bbr_output_wtime()
13994 if (bbr->rc_no_pacing) in bbr_output_wtime()
13997 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); in bbr_output_wtime()
13999 if (bbr->rc_use_google == 0) in bbr_output_wtime()
14001 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + in bbr_output_wtime()
14002 bbr->r_ctl.rc_lost_bytes))); in bbr_output_wtime()
14003 bbr->rc_output_starts_timer = 1; in bbr_output_wtime()
14004 if (bbr->bbr_use_rack_cheat && in bbr_output_wtime()
14006 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) { in bbr_output_wtime()
14011 if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) { in bbr_output_wtime()
14017 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++; in bbr_output_wtime()
14018 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) { in bbr_output_wtime()
14019 bbr->hw_pacing_set = 1; in bbr_output_wtime()
14024 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { in bbr_output_wtime()
14026 tp->snd_nxt = tp->snd_max; in bbr_output_wtime()
14061 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_mtu_chg()
14062 maxseg = tp->t_maxseg - bbr->rc_last_options; in bbr_mtu_chg()
14063 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); in bbr_mtu_chg()
14064 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { in bbr_mtu_chg()
14066 if (rsm->r_flags & BBR_ACKED) in bbr_mtu_chg()
14068 if ((rsm->r_end - rsm->r_start) > maxseg) { in bbr_mtu_chg()
14070 * We mark sack-passed on all the previous large in bbr_mtu_chg()
14073 rsm->r_flags |= BBR_SACK_PASSED; in bbr_mtu_chg()
14074 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) && in bbr_mtu_chg()
14075 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) { in bbr_mtu_chg()
14076 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; in bbr_mtu_chg()
14077 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; in bbr_mtu_chg()
14078 rsm->r_flags |= BBR_MARKED_LOST; in bbr_mtu_chg()
14085 bbr->r_ctl.rc_resend = frsm; in bbr_mtu_chg()
14117 tp->t_flags2 |= TF2_CANNOT_DO_ECN; in bbr_switch_failed()
14118 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ; in bbr_switch_failed()
14120 if (tp->t_in_hpts > IHPTS_NONE) { in bbr_switch_failed()
14123 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_switch_failed()
14125 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { in bbr_switch_failed()
14126 if (TSTMP_GT(bbr->rc_pacer_started, cts)) { in bbr_switch_failed()
14127 toval = bbr->rc_pacer_started - cts; in bbr_switch_failed()
14132 } else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { in bbr_switch_failed()
14133 if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { in bbr_switch_failed()
14134 toval = bbr->r_ctl.rc_timer_exp - cts; in bbr_switch_failed()
14166 * socket option arguments. When it re-acquires the lock after the copy, it
14178 switch (sopt->sopt_level) { in bbr_set_sockopt()
14184 switch (sopt->sopt_name) { in bbr_set_sockopt()
14233 if (inp->inp_flags & INP_DROPPED) { in bbr_set_sockopt()
14237 if (tp->t_fb != &__tcp_bbr) { in bbr_set_sockopt()
14241 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_set_sockopt()
14242 switch (sopt->sopt_name) { in bbr_set_sockopt()
14245 bbr->r_ctl.bbr_hptsi_per_second = optval; in bbr_set_sockopt()
14249 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval; in bbr_set_sockopt()
14253 bbr->r_ctl.bbr_hptsi_segments_max = optval; in bbr_set_sockopt()
14257 bbr->r_ctl.bbr_hptsi_bytes_min = optval; in bbr_set_sockopt()
14261 bbr->r_ctl.bbr_cross_over = optval; in bbr_set_sockopt()
14265 if (optval && (bbr->rc_use_google == 0)) { in bbr_set_sockopt()
14273 bbr->r_ctl.bbr_google_discount = optval; in bbr_set_sockopt()
14275 } else if ((optval == 0) && (bbr->rc_use_google == 1)) { in bbr_set_sockopt()
14283 bbr->rc_use_ts_limit = 1; in bbr_set_sockopt()
14285 bbr->rc_use_ts_limit = 0; in bbr_set_sockopt()
14295 bbr->rc_init_win = optval; in bbr_set_sockopt()
14297 if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd)) in bbr_set_sockopt()
14298 tp->snd_cwnd = twin; in bbr_set_sockopt()
14307 bbr->r_ctl.rc_startup_pg = optval; in bbr_set_sockopt()
14308 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { in bbr_set_sockopt()
14309 bbr->r_ctl.rc_bbr_hptsi_gain = optval; in bbr_set_sockopt()
14317 bbr->r_ctl.rc_drain_pg = optval; in bbr_set_sockopt()
14324 reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND)); in bbr_set_sockopt()
14331 bbr->r_ctl.bbr_rttprobe_gain_val = optval; in bbr_set_sockopt()
14338 bbr->r_ctl.rc_probertt_int = optval; in bbr_set_sockopt()
14345 bbr->no_pacing_until = 0; in bbr_set_sockopt()
14346 bbr->rc_no_pacing = 0; in bbr_set_sockopt()
14348 bbr->no_pacing_until = optval; in bbr_set_sockopt()
14349 if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) && in bbr_set_sockopt()
14350 (bbr->rc_bbr_state == BBR_STATE_STARTUP)){ in bbr_set_sockopt()
14352 bbr->rc_no_pacing = 1; in bbr_set_sockopt()
14359 bbr->rc_loss_exit = optval; in bbr_set_sockopt()
14366 bbr->r_ctl.rc_min_rto_ms = optval; in bbr_set_sockopt()
14370 bbr->rc_max_rto_sec = optval; in bbr_set_sockopt()
14373 /* Minimum time between rack t-o's in ms */ in bbr_set_sockopt()
14375 bbr->r_ctl.rc_min_to = optval; in bbr_set_sockopt()
14381 bbr->r_ctl.rc_reorder_shift = optval; in bbr_set_sockopt()
14388 bbr->r_ctl.rc_reorder_fade = optval; in bbr_set_sockopt()
14394 bbr->rc_tlp_threshold = optval; in bbr_set_sockopt()
14400 if (bbr->rc_use_google) { in bbr_set_sockopt()
14406 bbr->bbr_use_rack_cheat = 1; in bbr_set_sockopt()
14408 bbr->bbr_use_rack_cheat = 0; in bbr_set_sockopt()
14413 bbr->r_ctl.bbr_hptsi_segments_floor = optval; in bbr_set_sockopt()
14420 bbr->r_ctl.bbr_utter_max = optval; in bbr_set_sockopt()
14428 bbr->rc_use_idle_restart = 1; in bbr_set_sockopt()
14430 bbr->rc_use_idle_restart = 0; in bbr_set_sockopt()
14435 bbr->bbr_init_win_cheat = 1; in bbr_set_sockopt()
14436 if (bbr->rc_past_init_win == 0) { in bbr_set_sockopt()
14438 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_set_sockopt()
14442 bbr->bbr_init_win_cheat = 0; in bbr_set_sockopt()
14447 bbr->bbr_hdw_pace_ena = 1; in bbr_set_sockopt()
14448 bbr->bbr_attempt_hdwr_pace = 0; in bbr_set_sockopt()
14450 bbr->bbr_hdw_pace_ena = 0; in bbr_set_sockopt()
14452 if (bbr->r_ctl.crte != NULL) { in bbr_set_sockopt()
14453 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp); in bbr_set_sockopt()
14454 bbr->r_ctl.crte = NULL; in bbr_set_sockopt()
14464 tp->t_delayed_ack = 0; in bbr_set_sockopt()
14466 tp->t_delayed_ack = 2; in bbr_set_sockopt()
14468 tp->t_delayed_ack = optval; in bbr_set_sockopt()
14469 if (tp->t_flags & TF_DELACK) { in bbr_set_sockopt()
14470 tp->t_flags &= ~TF_DELACK; in bbr_set_sockopt()
14471 tp->t_flags |= TF_ACKNOW; in bbr_set_sockopt()
14480 /* RACK added ms i.e. rack-rtt + reord + N */ in bbr_set_sockopt()
14482 bbr->r_ctl.rc_pkt_delay = optval; in bbr_set_sockopt()
14488 bbr->rc_resends_use_tso = 1; in bbr_set_sockopt()
14490 bbr->rc_resends_use_tso = 0; in bbr_set_sockopt()
14495 bbr->rc_allow_data_af_clo = 1; in bbr_set_sockopt()
14497 bbr->rc_allow_data_af_clo = 0; in bbr_set_sockopt()
14501 if (bbr->rc_use_google == 0) in bbr_set_sockopt()
14504 bbr->r_use_policer = 1; in bbr_set_sockopt()
14506 bbr->r_use_policer = 0; in bbr_set_sockopt()
14512 bbr->ts_can_raise = 1; in bbr_set_sockopt()
14514 bbr->ts_can_raise = 0; in bbr_set_sockopt()
14518 if (bbr->rc_use_google) { in bbr_set_sockopt()
14522 bbr->r_ctl.rc_incr_tmrs = 1; in bbr_set_sockopt()
14524 bbr->r_ctl.rc_incr_tmrs = 0; in bbr_set_sockopt()
14529 if (bbr->rc_use_google) { in bbr_set_sockopt()
14539 bbr->r_ctl.rc_inc_tcp_oh = 1; in bbr_set_sockopt()
14541 bbr->r_ctl.rc_inc_tcp_oh = 0; in bbr_set_sockopt()
14543 bbr->r_ctl.rc_inc_ip_oh = 1; in bbr_set_sockopt()
14545 bbr->r_ctl.rc_inc_ip_oh = 0; in bbr_set_sockopt()
14547 bbr->r_ctl.rc_inc_enet_oh = 1; in bbr_set_sockopt()
14549 bbr->r_ctl.rc_inc_enet_oh = 0; in bbr_set_sockopt()
14556 tcp_log_socket_option(tp, sopt->sopt_name, optval, error); in bbr_set_sockopt()
14562 * return 0 on success, error-num on failure
14572 bbr = (struct tcp_bbr *)tp->t_fb_ptr; in bbr_get_sockopt()
14583 switch (sopt->sopt_name) { in bbr_get_sockopt()
14585 optval = bbr->r_ctl.bbr_hptsi_per_second; in bbr_get_sockopt()
14588 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar; in bbr_get_sockopt()
14591 optval = bbr->r_ctl.bbr_hptsi_segments_max; in bbr_get_sockopt()
14594 optval = bbr->no_pacing_until; in bbr_get_sockopt()
14597 optval = bbr->r_ctl.bbr_hptsi_bytes_min; in bbr_get_sockopt()
14600 optval = bbr->r_ctl.bbr_cross_over; in bbr_get_sockopt()
14603 optval = bbr->rc_use_google; in bbr_get_sockopt()
14606 optval = bbr->rc_use_ts_limit; in bbr_get_sockopt()
14609 optval = bbr->rc_init_win; in bbr_get_sockopt()
14612 optval = bbr->r_ctl.rc_startup_pg; in bbr_get_sockopt()
14615 optval = bbr->r_ctl.rc_drain_pg; in bbr_get_sockopt()
14618 optval = bbr->r_ctl.rc_probertt_int; in bbr_get_sockopt()
14621 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND); in bbr_get_sockopt()
14624 optval = bbr->r_ctl.bbr_rttprobe_gain_val; in bbr_get_sockopt()
14627 optval = bbr->rc_loss_exit; in bbr_get_sockopt()
14630 loptval = get_filter_value(&bbr->r_ctl.rc_delrate); in bbr_get_sockopt()
14633 optval = bbr->r_ctl.rc_min_rto_ms; in bbr_get_sockopt()
14636 optval = bbr->rc_max_rto_sec; in bbr_get_sockopt()
14640 optval = bbr->r_ctl.rc_pace_max_segs; in bbr_get_sockopt()
14643 /* Minimum time between rack t-o's in ms */ in bbr_get_sockopt()
14644 optval = bbr->r_ctl.rc_min_to; in bbr_get_sockopt()
14648 optval = bbr->r_ctl.rc_reorder_shift; in bbr_get_sockopt()
14652 optval = bbr->r_ctl.rc_reorder_fade; in bbr_get_sockopt()
14656 optval = bbr->bbr_use_rack_cheat; in bbr_get_sockopt()
14659 optval = bbr->r_ctl.bbr_hptsi_segments_floor; in bbr_get_sockopt()
14662 optval = bbr->r_ctl.bbr_utter_max; in bbr_get_sockopt()
14666 optval = bbr->bbr_init_win_cheat; in bbr_get_sockopt()
14669 optval = bbr->rc_use_idle_restart; in bbr_get_sockopt()
14673 optval = bbr->rc_tlp_threshold; in bbr_get_sockopt()
14676 /* RACK added ms i.e. rack-rtt + reord + N */ in bbr_get_sockopt()
14677 optval = bbr->r_ctl.rc_pkt_delay; in bbr_get_sockopt()
14680 optval = bbr->rc_resends_use_tso; in bbr_get_sockopt()
14683 optval = bbr->rc_allow_data_af_clo; in bbr_get_sockopt()
14686 optval = tp->t_delayed_ack; in bbr_get_sockopt()
14689 optval = bbr->bbr_hdw_pace_ena; in bbr_get_sockopt()
14692 optval = bbr->r_use_policer; in bbr_get_sockopt()
14695 optval = bbr->ts_can_raise; in bbr_get_sockopt()
14698 optval = bbr->r_ctl.rc_incr_tmrs; in bbr_get_sockopt()
14702 if (bbr->r_ctl.rc_inc_tcp_oh) in bbr_get_sockopt()
14704 if (bbr->r_ctl.rc_inc_ip_oh) in bbr_get_sockopt()
14706 if (bbr->r_ctl.rc_inc_enet_oh) in bbr_get_sockopt()
14714 if (sopt->sopt_name == TCP_BBR_USEDEL_RATE) in bbr_get_sockopt()
14722 * return 0 on success, error-num on failure
14727 if (sopt->sopt_dir == SOPT_SET) { in bbr_ctloutput()
14729 } else if (sopt->sopt_dir == SOPT_GET) { in bbr_ctloutput()
14732 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir); in bbr_ctloutput()