1 /*-
2  * Copyright (c) 2016-2020 Netflix, Inc.
29  * BBR - Congestion Based Congestion Control
157 /* Should the following be dynamic too -- loss wise */
191 						 * num-measures > min(0) and
225 /* thresholds for reduction on drain in sub-states/drain */
263  * means in non-recovery/retransmission scenarios
264  * cwnd will never be reached by the flight-size.
269 static int32_t bbr_sack_not_required = 0;	/* set to one to allow non-sack to use bbr */
290  * - Yuchung Cheng's RACK TCP (for which its named) that
293  * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
295  * - Van Jacobson's et.al BBR.
317  * TCP output is also over-written with a new version since it
329 static int32_t bbr_reorder_fade = 60000000;	/* 0 - never fade, def
330 						 * 60,000,000 - 60 seconds */
383 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
386 /* What is the max the 0 - bbr_cross_over MBPS TSO target
395 /* What is the min the 0 - bbr_cross-over MBPS  TSO target can be */
399 /* Cross over point from algo-a to algo-b */
515 	return(bbr->rc_bbr_substate);  in bbr_state_val()
523 	mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),  in get_min_cwnd()
524 		  bbr->r_ctl.rc_pace_max_segs);  in get_min_cwnd()
537 	bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;  in bbr_get_persists_timer_val()
538 	if (tp->t_srtt == 0) {  in bbr_get_persists_timer_val()
542 		srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_get_persists_timer_val()
543 		var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT);  in bbr_get_persists_timer_val()
545 	TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]),  in bbr_get_persists_timer_val()
564 	if (bbr->rc_all_timers_stopped) {  in bbr_timer_start()
568 	if (bbr->rc_in_persist) {  in bbr_timer_start()
572 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_timer_start()
574 	    ((tp->t_flags & TF_SACK_PERMIT) == 0) ||  in bbr_timer_start()
575 	    (tp->t_state < TCPS_ESTABLISHED)) {  in bbr_timer_start()
578 		if (SEQ_LT(tp->snd_una, tp->snd_max) ||  in bbr_timer_start()
579 		    sbavail(&tptosocket(tp)->so_snd)) {  in bbr_timer_start()
583 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_timer_start()
585 				idx = rsm->r_rtr_cnt - 1;  in bbr_timer_start()
586 				if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))  in bbr_timer_start()
587 					tstmp_touse = rsm->r_tim_lastsent[idx];  in bbr_timer_start()
589 					tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;  in bbr_timer_start()
591 				    time_since_sent = cts - tstmp_touse;  in bbr_timer_start()
593 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;  in bbr_timer_start()
594 			if (tp->t_srtt == 0)  in bbr_timer_start()
597 				tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) +  in bbr_timer_start()
598 				    ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT);  in bbr_timer_start()
599 			if (tp->t_rxtshift)  in bbr_timer_start()
600 				tov *= tcp_backoff[tp->t_rxtshift];  in bbr_timer_start()
602 				tov -= time_since_sent;  in bbr_timer_start()
604 				tov = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
606 			    (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),  in bbr_timer_start()
607 			    (bbr->rc_max_rto_sec * USECS_IN_SECOND));  in bbr_timer_start()
613 	if (rsm->r_flags & BBR_ACKED) {  in bbr_timer_start()
621 	if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_timer_start()
622 		if ((tp->t_flags & TF_SENTFIN) &&  in bbr_timer_start()
623 		    ((tp->snd_max - tp->snd_una) == 1) &&  in bbr_timer_start()
624 		    (rsm->r_flags & BBR_HAS_FIN)) {  in bbr_timer_start()
633 		idx = rsm->r_rtr_cnt - 1;  in bbr_timer_start()
634 		exp = rsm->r_tim_lastsent[idx] + thresh;  in bbr_timer_start()
636 			to = exp - cts;  in bbr_timer_start()
637 			if (to < bbr->r_ctl.rc_min_to) {  in bbr_timer_start()
638 				to = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
641 			to = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
645 		if (bbr->rc_tlp_in_progress != 0) {  in bbr_timer_start()
651 		rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);  in bbr_timer_start()
656 		if (rsm->r_flags & BBR_HAS_FIN) {  in bbr_timer_start()
662 		idx = rsm->r_rtr_cnt - 1;  in bbr_timer_start()
663 		if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))  in bbr_timer_start()
664 			tstmp_touse = rsm->r_tim_lastsent[idx];  in bbr_timer_start()
666 			tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;  in bbr_timer_start()
668 		    time_since_sent = cts - tstmp_touse;  in bbr_timer_start()
673 			to = thresh - time_since_sent;  in bbr_timer_start()
675 			to = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
676 		if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {  in bbr_timer_start()
683 		if ((bbr->rc_tlp_rtx_out == 1) &&  in bbr_timer_start()
684 		    (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {  in bbr_timer_start()
689 			bbr->rc_tlp_rtx_out = 0;  in bbr_timer_start()
692 		if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {  in bbr_timer_start()
697 			bbr->r_ctl.rc_tlp_seg_send_cnt = 0;  in bbr_timer_start()
698 			bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;  in bbr_timer_start()
703 		bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;  in bbr_timer_start()
706 		if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {  in bbr_timer_start()
714 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;  in bbr_timer_start()
723 	return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);  in bbr_minseg()
742 	if ((tp->t_state == TCPS_CLOSED) ||  in bbr_start_hpts_timer()
743 	    (tp->t_state == TCPS_LISTEN)) {  in bbr_start_hpts_timer()
746 	stopped = bbr->rc_tmr_stopped;  in bbr_start_hpts_timer()
747 	if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {  in bbr_start_hpts_timer()
748 		left = bbr->r_ctl.rc_timer_exp - cts;  in bbr_start_hpts_timer()
750 	bbr->r_ctl.rc_hpts_flags = 0;  in bbr_start_hpts_timer()
751 	bbr->r_ctl.rc_timer_exp = 0;  in bbr_start_hpts_timer()
752 	prev_delay = bbr->r_ctl.rc_last_delay_val;  in bbr_start_hpts_timer()
753 	if (bbr->r_ctl.rc_last_delay_val &&  in bbr_start_hpts_timer()
760 		pacing_delay = bbr->r_ctl.rc_last_delay_val;  in bbr_start_hpts_timer()
761 		if (TSTMP_GT(cts, bbr->rc_pacer_started)) {  in bbr_start_hpts_timer()
763 			delay_calc = cts - bbr->rc_pacer_started;  in bbr_start_hpts_timer()
765 				pacing_delay -= delay_calc;  in bbr_start_hpts_timer()
769 	if (bbr->r_agg_early_set) {  in bbr_start_hpts_timer()
770 …bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, pacing_delay, 0, bbr->r_agg_early_…  in bbr_start_hpts_timer()
771 		pacing_delay += bbr->r_ctl.rc_agg_early;  in bbr_start_hpts_timer()
772 		bbr->r_ctl.rc_agg_early = 0;  in bbr_start_hpts_timer()
773 		bbr->r_agg_early_set = 0;  in bbr_start_hpts_timer()
776 	if (bbr->r_ctl.rc_hptsi_agg_delay) {  in bbr_start_hpts_timer()
777 		if (pacing_delay > bbr->r_ctl.rc_hptsi_agg_delay) {  in bbr_start_hpts_timer()
779 			pacing_delay -= bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_start_hpts_timer()
780 			bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_start_hpts_timer()
783 			bbr->r_ctl.rc_hptsi_agg_delay -= pacing_delay;  in bbr_start_hpts_timer()
784 			bbr->r_ctl.rc_last_delay_val = pacing_delay = 100;  in bbr_start_hpts_timer()
787 	bbr->r_ctl.rc_last_delay_val = pacing_delay;  in bbr_start_hpts_timer()
789 	if (tp->t_flags & TF_DELACK) {  in bbr_start_hpts_timer()
790 		if (bbr->rc_in_persist == 0) {  in bbr_start_hpts_timer()
810 		bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;  in bbr_start_hpts_timer()
816 		bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;  in bbr_start_hpts_timer()
820 	 * wheel, we resort to a keep-alive timer if its configured.  in bbr_start_hpts_timer()
824 		if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_start_hpts_timer()
825 		    (tp->t_state <= TCPS_CLOSING)) {  in bbr_start_hpts_timer()
828 			 * del-ack), we don't have segments being paced. So  in bbr_start_hpts_timer()
831 			if (TCPS_HAVEESTABLISHED(tp->t_state)) {  in bbr_start_hpts_timer()
836 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;  in bbr_start_hpts_timer()
840 	    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {  in bbr_start_hpts_timer()
846 		 * keep-alive, delayed_ack we keep track of what was left  in bbr_start_hpts_timer()
852 	if (bbr->r_ctl.rc_incr_tmrs && pacing_delay &&  in bbr_start_hpts_timer()
853 	    (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {  in bbr_start_hpts_timer()
874 		 * Hack alert for now we can't time-out over 2147 seconds (a  in bbr_start_hpts_timer()
879 		bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;  in bbr_start_hpts_timer()
881 		bbr->r_ctl.rc_timer_exp = 0;  in bbr_start_hpts_timer()
883 	    (bbr->rc_use_google ||  in bbr_start_hpts_timer()
884 	     bbr->output_error_seen ||  in bbr_start_hpts_timer()
890 		bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY;  in bbr_start_hpts_timer()
891 		if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&  in bbr_start_hpts_timer()
892 		    (bbr->rc_cwnd_limited == 0)) {  in bbr_start_hpts_timer()
898 			tp->t_flags2 |= TF2_DONT_SACK_QUEUE;  in bbr_start_hpts_timer()
900 			tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;  in bbr_start_hpts_timer()
901 		bbr->rc_pacer_started = cts;  in bbr_start_hpts_timer()
904 		bbr->rc_timer_first = 0;  in bbr_start_hpts_timer()
905 		bbr->bbr_timer_src = frm;  in bbr_start_hpts_timer()
917 		 * on a keep-alive timer and a request comes in for  in bbr_start_hpts_timer()
921 			bbr->rc_pacer_started = cts;  in bbr_start_hpts_timer()
922 		if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&  in bbr_start_hpts_timer()
923 		    (bbr->rc_cwnd_limited == 0)) {  in bbr_start_hpts_timer()
929 			tp->t_flags2 |= (TF2_MBUF_QUEUE_READY |  in bbr_start_hpts_timer()
933 			tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY |  in bbr_start_hpts_timer()
936 		bbr->bbr_timer_src = frm;  in bbr_start_hpts_timer()
939 		bbr->rc_timer_first = 1;  in bbr_start_hpts_timer()
941 	bbr->rc_tmr_stopped = 0;  in bbr_start_hpts_timer()
958 	tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;  in bbr_timer_audit()
959 	if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))  in bbr_timer_audit()
961 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_timer_audit()
962 	if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&  in bbr_timer_audit()
967 	inp = bbr->rc_inp;  in bbr_timer_audit()
970 		if (tp->t_flags & TF_DELACK) {  in bbr_timer_audit()
978 			    inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_timer_audit()
979 			    (tp->t_state <= TCPS_CLOSING)) &&  in bbr_timer_audit()
981 		    (tp->snd_max == tp->snd_una)) {  in bbr_timer_audit()
986 	if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {  in bbr_timer_audit()
987 		if ((tp->t_flags & TF_SENTFIN) &&  in bbr_timer_audit()
988 		    ((tp->snd_max - tp->snd_una) == 1) &&  in bbr_timer_audit()
989 		    (rsm->r_flags & BBR_HAS_FIN)) {  in bbr_timer_audit()
1002 	} else if (SEQ_GT(tp->snd_max, tp->snd_una) &&  in bbr_timer_audit()
1006 		 * Either a TLP or RXT is fine if no sack-passed is in place  in bbr_timer_audit()
1019 	if (SEQ_GT(tp->snd_max, tp->snd_una) &&  in bbr_timer_audit()
1038 	if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {  in bbr_timer_audit()
1042 		bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,  in bbr_timer_audit()
1047 		 * timer. We don't bother with keep-alive, since when we  in bbr_timer_audit()
1048 		 * jump through the output, it will start the keep-alive if  in bbr_timer_audit()
1051 		 * We only need a delayed-ack added and or the hpts_timeout.  in bbr_timer_audit()
1054 		if (tp->t_flags & TF_DELACK) {  in bbr_timer_audit()
1057 				bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;  in bbr_timer_audit()
1061 				bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;  in bbr_timer_audit()
1067 			bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;  in bbr_timer_audit()
1089 	 * time-stamp taken in theory earlier return the difference. The  in bbr_calc_time()
1094 		return (cts - earlier_time);  in bbr_calc_time()
1107 	return (cts - earlier_time);  in bbr_calc_time()
1117 	if (error || req->newptr == NULL)  in sysctl_bbr_clear_lost()
1142 		printf("Clearing BBR out-size counters\n");  in sysctl_bbr_clear_lost()
1178 	    "How many mss's are outstanding during probe-rtt");  in bbr_init_sysctls()
1183 	    "If RTT has not shrank in this many micro-seconds enter probe-rtt");  in bbr_init_sysctls()
1188 	    "How many microseconds in probe-rtt");  in bbr_init_sysctls()
1203 	    "If we keep setting new low rtt's but delay going in probe-rtt can we force in??");  in bbr_init_sysctls()
1208 	    "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?");  in bbr_init_sysctls()
1213 	    "Can we dynamically adjust the probe-rtt limits and times?");  in bbr_init_sysctls()
1301 	    "For 0 -> 24Mbps what is floor number of segments for TSO");  in bbr_init_sysctls()
1306 	    "For 0 -> 24Mbps what is top number of segments for TSO");  in bbr_init_sysctls()
1388 	    "How many pkt-epoch's (0 is off) do we need before pacing is on?");  in bbr_init_sysctls()
1420 	    "Do we use a pkt-epoch for substate if 0 rttProp?");  in bbr_init_sysctls()
1425 	    "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?");  in bbr_init_sysctls()
1465 	    "How many packet-epochs does the b/w delivery rate last?");  in bbr_init_sysctls()
1470 	    "Does our sub-state drain invoke app limited if its long?");  in bbr_init_sysctls()
1475 	    "Should we set/recover cwnd for sub-state drain?");  in bbr_init_sysctls()
1480 	    "Should we set/recover cwnd for main-state drain?");  in bbr_init_sysctls()
1485 	    "Should we allow google probe-bw/drain to exit early at flight target?");  in bbr_init_sysctls()
1554 	    "What is the high-speed min cwnd (rttProp under 1ms)");  in bbr_init_sysctls()
1647 	    "Maximum RTO in seconds -- should be at least as large as min_rto");  in bbr_init_sysctls()
1704 	    "Do we call the policer detection code from a rack-timeout?");  in bbr_init_sysctls()
1709 	    "What packet epoch do we do false-positive detection at (0=no)?");  in bbr_init_sysctls()
1784 	    "Total number of enobufs for non-hardware paced flows");  in bbr_init_sysctls()
1848 	l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;  in bbr_fill_in_logging_data()
1849 	l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_fill_in_logging_data()
1850 	l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in bbr_fill_in_logging_data()
1851 	l->bw_inuse = bbr_get_bw(bbr);  in bbr_fill_in_logging_data()
1852 	l->inflight = ctf_flight_size(bbr->rc_tp,  in bbr_fill_in_logging_data()
1853 			  (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_fill_in_logging_data()
1854 	l->applimited = bbr->r_ctl.r_app_limited_until;  in bbr_fill_in_logging_data()
1855 	l->delivered = bbr->r_ctl.rc_delivered;  in bbr_fill_in_logging_data()
1856 	l->timeStamp = cts;  in bbr_fill_in_logging_data()
1857 	l->lost = bbr->r_ctl.rc_lost;  in bbr_fill_in_logging_data()
1858 	l->bbr_state = bbr->rc_bbr_state;  in bbr_fill_in_logging_data()
1859 	l->bbr_substate = bbr_state_val(bbr);  in bbr_fill_in_logging_data()
1860 	l->epoch = bbr->r_ctl.rc_rtt_epoch;  in bbr_fill_in_logging_data()
1861 	l->lt_epoch = bbr->r_ctl.rc_lt_epoch;  in bbr_fill_in_logging_data()
1862 	l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;  in bbr_fill_in_logging_data()
1863 	l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;  in bbr_fill_in_logging_data()
1864 	l->inhpts = tcp_in_hpts(bbr->rc_tp);  in bbr_fill_in_logging_data()
1865 	l->use_lt_bw = bbr->rc_lt_use_bw;  in bbr_fill_in_logging_data()
1866 	l->pkts_out = bbr->r_ctl.rc_flight_at_input;  in bbr_fill_in_logging_data()
1867 	l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_fill_in_logging_data()
1873 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bw_reduce()
1876 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_bw_reduce()
1881 		log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;  in bbr_log_type_bw_reduce()
1883 		log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;  in bbr_log_type_bw_reduce()
1885 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bw_reduce()
1886 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bw_reduce()
1887 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bw_reduce()
1889 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_bw_reduce()
1896 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_rwnd_collapse()
1899 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_rwnd_collapse()
1903 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_rwnd_collapse()
1904 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_rwnd_collapse()
1905 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_rwnd_collapse()
1907 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_rwnd_collapse()
1915 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_just_return()
1920 		log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_type_just_return()
1921 		log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;  in bbr_log_type_just_return()
1923 		log.u_bbr.flex5 = bbr->rc_in_persist;  in bbr_log_type_just_return()
1924 		log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;  in bbr_log_type_just_return()
1926 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_type_just_return()
1929 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_just_return()
1930 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_just_return()
1931 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_just_return()
1933 		    tlen, &log, false, &bbr->rc_tv);  in bbr_log_type_just_return()
1940 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_enter_rec()
1943 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_enter_rec()
1945 		log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;  in bbr_log_type_enter_rec()
1946 		log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;  in bbr_log_type_enter_rec()
1947 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_enter_rec()
1948 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_enter_rec()
1949 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_enter_rec()
1951 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_enter_rec()
1967 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_msgsize_fail()
1968 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_msgsize_fail()
1970 		    0, &log, false, &bbr->rc_tv);  in bbr_log_msgsize_fail()
1977 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_flowend()
1982 		if (bbr->rc_inp->inp_socket) {  in bbr_log_flowend()
1983 			r = &bbr->rc_inp->inp_socket->so_rcv;  in bbr_log_flowend()
1984 			s = &bbr->rc_inp->inp_socket->so_snd;  in bbr_log_flowend()
1989 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_flowend()
2000 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_pkt_epoch()
2006 		log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_log_pkt_epoch()
2007 		log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;  in bbr_log_pkt_epoch()
2008 		log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;  in bbr_log_pkt_epoch()
2009 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;  in bbr_log_pkt_epoch()
2012 		log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;  in bbr_log_pkt_epoch()
2013 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_pkt_epoch()
2014 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_pkt_epoch()
2015 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_pkt_epoch()
2017 		    0, &log, false, &bbr->rc_tv);  in bbr_log_pkt_epoch()
2024 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_time_epoch()
2028 		log.u_bbr.flex1 = bbr->r_ctl.rc_lost;  in bbr_log_time_epoch()
2029 		log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;  in bbr_log_time_epoch()
2030 		log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;  in bbr_log_time_epoch()
2032 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_time_epoch()
2033 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_time_epoch()
2034 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_time_epoch()
2036 		    0, &log, false, &bbr->rc_tv);  in bbr_log_time_epoch()
2043 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_set_of_state_target()
2046 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_set_of_state_target()
2047 		log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;  in bbr_log_set_of_state_target()
2050 		log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;  in bbr_log_set_of_state_target()
2052 		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;  in bbr_log_set_of_state_target()
2053 		log.u_bbr.flex7 = bbr->rc_last_options;  in bbr_log_set_of_state_target()
2055 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_set_of_state_target()
2056 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_set_of_state_target()
2057 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_set_of_state_target()
2059 		    0, &log, false, &bbr->rc_tv);  in bbr_log_set_of_state_target()
2067 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_statechange()
2072 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;  in bbr_log_type_statechange()
2073 		log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;  in bbr_log_type_statechange()
2078 		log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;  in bbr_log_type_statechange()
2079 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;  in bbr_log_type_statechange()
2080 		log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);  in bbr_log_type_statechange()
2081 		log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;  in bbr_log_type_statechange()
2082 		log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;  in bbr_log_type_statechange()
2083 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_statechange()
2084 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_statechange()
2085 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_statechange()
2087 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_statechange()
2095 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_rtt_shrinks()
2100 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;  in bbr_log_rtt_shrinks()
2101 		log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;  in bbr_log_rtt_shrinks()
2104 		log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;  in bbr_log_rtt_shrinks()
2107 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_rtt_shrinks()
2108 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_rtt_shrinks()
2109 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_rtt_shrinks()
2111 		    0, &log, false, &bbr->rc_tv);  in bbr_log_rtt_shrinks()
2118 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_exit_rec()
2121 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_exit_rec()
2122 		log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;  in bbr_log_type_exit_rec()
2123 		log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;  in bbr_log_type_exit_rec()
2124 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_type_exit_rec()
2125 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_exit_rec()
2126 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_exit_rec()
2127 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_exit_rec()
2129 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_exit_rec()
2137 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_cwndupd()
2140 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_cwndupd()
2148 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_cwndupd()
2149 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_cwndupd()
2150 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_cwndupd()
2152 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_cwndupd()
2163 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_rtt_sample()
2166 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_rtt_sample()
2168 		log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;  in bbr_log_rtt_sample()
2169 		log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_log_rtt_sample()
2170 		log.u_bbr.flex4 = bbr->rc_tp->ts_offset;  in bbr_log_rtt_sample()
2171 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_rtt_sample()
2172 		log.u_bbr.pkts_out = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_log_rtt_sample()
2175 		log.u_bbr.flex8 = bbr->rc_ack_was_delayed;  in bbr_log_rtt_sample()
2176 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_rtt_sample()
2177 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_rtt_sample()
2178 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_rtt_sample()
2180 		    0, &log, false, &bbr->rc_tv);  in bbr_log_rtt_sample()
2187 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_pesist()
2194 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_pesist()
2195 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_pesist()
2196 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_pesist()
2198 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_pesist()
2204 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_ack_clear()
2208 		log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;  in bbr_log_ack_clear()
2209 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;  in bbr_log_ack_clear()
2210 		log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;  in bbr_log_ack_clear()
2211 		log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;  in bbr_log_ack_clear()
2212 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_ack_clear()
2213 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_ack_clear()
2214 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_ack_clear()
2215 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_ack_clear()
2217 		    0, &log, false, &bbr->rc_tv);  in bbr_log_ack_clear()
2225 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_ack_event()
2231 		log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;  in bbr_log_ack_event()
2235 			log.u_bbr.flex3 = m->m_flags;  in bbr_log_ack_event()
2236 			if (m->m_flags & M_TSTMP) {  in bbr_log_ack_event()
2244 			if (m->m_flags & M_TSTMP_LRO) {  in bbr_log_ack_event()
2259 		log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;  in bbr_log_ack_event()
2260 		log.u_bbr.flex7 = bbr->r_wanted_output;  in bbr_log_ack_event()
2261 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_ack_event()
2262 		TCP_LOG_EVENTP(bbr->rc_tp, th,  in bbr_log_ack_event()
2263 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_ack_event()
2264 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_ack_event()
2266 		    tlen, &log, true, &bbr->rc_tv);  in bbr_log_ack_event()
2273 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_doseg_done()
2279 		log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;  in bbr_log_doseg_done()
2280 		log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_doseg_done()
2281 		log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;  in bbr_log_doseg_done()
2282 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;  in bbr_log_doseg_done()
2283 		log.u_bbr.flex7 = bbr->r_wanted_output;  in bbr_log_doseg_done()
2284 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_doseg_done()
2285 		log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;  in bbr_log_doseg_done()
2286 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_doseg_done()
2287 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_doseg_done()
2288 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_doseg_done()
2290 		    0, &log, true, &bbr->rc_tv);  in bbr_log_doseg_done()
2298 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_enobuf_jmp()
2306 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_enobuf_jmp()
2307 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_enobuf_jmp()
2308 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_enobuf_jmp()
2310 		    len, &log, true, &bbr->rc_tv);  in bbr_log_enobuf_jmp()
2317 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_processing()
2323 		log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;  in bbr_log_to_processing()
2324 		log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_processing()
2326 		log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;  in bbr_log_to_processing()
2328 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_processing()
2329 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_processing()
2330 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_processing()
2332 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_processing()
2339 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_event()
2344 		log.u_bbr.flex1 = bbr->bbr_timer_src;  in bbr_log_to_event()
2346 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_event()
2347 		ar = (uintptr_t)(bbr->r_ctl.rc_resend);  in bbr_log_to_event()
2351 		ar = (uintptr_t)bbr->r_ctl.rc_resend;  in bbr_log_to_event()
2354 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_to_event()
2356 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_event()
2357 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_event()
2358 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_event()
2360 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_event()
2367 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_startup_event()
2375 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_startup_event()
2376 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;  in bbr_log_startup_event()
2378 		log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_log_startup_event()
2379 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_startup_event()
2380 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_startup_event()
2381 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_startup_event()
2383 		    0, &log, false, &bbr->rc_tv);  in bbr_log_startup_event()
2390 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_hpts_diag()
2394 		log.u_bbr.flex1 = diag->p_nxt_slot;  in bbr_log_hpts_diag()
2395 		log.u_bbr.flex2 = diag->p_cur_slot;  in bbr_log_hpts_diag()
2396 		log.u_bbr.flex3 = diag->slot_req;  in bbr_log_hpts_diag()
2397 		log.u_bbr.flex4 = diag->inp_hptsslot;  in bbr_log_hpts_diag()
2398 		log.u_bbr.flex5 = diag->time_remaining;  in bbr_log_hpts_diag()
2399 		log.u_bbr.flex6 = diag->need_new_to;  in bbr_log_hpts_diag()
2400 		log.u_bbr.flex7 = diag->p_hpts_active;  in bbr_log_hpts_diag()
2401 		log.u_bbr.flex8 = diag->p_on_min_sleep;  in bbr_log_hpts_diag()
2403 		log.u_bbr.epoch = diag->have_slept;  in bbr_log_hpts_diag()
2404 		log.u_bbr.lt_epoch = diag->yet_to_sleep;  in bbr_log_hpts_diag()
2405 		log.u_bbr.pkts_out = diag->co_ret;  in bbr_log_hpts_diag()
2406 		log.u_bbr.applimited = diag->hpts_sleep_time;  in bbr_log_hpts_diag()
2407 		log.u_bbr.delivered = diag->p_prev_slot;  in bbr_log_hpts_diag()
2408 		log.u_bbr.inflight = diag->p_runningslot;  in bbr_log_hpts_diag()
2409 		log.u_bbr.bw_inuse = diag->wheel_slot;  in bbr_log_hpts_diag()
2410 		log.u_bbr.rttProp = diag->wheel_cts;  in bbr_log_hpts_diag()
2411 		log.u_bbr.delRate = diag->maxslots;  in bbr_log_hpts_diag()
2412 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_hpts_diag()
2413 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_hpts_diag()
2414 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_hpts_diag()
2416 		    0, &log, false, &bbr->rc_tv);  in bbr_log_hpts_diag()
2424 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_timer_var()
2428 		log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;  in bbr_log_timer_var()
2433 		log.u_bbr.flex6 = bbr->rc_tp->t_srtt;  in bbr_log_timer_var()
2435 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_timer_var()
2436 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_timer_var()
2437 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_timer_var()
2439 		    0, &log, false, &bbr->rc_tv);  in bbr_log_timer_var()
2447 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_pacing_delay_calc()
2462 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_pacing_delay_calc()
2463 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_pacing_delay_calc()
2464 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_pacing_delay_calc()
2466 		    len, &log, false, &bbr->rc_tv);  in bbr_log_pacing_delay_calc()
2473 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_start()
2478 		log.u_bbr.flex1 = bbr->bbr_timer_src;  in bbr_log_to_start()
2480 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_start()
2482 		log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot;  in bbr_log_to_start()
2483 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_to_start()
2484 		log.u_bbr.pkts_out = bbr->rc_tp->t_flags2;  in bbr_log_to_start()
2486 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_start()
2487 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_start()
2488 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_start()
2490 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_start()
2497 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_thresh_choice()
2503 		log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;  in bbr_log_thresh_choice()
2504 		log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];  in bbr_log_thresh_choice()
2505 		log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_thresh_choice()
2507 		log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;  in bbr_log_thresh_choice()
2509 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_thresh_choice()
2510 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_thresh_choice()
2511 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_thresh_choice()
2513 		    0, &log, false, &bbr->rc_tv);  in bbr_log_thresh_choice()
2520 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_cancel()
2525 		log.u_bbr.flex2 = bbr->bbr_timer_src;  in bbr_log_to_cancel()
2526 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_cancel()
2527 		log.u_bbr.flex4 = bbr->rc_in_persist;  in bbr_log_to_cancel()
2528 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_to_cancel()
2529 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_to_cancel()
2531 		log.u_bbr.pkts_out = bbr->rc_pacer_started;  in bbr_log_to_cancel()
2532 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_cancel()
2533 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_cancel()
2534 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_cancel()
2536 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_cancel()
2543 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_tstmp_validation()
2546 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_tstmp_validation()
2547 		log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;  in bbr_log_tstmp_validation()
2552 		log.u_bbr.flex7 = bbr->rc_ts_clock_set;  in bbr_log_tstmp_validation()
2553 		log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;  in bbr_log_tstmp_validation()
2554 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_tstmp_validation()
2555 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_tstmp_validation()
2556 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_tstmp_validation()
2558 		    0, &log, false, &bbr->rc_tv);  in bbr_log_tstmp_validation()
2565 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_tsosize()
2572 		log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;  in bbr_log_type_tsosize()
2575 		log.u_bbr.flex7 = bbr->rc_no_pacing;  in bbr_log_type_tsosize()
2577 		log.u_bbr.flex7 |= bbr->rc_past_init_win;  in bbr_log_type_tsosize()
2579 			log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;  in bbr_log_type_tsosize()
2581 			log.u_bbr.flex8 = bbr->rc_use_google;  in bbr_log_type_tsosize()
2582 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_tsosize()
2583 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_tsosize()
2584 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_tsosize()
2586 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_tsosize()
2594 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_rsmclear()
2599 		log.u_bbr.flex2 = rsm->r_start;  in bbr_log_type_rsmclear()
2600 		log.u_bbr.flex3 = rsm->r_end;  in bbr_log_type_rsmclear()
2601 		log.u_bbr.flex4 = rsm->r_delivered;  in bbr_log_type_rsmclear()
2602 		log.u_bbr.flex5 = rsm->r_rtr_cnt;  in bbr_log_type_rsmclear()
2603 		log.u_bbr.flex6 = rsm->r_dupack;  in bbr_log_type_rsmclear()
2604 		log.u_bbr.flex7 = rsm->r_tim_lastsent[0];  in bbr_log_type_rsmclear()
2605 		log.u_bbr.flex8 = rsm->r_flags;  in bbr_log_type_rsmclear()
2608 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_rsmclear()
2609 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_rsmclear()
2610 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_rsmclear()
2612 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_rsmclear()
2623 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bbrupd()
2637 		if (bbr->rc_ack_was_delayed)  in bbr_log_type_bbrupd()
2638 			log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_log_type_bbrupd()
2641 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bbrupd()
2642 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bbrupd()
2643 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bbrupd()
2645 		    flex2, &log, false, &bbr->rc_tv);  in bbr_log_type_bbrupd()
2654 	if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_ltbw()
2662 		log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;  in bbr_log_type_ltbw()
2663 		log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;  in bbr_log_type_ltbw()
2664 		log.u_bbr.flex7 = bbr->rc_lt_is_sampling;  in bbr_log_type_ltbw()
2666 		log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;  in bbr_log_type_ltbw()
2667 		if (bbr->rc_lt_use_bw == 0)  in bbr_log_type_ltbw()
2668 			log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;  in bbr_log_type_ltbw()
2670 			log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;  in bbr_log_type_ltbw()
2671 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_ltbw()
2672 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_ltbw()
2673 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_ltbw()
2675 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_ltbw()
2682 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_progress_event()
2685 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_progress_event()
2688 		log.u_bbr.flex3 = tp->t_maxunacktime;  in bbr_log_progress_event()
2689 		log.u_bbr.flex4 = tp->t_acktime;  in bbr_log_progress_event()
2691 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_progress_event()
2692 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_progress_event()
2693 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_progress_event()
2695 		    0, &log, false, &bbr->rc_tv);  in bbr_log_progress_event()
2704 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_type_log_hdwr_pacing()
2716 		log.u_bbr.flex8 = bbr->skip_gain;  in bbr_type_log_hdwr_pacing()
2718 		log.u_bbr.flex8 |= bbr->gain_is_limited;  in bbr_type_log_hdwr_pacing()
2720 		log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;  in bbr_type_log_hdwr_pacing()
2721 		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;  in bbr_type_log_hdwr_pacing()
2722 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_type_log_hdwr_pacing()
2723 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_type_log_hdwr_pacing()
2724 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_type_log_hdwr_pacing()
2726 		    0, &log, false, &bbr->rc_tv);  in bbr_type_log_hdwr_pacing()
2733 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bbrsnd()
2741 		log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;  in bbr_log_type_bbrsnd()
2742 		log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_log_type_bbrsnd()
2743 		log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);  in bbr_log_type_bbrsnd()
2744 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_type_bbrsnd()
2745 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bbrsnd()
2746 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bbrsnd()
2747 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bbrsnd()
2749 		    len, &log, false, &bbr->rc_tv);  in bbr_log_type_bbrsnd()
2756 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bbrrttprop()
2760 		log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;  in bbr_log_type_bbrrttprop()
2762 		log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;  in bbr_log_type_bbrrttprop()
2768 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bbrrttprop()
2769 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bbrrttprop()
2770 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bbrrttprop()
2772 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_bbrrttprop()
2779 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_exit_gain()
2783 		log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;  in bbr_log_exit_gain()
2784 		log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_log_exit_gain()
2785 		log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;  in bbr_log_exit_gain()
2786 		log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;  in bbr_log_exit_gain()
2787 		log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;  in bbr_log_exit_gain()
2788 		log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;  in bbr_log_exit_gain()
2791 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_exit_gain()
2792 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_exit_gain()
2793 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_exit_gain()
2795 		    0, &log, false, &bbr->rc_tv);  in bbr_log_exit_gain()
2802 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_settings_change()
2805 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_settings_change()
2806 		/* R-HU */  in bbr_log_settings_change()
2814 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_settings_change()
2815 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_settings_change()
2816 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_settings_change()
2818 		    0, &log, false, &bbr->rc_tv);  in bbr_log_settings_change()
2830 	bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_get_full_bw()
2841 	if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)  in bbr_set_pktepoch()
2842 		lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;  in bbr_set_pktepoch()
2845 	del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;  in bbr_set_pktepoch()
2856 	bbr->r_ctl.rc_pkt_epoch_loss_rate =  (uint32_t)calclr;  in bbr_set_pktepoch()
2857 	if (IN_RECOVERY(bbr->rc_tp->t_flags))  in bbr_set_pktepoch()
2858 		bbr->r_ctl.recovery_lr += (uint32_t)calclr;  in bbr_set_pktepoch()
2859 	bbr->r_ctl.rc_pkt_epoch++;  in bbr_set_pktepoch()
2860 	if (bbr->rc_no_pacing &&  in bbr_set_pktepoch()
2861 	    (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {  in bbr_set_pktepoch()
2862 		bbr->rc_no_pacing = 0;  in bbr_set_pktepoch()
2865 	bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);  in bbr_set_pktepoch()
2866 	bbr->r_ctl.rc_pkt_epoch_time = cts;  in bbr_set_pktepoch()
2869 	bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;  in bbr_set_pktepoch()
2870 	bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;  in bbr_set_pktepoch()
2879 	bbr->r_ctl.rc_rtt_epoch++;  in bbr_set_epoch()
2880 	epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;  in bbr_set_epoch()
2882 	bbr->r_ctl.rc_rcv_epoch_start = cts;  in bbr_set_epoch()
2888 	if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {  in bbr_isit_a_pkt_epoch()
2889 		bbr->rc_is_pkt_epoch_now = 1;  in bbr_isit_a_pkt_epoch()
2913 	if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in __bbr_get_bw()
2915 		rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in __bbr_get_bw()
2918 			min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *  in __bbr_get_bw()
2921 			if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {  in __bbr_get_bw()
2922 				min_bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2925 		} else if (bbr->rc_tp->t_srtt != 0) {  in __bbr_get_bw()
2930 			min_bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2935 	if ((bbr->rc_past_init_win == 0) &&  in __bbr_get_bw()
2936 	    (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))  in __bbr_get_bw()
2937 		bbr->rc_past_init_win = 1;  in __bbr_get_bw()
2938 	if ((bbr->rc_use_google)  && (bbr->r_ctl.r_measurement_count >= 1))  in __bbr_get_bw()
2941 	    ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||  in __bbr_get_bw()
2942 	     (bbr->rc_past_init_win == 0))) {  in __bbr_get_bw()
2946 		rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in __bbr_get_bw()
2953 			bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *  in __bbr_get_bw()
2956 			if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {  in __bbr_get_bw()
2957 				bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2961 			bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2971 	if (bbr->rc_lt_use_bw)  in __bbr_get_bw()
2972 		bw = bbr->r_ctl.rc_lt_bw;  in __bbr_get_bw()
2973 	else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))  in __bbr_get_bw()
2974 		bw = bbr->r_ctl.red_bw;  in __bbr_get_bw()
2976 		bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in __bbr_get_bw()
2998 	bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_reset_lt_bw_interval()
2999 	bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;  in bbr_reset_lt_bw_interval()
3000 	bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;  in bbr_reset_lt_bw_interval()
3001 	bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_reset_lt_bw_interval()
3007 	bbr->rc_lt_is_sampling = 0;  in bbr_reset_lt_bw_sampling()
3008 	bbr->rc_lt_use_bw = 0;  in bbr_reset_lt_bw_sampling()
3009 	bbr->r_ctl.rc_lt_bw = 0;  in bbr_reset_lt_bw_sampling()
3019 	if (bbr->r_ctl.rc_lt_bw) {  in bbr_lt_bw_samp_done()
3021 		if (bbr->r_ctl.rc_lt_bw > bw)  in bbr_lt_bw_samp_done()
3022 			diff = bbr->r_ctl.rc_lt_bw - bw;  in bbr_lt_bw_samp_done()
3024 			diff = bw - bbr->r_ctl.rc_lt_bw;  in bbr_lt_bw_samp_done()
3026 		    (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {  in bbr_lt_bw_samp_done()
3030 			saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;  in bbr_lt_bw_samp_done()
3031 			bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2;	/* average of two */  in bbr_lt_bw_samp_done()
3032 			bbr->rc_lt_use_bw = 1;  in bbr_lt_bw_samp_done()
3033 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_lt_bw_samp_done()
3038 			bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;  in bbr_lt_bw_samp_done()
3047 	bbr->r_ctl.rc_lt_bw = bw;  in bbr_lt_bw_samp_done()
3059 		deduct = bbr->r_ctl.rc_level_state_extra / ran;  in bbr_randomize_extra_state_time()
3060 		bbr->r_ctl.rc_level_state_extra -= deduct;  in bbr_randomize_extra_state_time()
3074 	bbr->r_ctl.rc_exta_time_gd = 0;  in bbr_pick_probebw_substate()
3075 	bbr->rc_hit_state_1 = 0;  in bbr_pick_probebw_substate()
3076 	bbr->r_ctl.rc_level_state_extra = 0;  in bbr_pick_probebw_substate()
3077 	ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1));  in bbr_pick_probebw_substate()
3082 	 * we fully enter the state. Note that the (8 - 1 - ran) assures that  in bbr_pick_probebw_substate()
3083 	 * we return 1 - 7, so we dont return 0 and end up starting in  in bbr_pick_probebw_substate()
3086 	ret_val = BBR_SUBSTATE_COUNT - 1 - ran;  in bbr_pick_probebw_substate()
3088 	if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))  in bbr_pick_probebw_substate()
3091 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_pick_probebw_substate()
3101 	if (bbr->r_use_policer == 0)  in bbr_lt_bw_sampling()
3103 	if (bbr->rc_lt_use_bw) {  in bbr_lt_bw_sampling()
3105 		diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;  in bbr_lt_bw_sampling()
3110 			if (bbr->rc_filled_pipe) {  in bbr_lt_bw_sampling()
3112 				bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_lt_bw_sampling()
3114 				bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_lt_bw_sampling()
3122 				bbr->rc_bbr_state = BBR_STATE_STARTUP;  in bbr_lt_bw_sampling()
3124 				bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_lt_bw_sampling()
3125 				bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_lt_bw_sampling()
3126 				bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;  in bbr_lt_bw_sampling()
3127 				bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;  in bbr_lt_bw_sampling()
3131 			/* reason 0 is to stop using lt-bw */  in bbr_lt_bw_sampling()
3136 			/* Not doing false-positive detection */  in bbr_lt_bw_sampling()
3142 			bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;  in bbr_lt_bw_sampling()
3143 			bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_lt_bw_sampling()
3146 			lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;  in bbr_lt_bw_sampling()
3147 			delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;  in bbr_lt_bw_sampling()
3162 	 * exhaust its tokens and estimate the steady-state rate allowed by  in bbr_lt_bw_sampling()
3164 	 * over-estimate the bw.  in bbr_lt_bw_sampling()
3166 	if (bbr->rc_lt_is_sampling == 0) {  in bbr_lt_bw_sampling()
3171 		bbr->rc_lt_is_sampling = 1;  in bbr_lt_bw_sampling()
3176 	if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))  in bbr_lt_bw_sampling()
3177 		d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;  in bbr_lt_bw_sampling()
3182 	if (bbr->r_ctl.r_app_limited_until) {  in bbr_lt_bw_sampling()
3183 		/* Can not measure in app-limited state */  in bbr_lt_bw_sampling()
3189 	diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;  in bbr_lt_bw_sampling()
3196 		/* 6 is not_enough time or no-loss */  in bbr_lt_bw_sampling()
3216 	 * tokens are exhausted under-estimates the policed rate.  in bbr_lt_bw_sampling()
3219 		/* 6 is not_enough time or no-loss */  in bbr_lt_bw_sampling()
3224 	lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;  in bbr_lt_bw_sampling()
3225 	delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;  in bbr_lt_bw_sampling()
3233 		/* 6 is not_enough time or no-loss */  in bbr_lt_bw_sampling()
3262 		bbr->r_ctl.rc_num_maps_alloced++;  in bbr_alloc()
3265 	if (bbr->r_ctl.rc_free_cnt) {  in bbr_alloc()
3267 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);  in bbr_alloc()
3268 		TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_alloc()
3269 		bbr->r_ctl.rc_free_cnt--;  in bbr_alloc()
3280 	    (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {  in bbr_alloc_full_limit()
3282 		if (!bbr->alloc_limit_reported) {  in bbr_alloc_full_limit()
3283 			bbr->alloc_limit_reported = 1;  in bbr_alloc_full_limit()
3300 		    bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {  in bbr_alloc_limit()
3302 			if (!bbr->alloc_limit_reported) {  in bbr_alloc_limit()
3303 				bbr->alloc_limit_reported = 1;  in bbr_alloc_limit()
3313 		rsm->r_limit_type = limit_type;  in bbr_alloc_limit()
3314 		bbr->r_ctl.rc_num_split_allocs++;  in bbr_alloc_limit()
3322 	if (rsm->r_limit_type) {  in bbr_free()
3324 		bbr->r_ctl.rc_num_split_allocs--;  in bbr_free()
3326 	if (rsm->r_is_smallmap)  in bbr_free()
3327 		bbr->r_ctl.rc_num_small_maps_alloced--;  in bbr_free()
3328 	if (bbr->r_ctl.rc_tlp_send == rsm)  in bbr_free()
3329 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_free()
3330 	if (bbr->r_ctl.rc_resend == rsm) {  in bbr_free()
3331 		bbr->r_ctl.rc_resend = NULL;  in bbr_free()
3333 	if (bbr->r_ctl.rc_next == rsm)  in bbr_free()
3334 		bbr->r_ctl.rc_next = NULL;  in bbr_free()
3335 	if (bbr->r_ctl.rc_sacklast == rsm)  in bbr_free()
3336 		bbr->r_ctl.rc_sacklast = NULL;  in bbr_free()
3337 	if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {  in bbr_free()
3339 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_free()
3340 		rsm->r_limit_type = 0;  in bbr_free()
3341 		bbr->r_ctl.rc_free_cnt++;  in bbr_free()
3344 	bbr->r_ctl.rc_num_maps_alloced--;  in bbr_free()
3377 	if (bbr->rc_init_win) {  in bbr_initial_cwnd()
3378 		i_cwnd = bbr->rc_init_win * tp->t_maxseg;  in bbr_initial_cwnd()
3380 		i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),  in bbr_initial_cwnd()
3381 		    max(2 * tp->t_maxseg, 14600));  in bbr_initial_cwnd()
3383 		i_cwnd = min(4 * tp->t_maxseg,  in bbr_initial_cwnd()
3384 		    max(2 * tp->t_maxseg, 4380));  in bbr_initial_cwnd()
3387 		if (tp->t_maxseg > 2190)  in bbr_initial_cwnd()
3388 			i_cwnd = 2 * tp->t_maxseg;  in bbr_initial_cwnd()
3389 		else if (tp->t_maxseg > 1095)  in bbr_initial_cwnd()
3390 			i_cwnd = 3 * tp->t_maxseg;  in bbr_initial_cwnd()
3392 			i_cwnd = 4 * tp->t_maxseg;  in bbr_initial_cwnd()
3407 	if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||  in bbr_get_raw_target_cwnd()
3410 		return (bbr_initial_cwnd(bbr, bbr->rc_tp));  in bbr_get_raw_target_cwnd()
3420 	cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT));  in bbr_get_raw_target_cwnd()
3430 	mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);  in bbr_get_target_cwnd()
3435 	 * fq layer to trap packets in) quanta's per the I-D  in bbr_get_target_cwnd()
3438 	cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);  in bbr_get_target_cwnd()
3439 	if (bbr->rc_use_google) {  in bbr_get_target_cwnd()
3440 		if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&  in bbr_get_target_cwnd()
3445 			 * is documented no-where except in the code.  in bbr_get_target_cwnd()
3475 	if (bbr->r_ctl.rc_inc_tcp_oh) {  in bbr_get_header_oh()
3477 		seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));  in bbr_get_header_oh()
3479 	if (bbr->r_ctl.rc_inc_ip_oh) {  in bbr_get_header_oh()
3482 		if (bbr->r_is_v6) {  in bbr_get_header_oh()
3493 	if (bbr->r_ctl.rc_inc_enet_oh) {  in bbr_get_header_oh()
3531 	maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in bbr_get_pacing_delay()
3532 	num_segs = (len + maxseg - 1) / maxseg;  in bbr_get_pacing_delay()
3533 	if (bbr->rc_use_google == 0) {  in bbr_get_pacing_delay()
3539 	if (bbr->rc_use_google) {  in bbr_get_pacing_delay()
3546 		cbw = bw *  (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);  in bbr_get_pacing_delay()
3561 	    (bbr->rc_use_google == 0) &&  in bbr_get_pacing_delay()
3586 	if ((tp->t_flags & TF_GPUTINPROG) &&  in bbr_ack_received()
3587 	    SEQ_GEQ(th->th_ack, tp->gput_ack)) {  in bbr_ack_received()
3597 		gput = (int64_t) (th->th_ack - tp->gput_seq) * 8;  in bbr_ack_received()
3598 		time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));  in bbr_ack_received()
3600 		stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,  in bbr_ack_received()
3602 		if (tp->t_stats_gput_prev > 0)  in bbr_ack_received()
3603 			stats_voi_update_abs_s32(tp->t_stats,  in bbr_ack_received()
3605 						 ((gput - tp->t_stats_gput_prev) * 100) /  in bbr_ack_received()
3606 						 tp->t_stats_gput_prev);  in bbr_ack_received()
3607 		tp->t_flags &= ~TF_GPUTINPROG;  in bbr_ack_received()
3608 		tp->t_stats_gput_prev = cgput;  in bbr_ack_received()
3611 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&  in bbr_ack_received()
3612 	    ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {  in bbr_ack_received()
3613 		/* We don't change anything in probe-rtt */  in bbr_ack_received()
3616 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_ack_received()
3620 		bytes_this_ack -= prev_acked;  in bbr_ack_received()
3625 		if ((bytes_this_ack < maxseg) && bbr->rc_use_google)  in bbr_ack_received()
3631 	cwnd = tp->snd_cwnd;  in bbr_ack_received()
3632 	bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_ack_received()
3636 						  (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);  in bbr_ack_received()
3638 		target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);  in bbr_ack_received()
3639 	if (IN_RECOVERY(tp->t_flags) &&  in bbr_ack_received()
3640 	    (bbr->bbr_prev_in_rec == 0)) {  in bbr_ack_received()
3645 		bbr->pkt_conservation = 1;  in bbr_ack_received()
3646 		bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;  in bbr_ack_received()
3648 				       (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_ack_received()
3651 	if (IN_RECOVERY(tp->t_flags)) {  in bbr_ack_received()
3654 		bbr->bbr_prev_in_rec = 1;  in bbr_ack_received()
3656 			cwnd -= losses;  in bbr_ack_received()
3662 					 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_ack_received()
3665 		if (bbr->pkt_conservation) {  in bbr_ack_received()
3668 			if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))  in bbr_ack_received()
3669 				time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;  in bbr_ack_received()
3675 				bbr->pkt_conservation = 0;  in bbr_ack_received()
3681 				tp->snd_cwnd = cwnd;  in bbr_ack_received()
3683 						     prev_acked, 1, target_cwnd, th->th_ack, line);  in bbr_ack_received()
3688 		bbr->bbr_prev_in_rec = 0;  in bbr_ack_received()
3689 	if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {  in bbr_ack_received()
3690 		bbr->r_ctl.restrict_growth--;  in bbr_ack_received()
3694 	if (bbr->rc_filled_pipe) {  in bbr_ack_received()
3706 		else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)  in bbr_ack_received()
3714 		    (bbr->rc_past_init_win == 0)) {  in bbr_ack_received()
3725 	tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));  in bbr_ack_received()
3726 …bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, li…  in bbr_ack_received()
3734 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in tcp_bbr_partialack()
3737 		(bbr->r_ctl.rc_sacked  + bbr->r_ctl.rc_lost_bytes)) <=  in tcp_bbr_partialack()
3738 	    tp->snd_cwnd) {  in tcp_bbr_partialack()
3739 		bbr->r_wanted_output = 1;  in tcp_bbr_partialack()
3750 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_post_recovery()
3754 	EXIT_RECOVERY(tp->t_flags);  in bbr_post_recovery()
3755 	/* Lock in our b/w reduction for the specified number of pkt-epochs */  in bbr_post_recovery()
3756 	bbr->r_recovery_bw = 0;  in bbr_post_recovery()
3757 	tp->snd_recover = tp->snd_una;  in bbr_post_recovery()
3758 	tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_post_recovery()
3759 	bbr->pkt_conservation = 0;  in bbr_post_recovery()
3760 	if (bbr->rc_use_google == 0) {  in bbr_post_recovery()
3762 		 * For non-google mode lets  in bbr_post_recovery()
3768 		bbr->bbr_prev_in_rec = 0;  in bbr_post_recovery()
3771 	if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {  in bbr_post_recovery()
3772 		tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);  in bbr_post_recovery()
3775 		/* For probe-rtt case lets fix up its saved_cwnd */  in bbr_post_recovery()
3776 		if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {  in bbr_post_recovery()
3777 			bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;  in bbr_post_recovery()
3782 		     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_post_recovery()
3783 	if ((bbr->rc_use_google == 0) &&  in bbr_post_recovery()
3797 				     bbr->r_ctl.recovery_lr, 21,  in bbr_post_recovery()
3799 				     bbr->r_ctl.rc_red_cwnd_pe,  in bbr_post_recovery()
3803 		if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&  in bbr_post_recovery()
3806 		     (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&  in bbr_post_recovery()
3807 		     bbr->rc_hit_state_1 &&  in bbr_post_recovery()
3809 		    ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&  in bbr_post_recovery()
3815 			cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_post_recovery()
3816 			cwnd_p = &bbr->r_ctl.rc_saved_cwnd;  in bbr_post_recovery()
3818  			cwnd = tp->snd_cwnd;  in bbr_post_recovery()
3819 			cwnd_p = &tp->snd_cwnd;  in bbr_post_recovery()
3821 		maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_post_recovery()
3823 		if (bbr->r_ctl.rc_lost == 0)  in bbr_post_recovery()
3825 		else if (bbr->r_ctl.rc_delivered == 0)  in bbr_post_recovery()
3828 			lr2use = (uint64_t)bbr->r_ctl.rc_lost * (uint64_t)1000;  in bbr_post_recovery()
3829 			lr2use /= bbr->r_ctl.rc_delivered;  in bbr_post_recovery()
3831 		lr2use += bbr->r_ctl.recovery_lr;  in bbr_post_recovery()
3838 			    bbr->r_ctl.restrict_growth += acks_inflight;  in bbr_post_recovery()
3844 				newcwnd = roundup((cwnd - val), maxseg);  in bbr_post_recovery()
3860 				newcwnd = (get_min_cwnd(bbr) - acks_inflight);  in bbr_post_recovery()
3872 		if (tp->snd_cwnd > newcwnd)  in bbr_post_recovery()
3873 			tp->snd_cwnd = newcwnd;  in bbr_post_recovery()
3877 		bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;  in bbr_post_recovery()
3880 	bbr->r_ctl.recovery_lr = 0;  in bbr_post_recovery()
3881 	if (flight <= tp->snd_cwnd) {  in bbr_post_recovery()
3882 		bbr->r_wanted_output = 1;  in bbr_post_recovery()
3884 	tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_post_recovery()
3890 	bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_setup_red_bw()
3892 	if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)  in bbr_setup_red_bw()
3893 		bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;  in bbr_setup_red_bw()
3894 	if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))  in bbr_setup_red_bw()
3895 		bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;  in bbr_setup_red_bw()
3906 	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);  in bbr_cong_signal()
3908 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_cong_signal()
3911 		if (!IN_RECOVERY(tp->t_flags)) {  in bbr_cong_signal()
3912 			tp->snd_recover = tp->snd_max;  in bbr_cong_signal()
3914 			bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_cong_signal()
3915 			if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {  in bbr_cong_signal()
3921 				bbr->r_ctl.rc_lt_epoch++;  in bbr_cong_signal()
3923 			if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in bbr_cong_signal()
3930 				bbr->r_ctl.rc_bbr_last_startup_epoch++;  in bbr_cong_signal()
3932 			bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;  in bbr_cong_signal()
3933 			ENTER_RECOVERY(tp->t_flags);  in bbr_cong_signal()
3934 			bbr->rc_tlp_rtx_out = 0;  in bbr_cong_signal()
3935 			bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;  in bbr_cong_signal()
3936 			tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_cong_signal()
3937 			if (tcp_in_hpts(bbr->rc_tp) &&  in bbr_cong_signal()
3938 			    ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {  in bbr_cong_signal()
3945 				bbr->rc_timer_first = 1;  in bbr_cong_signal()
3946 				bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_cong_signal()
3954 			if ((bbr->rc_use_google == 0) &&  in bbr_cong_signal()
3955 			    (bbr->r_ctl.bbr_rttprobe_gain_val ||  in bbr_cong_signal()
3956 			     (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {  in bbr_cong_signal()
3957 				tp->snd_cwnd = ctf_flight_size(tp,  in bbr_cong_signal()
3958 					           (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_cong_signal()
3959 					(tp->t_maxseg - bbr->rc_last_options);  in bbr_cong_signal()
3960 				if (tp->snd_cwnd < get_min_cwnd(bbr)) {  in bbr_cong_signal()
3962 					tp->snd_cwnd = get_min_cwnd(bbr);  in bbr_cong_signal()
3966 			bbr_log_type_enter_rec(bbr, rsm->r_start);  in bbr_cong_signal()
3972 		bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_cong_signal()
3973 		if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {  in bbr_cong_signal()
3974 			tp->snd_cwnd = tp->snd_cwnd_prev;  in bbr_cong_signal()
3975 			tp->snd_ssthresh = tp->snd_ssthresh_prev;  in bbr_cong_signal()
3976 			tp->snd_recover = tp->snd_recover_prev;  in bbr_cong_signal()
3977 			tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);  in bbr_cong_signal()
3980 		tp->t_badrxtwin = 0;  in bbr_cong_signal()
3988  *	- There is no delayed ack timer in progress.
3989  *	- Our last ack wasn't a 0-sized window. We never want to delay
3990  *	  the ack that opens up a 0-sized window.
3991  *	- LRO wasn't used for this segment. We make sure by checking that the
3993  *	- Delayed acks are enabled or this is a half-synchronized T/TCP
3995  *	- The data being acked is less than a full segment (a stretch ack
3997  *      - nsegs is 1 (if its more than that we received more than 1 ack).
4000 	(((tp->t_flags & TF_RXWIN0SENT) == 0) &&		\
4001 	 ((tp->t_flags & TF_DELACK) == 0) && 		 	\
4002 	 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) &&	\
4003 	 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4018 	 * Walk the time-order transmitted list looking for an rsm that is  in bbr_find_lowest_rsm()
4022 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {  in bbr_find_lowest_rsm()
4023 		if (rsm->r_flags & BBR_ACKED) {  in bbr_find_lowest_rsm()
4043 	TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_find_high_nonack()
4044 		if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) {  in bbr_find_high_nonack()
4065 	 * If reorder-fade is configured, then we track the last time we saw  in bbr_calc_thresh_rack()
4066 	 * re-ordering occur. If we reach the point where enough time as  in bbr_calc_thresh_rack()
4069 	 * Or if reorder-face is 0, then once we see reordering we consider  in bbr_calc_thresh_rack()
4073 	 * In the end if lro is non-zero we add the extra time for  in bbr_calc_thresh_rack()
4081 	if (bbr->r_ctl.rc_reorder_ts) {  in bbr_calc_thresh_rack()
4082 		if (bbr->r_ctl.rc_reorder_fade) {  in bbr_calc_thresh_rack()
4083 			if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {  in bbr_calc_thresh_rack()
4084 				lro = cts - bbr->r_ctl.rc_reorder_ts;  in bbr_calc_thresh_rack()
4096 			if (lro > bbr->r_ctl.rc_reorder_fade) {  in bbr_calc_thresh_rack()
4098 				bbr->r_ctl.rc_reorder_ts = 0;  in bbr_calc_thresh_rack()
4108 	thresh = srtt + bbr->r_ctl.rc_pkt_delay;  in bbr_calc_thresh_rack()
4111 		if (bbr->r_ctl.rc_reorder_shift)  in bbr_calc_thresh_rack()
4112 			thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);  in bbr_calc_thresh_rack()
4119 	if ((bbr->rc_tp)->t_srtt == 0)  in bbr_calc_thresh_rack()
4122 		t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_calc_thresh_rack()
4127 	if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {  in bbr_calc_thresh_rack()
4128 		thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);  in bbr_calc_thresh_rack()
4135  * Return to the caller the amount of time in mico-seconds
4149 	if (bbr->rc_tlp_threshold)  in bbr_calc_thresh_tlp()
4150 		thresh = srtt + (srtt / bbr->rc_tlp_threshold);  in bbr_calc_thresh_tlp()
4153 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_calc_thresh_tlp()
4155 	len = rsm->r_end - rsm->r_start;  in bbr_calc_thresh_tlp()
4162 		 * possible inter-packet delay (if any).  in bbr_calc_thresh_tlp()
4167 		idx = rsm->r_rtr_cnt - 1;  in bbr_calc_thresh_tlp()
4168 		nidx = prsm->r_rtr_cnt - 1;  in bbr_calc_thresh_tlp()
4169 		if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {  in bbr_calc_thresh_tlp()
4171 			inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];  in bbr_calc_thresh_tlp()
4176 		 * Possibly compensate for delayed-ack.  in bbr_calc_thresh_tlp()
4185 	if (tp->t_srtt == 0)  in bbr_calc_thresh_tlp()
4188 		t_rxtcur = TICKS_2_USEC(tp->t_rxtcur);  in bbr_calc_thresh_tlp()
4196 	if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {  in bbr_calc_thresh_tlp()
4197 		thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);  in bbr_calc_thresh_tlp()
4215 	f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in bbr_get_rtt()
4216 	if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {  in bbr_get_rtt()
4218 		if (bbr->rc_tp->t_srtt == 0)  in bbr_get_rtt()
4221 			f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_get_rtt()
4224 		 * delayed-ack min  in bbr_get_rtt()
4230 	/* Take the filter version or last measured pkt-rtt */  in bbr_get_rtt()
4234 		if (bbr->r_ctl.rc_pkt_epoch_rtt) {  in bbr_get_rtt()
4235 			srtt = bbr->r_ctl.rc_pkt_epoch_rtt;  in bbr_get_rtt()
4241 		srtt = bbr->r_ctl.rc_last_rtt;  in bbr_get_rtt()
4243 		if (bbr->rc_ack_was_delayed)  in bbr_get_rtt()
4244 			srtt += bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_get_rtt()
4246 		srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_get_rtt()
4264 	if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {  in bbr_is_lost()
4286 	if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {  in bbr_check_recovery_mode()
4290 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_check_recovery_mode()
4295 	if (tp->t_flags & TF_SENTFIN) {  in bbr_check_recovery_mode()
4299 	if (rsm->r_flags & BBR_ACKED) {  in bbr_check_recovery_mode()
4308 	idx = rsm->r_rtr_cnt - 1;  in bbr_check_recovery_mode()
4309 	if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {  in bbr_check_recovery_mode()
4315 	    ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||  in bbr_check_recovery_mode()
4316 	     (rsm->r_flags & BBR_SACK_PASSED))) {  in bbr_check_recovery_mode()
4317 		if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {  in bbr_check_recovery_mode()
4318 			rsm->r_flags |= BBR_MARKED_LOST;  in bbr_check_recovery_mode()
4319 			bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;  in bbr_check_recovery_mode()
4320 			bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;  in bbr_check_recovery_mode()
4324 		if ((rsm->r_end - rsm->r_start) == 0)  in bbr_check_recovery_mode()
4345 	 * retransmissions, if so we will enter fast-recovery. The output  in bbr_timeout_rack()
4351 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_rack()
4354 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {  in bbr_timeout_rack()
4359 	lost = bbr->r_ctl.rc_lost;  in bbr_timeout_rack()
4360 	if (bbr->r_state && (bbr->r_state != tp->t_state))  in bbr_timeout_rack()
4363 	if (bbr->r_ctl.rc_resend == NULL) {  in bbr_timeout_rack()
4365 		bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);  in bbr_timeout_rack()
4368 		bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));  in bbr_timeout_rack()
4369 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;  in bbr_timeout_rack()
4378 	nrsm->r_start = start;  in bbr_clone_rsm()
4379 	nrsm->r_end = rsm->r_end;  in bbr_clone_rsm()
4380 	nrsm->r_rtr_cnt = rsm->r_rtr_cnt;  in bbr_clone_rsm()
4381 	nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed;  in bbr_clone_rsm()
4382 	nrsm->r_flags = rsm->r_flags;  in bbr_clone_rsm()
4384 	nrsm->r_flags &= ~BBR_HAS_SYN;  in bbr_clone_rsm()
4386 	rsm->r_flags &= ~BBR_HAS_FIN;  in bbr_clone_rsm()
4387 	nrsm->r_dupack = rsm->r_dupack;  in bbr_clone_rsm()
4388 	nrsm->r_rtr_bytes = 0;  in bbr_clone_rsm()
4389 	nrsm->r_is_gain = rsm->r_is_gain;  in bbr_clone_rsm()
4390 	nrsm->r_is_drain = rsm->r_is_drain;  in bbr_clone_rsm()
4391 	nrsm->r_delivered = rsm->r_delivered;  in bbr_clone_rsm()
4392 	nrsm->r_ts_valid = rsm->r_ts_valid;  in bbr_clone_rsm()
4393 	nrsm->r_del_ack_ts = rsm->r_del_ack_ts;  in bbr_clone_rsm()
4394 	nrsm->r_del_time = rsm->r_del_time;  in bbr_clone_rsm()
4395 	nrsm->r_app_limited = rsm->r_app_limited;  in bbr_clone_rsm()
4396 	nrsm->r_first_sent_time = rsm->r_first_sent_time;  in bbr_clone_rsm()
4397 	nrsm->r_flight_at_send = rsm->r_flight_at_send;  in bbr_clone_rsm()
4399 	nrsm->r_bbr_state = rsm->r_bbr_state;  in bbr_clone_rsm()
4400 	for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {  in bbr_clone_rsm()
4401 		nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];  in bbr_clone_rsm()
4403 	rsm->r_end = nrsm->r_start;  in bbr_clone_rsm()
4404 	idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);  in bbr_clone_rsm()
4407 	if ((rsm->r_is_smallmap == 0) &&  in bbr_clone_rsm()
4408 	    ((rsm->r_end - rsm->r_start) <= idx)) {  in bbr_clone_rsm()
4409 		bbr->r_ctl.rc_num_small_maps_alloced++;  in bbr_clone_rsm()
4410 		rsm->r_is_smallmap = 1;  in bbr_clone_rsm()
4413 	if ((nrsm->r_end - nrsm->r_start) <= idx) {  in bbr_clone_rsm()
4414 		bbr->r_ctl.rc_num_small_maps_alloced++;  in bbr_clone_rsm()
4415 		nrsm->r_is_smallmap = 1;  in bbr_clone_rsm()
4438 	if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) {  in bbr_sack_mergable()
4440 		if ((l_rsm->r_end == start) ||  in bbr_sack_mergable()
4441 		    (SEQ_LT(start, l_rsm->r_end) &&  in bbr_sack_mergable()
4442 		     SEQ_GT(end, l_rsm->r_end))) {  in bbr_sack_mergable()
4444 			     * map blk   |------|  in bbr_sack_mergable()
4445 			     * sack blk         |------|  in bbr_sack_mergable()
4447 			     * map blk   |------|  in bbr_sack_mergable()
4448 			     * sack blk      |------|  in bbr_sack_mergable()
4453 	if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) {  in bbr_sack_mergable()
4455 		if ((r_rsm->r_start == end) ||  in bbr_sack_mergable()
4456 		    (SEQ_LT(start, r_rsm->r_start) &&  in bbr_sack_mergable()
4457 		     SEQ_GT(end, r_rsm->r_start))) {  in bbr_sack_mergable()
4459 			 * map blk          |---------|  in bbr_sack_mergable()
4460 			 * sack blk    |----|  in bbr_sack_mergable()
4462 			 * map blk          |---------|  in bbr_sack_mergable()
4463 			 * sack blk    |-------|  in bbr_sack_mergable()
4486 	l_rsm->r_end = r_rsm->r_end;  in bbr_merge_rsm()
4487 	if (l_rsm->r_dupack < r_rsm->r_dupack)  in bbr_merge_rsm()
4488 		l_rsm->r_dupack = r_rsm->r_dupack;  in bbr_merge_rsm()
4489 	if (r_rsm->r_rtr_bytes)  in bbr_merge_rsm()
4490 		l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;  in bbr_merge_rsm()
4491 	if (r_rsm->r_in_tmap) {  in bbr_merge_rsm()
4493 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);  in bbr_merge_rsm()
4495 	if (r_rsm->r_app_limited)  in bbr_merge_rsm()
4496 		l_rsm->r_app_limited = r_rsm->r_app_limited;  in bbr_merge_rsm()
4498 	if (r_rsm->r_flags & BBR_HAS_FIN)  in bbr_merge_rsm()
4499 		l_rsm->r_flags |= BBR_HAS_FIN;  in bbr_merge_rsm()
4500 	if (r_rsm->r_flags & BBR_TLP)  in bbr_merge_rsm()
4501 		l_rsm->r_flags |= BBR_TLP;  in bbr_merge_rsm()
4502 	if (r_rsm->r_flags & BBR_RWND_COLLAPSED)  in bbr_merge_rsm()
4503 		l_rsm->r_flags |= BBR_RWND_COLLAPSED;  in bbr_merge_rsm()
4504 	if (r_rsm->r_flags & BBR_MARKED_LOST) {  in bbr_merge_rsm()
4506 		bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;  in bbr_merge_rsm()
4508 	TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);  in bbr_merge_rsm()
4509 	if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {  in bbr_merge_rsm()
4511 		r_rsm->r_limit_type = l_rsm->r_limit_type;  in bbr_merge_rsm()
4512 		l_rsm->r_limit_type = 0;  in bbr_merge_rsm()
4539 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_tlp()
4542 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {  in bbr_timeout_tlp()
4548 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_tlp()
4551 	if (bbr->rc_in_persist) {  in bbr_timeout_tlp()
4554 	if (bbr->r_state && (bbr->r_state != tp->t_state))  in bbr_timeout_tlp()
4557 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_tlp()
4563 	avail = sbavail(&so->so_snd);  in bbr_timeout_tlp()
4565 	if (out > tp->snd_wnd) {  in bbr_timeout_tlp()
4572 		amm = avail - out;  in bbr_timeout_tlp()
4575 		} else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {  in bbr_timeout_tlp()
4576 			/* not enough to fill a MTU and no-delay is off */  in bbr_timeout_tlp()
4579 		/* Set the send-new override */  in bbr_timeout_tlp()
4580 		if ((out + amm) <= tp->snd_wnd) {  in bbr_timeout_tlp()
4581 			bbr->rc_tlp_new_data = 1;  in bbr_timeout_tlp()
4585 		bbr->r_ctl.rc_tlp_seg_send_cnt = 0;  in bbr_timeout_tlp()
4586 		bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;  in bbr_timeout_tlp()
4587 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_timeout_tlp()
4594 	 * Ok we need to arrange the last un-acked segment to be re-sent, or  in bbr_timeout_tlp()
4595 	 * optionally the first un-acked segment.  in bbr_timeout_tlp()
4598 		rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);  in bbr_timeout_tlp()
4599 		if (rsm && (rsm->r_flags & (BBR_ACKED | BBR_HAS_FIN))) {  in bbr_timeout_tlp()
4610 		TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_timeout_tlp()
4611 			if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {  in bbr_timeout_tlp()
4618 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_timeout_tlp()
4623 	if ((rsm->r_end - rsm->r_start) > maxseg) {  in bbr_timeout_tlp()
4634 			 * do the large send (BTLP :-) ).  in bbr_timeout_tlp()
4638 		bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));  in bbr_timeout_tlp()
4639 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_timeout_tlp()
4640 		if (rsm->r_in_tmap) {  in bbr_timeout_tlp()
4641 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_timeout_tlp()
4642 			nrsm->r_in_tmap = 1;  in bbr_timeout_tlp()
4644 		rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_timeout_tlp()
4648 	bbr->r_ctl.rc_tlp_send = rsm;  in bbr_timeout_tlp()
4649 	bbr->rc_tlp_rtx_out = 1;  in bbr_timeout_tlp()
4650 	if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {  in bbr_timeout_tlp()
4651 		bbr->r_ctl.rc_tlp_seg_send_cnt++;  in bbr_timeout_tlp()
4652 		tp->t_rxtshift++;  in bbr_timeout_tlp()
4654 		bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;  in bbr_timeout_tlp()
4655 		bbr->r_ctl.rc_tlp_seg_send_cnt = 1;  in bbr_timeout_tlp()
4658 	if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {  in bbr_timeout_tlp()
4664 		bbr->rc_tlp_new_data = 0;  in bbr_timeout_tlp()
4665 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_timeout_tlp()
4667 			rsm->r_flags &= ~BBR_TLP;  in bbr_timeout_tlp()
4671 		rsm->r_flags |= BBR_TLP;  in bbr_timeout_tlp()
4673 	if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&  in bbr_timeout_tlp()
4674 	    (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {  in bbr_timeout_tlp()
4682 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;  in bbr_timeout_tlp()
4697 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_delack()
4701 	tp->t_flags &= ~TF_DELACK;  in bbr_timeout_delack()
4702 	tp->t_flags |= TF_ACKNOW;  in bbr_timeout_delack()
4704 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;  in bbr_timeout_delack()
4709  * Here we send a KEEP-ALIVE like probe to the
4721 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_persist()
4724 	if (bbr->rc_in_persist == 0)  in bbr_timeout_persist()
4732 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;  in bbr_timeout_persist()
4739 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_persist()
4747 	if (tp->t_rxtshift >= V_tcp_retries &&  in bbr_timeout_persist()
4748 	    (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||  in bbr_timeout_persist()
4749 	    ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {  in bbr_timeout_persist()
4752 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_persist()
4754 	if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&  in bbr_timeout_persist()
4755 	    tp->snd_una == tp->snd_max) {  in bbr_timeout_persist()
4764 	if (tp->t_state > TCPS_CLOSE_WAIT &&  in bbr_timeout_persist()
4765 	    (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {  in bbr_timeout_persist()
4768 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_persist()
4770 	t_template = tcpip_maketemplate(bbr->rc_inp);  in bbr_timeout_persist()
4772 		tcp_respond(tp, t_template->tt_ipgen,  in bbr_timeout_persist()
4773 			    &t_template->tt_t, (struct mbuf *)NULL,  in bbr_timeout_persist()
4774 			    tp->rcv_nxt, tp->snd_una - 1, 0);  in bbr_timeout_persist()
4776 		if (tp->t_flags & TF_DELACK)  in bbr_timeout_persist()
4777 			tp->t_flags &= ~TF_DELACK;  in bbr_timeout_persist()
4780 	if (tp->t_rxtshift < V_tcp_retries)  in bbr_timeout_persist()
4781 		tp->t_rxtshift++;  in bbr_timeout_persist()
4799 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_keepalive()
4802 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;  in bbr_timeout_keepalive()
4805 	 * Keep-alive timer went off; send something or drop connection if  in bbr_timeout_keepalive()
4809 	if (tp->t_state < TCPS_ESTABLISHED)  in bbr_timeout_keepalive()
4811 	if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_timeout_keepalive()
4812 	    tp->t_state <= TCPS_CLOSING) {  in bbr_timeout_keepalive()
4813 		if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))  in bbr_timeout_keepalive()
4820 		 * number tp->snd_una-1 causes the transmitted zero-length  in bbr_timeout_keepalive()
4828 			tcp_respond(tp, t_template->tt_ipgen,  in bbr_timeout_keepalive()
4829 			    &t_template->tt_t, (struct mbuf *)NULL,  in bbr_timeout_keepalive()
4830 			    tp->rcv_nxt, tp->snd_una - 1, 0);  in bbr_timeout_keepalive()
4839 	return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_keepalive()
4851 	 * un-acked.  in bbr_remxt_tmr()
4857 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_remxt_tmr()
4858 	cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_remxt_tmr()
4859 	lost = bbr->r_ctl.rc_lost;  in bbr_remxt_tmr()
4860 	if (bbr->r_state && (bbr->r_state != tp->t_state))  in bbr_remxt_tmr()
4863 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_remxt_tmr()
4864 		if (rsm->r_flags & BBR_ACKED) {  in bbr_remxt_tmr()
4867 			rsm->r_dupack = 0;  in bbr_remxt_tmr()
4868 			if (rsm->r_in_tmap == 0) {  in bbr_remxt_tmr()
4869 				/* We must re-add it back to the tlist */  in bbr_remxt_tmr()
4871 					TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_remxt_tmr()
4873 					TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);  in bbr_remxt_tmr()
4875 				rsm->r_in_tmap = 1;  in bbr_remxt_tmr()
4877 			old_flags = rsm->r_flags;  in bbr_remxt_tmr()
4878 			rsm->r_flags |= BBR_RXT_CLEARED;  in bbr_remxt_tmr()
4879 			rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);  in bbr_remxt_tmr()
4882 			if ((tp->t_state < TCPS_ESTABLISHED) &&  in bbr_remxt_tmr()
4883 			    (rsm->r_start == tp->snd_una)) {  in bbr_remxt_tmr()
4891 			if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {  in bbr_remxt_tmr()
4892 				bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;  in bbr_remxt_tmr()
4893 				bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;  in bbr_remxt_tmr()
4900 				rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;  in bbr_remxt_tmr()
4901 				rsm->r_flags &= ~BBR_WAS_SACKPASS;  in bbr_remxt_tmr()
4910 				rsm->r_flags |= BBR_MARKED_LOST;  in bbr_remxt_tmr()
4911 				rsm->r_flags &= ~BBR_WAS_SACKPASS;  in bbr_remxt_tmr()
4912 				rsm->r_flags &= ~BBR_SACK_PASSED;  in bbr_remxt_tmr()
4917 	bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_remxt_tmr()
4918 	/* Clear the count (we just un-acked them) */  in bbr_remxt_tmr()
4920 	bbr->rc_tlp_new_data = 0;  in bbr_remxt_tmr()
4921 	bbr->r_ctl.rc_tlp_seg_send_cnt = 0;  in bbr_remxt_tmr()
4923 	bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_remxt_tmr()
4924 	bbr->r_agg_early_set = 0;  in bbr_remxt_tmr()
4925 	bbr->r_ctl.rc_agg_early = 0;  in bbr_remxt_tmr()
4926 	bbr->rc_tlp_rtx_out = 0;  in bbr_remxt_tmr()
4927 	bbr->r_ctl.rc_sacked = 0;  in bbr_remxt_tmr()
4928 	bbr->r_ctl.rc_sacklast = NULL;  in bbr_remxt_tmr()
4929 	bbr->r_timer_override = 1;  in bbr_remxt_tmr()
4930 	bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));  in bbr_remxt_tmr()
4934  * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4945 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;  in bbr_timeout_rxt()
4946 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_rxt()
4949 	if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_timeout_rxt()
4950 	    (tp->snd_una == tp->snd_max)) {  in bbr_timeout_rxt()
4961 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_rxt()
4964 	if ((bbr->r_ctl.rc_resend == NULL) ||  in bbr_timeout_rxt()
4965 	    ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {  in bbr_timeout_rxt()
4972 		tp->t_rxtshift++;  in bbr_timeout_rxt()
4974 	if (tp->t_rxtshift > V_tcp_retries) {  in bbr_timeout_rxt()
4975 		tp->t_rxtshift = V_tcp_retries;  in bbr_timeout_rxt()
4979 		MPASS(tp->t_softerror >= 0);  in bbr_timeout_rxt()
4980 		retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;  in bbr_timeout_rxt()
4983 	if (tp->t_state == TCPS_SYN_SENT) {  in bbr_timeout_rxt()
4988 		tp->snd_cwnd = 1;  in bbr_timeout_rxt()
4989 	} else if (tp->t_rxtshift == 1) {  in bbr_timeout_rxt()
4996 		 * End-to-End Network Path Properties" by Allman and Paxson  in bbr_timeout_rxt()
4999 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_rxt()
5000 		if (!IN_RECOVERY(tp->t_flags)) {  in bbr_timeout_rxt()
5001 			tp->snd_cwnd_prev = tp->snd_cwnd;  in bbr_timeout_rxt()
5002 			tp->snd_ssthresh_prev = tp->snd_ssthresh;  in bbr_timeout_rxt()
5003 			tp->snd_recover_prev = tp->snd_recover;  in bbr_timeout_rxt()
5004 			tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));  in bbr_timeout_rxt()
5005 			tp->t_flags |= TF_PREVVALID;  in bbr_timeout_rxt()
5007 			tp->t_flags &= ~TF_PREVVALID;  in bbr_timeout_rxt()
5009 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_rxt()
5011 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_rxt()
5012 		tp->t_flags &= ~TF_PREVVALID;  in bbr_timeout_rxt()
5015 	if ((tp->t_state == TCPS_SYN_SENT) ||  in bbr_timeout_rxt()
5016 	    (tp->t_state == TCPS_SYN_RECEIVED))  in bbr_timeout_rxt()
5017 		rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift];  in bbr_timeout_rxt()
5019 		rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];  in bbr_timeout_rxt()
5020 	TCPT_RANGESET(tp->t_rxtcur, rexmt,  in bbr_timeout_rxt()
5021 	    MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),  in bbr_timeout_rxt()
5022 	    MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));  in bbr_timeout_rxt()
5031 	isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false;  in bbr_timeout_rxt()
5038 	    ((tp->t_state == TCPS_ESTABLISHED) ||  in bbr_timeout_rxt()
5039 	    (tp->t_state == TCPS_FIN_WAIT_1))) {  in bbr_timeout_rxt()
5042 		 * 1448 -> 1188 -> 524) should be given 2 chances to recover  in bbr_timeout_rxt()
5043 		 * before further clamping down. 'tp->t_rxtshift % 2 == 0'  in bbr_timeout_rxt()
5046 		if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==  in bbr_timeout_rxt()
5048 		    (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&  in bbr_timeout_rxt()
5049 		    tp->t_rxtshift % 2 == 0)) {  in bbr_timeout_rxt()
5051 			 * Enter Path MTU Black-hole Detection mechanism: -  in bbr_timeout_rxt()
5052 			 * Disable Path MTU Discovery (IP "DF" bit). -  in bbr_timeout_rxt()
5056 			if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {  in bbr_timeout_rxt()
5061 				tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;  in bbr_timeout_rxt()
5063 				tp->t_pmtud_saved_maxseg = tp->t_maxseg;  in bbr_timeout_rxt()
5070 			isipv6 = bbr->r_is_v6;  in bbr_timeout_rxt()
5072 			    tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {  in bbr_timeout_rxt()
5074 				tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;  in bbr_timeout_rxt()
5078 				tp->t_maxseg = V_tcp_v6mssdflt;  in bbr_timeout_rxt()
5083 				tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_timeout_rxt()
5091 			if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {  in bbr_timeout_rxt()
5093 				tp->t_maxseg = V_tcp_pmtud_blackhole_mss;  in bbr_timeout_rxt()
5097 				tp->t_maxseg = V_tcp_mssdflt;  in bbr_timeout_rxt()
5102 				tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_timeout_rxt()
5115 			if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&  in bbr_timeout_rxt()
5116 			    (tp->t_rxtshift >= 6)) {  in bbr_timeout_rxt()
5117 				tp->t_flags2 |= TF2_PLPMTU_PMTUD;  in bbr_timeout_rxt()
5118 				tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;  in bbr_timeout_rxt()
5119 				tp->t_maxseg = tp->t_pmtud_saved_maxseg;  in bbr_timeout_rxt()
5120 				if (tp->t_maxseg < V_tcp_mssdflt) {  in bbr_timeout_rxt()
5126 					tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;  in bbr_timeout_rxt()
5128 					tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;  in bbr_timeout_rxt()
5136 	 * third SYN to work-around some broken terminal servers (most of  in bbr_timeout_rxt()
5139 	 * unknown-to-them TCP options.  in bbr_timeout_rxt()
5141 	if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&  in bbr_timeout_rxt()
5142 	    (tp->t_rxtshift == 3))  in bbr_timeout_rxt()
5143 		tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);  in bbr_timeout_rxt()
5150 	if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {  in bbr_timeout_rxt()
5152 		if (bbr->r_is_v6)  in bbr_timeout_rxt()
5157 		tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);  in bbr_timeout_rxt()
5158 		tp->t_srtt = 0;  in bbr_timeout_rxt()
5160 	sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_timeout_rxt()
5161 	tp->snd_recover = tp->snd_max;  in bbr_timeout_rxt()
5162 	tp->t_flags |= TF_ACKNOW;  in bbr_timeout_rxt()
5163 	tp->t_rtttime = 0;  in bbr_timeout_rxt()
5172 	int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);  in bbr_process_timers()
5177 	if (tp->t_state == TCPS_LISTEN) {  in bbr_process_timers()
5179 		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)  in bbr_process_timers()
5183 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {  in bbr_process_timers()
5186 		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {  in bbr_process_timers()
5187 			ret = -1;  in bbr_process_timers()
5192 			ret = -2;  in bbr_process_timers()
5200 		left = bbr->r_ctl.rc_timer_exp - cts;  in bbr_process_timers()
5201 		ret = -3;  in bbr_process_timers()
5206 	bbr->rc_tmr_stopped = 0;  in bbr_process_timers()
5207 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;  in bbr_process_timers()
5213 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_process_timers()
5216 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_process_timers()
5219 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_process_timers()
5231 	if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {  in bbr_timer_cancel()
5234 		if (tcp_in_hpts(bbr->rc_tp) &&  in bbr_timer_cancel()
5235 		    (bbr->rc_timer_first == 1)) {  in bbr_timer_cancel()
5242 			tcp_hpts_remove(bbr->rc_tp);  in bbr_timer_cancel()
5243 			if (bbr->r_ctl.rc_last_delay_val) {  in bbr_timer_cancel()
5247 				if (TSTMP_GT(cts, bbr->rc_pacer_started))  in bbr_timer_cancel()
5248 					time_since_send = cts - bbr->rc_pacer_started;  in bbr_timer_cancel()
5251 				if (bbr->r_ctl.rc_last_delay_val > time_since_send) {  in bbr_timer_cancel()
5253 					bbr->r_ctl.rc_last_delay_val -= time_since_send;  in bbr_timer_cancel()
5255 					bbr->r_ctl.rc_last_delay_val = 0;  in bbr_timer_cancel()
5257 				bbr->rc_pacer_started = cts;  in bbr_timer_cancel()
5260 		bbr->rc_timer_first = 0;  in bbr_timer_cancel()
5262 		bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;  in bbr_timer_cancel()
5263 		bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);  in bbr_timer_cancel()
5272 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_stopall()
5273 	bbr->rc_all_timers_stopped = 1;  in bbr_stopall()
5285 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_get_earliest_send_outstanding()
5288 	return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);  in bbr_get_earliest_send_outstanding()
5297 	rsm->r_rtr_cnt++;  in bbr_update_rsm()
5298 	rsm->r_dupack = 0;  in bbr_update_rsm()
5299 	if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {  in bbr_update_rsm()
5300 		rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;  in bbr_update_rsm()
5301 		rsm->r_flags |= BBR_OVERMAX;  in bbr_update_rsm()
5303 	if (rsm->r_flags & BBR_RWND_COLLAPSED) {  in bbr_update_rsm()
5305 		rsm->r_flags &= ~BBR_RWND_COLLAPSED;  in bbr_update_rsm()
5307 	if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_update_rsm()
5309 		rsm->r_flags &= ~BBR_MARKED_LOST;  in bbr_update_rsm()
5310 		bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_update_rsm()
5312 	if (rsm->r_flags & BBR_RXT_CLEARED) {  in bbr_update_rsm()
5321 		rsm->r_flags &= ~BBR_RXT_CLEARED;  in bbr_update_rsm()
5323 	if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {  in bbr_update_rsm()
5324 		bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);  in bbr_update_rsm()
5325 		rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);  in bbr_update_rsm()
5327 	idx = rsm->r_rtr_cnt - 1;  in bbr_update_rsm()
5328 	rsm->r_tim_lastsent[idx] = cts;  in bbr_update_rsm()
5329 	rsm->r_pacing_delay = pacing_time;  in bbr_update_rsm()
5330 	rsm->r_delivered = bbr->r_ctl.rc_delivered;  in bbr_update_rsm()
5331 	rsm->r_ts_valid = bbr->rc_ts_valid;  in bbr_update_rsm()
5332 	if (bbr->rc_ts_valid)  in bbr_update_rsm()
5333 		rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;  in bbr_update_rsm()
5334 	if (bbr->r_ctl.r_app_limited_until)  in bbr_update_rsm()
5335 		rsm->r_app_limited = 1;  in bbr_update_rsm()
5337 		rsm->r_app_limited = 0;  in bbr_update_rsm()
5338 	if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)  in bbr_update_rsm()
5339 		rsm->r_bbr_state = bbr_state_val(bbr);  in bbr_update_rsm()
5341 		rsm->r_bbr_state = 8;  in bbr_update_rsm()
5342 	if (rsm->r_flags & BBR_ACKED) {  in bbr_update_rsm()
5346 		old_flags = rsm->r_flags;  in bbr_update_rsm()
5347 		rsm->r_flags &= ~BBR_ACKED;  in bbr_update_rsm()
5349 		bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);  in bbr_update_rsm()
5350 		if (bbr->r_ctl.rc_sacked == 0)  in bbr_update_rsm()
5351 			bbr->r_ctl.rc_sacklast = NULL;  in bbr_update_rsm()
5353 	if (rsm->r_in_tmap) {  in bbr_update_rsm()
5354 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_update_rsm()
5356 	TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_update_rsm()
5357 	rsm->r_in_tmap = 1;  in bbr_update_rsm()
5358 	if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_update_rsm()
5360 		rsm->r_flags &= ~BBR_SACK_PASSED;  in bbr_update_rsm()
5361 		rsm->r_flags |= BBR_WAS_SACKPASS;  in bbr_update_rsm()
5363 	rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);  in bbr_update_rsm()
5364 	rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,  in bbr_update_rsm()
5365 						(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_update_rsm()
5366 	bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);  in bbr_update_rsm()
5367 	if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {  in bbr_update_rsm()
5368 		rsm->r_is_gain = 1;  in bbr_update_rsm()
5369 		rsm->r_is_drain = 0;  in bbr_update_rsm()
5370 	} else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {  in bbr_update_rsm()
5371 		rsm->r_is_drain = 1;  in bbr_update_rsm()
5372 		rsm->r_is_gain = 0;  in bbr_update_rsm()
5374 		rsm->r_is_drain = 0;  in bbr_update_rsm()
5375 		rsm->r_is_gain = 0;  in bbr_update_rsm()
5377 	rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */  in bbr_update_rsm()
5391 	 * We (re-)transmitted starting at rsm->r_start for some length  in bbr_update_entry()
5399 	c_end = rsm->r_start + len;  in bbr_update_entry()
5400 	if (SEQ_GEQ(c_end, rsm->r_end)) {  in bbr_update_entry()
5406 		if (c_end == rsm->r_end) {  in bbr_update_entry()
5413 			act_len = rsm->r_end - rsm->r_start;  in bbr_update_entry()
5414 			*lenp = (len - act_len);  in bbr_update_entry()
5415 			return (rsm->r_end);  in bbr_update_entry()
5436 	TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_update_entry()
5437 	nrsm->r_dupack = 0;  in bbr_update_entry()
5438 	if (rsm->r_in_tmap) {  in bbr_update_entry()
5439 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_update_entry()
5440 		nrsm->r_in_tmap = 1;  in bbr_update_entry()
5442 	rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_update_entry()
5467 	if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {  in bbr_setup_less_of_rate()
5471 		bbr->skip_gain = 1;  in bbr_setup_less_of_rate()
5472 		bbr->gain_is_limited = 0;  in bbr_setup_less_of_rate()
5473 		red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;  in bbr_setup_less_of_rate()
5475 			filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);  in bbr_setup_less_of_rate()
5478 		bbr->skip_gain = 0;  in bbr_setup_less_of_rate()
5479 		bbr->gain_is_limited = 1;  in bbr_setup_less_of_rate()
5487 	int error, rate = -1;  in bbr_update_hardware_pacing_rate()
5489 	if (bbr->r_ctl.crte == NULL)  in bbr_update_hardware_pacing_rate()
5491 	if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||  in bbr_update_hardware_pacing_rate()
5492 	    (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {  in bbr_update_hardware_pacing_rate()
5494 		/* Clear the way for a re-attempt */  in bbr_update_hardware_pacing_rate()
5495 		bbr->bbr_attempt_hdwr_pace = 0;  in bbr_update_hardware_pacing_rate()
5497 		bbr->gain_is_limited = 0;  in bbr_update_hardware_pacing_rate()
5498 		bbr->skip_gain = 0;  in bbr_update_hardware_pacing_rate()
5499 		bbr->bbr_hdrw_pacing = 0;  in bbr_update_hardware_pacing_rate()
5500 		counter_u64_add(bbr_flows_whdwr_pacing, -1);  in bbr_update_hardware_pacing_rate()
5506 	nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,  in bbr_update_hardware_pacing_rate()
5507 				   bbr->rc_tp,  in bbr_update_hardware_pacing_rate()
5508 				   bbr->rc_inp->inp_route.ro_nh->nh_ifp,  in bbr_update_hardware_pacing_rate()
5515 	if (nrte != bbr->r_ctl.crte) {  in bbr_update_hardware_pacing_rate()
5516 		bbr->r_ctl.crte = nrte;  in bbr_update_hardware_pacing_rate()
5519 			if (bbr->r_ctl.crte->rate < rate) {  in bbr_update_hardware_pacing_rate()
5522 						       bbr->r_ctl.crte->rate, rate);  in bbr_update_hardware_pacing_rate()
5525 				bbr->gain_is_limited = 0;  in bbr_update_hardware_pacing_rate()
5526 				bbr->skip_gain = 0;  in bbr_update_hardware_pacing_rate()
5531 			bbr->gain_is_limited = 0;  in bbr_update_hardware_pacing_rate()
5532 			bbr->skip_gain = 0;  in bbr_update_hardware_pacing_rate()
5533 			bbr->bbr_hdrw_pacing = 0;  in bbr_update_hardware_pacing_rate()
5536 					 bbr->r_ctl.crte->ptbl->rs_ifp,  in bbr_update_hardware_pacing_rate()
5538 					 bbr->r_ctl.crte->rate,  in bbr_update_hardware_pacing_rate()
5556 	if ((bbr->bbr_hdrw_pacing == 0) ||  in bbr_adjust_for_hw_pacing()
5557 	    (IN_RECOVERY(bbr->rc_tp->t_flags)) ||  in bbr_adjust_for_hw_pacing()
5558 	    (bbr->r_ctl.crte == NULL))  in bbr_adjust_for_hw_pacing()
5560 	if (bbr->hw_pacing_set == 0) {  in bbr_adjust_for_hw_pacing()
5568 	rlp = bbr->r_ctl.crte;  in bbr_adjust_for_hw_pacing()
5569 	if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)  in bbr_adjust_for_hw_pacing()
5570 		maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in bbr_adjust_for_hw_pacing()
5572 		maxseg = BBR_MIN_SEG - bbr->rc_last_options;  in bbr_adjust_for_hw_pacing()
5580 		        bbr->r_ctl.rc_pace_max_segs, cts, 1);  in bbr_adjust_for_hw_pacing()
5581 	hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;  in bbr_adjust_for_hw_pacing()
5582 	hdwr_delay *= rlp->time_between;  in bbr_adjust_for_hw_pacing()
5584 		delta = cur_delay - hdwr_delay;  in bbr_adjust_for_hw_pacing()
5588 			     (bbr->r_ctl.rc_pace_max_segs / maxseg),  in bbr_adjust_for_hw_pacing()
5591 	    (delta < (max(rlp->time_between,  in bbr_adjust_for_hw_pacing()
5592 			  bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {  in bbr_adjust_for_hw_pacing()
5606 		seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between),  in bbr_adjust_for_hw_pacing()
5607 			     (bbr->r_ctl.rc_pace_max_segs/maxseg));  in bbr_adjust_for_hw_pacing()
5610 		    (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {  in bbr_adjust_for_hw_pacing()
5616 			seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;  in bbr_adjust_for_hw_pacing()
5630 		seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;  in bbr_adjust_for_hw_pacing()
5632 		    (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {  in bbr_adjust_for_hw_pacing()
5638 			seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;  in bbr_adjust_for_hw_pacing()
5652 		seg_sz = bbr->r_ctl.rc_pace_max_segs;  in bbr_adjust_for_hw_pacing()
5654 	if (seg_sz > bbr->r_ctl.rc_pace_max_segs)  in bbr_adjust_for_hw_pacing()
5657 		new_tso = bbr->r_ctl.rc_pace_max_segs;  in bbr_adjust_for_hw_pacing()
5658 	if (new_tso >= (PACE_MAX_IP_BYTES-maxseg))  in bbr_adjust_for_hw_pacing()
5659 		new_tso = PACE_MAX_IP_BYTES - maxseg;  in bbr_adjust_for_hw_pacing()
5661 	if (new_tso != bbr->r_ctl.rc_pace_max_segs) {  in bbr_adjust_for_hw_pacing()
5662 		bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);  in bbr_adjust_for_hw_pacing()
5663 		bbr->r_ctl.rc_pace_max_segs = new_tso;  in bbr_adjust_for_hw_pacing()
5690 	 * a full iwnd just like new-reno/cubic.  in tcp_bbr_tso_size_check()
5695 	 *  if ( bw <= per-tcb-cross-over)  in tcp_bbr_tso_size_check()
5697 	 *                 can send in goal-time seconds.  in tcp_bbr_tso_size_check()
5703 	 *     if (tso > per-tcb-max)  in tcp_bbr_tso_size_check()
5704 	 *         tso = per-tcb-max  in tcp_bbr_tso_size_check()
5706 	 *     tso = max-tso (64k/mss)  in tcp_bbr_tso_size_check()
5708 	 *     goal_tso = bw / per-tcb-divsor  in tcp_bbr_tso_size_check()
5709 	 *     seg = (goal_tso + mss-1)/mss  in tcp_bbr_tso_size_check()
5712 	 * if (tso < per-tcb-floor)  in tcp_bbr_tso_size_check()
5713 	 *    tso = per-tcb-floor  in tcp_bbr_tso_size_check()
5714 	 * if (tso > per-tcb-utter_max)  in tcp_bbr_tso_size_check()
5715 	 *    tso = per-tcb-utter_max  in tcp_bbr_tso_size_check()
5717 	 * Note the default per-tcb-divisor is 1000 (same as google).  in tcp_bbr_tso_size_check()
5721 	 * cross-over = 23,168,000 bps  in tcp_bbr_tso_size_check()
5722 	 * goal-time = 18000  in tcp_bbr_tso_size_check()
5723 	 * per-tcb-max = 2  in tcp_bbr_tso_size_check()
5724 	 * per-tcb-divisor = 1000  in tcp_bbr_tso_size_check()
5725 	 * per-tcb-floor = 1  in tcp_bbr_tso_size_check()
5734 	if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {  in tcp_bbr_tso_size_check()
5735 		maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in tcp_bbr_tso_size_check()
5737 		maxseg = BBR_MIN_SEG - bbr->rc_last_options;  in tcp_bbr_tso_size_check()
5739 	old_tso = bbr->r_ctl.rc_pace_max_segs;  in tcp_bbr_tso_size_check()
5740 	if (bbr->rc_past_init_win == 0) {  in tcp_bbr_tso_size_check()
5746 		if (bbr->rc_use_google)  in tcp_bbr_tso_size_check()
5747 			bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);  in tcp_bbr_tso_size_check()
5748 		else if (bbr->bbr_init_win_cheat)  in tcp_bbr_tso_size_check()
5749 			bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);  in tcp_bbr_tso_size_check()
5751 			bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in tcp_bbr_tso_size_check()
5752 		if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)  in tcp_bbr_tso_size_check()
5753 			bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;  in tcp_bbr_tso_size_check()
5754 		if (bbr->r_ctl.rc_pace_max_segs == 0) {  in tcp_bbr_tso_size_check()
5755 			bbr->r_ctl.rc_pace_max_segs = maxseg;  in tcp_bbr_tso_size_check()
5757 		bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);  in tcp_bbr_tso_size_check()
5767 	if (IN_RECOVERY(bbr->rc_tp->t_flags) &&  in tcp_bbr_tso_size_check()
5768 	     (bbr->rc_use_google == 0)) {  in tcp_bbr_tso_size_check()
5771 	} else if (bbr->rc_use_google) {  in tcp_bbr_tso_size_check()
5775 		if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {  in tcp_bbr_tso_size_check()
5776 			bw *= bbr->r_ctl.rc_bbr_hptsi_gain;  in tcp_bbr_tso_size_check()
5790 	} else if (bbr->rc_no_pacing) {  in tcp_bbr_tso_size_check()
5792 	} else if (bw <= bbr->r_ctl.bbr_cross_over) {  in tcp_bbr_tso_size_check()
5799 		tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);  in tcp_bbr_tso_size_check()
5802 			if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)  in tcp_bbr_tso_size_check()
5803 				new_tso = bbr->r_ctl.bbr_hptsi_segments_max;  in tcp_bbr_tso_size_check()
5836 		bw /= bbr->r_ctl.bbr_hptsi_per_second;  in tcp_bbr_tso_size_check()
5843 		if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))  in tcp_bbr_tso_size_check()
5844 			new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);  in tcp_bbr_tso_size_check()
5846 …if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floo…  in tcp_bbr_tso_size_check()
5847 		new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;  in tcp_bbr_tso_size_check()
5851 	if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {  in tcp_bbr_tso_size_check()
5852 		new_tso = bbr->r_ctl.bbr_utter_max * maxseg;  in tcp_bbr_tso_size_check()
5857 		bbr->r_ctl.rc_pace_max_segs = new_tso;  in tcp_bbr_tso_size_check()
5892 		 * We don't log errors -- we could but snd_max does not  in bbr_log_output()
5905 	snd_una = tp->snd_una;  in bbr_log_output()
5913 		if ((th_flags & TH_SYN) && (tp->iss == seq_out))  in bbr_log_output()
5919 		/* Are sending an old segment to induce an ack (keep-alive)? */  in bbr_log_output()
5928 		len = end - seq_out;  in bbr_log_output()
5930 	snd_max = tp->snd_max;  in bbr_log_output()
5935 	pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);  in bbr_log_output()
5943 		rsm->r_flags = 0;  in bbr_log_output()
5945 			rsm->r_flags |= BBR_HAS_SYN;  in bbr_log_output()
5947 			rsm->r_flags |= BBR_HAS_FIN;  in bbr_log_output()
5948 		rsm->r_tim_lastsent[0] = cts;  in bbr_log_output()
5949 		rsm->r_rtr_cnt = 1;  in bbr_log_output()
5950 		rsm->r_rtr_bytes = 0;  in bbr_log_output()
5951 		rsm->r_start = seq_out;  in bbr_log_output()
5952 		rsm->r_end = rsm->r_start + len;  in bbr_log_output()
5953 		rsm->r_dupack = 0;  in bbr_log_output()
5954 		rsm->r_delivered = bbr->r_ctl.rc_delivered;  in bbr_log_output()
5955 		rsm->r_pacing_delay = pacing_time;  in bbr_log_output()
5956 		rsm->r_ts_valid = bbr->rc_ts_valid;  in bbr_log_output()
5957 		if (bbr->rc_ts_valid)  in bbr_log_output()
5958 			rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;  in bbr_log_output()
5959 		rsm->r_del_time = bbr->r_ctl.rc_del_time;  in bbr_log_output()
5960 		if (bbr->r_ctl.r_app_limited_until)  in bbr_log_output()
5961 			rsm->r_app_limited = 1;  in bbr_log_output()
5963 			rsm->r_app_limited = 0;  in bbr_log_output()
5964 		rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);  in bbr_log_output()
5965 		rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,  in bbr_log_output()
5966 						(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_log_output()
5971 		rsm->r_flight_at_send += len;  in bbr_log_output()
5972 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_log_output()
5973 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_log_output()
5974 		rsm->r_in_tmap = 1;  in bbr_log_output()
5975 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)  in bbr_log_output()
5976 			rsm->r_bbr_state = bbr_state_val(bbr);  in bbr_log_output()
5978 			rsm->r_bbr_state = 8;  in bbr_log_output()
5979 		if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {  in bbr_log_output()
5980 			rsm->r_is_gain = 1;  in bbr_log_output()
5981 			rsm->r_is_drain = 0;  in bbr_log_output()
5982 		} else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {  in bbr_log_output()
5983 			rsm->r_is_drain = 1;  in bbr_log_output()
5984 			rsm->r_is_gain = 0;  in bbr_log_output()
5986 			rsm->r_is_drain = 0;  in bbr_log_output()
5987 			rsm->r_is_gain = 0;  in bbr_log_output()
5995 	if (hintrsm && (hintrsm->r_start == seq_out)) {  in bbr_log_output()
5998 	} else if (bbr->r_ctl.rc_next) {  in bbr_log_output()
6000 		rsm = bbr->r_ctl.rc_next;  in bbr_log_output()
6005 	if ((rsm) && (rsm->r_start == seq_out)) {  in bbr_log_output()
6018 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_log_output()
6019 		if (rsm->r_start == seq_out) {  in bbr_log_output()
6021 			bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);  in bbr_log_output()
6028 		if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {  in bbr_log_output()
6044 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_log_output()
6045 			if (rsm->r_in_tmap) {  in bbr_log_output()
6046 				TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_log_output()
6047 				nrsm->r_in_tmap = 1;  in bbr_log_output()
6049 			rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_log_output()
6060 	if (seq_out == tp->snd_max) {  in bbr_log_output()
6062 	} else if (SEQ_LT(seq_out, tp->snd_max)) {  in bbr_log_output()
6064 		printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",  in bbr_log_output()
6065 		    seq_out, len, tp->snd_una, tp->snd_max);  in bbr_log_output()
6067 		TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_log_output()
6069 			    rsm, rsm->r_start, rsm->r_end);  in bbr_log_output()
6078 		 * Hmm beyond sndmax? (only if we are using the new rtt-pack  in bbr_log_output()
6082 		    seq_out, len, tp->snd_max, tp);  in bbr_log_output()
6091 	 * Collapse timeout back the cum-ack moved.  in bbr_collapse_rtt()
6093 	tp->t_rxtshift = 0;  in bbr_collapse_rtt()
6094 	tp->t_softerror = 0;  in bbr_collapse_rtt()
6100 	bbr->rtt_valid = 1;  in tcp_bbr_xmit_timer()
6101 	bbr->r_ctl.cur_rtt = rtt_usecs;  in tcp_bbr_xmit_timer()
6102 	bbr->r_ctl.ts_in = tsin;  in tcp_bbr_xmit_timer()
6104 		bbr->r_ctl.cur_rtt_send_time = rsm_send_time;  in tcp_bbr_xmit_timer()
6112 	 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp).  in bbr_make_timestamp_determination()
6113 	 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).  in bbr_make_timestamp_determination()
6114 	 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts)  in bbr_make_timestamp_determination()
6115 	 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time)  in bbr_make_timestamp_determination()
6119 	 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts  in bbr_make_timestamp_determination()
6123 	 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp  in bbr_make_timestamp_determination()
6130 	 * out time-slices of the actual b/w. This would mean we could not use  in bbr_make_timestamp_determination()
6134 	 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio.  in bbr_make_timestamp_determination()
6148 	delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;  in bbr_make_timestamp_determination()
6158 	peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;  in bbr_make_timestamp_determination()
6177 		 * then we saw then in micro-seconds.  in bbr_make_timestamp_determination()
6180 			/* well it looks like the peer is a micro-second clock. */  in bbr_make_timestamp_determination()
6181 			bbr->rc_ts_clock_set = 1;  in bbr_make_timestamp_determination()
6182 			bbr->r_ctl.bbr_peer_tsratio = 1;  in bbr_make_timestamp_determination()
6184 			bbr->rc_ts_cant_be_used = 1;  in bbr_make_timestamp_determination()
6185 			bbr->rc_ts_clock_set = 1;  in bbr_make_timestamp_determination()
6190 	bbr->rc_ts_clock_set = 1;  in bbr_make_timestamp_determination()
6195 		bbr->r_ctl.bbr_peer_tsratio = 1;  in bbr_make_timestamp_determination()
6202 		bbr->r_ctl.bbr_peer_tsratio = 10;  in bbr_make_timestamp_determination()
6209 		bbr->r_ctl.bbr_peer_tsratio = 100;  in bbr_make_timestamp_determination()
6216 		bbr->r_ctl.bbr_peer_tsratio = 1000;  in bbr_make_timestamp_determination()
6223 		bbr->r_ctl.bbr_peer_tsratio = 10000;  in bbr_make_timestamp_determination()
6227 	bbr->rc_ts_cant_be_used = 1;  in bbr_make_timestamp_determination()
6228 	bbr->r_ctl.bbr_peer_tsratio = 0;  in bbr_make_timestamp_determination()
6233  * Collect new round-trip time estimate
6243 	if (bbr->rtt_valid == 0)  in tcp_bbr_xmit_timer_commit()
6247 	rtt = bbr->r_ctl.cur_rtt;  in tcp_bbr_xmit_timer_commit()
6248 	tsin = bbr->r_ctl.ts_in;  in tcp_bbr_xmit_timer_commit()
6249 	if (bbr->rc_prtt_set_ts) {  in tcp_bbr_xmit_timer_commit()
6258 		 * and when they enter probe-rtt they update the  in tcp_bbr_xmit_timer_commit()
6263 		bbr->rc_prtt_set_ts = 0;  in tcp_bbr_xmit_timer_commit()
6264 		rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in tcp_bbr_xmit_timer_commit()
6266 			filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);  in tcp_bbr_xmit_timer_commit()
6268 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in tcp_bbr_xmit_timer_commit()
6271 	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt));  in tcp_bbr_xmit_timer_commit()
6273 	if (bbr->rc_ack_was_delayed)  in tcp_bbr_xmit_timer_commit()
6274 		rtt += bbr->r_ctl.rc_ack_hdwr_delay;  in tcp_bbr_xmit_timer_commit()
6276 	if (rtt < bbr->r_ctl.rc_lowest_rtt)  in tcp_bbr_xmit_timer_commit()
6277 		bbr->r_ctl.rc_lowest_rtt = rtt;  in tcp_bbr_xmit_timer_commit()
6279 	if (bbr->r_init_rtt) {  in tcp_bbr_xmit_timer_commit()
6281 		 * The initial rtt is not-trusted, nuke it and lets get  in tcp_bbr_xmit_timer_commit()
6284 		bbr->r_init_rtt = 0;  in tcp_bbr_xmit_timer_commit()
6285 		tp->t_srtt = 0;  in tcp_bbr_xmit_timer_commit()
6287 	if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {  in tcp_bbr_xmit_timer_commit()
6292 		 * series of cum-ack's to determine  in tcp_bbr_xmit_timer_commit()
6295 		if (bbr->rc_ack_is_cumack) {  in tcp_bbr_xmit_timer_commit()
6296 			if (bbr->rc_ts_data_set) {  in tcp_bbr_xmit_timer_commit()
6300 				bbr->rc_ts_data_set = 1;  in tcp_bbr_xmit_timer_commit()
6301 				bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;  in tcp_bbr_xmit_timer_commit()
6302 				bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;  in tcp_bbr_xmit_timer_commit()
6309 			bbr->rc_ts_data_set = 0;  in tcp_bbr_xmit_timer_commit()
6313 	rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1)));  in tcp_bbr_xmit_timer_commit()
6314 	if (tp->t_srtt != 0) {  in tcp_bbr_xmit_timer_commit()
6323 		delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT)  in tcp_bbr_xmit_timer_commit()
6324 		    - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));  in tcp_bbr_xmit_timer_commit()
6326 		tp->t_srtt += delta;  in tcp_bbr_xmit_timer_commit()
6327 		if (tp->t_srtt <= 0)  in tcp_bbr_xmit_timer_commit()
6328 			tp->t_srtt = 1;  in tcp_bbr_xmit_timer_commit()
6338 		 * wired-in beta.  in tcp_bbr_xmit_timer_commit()
6341 			delta = -delta;  in tcp_bbr_xmit_timer_commit()
6342 		delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);  in tcp_bbr_xmit_timer_commit()
6343 		tp->t_rttvar += delta;  in tcp_bbr_xmit_timer_commit()
6344 		if (tp->t_rttvar <= 0)  in tcp_bbr_xmit_timer_commit()
6345 			tp->t_rttvar = 1;  in tcp_bbr_xmit_timer_commit()
6348 		 * No rtt measurement yet - use the unsmoothed rtt. Set the  in tcp_bbr_xmit_timer_commit()
6352 		tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT;  in tcp_bbr_xmit_timer_commit()
6353 		tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1);  in tcp_bbr_xmit_timer_commit()
6356 	if (tp->t_rttupdated < UCHAR_MAX)  in tcp_bbr_xmit_timer_commit()
6357 		tp->t_rttupdated++;  in tcp_bbr_xmit_timer_commit()
6359 	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));  in tcp_bbr_xmit_timer_commit()
6365 	 * tick of rounding and 1 extra tick because of +-1/2 tick  in tcp_bbr_xmit_timer_commit()
6371 	TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),  in tcp_bbr_xmit_timer_commit()
6372 	    max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),  in tcp_bbr_xmit_timer_commit()
6373 	    MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));  in tcp_bbr_xmit_timer_commit()
6382 	tp->t_softerror = 0;  in tcp_bbr_xmit_timer_commit()
6383 	rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);  in tcp_bbr_xmit_timer_commit()
6384 	if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)  in tcp_bbr_xmit_timer_commit()
6385 		bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;  in tcp_bbr_xmit_timer_commit()
6391 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_set_reduced_rtt()
6393 	    (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&  in bbr_set_reduced_rtt()
6394 	    ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {  in bbr_set_reduced_rtt()
6396 		 * We should enter probe-rtt its been too long  in bbr_set_reduced_rtt()
6409 	if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {  in tcp_bbr_commit_bw()
6415 	if (bbr->r_ctl.r_measurement_count < 0xffffffff)  in tcp_bbr_commit_bw()
6416 		bbr->r_ctl.r_measurement_count++;  in tcp_bbr_commit_bw()
6417 	orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in tcp_bbr_commit_bw()
6418 	apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);  in tcp_bbr_commit_bw()
6420 			    (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),  in tcp_bbr_commit_bw()
6423 	    (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {  in tcp_bbr_commit_bw()
6424 		if (bbr->bbr_hdrw_pacing) {  in tcp_bbr_commit_bw()
6433 		if (bbr->r_recovery_bw)  {  in tcp_bbr_commit_bw()
6437 	} else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))  in tcp_bbr_commit_bw()
6444 	if (bbr->rc_in_persist == 0) {  in bbr_nf_measurement()
6450 		if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))  in bbr_nf_measurement()
6451 			tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);  in bbr_nf_measurement()
6459 		delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);  in bbr_nf_measurement()
6469 		 * can see in the trace viewer if it gets over-ridden.  in bbr_nf_measurement()
6471 		if (rsm->r_ts_valid &&  in bbr_nf_measurement()
6472 		    bbr->rc_ts_valid &&  in bbr_nf_measurement()
6473 		    bbr->rc_ts_clock_set &&  in bbr_nf_measurement()
6474 		    (bbr->rc_ts_cant_be_used == 0) &&  in bbr_nf_measurement()
6475 		    bbr->rc_use_ts_limit) {  in bbr_nf_measurement()
6476 			ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);  in bbr_nf_measurement()
6477 			ts_diff *= bbr->r_ctl.bbr_peer_tsratio;  in bbr_nf_measurement()
6483 						    bbr->r_ctl.last_inbound_ts,  in bbr_nf_measurement()
6484 						    rsm->r_del_ack_ts, 0,  in bbr_nf_measurement()
6494 				if ((bbr->ts_can_raise) &&  in bbr_nf_measurement()
6514 		if (rsm->r_first_sent_time &&  in bbr_nf_measurement()
6515 		    TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {  in bbr_nf_measurement()
6528 			sbw = (uint64_t)(rsm->r_flight_at_send);  in bbr_nf_measurement()
6530 			sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;  in bbr_nf_measurement()
6531 			sti += rsm->r_pacing_delay;  in bbr_nf_measurement()
6539 						    rsm->r_first_sent_time, 0, (sbw >> 32),  in bbr_nf_measurement()
6545 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;  in bbr_nf_measurement()
6546 		if ((rsm->r_app_limited == 0) ||  in bbr_nf_measurement()
6547 		    (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {  in bbr_nf_measurement()
6550 					    0, 0, 0, 0,  bbr->r_ctl.rc_del_time,  rsm->r_del_time);  in bbr_nf_measurement()
6558 	if (bbr->rc_in_persist == 0) {  in bbr_google_measurement()
6565 		if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))  in bbr_google_measurement()
6566 			tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);  in bbr_google_measurement()
6574 		delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);  in bbr_google_measurement()
6578 		if (tim < bbr->r_ctl.rc_lowest_rtt) {  in bbr_google_measurement()
6580 					    tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);  in bbr_google_measurement()
6586 		 * can see in the trace viewer if it gets over-ridden.  in bbr_google_measurement()
6588 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;  in bbr_google_measurement()
6590 		if (rsm->r_first_sent_time &&  in bbr_google_measurement()
6591 		    TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {  in bbr_google_measurement()
6604 			sbw = (uint64_t)(rsm->r_flight_at_send);  in bbr_google_measurement()
6606 			sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;  in bbr_google_measurement()
6607 			sti += rsm->r_pacing_delay;  in bbr_google_measurement()
6615 						    rsm->r_first_sent_time, 0, (sbw >> 32),  in bbr_google_measurement()
6620 			    (sti < bbr->r_ctl.rc_lowest_rtt)) {  in bbr_google_measurement()
6622 						    (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);  in bbr_google_measurement()
6627 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;  in bbr_google_measurement()
6629 		    ((rsm->r_app_limited == 0) ||  in bbr_google_measurement()
6630 		     (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {  in bbr_google_measurement()
6633 					    0, 0, 0, 0, bbr->r_ctl.rc_del_time,  rsm->r_del_time);  in bbr_google_measurement()
6645 	bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);  in bbr_update_bbr_info()
6646 	bbr->r_ctl.rc_del_time = cts;  in bbr_update_bbr_info()
6654 	if ((bbr->rc_use_google == 0) &&  in bbr_update_bbr_info()
6667 	if ((bbr_no_retran && bbr->rc_use_google) &&  in bbr_update_bbr_info()
6677 	tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);  in bbr_update_bbr_info()
6679 		bbr->rc_ack_is_cumack = 1;  in bbr_update_bbr_info()
6681 		bbr->rc_ack_is_cumack = 0;  in bbr_update_bbr_info()
6688 	 * going into probe-rtt (we were seeing cases where that  in bbr_update_bbr_info()
6697 	bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts,  in bbr_update_bbr_info()
6698 	    match, rsm->r_start, rsm->r_flags);  in bbr_update_bbr_info()
6699 	apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in bbr_update_bbr_info()
6702 		 * The RTT-prop moved, reset the target (may be a  in bbr_update_bbr_info()
6706 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)  in bbr_update_bbr_info()
6713 	if ((bbr->rc_use_google == 0) &&  in bbr_update_bbr_info()
6722 	if (bbr->r_ctl.r_app_limited_until &&  in bbr_update_bbr_info()
6723 	    (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {  in bbr_update_bbr_info()
6724 		/* We are no longer app-limited */  in bbr_update_bbr_info()
6725 		bbr->r_ctl.r_app_limited_until = 0;  in bbr_update_bbr_info()
6727 	if (bbr->rc_use_google) {  in bbr_update_bbr_info()
6744 	msec = cts - (MS_IN_USEC * sec);  in bbr_ts_convert()
6759 	if ((rsm->r_flags & BBR_ACKED) ||  in bbr_update_rtt()
6760 	    (rsm->r_flags & BBR_WAS_RENEGED) ||  in bbr_update_rtt()
6761 	    (rsm->r_flags & BBR_RXT_CLEARED)) {  in bbr_update_rtt()
6765 	if (rsm->r_rtt_not_allowed) {  in bbr_update_rtt()
6769 	if (rsm->r_rtr_cnt == 1) {  in bbr_update_rtt()
6773 		if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))  in bbr_update_rtt()
6774 			t = cts - rsm->r_tim_lastsent[0];  in bbr_update_rtt()
6777 		bbr->r_ctl.rc_last_rtt = t;  in bbr_update_rtt()
6778 		bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,  in bbr_update_rtt()
6779 				    BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);  in bbr_update_rtt()
6784 	    (bbr->rc_use_google == 1) &&  in bbr_update_rtt()
6786 	    (to->to_flags & TOF_TS) &&  in bbr_update_rtt()
6787 	    (to->to_tsecr != 0)) {  in bbr_update_rtt()
6788 		t = tcp_tv_to_msec(&bbr->rc_tv) - to->to_tsecr;  in bbr_update_rtt()
6792 		bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,  in bbr_update_rtt()
6794 				    rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],  in bbr_update_rtt()
6798 	uts = bbr_ts_convert(to->to_tsecr);  in bbr_update_rtt()
6799 	if ((to->to_flags & TOF_TS) &&  in bbr_update_rtt()
6800 	    (to->to_tsecr != 0) &&  in bbr_update_rtt()
6802 	    ((rsm->r_flags & BBR_OVERMAX) == 0)) {  in bbr_update_rtt()
6810 		for (i = 0; i < rsm->r_rtr_cnt; i++) {  in bbr_update_rtt()
6811 			if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&  in bbr_update_rtt()
6812 			    (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {  in bbr_update_rtt()
6813 				if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))  in bbr_update_rtt()
6814 					t = cts - rsm->r_tim_lastsent[i];  in bbr_update_rtt()
6817 				bbr->r_ctl.rc_last_rtt = t;  in bbr_update_rtt()
6818 				bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,  in bbr_update_rtt()
6819 						    rsm->r_tim_lastsent[i], ack_type, to);  in bbr_update_rtt()
6820 				if ((i + 1) < rsm->r_rtr_cnt) {  in bbr_update_rtt()
6823 				} else if (rsm->r_flags & BBR_TLP) {  in bbr_update_rtt()
6824 					bbr->rc_tlp_rtx_out = 0;  in bbr_update_rtt()
6834 	 * time-stamp since its not there or the time the peer last  in bbr_update_rtt()
6835 	 * received a segment that moved forward its cum-ack point.  in bbr_update_rtt()
6841 	i = rsm->r_rtr_cnt - 1;  in bbr_update_rtt()
6842 	if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))  in bbr_update_rtt()
6843 		t = cts - rsm->r_tim_lastsent[i];  in bbr_update_rtt()
6846 	if (t < bbr->r_ctl.rc_lowest_rtt) {  in bbr_update_rtt()
6852 		 * the rack-draft.  in bbr_update_rtt()
6858 		if ((rsm->r_flags & BBR_OVERMAX) == 0) {  in bbr_update_rtt()
6860 			if (rsm->r_rtr_cnt == 1)  in bbr_update_rtt()
6861 				panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);  in bbr_update_rtt()
6863 			i = rsm->r_rtr_cnt - 2;  in bbr_update_rtt()
6864 			if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))  in bbr_update_rtt()
6865 				t = cts - rsm->r_tim_lastsent[i];  in bbr_update_rtt()
6868 			bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,  in bbr_update_rtt()
6869 					    rsm->r_tim_lastsent[i], ack_type, to);  in bbr_update_rtt()
6877 			bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,  in bbr_update_rtt()
6885 		if (rsm->r_flags & BBR_TLP)  in bbr_update_rtt()
6886 			bbr->rc_tlp_rtx_out = 0;  in bbr_update_rtt()
6887 		if ((rsm->r_flags & BBR_OVERMAX) == 0)  in bbr_update_rtt()
6888 			bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,  in bbr_update_rtt()
6891 			bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,  in bbr_update_rtt()
6908 	TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,  in bbr_log_sack_passed()
6914 		if (nrsm->r_flags & BBR_ACKED) {  in bbr_log_sack_passed()
6918 		if (nrsm->r_flags & BBR_SACK_PASSED) {  in bbr_log_sack_passed()
6927 		nrsm->r_flags |= BBR_SACK_PASSED;  in bbr_log_sack_passed()
6928 		if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) &&  in bbr_log_sack_passed()
6929 		    bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {  in bbr_log_sack_passed()
6930 			bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;  in bbr_log_sack_passed()
6931 			bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;  in bbr_log_sack_passed()
6932 			nrsm->r_flags |= BBR_MARKED_LOST;  in bbr_log_sack_passed()
6934 		nrsm->r_flags &= ~BBR_WAS_SACKPASS;  in bbr_log_sack_passed()
6952 	start = sack->start;  in bbr_proc_sack_blk()
6953 	end = sack->end;  in bbr_proc_sack_blk()
6959 	if (rsm && SEQ_LT(start, rsm->r_start)) {  in bbr_proc_sack_blk()
6961 		TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_proc_sack_blk()
6962 			if (SEQ_GEQ(start, rsm->r_start) &&  in bbr_proc_sack_blk()
6963 			    SEQ_LT(start, rsm->r_end)) {  in bbr_proc_sack_blk()
6974 	TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_proc_sack_blk()
6975 		if (SEQ_GEQ(start, rsm->r_start) &&  in bbr_proc_sack_blk()
6976 		    SEQ_LT(start, rsm->r_end)) {  in bbr_proc_sack_blk()
6988 		if (tp->t_flags & TF_SENTFIN) {  in bbr_proc_sack_blk()
6993 			nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);  in bbr_proc_sack_blk()
6994 			if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {  in bbr_proc_sack_blk()
6998 				nrsm->r_end++;  in bbr_proc_sack_blk()
7017 	if (rsm->r_start != start) {  in bbr_proc_sack_blk()
7031 			sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);  in bbr_proc_sack_blk()
7035 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_proc_sack_blk()
7036 		if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7037 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_proc_sack_blk()
7038 			nrsm->r_in_tmap = 1;  in bbr_proc_sack_blk()
7040 		rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_proc_sack_blk()
7043 	if (SEQ_GEQ(end, rsm->r_end)) {  in bbr_proc_sack_blk()
7048 		if ((rsm->r_flags & BBR_ACKED) == 0) {  in bbr_proc_sack_blk()
7050 			changed += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7051 			bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7053 			if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7054 				bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7057 			if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_proc_sack_blk()
7059 				bbr->r_ctl.rc_reorder_ts = cts;  in bbr_proc_sack_blk()
7060 				if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7061 					bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7062 					if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_proc_sack_blk()
7064 						bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_proc_sack_blk()
7067 			rsm->r_flags |= BBR_ACKED;  in bbr_proc_sack_blk()
7068 			rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);  in bbr_proc_sack_blk()
7069 			if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7070 				TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_proc_sack_blk()
7071 				rsm->r_in_tmap = 0;  in bbr_proc_sack_blk()
7075 		if (end == rsm->r_end) {  in bbr_proc_sack_blk()
7076 			/* This block only - done */  in bbr_proc_sack_blk()
7080 		start = rsm->r_end;  in bbr_proc_sack_blk()
7086 	if (rsm->r_flags & BBR_ACKED) {  in bbr_proc_sack_blk()
7101 		sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);  in bbr_proc_sack_blk()
7107 	rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_proc_sack_blk()
7108 	TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_proc_sack_blk()
7109 	if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7110 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_proc_sack_blk()
7111 		nrsm->r_in_tmap = 1;  in bbr_proc_sack_blk()
7113 	nrsm->r_dupack = 0;  in bbr_proc_sack_blk()
7116 	changed += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7117 	bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7120 	if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7121 		bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7123 	if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_proc_sack_blk()
7125 		bbr->r_ctl.rc_reorder_ts = cts;  in bbr_proc_sack_blk()
7126 		if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7127 			bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7128 			if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_proc_sack_blk()
7130 				bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_proc_sack_blk()
7133 	rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);  in bbr_proc_sack_blk()
7134 	rsm->r_flags |= BBR_ACKED;  in bbr_proc_sack_blk()
7135 	if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7136 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_proc_sack_blk()
7137 		rsm->r_in_tmap = 0;  in bbr_proc_sack_blk()
7140 	if (rsm && (rsm->r_flags & BBR_ACKED)) {  in bbr_proc_sack_blk()
7148 		    (nrsm->r_flags & BBR_ACKED)) {  in bbr_proc_sack_blk()
7155 		    (nrsm->r_flags & BBR_ACKED)) {  in bbr_proc_sack_blk()
7174 		bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);  in bbr_proc_sack_blk()
7176 		bbr->r_ctl.rc_sacklast = NULL;  in bbr_proc_sack_blk()
7188 	while (rsm && (rsm->r_flags & BBR_ACKED)) {  in bbr_peer_reneges()
7191 		bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);  in bbr_peer_reneges()
7193 		if (rsm->r_in_tmap) {  in bbr_peer_reneges()
7195 			    bbr, rsm, rsm->r_flags);  in bbr_peer_reneges()
7198 		oflags = rsm->r_flags;  in bbr_peer_reneges()
7199 		if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_peer_reneges()
7200 			bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_peer_reneges()
7201 			bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_peer_reneges()
7202 			if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_peer_reneges()
7204 				bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_peer_reneges()
7206 		rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);  in bbr_peer_reneges()
7207 		rsm->r_flags |= BBR_WAS_RENEGED;  in bbr_peer_reneges()
7208 		rsm->r_flags |= BBR_RXT_CLEARED;  in bbr_peer_reneges()
7209 		bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);  in bbr_peer_reneges()
7212 			TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_peer_reneges()
7215 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);  in bbr_peer_reneges()
7218 		tmap->r_in_tmap = 1;  in bbr_peer_reneges()
7233 	sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);  in bbr_peer_reneges()
7243 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_log_syn()
7244 	cts = bbr->r_ctl.rc_rcvtime;  in bbr_log_syn()
7245 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_syn()
7246 	if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {  in bbr_log_syn()
7247 		if ((rsm->r_end - rsm->r_start) <= 1) {  in bbr_log_syn()
7249 			bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;  in bbr_log_syn()
7250 			rsm->r_rtr_bytes = 0;  in bbr_log_syn()
7251 			TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_log_syn()
7252 			if (rsm->r_in_tmap) {  in bbr_log_syn()
7253 				TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_log_syn()
7254 				rsm->r_in_tmap = 0;  in bbr_log_syn()
7256 			if (bbr->r_ctl.rc_next == rsm) {  in bbr_log_syn()
7258 				bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_syn()
7265 			rsm->r_flags &= ~BBR_HAS_SYN;  in bbr_log_syn()
7266 			rsm->r_start++;  in bbr_log_syn()
7294 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_log_ack()
7295 	cts = bbr->r_ctl.rc_rcvtime;  in bbr_log_ack()
7297 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7299 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_log_ack()
7300 	p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);  in bbr_log_ack()
7301 	th_ack = th->th_ack;  in bbr_log_ack()
7302 	if (SEQ_GT(th_ack, tp->snd_una)) {  in bbr_log_ack()
7304 		bbr->rc_tp->t_acktime = ticks;  in bbr_log_ack()
7306 	if (SEQ_LEQ(th_ack, tp->snd_una)) {  in bbr_log_ack()
7310 	if (rsm && SEQ_GT(th_ack, rsm->r_start)) {  in bbr_log_ack()
7311 		changed = th_ack - rsm->r_start;  in bbr_log_ack()
7312 	} else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {  in bbr_log_ack()
7318 		if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) {  in bbr_log_ack()
7325 			ts = bbr_ts_convert(to->to_tsecr);  in bbr_log_ack()
7326 			now = bbr_ts_convert(tcp_tv_to_msec(&bbr->rc_tv));  in bbr_log_ack()
7327 			rtt = now - ts;  in bbr_log_ack()
7331 						tp->iss, 0, cts,  in bbr_log_ack()
7332 						BBR_RTT_BY_TIMESTAMP, tp->iss, 0);  in bbr_log_ack()
7333 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in bbr_log_ack()
7335 			bbr->r_wanted_output = 1;  in bbr_log_ack()
7348 		bbr->r_wanted_output = 1;  in bbr_log_ack()
7351 			if (tp->t_flags & TF_SENTFIN) {  in bbr_log_ack()
7358 			    th, tp->t_state, bbr,  in bbr_log_ack()
7359 			    tp->snd_una, tp->snd_max, changed);  in bbr_log_ack()
7364 	if (SEQ_LT(th_ack, rsm->r_start)) {  in bbr_log_ack()
7368 		    rsm->r_start,  in bbr_log_ack()
7369 		    th_ack, tp->t_state,  in bbr_log_ack()
7370 		    bbr->r_state, bbr);  in bbr_log_ack()
7371 		panic("th-ack is bad bbr:%p tp:%p", bbr, tp);  in bbr_log_ack()
7374 	} else if (th_ack == rsm->r_start) {  in bbr_log_ack()
7383 	rsm->r_dupack = 0;  in bbr_log_ack()
7385 	if (SEQ_GEQ(th_ack, rsm->r_end)) {  in bbr_log_ack()
7389 		if (rsm->r_flags & BBR_ACKED) {  in bbr_log_ack()
7391 			 * It was acked on the scoreboard -- remove it from  in bbr_log_ack()
7394 			p_acked += (rsm->r_end - rsm->r_start);  in bbr_log_ack()
7395 			bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);  in bbr_log_ack()
7396 			if (bbr->r_ctl.rc_sacked == 0)  in bbr_log_ack()
7397 				bbr->r_ctl.rc_sacklast = NULL;  in bbr_log_ack()
7400 			if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_log_ack()
7401 				bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_log_ack()
7403 			if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_log_ack()
7410 				bbr->r_ctl.rc_reorder_ts = cts;  in bbr_log_ack()
7411 				if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_log_ack()
7412 					bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_log_ack()
7413 					if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_log_ack()
7415 						bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_log_ack()
7418 			rsm->r_flags &= ~BBR_MARKED_LOST;  in bbr_log_ack()
7420 		bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;  in bbr_log_ack()
7421 		rsm->r_rtr_bytes = 0;  in bbr_log_ack()
7422 		TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_log_ack()
7423 		if (rsm->r_in_tmap) {  in bbr_log_ack()
7424 			TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_log_ack()
7425 			rsm->r_in_tmap = 0;  in bbr_log_ack()
7427 		if (bbr->r_ctl.rc_next == rsm) {  in bbr_log_ack()
7429 			bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7433 		left = th_ack - rsm->r_end;  in bbr_log_ack()
7437 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7442 	if (rsm->r_flags & BBR_ACKED) {  in bbr_log_ack()
7444 		 * It was acked on the scoreboard -- remove it from total  in bbr_log_ack()
7445 		 * for the part being cum-acked.  in bbr_log_ack()
7447 		p_acked += (rsm->r_end - rsm->r_start);  in bbr_log_ack()
7448 		bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);  in bbr_log_ack()
7449 		if (bbr->r_ctl.rc_sacked == 0)  in bbr_log_ack()
7450 			bbr->r_ctl.rc_sacklast = NULL;  in bbr_log_ack()
7461 	if ((rsm->r_flags & BBR_MARKED_LOST) &&  in bbr_log_ack()
7462 	    ((rsm->r_flags & BBR_ACKED) == 0)) {  in bbr_log_ack()
7468 		bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;  in bbr_log_ack()
7471 	bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;  in bbr_log_ack()
7472 	rsm->r_rtr_bytes = 0;  in bbr_log_ack()
7474 	rsm->r_start = th_ack;  in bbr_log_ack()
7477 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7478 	if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {  in bbr_log_ack()
7484 		 * us snd_una up to (rsm->r_end). We need to undo the acked  in bbr_log_ack()
7488 		 * rsm->r_start in case we get an old ack where th_ack is  in bbr_log_ack()
7491 		bbr_peer_reneges(bbr, rsm, th->th_ack);  in bbr_log_ack()
7493 	if ((to->to_flags & TOF_SACK) == 0) {  in bbr_log_ack()
7497 	rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);  in bbr_log_ack()
7499 		last_seq = rsm->r_end;  in bbr_log_ack()
7501 		last_seq = tp->snd_max;  in bbr_log_ack()
7504 	if (SEQ_GT(th_ack, tp->snd_una))  in bbr_log_ack()
7507 		ack_point = tp->snd_una;  in bbr_log_ack()
7508 	for (i = 0; i < to->to_nsacks; i++) {  in bbr_log_ack()
7509 		bcopy((to->to_sacks + i * TCPOLEN_SACK),  in bbr_log_ack()
7515 		    SEQ_LT(sack.start, tp->snd_max) &&  in bbr_log_ack()
7517 		    SEQ_LEQ(sack.end, tp->snd_max)) {  in bbr_log_ack()
7518 			if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&  in bbr_log_ack()
7520 			    ((sack.end - sack.start) < (p_maxseg / 8))) {  in bbr_log_ack()
7533 			 * Its a D-SACK block.  in bbr_log_ack()
7544 	new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks,  in bbr_log_ack()
7545 				  num_sack_blks, th->th_ack);  in bbr_log_ack()
7546 	ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);  in bbr_log_ack()
7548 	BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb));  in bbr_log_ack()
7586 					 * Now collapse out the dup-sack and  in bbr_log_ack()
7594 					num_sack_blks--;  in bbr_log_ack()
7601 	rsm = bbr->r_ctl.rc_sacklast;  in bbr_log_ack()
7605 			bbr->r_wanted_output = 1;  in bbr_log_ack()
7612 	if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {  in bbr_log_ack()
7623 			bbr->r_wanted_output = 1;  in bbr_log_ack()
7628 			if (bbr->r_ctl.rc_resend == NULL) {  in bbr_log_ack()
7629 				bbr->r_ctl.rc_resend = rsm;  in bbr_log_ack()
7633 	if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {  in bbr_log_ack()
7635 		 * See if we need to rack-retransmit anything if so set it  in bbr_log_ack()
7639 		if (bbr->r_ctl.rc_resend == NULL) {  in bbr_log_ack()
7640 			bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);  in bbr_log_ack()
7645 	 * ack-received code to augment what was changed between th_ack <->  in bbr_log_ack()
7656 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_strike_dupack()
7657 	if (rsm && (rsm->r_dupack < 0xff)) {  in bbr_strike_dupack()
7658 		rsm->r_dupack++;  in bbr_strike_dupack()
7659 		if (rsm->r_dupack >= DUP_ACK_THRESHOLD)  in bbr_strike_dupack()
7660 			bbr->r_wanted_output = 1;  in bbr_strike_dupack()
7667  * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7685 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_process_ack()
7686 	lost = bbr->r_ctl.rc_lost;  in bbr_process_ack()
7687 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_process_ack()
7688 	if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) {  in bbr_process_ack()
7690 		tp->t_flags2 |= TF2_NO_ISS_CHECK;  in bbr_process_ack()
7696 		if (tp->t_flags2 & TF2_NO_ISS_CHECK) {  in bbr_process_ack()
7698 			seq_min = tp->snd_una - tp->max_sndwnd;  in bbr_process_ack()
7701 			if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) {  in bbr_process_ack()
7703 				seq_min = tp->iss + 1;  in bbr_process_ack()
7710 				seq_min = tp->snd_una - tp->max_sndwnd;  in bbr_process_ack()
7714 		if (SEQ_LT(th->th_ack, seq_min)) {  in bbr_process_ack()
7721 			bbr->r_wanted_output = 1;  in bbr_process_ack()
7725 	if (SEQ_GT(th->th_ack, tp->snd_max)) {  in bbr_process_ack()
7727 		bbr->r_wanted_output = 1;  in bbr_process_ack()
7730 	if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {  in bbr_process_ack()
7732 		if (bbr->rc_in_persist)  in bbr_process_ack()
7733 			tp->t_rxtshift = 0;  in bbr_process_ack()
7734 		if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd))  in bbr_process_ack()
7738 	bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));  in bbr_process_ack()
7739 	if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {  in bbr_process_ack()
7744 		if (th->th_ack == tp->snd_una) {  in bbr_process_ack()
7746 			if (bbr->r_state == TCPS_SYN_SENT) {  in bbr_process_ack()
7749 				 * the SYN-ACK is processed in syn_sent  in bbr_process_ack()
7766 	if (tp->t_flags & TF_NEEDSYN) {  in bbr_process_ack()
7768 		 * T/TCP: Connection was half-synchronized, and our SYN has  in bbr_process_ack()
7770 		 * to non-starred state, increment snd_una for ACK of SYN,  in bbr_process_ack()
7773 		tp->t_flags &= ~TF_NEEDSYN;  in bbr_process_ack()
7774 		tp->snd_una++;  in bbr_process_ack()
7776 		if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==  in bbr_process_ack()
7778 			tp->rcv_scale = tp->request_r_scale;  in bbr_process_ack()
7794 	if (tp->t_flags & TF_PREVVALID) {  in bbr_process_ack()
7795 		tp->t_flags &= ~TF_PREVVALID;  in bbr_process_ack()
7796 		if (tp->t_rxtshift == 1 &&  in bbr_process_ack()
7797 		    (int)(ticks - tp->t_badrxtwin) < 0)  in bbr_process_ack()
7801 	acked_amount = min(acked, (int)sbavail(&so->so_snd));  in bbr_process_ack()
7802 	tp->snd_wnd -= acked_amount;  in bbr_process_ack()
7803 	mfree = sbcut_locked(&so->so_snd, acked_amount);  in bbr_process_ack()
7807 	if (SEQ_GT(th->th_ack, tp->snd_una)) {  in bbr_process_ack()
7810 	tp->snd_una = th->th_ack;  in bbr_process_ack()
7811 …bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - los…  in bbr_process_ack()
7812 	if (IN_RECOVERY(tp->t_flags)) {  in bbr_process_ack()
7813 		if (SEQ_LT(th->th_ack, tp->snd_recover) &&  in bbr_process_ack()
7814 		    (SEQ_LT(th->th_ack, tp->snd_max))) {  in bbr_process_ack()
7820 	if (SEQ_GT(tp->snd_una, tp->snd_recover)) {  in bbr_process_ack()
7821 		tp->snd_recover = tp->snd_una;  in bbr_process_ack()
7823 	if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_process_ack()
7824 		tp->snd_nxt = tp->snd_max;  in bbr_process_ack()
7826 	if (tp->snd_una == tp->snd_max) {  in bbr_process_ack()
7830 		if (sbavail(&so->so_snd) == 0)  in bbr_process_ack()
7831 			bbr->rc_tp->t_acktime = 0;  in bbr_process_ack()
7832 		if ((sbused(&so->so_snd) == 0) &&  in bbr_process_ack()
7833 		    (tp->t_flags & TF_SENTFIN)) {  in bbr_process_ack()
7836 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_ack()
7837 		if (bbr->rc_in_persist == 0) {  in bbr_process_ack()
7838 			bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;  in bbr_process_ack()
7840 		sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_process_ack()
7841 		bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_process_ack()
7847 		if ((tp->t_state >= TCPS_FIN_WAIT_1) &&  in bbr_process_ack()
7848 		    (sbavail(&so->so_snd) == 0) &&  in bbr_process_ack()
7849 		    (tp->t_flags2 & TF2_DROP_AF_DATA)) {  in bbr_process_ack()
7856 			/* tcp_close will kill the inp pre-log the Reset */  in bbr_process_ack()
7864 		bbr->r_wanted_output = 1;  in bbr_process_ack()
7874 	if (bbr->rc_in_persist == 0) {  in bbr_enter_persist()
7876 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_enter_persist()
7877 		tp->t_rxtshift = 0;  in bbr_enter_persist()
7878 		bbr->rc_in_persist = 1;  in bbr_enter_persist()
7879 		bbr->r_ctl.rc_went_idle_time = cts;  in bbr_enter_persist()
7883 		if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_enter_persist()
7886 			time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_enter_persist()
7887 			if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_enter_persist()
7893 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_enter_persist()
7896 		bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_enter_persist()
7909 		if (bbr->rc_use_idle_restart) {  in bbr_restart_after_idle()
7910 			bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;  in bbr_restart_after_idle()
7917 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_restart_after_idle()
7918 			bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_restart_after_idle()
7921 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;  in bbr_restart_after_idle()
7922 			bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;  in bbr_restart_after_idle()
7924 		} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_restart_after_idle()
7935 	if (bbr->rc_in_persist == 0)  in bbr_exit_persist()
7937 	idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);  in bbr_exit_persist()
7938 	bbr->rc_in_persist = 0;  in bbr_exit_persist()
7939 	bbr->rc_hit_state_1 = 0;  in bbr_exit_persist()
7940 	bbr->r_ctl.rc_del_time = cts;  in bbr_exit_persist()
7946 	if (tcp_in_hpts(bbr->rc_tp)) {  in bbr_exit_persist()
7947 		tcp_hpts_remove(bbr->rc_tp);  in bbr_exit_persist()
7948 		bbr->rc_timer_first = 0;  in bbr_exit_persist()
7949 		bbr->r_ctl.rc_hpts_flags = 0;  in bbr_exit_persist()
7950 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_exit_persist()
7951 		bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_exit_persist()
7952 		bbr->r_agg_early_set = 0;  in bbr_exit_persist()
7953 		bbr->r_ctl.rc_agg_early = 0;  in bbr_exit_persist()
7963 		bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_exit_persist()
7965 	tp->t_rxtshift = 0;  in bbr_exit_persist()
7976 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_exit_persist()
7977 	/* Time un-freezes for the state */  in bbr_exit_persist()
7978 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_exit_persist()
7979 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||  in bbr_exit_persist()
7980 	    (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {  in bbr_exit_persist()
7982 		 * If we are going back to probe-bw  in bbr_exit_persist()
8015 	maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in bbr_collapsed_window()
8016 	max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;  in bbr_collapsed_window()
8018 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_collapsed_window()
8020 		if (rsm->r_flags & BBR_RWND_COLLAPSED)  in bbr_collapsed_window()
8021 			rsm->r_flags &= ~BBR_RWND_COLLAPSED;  in bbr_collapsed_window()
8022 		if (SEQ_GEQ(max_seq, rsm->r_start) &&  in bbr_collapsed_window()
8023 		    SEQ_GEQ(rsm->r_end, max_seq)) {  in bbr_collapsed_window()
8028 	bbr->rc_has_collapsed = 0;  in bbr_collapsed_window()
8041 	if ((max_seq != rsm->r_start) &&  in bbr_collapsed_window()
8042 	    (max_seq != rsm->r_end)){  in bbr_collapsed_window()
8046 		res1 = max_seq - rsm->r_start;  in bbr_collapsed_window()
8047 		res2 = rsm->r_end - max_seq;  in bbr_collapsed_window()
8052 		} else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {  in bbr_collapsed_window()
8058 	if (max_seq == rsm->r_start) {  in bbr_collapsed_window()
8061 	} else if (max_seq == rsm->r_end) {  in bbr_collapsed_window()
8068 	} else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {  in bbr_collapsed_window()
8079 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_collapsed_window()
8080 		if (rsm->r_in_tmap) {  in bbr_collapsed_window()
8081 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_collapsed_window()
8082 			nrsm->r_in_tmap = 1;  in bbr_collapsed_window()
8095 	TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_collapsed_window()
8096 		nrsm->r_flags |= BBR_RWND_COLLAPSED;  in bbr_collapsed_window()
8098 		bbr->rc_has_collapsed = 1;  in bbr_collapsed_window()
8109 	TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_un_collapse_window()
8110 		if (rsm->r_flags & BBR_RWND_COLLAPSED) {  in bbr_un_collapse_window()
8112 			rsm->r_flags &= ~BBR_RWND_COLLAPSED;  in bbr_un_collapse_window()
8118 				   (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);  in bbr_un_collapse_window()
8119 	bbr->rc_has_collapsed = 0;  in bbr_un_collapse_window()
8140 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_process_data()
8142 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_process_data()
8144 	    (SEQ_LT(tp->snd_wl1, th->th_seq) ||  in bbr_process_data()
8145 	    (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||  in bbr_process_data()
8146 	    (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {  in bbr_process_data()
8149 		    tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)  in bbr_process_data()
8151 		tp->snd_wnd = tiwin;  in bbr_process_data()
8152 		tp->snd_wl1 = th->th_seq;  in bbr_process_data()
8153 		tp->snd_wl2 = th->th_ack;  in bbr_process_data()
8154 		if (tp->snd_wnd > tp->max_sndwnd)  in bbr_process_data()
8155 			tp->max_sndwnd = tp->snd_wnd;  in bbr_process_data()
8156 		bbr->r_wanted_output = 1;  in bbr_process_data()
8158 		if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {  in bbr_process_data()
8159 			tp->snd_wnd = tiwin;  in bbr_process_data()
8160 			tp->snd_wl1 = th->th_seq;  in bbr_process_data()
8161 			tp->snd_wl2 = th->th_ack;  in bbr_process_data()
8164 	if (tp->snd_wnd < ctf_outstanding(tp))  in bbr_process_data()
8167  	else if (bbr->rc_has_collapsed)  in bbr_process_data()
8170 	if ((bbr->rc_in_persist != 0) &&  in bbr_process_data()
8171 	    (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),  in bbr_process_data()
8177 		bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_process_data()
8180 		bbr->r_wanted_output = 1;  in bbr_process_data()
8183 	if ((bbr->rc_in_persist == 0) &&  in bbr_process_data()
8184 	    (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_process_data()
8185 	    TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_process_data()
8186 	    (tp->snd_max == tp->snd_una) &&  in bbr_process_data()
8187 	    sbavail(&so->so_snd) &&  in bbr_process_data()
8188 	    (sbavail(&so->so_snd) > tp->snd_wnd)) {  in bbr_process_data()
8190 		bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_process_data()
8192 	if (tp->t_flags2 & TF2_DROP_AF_DATA) {  in bbr_process_data()
8202 	tp->rcv_up = tp->rcv_nxt;  in bbr_process_data()
8207 	 * This process logically involves adjusting tp->rcv_wnd as data is  in bbr_process_data()
8212 	tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&  in bbr_process_data()
8213 	    (tp->t_flags & TF_FASTOPEN));  in bbr_process_data()
8215 	    TCPS_HAVERCVDFIN(tp->t_state) == 0) {  in bbr_process_data()
8216 		tcp_seq save_start = th->th_seq;  in bbr_process_data()
8217 		tcp_seq save_rnxt  = tp->rcv_nxt;  in bbr_process_data()
8232 		if (th->th_seq == tp->rcv_nxt &&  in bbr_process_data()
8234 		    (TCPS_HAVEESTABLISHED(tp->t_state) ||  in bbr_process_data()
8239 			if (so->so_rcv.sb_shlim) {  in bbr_process_data()
8242 				if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,  in bbr_process_data()
8252 				bbr->bbr_segs_rcvd += max(1, nsegs);  in bbr_process_data()
8253 				tp->t_flags |= TF_DELACK;  in bbr_process_data()
8254 				bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_data()
8256 				bbr->r_wanted_output = 1;  in bbr_process_data()
8257 				tp->t_flags |= TF_ACKNOW;  in bbr_process_data()
8259 			tp->rcv_nxt += tlen;  in bbr_process_data()
8261 			    ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&  in bbr_process_data()
8262 			    (tp->t_fbyte_in == 0)) {  in bbr_process_data()
8263 				tp->t_fbyte_in = ticks;  in bbr_process_data()
8264 				if (tp->t_fbyte_in == 0)  in bbr_process_data()
8265 					tp->t_fbyte_in = 1;  in bbr_process_data()
8266 				if (tp->t_fbyte_out && tp->t_fbyte_in)  in bbr_process_data()
8267 					tp->t_flags2 |= TF2_FBYTES_COMPLETE;  in bbr_process_data()
8273 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)  in bbr_process_data()
8279 					sbappendstream_locked(&so->so_rcv, m, 0);  in bbr_process_data()
8283 			if (so->so_rcv.sb_shlim && appended != mcnt)  in bbr_process_data()
8284 				counter_fo_release(so->so_rcv.sb_shlim,  in bbr_process_data()
8285 				    mcnt - appended);  in bbr_process_data()
8298 			tp->t_flags |= TF_ACKNOW;  in bbr_process_data()
8299 			if (tp->t_flags & TF_WAKESOR) {  in bbr_process_data()
8300 				tp->t_flags &= ~TF_WAKESOR;  in bbr_process_data()
8305 		if ((tp->t_flags & TF_SACK_PERMIT) &&  in bbr_process_data()
8307 		    TCPS_HAVEESTABLISHED(tp->t_state)) {  in bbr_process_data()
8315 			} else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {  in bbr_process_data()
8316 				if ((tp->rcv_numsacks >= 1) &&  in bbr_process_data()
8317 				    (tp->sackblks[0].end == save_start)) {  in bbr_process_data()
8323 					    tp->sackblks[0].start,  in bbr_process_data()
8324 					    tp->sackblks[0].end);  in bbr_process_data()
8348 		if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {  in bbr_process_data()
8352 			 * If connection is half-synchronized (ie NEEDSYN  in bbr_process_data()
8358 			if (tp->t_flags & TF_NEEDSYN) {  in bbr_process_data()
8359 				tp->t_flags |= TF_DELACK;  in bbr_process_data()
8361 				    __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_data()
8363 				tp->t_flags |= TF_ACKNOW;  in bbr_process_data()
8365 			tp->rcv_nxt++;  in bbr_process_data()
8367 		switch (tp->t_state) {  in bbr_process_data()
8373 			tp->t_starttime = ticks;  in bbr_process_data()
8389 			 * starting the time-wait timer, turning off the  in bbr_process_data()
8393 			bbr->rc_timer_first = 1;  in bbr_process_data()
8395 			    __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_data()
8403 	if ((tp->t_flags & TF_ACKNOW) ||  in bbr_process_data()
8404 	    (sbavail(&so->so_snd) > ctf_outstanding(tp))) {  in bbr_process_data()
8405 		bbr->r_wanted_output = 1;  in bbr_process_data()
8412  * have broken out the fast-data path also just like
8413  * the fast-ack. Return 1 if we processed the packet
8414  * return 0 if you need to take the "slow-path".
8429 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_fastnewdata()
8436 	if (bbr->r_ctl.rc_resend != NULL) {  in bbr_do_fastnewdata()
8439 	if (tiwin && tiwin != tp->snd_wnd) {  in bbr_do_fastnewdata()
8442 	if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {  in bbr_do_fastnewdata()
8445 	if (__predict_false((to->to_flags & TOF_TS) &&  in bbr_do_fastnewdata()
8446 	    (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {  in bbr_do_fastnewdata()
8449 	if (__predict_false((th->th_ack != tp->snd_una))) {  in bbr_do_fastnewdata()
8452 	if (__predict_false(tlen > sbspace(&so->so_rcv))) {  in bbr_do_fastnewdata()
8455 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_fastnewdata()
8456 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {  in bbr_do_fastnewdata()
8457 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_fastnewdata()
8458 		tp->ts_recent = to->to_tsval;  in bbr_do_fastnewdata()
8461 	 * This is a pure, in-sequence data packet with nothing on the  in bbr_do_fastnewdata()
8464 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_do_fastnewdata()
8467 	if (so->so_rcv.sb_shlim) {  in bbr_do_fastnewdata()
8470 		if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,  in bbr_do_fastnewdata()
8479 	if (tp->rcv_numsacks)  in bbr_do_fastnewdata()
8482 	tp->rcv_nxt += tlen;  in bbr_do_fastnewdata()
8484 	    ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&  in bbr_do_fastnewdata()
8485 	    (tp->t_fbyte_in == 0)) {  in bbr_do_fastnewdata()
8486 		tp->t_fbyte_in = ticks;  in bbr_do_fastnewdata()
8487 		if (tp->t_fbyte_in == 0)  in bbr_do_fastnewdata()
8488 			tp->t_fbyte_in = 1;  in bbr_do_fastnewdata()
8489 		if (tp->t_fbyte_out && tp->t_fbyte_in)  in bbr_do_fastnewdata()
8490 			tp->t_flags2 |= TF2_FBYTES_COMPLETE;  in bbr_do_fastnewdata()
8495 	tp->snd_wl1 = th->th_seq;  in bbr_do_fastnewdata()
8499 	tp->rcv_up = tp->rcv_nxt;  in bbr_do_fastnewdata()
8506 	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_fastnewdata()
8515 				so->so_rcv.sb_flags &= ~SB_AUTOSIZE;  in bbr_do_fastnewdata()
8521 			sbappendstream_locked(&so->so_rcv, m, 0);  in bbr_do_fastnewdata()
8527 	if (so->so_rcv.sb_shlim && mcnt != appended)  in bbr_do_fastnewdata()
8528 		counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);  in bbr_do_fastnewdata()
8531 		bbr->bbr_segs_rcvd += max(1, nsegs);  in bbr_do_fastnewdata()
8532 		tp->t_flags |= TF_DELACK;  in bbr_do_fastnewdata()
8533 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_do_fastnewdata()
8535 		bbr->r_wanted_output = 1;  in bbr_do_fastnewdata()
8536 		tp->t_flags |= TF_ACKNOW;  in bbr_do_fastnewdata()
8544  * in sequence to remain in the fast-path. We also add
8548  * slow-path. If we return 1, then all is well and
8562 	if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {  in bbr_fastack()
8566 	if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_fastack()
8574 	if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {  in bbr_fastack()
8578 	if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {  in bbr_fastack()
8582 	if (__predict_false(IN_RECOVERY(tp->t_flags))) {  in bbr_fastack()
8586 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_fastack()
8587 	if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {  in bbr_fastack()
8591 	if (__predict_false(bbr->rc_in_persist != 0)) {  in bbr_fastack()
8595 	if (bbr->r_ctl.rc_sacked) {  in bbr_fastack()
8599 	/* Ok if we reach here, we can process a fast-ack */  in bbr_fastack()
8600 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_fastack()
8607 	bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);  in bbr_fastack()
8609 	if (tiwin != tp->snd_wnd) {  in bbr_fastack()
8610 		tp->snd_wnd = tiwin;  in bbr_fastack()
8611 		tp->snd_wl1 = th->th_seq;  in bbr_fastack()
8612 		if (tp->snd_wnd > tp->max_sndwnd)  in bbr_fastack()
8613 			tp->max_sndwnd = tp->snd_wnd;  in bbr_fastack()
8616 	if ((bbr->rc_in_persist != 0) &&  in bbr_fastack()
8617 	    (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),  in bbr_fastack()
8619 		bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_fastack()
8620 		bbr->r_wanted_output = 1;  in bbr_fastack()
8623 	if ((bbr->rc_in_persist == 0) &&  in bbr_fastack()
8624 	    (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_fastack()
8625 	    TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_fastack()
8626 	    (tp->snd_max == tp->snd_una) &&  in bbr_fastack()
8627 	    sbavail(&so->so_snd) &&  in bbr_fastack()
8628 	    (sbavail(&so->so_snd) > tp->snd_wnd)) {  in bbr_fastack()
8630 		bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_fastack()
8637 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_fastack()
8638 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {  in bbr_fastack()
8639 		tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;  in bbr_fastack()
8640 		tp->ts_recent = to->to_tsval;  in bbr_fastack()
8650 	if (tp->t_flags & TF_PREVVALID) {  in bbr_fastack()
8651 		tp->t_flags &= ~TF_PREVVALID;  in bbr_fastack()
8652 		if (tp->t_rxtshift == 1 &&  in bbr_fastack()
8653 		    (int)(ticks - tp->t_badrxtwin) < 0)  in bbr_fastack()
8672 	sbdrop(&so->so_snd, acked);  in bbr_fastack()
8674 	if (SEQ_GT(th->th_ack, tp->snd_una))  in bbr_fastack()
8676 	tp->snd_una = th->th_ack;  in bbr_fastack()
8677 	if (tp->snd_wnd < ctf_outstanding(tp))  in bbr_fastack()
8680 	else if (bbr->rc_has_collapsed)  in bbr_fastack()
8683 	if (SEQ_GT(tp->snd_una, tp->snd_recover)) {  in bbr_fastack()
8684 		tp->snd_recover = tp->snd_una;  in bbr_fastack()
8690 	tp->snd_wl2 = th->th_ack;  in bbr_fastack()
8694 	 * otherwise restart timer using current (possibly backed-off)  in bbr_fastack()
8701 	if (tp->snd_una == tp->snd_max) {  in bbr_fastack()
8704 		if (sbavail(&so->so_snd) == 0)  in bbr_fastack()
8705 			bbr->rc_tp->t_acktime = 0;  in bbr_fastack()
8706 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_fastack()
8707 		if (bbr->rc_in_persist == 0) {  in bbr_fastack()
8708 			bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;  in bbr_fastack()
8710 		sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_fastack()
8711 		bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_fastack()
8717 		bbr->r_wanted_output = 1;  in bbr_fastack()
8719 	if (sbavail(&so->so_snd)) {  in bbr_fastack()
8720 		bbr->r_wanted_output = 1;  in bbr_fastack()
8742 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_syn_sent()
8748 	 * this is an acceptable SYN segment initialize tp->rcv_nxt and  in bbr_do_syn_sent()
8749 	 * tp->irs if seg contains ack then advance tp->snd_una. BRR does  in bbr_do_syn_sent()
8756 	    (SEQ_LEQ(th->th_ack, tp->iss) ||  in bbr_do_syn_sent()
8757 	    SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_do_syn_sent()
8777 	tp->irs = th->th_seq;  in bbr_do_syn_sent()
8788 		if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==  in bbr_do_syn_sent()
8790 			tp->rcv_scale = tp->request_r_scale;  in bbr_do_syn_sent()
8792 		tp->rcv_adv += min(tp->rcv_wnd,  in bbr_do_syn_sent()
8793 		    TCP_MAXWIN << tp->rcv_scale);  in bbr_do_syn_sent()
8798 		if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_do_syn_sent()
8799 		    (tp->snd_una != tp->snd_max)) {  in bbr_do_syn_sent()
8800 			tp->snd_nxt = th->th_ack;  in bbr_do_syn_sent()
8808 			bbr->bbr_segs_rcvd += 1;  in bbr_do_syn_sent()
8809 			tp->t_flags |= TF_DELACK;  in bbr_do_syn_sent()
8810 			bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_do_syn_sent()
8812 			bbr->r_wanted_output = 1;  in bbr_do_syn_sent()
8813 			tp->t_flags |= TF_ACKNOW;  in bbr_do_syn_sent()
8815 		if (SEQ_GT(th->th_ack, tp->iss)) {  in bbr_do_syn_sent()
8822 		if (SEQ_GT(th->th_ack, tp->snd_una)) {  in bbr_do_syn_sent()
8828 			 * ack-processing since the  in bbr_do_syn_sent()
8829 			 * data stream in our send-map  in bbr_do_syn_sent()
8835 			tp->snd_una++;  in bbr_do_syn_sent()
8839 		 * SYN_SENT  --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1  in bbr_do_syn_sent()
8841 		tp->t_starttime = ticks;  in bbr_do_syn_sent()
8842 		if (tp->t_flags & TF_NEEDFIN) {  in bbr_do_syn_sent()
8844 			tp->t_flags &= ~TF_NEEDFIN;  in bbr_do_syn_sent()
8854 		 * Received initial SYN in SYN-SENT[*] state => simultaneous  in bbr_do_syn_sent()
8857 		 * half-synchronized. Otherwise, do 3-way handshake:  in bbr_do_syn_sent()
8858 		 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If  in bbr_do_syn_sent()
8861 		tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN);  in bbr_do_syn_sent()
8865 	 * Advance th->th_seq to correspond to first data byte. If data,  in bbr_do_syn_sent()
8868 	th->th_seq++;  in bbr_do_syn_sent()
8869 	if (tlen > tp->rcv_wnd) {  in bbr_do_syn_sent()
8870 		todrop = tlen - tp->rcv_wnd;  in bbr_do_syn_sent()
8871 		m_adj(m, -todrop);  in bbr_do_syn_sent()
8872 		tlen = tp->rcv_wnd;  in bbr_do_syn_sent()
8877 	tp->snd_wl1 = th->th_seq - 1;  in bbr_do_syn_sent()
8878 	tp->rcv_up = th->th_seq;  in bbr_do_syn_sent()
8886 		if ((to->to_flags & TOF_TS) != 0) {  in bbr_do_syn_sent()
8889 			t = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_syn_sent()
8890 			if (TSTMP_GEQ(t, to->to_tsecr)) {  in bbr_do_syn_sent()
8891 				rtt = t - to->to_tsecr;  in bbr_do_syn_sent()
8897 				apply_filter_min_small(&bbr->r_ctl.rc_rttprop,  in bbr_do_syn_sent()
8898 						       rtt, bbr->r_ctl.rc_rcvtime);  in bbr_do_syn_sent()
8904 		if (tp->t_state == TCPS_FIN_WAIT_1) {  in bbr_do_syn_sent()
8921 				if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_syn_sent()
8952 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_syn_recv()
8955 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_syn_recv()
8958 	    (SEQ_LEQ(th->th_ack, tp->snd_una) ||  in bbr_do_syn_recv()
8959 	     SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_do_syn_recv()
8964 	if (tp->t_flags & TF_FASTOPEN) {  in bbr_do_syn_recv()
8976 			/* non-initial SYN is ignored */  in bbr_do_syn_recv()
8977 			if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||  in bbr_do_syn_recv()
8978 			    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||  in bbr_do_syn_recv()
8979 			    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {  in bbr_do_syn_recv()
8992 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_syn_recv()
8993 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_syn_recv()
8998 	 * In the SYN-RECEIVED state, validate that the packet belongs to  in bbr_do_syn_recv()
9004 	if (SEQ_LT(th->th_seq, tp->irs)) {  in bbr_do_syn_recv()
9026 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_syn_recv()
9027 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_syn_recv()
9028 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_syn_recv()
9030 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_syn_recv()
9031 		tp->ts_recent = to->to_tsval;  in bbr_do_syn_recv()
9033 	tp->snd_wnd = tiwin;  in bbr_do_syn_recv()
9035 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_syn_recv()
9036 	 * is on (half-synchronized state), then queue data for later  in bbr_do_syn_recv()
9040 		if (tp->t_flags & TF_FASTOPEN) {  in bbr_do_syn_recv()
9047 	if (tp->t_flags & TF_SONOTCONN) {  in bbr_do_syn_recv()
9048 		tp->t_flags &= ~TF_SONOTCONN;  in bbr_do_syn_recv()
9052 	if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==  in bbr_do_syn_recv()
9054 		tp->rcv_scale = tp->request_r_scale;  in bbr_do_syn_recv()
9060 	if ((to->to_flags & TOF_TS) != 0) {  in bbr_do_syn_recv()
9063 		t = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_syn_recv()
9064 		if (TSTMP_GEQ(t, to->to_tsecr)) {  in bbr_do_syn_recv()
9065 			rtt = t - to->to_tsecr;  in bbr_do_syn_recv()
9071 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);  in bbr_do_syn_recv()
9077 	if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) {  in bbr_do_syn_recv()
9078 		tcp_fastopen_decrement_counter(tp->t_tfo_pending);  in bbr_do_syn_recv()
9079 		tp->t_tfo_pending = NULL;  in bbr_do_syn_recv()
9082 	 * Make transitions: SYN-RECEIVED  -> ESTABLISHED SYN-RECEIVED* ->  in bbr_do_syn_recv()
9083 	 * FIN-WAIT-1  in bbr_do_syn_recv()
9085 	tp->t_starttime = ticks;  in bbr_do_syn_recv()
9086 	if (tp->t_flags & TF_NEEDFIN) {  in bbr_do_syn_recv()
9088 		tp->t_flags &= ~TF_NEEDFIN;  in bbr_do_syn_recv()
9099 		if (!(tp->t_flags & TF_FASTOPEN))  in bbr_do_syn_recv()
9107 	if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))  in bbr_do_syn_recv()
9108 		tp->snd_una++;  in bbr_do_syn_recv()
9116 		if (tp->t_flags & TF_WAKESOR) {  in bbr_do_syn_recv()
9117 			tp->t_flags &= ~TF_WAKESOR;  in bbr_do_syn_recv()
9122 	tp->snd_wl1 = th->th_seq - 1;  in bbr_do_syn_recv()
9126 	if (tp->t_state == TCPS_FIN_WAIT_1) {  in bbr_do_syn_recv()
9143 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_syn_recv()
9174 	 * uni-directional data xfer.  If the packet has no control flags,  in bbr_do_established()
9175 	 * is in-sequence, the window didn't change and we're not  in bbr_do_established()
9179 	 * waiting for space.  If the length is non-zero and the ack didn't  in bbr_do_established()
9180 	 * move, we're the receiver side.  If we're getting packets in-order  in bbr_do_established()
9183 	 * hidden state-flags are also off. Since we check for  in bbr_do_established()
9186 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_established()
9187 	if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {  in bbr_do_established()
9193 		bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);  in bbr_do_established()
9195 	if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&  in bbr_do_established()
9198 	    __predict_true(th->th_seq == tp->rcv_nxt)) {  in bbr_do_established()
9214 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_established()
9228 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_established()
9229 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_established()
9250 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_established()
9251 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_established()
9252 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_established()
9254 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_established()
9255 		tp->ts_recent = to->to_tsval;  in bbr_do_established()
9258 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_established()
9259 	 * is on (half-synchronized state), then queue data for later  in bbr_do_established()
9263 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_established()
9266 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_established()
9268 			bbr->r_wanted_output = 1;  in bbr_do_established()
9281 	if (sbavail(&so->so_snd)) {  in bbr_do_established()
9308 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_close_wait()
9311 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_close_wait()
9325 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_close_wait()
9326 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_close_wait()
9347 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_close_wait()
9348 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_close_wait()
9349 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_close_wait()
9351 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_close_wait()
9352 		tp->ts_recent = to->to_tsval;  in bbr_do_close_wait()
9355 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_close_wait()
9356 	 * is on (half-synchronized state), then queue data for later  in bbr_do_close_wait()
9360 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_close_wait()
9363 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_close_wait()
9365 			bbr->r_wanted_output = 1;  in bbr_do_close_wait()
9378 	if (sbavail(&so->so_snd)) {  in bbr_do_close_wait()
9394 	if (bbr->rc_allow_data_af_clo == 0) {  in bbr_check_data_after_close()
9397 		/* tcp_close will kill the inp pre-log the Reset */  in bbr_check_data_after_close()
9404 	if (sbavail(&so->so_snd) == 0)  in bbr_check_data_after_close()
9407 	tp->rcv_nxt = th->th_seq + *tlen;  in bbr_check_data_after_close()
9408 	tp->t_flags2 |= TF2_DROP_AF_DATA;  in bbr_check_data_after_close()
9409 	bbr->r_wanted_output = 1;  in bbr_check_data_after_close()
9430 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_fin_wait_1()
9433 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_fin_wait_1()
9447 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_fin_wait_1()
9448 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_fin_wait_1()
9461 	if ((tp->t_flags & TF_CLOSED) && tlen &&  in bbr_do_fin_wait_1()
9478 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_fin_wait_1()
9479 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_fin_wait_1()
9480 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_fin_wait_1()
9482 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_fin_wait_1()
9483 		tp->ts_recent = to->to_tsval;  in bbr_do_fin_wait_1()
9486 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_fin_wait_1()
9487 	 * is on (half-synchronized state), then queue data for later  in bbr_do_fin_wait_1()
9491 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_fin_wait_1()
9494 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_fin_wait_1()
9496 			bbr->r_wanted_output = 1;  in bbr_do_fin_wait_1()
9519 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_fin_wait_1()
9528 	if (sbavail(&so->so_snd)) {  in bbr_do_fin_wait_1()
9555 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_closing()
9558 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_closing()
9572 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_closing()
9573 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_closing()
9594 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_closing()
9595 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_closing()
9596 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_closing()
9598 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_closing()
9599 		tp->ts_recent = to->to_tsval;  in bbr_do_closing()
9602 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_closing()
9603 	 * is on (half-synchronized state), then queue data for later  in bbr_do_closing()
9607 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_closing()
9610 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_closing()
9612 			bbr->r_wanted_output = 1;  in bbr_do_closing()
9630 	if (sbavail(&so->so_snd)) {  in bbr_do_closing()
9657 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_lastack()
9660 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_lastack()
9674 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_lastack()
9675 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_lastack()
9696 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_lastack()
9697 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_lastack()
9698 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_lastack()
9700 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_lastack()
9701 		tp->ts_recent = to->to_tsval;  in bbr_do_lastack()
9704 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_lastack()
9705 	 * is on (half-synchronized state), then queue data for later  in bbr_do_lastack()
9709 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_lastack()
9712 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_lastack()
9714 			bbr->r_wanted_output = 1;  in bbr_do_lastack()
9732 	if (sbavail(&so->so_snd)) {  in bbr_do_lastack()
9759 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_fin_wait_2()
9763 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_fin_wait_2()
9778 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_fin_wait_2()
9779 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_fin_wait_2()
9793 	if ((tp->t_flags & TF_CLOSED) && tlen &&  in bbr_do_fin_wait_2()
9810 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_fin_wait_2()
9811 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_fin_wait_2()
9812 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_fin_wait_2()
9814 		tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_fin_wait_2()
9815 		tp->ts_recent = to->to_tsval;  in bbr_do_fin_wait_2()
9818 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_fin_wait_2()
9819 	 * is on (half-synchronized state), then queue data for later  in bbr_do_fin_wait_2()
9823 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_fin_wait_2()
9826 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_fin_wait_2()
9828 			bbr->r_wanted_output = 1;  in bbr_do_fin_wait_2()
9841 	if (sbavail(&so->so_snd)) {  in bbr_do_fin_wait_2()
9860 		bbr->rc_in_persist = 1;  in bbr_stop_all_timers()
9862 	if (tcp_in_hpts(bbr->rc_tp)) {  in bbr_stop_all_timers()
9863 		tcp_hpts_remove(bbr->rc_tp);  in bbr_stop_all_timers()
9870 	bbr->rc_use_google = 1;  in bbr_google_mode_on()
9871 	bbr->rc_no_pacing = 0;  in bbr_google_mode_on()
9872 	bbr->r_ctl.bbr_google_discount = bbr_google_discount;  in bbr_google_mode_on()
9873 	bbr->r_use_policer = bbr_policer_detection_enabled;  in bbr_google_mode_on()
9874 	bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);  in bbr_google_mode_on()
9875 	bbr->bbr_use_rack_cheat = 0;  in bbr_google_mode_on()
9876 	bbr->r_ctl.rc_incr_tmrs = 0;  in bbr_google_mode_on()
9877 	bbr->r_ctl.rc_inc_tcp_oh = 0;  in bbr_google_mode_on()
9878 	bbr->r_ctl.rc_inc_ip_oh = 0;  in bbr_google_mode_on()
9879 	bbr->r_ctl.rc_inc_enet_oh = 0;  in bbr_google_mode_on()
9880 	reset_time(&bbr->r_ctl.rc_delrate,  in bbr_google_mode_on()
9882 	reset_time_small(&bbr->r_ctl.rc_rttprop,  in bbr_google_mode_on()
9884 	tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));  in bbr_google_mode_on()
9890 	bbr->rc_use_google = 0;  in bbr_google_mode_off()
9891 	bbr->r_ctl.bbr_google_discount = 0;  in bbr_google_mode_off()
9892 	bbr->no_pacing_until = bbr_no_pacing_until;  in bbr_google_mode_off()
9893 	bbr->r_use_policer = 0;  in bbr_google_mode_off()
9894 	if (bbr->no_pacing_until)  in bbr_google_mode_off()
9895 		bbr->rc_no_pacing = 1;  in bbr_google_mode_off()
9897 		bbr->rc_no_pacing = 0;  in bbr_google_mode_off()
9899 		bbr->bbr_use_rack_cheat = 1;  in bbr_google_mode_off()
9901 		bbr->bbr_use_rack_cheat = 0;  in bbr_google_mode_off()
9903 		bbr->r_ctl.rc_incr_tmrs = 1;  in bbr_google_mode_off()
9905 		bbr->r_ctl.rc_incr_tmrs = 0;  in bbr_google_mode_off()
9907 		bbr->r_ctl.rc_inc_tcp_oh = 1;  in bbr_google_mode_off()
9909 		bbr->r_ctl.rc_inc_tcp_oh = 0;  in bbr_google_mode_off()
9911 		bbr->r_ctl.rc_inc_ip_oh = 1;  in bbr_google_mode_off()
9913 		bbr->r_ctl.rc_inc_ip_oh = 0;  in bbr_google_mode_off()
9915 		bbr->r_ctl.rc_inc_enet_oh = 1;  in bbr_google_mode_off()
9917 		bbr->r_ctl.rc_inc_enet_oh = 0;  in bbr_google_mode_off()
9918 	bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;  in bbr_google_mode_off()
9919 	reset_time(&bbr->r_ctl.rc_delrate,  in bbr_google_mode_off()
9921 	reset_time_small(&bbr->r_ctl.rc_rttprop,  in bbr_google_mode_off()
9923 	tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));  in bbr_google_mode_off()
9926  * Return 0 on success, non-zero on failure
9949 	bbr->rtt_valid = 0;  in bbr_init()
9950 	tp->t_flags2 |= TF2_CANNOT_DO_ECN;  in bbr_init()
9951 	tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;  in bbr_init()
9953 	tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;  in bbr_init()
9954 	tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;  in bbr_init()
9955 	tp->t_flags2 &= ~TF2_MBUF_ACKCMP;  in bbr_init()
9956 	tp->t_flags2 &= ~TF2_MBUF_L_ACKS;  in bbr_init()
9958 	TAILQ_INIT(&bbr->r_ctl.rc_map);  in bbr_init()
9959 	TAILQ_INIT(&bbr->r_ctl.rc_free);  in bbr_init()
9960 	TAILQ_INIT(&bbr->r_ctl.rc_tmap);  in bbr_init()
9961 	bbr->rc_tp = tp;  in bbr_init()
9962 	bbr->rc_inp = inp;  in bbr_init()
9963 	cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_init()
9964 	tp->t_acktime = 0;  in bbr_init()
9965 	bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;  in bbr_init()
9966 	bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;  in bbr_init()
9967 	bbr->rc_tlp_threshold = bbr_tlp_thresh;  in bbr_init()
9968 	bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;  in bbr_init()
9969 	bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;  in bbr_init()
9970 	bbr->r_ctl.rc_min_to = bbr_min_to;  in bbr_init()
9971 	bbr->rc_bbr_state = BBR_STATE_STARTUP;  in bbr_init()
9972 	bbr->r_ctl.bbr_lost_at_state = 0;  in bbr_init()
9973 	bbr->r_ctl.rc_lost_at_startup = 0;  in bbr_init()
9974 	bbr->rc_all_timers_stopped = 0;  in bbr_init()
9975 	bbr->r_ctl.rc_bbr_lastbtlbw = 0;  in bbr_init()
9976 	bbr->r_ctl.rc_pkt_epoch_del = 0;  in bbr_init()
9977 	bbr->r_ctl.rc_pkt_epoch = 0;  in bbr_init()
9978 	bbr->r_ctl.rc_lowest_rtt = 0xffffffff;  in bbr_init()
9979 	bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;  in bbr_init()
9980 	bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;  in bbr_init()
9981 	bbr->r_ctl.rc_went_idle_time = cts;  in bbr_init()
9982 	bbr->rc_pacer_started = cts;  in bbr_init()
9983 	bbr->r_ctl.rc_pkt_epoch_time = cts;  in bbr_init()
9984 	bbr->r_ctl.rc_rcvtime = cts;  in bbr_init()
9985 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_init()
9986 	bbr->r_ctl.rc_del_time = cts;  in bbr_init()
9987 	bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_init()
9988 	bbr->r_ctl.last_in_probertt = cts;  in bbr_init()
9989 	bbr->skip_gain = 0;  in bbr_init()
9990 	bbr->gain_is_limited = 0;  in bbr_init()
9991 	bbr->no_pacing_until = bbr_no_pacing_until;  in bbr_init()
9992 	if (bbr->no_pacing_until)  in bbr_init()
9993 		bbr->rc_no_pacing = 1;  in bbr_init()
9995 		bbr->rc_no_pacing = 0;  in bbr_init()
9996 		bbr->rc_use_google = 1;  in bbr_init()
9997 		bbr->r_ctl.bbr_google_discount = bbr_google_discount;  in bbr_init()
9998 		bbr->r_use_policer = bbr_policer_detection_enabled;  in bbr_init()
10000 		bbr->rc_use_google = 0;  in bbr_init()
10001 		bbr->r_ctl.bbr_google_discount = 0;  in bbr_init()
10002 		bbr->r_use_policer = 0;  in bbr_init()
10005 		bbr->rc_use_ts_limit = 1;  in bbr_init()
10007 		bbr->rc_use_ts_limit = 0;  in bbr_init()
10009 		bbr->ts_can_raise = 1;  in bbr_init()
10011 		bbr->ts_can_raise = 0;  in bbr_init()
10013 		tp->t_delayed_ack = 2;  in bbr_init()
10015 		tp->t_delayed_ack = 0;  in bbr_init()
10017 		tp->t_delayed_ack = V_tcp_delack_enabled;  in bbr_init()
10019 		tp->t_delayed_ack = 2;  in bbr_init()
10020 	if (bbr->rc_use_google == 0)  in bbr_init()
10021 		bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;  in bbr_init()
10023 		bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);  in bbr_init()
10024 	bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;  in bbr_init()
10025 	bbr->rc_max_rto_sec = bbr_rto_max_sec;  in bbr_init()
10026 	bbr->rc_init_win = bbr_def_init_win;  in bbr_init()
10027 	if (tp->t_flags & TF_REQ_TSTMP)  in bbr_init()
10028 		bbr->rc_last_options = TCP_TS_OVERHEAD;  in bbr_init()
10029 	bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;  in bbr_init()
10030 	bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;  in bbr_init()
10031 	bbr->r_init_rtt = 1;  in bbr_init()
10035 		bbr->bbr_hdw_pace_ena = 1;  in bbr_init()
10037 		bbr->bbr_hdw_pace_ena = 0;  in bbr_init()
10039 		bbr->bbr_init_win_cheat = 1;  in bbr_init()
10041 		bbr->bbr_init_win_cheat = 0;  in bbr_init()
10042 	bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;  in bbr_init()
10043 	bbr->r_ctl.rc_drain_pg = bbr_drain_gain;  in bbr_init()
10044 	bbr->r_ctl.rc_startup_pg = bbr_high_gain;  in bbr_init()
10045 	bbr->rc_loss_exit = bbr_exit_startup_at_loss;  in bbr_init()
10046 	bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;  in bbr_init()
10047 	bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;  in bbr_init()
10048 	bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;  in bbr_init()
10049 	bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;  in bbr_init()
10050 	bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;  in bbr_init()
10051 	bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;  in bbr_init()
10052 	bbr->r_ctl.bbr_cross_over = bbr_cross_over;  in bbr_init()
10053 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_init()
10054 	if (bbr->rc_use_google) {  in bbr_init()
10055 		setup_time_filter(&bbr->r_ctl.rc_delrate,  in bbr_init()
10058 		setup_time_filter_small(&bbr->r_ctl.rc_rttprop,  in bbr_init()
10061 		setup_time_filter(&bbr->r_ctl.rc_delrate,  in bbr_init()
10064 		setup_time_filter_small(&bbr->r_ctl.rc_rttprop,  in bbr_init()
10069 		bbr->rc_use_idle_restart = 1;  in bbr_init()
10071 		bbr->rc_use_idle_restart = 0;  in bbr_init()
10072 	bbr->r_ctl.rc_bbr_cur_del_rate = 0;  in bbr_init()
10073 	bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;  in bbr_init()
10075 		bbr->rc_resends_use_tso = 1;  in bbr_init()
10076 	if (tp->snd_una != tp->snd_max) {  in bbr_init()
10086 		rsm->r_rtt_not_allowed = 1;  in bbr_init()
10087 		rsm->r_tim_lastsent[0] = cts;  in bbr_init()
10088 		rsm->r_rtr_cnt = 1;  in bbr_init()
10089 		rsm->r_rtr_bytes = 0;  in bbr_init()
10090 		rsm->r_start = tp->snd_una;  in bbr_init()
10091 		rsm->r_end = tp->snd_max;  in bbr_init()
10092 		rsm->r_dupack = 0;  in bbr_init()
10093 		rsm->r_delivered = bbr->r_ctl.rc_delivered;  in bbr_init()
10094 		rsm->r_ts_valid = 0;  in bbr_init()
10095 		rsm->r_del_ack_ts = tp->ts_recent;  in bbr_init()
10096 		rsm->r_del_time = cts;  in bbr_init()
10097 		if (bbr->r_ctl.r_app_limited_until)  in bbr_init()
10098 			rsm->r_app_limited = 1;  in bbr_init()
10100 			rsm->r_app_limited = 0;  in bbr_init()
10101 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_init()
10102 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_init()
10103 		rsm->r_in_tmap = 1;  in bbr_init()
10104 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)  in bbr_init()
10105 			rsm->r_bbr_state = bbr_state_val(bbr);  in bbr_init()
10107 			rsm->r_bbr_state = 8;  in bbr_init()
10109 	if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))  in bbr_init()
10110 		bbr->bbr_use_rack_cheat = 1;  in bbr_init()
10111 	if (bbr_incr_timers && (bbr->rc_use_google == 0))  in bbr_init()
10112 		bbr->r_ctl.rc_incr_tmrs = 1;  in bbr_init()
10113 	if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))  in bbr_init()
10114 		bbr->r_ctl.rc_inc_tcp_oh = 1;  in bbr_init()
10115 	if (bbr_include_ip_oh && (bbr->rc_use_google == 0))  in bbr_init()
10116 		bbr->r_ctl.rc_inc_ip_oh = 1;  in bbr_init()
10117 	if (bbr_include_enet_oh && (bbr->rc_use_google == 0))  in bbr_init()
10118 		bbr->r_ctl.rc_inc_enet_oh = 1;  in bbr_init()
10121 	if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_init()
10122 	    (tp->t_srtt)) {  in bbr_init()
10125 		rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_init()
10126 		apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in bbr_init()
10144 	TCPT_RANGESET(tp->t_rxtcur,  in bbr_init()
10145 	    ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,  in bbr_init()
10146 	    tp->t_rttmin, tcp_rexmit_max);  in bbr_init()
10153  * non-zero if we can't handle the connection. A EAGAIN
10161 	if ((tp->t_state == TCPS_CLOSED) ||  in bbr_handoff_ok()
10162 	    (tp->t_state == TCPS_LISTEN)) {  in bbr_handoff_ok()
10166 	if ((tp->t_state == TCPS_SYN_SENT) ||  in bbr_handoff_ok()
10167 	    (tp->t_state == TCPS_SYN_RECEIVED)) {  in bbr_handoff_ok()
10174 	if (tp->t_flags & TF_SENTFIN)  in bbr_handoff_ok()
10176 	if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) {  in bbr_handoff_ok()
10189 	if (tp->t_fb_ptr) {  in bbr_fini()
10194 		bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_fini()
10195 		if (bbr->r_ctl.crte)  in bbr_fini()
10196 			tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);  in bbr_fini()
10198 		bbr->rc_tp = NULL;  in bbr_fini()
10199 		if (bbr->bbr_hdrw_pacing)  in bbr_fini()
10200 			counter_u64_add(bbr_flows_whdwr_pacing, -1);  in bbr_fini()
10202 			counter_u64_add(bbr_flows_nohdwr_pacing, -1);  in bbr_fini()
10203 		if (bbr->r_ctl.crte != NULL) {  in bbr_fini()
10204 			tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);  in bbr_fini()
10205 			bbr->r_ctl.crte = NULL;  in bbr_fini()
10207 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_fini()
10209 			TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_fini()
10211 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_fini()
10213 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);  in bbr_fini()
10215 			TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_fini()
10217 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);  in bbr_fini()
10219 		calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;  in bbr_fini()
10220 		if (calc > (bbr->r_ctl.rc_init_rwnd / 10))  in bbr_fini()
10224 		bbr->r_ctl.rc_free_cnt = 0;  in bbr_fini()
10225 		uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);  in bbr_fini()
10226 		tp->t_fb_ptr = NULL;  in bbr_fini()
10229 	tp->snd_nxt = tp->snd_max;  in bbr_fini()
10235 	switch (tp->t_state) {  in bbr_set_state()
10237 		bbr->r_state = TCPS_SYN_SENT;  in bbr_set_state()
10238 		bbr->r_substate = bbr_do_syn_sent;  in bbr_set_state()
10241 		bbr->r_state = TCPS_SYN_RECEIVED;  in bbr_set_state()
10242 		bbr->r_substate = bbr_do_syn_recv;  in bbr_set_state()
10245 		bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);  in bbr_set_state()
10246 		bbr->r_state = TCPS_ESTABLISHED;  in bbr_set_state()
10247 		bbr->r_substate = bbr_do_established;  in bbr_set_state()
10250 		bbr->r_state = TCPS_CLOSE_WAIT;  in bbr_set_state()
10251 		bbr->r_substate = bbr_do_close_wait;  in bbr_set_state()
10254 		bbr->r_state = TCPS_FIN_WAIT_1;  in bbr_set_state()
10255 		bbr->r_substate = bbr_do_fin_wait_1;  in bbr_set_state()
10258 		bbr->r_state = TCPS_CLOSING;  in bbr_set_state()
10259 		bbr->r_substate = bbr_do_closing;  in bbr_set_state()
10262 		bbr->r_state = TCPS_LAST_ACK;  in bbr_set_state()
10263 		bbr->r_substate = bbr_do_lastack;  in bbr_set_state()
10266 		bbr->r_state = TCPS_FIN_WAIT_2;  in bbr_set_state()
10267 		bbr->r_substate = bbr_do_fin_wait_2;  in bbr_set_state()
10288 		/* Save the lowest srtt we saw in our end of the sub-state */  in bbr_substate_change()
10289 		bbr->rc_hit_state_1 = 0;  in bbr_substate_change()
10290 		if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)  in bbr_substate_change()
10291 			bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;  in bbr_substate_change()
10293 	bbr->rc_bbr_substate++;  in bbr_substate_change()
10299 		if (bbr->skip_gain) {  in bbr_substate_change()
10305 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];  in bbr_substate_change()
10306 		} else if (bbr->gain_is_limited &&  in bbr_substate_change()
10307 			   bbr->bbr_hdrw_pacing &&  in bbr_substate_change()
10308 			   bbr->r_ctl.crte) {  in bbr_substate_change()
10318 			rate = bbr->r_ctl.crte->rate;  in bbr_substate_change()
10324 				bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;  in bbr_substate_change()
10326 				bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];  in bbr_substate_change()
10329 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];  in bbr_substate_change()
10330 		if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {  in bbr_substate_change()
10331 			bbr->r_ctl.rc_bbr_state_atflight = cts;  in bbr_substate_change()
10333 			bbr->r_ctl.rc_bbr_state_atflight = 0;  in bbr_substate_change()
10335 		bbr->rc_hit_state_1 = 1;  in bbr_substate_change()
10336 		bbr->r_ctl.rc_exta_time_gd = 0;  in bbr_substate_change()
10337 		bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,  in bbr_substate_change()
10338 						     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_substate_change()
10340 			bbr->r_ctl.rc_bbr_state_atflight = 0;  in bbr_substate_change()
10342 			bbr->r_ctl.rc_bbr_state_atflight = cts;  in bbr_substate_change()
10343 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];  in bbr_substate_change()
10345 		/* All other cycles hit here 2-7 */  in bbr_substate_change()
10346 		if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {  in bbr_substate_change()
10348 			    (bbr->rc_use_google == 0) &&  in bbr_substate_change()
10349 			    (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {  in bbr_substate_change()
10350 				bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_substate_change()
10353 			if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))  in bbr_substate_change()
10354 				bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -  in bbr_substate_change()
10357 				bbr->r_ctl.rc_exta_time_gd = 0;  in bbr_substate_change()
10358 			if (bbr->r_ctl.rc_exta_time_gd) {  in bbr_substate_change()
10359 				bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;  in bbr_substate_change()
10361 				bbr->r_ctl.rc_level_state_extra /= 7;  in bbr_substate_change()
10362 				if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {  in bbr_substate_change()
10368 		bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);  in bbr_substate_change()
10369 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];  in bbr_substate_change()
10371 	if (bbr->rc_use_google) {  in bbr_substate_change()
10372 		bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);  in bbr_substate_change()
10374 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_substate_change()
10375 	bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;  in bbr_substate_change()
10379 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_substate_change()
10382 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_substate_change()
10383 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_substate_change()
10386 			counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_substate_change()
10389 	bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;  in bbr_substate_change()
10392 	    (bbr->rc_use_google == 0) &&  in bbr_substate_change()
10395 		bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_substate_change()
10396 		bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_substate_change()
10399 			bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +  in bbr_substate_change()
10400 							  ctf_flight_size(bbr->rc_tp,  in bbr_substate_change()
10401 							      (bbr->r_ctl.rc_sacked +  in bbr_substate_change()
10402 							       bbr->r_ctl.rc_lost_bytes)));  in bbr_substate_change()
10406 	if (bbr->rc_lt_use_bw) {  in bbr_substate_change()
10408 		bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_substate_change()
10411 	if (bbr->rc_use_google)  in bbr_substate_change()
10413 	bbr->r_ctl.gain_epoch = cts;  in bbr_substate_change()
10414 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_substate_change()
10415 	bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;  in bbr_substate_change()
10423 	    (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {  in bbr_set_probebw_google_gains()
10427 	if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_set_probebw_google_gains()
10430 	if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {  in bbr_set_probebw_google_gains()
10443 		 *    we stay in GAIN (gain-to-target).  in bbr_set_probebw_google_gains()
10447 		if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {  in bbr_set_probebw_google_gains()
10461 	if (bbr->rc_use_google) {  in bbr_set_probebw_gains()
10478 	if (bbr->r_ctl.rc_bbr_state_atflight == 0) {  in bbr_set_probebw_gains()
10480 			flight = ctf_flight_size(bbr->rc_tp,  in bbr_set_probebw_gains()
10481 				     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_set_probebw_gains()
10482 			if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {  in bbr_set_probebw_gains()
10484 				if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {  in bbr_set_probebw_gains()
10485 					bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_set_probebw_gains()
10490 					bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);  in bbr_set_probebw_gains()
10493 			if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&  in bbr_set_probebw_gains()
10494 			    (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||  in bbr_set_probebw_gains()
10495 			     (flight >= bbr->r_ctl.flightsize_at_drain))) {  in bbr_set_probebw_gains()
10503 				bbr->r_ctl.flightsize_at_drain = flight;  in bbr_set_probebw_gains()
10508 					bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;  in bbr_set_probebw_gains()
10509 					bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;  in bbr_set_probebw_gains()
10512 					bbr->r_ctl.rc_bbr_hptsi_gain *= 4;  in bbr_set_probebw_gains()
10513 					bbr->r_ctl.rc_bbr_hptsi_gain /= 5;  in bbr_set_probebw_gains()
10515 				if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {  in bbr_set_probebw_gains()
10517 					bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);  in bbr_set_probebw_gains()
10524 				bbr->r_ctl.gain_epoch = cts;  in bbr_set_probebw_gains()
10526 			if (flight <= bbr->r_ctl.rc_target_at_state) {  in bbr_set_probebw_gains()
10528 				    (bbr->rc_use_google == 0) &&  in bbr_set_probebw_gains()
10529 				    (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {  in bbr_set_probebw_gains()
10530 					bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_set_probebw_gains()
10533 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);  in bbr_set_probebw_gains()
10538 			if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {  in bbr_set_probebw_gains()
10539 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);  in bbr_set_probebw_gains()
10542 			if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||  in bbr_set_probebw_gains()
10543 			    ((ctf_outstanding(bbr->rc_tp) +  bbr->rc_tp->t_maxseg - 1) >=  in bbr_set_probebw_gains()
10544 			     bbr->rc_tp->snd_wnd)) {  in bbr_set_probebw_gains()
10545 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);  in bbr_set_probebw_gains()
10561 	if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))  in bbr_set_probebw_gains()
10563 	if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {  in bbr_set_probebw_gains()
10564 		/* Less than a full time-period has passed */  in bbr_set_probebw_gains()
10567 	if (bbr->r_ctl.rc_level_state_extra &&  in bbr_set_probebw_gains()
10569 	    ((cts - bbr->r_ctl.rc_bbr_state_time) <  in bbr_set_probebw_gains()
10570 	     (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {  in bbr_set_probebw_gains()
10571 		/* Less than a full time-period + extra has passed */  in bbr_set_probebw_gains()
10575 	    bbr->r_ctl.rc_level_state_extra &&  in bbr_set_probebw_gains()
10577 	    ((cts - bbr->r_ctl.rc_bbr_state_time) <  in bbr_set_probebw_gains()
10578 	     (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {  in bbr_set_probebw_gains()
10579 		/* Less than a full time-period + extra has passed */  in bbr_set_probebw_gains()
10590 	if (bbr->rc_use_google) {  in bbr_get_a_state_target()
10594 		mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),  in bbr_get_a_state_target()
10595 			  bbr->r_ctl.rc_pace_max_segs);  in bbr_get_a_state_target()
10611 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&  in bbr_set_state_target()
10612 	    ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {  in bbr_set_state_target()
10613 		/* Special case using old probe-rtt method */  in bbr_set_state_target()
10614 		tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_set_state_target()
10617 		/* Non-probe-rtt case and reduced probe-rtt  */  in bbr_set_state_target()
10618 		if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&  in bbr_set_state_target()
10619 		    (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {  in bbr_set_state_target()
10621 			tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);  in bbr_set_state_target()
10623 		} else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {  in bbr_set_state_target()
10633 			 * for non-google mode and default (non-configured).  in bbr_set_state_target()
10636 			if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN])  {  in bbr_set_state_target()
10640 				tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);  in bbr_set_state_target()
10646 	bbr->r_ctl.rc_target_at_state = tar;  in bbr_set_state_target()
10655 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_enter_probe_rtt()
10656 	bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,  in bbr_enter_probe_rtt()
10657 					     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_enter_probe_rtt()
10658 	bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain  in bbr_enter_probe_rtt()
10659 					  + bbr->r_ctl.rc_delivered);  in bbr_enter_probe_rtt()
10661 	if (bbr->rc_use_google || bbr_probertt_sets_rtt)  in bbr_enter_probe_rtt()
10662 		bbr->rc_prtt_set_ts = 1;  in bbr_enter_probe_rtt()
10663 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_enter_probe_rtt()
10664 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_enter_probe_rtt()
10665 		counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_enter_probe_rtt()
10668 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_enter_probe_rtt()
10669 	bbr->r_ctl.last_in_probertt = cts;  in bbr_enter_probe_rtt()
10670 	bbr->r_ctl.rc_probertt_srttchktim = cts;  in bbr_enter_probe_rtt()
10671 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_enter_probe_rtt()
10672 	bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;  in bbr_enter_probe_rtt()
10676 	    bbr->rc_hit_state_1 &&  in bbr_enter_probe_rtt()
10677 	    (bbr->rc_use_google == 0) &&  in bbr_enter_probe_rtt()
10679 		if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)  in bbr_enter_probe_rtt()
10680 			bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_enter_probe_rtt()
10682 		bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_enter_probe_rtt()
10684 	bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_enter_probe_rtt()
10685 	if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){  in bbr_enter_probe_rtt()
10686 		/* Set to the non-configurable default of 4 (PROBE_RTT_MIN)  */  in bbr_enter_probe_rtt()
10687 		bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_enter_probe_rtt()
10689 		bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_enter_probe_rtt()
10690 		bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_enter_probe_rtt()
10691 		bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);  in bbr_enter_probe_rtt()
10692 		bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;  in bbr_enter_probe_rtt()
10699 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;  in bbr_enter_probe_rtt()
10700 		bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_enter_probe_rtt()
10703 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_enter_probe_rtt()
10704 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_enter_probe_rtt()
10708 	if (ctf_flight_size(bbr->rc_tp,  in bbr_enter_probe_rtt()
10709 		(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=  in bbr_enter_probe_rtt()
10710 	    bbr->r_ctl.rc_target_at_state) {  in bbr_enter_probe_rtt()
10712 		bbr->r_ctl.rc_bbr_enters_probertt = cts;  in bbr_enter_probe_rtt()
10715 		bbr->r_ctl.rc_bbr_enters_probertt = 0;  in bbr_enter_probe_rtt()
10717 	bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;  in bbr_enter_probe_rtt()
10727 	 * Sanity check on probe-rtt intervals.  in bbr_check_probe_rtt_limits()
10729 	 * against new-reno flows with huge buffers  in bbr_check_probe_rtt_limits()
10730 	 * our rtt-prop interval could come to dominate  in bbr_check_probe_rtt_limits()
10735 	    (bbr->rc_use_google == 0)) {  in bbr_check_probe_rtt_limits()
10739 		/* Are we to small and go into probe-rtt to often? */  in bbr_check_probe_rtt_limits()
10745 				newval = cur_rttp + (fval - bbr_rtt_probe_limit);  in bbr_check_probe_rtt_limits()
10754 		if (cur_rttp > 	bbr->r_ctl.rc_probertt_int) {  in bbr_check_probe_rtt_limits()
10755 			bbr->r_ctl.rc_probertt_int = cur_rttp;  in bbr_check_probe_rtt_limits()
10756 			reset_time_small(&bbr->r_ctl.rc_rttprop, newval);  in bbr_check_probe_rtt_limits()
10763 			if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {  in bbr_check_probe_rtt_limits()
10769 					bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;  in bbr_check_probe_rtt_limits()
10770 					reset_time_small(&bbr->r_ctl.rc_rttprop,  in bbr_check_probe_rtt_limits()
10779 					if (cur_rttp < bbr->r_ctl.rc_probertt_int) {  in bbr_check_probe_rtt_limits()
10781 						bbr->r_ctl.rc_probertt_int = cur_rttp;  in bbr_check_probe_rtt_limits()
10782 						reset_time_small(&bbr->r_ctl.rc_rttprop, newval);  in bbr_check_probe_rtt_limits()
10796 	/* Exit probe-rtt */  in bbr_exit_probe_rtt()
10798 	if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {  in bbr_exit_probe_rtt()
10799 		tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_exit_probe_rtt()
10803 	bbr->rc_hit_state_1 = 0;  in bbr_exit_probe_rtt()
10804 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_exit_probe_rtt()
10805 	bbr->r_ctl.last_in_probertt = cts;  in bbr_exit_probe_rtt()
10807 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_exit_probe_rtt()
10808 	bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,  in bbr_exit_probe_rtt()
10809 					      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_exit_probe_rtt()
10810 					  bbr->r_ctl.rc_delivered);  in bbr_exit_probe_rtt()
10811 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_exit_probe_rtt()
10814 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_exit_probe_rtt()
10815 		counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_exit_probe_rtt()
10817 	if (bbr->rc_filled_pipe) {  in bbr_exit_probe_rtt()
10819 		bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_exit_probe_rtt()
10820 		bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_exit_probe_rtt()
10821 		bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;  in bbr_exit_probe_rtt()
10826 		bbr->rc_bbr_state = BBR_STATE_STARTUP;  in bbr_exit_probe_rtt()
10827 		bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_exit_probe_rtt()
10831 		 * the number of pe's we were in probe-rtt  in bbr_exit_probe_rtt()
10835 		bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);  in bbr_exit_probe_rtt()
10836 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_exit_probe_rtt()
10838 		if (bbr->r_ctl.rc_lost &&  in bbr_exit_probe_rtt()
10840 		    (bbr->rc_use_google == 0))  in bbr_exit_probe_rtt()
10841 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;  in bbr_exit_probe_rtt()
10843 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;  in bbr_exit_probe_rtt()
10844 		bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;  in bbr_exit_probe_rtt()
10848 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_exit_probe_rtt()
10849 		    bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);  in bbr_exit_probe_rtt()
10857 	if ((bbr->rc_past_init_win == 1) &&  in bbr_should_enter_probe_rtt()
10858 	    (bbr->rc_in_persist == 0) &&  in bbr_should_enter_probe_rtt()
10859 	    (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {  in bbr_should_enter_probe_rtt()
10863 	    (bbr->rc_in_persist == 0) &&  in bbr_should_enter_probe_rtt()
10864 	    (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&  in bbr_should_enter_probe_rtt()
10865 	    ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {  in bbr_should_enter_probe_rtt()
10877 		 * Need to be on a pkt-epoch to continue.  in bbr_google_startup()
10882 	gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *  in bbr_google_startup()
10883 		 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_google_startup()
10885 		bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_google_startup()
10886 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_google_startup()
10887 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);  in bbr_google_startup()
10888 		bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;  in bbr_google_startup()
10890 	if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)  in bbr_google_startup()
10892 	bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_google_startup()
10893 			      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);  in bbr_google_startup()
10905 	if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&  in bbr_state_startup()
10906 	    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {  in bbr_state_startup()
10919 	if (bbr->rc_use_google)  in bbr_state_startup()
10922 	if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&  in bbr_state_startup()
10925 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;  in bbr_state_startup()
10929 		 * Need to be on a pkt-epoch to continue.  in bbr_state_startup()
10939 		if (bbr->r_ctl.rc_pkt_epoch_rtt &&  in bbr_state_startup()
10940 		    bbr->r_ctl.startup_last_srtt &&  in bbr_state_startup()
10941 		    (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {  in bbr_state_startup()
10942 			delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;  in bbr_state_startup()
10943 			rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;  in bbr_state_startup()
10946 		if ((bbr->r_ctl.startup_last_srtt == 0)  ||  in bbr_state_startup()
10947 		    (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))  in bbr_state_startup()
10949 			bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;  in bbr_state_startup()
10951 		if ((bbr->r_ctl.rc_lost == 0) &&  in bbr_state_startup()
10958 			if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)  in bbr_state_startup()
10959 				bbr->r_ctl.rc_bbr_last_startup_epoch++;  in bbr_state_startup()
10961 					      delta, bbr->r_ctl.startup_last_srtt, 10);  in bbr_state_startup()
10965 	if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&  in bbr_state_startup()
10966 	    (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&  in bbr_state_startup()
10967 	    (!IN_RECOVERY(bbr->rc_tp->t_flags))) {  in bbr_state_startup()
10972 		 * the number of non-gain we have already accumulated.  in bbr_state_startup()
10974 		if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)  in bbr_state_startup()
10975 			bbr->r_ctl.rc_bbr_last_startup_epoch++;  in bbr_state_startup()
10976 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
10977 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);  in bbr_state_startup()
10981 	if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)  in bbr_state_startup()
10982 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
10983 	bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;  in bbr_state_startup()
10985 	if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)  in bbr_state_startup()
10986 		gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *  in bbr_state_startup()
10987 			 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_state_startup()
10989 		gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *  in bbr_state_startup()
10990 			 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_state_startup()
10992 	if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)  in bbr_state_startup()
10993 		bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;  in bbr_state_startup()
10995 		bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_state_startup()
10997 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
10998 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
10999 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);  in bbr_state_startup()
11001 	if ((bbr->rc_loss_exit &&  in bbr_state_startup()
11002 	     (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&  in bbr_state_startup()
11003 	     (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&  in bbr_state_startup()
11004 	    ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {  in bbr_state_startup()
11012 		if ((ctf_flight_size(bbr->rc_tp,  in bbr_state_startup()
11013 			 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_state_startup()
11014 		     (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {  in bbr_state_startup()
11016 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11017 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);  in bbr_state_startup()
11020 			bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
11021 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11022 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);  in bbr_state_startup()
11025 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
11026 	if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||  in bbr_state_startup()
11032 	bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11033 			      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);  in bbr_state_startup()
11044 	if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&  in bbr_state_change()
11045 	    (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&  in bbr_state_change()
11046 	    (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&  in bbr_state_change()
11047 	    (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&  in bbr_state_change()
11048 	    (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {  in bbr_state_change()
11050 		panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);  in bbr_state_change()
11053 	if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in bbr_state_change()
11058 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_change()
11059 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);  in bbr_state_change()
11060 			bbr->rc_filled_pipe = 1;  in bbr_state_change()
11061 			bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_state_change()
11062 			if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_state_change()
11063 				time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_state_change()
11064 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_state_change()
11067 			if (bbr->rc_no_pacing)  in bbr_state_change()
11068 				bbr->rc_no_pacing = 0;  in bbr_state_change()
11069 			bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_state_change()
11070 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;  in bbr_state_change()
11071 			bbr->rc_bbr_state = BBR_STATE_DRAIN;  in bbr_state_change()
11073 			if ((bbr->rc_use_google == 0) &&  in bbr_state_change()
11075 				/* Here we don't have to worry about probe-rtt */  in bbr_state_change()
11076 				bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_state_change()
11077 				bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_state_change()
11080 			bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;  in bbr_state_change()
11082 			if (ctf_flight_size(bbr->rc_tp,  in bbr_state_change()
11083 			        (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=  in bbr_state_change()
11084 			    bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11089 				bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_state_change()
11091 				bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_state_change()
11095 	} else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {  in bbr_state_change()
11099 		tp = bbr->rc_tp;  in bbr_state_change()
11101 			      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_state_change()
11102 		if (inflight >= bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11104 			bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_state_change()
11105 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_state_change()
11106 			bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_state_change()
11112 			bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;  in bbr_state_change()
11113 			bbr->rc_bbr_substate = BBR_SUB_LEVEL6;  in bbr_state_change()
11116 	} else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {  in bbr_state_change()
11117 		/* Has in-flight reached the bdp (or less)? */  in bbr_state_change()
11121 		tp = bbr->rc_tp;  in bbr_state_change()
11123 			      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_state_change()
11124 		if ((bbr->rc_use_google == 0) &&  in bbr_state_change()
11126 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_state_change()
11128 			 * Here we don't have to worry about probe-rtt  in bbr_state_change()
11129 			 * re-slam it, but keep it slammed down.  in bbr_state_change()
11131 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_state_change()
11134 		if (inflight <= bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11136 			bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_state_change()
11137 			bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_state_change()
11138 			if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_state_change()
11141 				time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_state_change()
11142 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_state_change()
11144 			if ((bbr->rc_use_google == 0) &&  in bbr_state_change()
11146 			    (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {  in bbr_state_change()
11148 				tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_state_change()
11151 			/* Setup probe-rtt has being done now RRS-HERE */  in bbr_state_change()
11152 			bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_state_change()
11153 			bbr->r_ctl.last_in_probertt = cts;  in bbr_state_change()
11155 			/* Randomly pick a sub-state */  in bbr_state_change()
11156 			bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_state_change()
11160 	} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {  in bbr_state_change()
11163 		flight = ctf_flight_size(bbr->rc_tp,  in bbr_state_change()
11164 			     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_state_change()
11165 		bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);  in bbr_state_change()
11166 		if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&  in bbr_state_change()
11167 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_state_change()
11171 			bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_state_change()
11174 			   (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_state_change()
11175 			/* Re-slam it */  in bbr_state_change()
11176 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_state_change()
11179 		if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {  in bbr_state_change()
11181 			if (flight <= bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11183 				bbr->r_ctl.rc_bbr_enters_probertt = cts;  in bbr_state_change()
11185 				if (bbr->r_ctl.rc_bbr_enters_probertt == 0)  in bbr_state_change()
11186 					bbr->r_ctl.rc_bbr_enters_probertt = 1;  in bbr_state_change()
11187 				if (bbr->rc_use_google == 0) {  in bbr_state_change()
11192 					if (bbr->r_ctl.bbr_rttprobe_gain_val)  in bbr_state_change()
11193 						bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;  in bbr_state_change()
11195 						bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_state_change()
11198 			if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&  in bbr_state_change()
11199 			    (bbr->rc_use_google == 0) &&  in bbr_state_change()
11200 			    bbr->r_ctl.bbr_rttprobe_gain_val &&  in bbr_state_change()
11201 			    (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||  in bbr_state_change()
11202 			     (flight >= bbr->r_ctl.flightsize_at_drain))) {  in bbr_state_change()
11213 				bbr->r_ctl.flightsize_at_drain = flight;  in bbr_state_change()
11214 				bbr->r_ctl.rc_probertt_srttchktim = cts;  in bbr_state_change()
11215 				red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);  in bbr_state_change()
11216 				if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {  in bbr_state_change()
11218 					bbr->r_ctl.rc_bbr_hptsi_gain -= red;  in bbr_state_change()
11220 				} else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {  in bbr_state_change()
11222 					bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);  in bbr_state_change()
11226 					bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);  in bbr_state_change()
11230 		if (bbr->r_ctl.rc_bbr_enters_probertt &&  in bbr_state_change()
11231 		    (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&  in bbr_state_change()
11232 		    ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {  in bbr_state_change()
11234 			bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);  in bbr_state_change()
11236 	} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_state_change()
11237 		if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&  in bbr_state_change()
11238 		    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {  in bbr_state_change()
11260 	if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {  in bbr_check_bbr_for_state()
11265 	bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);  in bbr_check_bbr_for_state()
11287 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_do_segment_nounlock()
11288 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_segment_nounlock()
11294 	 * If this is either a state-changing packet or current state isn't  in bbr_do_segment_nounlock()
11301 	KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",  in bbr_do_segment_nounlock()
11303 	KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",  in bbr_do_segment_nounlock()
11306 	tp->t_rcvtime = ticks;  in bbr_do_segment_nounlock()
11308 	 * Unscale the window into a 32-bit value. For the SYN_SENT state  in bbr_do_segment_nounlock()
11311 	tiwin = th->th_win << tp->snd_scale;  in bbr_do_segment_nounlock()
11313 	stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);  in bbr_do_segment_nounlock()
11316 	if (m->m_flags & M_TSTMP) {  in bbr_do_segment_nounlock()
11321 		bbr->rc_tv.tv_sec = ts.tv_sec;  in bbr_do_segment_nounlock()
11322 		bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;  in bbr_do_segment_nounlock()
11323 		bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usec(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11324 	} else if (m->m_flags & M_TSTMP_LRO) {  in bbr_do_segment_nounlock()
11329 		bbr->rc_tv.tv_sec = ts.tv_sec;  in bbr_do_segment_nounlock()
11330 		bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;  in bbr_do_segment_nounlock()
11331 		bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usec(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11336 		bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11342 	    (th->th_off << 2) - sizeof(struct tcphdr),  in bbr_do_segment_nounlock()
11344 	if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) {  in bbr_do_segment_nounlock()
11359 	if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&  in bbr_do_segment_nounlock()
11371 		to.to_tsecr -= tp->ts_offset;  in bbr_do_segment_nounlock()
11372 		if (TSTMP_GT(to.to_tsecr, tcp_tv_to_msec(&bbr->rc_tv)))  in bbr_do_segment_nounlock()
11379 	if (bbr->r_state == 0) {  in bbr_do_segment_nounlock()
11387 		if (bbr->rc_inp == NULL) {  in bbr_do_segment_nounlock()
11388 			bbr->rc_inp = inp;  in bbr_do_segment_nounlock()
11394 		if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {  in bbr_do_segment_nounlock()
11396 			    (tp->t_flags & TF_REQ_SCALE)) {  in bbr_do_segment_nounlock()
11397 				tp->t_flags |= TF_RCVD_SCALE;  in bbr_do_segment_nounlock()
11398 				tp->snd_scale = to.to_wscale;  in bbr_do_segment_nounlock()
11400 				tp->t_flags &= ~TF_REQ_SCALE;  in bbr_do_segment_nounlock()
11405 			tp->snd_wnd = th->th_win;  in bbr_do_segment_nounlock()
11407 			    (tp->t_flags & TF_REQ_TSTMP)) {  in bbr_do_segment_nounlock()
11408 				tp->t_flags |= TF_RCVD_TSTMP;  in bbr_do_segment_nounlock()
11409 				tp->ts_recent = to.to_tsval;  in bbr_do_segment_nounlock()
11410 				tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11412 			    tp->t_flags &= ~TF_REQ_TSTMP;  in bbr_do_segment_nounlock()
11415 			if ((tp->t_flags & TF_SACK_PERMIT) &&  in bbr_do_segment_nounlock()
11417 				tp->t_flags &= ~TF_SACK_PERMIT;  in bbr_do_segment_nounlock()
11418 			if (tp->t_flags & TF_FASTOPEN) {  in bbr_do_segment_nounlock()
11425 						if ((inp->inp_vflag & INP_IPV6) != 0)  in bbr_do_segment_nounlock()
11441 		if ((tp->t_flags & TF_SACK_PERMIT) == 0) {  in bbr_do_segment_nounlock()
11444 			(*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen,  in bbr_do_segment_nounlock()
11449 		bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0;  in bbr_do_segment_nounlock()
11451 		sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);  in bbr_do_segment_nounlock()
11462 	 * always. All other times (timers etc) we must have a rack-state  in bbr_do_segment_nounlock()
11467 	if (bbr->r_state != tp->t_state)  in bbr_do_segment_nounlock()
11470 	if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)  in bbr_do_segment_nounlock()
11472 	prev_state = bbr->r_state;  in bbr_do_segment_nounlock()
11473 	bbr->rc_ack_was_delayed = 0;  in bbr_do_segment_nounlock()
11474 	lost = bbr->r_ctl.rc_lost;  in bbr_do_segment_nounlock()
11475 	bbr->rc_is_pkt_epoch_now = 0;  in bbr_do_segment_nounlock()
11476 	if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) {  in bbr_do_segment_nounlock()
11480 			bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;  in bbr_do_segment_nounlock()
11481 			bbr->rc_ack_was_delayed = 1;  in bbr_do_segment_nounlock()
11482 			if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,  in bbr_do_segment_nounlock()
11483 				     bbr->r_ctl.highest_hdwr_delay))  in bbr_do_segment_nounlock()
11484 				bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_do_segment_nounlock()
11486 			bbr->r_ctl.rc_ack_hdwr_delay = 0;  in bbr_do_segment_nounlock()
11487 			bbr->rc_ack_was_delayed = 0;  in bbr_do_segment_nounlock()
11490 		bbr->r_ctl.rc_ack_hdwr_delay = 0;  in bbr_do_segment_nounlock()
11491 		bbr->rc_ack_was_delayed = 0;  in bbr_do_segment_nounlock()
11500 	 * If a segment with the ACK-bit set arrives in the SYN-SENT state  in bbr_do_segment_nounlock()
11503 	if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&  in bbr_do_segment_nounlock()
11504 	    (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_do_segment_nounlock()
11509 	if (tiwin > bbr->r_ctl.rc_high_rwnd)  in bbr_do_segment_nounlock()
11510 		bbr->r_ctl.rc_high_rwnd = tiwin;  in bbr_do_segment_nounlock()
11511 	bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,  in bbr_do_segment_nounlock()
11512 					    (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_do_segment_nounlock()
11513 	bbr->rtt_valid = 0;  in bbr_do_segment_nounlock()
11515 		bbr->rc_ts_valid = 1;  in bbr_do_segment_nounlock()
11516 		bbr->r_ctl.last_inbound_ts = to.to_tsval;  in bbr_do_segment_nounlock()
11518 		bbr->rc_ts_valid = 0;  in bbr_do_segment_nounlock()
11519 		bbr->r_ctl.last_inbound_ts = 0;  in bbr_do_segment_nounlock()
11521 	retval = (*bbr->r_substate) (m, th, so,  in bbr_do_segment_nounlock()
11535 		if (bbr->rc_is_pkt_epoch_now)  in bbr_do_segment_nounlock()
11537 		bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));  in bbr_do_segment_nounlock()
11539 			if ((bbr->r_wanted_output != 0) ||  in bbr_do_segment_nounlock()
11540 			    (tp->t_flags & TF_ACKNOW)) {  in bbr_do_segment_nounlock()
11542 				bbr->rc_output_starts_timer = 0;  in bbr_do_segment_nounlock()
11550 		    ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&  in bbr_do_segment_nounlock()
11551 		    (SEQ_GT(tp->snd_max, tp->snd_una) ||  in bbr_do_segment_nounlock()
11552 		     (tp->t_flags & TF_DELACK) ||  in bbr_do_segment_nounlock()
11553 		     ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_do_segment_nounlock()
11554 		      (tp->t_state <= TCPS_CLOSING)))) {  in bbr_do_segment_nounlock()
11559 			if ((tp->snd_max == tp->snd_una) &&  in bbr_do_segment_nounlock()
11560 			    ((tp->t_flags & TF_DELACK) == 0) &&  in bbr_do_segment_nounlock()
11562 			    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {  in bbr_do_segment_nounlock()
11571 					if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&  in bbr_do_segment_nounlock()
11572 					    (TSTMP_GT(lcts, bbr->rc_pacer_started))) {  in bbr_do_segment_nounlock()
11575 						del = lcts - bbr->rc_pacer_started;  in bbr_do_segment_nounlock()
11576 						if (bbr->r_ctl.rc_last_delay_val > del) {  in bbr_do_segment_nounlock()
11578 							bbr->r_ctl.rc_last_delay_val -= del;  in bbr_do_segment_nounlock()
11579 							bbr->rc_pacer_started = lcts;  in bbr_do_segment_nounlock()
11582 							bbr->r_ctl.rc_last_delay_val = 0;  in bbr_do_segment_nounlock()
11589 				bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,  in bbr_do_segment_nounlock()
11592 		} else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {  in bbr_do_segment_nounlock()
11594 			bbr_timer_audit(tp, bbr, lcts, &so->so_snd);  in bbr_do_segment_nounlock()
11597 		if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS))  in bbr_do_segment_nounlock()
11598 			tp->t_flags2 &= ~TF2_HPTS_CALLS;  in bbr_do_segment_nounlock()
11600 		if (bbr->r_state != tp->t_state)  in bbr_do_segment_nounlock()
11605 			bbr->r_wanted_output = 0;  in bbr_do_segment_nounlock()
11618 	if (!STAILQ_EMPTY(&tp->t_inqueue)) {  in bbr_do_segment()
11624 	if (m->m_flags & M_TSTMP_LRO) {  in bbr_do_segment()
11648 	if (ctf_outstanding(tp) >= tp->snd_wnd) {  in bbr_what_can_we_send()
11649 		/* We never want to go over our peers rcv-window */  in bbr_what_can_we_send()
11654 		flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_what_can_we_send()
11659 			 * >= tp->snd_wnd).  in bbr_what_can_we_send()
11663 		len = sendwin - flight;  in bbr_what_can_we_send()
11664 		if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {  in bbr_what_can_we_send()
11666 			len = tp->snd_wnd - ctf_outstanding(tp);  in bbr_what_can_we_send()
11673 			len = avail - sb_offset;  in bbr_what_can_we_send()
11686 		if (rsm->r_flags & BBR_TLP) {  in bbr_do_send_accounting()
11695 			tp->t_sndrexmitpack++;  in bbr_do_send_accounting()
11699 			stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,  in bbr_do_send_accounting()
11704 		 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is  in bbr_do_send_accounting()
11705 		 * sub-state  in bbr_do_send_accounting()
11707 		counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);  in bbr_do_send_accounting()
11708 		if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {  in bbr_do_send_accounting()
11710 			counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);  in bbr_do_send_accounting()
11713 			 * Log our probe state 3, and log also 5-13 to show  in bbr_do_send_accounting()
11714 			 * us the recovery sub-state for the send. This  in bbr_do_send_accounting()
11735 		stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,  in bbr_do_send_accounting()
11744 	if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {  in bbr_cwnd_limiting()
11755 		if (tp->snd_cwnd > target)  in bbr_cwnd_limiting()
11756 			tp->snd_cwnd = target;  in bbr_cwnd_limiting()
11766 	 * account that we are limited by TCP_MAXWIN << tp->rcv_scale.  in bbr_window_update_needed()
11772 	if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {  in bbr_window_update_needed()
11773 		oldwin = (tp->rcv_adv - tp->rcv_nxt);  in bbr_window_update_needed()
11775 			adv -= oldwin;  in bbr_window_update_needed()
11788 	if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)  in bbr_window_update_needed()
11792 	    (adv >= (so->so_rcv.sb_hiwat / 4) ||  in bbr_window_update_needed()
11793 	    recwin <= (so->so_rcv.sb_hiwat / 8) ||  in bbr_window_update_needed()
11794 	    so->so_rcv.sb_hiwat <= 8 * maxseg)) {  in bbr_window_update_needed()
11797 	if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat)  in bbr_window_update_needed()
11805  * if the TCB was on the hpts. A non-zero return
11863 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_output_wtime()
11865 	memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));  in bbr_output_wtime()
11866 	cts = tcp_tv_to_usec(&bbr->rc_tv);  in bbr_output_wtime()
11867 	inp = bbr->rc_inp;  in bbr_output_wtime()
11868 	hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS);  in bbr_output_wtime()
11869 	tp->t_flags2 &= ~TF2_HPTS_CALLS;  in bbr_output_wtime()
11870 	so = inp->inp_socket;  in bbr_output_wtime()
11871 	sb = &so->so_snd;  in bbr_output_wtime()
11872 	if (tp->t_nic_ktls_xmit)  in bbr_output_wtime()
11877 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_output_wtime()
11882 	pace_max_segs = bbr->r_ctl.rc_pace_max_segs;  in bbr_output_wtime()
11886 	if (tp->t_flags & TF_TOE)  in bbr_output_wtime()
11891 	if (bbr->r_state) {  in bbr_output_wtime()
11893 		isipv6 = bbr->r_is_v6;  in bbr_output_wtime()
11895 		isipv6 = (inp->inp_vflag & INP_IPV6) != 0;  in bbr_output_wtime()
11898 	if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&  in bbr_output_wtime()
11904 		if ((tp->t_flags & TF_ACKNOW) == 0) {  in bbr_output_wtime()
11909 			 * whatever timer is running (KEEP/DEL-ACK?) and  in bbr_output_wtime()
11914 			recwin = lmin(lmax(sbspace(&so->so_rcv), 0),  in bbr_output_wtime()
11915 				      (long)TCP_MAXWIN << tp->rcv_scale);  in bbr_output_wtime()
11917 			    ((tcp_outflags[tp->t_state] & TH_RST) == 0) &&  in bbr_output_wtime()
11918 			    ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <=  in bbr_output_wtime()
11919 			    (tp->snd_max - tp->snd_una))) {  in bbr_output_wtime()
11923 				 * let the timer-run off.  in bbr_output_wtime()
11931 	if (bbr->r_ctl.rc_last_delay_val) {  in bbr_output_wtime()
11933 		if (SEQ_GT(cts, bbr->rc_pacer_started))  in bbr_output_wtime()
11934 			delay_calc = cts - bbr->rc_pacer_started;  in bbr_output_wtime()
11935 		if (delay_calc >= bbr->r_ctl.rc_last_delay_val)  in bbr_output_wtime()
11936 			delay_calc -= bbr->r_ctl.rc_last_delay_val;  in bbr_output_wtime()
11941 	if ((bbr->r_timer_override) ||  in bbr_output_wtime()
11942 	    (tp->t_state < TCPS_ESTABLISHED)) {  in bbr_output_wtime()
11947 		if ((bbr->r_ctl.rc_last_delay_val) &&  in bbr_output_wtime()
11948 		    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&  in bbr_output_wtime()
11957 			bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
11959 		} else if (tp->t_state == TCPS_CLOSED) {  in bbr_output_wtime()
11960 			bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
11972 	bbr->rc_cwnd_limited = 0;  in bbr_output_wtime()
11973 	if (bbr->r_ctl.rc_last_delay_val) {  in bbr_output_wtime()
11975 		if (SEQ_GT(cts, bbr->rc_pacer_started))  in bbr_output_wtime()
11976 			delay_calc = cts - bbr->rc_pacer_started;  in bbr_output_wtime()
11979 		if (delay_calc >= bbr->r_ctl.rc_last_delay_val)  in bbr_output_wtime()
11981 			delay_calc -= bbr->r_ctl.rc_last_delay_val;  in bbr_output_wtime()
11988 			bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);  in bbr_output_wtime()
11989 			bbr->r_agg_early_set = 1;  in bbr_output_wtime()
11990 			if (bbr->r_ctl.rc_hptsi_agg_delay) {  in bbr_output_wtime()
11991 				if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {  in bbr_output_wtime()
11993 					bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;  in bbr_output_wtime()
11994 					bbr->r_agg_early_set = 0;  in bbr_output_wtime()
11995 					bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
11997 					bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_output_wtime()
11998 					bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_output_wtime()
12001 			merged_val = bbr->rc_pacer_started;  in bbr_output_wtime()
12003 			merged_val |= bbr->r_ctl.rc_last_delay_val;  in bbr_output_wtime()
12005 						 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,  in bbr_output_wtime()
12006 						 bbr->r_agg_early_set, 3);  in bbr_output_wtime()
12007 			bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12013 		if (bbr->r_agg_early_set)  in bbr_output_wtime()
12014 			bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
12015 		bbr->r_agg_early_set = 0;  in bbr_output_wtime()
12024 		if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)  in bbr_output_wtime()
12025 			bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;  in bbr_output_wtime()
12027 			bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;  in bbr_output_wtime()
12029 	sendwin = min(tp->snd_wnd, tp->snd_cwnd);  in bbr_output_wtime()
12030 	if ((tp->snd_una == tp->snd_max) &&  in bbr_output_wtime()
12031 	    (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&  in bbr_output_wtime()
12036 		 * suite of states or a fast-ramp up.  in bbr_output_wtime()
12039 				       cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));  in bbr_output_wtime()
12047 	if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {  in bbr_output_wtime()
12060 	bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;  in bbr_output_wtime()
12062 	    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {  in bbr_output_wtime()
12063 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12065 	bbr->r_timer_override = 0;  in bbr_output_wtime()
12066 	bbr->r_wanted_output = 0;  in bbr_output_wtime()
12071 	if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12072 	    ((tp->t_state == TCPS_SYN_RECEIVED) ||  in bbr_output_wtime()
12073 	     (tp->t_state == TCPS_SYN_SENT)) &&  in bbr_output_wtime()
12074 	    SEQ_GT(tp->snd_max, tp->snd_una) &&	/* initial SYN or SYN|ACK sent */  in bbr_output_wtime()
12075 	    (tp->t_rxtshift == 0)) {	/* not a retransmit */  in bbr_output_wtime()
12084 	if (bbr->rc_use_google == 0)  in bbr_output_wtime()
12103 	sendwin = min(tp->snd_wnd, tp->snd_cwnd);  in bbr_output_wtime()
12104 	sb_offset = tp->snd_max - tp->snd_una;  in bbr_output_wtime()
12105 	flags = tcp_outflags[tp->t_state];  in bbr_output_wtime()
12114 	while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {  in bbr_output_wtime()
12120 			tot_len = tp->t_maxseg;  in bbr_output_wtime()
12126 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_output_wtime()
12127 		bbr->r_ctl.rc_free_cnt++;  in bbr_output_wtime()
12131 	if (bbr->r_ctl.rc_resend == NULL) {  in bbr_output_wtime()
12133 		bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);  in bbr_output_wtime()
12134 		if (bbr->r_ctl.rc_resend) {  in bbr_output_wtime()
12138 			bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);  in bbr_output_wtime()
12141 	if (bbr->r_ctl.rc_resend) {  in bbr_output_wtime()
12142 		rsm = bbr->r_ctl.rc_resend;  in bbr_output_wtime()
12146 		/* Remove any TLP flags its a RACK or T-O */  in bbr_output_wtime()
12147 		rsm->r_flags &= ~BBR_TLP;  in bbr_output_wtime()
12148 		bbr->r_ctl.rc_resend = NULL;  in bbr_output_wtime()
12149 		if (SEQ_LT(rsm->r_start, tp->snd_una)) {  in bbr_output_wtime()
12152 			    tp, bbr, rsm, rsm->r_start, tp->snd_una);  in bbr_output_wtime()
12160 		if (rsm->r_flags & BBR_HAS_SYN) {  in bbr_output_wtime()
12165 				rsm->r_start++;  in bbr_output_wtime()
12166 				if (rsm->r_start == rsm->r_end) {  in bbr_output_wtime()
12171 					rsm->r_flags &= ~BBR_HAS_SYN;  in bbr_output_wtime()
12172 					len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12181 			len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12182 		if ((bbr->rc_resends_use_tso == 0) &&  in bbr_output_wtime()
12187 		sb_offset = rsm->r_start - tp->snd_una;  in bbr_output_wtime()
12199 	} else if (bbr->r_ctl.rc_tlp_send) {  in bbr_output_wtime()
12204 		rsm = bbr->r_ctl.rc_tlp_send;  in bbr_output_wtime()
12205 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_output_wtime()
12207 		len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12208 		if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))  in bbr_output_wtime()
12211 		if (SEQ_GT(tp->snd_una, rsm->r_start)) {  in bbr_output_wtime()
12214 			    tp, bbr, tp->snd_una, rsm, rsm->r_start);  in bbr_output_wtime()
12221 		sb_offset = rsm->r_start - tp->snd_una;  in bbr_output_wtime()
12230 	    (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {  in bbr_output_wtime()
12232 		if (!bbr->alloc_limit_reported) {  in bbr_output_wtime()
12233 			bbr->alloc_limit_reported = 1;  in bbr_output_wtime()
12239 	if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {  in bbr_output_wtime()
12248 	if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))  in bbr_output_wtime()
12250 	if (tp->t_flags & TF_NEEDSYN)  in bbr_output_wtime()
12253 	if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {  in bbr_output_wtime()
12255 		len--;  in bbr_output_wtime()
12271 		end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);  in bbr_output_wtime()
12282 	 * fast-retransmit because TCP will reset snd_nxt to snd_max after  in bbr_output_wtime()
12283 	 * the fast-retransmit.  in bbr_output_wtime()
12285 	 * In the normal retransmit-FIN-only case, however, snd_nxt will be  in bbr_output_wtime()
12296 		if (SEQ_GT(tp->snd_max, tp->snd_una))  in bbr_output_wtime()
12297 			sb_offset = tp->snd_max - tp->snd_una;  in bbr_output_wtime()
12300 		if (bbr->rc_tlp_new_data) {  in bbr_output_wtime()
12307 			if (tlplen > (uint32_t)(avail - sb_offset)) {  in bbr_output_wtime()
12308 				tlplen = (uint32_t)(avail - sb_offset);  in bbr_output_wtime()
12310 			if (tlplen > tp->snd_wnd) {  in bbr_output_wtime()
12311 				len = tp->snd_wnd;  in bbr_output_wtime()
12315 			bbr->rc_tlp_new_data = 0;  in bbr_output_wtime()
12319 			    (bbr->rc_in_persist == 0) &&  in bbr_output_wtime()
12321 			    ((avail - sb_offset) >= p_maxseg)) {  in bbr_output_wtime()
12333 			if (bbr->rc_in_persist) {  in bbr_output_wtime()
12340 				rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_output_wtime()
12342 					len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12343 					if (rsm->r_flags & BBR_HAS_FIN)  in bbr_output_wtime()
12344 						len--;  in bbr_output_wtime()
12345 					if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))  in bbr_output_wtime()
12358 					sb_offset = rsm->r_start - tp->snd_una;  in bbr_output_wtime()
12379 	 * SYN-SENT state and if segment contains data and if we don't know  in bbr_output_wtime()
12383 	    SEQ_GT(tp->snd_max, tp->snd_una)) {  in bbr_output_wtime()
12384 		if (tp->t_state != TCPS_SYN_RECEIVED)  in bbr_output_wtime()
12390 		if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12391 		    (tp->t_state == TCPS_SYN_RECEIVED))  in bbr_output_wtime()
12393 		sb_offset--, len++;  in bbr_output_wtime()
12401 		len--;  in bbr_output_wtime()
12408 	if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {  in bbr_output_wtime()
12415 	 *  - When retransmitting SYN|ACK on a passively-created socket  in bbr_output_wtime()
12416 	 *  - When retransmitting SYN on an actively created socket  in bbr_output_wtime()
12417 	 *  - When sending a zero-length cookie (cookie request) on an  in bbr_output_wtime()
12419 	 *  - When the socket is in the CLOSED state (RST is being sent)  in bbr_output_wtime()
12421 	if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12422 	    (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||  in bbr_output_wtime()
12423 	     ((tp->t_state == TCPS_SYN_SENT) &&  in bbr_output_wtime()
12424 	      (tp->t_tfo_client_cookie_len == 0)) ||  in bbr_output_wtime()
12430 	/* Without fast-open there should never be data sent on a SYN */  in bbr_output_wtime()
12431 	if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN))  in bbr_output_wtime()
12445 		 * 0-byte window. This makes sure the persist timer is set  in bbr_output_wtime()
12450 		if ((tp->snd_wnd == 0) &&  in bbr_output_wtime()
12451 		    (TCPS_HAVEESTABLISHED(tp->t_state)) &&  in bbr_output_wtime()
12452 		    (tp->snd_una == tp->snd_max) &&  in bbr_output_wtime()
12462 		   (len < bbr->r_ctl.rc_pace_max_segs)) {  in bbr_output_wtime()
12468 		if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_output_wtime()
12469 		    (TCPS_HAVEESTABLISHED(tp->t_state)) &&  in bbr_output_wtime()
12470 		    (len < (int)(sbavail(sb) - sb_offset))) {  in bbr_output_wtime()
12479 			if (tp->snd_max == tp->snd_una)  in bbr_output_wtime()
12481 		} else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&  in bbr_output_wtime()
12482 			   (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12483 						 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&  in bbr_output_wtime()
12484 			   (len < (int)(sbavail(sb) - sb_offset)) &&  in bbr_output_wtime()
12496 			bbr->rc_cwnd_limited = 1;  in bbr_output_wtime()
12498 		} else if (((tp->snd_wnd - ctf_outstanding(tp)) <  in bbr_output_wtime()
12499 			    min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_output_wtime()
12500 			   (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12501 						 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&  in bbr_output_wtime()
12502 			   (len < (int)(sbavail(sb) - sb_offset)) &&  in bbr_output_wtime()
12503 			   (TCPS_HAVEESTABLISHED(tp->t_state))) {  in bbr_output_wtime()
12521 	if (bbr->rc_in_persist &&  in bbr_output_wtime()
12524 	    (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {  in bbr_output_wtime()
12532 			sbleft = sbavail(sb) - sb_offset;  in bbr_output_wtime()
12535 		if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {  in bbr_output_wtime()
12545 	 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP  in bbr_output_wtime()
12562 	if (inp->inp_options)  in bbr_output_wtime()
12563 		ipoptlen = inp->inp_options->m_len -  in bbr_output_wtime()
12569 	 * Pre-calculate here as we save another lookup into the darknesses  in bbr_output_wtime()
12587 	if ((tp->t_flags & TF_TSO) && V_tcp_do_tso &&  in bbr_output_wtime()
12589 	    (tp->t_port == 0) &&  in bbr_output_wtime()
12590 	    ((tp->t_flags & TF_SIGNATURE) == 0) &&  in bbr_output_wtime()
12594 	recwin = lmin(lmax(sbspace(&so->so_rcv), 0),  in bbr_output_wtime()
12595 	    (long)TCP_MAXWIN << tp->rcv_scale);  in bbr_output_wtime()
12598 	 * conditions when len is non-zero:  in bbr_output_wtime()
12600 	 * - We have a full segment (or more with TSO) - This is the last  in bbr_output_wtime()
12602 	 * NODELAY - we've timed out (e.g. persist timer) - we have more  in bbr_output_wtime()
12604 	 * limited the window size) - we need to retransmit  in bbr_output_wtime()
12619 		if (((tp->t_flags & TF_MORETOCOME) == 0) &&	/* normal case */  in bbr_output_wtime()
12620 		    ((tp->t_flags & TF_NODELAY) ||  in bbr_output_wtime()
12621 		    ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) &&  in bbr_output_wtime()
12622 		    (tp->t_flags & TF_NOPUSH) == 0) {  in bbr_output_wtime()
12625 		if ((tp->snd_una == tp->snd_max) && len) {	/* Nothing outstanding */  in bbr_output_wtime()
12628 		if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {  in bbr_output_wtime()
12656 	 * pending (it will get piggy-backed on it) or the remote side  in bbr_output_wtime()
12657 	 * already has done a half-close and won't send more data.  Skip  in bbr_output_wtime()
12658 	 * this if the connection is in T/TCP half-open state.  in bbr_output_wtime()
12660 	if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&  in bbr_output_wtime()
12661 	    !(tp->t_flags & TF_DELACK) &&  in bbr_output_wtime()
12662 	    !TCPS_HAVERCVDFIN(tp->t_state)) {  in bbr_output_wtime()
12669 	 * is also a catch-all for the retransmit timer timeout case.  in bbr_output_wtime()
12671 	if (tp->t_flags & TF_ACKNOW) {  in bbr_output_wtime()
12678 	if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) {  in bbr_output_wtime()
12686 	    ((tp->t_flags & TF_SENTFIN) == 0)) {  in bbr_output_wtime()
12696 		pacing_delay = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);  in bbr_output_wtime()
12697 	if (bbr->rc_no_pacing)  in bbr_output_wtime()
12700 		if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=  in bbr_output_wtime()
12701 		    tp->snd_wnd) {  in bbr_output_wtime()
12705 			if ((bbr->rc_in_persist == 0) &&  in bbr_output_wtime()
12706 			    TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_output_wtime()
12707 			    (tp->snd_max == tp->snd_una) &&  in bbr_output_wtime()
12708 			    sbavail(&so->so_snd)) {  in bbr_output_wtime()
12710 				bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_output_wtime()
12716 		} else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12717 						 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {  in bbr_output_wtime()
12720 			bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12721 									bbr->r_ctl.rc_lost_bytes)));  in bbr_output_wtime()
12722 			bbr->rc_cwnd_limited = 1;  in bbr_output_wtime()
12728 		bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_output_wtime()
12729 		bbr->r_agg_early_set = 0;  in bbr_output_wtime()
12730 		bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
12731 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12732 	} else if (bbr->rc_use_google == 0)  in bbr_output_wtime()
12740 		bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12741 								       bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);  in bbr_output_wtime()
12746 	bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12747 	bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
12750 	if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_output_wtime()
12752 		tp->snd_nxt = tp->snd_max;  in bbr_output_wtime()
12765 		bbr->rc_tlp_in_progress = 0;  in bbr_output_wtime()
12766 		bbr->rc_tlp_rtx_out = 0;  in bbr_output_wtime()
12771 		bbr->rc_tlp_in_progress = 1;  in bbr_output_wtime()
12777 			 * This is sub-optimal. We only send a stand alone  in bbr_output_wtime()
12782 				if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) {  in bbr_output_wtime()
12795 		    (((rsm->r_flags & BBR_HAS_FIN) == 0) &&  in bbr_output_wtime()
12798 			len--;  in bbr_output_wtime()
12803 		if ((tp->snd_una == tp->snd_max) &&  in bbr_output_wtime()
12804 		    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {  in bbr_output_wtime()
12814 			tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;  in bbr_output_wtime()
12816 			tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;  in bbr_output_wtime()
12835 	 * established connection segments.  Options for SYN-ACK segments  in bbr_output_wtime()
12840 	if ((tp->t_flags & TF_NOOPT) == 0) {  in bbr_output_wtime()
12843 			to.to_mss = tcp_mssopt(&inp->inp_inc);  in bbr_output_wtime()
12844 			if (tp->t_port)  in bbr_output_wtime()
12845 				to.to_mss -= V_tcp_udp_tunneling_overhead;  in bbr_output_wtime()
12854 			if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12855 			    (tp->t_rxtshift == 0)) {  in bbr_output_wtime()
12856 				if (tp->t_state == TCPS_SYN_RECEIVED) {  in bbr_output_wtime()
12859 					    (u_int8_t *)&tp->t_tfo_cookie.server;  in bbr_output_wtime()
12862 				} else if (tp->t_state == TCPS_SYN_SENT) {  in bbr_output_wtime()
12864 					    tp->t_tfo_client_cookie_len;  in bbr_output_wtime()
12866 					    tp->t_tfo_cookie.client;  in bbr_output_wtime()
12873 		if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {  in bbr_output_wtime()
12874 			to.to_wscale = tp->request_r_scale;  in bbr_output_wtime()
12878 		if ((tp->t_flags & TF_RCVD_TSTMP) ||  in bbr_output_wtime()
12879 		    ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {  in bbr_output_wtime()
12880 			to.to_tsval = 	tcp_tv_to_msec(&bbr->rc_tv) + tp->ts_offset;  in bbr_output_wtime()
12881 			to.to_tsecr = tp->ts_recent;  in bbr_output_wtime()
12886 		if (tp->rfbuf_ts == 0 &&  in bbr_output_wtime()
12887 		    (so->so_rcv.sb_flags & SB_AUTOSIZE))  in bbr_output_wtime()
12888 			tp->rfbuf_ts = 	tcp_tv_to_msec(&bbr->rc_tv);  in bbr_output_wtime()
12892 		else if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_output_wtime()
12893 		    tp->rcv_numsacks > 0) {  in bbr_output_wtime()
12895 			to.to_nsacks = tp->rcv_numsacks;  in bbr_output_wtime()
12896 			to.to_sacks = (u_char *)tp->sackblks;  in bbr_output_wtime()
12899 		/* TCP-MD5 (RFC2385). */  in bbr_output_wtime()
12900 		if (tp->t_flags & TF_SIGNATURE)  in bbr_output_wtime()
12910 		if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie &&  in bbr_output_wtime()
12914 	if (tp->t_port) {  in bbr_output_wtime()
12927 	if (inp->inp_options)  in bbr_output_wtime()
12928 		ipoptlen = inp->inp_options->m_len -  in bbr_output_wtime()
12936 	if (bbr->rc_last_options != local_options) {  in bbr_output_wtime()
12941 		bbr->rc_last_options = local_options;  in bbr_output_wtime()
12943 	maxseg = tp->t_maxseg - (ipoptlen + optlen);  in bbr_output_wtime()
12959 			if_hw_tsomax = tp->t_tsomax;  in bbr_output_wtime()
12960 			if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;  in bbr_output_wtime()
12961 			if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;  in bbr_output_wtime()
12971 				max_len = (if_hw_tsomax - hdrlen -  in bbr_output_wtime()
12986 					len -= moff;  in bbr_output_wtime()
12999 			if (optlen + ipoptlen >= tp->t_maxseg) {  in bbr_output_wtime()
13037 		if (SEQ_LT(rsm->r_start, tp->snd_una)) {  in bbr_output_wtime()
13039 			    rsm, tp, bbr, rsm->r_start, tp->snd_una);  in bbr_output_wtime()
13083 		m->m_data += max_linkhdr;  in bbr_output_wtime()
13084 		m->m_len = hdrlen;  in bbr_output_wtime()
13109 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_output_wtime()
13116 				if (rsm->r_start != tp->snd_una) {  in bbr_output_wtime()
13124 					return (-EFAULT); /* tcp_drop() */  in bbr_output_wtime()
13126 				len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
13134 		if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {  in bbr_output_wtime()
13139 			m->m_len += len;  in bbr_output_wtime()
13153 					    tp->snd_una, rsm->r_flags, rsm->r_start,  in bbr_output_wtime()
13159 					    tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);  in bbr_output_wtime()
13163 			m->m_next = tcp_m_copym(  in bbr_output_wtime()
13177 			if (m->m_next == NULL) {  in bbr_output_wtime()
13197 				foo = foo->m_next;  in bbr_output_wtime()
13217 		if (tp->t_flags & TF_ACKNOW)  in bbr_output_wtime()
13239 			m->m_data += max_linkhdr;  in bbr_output_wtime()
13240 		m->m_len = hdrlen;  in bbr_output_wtime()
13243 	m->m_pkthdr.rcvif = (struct ifnet *)0;  in bbr_output_wtime()
13250 		if (tp->t_port) {  in bbr_output_wtime()
13252 			udp->uh_sport = htons(V_tcp_udp_tunneling_port);  in bbr_output_wtime()
13253 			udp->uh_dport = tp->t_port;  in bbr_output_wtime()
13254 			ulen = hdrlen + len - sizeof(struct ip6_hdr);  in bbr_output_wtime()
13255 			udp->uh_ulen = htons(ulen);  in bbr_output_wtime()
13260 		tcpip_fillheaders(inp, tp->t_port, ip6, th);  in bbr_output_wtime()
13265 		if (tp->t_port) {  in bbr_output_wtime()
13267 			udp->uh_sport = htons(V_tcp_udp_tunneling_port);  in bbr_output_wtime()
13268 			udp->uh_dport = tp->t_port;  in bbr_output_wtime()
13269 			ulen = hdrlen + len - sizeof(struct ip);  in bbr_output_wtime()
13270 			udp->uh_ulen = htons(ulen);  in bbr_output_wtime()
13275 		tcpip_fillheaders(inp, tp->t_port, ip, th);  in bbr_output_wtime()
13291 			th->th_seq = htonl(tp->snd_max);  in bbr_output_wtime()
13292 			bbr_seq = tp->snd_max;  in bbr_output_wtime()
13295 			th->th_seq = htonl(tp->iss);  in bbr_output_wtime()
13296 			bbr_seq = tp->iss;  in bbr_output_wtime()
13298 			if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) {  in bbr_output_wtime()
13303 				th->th_seq = (htonl(tp->snd_max - 1));  in bbr_output_wtime()
13304 				bbr_seq = (tp->snd_max - 1);  in bbr_output_wtime()
13307 				th->th_seq = htonl(tp->snd_max);  in bbr_output_wtime()
13308 				bbr_seq = tp->snd_max;  in bbr_output_wtime()
13320 			 * might be better to send (tp->snd_una - 1) which  in bbr_output_wtime()
13323 			if (tp->t_flags & TF_SENTFIN) {  in bbr_output_wtime()
13324 				th->th_seq = htonl(tp->snd_max - 1);  in bbr_output_wtime()
13325 				bbr_seq = (tp->snd_max - 1);  in bbr_output_wtime()
13327 				th->th_seq = htonl(tp->snd_max);  in bbr_output_wtime()
13328 				bbr_seq = tp->snd_max;  in bbr_output_wtime()
13333 		th->th_seq = htonl(rsm->r_start);  in bbr_output_wtime()
13334 		bbr_seq = rsm->r_start;  in bbr_output_wtime()
13336 	th->th_ack = htonl(tp->rcv_nxt);  in bbr_output_wtime()
13339 		th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;  in bbr_output_wtime()
13346 	if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) &&  in bbr_output_wtime()
13349 	if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&  in bbr_output_wtime()
13350 	    recwin < (tp->rcv_adv - tp->rcv_nxt))  in bbr_output_wtime()
13351 		recwin = (tp->rcv_adv - tp->rcv_nxt);  in bbr_output_wtime()
13352 	if (recwin > TCP_MAXWIN << tp->rcv_scale)  in bbr_output_wtime()
13353 		recwin = TCP_MAXWIN << tp->rcv_scale;  in bbr_output_wtime()
13361 		th->th_win = htons((u_short)  in bbr_output_wtime()
13362 		    (min(sbspace(&so->so_rcv), TCP_MAXWIN)));  in bbr_output_wtime()
13365 		recwin = roundup2(recwin, 1 << tp->rcv_scale);  in bbr_output_wtime()
13366 		th->th_win = htons((u_short)(recwin >> tp->rcv_scale));  in bbr_output_wtime()
13369 	 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0  in bbr_output_wtime()
13376 	if (th->th_win == 0) {  in bbr_output_wtime()
13377 		tp->t_sndzerowin++;  in bbr_output_wtime()
13378 		tp->t_flags |= TF_RXWIN0SENT;  in bbr_output_wtime()
13380 		tp->t_flags &= ~TF_RXWIN0SENT;  in bbr_output_wtime()
13385 	tp->snd_up = tp->snd_una;  in bbr_output_wtime()
13390 	m->m_pkthdr.len = hdrlen + len;	/* in6_cksum() need this */  in bbr_output_wtime()
13400 		    (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {  in bbr_output_wtime()
13416 		if (tp->t_port) {  in bbr_output_wtime()
13417 			m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;  in bbr_output_wtime()
13418 			m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);  in bbr_output_wtime()
13419 			udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);  in bbr_output_wtime()
13420 			th->th_sum = htons(0);  in bbr_output_wtime()
13423 			csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;  in bbr_output_wtime()
13424 			m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);  in bbr_output_wtime()
13425 			th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) +  in bbr_output_wtime()
13435 		if (tp->t_port) {  in bbr_output_wtime()
13436 			m->m_pkthdr.csum_flags = CSUM_UDP;  in bbr_output_wtime()
13437 			m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);  in bbr_output_wtime()
13438 			udp->uh_sum = in_pseudo(ip->ip_src.s_addr,  in bbr_output_wtime()
13439 			    ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));  in bbr_output_wtime()
13440 			th->th_sum = htons(0);  in bbr_output_wtime()
13443 			csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP;  in bbr_output_wtime()
13444 			m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);  in bbr_output_wtime()
13445 			th->th_sum = in_pseudo(ip->ip_src.s_addr,  in bbr_output_wtime()
13446 			    ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +  in bbr_output_wtime()
13450 		KASSERT(ip->ip_v == IPVERSION,  in bbr_output_wtime()
13451 		    ("%s: IP version incorrect: %d", __func__, ip->ip_v));  in bbr_output_wtime()
13463 		m->m_pkthdr.csum_flags |= CSUM_TSO;  in bbr_output_wtime()
13465 		m->m_pkthdr.tso_segsz = maxseg;  in bbr_output_wtime()
13482 		log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_output_wtime()
13483 		log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);  in bbr_output_wtime()
13486 		log.u_bbr.flex5 = bbr->rc_past_init_win;  in bbr_output_wtime()
13488 		log.u_bbr.flex5 |= bbr->rc_no_pacing;  in bbr_output_wtime()
13490 		log.u_bbr.flex5 |= tp->t_maxseg;  in bbr_output_wtime()
13491 		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;  in bbr_output_wtime()
13492 		log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);  in bbr_output_wtime()
13494 		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;  in bbr_output_wtime()
13503 		lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,  in bbr_output_wtime()
13514 	 * m->m_pkthdr.len should have been set before cksum calcuration,  in bbr_output_wtime()
13525 		ip6->ip6_hlim = in6_selecthlim(inp, NULL);  in bbr_output_wtime()
13532 		ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));  in bbr_output_wtime()
13535 			tp->t_flags2 |= TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13537 			tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13539 		if (tp->t_state == TCPS_SYN_SENT)  in bbr_output_wtime()
13544 		error = ip6_output(m, inp->in6p_outputopts,  in bbr_output_wtime()
13545 		    &inp->inp_route6,  in bbr_output_wtime()
13549 		if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)  in bbr_output_wtime()
13550 			mtu = inp->inp_route6.ro_nh->nh_mtu;  in bbr_output_wtime()
13558 		ip->ip_len = htons(m->m_pkthdr.len);  in bbr_output_wtime()
13561 			ip->ip_ttl = in6_selecthlim(inp, NULL);  in bbr_output_wtime()
13573 		if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {  in bbr_output_wtime()
13574 			tp->t_flags2 |= TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13575 			if (tp->t_port == 0 || len < V_tcp_minmss) {  in bbr_output_wtime()
13576 				ip->ip_off |= htons(IP_DF);  in bbr_output_wtime()
13579 			tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13582 		if (tp->t_state == TCPS_SYN_SENT)  in bbr_output_wtime()
13587 		error = ip_output(m, inp->inp_options, &inp->inp_route,  in bbr_output_wtime()
13590 		if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)  in bbr_output_wtime()
13591 			mtu = inp->inp_route.ro_nh->nh_mtu;  in bbr_output_wtime()
13595 		lgb->tlb_errno = error;  in bbr_output_wtime()
13606 		if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_output_wtime()
13607 		    (tp->t_flags & TF_SACK_PERMIT) &&  in bbr_output_wtime()
13608 		    tp->rcv_numsacks > 0)  in bbr_output_wtime()
13611 		bbr->output_error_seen = 0;  in bbr_output_wtime()
13612 		bbr->oerror_cnt = 0;  in bbr_output_wtime()
13613 		bbr->bbr_segs_rcvd = 0;  in bbr_output_wtime()
13619 			if (tp->snd_una == tp->snd_max) {  in bbr_output_wtime()
13626 				bbr->r_ctl.rc_del_time = cts;  in bbr_output_wtime()
13631 					counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1);  in bbr_output_wtime()
13636 				idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);  in bbr_output_wtime()
13638 					idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1);  in bbr_output_wtime()
13652 			if (tp->snd_una == tp->snd_max)  in bbr_output_wtime()
13653 				bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_output_wtime()
13665 	if (bbr->rc_in_persist == 0) {  in bbr_output_wtime()
13673 		if (tp->snd_una == tp->snd_max &&  in bbr_output_wtime()
13680 			bbr->rc_tp->t_acktime  = ticks;  in bbr_output_wtime()
13688 				tp->snd_max = tp->iss + 1;  in bbr_output_wtime()
13690 			if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {  in bbr_output_wtime()
13691 				tp->snd_max++;  in bbr_output_wtime()
13692 				tp->t_flags |= TF_SENTFIN;  in bbr_output_wtime()
13696 			tp->snd_max += len;  in bbr_output_wtime()
13709 		if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {  in bbr_output_wtime()
13711 			tp->t_flags |= TF_SENTFIN;  in bbr_output_wtime()
13713 		if (xlen && (tp->snd_una == tp->snd_max)) {  in bbr_output_wtime()
13719 			bbr->rc_tp->t_acktime = ticks;  in bbr_output_wtime()
13722 			tp->snd_max += xlen;  in bbr_output_wtime()
13729 		 * case of ENOBUFS we will fall out and become ack-clocked.  in bbr_output_wtime()
13737 		bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_output_wtime()
13738 		bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
13739 		bbr->r_agg_early_set = 0;  in bbr_output_wtime()
13740 		bbr->output_error_seen = 1;  in bbr_output_wtime()
13741 		if (bbr->oerror_cnt < 0xf)  in bbr_output_wtime()
13742 			bbr->oerror_cnt++;  in bbr_output_wtime()
13743 		if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {  in bbr_output_wtime()
13745 			return (-ENETDOWN);  in bbr_output_wtime()
13752 			 * slam him below a T-O (1MSS).  in bbr_output_wtime()
13754 			if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {  in bbr_output_wtime()
13755 				tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
13756 								    bbr->r_ctl.rc_lost_bytes)) - maxseg;  in bbr_output_wtime()
13757 				if (tp->snd_cwnd < maxseg)  in bbr_output_wtime()
13758 					tp->snd_cwnd = maxseg;  in bbr_output_wtime()
13760 			pacing_delay = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;  in bbr_output_wtime()
13762 			if (bbr->bbr_hdrw_pacing)  in bbr_output_wtime()
13787 				old_maxseg = tp->t_maxseg;  in bbr_output_wtime()
13791 					tcp_mss_update(tp, -1, mtu, NULL, NULL);  in bbr_output_wtime()
13792 				if (old_maxseg <= tp->t_maxseg) {  in bbr_output_wtime()
13794 					tp->t_maxseg = old_maxseg - 40;  in bbr_output_wtime()
13795 					if (tp->t_maxseg < V_tcp_mssdflt) {  in bbr_output_wtime()
13801 						tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;  in bbr_output_wtime()
13803 						tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;  in bbr_output_wtime()
13811 				if ((tot_len + len) && (len >= tp->t_maxseg)) {  in bbr_output_wtime()
13813 					    bbr->r_ctl.rc_bbr_hptsi_gain,  in bbr_output_wtime()
13816 						pacing_delay = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;  in bbr_output_wtime()
13818 					pacing_delay = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;  in bbr_output_wtime()
13819 				bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
13826 			tp->t_softerror = error;  in bbr_output_wtime()
13832 			if (TCPS_HAVERCVDSYN(tp->t_state)) {  in bbr_output_wtime()
13833 				tp->t_softerror = error;  in bbr_output_wtime()
13838 			pacing_delay = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;  in bbr_output_wtime()
13839 			bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
13844 	} else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&  in bbr_output_wtime()
13847 	    (bbr->rc_in_persist == 0)) {  in bbr_output_wtime()
13848 		tp->gput_seq = bbr_seq;  in bbr_output_wtime()
13849 		tp->gput_ack = bbr_seq +  in bbr_output_wtime()
13850 		    min(sbavail(&so->so_snd) - sb_offset, sendwin);  in bbr_output_wtime()
13851 		tp->gput_ts = cts;  in bbr_output_wtime()
13852 		tp->t_flags |= TF_GPUTINPROG;  in bbr_output_wtime()
13856 	if ((bbr->bbr_hdw_pace_ena) &&  in bbr_output_wtime()
13857 	    (bbr->bbr_attempt_hdwr_pace == 0) &&  in bbr_output_wtime()
13858 	    (bbr->rc_past_init_win) &&  in bbr_output_wtime()
13859 	    (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&  in bbr_output_wtime()
13860 	    (get_filter_value(&bbr->r_ctl.rc_delrate)) &&  in bbr_output_wtime()
13861 	    (inp->inp_route.ro_nh &&  in bbr_output_wtime()
13862 	     inp->inp_route.ro_nh->nh_ifp)) {  in bbr_output_wtime()
13874 		bbr->bbr_attempt_hdwr_pace = 1;  in bbr_output_wtime()
13875 		bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,  in bbr_output_wtime()
13876 						      inp->inp_route.ro_nh->nh_ifp,  in bbr_output_wtime()
13880 		if (bbr->r_ctl.crte) {  in bbr_output_wtime()
13882 						 bbr->r_ctl.crte->ptbl->rs_ifp,  in bbr_output_wtime()
13884 						 bbr->r_ctl.crte->rate,  in bbr_output_wtime()
13887 			counter_u64_add(bbr_flows_nohdwr_pacing, -1);  in bbr_output_wtime()
13889 			bbr->bbr_hdrw_pacing = 1;  in bbr_output_wtime()
13891 			if (bbr->r_ctl.crte->rate < rate_wanted) {  in bbr_output_wtime()
13894 						       bbr->r_ctl.crte->rate, rate_wanted);  in bbr_output_wtime()
13897 				bbr->gain_is_limited = 0;  in bbr_output_wtime()
13898 				bbr->skip_gain = 0;  in bbr_output_wtime()
13903 						 inp->inp_route.ro_nh->nh_ifp,  in bbr_output_wtime()
13910 	if (bbr->bbr_hdrw_pacing) {  in bbr_output_wtime()
13917 		if (inp->inp_snd_tag == NULL) {  in bbr_output_wtime()
13919 			bbr->bbr_hdrw_pacing = 0;  in bbr_output_wtime()
13920 		} else if ((inp->inp_route.ro_nh == NULL) ||  in bbr_output_wtime()
13921 		    (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) {  in bbr_output_wtime()
13925 			 * and setup to re-attempt next go  in bbr_output_wtime()
13928 			bbr->bbr_hdrw_pacing = 0;  in bbr_output_wtime()
13929 			bbr->bbr_attempt_hdwr_pace = 0;  in bbr_output_wtime()
13930 			tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);  in bbr_output_wtime()
13939 	if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))  in bbr_output_wtime()
13940 		tp->rcv_adv = tp->rcv_nxt + recwin;  in bbr_output_wtime()
13942 	tp->last_ack_sent = tp->rcv_nxt;  in bbr_output_wtime()
13944 	    (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&  in bbr_output_wtime()
13950 	    (IN_RECOVERY(tp->t_flags) == 0) &&  in bbr_output_wtime()
13951 	    (bbr->rc_in_persist == 0) &&  in bbr_output_wtime()
13952 	    (tot_len < bbr->r_ctl.rc_pace_max_segs)) {  in bbr_output_wtime()
13954 		 * For non-tso we need to goto again until we have sent out  in bbr_output_wtime()
13958 		if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_output_wtime()
13960 			tp->snd_nxt = tp->snd_max;  in bbr_output_wtime()
13968 		tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);  in bbr_output_wtime()
13978 		 * Calculate/Re-Calculate the hptsi timeout in usecs based on  in bbr_output_wtime()
13981 		pacing_delay = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);  in bbr_output_wtime()
13982 		if (bbr->rc_no_pacing)  in bbr_output_wtime()
13985 	tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);  in bbr_output_wtime()
13987 	if (bbr->rc_use_google == 0)  in bbr_output_wtime()
13989 	bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
13990 							bbr->r_ctl.rc_lost_bytes)));  in bbr_output_wtime()
13991 	bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
13992 	if (bbr->bbr_use_rack_cheat &&  in bbr_output_wtime()
13994 	     ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {  in bbr_output_wtime()
13999 	if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {  in bbr_output_wtime()
14005 		bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;  in bbr_output_wtime()
14006 		if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {  in bbr_output_wtime()
14007 			bbr->hw_pacing_set = 1;  in bbr_output_wtime()
14012 	if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_output_wtime()
14014 		tp->snd_nxt = tp->snd_max;  in bbr_output_wtime()
14049 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_mtu_chg()
14050 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_mtu_chg()
14051 	sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_mtu_chg()
14052 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_mtu_chg()
14054 		if (rsm->r_flags & BBR_ACKED)  in bbr_mtu_chg()
14056 		if ((rsm->r_end - rsm->r_start) > maxseg) {  in bbr_mtu_chg()
14058 			 * We mark sack-passed on all the previous large  in bbr_mtu_chg()
14061 			rsm->r_flags |= BBR_SACK_PASSED;  in bbr_mtu_chg()
14062 			if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&  in bbr_mtu_chg()
14063 			    bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {  in bbr_mtu_chg()
14064 				bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;  in bbr_mtu_chg()
14065 				bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;  in bbr_mtu_chg()
14066 				rsm->r_flags |= BBR_MARKED_LOST;  in bbr_mtu_chg()
14073 		bbr->r_ctl.rc_resend = frsm;  in bbr_mtu_chg()
14105 	tp->t_flags2 |= TF2_CANNOT_DO_ECN;  in bbr_switch_failed()
14106 	tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;  in bbr_switch_failed()
14108 	if (tp->t_in_hpts > IHPTS_NONE) {  in bbr_switch_failed()
14111 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_switch_failed()
14113 	if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {  in bbr_switch_failed()
14114 		if (TSTMP_GT(bbr->rc_pacer_started, cts)) {  in bbr_switch_failed()
14115 			toval = bbr->rc_pacer_started - cts;  in bbr_switch_failed()
14120 	} else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {  in bbr_switch_failed()
14121 		if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {  in bbr_switch_failed()
14122 			toval = bbr->r_ctl.rc_timer_exp - cts;  in bbr_switch_failed()
14153  * socket option arguments.  When it re-acquires the lock after the copy, it
14165 	switch (sopt->sopt_level) {  in bbr_set_sockopt()
14171 	switch (sopt->sopt_name) {  in bbr_set_sockopt()
14220 	if (inp->inp_flags & INP_DROPPED) {  in bbr_set_sockopt()
14224 	if (tp->t_fb != &__tcp_bbr) {  in bbr_set_sockopt()
14228 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_set_sockopt()
14229 	switch (sopt->sopt_name) {  in bbr_set_sockopt()
14232 		bbr->r_ctl.bbr_hptsi_per_second = optval;  in bbr_set_sockopt()
14236 		bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;  in bbr_set_sockopt()
14240 		bbr->r_ctl.bbr_hptsi_segments_max = optval;  in bbr_set_sockopt()
14244 		bbr->r_ctl.bbr_hptsi_bytes_min = optval;  in bbr_set_sockopt()
14248 		bbr->r_ctl.bbr_cross_over = optval;  in bbr_set_sockopt()
14252 		if (optval && (bbr->rc_use_google == 0)) {  in bbr_set_sockopt()
14260 				bbr->r_ctl.bbr_google_discount = optval;  in bbr_set_sockopt()
14262 		} else if ((optval == 0) && (bbr->rc_use_google == 1)) {  in bbr_set_sockopt()
14270 			bbr->rc_use_ts_limit = 1;  in bbr_set_sockopt()
14272 			bbr->rc_use_ts_limit = 0;  in bbr_set_sockopt()
14282 			bbr->rc_init_win = optval;  in bbr_set_sockopt()
14284 			if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))  in bbr_set_sockopt()
14285 				tp->snd_cwnd = twin;  in bbr_set_sockopt()
14294 			bbr->r_ctl.rc_startup_pg = optval;  in bbr_set_sockopt()
14295 			if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in bbr_set_sockopt()
14296 				bbr->r_ctl.rc_bbr_hptsi_gain = optval;  in bbr_set_sockopt()
14304 			bbr->r_ctl.rc_drain_pg = optval;  in bbr_set_sockopt()
14311 			reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));  in bbr_set_sockopt()
14318 			bbr->r_ctl.bbr_rttprobe_gain_val = optval;  in bbr_set_sockopt()
14325 			bbr->r_ctl.rc_probertt_int = optval;  in bbr_set_sockopt()
14332 			bbr->no_pacing_until = 0;  in bbr_set_sockopt()
14333 			bbr->rc_no_pacing = 0;  in bbr_set_sockopt()
14335 			bbr->no_pacing_until = optval;  in bbr_set_sockopt()
14336 			if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&  in bbr_set_sockopt()
14337 			    (bbr->rc_bbr_state == BBR_STATE_STARTUP)){  in bbr_set_sockopt()
14339 				bbr->rc_no_pacing = 1;  in bbr_set_sockopt()
14346 		bbr->rc_loss_exit = optval;  in bbr_set_sockopt()
14353 		bbr->r_ctl.rc_min_rto_ms = optval;  in bbr_set_sockopt()
14357 		bbr->rc_max_rto_sec = optval;  in bbr_set_sockopt()
14360 		/* Minimum time between rack t-o's in ms */  in bbr_set_sockopt()
14362 		bbr->r_ctl.rc_min_to = optval;  in bbr_set_sockopt()
14368 			bbr->r_ctl.rc_reorder_shift = optval;  in bbr_set_sockopt()
14375 		bbr->r_ctl.rc_reorder_fade = optval;  in bbr_set_sockopt()
14381 			bbr->rc_tlp_threshold = optval;  in bbr_set_sockopt()
14387 		if (bbr->rc_use_google) {  in bbr_set_sockopt()
14393 			bbr->bbr_use_rack_cheat = 1;  in bbr_set_sockopt()
14395 			bbr->bbr_use_rack_cheat = 0;  in bbr_set_sockopt()
14400 			bbr->r_ctl.bbr_hptsi_segments_floor = optval;  in bbr_set_sockopt()
14407 			bbr->r_ctl.bbr_utter_max = optval;  in bbr_set_sockopt()
14415 			bbr->rc_use_idle_restart = 1;  in bbr_set_sockopt()
14417 			bbr->rc_use_idle_restart = 0;  in bbr_set_sockopt()
14422 			bbr->bbr_init_win_cheat = 1;  in bbr_set_sockopt()
14423 			if (bbr->rc_past_init_win == 0) {  in bbr_set_sockopt()
14425 				cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_set_sockopt()
14429 			bbr->bbr_init_win_cheat = 0;  in bbr_set_sockopt()
14434 			bbr->bbr_hdw_pace_ena = 1;  in bbr_set_sockopt()
14435 			bbr->bbr_attempt_hdwr_pace = 0;  in bbr_set_sockopt()
14437 			bbr->bbr_hdw_pace_ena = 0;  in bbr_set_sockopt()
14439 			if (bbr->r_ctl.crte != NULL) {  in bbr_set_sockopt()
14440 				tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);  in bbr_set_sockopt()
14441 				bbr->r_ctl.crte = NULL;  in bbr_set_sockopt()
14451 				tp->t_delayed_ack = 0;  in bbr_set_sockopt()
14453 				tp->t_delayed_ack = 2;  in bbr_set_sockopt()
14455 				tp->t_delayed_ack = optval;  in bbr_set_sockopt()
14456 			if (tp->t_flags & TF_DELACK) {  in bbr_set_sockopt()
14457 				tp->t_flags &= ~TF_DELACK;  in bbr_set_sockopt()
14458 				tp->t_flags |= TF_ACKNOW;  in bbr_set_sockopt()
14467 		/* RACK added ms i.e. rack-rtt + reord + N */  in bbr_set_sockopt()
14469 		bbr->r_ctl.rc_pkt_delay = optval;  in bbr_set_sockopt()
14475 			bbr->rc_resends_use_tso = 1;  in bbr_set_sockopt()
14477 			bbr->rc_resends_use_tso = 0;  in bbr_set_sockopt()
14482 			bbr->rc_allow_data_af_clo = 1;  in bbr_set_sockopt()
14484 			bbr->rc_allow_data_af_clo = 0;  in bbr_set_sockopt()
14488 		if (bbr->rc_use_google == 0)  in bbr_set_sockopt()
14491 			bbr->r_use_policer = 1;  in bbr_set_sockopt()
14493 			bbr->r_use_policer = 0;  in bbr_set_sockopt()
14499 			bbr->ts_can_raise = 1;  in bbr_set_sockopt()
14501 			bbr->ts_can_raise = 0;  in bbr_set_sockopt()
14505 		if (bbr->rc_use_google) {  in bbr_set_sockopt()
14509 				bbr->r_ctl.rc_incr_tmrs = 1;  in bbr_set_sockopt()
14511 				bbr->r_ctl.rc_incr_tmrs = 0;  in bbr_set_sockopt()
14516 		if (bbr->rc_use_google) {  in bbr_set_sockopt()
14526 				bbr->r_ctl.rc_inc_tcp_oh = 1;  in bbr_set_sockopt()
14528 				bbr->r_ctl.rc_inc_tcp_oh = 0;  in bbr_set_sockopt()
14530 				bbr->r_ctl.rc_inc_ip_oh = 1;  in bbr_set_sockopt()
14532 				bbr->r_ctl.rc_inc_ip_oh = 0;  in bbr_set_sockopt()
14534 				bbr->r_ctl.rc_inc_enet_oh = 1;  in bbr_set_sockopt()
14536 				bbr->r_ctl.rc_inc_enet_oh = 0;  in bbr_set_sockopt()
14543 	tcp_log_socket_option(tp, sopt->sopt_name, optval, error);  in bbr_set_sockopt()
14549  * return 0 on success, error-num on failure
14559 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_get_sockopt()
14570 	switch (sopt->sopt_name) {  in bbr_get_sockopt()
14572 		optval = bbr->r_ctl.bbr_hptsi_per_second;  in bbr_get_sockopt()
14575 		optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;  in bbr_get_sockopt()
14578 		optval = bbr->r_ctl.bbr_hptsi_segments_max;  in bbr_get_sockopt()
14581 		optval = bbr->no_pacing_until;  in bbr_get_sockopt()
14584 		optval = bbr->r_ctl.bbr_hptsi_bytes_min;  in bbr_get_sockopt()
14587 		optval = bbr->r_ctl.bbr_cross_over;  in bbr_get_sockopt()
14590 		optval = bbr->rc_use_google;  in bbr_get_sockopt()
14593 		optval = bbr->rc_use_ts_limit;  in bbr_get_sockopt()
14596 		optval = bbr->rc_init_win;  in bbr_get_sockopt()
14599 		optval = bbr->r_ctl.rc_startup_pg;  in bbr_get_sockopt()
14602 		optval = bbr->r_ctl.rc_drain_pg;  in bbr_get_sockopt()
14605 		optval = bbr->r_ctl.rc_probertt_int;  in bbr_get_sockopt()
14608 		optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);  in bbr_get_sockopt()
14611 		optval = bbr->r_ctl.bbr_rttprobe_gain_val;  in bbr_get_sockopt()
14614 		optval = bbr->rc_loss_exit;  in bbr_get_sockopt()
14617 		loptval = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_get_sockopt()
14620 		optval = bbr->r_ctl.rc_min_rto_ms;  in bbr_get_sockopt()
14623 		optval = bbr->rc_max_rto_sec;  in bbr_get_sockopt()
14627 		optval = bbr->r_ctl.rc_pace_max_segs;  in bbr_get_sockopt()
14630 		/* Minimum time between rack t-o's in ms */  in bbr_get_sockopt()
14631 		optval = bbr->r_ctl.rc_min_to;  in bbr_get_sockopt()
14635 		optval = bbr->r_ctl.rc_reorder_shift;  in bbr_get_sockopt()
14639 		optval = bbr->r_ctl.rc_reorder_fade;  in bbr_get_sockopt()
14643 		optval = bbr->bbr_use_rack_cheat;  in bbr_get_sockopt()
14646 		optval = bbr->r_ctl.bbr_hptsi_segments_floor;  in bbr_get_sockopt()
14649 		optval = bbr->r_ctl.bbr_utter_max;  in bbr_get_sockopt()
14653 		optval = bbr->bbr_init_win_cheat;  in bbr_get_sockopt()
14656 		optval = bbr->rc_use_idle_restart;  in bbr_get_sockopt()
14660 		optval = bbr->rc_tlp_threshold;  in bbr_get_sockopt()
14663 		/* RACK added ms i.e. rack-rtt + reord + N */  in bbr_get_sockopt()
14664 		optval = bbr->r_ctl.rc_pkt_delay;  in bbr_get_sockopt()
14667 		optval = bbr->rc_resends_use_tso;  in bbr_get_sockopt()
14670 		optval = bbr->rc_allow_data_af_clo;  in bbr_get_sockopt()
14673 		optval = tp->t_delayed_ack;  in bbr_get_sockopt()
14676 		optval = bbr->bbr_hdw_pace_ena;  in bbr_get_sockopt()
14679 		optval = bbr->r_use_policer;  in bbr_get_sockopt()
14682 		optval = bbr->ts_can_raise;  in bbr_get_sockopt()
14685 		optval = bbr->r_ctl.rc_incr_tmrs;  in bbr_get_sockopt()
14689 		if (bbr->r_ctl.rc_inc_tcp_oh)  in bbr_get_sockopt()
14691 		if (bbr->r_ctl.rc_inc_ip_oh)  in bbr_get_sockopt()
14693 		if (bbr->r_ctl.rc_inc_enet_oh)  in bbr_get_sockopt()
14701 	if (sopt->sopt_name == TCP_BBR_USEDEL_RATE)  in bbr_get_sockopt()
14709  * return 0 on success, error-num on failure
14714 	if (sopt->sopt_dir == SOPT_SET) {  in bbr_ctloutput()
14716 	} else if (sopt->sopt_dir == SOPT_GET) {  in bbr_ctloutput()
14719 		panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);  in bbr_ctloutput()