1 /*-
2  * Copyright (c) 2016-2020 Netflix, Inc.
29  * BBR - Congestion Based Congestion Control
159 /* Should the following be dynamic too -- loss wise */
193 						 * num-measures > min(0) and
227 /* thresholds for reduction on drain in sub-states/drain */
265  * means in non-recovery/retransmission scenarios
266  * cwnd will never be reached by the flight-size.
271 static int32_t bbr_sack_not_required = 0;	/* set to one to allow non-sack to use bbr */
292  * - Yuchung Cheng's RACK TCP (for which its named) that
295  * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
297  * - Van Jacobson's et.al BBR.
319  * TCP output is also over-written with a new version since it
331 static int32_t bbr_reorder_fade = 60000000;	/* 0 - never fade, def
332 						 * 60,000,000 - 60 seconds */
385 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
388 /* What is the max the 0 - bbr_cross_over MBPS TSO target
397 /* What is the min the 0 - bbr_cross-over MBPS  TSO target can be */
401 /* Cross over point from algo-a to algo-b */
517 	return(bbr->rc_bbr_substate);  in bbr_state_val()
525 	mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),  in get_min_cwnd()
526 		  bbr->r_ctl.rc_pace_max_segs);  in get_min_cwnd()
539 	bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;  in bbr_get_persists_timer_val()
540 	if (tp->t_srtt == 0) {  in bbr_get_persists_timer_val()
544 		srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_get_persists_timer_val()
545 		var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT);  in bbr_get_persists_timer_val()
547 	TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]),  in bbr_get_persists_timer_val()
566 	if (bbr->rc_all_timers_stopped) {  in bbr_timer_start()
570 	if (bbr->rc_in_persist) {  in bbr_timer_start()
574 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_timer_start()
576 	    ((tp->t_flags & TF_SACK_PERMIT) == 0) ||  in bbr_timer_start()
577 	    (tp->t_state < TCPS_ESTABLISHED)) {  in bbr_timer_start()
580 		if (SEQ_LT(tp->snd_una, tp->snd_max) ||  in bbr_timer_start()
581 		    sbavail(&tptosocket(tp)->so_snd)) {  in bbr_timer_start()
585 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_timer_start()
587 				idx = rsm->r_rtr_cnt - 1;  in bbr_timer_start()
588 				if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))  in bbr_timer_start()
589 					tstmp_touse = rsm->r_tim_lastsent[idx];  in bbr_timer_start()
591 					tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;  in bbr_timer_start()
593 				    time_since_sent = cts - tstmp_touse;  in bbr_timer_start()
595 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;  in bbr_timer_start()
596 			if (tp->t_srtt == 0)  in bbr_timer_start()
599 				tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) +  in bbr_timer_start()
600 				    ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT);  in bbr_timer_start()
601 			if (tp->t_rxtshift)  in bbr_timer_start()
602 				tov *= tcp_backoff[tp->t_rxtshift];  in bbr_timer_start()
604 				tov -= time_since_sent;  in bbr_timer_start()
606 				tov = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
608 			    (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),  in bbr_timer_start()
609 			    (bbr->rc_max_rto_sec * USECS_IN_SECOND));  in bbr_timer_start()
615 	if (rsm->r_flags & BBR_ACKED) {  in bbr_timer_start()
623 	if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_timer_start()
624 		if ((tp->t_flags & TF_SENTFIN) &&  in bbr_timer_start()
625 		    ((tp->snd_max - tp->snd_una) == 1) &&  in bbr_timer_start()
626 		    (rsm->r_flags & BBR_HAS_FIN)) {  in bbr_timer_start()
635 		idx = rsm->r_rtr_cnt - 1;  in bbr_timer_start()
636 		exp = rsm->r_tim_lastsent[idx] + thresh;  in bbr_timer_start()
638 			to = exp - cts;  in bbr_timer_start()
639 			if (to < bbr->r_ctl.rc_min_to) {  in bbr_timer_start()
640 				to = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
643 			to = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
647 		if (bbr->rc_tlp_in_progress != 0) {  in bbr_timer_start()
653 		rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);  in bbr_timer_start()
658 		if (rsm->r_flags & BBR_HAS_FIN) {  in bbr_timer_start()
664 		idx = rsm->r_rtr_cnt - 1;  in bbr_timer_start()
665 		if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))  in bbr_timer_start()
666 			tstmp_touse = rsm->r_tim_lastsent[idx];  in bbr_timer_start()
668 			tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;  in bbr_timer_start()
670 		    time_since_sent = cts - tstmp_touse;  in bbr_timer_start()
675 			to = thresh - time_since_sent;  in bbr_timer_start()
677 			to = bbr->r_ctl.rc_min_to;  in bbr_timer_start()
678 		if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {  in bbr_timer_start()
685 		if ((bbr->rc_tlp_rtx_out == 1) &&  in bbr_timer_start()
686 		    (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {  in bbr_timer_start()
691 			bbr->rc_tlp_rtx_out = 0;  in bbr_timer_start()
694 		if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {  in bbr_timer_start()
699 			bbr->r_ctl.rc_tlp_seg_send_cnt = 0;  in bbr_timer_start()
700 			bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;  in bbr_timer_start()
705 		bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;  in bbr_timer_start()
708 		if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {  in bbr_timer_start()
716 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;  in bbr_timer_start()
725 	return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);  in bbr_minseg()
744 	if ((tp->t_state == TCPS_CLOSED) ||  in bbr_start_hpts_timer()
745 	    (tp->t_state == TCPS_LISTEN)) {  in bbr_start_hpts_timer()
748 	stopped = bbr->rc_tmr_stopped;  in bbr_start_hpts_timer()
749 	if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {  in bbr_start_hpts_timer()
750 		left = bbr->r_ctl.rc_timer_exp - cts;  in bbr_start_hpts_timer()
752 	bbr->r_ctl.rc_hpts_flags = 0;  in bbr_start_hpts_timer()
753 	bbr->r_ctl.rc_timer_exp = 0;  in bbr_start_hpts_timer()
754 	prev_delay = bbr->r_ctl.rc_last_delay_val;  in bbr_start_hpts_timer()
755 	if (bbr->r_ctl.rc_last_delay_val &&  in bbr_start_hpts_timer()
762 		slot = bbr->r_ctl.rc_last_delay_val;  in bbr_start_hpts_timer()
763 		if (TSTMP_GT(cts, bbr->rc_pacer_started)) {  in bbr_start_hpts_timer()
765 			delay_calc = cts - bbr->rc_pacer_started;  in bbr_start_hpts_timer()
767 				slot -= delay_calc;  in bbr_start_hpts_timer()
771 	if (bbr->r_agg_early_set) {  in bbr_start_hpts_timer()
772 		bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2);  in bbr_start_hpts_timer()
773 		slot += bbr->r_ctl.rc_agg_early;  in bbr_start_hpts_timer()
774 		bbr->r_ctl.rc_agg_early = 0;  in bbr_start_hpts_timer()
775 		bbr->r_agg_early_set = 0;  in bbr_start_hpts_timer()
778 	if (bbr->r_ctl.rc_hptsi_agg_delay) {  in bbr_start_hpts_timer()
779 		if (slot > bbr->r_ctl.rc_hptsi_agg_delay) {  in bbr_start_hpts_timer()
781 			slot -= bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_start_hpts_timer()
782 			bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_start_hpts_timer()
785 			bbr->r_ctl.rc_hptsi_agg_delay -= slot;  in bbr_start_hpts_timer()
786 			bbr->r_ctl.rc_last_delay_val = slot = 100;  in bbr_start_hpts_timer()
789 	bbr->r_ctl.rc_last_delay_val = slot;  in bbr_start_hpts_timer()
791 	if (tp->t_flags & TF_DELACK) {  in bbr_start_hpts_timer()
792 		if (bbr->rc_in_persist == 0) {  in bbr_start_hpts_timer()
812 		bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;  in bbr_start_hpts_timer()
818 		bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;  in bbr_start_hpts_timer()
822 	 * wheel, we resort to a keep-alive timer if its configured.  in bbr_start_hpts_timer()
826 		if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_start_hpts_timer()
827 		    (tp->t_state <= TCPS_CLOSING)) {  in bbr_start_hpts_timer()
830 			 * del-ack), we don't have segments being paced. So  in bbr_start_hpts_timer()
833 			if (TCPS_HAVEESTABLISHED(tp->t_state)) {  in bbr_start_hpts_timer()
838 			bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;  in bbr_start_hpts_timer()
842 	    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {  in bbr_start_hpts_timer()
848 		 * keep-alive, delayed_ack we keep track of what was left  in bbr_start_hpts_timer()
854 	if (bbr->r_ctl.rc_incr_tmrs && slot &&  in bbr_start_hpts_timer()
855 	    (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {  in bbr_start_hpts_timer()
876 		 * Hack alert for now we can't time-out over 2147 seconds (a  in bbr_start_hpts_timer()
881 		bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;  in bbr_start_hpts_timer()
883 		bbr->r_ctl.rc_timer_exp = 0;  in bbr_start_hpts_timer()
885 	    (bbr->rc_use_google ||  in bbr_start_hpts_timer()
886 	     bbr->output_error_seen ||  in bbr_start_hpts_timer()
892 		bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY;  in bbr_start_hpts_timer()
893 		if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&  in bbr_start_hpts_timer()
894 		    (bbr->rc_cwnd_limited == 0)) {  in bbr_start_hpts_timer()
900 			tp->t_flags2 |= TF2_DONT_SACK_QUEUE;  in bbr_start_hpts_timer()
902 			tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;  in bbr_start_hpts_timer()
903 		bbr->rc_pacer_started = cts;  in bbr_start_hpts_timer()
907 		bbr->rc_timer_first = 0;  in bbr_start_hpts_timer()
908 		bbr->bbr_timer_src = frm;  in bbr_start_hpts_timer()
921 		 * on a keep-alive timer and a request comes in for  in bbr_start_hpts_timer()
925 			bbr->rc_pacer_started = cts;  in bbr_start_hpts_timer()
926 		if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&  in bbr_start_hpts_timer()
927 		    (bbr->rc_cwnd_limited == 0)) {  in bbr_start_hpts_timer()
933 			tp->t_flags2 |= (TF2_MBUF_QUEUE_READY |  in bbr_start_hpts_timer()
937 			tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY |  in bbr_start_hpts_timer()
940 		bbr->bbr_timer_src = frm;  in bbr_start_hpts_timer()
943 		bbr->rc_timer_first = 1;  in bbr_start_hpts_timer()
945 	bbr->rc_tmr_stopped = 0;  in bbr_start_hpts_timer()
962 	tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;  in bbr_timer_audit()
963 	if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))  in bbr_timer_audit()
965 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_timer_audit()
966 	if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&  in bbr_timer_audit()
971 	inp = bbr->rc_inp;  in bbr_timer_audit()
974 		if (tp->t_flags & TF_DELACK) {  in bbr_timer_audit()
981 		} else if (sbavail(&inp->inp_socket->so_snd) &&  in bbr_timer_audit()
990 			    inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_timer_audit()
991 			    (tp->t_state <= TCPS_CLOSING)) &&  in bbr_timer_audit()
993 		    (tp->snd_max == tp->snd_una)) {  in bbr_timer_audit()
998 	if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {  in bbr_timer_audit()
999 		if ((tp->t_flags & TF_SENTFIN) &&  in bbr_timer_audit()
1000 		    ((tp->snd_max - tp->snd_una) == 1) &&  in bbr_timer_audit()
1001 		    (rsm->r_flags & BBR_HAS_FIN)) {  in bbr_timer_audit()
1014 	} else if (SEQ_GT(tp->snd_max, tp->snd_una) &&  in bbr_timer_audit()
1018 		 * Either a TLP or RXT is fine if no sack-passed is in place  in bbr_timer_audit()
1031 	if (SEQ_GT(tp->snd_max, tp->snd_una) &&  in bbr_timer_audit()
1050 	if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {  in bbr_timer_audit()
1054 		bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,  in bbr_timer_audit()
1059 		 * timer. We don't bother with keep-alive, since when we  in bbr_timer_audit()
1060 		 * jump through the output, it will start the keep-alive if  in bbr_timer_audit()
1063 		 * We only need a delayed-ack added and or the hpts_timeout.  in bbr_timer_audit()
1066 		if (tp->t_flags & TF_DELACK) {  in bbr_timer_audit()
1069 				bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;  in bbr_timer_audit()
1073 				bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;  in bbr_timer_audit()
1079 			bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;  in bbr_timer_audit()
1101 	 * time-stamp taken in theory earlier return the difference. The  in bbr_calc_time()
1106 		return (cts - earlier_time);  in bbr_calc_time()
1119 	return (cts - earlier_time);  in bbr_calc_time()
1129 	if (error || req->newptr == NULL)  in sysctl_bbr_clear_lost()
1154 		printf("Clearing BBR out-size counters\n");  in sysctl_bbr_clear_lost()
1190 	    "How many mss's are outstanding during probe-rtt");  in bbr_init_sysctls()
1195 	    "If RTT has not shrank in this many micro-seconds enter probe-rtt");  in bbr_init_sysctls()
1200 	    "How many microseconds in probe-rtt");  in bbr_init_sysctls()
1215 	    "If we keep setting new low rtt's but delay going in probe-rtt can we force in??");  in bbr_init_sysctls()
1220 	    "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?");  in bbr_init_sysctls()
1225 	    "Can we dynamically adjust the probe-rtt limits and times?");  in bbr_init_sysctls()
1313 	    "For 0 -> 24Mbps what is floor number of segments for TSO");  in bbr_init_sysctls()
1318 	    "For 0 -> 24Mbps what is top number of segments for TSO");  in bbr_init_sysctls()
1400 	    "How many pkt-epoch's (0 is off) do we need before pacing is on?");  in bbr_init_sysctls()
1432 	    "Do we use a pkt-epoch for substate if 0 rttProp?");  in bbr_init_sysctls()
1437 	    "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?");  in bbr_init_sysctls()
1477 	    "How many packet-epochs does the b/w delivery rate last?");  in bbr_init_sysctls()
1482 	    "Does our sub-state drain invoke app limited if its long?");  in bbr_init_sysctls()
1487 	    "Should we set/recover cwnd for sub-state drain?");  in bbr_init_sysctls()
1492 	    "Should we set/recover cwnd for main-state drain?");  in bbr_init_sysctls()
1497 	    "Should we allow google probe-bw/drain to exit early at flight target?");  in bbr_init_sysctls()
1566 	    "What is the high-speed min cwnd (rttProp under 1ms)");  in bbr_init_sysctls()
1659 	    "Maximum RTO in seconds -- should be at least as large as min_rto");  in bbr_init_sysctls()
1716 	    "Do we call the policer detection code from a rack-timeout?");  in bbr_init_sysctls()
1721 	    "What packet epoch do we do false-positive detection at (0=no)?");  in bbr_init_sysctls()
1796 	    "Total number of enobufs for non-hardware paced flows");  in bbr_init_sysctls()
1860 	l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;  in bbr_fill_in_logging_data()
1861 	l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_fill_in_logging_data()
1862 	l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in bbr_fill_in_logging_data()
1863 	l->bw_inuse = bbr_get_bw(bbr);  in bbr_fill_in_logging_data()
1864 	l->inflight = ctf_flight_size(bbr->rc_tp,  in bbr_fill_in_logging_data()
1865 			  (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_fill_in_logging_data()
1866 	l->applimited = bbr->r_ctl.r_app_limited_until;  in bbr_fill_in_logging_data()
1867 	l->delivered = bbr->r_ctl.rc_delivered;  in bbr_fill_in_logging_data()
1868 	l->timeStamp = cts;  in bbr_fill_in_logging_data()
1869 	l->lost = bbr->r_ctl.rc_lost;  in bbr_fill_in_logging_data()
1870 	l->bbr_state = bbr->rc_bbr_state;  in bbr_fill_in_logging_data()
1871 	l->bbr_substate = bbr_state_val(bbr);  in bbr_fill_in_logging_data()
1872 	l->epoch = bbr->r_ctl.rc_rtt_epoch;  in bbr_fill_in_logging_data()
1873 	l->lt_epoch = bbr->r_ctl.rc_lt_epoch;  in bbr_fill_in_logging_data()
1874 	l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;  in bbr_fill_in_logging_data()
1875 	l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;  in bbr_fill_in_logging_data()
1876 	l->inhpts = tcp_in_hpts(bbr->rc_tp);  in bbr_fill_in_logging_data()
1877 	l->use_lt_bw = bbr->rc_lt_use_bw;  in bbr_fill_in_logging_data()
1878 	l->pkts_out = bbr->r_ctl.rc_flight_at_input;  in bbr_fill_in_logging_data()
1879 	l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_fill_in_logging_data()
1885 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bw_reduce()
1888 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_bw_reduce()
1893 		log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;  in bbr_log_type_bw_reduce()
1895 		log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;  in bbr_log_type_bw_reduce()
1897 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bw_reduce()
1898 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bw_reduce()
1899 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bw_reduce()
1901 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_bw_reduce()
1908 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_rwnd_collapse()
1911 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_rwnd_collapse()
1915 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_rwnd_collapse()
1916 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_rwnd_collapse()
1917 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_rwnd_collapse()
1919 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_rwnd_collapse()
1927 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_just_return()
1932 		log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_type_just_return()
1933 		log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;  in bbr_log_type_just_return()
1935 		log.u_bbr.flex5 = bbr->rc_in_persist;  in bbr_log_type_just_return()
1936 		log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;  in bbr_log_type_just_return()
1938 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_type_just_return()
1941 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_just_return()
1942 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_just_return()
1943 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_just_return()
1945 		    tlen, &log, false, &bbr->rc_tv);  in bbr_log_type_just_return()
1952 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_enter_rec()
1955 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_enter_rec()
1957 		log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;  in bbr_log_type_enter_rec()
1958 		log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;  in bbr_log_type_enter_rec()
1959 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_enter_rec()
1960 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_enter_rec()
1961 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_enter_rec()
1963 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_enter_rec()
1979 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_msgsize_fail()
1980 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_msgsize_fail()
1982 		    0, &log, false, &bbr->rc_tv);  in bbr_log_msgsize_fail()
1989 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_flowend()
1994 		if (bbr->rc_inp->inp_socket) {  in bbr_log_flowend()
1995 			r = &bbr->rc_inp->inp_socket->so_rcv;  in bbr_log_flowend()
1996 			s = &bbr->rc_inp->inp_socket->so_snd;  in bbr_log_flowend()
2001 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_flowend()
2012 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_pkt_epoch()
2018 		log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_log_pkt_epoch()
2019 		log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;  in bbr_log_pkt_epoch()
2020 		log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;  in bbr_log_pkt_epoch()
2021 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;  in bbr_log_pkt_epoch()
2024 		log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;  in bbr_log_pkt_epoch()
2025 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_pkt_epoch()
2026 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_pkt_epoch()
2027 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_pkt_epoch()
2029 		    0, &log, false, &bbr->rc_tv);  in bbr_log_pkt_epoch()
2036 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_time_epoch()
2040 		log.u_bbr.flex1 = bbr->r_ctl.rc_lost;  in bbr_log_time_epoch()
2041 		log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;  in bbr_log_time_epoch()
2042 		log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;  in bbr_log_time_epoch()
2044 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_time_epoch()
2045 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_time_epoch()
2046 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_time_epoch()
2048 		    0, &log, false, &bbr->rc_tv);  in bbr_log_time_epoch()
2055 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_set_of_state_target()
2058 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_set_of_state_target()
2059 		log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;  in bbr_log_set_of_state_target()
2062 		log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;  in bbr_log_set_of_state_target()
2064 		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;  in bbr_log_set_of_state_target()
2065 		log.u_bbr.flex7 = bbr->rc_last_options;  in bbr_log_set_of_state_target()
2067 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_set_of_state_target()
2068 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_set_of_state_target()
2069 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_set_of_state_target()
2071 		    0, &log, false, &bbr->rc_tv);  in bbr_log_set_of_state_target()
2079 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_statechange()
2084 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;  in bbr_log_type_statechange()
2085 		log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;  in bbr_log_type_statechange()
2090 		log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;  in bbr_log_type_statechange()
2091 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;  in bbr_log_type_statechange()
2092 		log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);  in bbr_log_type_statechange()
2093 		log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;  in bbr_log_type_statechange()
2094 		log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;  in bbr_log_type_statechange()
2095 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_statechange()
2096 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_statechange()
2097 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_statechange()
2099 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_statechange()
2107 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_rtt_shrinks()
2112 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;  in bbr_log_rtt_shrinks()
2113 		log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;  in bbr_log_rtt_shrinks()
2116 		log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;  in bbr_log_rtt_shrinks()
2119 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_rtt_shrinks()
2120 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_rtt_shrinks()
2121 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_rtt_shrinks()
2123 		    0, &log, false, &bbr->rc_tv);  in bbr_log_rtt_shrinks()
2130 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_exit_rec()
2133 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_exit_rec()
2134 		log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;  in bbr_log_type_exit_rec()
2135 		log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;  in bbr_log_type_exit_rec()
2136 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_type_exit_rec()
2137 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_exit_rec()
2138 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_exit_rec()
2139 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_exit_rec()
2141 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_exit_rec()
2146 bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg,  in bbr_log_type_cwndupd()  argument
2149 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_cwndupd()
2152 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_type_cwndupd()
2156 		log.u_bbr.flex4 = chg;  in bbr_log_type_cwndupd()
2160 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_cwndupd()
2161 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_cwndupd()
2162 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_cwndupd()
2164 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_cwndupd()
2175 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_rtt_sample()
2178 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_rtt_sample()
2180 		log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;  in bbr_log_rtt_sample()
2181 		log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_log_rtt_sample()
2182 		log.u_bbr.flex4 = bbr->rc_tp->ts_offset;  in bbr_log_rtt_sample()
2183 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_rtt_sample()
2184 		log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_log_rtt_sample()
2187 		log.u_bbr.flex8 = bbr->rc_ack_was_delayed;  in bbr_log_rtt_sample()
2188 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_rtt_sample()
2189 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_rtt_sample()
2190 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_rtt_sample()
2192 		    0, &log, false, &bbr->rc_tv);  in bbr_log_rtt_sample()
2199 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_pesist()
2206 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_pesist()
2207 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_pesist()
2208 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_pesist()
2210 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_pesist()
2216 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_ack_clear()
2220 		log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;  in bbr_log_ack_clear()
2221 		log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;  in bbr_log_ack_clear()
2222 		log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;  in bbr_log_ack_clear()
2223 		log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;  in bbr_log_ack_clear()
2224 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_ack_clear()
2225 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_ack_clear()
2226 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_ack_clear()
2227 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_ack_clear()
2229 		    0, &log, false, &bbr->rc_tv);  in bbr_log_ack_clear()
2237 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_ack_event()
2243 		log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;  in bbr_log_ack_event()
2247 			log.u_bbr.flex3 = m->m_flags;  in bbr_log_ack_event()
2248 			if (m->m_flags & M_TSTMP) {  in bbr_log_ack_event()
2256 			if (m->m_flags & M_TSTMP_LRO) {  in bbr_log_ack_event()
2271 		log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;  in bbr_log_ack_event()
2272 		log.u_bbr.flex7 = bbr->r_wanted_output;  in bbr_log_ack_event()
2273 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_ack_event()
2274 		TCP_LOG_EVENTP(bbr->rc_tp, th,  in bbr_log_ack_event()
2275 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_ack_event()
2276 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_ack_event()
2278 		    tlen, &log, true, &bbr->rc_tv);  in bbr_log_ack_event()
2285 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_doseg_done()
2291 		log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;  in bbr_log_doseg_done()
2292 		log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_doseg_done()
2293 		log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;  in bbr_log_doseg_done()
2294 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;  in bbr_log_doseg_done()
2295 		log.u_bbr.flex7 = bbr->r_wanted_output;  in bbr_log_doseg_done()
2296 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_doseg_done()
2297 		log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;  in bbr_log_doseg_done()
2298 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_doseg_done()
2299 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_doseg_done()
2300 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_doseg_done()
2302 		    0, &log, true, &bbr->rc_tv);  in bbr_log_doseg_done()
2310 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_enobuf_jmp()
2318 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_enobuf_jmp()
2319 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_enobuf_jmp()
2320 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_enobuf_jmp()
2322 		    len, &log, true, &bbr->rc_tv);  in bbr_log_enobuf_jmp()
2329 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_processing()
2335 		log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;  in bbr_log_to_processing()
2336 		log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_processing()
2338 		log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;  in bbr_log_to_processing()
2340 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_processing()
2341 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_processing()
2342 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_processing()
2344 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_processing()
2351 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_event()
2356 		log.u_bbr.flex1 = bbr->bbr_timer_src;  in bbr_log_to_event()
2358 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_event()
2359 		ar = (uintptr_t)(bbr->r_ctl.rc_resend);  in bbr_log_to_event()
2363 		ar = (uintptr_t)bbr->r_ctl.rc_resend;  in bbr_log_to_event()
2366 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_to_event()
2368 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_event()
2369 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_event()
2370 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_event()
2372 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_event()
2379 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_startup_event()
2387 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_startup_event()
2388 		log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;  in bbr_log_startup_event()
2390 		log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_log_startup_event()
2391 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_startup_event()
2392 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_startup_event()
2393 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_startup_event()
2395 		    0, &log, false, &bbr->rc_tv);  in bbr_log_startup_event()
2402 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_hpts_diag()
2406 		log.u_bbr.flex1 = diag->p_nxt_slot;  in bbr_log_hpts_diag()
2407 		log.u_bbr.flex2 = diag->p_cur_slot;  in bbr_log_hpts_diag()
2408 		log.u_bbr.flex3 = diag->slot_req;  in bbr_log_hpts_diag()
2409 		log.u_bbr.flex4 = diag->inp_hptsslot;  in bbr_log_hpts_diag()
2410 		log.u_bbr.flex5 = diag->slot_remaining;  in bbr_log_hpts_diag()
2411 		log.u_bbr.flex6 = diag->need_new_to;  in bbr_log_hpts_diag()
2412 		log.u_bbr.flex7 = diag->p_hpts_active;  in bbr_log_hpts_diag()
2413 		log.u_bbr.flex8 = diag->p_on_min_sleep;  in bbr_log_hpts_diag()
2415 		log.u_bbr.epoch = diag->have_slept;  in bbr_log_hpts_diag()
2416 		log.u_bbr.lt_epoch = diag->yet_to_sleep;  in bbr_log_hpts_diag()
2417 		log.u_bbr.pkts_out = diag->co_ret;  in bbr_log_hpts_diag()
2418 		log.u_bbr.applimited = diag->hpts_sleep_time;  in bbr_log_hpts_diag()
2419 		log.u_bbr.delivered = diag->p_prev_slot;  in bbr_log_hpts_diag()
2420 		log.u_bbr.inflight = diag->p_runningslot;  in bbr_log_hpts_diag()
2421 		log.u_bbr.bw_inuse = diag->wheel_slot;  in bbr_log_hpts_diag()
2422 		log.u_bbr.rttProp = diag->wheel_cts;  in bbr_log_hpts_diag()
2423 		log.u_bbr.delRate = diag->maxslots;  in bbr_log_hpts_diag()
2424 		log.u_bbr.cur_del_rate = diag->p_curtick;  in bbr_log_hpts_diag()
2426 		log.u_bbr.cur_del_rate |= diag->p_lasttick;  in bbr_log_hpts_diag()
2427 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_hpts_diag()
2428 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_hpts_diag()
2429 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_hpts_diag()
2431 		    0, &log, false, &bbr->rc_tv);  in bbr_log_hpts_diag()
2439 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_timer_var()
2443 		log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;  in bbr_log_timer_var()
2448 		log.u_bbr.flex6 = bbr->rc_tp->t_srtt;  in bbr_log_timer_var()
2450 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_timer_var()
2451 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_timer_var()
2452 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_timer_var()
2454 		    0, &log, false, &bbr->rc_tv);  in bbr_log_timer_var()
2462 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_pacing_delay_calc()
2477 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_pacing_delay_calc()
2478 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_pacing_delay_calc()
2479 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_pacing_delay_calc()
2481 		    len, &log, false, &bbr->rc_tv);  in bbr_log_pacing_delay_calc()
2488 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_start()
2493 		log.u_bbr.flex1 = bbr->bbr_timer_src;  in bbr_log_to_start()
2495 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_start()
2497 		log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot;  in bbr_log_to_start()
2498 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_to_start()
2499 		log.u_bbr.pkts_out = bbr->rc_tp->t_flags2;  in bbr_log_to_start()
2501 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_start()
2502 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_start()
2503 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_start()
2505 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_start()
2512 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_thresh_choice()
2518 		log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;  in bbr_log_thresh_choice()
2519 		log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];  in bbr_log_thresh_choice()
2520 		log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_thresh_choice()
2522 		log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;  in bbr_log_thresh_choice()
2524 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_thresh_choice()
2525 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_thresh_choice()
2526 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_thresh_choice()
2528 		    0, &log, false, &bbr->rc_tv);  in bbr_log_thresh_choice()
2535 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_to_cancel()
2540 		log.u_bbr.flex2 = bbr->bbr_timer_src;  in bbr_log_to_cancel()
2541 		log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;  in bbr_log_to_cancel()
2542 		log.u_bbr.flex4 = bbr->rc_in_persist;  in bbr_log_to_cancel()
2543 		log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;  in bbr_log_to_cancel()
2544 		log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_log_to_cancel()
2546 		log.u_bbr.pkts_out = bbr->rc_pacer_started;  in bbr_log_to_cancel()
2547 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_to_cancel()
2548 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_to_cancel()
2549 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_to_cancel()
2551 		    0, &log, false, &bbr->rc_tv);  in bbr_log_to_cancel()
2558 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_tstmp_validation()
2561 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_tstmp_validation()
2562 		log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;  in bbr_log_tstmp_validation()
2567 		log.u_bbr.flex7 = bbr->rc_ts_clock_set;  in bbr_log_tstmp_validation()
2568 		log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;  in bbr_log_tstmp_validation()
2569 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_tstmp_validation()
2570 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_tstmp_validation()
2571 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_tstmp_validation()
2573 		    0, &log, false, &bbr->rc_tv);  in bbr_log_tstmp_validation()
2580 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_tsosize()
2587 		log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;  in bbr_log_type_tsosize()
2590 		log.u_bbr.flex7 = bbr->rc_no_pacing;  in bbr_log_type_tsosize()
2592 		log.u_bbr.flex7 |= bbr->rc_past_init_win;  in bbr_log_type_tsosize()
2594 			log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;  in bbr_log_type_tsosize()
2596 			log.u_bbr.flex8 = bbr->rc_use_google;  in bbr_log_type_tsosize()
2597 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_tsosize()
2598 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_tsosize()
2599 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_tsosize()
2601 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_tsosize()
2609 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_rsmclear()
2614 		log.u_bbr.flex2 = rsm->r_start;  in bbr_log_type_rsmclear()
2615 		log.u_bbr.flex3 = rsm->r_end;  in bbr_log_type_rsmclear()
2616 		log.u_bbr.flex4 = rsm->r_delivered;  in bbr_log_type_rsmclear()
2617 		log.u_bbr.flex5 = rsm->r_rtr_cnt;  in bbr_log_type_rsmclear()
2618 		log.u_bbr.flex6 = rsm->r_dupack;  in bbr_log_type_rsmclear()
2619 		log.u_bbr.flex7 = rsm->r_tim_lastsent[0];  in bbr_log_type_rsmclear()
2620 		log.u_bbr.flex8 = rsm->r_flags;  in bbr_log_type_rsmclear()
2623 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_rsmclear()
2624 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_rsmclear()
2625 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_rsmclear()
2627 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_rsmclear()
2638 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bbrupd()
2652 		if (bbr->rc_ack_was_delayed)  in bbr_log_type_bbrupd()
2653 			log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_log_type_bbrupd()
2656 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bbrupd()
2657 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bbrupd()
2658 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bbrupd()
2660 		    flex2, &log, false, &bbr->rc_tv);  in bbr_log_type_bbrupd()
2669 	if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_ltbw()
2677 		log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;  in bbr_log_type_ltbw()
2678 		log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;  in bbr_log_type_ltbw()
2679 		log.u_bbr.flex7 = bbr->rc_lt_is_sampling;  in bbr_log_type_ltbw()
2681 		log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;  in bbr_log_type_ltbw()
2682 		if (bbr->rc_lt_use_bw == 0)  in bbr_log_type_ltbw()
2683 			log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;  in bbr_log_type_ltbw()
2685 			log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;  in bbr_log_type_ltbw()
2686 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_ltbw()
2687 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_ltbw()
2688 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_ltbw()
2690 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_ltbw()
2697 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_progress_event()
2700 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_progress_event()
2703 		log.u_bbr.flex3 = tp->t_maxunacktime;  in bbr_log_progress_event()
2704 		log.u_bbr.flex4 = tp->t_acktime;  in bbr_log_progress_event()
2706 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_progress_event()
2707 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_progress_event()
2708 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_progress_event()
2710 		    0, &log, false, &bbr->rc_tv);  in bbr_log_progress_event()
2719 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_type_log_hdwr_pacing()
2731 		log.u_bbr.flex8 = bbr->skip_gain;  in bbr_type_log_hdwr_pacing()
2733 		log.u_bbr.flex8 |= bbr->gain_is_limited;  in bbr_type_log_hdwr_pacing()
2735 		log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;  in bbr_type_log_hdwr_pacing()
2736 		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;  in bbr_type_log_hdwr_pacing()
2737 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_type_log_hdwr_pacing()
2738 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_type_log_hdwr_pacing()
2739 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_type_log_hdwr_pacing()
2741 		    0, &log, false, &bbr->rc_tv);  in bbr_type_log_hdwr_pacing()
2748 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bbrsnd()
2756 		log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;  in bbr_log_type_bbrsnd()
2757 		log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_log_type_bbrsnd()
2758 		log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);  in bbr_log_type_bbrsnd()
2759 		log.u_bbr.flex8 = bbr->rc_in_persist;  in bbr_log_type_bbrsnd()
2760 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bbrsnd()
2761 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bbrsnd()
2762 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bbrsnd()
2764 		    len, &log, false, &bbr->rc_tv);  in bbr_log_type_bbrsnd()
2771 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_type_bbrrttprop()
2775 		log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;  in bbr_log_type_bbrrttprop()
2777 		log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;  in bbr_log_type_bbrrttprop()
2783 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_type_bbrrttprop()
2784 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_type_bbrrttprop()
2785 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_type_bbrrttprop()
2787 		    0, &log, false, &bbr->rc_tv);  in bbr_log_type_bbrrttprop()
2794 	if (tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_exit_gain()
2798 		log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;  in bbr_log_exit_gain()
2799 		log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_log_exit_gain()
2800 		log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;  in bbr_log_exit_gain()
2801 		log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;  in bbr_log_exit_gain()
2802 		log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;  in bbr_log_exit_gain()
2803 		log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;  in bbr_log_exit_gain()
2806 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_exit_gain()
2807 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_exit_gain()
2808 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_exit_gain()
2810 		    0, &log, false, &bbr->rc_tv);  in bbr_log_exit_gain()
2817 	if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {  in bbr_log_settings_change()
2820 		bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);  in bbr_log_settings_change()
2821 		/* R-HU */  in bbr_log_settings_change()
2829 		TCP_LOG_EVENTP(bbr->rc_tp, NULL,  in bbr_log_settings_change()
2830 		    &bbr->rc_inp->inp_socket->so_rcv,  in bbr_log_settings_change()
2831 		    &bbr->rc_inp->inp_socket->so_snd,  in bbr_log_settings_change()
2833 		    0, &log, false, &bbr->rc_tv);  in bbr_log_settings_change()
2845 	bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_get_full_bw()
2856 	if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)  in bbr_set_pktepoch()
2857 		lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;  in bbr_set_pktepoch()
2860 	del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;  in bbr_set_pktepoch()
2871 	bbr->r_ctl.rc_pkt_epoch_loss_rate =  (uint32_t)calclr;  in bbr_set_pktepoch()
2872 	if (IN_RECOVERY(bbr->rc_tp->t_flags))  in bbr_set_pktepoch()
2873 		bbr->r_ctl.recovery_lr += (uint32_t)calclr;  in bbr_set_pktepoch()
2874 	bbr->r_ctl.rc_pkt_epoch++;  in bbr_set_pktepoch()
2875 	if (bbr->rc_no_pacing &&  in bbr_set_pktepoch()
2876 	    (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {  in bbr_set_pktepoch()
2877 		bbr->rc_no_pacing = 0;  in bbr_set_pktepoch()
2880 	bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);  in bbr_set_pktepoch()
2881 	bbr->r_ctl.rc_pkt_epoch_time = cts;  in bbr_set_pktepoch()
2884 	bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;  in bbr_set_pktepoch()
2885 	bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;  in bbr_set_pktepoch()
2894 	bbr->r_ctl.rc_rtt_epoch++;  in bbr_set_epoch()
2895 	epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;  in bbr_set_epoch()
2897 	bbr->r_ctl.rc_rcv_epoch_start = cts;  in bbr_set_epoch()
2903 	if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {  in bbr_isit_a_pkt_epoch()
2904 		bbr->rc_is_pkt_epoch_now = 1;  in bbr_isit_a_pkt_epoch()
2928 	if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in __bbr_get_bw()
2930 		rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in __bbr_get_bw()
2933 			min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *  in __bbr_get_bw()
2936 			if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {  in __bbr_get_bw()
2937 				min_bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2940 		} else if (bbr->rc_tp->t_srtt != 0) {  in __bbr_get_bw()
2945 			min_bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2950 	if ((bbr->rc_past_init_win == 0) &&  in __bbr_get_bw()
2951 	    (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))  in __bbr_get_bw()
2952 		bbr->rc_past_init_win = 1;  in __bbr_get_bw()
2953 	if ((bbr->rc_use_google)  && (bbr->r_ctl.r_measurement_count >= 1))  in __bbr_get_bw()
2956 	    ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||  in __bbr_get_bw()
2957 	     (bbr->rc_past_init_win == 0))) {  in __bbr_get_bw()
2961 		rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in __bbr_get_bw()
2968 			bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *  in __bbr_get_bw()
2971 			if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {  in __bbr_get_bw()
2972 				bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2976 			bw = bbr->r_ctl.rc_initial_hptsi_bw;  in __bbr_get_bw()
2986 	if (bbr->rc_lt_use_bw)  in __bbr_get_bw()
2987 		bw = bbr->r_ctl.rc_lt_bw;  in __bbr_get_bw()
2988 	else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))  in __bbr_get_bw()
2989 		bw = bbr->r_ctl.red_bw;  in __bbr_get_bw()
2991 		bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in __bbr_get_bw()
3016 	bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_reset_lt_bw_interval()
3017 	bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;  in bbr_reset_lt_bw_interval()
3018 	bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;  in bbr_reset_lt_bw_interval()
3019 	bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_reset_lt_bw_interval()
3025 	bbr->rc_lt_is_sampling = 0;  in bbr_reset_lt_bw_sampling()
3026 	bbr->rc_lt_use_bw = 0;  in bbr_reset_lt_bw_sampling()
3027 	bbr->r_ctl.rc_lt_bw = 0;  in bbr_reset_lt_bw_sampling()
3037 	if (bbr->r_ctl.rc_lt_bw) {  in bbr_lt_bw_samp_done()
3039 		if (bbr->r_ctl.rc_lt_bw > bw)  in bbr_lt_bw_samp_done()
3040 			diff = bbr->r_ctl.rc_lt_bw - bw;  in bbr_lt_bw_samp_done()
3042 			diff = bw - bbr->r_ctl.rc_lt_bw;  in bbr_lt_bw_samp_done()
3044 		    (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {  in bbr_lt_bw_samp_done()
3048 			saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;  in bbr_lt_bw_samp_done()
3049 			bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2;	/* average of two */  in bbr_lt_bw_samp_done()
3050 			bbr->rc_lt_use_bw = 1;  in bbr_lt_bw_samp_done()
3051 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_lt_bw_samp_done()
3056 			bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;  in bbr_lt_bw_samp_done()
3065 	bbr->r_ctl.rc_lt_bw = bw;  in bbr_lt_bw_samp_done()
3077 		deduct = bbr->r_ctl.rc_level_state_extra / ran;  in bbr_randomize_extra_state_time()
3078 		bbr->r_ctl.rc_level_state_extra -= deduct;  in bbr_randomize_extra_state_time()
3092 	bbr->r_ctl.rc_exta_time_gd = 0;  in bbr_pick_probebw_substate()
3093 	bbr->rc_hit_state_1 = 0;  in bbr_pick_probebw_substate()
3094 	bbr->r_ctl.rc_level_state_extra = 0;  in bbr_pick_probebw_substate()
3095 	ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1));  in bbr_pick_probebw_substate()
3100 	 * we fully enter the state. Note that the (8 - 1 - ran) assures that  in bbr_pick_probebw_substate()
3101 	 * we return 1 - 7, so we dont return 0 and end up starting in  in bbr_pick_probebw_substate()
3104 	ret_val = BBR_SUBSTATE_COUNT - 1 - ran;  in bbr_pick_probebw_substate()
3106 	if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))  in bbr_pick_probebw_substate()
3109 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_pick_probebw_substate()
3119 	if (bbr->r_use_policer == 0)  in bbr_lt_bw_sampling()
3121 	if (bbr->rc_lt_use_bw) {  in bbr_lt_bw_sampling()
3123 		diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;  in bbr_lt_bw_sampling()
3128 			if (bbr->rc_filled_pipe) {  in bbr_lt_bw_sampling()
3130 				bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_lt_bw_sampling()
3132 				bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_lt_bw_sampling()
3140 				bbr->rc_bbr_state = BBR_STATE_STARTUP;  in bbr_lt_bw_sampling()
3142 				bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_lt_bw_sampling()
3143 				bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_lt_bw_sampling()
3144 				bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;  in bbr_lt_bw_sampling()
3145 				bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;  in bbr_lt_bw_sampling()
3149 			/* reason 0 is to stop using lt-bw */  in bbr_lt_bw_sampling()
3154 			/* Not doing false-positive detection */  in bbr_lt_bw_sampling()
3160 			bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;  in bbr_lt_bw_sampling()
3161 			bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_lt_bw_sampling()
3164 			lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;  in bbr_lt_bw_sampling()
3165 			delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;  in bbr_lt_bw_sampling()
3180 	 * exhaust its tokens and estimate the steady-state rate allowed by  in bbr_lt_bw_sampling()
3182 	 * over-estimate the bw.  in bbr_lt_bw_sampling()
3184 	if (bbr->rc_lt_is_sampling == 0) {  in bbr_lt_bw_sampling()
3189 		bbr->rc_lt_is_sampling = 1;  in bbr_lt_bw_sampling()
3194 	if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))  in bbr_lt_bw_sampling()
3195 		d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;  in bbr_lt_bw_sampling()
3200 	if (bbr->r_ctl.r_app_limited_until) {  in bbr_lt_bw_sampling()
3201 		/* Can not measure in app-limited state */  in bbr_lt_bw_sampling()
3207 	diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;  in bbr_lt_bw_sampling()
3214 		/* 6 is not_enough time or no-loss */  in bbr_lt_bw_sampling()
3234 	 * tokens are exhausted under-estimates the policed rate.  in bbr_lt_bw_sampling()
3237 		/* 6 is not_enough time or no-loss */  in bbr_lt_bw_sampling()
3242 	lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;  in bbr_lt_bw_sampling()
3243 	delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;  in bbr_lt_bw_sampling()
3251 		/* 6 is not_enough time or no-loss */  in bbr_lt_bw_sampling()
3280 		bbr->r_ctl.rc_num_maps_alloced++;  in bbr_alloc()
3283 	if (bbr->r_ctl.rc_free_cnt) {  in bbr_alloc()
3285 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);  in bbr_alloc()
3286 		TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_alloc()
3287 		bbr->r_ctl.rc_free_cnt--;  in bbr_alloc()
3298 	    (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {  in bbr_alloc_full_limit()
3300 		if (!bbr->alloc_limit_reported) {  in bbr_alloc_full_limit()
3301 			bbr->alloc_limit_reported = 1;  in bbr_alloc_full_limit()
3318 		    bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {  in bbr_alloc_limit()
3320 			if (!bbr->alloc_limit_reported) {  in bbr_alloc_limit()
3321 				bbr->alloc_limit_reported = 1;  in bbr_alloc_limit()
3331 		rsm->r_limit_type = limit_type;  in bbr_alloc_limit()
3332 		bbr->r_ctl.rc_num_split_allocs++;  in bbr_alloc_limit()
3340 	if (rsm->r_limit_type) {  in bbr_free()
3342 		bbr->r_ctl.rc_num_split_allocs--;  in bbr_free()
3344 	if (rsm->r_is_smallmap)  in bbr_free()
3345 		bbr->r_ctl.rc_num_small_maps_alloced--;  in bbr_free()
3346 	if (bbr->r_ctl.rc_tlp_send == rsm)  in bbr_free()
3347 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_free()
3348 	if (bbr->r_ctl.rc_resend == rsm) {  in bbr_free()
3349 		bbr->r_ctl.rc_resend = NULL;  in bbr_free()
3351 	if (bbr->r_ctl.rc_next == rsm)  in bbr_free()
3352 		bbr->r_ctl.rc_next = NULL;  in bbr_free()
3353 	if (bbr->r_ctl.rc_sacklast == rsm)  in bbr_free()
3354 		bbr->r_ctl.rc_sacklast = NULL;  in bbr_free()
3355 	if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {  in bbr_free()
3357 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_free()
3358 		rsm->r_limit_type = 0;  in bbr_free()
3359 		bbr->r_ctl.rc_free_cnt++;  in bbr_free()
3362 	bbr->r_ctl.rc_num_maps_alloced--;  in bbr_free()
3395 	if (bbr->rc_init_win) {  in bbr_initial_cwnd()
3396 		i_cwnd = bbr->rc_init_win * tp->t_maxseg;  in bbr_initial_cwnd()
3398 		i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),  in bbr_initial_cwnd()
3399 		    max(2 * tp->t_maxseg, 14600));  in bbr_initial_cwnd()
3401 		i_cwnd = min(4 * tp->t_maxseg,  in bbr_initial_cwnd()
3402 		    max(2 * tp->t_maxseg, 4380));  in bbr_initial_cwnd()
3405 		if (tp->t_maxseg > 2190)  in bbr_initial_cwnd()
3406 			i_cwnd = 2 * tp->t_maxseg;  in bbr_initial_cwnd()
3407 		else if (tp->t_maxseg > 1095)  in bbr_initial_cwnd()
3408 			i_cwnd = 3 * tp->t_maxseg;  in bbr_initial_cwnd()
3410 			i_cwnd = 4 * tp->t_maxseg;  in bbr_initial_cwnd()
3425 	if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||  in bbr_get_raw_target_cwnd()
3428 		return (bbr_initial_cwnd(bbr, bbr->rc_tp));  in bbr_get_raw_target_cwnd()
3438 	cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT));  in bbr_get_raw_target_cwnd()
3448 	mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);  in bbr_get_target_cwnd()
3453 	 * fq layer to trap packets in) quanta's per the I-D  in bbr_get_target_cwnd()
3456 	cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);  in bbr_get_target_cwnd()
3457 	if (bbr->rc_use_google) {  in bbr_get_target_cwnd()
3458 		if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&  in bbr_get_target_cwnd()
3463 			 * is documented no-where except in the code.  in bbr_get_target_cwnd()
3493 	if (bbr->r_ctl.rc_inc_tcp_oh) {  in bbr_get_header_oh()
3495 		seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));  in bbr_get_header_oh()
3497 	if (bbr->r_ctl.rc_inc_ip_oh) {  in bbr_get_header_oh()
3500 		if (bbr->r_is_v6) {  in bbr_get_header_oh()
3511 	if (bbr->r_ctl.rc_inc_enet_oh) {  in bbr_get_header_oh()
3549 	maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in bbr_get_pacing_delay()
3550 	num_segs = (len + maxseg - 1) / maxseg;  in bbr_get_pacing_delay()
3551 	if (bbr->rc_use_google == 0) {  in bbr_get_pacing_delay()
3557 	if (bbr->rc_use_google) {  in bbr_get_pacing_delay()
3564 		cbw = bw *  (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);  in bbr_get_pacing_delay()
3579 	    (bbr->rc_use_google == 0) &&  in bbr_get_pacing_delay()
3604 	if ((tp->t_flags & TF_GPUTINPROG) &&  in bbr_ack_received()
3605 	    SEQ_GEQ(th->th_ack, tp->gput_ack)) {  in bbr_ack_received()
3615 		gput = (int64_t) (th->th_ack - tp->gput_seq) * 8;  in bbr_ack_received()
3616 		time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));  in bbr_ack_received()
3618 		stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,  in bbr_ack_received()
3620 		if (tp->t_stats_gput_prev > 0)  in bbr_ack_received()
3621 			stats_voi_update_abs_s32(tp->t_stats,  in bbr_ack_received()
3623 						 ((gput - tp->t_stats_gput_prev) * 100) /  in bbr_ack_received()
3624 						 tp->t_stats_gput_prev);  in bbr_ack_received()
3625 		tp->t_flags &= ~TF_GPUTINPROG;  in bbr_ack_received()
3626 		tp->t_stats_gput_prev = cgput;  in bbr_ack_received()
3629 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&  in bbr_ack_received()
3630 	    ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {  in bbr_ack_received()
3631 		/* We don't change anything in probe-rtt */  in bbr_ack_received()
3634 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_ack_received()
3638 		bytes_this_ack -= prev_acked;  in bbr_ack_received()
3643 		if ((bytes_this_ack < maxseg) && bbr->rc_use_google)  in bbr_ack_received()
3649 	cwnd = tp->snd_cwnd;  in bbr_ack_received()
3650 	bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_ack_received()
3654 						  (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);  in bbr_ack_received()
3656 		target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);  in bbr_ack_received()
3657 	if (IN_RECOVERY(tp->t_flags) &&  in bbr_ack_received()
3658 	    (bbr->bbr_prev_in_rec == 0)) {  in bbr_ack_received()
3663 		bbr->pkt_conservation = 1;  in bbr_ack_received()
3664 		bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;  in bbr_ack_received()
3666 				       (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_ack_received()
3669 	if (IN_RECOVERY(tp->t_flags)) {  in bbr_ack_received()
3672 		bbr->bbr_prev_in_rec = 1;  in bbr_ack_received()
3674 			cwnd -= losses;  in bbr_ack_received()
3680 					 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_ack_received()
3683 		if (bbr->pkt_conservation) {  in bbr_ack_received()
3686 			if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))  in bbr_ack_received()
3687 				time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;  in bbr_ack_received()
3693 				bbr->pkt_conservation = 0;  in bbr_ack_received()
3699 				tp->snd_cwnd = cwnd;  in bbr_ack_received()
3701 						     prev_acked, 1, target_cwnd, th->th_ack, line);  in bbr_ack_received()
3706 		bbr->bbr_prev_in_rec = 0;  in bbr_ack_received()
3707 	if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {  in bbr_ack_received()
3708 		bbr->r_ctl.restrict_growth--;  in bbr_ack_received()
3712 	if (bbr->rc_filled_pipe) {  in bbr_ack_received()
3724 		else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)  in bbr_ack_received()
3732 		    (bbr->rc_past_init_win == 0)) {  in bbr_ack_received()
3743 	tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));  in bbr_ack_received()
3744 …bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, li…  in bbr_ack_received()
3752 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in tcp_bbr_partialack()
3755 		(bbr->r_ctl.rc_sacked  + bbr->r_ctl.rc_lost_bytes)) <=  in tcp_bbr_partialack()
3756 	    tp->snd_cwnd) {  in tcp_bbr_partialack()
3757 		bbr->r_wanted_output = 1;  in tcp_bbr_partialack()
3768 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_post_recovery()
3772 	EXIT_RECOVERY(tp->t_flags);  in bbr_post_recovery()
3773 	/* Lock in our b/w reduction for the specified number of pkt-epochs */  in bbr_post_recovery()
3774 	bbr->r_recovery_bw = 0;  in bbr_post_recovery()
3775 	tp->snd_recover = tp->snd_una;  in bbr_post_recovery()
3776 	tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_post_recovery()
3777 	bbr->pkt_conservation = 0;  in bbr_post_recovery()
3778 	if (bbr->rc_use_google == 0) {  in bbr_post_recovery()
3780 		 * For non-google mode lets  in bbr_post_recovery()
3786 		bbr->bbr_prev_in_rec = 0;  in bbr_post_recovery()
3789 	if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {  in bbr_post_recovery()
3790 		tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);  in bbr_post_recovery()
3793 		/* For probe-rtt case lets fix up its saved_cwnd */  in bbr_post_recovery()
3794 		if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {  in bbr_post_recovery()
3795 			bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;  in bbr_post_recovery()
3800 		     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_post_recovery()
3801 	if ((bbr->rc_use_google == 0) &&  in bbr_post_recovery()
3815 				     bbr->r_ctl.recovery_lr, 21,  in bbr_post_recovery()
3817 				     bbr->r_ctl.rc_red_cwnd_pe,  in bbr_post_recovery()
3821 		if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&  in bbr_post_recovery()
3824 		     (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&  in bbr_post_recovery()
3825 		     bbr->rc_hit_state_1 &&  in bbr_post_recovery()
3827 		    ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&  in bbr_post_recovery()
3833 			cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_post_recovery()
3834 			cwnd_p = &bbr->r_ctl.rc_saved_cwnd;  in bbr_post_recovery()
3836  			cwnd = tp->snd_cwnd;  in bbr_post_recovery()
3837 			cwnd_p = &tp->snd_cwnd;  in bbr_post_recovery()
3839 		maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_post_recovery()
3841 		if (bbr->r_ctl.rc_lost == 0)  in bbr_post_recovery()
3843 		else if (bbr->r_ctl.rc_delivered == 0)  in bbr_post_recovery()
3846 			lr2use = bbr->r_ctl.rc_lost * 1000;  in bbr_post_recovery()
3847 			lr2use /= bbr->r_ctl.rc_delivered;  in bbr_post_recovery()
3849 		lr2use += bbr->r_ctl.recovery_lr;  in bbr_post_recovery()
3856 			    bbr->r_ctl.restrict_growth += acks_inflight;  in bbr_post_recovery()
3862 				newcwnd = roundup((cwnd - val), maxseg);  in bbr_post_recovery()
3878 				newcwnd = (get_min_cwnd(bbr) - acks_inflight);  in bbr_post_recovery()
3890 		if (tp->snd_cwnd > newcwnd)  in bbr_post_recovery()
3891 			tp->snd_cwnd = newcwnd;  in bbr_post_recovery()
3895 		bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;  in bbr_post_recovery()
3898 	bbr->r_ctl.recovery_lr = 0;  in bbr_post_recovery()
3899 	if (flight <= tp->snd_cwnd) {  in bbr_post_recovery()
3900 		bbr->r_wanted_output = 1;  in bbr_post_recovery()
3902 	tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_post_recovery()
3908 	bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in bbr_setup_red_bw()
3910 	if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)  in bbr_setup_red_bw()
3911 		bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;  in bbr_setup_red_bw()
3912 	if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))  in bbr_setup_red_bw()
3913 		bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;  in bbr_setup_red_bw()
3924 	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);  in bbr_cong_signal()
3926 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_cong_signal()
3929 		if (!IN_RECOVERY(tp->t_flags)) {  in bbr_cong_signal()
3930 			tp->snd_recover = tp->snd_max;  in bbr_cong_signal()
3932 			bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_cong_signal()
3933 			if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {  in bbr_cong_signal()
3939 				bbr->r_ctl.rc_lt_epoch++;  in bbr_cong_signal()
3941 			if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in bbr_cong_signal()
3948 				bbr->r_ctl.rc_bbr_last_startup_epoch++;  in bbr_cong_signal()
3950 			bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;  in bbr_cong_signal()
3951 			ENTER_RECOVERY(tp->t_flags);  in bbr_cong_signal()
3952 			bbr->rc_tlp_rtx_out = 0;  in bbr_cong_signal()
3953 			bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;  in bbr_cong_signal()
3954 			tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_cong_signal()
3955 			if (tcp_in_hpts(bbr->rc_tp) &&  in bbr_cong_signal()
3956 			    ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {  in bbr_cong_signal()
3963 				bbr->rc_timer_first = 1;  in bbr_cong_signal()
3964 				bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_cong_signal()
3972 			if ((bbr->rc_use_google == 0) &&  in bbr_cong_signal()
3973 			    (bbr->r_ctl.bbr_rttprobe_gain_val ||  in bbr_cong_signal()
3974 			     (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {  in bbr_cong_signal()
3975 				tp->snd_cwnd = ctf_flight_size(tp,  in bbr_cong_signal()
3976 					           (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_cong_signal()
3977 					(tp->t_maxseg - bbr->rc_last_options);  in bbr_cong_signal()
3978 				if (tp->snd_cwnd < get_min_cwnd(bbr)) {  in bbr_cong_signal()
3980 					tp->snd_cwnd = get_min_cwnd(bbr);  in bbr_cong_signal()
3984 			bbr_log_type_enter_rec(bbr, rsm->r_start);  in bbr_cong_signal()
3990 		bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_cong_signal()
3991 		if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {  in bbr_cong_signal()
3992 			tp->snd_cwnd = tp->snd_cwnd_prev;  in bbr_cong_signal()
3993 			tp->snd_ssthresh = tp->snd_ssthresh_prev;  in bbr_cong_signal()
3994 			tp->snd_recover = tp->snd_recover_prev;  in bbr_cong_signal()
3995 			tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);  in bbr_cong_signal()
3998 		tp->t_badrxtwin = 0;  in bbr_cong_signal()
4006  *	- There is no delayed ack timer in progress.
4007  *	- Our last ack wasn't a 0-sized window. We never want to delay
4008  *	  the ack that opens up a 0-sized window.
4009  *	- LRO wasn't used for this segment. We make sure by checking that the
4011  *	- Delayed acks are enabled or this is a half-synchronized T/TCP
4013  *	- The data being acked is less than a full segment (a stretch ack
4015  *      - nsegs is 1 (if its more than that we received more than 1 ack).
4018 	(((tp->t_flags & TF_RXWIN0SENT) == 0) &&		\
4019 	 ((tp->t_flags & TF_DELACK) == 0) && 		 	\
4020 	 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) &&	\
4021 	 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4036 	 * Walk the time-order transmitted list looking for an rsm that is  in bbr_find_lowest_rsm()
4040 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {  in bbr_find_lowest_rsm()
4041 		if (rsm->r_flags & BBR_ACKED) {  in bbr_find_lowest_rsm()
4061 	TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_find_high_nonack()
4062 		if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) {  in bbr_find_high_nonack()
4083 	 * If reorder-fade is configured, then we track the last time we saw  in bbr_calc_thresh_rack()
4084 	 * re-ordering occur. If we reach the point where enough time as  in bbr_calc_thresh_rack()
4087 	 * Or if reorder-face is 0, then once we see reordering we consider  in bbr_calc_thresh_rack()
4091 	 * In the end if lro is non-zero we add the extra time for  in bbr_calc_thresh_rack()
4099 	if (bbr->r_ctl.rc_reorder_ts) {  in bbr_calc_thresh_rack()
4100 		if (bbr->r_ctl.rc_reorder_fade) {  in bbr_calc_thresh_rack()
4101 			if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {  in bbr_calc_thresh_rack()
4102 				lro = cts - bbr->r_ctl.rc_reorder_ts;  in bbr_calc_thresh_rack()
4114 			if (lro > bbr->r_ctl.rc_reorder_fade) {  in bbr_calc_thresh_rack()
4116 				bbr->r_ctl.rc_reorder_ts = 0;  in bbr_calc_thresh_rack()
4126 	thresh = srtt + bbr->r_ctl.rc_pkt_delay;  in bbr_calc_thresh_rack()
4129 		if (bbr->r_ctl.rc_reorder_shift)  in bbr_calc_thresh_rack()
4130 			thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);  in bbr_calc_thresh_rack()
4137 	if ((bbr->rc_tp)->t_srtt == 0)  in bbr_calc_thresh_rack()
4140 		t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);  in bbr_calc_thresh_rack()
4145 	if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {  in bbr_calc_thresh_rack()
4146 		thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);  in bbr_calc_thresh_rack()
4153  * Return to the caller the amount of time in mico-seconds
4167 	if (bbr->rc_tlp_threshold)  in bbr_calc_thresh_tlp()
4168 		thresh = srtt + (srtt / bbr->rc_tlp_threshold);  in bbr_calc_thresh_tlp()
4171 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_calc_thresh_tlp()
4173 	len = rsm->r_end - rsm->r_start;  in bbr_calc_thresh_tlp()
4180 		 * possible inter-packet delay (if any).  in bbr_calc_thresh_tlp()
4185 		idx = rsm->r_rtr_cnt - 1;  in bbr_calc_thresh_tlp()
4186 		nidx = prsm->r_rtr_cnt - 1;  in bbr_calc_thresh_tlp()
4187 		if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {  in bbr_calc_thresh_tlp()
4189 			inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];  in bbr_calc_thresh_tlp()
4194 		 * Possibly compensate for delayed-ack.  in bbr_calc_thresh_tlp()
4203 	if (tp->t_srtt == 0)  in bbr_calc_thresh_tlp()
4206 		t_rxtcur = TICKS_2_USEC(tp->t_rxtcur);  in bbr_calc_thresh_tlp()
4214 	if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {  in bbr_calc_thresh_tlp()
4215 		thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);  in bbr_calc_thresh_tlp()
4233 	f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in bbr_get_rtt()
4234 	if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {  in bbr_get_rtt()
4236 		if (bbr->rc_tp->t_srtt == 0)  in bbr_get_rtt()
4239 			f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_get_rtt()
4242 		 * delayed-ack min  in bbr_get_rtt()
4248 	/* Take the filter version or last measured pkt-rtt */  in bbr_get_rtt()
4252 		if (bbr->r_ctl.rc_pkt_epoch_rtt) {  in bbr_get_rtt()
4253 			srtt = bbr->r_ctl.rc_pkt_epoch_rtt;  in bbr_get_rtt()
4259 		srtt = bbr->r_ctl.rc_last_rtt;  in bbr_get_rtt()
4261 		if (bbr->rc_ack_was_delayed)  in bbr_get_rtt()
4262 			srtt += bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_get_rtt()
4264 		srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_get_rtt()
4282 	if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {  in bbr_is_lost()
4304 	if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {  in bbr_check_recovery_mode()
4308 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_check_recovery_mode()
4313 	if (tp->t_flags & TF_SENTFIN) {  in bbr_check_recovery_mode()
4317 	if (rsm->r_flags & BBR_ACKED) {  in bbr_check_recovery_mode()
4326 	idx = rsm->r_rtr_cnt - 1;  in bbr_check_recovery_mode()
4327 	if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {  in bbr_check_recovery_mode()
4333 	    ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||  in bbr_check_recovery_mode()
4334 	     (rsm->r_flags & BBR_SACK_PASSED))) {  in bbr_check_recovery_mode()
4335 		if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {  in bbr_check_recovery_mode()
4336 			rsm->r_flags |= BBR_MARKED_LOST;  in bbr_check_recovery_mode()
4337 			bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;  in bbr_check_recovery_mode()
4338 			bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;  in bbr_check_recovery_mode()
4342 		if ((rsm->r_end - rsm->r_start) == 0)  in bbr_check_recovery_mode()
4363 	 * retransmissions, if so we will enter fast-recovery. The output  in bbr_timeout_rack()
4369 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_rack()
4372 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {  in bbr_timeout_rack()
4377 	lost = bbr->r_ctl.rc_lost;  in bbr_timeout_rack()
4378 	if (bbr->r_state && (bbr->r_state != tp->t_state))  in bbr_timeout_rack()
4381 	if (bbr->r_ctl.rc_resend == NULL) {  in bbr_timeout_rack()
4383 		bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);  in bbr_timeout_rack()
4386 		bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));  in bbr_timeout_rack()
4387 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;  in bbr_timeout_rack()
4396 	nrsm->r_start = start;  in bbr_clone_rsm()
4397 	nrsm->r_end = rsm->r_end;  in bbr_clone_rsm()
4398 	nrsm->r_rtr_cnt = rsm->r_rtr_cnt;  in bbr_clone_rsm()
4399 	nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed;  in bbr_clone_rsm()
4400 	nrsm->r_flags = rsm->r_flags;  in bbr_clone_rsm()
4402 	nrsm->r_flags &= ~BBR_HAS_SYN;  in bbr_clone_rsm()
4404 	rsm->r_flags &= ~BBR_HAS_FIN;  in bbr_clone_rsm()
4405 	nrsm->r_dupack = rsm->r_dupack;  in bbr_clone_rsm()
4406 	nrsm->r_rtr_bytes = 0;  in bbr_clone_rsm()
4407 	nrsm->r_is_gain = rsm->r_is_gain;  in bbr_clone_rsm()
4408 	nrsm->r_is_drain = rsm->r_is_drain;  in bbr_clone_rsm()
4409 	nrsm->r_delivered = rsm->r_delivered;  in bbr_clone_rsm()
4410 	nrsm->r_ts_valid = rsm->r_ts_valid;  in bbr_clone_rsm()
4411 	nrsm->r_del_ack_ts = rsm->r_del_ack_ts;  in bbr_clone_rsm()
4412 	nrsm->r_del_time = rsm->r_del_time;  in bbr_clone_rsm()
4413 	nrsm->r_app_limited = rsm->r_app_limited;  in bbr_clone_rsm()
4414 	nrsm->r_first_sent_time = rsm->r_first_sent_time;  in bbr_clone_rsm()
4415 	nrsm->r_flight_at_send = rsm->r_flight_at_send;  in bbr_clone_rsm()
4417 	nrsm->r_bbr_state = rsm->r_bbr_state;  in bbr_clone_rsm()
4418 	for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {  in bbr_clone_rsm()
4419 		nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];  in bbr_clone_rsm()
4421 	rsm->r_end = nrsm->r_start;  in bbr_clone_rsm()
4422 	idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);  in bbr_clone_rsm()
4425 	if ((rsm->r_is_smallmap == 0) &&  in bbr_clone_rsm()
4426 	    ((rsm->r_end - rsm->r_start) <= idx)) {  in bbr_clone_rsm()
4427 		bbr->r_ctl.rc_num_small_maps_alloced++;  in bbr_clone_rsm()
4428 		rsm->r_is_smallmap = 1;  in bbr_clone_rsm()
4431 	if ((nrsm->r_end - nrsm->r_start) <= idx) {  in bbr_clone_rsm()
4432 		bbr->r_ctl.rc_num_small_maps_alloced++;  in bbr_clone_rsm()
4433 		nrsm->r_is_smallmap = 1;  in bbr_clone_rsm()
4456 	if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) {  in bbr_sack_mergable()
4458 		if ((l_rsm->r_end == start) ||  in bbr_sack_mergable()
4459 		    (SEQ_LT(start, l_rsm->r_end) &&  in bbr_sack_mergable()
4460 		     SEQ_GT(end, l_rsm->r_end))) {  in bbr_sack_mergable()
4462 			     * map blk   |------|  in bbr_sack_mergable()
4463 			     * sack blk         |------|  in bbr_sack_mergable()
4465 			     * map blk   |------|  in bbr_sack_mergable()
4466 			     * sack blk      |------|  in bbr_sack_mergable()
4471 	if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) {  in bbr_sack_mergable()
4473 		if ((r_rsm->r_start == end) ||  in bbr_sack_mergable()
4474 		    (SEQ_LT(start, r_rsm->r_start) &&  in bbr_sack_mergable()
4475 		     SEQ_GT(end, r_rsm->r_start))) {  in bbr_sack_mergable()
4477 			 * map blk          |---------|  in bbr_sack_mergable()
4478 			 * sack blk    |----|  in bbr_sack_mergable()
4480 			 * map blk          |---------|  in bbr_sack_mergable()
4481 			 * sack blk    |-------|  in bbr_sack_mergable()
4504 	l_rsm->r_end = r_rsm->r_end;  in bbr_merge_rsm()
4505 	if (l_rsm->r_dupack < r_rsm->r_dupack)  in bbr_merge_rsm()
4506 		l_rsm->r_dupack = r_rsm->r_dupack;  in bbr_merge_rsm()
4507 	if (r_rsm->r_rtr_bytes)  in bbr_merge_rsm()
4508 		l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;  in bbr_merge_rsm()
4509 	if (r_rsm->r_in_tmap) {  in bbr_merge_rsm()
4511 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);  in bbr_merge_rsm()
4513 	if (r_rsm->r_app_limited)  in bbr_merge_rsm()
4514 		l_rsm->r_app_limited = r_rsm->r_app_limited;  in bbr_merge_rsm()
4516 	if (r_rsm->r_flags & BBR_HAS_FIN)  in bbr_merge_rsm()
4517 		l_rsm->r_flags |= BBR_HAS_FIN;  in bbr_merge_rsm()
4518 	if (r_rsm->r_flags & BBR_TLP)  in bbr_merge_rsm()
4519 		l_rsm->r_flags |= BBR_TLP;  in bbr_merge_rsm()
4520 	if (r_rsm->r_flags & BBR_RWND_COLLAPSED)  in bbr_merge_rsm()
4521 		l_rsm->r_flags |= BBR_RWND_COLLAPSED;  in bbr_merge_rsm()
4522 	if (r_rsm->r_flags & BBR_MARKED_LOST) {  in bbr_merge_rsm()
4524 		bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;  in bbr_merge_rsm()
4526 	TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);  in bbr_merge_rsm()
4527 	if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {  in bbr_merge_rsm()
4529 		r_rsm->r_limit_type = l_rsm->r_limit_type;  in bbr_merge_rsm()
4530 		l_rsm->r_limit_type = 0;  in bbr_merge_rsm()
4557 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_tlp()
4560 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {  in bbr_timeout_tlp()
4566 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_tlp()
4569 	if (bbr->rc_in_persist) {  in bbr_timeout_tlp()
4572 	if (bbr->r_state && (bbr->r_state != tp->t_state))  in bbr_timeout_tlp()
4575 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_tlp()
4581 	avail = sbavail(&so->so_snd);  in bbr_timeout_tlp()
4583 	if (out > tp->snd_wnd) {  in bbr_timeout_tlp()
4590 		amm = avail - out;  in bbr_timeout_tlp()
4593 		} else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {  in bbr_timeout_tlp()
4594 			/* not enough to fill a MTU and no-delay is off */  in bbr_timeout_tlp()
4597 		/* Set the send-new override */  in bbr_timeout_tlp()
4598 		if ((out + amm) <= tp->snd_wnd) {  in bbr_timeout_tlp()
4599 			bbr->rc_tlp_new_data = 1;  in bbr_timeout_tlp()
4603 		bbr->r_ctl.rc_tlp_seg_send_cnt = 0;  in bbr_timeout_tlp()
4604 		bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;  in bbr_timeout_tlp()
4605 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_timeout_tlp()
4612 	 * Ok we need to arrange the last un-acked segment to be re-sent, or  in bbr_timeout_tlp()
4613 	 * optionally the first un-acked segment.  in bbr_timeout_tlp()
4616 		rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);  in bbr_timeout_tlp()
4628 		TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_timeout_tlp()
4629 			if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {  in bbr_timeout_tlp()
4636 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_timeout_tlp()
4641 	if ((rsm->r_end - rsm->r_start) > maxseg) {  in bbr_timeout_tlp()
4652 			 * do the large send (BTLP :-) ).  in bbr_timeout_tlp()
4656 		bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));  in bbr_timeout_tlp()
4657 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_timeout_tlp()
4658 		if (rsm->r_in_tmap) {  in bbr_timeout_tlp()
4659 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_timeout_tlp()
4660 			nrsm->r_in_tmap = 1;  in bbr_timeout_tlp()
4662 		rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_timeout_tlp()
4666 	bbr->r_ctl.rc_tlp_send = rsm;  in bbr_timeout_tlp()
4667 	bbr->rc_tlp_rtx_out = 1;  in bbr_timeout_tlp()
4668 	if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {  in bbr_timeout_tlp()
4669 		bbr->r_ctl.rc_tlp_seg_send_cnt++;  in bbr_timeout_tlp()
4670 		tp->t_rxtshift++;  in bbr_timeout_tlp()
4672 		bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;  in bbr_timeout_tlp()
4673 		bbr->r_ctl.rc_tlp_seg_send_cnt = 1;  in bbr_timeout_tlp()
4676 	if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {  in bbr_timeout_tlp()
4682 		bbr->rc_tlp_new_data = 0;  in bbr_timeout_tlp()
4683 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_timeout_tlp()
4685 			rsm->r_flags &= ~BBR_TLP;  in bbr_timeout_tlp()
4689 		rsm->r_flags |= BBR_TLP;  in bbr_timeout_tlp()
4691 	if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&  in bbr_timeout_tlp()
4692 	    (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {  in bbr_timeout_tlp()
4700 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;  in bbr_timeout_tlp()
4715 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_delack()
4719 	tp->t_flags &= ~TF_DELACK;  in bbr_timeout_delack()
4720 	tp->t_flags |= TF_ACKNOW;  in bbr_timeout_delack()
4722 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;  in bbr_timeout_delack()
4727  * Here we send a KEEP-ALIVE like probe to the
4739 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_persist()
4742 	if (bbr->rc_in_persist == 0)  in bbr_timeout_persist()
4750 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;  in bbr_timeout_persist()
4757 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_persist()
4765 	if (tp->t_rxtshift >= V_tcp_retries &&  in bbr_timeout_persist()
4766 	    (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||  in bbr_timeout_persist()
4767 	    ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {  in bbr_timeout_persist()
4770 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_persist()
4772 	if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&  in bbr_timeout_persist()
4773 	    tp->snd_una == tp->snd_max) {  in bbr_timeout_persist()
4782 	if (tp->t_state > TCPS_CLOSE_WAIT &&  in bbr_timeout_persist()
4783 	    (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {  in bbr_timeout_persist()
4786 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_persist()
4788 	t_template = tcpip_maketemplate(bbr->rc_inp);  in bbr_timeout_persist()
4790 		tcp_respond(tp, t_template->tt_ipgen,  in bbr_timeout_persist()
4791 			    &t_template->tt_t, (struct mbuf *)NULL,  in bbr_timeout_persist()
4792 			    tp->rcv_nxt, tp->snd_una - 1, 0);  in bbr_timeout_persist()
4794 		if (tp->t_flags & TF_DELACK)  in bbr_timeout_persist()
4795 			tp->t_flags &= ~TF_DELACK;  in bbr_timeout_persist()
4798 	if (tp->t_rxtshift < V_tcp_retries)  in bbr_timeout_persist()
4799 		tp->t_rxtshift++;  in bbr_timeout_persist()
4817 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_keepalive()
4820 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;  in bbr_timeout_keepalive()
4823 	 * Keep-alive timer went off; send something or drop connection if  in bbr_timeout_keepalive()
4827 	if (tp->t_state < TCPS_ESTABLISHED)  in bbr_timeout_keepalive()
4829 	if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_timeout_keepalive()
4830 	    tp->t_state <= TCPS_CLOSING) {  in bbr_timeout_keepalive()
4831 		if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))  in bbr_timeout_keepalive()
4838 		 * number tp->snd_una-1 causes the transmitted zero-length  in bbr_timeout_keepalive()
4846 			tcp_respond(tp, t_template->tt_ipgen,  in bbr_timeout_keepalive()
4847 			    &t_template->tt_t, (struct mbuf *)NULL,  in bbr_timeout_keepalive()
4848 			    tp->rcv_nxt, tp->snd_una - 1, 0);  in bbr_timeout_keepalive()
4857 	return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_keepalive()
4869 	 * un-acked.  in bbr_remxt_tmr()
4875 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_remxt_tmr()
4876 	cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_remxt_tmr()
4877 	lost = bbr->r_ctl.rc_lost;  in bbr_remxt_tmr()
4878 	if (bbr->r_state && (bbr->r_state != tp->t_state))  in bbr_remxt_tmr()
4881 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_remxt_tmr()
4882 		if (rsm->r_flags & BBR_ACKED) {  in bbr_remxt_tmr()
4885 			rsm->r_dupack = 0;  in bbr_remxt_tmr()
4886 			if (rsm->r_in_tmap == 0) {  in bbr_remxt_tmr()
4887 				/* We must re-add it back to the tlist */  in bbr_remxt_tmr()
4889 					TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_remxt_tmr()
4891 					TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);  in bbr_remxt_tmr()
4893 				rsm->r_in_tmap = 1;  in bbr_remxt_tmr()
4895 			old_flags = rsm->r_flags;  in bbr_remxt_tmr()
4896 			rsm->r_flags |= BBR_RXT_CLEARED;  in bbr_remxt_tmr()
4897 			rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);  in bbr_remxt_tmr()
4900 			if ((tp->t_state < TCPS_ESTABLISHED) &&  in bbr_remxt_tmr()
4901 			    (rsm->r_start == tp->snd_una)) {  in bbr_remxt_tmr()
4909 			if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {  in bbr_remxt_tmr()
4910 				bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;  in bbr_remxt_tmr()
4911 				bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;  in bbr_remxt_tmr()
4918 				rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;  in bbr_remxt_tmr()
4919 				rsm->r_flags &= ~BBR_WAS_SACKPASS;  in bbr_remxt_tmr()
4928 				rsm->r_flags |= BBR_MARKED_LOST;  in bbr_remxt_tmr()
4929 				rsm->r_flags &= ~BBR_WAS_SACKPASS;  in bbr_remxt_tmr()
4930 				rsm->r_flags &= ~BBR_SACK_PASSED;  in bbr_remxt_tmr()
4935 	bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_remxt_tmr()
4936 	/* Clear the count (we just un-acked them) */  in bbr_remxt_tmr()
4938 	bbr->rc_tlp_new_data = 0;  in bbr_remxt_tmr()
4939 	bbr->r_ctl.rc_tlp_seg_send_cnt = 0;  in bbr_remxt_tmr()
4941 	bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_remxt_tmr()
4942 	bbr->r_agg_early_set = 0;  in bbr_remxt_tmr()
4943 	bbr->r_ctl.rc_agg_early = 0;  in bbr_remxt_tmr()
4944 	bbr->rc_tlp_rtx_out = 0;  in bbr_remxt_tmr()
4945 	bbr->r_ctl.rc_sacked = 0;  in bbr_remxt_tmr()
4946 	bbr->r_ctl.rc_sacklast = NULL;  in bbr_remxt_tmr()
4947 	bbr->r_timer_override = 1;  in bbr_remxt_tmr()
4948 	bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));  in bbr_remxt_tmr()
4952  * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4963 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;  in bbr_timeout_rxt()
4964 	if (bbr->rc_all_timers_stopped) {  in bbr_timeout_rxt()
4967 	if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_timeout_rxt()
4968 	    (tp->snd_una == tp->snd_max)) {  in bbr_timeout_rxt()
4979 		return (-ETIMEDOUT);	/* tcp_drop() */  in bbr_timeout_rxt()
4982 	if ((bbr->r_ctl.rc_resend == NULL) ||  in bbr_timeout_rxt()
4983 	    ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {  in bbr_timeout_rxt()
4990 		tp->t_rxtshift++;  in bbr_timeout_rxt()
4992 	if (tp->t_rxtshift > V_tcp_retries) {  in bbr_timeout_rxt()
4993 		tp->t_rxtshift = V_tcp_retries;  in bbr_timeout_rxt()
4997 		MPASS(tp->t_softerror >= 0);  in bbr_timeout_rxt()
4998 		retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;  in bbr_timeout_rxt()
5001 	if (tp->t_state == TCPS_SYN_SENT) {  in bbr_timeout_rxt()
5006 		tp->snd_cwnd = 1;  in bbr_timeout_rxt()
5007 	} else if (tp->t_rxtshift == 1) {  in bbr_timeout_rxt()
5014 		 * End-to-End Network Path Properties" by Allman and Paxson  in bbr_timeout_rxt()
5017 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_rxt()
5018 		if (!IN_RECOVERY(tp->t_flags)) {  in bbr_timeout_rxt()
5019 			tp->snd_cwnd_prev = tp->snd_cwnd;  in bbr_timeout_rxt()
5020 			tp->snd_ssthresh_prev = tp->snd_ssthresh;  in bbr_timeout_rxt()
5021 			tp->snd_recover_prev = tp->snd_recover;  in bbr_timeout_rxt()
5022 			tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));  in bbr_timeout_rxt()
5023 			tp->t_flags |= TF_PREVVALID;  in bbr_timeout_rxt()
5025 			tp->t_flags &= ~TF_PREVVALID;  in bbr_timeout_rxt()
5027 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_rxt()
5029 		tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;  in bbr_timeout_rxt()
5030 		tp->t_flags &= ~TF_PREVVALID;  in bbr_timeout_rxt()
5033 	if ((tp->t_state == TCPS_SYN_SENT) ||  in bbr_timeout_rxt()
5034 	    (tp->t_state == TCPS_SYN_RECEIVED))  in bbr_timeout_rxt()
5035 		rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift];  in bbr_timeout_rxt()
5037 		rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];  in bbr_timeout_rxt()
5038 	TCPT_RANGESET(tp->t_rxtcur, rexmt,  in bbr_timeout_rxt()
5039 	    MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),  in bbr_timeout_rxt()
5040 	    MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));  in bbr_timeout_rxt()
5049 	isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false;  in bbr_timeout_rxt()
5056 	    ((tp->t_state == TCPS_ESTABLISHED) ||  in bbr_timeout_rxt()
5057 	    (tp->t_state == TCPS_FIN_WAIT_1))) {  in bbr_timeout_rxt()
5060 		 * 1448 -> 1188 -> 524) should be given 2 chances to recover  in bbr_timeout_rxt()
5061 		 * before further clamping down. 'tp->t_rxtshift % 2 == 0'  in bbr_timeout_rxt()
5064 		if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==  in bbr_timeout_rxt()
5066 		    (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&  in bbr_timeout_rxt()
5067 		    tp->t_rxtshift % 2 == 0)) {  in bbr_timeout_rxt()
5069 			 * Enter Path MTU Black-hole Detection mechanism: -  in bbr_timeout_rxt()
5070 			 * Disable Path MTU Discovery (IP "DF" bit). -  in bbr_timeout_rxt()
5074 			if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {  in bbr_timeout_rxt()
5079 				tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;  in bbr_timeout_rxt()
5081 				tp->t_pmtud_saved_maxseg = tp->t_maxseg;  in bbr_timeout_rxt()
5088 			isipv6 = bbr->r_is_v6;  in bbr_timeout_rxt()
5090 			    tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {  in bbr_timeout_rxt()
5092 				tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;  in bbr_timeout_rxt()
5096 				tp->t_maxseg = V_tcp_v6mssdflt;  in bbr_timeout_rxt()
5101 				tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_timeout_rxt()
5109 			if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {  in bbr_timeout_rxt()
5111 				tp->t_maxseg = V_tcp_pmtud_blackhole_mss;  in bbr_timeout_rxt()
5115 				tp->t_maxseg = V_tcp_mssdflt;  in bbr_timeout_rxt()
5120 				tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_timeout_rxt()
5133 			if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&  in bbr_timeout_rxt()
5134 			    (tp->t_rxtshift >= 6)) {  in bbr_timeout_rxt()
5135 				tp->t_flags2 |= TF2_PLPMTU_PMTUD;  in bbr_timeout_rxt()
5136 				tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;  in bbr_timeout_rxt()
5137 				tp->t_maxseg = tp->t_pmtud_saved_maxseg;  in bbr_timeout_rxt()
5138 				if (tp->t_maxseg < V_tcp_mssdflt) {  in bbr_timeout_rxt()
5144 					tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;  in bbr_timeout_rxt()
5146 					tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;  in bbr_timeout_rxt()
5154 	 * third SYN to work-around some broken terminal servers (most of  in bbr_timeout_rxt()
5157 	 * unknown-to-them TCP options.  in bbr_timeout_rxt()
5159 	if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&  in bbr_timeout_rxt()
5160 	    (tp->t_rxtshift == 3))  in bbr_timeout_rxt()
5161 		tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);  in bbr_timeout_rxt()
5168 	if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {  in bbr_timeout_rxt()
5170 		if (bbr->r_is_v6)  in bbr_timeout_rxt()
5175 		tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);  in bbr_timeout_rxt()
5176 		tp->t_srtt = 0;  in bbr_timeout_rxt()
5178 	sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_timeout_rxt()
5179 	tp->snd_recover = tp->snd_max;  in bbr_timeout_rxt()
5180 	tp->t_flags |= TF_ACKNOW;  in bbr_timeout_rxt()
5181 	tp->t_rtttime = 0;  in bbr_timeout_rxt()
5190 	int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);  in bbr_process_timers()
5195 	if (tp->t_state == TCPS_LISTEN) {  in bbr_process_timers()
5197 		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)  in bbr_process_timers()
5201 	if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {  in bbr_process_timers()
5204 		if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {  in bbr_process_timers()
5205 			ret = -1;  in bbr_process_timers()
5210 			ret = -2;  in bbr_process_timers()
5218 		left = bbr->r_ctl.rc_timer_exp - cts;  in bbr_process_timers()
5219 		ret = -3;  in bbr_process_timers()
5224 	bbr->rc_tmr_stopped = 0;  in bbr_process_timers()
5225 	bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;  in bbr_process_timers()
5231 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_process_timers()
5234 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_process_timers()
5237 		bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_process_timers()
5249 	if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {  in bbr_timer_cancel()
5252 		if (tcp_in_hpts(bbr->rc_tp) &&  in bbr_timer_cancel()
5253 		    (bbr->rc_timer_first == 1)) {  in bbr_timer_cancel()
5260 			tcp_hpts_remove(bbr->rc_tp);  in bbr_timer_cancel()
5261 			if (bbr->r_ctl.rc_last_delay_val) {  in bbr_timer_cancel()
5265 				if (TSTMP_GT(cts, bbr->rc_pacer_started))  in bbr_timer_cancel()
5266 					time_since_send = cts - bbr->rc_pacer_started;  in bbr_timer_cancel()
5269 				if (bbr->r_ctl.rc_last_delay_val > time_since_send) {  in bbr_timer_cancel()
5271 					bbr->r_ctl.rc_last_delay_val -= time_since_send;  in bbr_timer_cancel()
5273 					bbr->r_ctl.rc_last_delay_val = 0;  in bbr_timer_cancel()
5275 				bbr->rc_pacer_started = cts;  in bbr_timer_cancel()
5278 		bbr->rc_timer_first = 0;  in bbr_timer_cancel()
5280 		bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;  in bbr_timer_cancel()
5281 		bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);  in bbr_timer_cancel()
5290 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_stopall()
5291 	bbr->rc_all_timers_stopped = 1;  in bbr_stopall()
5303 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_get_earliest_send_outstanding()
5306 	return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);  in bbr_get_earliest_send_outstanding()
5315 	rsm->r_rtr_cnt++;  in bbr_update_rsm()
5316 	rsm->r_dupack = 0;  in bbr_update_rsm()
5317 	if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {  in bbr_update_rsm()
5318 		rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;  in bbr_update_rsm()
5319 		rsm->r_flags |= BBR_OVERMAX;  in bbr_update_rsm()
5321 	if (rsm->r_flags & BBR_RWND_COLLAPSED) {  in bbr_update_rsm()
5323 		rsm->r_flags &= ~BBR_RWND_COLLAPSED;  in bbr_update_rsm()
5325 	if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_update_rsm()
5327 		rsm->r_flags &= ~BBR_MARKED_LOST;  in bbr_update_rsm()
5328 		bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_update_rsm()
5330 	if (rsm->r_flags & BBR_RXT_CLEARED) {  in bbr_update_rsm()
5339 		rsm->r_flags &= ~BBR_RXT_CLEARED;  in bbr_update_rsm()
5341 	if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {  in bbr_update_rsm()
5342 		bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);  in bbr_update_rsm()
5343 		rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);  in bbr_update_rsm()
5345 	idx = rsm->r_rtr_cnt - 1;  in bbr_update_rsm()
5346 	rsm->r_tim_lastsent[idx] = cts;  in bbr_update_rsm()
5347 	rsm->r_pacing_delay = pacing_time;  in bbr_update_rsm()
5348 	rsm->r_delivered = bbr->r_ctl.rc_delivered;  in bbr_update_rsm()
5349 	rsm->r_ts_valid = bbr->rc_ts_valid;  in bbr_update_rsm()
5350 	if (bbr->rc_ts_valid)  in bbr_update_rsm()
5351 		rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;  in bbr_update_rsm()
5352 	if (bbr->r_ctl.r_app_limited_until)  in bbr_update_rsm()
5353 		rsm->r_app_limited = 1;  in bbr_update_rsm()
5355 		rsm->r_app_limited = 0;  in bbr_update_rsm()
5356 	if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)  in bbr_update_rsm()
5357 		rsm->r_bbr_state = bbr_state_val(bbr);  in bbr_update_rsm()
5359 		rsm->r_bbr_state = 8;  in bbr_update_rsm()
5360 	if (rsm->r_flags & BBR_ACKED) {  in bbr_update_rsm()
5364 		old_flags = rsm->r_flags;  in bbr_update_rsm()
5365 		rsm->r_flags &= ~BBR_ACKED;  in bbr_update_rsm()
5367 		bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);  in bbr_update_rsm()
5368 		if (bbr->r_ctl.rc_sacked == 0)  in bbr_update_rsm()
5369 			bbr->r_ctl.rc_sacklast = NULL;  in bbr_update_rsm()
5371 	if (rsm->r_in_tmap) {  in bbr_update_rsm()
5372 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_update_rsm()
5374 	TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_update_rsm()
5375 	rsm->r_in_tmap = 1;  in bbr_update_rsm()
5376 	if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_update_rsm()
5378 		rsm->r_flags &= ~BBR_SACK_PASSED;  in bbr_update_rsm()
5379 		rsm->r_flags |= BBR_WAS_SACKPASS;  in bbr_update_rsm()
5381 	rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);  in bbr_update_rsm()
5382 	rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,  in bbr_update_rsm()
5383 						(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_update_rsm()
5384 	bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);  in bbr_update_rsm()
5385 	if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {  in bbr_update_rsm()
5386 		rsm->r_is_gain = 1;  in bbr_update_rsm()
5387 		rsm->r_is_drain = 0;  in bbr_update_rsm()
5388 	} else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {  in bbr_update_rsm()
5389 		rsm->r_is_drain = 1;  in bbr_update_rsm()
5390 		rsm->r_is_gain = 0;  in bbr_update_rsm()
5392 		rsm->r_is_drain = 0;  in bbr_update_rsm()
5393 		rsm->r_is_gain = 0;  in bbr_update_rsm()
5395 	rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */  in bbr_update_rsm()
5409 	 * We (re-)transmitted starting at rsm->r_start for some length  in bbr_update_entry()
5417 	c_end = rsm->r_start + len;  in bbr_update_entry()
5418 	if (SEQ_GEQ(c_end, rsm->r_end)) {  in bbr_update_entry()
5424 		if (c_end == rsm->r_end) {  in bbr_update_entry()
5431 			act_len = rsm->r_end - rsm->r_start;  in bbr_update_entry()
5432 			*lenp = (len - act_len);  in bbr_update_entry()
5433 			return (rsm->r_end);  in bbr_update_entry()
5454 	TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_update_entry()
5455 	nrsm->r_dupack = 0;  in bbr_update_entry()
5456 	if (rsm->r_in_tmap) {  in bbr_update_entry()
5457 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_update_entry()
5458 		nrsm->r_in_tmap = 1;  in bbr_update_entry()
5460 	rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_update_entry()
5485 	if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {  in bbr_setup_less_of_rate()
5489 		bbr->skip_gain = 1;  in bbr_setup_less_of_rate()
5490 		bbr->gain_is_limited = 0;  in bbr_setup_less_of_rate()
5491 		red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;  in bbr_setup_less_of_rate()
5493 			filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);  in bbr_setup_less_of_rate()
5496 		bbr->skip_gain = 0;  in bbr_setup_less_of_rate()
5497 		bbr->gain_is_limited = 1;  in bbr_setup_less_of_rate()
5505 	int error, rate = -1;  in bbr_update_hardware_pacing_rate()
5507 	if (bbr->r_ctl.crte == NULL)  in bbr_update_hardware_pacing_rate()
5509 	if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||  in bbr_update_hardware_pacing_rate()
5510 	    (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {  in bbr_update_hardware_pacing_rate()
5512 		/* Clear the way for a re-attempt */  in bbr_update_hardware_pacing_rate()
5513 		bbr->bbr_attempt_hdwr_pace = 0;  in bbr_update_hardware_pacing_rate()
5515 		bbr->gain_is_limited = 0;  in bbr_update_hardware_pacing_rate()
5516 		bbr->skip_gain = 0;  in bbr_update_hardware_pacing_rate()
5517 		bbr->bbr_hdrw_pacing = 0;  in bbr_update_hardware_pacing_rate()
5518 		counter_u64_add(bbr_flows_whdwr_pacing, -1);  in bbr_update_hardware_pacing_rate()
5524 	nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,  in bbr_update_hardware_pacing_rate()
5525 				   bbr->rc_tp,  in bbr_update_hardware_pacing_rate()
5526 				   bbr->rc_inp->inp_route.ro_nh->nh_ifp,  in bbr_update_hardware_pacing_rate()
5533 	if (nrte != bbr->r_ctl.crte) {  in bbr_update_hardware_pacing_rate()
5534 		bbr->r_ctl.crte = nrte;  in bbr_update_hardware_pacing_rate()
5537 			if (bbr->r_ctl.crte->rate < rate) {  in bbr_update_hardware_pacing_rate()
5540 						       bbr->r_ctl.crte->rate, rate);  in bbr_update_hardware_pacing_rate()
5543 				bbr->gain_is_limited = 0;  in bbr_update_hardware_pacing_rate()
5544 				bbr->skip_gain = 0;  in bbr_update_hardware_pacing_rate()
5549 			bbr->gain_is_limited = 0;  in bbr_update_hardware_pacing_rate()
5550 			bbr->skip_gain = 0;  in bbr_update_hardware_pacing_rate()
5551 			bbr->bbr_hdrw_pacing = 0;  in bbr_update_hardware_pacing_rate()
5554 					 bbr->r_ctl.crte->ptbl->rs_ifp,  in bbr_update_hardware_pacing_rate()
5556 					 ((bbr->r_ctl.crte == NULL) ? 0 : bbr->r_ctl.crte->rate),  in bbr_update_hardware_pacing_rate()
5574 	if ((bbr->bbr_hdrw_pacing == 0) ||  in bbr_adjust_for_hw_pacing()
5575 	    (IN_RECOVERY(bbr->rc_tp->t_flags)) ||  in bbr_adjust_for_hw_pacing()
5576 	    (bbr->r_ctl.crte == NULL))  in bbr_adjust_for_hw_pacing()
5578 	if (bbr->hw_pacing_set == 0) {  in bbr_adjust_for_hw_pacing()
5586 	rlp = bbr->r_ctl.crte;  in bbr_adjust_for_hw_pacing()
5587 	if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)  in bbr_adjust_for_hw_pacing()
5588 		maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in bbr_adjust_for_hw_pacing()
5590 		maxseg = BBR_MIN_SEG - bbr->rc_last_options;  in bbr_adjust_for_hw_pacing()
5598 		        bbr->r_ctl.rc_pace_max_segs, cts, 1);  in bbr_adjust_for_hw_pacing()
5599 	hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;  in bbr_adjust_for_hw_pacing()
5600 	hdwr_delay *= rlp->time_between;  in bbr_adjust_for_hw_pacing()
5602 		delta = cur_delay - hdwr_delay;  in bbr_adjust_for_hw_pacing()
5606 			     (bbr->r_ctl.rc_pace_max_segs / maxseg),  in bbr_adjust_for_hw_pacing()
5609 	    (delta < (max(rlp->time_between,  in bbr_adjust_for_hw_pacing()
5610 			  bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {  in bbr_adjust_for_hw_pacing()
5624 		seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between),  in bbr_adjust_for_hw_pacing()
5625 			     (bbr->r_ctl.rc_pace_max_segs/maxseg));  in bbr_adjust_for_hw_pacing()
5628 		    (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {  in bbr_adjust_for_hw_pacing()
5634 			seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;  in bbr_adjust_for_hw_pacing()
5641 		 * we probably are going quite fast at  in bbr_adjust_for_hw_pacing()
5648 		seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;  in bbr_adjust_for_hw_pacing()
5650 		    (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {  in bbr_adjust_for_hw_pacing()
5656 			seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;  in bbr_adjust_for_hw_pacing()
5670 		seg_sz = bbr->r_ctl.rc_pace_max_segs;  in bbr_adjust_for_hw_pacing()
5672 	if (seg_sz > bbr->r_ctl.rc_pace_max_segs)  in bbr_adjust_for_hw_pacing()
5675 		new_tso = bbr->r_ctl.rc_pace_max_segs;  in bbr_adjust_for_hw_pacing()
5676 	if (new_tso >= (PACE_MAX_IP_BYTES-maxseg))  in bbr_adjust_for_hw_pacing()
5677 		new_tso = PACE_MAX_IP_BYTES - maxseg;  in bbr_adjust_for_hw_pacing()
5679 	if (new_tso != bbr->r_ctl.rc_pace_max_segs) {  in bbr_adjust_for_hw_pacing()
5680 		bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);  in bbr_adjust_for_hw_pacing()
5681 		bbr->r_ctl.rc_pace_max_segs = new_tso;  in bbr_adjust_for_hw_pacing()
5708 	 * a full iwnd just like new-reno/cubic.  in tcp_bbr_tso_size_check()
5713 	 *  if ( bw <= per-tcb-cross-over)  in tcp_bbr_tso_size_check()
5715 	 *                 can send in goal-time seconds.  in tcp_bbr_tso_size_check()
5721 	 *     if (tso > per-tcb-max)  in tcp_bbr_tso_size_check()
5722 	 *         tso = per-tcb-max  in tcp_bbr_tso_size_check()
5724 	 *     tso = max-tso (64k/mss)  in tcp_bbr_tso_size_check()
5726 	 *     goal_tso = bw / per-tcb-divsor  in tcp_bbr_tso_size_check()
5727 	 *     seg = (goal_tso + mss-1)/mss  in tcp_bbr_tso_size_check()
5730 	 * if (tso < per-tcb-floor)  in tcp_bbr_tso_size_check()
5731 	 *    tso = per-tcb-floor  in tcp_bbr_tso_size_check()
5732 	 * if (tso > per-tcb-utter_max)  in tcp_bbr_tso_size_check()
5733 	 *    tso = per-tcb-utter_max  in tcp_bbr_tso_size_check()
5735 	 * Note the default per-tcb-divisor is 1000 (same as google).  in tcp_bbr_tso_size_check()
5739 	 * cross-over = 23,168,000 bps  in tcp_bbr_tso_size_check()
5740 	 * goal-time = 18000  in tcp_bbr_tso_size_check()
5741 	 * per-tcb-max = 2  in tcp_bbr_tso_size_check()
5742 	 * per-tcb-divisor = 1000  in tcp_bbr_tso_size_check()
5743 	 * per-tcb-floor = 1  in tcp_bbr_tso_size_check()
5752 	if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {  in tcp_bbr_tso_size_check()
5753 		maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in tcp_bbr_tso_size_check()
5755 		maxseg = BBR_MIN_SEG - bbr->rc_last_options;  in tcp_bbr_tso_size_check()
5757 	old_tso = bbr->r_ctl.rc_pace_max_segs;  in tcp_bbr_tso_size_check()
5758 	if (bbr->rc_past_init_win == 0) {  in tcp_bbr_tso_size_check()
5764 		if (bbr->rc_use_google)  in tcp_bbr_tso_size_check()
5765 			bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);  in tcp_bbr_tso_size_check()
5766 		else if (bbr->bbr_init_win_cheat)  in tcp_bbr_tso_size_check()
5767 			bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);  in tcp_bbr_tso_size_check()
5769 			bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in tcp_bbr_tso_size_check()
5770 		if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)  in tcp_bbr_tso_size_check()
5771 			bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;  in tcp_bbr_tso_size_check()
5772 		if (bbr->r_ctl.rc_pace_max_segs == 0) {  in tcp_bbr_tso_size_check()
5773 			bbr->r_ctl.rc_pace_max_segs = maxseg;  in tcp_bbr_tso_size_check()
5775 		bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);  in tcp_bbr_tso_size_check()
5785 	if (IN_RECOVERY(bbr->rc_tp->t_flags) &&  in tcp_bbr_tso_size_check()
5786 	     (bbr->rc_use_google == 0)) {  in tcp_bbr_tso_size_check()
5789 	} else if (bbr->rc_use_google) {  in tcp_bbr_tso_size_check()
5793 		if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {  in tcp_bbr_tso_size_check()
5794 			bw *= bbr->r_ctl.rc_bbr_hptsi_gain;  in tcp_bbr_tso_size_check()
5808 	} else if (bbr->rc_no_pacing) {  in tcp_bbr_tso_size_check()
5810 	} else if (bw <= bbr->r_ctl.bbr_cross_over) {  in tcp_bbr_tso_size_check()
5817 		tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);  in tcp_bbr_tso_size_check()
5820 			if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)  in tcp_bbr_tso_size_check()
5821 				new_tso = bbr->r_ctl.bbr_hptsi_segments_max;  in tcp_bbr_tso_size_check()
5836 		 * This guy is so fast b/w wise that we can TSO as large as  in tcp_bbr_tso_size_check()
5854 		bw /= bbr->r_ctl.bbr_hptsi_per_second;  in tcp_bbr_tso_size_check()
5861 		if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))  in tcp_bbr_tso_size_check()
5862 			new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);  in tcp_bbr_tso_size_check()
5864 …if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floo…  in tcp_bbr_tso_size_check()
5865 		new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;  in tcp_bbr_tso_size_check()
5869 	if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {  in tcp_bbr_tso_size_check()
5870 		new_tso = bbr->r_ctl.bbr_utter_max * maxseg;  in tcp_bbr_tso_size_check()
5875 		bbr->r_ctl.rc_pace_max_segs = new_tso;  in tcp_bbr_tso_size_check()
5910 		 * We don't log errors -- we could but snd_max does not  in bbr_log_output()
5923 	snd_una = tp->snd_una;  in bbr_log_output()
5931 		if ((th_flags & TH_SYN) && (tp->iss == seq_out))  in bbr_log_output()
5937 		/* Are sending an old segment to induce an ack (keep-alive)? */  in bbr_log_output()
5946 		len = end - seq_out;  in bbr_log_output()
5948 	snd_max = tp->snd_max;  in bbr_log_output()
5953 	pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);  in bbr_log_output()
5961 		rsm->r_flags = 0;  in bbr_log_output()
5963 			rsm->r_flags |= BBR_HAS_SYN;  in bbr_log_output()
5965 			rsm->r_flags |= BBR_HAS_FIN;  in bbr_log_output()
5966 		rsm->r_tim_lastsent[0] = cts;  in bbr_log_output()
5967 		rsm->r_rtr_cnt = 1;  in bbr_log_output()
5968 		rsm->r_rtr_bytes = 0;  in bbr_log_output()
5969 		rsm->r_start = seq_out;  in bbr_log_output()
5970 		rsm->r_end = rsm->r_start + len;  in bbr_log_output()
5971 		rsm->r_dupack = 0;  in bbr_log_output()
5972 		rsm->r_delivered = bbr->r_ctl.rc_delivered;  in bbr_log_output()
5973 		rsm->r_pacing_delay = pacing_time;  in bbr_log_output()
5974 		rsm->r_ts_valid = bbr->rc_ts_valid;  in bbr_log_output()
5975 		if (bbr->rc_ts_valid)  in bbr_log_output()
5976 			rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;  in bbr_log_output()
5977 		rsm->r_del_time = bbr->r_ctl.rc_del_time;  in bbr_log_output()
5978 		if (bbr->r_ctl.r_app_limited_until)  in bbr_log_output()
5979 			rsm->r_app_limited = 1;  in bbr_log_output()
5981 			rsm->r_app_limited = 0;  in bbr_log_output()
5982 		rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);  in bbr_log_output()
5983 		rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,  in bbr_log_output()
5984 						(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_log_output()
5989 		rsm->r_flight_at_send += len;  in bbr_log_output()
5990 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_log_output()
5991 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_log_output()
5992 		rsm->r_in_tmap = 1;  in bbr_log_output()
5993 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)  in bbr_log_output()
5994 			rsm->r_bbr_state = bbr_state_val(bbr);  in bbr_log_output()
5996 			rsm->r_bbr_state = 8;  in bbr_log_output()
5997 		if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {  in bbr_log_output()
5998 			rsm->r_is_gain = 1;  in bbr_log_output()
5999 			rsm->r_is_drain = 0;  in bbr_log_output()
6000 		} else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {  in bbr_log_output()
6001 			rsm->r_is_drain = 1;  in bbr_log_output()
6002 			rsm->r_is_gain = 0;  in bbr_log_output()
6004 			rsm->r_is_drain = 0;  in bbr_log_output()
6005 			rsm->r_is_gain = 0;  in bbr_log_output()
6013 	if (hintrsm && (hintrsm->r_start == seq_out)) {  in bbr_log_output()
6016 	} else if (bbr->r_ctl.rc_next) {  in bbr_log_output()
6018 		rsm = bbr->r_ctl.rc_next;  in bbr_log_output()
6023 	if ((rsm) && (rsm->r_start == seq_out)) {  in bbr_log_output()
6036 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_log_output()
6037 		if (rsm->r_start == seq_out) {  in bbr_log_output()
6039 			bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);  in bbr_log_output()
6046 		if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {  in bbr_log_output()
6062 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_log_output()
6063 			if (rsm->r_in_tmap) {  in bbr_log_output()
6064 				TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_log_output()
6065 				nrsm->r_in_tmap = 1;  in bbr_log_output()
6067 			rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_log_output()
6078 	if (seq_out == tp->snd_max) {  in bbr_log_output()
6080 	} else if (SEQ_LT(seq_out, tp->snd_max)) {  in bbr_log_output()
6082 		printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",  in bbr_log_output()
6083 		    seq_out, len, tp->snd_una, tp->snd_max);  in bbr_log_output()
6085 		TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_log_output()
6087 			    rsm, rsm->r_start, rsm->r_end);  in bbr_log_output()
6096 		 * Hmm beyond sndmax? (only if we are using the new rtt-pack  in bbr_log_output()
6100 		    seq_out, len, tp->snd_max, tp);  in bbr_log_output()
6109 	 * Collapse timeout back the cum-ack moved.  in bbr_collapse_rtt()
6111 	tp->t_rxtshift = 0;  in bbr_collapse_rtt()
6112 	tp->t_softerror = 0;  in bbr_collapse_rtt()
6118 	bbr->rtt_valid = 1;  in tcp_bbr_xmit_timer()
6119 	bbr->r_ctl.cur_rtt = rtt_usecs;  in tcp_bbr_xmit_timer()
6120 	bbr->r_ctl.ts_in = tsin;  in tcp_bbr_xmit_timer()
6122 		bbr->r_ctl.cur_rtt_send_time = rsm_send_time;  in tcp_bbr_xmit_timer()
6130 	 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp).  in bbr_make_timestamp_determination()
6131 	 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).  in bbr_make_timestamp_determination()
6132 	 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts)  in bbr_make_timestamp_determination()
6133 	 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time)  in bbr_make_timestamp_determination()
6137 	 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts  in bbr_make_timestamp_determination()
6141 	 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp  in bbr_make_timestamp_determination()
6148 	 * out time-slices of the actual b/w. This would mean we could not use  in bbr_make_timestamp_determination()
6152 	 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio.  in bbr_make_timestamp_determination()
6166 	delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;  in bbr_make_timestamp_determination()
6176 	peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;  in bbr_make_timestamp_determination()
6195 		 * then we saw then in micro-seconds.  in bbr_make_timestamp_determination()
6198 			/* well it looks like the peer is a micro-second clock. */  in bbr_make_timestamp_determination()
6199 			bbr->rc_ts_clock_set = 1;  in bbr_make_timestamp_determination()
6200 			bbr->r_ctl.bbr_peer_tsratio = 1;  in bbr_make_timestamp_determination()
6202 			bbr->rc_ts_cant_be_used = 1;  in bbr_make_timestamp_determination()
6203 			bbr->rc_ts_clock_set = 1;  in bbr_make_timestamp_determination()
6208 	bbr->rc_ts_clock_set = 1;  in bbr_make_timestamp_determination()
6213 		bbr->r_ctl.bbr_peer_tsratio = 1;  in bbr_make_timestamp_determination()
6220 		bbr->r_ctl.bbr_peer_tsratio = 10;  in bbr_make_timestamp_determination()
6227 		bbr->r_ctl.bbr_peer_tsratio = 100;  in bbr_make_timestamp_determination()
6234 		bbr->r_ctl.bbr_peer_tsratio = 1000;  in bbr_make_timestamp_determination()
6241 		bbr->r_ctl.bbr_peer_tsratio = 10000;  in bbr_make_timestamp_determination()
6245 	bbr->rc_ts_cant_be_used = 1;  in bbr_make_timestamp_determination()
6246 	bbr->r_ctl.bbr_peer_tsratio = 0;  in bbr_make_timestamp_determination()
6251  * Collect new round-trip time estimate
6261 	if (bbr->rtt_valid == 0)  in tcp_bbr_xmit_timer_commit()
6265 	rtt = bbr->r_ctl.cur_rtt;  in tcp_bbr_xmit_timer_commit()
6266 	tsin = bbr->r_ctl.ts_in;  in tcp_bbr_xmit_timer_commit()
6267 	if (bbr->rc_prtt_set_ts) {  in tcp_bbr_xmit_timer_commit()
6276 		 * and when they enter probe-rtt they update the  in tcp_bbr_xmit_timer_commit()
6281 		bbr->rc_prtt_set_ts = 0;  in tcp_bbr_xmit_timer_commit()
6282 		rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);  in tcp_bbr_xmit_timer_commit()
6284 			filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);  in tcp_bbr_xmit_timer_commit()
6286 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in tcp_bbr_xmit_timer_commit()
6289 	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt));  in tcp_bbr_xmit_timer_commit()
6291 	if (bbr->rc_ack_was_delayed)  in tcp_bbr_xmit_timer_commit()
6292 		rtt += bbr->r_ctl.rc_ack_hdwr_delay;  in tcp_bbr_xmit_timer_commit()
6294 	if (rtt < bbr->r_ctl.rc_lowest_rtt)  in tcp_bbr_xmit_timer_commit()
6295 		bbr->r_ctl.rc_lowest_rtt = rtt;  in tcp_bbr_xmit_timer_commit()
6297 	if (bbr->r_init_rtt) {  in tcp_bbr_xmit_timer_commit()
6299 		 * The initial rtt is not-trusted, nuke it and lets get  in tcp_bbr_xmit_timer_commit()
6302 		bbr->r_init_rtt = 0;  in tcp_bbr_xmit_timer_commit()
6303 		tp->t_srtt = 0;  in tcp_bbr_xmit_timer_commit()
6305 	if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {  in tcp_bbr_xmit_timer_commit()
6310 		 * series of cum-ack's to determine  in tcp_bbr_xmit_timer_commit()
6313 		if (bbr->rc_ack_is_cumack) {  in tcp_bbr_xmit_timer_commit()
6314 			if (bbr->rc_ts_data_set) {  in tcp_bbr_xmit_timer_commit()
6318 				bbr->rc_ts_data_set = 1;  in tcp_bbr_xmit_timer_commit()
6319 				bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;  in tcp_bbr_xmit_timer_commit()
6320 				bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;  in tcp_bbr_xmit_timer_commit()
6327 			bbr->rc_ts_data_set = 0;  in tcp_bbr_xmit_timer_commit()
6331 	rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1)));  in tcp_bbr_xmit_timer_commit()
6334 	if (tp->t_srtt != 0) {  in tcp_bbr_xmit_timer_commit()
6343 		delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT)  in tcp_bbr_xmit_timer_commit()
6344 		    - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));  in tcp_bbr_xmit_timer_commit()
6346 		tp->t_srtt += delta;  in tcp_bbr_xmit_timer_commit()
6347 		if (tp->t_srtt <= 0)  in tcp_bbr_xmit_timer_commit()
6348 			tp->t_srtt = 1;  in tcp_bbr_xmit_timer_commit()
6358 		 * wired-in beta.  in tcp_bbr_xmit_timer_commit()
6361 			delta = -delta;  in tcp_bbr_xmit_timer_commit()
6362 		delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);  in tcp_bbr_xmit_timer_commit()
6363 		tp->t_rttvar += delta;  in tcp_bbr_xmit_timer_commit()
6364 		if (tp->t_rttvar <= 0)  in tcp_bbr_xmit_timer_commit()
6365 			tp->t_rttvar = 1;  in tcp_bbr_xmit_timer_commit()
6368 		 * No rtt measurement yet - use the unsmoothed rtt. Set the  in tcp_bbr_xmit_timer_commit()
6372 		tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT;  in tcp_bbr_xmit_timer_commit()
6373 		tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1);  in tcp_bbr_xmit_timer_commit()
6376 	if (tp->t_rttupdated < UCHAR_MAX)  in tcp_bbr_xmit_timer_commit()
6377 		tp->t_rttupdated++;  in tcp_bbr_xmit_timer_commit()
6379 	stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));  in tcp_bbr_xmit_timer_commit()
6385 	 * tick of rounding and 1 extra tick because of +-1/2 tick  in tcp_bbr_xmit_timer_commit()
6391 	TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),  in tcp_bbr_xmit_timer_commit()
6392 	    max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),  in tcp_bbr_xmit_timer_commit()
6393 	    MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));  in tcp_bbr_xmit_timer_commit()
6402 	tp->t_softerror = 0;  in tcp_bbr_xmit_timer_commit()
6403 	rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);  in tcp_bbr_xmit_timer_commit()
6404 	if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)  in tcp_bbr_xmit_timer_commit()
6405 		bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;  in tcp_bbr_xmit_timer_commit()
6411 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_set_reduced_rtt()
6413 	    (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&  in bbr_set_reduced_rtt()
6414 	    ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {  in bbr_set_reduced_rtt()
6416 		 * We should enter probe-rtt its been too long  in bbr_set_reduced_rtt()
6429 	if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {  in tcp_bbr_commit_bw()
6435 	if (bbr->r_ctl.r_measurement_count < 0xffffffff)  in tcp_bbr_commit_bw()
6436 		bbr->r_ctl.r_measurement_count++;  in tcp_bbr_commit_bw()
6437 	orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);  in tcp_bbr_commit_bw()
6438 	apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);  in tcp_bbr_commit_bw()
6440 			    (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),  in tcp_bbr_commit_bw()
6443 	    (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {  in tcp_bbr_commit_bw()
6444 		if (bbr->bbr_hdrw_pacing) {  in tcp_bbr_commit_bw()
6453 		if (bbr->r_recovery_bw)  {  in tcp_bbr_commit_bw()
6457 	} else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))  in tcp_bbr_commit_bw()
6464 	if (bbr->rc_in_persist == 0) {  in bbr_nf_measurement()
6470 		if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))  in bbr_nf_measurement()
6471 			tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);  in bbr_nf_measurement()
6479 		delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);  in bbr_nf_measurement()
6489 		 * can see in the trace viewer if it gets over-ridden.  in bbr_nf_measurement()
6491 		if (rsm->r_ts_valid &&  in bbr_nf_measurement()
6492 		    bbr->rc_ts_valid &&  in bbr_nf_measurement()
6493 		    bbr->rc_ts_clock_set &&  in bbr_nf_measurement()
6494 		    (bbr->rc_ts_cant_be_used == 0) &&  in bbr_nf_measurement()
6495 		    bbr->rc_use_ts_limit) {  in bbr_nf_measurement()
6496 			ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);  in bbr_nf_measurement()
6497 			ts_diff *= bbr->r_ctl.bbr_peer_tsratio;  in bbr_nf_measurement()
6503 						    bbr->r_ctl.last_inbound_ts,  in bbr_nf_measurement()
6504 						    rsm->r_del_ack_ts, 0,  in bbr_nf_measurement()
6514 				if ((bbr->ts_can_raise) &&  in bbr_nf_measurement()
6534 		if (rsm->r_first_sent_time &&  in bbr_nf_measurement()
6535 		    TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {  in bbr_nf_measurement()
6548 			sbw = (uint64_t)(rsm->r_flight_at_send);  in bbr_nf_measurement()
6550 			sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;  in bbr_nf_measurement()
6551 			sti += rsm->r_pacing_delay;  in bbr_nf_measurement()
6559 						    rsm->r_first_sent_time, 0, (sbw >> 32),  in bbr_nf_measurement()
6565 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;  in bbr_nf_measurement()
6566 		if ((rsm->r_app_limited == 0) ||  in bbr_nf_measurement()
6567 		    (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {  in bbr_nf_measurement()
6570 					    0, 0, 0, 0,  bbr->r_ctl.rc_del_time,  rsm->r_del_time);  in bbr_nf_measurement()
6578 	if (bbr->rc_in_persist == 0) {  in bbr_google_measurement()
6585 		if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))  in bbr_google_measurement()
6586 			tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);  in bbr_google_measurement()
6594 		delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);  in bbr_google_measurement()
6598 		if (tim < bbr->r_ctl.rc_lowest_rtt) {  in bbr_google_measurement()
6600 					    tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);  in bbr_google_measurement()
6606 		 * can see in the trace viewer if it gets over-ridden.  in bbr_google_measurement()
6608 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;  in bbr_google_measurement()
6610 		if (rsm->r_first_sent_time &&  in bbr_google_measurement()
6611 		    TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {  in bbr_google_measurement()
6624 			sbw = (uint64_t)(rsm->r_flight_at_send);  in bbr_google_measurement()
6626 			sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;  in bbr_google_measurement()
6627 			sti += rsm->r_pacing_delay;  in bbr_google_measurement()
6635 						    rsm->r_first_sent_time, 0, (sbw >> 32),  in bbr_google_measurement()
6640 			    (sti < bbr->r_ctl.rc_lowest_rtt)) {  in bbr_google_measurement()
6642 						    (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);  in bbr_google_measurement()
6647 		bbr->r_ctl.rc_bbr_cur_del_rate = bw;  in bbr_google_measurement()
6649 		    ((rsm->r_app_limited == 0) ||  in bbr_google_measurement()
6650 		     (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {  in bbr_google_measurement()
6653 					    0, 0, 0, 0, bbr->r_ctl.rc_del_time,  rsm->r_del_time);  in bbr_google_measurement()
6665 	bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);  in bbr_update_bbr_info()
6666 	bbr->r_ctl.rc_del_time = cts;  in bbr_update_bbr_info()
6674 	if ((bbr->rc_use_google == 0) &&  in bbr_update_bbr_info()
6687 	if ((bbr_no_retran && bbr->rc_use_google) &&  in bbr_update_bbr_info()
6697 	tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);  in bbr_update_bbr_info()
6699 		bbr->rc_ack_is_cumack = 1;  in bbr_update_bbr_info()
6701 		bbr->rc_ack_is_cumack = 0;  in bbr_update_bbr_info()
6708 	 * going into probe-rtt (we were seeing cases where that  in bbr_update_bbr_info()
6717 	bbr_log_type_bbrrttprop(bbr, rtt, (rsm ? rsm->r_end : 0), uts, cts,  in bbr_update_bbr_info()
6718 	    match, rsm->r_start, rsm->r_flags);  in bbr_update_bbr_info()
6719 	apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in bbr_update_bbr_info()
6722 		 * The RTT-prop moved, reset the target (may be a  in bbr_update_bbr_info()
6726 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)  in bbr_update_bbr_info()
6733 	if ((bbr->rc_use_google == 0) &&  in bbr_update_bbr_info()
6742 	if (bbr->r_ctl.r_app_limited_until &&  in bbr_update_bbr_info()
6743 	    (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {  in bbr_update_bbr_info()
6744 		/* We are no longer app-limited */  in bbr_update_bbr_info()
6745 		bbr->r_ctl.r_app_limited_until = 0;  in bbr_update_bbr_info()
6747 	if (bbr->rc_use_google) {  in bbr_update_bbr_info()
6764 	msec = cts - (MS_IN_USEC * sec);  in bbr_ts_convert()
6779 	if ((rsm->r_flags & BBR_ACKED) ||  in bbr_update_rtt()
6780 	    (rsm->r_flags & BBR_WAS_RENEGED) ||  in bbr_update_rtt()
6781 	    (rsm->r_flags & BBR_RXT_CLEARED)) {  in bbr_update_rtt()
6785 	if (rsm->r_rtt_not_allowed) {  in bbr_update_rtt()
6789 	if (rsm->r_rtr_cnt == 1) {  in bbr_update_rtt()
6793 		if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))  in bbr_update_rtt()
6794 			t = cts - rsm->r_tim_lastsent[0];  in bbr_update_rtt()
6799 		bbr->r_ctl.rc_last_rtt = t;  in bbr_update_rtt()
6800 		bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,  in bbr_update_rtt()
6801 				    BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);  in bbr_update_rtt()
6806 	    (bbr->rc_use_google == 1) &&  in bbr_update_rtt()
6808 	    (to->to_flags & TOF_TS) &&  in bbr_update_rtt()
6809 	    (to->to_tsecr != 0)) {  in bbr_update_rtt()
6810 		t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr;  in bbr_update_rtt()
6814 		bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,  in bbr_update_rtt()
6816 				    rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],  in bbr_update_rtt()
6820 	uts = bbr_ts_convert(to->to_tsecr);  in bbr_update_rtt()
6821 	if ((to->to_flags & TOF_TS) &&  in bbr_update_rtt()
6822 	    (to->to_tsecr != 0) &&  in bbr_update_rtt()
6824 	    ((rsm->r_flags & BBR_OVERMAX) == 0)) {  in bbr_update_rtt()
6832 		for (i = 0; i < rsm->r_rtr_cnt; i++) {  in bbr_update_rtt()
6833 			if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&  in bbr_update_rtt()
6834 			    (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {  in bbr_update_rtt()
6835 				if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))  in bbr_update_rtt()
6836 					t = cts - rsm->r_tim_lastsent[i];  in bbr_update_rtt()
6841 				bbr->r_ctl.rc_last_rtt = t;  in bbr_update_rtt()
6842 				bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,  in bbr_update_rtt()
6843 						    rsm->r_tim_lastsent[i], ack_type, to);  in bbr_update_rtt()
6844 				if ((i + 1) < rsm->r_rtr_cnt) {  in bbr_update_rtt()
6847 				} else if (rsm->r_flags & BBR_TLP) {  in bbr_update_rtt()
6848 					bbr->rc_tlp_rtx_out = 0;  in bbr_update_rtt()
6858 	 * time-stamp since its not there or the time the peer last  in bbr_update_rtt()
6859 	 * received a segment that moved forward its cum-ack point.  in bbr_update_rtt()
6865 	i = rsm->r_rtr_cnt - 1;  in bbr_update_rtt()
6866 	if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))  in bbr_update_rtt()
6867 		t = cts - rsm->r_tim_lastsent[i];  in bbr_update_rtt()
6870 	if (t < bbr->r_ctl.rc_lowest_rtt) {  in bbr_update_rtt()
6876 		 * the rack-draft.  in bbr_update_rtt()
6882 		if ((rsm->r_flags & BBR_OVERMAX) == 0) {  in bbr_update_rtt()
6884 			if (rsm->r_rtr_cnt == 1)  in bbr_update_rtt()
6885 				panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);  in bbr_update_rtt()
6887 			i = rsm->r_rtr_cnt - 2;  in bbr_update_rtt()
6888 			if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))  in bbr_update_rtt()
6889 				t = cts - rsm->r_tim_lastsent[i];  in bbr_update_rtt()
6892 			bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,  in bbr_update_rtt()
6893 					    rsm->r_tim_lastsent[i], ack_type, to);  in bbr_update_rtt()
6901 			bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,  in bbr_update_rtt()
6909 		if (rsm->r_flags & BBR_TLP)  in bbr_update_rtt()
6910 			bbr->rc_tlp_rtx_out = 0;  in bbr_update_rtt()
6911 		if ((rsm->r_flags & BBR_OVERMAX) == 0)  in bbr_update_rtt()
6912 			bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,  in bbr_update_rtt()
6915 			bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,  in bbr_update_rtt()
6932 	TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,  in bbr_log_sack_passed()
6938 		if (nrsm->r_flags & BBR_ACKED) {  in bbr_log_sack_passed()
6942 		if (nrsm->r_flags & BBR_SACK_PASSED) {  in bbr_log_sack_passed()
6951 		nrsm->r_flags |= BBR_SACK_PASSED;  in bbr_log_sack_passed()
6952 		if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) &&  in bbr_log_sack_passed()
6953 		    bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {  in bbr_log_sack_passed()
6954 			bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;  in bbr_log_sack_passed()
6955 			bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;  in bbr_log_sack_passed()
6956 			nrsm->r_flags |= BBR_MARKED_LOST;  in bbr_log_sack_passed()
6958 		nrsm->r_flags &= ~BBR_WAS_SACKPASS;  in bbr_log_sack_passed()
6976 	start = sack->start;  in bbr_proc_sack_blk()
6977 	end = sack->end;  in bbr_proc_sack_blk()
6983 	if (rsm && SEQ_LT(start, rsm->r_start)) {  in bbr_proc_sack_blk()
6985 		TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_proc_sack_blk()
6986 			if (SEQ_GEQ(start, rsm->r_start) &&  in bbr_proc_sack_blk()
6987 			    SEQ_LT(start, rsm->r_end)) {  in bbr_proc_sack_blk()
6998 	TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_proc_sack_blk()
6999 		if (SEQ_GEQ(start, rsm->r_start) &&  in bbr_proc_sack_blk()
7000 		    SEQ_LT(start, rsm->r_end)) {  in bbr_proc_sack_blk()
7012 		if (tp->t_flags & TF_SENTFIN) {  in bbr_proc_sack_blk()
7017 			nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);  in bbr_proc_sack_blk()
7018 			if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {  in bbr_proc_sack_blk()
7022 				nrsm->r_end++;  in bbr_proc_sack_blk()
7041 	if (rsm->r_start != start) {  in bbr_proc_sack_blk()
7055 			sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);  in bbr_proc_sack_blk()
7059 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_proc_sack_blk()
7060 		if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7061 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_proc_sack_blk()
7062 			nrsm->r_in_tmap = 1;  in bbr_proc_sack_blk()
7064 		rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_proc_sack_blk()
7067 	if (SEQ_GEQ(end, rsm->r_end)) {  in bbr_proc_sack_blk()
7072 		if ((rsm->r_flags & BBR_ACKED) == 0) {  in bbr_proc_sack_blk()
7074 			changed += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7075 			bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7077 			if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7078 				bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7081 			if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_proc_sack_blk()
7083 				bbr->r_ctl.rc_reorder_ts = cts;  in bbr_proc_sack_blk()
7084 				if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7085 					bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7086 					if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_proc_sack_blk()
7088 						bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_proc_sack_blk()
7091 			rsm->r_flags |= BBR_ACKED;  in bbr_proc_sack_blk()
7092 			rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);  in bbr_proc_sack_blk()
7093 			if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7094 				TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_proc_sack_blk()
7095 				rsm->r_in_tmap = 0;  in bbr_proc_sack_blk()
7099 		if (end == rsm->r_end) {  in bbr_proc_sack_blk()
7100 			/* This block only - done */  in bbr_proc_sack_blk()
7104 		start = rsm->r_end;  in bbr_proc_sack_blk()
7110 	if (rsm->r_flags & BBR_ACKED) {  in bbr_proc_sack_blk()
7125 		sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);  in bbr_proc_sack_blk()
7131 	rsm->r_flags &= (~BBR_HAS_FIN);  in bbr_proc_sack_blk()
7132 	TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_proc_sack_blk()
7133 	if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7134 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_proc_sack_blk()
7135 		nrsm->r_in_tmap = 1;  in bbr_proc_sack_blk()
7137 	nrsm->r_dupack = 0;  in bbr_proc_sack_blk()
7140 	changed += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7141 	bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);  in bbr_proc_sack_blk()
7144 	if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7145 		bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7147 	if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_proc_sack_blk()
7149 		bbr->r_ctl.rc_reorder_ts = cts;  in bbr_proc_sack_blk()
7150 		if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_proc_sack_blk()
7151 			bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_proc_sack_blk()
7152 			if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_proc_sack_blk()
7154 				bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_proc_sack_blk()
7157 	rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);  in bbr_proc_sack_blk()
7158 	rsm->r_flags |= BBR_ACKED;  in bbr_proc_sack_blk()
7159 	if (rsm->r_in_tmap) {  in bbr_proc_sack_blk()
7160 		TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_proc_sack_blk()
7161 		rsm->r_in_tmap = 0;  in bbr_proc_sack_blk()
7164 	if (rsm && (rsm->r_flags & BBR_ACKED)) {  in bbr_proc_sack_blk()
7172 		    (nrsm->r_flags & BBR_ACKED)) {  in bbr_proc_sack_blk()
7179 		    (nrsm->r_flags & BBR_ACKED)) {  in bbr_proc_sack_blk()
7198 		bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);  in bbr_proc_sack_blk()
7200 		bbr->r_ctl.rc_sacklast = NULL;  in bbr_proc_sack_blk()
7212 	while (rsm && (rsm->r_flags & BBR_ACKED)) {  in bbr_peer_reneges()
7215 		bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);  in bbr_peer_reneges()
7217 		if (rsm->r_in_tmap) {  in bbr_peer_reneges()
7219 			    bbr, rsm, rsm->r_flags);  in bbr_peer_reneges()
7222 		oflags = rsm->r_flags;  in bbr_peer_reneges()
7223 		if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_peer_reneges()
7224 			bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_peer_reneges()
7225 			bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_peer_reneges()
7226 			if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_peer_reneges()
7228 				bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_peer_reneges()
7230 		rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);  in bbr_peer_reneges()
7231 		rsm->r_flags |= BBR_WAS_RENEGED;  in bbr_peer_reneges()
7232 		rsm->r_flags |= BBR_RXT_CLEARED;  in bbr_peer_reneges()
7233 		bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);  in bbr_peer_reneges()
7236 			TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_peer_reneges()
7239 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);  in bbr_peer_reneges()
7242 		tmap->r_in_tmap = 1;  in bbr_peer_reneges()
7257 	sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);  in bbr_peer_reneges()
7267 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_log_syn()
7268 	cts = bbr->r_ctl.rc_rcvtime;  in bbr_log_syn()
7269 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_syn()
7270 	if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {  in bbr_log_syn()
7271 		if ((rsm->r_end - rsm->r_start) <= 1) {  in bbr_log_syn()
7273 			bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;  in bbr_log_syn()
7274 			rsm->r_rtr_bytes = 0;  in bbr_log_syn()
7275 			TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_log_syn()
7276 			if (rsm->r_in_tmap) {  in bbr_log_syn()
7277 				TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_log_syn()
7278 				rsm->r_in_tmap = 0;  in bbr_log_syn()
7280 			if (bbr->r_ctl.rc_next == rsm) {  in bbr_log_syn()
7282 				bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_syn()
7288 			/* There is more (Fast open)? strip out SYN. */  in bbr_log_syn()
7289 			rsm->r_flags &= ~BBR_HAS_SYN;  in bbr_log_syn()
7290 			rsm->r_start++;  in bbr_log_syn()
7318 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_log_ack()
7319 	cts = bbr->r_ctl.rc_rcvtime;  in bbr_log_ack()
7321 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7323 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_log_ack()
7324 	p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);  in bbr_log_ack()
7325 	th_ack = th->th_ack;  in bbr_log_ack()
7326 	if (SEQ_GT(th_ack, tp->snd_una)) {  in bbr_log_ack()
7327 		acked = th_ack - tp->snd_una;  in bbr_log_ack()
7329 		bbr->rc_tp->t_acktime = ticks;  in bbr_log_ack()
7332 	if (SEQ_LEQ(th_ack, tp->snd_una)) {  in bbr_log_ack()
7336 	if (rsm && SEQ_GT(th_ack, rsm->r_start)) {  in bbr_log_ack()
7337 		changed = th_ack - rsm->r_start;  in bbr_log_ack()
7338 	} else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {  in bbr_log_ack()
7344 		if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) {  in bbr_log_ack()
7351 			ts = bbr_ts_convert(to->to_tsecr);  in bbr_log_ack()
7352 			now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv));  in bbr_log_ack()
7353 			rtt = now - ts;  in bbr_log_ack()
7357 						tp->iss, 0, cts,  in bbr_log_ack()
7358 						BBR_RTT_BY_TIMESTAMP, tp->iss, 0);  in bbr_log_ack()
7359 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in bbr_log_ack()
7361 			bbr->r_wanted_output = 1;  in bbr_log_ack()
7374 		bbr->r_wanted_output = 1;  in bbr_log_ack()
7377 			if (tp->t_flags & TF_SENTFIN) {  in bbr_log_ack()
7382 			panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n",  in bbr_log_ack()
7384 			    th, tp->t_state, bbr,  in bbr_log_ack()
7385 			    tp->snd_una, tp->snd_max, changed);  in bbr_log_ack()
7390 	if (SEQ_LT(th_ack, rsm->r_start)) {  in bbr_log_ack()
7394 		    rsm->r_start,  in bbr_log_ack()
7395 		    th_ack, tp->t_state,  in bbr_log_ack()
7396 		    bbr->r_state, bbr);  in bbr_log_ack()
7397 		panic("th-ack is bad bbr:%p tp:%p", bbr, tp);  in bbr_log_ack()
7400 	} else if (th_ack == rsm->r_start) {  in bbr_log_ack()
7409 	rsm->r_dupack = 0;  in bbr_log_ack()
7411 	if (SEQ_GEQ(th_ack, rsm->r_end)) {  in bbr_log_ack()
7415 		if (rsm->r_flags & BBR_ACKED) {  in bbr_log_ack()
7417 			 * It was acked on the scoreboard -- remove it from  in bbr_log_ack()
7420 			p_acked += (rsm->r_end - rsm->r_start);  in bbr_log_ack()
7421 			bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);  in bbr_log_ack()
7422 			if (bbr->r_ctl.rc_sacked == 0)  in bbr_log_ack()
7423 				bbr->r_ctl.rc_sacklast = NULL;  in bbr_log_ack()
7426 			if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_log_ack()
7427 				bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;  in bbr_log_ack()
7429 			if (rsm->r_flags & BBR_SACK_PASSED) {  in bbr_log_ack()
7436 				bbr->r_ctl.rc_reorder_ts = cts;  in bbr_log_ack()
7437 				if (rsm->r_flags & BBR_MARKED_LOST) {  in bbr_log_ack()
7438 					bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;  in bbr_log_ack()
7439 					if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))  in bbr_log_ack()
7441 						bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;  in bbr_log_ack()
7444 			rsm->r_flags &= ~BBR_MARKED_LOST;  in bbr_log_ack()
7446 		bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;  in bbr_log_ack()
7447 		rsm->r_rtr_bytes = 0;  in bbr_log_ack()
7448 		TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_log_ack()
7449 		if (rsm->r_in_tmap) {  in bbr_log_ack()
7450 			TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_log_ack()
7451 			rsm->r_in_tmap = 0;  in bbr_log_ack()
7453 		if (bbr->r_ctl.rc_next == rsm) {  in bbr_log_ack()
7455 			bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7459 		left = th_ack - rsm->r_end;  in bbr_log_ack()
7463 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7468 	if (rsm->r_flags & BBR_ACKED) {  in bbr_log_ack()
7470 		 * It was acked on the scoreboard -- remove it from total  in bbr_log_ack()
7471 		 * for the part being cum-acked.  in bbr_log_ack()
7473 		p_acked += (rsm->r_end - rsm->r_start);  in bbr_log_ack()
7474 		bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);  in bbr_log_ack()
7475 		if (bbr->r_ctl.rc_sacked == 0)  in bbr_log_ack()
7476 			bbr->r_ctl.rc_sacklast = NULL;  in bbr_log_ack()
7487 	if ((rsm->r_flags & BBR_MARKED_LOST) &&  in bbr_log_ack()
7488 	    ((rsm->r_flags & BBR_ACKED) == 0)) {  in bbr_log_ack()
7494 		bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;  in bbr_log_ack()
7497 	bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;  in bbr_log_ack()
7498 	rsm->r_rtr_bytes = 0;  in bbr_log_ack()
7500 	rsm->r_start = th_ack;  in bbr_log_ack()
7503 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_log_ack()
7504 	if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {  in bbr_log_ack()
7510 		 * us snd_una up to (rsm->r_end). We need to undo the acked  in bbr_log_ack()
7514 		 * rsm->r_start in case we get an old ack where th_ack is  in bbr_log_ack()
7517 		bbr_peer_reneges(bbr, rsm, th->th_ack);  in bbr_log_ack()
7519 	if ((to->to_flags & TOF_SACK) == 0) {  in bbr_log_ack()
7523 	rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);  in bbr_log_ack()
7525 		last_seq = rsm->r_end;  in bbr_log_ack()
7527 		last_seq = tp->snd_max;  in bbr_log_ack()
7530 	if (SEQ_GT(th_ack, tp->snd_una))  in bbr_log_ack()
7533 		ack_point = tp->snd_una;  in bbr_log_ack()
7534 	for (i = 0; i < to->to_nsacks; i++) {  in bbr_log_ack()
7535 		bcopy((to->to_sacks + i * TCPOLEN_SACK),  in bbr_log_ack()
7541 		    SEQ_LT(sack.start, tp->snd_max) &&  in bbr_log_ack()
7543 		    SEQ_LEQ(sack.end, tp->snd_max)) {  in bbr_log_ack()
7544 			if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&  in bbr_log_ack()
7546 			    ((sack.end - sack.start) < (p_maxseg / 8))) {  in bbr_log_ack()
7559 			 * Its a D-SACK block.  in bbr_log_ack()
7570 	new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks,  in bbr_log_ack()
7571 				  num_sack_blks, th->th_ack);  in bbr_log_ack()
7572 	ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);  in bbr_log_ack()
7574 	BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb));  in bbr_log_ack()
7612 					 * Now collapse out the dup-sack and  in bbr_log_ack()
7620 					num_sack_blks--;  in bbr_log_ack()
7627 	rsm = bbr->r_ctl.rc_sacklast;  in bbr_log_ack()
7631 			bbr->r_wanted_output = 1;  in bbr_log_ack()
7638 	if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {  in bbr_log_ack()
7649 			bbr->r_wanted_output = 1;  in bbr_log_ack()
7654 			if (bbr->r_ctl.rc_resend == NULL) {  in bbr_log_ack()
7655 				bbr->r_ctl.rc_resend = rsm;  in bbr_log_ack()
7659 	if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {  in bbr_log_ack()
7661 		 * See if we need to rack-retransmit anything if so set it  in bbr_log_ack()
7665 		if (bbr->r_ctl.rc_resend == NULL) {  in bbr_log_ack()
7666 			bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);  in bbr_log_ack()
7671 	 * ack-received code to augment what was changed between th_ack <->  in bbr_log_ack()
7682 	rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);  in bbr_strike_dupack()
7683 	if (rsm && (rsm->r_dupack < 0xff)) {  in bbr_strike_dupack()
7684 		rsm->r_dupack++;  in bbr_strike_dupack()
7685 		if (rsm->r_dupack >= DUP_ACK_THRESHOLD)  in bbr_strike_dupack()
7686 			bbr->r_wanted_output = 1;  in bbr_strike_dupack()
7693  * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7711 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_process_ack()
7712 	lost = bbr->r_ctl.rc_lost;  in bbr_process_ack()
7713 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_process_ack()
7714 	if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) {  in bbr_process_ack()
7716 		tp->t_flags2 |= TF2_NO_ISS_CHECK;  in bbr_process_ack()
7722 		if (tp->t_flags2 & TF2_NO_ISS_CHECK) {  in bbr_process_ack()
7724 			seq_min = tp->snd_una - tp->max_sndwnd;  in bbr_process_ack()
7727 			if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) {  in bbr_process_ack()
7729 				seq_min = tp->iss + 1;  in bbr_process_ack()
7736 				seq_min = tp->snd_una - tp->max_sndwnd;  in bbr_process_ack()
7740 		if (SEQ_LT(th->th_ack, seq_min)) {  in bbr_process_ack()
7747 			bbr->r_wanted_output = 1;  in bbr_process_ack()
7751 	if (SEQ_GT(th->th_ack, tp->snd_max)) {  in bbr_process_ack()
7753 		bbr->r_wanted_output = 1;  in bbr_process_ack()
7756 	if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {  in bbr_process_ack()
7758 		if (bbr->rc_in_persist)  in bbr_process_ack()
7759 			tp->t_rxtshift = 0;  in bbr_process_ack()
7760 		if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd))  in bbr_process_ack()
7764 	bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));  in bbr_process_ack()
7765 	if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {  in bbr_process_ack()
7770 		if (th->th_ack == tp->snd_una) {  in bbr_process_ack()
7772 			if (bbr->r_state == TCPS_SYN_SENT) {  in bbr_process_ack()
7775 				 * the SYN-ACK is processed in syn_sent  in bbr_process_ack()
7792 	if (tp->t_flags & TF_NEEDSYN) {  in bbr_process_ack()
7794 		 * T/TCP: Connection was half-synchronized, and our SYN has  in bbr_process_ack()
7796 		 * to non-starred state, increment snd_una for ACK of SYN,  in bbr_process_ack()
7799 		tp->t_flags &= ~TF_NEEDSYN;  in bbr_process_ack()
7800 		tp->snd_una++;  in bbr_process_ack()
7802 		if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==  in bbr_process_ack()
7804 			tp->rcv_scale = tp->request_r_scale;  in bbr_process_ack()
7820 	if (tp->t_flags & TF_PREVVALID) {  in bbr_process_ack()
7821 		tp->t_flags &= ~TF_PREVVALID;  in bbr_process_ack()
7822 		if (tp->t_rxtshift == 1 &&  in bbr_process_ack()
7823 		    (int)(ticks - tp->t_badrxtwin) < 0)  in bbr_process_ack()
7827 	acked_amount = min(acked, (int)sbavail(&so->so_snd));  in bbr_process_ack()
7828 	tp->snd_wnd -= acked_amount;  in bbr_process_ack()
7829 	mfree = sbcut_locked(&so->so_snd, acked_amount);  in bbr_process_ack()
7833 	if (SEQ_GT(th->th_ack, tp->snd_una)) {  in bbr_process_ack()
7836 	tp->snd_una = th->th_ack;  in bbr_process_ack()
7837 …bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - los…  in bbr_process_ack()
7838 	if (IN_RECOVERY(tp->t_flags)) {  in bbr_process_ack()
7839 		if (SEQ_LT(th->th_ack, tp->snd_recover) &&  in bbr_process_ack()
7840 		    (SEQ_LT(th->th_ack, tp->snd_max))) {  in bbr_process_ack()
7846 	if (SEQ_GT(tp->snd_una, tp->snd_recover)) {  in bbr_process_ack()
7847 		tp->snd_recover = tp->snd_una;  in bbr_process_ack()
7849 	if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_process_ack()
7850 		tp->snd_nxt = tp->snd_max;  in bbr_process_ack()
7852 	if (tp->snd_una == tp->snd_max) {  in bbr_process_ack()
7856 		if (sbavail(&so->so_snd) == 0)  in bbr_process_ack()
7857 			bbr->rc_tp->t_acktime = 0;  in bbr_process_ack()
7858 		if ((sbused(&so->so_snd) == 0) &&  in bbr_process_ack()
7859 		    (tp->t_flags & TF_SENTFIN)) {  in bbr_process_ack()
7862 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_ack()
7863 		if (bbr->rc_in_persist == 0) {  in bbr_process_ack()
7864 			bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;  in bbr_process_ack()
7866 		sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_process_ack()
7867 		bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_process_ack()
7873 		if ((tp->t_state >= TCPS_FIN_WAIT_1) &&  in bbr_process_ack()
7874 		    (sbavail(&so->so_snd) == 0) &&  in bbr_process_ack()
7875 		    (tp->t_flags2 & TF2_DROP_AF_DATA)) {  in bbr_process_ack()
7882 			/* tcp_close will kill the inp pre-log the Reset */  in bbr_process_ack()
7890 		bbr->r_wanted_output = 1;  in bbr_process_ack()
7900 	if (bbr->rc_in_persist == 0) {  in bbr_enter_persist()
7902 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_enter_persist()
7903 		tp->t_rxtshift = 0;  in bbr_enter_persist()
7904 		bbr->rc_in_persist = 1;  in bbr_enter_persist()
7905 		bbr->r_ctl.rc_went_idle_time = cts;  in bbr_enter_persist()
7909 		if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_enter_persist()
7912 			time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_enter_persist()
7913 			if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_enter_persist()
7919 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_enter_persist()
7922 		bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_enter_persist()
7935 		if (bbr->rc_use_idle_restart) {  in bbr_restart_after_idle()
7936 			bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;  in bbr_restart_after_idle()
7943 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_restart_after_idle()
7944 			bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_restart_after_idle()
7947 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;  in bbr_restart_after_idle()
7948 			bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;  in bbr_restart_after_idle()
7950 		} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_restart_after_idle()
7961 	if (bbr->rc_in_persist == 0)  in bbr_exit_persist()
7963 	idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);  in bbr_exit_persist()
7964 	bbr->rc_in_persist = 0;  in bbr_exit_persist()
7965 	bbr->rc_hit_state_1 = 0;  in bbr_exit_persist()
7966 	bbr->r_ctl.rc_del_time = cts;  in bbr_exit_persist()
7972 	if (tcp_in_hpts(bbr->rc_tp)) {  in bbr_exit_persist()
7973 		tcp_hpts_remove(bbr->rc_tp);  in bbr_exit_persist()
7974 		bbr->rc_timer_first = 0;  in bbr_exit_persist()
7975 		bbr->r_ctl.rc_hpts_flags = 0;  in bbr_exit_persist()
7976 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_exit_persist()
7977 		bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_exit_persist()
7978 		bbr->r_agg_early_set = 0;  in bbr_exit_persist()
7979 		bbr->r_ctl.rc_agg_early = 0;  in bbr_exit_persist()
7989 		bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_exit_persist()
7991 	tp->t_rxtshift = 0;  in bbr_exit_persist()
8002 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_exit_persist()
8003 	/* Time un-freezes for the state */  in bbr_exit_persist()
8004 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_exit_persist()
8005 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||  in bbr_exit_persist()
8006 	    (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {  in bbr_exit_persist()
8008 		 * If we are going back to probe-bw  in bbr_exit_persist()
8010 		 * do a fast restart.  in bbr_exit_persist()
8041 	maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;  in bbr_collapsed_window()
8042 	max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;  in bbr_collapsed_window()
8044 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_collapsed_window()
8046 		if (rsm->r_flags & BBR_RWND_COLLAPSED)  in bbr_collapsed_window()
8047 			rsm->r_flags &= ~BBR_RWND_COLLAPSED;  in bbr_collapsed_window()
8048 		if (SEQ_GEQ(max_seq, rsm->r_start) &&  in bbr_collapsed_window()
8049 		    SEQ_GEQ(rsm->r_end, max_seq)) {  in bbr_collapsed_window()
8054 	bbr->rc_has_collapsed = 0;  in bbr_collapsed_window()
8067 	if ((max_seq != rsm->r_start) &&  in bbr_collapsed_window()
8068 	    (max_seq != rsm->r_end)){  in bbr_collapsed_window()
8072 		res1 = max_seq - rsm->r_start;  in bbr_collapsed_window()
8073 		res2 = rsm->r_end - max_seq;  in bbr_collapsed_window()
8078 		} else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {  in bbr_collapsed_window()
8084 	if (max_seq == rsm->r_start) {  in bbr_collapsed_window()
8087 	} else if (max_seq == rsm->r_end) {  in bbr_collapsed_window()
8094 	} else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {  in bbr_collapsed_window()
8105 		TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);  in bbr_collapsed_window()
8106 		if (rsm->r_in_tmap) {  in bbr_collapsed_window()
8107 			TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);  in bbr_collapsed_window()
8108 			nrsm->r_in_tmap = 1;  in bbr_collapsed_window()
8121 	TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_collapsed_window()
8122 		nrsm->r_flags |= BBR_RWND_COLLAPSED;  in bbr_collapsed_window()
8124 		bbr->rc_has_collapsed = 1;  in bbr_collapsed_window()
8135 	TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {  in bbr_un_collapse_window()
8136 		if (rsm->r_flags & BBR_RWND_COLLAPSED) {  in bbr_un_collapse_window()
8138 			rsm->r_flags &= ~BBR_RWND_COLLAPSED;  in bbr_un_collapse_window()
8144 				   (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);  in bbr_un_collapse_window()
8145 	bbr->rc_has_collapsed = 0;  in bbr_un_collapse_window()
8166 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_process_data()
8168 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_process_data()
8170 	    (SEQ_LT(tp->snd_wl1, th->th_seq) ||  in bbr_process_data()
8171 	    (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||  in bbr_process_data()
8172 	    (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {  in bbr_process_data()
8175 		    tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)  in bbr_process_data()
8177 		tp->snd_wnd = tiwin;  in bbr_process_data()
8178 		tp->snd_wl1 = th->th_seq;  in bbr_process_data()
8179 		tp->snd_wl2 = th->th_ack;  in bbr_process_data()
8180 		if (tp->snd_wnd > tp->max_sndwnd)  in bbr_process_data()
8181 			tp->max_sndwnd = tp->snd_wnd;  in bbr_process_data()
8182 		bbr->r_wanted_output = 1;  in bbr_process_data()
8184 		if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {  in bbr_process_data()
8185 			tp->snd_wnd = tiwin;  in bbr_process_data()
8186 			tp->snd_wl1 = th->th_seq;  in bbr_process_data()
8187 			tp->snd_wl2 = th->th_ack;  in bbr_process_data()
8190 	if (tp->snd_wnd < ctf_outstanding(tp))  in bbr_process_data()
8193  	else if (bbr->rc_has_collapsed)  in bbr_process_data()
8196 	if ((bbr->rc_in_persist != 0) &&  in bbr_process_data()
8197 	    (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),  in bbr_process_data()
8203 		bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_process_data()
8206 		bbr->r_wanted_output = 1;  in bbr_process_data()
8209 	if ((bbr->rc_in_persist == 0) &&  in bbr_process_data()
8210 	    (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_process_data()
8211 	    TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_process_data()
8212 	    (tp->snd_max == tp->snd_una) &&  in bbr_process_data()
8213 	    sbavail(&so->so_snd) &&  in bbr_process_data()
8214 	    (sbavail(&so->so_snd) > tp->snd_wnd)) {  in bbr_process_data()
8216 		bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_process_data()
8218 	if (tp->t_flags2 & TF2_DROP_AF_DATA) {  in bbr_process_data()
8228 	tp->rcv_up = tp->rcv_nxt;  in bbr_process_data()
8233 	 * This process logically involves adjusting tp->rcv_wnd as data is  in bbr_process_data()
8238 	tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&  in bbr_process_data()
8239 	    (tp->t_flags & TF_FASTOPEN));  in bbr_process_data()
8241 	    TCPS_HAVERCVDFIN(tp->t_state) == 0) {  in bbr_process_data()
8242 		tcp_seq save_start = th->th_seq;  in bbr_process_data()
8243 		tcp_seq save_rnxt  = tp->rcv_nxt;  in bbr_process_data()
8256 		 * segments are out of order (so fast retransmit can work).  in bbr_process_data()
8258 		if (th->th_seq == tp->rcv_nxt &&  in bbr_process_data()
8260 		    (TCPS_HAVEESTABLISHED(tp->t_state) ||  in bbr_process_data()
8265 			if (so->so_rcv.sb_shlim) {  in bbr_process_data()
8268 				if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,  in bbr_process_data()
8278 				bbr->bbr_segs_rcvd += max(1, nsegs);  in bbr_process_data()
8279 				tp->t_flags |= TF_DELACK;  in bbr_process_data()
8280 				bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_data()
8282 				bbr->r_wanted_output = 1;  in bbr_process_data()
8283 				tp->t_flags |= TF_ACKNOW;  in bbr_process_data()
8285 			tp->rcv_nxt += tlen;  in bbr_process_data()
8287 			    ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&  in bbr_process_data()
8288 			    (tp->t_fbyte_in == 0)) {  in bbr_process_data()
8289 				tp->t_fbyte_in = ticks;  in bbr_process_data()
8290 				if (tp->t_fbyte_in == 0)  in bbr_process_data()
8291 					tp->t_fbyte_in = 1;  in bbr_process_data()
8292 				if (tp->t_fbyte_out && tp->t_fbyte_in)  in bbr_process_data()
8293 					tp->t_flags2 |= TF2_FBYTES_COMPLETE;  in bbr_process_data()
8299 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)  in bbr_process_data()
8305 					sbappendstream_locked(&so->so_rcv, m, 0);  in bbr_process_data()
8309 			if (so->so_rcv.sb_shlim && appended != mcnt)  in bbr_process_data()
8310 				counter_fo_release(so->so_rcv.sb_shlim,  in bbr_process_data()
8311 				    mcnt - appended);  in bbr_process_data()
8324 			tp->t_flags |= TF_ACKNOW;  in bbr_process_data()
8325 			if (tp->t_flags & TF_WAKESOR) {  in bbr_process_data()
8326 				tp->t_flags &= ~TF_WAKESOR;  in bbr_process_data()
8331 		if ((tp->t_flags & TF_SACK_PERMIT) &&  in bbr_process_data()
8333 		    TCPS_HAVEESTABLISHED(tp->t_state)) {  in bbr_process_data()
8341 			} else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {  in bbr_process_data()
8342 				if ((tp->rcv_numsacks >= 1) &&  in bbr_process_data()
8343 				    (tp->sackblks[0].end == save_start)) {  in bbr_process_data()
8349 					    tp->sackblks[0].start,  in bbr_process_data()
8350 					    tp->sackblks[0].end);  in bbr_process_data()
8374 		if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {  in bbr_process_data()
8378 			 * If connection is half-synchronized (ie NEEDSYN  in bbr_process_data()
8384 			if (tp->t_flags & TF_NEEDSYN) {  in bbr_process_data()
8385 				tp->t_flags |= TF_DELACK;  in bbr_process_data()
8387 				    __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_data()
8389 				tp->t_flags |= TF_ACKNOW;  in bbr_process_data()
8391 			tp->rcv_nxt++;  in bbr_process_data()
8393 		switch (tp->t_state) {  in bbr_process_data()
8399 			tp->t_starttime = ticks;  in bbr_process_data()
8415 			 * starting the time-wait timer, turning off the  in bbr_process_data()
8419 			bbr->rc_timer_first = 1;  in bbr_process_data()
8421 			    __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_process_data()
8429 	if ((tp->t_flags & TF_ACKNOW) ||  in bbr_process_data()
8430 	    (sbavail(&so->so_snd) > ctf_outstanding(tp))) {  in bbr_process_data()
8431 		bbr->r_wanted_output = 1;  in bbr_process_data()
8438  * have broken out the fast-data path also just like
8439  * the fast-ack. Return 1 if we processed the packet
8440  * return 0 if you need to take the "slow-path".
8455 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_fastnewdata()
8462 	if (bbr->r_ctl.rc_resend != NULL) {  in bbr_do_fastnewdata()
8465 	if (tiwin && tiwin != tp->snd_wnd) {  in bbr_do_fastnewdata()
8468 	if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {  in bbr_do_fastnewdata()
8471 	if (__predict_false((to->to_flags & TOF_TS) &&  in bbr_do_fastnewdata()
8472 	    (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {  in bbr_do_fastnewdata()
8475 	if (__predict_false((th->th_ack != tp->snd_una))) {  in bbr_do_fastnewdata()
8478 	if (__predict_false(tlen > sbspace(&so->so_rcv))) {  in bbr_do_fastnewdata()
8481 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_fastnewdata()
8482 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {  in bbr_do_fastnewdata()
8483 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_fastnewdata()
8484 		tp->ts_recent = to->to_tsval;  in bbr_do_fastnewdata()
8487 	 * This is a pure, in-sequence data packet with nothing on the  in bbr_do_fastnewdata()
8490 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_do_fastnewdata()
8493 	if (so->so_rcv.sb_shlim) {  in bbr_do_fastnewdata()
8496 		if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,  in bbr_do_fastnewdata()
8505 	if (tp->rcv_numsacks)  in bbr_do_fastnewdata()
8508 	tp->rcv_nxt += tlen;  in bbr_do_fastnewdata()
8510 	    ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&  in bbr_do_fastnewdata()
8511 	    (tp->t_fbyte_in == 0)) {  in bbr_do_fastnewdata()
8512 		tp->t_fbyte_in = ticks;  in bbr_do_fastnewdata()
8513 		if (tp->t_fbyte_in == 0)  in bbr_do_fastnewdata()
8514 			tp->t_fbyte_in = 1;  in bbr_do_fastnewdata()
8515 		if (tp->t_fbyte_out && tp->t_fbyte_in)  in bbr_do_fastnewdata()
8516 			tp->t_flags2 |= TF2_FBYTES_COMPLETE;  in bbr_do_fastnewdata()
8521 	tp->snd_wl1 = th->th_seq;  in bbr_do_fastnewdata()
8525 	tp->rcv_up = tp->rcv_nxt;  in bbr_do_fastnewdata()
8532 	if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_fastnewdata()
8541 				so->so_rcv.sb_flags &= ~SB_AUTOSIZE;  in bbr_do_fastnewdata()
8547 			sbappendstream_locked(&so->so_rcv, m, 0);  in bbr_do_fastnewdata()
8553 	if (so->so_rcv.sb_shlim && mcnt != appended)  in bbr_do_fastnewdata()
8554 		counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);  in bbr_do_fastnewdata()
8557 		bbr->bbr_segs_rcvd += max(1, nsegs);  in bbr_do_fastnewdata()
8558 		tp->t_flags |= TF_DELACK;  in bbr_do_fastnewdata()
8559 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_do_fastnewdata()
8561 		bbr->r_wanted_output = 1;  in bbr_do_fastnewdata()
8562 		tp->t_flags |= TF_ACKNOW;  in bbr_do_fastnewdata()
8569  * fast path. We again allow window updates that are
8570  * in sequence to remain in the fast-path. We also add
8574  * slow-path. If we return 1, then all is well and
8588 	if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {  in bbr_fastack()
8592 	if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_fastack()
8600 	if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {  in bbr_fastack()
8604 	if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {  in bbr_fastack()
8608 	if (__predict_false(IN_RECOVERY(tp->t_flags))) {  in bbr_fastack()
8612 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_fastack()
8613 	if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {  in bbr_fastack()
8617 	if (__predict_false(bbr->rc_in_persist != 0)) {  in bbr_fastack()
8621 	if (bbr->r_ctl.rc_sacked) {  in bbr_fastack()
8625 	/* Ok if we reach here, we can process a fast-ack */  in bbr_fastack()
8626 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_fastack()
8629 	 * We never detect loss in fast ack [we can't  in bbr_fastack()
8633 	bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);  in bbr_fastack()
8635 	if (tiwin != tp->snd_wnd) {  in bbr_fastack()
8636 		tp->snd_wnd = tiwin;  in bbr_fastack()
8637 		tp->snd_wl1 = th->th_seq;  in bbr_fastack()
8638 		if (tp->snd_wnd > tp->max_sndwnd)  in bbr_fastack()
8639 			tp->max_sndwnd = tp->snd_wnd;  in bbr_fastack()
8642 	if ((bbr->rc_in_persist != 0) &&  in bbr_fastack()
8643 	    (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),  in bbr_fastack()
8645 		bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_fastack()
8646 		bbr->r_wanted_output = 1;  in bbr_fastack()
8649 	if ((bbr->rc_in_persist == 0) &&  in bbr_fastack()
8650 	    (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_fastack()
8651 	    TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_fastack()
8652 	    (tp->snd_max == tp->snd_una) &&  in bbr_fastack()
8653 	    sbavail(&so->so_snd) &&  in bbr_fastack()
8654 	    (sbavail(&so->so_snd) > tp->snd_wnd)) {  in bbr_fastack()
8656 		bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_fastack()
8663 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_fastack()
8664 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {  in bbr_fastack()
8665 		tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;  in bbr_fastack()
8666 		tp->ts_recent = to->to_tsval;  in bbr_fastack()
8676 	if (tp->t_flags & TF_PREVVALID) {  in bbr_fastack()
8677 		tp->t_flags &= ~TF_PREVVALID;  in bbr_fastack()
8678 		if (tp->t_rxtshift == 1 &&  in bbr_fastack()
8679 		    (int)(ticks - tp->t_badrxtwin) < 0)  in bbr_fastack()
8698 	sbdrop(&so->so_snd, acked);  in bbr_fastack()
8700 	if (SEQ_GT(th->th_ack, tp->snd_una))  in bbr_fastack()
8702 	tp->snd_una = th->th_ack;  in bbr_fastack()
8703 	if (tp->snd_wnd < ctf_outstanding(tp))  in bbr_fastack()
8706 	else if (bbr->rc_has_collapsed)  in bbr_fastack()
8709 	if (SEQ_GT(tp->snd_una, tp->snd_recover)) {  in bbr_fastack()
8710 		tp->snd_recover = tp->snd_una;  in bbr_fastack()
8716 	tp->snd_wl2 = th->th_ack;  in bbr_fastack()
8720 	 * otherwise restart timer using current (possibly backed-off)  in bbr_fastack()
8727 	if (tp->snd_una == tp->snd_max) {  in bbr_fastack()
8730 		if (sbavail(&so->so_snd) == 0)  in bbr_fastack()
8731 			bbr->rc_tp->t_acktime = 0;  in bbr_fastack()
8732 		bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_fastack()
8733 		if (bbr->rc_in_persist == 0) {  in bbr_fastack()
8734 			bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;  in bbr_fastack()
8736 		sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_fastack()
8737 		bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);  in bbr_fastack()
8743 		bbr->r_wanted_output = 1;  in bbr_fastack()
8745 	if (sbavail(&so->so_snd)) {  in bbr_fastack()
8746 		bbr->r_wanted_output = 1;  in bbr_fastack()
8768 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_syn_sent()
8774 	 * this is an acceptable SYN segment initialize tp->rcv_nxt and  in bbr_do_syn_sent()
8775 	 * tp->irs if seg contains ack then advance tp->snd_una. BRR does  in bbr_do_syn_sent()
8782 	    (SEQ_LEQ(th->th_ack, tp->iss) ||  in bbr_do_syn_sent()
8783 	    SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_do_syn_sent()
8803 	tp->irs = th->th_seq;  in bbr_do_syn_sent()
8814 		if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==  in bbr_do_syn_sent()
8816 			tp->rcv_scale = tp->request_r_scale;  in bbr_do_syn_sent()
8818 		tp->rcv_adv += min(tp->rcv_wnd,  in bbr_do_syn_sent()
8819 		    TCP_MAXWIN << tp->rcv_scale);  in bbr_do_syn_sent()
8824 		if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_do_syn_sent()
8825 		    (tp->snd_una != tp->snd_max)) {  in bbr_do_syn_sent()
8826 			tp->snd_nxt = th->th_ack;  in bbr_do_syn_sent()
8834 			bbr->bbr_segs_rcvd += 1;  in bbr_do_syn_sent()
8835 			tp->t_flags |= TF_DELACK;  in bbr_do_syn_sent()
8836 			bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);  in bbr_do_syn_sent()
8838 			bbr->r_wanted_output = 1;  in bbr_do_syn_sent()
8839 			tp->t_flags |= TF_ACKNOW;  in bbr_do_syn_sent()
8841 		if (SEQ_GT(th->th_ack, tp->iss)) {  in bbr_do_syn_sent()
8848 		if (SEQ_GT(th->th_ack, tp->snd_una)) {  in bbr_do_syn_sent()
8851 			 * fast open case. If th_ack is  in bbr_do_syn_sent()
8854 			 * ack-processing since the  in bbr_do_syn_sent()
8855 			 * data stream in our send-map  in bbr_do_syn_sent()
8861 			tp->snd_una++;  in bbr_do_syn_sent()
8865 		 * SYN_SENT  --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1  in bbr_do_syn_sent()
8867 		tp->t_starttime = ticks;  in bbr_do_syn_sent()
8868 		if (tp->t_flags & TF_NEEDFIN) {  in bbr_do_syn_sent()
8870 			tp->t_flags &= ~TF_NEEDFIN;  in bbr_do_syn_sent()
8880 		 * Received initial SYN in SYN-SENT[*] state => simultaneous  in bbr_do_syn_sent()
8883 		 * half-synchronized. Otherwise, do 3-way handshake:  in bbr_do_syn_sent()
8884 		 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If  in bbr_do_syn_sent()
8887 		tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN);  in bbr_do_syn_sent()
8891 	 * Advance th->th_seq to correspond to first data byte. If data,  in bbr_do_syn_sent()
8894 	th->th_seq++;  in bbr_do_syn_sent()
8895 	if (tlen > tp->rcv_wnd) {  in bbr_do_syn_sent()
8896 		todrop = tlen - tp->rcv_wnd;  in bbr_do_syn_sent()
8897 		m_adj(m, -todrop);  in bbr_do_syn_sent()
8898 		tlen = tp->rcv_wnd;  in bbr_do_syn_sent()
8903 	tp->snd_wl1 = th->th_seq - 1;  in bbr_do_syn_sent()
8904 	tp->rcv_up = th->th_seq;  in bbr_do_syn_sent()
8912 		if ((to->to_flags & TOF_TS) != 0) {  in bbr_do_syn_sent()
8915 			t = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_syn_sent()
8916 			if (TSTMP_GEQ(t, to->to_tsecr)) {  in bbr_do_syn_sent()
8917 				rtt = t - to->to_tsecr;  in bbr_do_syn_sent()
8923 				apply_filter_min_small(&bbr->r_ctl.rc_rttprop,  in bbr_do_syn_sent()
8924 						       rtt, bbr->r_ctl.rc_rcvtime);  in bbr_do_syn_sent()
8930 		if (tp->t_state == TCPS_FIN_WAIT_1) {  in bbr_do_syn_sent()
8947 				if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_syn_sent()
8978 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_syn_recv()
8981 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_syn_recv()
8984 	    (SEQ_LEQ(th->th_ack, tp->snd_una) ||  in bbr_do_syn_recv()
8985 	     SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_do_syn_recv()
8990 	if (tp->t_flags & TF_FASTOPEN) {  in bbr_do_syn_recv()
9002 			/* non-initial SYN is ignored */  in bbr_do_syn_recv()
9003 			if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||  in bbr_do_syn_recv()
9004 			    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||  in bbr_do_syn_recv()
9005 			    (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {  in bbr_do_syn_recv()
9018 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_syn_recv()
9019 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_syn_recv()
9024 	 * In the SYN-RECEIVED state, validate that the packet belongs to  in bbr_do_syn_recv()
9030 	if (SEQ_LT(th->th_seq, tp->irs)) {  in bbr_do_syn_recv()
9052 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_syn_recv()
9053 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_syn_recv()
9054 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_syn_recv()
9056 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_syn_recv()
9057 		tp->ts_recent = to->to_tsval;  in bbr_do_syn_recv()
9059 	tp->snd_wnd = tiwin;  in bbr_do_syn_recv()
9061 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_syn_recv()
9062 	 * is on (half-synchronized state), then queue data for later  in bbr_do_syn_recv()
9066 		if (tp->t_flags & TF_FASTOPEN) {  in bbr_do_syn_recv()
9073 	if (tp->t_flags & TF_SONOTCONN) {  in bbr_do_syn_recv()
9074 		tp->t_flags &= ~TF_SONOTCONN;  in bbr_do_syn_recv()
9078 	if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==  in bbr_do_syn_recv()
9080 		tp->rcv_scale = tp->request_r_scale;  in bbr_do_syn_recv()
9086 	if ((to->to_flags & TOF_TS) != 0) {  in bbr_do_syn_recv()
9089 		t = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_syn_recv()
9090 		if (TSTMP_GEQ(t, to->to_tsecr)) {  in bbr_do_syn_recv()
9091 			rtt = t - to->to_tsecr;  in bbr_do_syn_recv()
9097 			apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);  in bbr_do_syn_recv()
9103 	if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) {  in bbr_do_syn_recv()
9104 		tcp_fastopen_decrement_counter(tp->t_tfo_pending);  in bbr_do_syn_recv()
9105 		tp->t_tfo_pending = NULL;  in bbr_do_syn_recv()
9108 	 * Make transitions: SYN-RECEIVED  -> ESTABLISHED SYN-RECEIVED* ->  in bbr_do_syn_recv()
9109 	 * FIN-WAIT-1  in bbr_do_syn_recv()
9111 	tp->t_starttime = ticks;  in bbr_do_syn_recv()
9112 	if (tp->t_flags & TF_NEEDFIN) {  in bbr_do_syn_recv()
9114 		tp->t_flags &= ~TF_NEEDFIN;  in bbr_do_syn_recv()
9125 		if (!(tp->t_flags & TF_FASTOPEN))  in bbr_do_syn_recv()
9133 	if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))  in bbr_do_syn_recv()
9134 		tp->snd_una++;  in bbr_do_syn_recv()
9142 		if (tp->t_flags & TF_WAKESOR) {  in bbr_do_syn_recv()
9143 			tp->t_flags &= ~TF_WAKESOR;  in bbr_do_syn_recv()
9148 	tp->snd_wl1 = th->th_seq - 1;  in bbr_do_syn_recv()
9152 	if (tp->t_state == TCPS_FIN_WAIT_1) {  in bbr_do_syn_recv()
9169 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_syn_recv()
9200 	 * uni-directional data xfer.  If the packet has no control flags,  in bbr_do_established()
9201 	 * is in-sequence, the window didn't change and we're not  in bbr_do_established()
9205 	 * waiting for space.  If the length is non-zero and the ack didn't  in bbr_do_established()
9206 	 * move, we're the receiver side.  If we're getting packets in-order  in bbr_do_established()
9209 	 * hidden state-flags are also off. Since we check for  in bbr_do_established()
9212 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_established()
9213 	if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {  in bbr_do_established()
9219 		bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);  in bbr_do_established()
9221 	if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&  in bbr_do_established()
9224 	    __predict_true(th->th_seq == tp->rcv_nxt)) {  in bbr_do_established()
9240 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_established()
9254 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_established()
9255 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_established()
9276 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_established()
9277 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_established()
9278 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_established()
9280 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_established()
9281 		tp->ts_recent = to->to_tsval;  in bbr_do_established()
9284 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_established()
9285 	 * is on (half-synchronized state), then queue data for later  in bbr_do_established()
9289 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_established()
9292 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_established()
9294 			bbr->r_wanted_output = 1;  in bbr_do_established()
9307 	if (sbavail(&so->so_snd)) {  in bbr_do_established()
9334 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_close_wait()
9337 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_close_wait()
9351 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_close_wait()
9352 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_close_wait()
9373 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_close_wait()
9374 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_close_wait()
9375 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_close_wait()
9377 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_close_wait()
9378 		tp->ts_recent = to->to_tsval;  in bbr_do_close_wait()
9381 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_close_wait()
9382 	 * is on (half-synchronized state), then queue data for later  in bbr_do_close_wait()
9386 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_close_wait()
9389 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_close_wait()
9391 			bbr->r_wanted_output = 1;  in bbr_do_close_wait()
9404 	if (sbavail(&so->so_snd)) {  in bbr_do_close_wait()
9420 	if (bbr->rc_allow_data_af_clo == 0) {  in bbr_check_data_after_close()
9423 		/* tcp_close will kill the inp pre-log the Reset */  in bbr_check_data_after_close()
9430 	if (sbavail(&so->so_snd) == 0)  in bbr_check_data_after_close()
9433 	tp->rcv_nxt = th->th_seq + *tlen;  in bbr_check_data_after_close()
9434 	tp->t_flags2 |= TF2_DROP_AF_DATA;  in bbr_check_data_after_close()
9435 	bbr->r_wanted_output = 1;  in bbr_check_data_after_close()
9456 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_fin_wait_1()
9459 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_fin_wait_1()
9473 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_fin_wait_1()
9474 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_fin_wait_1()
9487 	if ((tp->t_flags & TF_CLOSED) && tlen &&  in bbr_do_fin_wait_1()
9504 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_fin_wait_1()
9505 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_fin_wait_1()
9506 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_fin_wait_1()
9508 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_fin_wait_1()
9509 		tp->ts_recent = to->to_tsval;  in bbr_do_fin_wait_1()
9512 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_fin_wait_1()
9513 	 * is on (half-synchronized state), then queue data for later  in bbr_do_fin_wait_1()
9517 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_fin_wait_1()
9520 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_fin_wait_1()
9522 			bbr->r_wanted_output = 1;  in bbr_do_fin_wait_1()
9545 		if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {  in bbr_do_fin_wait_1()
9554 	if (sbavail(&so->so_snd)) {  in bbr_do_fin_wait_1()
9581 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_closing()
9584 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_closing()
9598 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_closing()
9599 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_closing()
9620 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_closing()
9621 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_closing()
9622 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_closing()
9624 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_closing()
9625 		tp->ts_recent = to->to_tsval;  in bbr_do_closing()
9628 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_closing()
9629 	 * is on (half-synchronized state), then queue data for later  in bbr_do_closing()
9633 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_closing()
9636 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_closing()
9638 			bbr->r_wanted_output = 1;  in bbr_do_closing()
9656 	if (sbavail(&so->so_snd)) {  in bbr_do_closing()
9683 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_lastack()
9686 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_lastack()
9700 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_lastack()
9701 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_lastack()
9722 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_lastack()
9723 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_lastack()
9724 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_lastack()
9726 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_lastack()
9727 		tp->ts_recent = to->to_tsval;  in bbr_do_lastack()
9730 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_lastack()
9731 	 * is on (half-synchronized state), then queue data for later  in bbr_do_lastack()
9735 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_lastack()
9738 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_lastack()
9740 			bbr->r_wanted_output = 1;  in bbr_do_lastack()
9758 	if (sbavail(&so->so_snd)) {  in bbr_do_lastack()
9785 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_fin_wait_2()
9789 	    (tp->t_fin_is_rst && (thflags & TH_FIN)))  in bbr_do_fin_wait_2()
9804 	if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&  in bbr_do_fin_wait_2()
9805 	    TSTMP_LT(to->to_tsval, tp->ts_recent)) {  in bbr_do_fin_wait_2()
9819 	if ((tp->t_flags & TF_CLOSED) && tlen &&  in bbr_do_fin_wait_2()
9836 	if ((to->to_flags & TOF_TS) != 0 &&  in bbr_do_fin_wait_2()
9837 	    SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&  in bbr_do_fin_wait_2()
9838 	    SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +  in bbr_do_fin_wait_2()
9840 		tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_fin_wait_2()
9841 		tp->ts_recent = to->to_tsval;  in bbr_do_fin_wait_2()
9844 	 * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag  in bbr_do_fin_wait_2()
9845 	 * is on (half-synchronized state), then queue data for later  in bbr_do_fin_wait_2()
9849 		if (tp->t_flags & TF_NEEDSYN) {  in bbr_do_fin_wait_2()
9852 		} else if (tp->t_flags & TF_ACKNOW) {  in bbr_do_fin_wait_2()
9854 			bbr->r_wanted_output = 1;  in bbr_do_fin_wait_2()
9867 	if (sbavail(&so->so_snd)) {  in bbr_do_fin_wait_2()
9886 		bbr->rc_in_persist = 1;  in bbr_stop_all_timers()
9888 	if (tcp_in_hpts(bbr->rc_tp)) {  in bbr_stop_all_timers()
9889 		tcp_hpts_remove(bbr->rc_tp);  in bbr_stop_all_timers()
9896 	bbr->rc_use_google = 1;  in bbr_google_mode_on()
9897 	bbr->rc_no_pacing = 0;  in bbr_google_mode_on()
9898 	bbr->r_ctl.bbr_google_discount = bbr_google_discount;  in bbr_google_mode_on()
9899 	bbr->r_use_policer = bbr_policer_detection_enabled;  in bbr_google_mode_on()
9900 	bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);  in bbr_google_mode_on()
9901 	bbr->bbr_use_rack_cheat = 0;  in bbr_google_mode_on()
9902 	bbr->r_ctl.rc_incr_tmrs = 0;  in bbr_google_mode_on()
9903 	bbr->r_ctl.rc_inc_tcp_oh = 0;  in bbr_google_mode_on()
9904 	bbr->r_ctl.rc_inc_ip_oh = 0;  in bbr_google_mode_on()
9905 	bbr->r_ctl.rc_inc_enet_oh = 0;  in bbr_google_mode_on()
9906 	reset_time(&bbr->r_ctl.rc_delrate,  in bbr_google_mode_on()
9908 	reset_time_small(&bbr->r_ctl.rc_rttprop,  in bbr_google_mode_on()
9910 	tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));  in bbr_google_mode_on()
9916 	bbr->rc_use_google = 0;  in bbr_google_mode_off()
9917 	bbr->r_ctl.bbr_google_discount = 0;  in bbr_google_mode_off()
9918 	bbr->no_pacing_until = bbr_no_pacing_until;  in bbr_google_mode_off()
9919 	bbr->r_use_policer = 0;  in bbr_google_mode_off()
9920 	if (bbr->no_pacing_until)  in bbr_google_mode_off()
9921 		bbr->rc_no_pacing = 1;  in bbr_google_mode_off()
9923 		bbr->rc_no_pacing = 0;  in bbr_google_mode_off()
9925 		bbr->bbr_use_rack_cheat = 1;  in bbr_google_mode_off()
9927 		bbr->bbr_use_rack_cheat = 0;  in bbr_google_mode_off()
9929 		bbr->r_ctl.rc_incr_tmrs = 1;  in bbr_google_mode_off()
9931 		bbr->r_ctl.rc_incr_tmrs = 0;  in bbr_google_mode_off()
9933 		bbr->r_ctl.rc_inc_tcp_oh = 1;  in bbr_google_mode_off()
9935 		bbr->r_ctl.rc_inc_tcp_oh = 0;  in bbr_google_mode_off()
9937 		bbr->r_ctl.rc_inc_ip_oh = 1;  in bbr_google_mode_off()
9939 		bbr->r_ctl.rc_inc_ip_oh = 0;  in bbr_google_mode_off()
9941 		bbr->r_ctl.rc_inc_enet_oh = 1;  in bbr_google_mode_off()
9943 		bbr->r_ctl.rc_inc_enet_oh = 0;  in bbr_google_mode_off()
9944 	bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;  in bbr_google_mode_off()
9945 	reset_time(&bbr->r_ctl.rc_delrate,  in bbr_google_mode_off()
9947 	reset_time_small(&bbr->r_ctl.rc_rttprop,  in bbr_google_mode_off()
9949 	tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));  in bbr_google_mode_off()
9952  * Return 0 on success, non-zero on failure
9975 	bbr->rtt_valid = 0;  in bbr_init()
9976 	tp->t_flags2 |= TF2_CANNOT_DO_ECN;  in bbr_init()
9977 	tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;  in bbr_init()
9979 	tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;  in bbr_init()
9980 	tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;  in bbr_init()
9981 	tp->t_flags2 &= ~TF2_MBUF_ACKCMP;  in bbr_init()
9982 	tp->t_flags2 &= ~TF2_MBUF_L_ACKS;  in bbr_init()
9984 	TAILQ_INIT(&bbr->r_ctl.rc_map);  in bbr_init()
9985 	TAILQ_INIT(&bbr->r_ctl.rc_free);  in bbr_init()
9986 	TAILQ_INIT(&bbr->r_ctl.rc_tmap);  in bbr_init()
9987 	bbr->rc_tp = tp;  in bbr_init()
9988 	bbr->rc_inp = inp;  in bbr_init()
9989 	cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_init()
9990 	tp->t_acktime = 0;  in bbr_init()
9991 	bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;  in bbr_init()
9992 	bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;  in bbr_init()
9993 	bbr->rc_tlp_threshold = bbr_tlp_thresh;  in bbr_init()
9994 	bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;  in bbr_init()
9995 	bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;  in bbr_init()
9996 	bbr->r_ctl.rc_min_to = bbr_min_to;  in bbr_init()
9997 	bbr->rc_bbr_state = BBR_STATE_STARTUP;  in bbr_init()
9998 	bbr->r_ctl.bbr_lost_at_state = 0;  in bbr_init()
9999 	bbr->r_ctl.rc_lost_at_startup = 0;  in bbr_init()
10000 	bbr->rc_all_timers_stopped = 0;  in bbr_init()
10001 	bbr->r_ctl.rc_bbr_lastbtlbw = 0;  in bbr_init()
10002 	bbr->r_ctl.rc_pkt_epoch_del = 0;  in bbr_init()
10003 	bbr->r_ctl.rc_pkt_epoch = 0;  in bbr_init()
10004 	bbr->r_ctl.rc_lowest_rtt = 0xffffffff;  in bbr_init()
10005 	bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;  in bbr_init()
10006 	bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;  in bbr_init()
10007 	bbr->r_ctl.rc_went_idle_time = cts;  in bbr_init()
10008 	bbr->rc_pacer_started = cts;  in bbr_init()
10009 	bbr->r_ctl.rc_pkt_epoch_time = cts;  in bbr_init()
10010 	bbr->r_ctl.rc_rcvtime = cts;  in bbr_init()
10011 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_init()
10012 	bbr->r_ctl.rc_del_time = cts;  in bbr_init()
10013 	bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_init()
10014 	bbr->r_ctl.last_in_probertt = cts;  in bbr_init()
10015 	bbr->skip_gain = 0;  in bbr_init()
10016 	bbr->gain_is_limited = 0;  in bbr_init()
10017 	bbr->no_pacing_until = bbr_no_pacing_until;  in bbr_init()
10018 	if (bbr->no_pacing_until)  in bbr_init()
10019 		bbr->rc_no_pacing = 1;  in bbr_init()
10021 		bbr->rc_no_pacing = 0;  in bbr_init()
10022 		bbr->rc_use_google = 1;  in bbr_init()
10023 		bbr->r_ctl.bbr_google_discount = bbr_google_discount;  in bbr_init()
10024 		bbr->r_use_policer = bbr_policer_detection_enabled;  in bbr_init()
10026 		bbr->rc_use_google = 0;  in bbr_init()
10027 		bbr->r_ctl.bbr_google_discount = 0;  in bbr_init()
10028 		bbr->r_use_policer = 0;  in bbr_init()
10031 		bbr->rc_use_ts_limit = 1;  in bbr_init()
10033 		bbr->rc_use_ts_limit = 0;  in bbr_init()
10035 		bbr->ts_can_raise = 1;  in bbr_init()
10037 		bbr->ts_can_raise = 0;  in bbr_init()
10039 		tp->t_delayed_ack = 2;  in bbr_init()
10041 		tp->t_delayed_ack = 0;  in bbr_init()
10043 		tp->t_delayed_ack = V_tcp_delack_enabled;  in bbr_init()
10045 		tp->t_delayed_ack = 2;  in bbr_init()
10046 	if (bbr->rc_use_google == 0)  in bbr_init()
10047 		bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;  in bbr_init()
10049 		bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);  in bbr_init()
10050 	bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;  in bbr_init()
10051 	bbr->rc_max_rto_sec = bbr_rto_max_sec;  in bbr_init()
10052 	bbr->rc_init_win = bbr_def_init_win;  in bbr_init()
10053 	if (tp->t_flags & TF_REQ_TSTMP)  in bbr_init()
10054 		bbr->rc_last_options = TCP_TS_OVERHEAD;  in bbr_init()
10055 	bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;  in bbr_init()
10056 	bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;  in bbr_init()
10057 	bbr->r_init_rtt = 1;  in bbr_init()
10061 		bbr->bbr_hdw_pace_ena = 1;  in bbr_init()
10063 		bbr->bbr_hdw_pace_ena = 0;  in bbr_init()
10065 		bbr->bbr_init_win_cheat = 1;  in bbr_init()
10067 		bbr->bbr_init_win_cheat = 0;  in bbr_init()
10068 	bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;  in bbr_init()
10069 	bbr->r_ctl.rc_drain_pg = bbr_drain_gain;  in bbr_init()
10070 	bbr->r_ctl.rc_startup_pg = bbr_high_gain;  in bbr_init()
10071 	bbr->rc_loss_exit = bbr_exit_startup_at_loss;  in bbr_init()
10072 	bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;  in bbr_init()
10073 	bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;  in bbr_init()
10074 	bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;  in bbr_init()
10075 	bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;  in bbr_init()
10076 	bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;  in bbr_init()
10077 	bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;  in bbr_init()
10078 	bbr->r_ctl.bbr_cross_over = bbr_cross_over;  in bbr_init()
10079 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_init()
10080 	if (bbr->rc_use_google) {  in bbr_init()
10081 		setup_time_filter(&bbr->r_ctl.rc_delrate,  in bbr_init()
10084 		setup_time_filter_small(&bbr->r_ctl.rc_rttprop,  in bbr_init()
10087 		setup_time_filter(&bbr->r_ctl.rc_delrate,  in bbr_init()
10090 		setup_time_filter_small(&bbr->r_ctl.rc_rttprop,  in bbr_init()
10095 		bbr->rc_use_idle_restart = 1;  in bbr_init()
10097 		bbr->rc_use_idle_restart = 0;  in bbr_init()
10098 	bbr->r_ctl.rc_bbr_cur_del_rate = 0;  in bbr_init()
10099 	bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;  in bbr_init()
10101 		bbr->rc_resends_use_tso = 1;  in bbr_init()
10102 	if (tp->snd_una != tp->snd_max) {  in bbr_init()
10112 		rsm->r_rtt_not_allowed = 1;  in bbr_init()
10113 		rsm->r_tim_lastsent[0] = cts;  in bbr_init()
10114 		rsm->r_rtr_cnt = 1;  in bbr_init()
10115 		rsm->r_rtr_bytes = 0;  in bbr_init()
10116 		rsm->r_start = tp->snd_una;  in bbr_init()
10117 		rsm->r_end = tp->snd_max;  in bbr_init()
10118 		rsm->r_dupack = 0;  in bbr_init()
10119 		rsm->r_delivered = bbr->r_ctl.rc_delivered;  in bbr_init()
10120 		rsm->r_ts_valid = 0;  in bbr_init()
10121 		rsm->r_del_ack_ts = tp->ts_recent;  in bbr_init()
10122 		rsm->r_del_time = cts;  in bbr_init()
10123 		if (bbr->r_ctl.r_app_limited_until)  in bbr_init()
10124 			rsm->r_app_limited = 1;  in bbr_init()
10126 			rsm->r_app_limited = 0;  in bbr_init()
10127 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_init()
10128 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);  in bbr_init()
10129 		rsm->r_in_tmap = 1;  in bbr_init()
10130 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)  in bbr_init()
10131 			rsm->r_bbr_state = bbr_state_val(bbr);  in bbr_init()
10133 			rsm->r_bbr_state = 8;  in bbr_init()
10135 	if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))  in bbr_init()
10136 		bbr->bbr_use_rack_cheat = 1;  in bbr_init()
10137 	if (bbr_incr_timers && (bbr->rc_use_google == 0))  in bbr_init()
10138 		bbr->r_ctl.rc_incr_tmrs = 1;  in bbr_init()
10139 	if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))  in bbr_init()
10140 		bbr->r_ctl.rc_inc_tcp_oh = 1;  in bbr_init()
10141 	if (bbr_include_ip_oh && (bbr->rc_use_google == 0))  in bbr_init()
10142 		bbr->r_ctl.rc_inc_ip_oh = 1;  in bbr_init()
10143 	if (bbr_include_enet_oh && (bbr->rc_use_google == 0))  in bbr_init()
10144 		bbr->r_ctl.rc_inc_enet_oh = 1;  in bbr_init()
10147 	if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_init()
10148 	    (tp->t_srtt)) {  in bbr_init()
10151 		rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);  in bbr_init()
10152 		apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);  in bbr_init()
10170 	TCPT_RANGESET(tp->t_rxtcur,  in bbr_init()
10171 	    ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,  in bbr_init()
10172 	    tp->t_rttmin, TCPTV_REXMTMAX);  in bbr_init()
10179  * non-zero if we can't handle the connection. A EAGAIN
10187 	if ((tp->t_state == TCPS_CLOSED) ||  in bbr_handoff_ok()
10188 	    (tp->t_state == TCPS_LISTEN)) {  in bbr_handoff_ok()
10192 	if ((tp->t_state == TCPS_SYN_SENT) ||  in bbr_handoff_ok()
10193 	    (tp->t_state == TCPS_SYN_RECEIVED)) {  in bbr_handoff_ok()
10200 	if (tp->t_flags & TF_SENTFIN)  in bbr_handoff_ok()
10202 	if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) {  in bbr_handoff_ok()
10215 	if (tp->t_fb_ptr) {  in bbr_fini()
10220 		bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_fini()
10221 		if (bbr->r_ctl.crte)  in bbr_fini()
10222 			tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);  in bbr_fini()
10224 		bbr->rc_tp = NULL;  in bbr_fini()
10225 		if (bbr->bbr_hdrw_pacing)  in bbr_fini()
10226 			counter_u64_add(bbr_flows_whdwr_pacing, -1);  in bbr_fini()
10228 			counter_u64_add(bbr_flows_nohdwr_pacing, -1);  in bbr_fini()
10229 		if (bbr->r_ctl.crte != NULL) {  in bbr_fini()
10230 			tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);  in bbr_fini()
10231 			bbr->r_ctl.crte = NULL;  in bbr_fini()
10233 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_fini()
10235 			TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);  in bbr_fini()
10237 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_fini()
10239 		rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);  in bbr_fini()
10241 			TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_fini()
10243 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);  in bbr_fini()
10245 		calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;  in bbr_fini()
10246 		if (calc > (bbr->r_ctl.rc_init_rwnd / 10))  in bbr_fini()
10250 		bbr->r_ctl.rc_free_cnt = 0;  in bbr_fini()
10251 		uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);  in bbr_fini()
10252 		tp->t_fb_ptr = NULL;  in bbr_fini()
10255 	tp->snd_nxt = tp->snd_max;  in bbr_fini()
10261 	switch (tp->t_state) {  in bbr_set_state()
10263 		bbr->r_state = TCPS_SYN_SENT;  in bbr_set_state()
10264 		bbr->r_substate = bbr_do_syn_sent;  in bbr_set_state()
10267 		bbr->r_state = TCPS_SYN_RECEIVED;  in bbr_set_state()
10268 		bbr->r_substate = bbr_do_syn_recv;  in bbr_set_state()
10271 		bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);  in bbr_set_state()
10272 		bbr->r_state = TCPS_ESTABLISHED;  in bbr_set_state()
10273 		bbr->r_substate = bbr_do_established;  in bbr_set_state()
10276 		bbr->r_state = TCPS_CLOSE_WAIT;  in bbr_set_state()
10277 		bbr->r_substate = bbr_do_close_wait;  in bbr_set_state()
10280 		bbr->r_state = TCPS_FIN_WAIT_1;  in bbr_set_state()
10281 		bbr->r_substate = bbr_do_fin_wait_1;  in bbr_set_state()
10284 		bbr->r_state = TCPS_CLOSING;  in bbr_set_state()
10285 		bbr->r_substate = bbr_do_closing;  in bbr_set_state()
10288 		bbr->r_state = TCPS_LAST_ACK;  in bbr_set_state()
10289 		bbr->r_substate = bbr_do_lastack;  in bbr_set_state()
10292 		bbr->r_state = TCPS_FIN_WAIT_2;  in bbr_set_state()
10293 		bbr->r_substate = bbr_do_fin_wait_2;  in bbr_set_state()
10314 		/* Save the lowest srtt we saw in our end of the sub-state */  in bbr_substate_change()
10315 		bbr->rc_hit_state_1 = 0;  in bbr_substate_change()
10316 		if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)  in bbr_substate_change()
10317 			bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;  in bbr_substate_change()
10319 	bbr->rc_bbr_substate++;  in bbr_substate_change()
10320 	if (bbr->rc_bbr_substate >= BBR_SUBSTATE_COUNT) {  in bbr_substate_change()
10321 		/* Cycle back to first state-> gain */  in bbr_substate_change()
10322 		bbr->rc_bbr_substate = 0;  in bbr_substate_change()
10329 		if (bbr->skip_gain) {  in bbr_substate_change()
10335 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];  in bbr_substate_change()
10336 		} else if (bbr->gain_is_limited &&  in bbr_substate_change()
10337 			   bbr->bbr_hdrw_pacing &&  in bbr_substate_change()
10338 			   bbr->r_ctl.crte) {  in bbr_substate_change()
10348 			rate = bbr->r_ctl.crte->rate;  in bbr_substate_change()
10354 				bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;  in bbr_substate_change()
10356 				bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];  in bbr_substate_change()
10359 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];  in bbr_substate_change()
10360 		if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {  in bbr_substate_change()
10361 			bbr->r_ctl.rc_bbr_state_atflight = cts;  in bbr_substate_change()
10363 			bbr->r_ctl.rc_bbr_state_atflight = 0;  in bbr_substate_change()
10365 		bbr->rc_hit_state_1 = 1;  in bbr_substate_change()
10366 		bbr->r_ctl.rc_exta_time_gd = 0;  in bbr_substate_change()
10367 		bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,  in bbr_substate_change()
10368 						     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_substate_change()
10370 			bbr->r_ctl.rc_bbr_state_atflight = 0;  in bbr_substate_change()
10372 			bbr->r_ctl.rc_bbr_state_atflight = cts;  in bbr_substate_change()
10373 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];  in bbr_substate_change()
10375 		/* All other cycles hit here 2-7 */  in bbr_substate_change()
10376 		if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {  in bbr_substate_change()
10378 			    (bbr->rc_use_google == 0) &&  in bbr_substate_change()
10379 			    (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {  in bbr_substate_change()
10380 				bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_substate_change()
10383 			if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))  in bbr_substate_change()
10384 				bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -  in bbr_substate_change()
10387 				bbr->r_ctl.rc_exta_time_gd = 0;  in bbr_substate_change()
10388 			if (bbr->r_ctl.rc_exta_time_gd) {  in bbr_substate_change()
10389 				bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;  in bbr_substate_change()
10391 				bbr->r_ctl.rc_level_state_extra /= 7;  in bbr_substate_change()
10392 				if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {  in bbr_substate_change()
10398 		bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);  in bbr_substate_change()
10399 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];  in bbr_substate_change()
10401 	if (bbr->rc_use_google) {  in bbr_substate_change()
10402 		bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);  in bbr_substate_change()
10404 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_substate_change()
10405 	bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;  in bbr_substate_change()
10409 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_substate_change()
10412 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_substate_change()
10413 		if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_substate_change()
10416 			counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_substate_change()
10419 	bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;  in bbr_substate_change()
10422 	    (bbr->rc_use_google == 0) &&  in bbr_substate_change()
10425 		bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_substate_change()
10426 		bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_substate_change()
10429 			bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +  in bbr_substate_change()
10430 							  ctf_flight_size(bbr->rc_tp,  in bbr_substate_change()
10431 							      (bbr->r_ctl.rc_sacked +  in bbr_substate_change()
10432 							       bbr->r_ctl.rc_lost_bytes)));  in bbr_substate_change()
10436 	if (bbr->rc_lt_use_bw) {  in bbr_substate_change()
10438 		bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_substate_change()
10441 	if (bbr->rc_use_google)  in bbr_substate_change()
10443 	bbr->r_ctl.gain_epoch = cts;  in bbr_substate_change()
10444 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_substate_change()
10445 	bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;  in bbr_substate_change()
10453 	    (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {  in bbr_set_probebw_google_gains()
10457 	if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_set_probebw_google_gains()
10460 	if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {  in bbr_set_probebw_google_gains()
10473 		 *    we stay in GAIN (gain-to-target).  in bbr_set_probebw_google_gains()
10477 		if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {  in bbr_set_probebw_google_gains()
10491 	if (bbr->rc_use_google) {  in bbr_set_probebw_gains()
10508 	if (bbr->r_ctl.rc_bbr_state_atflight == 0) {  in bbr_set_probebw_gains()
10510 			flight = ctf_flight_size(bbr->rc_tp,  in bbr_set_probebw_gains()
10511 				     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_set_probebw_gains()
10512 			if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {  in bbr_set_probebw_gains()
10514 				if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {  in bbr_set_probebw_gains()
10515 					bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_set_probebw_gains()
10520 					bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);  in bbr_set_probebw_gains()
10523 			if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&  in bbr_set_probebw_gains()
10524 			    (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||  in bbr_set_probebw_gains()
10525 			     (flight >= bbr->r_ctl.flightsize_at_drain))) {  in bbr_set_probebw_gains()
10533 				bbr->r_ctl.flightsize_at_drain = flight;  in bbr_set_probebw_gains()
10538 					bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;  in bbr_set_probebw_gains()
10539 					bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;  in bbr_set_probebw_gains()
10542 					bbr->r_ctl.rc_bbr_hptsi_gain *= 4;  in bbr_set_probebw_gains()
10543 					bbr->r_ctl.rc_bbr_hptsi_gain /= 5;  in bbr_set_probebw_gains()
10545 				if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {  in bbr_set_probebw_gains()
10547 					bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);  in bbr_set_probebw_gains()
10554 				bbr->r_ctl.gain_epoch = cts;  in bbr_set_probebw_gains()
10556 			if (flight <= bbr->r_ctl.rc_target_at_state) {  in bbr_set_probebw_gains()
10558 				    (bbr->rc_use_google == 0) &&  in bbr_set_probebw_gains()
10559 				    (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {  in bbr_set_probebw_gains()
10560 					bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_set_probebw_gains()
10563 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);  in bbr_set_probebw_gains()
10568 			if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {  in bbr_set_probebw_gains()
10569 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);  in bbr_set_probebw_gains()
10572 			if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||  in bbr_set_probebw_gains()
10573 			    ((ctf_outstanding(bbr->rc_tp) +  bbr->rc_tp->t_maxseg - 1) >=  in bbr_set_probebw_gains()
10574 			     bbr->rc_tp->snd_wnd)) {  in bbr_set_probebw_gains()
10575 				bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);  in bbr_set_probebw_gains()
10591 	if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))  in bbr_set_probebw_gains()
10593 	if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {  in bbr_set_probebw_gains()
10594 		/* Less than a full time-period has passed */  in bbr_set_probebw_gains()
10597 	if (bbr->r_ctl.rc_level_state_extra &&  in bbr_set_probebw_gains()
10599 	    ((cts - bbr->r_ctl.rc_bbr_state_time) <  in bbr_set_probebw_gains()
10600 	     (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {  in bbr_set_probebw_gains()
10601 		/* Less than a full time-period + extra has passed */  in bbr_set_probebw_gains()
10605 	    bbr->r_ctl.rc_level_state_extra &&  in bbr_set_probebw_gains()
10607 	    ((cts - bbr->r_ctl.rc_bbr_state_time) <  in bbr_set_probebw_gains()
10608 	     (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {  in bbr_set_probebw_gains()
10609 		/* Less than a full time-period + extra has passed */  in bbr_set_probebw_gains()
10620 	if (bbr->rc_use_google) {  in bbr_get_a_state_target()
10624 		mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),  in bbr_get_a_state_target()
10625 			  bbr->r_ctl.rc_pace_max_segs);  in bbr_get_a_state_target()
10641 	if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&  in bbr_set_state_target()
10642 	    ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {  in bbr_set_state_target()
10643 		/* Special case using old probe-rtt method */  in bbr_set_state_target()
10644 		tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_set_state_target()
10647 		/* Non-probe-rtt case and reduced probe-rtt  */  in bbr_set_state_target()
10648 		if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&  in bbr_set_state_target()
10649 		    (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {  in bbr_set_state_target()
10651 			tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);  in bbr_set_state_target()
10653 		} else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {  in bbr_set_state_target()
10663 			 * for non-google mode and default (non-configured).  in bbr_set_state_target()
10666 			if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN])  {  in bbr_set_state_target()
10670 				tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);  in bbr_set_state_target()
10676 	bbr->r_ctl.rc_target_at_state = tar;  in bbr_set_state_target()
10685 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_enter_probe_rtt()
10686 	bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,  in bbr_enter_probe_rtt()
10687 					     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_enter_probe_rtt()
10688 	bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain  in bbr_enter_probe_rtt()
10689 					  + bbr->r_ctl.rc_delivered);  in bbr_enter_probe_rtt()
10691 	if (bbr->rc_use_google || bbr_probertt_sets_rtt)  in bbr_enter_probe_rtt()
10692 		bbr->rc_prtt_set_ts = 1;  in bbr_enter_probe_rtt()
10693 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_enter_probe_rtt()
10694 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_enter_probe_rtt()
10695 		counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_enter_probe_rtt()
10698 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_enter_probe_rtt()
10699 	bbr->r_ctl.last_in_probertt = cts;  in bbr_enter_probe_rtt()
10700 	bbr->r_ctl.rc_probertt_srttchktim = cts;  in bbr_enter_probe_rtt()
10701 	bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_enter_probe_rtt()
10702 	bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;  in bbr_enter_probe_rtt()
10706 	    bbr->rc_hit_state_1 &&  in bbr_enter_probe_rtt()
10707 	    (bbr->rc_use_google == 0) &&  in bbr_enter_probe_rtt()
10709 		if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)  in bbr_enter_probe_rtt()
10710 			bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_enter_probe_rtt()
10712 		bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_enter_probe_rtt()
10714 	bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_enter_probe_rtt()
10715 	if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){  in bbr_enter_probe_rtt()
10716 		/* Set to the non-configurable default of 4 (PROBE_RTT_MIN)  */  in bbr_enter_probe_rtt()
10717 		bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_enter_probe_rtt()
10719 		bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_enter_probe_rtt()
10720 		bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_enter_probe_rtt()
10721 		bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);  in bbr_enter_probe_rtt()
10722 		bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;  in bbr_enter_probe_rtt()
10729 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;  in bbr_enter_probe_rtt()
10730 		bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_enter_probe_rtt()
10733 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_enter_probe_rtt()
10734 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_enter_probe_rtt()
10738 	if (ctf_flight_size(bbr->rc_tp,  in bbr_enter_probe_rtt()
10739 		(bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=  in bbr_enter_probe_rtt()
10740 	    bbr->r_ctl.rc_target_at_state) {  in bbr_enter_probe_rtt()
10742 		bbr->r_ctl.rc_bbr_enters_probertt = cts;  in bbr_enter_probe_rtt()
10745 		bbr->r_ctl.rc_bbr_enters_probertt = 0;  in bbr_enter_probe_rtt()
10747 	bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;  in bbr_enter_probe_rtt()
10757 	 * Sanity check on probe-rtt intervals.  in bbr_check_probe_rtt_limits()
10759 	 * against new-reno flows with huge buffers  in bbr_check_probe_rtt_limits()
10760 	 * our rtt-prop interval could come to dominate  in bbr_check_probe_rtt_limits()
10765 	    (bbr->rc_use_google == 0)) {  in bbr_check_probe_rtt_limits()
10769 		/* Are we to small and go into probe-rtt to often? */  in bbr_check_probe_rtt_limits()
10775 				newval = cur_rttp + (fval - bbr_rtt_probe_limit);  in bbr_check_probe_rtt_limits()
10784 		if (cur_rttp > 	bbr->r_ctl.rc_probertt_int) {  in bbr_check_probe_rtt_limits()
10785 			bbr->r_ctl.rc_probertt_int = cur_rttp;  in bbr_check_probe_rtt_limits()
10786 			reset_time_small(&bbr->r_ctl.rc_rttprop, newval);  in bbr_check_probe_rtt_limits()
10793 			if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {  in bbr_check_probe_rtt_limits()
10799 					bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;  in bbr_check_probe_rtt_limits()
10800 					reset_time_small(&bbr->r_ctl.rc_rttprop,  in bbr_check_probe_rtt_limits()
10809 					if (cur_rttp < bbr->r_ctl.rc_probertt_int) {  in bbr_check_probe_rtt_limits()
10811 						bbr->r_ctl.rc_probertt_int = cur_rttp;  in bbr_check_probe_rtt_limits()
10812 						reset_time_small(&bbr->r_ctl.rc_rttprop, newval);  in bbr_check_probe_rtt_limits()
10826 	/* Exit probe-rtt */  in bbr_exit_probe_rtt()
10828 	if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {  in bbr_exit_probe_rtt()
10829 		tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_exit_probe_rtt()
10833 	bbr->rc_hit_state_1 = 0;  in bbr_exit_probe_rtt()
10834 	bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_exit_probe_rtt()
10835 	bbr->r_ctl.last_in_probertt = cts;  in bbr_exit_probe_rtt()
10837 	bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_exit_probe_rtt()
10838 	bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,  in bbr_exit_probe_rtt()
10839 					      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_exit_probe_rtt()
10840 					  bbr->r_ctl.rc_delivered);  in bbr_exit_probe_rtt()
10841 	if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_exit_probe_rtt()
10844 		time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_exit_probe_rtt()
10845 		counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_exit_probe_rtt()
10847 	if (bbr->rc_filled_pipe) {  in bbr_exit_probe_rtt()
10849 		bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_exit_probe_rtt()
10850 		bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_exit_probe_rtt()
10851 		bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;  in bbr_exit_probe_rtt()
10856 		bbr->rc_bbr_state = BBR_STATE_STARTUP;  in bbr_exit_probe_rtt()
10857 		bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_exit_probe_rtt()
10861 		 * the number of pe's we were in probe-rtt  in bbr_exit_probe_rtt()
10865 		bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);  in bbr_exit_probe_rtt()
10866 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_exit_probe_rtt()
10868 		if (bbr->r_ctl.rc_lost &&  in bbr_exit_probe_rtt()
10870 		    (bbr->rc_use_google == 0))  in bbr_exit_probe_rtt()
10871 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;  in bbr_exit_probe_rtt()
10873 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;  in bbr_exit_probe_rtt()
10874 		bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;  in bbr_exit_probe_rtt()
10878 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_exit_probe_rtt()
10879 		    bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);  in bbr_exit_probe_rtt()
10887 	if ((bbr->rc_past_init_win == 1) &&  in bbr_should_enter_probe_rtt()
10888 	    (bbr->rc_in_persist == 0) &&  in bbr_should_enter_probe_rtt()
10889 	    (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {  in bbr_should_enter_probe_rtt()
10893 	    (bbr->rc_in_persist == 0) &&  in bbr_should_enter_probe_rtt()
10894 	    (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&  in bbr_should_enter_probe_rtt()
10895 	    ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {  in bbr_should_enter_probe_rtt()
10907 		 * Need to be on a pkt-epoch to continue.  in bbr_google_startup()
10912 	gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *  in bbr_google_startup()
10913 		 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_google_startup()
10915 		bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_google_startup()
10916 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_google_startup()
10917 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);  in bbr_google_startup()
10918 		bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;  in bbr_google_startup()
10920 	if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)  in bbr_google_startup()
10922 	bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_google_startup()
10923 			      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);  in bbr_google_startup()
10935 	if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&  in bbr_state_startup()
10936 	    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {  in bbr_state_startup()
10949 	if (bbr->rc_use_google)  in bbr_state_startup()
10952 	if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&  in bbr_state_startup()
10955 		bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;  in bbr_state_startup()
10959 		 * Need to be on a pkt-epoch to continue.  in bbr_state_startup()
10969 		if (bbr->r_ctl.rc_pkt_epoch_rtt &&  in bbr_state_startup()
10970 		    bbr->r_ctl.startup_last_srtt &&  in bbr_state_startup()
10971 		    (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {  in bbr_state_startup()
10972 			delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;  in bbr_state_startup()
10973 			rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;  in bbr_state_startup()
10976 		if ((bbr->r_ctl.startup_last_srtt == 0)  ||  in bbr_state_startup()
10977 		    (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))  in bbr_state_startup()
10979 			bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;  in bbr_state_startup()
10981 		if ((bbr->r_ctl.rc_lost == 0) &&  in bbr_state_startup()
10988 			if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)  in bbr_state_startup()
10989 				bbr->r_ctl.rc_bbr_last_startup_epoch++;  in bbr_state_startup()
10991 					      delta, bbr->r_ctl.startup_last_srtt, 10);  in bbr_state_startup()
10995 	if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&  in bbr_state_startup()
10996 	    (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&  in bbr_state_startup()
10997 	    (!IN_RECOVERY(bbr->rc_tp->t_flags))) {  in bbr_state_startup()
11002 		 * the number of non-gain we have already accumulated.  in bbr_state_startup()
11004 		if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)  in bbr_state_startup()
11005 			bbr->r_ctl.rc_bbr_last_startup_epoch++;  in bbr_state_startup()
11006 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11007 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);  in bbr_state_startup()
11011 	if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)  in bbr_state_startup()
11012 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
11013 	bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;  in bbr_state_startup()
11015 	if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)  in bbr_state_startup()
11016 		gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *  in bbr_state_startup()
11017 			 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_state_startup()
11019 		gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *  in bbr_state_startup()
11020 			 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;  in bbr_state_startup()
11022 	if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)  in bbr_state_startup()
11023 		bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;  in bbr_state_startup()
11025 		bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;  in bbr_state_startup()
11027 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
11028 		bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11029 				      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);  in bbr_state_startup()
11031 	if ((bbr->rc_loss_exit &&  in bbr_state_startup()
11032 	     (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&  in bbr_state_startup()
11033 	     (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&  in bbr_state_startup()
11034 	    ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {  in bbr_state_startup()
11042 		if ((ctf_flight_size(bbr->rc_tp,  in bbr_state_startup()
11043 			 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +  in bbr_state_startup()
11044 		     (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {  in bbr_state_startup()
11046 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11047 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);  in bbr_state_startup()
11050 			bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
11051 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11052 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);  in bbr_state_startup()
11055 		bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;  in bbr_state_startup()
11056 	if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||  in bbr_state_startup()
11062 	bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_startup()
11063 			      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);  in bbr_state_startup()
11074 	if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&  in bbr_state_change()
11075 	    (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&  in bbr_state_change()
11076 	    (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&  in bbr_state_change()
11077 	    (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&  in bbr_state_change()
11078 	    (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {  in bbr_state_change()
11080 		panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);  in bbr_state_change()
11083 	if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in bbr_state_change()
11088 			bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,  in bbr_state_change()
11089 					      bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);  in bbr_state_change()
11090 			bbr->rc_filled_pipe = 1;  in bbr_state_change()
11091 			bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_state_change()
11092 			if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_state_change()
11093 				time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_state_change()
11094 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_state_change()
11097 			if (bbr->rc_no_pacing)  in bbr_state_change()
11098 				bbr->rc_no_pacing = 0;  in bbr_state_change()
11099 			bbr->r_ctl.rc_bbr_state_time = cts;  in bbr_state_change()
11100 			bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;  in bbr_state_change()
11101 			bbr->rc_bbr_state = BBR_STATE_DRAIN;  in bbr_state_change()
11103 			if ((bbr->rc_use_google == 0) &&  in bbr_state_change()
11105 				/* Here we don't have to worry about probe-rtt */  in bbr_state_change()
11106 				bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;  in bbr_state_change()
11107 				bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_state_change()
11110 			bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;  in bbr_state_change()
11112 			if (ctf_flight_size(bbr->rc_tp,  in bbr_state_change()
11113 			        (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=  in bbr_state_change()
11114 			    bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11119 				bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_state_change()
11121 				bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_state_change()
11125 	} else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {  in bbr_state_change()
11129 		tp = bbr->rc_tp;  in bbr_state_change()
11131 			      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_state_change()
11132 		if (inflight >= bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11134 			bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_state_change()
11135 			bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_state_change()
11136 			bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;  in bbr_state_change()
11142 			bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;  in bbr_state_change()
11143 			bbr->rc_bbr_substate = BBR_SUB_LEVEL6;  in bbr_state_change()
11146 	} else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {  in bbr_state_change()
11147 		/* Has in-flight reached the bdp (or less)? */  in bbr_state_change()
11151 		tp = bbr->rc_tp;  in bbr_state_change()
11153 			      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_state_change()
11154 		if ((bbr->rc_use_google == 0) &&  in bbr_state_change()
11156 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_state_change()
11158 			 * Here we don't have to worry about probe-rtt  in bbr_state_change()
11159 			 * re-slam it, but keep it slammed down.  in bbr_state_change()
11161 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_state_change()
11164 		if (inflight <= bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11166 			bbr->rc_bbr_state = BBR_STATE_PROBE_BW;  in bbr_state_change()
11167 			bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;  in bbr_state_change()
11168 			if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {  in bbr_state_change()
11171 				time_in = cts - bbr->r_ctl.rc_bbr_state_time;  in bbr_state_change()
11172 				counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);  in bbr_state_change()
11174 			if ((bbr->rc_use_google == 0) &&  in bbr_state_change()
11176 			    (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {  in bbr_state_change()
11178 				tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;  in bbr_state_change()
11181 			/* Setup probe-rtt has being done now RRS-HERE */  in bbr_state_change()
11182 			bbr->r_ctl.rc_rtt_shrinks = cts;  in bbr_state_change()
11183 			bbr->r_ctl.last_in_probertt = cts;  in bbr_state_change()
11185 			/* Randomly pick a sub-state */  in bbr_state_change()
11186 			bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);  in bbr_state_change()
11190 	} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {  in bbr_state_change()
11193 		flight = ctf_flight_size(bbr->rc_tp,  in bbr_state_change()
11194 			     (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_state_change()
11195 		bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);  in bbr_state_change()
11196 		if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&  in bbr_state_change()
11197 		    (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_state_change()
11201 			bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);  in bbr_state_change()
11204 			   (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {  in bbr_state_change()
11205 			/* Re-slam it */  in bbr_state_change()
11206 			bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;  in bbr_state_change()
11209 		if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {  in bbr_state_change()
11211 			if (flight <= bbr->r_ctl.rc_target_at_state) {  in bbr_state_change()
11213 				bbr->r_ctl.rc_bbr_enters_probertt = cts;  in bbr_state_change()
11215 				if (bbr->r_ctl.rc_bbr_enters_probertt == 0)  in bbr_state_change()
11216 					bbr->r_ctl.rc_bbr_enters_probertt = 1;  in bbr_state_change()
11217 				if (bbr->rc_use_google == 0) {  in bbr_state_change()
11222 					if (bbr->r_ctl.bbr_rttprobe_gain_val)  in bbr_state_change()
11223 						bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;  in bbr_state_change()
11225 						bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;  in bbr_state_change()
11228 			if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&  in bbr_state_change()
11229 			    (bbr->rc_use_google == 0) &&  in bbr_state_change()
11230 			    bbr->r_ctl.bbr_rttprobe_gain_val &&  in bbr_state_change()
11231 			    (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||  in bbr_state_change()
11232 			     (flight >= bbr->r_ctl.flightsize_at_drain))) {  in bbr_state_change()
11243 				bbr->r_ctl.flightsize_at_drain = flight;  in bbr_state_change()
11244 				bbr->r_ctl.rc_probertt_srttchktim = cts;  in bbr_state_change()
11245 				red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);  in bbr_state_change()
11246 				if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {  in bbr_state_change()
11248 					bbr->r_ctl.rc_bbr_hptsi_gain -= red;  in bbr_state_change()
11250 				} else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {  in bbr_state_change()
11252 					bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);  in bbr_state_change()
11256 					bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);  in bbr_state_change()
11260 		if (bbr->r_ctl.rc_bbr_enters_probertt &&  in bbr_state_change()
11261 		    (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&  in bbr_state_change()
11262 		    ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {  in bbr_state_change()
11264 			bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);  in bbr_state_change()
11266 	} else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {  in bbr_state_change()
11267 		if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&  in bbr_state_change()
11268 		    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {  in bbr_state_change()
11290 	if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {  in bbr_check_bbr_for_state()
11295 	bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);  in bbr_check_bbr_for_state()
11317 	nsegs = max(1, m->m_pkthdr.lro_nsegs);  in bbr_do_segment_nounlock()
11318 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_do_segment_nounlock()
11324 	 * If this is either a state-changing packet or current state isn't  in bbr_do_segment_nounlock()
11331 	KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",  in bbr_do_segment_nounlock()
11333 	KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",  in bbr_do_segment_nounlock()
11336 	tp->t_rcvtime = ticks;  in bbr_do_segment_nounlock()
11338 	 * Unscale the window into a 32-bit value. For the SYN_SENT state  in bbr_do_segment_nounlock()
11341 	tiwin = th->th_win << tp->snd_scale;  in bbr_do_segment_nounlock()
11343 	stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);  in bbr_do_segment_nounlock()
11346 	if (m->m_flags & M_TSTMP) {  in bbr_do_segment_nounlock()
11351 		bbr->rc_tv.tv_sec = ts.tv_sec;  in bbr_do_segment_nounlock()
11352 		bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;  in bbr_do_segment_nounlock()
11353 		bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11354 	} else if (m->m_flags & M_TSTMP_LRO) {  in bbr_do_segment_nounlock()
11359 		bbr->rc_tv.tv_sec = ts.tv_sec;  in bbr_do_segment_nounlock()
11360 		bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;  in bbr_do_segment_nounlock()
11361 		bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11366 		bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11372 	    (th->th_off << 2) - sizeof(struct tcphdr),  in bbr_do_segment_nounlock()
11374 	if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) {  in bbr_do_segment_nounlock()
11389 	if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&  in bbr_do_segment_nounlock()
11401 		to.to_tsecr -= tp->ts_offset;  in bbr_do_segment_nounlock()
11402 		if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv)))  in bbr_do_segment_nounlock()
11409 	if (bbr->r_state == 0) {  in bbr_do_segment_nounlock()
11417 		if (bbr->rc_inp == NULL) {  in bbr_do_segment_nounlock()
11418 			bbr->rc_inp = inp;  in bbr_do_segment_nounlock()
11424 		if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {  in bbr_do_segment_nounlock()
11426 			    (tp->t_flags & TF_REQ_SCALE)) {  in bbr_do_segment_nounlock()
11427 				tp->t_flags |= TF_RCVD_SCALE;  in bbr_do_segment_nounlock()
11428 				tp->snd_scale = to.to_wscale;  in bbr_do_segment_nounlock()
11430 				tp->t_flags &= ~TF_REQ_SCALE;  in bbr_do_segment_nounlock()
11435 			tp->snd_wnd = th->th_win;  in bbr_do_segment_nounlock()
11437 			    (tp->t_flags & TF_REQ_TSTMP)) {  in bbr_do_segment_nounlock()
11438 				tp->t_flags |= TF_RCVD_TSTMP;  in bbr_do_segment_nounlock()
11439 				tp->ts_recent = to.to_tsval;  in bbr_do_segment_nounlock()
11440 				tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_do_segment_nounlock()
11442 			    tp->t_flags &= ~TF_REQ_TSTMP;  in bbr_do_segment_nounlock()
11445 			if ((tp->t_flags & TF_SACK_PERMIT) &&  in bbr_do_segment_nounlock()
11447 				tp->t_flags &= ~TF_SACK_PERMIT;  in bbr_do_segment_nounlock()
11448 			if (tp->t_flags & TF_FASTOPEN) {  in bbr_do_segment_nounlock()
11455 						if ((inp->inp_vflag & INP_IPV6) != 0)  in bbr_do_segment_nounlock()
11471 		if ((tp->t_flags & TF_SACK_PERMIT) == 0) {  in bbr_do_segment_nounlock()
11474 			(*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen,  in bbr_do_segment_nounlock()
11479 		bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0;  in bbr_do_segment_nounlock()
11481 		sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);  in bbr_do_segment_nounlock()
11492 	 * always. All other times (timers etc) we must have a rack-state  in bbr_do_segment_nounlock()
11497 	if (bbr->r_state != tp->t_state)  in bbr_do_segment_nounlock()
11500 	if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)  in bbr_do_segment_nounlock()
11502 	prev_state = bbr->r_state;  in bbr_do_segment_nounlock()
11503 	bbr->rc_ack_was_delayed = 0;  in bbr_do_segment_nounlock()
11504 	lost = bbr->r_ctl.rc_lost;  in bbr_do_segment_nounlock()
11505 	bbr->rc_is_pkt_epoch_now = 0;  in bbr_do_segment_nounlock()
11506 	if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) {  in bbr_do_segment_nounlock()
11510 			bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;  in bbr_do_segment_nounlock()
11511 			bbr->rc_ack_was_delayed = 1;  in bbr_do_segment_nounlock()
11512 			if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,  in bbr_do_segment_nounlock()
11513 				     bbr->r_ctl.highest_hdwr_delay))  in bbr_do_segment_nounlock()
11514 				bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;  in bbr_do_segment_nounlock()
11516 			bbr->r_ctl.rc_ack_hdwr_delay = 0;  in bbr_do_segment_nounlock()
11517 			bbr->rc_ack_was_delayed = 0;  in bbr_do_segment_nounlock()
11520 		bbr->r_ctl.rc_ack_hdwr_delay = 0;  in bbr_do_segment_nounlock()
11521 		bbr->rc_ack_was_delayed = 0;  in bbr_do_segment_nounlock()
11530 	 * If a segment with the ACK-bit set arrives in the SYN-SENT state  in bbr_do_segment_nounlock()
11533 	if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&  in bbr_do_segment_nounlock()
11534 	    (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {  in bbr_do_segment_nounlock()
11539 	if (tiwin > bbr->r_ctl.rc_high_rwnd)  in bbr_do_segment_nounlock()
11540 		bbr->r_ctl.rc_high_rwnd = tiwin;  in bbr_do_segment_nounlock()
11541 	bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,  in bbr_do_segment_nounlock()
11542 					    (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_do_segment_nounlock()
11543 	bbr->rtt_valid = 0;  in bbr_do_segment_nounlock()
11545 		bbr->rc_ts_valid = 1;  in bbr_do_segment_nounlock()
11546 		bbr->r_ctl.last_inbound_ts = to.to_tsval;  in bbr_do_segment_nounlock()
11548 		bbr->rc_ts_valid = 0;  in bbr_do_segment_nounlock()
11549 		bbr->r_ctl.last_inbound_ts = 0;  in bbr_do_segment_nounlock()
11551 	retval = (*bbr->r_substate) (m, th, so,  in bbr_do_segment_nounlock()
11565 		if (bbr->rc_is_pkt_epoch_now)  in bbr_do_segment_nounlock()
11567 		bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));  in bbr_do_segment_nounlock()
11569 			if ((bbr->r_wanted_output != 0) ||  in bbr_do_segment_nounlock()
11570 			    (tp->t_flags & TF_ACKNOW)) {   in bbr_do_segment_nounlock()
11572 				bbr->rc_output_starts_timer = 0;  in bbr_do_segment_nounlock()
11580 		    ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&  in bbr_do_segment_nounlock()
11581 		    (SEQ_GT(tp->snd_max, tp->snd_una) ||  in bbr_do_segment_nounlock()
11582 		     (tp->t_flags & TF_DELACK) ||  in bbr_do_segment_nounlock()
11583 		     ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&  in bbr_do_segment_nounlock()
11584 		      (tp->t_state <= TCPS_CLOSING)))) {  in bbr_do_segment_nounlock()
11589 			if ((tp->snd_max == tp->snd_una) &&  in bbr_do_segment_nounlock()
11590 			    ((tp->t_flags & TF_DELACK) == 0) &&  in bbr_do_segment_nounlock()
11592 			    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {  in bbr_do_segment_nounlock()
11601 					if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&  in bbr_do_segment_nounlock()
11602 					    (TSTMP_GT(lcts, bbr->rc_pacer_started))) {  in bbr_do_segment_nounlock()
11605 						del = lcts - bbr->rc_pacer_started;  in bbr_do_segment_nounlock()
11606 						if (bbr->r_ctl.rc_last_delay_val > del) {  in bbr_do_segment_nounlock()
11608 							bbr->r_ctl.rc_last_delay_val -= del;  in bbr_do_segment_nounlock()
11609 							bbr->rc_pacer_started = lcts;  in bbr_do_segment_nounlock()
11612 							bbr->r_ctl.rc_last_delay_val = 0;  in bbr_do_segment_nounlock()
11619 				bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,  in bbr_do_segment_nounlock()
11622 		} else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {  in bbr_do_segment_nounlock()
11624 			bbr_timer_audit(tp, bbr, lcts, &so->so_snd);  in bbr_do_segment_nounlock()
11627 		if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS))  in bbr_do_segment_nounlock()
11628 			tp->t_flags2 &= ~TF2_HPTS_CALLS;  in bbr_do_segment_nounlock()
11630 		if (bbr->r_state != tp->t_state)  in bbr_do_segment_nounlock()
11635 			bbr->r_wanted_output = 0;  in bbr_do_segment_nounlock()
11648 	if (!STAILQ_EMPTY(&tp->t_inqueue)) {  in bbr_do_segment()
11654 	if (m->m_flags & M_TSTMP_LRO) {  in bbr_do_segment()
11678 	if (ctf_outstanding(tp) >= tp->snd_wnd) {  in bbr_what_can_we_send()
11679 		/* We never want to go over our peers rcv-window */  in bbr_what_can_we_send()
11684 		flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));  in bbr_what_can_we_send()
11689 			 * >= tp->snd_wnd).  in bbr_what_can_we_send()
11693 		len = sendwin - flight;  in bbr_what_can_we_send()
11694 		if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {  in bbr_what_can_we_send()
11696 			len = tp->snd_wnd - ctf_outstanding(tp);  in bbr_what_can_we_send()
11703 			len = avail - sb_offset;  in bbr_what_can_we_send()
11716 		if (rsm->r_flags & BBR_TLP) {  in bbr_do_send_accounting()
11725 			tp->t_sndrexmitpack++;  in bbr_do_send_accounting()
11729 			stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,  in bbr_do_send_accounting()
11734 		 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is  in bbr_do_send_accounting()
11735 		 * sub-state  in bbr_do_send_accounting()
11737 		counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);  in bbr_do_send_accounting()
11738 		if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {  in bbr_do_send_accounting()
11740 			counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);  in bbr_do_send_accounting()
11743 			 * Log our probe state 3, and log also 5-13 to show  in bbr_do_send_accounting()
11744 			 * us the recovery sub-state for the send. This  in bbr_do_send_accounting()
11765 		stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,  in bbr_do_send_accounting()
11774 	if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {  in bbr_cwnd_limiting()
11785 		if (tp->snd_cwnd > target)  in bbr_cwnd_limiting()
11786 			tp->snd_cwnd = target;  in bbr_cwnd_limiting()
11796 	 * account that we are limited by TCP_MAXWIN << tp->rcv_scale.  in bbr_window_update_needed()
11802 	if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {  in bbr_window_update_needed()
11803 		oldwin = (tp->rcv_adv - tp->rcv_nxt);  in bbr_window_update_needed()
11805 			adv -= oldwin;  in bbr_window_update_needed()
11818 	if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)  in bbr_window_update_needed()
11822 	    (adv >= (so->so_rcv.sb_hiwat / 4) ||  in bbr_window_update_needed()
11823 	    recwin <= (so->so_rcv.sb_hiwat / 8) ||  in bbr_window_update_needed()
11824 	    so->so_rcv.sb_hiwat <= 8 * maxseg)) {  in bbr_window_update_needed()
11827 	if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat)  in bbr_window_update_needed()
11835  * if the TCB was on the hpts. A non-zero return
11893 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_output_wtime()
11895 	memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));  in bbr_output_wtime()
11896 	cts = tcp_tv_to_usectick(&bbr->rc_tv);  in bbr_output_wtime()
11897 	inp = bbr->rc_inp;  in bbr_output_wtime()
11898 	hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS);  in bbr_output_wtime()
11899 	tp->t_flags2 &= ~TF2_HPTS_CALLS;  in bbr_output_wtime()
11900 	so = inp->inp_socket;  in bbr_output_wtime()
11901 	sb = &so->so_snd;  in bbr_output_wtime()
11902 	if (tp->t_nic_ktls_xmit)  in bbr_output_wtime()
11907 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_output_wtime()
11912 	pace_max_segs = bbr->r_ctl.rc_pace_max_segs;  in bbr_output_wtime()
11916 	if (tp->t_flags & TF_TOE)  in bbr_output_wtime()
11921 	if (bbr->r_state) {  in bbr_output_wtime()
11923 		isipv6 = bbr->r_is_v6;  in bbr_output_wtime()
11925 		isipv6 = (inp->inp_vflag & INP_IPV6) != 0;  in bbr_output_wtime()
11928 	if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&  in bbr_output_wtime()
11934 		if ((tp->t_flags & TF_ACKNOW) == 0) {  in bbr_output_wtime()
11939 			 * whatever timer is running (KEEP/DEL-ACK?) and  in bbr_output_wtime()
11944 			recwin = lmin(lmax(sbspace(&so->so_rcv), 0),  in bbr_output_wtime()
11945 				      (long)TCP_MAXWIN << tp->rcv_scale);  in bbr_output_wtime()
11947 			    ((tcp_outflags[tp->t_state] & TH_RST) == 0) &&  in bbr_output_wtime()
11948 			    ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <=  in bbr_output_wtime()
11949 			    (tp->snd_max - tp->snd_una))) {  in bbr_output_wtime()
11953 				 * let the timer-run off.  in bbr_output_wtime()
11961 	if (bbr->r_ctl.rc_last_delay_val) {  in bbr_output_wtime()
11963 		if (SEQ_GT(cts, bbr->rc_pacer_started))  in bbr_output_wtime()
11964 			delay_calc = cts - bbr->rc_pacer_started;  in bbr_output_wtime()
11965 		if (delay_calc >= bbr->r_ctl.rc_last_delay_val)  in bbr_output_wtime()
11966 			delay_calc -= bbr->r_ctl.rc_last_delay_val;  in bbr_output_wtime()
11971 	if ((bbr->r_timer_override) ||  in bbr_output_wtime()
11972 	    (tp->t_state < TCPS_ESTABLISHED)) {  in bbr_output_wtime()
11977 		if ((bbr->r_ctl.rc_last_delay_val) &&  in bbr_output_wtime()
11978 		    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&  in bbr_output_wtime()
11987 			bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
11989 		} else if (tp->t_state == TCPS_CLOSED) {  in bbr_output_wtime()
11990 			bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12002 	bbr->rc_cwnd_limited = 0;  in bbr_output_wtime()
12003 	if (bbr->r_ctl.rc_last_delay_val) {  in bbr_output_wtime()
12005 		if (SEQ_GT(cts, bbr->rc_pacer_started))  in bbr_output_wtime()
12006 			delay_calc = cts - bbr->rc_pacer_started;  in bbr_output_wtime()
12009 		if (delay_calc >= bbr->r_ctl.rc_last_delay_val)  in bbr_output_wtime()
12011 			delay_calc -= bbr->r_ctl.rc_last_delay_val;  in bbr_output_wtime()
12019 			bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);  in bbr_output_wtime()
12020 			bbr->r_agg_early_set = 1;  in bbr_output_wtime()
12021 			if (bbr->r_ctl.rc_hptsi_agg_delay) {  in bbr_output_wtime()
12022 				if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {  in bbr_output_wtime()
12024 					bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;  in bbr_output_wtime()
12025 					bbr->r_agg_early_set = 0;  in bbr_output_wtime()
12026 					bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
12028 					bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_output_wtime()
12029 					bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_output_wtime()
12032 			merged_val = bbr->rc_pacer_started;  in bbr_output_wtime()
12034 			merged_val |= bbr->r_ctl.rc_last_delay_val;  in bbr_output_wtime()
12036 						 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,  in bbr_output_wtime()
12037 						 bbr->r_agg_early_set, 3);  in bbr_output_wtime()
12038 			bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12044 		if (bbr->r_agg_early_set)  in bbr_output_wtime()
12045 			bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
12046 		bbr->r_agg_early_set = 0;  in bbr_output_wtime()
12055 		if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)  in bbr_output_wtime()
12056 			bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;  in bbr_output_wtime()
12058 			bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;  in bbr_output_wtime()
12060 	sendwin = min(tp->snd_wnd, tp->snd_cwnd);  in bbr_output_wtime()
12061 	if ((tp->snd_una == tp->snd_max) &&  in bbr_output_wtime()
12062 	    (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&  in bbr_output_wtime()
12067 		 * suite of states or a fast-ramp up.  in bbr_output_wtime()
12070 				       cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));  in bbr_output_wtime()
12078 	if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {  in bbr_output_wtime()
12091 	bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;  in bbr_output_wtime()
12093 	    (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {  in bbr_output_wtime()
12094 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12096 	bbr->r_timer_override = 0;  in bbr_output_wtime()
12097 	bbr->r_wanted_output = 0;  in bbr_output_wtime()
12102 	if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12103 	    ((tp->t_state == TCPS_SYN_RECEIVED) ||  in bbr_output_wtime()
12104 	     (tp->t_state == TCPS_SYN_SENT)) &&  in bbr_output_wtime()
12105 	    SEQ_GT(tp->snd_max, tp->snd_una) &&	/* initial SYN or SYN|ACK sent */  in bbr_output_wtime()
12106 	    (tp->t_rxtshift == 0)) {	/* not a retransmit */  in bbr_output_wtime()
12115 	if (bbr->rc_use_google == 0)  in bbr_output_wtime()
12134 	sendwin = min(tp->snd_wnd, tp->snd_cwnd);  in bbr_output_wtime()
12135 	sb_offset = tp->snd_max - tp->snd_una;  in bbr_output_wtime()
12136 	flags = tcp_outflags[tp->t_state];  in bbr_output_wtime()
12145 	while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {  in bbr_output_wtime()
12151 			tot_len = tp->t_maxseg;  in bbr_output_wtime()
12157 		TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);  in bbr_output_wtime()
12158 		bbr->r_ctl.rc_free_cnt++;  in bbr_output_wtime()
12162 	if (bbr->r_ctl.rc_resend == NULL) {  in bbr_output_wtime()
12164 		bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);  in bbr_output_wtime()
12165 		if (bbr->r_ctl.rc_resend) {  in bbr_output_wtime()
12169 			bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);  in bbr_output_wtime()
12172 	if (bbr->r_ctl.rc_resend) {  in bbr_output_wtime()
12173 		rsm = bbr->r_ctl.rc_resend;  in bbr_output_wtime()
12177 		/* Remove any TLP flags its a RACK or T-O */  in bbr_output_wtime()
12178 		rsm->r_flags &= ~BBR_TLP;  in bbr_output_wtime()
12179 		bbr->r_ctl.rc_resend = NULL;  in bbr_output_wtime()
12180 		if (SEQ_LT(rsm->r_start, tp->snd_una)) {  in bbr_output_wtime()
12183 			    tp, bbr, rsm, rsm->r_start, tp->snd_una);  in bbr_output_wtime()
12191 		if (rsm->r_flags & BBR_HAS_SYN) {  in bbr_output_wtime()
12196 				rsm->r_start++;  in bbr_output_wtime()
12197 				if (rsm->r_start == rsm->r_end) {  in bbr_output_wtime()
12202 					rsm->r_flags &= ~BBR_HAS_SYN;  in bbr_output_wtime()
12203 					len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12212 			len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12213 		if ((bbr->rc_resends_use_tso == 0) &&  in bbr_output_wtime()
12218 		sb_offset = rsm->r_start - tp->snd_una;  in bbr_output_wtime()
12230 	} else if (bbr->r_ctl.rc_tlp_send) {  in bbr_output_wtime()
12235 		rsm = bbr->r_ctl.rc_tlp_send;  in bbr_output_wtime()
12236 		bbr->r_ctl.rc_tlp_send = NULL;  in bbr_output_wtime()
12238 		len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12239 		if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))  in bbr_output_wtime()
12242 		if (SEQ_GT(tp->snd_una, rsm->r_start)) {  in bbr_output_wtime()
12245 			    tp, bbr, tp->snd_una, rsm, rsm->r_start);  in bbr_output_wtime()
12252 		sb_offset = rsm->r_start - tp->snd_una;  in bbr_output_wtime()
12261 	    (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {  in bbr_output_wtime()
12263 		if (!bbr->alloc_limit_reported) {  in bbr_output_wtime()
12264 			bbr->alloc_limit_reported = 1;  in bbr_output_wtime()
12270 	if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {  in bbr_output_wtime()
12279 	if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))  in bbr_output_wtime()
12281 	if (tp->t_flags & TF_NEEDSYN)  in bbr_output_wtime()
12284 	if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {  in bbr_output_wtime()
12286 		len--;  in bbr_output_wtime()
12302 		end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);  in bbr_output_wtime()
12313 	 * fast-retransmit because TCP will reset snd_nxt to snd_max after  in bbr_output_wtime()
12314 	 * the fast-retransmit.  in bbr_output_wtime()
12316 	 * In the normal retransmit-FIN-only case, however, snd_nxt will be  in bbr_output_wtime()
12327 		if (SEQ_GT(tp->snd_max, tp->snd_una))  in bbr_output_wtime()
12328 			sb_offset = tp->snd_max - tp->snd_una;  in bbr_output_wtime()
12331 		if (bbr->rc_tlp_new_data) {  in bbr_output_wtime()
12338 			if (tlplen > (uint32_t)(avail - sb_offset)) {  in bbr_output_wtime()
12339 				tlplen = (uint32_t)(avail - sb_offset);  in bbr_output_wtime()
12341 			if (tlplen > tp->snd_wnd) {  in bbr_output_wtime()
12342 				len = tp->snd_wnd;  in bbr_output_wtime()
12346 			bbr->rc_tlp_new_data = 0;  in bbr_output_wtime()
12350 			    (bbr->rc_in_persist == 0) &&  in bbr_output_wtime()
12352 			    ((avail - sb_offset) >= p_maxseg)) {  in bbr_output_wtime()
12364 			if (bbr->rc_in_persist) {  in bbr_output_wtime()
12371 				rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_output_wtime()
12373 					len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
12374 					if (rsm->r_flags & BBR_HAS_FIN)  in bbr_output_wtime()
12375 						len--;  in bbr_output_wtime()
12376 					if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))  in bbr_output_wtime()
12389 					sb_offset = rsm->r_start - tp->snd_una;  in bbr_output_wtime()
12410 	 * SYN-SENT state and if segment contains data and if we don't know  in bbr_output_wtime()
12414 	    SEQ_GT(tp->snd_max, tp->snd_una)) {  in bbr_output_wtime()
12415 		if (tp->t_state != TCPS_SYN_RECEIVED)  in bbr_output_wtime()
12421 		if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12422 		    (tp->t_state == TCPS_SYN_RECEIVED))  in bbr_output_wtime()
12424 		sb_offset--, len++;  in bbr_output_wtime()
12432 		len--;  in bbr_output_wtime()
12439 	if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {  in bbr_output_wtime()
12446 	 *  - When retransmitting SYN|ACK on a passively-created socket  in bbr_output_wtime()
12447 	 *  - When retransmitting SYN on an actively created socket  in bbr_output_wtime()
12448 	 *  - When sending a zero-length cookie (cookie request) on an  in bbr_output_wtime()
12450 	 *  - When the socket is in the CLOSED state (RST is being sent)  in bbr_output_wtime()
12452 	if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12453 	    (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||  in bbr_output_wtime()
12454 	     ((tp->t_state == TCPS_SYN_SENT) &&  in bbr_output_wtime()
12455 	      (tp->t_tfo_client_cookie_len == 0)) ||  in bbr_output_wtime()
12461 	/* Without fast-open there should never be data sent on a SYN */  in bbr_output_wtime()
12462 	if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN))  in bbr_output_wtime()
12476 		 * 0-byte window. This makes sure the persist timer is set  in bbr_output_wtime()
12481 		if ((tp->snd_wnd == 0) &&  in bbr_output_wtime()
12482 		    (TCPS_HAVEESTABLISHED(tp->t_state)) &&  in bbr_output_wtime()
12483 		    (tp->snd_una == tp->snd_max) &&  in bbr_output_wtime()
12493 		   (len < bbr->r_ctl.rc_pace_max_segs)) {  in bbr_output_wtime()
12499 		if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_output_wtime()
12500 		    (TCPS_HAVEESTABLISHED(tp->t_state)) &&  in bbr_output_wtime()
12501 		    (len < (int)(sbavail(sb) - sb_offset))) {  in bbr_output_wtime()
12510 			if (tp->snd_max == tp->snd_una)  in bbr_output_wtime()
12512 		} else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&  in bbr_output_wtime()
12513 			   (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12514 						 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&  in bbr_output_wtime()
12515 			   (len < (int)(sbavail(sb) - sb_offset)) &&  in bbr_output_wtime()
12527 			bbr->rc_cwnd_limited = 1;  in bbr_output_wtime()
12529 		} else if (((tp->snd_wnd - ctf_outstanding(tp)) <  in bbr_output_wtime()
12530 			    min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&  in bbr_output_wtime()
12531 			   (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12532 						 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&  in bbr_output_wtime()
12533 			   (len < (int)(sbavail(sb) - sb_offset)) &&  in bbr_output_wtime()
12534 			   (TCPS_HAVEESTABLISHED(tp->t_state))) {  in bbr_output_wtime()
12552 	if (bbr->rc_in_persist &&  in bbr_output_wtime()
12555 	    (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {  in bbr_output_wtime()
12563 			sbleft = sbavail(sb) - sb_offset;  in bbr_output_wtime()
12566 		if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {  in bbr_output_wtime()
12576 	 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP  in bbr_output_wtime()
12593 	if (inp->inp_options)  in bbr_output_wtime()
12594 		ipoptlen = inp->inp_options->m_len -  in bbr_output_wtime()
12600 	 * Pre-calculate here as we save another lookup into the darknesses  in bbr_output_wtime()
12618 	if ((tp->t_flags & TF_TSO) && V_tcp_do_tso &&  in bbr_output_wtime()
12620 	    (tp->t_port == 0) &&  in bbr_output_wtime()
12621 	    ((tp->t_flags & TF_SIGNATURE) == 0) &&  in bbr_output_wtime()
12625 	recwin = lmin(lmax(sbspace(&so->so_rcv), 0),  in bbr_output_wtime()
12626 	    (long)TCP_MAXWIN << tp->rcv_scale);  in bbr_output_wtime()
12629 	 * conditions when len is non-zero:  in bbr_output_wtime()
12631 	 * - We have a full segment (or more with TSO) - This is the last  in bbr_output_wtime()
12633 	 * NODELAY - we've timed out (e.g. persist timer) - we have more  in bbr_output_wtime()
12635 	 * limited the window size) - we need to retransmit  in bbr_output_wtime()
12650 		if (((tp->t_flags & TF_MORETOCOME) == 0) &&	/* normal case */  in bbr_output_wtime()
12651 		    ((tp->t_flags & TF_NODELAY) ||  in bbr_output_wtime()
12652 		    ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) &&  in bbr_output_wtime()
12653 		    (tp->t_flags & TF_NOPUSH) == 0) {  in bbr_output_wtime()
12656 		if ((tp->snd_una == tp->snd_max) && len) {	/* Nothing outstanding */  in bbr_output_wtime()
12659 		if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {  in bbr_output_wtime()
12687 	 * pending (it will get piggy-backed on it) or the remote side  in bbr_output_wtime()
12688 	 * already has done a half-close and won't send more data.  Skip  in bbr_output_wtime()
12689 	 * this if the connection is in T/TCP half-open state.  in bbr_output_wtime()
12691 	if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&  in bbr_output_wtime()
12692 	    !(tp->t_flags & TF_DELACK) &&  in bbr_output_wtime()
12693 	    !TCPS_HAVERCVDFIN(tp->t_state)) {  in bbr_output_wtime()
12700 	 * is also a catch-all for the retransmit timer timeout case.  in bbr_output_wtime()
12702 	if (tp->t_flags & TF_ACKNOW) {  in bbr_output_wtime()
12709 	if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) {  in bbr_output_wtime()
12717 	    ((tp->t_flags & TF_SENTFIN) == 0)) {  in bbr_output_wtime()
12727 		slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);  in bbr_output_wtime()
12728 	if (bbr->rc_no_pacing)  in bbr_output_wtime()
12731 		if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=  in bbr_output_wtime()
12732 		    tp->snd_wnd) {  in bbr_output_wtime()
12736 			if ((bbr->rc_in_persist == 0) &&  in bbr_output_wtime()
12737 			    TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_output_wtime()
12738 			    (tp->snd_max == tp->snd_una) &&  in bbr_output_wtime()
12739 			    sbavail(&so->so_snd)) {  in bbr_output_wtime()
12741 				bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);  in bbr_output_wtime()
12747 		} else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12748 						 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {  in bbr_output_wtime()
12751 			bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12752 									bbr->r_ctl.rc_lost_bytes)));  in bbr_output_wtime()
12753 			bbr->rc_cwnd_limited = 1;  in bbr_output_wtime()
12759 		bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_output_wtime()
12760 		bbr->r_agg_early_set = 0;  in bbr_output_wtime()
12761 		bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
12762 		bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12763 	} else if (bbr->rc_use_google == 0)  in bbr_output_wtime()
12771 		bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
12772 								       bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);  in bbr_output_wtime()
12777 	bbr->r_ctl.rc_last_delay_val = 0;  in bbr_output_wtime()
12778 	bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
12781 	if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_output_wtime()
12783 		tp->snd_nxt = tp->snd_max;  in bbr_output_wtime()
12796 		bbr->rc_tlp_in_progress = 0;  in bbr_output_wtime()
12797 		bbr->rc_tlp_rtx_out = 0;  in bbr_output_wtime()
12802 		bbr->rc_tlp_in_progress = 1;  in bbr_output_wtime()
12808 			 * This is sub-optimal. We only send a stand alone  in bbr_output_wtime()
12813 				if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) {  in bbr_output_wtime()
12826 		    (((rsm->r_flags & BBR_HAS_FIN) == 0) &&  in bbr_output_wtime()
12829 			len--;  in bbr_output_wtime()
12834 		if ((tp->snd_una == tp->snd_max) &&  in bbr_output_wtime()
12835 		    (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {  in bbr_output_wtime()
12845 			tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;  in bbr_output_wtime()
12847 			tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;  in bbr_output_wtime()
12866 	 * established connection segments.  Options for SYN-ACK segments  in bbr_output_wtime()
12871 	if ((tp->t_flags & TF_NOOPT) == 0) {  in bbr_output_wtime()
12874 			to.to_mss = tcp_mssopt(&inp->inp_inc);  in bbr_output_wtime()
12875 			if (tp->t_port)  in bbr_output_wtime()
12876 				to.to_mss -= V_tcp_udp_tunneling_overhead;  in bbr_output_wtime()
12885 			if ((tp->t_flags & TF_FASTOPEN) &&  in bbr_output_wtime()
12886 			    (tp->t_rxtshift == 0)) {  in bbr_output_wtime()
12887 				if (tp->t_state == TCPS_SYN_RECEIVED) {  in bbr_output_wtime()
12890 					    (u_int8_t *)&tp->t_tfo_cookie.server;  in bbr_output_wtime()
12893 				} else if (tp->t_state == TCPS_SYN_SENT) {  in bbr_output_wtime()
12895 					    tp->t_tfo_client_cookie_len;  in bbr_output_wtime()
12897 					    tp->t_tfo_cookie.client;  in bbr_output_wtime()
12904 		if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {  in bbr_output_wtime()
12905 			to.to_wscale = tp->request_r_scale;  in bbr_output_wtime()
12909 		if ((tp->t_flags & TF_RCVD_TSTMP) ||  in bbr_output_wtime()
12910 		    ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {  in bbr_output_wtime()
12911 			to.to_tsval = 	tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset;  in bbr_output_wtime()
12912 			to.to_tsecr = tp->ts_recent;  in bbr_output_wtime()
12917 		if (tp->rfbuf_ts == 0 &&  in bbr_output_wtime()
12918 		    (so->so_rcv.sb_flags & SB_AUTOSIZE))  in bbr_output_wtime()
12919 			tp->rfbuf_ts = 	tcp_tv_to_mssectick(&bbr->rc_tv);  in bbr_output_wtime()
12923 		else if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_output_wtime()
12924 		    tp->rcv_numsacks > 0) {  in bbr_output_wtime()
12926 			to.to_nsacks = tp->rcv_numsacks;  in bbr_output_wtime()
12927 			to.to_sacks = (u_char *)tp->sackblks;  in bbr_output_wtime()
12930 		/* TCP-MD5 (RFC2385). */  in bbr_output_wtime()
12931 		if (tp->t_flags & TF_SIGNATURE)  in bbr_output_wtime()
12941 		if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie &&  in bbr_output_wtime()
12945 	if (tp->t_port) {  in bbr_output_wtime()
12958 	if (inp->inp_options)  in bbr_output_wtime()
12959 		ipoptlen = inp->inp_options->m_len -  in bbr_output_wtime()
12967 	if (bbr->rc_last_options != local_options) {  in bbr_output_wtime()
12972 		bbr->rc_last_options = local_options;  in bbr_output_wtime()
12974 	maxseg = tp->t_maxseg - (ipoptlen + optlen);  in bbr_output_wtime()
12990 			if_hw_tsomax = tp->t_tsomax;  in bbr_output_wtime()
12991 			if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;  in bbr_output_wtime()
12992 			if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;  in bbr_output_wtime()
13002 				max_len = (if_hw_tsomax - hdrlen -  in bbr_output_wtime()
13017 					len -= moff;  in bbr_output_wtime()
13030 			if (optlen + ipoptlen >= tp->t_maxseg) {  in bbr_output_wtime()
13068 		if (SEQ_LT(rsm->r_start, tp->snd_una)) {  in bbr_output_wtime()
13070 			    rsm, tp, bbr, rsm->r_start, tp->snd_una);  in bbr_output_wtime()
13114 		m->m_data += max_linkhdr;  in bbr_output_wtime()
13115 		m->m_len = hdrlen;  in bbr_output_wtime()
13140 			rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);  in bbr_output_wtime()
13147 				if (rsm->r_start != tp->snd_una) {  in bbr_output_wtime()
13155 					return (-EFAULT); /* tcp_drop() */  in bbr_output_wtime()
13157 				len = rsm->r_end - rsm->r_start;  in bbr_output_wtime()
13165 		if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {  in bbr_output_wtime()
13170 			m->m_len += len;  in bbr_output_wtime()
13184 					    tp->snd_una, rsm->r_flags, rsm->r_start,  in bbr_output_wtime()
13190 					    tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);  in bbr_output_wtime()
13194 			m->m_next = tcp_m_copym(  in bbr_output_wtime()
13212 			if (m->m_next == NULL) {  in bbr_output_wtime()
13232 				foo = foo->m_next;  in bbr_output_wtime()
13252 		if (tp->t_flags & TF_ACKNOW)  in bbr_output_wtime()
13274 			m->m_data += max_linkhdr;  in bbr_output_wtime()
13275 		m->m_len = hdrlen;  in bbr_output_wtime()
13278 	m->m_pkthdr.rcvif = (struct ifnet *)0;  in bbr_output_wtime()
13285 		if (tp->t_port) {  in bbr_output_wtime()
13287 			udp->uh_sport = htons(V_tcp_udp_tunneling_port);  in bbr_output_wtime()
13288 			udp->uh_dport = tp->t_port;  in bbr_output_wtime()
13289 			ulen = hdrlen + len - sizeof(struct ip6_hdr);  in bbr_output_wtime()
13290 			udp->uh_ulen = htons(ulen);  in bbr_output_wtime()
13295 		tcpip_fillheaders(inp, tp->t_port, ip6, th);  in bbr_output_wtime()
13300 		if (tp->t_port) {  in bbr_output_wtime()
13302 			udp->uh_sport = htons(V_tcp_udp_tunneling_port);  in bbr_output_wtime()
13303 			udp->uh_dport = tp->t_port;  in bbr_output_wtime()
13304 			ulen = hdrlen + len - sizeof(struct ip);  in bbr_output_wtime()
13305 			udp->uh_ulen = htons(ulen);  in bbr_output_wtime()
13310 		tcpip_fillheaders(inp, tp->t_port, ip, th);  in bbr_output_wtime()
13326 			th->th_seq = htonl(tp->snd_max);  in bbr_output_wtime()
13327 			bbr_seq = tp->snd_max;  in bbr_output_wtime()
13330 			th->th_seq = htonl(tp->iss);  in bbr_output_wtime()
13331 			bbr_seq = tp->iss;  in bbr_output_wtime()
13333 			if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) {  in bbr_output_wtime()
13338 				th->th_seq = (htonl(tp->snd_max - 1));  in bbr_output_wtime()
13339 				bbr_seq = (tp->snd_max - 1);  in bbr_output_wtime()
13342 				th->th_seq = htonl(tp->snd_max);  in bbr_output_wtime()
13343 				bbr_seq = tp->snd_max;  in bbr_output_wtime()
13355 			 * might be better to send (tp->snd_una - 1) which  in bbr_output_wtime()
13358 			if (tp->t_flags & TF_SENTFIN) {  in bbr_output_wtime()
13359 				th->th_seq = htonl(tp->snd_max - 1);  in bbr_output_wtime()
13360 				bbr_seq = (tp->snd_max - 1);  in bbr_output_wtime()
13362 				th->th_seq = htonl(tp->snd_max);  in bbr_output_wtime()
13363 				bbr_seq = tp->snd_max;  in bbr_output_wtime()
13368 		th->th_seq = htonl(rsm->r_start);  in bbr_output_wtime()
13369 		bbr_seq = rsm->r_start;  in bbr_output_wtime()
13371 	th->th_ack = htonl(tp->rcv_nxt);  in bbr_output_wtime()
13374 		th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;  in bbr_output_wtime()
13381 	if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) &&  in bbr_output_wtime()
13384 	if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&  in bbr_output_wtime()
13385 	    recwin < (tp->rcv_adv - tp->rcv_nxt))  in bbr_output_wtime()
13386 		recwin = (tp->rcv_adv - tp->rcv_nxt);  in bbr_output_wtime()
13387 	if (recwin > TCP_MAXWIN << tp->rcv_scale)  in bbr_output_wtime()
13388 		recwin = TCP_MAXWIN << tp->rcv_scale;  in bbr_output_wtime()
13396 		th->th_win = htons((u_short)  in bbr_output_wtime()
13397 		    (min(sbspace(&so->so_rcv), TCP_MAXWIN)));  in bbr_output_wtime()
13400 		recwin = roundup2(recwin, 1 << tp->rcv_scale);  in bbr_output_wtime()
13401 		th->th_win = htons((u_short)(recwin >> tp->rcv_scale));  in bbr_output_wtime()
13404 	 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0  in bbr_output_wtime()
13411 	if (th->th_win == 0) {  in bbr_output_wtime()
13412 		tp->t_sndzerowin++;  in bbr_output_wtime()
13413 		tp->t_flags |= TF_RXWIN0SENT;  in bbr_output_wtime()
13415 		tp->t_flags &= ~TF_RXWIN0SENT;  in bbr_output_wtime()
13420 	tp->snd_up = tp->snd_una;  in bbr_output_wtime()
13425 	m->m_pkthdr.len = hdrlen + len;	/* in6_cksum() need this */  in bbr_output_wtime()
13435 		    (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {  in bbr_output_wtime()
13451 		if (tp->t_port) {  in bbr_output_wtime()
13452 			m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;  in bbr_output_wtime()
13453 			m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);  in bbr_output_wtime()
13454 			udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);  in bbr_output_wtime()
13455 			th->th_sum = htons(0);  in bbr_output_wtime()
13458 			csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;  in bbr_output_wtime()
13459 			m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);  in bbr_output_wtime()
13460 			th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) +  in bbr_output_wtime()
13470 		if (tp->t_port) {  in bbr_output_wtime()
13471 			m->m_pkthdr.csum_flags = CSUM_UDP;  in bbr_output_wtime()
13472 			m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);  in bbr_output_wtime()
13473 			udp->uh_sum = in_pseudo(ip->ip_src.s_addr,  in bbr_output_wtime()
13474 			    ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));  in bbr_output_wtime()
13475 			th->th_sum = htons(0);  in bbr_output_wtime()
13478 			csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP;  in bbr_output_wtime()
13479 			m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);  in bbr_output_wtime()
13480 			th->th_sum = in_pseudo(ip->ip_src.s_addr,  in bbr_output_wtime()
13481 			    ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +  in bbr_output_wtime()
13485 		KASSERT(ip->ip_v == IPVERSION,  in bbr_output_wtime()
13486 		    ("%s: IP version incorrect: %d", __func__, ip->ip_v));  in bbr_output_wtime()
13498 		m->m_pkthdr.csum_flags |= CSUM_TSO;  in bbr_output_wtime()
13500 		m->m_pkthdr.tso_segsz = maxseg;  in bbr_output_wtime()
13517 		log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;  in bbr_output_wtime()
13518 		log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);  in bbr_output_wtime()
13521 		log.u_bbr.flex5 = bbr->rc_past_init_win;  in bbr_output_wtime()
13523 		log.u_bbr.flex5 |= bbr->rc_no_pacing;  in bbr_output_wtime()
13525 		log.u_bbr.flex5 |= tp->t_maxseg;  in bbr_output_wtime()
13526 		log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;  in bbr_output_wtime()
13527 		log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);  in bbr_output_wtime()
13529 		log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;  in bbr_output_wtime()
13538 		lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,  in bbr_output_wtime()
13549 	 * m->m_pkthdr.len should have been set before cksum calcuration,  in bbr_output_wtime()
13560 		ip6->ip6_hlim = in6_selecthlim(inp, NULL);  in bbr_output_wtime()
13567 		ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));  in bbr_output_wtime()
13570 			tp->t_flags2 |= TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13572 			tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13574 		if (tp->t_state == TCPS_SYN_SENT)  in bbr_output_wtime()
13579 		error = ip6_output(m, inp->in6p_outputopts,  in bbr_output_wtime()
13580 		    &inp->inp_route6,  in bbr_output_wtime()
13584 		if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)  in bbr_output_wtime()
13585 			mtu = inp->inp_route6.ro_nh->nh_mtu;  in bbr_output_wtime()
13593 		ip->ip_len = htons(m->m_pkthdr.len);  in bbr_output_wtime()
13596 			ip->ip_ttl = in6_selecthlim(inp, NULL);  in bbr_output_wtime()
13608 		if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {  in bbr_output_wtime()
13609 			tp->t_flags2 |= TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13610 			if (tp->t_port == 0 || len < V_tcp_minmss) {  in bbr_output_wtime()
13611 				ip->ip_off |= htons(IP_DF);  in bbr_output_wtime()
13614 			tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;  in bbr_output_wtime()
13617 		if (tp->t_state == TCPS_SYN_SENT)  in bbr_output_wtime()
13622 		error = ip_output(m, inp->inp_options, &inp->inp_route,  in bbr_output_wtime()
13625 		if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)  in bbr_output_wtime()
13626 			mtu = inp->inp_route.ro_nh->nh_mtu;  in bbr_output_wtime()
13630 		lgb->tlb_errno = error;  in bbr_output_wtime()
13641 		if (TCPS_HAVEESTABLISHED(tp->t_state) &&  in bbr_output_wtime()
13642 		    (tp->t_flags & TF_SACK_PERMIT) &&  in bbr_output_wtime()
13643 		    tp->rcv_numsacks > 0)  in bbr_output_wtime()
13646 		bbr->output_error_seen = 0;  in bbr_output_wtime()
13647 		bbr->oerror_cnt = 0;  in bbr_output_wtime()
13648 		bbr->bbr_segs_rcvd = 0;  in bbr_output_wtime()
13654 			if (tp->snd_una == tp->snd_max) {  in bbr_output_wtime()
13661 				bbr->r_ctl.rc_del_time = cts;  in bbr_output_wtime()
13666 					counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1);  in bbr_output_wtime()
13671 				idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);  in bbr_output_wtime()
13673 					idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1);  in bbr_output_wtime()
13687 			if (tp->snd_una == tp->snd_max)  in bbr_output_wtime()
13688 				bbr->r_ctl.rc_tlp_rxt_last_time = cts;  in bbr_output_wtime()
13700 	if (bbr->rc_in_persist == 0) {  in bbr_output_wtime()
13708 		if (tp->snd_una == tp->snd_max &&  in bbr_output_wtime()
13715 			bbr->rc_tp->t_acktime  = ticks;  in bbr_output_wtime()
13723 				tp->snd_max = tp->iss + 1;  in bbr_output_wtime()
13725 			if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {  in bbr_output_wtime()
13726 				tp->snd_max++;  in bbr_output_wtime()
13727 				tp->t_flags |= TF_SENTFIN;  in bbr_output_wtime()
13731 			tp->snd_max += len;  in bbr_output_wtime()
13744 		if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {  in bbr_output_wtime()
13746 			tp->t_flags |= TF_SENTFIN;  in bbr_output_wtime()
13748 		if (xlen && (tp->snd_una == tp->snd_max)) {  in bbr_output_wtime()
13754 			bbr->rc_tp->t_acktime = ticks;  in bbr_output_wtime()
13757 			tp->snd_max += xlen;  in bbr_output_wtime()
13764 		 * case of ENOBUFS we will fall out and become ack-clocked.  in bbr_output_wtime()
13772 		bbr->r_ctl.rc_hptsi_agg_delay = 0;  in bbr_output_wtime()
13773 		bbr->r_ctl.rc_agg_early = 0;  in bbr_output_wtime()
13774 		bbr->r_agg_early_set = 0;  in bbr_output_wtime()
13775 		bbr->output_error_seen = 1;  in bbr_output_wtime()
13776 		if (bbr->oerror_cnt < 0xf)  in bbr_output_wtime()
13777 			bbr->oerror_cnt++;  in bbr_output_wtime()
13778 		if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {  in bbr_output_wtime()
13780 			return (-ENETDOWN);  in bbr_output_wtime()
13787 			 * slam him below a T-O (1MSS).  in bbr_output_wtime()
13789 			if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {  in bbr_output_wtime()
13790 				tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
13791 								    bbr->r_ctl.rc_lost_bytes)) - maxseg;  in bbr_output_wtime()
13792 				if (tp->snd_cwnd < maxseg)  in bbr_output_wtime()
13793 					tp->snd_cwnd = maxseg;  in bbr_output_wtime()
13795 			slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;  in bbr_output_wtime()
13797 			if (bbr->bbr_hdrw_pacing)  in bbr_output_wtime()
13822 				old_maxseg = tp->t_maxseg;  in bbr_output_wtime()
13826 					tcp_mss_update(tp, -1, mtu, NULL, NULL);  in bbr_output_wtime()
13827 				if (old_maxseg <= tp->t_maxseg) {  in bbr_output_wtime()
13829 					tp->t_maxseg = old_maxseg - 40;  in bbr_output_wtime()
13830 					if (tp->t_maxseg < V_tcp_mssdflt) {  in bbr_output_wtime()
13836 						tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;  in bbr_output_wtime()
13838 						tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;  in bbr_output_wtime()
13846 				if ((tot_len + len) && (len >= tp->t_maxseg)) {  in bbr_output_wtime()
13848 					    bbr->r_ctl.rc_bbr_hptsi_gain,  in bbr_output_wtime()
13851 						slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;  in bbr_output_wtime()
13853 					slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;  in bbr_output_wtime()
13854 				bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
13861 			tp->t_softerror = error;  in bbr_output_wtime()
13867 			if (TCPS_HAVERCVDSYN(tp->t_state)) {  in bbr_output_wtime()
13868 				tp->t_softerror = error;  in bbr_output_wtime()
13873 			slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;  in bbr_output_wtime()
13874 			bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
13879 	} else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&  in bbr_output_wtime()
13882 	    (bbr->rc_in_persist == 0)) {  in bbr_output_wtime()
13883 		tp->gput_seq = bbr_seq;  in bbr_output_wtime()
13884 		tp->gput_ack = bbr_seq +  in bbr_output_wtime()
13885 		    min(sbavail(&so->so_snd) - sb_offset, sendwin);  in bbr_output_wtime()
13886 		tp->gput_ts = cts;  in bbr_output_wtime()
13887 		tp->t_flags |= TF_GPUTINPROG;  in bbr_output_wtime()
13891 	if ((bbr->bbr_hdw_pace_ena) &&  in bbr_output_wtime()
13892 	    (bbr->bbr_attempt_hdwr_pace == 0) &&  in bbr_output_wtime()
13893 	    (bbr->rc_past_init_win) &&  in bbr_output_wtime()
13894 	    (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&  in bbr_output_wtime()
13895 	    (get_filter_value(&bbr->r_ctl.rc_delrate)) &&  in bbr_output_wtime()
13896 	    (inp->inp_route.ro_nh &&  in bbr_output_wtime()
13897 	     inp->inp_route.ro_nh->nh_ifp)) {  in bbr_output_wtime()
13909 		bbr->bbr_attempt_hdwr_pace = 1;  in bbr_output_wtime()
13910 		bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,  in bbr_output_wtime()
13911 						      inp->inp_route.ro_nh->nh_ifp,  in bbr_output_wtime()
13915 		if (bbr->r_ctl.crte) {  in bbr_output_wtime()
13917 						 bbr->r_ctl.crte->ptbl->rs_ifp,  in bbr_output_wtime()
13919 						 bbr->r_ctl.crte->rate,  in bbr_output_wtime()
13922 			counter_u64_add(bbr_flows_nohdwr_pacing, -1);  in bbr_output_wtime()
13924 			bbr->bbr_hdrw_pacing = 1;  in bbr_output_wtime()
13926 			if (bbr->r_ctl.crte->rate < rate_wanted) {  in bbr_output_wtime()
13929 						       bbr->r_ctl.crte->rate, rate_wanted);  in bbr_output_wtime()
13932 				bbr->gain_is_limited = 0;  in bbr_output_wtime()
13933 				bbr->skip_gain = 0;  in bbr_output_wtime()
13938 						 inp->inp_route.ro_nh->nh_ifp,  in bbr_output_wtime()
13945 	if (bbr->bbr_hdrw_pacing) {  in bbr_output_wtime()
13952 		if (inp->inp_snd_tag == NULL) {  in bbr_output_wtime()
13954 			bbr->bbr_hdrw_pacing = 0;  in bbr_output_wtime()
13955 		} else if ((inp->inp_route.ro_nh == NULL) ||  in bbr_output_wtime()
13956 		    (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) {  in bbr_output_wtime()
13960 			 * and setup to re-attempt next go  in bbr_output_wtime()
13963 			bbr->bbr_hdrw_pacing = 0;  in bbr_output_wtime()
13964 			bbr->bbr_attempt_hdwr_pace = 0;  in bbr_output_wtime()
13965 			tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);  in bbr_output_wtime()
13974 	if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))  in bbr_output_wtime()
13975 		tp->rcv_adv = tp->rcv_nxt + recwin;  in bbr_output_wtime()
13977 	tp->last_ack_sent = tp->rcv_nxt;  in bbr_output_wtime()
13979 	    (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&  in bbr_output_wtime()
13985 	    (IN_RECOVERY(tp->t_flags) == 0) &&  in bbr_output_wtime()
13986 	    (bbr->rc_in_persist == 0) &&  in bbr_output_wtime()
13987 	    (tot_len < bbr->r_ctl.rc_pace_max_segs)) {  in bbr_output_wtime()
13989 		 * For non-tso we need to goto again until we have sent out  in bbr_output_wtime()
13993 		if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_output_wtime()
13995 			tp->snd_nxt = tp->snd_max;  in bbr_output_wtime()
14003 		tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);  in bbr_output_wtime()
14013 		 * Calculate/Re-Calculate the hptsi slot in usecs based on  in bbr_output_wtime()
14016 		slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);  in bbr_output_wtime()
14017 		if (bbr->rc_no_pacing)  in bbr_output_wtime()
14020 	tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);  in bbr_output_wtime()
14022 	if (bbr->rc_use_google == 0)  in bbr_output_wtime()
14024 	bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +  in bbr_output_wtime()
14025 							bbr->r_ctl.rc_lost_bytes)));  in bbr_output_wtime()
14026 	bbr->rc_output_starts_timer = 1;  in bbr_output_wtime()
14027 	if (bbr->bbr_use_rack_cheat &&  in bbr_output_wtime()
14029 	     ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {  in bbr_output_wtime()
14034 	if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {  in bbr_output_wtime()
14040 		bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;  in bbr_output_wtime()
14041 		if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {  in bbr_output_wtime()
14042 			bbr->hw_pacing_set = 1;  in bbr_output_wtime()
14047 	if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {  in bbr_output_wtime()
14049 		tp->snd_nxt = tp->snd_max;  in bbr_output_wtime()
14084 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_mtu_chg()
14085 	maxseg = tp->t_maxseg - bbr->rc_last_options;  in bbr_mtu_chg()
14086 	sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);  in bbr_mtu_chg()
14087 	TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {  in bbr_mtu_chg()
14089 		if (rsm->r_flags & BBR_ACKED)  in bbr_mtu_chg()
14091 		if ((rsm->r_end - rsm->r_start) > maxseg) {  in bbr_mtu_chg()
14093 			 * We mark sack-passed on all the previous large  in bbr_mtu_chg()
14096 			rsm->r_flags |= BBR_SACK_PASSED;  in bbr_mtu_chg()
14097 			if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&  in bbr_mtu_chg()
14098 			    bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {  in bbr_mtu_chg()
14099 				bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;  in bbr_mtu_chg()
14100 				bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;  in bbr_mtu_chg()
14101 				rsm->r_flags |= BBR_MARKED_LOST;  in bbr_mtu_chg()
14108 		bbr->r_ctl.rc_resend = frsm;  in bbr_mtu_chg()
14140 	tp->t_flags2 |= TF2_CANNOT_DO_ECN;  in bbr_switch_failed()
14141 	tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;  in bbr_switch_failed()
14143 	if (tp->t_in_hpts > IHPTS_NONE) {  in bbr_switch_failed()
14146 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_switch_failed()
14148 	if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {  in bbr_switch_failed()
14149 		if (TSTMP_GT(bbr->rc_pacer_started, cts)) {  in bbr_switch_failed()
14150 			toval = bbr->rc_pacer_started - cts;  in bbr_switch_failed()
14155 	} else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {  in bbr_switch_failed()
14156 		if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {  in bbr_switch_failed()
14157 			toval = bbr->r_ctl.rc_timer_exp - cts;  in bbr_switch_failed()
14189  * socket option arguments.  When it re-acquires the lock after the copy, it
14201 	switch (sopt->sopt_level) {  in bbr_set_sockopt()
14207 	switch (sopt->sopt_name) {  in bbr_set_sockopt()
14256 	if (inp->inp_flags & INP_DROPPED) {  in bbr_set_sockopt()
14260 	if (tp->t_fb != &__tcp_bbr) {  in bbr_set_sockopt()
14264 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_set_sockopt()
14265 	switch (sopt->sopt_name) {  in bbr_set_sockopt()
14268 		bbr->r_ctl.bbr_hptsi_per_second = optval;  in bbr_set_sockopt()
14272 		bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;  in bbr_set_sockopt()
14276 		bbr->r_ctl.bbr_hptsi_segments_max = optval;  in bbr_set_sockopt()
14280 		bbr->r_ctl.bbr_hptsi_bytes_min = optval;  in bbr_set_sockopt()
14284 		bbr->r_ctl.bbr_cross_over = optval;  in bbr_set_sockopt()
14288 		if (optval && (bbr->rc_use_google == 0)) {  in bbr_set_sockopt()
14296 				bbr->r_ctl.bbr_google_discount = optval;  in bbr_set_sockopt()
14298 		} else if ((optval == 0) && (bbr->rc_use_google == 1)) {  in bbr_set_sockopt()
14306 			bbr->rc_use_ts_limit = 1;  in bbr_set_sockopt()
14308 			bbr->rc_use_ts_limit = 0;  in bbr_set_sockopt()
14318 			bbr->rc_init_win = optval;  in bbr_set_sockopt()
14320 			if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))  in bbr_set_sockopt()
14321 				tp->snd_cwnd = twin;  in bbr_set_sockopt()
14330 			bbr->r_ctl.rc_startup_pg = optval;  in bbr_set_sockopt()
14331 			if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {  in bbr_set_sockopt()
14332 				bbr->r_ctl.rc_bbr_hptsi_gain = optval;  in bbr_set_sockopt()
14340 			bbr->r_ctl.rc_drain_pg = optval;  in bbr_set_sockopt()
14347 			reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));  in bbr_set_sockopt()
14354 			bbr->r_ctl.bbr_rttprobe_gain_val = optval;  in bbr_set_sockopt()
14361 			bbr->r_ctl.rc_probertt_int = optval;  in bbr_set_sockopt()
14368 			bbr->no_pacing_until = 0;  in bbr_set_sockopt()
14369 			bbr->rc_no_pacing = 0;  in bbr_set_sockopt()
14371 			bbr->no_pacing_until = optval;  in bbr_set_sockopt()
14372 			if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&  in bbr_set_sockopt()
14373 			    (bbr->rc_bbr_state == BBR_STATE_STARTUP)){  in bbr_set_sockopt()
14375 				bbr->rc_no_pacing = 1;  in bbr_set_sockopt()
14382 		bbr->rc_loss_exit = optval;  in bbr_set_sockopt()
14389 		bbr->r_ctl.rc_min_rto_ms = optval;  in bbr_set_sockopt()
14393 		bbr->rc_max_rto_sec = optval;  in bbr_set_sockopt()
14396 		/* Minimum time between rack t-o's in ms */  in bbr_set_sockopt()
14398 		bbr->r_ctl.rc_min_to = optval;  in bbr_set_sockopt()
14404 			bbr->r_ctl.rc_reorder_shift = optval;  in bbr_set_sockopt()
14411 		bbr->r_ctl.rc_reorder_fade = optval;  in bbr_set_sockopt()
14417 			bbr->rc_tlp_threshold = optval;  in bbr_set_sockopt()
14423 		if (bbr->rc_use_google) {  in bbr_set_sockopt()
14429 			bbr->bbr_use_rack_cheat = 1;  in bbr_set_sockopt()
14431 			bbr->bbr_use_rack_cheat = 0;  in bbr_set_sockopt()
14436 			bbr->r_ctl.bbr_hptsi_segments_floor = optval;  in bbr_set_sockopt()
14443 			bbr->r_ctl.bbr_utter_max = optval;  in bbr_set_sockopt()
14451 			bbr->rc_use_idle_restart = 1;  in bbr_set_sockopt()
14453 			bbr->rc_use_idle_restart = 0;  in bbr_set_sockopt()
14458 			bbr->bbr_init_win_cheat = 1;  in bbr_set_sockopt()
14459 			if (bbr->rc_past_init_win == 0) {  in bbr_set_sockopt()
14461 				cts = tcp_get_usecs(&bbr->rc_tv);  in bbr_set_sockopt()
14465 			bbr->bbr_init_win_cheat = 0;  in bbr_set_sockopt()
14470 			bbr->bbr_hdw_pace_ena = 1;  in bbr_set_sockopt()
14471 			bbr->bbr_attempt_hdwr_pace = 0;  in bbr_set_sockopt()
14473 			bbr->bbr_hdw_pace_ena = 0;  in bbr_set_sockopt()
14475 			if (bbr->r_ctl.crte != NULL) {  in bbr_set_sockopt()
14476 				tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);  in bbr_set_sockopt()
14477 				bbr->r_ctl.crte = NULL;  in bbr_set_sockopt()
14487 				tp->t_delayed_ack = 0;  in bbr_set_sockopt()
14489 				tp->t_delayed_ack = 2;  in bbr_set_sockopt()
14491 				tp->t_delayed_ack = optval;  in bbr_set_sockopt()
14492 			if (tp->t_flags & TF_DELACK) {  in bbr_set_sockopt()
14493 				tp->t_flags &= ~TF_DELACK;  in bbr_set_sockopt()
14494 				tp->t_flags |= TF_ACKNOW;  in bbr_set_sockopt()
14503 		/* RACK added ms i.e. rack-rtt + reord + N */  in bbr_set_sockopt()
14505 		bbr->r_ctl.rc_pkt_delay = optval;  in bbr_set_sockopt()
14511 			bbr->rc_resends_use_tso = 1;  in bbr_set_sockopt()
14513 			bbr->rc_resends_use_tso = 0;  in bbr_set_sockopt()
14518 			bbr->rc_allow_data_af_clo = 1;  in bbr_set_sockopt()
14520 			bbr->rc_allow_data_af_clo = 0;  in bbr_set_sockopt()
14524 		if (bbr->rc_use_google == 0)  in bbr_set_sockopt()
14527 			bbr->r_use_policer = 1;  in bbr_set_sockopt()
14529 			bbr->r_use_policer = 0;  in bbr_set_sockopt()
14535 			bbr->ts_can_raise = 1;  in bbr_set_sockopt()
14537 			bbr->ts_can_raise = 0;  in bbr_set_sockopt()
14541 		if (bbr->rc_use_google) {  in bbr_set_sockopt()
14545 				bbr->r_ctl.rc_incr_tmrs = 1;  in bbr_set_sockopt()
14547 				bbr->r_ctl.rc_incr_tmrs = 0;  in bbr_set_sockopt()
14552 		if (bbr->rc_use_google) {  in bbr_set_sockopt()
14562 				bbr->r_ctl.rc_inc_tcp_oh = 1;  in bbr_set_sockopt()
14564 				bbr->r_ctl.rc_inc_tcp_oh = 0;  in bbr_set_sockopt()
14566 				bbr->r_ctl.rc_inc_ip_oh = 1;  in bbr_set_sockopt()
14568 				bbr->r_ctl.rc_inc_ip_oh = 0;  in bbr_set_sockopt()
14570 				bbr->r_ctl.rc_inc_enet_oh = 1;  in bbr_set_sockopt()
14572 				bbr->r_ctl.rc_inc_enet_oh = 0;  in bbr_set_sockopt()
14579 	tcp_log_socket_option(tp, sopt->sopt_name, optval, error);  in bbr_set_sockopt()
14585  * return 0 on success, error-num on failure
14594 	bbr = (struct tcp_bbr *)tp->t_fb_ptr;  in bbr_get_sockopt()
14605 	switch (sopt->sopt_name) {  in bbr_get_sockopt()
14607 		optval = bbr->r_ctl.bbr_hptsi_per_second;  in bbr_get_sockopt()
14610 		optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;  in bbr_get_sockopt()
14613 		optval = bbr->r_ctl.bbr_hptsi_segments_max;  in bbr_get_sockopt()
14616 		optval = bbr->no_pacing_until;  in bbr_get_sockopt()
14619 		optval = bbr->r_ctl.bbr_hptsi_bytes_min;  in bbr_get_sockopt()
14622 		optval = bbr->r_ctl.bbr_cross_over;  in bbr_get_sockopt()
14625 		optval = bbr->rc_use_google;  in bbr_get_sockopt()
14628 		optval = bbr->rc_use_ts_limit;  in bbr_get_sockopt()
14631 		optval = bbr->rc_init_win;  in bbr_get_sockopt()
14634 		optval = bbr->r_ctl.rc_startup_pg;  in bbr_get_sockopt()
14637 		optval = bbr->r_ctl.rc_drain_pg;  in bbr_get_sockopt()
14640 		optval = bbr->r_ctl.rc_probertt_int;  in bbr_get_sockopt()
14643 		optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);  in bbr_get_sockopt()
14646 		optval = bbr->r_ctl.bbr_rttprobe_gain_val;  in bbr_get_sockopt()
14649 		optval = bbr->rc_loss_exit;  in bbr_get_sockopt()
14655 		optval = bbr->r_ctl.rc_min_rto_ms;  in bbr_get_sockopt()
14658 		optval = bbr->rc_max_rto_sec;  in bbr_get_sockopt()
14662 		optval = bbr->r_ctl.rc_pace_max_segs;  in bbr_get_sockopt()
14665 		/* Minimum time between rack t-o's in ms */  in bbr_get_sockopt()
14666 		optval = bbr->r_ctl.rc_min_to;  in bbr_get_sockopt()
14670 		optval = bbr->r_ctl.rc_reorder_shift;  in bbr_get_sockopt()
14674 		optval = bbr->r_ctl.rc_reorder_fade;  in bbr_get_sockopt()
14678 		optval = bbr->bbr_use_rack_cheat;  in bbr_get_sockopt()
14681 		optval = bbr->r_ctl.bbr_hptsi_segments_floor;  in bbr_get_sockopt()
14684 		optval = bbr->r_ctl.bbr_utter_max;  in bbr_get_sockopt()
14688 		optval = bbr->bbr_init_win_cheat;  in bbr_get_sockopt()
14691 		optval = bbr->rc_use_idle_restart;  in bbr_get_sockopt()
14695 		optval = bbr->rc_tlp_threshold;  in bbr_get_sockopt()
14698 		/* RACK added ms i.e. rack-rtt + reord + N */  in bbr_get_sockopt()
14699 		optval = bbr->r_ctl.rc_pkt_delay;  in bbr_get_sockopt()
14702 		optval = bbr->rc_resends_use_tso;  in bbr_get_sockopt()
14705 		optval = bbr->rc_allow_data_af_clo;  in bbr_get_sockopt()
14708 		optval = tp->t_delayed_ack;  in bbr_get_sockopt()
14711 		optval = bbr->bbr_hdw_pace_ena;  in bbr_get_sockopt()
14714 		optval = bbr->r_use_policer;  in bbr_get_sockopt()
14717 		optval = bbr->ts_can_raise;  in bbr_get_sockopt()
14720 		optval = bbr->r_ctl.rc_incr_tmrs;  in bbr_get_sockopt()
14724 		if (bbr->r_ctl.rc_inc_tcp_oh)  in bbr_get_sockopt()
14726 		if (bbr->r_ctl.rc_inc_ip_oh)  in bbr_get_sockopt()
14728 		if (bbr->r_ctl.rc_inc_enet_oh)  in bbr_get_sockopt()
14741  * return 0 on success, error-num on failure
14746 	if (sopt->sopt_dir == SOPT_SET) {  in bbr_ctloutput()
14748 	} else if (sopt->sopt_dir == SOPT_GET) {  in bbr_ctloutput()
14751 		panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);  in bbr_ctloutput()