Lines Matching full:cts
443 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses);
445 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line,
450 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch,
453 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts,
459 struct bbr_sendmap *rsm, uint32_t srtt, uint32_t cts);
461 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
466 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line);
471 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts);
473 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts);
475 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
483 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot,
486 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts,
490 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag);
493 uint32_t del_by, uint32_t cts, uint32_t sloton,
496 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
501 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts);
503 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);
505 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);
508 uint32_t cts, uint32_t usecs, uint64_t bw,
551 bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timer_start() argument
590 if (TSTMP_GT(tstmp_touse, cts)) in bbr_timer_start()
591 time_since_sent = cts - tstmp_touse; in bbr_timer_start()
608 bbr_log_timer_var(bbr, 2, cts, 0, bbr_get_rtt(bbr, BBR_SRTT), 0, to); in bbr_timer_start()
632 thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm); in bbr_timer_start()
635 if (SEQ_GEQ(exp, cts)) { in bbr_timer_start()
636 to = exp - cts; in bbr_timer_start()
667 if (TSTMP_GT(tstmp_touse, cts)) in bbr_timer_start()
668 time_since_sent = cts - tstmp_touse; in bbr_timer_start()
671 thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts); in bbr_timer_start()
705 bbr_log_timer_var(bbr, 1, cts, time_since_sent, srtt, thresh, to); in bbr_timer_start()
727 bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t slot… in bbr_start_hpts_timer() argument
747 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { in bbr_start_hpts_timer()
748 left = bbr->r_ctl.rc_timer_exp - cts; in bbr_start_hpts_timer()
761 if (TSTMP_GT(cts, bbr->rc_pacer_started)) { in bbr_start_hpts_timer()
763 delay_calc = cts - bbr->rc_pacer_started; in bbr_start_hpts_timer()
770 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2); in bbr_start_hpts_timer()
788 hpts_timeout = bbr_timer_start(tp, bbr, cts); in bbr_start_hpts_timer()
879 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; in bbr_start_hpts_timer()
901 bbr->rc_pacer_started = cts; in bbr_start_hpts_timer()
907 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 1); in bbr_start_hpts_timer()
908 bbr_log_hpts_diag(bbr, cts, &diag); in bbr_start_hpts_timer()
923 bbr->rc_pacer_started = cts; in bbr_start_hpts_timer()
939 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 0); in bbr_start_hpts_timer()
940 bbr_log_hpts_diag(bbr, cts, &diag); in bbr_start_hpts_timer()
944 bbr_log_type_bbrsnd(bbr, tot_len, slot, delay_calc, cts, frm, prev_delay); in bbr_start_hpts_timer()
948 bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb) in bbr_timer_audit() argument
1043 bbr_timer_cancel(bbr, __LINE__, cts); in bbr_timer_audit()
1044 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, in bbr_timer_audit()
1055 hpts_timeout = bbr_timer_start(tp, bbr, cts); in bbr_timer_audit()
1069 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; in bbr_timer_audit()
1077 * Considers the two time values now (cts) and earlier.
1078 * If cts is smaller than earlier, we could have
1083 * a case (where cts is behind earlier time by
1088 bbr_calc_time(uint32_t cts, uint32_t earlier_time) { in bbr_calc_time() argument
1090 * Given two timestamps, the current time stamp cts, and some other in bbr_calc_time()
1093 * after the cts. If this occurs we need to return 0. in bbr_calc_time()
1095 if (TSTMP_GEQ(cts, earlier_time)) in bbr_calc_time()
1096 return (cts - earlier_time); in bbr_calc_time()
1098 * cts is behind earlier_time if its less than 10ms consider it 0. in bbr_calc_time()
1103 if (TSTMP_GEQ((cts + 10000), earlier_time)) in bbr_calc_time()
1109 return (cts - earlier_time); in bbr_calc_time()
1847 bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts) in bbr_fill_in_logging_data() argument
1858 l->timeStamp = cts; in bbr_fill_in_logging_data()
1914 bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling, in bbr_log_type_just_return() argument
1920 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_just_return()
1958 …cb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts) in bbr_log_msgsize_fail() argument
1963 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_msgsize_fail()
1999 bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, in bbr_log_pkt_epoch() argument
2005 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_pkt_epoch()
2024 bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time) in bbr_log_time_epoch() argument
2029 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_time_epoch()
2067 bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line) in bbr_log_type_statechange() argument
2072 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_statechange()
2094 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, in bbr_log_rtt_shrinks() argument
2100 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_rtt_shrinks()
2187 bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t ente… in bbr_log_type_pesist() argument
2192 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_pesist()
2204 bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts) in bbr_log_ack_clear() argument
2209 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_ack_clear()
2225 uint16_t nsegs, uint32_t cts, int32_t nxt_pkt, struct mbuf *m) in bbr_log_ack_event() argument
2231 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_ack_event()
2273 bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out) in bbr_log_doseg_done() argument
2278 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_doseg_done()
2297 bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts, in bbr_log_enobuf_jmp() argument
2303 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_enobuf_jmp()
2317 bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_… in bbr_log_to_processing() argument
2322 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_to_processing()
2327 log.u_bbr.flex5 = cts; in bbr_log_to_processing()
2339 bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num) in bbr_log_to_event() argument
2345 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_to_event()
2367 bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t f… in bbr_log_startup_event() argument
2372 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_startup_event()
2390 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag) in bbr_log_hpts_diag() argument
2395 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_hpts_diag()
2426 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t s… in bbr_log_timer_var() argument
2432 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_timer_var()
2450 uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod) in bbr_log_pacing_delay_calc() argument
2455 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_pacing_delay_calc()
2476 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which) in bbr_log_to_start() argument
2481 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_to_start()
2500 bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t sr… in bbr_log_thresh_choice() argument
2505 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_thresh_choice()
2523 bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed) in bbr_log_to_cancel() argument
2528 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_to_cancel()
2568 bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_… in bbr_log_type_tsosize() argument
2573 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_tsosize()
2596 bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, in bbr_log_type_rsmclear() argument
2602 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_rsmclear()
2622 bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts, in bbr_log_type_bbrupd() argument
2631 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_bbrupd()
2655 bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason, in bbr_log_type_ltbw() argument
2662 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_ltbw()
2706 uint64_t rate, uint64_t hw_rate, int line, uint32_t cts, in bbr_type_log_hdwr_pacing() argument
2713 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_type_log_hdwr_pacing()
2736 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, uint32_t del_by, uint32_t cts… in bbr_log_type_bbrsnd() argument
2741 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_bbrsnd()
2759 …p(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uin… in bbr_log_type_bbrrttprop() argument
2764 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_type_bbrrttprop()
2782 bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method) in bbr_log_exit_gain() argument
2787 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_log_exit_gain()
2841 bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line) in bbr_set_pktepoch() argument
2868 tcp_bbr_tso_size_check(bbr, cts); in bbr_set_pktepoch()
2870 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time); in bbr_set_pktepoch()
2871 bbr->r_ctl.rc_pkt_epoch_time = cts; in bbr_set_pktepoch()
2873 bbr_log_pkt_epoch(bbr, cts, line, lost, del); in bbr_set_pktepoch()
2879 bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line) in bbr_set_epoch() argument
2885 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start; in bbr_set_epoch()
2886 bbr_log_time_epoch(bbr, cts, line, epoch_time); in bbr_set_epoch()
2887 bbr->r_ctl.rc_rcv_epoch_start = cts; in bbr_set_epoch()
2891 bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int3… in bbr_isit_a_pkt_epoch() argument
3001 bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts) in bbr_reset_lt_bw_interval() argument
3010 bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts) in bbr_reset_lt_bw_sampling() argument
3015 bbr_reset_lt_bw_interval(bbr, cts); in bbr_reset_lt_bw_sampling()
3019 bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin) in bbr_lt_bw_samp_done() argument
3048 bbr_log_type_ltbw(bbr, cts, 4, (uint32_t)bw, saved_bw, (uint32_t)diff, timin); in bbr_lt_bw_samp_done()
3053 bbr_reset_lt_bw_interval(bbr, cts); in bbr_lt_bw_samp_done()
3054 bbr_log_type_ltbw(bbr, cts, 5, 0, (uint32_t)bw, 0, timin); in bbr_lt_bw_samp_done()
3073 bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts) in bbr_pick_probebw_substate() argument
3093 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) in bbr_pick_probebw_substate()
3094 bbr_set_epoch(bbr, cts, __LINE__); in bbr_pick_probebw_substate()
3101 bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected) in bbr_lt_bw_sampling() argument
3114 bbr_reset_lt_bw_sampling(bbr, cts); in bbr_lt_bw_sampling()
3116 bbr_set_epoch(bbr, cts, __LINE__); in bbr_lt_bw_sampling()
3117 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_lt_bw_sampling()
3118 bbr_substate_change(bbr, cts, __LINE__, 0); in bbr_lt_bw_sampling()
3120 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_lt_bw_sampling()
3128 bbr_set_epoch(bbr, cts, __LINE__); in bbr_lt_bw_sampling()
3129 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_lt_bw_sampling()
3134 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_lt_bw_sampling()
3137 bbr_log_type_ltbw(bbr, cts, 0, 0, 0, 0, 0); in bbr_lt_bw_sampling()
3156 bbr_log_type_ltbw(bbr, cts, 7, lost, delivered, 0, 0); in bbr_lt_bw_sampling()
3159 bbr_log_type_ltbw(bbr, cts, 8, lost, delivered, 0, 0); in bbr_lt_bw_sampling()
3175 bbr_reset_lt_bw_interval(bbr, cts); in bbr_lt_bw_sampling()
3177 bbr_log_type_ltbw(bbr, cts, 1, 0, 0, 0, 0); in bbr_lt_bw_sampling()
3189 bbr_reset_lt_bw_sampling(bbr, cts); in bbr_lt_bw_sampling()
3191 bbr_log_type_ltbw(bbr, cts, 2, 0, 0, 0, d_time); in bbr_lt_bw_sampling()
3202 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time); in bbr_lt_bw_sampling()
3213 bbr_reset_lt_bw_sampling(bbr, cts); in bbr_lt_bw_sampling()
3215 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time); in bbr_lt_bw_sampling()
3225 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time); in bbr_lt_bw_sampling()
3233 bbr_log_type_ltbw(bbr, cts, 6, lost, delivered, 0, d_time); in bbr_lt_bw_sampling()
3239 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time); in bbr_lt_bw_sampling()
3244 bbr_reset_lt_bw_sampling(bbr, cts); in bbr_lt_bw_sampling()
3246 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time); in bbr_lt_bw_sampling()
3253 bbr_lt_bw_samp_done(bbr, bw, cts, d_time); in bbr_lt_bw_sampling()
3527 bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog) in bbr_get_pacing_delay() argument
3576 bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1); in bbr_get_pacing_delay()
3893 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts) in bbr_setup_red_bw() argument
3901 tcp_bbr_tso_size_check(bbr, cts); in bbr_setup_red_bw()
4063 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm) in bbr_calc_thresh_rack() argument
4088 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) { in bbr_calc_thresh_rack()
4089 lro = cts - bbr->r_ctl.rc_reorder_ts; in bbr_calc_thresh_rack()
4135 bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK); in bbr_calc_thresh_rack()
4147 uint32_t cts) in bbr_calc_thresh_tlp() argument
4195 bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP); in bbr_calc_thresh_tlp()
4263 bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts) in bbr_is_lost() argument
4268 cts, rsm); in bbr_is_lost()
4269 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) { in bbr_is_lost()
4280 bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_check_recovery_mode() argument
4314 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) { in bbr_check_recovery_mode()
4319 if (bbr_is_lost(bbr, rsm, cts) && in bbr_check_recovery_mode()
4345 bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timeout_rack() argument
4359 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_timeout_rack()
4367 bbr_log_to_event(bbr, cts, BBR_TO_FRM_RACK); in bbr_timeout_rack()
4370 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_timeout_rack()
4373 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); in bbr_timeout_rack()
4532 bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timeout_tlp() argument
4547 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_timeout_tlp()
4686 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TLP); in bbr_timeout_tlp()
4700 bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timeout_delack() argument
4705 bbr_log_to_event(bbr, cts, BBR_TO_FRM_DELACK); in bbr_timeout_delack()
4721 bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timeout_persist() argument
4736 bbr_log_to_event(bbr, cts, BBR_TO_FRM_PERSIST); in bbr_timeout_persist()
4761 bbr_exit_persist(tp, bbr, cts, __LINE__); in bbr_timeout_persist()
4787 bbr_start_hpts_timer(bbr, tp, cts, 3, 0, 0); in bbr_timeout_persist()
4799 bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timeout_keepalive() argument
4808 bbr_log_to_event(bbr, cts, BBR_TO_FRM_KEEP); in bbr_timeout_keepalive()
4839 bbr_start_hpts_timer(bbr, tp, cts, 4, 0, 0); in bbr_timeout_keepalive()
4860 uint32_t cts, lost; in bbr_remxt_tmr() local
4863 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_remxt_tmr()
4885 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__); in bbr_remxt_tmr()
4924 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TMR); in bbr_remxt_tmr()
4935 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); in bbr_remxt_tmr()
4943 bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_timeout_rxt() argument
5174 bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling) in bbr_process_timers() argument
5188 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { in bbr_process_timers()
5193 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling); in bbr_process_timers()
5198 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling); in bbr_process_timers()
5205 left = bbr->r_ctl.rc_timer_exp - cts; in bbr_process_timers()
5207 bbr_log_to_processing(bbr, cts, ret, left, hpts_calling); in bbr_process_timers()
5214 ret = bbr_timeout_delack(tp, bbr, cts); in bbr_process_timers()
5216 ret = bbr_timeout_persist(tp, bbr, cts); in bbr_process_timers()
5218 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5219 ret = bbr_timeout_rack(tp, bbr, cts); in bbr_process_timers()
5221 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5222 ret = bbr_timeout_tlp(tp, bbr, cts); in bbr_process_timers()
5224 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_process_timers()
5225 ret = bbr_timeout_rxt(tp, bbr, cts); in bbr_process_timers()
5227 ret = bbr_timeout_keepalive(tp, bbr, cts); in bbr_process_timers()
5229 bbr_log_to_processing(bbr, cts, ret, timers, hpts_calling); in bbr_process_timers()
5234 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts) in bbr_timer_cancel() argument
5252 if (TSTMP_GT(cts, bbr->rc_pacer_started)) in bbr_timer_cancel()
5253 time_since_send = cts - bbr->rc_pacer_started; in bbr_timer_cancel()
5262 bbr->rc_pacer_started = cts; in bbr_timer_cancel()
5266 bbr_log_to_cancel(bbr, line, cts, hpts_removed); in bbr_timer_cancel()
5286 bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts) in bbr_get_earliest_send_outstanding() argument
5292 return (cts); in bbr_get_earliest_send_outstanding()
5298 struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time) in bbr_update_rsm() argument
5333 rsm->r_tim_lastsent[idx] = cts; in bbr_update_rsm()
5353 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__); in bbr_update_rsm()
5368 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); in bbr_update_rsm()
5393 struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time) in bbr_update_entry() argument
5410 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time); in bbr_update_entry()
5448 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time); in bbr_update_entry()
5465 bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts, in bbr_setup_less_of_rate() argument
5480 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts); in bbr_setup_less_of_rate()
5489 bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts) in bbr_update_hardware_pacing_rate() argument
5507 tcp_bbr_tso_size_check(bbr, cts); in bbr_update_hardware_pacing_rate()
5526 bbr_setup_less_of_rate(bbr, cts, in bbr_update_hardware_pacing_rate()
5545 cts, in bbr_update_hardware_pacing_rate()
5551 bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts) in bbr_adjust_for_hw_pacing() argument
5585 bbr->r_ctl.rc_pace_max_segs, cts, 1); in bbr_adjust_for_hw_pacing()
5592 bbr_log_type_tsosize(bbr, cts, delta, cur_delay, hdwr_delay, in bbr_adjust_for_hw_pacing()
5667 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0); in bbr_adjust_for_hw_pacing()
5673 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts) in tcp_bbr_tso_size_check() argument
5762 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0); in tcp_bbr_tso_size_check()
5763 bbr_adjust_for_hw_pacing(bbr, cts); in tcp_bbr_tso_size_check()
5861 bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0); in tcp_bbr_tso_size_check()
5865 bbr_adjust_for_hw_pacing(bbr, cts); in tcp_bbr_tso_size_check()
5870 uint32_t seq_out, uint16_t th_flags, int32_t err, uint32_t cts, in bbr_log_output() argument
5940 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1); in bbr_log_output()
5953 rsm->r_tim_lastsent[0] = cts; in bbr_log_output()
5969 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); in bbr_log_output()
6015 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time); in bbr_log_output()
6025 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time); in bbr_log_output()
6041 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time); in bbr_log_output()
6055 seq_out = bbr_update_entry(tp, bbr, nrsm, cts, &len, pacing_time); in bbr_log_output()
6242 tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts) in tcp_bbr_xmit_timer_commit() argument
6271 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts); in tcp_bbr_xmit_timer_commit()
6273 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in tcp_bbr_xmit_timer_commit()
6394 bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line) in bbr_set_reduced_rtt() argument
6396 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_set_reduced_rtt()
6398 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && in bbr_set_reduced_rtt()
6399 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { in bbr_set_reduced_rtt()
6404 bbr_enter_probe_rtt(bbr, cts, __LINE__); in bbr_set_reduced_rtt()
6406 bbr_check_probe_rtt_limits(bbr, cts); in bbr_set_reduced_rtt()
6410 tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts) in tcp_bbr_commit_bw() argument
6416 bbr_log_type_bbrupd(bbr, 20, cts, 0, 0, in tcp_bbr_commit_bw()
6424 bbr_log_type_bbrupd(bbr, 21, cts, (uint32_t)orig_bw, in tcp_bbr_commit_bw()
6434 bbr_update_hardware_pacing_rate(bbr, cts); in tcp_bbr_commit_bw()
6437 tcp_bbr_tso_size_check(bbr, cts); in tcp_bbr_commit_bw()
6439 bbr_setup_red_bw(bbr, cts); in tcp_bbr_commit_bw()
6443 tcp_bbr_tso_size_check(bbr, cts); in tcp_bbr_commit_bw()
6447 bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts) in bbr_nf_measurement() argument
6486 bbr_log_type_bbrupd(bbr, 61, cts, in bbr_nf_measurement()
6495 bbr_log_type_bbrupd(bbr, 62, cts, in bbr_nf_measurement()
6501 bbr_log_type_bbrupd(bbr, 8, cts, in bbr_nf_measurement()
6509 bbr_log_type_bbrupd(bbr, 7, cts, in bbr_nf_measurement()
6539 bbr_log_type_bbrupd(bbr, 6, cts, in bbr_nf_measurement()
6553 tcp_bbr_commit_bw(bbr, cts); in bbr_nf_measurement()
6554 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered, in bbr_nf_measurement()
6561 bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts) in bbr_google_measurement() argument
6584 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered, in bbr_google_measurement()
6615 bbr_log_type_bbrupd(bbr, 6, cts, in bbr_google_measurement()
6626 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered, in bbr_google_measurement()
6636 tcp_bbr_commit_bw(bbr, cts); in bbr_google_measurement()
6637 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered, in bbr_google_measurement()
6644 bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint3… in bbr_update_bbr_info() argument
6651 bbr->r_ctl.rc_del_time = cts; in bbr_update_bbr_info()
6699 bbr_log_rtt_shrinks(bbr, cts, 0, rtt, __LINE__, BBR_RTTS_NEWRTT, 0); in bbr_update_bbr_info()
6700 bbr_set_reduced_rtt(bbr, cts, __LINE__); in bbr_update_bbr_info()
6702 bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts, in bbr_update_bbr_info()
6704 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_update_bbr_info()
6712 bbr_log_rtt_shrinks(bbr, cts, 0, 0, in bbr_update_bbr_info()
6716 bbr_check_probe_rtt_limits(bbr, cts); in bbr_update_bbr_info()
6733 bbr_google_measurement(bbr, rsm, rtt, cts); in bbr_update_bbr_info()
6735 bbr_nf_measurement(bbr, rsm, rtt, cts); in bbr_update_bbr_info()
6745 bbr_ts_convert(uint32_t cts) { in bbr_ts_convert() argument
6748 sec = cts / MS_IN_USEC; in bbr_ts_convert()
6749 msec = cts - (MS_IN_USEC * sec); in bbr_ts_convert()
6759 struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack) in bbr_update_rtt() argument
6778 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0])) in bbr_update_rtt()
6779 t = cts - rsm->r_tim_lastsent[0]; in bbr_update_rtt()
6783 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, in bbr_update_rtt()
6797 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, in bbr_update_rtt()
6818 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6819 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6823 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING, in bbr_update_rtt()
6847 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6848 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6869 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) in bbr_update_rtt()
6870 t = cts - rsm->r_tim_lastsent[i]; in bbr_update_rtt()
6873 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET, in bbr_update_rtt()
6882 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, in bbr_update_rtt()
6893 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, in bbr_update_rtt()
6896 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, in bbr_update_rtt()
6949 struct tcpopt *to, struct bbr_sendmap **prsm, uint32_t cts) in bbr_proc_sack_blk() argument
7054 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0); in bbr_proc_sack_blk()
7064 bbr->r_ctl.rc_reorder_ts = cts; in bbr_proc_sack_blk()
7079 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED); in bbr_proc_sack_blk()
7119 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0); in bbr_proc_sack_blk()
7120 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED); in bbr_proc_sack_blk()
7130 bbr->r_ctl.rc_reorder_ts = cts; in bbr_proc_sack_blk()
7246 uint32_t cts; in bbr_log_syn() local
7249 cts = bbr->r_ctl.rc_rcvtime; in bbr_log_syn()
7266 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0); in bbr_log_syn()
7291 uint32_t cts, acked, ack_point, sack_changed = 0; in bbr_log_ack() local
7300 cts = bbr->r_ctl.rc_rcvtime; in bbr_log_ack()
7336 tp->iss, 0, cts, in bbr_log_ack()
7338 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_log_ack()
7404 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack); in bbr_log_ack()
7415 bbr->r_ctl.rc_reorder_ts = cts; in bbr_log_ack()
7436 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED); in bbr_log_ack()
7464 bbr_update_rtt(tp, bbr, &lrsm, to, cts, BBR_CUM_ACKED, th_ack); in bbr_log_ack()
7475 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED); in bbr_log_ack()
7608 acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts); in bbr_log_ack()
7624 rsm = bbr_check_recovery_mode(tp, bbr, cts); in bbr_log_ack()
7645 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_log_ack()
7877 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line) in bbr_enter_persist() argument
7880 bbr_timer_cancel(bbr, __LINE__, cts); in bbr_enter_persist()
7884 bbr->r_ctl.rc_went_idle_time = cts; in bbr_enter_persist()
7886 bbr_log_type_pesist(bbr, cts, 0, line, 1); in bbr_enter_persist()
7888 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_enter_persist()
7891 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_enter_persist()
7901 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_enter_persist()
7906 bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time) in bbr_restart_after_idle() argument
7928 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_restart_after_idle()
7930 bbr_substate_change(bbr, cts, __LINE__, 1); in bbr_restart_after_idle()
7936 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line) in bbr_exit_persist() argument
7942 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time); in bbr_exit_persist()
7945 bbr->r_ctl.rc_del_time = cts; in bbr_exit_persist()
7960 bbr_log_type_pesist(bbr, cts, idle_time, line, 0); in bbr_exit_persist()
7967 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_PERSIST, 0); in bbr_exit_persist()
7968 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_exit_persist()
7976 bbr_set_epoch(bbr, cts, __LINE__); in bbr_exit_persist()
7983 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_exit_persist()
7991 bbr_restart_after_idle(bbr, cts, idle_time); in bbr_exit_persist()
9939 uint32_t cts; in bbr_init() local
9968 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_init()
9986 bbr->r_ctl.rc_went_idle_time = cts; in bbr_init()
9987 bbr->rc_pacer_started = cts; in bbr_init()
9988 bbr->r_ctl.rc_pkt_epoch_time = cts; in bbr_init()
9989 bbr->r_ctl.rc_rcvtime = cts; in bbr_init()
9990 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_init()
9991 bbr->r_ctl.rc_del_time = cts; in bbr_init()
9992 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_init()
9993 bbr->r_ctl.last_in_probertt = cts; in bbr_init()
10058 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_init()
10072 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_INIT, 0); in bbr_init()
10092 rsm->r_tim_lastsent[0] = cts; in bbr_init()
10101 rsm->r_del_time = cts; in bbr_init()
10125 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_init()
10131 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); in bbr_init()
10135 tcp_bbr_tso_size_check(bbr, cts); in bbr_init()
10152 bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0); in bbr_init()
10283 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int32_t line, int dolog) in bbr_substate_change() argument
10336 bbr->r_ctl.rc_bbr_state_atflight = cts; in bbr_substate_change()
10347 bbr->r_ctl.rc_bbr_state_atflight = cts; in bbr_substate_change()
10358 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP)) in bbr_substate_change()
10359 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) - in bbr_substate_change()
10373 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); in bbr_substate_change()
10377 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); in bbr_substate_change()
10382 bbr_log_type_statechange(bbr, cts, line); in bbr_substate_change()
10384 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_substate_change()
10387 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_substate_change()
10417 tcp_bbr_tso_size_check(bbr, cts); in bbr_substate_change()
10418 bbr->r_ctl.gain_epoch = cts; in bbr_substate_change()
10419 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_substate_change()
10424 bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses) in bbr_set_probebw_google_gains() argument
10432 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_set_probebw_google_gains()
10435 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) { in bbr_set_probebw_google_gains()
10458 bbr_substate_change(bbr, cts, __LINE__, 1); in bbr_set_probebw_google_gains()
10462 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses) in bbr_set_probebw_gains() argument
10467 bbr_set_probebw_google_gains(bbr, cts, losses); in bbr_set_probebw_gains()
10470 if (cts == 0) { in bbr_set_probebw_gains()
10472 * Never alow cts to be 0 we in bbr_set_probebw_gains()
10476 cts = 1; in bbr_set_probebw_gains()
10498 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) && in bbr_set_probebw_gains()
10499 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) || in bbr_set_probebw_gains()
10524 bbr_log_exit_gain(bbr, cts, 4); in bbr_set_probebw_gains()
10529 bbr->r_ctl.gain_epoch = cts; in bbr_set_probebw_gains()
10538 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10539 bbr_log_exit_gain(bbr, cts, 3); in bbr_set_probebw_gains()
10544 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10550 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); in bbr_set_probebw_gains()
10551 bbr_log_exit_gain(bbr, cts, 2); in bbr_set_probebw_gains()
10566 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) in bbr_set_probebw_gains()
10568 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) { in bbr_set_probebw_gains()
10574 ((cts - bbr->r_ctl.rc_bbr_state_time) < in bbr_set_probebw_gains()
10582 ((cts - bbr->r_ctl.rc_bbr_state_time) < in bbr_set_probebw_gains()
10587 bbr_substate_change(bbr, cts, __LINE__, 1); in bbr_set_probebw_gains()
10655 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line) in bbr_enter_probe_rtt() argument
10668 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_enter_probe_rtt()
10669 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_enter_probe_rtt()
10672 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_ENTERPROBE, 0); in bbr_enter_probe_rtt()
10673 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_enter_probe_rtt()
10674 bbr->r_ctl.last_in_probertt = cts; in bbr_enter_probe_rtt()
10675 bbr->r_ctl.rc_probertt_srttchktim = cts; in bbr_enter_probe_rtt()
10676 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_enter_probe_rtt()
10717 bbr->r_ctl.rc_bbr_enters_probertt = cts; in bbr_enter_probe_rtt()
10724 bbr_log_exit_gain(bbr, cts, 0); in bbr_enter_probe_rtt()
10725 bbr_log_type_statechange(bbr, cts, line); in bbr_enter_probe_rtt()
10729 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts) in bbr_check_probe_rtt_limits() argument
10794 bbr_log_rtt_shrinks(bbr, cts, cur_rttp, newval, __LINE__, BBR_RTTS_RESETS_VALUES, val); in bbr_check_probe_rtt_limits()
10799 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) in bbr_exit_probe_rtt() argument
10807 bbr_log_exit_gain(bbr, cts, 1); in bbr_exit_probe_rtt()
10809 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_exit_probe_rtt()
10810 bbr->r_ctl.last_in_probertt = cts; in bbr_exit_probe_rtt()
10811 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_RTTPROBE, 0); in bbr_exit_probe_rtt()
10816 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_exit_probe_rtt()
10819 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_exit_probe_rtt()
10825 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_exit_probe_rtt()
10827 bbr_substate_change(bbr, cts, __LINE__, 0); in bbr_exit_probe_rtt()
10828 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_exit_probe_rtt()
10832 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_exit_probe_rtt()
10852 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_exit_probe_rtt()
10853 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_exit_probe_rtt()
10856 bbr_check_probe_rtt_limits(bbr, cts); in bbr_exit_probe_rtt()
10860 bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts) in bbr_should_enter_probe_rtt() argument
10864 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) { in bbr_should_enter_probe_rtt()
10869 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && in bbr_should_enter_probe_rtt()
10870 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { in bbr_should_enter_probe_rtt()
10877 bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t pkt_epoch) in bbr_google_startup() argument
10891 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_google_startup()
10897 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_google_startup()
10903 bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch) in bbr_state_startup() argument
10911 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_state_startup()
10917 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0); in bbr_state_startup()
10918 bbr_set_reduced_rtt(bbr, cts, __LINE__); in bbr_state_startup()
10920 if (bbr_should_enter_probe_rtt(bbr, cts)) { in bbr_state_startup()
10921 bbr_enter_probe_rtt(bbr, cts, __LINE__); in bbr_state_startup()
10925 return (bbr_google_startup(bbr, cts, pkt_epoch)); in bbr_state_startup()
10965 bbr_log_startup_event(bbr, cts, rtt_gain, in bbr_state_startup()
10981 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11003 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11021 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11026 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11037 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_startup()
11043 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t loss… in bbr_state_change() argument
11060 if (bbr_state_startup(bbr, cts, epoch, pkt_epoch)) { in bbr_state_change()
11063 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, in bbr_state_change()
11067 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_state_change()
11068 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_state_change()
11074 bbr->r_ctl.rc_bbr_state_time = cts; in bbr_state_change()
11086 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_state_change()
11094 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_state_change()
11095 bbr_substate_change(bbr, cts, __LINE__, 0); in bbr_state_change()
11097 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_state_change()
11119 bbr_substate_change(bbr, cts, __LINE__, 1); in bbr_state_change()
11143 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { in bbr_state_change()
11146 time_in = cts - bbr->r_ctl.rc_bbr_state_time; in bbr_state_change()
11157 bbr->r_ctl.rc_rtt_shrinks = cts; in bbr_state_change()
11158 bbr->r_ctl.last_in_probertt = cts; in bbr_state_change()
11159 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_LEAVE_DRAIN, 0); in bbr_state_change()
11161 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); in bbr_state_change()
11162 bbr_substate_change(bbr, cts, __LINE__, 0); in bbr_state_change()
11163 bbr_log_type_statechange(bbr, cts, __LINE__); in bbr_state_change()
11187 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_REACHTAR, 0); in bbr_state_change()
11188 bbr->r_ctl.rc_bbr_enters_probertt = cts; in bbr_state_change()
11206 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) || in bbr_state_change()
11219 bbr->r_ctl.rc_probertt_srttchktim = cts; in bbr_state_change()
11224 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG, 0); in bbr_state_change()
11228 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG_FINAL, 0); in bbr_state_change()
11236 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) && in bbr_state_change()
11237 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) { in bbr_state_change()
11239 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts); in bbr_state_change()
11243 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_state_change()
11249 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0); in bbr_state_change()
11250 bbr_set_reduced_rtt(bbr, cts, __LINE__); in bbr_state_change()
11252 if (bbr_should_enter_probe_rtt(bbr, cts)) { in bbr_state_change()
11253 bbr_enter_probe_rtt(bbr, cts, __LINE__); in bbr_state_change()
11255 bbr_set_probebw_gains(bbr, cts, losses); in bbr_state_change()
11261 bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses) in bbr_check_bbr_for_state() argument
11265 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) { in bbr_check_bbr_for_state()
11266 bbr_set_epoch(bbr, cts, line); in bbr_check_bbr_for_state()
11270 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses); in bbr_check_bbr_for_state()
11281 uint32_t cts, lcts; in bbr_do_segment_nounlock() local
11328 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usec(&bbr->rc_tv); in bbr_do_segment_nounlock()
11336 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usec(&bbr->rc_tv); in bbr_do_segment_nounlock()
11341 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv); in bbr_do_segment_nounlock()
11484 if (TSTMP_GT(lcts, cts)) { in bbr_do_segment_nounlock()
11485 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts; in bbr_do_segment_nounlock()
11498 bbr_log_ack_event(bbr, th, &to, tlen, nsegs, cts, nxt_pkt, m); in bbr_do_segment_nounlock()
11539 tcp_bbr_xmit_timer_commit(bbr, tp, cts); in bbr_do_segment_nounlock()
11541 bbr_set_pktepoch(bbr, cts, __LINE__); in bbr_do_segment_nounlock()
11542 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost)); in bbr_do_segment_nounlock()
11552 bbr_start_hpts_timer(bbr, tp, cts, 6, 0, 0); in bbr_do_segment_nounlock()
11594 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val, in bbr_do_segment_nounlock()
11608 bbr_log_doseg_done(bbr, cts, nxt_pkt, did_out); in bbr_do_segment_nounlock()
11649 uint32_t avail, int32_t sb_offset, uint32_t cts) in bbr_what_can_we_send() argument
11818 uint32_t cts; in bbr_output_wtime() local
11871 cts = tcp_tv_to_usec(&bbr->rc_tv); in bbr_output_wtime()
11884 tcp_bbr_tso_size_check(bbr, cts); in bbr_output_wtime()
11934 bbr_timer_cancel(bbr, __LINE__, cts); in bbr_output_wtime()
11938 if (SEQ_GT(cts, bbr->rc_pacer_started)) in bbr_output_wtime()
11939 delay_calc = cts - bbr->rc_pacer_started; in bbr_output_wtime()
11980 if (SEQ_GT(cts, bbr->rc_pacer_started)) in bbr_output_wtime()
11981 delay_calc = cts - bbr->rc_pacer_started; in bbr_output_wtime()
12011 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val, in bbr_output_wtime()
12045 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time)); in bbr_output_wtime()
12056 retval = bbr_process_timers(tp, bbr, cts, hpts_calling); in bbr_output_wtime()
12091 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0); in bbr_output_wtime()
12139 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); in bbr_output_wtime()
12323 len = bbr_what_can_we_send(tp, bbr, sendwin, avail, sb_offset, cts); in bbr_output_wtime()
12464 bbr_enter_persist(tp, bbr, cts, __LINE__); in bbr_output_wtime()
12486 bbr_enter_persist(tp, bbr, cts, __LINE__); in bbr_output_wtime()
12702 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); in bbr_output_wtime()
12739 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0); in bbr_output_wtime()
12754 bbr_start_hpts_timer(bbr, tp, cts, 9, slot, tot_len); in bbr_output_wtime()
12755 bbr_log_type_just_return(bbr, cts, tot_len, hpts_calling, app_limited, p_maxseg, len); in bbr_output_wtime()
12779 bbr_timer_cancel(bbr, __LINE__, cts); in bbr_output_wtime()
12810 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { in bbr_output_wtime()
12816 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0); in bbr_output_wtime()
12817 bbr_set_reduced_rtt(bbr, cts, __LINE__); in bbr_output_wtime()
13083 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0); in bbr_output_wtime()
13233 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0); in bbr_output_wtime()
13486 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); in bbr_output_wtime()
13632 bbr->r_ctl.rc_del_time = cts; in bbr_output_wtime()
13659 bbr->r_ctl.rc_tlp_rxt_last_time = cts; in bbr_output_wtime()
13663 cts, mb, &abandon, rsm, 0, sb); in bbr_output_wtime()
13795 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, csum_flags, tso, cts); in bbr_output_wtime()
13811 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, 0, tso, cts); in bbr_output_wtime()
13820 (tot_len + len), cts, 0); in bbr_output_wtime()
13826 bbr_start_hpts_timer(bbr, tp, cts, 10, slot, in bbr_output_wtime()
13846 bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0); in bbr_output_wtime()
13857 tp->gput_ts = cts; in bbr_output_wtime()
13891 __LINE__, cts, err); in bbr_output_wtime()
13899 bbr_setup_less_of_rate(bbr, cts, in bbr_output_wtime()
13906 tcp_bbr_tso_size_check(bbr, cts); in bbr_output_wtime()
13912 __LINE__, cts, err); in bbr_output_wtime()
13937 tcp_bbr_tso_size_check(bbr, cts); in bbr_output_wtime()
13987 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); in bbr_output_wtime()
13994 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0); in bbr_output_wtime()
14000 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) { in bbr_output_wtime()
14014 tcp_bbr_tso_size_check(bbr, cts); in bbr_output_wtime()
14017 bbr_start_hpts_timer(bbr, tp, cts, 12, slot, tot_len); in bbr_output_wtime()
14106 uint32_t cts; in bbr_switch_failed() local
14118 cts = tcp_get_usecs(&tv); in bbr_switch_failed()
14120 if (TSTMP_GT(bbr->rc_pacer_started, cts)) { in bbr_switch_failed()
14121 toval = bbr->rc_pacer_started - cts; in bbr_switch_failed()
14127 if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { in bbr_switch_failed()
14128 toval = bbr->r_ctl.rc_timer_exp - cts; in bbr_switch_failed()
14137 bbr_log_hpts_diag(bbr, cts, &diag); in bbr_switch_failed()
14431 uint32_t cts; in bbr_set_sockopt() local
14432 cts = tcp_get_usecs(&bbr->rc_tv); in bbr_set_sockopt()
14433 tcp_bbr_tso_size_check(bbr, cts); in bbr_set_sockopt()