1 /*- 2 * Copyright (c) 2016-2020 Netflix, Inc. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 /** 27 * Author: Randall Stewart <rrs@netflix.com> 28 * This work is based on the ACM Queue paper 29 * BBR - Congestion Based Congestion Control 30 * and also numerous discussions with Neal, Yuchung and Van. 31 */ 32 33 #include <sys/cdefs.h> 34 __FBSDID("$FreeBSD$"); 35 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 #include "opt_ipsec.h" 39 #include "opt_tcpdebug.h" 40 #include "opt_ratelimit.h" 41 #include "opt_kern_tls.h" 42 #include <sys/param.h> 43 #include <sys/arb.h> 44 #include <sys/module.h> 45 #include <sys/kernel.h> 46 #include <sys/libkern.h> 47 #ifdef TCP_HHOOK 48 #include <sys/hhook.h> 49 #endif 50 #include <sys/malloc.h> 51 #include <sys/mbuf.h> 52 #include <sys/proc.h> 53 #include <sys/socket.h> 54 #include <sys/socketvar.h> 55 #ifdef KERN_TLS 56 #include <sys/ktls.h> 57 #endif 58 #include <sys/sysctl.h> 59 #include <sys/systm.h> 60 #ifdef STATS 61 #include <sys/qmath.h> 62 #include <sys/tree.h> 63 #include <sys/stats.h> /* Must come after qmath.h and tree.h */ 64 #endif 65 #include <sys/refcount.h> 66 #include <sys/queue.h> 67 #include <sys/eventhandler.h> 68 #include <sys/smp.h> 69 #include <sys/kthread.h> 70 #include <sys/lock.h> 71 #include <sys/mutex.h> 72 #include <sys/tim_filter.h> 73 #include <sys/time.h> 74 #include <sys/protosw.h> 75 #include <vm/uma.h> 76 #include <sys/kern_prefetch.h> 77 78 #include <net/route.h> 79 #include <net/route/nhop.h> 80 #include <net/vnet.h> 81 82 #define TCPSTATES /* for logging */ 83 84 #include <netinet/in.h> 85 #include <netinet/in_kdtrace.h> 86 #include <netinet/in_pcb.h> 87 #include <netinet/ip.h> 88 #include <netinet/ip_icmp.h> /* required for icmp_var.h */ 89 #include <netinet/icmp_var.h> /* for ICMP_BANDLIM */ 90 #include <netinet/ip_var.h> 91 #include <netinet/ip6.h> 92 #include <netinet6/in6_pcb.h> 93 #include <netinet6/ip6_var.h> 94 #define TCPOUTFLAGS 95 #include <netinet/tcp.h> 96 #include <netinet/tcp_fsm.h> 97 #include <netinet/tcp_seq.h> 98 #include <netinet/tcp_timer.h> 99 #include <netinet/tcp_var.h> 100 #include <netinet/tcpip.h> 101 #include <netinet/tcp_hpts.h> 102 #include <netinet/cc/cc.h> 103 #include <netinet/tcp_log_buf.h> 104 #include <netinet/tcp_ratelimit.h> 105 #include <netinet/tcp_lro.h> 106 #ifdef TCPDEBUG 107 #include <netinet/tcp_debug.h> 108 #endif /* TCPDEBUG */ 109 #ifdef TCP_OFFLOAD 110 #include <netinet/tcp_offload.h> 111 #endif 112 #ifdef INET6 113 #include <netinet6/tcp6_var.h> 114 #endif 115 #include <netinet/tcp_fastopen.h> 116 117 #include <netipsec/ipsec_support.h> 118 #include <net/if.h> 119 #include <net/if_var.h> 120 #include <net/ethernet.h> 121 122 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 123 #include <netipsec/ipsec.h> 124 #include <netipsec/ipsec6.h> 125 #endif /* IPSEC */ 126 127 #include <netinet/udp.h> 128 #include <netinet/udp_var.h> 129 #include <machine/in_cksum.h> 130 131 #ifdef MAC 132 #include <security/mac/mac_framework.h> 133 #endif 134 135 #include "sack_filter.h" 136 #include "tcp_bbr.h" 137 #include "rack_bbr_common.h" 138 uma_zone_t bbr_zone; 139 uma_zone_t bbr_pcb_zone; 140 141 struct sysctl_ctx_list bbr_sysctl_ctx; 142 struct sysctl_oid *bbr_sysctl_root; 143 144 #define TCPT_RANGESET_NOSLOP(tv, value, tvmin, tvmax) do { \ 145 (tv) = (value); \ 146 if ((u_long)(tv) < (u_long)(tvmin)) \ 147 (tv) = (tvmin); \ 148 if ((u_long)(tv) > (u_long)(tvmax)) \ 149 (tv) = (tvmax); \ 150 } while(0) 151 152 /*#define BBR_INVARIANT 1*/ 153 154 /* 155 * initial window 156 */ 157 static uint32_t bbr_def_init_win = 10; 158 static int32_t bbr_persist_min = 250000; /* 250ms */ 159 static int32_t bbr_persist_max = 1000000; /* 1 Second */ 160 static int32_t bbr_cwnd_may_shrink = 0; 161 static int32_t bbr_cwndtarget_rtt_touse = BBR_RTT_PROP; 162 static int32_t bbr_num_pktepo_for_del_limit = BBR_NUM_RTTS_FOR_DEL_LIMIT; 163 static int32_t bbr_hardware_pacing_limit = 8000; 164 static int32_t bbr_quanta = 3; /* How much extra quanta do we get? */ 165 static int32_t bbr_no_retran = 0; 166 167 168 static int32_t bbr_error_base_paceout = 10000; /* usec to pace */ 169 static int32_t bbr_max_net_error_cnt = 10; 170 /* Should the following be dynamic too -- loss wise */ 171 static int32_t bbr_rtt_gain_thresh = 0; 172 /* Measurement controls */ 173 static int32_t bbr_use_google_algo = 1; 174 static int32_t bbr_ts_limiting = 1; 175 static int32_t bbr_ts_can_raise = 0; 176 static int32_t bbr_do_red = 600; 177 static int32_t bbr_red_scale = 20000; 178 static int32_t bbr_red_mul = 1; 179 static int32_t bbr_red_div = 2; 180 static int32_t bbr_red_growth_restrict = 1; 181 static int32_t bbr_target_is_bbunit = 0; 182 static int32_t bbr_drop_limit = 0; 183 /* 184 * How much gain do we need to see to 185 * stay in startup? 186 */ 187 static int32_t bbr_marks_rxt_sack_passed = 0; 188 static int32_t bbr_start_exit = 25; 189 static int32_t bbr_low_start_exit = 25; /* When we are in reduced gain */ 190 static int32_t bbr_startup_loss_thresh = 2000; /* 20.00% loss */ 191 static int32_t bbr_hptsi_max_mul = 1; /* These two mul/div assure a min pacing */ 192 static int32_t bbr_hptsi_max_div = 2; /* time, 0 means turned off. We need this 193 * if we go back ever to where the pacer 194 * has priority over timers. 195 */ 196 static int32_t bbr_policer_call_from_rack_to = 0; 197 static int32_t bbr_policer_detection_enabled = 1; 198 static int32_t bbr_min_measurements_req = 1; /* We need at least 2 199 * measurments before we are 200 * "good" note that 2 == 1. 201 * This is because we use a > 202 * comparison. This means if 203 * min_measure was 0, it takes 204 * num-measures > min(0) and 205 * you get 1 measurement and 206 * you are good. Set to 1, you 207 * have to have two 208 * measurements (this is done 209 * to prevent it from being ok 210 * to have no measurements). */ 211 static int32_t bbr_no_pacing_until = 4; 212 213 static int32_t bbr_min_usec_delta = 20000; /* 20,000 usecs */ 214 static int32_t bbr_min_peer_delta = 20; /* 20 units */ 215 static int32_t bbr_delta_percent = 150; /* 15.0 % */ 216 217 static int32_t bbr_target_cwnd_mult_limit = 8; 218 /* 219 * bbr_cwnd_min_val is the number of 220 * segments we hold to in the RTT probe 221 * state typically 4. 222 */ 223 static int32_t bbr_cwnd_min_val = BBR_PROBERTT_NUM_MSS; 224 225 226 static int32_t bbr_cwnd_min_val_hs = BBR_HIGHSPEED_NUM_MSS; 227 228 static int32_t bbr_gain_to_target = 1; 229 static int32_t bbr_gain_gets_extra_too = 1; 230 /* 231 * bbr_high_gain is the 2/ln(2) value we need 232 * to double the sending rate in startup. This 233 * is used for both cwnd and hptsi gain's. 234 */ 235 static int32_t bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1; 236 static int32_t bbr_startup_lower = BBR_UNIT * 1500 / 1000 + 1; 237 static int32_t bbr_use_lower_gain_in_startup = 1; 238 239 /* thresholds for reduction on drain in sub-states/drain */ 240 static int32_t bbr_drain_rtt = BBR_SRTT; 241 static int32_t bbr_drain_floor = 88; 242 static int32_t google_allow_early_out = 1; 243 static int32_t google_consider_lost = 1; 244 static int32_t bbr_drain_drop_mul = 4; 245 static int32_t bbr_drain_drop_div = 5; 246 static int32_t bbr_rand_ot = 50; 247 static int32_t bbr_can_force_probertt = 0; 248 static int32_t bbr_can_adjust_probertt = 1; 249 static int32_t bbr_probertt_sets_rtt = 0; 250 static int32_t bbr_can_use_ts_for_rtt = 1; 251 static int32_t bbr_is_ratio = 0; 252 static int32_t bbr_sub_drain_app_limit = 1; 253 static int32_t bbr_prtt_slam_cwnd = 1; 254 static int32_t bbr_sub_drain_slam_cwnd = 1; 255 static int32_t bbr_slam_cwnd_in_main_drain = 1; 256 static int32_t bbr_filter_len_sec = 6; /* How long does the rttProp filter 257 * hold */ 258 static uint32_t bbr_rtt_probe_limit = (USECS_IN_SECOND * 4); 259 /* 260 * bbr_drain_gain is the reverse of the high_gain 261 * designed to drain back out the standing queue 262 * that is formed in startup by causing a larger 263 * hptsi gain and thus drainging the packets 264 * in flight. 265 */ 266 static int32_t bbr_drain_gain = BBR_UNIT * 1000 / 2885; 267 static int32_t bbr_rttprobe_gain = 192; 268 269 /* 270 * The cwnd_gain is the default cwnd gain applied when 271 * calculating a target cwnd. Note that the cwnd is 272 * a secondary factor in the way BBR works (see the 273 * paper and think about it, it will take some time). 274 * Basically the hptsi_gain spreads the packets out 275 * so you never get more than BDP to the peer even 276 * if the cwnd is high. In our implemenation that 277 * means in non-recovery/retransmission scenarios 278 * cwnd will never be reached by the flight-size. 279 */ 280 static int32_t bbr_cwnd_gain = BBR_UNIT * 2; 281 static int32_t bbr_tlp_type_to_use = BBR_SRTT; 282 static int32_t bbr_delack_time = 100000; /* 100ms in useconds */ 283 static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sack to use bbr */ 284 static int32_t bbr_initial_bw_bps = 62500; /* 500kbps in bytes ps */ 285 static int32_t bbr_ignore_data_after_close = 1; 286 static int16_t bbr_hptsi_gain[] = { 287 (BBR_UNIT *5 / 4), 288 (BBR_UNIT * 3 / 4), 289 BBR_UNIT, 290 BBR_UNIT, 291 BBR_UNIT, 292 BBR_UNIT, 293 BBR_UNIT, 294 BBR_UNIT 295 }; 296 int32_t bbr_use_rack_resend_cheat = 1; 297 int32_t bbr_sends_full_iwnd = 1; 298 299 #define BBR_HPTSI_GAIN_MAX 8 300 /* 301 * The BBR module incorporates a number of 302 * TCP ideas that have been put out into the IETF 303 * over the last few years: 304 * - Yuchung Cheng's RACK TCP (for which its named) that 305 * will stop us using the number of dup acks and instead 306 * use time as the gage of when we retransmit. 307 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft 308 * of Dukkipati et.al. 309 * - Van Jacobson's et.al BBR. 310 * 311 * RACK depends on SACK, so if an endpoint arrives that 312 * cannot do SACK the state machine below will shuttle the 313 * connection back to using the "default" TCP stack that is 314 * in FreeBSD. 315 * 316 * To implement BBR and RACK the original TCP stack was first decomposed 317 * into a functional state machine with individual states 318 * for each of the possible TCP connection states. The do_segement 319 * functions role in life is to mandate the connection supports SACK 320 * initially and then assure that the RACK state matches the conenction 321 * state before calling the states do_segment function. Data processing 322 * of inbound segments also now happens in the hpts_do_segment in general 323 * with only one exception. This is so we can keep the connection on 324 * a single CPU. 325 * 326 * Each state is simplified due to the fact that the original do_segment 327 * has been decomposed and we *know* what state we are in (no 328 * switches on the state) and all tests for SACK are gone. This 329 * greatly simplifies what each state does. 330 * 331 * TCP output is also over-written with a new version since it 332 * must maintain the new rack scoreboard and has had hptsi 333 * integrated as a requirment. Still todo is to eliminate the 334 * use of the callout_() system and use the hpts for all 335 * timers as well. 336 */ 337 static uint32_t bbr_rtt_probe_time = 200000; /* 200ms in micro seconds */ 338 static uint32_t bbr_rtt_probe_cwndtarg = 4; /* How many mss's outstanding */ 339 static const int32_t bbr_min_req_free = 2; /* The min we must have on the 340 * free list */ 341 static int32_t bbr_tlp_thresh = 1; 342 static int32_t bbr_reorder_thresh = 2; 343 static int32_t bbr_reorder_fade = 60000000; /* 0 - never fade, def 344 * 60,000,000 - 60 seconds */ 345 static int32_t bbr_pkt_delay = 1000; 346 static int32_t bbr_min_to = 1000; /* Number of usec's minimum timeout */ 347 static int32_t bbr_incr_timers = 1; 348 349 static int32_t bbr_tlp_min = 10000; /* 10ms in usecs */ 350 static int32_t bbr_delayed_ack_time = 200000; /* 200ms in usecs */ 351 static int32_t bbr_exit_startup_at_loss = 1; 352 353 /* 354 * bbr_lt_bw_ratio is 1/8th 355 * bbr_lt_bw_diff is < 4 Kbit/sec 356 */ 357 static uint64_t bbr_lt_bw_diff = 4000 / 8; /* In bytes per second */ 358 static uint64_t bbr_lt_bw_ratio = 8; /* For 1/8th */ 359 static uint32_t bbr_lt_bw_max_rtts = 48; /* How many rtt's do we use 360 * the lt_bw for */ 361 static uint32_t bbr_lt_intvl_min_rtts = 4; /* Min num of RTT's to measure 362 * lt_bw */ 363 static int32_t bbr_lt_intvl_fp = 0; /* False positive epoch diff */ 364 static int32_t bbr_lt_loss_thresh = 196; /* Lost vs delivered % */ 365 static int32_t bbr_lt_fd_thresh = 100; /* false detection % */ 366 367 static int32_t bbr_verbose_logging = 0; 368 /* 369 * Currently regular tcp has a rto_min of 30ms 370 * the backoff goes 12 times so that ends up 371 * being a total of 122.850 seconds before a 372 * connection is killed. 373 */ 374 static int32_t bbr_rto_min_ms = 30; /* 30ms same as main freebsd */ 375 static int32_t bbr_rto_max_sec = 4; /* 4 seconds */ 376 377 /****************************************************/ 378 /* DEFAULT TSO SIZING (cpu performance impacting) */ 379 /****************************************************/ 380 /* What amount is our formula using to get TSO size */ 381 static int32_t bbr_hptsi_per_second = 1000; 382 383 /* 384 * For hptsi under bbr_cross_over connections what is delay 385 * target 7ms (in usec) combined with a seg_max of 2 386 * gets us close to identical google behavior in 387 * TSO size selection (possibly more 1MSS sends). 388 */ 389 static int32_t bbr_hptsi_segments_delay_tar = 7000; 390 391 /* Does pacing delay include overhead's in its time calculations? */ 392 static int32_t bbr_include_enet_oh = 0; 393 static int32_t bbr_include_ip_oh = 1; 394 static int32_t bbr_include_tcp_oh = 1; 395 static int32_t bbr_google_discount = 10; 396 397 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */ 398 static int32_t bbr_state_is_pkt_epoch = 0; 399 static int32_t bbr_state_drain_2_tar = 1; 400 /* What is the max the 0 - bbr_cross_over MBPS TSO target 401 * can reach using our delay target. Note that this 402 * value becomes the floor for the cross over 403 * algorithm. 404 */ 405 static int32_t bbr_hptsi_segments_max = 2; 406 static int32_t bbr_hptsi_segments_floor = 1; 407 static int32_t bbr_hptsi_utter_max = 0; 408 409 /* What is the min the 0 - bbr_cross-over MBPS TSO target can be */ 410 static int32_t bbr_hptsi_bytes_min = 1460; 411 static int32_t bbr_all_get_min = 0; 412 413 /* Cross over point from algo-a to algo-b */ 414 static uint32_t bbr_cross_over = TWENTY_THREE_MBPS; 415 416 /* Do we deal with our restart state? */ 417 static int32_t bbr_uses_idle_restart = 0; 418 static int32_t bbr_idle_restart_threshold = 100000; /* 100ms in useconds */ 419 420 /* Do we allow hardware pacing? */ 421 static int32_t bbr_allow_hdwr_pacing = 0; 422 static int32_t bbr_hdwr_pace_adjust = 2; /* multipler when we calc the tso size */ 423 static int32_t bbr_hdwr_pace_floor = 1; 424 static int32_t bbr_hdwr_pacing_delay_cnt = 10; 425 426 /****************************************************/ 427 static int32_t bbr_resends_use_tso = 0; 428 static int32_t bbr_tlp_max_resend = 2; 429 static int32_t bbr_sack_block_limit = 128; 430 431 #define BBR_MAX_STAT 19 432 counter_u64_t bbr_state_time[BBR_MAX_STAT]; 433 counter_u64_t bbr_state_lost[BBR_MAX_STAT]; 434 counter_u64_t bbr_state_resend[BBR_MAX_STAT]; 435 counter_u64_t bbr_stat_arry[BBR_STAT_SIZE]; 436 counter_u64_t bbr_opts_arry[BBR_OPTS_SIZE]; 437 counter_u64_t bbr_out_size[TCP_MSS_ACCT_SIZE]; 438 counter_u64_t bbr_flows_whdwr_pacing; 439 counter_u64_t bbr_flows_nohdwr_pacing; 440 441 counter_u64_t bbr_nohdwr_pacing_enobuf; 442 counter_u64_t bbr_hdwr_pacing_enobuf; 443 444 static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr); 445 446 /* 447 * Static defintions we need for forward declarations. 448 */ 449 static uint32_t 450 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, 451 uint32_t useconds_time, uint64_t bw); 452 static uint32_t 453 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain); 454 static void 455 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win); 456 static void 457 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses); 458 static void 459 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line, 460 int dolog); 461 static uint32_t 462 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain); 463 static void 464 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, 465 int32_t pkt_epoch, uint32_t losses); 466 static uint32_t 467 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm); 468 static uint32_t bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp); 469 static uint32_t 470 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, 471 struct bbr_sendmap *rsm, uint32_t srtt, 472 uint32_t cts); 473 static void 474 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, 475 int32_t line); 476 static void 477 bbr_set_state_target(struct tcp_bbr *bbr, int line); 478 static void 479 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line); 480 481 static void 482 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line); 483 484 static void 485 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts); 486 487 static void 488 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts); 489 490 static void 491 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, uint32_t rtt, 492 uint32_t line, uint8_t is_start, uint16_t set); 493 494 static struct bbr_sendmap * 495 bbr_find_lowest_rsm(struct tcp_bbr *bbr); 496 static __inline uint32_t 497 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type); 498 static void 499 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which); 500 501 static void 502 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt, 503 uint32_t thresh, uint32_t to); 504 static void 505 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag); 506 507 static void 508 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, 509 uint32_t del_by, uint32_t cts, uint32_t sloton, uint32_t prev_delay); 510 511 static void 512 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, 513 uint32_t cts, int32_t line); 514 static void 515 bbr_stop_all_timers(struct tcpcb *tp); 516 static void 517 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts); 518 static void 519 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts); 520 static void 521 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts); 522 523 524 static void 525 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len, 526 uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod); 527 528 static inline uint8_t 529 bbr_state_val(struct tcp_bbr *bbr) 530 { 531 return(bbr->rc_bbr_substate); 532 } 533 534 static inline uint32_t 535 get_min_cwnd(struct tcp_bbr *bbr) 536 { 537 int mss; 538 539 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); 540 if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED) 541 return (bbr_cwnd_min_val_hs * mss); 542 else 543 return (bbr_cwnd_min_val * mss); 544 } 545 546 static uint32_t 547 bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr) 548 { 549 uint64_t srtt, var; 550 uint64_t ret_val; 551 552 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT; 553 if (tp->t_srtt == 0) { 554 srtt = (uint64_t)BBR_INITIAL_RTO; 555 var = 0; 556 } else { 557 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT); 558 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT); 559 } 560 TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]), 561 bbr_persist_min, bbr_persist_max); 562 return ((uint32_t)ret_val); 563 } 564 565 static uint32_t 566 bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 567 { 568 /* 569 * Start the FR timer, we do this based on getting the first one in 570 * the rc_tmap. Note that if its NULL we must stop the timer. in all 571 * events we need to stop the running timer (if its running) before 572 * starting the new one. 573 */ 574 uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse; 575 int32_t idx; 576 int32_t is_tlp_timer = 0; 577 struct bbr_sendmap *rsm; 578 579 if (bbr->rc_all_timers_stopped) { 580 /* All timers have been stopped none are to run */ 581 return (0); 582 } 583 if (bbr->rc_in_persist) { 584 /* We can't start any timer in persists */ 585 return (bbr_get_persists_timer_val(tp, bbr)); 586 } 587 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); 588 if ((rsm == NULL) || 589 ((tp->t_flags & TF_SACK_PERMIT) == 0) || 590 (tp->t_state < TCPS_ESTABLISHED)) { 591 /* Nothing on the send map */ 592 activate_rxt: 593 if (SEQ_LT(tp->snd_una, tp->snd_max) || sbavail(&(tp->t_inpcb->inp_socket->so_snd))) { 594 uint64_t tov; 595 596 time_since_sent = 0; 597 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); 598 if (rsm) { 599 idx = rsm->r_rtr_cnt - 1; 600 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time)) 601 tstmp_touse = rsm->r_tim_lastsent[idx]; 602 else 603 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time; 604 if (TSTMP_GT(tstmp_touse, cts)) 605 time_since_sent = cts - tstmp_touse; 606 } 607 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT; 608 if (tp->t_srtt == 0) 609 tov = BBR_INITIAL_RTO; 610 else 611 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) + 612 ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT); 613 if (tp->t_rxtshift) 614 tov *= tcp_backoff[tp->t_rxtshift]; 615 if (tov > time_since_sent) 616 tov -= time_since_sent; 617 else 618 tov = bbr->r_ctl.rc_min_to; 619 TCPT_RANGESET_NOSLOP(to, tov, 620 (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC), 621 (bbr->rc_max_rto_sec * USECS_IN_SECOND)); 622 bbr_log_timer_var(bbr, 2, cts, 0, srtt, 0, to); 623 return (to); 624 } 625 return (0); 626 } 627 if (rsm->r_flags & BBR_ACKED) { 628 rsm = bbr_find_lowest_rsm(bbr); 629 if (rsm == NULL) { 630 /* No lowest? */ 631 goto activate_rxt; 632 } 633 } 634 /* Convert from ms to usecs */ 635 if (rsm->r_flags & BBR_SACK_PASSED) { 636 if ((tp->t_flags & TF_SENTFIN) && 637 ((tp->snd_max - tp->snd_una) == 1) && 638 (rsm->r_flags & BBR_HAS_FIN)) { 639 /* 640 * We don't start a bbr rack timer if all we have is 641 * a FIN outstanding. 642 */ 643 goto activate_rxt; 644 } 645 srtt = bbr_get_rtt(bbr, BBR_RTT_RACK); 646 thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm); 647 idx = rsm->r_rtr_cnt - 1; 648 exp = rsm->r_tim_lastsent[idx] + thresh; 649 if (SEQ_GEQ(exp, cts)) { 650 to = exp - cts; 651 if (to < bbr->r_ctl.rc_min_to) { 652 to = bbr->r_ctl.rc_min_to; 653 } 654 } else { 655 to = bbr->r_ctl.rc_min_to; 656 } 657 } else { 658 /* Ok we need to do a TLP not RACK */ 659 if (bbr->rc_tlp_in_progress != 0) { 660 /* 661 * The previous send was a TLP. 662 */ 663 goto activate_rxt; 664 } 665 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext); 666 if (rsm == NULL) { 667 /* We found no rsm to TLP with. */ 668 goto activate_rxt; 669 } 670 if (rsm->r_flags & BBR_HAS_FIN) { 671 /* If its a FIN we don't do TLP */ 672 rsm = NULL; 673 goto activate_rxt; 674 } 675 time_since_sent = 0; 676 idx = rsm->r_rtr_cnt - 1; 677 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time)) 678 tstmp_touse = rsm->r_tim_lastsent[idx]; 679 else 680 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time; 681 if (TSTMP_GT(tstmp_touse, cts)) 682 time_since_sent = cts - tstmp_touse; 683 is_tlp_timer = 1; 684 srtt = bbr_get_rtt(bbr, bbr_tlp_type_to_use); 685 thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts); 686 if (thresh > time_since_sent) 687 to = thresh - time_since_sent; 688 else 689 to = bbr->r_ctl.rc_min_to; 690 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { 691 /* 692 * If the TLP time works out to larger than the max 693 * RTO lets not do TLP.. just RTO. 694 */ 695 goto activate_rxt; 696 } 697 if ((bbr->rc_tlp_rtx_out == 1) && 698 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) { 699 /* 700 * Second retransmit of the same TLP 701 * lets not. 702 */ 703 bbr->rc_tlp_rtx_out = 0; 704 goto activate_rxt; 705 } 706 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) { 707 /* 708 * The tail is no longer the last one I did a probe 709 * on 710 */ 711 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; 712 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start; 713 } 714 } 715 if (is_tlp_timer == 0) { 716 BBR_STAT_INC(bbr_to_arm_rack); 717 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK; 718 } else { 719 bbr_log_timer_var(bbr, 1, cts, time_since_sent, srtt, thresh, to); 720 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) { 721 /* 722 * We have exceeded how many times we can retran the 723 * current TLP timer, switch to the RTO timer. 724 */ 725 goto activate_rxt; 726 } else { 727 BBR_STAT_INC(bbr_to_arm_tlp); 728 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP; 729 } 730 } 731 return (to); 732 } 733 734 static inline int32_t 735 bbr_minseg(struct tcp_bbr *bbr) 736 { 737 return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options); 738 } 739 740 static void 741 bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t slot, uint32_t tot_len) 742 { 743 struct inpcb *inp; 744 struct hpts_diag diag; 745 uint32_t delayed_ack = 0; 746 uint32_t left = 0; 747 uint32_t hpts_timeout; 748 uint8_t stopped; 749 int32_t delay_calc = 0; 750 uint32_t prev_delay = 0; 751 752 inp = tp->t_inpcb; 753 if (inp->inp_in_hpts) { 754 /* A previous call is already set up */ 755 return; 756 } 757 if ((tp->t_state == TCPS_CLOSED) || 758 (tp->t_state == TCPS_LISTEN)) { 759 return; 760 } 761 stopped = bbr->rc_tmr_stopped; 762 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) { 763 left = bbr->r_ctl.rc_timer_exp - cts; 764 } 765 bbr->r_ctl.rc_hpts_flags = 0; 766 bbr->r_ctl.rc_timer_exp = 0; 767 prev_delay = bbr->r_ctl.rc_last_delay_val; 768 if (bbr->r_ctl.rc_last_delay_val && 769 (slot == 0)) { 770 /* 771 * If a previous pacer delay was in place we 772 * are not coming from the output side (where 773 * we calculate a delay, more likely a timer). 774 */ 775 slot = bbr->r_ctl.rc_last_delay_val; 776 if (TSTMP_GT(cts, bbr->rc_pacer_started)) { 777 /* Compensate for time passed */ 778 delay_calc = cts - bbr->rc_pacer_started; 779 if (delay_calc <= slot) 780 slot -= delay_calc; 781 } 782 } 783 /* Do we have early to make up for by pushing out the pacing time? */ 784 if (bbr->r_agg_early_set) { 785 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2); 786 slot += bbr->r_ctl.rc_agg_early; 787 bbr->r_ctl.rc_agg_early = 0; 788 bbr->r_agg_early_set = 0; 789 } 790 /* Are we running a total debt that needs to be compensated for? */ 791 if (bbr->r_ctl.rc_hptsi_agg_delay) { 792 if (slot > bbr->r_ctl.rc_hptsi_agg_delay) { 793 /* We nuke the delay */ 794 slot -= bbr->r_ctl.rc_hptsi_agg_delay; 795 bbr->r_ctl.rc_hptsi_agg_delay = 0; 796 } else { 797 /* We nuke some of the delay, put in a minimal 100usecs */ 798 bbr->r_ctl.rc_hptsi_agg_delay -= slot; 799 bbr->r_ctl.rc_last_delay_val = slot = 100; 800 } 801 } 802 bbr->r_ctl.rc_last_delay_val = slot; 803 hpts_timeout = bbr_timer_start(tp, bbr, cts); 804 if (tp->t_flags & TF_DELACK) { 805 if (bbr->rc_in_persist == 0) { 806 delayed_ack = bbr_delack_time; 807 } else { 808 /* 809 * We are in persists and have 810 * gotten a new data element. 811 */ 812 if (hpts_timeout > bbr_delack_time) { 813 /* 814 * Lets make the persists timer (which acks) 815 * be the smaller of hpts_timeout and bbr_delack_time. 816 */ 817 hpts_timeout = bbr_delack_time; 818 } 819 } 820 } 821 if (delayed_ack && 822 ((hpts_timeout == 0) || 823 (delayed_ack < hpts_timeout))) { 824 /* We need a Delayed ack timer */ 825 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; 826 hpts_timeout = delayed_ack; 827 } 828 if (slot) { 829 /* Mark that we have a pacing timer up */ 830 BBR_STAT_INC(bbr_paced_segments); 831 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT; 832 } 833 /* 834 * If no timers are going to run and we will fall off thfe hptsi 835 * wheel, we resort to a keep-alive timer if its configured. 836 */ 837 if ((hpts_timeout == 0) && 838 (slot == 0)) { 839 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && 840 (tp->t_state <= TCPS_CLOSING)) { 841 /* 842 * Ok we have no timer (persists, rack, tlp, rxt or 843 * del-ack), we don't have segments being paced. So 844 * all that is left is the keepalive timer. 845 */ 846 if (TCPS_HAVEESTABLISHED(tp->t_state)) { 847 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp)); 848 } else { 849 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp)); 850 } 851 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP; 852 } 853 } 854 if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) == 855 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) { 856 /* 857 * RACK, TLP, persists and RXT timers all are restartable 858 * based on actions input .. i.e we received a packet (ack 859 * or sack) and that changes things (rw, or snd_una etc). 860 * Thus we can restart them with a new value. For 861 * keep-alive, delayed_ack we keep track of what was left 862 * and restart the timer with a smaller value. 863 */ 864 if (left < hpts_timeout) 865 hpts_timeout = left; 866 } 867 if (bbr->r_ctl.rc_incr_tmrs && slot && 868 (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) { 869 /* 870 * If configured to do so, and the timer is either 871 * the TLP or RXT timer, we need to increase the timeout 872 * by the pacing time. Consider the bottleneck at my 873 * machine as an example, we are sending something 874 * to start a TLP on. The last packet won't be emitted 875 * fully until the pacing time (the bottleneck will hold 876 * the data in place). Once the packet is emitted that 877 * is when we want to start waiting for the TLP. This 878 * is most evident with hardware pacing (where the nic 879 * is holding the packet(s) before emitting). But it 880 * can also show up in the network so we do it for all 881 * cases. Technically we would take off one packet from 882 * this extra delay but this is easier and being more 883 * conservative is probably better. 884 */ 885 hpts_timeout += slot; 886 } 887 if (hpts_timeout) { 888 /* 889 * Hack alert for now we can't time-out over 2147 seconds (a 890 * bit more than 35min) 891 */ 892 if (hpts_timeout > 0x7ffffffe) 893 hpts_timeout = 0x7ffffffe; 894 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; 895 } else 896 bbr->r_ctl.rc_timer_exp = 0; 897 if ((slot) && 898 (bbr->rc_use_google || 899 bbr->output_error_seen || 900 (slot <= hpts_timeout)) ) { 901 /* 902 * Tell LRO that it can queue packets while 903 * we pace. 904 */ 905 bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY; 906 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && 907 (bbr->rc_cwnd_limited == 0)) { 908 /* 909 * If we are not cwnd limited and we 910 * are running a rack timer we put on 911 * the do not disturbe even for sack. 912 */ 913 inp->inp_flags2 |= INP_DONT_SACK_QUEUE; 914 } else 915 inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE; 916 bbr->rc_pacer_started = cts; 917 918 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(slot), 919 __LINE__, &diag); 920 bbr->rc_timer_first = 0; 921 bbr->bbr_timer_src = frm; 922 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 1); 923 bbr_log_hpts_diag(bbr, cts, &diag); 924 } else if (hpts_timeout) { 925 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(hpts_timeout), 926 __LINE__, &diag); 927 /* 928 * We add the flag here as well if the slot is set, 929 * since hpts will call in to clear the queue first before 930 * calling the output routine (which does our timers). 931 * We don't want to set the flag if its just a timer 932 * else the arrival of data might (that causes us 933 * to send more) might get delayed. Imagine being 934 * on a keep-alive timer and a request comes in for 935 * more data. 936 */ 937 if (slot) 938 bbr->rc_pacer_started = cts; 939 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && 940 (bbr->rc_cwnd_limited == 0)) { 941 /* 942 * For a rack timer, don't wake us even 943 * if a sack arrives as long as we are 944 * not cwnd limited. 945 */ 946 bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY; 947 inp->inp_flags2 |= INP_DONT_SACK_QUEUE; 948 } else { 949 /* All other timers wake us up */ 950 bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY; 951 inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE; 952 } 953 bbr->bbr_timer_src = frm; 954 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 0); 955 bbr_log_hpts_diag(bbr, cts, &diag); 956 bbr->rc_timer_first = 1; 957 } 958 bbr->rc_tmr_stopped = 0; 959 bbr_log_type_bbrsnd(bbr, tot_len, slot, delay_calc, cts, frm, prev_delay); 960 } 961 962 static void 963 bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb) 964 { 965 /* 966 * We received an ack, and then did not call send or were bounced 967 * out due to the hpts was running. Now a timer is up as well, is it 968 * the right timer? 969 */ 970 struct inpcb *inp; 971 struct bbr_sendmap *rsm; 972 uint32_t hpts_timeout; 973 int tmr_up; 974 975 tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK; 976 if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT)) 977 return; 978 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); 979 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) && 980 (tmr_up == PACE_TMR_RXT)) { 981 /* Should be an RXT */ 982 return; 983 } 984 inp = bbr->rc_inp; 985 if (rsm == NULL) { 986 /* Nothing outstanding? */ 987 if (tp->t_flags & TF_DELACK) { 988 if (tmr_up == PACE_TMR_DELACK) 989 /* 990 * We are supposed to have delayed ack up 991 * and we do 992 */ 993 return; 994 } else if (sbavail(&inp->inp_socket->so_snd) && 995 (tmr_up == PACE_TMR_RXT)) { 996 /* 997 * if we hit enobufs then we would expect the 998 * possiblity of nothing outstanding and the RXT up 999 * (and the hptsi timer). 1000 */ 1001 return; 1002 } else if (((V_tcp_always_keepalive || 1003 inp->inp_socket->so_options & SO_KEEPALIVE) && 1004 (tp->t_state <= TCPS_CLOSING)) && 1005 (tmr_up == PACE_TMR_KEEP) && 1006 (tp->snd_max == tp->snd_una)) { 1007 /* We should have keep alive up and we do */ 1008 return; 1009 } 1010 } 1011 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) { 1012 if ((tp->t_flags & TF_SENTFIN) && 1013 ((tp->snd_max - tp->snd_una) == 1) && 1014 (rsm->r_flags & BBR_HAS_FIN)) { 1015 /* needs to be a RXT */ 1016 if (tmr_up == PACE_TMR_RXT) 1017 return; 1018 else 1019 goto wrong_timer; 1020 } else if (tmr_up == PACE_TMR_RACK) 1021 return; 1022 else 1023 goto wrong_timer; 1024 } else if (rsm && (tmr_up == PACE_TMR_RACK)) { 1025 /* Rack timer has priority if we have data out */ 1026 return; 1027 } else if (SEQ_GT(tp->snd_max, tp->snd_una) && 1028 ((tmr_up == PACE_TMR_TLP) || 1029 (tmr_up == PACE_TMR_RXT))) { 1030 /* 1031 * Either a TLP or RXT is fine if no sack-passed is in place 1032 * and data is outstanding. 1033 */ 1034 return; 1035 } else if (tmr_up == PACE_TMR_DELACK) { 1036 /* 1037 * If the delayed ack was going to go off before the 1038 * rtx/tlp/rack timer were going to expire, then that would 1039 * be the timer in control. Note we don't check the time 1040 * here trusting the code is correct. 1041 */ 1042 return; 1043 } 1044 if (SEQ_GT(tp->snd_max, tp->snd_una) && 1045 ((tmr_up == PACE_TMR_RXT) || 1046 (tmr_up == PACE_TMR_TLP) || 1047 (tmr_up == PACE_TMR_RACK))) { 1048 /* 1049 * We have outstanding data and 1050 * we *do* have a RACK, TLP or RXT 1051 * timer running. We won't restart 1052 * anything here since thats probably ok we 1053 * will get called with some timer here shortly. 1054 */ 1055 return; 1056 } 1057 /* 1058 * Ok the timer originally started is not what we want now. We will 1059 * force the hpts to be stopped if any, and restart with the slot 1060 * set to what was in the saved slot. 1061 */ 1062 wrong_timer: 1063 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) { 1064 if (inp->inp_in_hpts) 1065 tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); 1066 bbr_timer_cancel(bbr, __LINE__, cts); 1067 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val, 1068 0); 1069 } else { 1070 /* 1071 * Output is hptsi so we just need to switch the type of 1072 * timer. We don't bother with keep-alive, since when we 1073 * jump through the output, it will start the keep-alive if 1074 * nothing is sent. 1075 * 1076 * We only need a delayed-ack added and or the hpts_timeout. 1077 */ 1078 hpts_timeout = bbr_timer_start(tp, bbr, cts); 1079 if (tp->t_flags & TF_DELACK) { 1080 if (hpts_timeout == 0) { 1081 hpts_timeout = bbr_delack_time; 1082 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; 1083 } 1084 else if (hpts_timeout > bbr_delack_time) { 1085 hpts_timeout = bbr_delack_time; 1086 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK; 1087 } 1088 } 1089 if (hpts_timeout) { 1090 if (hpts_timeout > 0x7ffffffe) 1091 hpts_timeout = 0x7ffffffe; 1092 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout; 1093 } 1094 } 1095 } 1096 1097 int32_t bbr_clear_lost = 0; 1098 1099 /* 1100 * Considers the two time values now (cts) and earlier. 1101 * If cts is smaller than earlier, we could have 1102 * had a sequence wrap (our counter wraps every 1103 * 70 min or so) or it could be just clock skew 1104 * getting us two differnt time values. Clock skew 1105 * will show up within 10ms or so. So in such 1106 * a case (where cts is behind earlier time by 1107 * less than 10ms) we return 0. Otherwise we 1108 * return the true difference between them. 1109 */ 1110 static inline uint32_t 1111 bbr_calc_time(uint32_t cts, uint32_t earlier_time) { 1112 /* 1113 * Given two timestamps, the current time stamp cts, and some other 1114 * time-stamp taken in theory earlier return the difference. The 1115 * trick is here sometimes locking will get the other timestamp 1116 * after the cts. If this occurs we need to return 0. 1117 */ 1118 if (TSTMP_GEQ(cts, earlier_time)) 1119 return (cts - earlier_time); 1120 /* 1121 * cts is behind earlier_time if its less than 10ms consider it 0. 1122 * If its more than 10ms difference then we had a time wrap. Else 1123 * its just the normal locking foo. I wonder if we should not go to 1124 * 64bit TS and get rid of this issue. 1125 */ 1126 if (TSTMP_GEQ((cts + 10000), earlier_time)) 1127 return (0); 1128 /* 1129 * Ok the time must have wrapped. So we need to answer a large 1130 * amount of time, which the normal subtraction should do. 1131 */ 1132 return (cts - earlier_time); 1133 } 1134 1135 1136 1137 static int 1138 sysctl_bbr_clear_lost(SYSCTL_HANDLER_ARGS) 1139 { 1140 uint32_t stat; 1141 int32_t error; 1142 1143 error = SYSCTL_OUT(req, &bbr_clear_lost, sizeof(uint32_t)); 1144 if (error || req->newptr == NULL) 1145 return error; 1146 1147 error = SYSCTL_IN(req, &stat, sizeof(uint32_t)); 1148 if (error) 1149 return (error); 1150 if (stat == 1) { 1151 #ifdef BBR_INVARIANTS 1152 printf("Clearing BBR lost counters\n"); 1153 #endif 1154 COUNTER_ARRAY_ZERO(bbr_state_lost, BBR_MAX_STAT); 1155 COUNTER_ARRAY_ZERO(bbr_state_time, BBR_MAX_STAT); 1156 COUNTER_ARRAY_ZERO(bbr_state_resend, BBR_MAX_STAT); 1157 } else if (stat == 2) { 1158 #ifdef BBR_INVARIANTS 1159 printf("Clearing BBR option counters\n"); 1160 #endif 1161 COUNTER_ARRAY_ZERO(bbr_opts_arry, BBR_OPTS_SIZE); 1162 } else if (stat == 3) { 1163 #ifdef BBR_INVARIANTS 1164 printf("Clearing BBR stats counters\n"); 1165 #endif 1166 COUNTER_ARRAY_ZERO(bbr_stat_arry, BBR_STAT_SIZE); 1167 } else if (stat == 4) { 1168 #ifdef BBR_INVARIANTS 1169 printf("Clearing BBR out-size counters\n"); 1170 #endif 1171 COUNTER_ARRAY_ZERO(bbr_out_size, TCP_MSS_ACCT_SIZE); 1172 } 1173 bbr_clear_lost = 0; 1174 return (0); 1175 } 1176 1177 static void 1178 bbr_init_sysctls(void) 1179 { 1180 struct sysctl_oid *bbr_probertt; 1181 struct sysctl_oid *bbr_hptsi; 1182 struct sysctl_oid *bbr_measure; 1183 struct sysctl_oid *bbr_cwnd; 1184 struct sysctl_oid *bbr_timeout; 1185 struct sysctl_oid *bbr_states; 1186 struct sysctl_oid *bbr_startup; 1187 struct sysctl_oid *bbr_policer; 1188 1189 /* Probe rtt controls */ 1190 bbr_probertt = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1191 SYSCTL_CHILDREN(bbr_sysctl_root), 1192 OID_AUTO, 1193 "probertt", 1194 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1195 ""); 1196 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1197 SYSCTL_CHILDREN(bbr_probertt), 1198 OID_AUTO, "gain", CTLFLAG_RW, 1199 &bbr_rttprobe_gain, 192, 1200 "What is the filter gain drop in probe_rtt (0=disable)?"); 1201 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1202 SYSCTL_CHILDREN(bbr_probertt), 1203 OID_AUTO, "cwnd", CTLFLAG_RW, 1204 &bbr_rtt_probe_cwndtarg, 4, 1205 "How many mss's are outstanding during probe-rtt"); 1206 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1207 SYSCTL_CHILDREN(bbr_probertt), 1208 OID_AUTO, "int", CTLFLAG_RW, 1209 &bbr_rtt_probe_limit, 4000000, 1210 "If RTT has not shrank in this many micro-seconds enter probe-rtt"); 1211 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1212 SYSCTL_CHILDREN(bbr_probertt), 1213 OID_AUTO, "mintime", CTLFLAG_RW, 1214 &bbr_rtt_probe_time, 200000, 1215 "How many microseconds in probe-rtt"); 1216 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1217 SYSCTL_CHILDREN(bbr_probertt), 1218 OID_AUTO, "filter_len_sec", CTLFLAG_RW, 1219 &bbr_filter_len_sec, 6, 1220 "How long in seconds does the rttProp filter run?"); 1221 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1222 SYSCTL_CHILDREN(bbr_probertt), 1223 OID_AUTO, "drain_rtt", CTLFLAG_RW, 1224 &bbr_drain_rtt, BBR_SRTT, 1225 "What is the drain rtt to use in probeRTT (rtt_prop=0, rtt_rack=1, rtt_pkt=2, rtt_srtt=3?"); 1226 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1227 SYSCTL_CHILDREN(bbr_probertt), 1228 OID_AUTO, "can_force", CTLFLAG_RW, 1229 &bbr_can_force_probertt, 0, 1230 "If we keep setting new low rtt's but delay going in probe-rtt can we force in??"); 1231 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1232 SYSCTL_CHILDREN(bbr_probertt), 1233 OID_AUTO, "enter_sets_force", CTLFLAG_RW, 1234 &bbr_probertt_sets_rtt, 0, 1235 "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?"); 1236 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1237 SYSCTL_CHILDREN(bbr_probertt), 1238 OID_AUTO, "can_adjust", CTLFLAG_RW, 1239 &bbr_can_adjust_probertt, 1, 1240 "Can we dynamically adjust the probe-rtt limits and times?"); 1241 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1242 SYSCTL_CHILDREN(bbr_probertt), 1243 OID_AUTO, "is_ratio", CTLFLAG_RW, 1244 &bbr_is_ratio, 0, 1245 "is the limit to filter a ratio?"); 1246 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1247 SYSCTL_CHILDREN(bbr_probertt), 1248 OID_AUTO, "use_cwnd", CTLFLAG_RW, 1249 &bbr_prtt_slam_cwnd, 0, 1250 "Should we set/recover cwnd?"); 1251 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1252 SYSCTL_CHILDREN(bbr_probertt), 1253 OID_AUTO, "can_use_ts", CTLFLAG_RW, 1254 &bbr_can_use_ts_for_rtt, 1, 1255 "Can we use the ms timestamp if available for retransmistted rtt calculations?"); 1256 1257 /* Pacing controls */ 1258 bbr_hptsi = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1259 SYSCTL_CHILDREN(bbr_sysctl_root), 1260 OID_AUTO, 1261 "pacing", 1262 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1263 ""); 1264 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1265 SYSCTL_CHILDREN(bbr_hptsi), 1266 OID_AUTO, "hw_pacing", CTLFLAG_RW, 1267 &bbr_allow_hdwr_pacing, 1, 1268 "Do we allow hardware pacing?"); 1269 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1270 SYSCTL_CHILDREN(bbr_hptsi), 1271 OID_AUTO, "hw_pacing_limit", CTLFLAG_RW, 1272 &bbr_hardware_pacing_limit, 4000, 1273 "Do we have a limited number of connections for pacing chelsio (0=no limit)?"); 1274 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1275 SYSCTL_CHILDREN(bbr_hptsi), 1276 OID_AUTO, "hw_pacing_adj", CTLFLAG_RW, 1277 &bbr_hdwr_pace_adjust, 2, 1278 "Multiplier to calculated tso size?"); 1279 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1280 SYSCTL_CHILDREN(bbr_hptsi), 1281 OID_AUTO, "hw_pacing_floor", CTLFLAG_RW, 1282 &bbr_hdwr_pace_floor, 1, 1283 "Do we invoke the hardware pacing floor?"); 1284 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1285 SYSCTL_CHILDREN(bbr_hptsi), 1286 OID_AUTO, "hw_pacing_delay_cnt", CTLFLAG_RW, 1287 &bbr_hdwr_pacing_delay_cnt, 10, 1288 "How many packets must be sent after hdwr pacing is enabled"); 1289 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1290 SYSCTL_CHILDREN(bbr_hptsi), 1291 OID_AUTO, "bw_cross", CTLFLAG_RW, 1292 &bbr_cross_over, 3000000, 1293 "What is the point where we cross over to linux like TSO size set"); 1294 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1295 SYSCTL_CHILDREN(bbr_hptsi), 1296 OID_AUTO, "seg_deltarg", CTLFLAG_RW, 1297 &bbr_hptsi_segments_delay_tar, 7000, 1298 "What is the worse case delay target for hptsi < 48Mbp connections"); 1299 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1300 SYSCTL_CHILDREN(bbr_hptsi), 1301 OID_AUTO, "enet_oh", CTLFLAG_RW, 1302 &bbr_include_enet_oh, 0, 1303 "Do we include the ethernet overhead in calculating pacing delay?"); 1304 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1305 SYSCTL_CHILDREN(bbr_hptsi), 1306 OID_AUTO, "ip_oh", CTLFLAG_RW, 1307 &bbr_include_ip_oh, 1, 1308 "Do we include the IP overhead in calculating pacing delay?"); 1309 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1310 SYSCTL_CHILDREN(bbr_hptsi), 1311 OID_AUTO, "tcp_oh", CTLFLAG_RW, 1312 &bbr_include_tcp_oh, 0, 1313 "Do we include the TCP overhead in calculating pacing delay?"); 1314 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1315 SYSCTL_CHILDREN(bbr_hptsi), 1316 OID_AUTO, "google_discount", CTLFLAG_RW, 1317 &bbr_google_discount, 10, 1318 "What is the default google discount percentage wise for pacing (11 = 1.1%%)?"); 1319 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1320 SYSCTL_CHILDREN(bbr_hptsi), 1321 OID_AUTO, "all_get_min", CTLFLAG_RW, 1322 &bbr_all_get_min, 0, 1323 "If you are less than a MSS do you just get the min?"); 1324 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1325 SYSCTL_CHILDREN(bbr_hptsi), 1326 OID_AUTO, "tso_min", CTLFLAG_RW, 1327 &bbr_hptsi_bytes_min, 1460, 1328 "For 0 -> 24Mbps what is floor number of segments for TSO"); 1329 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1330 SYSCTL_CHILDREN(bbr_hptsi), 1331 OID_AUTO, "seg_tso_max", CTLFLAG_RW, 1332 &bbr_hptsi_segments_max, 6, 1333 "For 0 -> 24Mbps what is top number of segments for TSO"); 1334 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1335 SYSCTL_CHILDREN(bbr_hptsi), 1336 OID_AUTO, "seg_floor", CTLFLAG_RW, 1337 &bbr_hptsi_segments_floor, 1, 1338 "Minimum TSO size we will fall too in segments"); 1339 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1340 SYSCTL_CHILDREN(bbr_hptsi), 1341 OID_AUTO, "utter_max", CTLFLAG_RW, 1342 &bbr_hptsi_utter_max, 0, 1343 "The absolute maximum that any pacing (outside of hardware) can be"); 1344 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1345 SYSCTL_CHILDREN(bbr_hptsi), 1346 OID_AUTO, "seg_divisor", CTLFLAG_RW, 1347 &bbr_hptsi_per_second, 100, 1348 "What is the divisor in our hptsi TSO calculation 512Mbps < X > 24Mbps "); 1349 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1350 SYSCTL_CHILDREN(bbr_hptsi), 1351 OID_AUTO, "srtt_mul", CTLFLAG_RW, 1352 &bbr_hptsi_max_mul, 1, 1353 "The multiplier for pace len max"); 1354 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1355 SYSCTL_CHILDREN(bbr_hptsi), 1356 OID_AUTO, "srtt_div", CTLFLAG_RW, 1357 &bbr_hptsi_max_div, 2, 1358 "The divisor for pace len max"); 1359 /* Measurement controls */ 1360 bbr_measure = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1361 SYSCTL_CHILDREN(bbr_sysctl_root), 1362 OID_AUTO, 1363 "measure", 1364 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1365 "Measurement controls"); 1366 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1367 SYSCTL_CHILDREN(bbr_measure), 1368 OID_AUTO, "min_i_bw", CTLFLAG_RW, 1369 &bbr_initial_bw_bps, 62500, 1370 "Minimum initial b/w in bytes per second"); 1371 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1372 SYSCTL_CHILDREN(bbr_measure), 1373 OID_AUTO, "no_sack_needed", CTLFLAG_RW, 1374 &bbr_sack_not_required, 0, 1375 "Do we allow bbr to run on connections not supporting SACK?"); 1376 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1377 SYSCTL_CHILDREN(bbr_measure), 1378 OID_AUTO, "use_google", CTLFLAG_RW, 1379 &bbr_use_google_algo, 0, 1380 "Use has close to google V1.0 has possible?"); 1381 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1382 SYSCTL_CHILDREN(bbr_measure), 1383 OID_AUTO, "ts_limiting", CTLFLAG_RW, 1384 &bbr_ts_limiting, 1, 1385 "Do we attempt to use the peers timestamp to limit b/w caculations?"); 1386 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1387 SYSCTL_CHILDREN(bbr_measure), 1388 OID_AUTO, "ts_can_raise", CTLFLAG_RW, 1389 &bbr_ts_can_raise, 0, 1390 "Can we raise the b/w via timestamp b/w calculation?"); 1391 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1392 SYSCTL_CHILDREN(bbr_measure), 1393 OID_AUTO, "ts_delta", CTLFLAG_RW, 1394 &bbr_min_usec_delta, 20000, 1395 "How long in usec between ts of our sends in ts validation code?"); 1396 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1397 SYSCTL_CHILDREN(bbr_measure), 1398 OID_AUTO, "ts_peer_delta", CTLFLAG_RW, 1399 &bbr_min_peer_delta, 20, 1400 "What min numerical value should be between the peer deltas?"); 1401 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1402 SYSCTL_CHILDREN(bbr_measure), 1403 OID_AUTO, "ts_delta_percent", CTLFLAG_RW, 1404 &bbr_delta_percent, 150, 1405 "What percentage (150 = 15.0) do we allow variance for?"); 1406 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1407 SYSCTL_CHILDREN(bbr_measure), 1408 OID_AUTO, "min_measure_good_bw", CTLFLAG_RW, 1409 &bbr_min_measurements_req, 1, 1410 "What is the minimum measurment count we need before we switch to our b/w estimate"); 1411 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1412 SYSCTL_CHILDREN(bbr_measure), 1413 OID_AUTO, "min_measure_before_pace", CTLFLAG_RW, 1414 &bbr_no_pacing_until, 4, 1415 "How many pkt-epoch's (0 is off) do we need before pacing is on?"); 1416 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1417 SYSCTL_CHILDREN(bbr_measure), 1418 OID_AUTO, "quanta", CTLFLAG_RW, 1419 &bbr_quanta, 2, 1420 "Extra quanta to add when calculating the target (ID section 4.2.3.2)."); 1421 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1422 SYSCTL_CHILDREN(bbr_measure), 1423 OID_AUTO, "noretran", CTLFLAG_RW, 1424 &bbr_no_retran, 0, 1425 "Should google mode not use retransmission measurements for the b/w estimation?"); 1426 /* State controls */ 1427 bbr_states = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1428 SYSCTL_CHILDREN(bbr_sysctl_root), 1429 OID_AUTO, 1430 "states", 1431 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1432 "State controls"); 1433 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1434 SYSCTL_CHILDREN(bbr_states), 1435 OID_AUTO, "idle_restart", CTLFLAG_RW, 1436 &bbr_uses_idle_restart, 0, 1437 "Do we use a new special idle_restart state to ramp back up quickly?"); 1438 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1439 SYSCTL_CHILDREN(bbr_states), 1440 OID_AUTO, "idle_restart_threshold", CTLFLAG_RW, 1441 &bbr_idle_restart_threshold, 100000, 1442 "How long must we be idle before we restart??"); 1443 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1444 SYSCTL_CHILDREN(bbr_states), 1445 OID_AUTO, "use_pkt_epoch", CTLFLAG_RW, 1446 &bbr_state_is_pkt_epoch, 0, 1447 "Do we use a pkt-epoch for substate if 0 rttProp?"); 1448 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1449 SYSCTL_CHILDREN(bbr_states), 1450 OID_AUTO, "startup_rtt_gain", CTLFLAG_RW, 1451 &bbr_rtt_gain_thresh, 0, 1452 "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?"); 1453 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1454 SYSCTL_CHILDREN(bbr_states), 1455 OID_AUTO, "drain_floor", CTLFLAG_RW, 1456 &bbr_drain_floor, 88, 1457 "What is the lowest we can drain (pg) too?"); 1458 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1459 SYSCTL_CHILDREN(bbr_states), 1460 OID_AUTO, "drain_2_target", CTLFLAG_RW, 1461 &bbr_state_drain_2_tar, 1, 1462 "Do we drain to target in drain substate?"); 1463 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1464 SYSCTL_CHILDREN(bbr_states), 1465 OID_AUTO, "gain_2_target", CTLFLAG_RW, 1466 &bbr_gain_to_target, 1, 1467 "Does probe bw gain to target??"); 1468 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1469 SYSCTL_CHILDREN(bbr_states), 1470 OID_AUTO, "gain_extra_time", CTLFLAG_RW, 1471 &bbr_gain_gets_extra_too, 1, 1472 "Does probe bw gain get the extra time too?"); 1473 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1474 SYSCTL_CHILDREN(bbr_states), 1475 OID_AUTO, "ld_div", CTLFLAG_RW, 1476 &bbr_drain_drop_div, 5, 1477 "Long drain drop divider?"); 1478 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1479 SYSCTL_CHILDREN(bbr_states), 1480 OID_AUTO, "ld_mul", CTLFLAG_RW, 1481 &bbr_drain_drop_mul, 4, 1482 "Long drain drop multiplier?"); 1483 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1484 SYSCTL_CHILDREN(bbr_states), 1485 OID_AUTO, "rand_ot_disc", CTLFLAG_RW, 1486 &bbr_rand_ot, 50, 1487 "Random discount of the ot?"); 1488 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1489 SYSCTL_CHILDREN(bbr_states), 1490 OID_AUTO, "dr_filter_life", CTLFLAG_RW, 1491 &bbr_num_pktepo_for_del_limit, BBR_NUM_RTTS_FOR_DEL_LIMIT, 1492 "How many packet-epochs does the b/w delivery rate last?"); 1493 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1494 SYSCTL_CHILDREN(bbr_states), 1495 OID_AUTO, "subdrain_applimited", CTLFLAG_RW, 1496 &bbr_sub_drain_app_limit, 0, 1497 "Does our sub-state drain invoke app limited if its long?"); 1498 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1499 SYSCTL_CHILDREN(bbr_states), 1500 OID_AUTO, "use_cwnd_subdrain", CTLFLAG_RW, 1501 &bbr_sub_drain_slam_cwnd, 0, 1502 "Should we set/recover cwnd for sub-state drain?"); 1503 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1504 SYSCTL_CHILDREN(bbr_states), 1505 OID_AUTO, "use_cwnd_maindrain", CTLFLAG_RW, 1506 &bbr_slam_cwnd_in_main_drain, 0, 1507 "Should we set/recover cwnd for main-state drain?"); 1508 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1509 SYSCTL_CHILDREN(bbr_states), 1510 OID_AUTO, "google_gets_earlyout", CTLFLAG_RW, 1511 &google_allow_early_out, 1, 1512 "Should we allow google probe-bw/drain to exit early at flight target?"); 1513 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1514 SYSCTL_CHILDREN(bbr_states), 1515 OID_AUTO, "google_exit_loss", CTLFLAG_RW, 1516 &google_consider_lost, 1, 1517 "Should we have losses exit gain of probebw in google mode??"); 1518 /* Startup controls */ 1519 bbr_startup = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1520 SYSCTL_CHILDREN(bbr_sysctl_root), 1521 OID_AUTO, 1522 "startup", 1523 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1524 "Startup controls"); 1525 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1526 SYSCTL_CHILDREN(bbr_startup), 1527 OID_AUTO, "cheat_iwnd", CTLFLAG_RW, 1528 &bbr_sends_full_iwnd, 1, 1529 "Do we not pace but burst out initial windows has our TSO size?"); 1530 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1531 SYSCTL_CHILDREN(bbr_startup), 1532 OID_AUTO, "loss_threshold", CTLFLAG_RW, 1533 &bbr_startup_loss_thresh, 2000, 1534 "In startup what is the loss threshold in a pe that will exit us from startup?"); 1535 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1536 SYSCTL_CHILDREN(bbr_startup), 1537 OID_AUTO, "use_lowerpg", CTLFLAG_RW, 1538 &bbr_use_lower_gain_in_startup, 1, 1539 "Should we use a lower hptsi gain if we see loss in startup?"); 1540 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1541 SYSCTL_CHILDREN(bbr_startup), 1542 OID_AUTO, "gain", CTLFLAG_RW, 1543 &bbr_start_exit, 25, 1544 "What gain percent do we need to see to stay in startup??"); 1545 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1546 SYSCTL_CHILDREN(bbr_startup), 1547 OID_AUTO, "low_gain", CTLFLAG_RW, 1548 &bbr_low_start_exit, 15, 1549 "What gain percent do we need to see to stay in the lower gain startup??"); 1550 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1551 SYSCTL_CHILDREN(bbr_startup), 1552 OID_AUTO, "loss_exit", CTLFLAG_RW, 1553 &bbr_exit_startup_at_loss, 1, 1554 "Should we exit startup at loss in an epoch if we are not gaining?"); 1555 /* CWND controls */ 1556 bbr_cwnd = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1557 SYSCTL_CHILDREN(bbr_sysctl_root), 1558 OID_AUTO, 1559 "cwnd", 1560 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1561 "Cwnd controls"); 1562 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1563 SYSCTL_CHILDREN(bbr_cwnd), 1564 OID_AUTO, "tar_rtt", CTLFLAG_RW, 1565 &bbr_cwndtarget_rtt_touse, 0, 1566 "Target cwnd rtt measurment to use (0=rtt_prop, 1=rtt_rack, 2=pkt_rtt, 3=srtt)?"); 1567 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1568 SYSCTL_CHILDREN(bbr_cwnd), 1569 OID_AUTO, "may_shrink", CTLFLAG_RW, 1570 &bbr_cwnd_may_shrink, 0, 1571 "Can the cwnd shrink if it would grow to more than the target?"); 1572 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1573 SYSCTL_CHILDREN(bbr_cwnd), 1574 OID_AUTO, "max_target_limit", CTLFLAG_RW, 1575 &bbr_target_cwnd_mult_limit, 8, 1576 "Do we limit the cwnd to some multiple of the cwnd target if cwnd can't shrink 0=no?"); 1577 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1578 SYSCTL_CHILDREN(bbr_cwnd), 1579 OID_AUTO, "highspeed_min", CTLFLAG_RW, 1580 &bbr_cwnd_min_val_hs, BBR_HIGHSPEED_NUM_MSS, 1581 "What is the high-speed min cwnd (rttProp under 1ms)"); 1582 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1583 SYSCTL_CHILDREN(bbr_cwnd), 1584 OID_AUTO, "lowspeed_min", CTLFLAG_RW, 1585 &bbr_cwnd_min_val, BBR_PROBERTT_NUM_MSS, 1586 "What is the min cwnd (rttProp > 1ms)"); 1587 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1588 SYSCTL_CHILDREN(bbr_cwnd), 1589 OID_AUTO, "initwin", CTLFLAG_RW, 1590 &bbr_def_init_win, 10, 1591 "What is the BBR initial window, if 0 use tcp version"); 1592 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1593 SYSCTL_CHILDREN(bbr_cwnd), 1594 OID_AUTO, "do_loss_red", CTLFLAG_RW, 1595 &bbr_do_red, 600, 1596 "Do we reduce the b/w at exit from recovery based on ratio of prop/srtt (800=80.0, 0=off)?"); 1597 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1598 SYSCTL_CHILDREN(bbr_cwnd), 1599 OID_AUTO, "red_scale", CTLFLAG_RW, 1600 &bbr_red_scale, 20000, 1601 "What RTT do we scale with?"); 1602 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1603 SYSCTL_CHILDREN(bbr_cwnd), 1604 OID_AUTO, "red_growslow", CTLFLAG_RW, 1605 &bbr_red_growth_restrict, 1, 1606 "Do we restrict cwnd growth for whats in flight?"); 1607 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1608 SYSCTL_CHILDREN(bbr_cwnd), 1609 OID_AUTO, "red_div", CTLFLAG_RW, 1610 &bbr_red_div, 2, 1611 "If we reduce whats the divisor?"); 1612 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1613 SYSCTL_CHILDREN(bbr_cwnd), 1614 OID_AUTO, "red_mul", CTLFLAG_RW, 1615 &bbr_red_mul, 1, 1616 "If we reduce whats the mulitiplier?"); 1617 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1618 SYSCTL_CHILDREN(bbr_cwnd), 1619 OID_AUTO, "target_is_unit", CTLFLAG_RW, 1620 &bbr_target_is_bbunit, 0, 1621 "Is the state target the pacing_gain or BBR_UNIT?"); 1622 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1623 SYSCTL_CHILDREN(bbr_cwnd), 1624 OID_AUTO, "drop_limit", CTLFLAG_RW, 1625 &bbr_drop_limit, 0, 1626 "Number of segments limit for drop (0=use min_cwnd w/flight)?"); 1627 1628 /* Timeout controls */ 1629 bbr_timeout = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1630 SYSCTL_CHILDREN(bbr_sysctl_root), 1631 OID_AUTO, 1632 "timeout", 1633 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1634 "Time out controls"); 1635 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1636 SYSCTL_CHILDREN(bbr_timeout), 1637 OID_AUTO, "delack", CTLFLAG_RW, 1638 &bbr_delack_time, 100000, 1639 "BBR's delayed ack time"); 1640 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1641 SYSCTL_CHILDREN(bbr_timeout), 1642 OID_AUTO, "tlp_uses", CTLFLAG_RW, 1643 &bbr_tlp_type_to_use, 3, 1644 "RTT that TLP uses in its calculations, 0=rttProp, 1=Rack_rtt, 2=pkt_rtt and 3=srtt"); 1645 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1646 SYSCTL_CHILDREN(bbr_timeout), 1647 OID_AUTO, "persmin", CTLFLAG_RW, 1648 &bbr_persist_min, 250000, 1649 "What is the minimum time in microseconds between persists"); 1650 SYSCTL_ADD_U32(&bbr_sysctl_ctx, 1651 SYSCTL_CHILDREN(bbr_timeout), 1652 OID_AUTO, "persmax", CTLFLAG_RW, 1653 &bbr_persist_max, 1000000, 1654 "What is the largest delay in microseconds between persists"); 1655 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1656 SYSCTL_CHILDREN(bbr_timeout), 1657 OID_AUTO, "tlp_minto", CTLFLAG_RW, 1658 &bbr_tlp_min, 10000, 1659 "TLP Min timeout in usecs"); 1660 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1661 SYSCTL_CHILDREN(bbr_timeout), 1662 OID_AUTO, "tlp_dack_time", CTLFLAG_RW, 1663 &bbr_delayed_ack_time, 200000, 1664 "TLP delayed ack compensation value"); 1665 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1666 SYSCTL_CHILDREN(bbr_sysctl_root), 1667 OID_AUTO, "minrto", CTLFLAG_RW, 1668 &bbr_rto_min_ms, 30, 1669 "Minimum RTO in ms"); 1670 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1671 SYSCTL_CHILDREN(bbr_timeout), 1672 OID_AUTO, "maxrto", CTLFLAG_RW, 1673 &bbr_rto_max_sec, 4, 1674 "Maxiumum RTO in seconds -- should be at least as large as min_rto"); 1675 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1676 SYSCTL_CHILDREN(bbr_timeout), 1677 OID_AUTO, "tlp_retry", CTLFLAG_RW, 1678 &bbr_tlp_max_resend, 2, 1679 "How many times does TLP retry a single segment or multiple with no ACK"); 1680 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1681 SYSCTL_CHILDREN(bbr_timeout), 1682 OID_AUTO, "minto", CTLFLAG_RW, 1683 &bbr_min_to, 1000, 1684 "Minimum rack timeout in useconds"); 1685 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1686 SYSCTL_CHILDREN(bbr_timeout), 1687 OID_AUTO, "pktdelay", CTLFLAG_RW, 1688 &bbr_pkt_delay, 1000, 1689 "Extra RACK time (in useconds) besides reordering thresh"); 1690 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1691 SYSCTL_CHILDREN(bbr_timeout), 1692 OID_AUTO, "incr_tmrs", CTLFLAG_RW, 1693 &bbr_incr_timers, 1, 1694 "Increase the RXT/TLP timer by the pacing time used?"); 1695 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1696 SYSCTL_CHILDREN(bbr_timeout), 1697 OID_AUTO, "rxtmark_sackpassed", CTLFLAG_RW, 1698 &bbr_marks_rxt_sack_passed, 0, 1699 "Mark sack passed on all those not ack'd when a RXT hits?"); 1700 /* Policer controls */ 1701 bbr_policer = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 1702 SYSCTL_CHILDREN(bbr_sysctl_root), 1703 OID_AUTO, 1704 "policer", 1705 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1706 "Policer controls"); 1707 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1708 SYSCTL_CHILDREN(bbr_policer), 1709 OID_AUTO, "detect_enable", CTLFLAG_RW, 1710 &bbr_policer_detection_enabled, 1, 1711 "Is policer detection enabled??"); 1712 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1713 SYSCTL_CHILDREN(bbr_policer), 1714 OID_AUTO, "min_pes", CTLFLAG_RW, 1715 &bbr_lt_intvl_min_rtts, 4, 1716 "Minimum number of PE's?"); 1717 SYSCTL_ADD_U64(&bbr_sysctl_ctx, 1718 SYSCTL_CHILDREN(bbr_policer), 1719 OID_AUTO, "bwdiff", CTLFLAG_RW, 1720 &bbr_lt_bw_diff, (4000/8), 1721 "Minimal bw diff?"); 1722 SYSCTL_ADD_U64(&bbr_sysctl_ctx, 1723 SYSCTL_CHILDREN(bbr_policer), 1724 OID_AUTO, "bwratio", CTLFLAG_RW, 1725 &bbr_lt_bw_ratio, 8, 1726 "Minimal bw diff?"); 1727 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1728 SYSCTL_CHILDREN(bbr_policer), 1729 OID_AUTO, "from_rack_rxt", CTLFLAG_RW, 1730 &bbr_policer_call_from_rack_to, 0, 1731 "Do we call the policer detection code from a rack-timeout?"); 1732 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1733 SYSCTL_CHILDREN(bbr_policer), 1734 OID_AUTO, "false_postive", CTLFLAG_RW, 1735 &bbr_lt_intvl_fp, 0, 1736 "What packet epoch do we do false-postive detection at (0=no)?"); 1737 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1738 SYSCTL_CHILDREN(bbr_policer), 1739 OID_AUTO, "loss_thresh", CTLFLAG_RW, 1740 &bbr_lt_loss_thresh, 196, 1741 "Loss threshold 196 = 19.6%?"); 1742 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1743 SYSCTL_CHILDREN(bbr_policer), 1744 OID_AUTO, "false_postive_thresh", CTLFLAG_RW, 1745 &bbr_lt_fd_thresh, 100, 1746 "What percentage is the false detection threshold (150=15.0)?"); 1747 /* All the rest */ 1748 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1749 SYSCTL_CHILDREN(bbr_sysctl_root), 1750 OID_AUTO, "cheat_rxt", CTLFLAG_RW, 1751 &bbr_use_rack_resend_cheat, 0, 1752 "Do we burst 1ms between sends on retransmissions (like rack)?"); 1753 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1754 SYSCTL_CHILDREN(bbr_sysctl_root), 1755 OID_AUTO, "error_paceout", CTLFLAG_RW, 1756 &bbr_error_base_paceout, 10000, 1757 "When we hit an error what is the min to pace out in usec's?"); 1758 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1759 SYSCTL_CHILDREN(bbr_sysctl_root), 1760 OID_AUTO, "kill_paceout", CTLFLAG_RW, 1761 &bbr_max_net_error_cnt, 10, 1762 "When we hit this many errors in a row, kill the session?"); 1763 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1764 SYSCTL_CHILDREN(bbr_sysctl_root), 1765 OID_AUTO, "data_after_close", CTLFLAG_RW, 1766 &bbr_ignore_data_after_close, 1, 1767 "Do we hold off sending a RST until all pending data is ack'd"); 1768 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1769 SYSCTL_CHILDREN(bbr_sysctl_root), 1770 OID_AUTO, "resend_use_tso", CTLFLAG_RW, 1771 &bbr_resends_use_tso, 0, 1772 "Can resends use TSO?"); 1773 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1774 SYSCTL_CHILDREN(bbr_sysctl_root), 1775 OID_AUTO, "sblklimit", CTLFLAG_RW, 1776 &bbr_sack_block_limit, 128, 1777 "When do we start ignoring small sack blocks"); 1778 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1779 SYSCTL_CHILDREN(bbr_sysctl_root), 1780 OID_AUTO, "bb_verbose", CTLFLAG_RW, 1781 &bbr_verbose_logging, 0, 1782 "Should BBR black box logging be verbose"); 1783 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1784 SYSCTL_CHILDREN(bbr_sysctl_root), 1785 OID_AUTO, "reorder_thresh", CTLFLAG_RW, 1786 &bbr_reorder_thresh, 2, 1787 "What factor for rack will be added when seeing reordering (shift right)"); 1788 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1789 SYSCTL_CHILDREN(bbr_sysctl_root), 1790 OID_AUTO, "reorder_fade", CTLFLAG_RW, 1791 &bbr_reorder_fade, 0, 1792 "Does reorder detection fade, if so how many ms (0 means never)"); 1793 SYSCTL_ADD_S32(&bbr_sysctl_ctx, 1794 SYSCTL_CHILDREN(bbr_sysctl_root), 1795 OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW, 1796 &bbr_tlp_thresh, 1, 1797 "what divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)"); 1798 /* Stats and counters */ 1799 /* The pacing counters for hdwr/software can't be in the array */ 1800 bbr_nohdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK); 1801 bbr_hdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK); 1802 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx, 1803 SYSCTL_CHILDREN(bbr_sysctl_root), 1804 OID_AUTO, "enob_hdwr_pacing", CTLFLAG_RD, 1805 &bbr_hdwr_pacing_enobuf, 1806 "Total number of enobufs for hardware paced flows"); 1807 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx, 1808 SYSCTL_CHILDREN(bbr_sysctl_root), 1809 OID_AUTO, "enob_no_hdwr_pacing", CTLFLAG_RD, 1810 &bbr_nohdwr_pacing_enobuf, 1811 "Total number of enobufs for non-hardware paced flows"); 1812 1813 1814 bbr_flows_whdwr_pacing = counter_u64_alloc(M_WAITOK); 1815 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx, 1816 SYSCTL_CHILDREN(bbr_sysctl_root), 1817 OID_AUTO, "hdwr_pacing", CTLFLAG_RD, 1818 &bbr_flows_whdwr_pacing, 1819 "Total number of hardware paced flows"); 1820 bbr_flows_nohdwr_pacing = counter_u64_alloc(M_WAITOK); 1821 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx, 1822 SYSCTL_CHILDREN(bbr_sysctl_root), 1823 OID_AUTO, "software_pacing", CTLFLAG_RD, 1824 &bbr_flows_nohdwr_pacing, 1825 "Total number of software paced flows"); 1826 COUNTER_ARRAY_ALLOC(bbr_stat_arry, BBR_STAT_SIZE, M_WAITOK); 1827 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root), 1828 OID_AUTO, "stats", CTLFLAG_RD, 1829 bbr_stat_arry, BBR_STAT_SIZE, "BBR Stats"); 1830 COUNTER_ARRAY_ALLOC(bbr_opts_arry, BBR_OPTS_SIZE, M_WAITOK); 1831 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root), 1832 OID_AUTO, "opts", CTLFLAG_RD, 1833 bbr_opts_arry, BBR_OPTS_SIZE, "BBR Option Stats"); 1834 COUNTER_ARRAY_ALLOC(bbr_state_lost, BBR_MAX_STAT, M_WAITOK); 1835 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root), 1836 OID_AUTO, "lost", CTLFLAG_RD, 1837 bbr_state_lost, BBR_MAX_STAT, "Stats of when losses occur"); 1838 COUNTER_ARRAY_ALLOC(bbr_state_resend, BBR_MAX_STAT, M_WAITOK); 1839 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root), 1840 OID_AUTO, "stateresend", CTLFLAG_RD, 1841 bbr_state_resend, BBR_MAX_STAT, "Stats of what states resend"); 1842 COUNTER_ARRAY_ALLOC(bbr_state_time, BBR_MAX_STAT, M_WAITOK); 1843 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root), 1844 OID_AUTO, "statetime", CTLFLAG_RD, 1845 bbr_state_time, BBR_MAX_STAT, "Stats of time spent in the states"); 1846 COUNTER_ARRAY_ALLOC(bbr_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK); 1847 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root), 1848 OID_AUTO, "outsize", CTLFLAG_RD, 1849 bbr_out_size, TCP_MSS_ACCT_SIZE, "Size of output calls"); 1850 SYSCTL_ADD_PROC(&bbr_sysctl_ctx, 1851 SYSCTL_CHILDREN(bbr_sysctl_root), 1852 OID_AUTO, "clrlost", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 1853 &bbr_clear_lost, 0, sysctl_bbr_clear_lost, "IU", "Clear lost counters"); 1854 } 1855 1856 static void 1857 bbr_counter_destroy(void) 1858 { 1859 COUNTER_ARRAY_FREE(bbr_stat_arry, BBR_STAT_SIZE); 1860 COUNTER_ARRAY_FREE(bbr_opts_arry, BBR_OPTS_SIZE); 1861 COUNTER_ARRAY_FREE(bbr_out_size, TCP_MSS_ACCT_SIZE); 1862 COUNTER_ARRAY_FREE(bbr_state_lost, BBR_MAX_STAT); 1863 COUNTER_ARRAY_FREE(bbr_state_time, BBR_MAX_STAT); 1864 COUNTER_ARRAY_FREE(bbr_state_resend, BBR_MAX_STAT); 1865 counter_u64_free(bbr_nohdwr_pacing_enobuf); 1866 counter_u64_free(bbr_hdwr_pacing_enobuf); 1867 counter_u64_free(bbr_flows_whdwr_pacing); 1868 counter_u64_free(bbr_flows_nohdwr_pacing); 1869 1870 } 1871 1872 static __inline void 1873 bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts) 1874 { 1875 memset(l, 0, sizeof(union tcp_log_stackspecific)); 1876 l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate; 1877 l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate); 1878 l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop); 1879 l->bw_inuse = bbr_get_bw(bbr); 1880 l->inflight = ctf_flight_size(bbr->rc_tp, 1881 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 1882 l->applimited = bbr->r_ctl.r_app_limited_until; 1883 l->delivered = bbr->r_ctl.rc_delivered; 1884 l->timeStamp = cts; 1885 l->lost = bbr->r_ctl.rc_lost; 1886 l->bbr_state = bbr->rc_bbr_state; 1887 l->bbr_substate = bbr_state_val(bbr); 1888 l->epoch = bbr->r_ctl.rc_rtt_epoch; 1889 l->lt_epoch = bbr->r_ctl.rc_lt_epoch; 1890 l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain; 1891 l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain; 1892 l->inhpts = bbr->rc_inp->inp_in_hpts; 1893 l->ininput = bbr->rc_inp->inp_in_input; 1894 l->use_lt_bw = bbr->rc_lt_use_bw; 1895 l->pkts_out = bbr->r_ctl.rc_flight_at_input; 1896 l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch; 1897 } 1898 1899 static void 1900 bbr_log_type_bw_reduce(struct tcp_bbr *bbr, int reason) 1901 { 1902 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 1903 union tcp_log_stackspecific log; 1904 1905 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 1906 log.u_bbr.flex1 = 0; 1907 log.u_bbr.flex2 = 0; 1908 log.u_bbr.flex5 = 0; 1909 log.u_bbr.flex3 = 0; 1910 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate; 1911 log.u_bbr.flex7 = reason; 1912 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt; 1913 log.u_bbr.flex8 = 0; 1914 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 1915 &bbr->rc_inp->inp_socket->so_rcv, 1916 &bbr->rc_inp->inp_socket->so_snd, 1917 BBR_LOG_BW_RED_EV, 0, 1918 0, &log, false, &bbr->rc_tv); 1919 } 1920 } 1921 1922 static void 1923 bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count) 1924 { 1925 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 1926 union tcp_log_stackspecific log; 1927 1928 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 1929 log.u_bbr.flex1 = seq; 1930 log.u_bbr.flex2 = count; 1931 log.u_bbr.flex8 = mode; 1932 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 1933 &bbr->rc_inp->inp_socket->so_rcv, 1934 &bbr->rc_inp->inp_socket->so_snd, 1935 BBR_LOG_LOWGAIN, 0, 1936 0, &log, false, &bbr->rc_tv); 1937 } 1938 } 1939 1940 1941 1942 static void 1943 bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling, 1944 uint8_t reason, uint32_t p_maxseg, int len) 1945 { 1946 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 1947 union tcp_log_stackspecific log; 1948 1949 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 1950 log.u_bbr.flex1 = p_maxseg; 1951 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags; 1952 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp; 1953 log.u_bbr.flex4 = reason; 1954 log.u_bbr.flex5 = bbr->rc_in_persist; 1955 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val; 1956 log.u_bbr.flex7 = p_maxseg; 1957 log.u_bbr.flex8 = bbr->rc_in_persist; 1958 log.u_bbr.pkts_out = 0; 1959 log.u_bbr.applimited = len; 1960 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 1961 &bbr->rc_inp->inp_socket->so_rcv, 1962 &bbr->rc_inp->inp_socket->so_snd, 1963 BBR_LOG_JUSTRET, 0, 1964 tlen, &log, false, &bbr->rc_tv); 1965 } 1966 } 1967 1968 1969 static void 1970 bbr_log_type_enter_rec(struct tcp_bbr *bbr, uint32_t seq) 1971 { 1972 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 1973 union tcp_log_stackspecific log; 1974 1975 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 1976 log.u_bbr.flex1 = seq; 1977 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent; 1978 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start; 1979 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 1980 &bbr->rc_inp->inp_socket->so_rcv, 1981 &bbr->rc_inp->inp_socket->so_snd, 1982 BBR_LOG_ENTREC, 0, 1983 0, &log, false, &bbr->rc_tv); 1984 } 1985 } 1986 1987 static void 1988 bbr_log_msgsize_fail(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts) 1989 { 1990 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 1991 union tcp_log_stackspecific log; 1992 1993 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 1994 log.u_bbr.flex1 = tso; 1995 log.u_bbr.flex2 = maxseg; 1996 log.u_bbr.flex3 = mtu; 1997 log.u_bbr.flex4 = csum_flags; 1998 TCP_LOG_EVENTP(tp, NULL, 1999 &bbr->rc_inp->inp_socket->so_rcv, 2000 &bbr->rc_inp->inp_socket->so_snd, 2001 BBR_LOG_MSGSIZE, 0, 2002 0, &log, false, &bbr->rc_tv); 2003 } 2004 } 2005 2006 static void 2007 bbr_log_flowend(struct tcp_bbr *bbr) 2008 { 2009 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2010 union tcp_log_stackspecific log; 2011 struct sockbuf *r, *s; 2012 struct timeval tv; 2013 2014 if (bbr->rc_inp->inp_socket) { 2015 r = &bbr->rc_inp->inp_socket->so_rcv; 2016 s = &bbr->rc_inp->inp_socket->so_snd; 2017 } else { 2018 r = s = NULL; 2019 } 2020 bbr_fill_in_logging_data(bbr, &log.u_bbr, tcp_get_usecs(&tv)); 2021 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2022 r, s, 2023 TCP_LOG_FLOWEND, 0, 2024 0, &log, false, &tv); 2025 } 2026 } 2027 2028 static void 2029 bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, 2030 uint32_t lost, uint32_t del) 2031 { 2032 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2033 union tcp_log_stackspecific log; 2034 2035 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2036 log.u_bbr.flex1 = lost; 2037 log.u_bbr.flex2 = del; 2038 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw; 2039 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt; 2040 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch; 2041 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; 2042 log.u_bbr.flex7 = line; 2043 log.u_bbr.flex8 = 0; 2044 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count; 2045 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2046 &bbr->rc_inp->inp_socket->so_rcv, 2047 &bbr->rc_inp->inp_socket->so_snd, 2048 BBR_LOG_PKT_EPOCH, 0, 2049 0, &log, false, &bbr->rc_tv); 2050 } 2051 } 2052 2053 static void 2054 bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time) 2055 { 2056 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2057 union tcp_log_stackspecific log; 2058 2059 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2060 log.u_bbr.flex1 = bbr->r_ctl.rc_lost; 2061 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat; 2062 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat; 2063 log.u_bbr.flex7 = line; 2064 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2065 &bbr->rc_inp->inp_socket->so_rcv, 2066 &bbr->rc_inp->inp_socket->so_snd, 2067 BBR_LOG_TIME_EPOCH, 0, 2068 0, &log, false, &bbr->rc_tv); 2069 } 2070 } 2071 2072 static void 2073 bbr_log_set_of_state_target(struct tcp_bbr *bbr, uint32_t new_tar, int line, int meth) 2074 { 2075 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2076 union tcp_log_stackspecific log; 2077 2078 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2079 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state; 2080 log.u_bbr.flex2 = new_tar; 2081 log.u_bbr.flex3 = line; 2082 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs; 2083 log.u_bbr.flex5 = bbr_quanta; 2084 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs; 2085 log.u_bbr.flex7 = bbr->rc_last_options; 2086 log.u_bbr.flex8 = meth; 2087 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2088 &bbr->rc_inp->inp_socket->so_rcv, 2089 &bbr->rc_inp->inp_socket->so_snd, 2090 BBR_LOG_STATE_TARGET, 0, 2091 0, &log, false, &bbr->rc_tv); 2092 } 2093 2094 } 2095 2096 static void 2097 bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line) 2098 { 2099 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2100 union tcp_log_stackspecific log; 2101 2102 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2103 log.u_bbr.flex1 = line; 2104 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; 2105 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int; 2106 if (bbr_state_is_pkt_epoch) 2107 log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PKTRTT); 2108 else 2109 log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PROP); 2110 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch; 2111 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; 2112 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000); 2113 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra; 2114 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state; 2115 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2116 &bbr->rc_inp->inp_socket->so_rcv, 2117 &bbr->rc_inp->inp_socket->so_snd, 2118 BBR_LOG_STATE, 0, 2119 0, &log, false, &bbr->rc_tv); 2120 } 2121 } 2122 2123 static void 2124 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, 2125 uint32_t rtt, uint32_t line, uint8_t reas, uint16_t cond) 2126 { 2127 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2128 union tcp_log_stackspecific log; 2129 2130 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2131 log.u_bbr.flex1 = line; 2132 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; 2133 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt; 2134 log.u_bbr.flex4 = applied; 2135 log.u_bbr.flex5 = rtt; 2136 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state; 2137 log.u_bbr.flex7 = cond; 2138 log.u_bbr.flex8 = reas; 2139 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2140 &bbr->rc_inp->inp_socket->so_rcv, 2141 &bbr->rc_inp->inp_socket->so_snd, 2142 BBR_LOG_RTT_SHRINKS, 0, 2143 0, &log, false, &bbr->rc_tv); 2144 } 2145 } 2146 2147 static void 2148 bbr_log_type_exit_rec(struct tcp_bbr *bbr) 2149 { 2150 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2151 union tcp_log_stackspecific log; 2152 2153 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2154 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start; 2155 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent; 2156 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; 2157 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2158 &bbr->rc_inp->inp_socket->so_rcv, 2159 &bbr->rc_inp->inp_socket->so_snd, 2160 BBR_LOG_EXITREC, 0, 2161 0, &log, false, &bbr->rc_tv); 2162 } 2163 } 2164 2165 static void 2166 bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg, 2167 uint32_t prev_acked, int32_t meth, uint32_t target, uint32_t th_ack, int32_t line) 2168 { 2169 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2170 union tcp_log_stackspecific log; 2171 2172 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2173 log.u_bbr.flex1 = line; 2174 log.u_bbr.flex2 = prev_acked; 2175 log.u_bbr.flex3 = bytes_this_ack; 2176 log.u_bbr.flex4 = chg; 2177 log.u_bbr.flex5 = th_ack; 2178 log.u_bbr.flex6 = target; 2179 log.u_bbr.flex8 = meth; 2180 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2181 &bbr->rc_inp->inp_socket->so_rcv, 2182 &bbr->rc_inp->inp_socket->so_snd, 2183 BBR_LOG_CWND, 0, 2184 0, &log, false, &bbr->rc_tv); 2185 } 2186 } 2187 2188 static void 2189 bbr_log_rtt_sample(struct tcp_bbr *bbr, uint32_t rtt, uint32_t tsin) 2190 { 2191 /* 2192 * Log the rtt sample we are applying to the srtt algorithm in 2193 * useconds. 2194 */ 2195 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2196 union tcp_log_stackspecific log; 2197 2198 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2199 log.u_bbr.flex1 = rtt; 2200 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time; 2201 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay; 2202 log.u_bbr.flex4 = bbr->rc_tp->ts_offset; 2203 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; 2204 log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv); 2205 log.u_bbr.flex6 = tsin; 2206 log.u_bbr.flex7 = 0; 2207 log.u_bbr.flex8 = bbr->rc_ack_was_delayed; 2208 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2209 &bbr->rc_inp->inp_socket->so_rcv, 2210 &bbr->rc_inp->inp_socket->so_snd, 2211 TCP_LOG_RTT, 0, 2212 0, &log, false, &bbr->rc_tv); 2213 } 2214 } 2215 2216 static void 2217 bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t enter_exit) 2218 { 2219 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2220 union tcp_log_stackspecific log; 2221 2222 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2223 log.u_bbr.flex1 = time_in; 2224 log.u_bbr.flex2 = line; 2225 log.u_bbr.flex8 = enter_exit; 2226 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2227 &bbr->rc_inp->inp_socket->so_rcv, 2228 &bbr->rc_inp->inp_socket->so_snd, 2229 BBR_LOG_PERSIST, 0, 2230 0, &log, false, &bbr->rc_tv); 2231 } 2232 } 2233 static void 2234 bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts) 2235 { 2236 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2237 union tcp_log_stackspecific log; 2238 2239 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2240 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age; 2241 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks; 2242 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int; 2243 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time; 2244 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; 2245 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2246 &bbr->rc_inp->inp_socket->so_rcv, 2247 &bbr->rc_inp->inp_socket->so_snd, 2248 BBR_LOG_ACKCLEAR, 0, 2249 0, &log, false, &bbr->rc_tv); 2250 } 2251 } 2252 2253 static void 2254 bbr_log_ack_event(struct tcp_bbr *bbr, struct tcphdr *th, struct tcpopt *to, uint32_t tlen, 2255 uint16_t nsegs, uint32_t cts, int32_t nxt_pkt, struct mbuf *m) 2256 { 2257 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2258 union tcp_log_stackspecific log; 2259 struct timeval tv; 2260 2261 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2262 log.u_bbr.flex1 = nsegs; 2263 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes; 2264 if (m) { 2265 struct timespec ts; 2266 2267 log.u_bbr.flex3 = m->m_flags; 2268 if (m->m_flags & M_TSTMP) { 2269 mbuf_tstmp2timespec(m, &ts); 2270 tv.tv_sec = ts.tv_sec; 2271 tv.tv_usec = ts.tv_nsec / 1000; 2272 log.u_bbr.lt_epoch = tcp_tv_to_usectick(&tv); 2273 } else { 2274 log.u_bbr.lt_epoch = 0; 2275 } 2276 if (m->m_flags & M_TSTMP_LRO) { 2277 tv.tv_sec = m->m_pkthdr.rcv_tstmp / 1000000000; 2278 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000) / 1000; 2279 log.u_bbr.flex5 = tcp_tv_to_usectick(&tv); 2280 } else { 2281 /* No arrival timestamp */ 2282 log.u_bbr.flex5 = 0; 2283 } 2284 2285 log.u_bbr.pkts_out = tcp_get_usecs(&tv); 2286 } else { 2287 log.u_bbr.flex3 = 0; 2288 log.u_bbr.flex5 = 0; 2289 log.u_bbr.flex6 = 0; 2290 log.u_bbr.pkts_out = 0; 2291 } 2292 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state; 2293 log.u_bbr.flex7 = bbr->r_wanted_output; 2294 log.u_bbr.flex8 = bbr->rc_in_persist; 2295 TCP_LOG_EVENTP(bbr->rc_tp, th, 2296 &bbr->rc_inp->inp_socket->so_rcv, 2297 &bbr->rc_inp->inp_socket->so_snd, 2298 TCP_LOG_IN, 0, 2299 tlen, &log, true, &bbr->rc_tv); 2300 } 2301 } 2302 2303 static void 2304 bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out) 2305 { 2306 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2307 union tcp_log_stackspecific log; 2308 2309 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2310 log.u_bbr.flex1 = did_out; 2311 log.u_bbr.flex2 = nxt_pkt; 2312 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val; 2313 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags; 2314 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp; 2315 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes; 2316 log.u_bbr.flex7 = bbr->r_wanted_output; 2317 log.u_bbr.flex8 = bbr->rc_in_persist; 2318 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay; 2319 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2320 &bbr->rc_inp->inp_socket->so_rcv, 2321 &bbr->rc_inp->inp_socket->so_snd, 2322 BBR_LOG_DOSEG_DONE, 0, 2323 0, &log, true, &bbr->rc_tv); 2324 } 2325 } 2326 2327 static void 2328 bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts, 2329 int32_t line, uint32_t o_len, uint32_t segcnt, uint32_t segsiz) 2330 { 2331 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2332 union tcp_log_stackspecific log; 2333 2334 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2335 log.u_bbr.flex1 = line; 2336 log.u_bbr.flex2 = o_len; 2337 log.u_bbr.flex3 = segcnt; 2338 log.u_bbr.flex4 = segsiz; 2339 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2340 &bbr->rc_inp->inp_socket->so_rcv, 2341 &bbr->rc_inp->inp_socket->so_snd, 2342 BBR_LOG_ENOBUF_JMP, ENOBUFS, 2343 len, &log, true, &bbr->rc_tv); 2344 } 2345 } 2346 2347 static void 2348 bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_calling) 2349 { 2350 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2351 union tcp_log_stackspecific log; 2352 2353 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2354 log.u_bbr.flex1 = timers; 2355 log.u_bbr.flex2 = ret; 2356 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp; 2357 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags; 2358 log.u_bbr.flex5 = cts; 2359 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state; 2360 log.u_bbr.flex8 = hpts_calling; 2361 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2362 &bbr->rc_inp->inp_socket->so_rcv, 2363 &bbr->rc_inp->inp_socket->so_snd, 2364 BBR_LOG_TO_PROCESS, 0, 2365 0, &log, false, &bbr->rc_tv); 2366 } 2367 } 2368 2369 static void 2370 bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num) 2371 { 2372 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2373 union tcp_log_stackspecific log; 2374 uint64_t ar; 2375 2376 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2377 log.u_bbr.flex1 = bbr->bbr_timer_src; 2378 log.u_bbr.flex2 = 0; 2379 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; 2380 ar = (uint64_t)(bbr->r_ctl.rc_resend); 2381 ar >>= 32; 2382 ar &= 0x00000000ffffffff; 2383 log.u_bbr.flex4 = (uint32_t)ar; 2384 ar = (uint64_t)bbr->r_ctl.rc_resend; 2385 ar &= 0x00000000ffffffff; 2386 log.u_bbr.flex5 = (uint32_t)ar; 2387 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); 2388 log.u_bbr.flex8 = to_num; 2389 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2390 &bbr->rc_inp->inp_socket->so_rcv, 2391 &bbr->rc_inp->inp_socket->so_snd, 2392 BBR_LOG_RTO, 0, 2393 0, &log, false, &bbr->rc_tv); 2394 } 2395 } 2396 2397 static void 2398 bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t flex3, uint8_t reason) 2399 { 2400 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2401 union tcp_log_stackspecific log; 2402 2403 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2404 log.u_bbr.flex1 = flex1; 2405 log.u_bbr.flex2 = flex2; 2406 log.u_bbr.flex3 = flex3; 2407 log.u_bbr.flex4 = 0; 2408 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; 2409 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup; 2410 log.u_bbr.flex8 = reason; 2411 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw; 2412 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2413 &bbr->rc_inp->inp_socket->so_rcv, 2414 &bbr->rc_inp->inp_socket->so_snd, 2415 BBR_LOG_REDUCE, 0, 2416 0, &log, false, &bbr->rc_tv); 2417 } 2418 } 2419 2420 static void 2421 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag) 2422 { 2423 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2424 union tcp_log_stackspecific log; 2425 2426 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2427 log.u_bbr.flex1 = diag->p_nxt_slot; 2428 log.u_bbr.flex2 = diag->p_cur_slot; 2429 log.u_bbr.flex3 = diag->slot_req; 2430 log.u_bbr.flex4 = diag->inp_hptsslot; 2431 log.u_bbr.flex5 = diag->slot_remaining; 2432 log.u_bbr.flex6 = diag->need_new_to; 2433 log.u_bbr.flex7 = diag->p_hpts_active; 2434 log.u_bbr.flex8 = diag->p_on_min_sleep; 2435 /* Hijack other fields as needed */ 2436 log.u_bbr.epoch = diag->have_slept; 2437 log.u_bbr.lt_epoch = diag->yet_to_sleep; 2438 log.u_bbr.pkts_out = diag->co_ret; 2439 log.u_bbr.applimited = diag->hpts_sleep_time; 2440 log.u_bbr.delivered = diag->p_prev_slot; 2441 log.u_bbr.inflight = diag->p_runningtick; 2442 log.u_bbr.bw_inuse = diag->wheel_tick; 2443 log.u_bbr.rttProp = diag->wheel_cts; 2444 log.u_bbr.delRate = diag->maxticks; 2445 log.u_bbr.cur_del_rate = diag->p_curtick; 2446 log.u_bbr.cur_del_rate <<= 32; 2447 log.u_bbr.cur_del_rate |= diag->p_lasttick; 2448 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2449 &bbr->rc_inp->inp_socket->so_rcv, 2450 &bbr->rc_inp->inp_socket->so_snd, 2451 BBR_LOG_HPTSDIAG, 0, 2452 0, &log, false, &bbr->rc_tv); 2453 } 2454 } 2455 2456 static void 2457 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt, 2458 uint32_t thresh, uint32_t to) 2459 { 2460 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2461 union tcp_log_stackspecific log; 2462 2463 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2464 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar; 2465 log.u_bbr.flex2 = time_since_sent; 2466 log.u_bbr.flex3 = srtt; 2467 log.u_bbr.flex4 = thresh; 2468 log.u_bbr.flex5 = to; 2469 log.u_bbr.flex6 = bbr->rc_tp->t_srtt; 2470 log.u_bbr.flex8 = mode; 2471 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2472 &bbr->rc_inp->inp_socket->so_rcv, 2473 &bbr->rc_inp->inp_socket->so_snd, 2474 BBR_LOG_TIMERPREP, 0, 2475 0, &log, false, &bbr->rc_tv); 2476 } 2477 } 2478 2479 static void 2480 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len, 2481 uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod) 2482 { 2483 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2484 union tcp_log_stackspecific log; 2485 2486 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2487 log.u_bbr.flex1 = usecs; 2488 log.u_bbr.flex2 = len; 2489 log.u_bbr.flex3 = (uint32_t)((bw >> 32) & 0x00000000ffffffff); 2490 log.u_bbr.flex4 = (uint32_t)(bw & 0x00000000ffffffff); 2491 if (override) 2492 log.u_bbr.flex5 = (1 << 2); 2493 else 2494 log.u_bbr.flex5 = 0; 2495 log.u_bbr.flex6 = override; 2496 log.u_bbr.flex7 = gain; 2497 log.u_bbr.flex8 = mod; 2498 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2499 &bbr->rc_inp->inp_socket->so_rcv, 2500 &bbr->rc_inp->inp_socket->so_snd, 2501 BBR_LOG_HPTSI_CALC, 0, 2502 len, &log, false, &bbr->rc_tv); 2503 } 2504 } 2505 2506 static void 2507 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which) 2508 { 2509 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2510 union tcp_log_stackspecific log; 2511 2512 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2513 2514 log.u_bbr.flex1 = bbr->bbr_timer_src; 2515 log.u_bbr.flex2 = to; 2516 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; 2517 log.u_bbr.flex4 = slot; 2518 log.u_bbr.flex5 = bbr->rc_inp->inp_hptsslot; 2519 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); 2520 log.u_bbr.pkts_out = bbr->rc_inp->inp_flags2; 2521 log.u_bbr.flex8 = which; 2522 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2523 &bbr->rc_inp->inp_socket->so_rcv, 2524 &bbr->rc_inp->inp_socket->so_snd, 2525 BBR_LOG_TIMERSTAR, 0, 2526 0, &log, false, &bbr->rc_tv); 2527 } 2528 } 2529 2530 static void 2531 bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm) 2532 { 2533 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2534 union tcp_log_stackspecific log; 2535 2536 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2537 log.u_bbr.flex1 = thresh; 2538 log.u_bbr.flex2 = lro; 2539 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts; 2540 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]; 2541 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); 2542 log.u_bbr.flex6 = srtt; 2543 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift; 2544 log.u_bbr.flex8 = frm; 2545 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2546 &bbr->rc_inp->inp_socket->so_rcv, 2547 &bbr->rc_inp->inp_socket->so_snd, 2548 BBR_LOG_THRESH_CALC, 0, 2549 0, &log, false, &bbr->rc_tv); 2550 } 2551 } 2552 2553 static void 2554 bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed) 2555 { 2556 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2557 union tcp_log_stackspecific log; 2558 2559 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2560 log.u_bbr.flex1 = line; 2561 log.u_bbr.flex2 = bbr->bbr_timer_src; 2562 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags; 2563 log.u_bbr.flex4 = bbr->rc_in_persist; 2564 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state; 2565 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); 2566 log.u_bbr.flex8 = hpts_removed; 2567 log.u_bbr.pkts_out = bbr->rc_pacer_started; 2568 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2569 &bbr->rc_inp->inp_socket->so_rcv, 2570 &bbr->rc_inp->inp_socket->so_snd, 2571 BBR_LOG_TIMERCANC, 0, 2572 0, &log, false, &bbr->rc_tv); 2573 } 2574 } 2575 2576 2577 static void 2578 bbr_log_tstmp_validation(struct tcp_bbr *bbr, uint64_t peer_delta, uint64_t delta) 2579 { 2580 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2581 union tcp_log_stackspecific log; 2582 2583 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2584 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio; 2585 log.u_bbr.flex2 = (peer_delta >> 32); 2586 log.u_bbr.flex3 = (peer_delta & 0x00000000ffffffff); 2587 log.u_bbr.flex4 = (delta >> 32); 2588 log.u_bbr.flex5 = (delta & 0x00000000ffffffff); 2589 log.u_bbr.flex7 = bbr->rc_ts_clock_set; 2590 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used; 2591 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2592 &bbr->rc_inp->inp_socket->so_rcv, 2593 &bbr->rc_inp->inp_socket->so_snd, 2594 BBR_LOG_TSTMP_VAL, 0, 2595 0, &log, false, &bbr->rc_tv); 2596 2597 } 2598 } 2599 2600 static void 2601 bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_val, uint32_t maxseg, int hdwr) 2602 { 2603 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2604 union tcp_log_stackspecific log; 2605 2606 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2607 log.u_bbr.flex1 = tsosz; 2608 log.u_bbr.flex2 = tls; 2609 log.u_bbr.flex3 = tcp_min_hptsi_time; 2610 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min; 2611 log.u_bbr.flex5 = old_val; 2612 log.u_bbr.flex6 = maxseg; 2613 log.u_bbr.flex7 = bbr->rc_no_pacing; 2614 log.u_bbr.flex7 <<= 1; 2615 log.u_bbr.flex7 |= bbr->rc_past_init_win; 2616 if (hdwr) 2617 log.u_bbr.flex8 = 0x80 | bbr->rc_use_google; 2618 else 2619 log.u_bbr.flex8 = bbr->rc_use_google; 2620 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2621 &bbr->rc_inp->inp_socket->so_rcv, 2622 &bbr->rc_inp->inp_socket->so_snd, 2623 BBR_LOG_BBRTSO, 0, 2624 0, &log, false, &bbr->rc_tv); 2625 } 2626 } 2627 2628 static void 2629 bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, 2630 uint32_t flags, uint32_t line) 2631 { 2632 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2633 union tcp_log_stackspecific log; 2634 2635 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2636 log.u_bbr.flex1 = line; 2637 log.u_bbr.flex2 = rsm->r_start; 2638 log.u_bbr.flex3 = rsm->r_end; 2639 log.u_bbr.flex4 = rsm->r_delivered; 2640 log.u_bbr.flex5 = rsm->r_rtr_cnt; 2641 log.u_bbr.flex6 = rsm->r_dupack; 2642 log.u_bbr.flex7 = rsm->r_tim_lastsent[0]; 2643 log.u_bbr.flex8 = rsm->r_flags; 2644 /* Hijack the pkts_out fids */ 2645 log.u_bbr.applimited = flags; 2646 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2647 &bbr->rc_inp->inp_socket->so_rcv, 2648 &bbr->rc_inp->inp_socket->so_snd, 2649 BBR_RSM_CLEARED, 0, 2650 0, &log, false, &bbr->rc_tv); 2651 } 2652 } 2653 2654 static void 2655 bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts, 2656 uint32_t flex3, uint32_t flex2, uint32_t flex5, 2657 uint32_t flex6, uint32_t pkts_out, int flex7, 2658 uint32_t flex4, uint32_t flex1) 2659 { 2660 2661 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2662 union tcp_log_stackspecific log; 2663 2664 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2665 log.u_bbr.flex1 = flex1; 2666 log.u_bbr.flex2 = flex2; 2667 log.u_bbr.flex3 = flex3; 2668 log.u_bbr.flex4 = flex4; 2669 log.u_bbr.flex5 = flex5; 2670 log.u_bbr.flex6 = flex6; 2671 log.u_bbr.flex7 = flex7; 2672 /* Hijack the pkts_out fids */ 2673 log.u_bbr.pkts_out = pkts_out; 2674 log.u_bbr.flex8 = flex8; 2675 if (bbr->rc_ack_was_delayed) 2676 log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay; 2677 else 2678 log.u_bbr.epoch = 0; 2679 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2680 &bbr->rc_inp->inp_socket->so_rcv, 2681 &bbr->rc_inp->inp_socket->so_snd, 2682 BBR_LOG_BBRUPD, 0, 2683 flex2, &log, false, &bbr->rc_tv); 2684 } 2685 } 2686 2687 2688 static void 2689 bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason, 2690 uint32_t newbw, uint32_t obw, uint32_t diff, 2691 uint32_t tim) 2692 { 2693 if (/*bbr_verbose_logging && */(bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2694 union tcp_log_stackspecific log; 2695 2696 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2697 log.u_bbr.flex1 = reason; 2698 log.u_bbr.flex2 = newbw; 2699 log.u_bbr.flex3 = obw; 2700 log.u_bbr.flex4 = diff; 2701 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost; 2702 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del; 2703 log.u_bbr.flex7 = bbr->rc_lt_is_sampling; 2704 log.u_bbr.pkts_out = tim; 2705 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw; 2706 if (bbr->rc_lt_use_bw == 0) 2707 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch; 2708 else 2709 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use; 2710 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2711 &bbr->rc_inp->inp_socket->so_rcv, 2712 &bbr->rc_inp->inp_socket->so_snd, 2713 BBR_LOG_BWSAMP, 0, 2714 0, &log, false, &bbr->rc_tv); 2715 } 2716 } 2717 2718 static inline void 2719 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line) 2720 { 2721 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2722 union tcp_log_stackspecific log; 2723 2724 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2725 log.u_bbr.flex1 = line; 2726 log.u_bbr.flex2 = tick; 2727 log.u_bbr.flex3 = tp->t_maxunacktime; 2728 log.u_bbr.flex4 = tp->t_acktime; 2729 log.u_bbr.flex8 = event; 2730 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2731 &bbr->rc_inp->inp_socket->so_rcv, 2732 &bbr->rc_inp->inp_socket->so_snd, 2733 BBR_LOG_PROGRESS, 0, 2734 0, &log, false, &bbr->rc_tv); 2735 } 2736 } 2737 2738 static void 2739 bbr_type_log_hdwr_pacing(struct tcp_bbr *bbr, const struct ifnet *ifp, 2740 uint64_t rate, uint64_t hw_rate, int line, uint32_t cts, 2741 int error) 2742 { 2743 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2744 union tcp_log_stackspecific log; 2745 2746 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2747 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff); 2748 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff); 2749 log.u_bbr.flex3 = (((uint64_t)ifp >> 32) & 0x00000000ffffffff); 2750 log.u_bbr.flex4 = ((uint64_t)ifp & 0x00000000ffffffff); 2751 log.u_bbr.bw_inuse = rate; 2752 log.u_bbr.flex5 = line; 2753 log.u_bbr.flex6 = error; 2754 log.u_bbr.flex8 = bbr->skip_gain; 2755 log.u_bbr.flex8 <<= 1; 2756 log.u_bbr.flex8 |= bbr->gain_is_limited; 2757 log.u_bbr.flex8 <<= 1; 2758 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing; 2759 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg; 2760 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2761 &bbr->rc_inp->inp_socket->so_rcv, 2762 &bbr->rc_inp->inp_socket->so_snd, 2763 BBR_LOG_HDWR_PACE, 0, 2764 0, &log, false, &bbr->rc_tv); 2765 } 2766 } 2767 2768 static void 2769 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, uint32_t del_by, uint32_t cts, uint32_t line, uint32_t prev_delay) 2770 { 2771 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2772 union tcp_log_stackspecific log; 2773 2774 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2775 log.u_bbr.flex1 = slot; 2776 log.u_bbr.flex2 = del_by; 2777 log.u_bbr.flex3 = prev_delay; 2778 log.u_bbr.flex4 = line; 2779 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val; 2780 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay; 2781 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags); 2782 log.u_bbr.flex8 = bbr->rc_in_persist; 2783 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2784 &bbr->rc_inp->inp_socket->so_rcv, 2785 &bbr->rc_inp->inp_socket->so_snd, 2786 BBR_LOG_BBRSND, 0, 2787 len, &log, false, &bbr->rc_tv); 2788 } 2789 } 2790 2791 static void 2792 bbr_log_type_bbrrttprop(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uint32_t seq, uint8_t flags) 2793 { 2794 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2795 union tcp_log_stackspecific log; 2796 2797 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2798 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered; 2799 log.u_bbr.flex2 = 0; 2800 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt; 2801 log.u_bbr.flex4 = end; 2802 log.u_bbr.flex5 = seq; 2803 log.u_bbr.flex6 = t; 2804 log.u_bbr.flex7 = match; 2805 log.u_bbr.flex8 = flags; 2806 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2807 &bbr->rc_inp->inp_socket->so_rcv, 2808 &bbr->rc_inp->inp_socket->so_snd, 2809 BBR_LOG_BBRRTT, 0, 2810 0, &log, false, &bbr->rc_tv); 2811 } 2812 } 2813 2814 static void 2815 bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method) 2816 { 2817 if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2818 union tcp_log_stackspecific log; 2819 2820 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 2821 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state; 2822 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options); 2823 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch; 2824 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs; 2825 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs; 2826 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight; 2827 log.u_bbr.flex7 = 0; 2828 log.u_bbr.flex8 = entry_method; 2829 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2830 &bbr->rc_inp->inp_socket->so_rcv, 2831 &bbr->rc_inp->inp_socket->so_snd, 2832 BBR_LOG_EXIT_GAIN, 0, 2833 0, &log, false, &bbr->rc_tv); 2834 } 2835 } 2836 2837 static void 2838 bbr_log_settings_change(struct tcp_bbr *bbr, int settings_desired) 2839 { 2840 if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2841 union tcp_log_stackspecific log; 2842 2843 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime); 2844 /* R-HU */ 2845 log.u_bbr.flex1 = 0; 2846 log.u_bbr.flex2 = 0; 2847 log.u_bbr.flex3 = 0; 2848 log.u_bbr.flex4 = 0; 2849 log.u_bbr.flex7 = 0; 2850 log.u_bbr.flex8 = settings_desired; 2851 2852 TCP_LOG_EVENTP(bbr->rc_tp, NULL, 2853 &bbr->rc_inp->inp_socket->so_rcv, 2854 &bbr->rc_inp->inp_socket->so_snd, 2855 BBR_LOG_SETTINGS_CHG, 0, 2856 0, &log, false, &bbr->rc_tv); 2857 } 2858 } 2859 2860 /* 2861 * Returns the bw from the our filter. 2862 */ 2863 static inline uint64_t 2864 bbr_get_full_bw(struct tcp_bbr *bbr) 2865 { 2866 uint64_t bw; 2867 2868 bw = get_filter_value(&bbr->r_ctl.rc_delrate); 2869 2870 return (bw); 2871 } 2872 2873 static inline void 2874 bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line) 2875 { 2876 uint64_t calclr; 2877 uint32_t lost, del; 2878 2879 if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch) 2880 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch; 2881 else 2882 lost = 0; 2883 del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del; 2884 if (lost == 0) { 2885 calclr = 0; 2886 } else if (del) { 2887 calclr = lost; 2888 calclr *= (uint64_t)1000; 2889 calclr /= (uint64_t)del; 2890 } else { 2891 /* Nothing delivered? 100.0% loss */ 2892 calclr = 1000; 2893 } 2894 bbr->r_ctl.rc_pkt_epoch_loss_rate = (uint32_t)calclr; 2895 if (IN_RECOVERY(bbr->rc_tp->t_flags)) 2896 bbr->r_ctl.recovery_lr += (uint32_t)calclr; 2897 bbr->r_ctl.rc_pkt_epoch++; 2898 if (bbr->rc_no_pacing && 2899 (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) { 2900 bbr->rc_no_pacing = 0; 2901 tcp_bbr_tso_size_check(bbr, cts); 2902 } 2903 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time); 2904 bbr->r_ctl.rc_pkt_epoch_time = cts; 2905 /* What was our loss rate */ 2906 bbr_log_pkt_epoch(bbr, cts, line, lost, del); 2907 bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered; 2908 bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost; 2909 } 2910 2911 static inline void 2912 bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line) 2913 { 2914 uint32_t epoch_time; 2915 2916 /* Tick the RTT clock */ 2917 bbr->r_ctl.rc_rtt_epoch++; 2918 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start; 2919 bbr_log_time_epoch(bbr, cts, line, epoch_time); 2920 bbr->r_ctl.rc_rcv_epoch_start = cts; 2921 } 2922 2923 2924 static inline void 2925 bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked) 2926 { 2927 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) { 2928 bbr->rc_is_pkt_epoch_now = 1; 2929 } 2930 } 2931 2932 /* 2933 * Returns the bw from either the b/w filter 2934 * or from the lt_bw (if the connection is being 2935 * policed). 2936 */ 2937 static inline uint64_t 2938 __bbr_get_bw(struct tcp_bbr *bbr) 2939 { 2940 uint64_t bw, min_bw; 2941 uint64_t rtt; 2942 int gm_measure_cnt = 1; 2943 2944 /* 2945 * For startup we make, like google, a 2946 * minimum b/w. This is generated from the 2947 * IW and the rttProp. We do fall back to srtt 2948 * if for some reason (initial handshake) we don't 2949 * have a rttProp. We, in the worst case, fall back 2950 * to the configured min_bw (rc_initial_hptsi_bw). 2951 */ 2952 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { 2953 /* Attempt first to use rttProp */ 2954 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop); 2955 if (rtt && (rtt < 0xffffffff)) { 2956 measure: 2957 min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) * 2958 ((uint64_t)1000000); 2959 min_bw /= rtt; 2960 if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) { 2961 min_bw = bbr->r_ctl.rc_initial_hptsi_bw; 2962 } 2963 2964 } else if (bbr->rc_tp->t_srtt != 0) { 2965 /* No rttProp, use srtt? */ 2966 rtt = bbr_get_rtt(bbr, BBR_SRTT); 2967 goto measure; 2968 } else { 2969 min_bw = bbr->r_ctl.rc_initial_hptsi_bw; 2970 } 2971 } else 2972 min_bw = 0; 2973 2974 if ((bbr->rc_past_init_win == 0) && 2975 (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp))) 2976 bbr->rc_past_init_win = 1; 2977 if ((bbr->rc_use_google) && (bbr->r_ctl.r_measurement_count >= 1)) 2978 gm_measure_cnt = 0; 2979 if (gm_measure_cnt && 2980 ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) || 2981 (bbr->rc_past_init_win == 0))) { 2982 /* For google we use our guess rate until we get 1 measurement */ 2983 2984 use_initial_window: 2985 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop); 2986 if (rtt && (rtt < 0xffffffff)) { 2987 /* 2988 * We have an RTT measurment. Use that in 2989 * combination with our initial window to calculate 2990 * a b/w. 2991 */ 2992 bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) * 2993 ((uint64_t)1000000); 2994 bw /= rtt; 2995 if (bw < bbr->r_ctl.rc_initial_hptsi_bw) { 2996 bw = bbr->r_ctl.rc_initial_hptsi_bw; 2997 } 2998 } else { 2999 /* Drop back to the 40 and punt to a default */ 3000 bw = bbr->r_ctl.rc_initial_hptsi_bw; 3001 } 3002 if (bw < 1) 3003 /* Probably should panic */ 3004 bw = 1; 3005 if (bw > min_bw) 3006 return (bw); 3007 else 3008 return (min_bw); 3009 } 3010 if (bbr->rc_lt_use_bw) 3011 bw = bbr->r_ctl.rc_lt_bw; 3012 else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0)) 3013 bw = bbr->r_ctl.red_bw; 3014 else 3015 bw = get_filter_value(&bbr->r_ctl.rc_delrate); 3016 if (bbr->rc_tp->t_peakrate_thr && (bbr->rc_use_google == 0)) { 3017 /* 3018 * Enforce user set rate limit, keep in mind that 3019 * t_peakrate_thr is in B/s already 3020 */ 3021 bw = uqmin((uint64_t)bbr->rc_tp->t_peakrate_thr, bw); 3022 } 3023 if (bw == 0) { 3024 /* We should not be at 0, go to the initial window then */ 3025 goto use_initial_window; 3026 } 3027 if (bw < 1) 3028 /* Probably should panic */ 3029 bw = 1; 3030 if (bw < min_bw) 3031 bw = min_bw; 3032 return (bw); 3033 } 3034 3035 static inline uint64_t 3036 bbr_get_bw(struct tcp_bbr *bbr) 3037 { 3038 uint64_t bw; 3039 3040 bw = __bbr_get_bw(bbr); 3041 return (bw); 3042 } 3043 3044 static inline void 3045 bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts) 3046 { 3047 bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch; 3048 bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time; 3049 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered; 3050 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 3051 } 3052 3053 static inline void 3054 bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts) 3055 { 3056 bbr->rc_lt_is_sampling = 0; 3057 bbr->rc_lt_use_bw = 0; 3058 bbr->r_ctl.rc_lt_bw = 0; 3059 bbr_reset_lt_bw_interval(bbr, cts); 3060 } 3061 3062 static inline void 3063 bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin) 3064 { 3065 uint64_t diff; 3066 3067 /* Do we have a previous sample? */ 3068 if (bbr->r_ctl.rc_lt_bw) { 3069 /* Get the diff in bytes per second */ 3070 if (bbr->r_ctl.rc_lt_bw > bw) 3071 diff = bbr->r_ctl.rc_lt_bw - bw; 3072 else 3073 diff = bw - bbr->r_ctl.rc_lt_bw; 3074 if ((diff <= bbr_lt_bw_diff) || 3075 (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) { 3076 /* Consider us policed */ 3077 uint32_t saved_bw; 3078 3079 saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw; 3080 bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2; /* average of two */ 3081 bbr->rc_lt_use_bw = 1; 3082 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; 3083 /* 3084 * Use pkt based epoch for measuring length of 3085 * policer up 3086 */ 3087 bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch; 3088 /* 3089 * reason 4 is we need to start consider being 3090 * policed 3091 */ 3092 bbr_log_type_ltbw(bbr, cts, 4, (uint32_t)bw, saved_bw, (uint32_t)diff, timin); 3093 return; 3094 } 3095 } 3096 bbr->r_ctl.rc_lt_bw = bw; 3097 bbr_reset_lt_bw_interval(bbr, cts); 3098 bbr_log_type_ltbw(bbr, cts, 5, 0, (uint32_t)bw, 0, timin); 3099 } 3100 3101 static void 3102 bbr_randomize_extra_state_time(struct tcp_bbr *bbr) 3103 { 3104 uint32_t ran, deduct; 3105 3106 ran = arc4random_uniform(bbr_rand_ot); 3107 if (ran) { 3108 deduct = bbr->r_ctl.rc_level_state_extra / ran; 3109 bbr->r_ctl.rc_level_state_extra -= deduct; 3110 } 3111 } 3112 /* 3113 * Return randomly the starting state 3114 * to use in probebw. 3115 */ 3116 static uint8_t 3117 bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts) 3118 { 3119 uint32_t ran; 3120 uint8_t ret_val; 3121 3122 /* Initialize the offset to 0 */ 3123 bbr->r_ctl.rc_exta_time_gd = 0; 3124 bbr->rc_hit_state_1 = 0; 3125 bbr->r_ctl.rc_level_state_extra = 0; 3126 ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1)); 3127 /* 3128 * The math works funny here :) the return value is used to set the 3129 * substate and then the state change is called which increments by 3130 * one. So if we return 1 (DRAIN) we will increment to 2 (LEVEL1) when 3131 * we fully enter the state. Note that the (8 - 1 - ran) assures that 3132 * we return 1 - 7, so we dont return 0 and end up starting in 3133 * state 1 (DRAIN). 3134 */ 3135 ret_val = BBR_SUBSTATE_COUNT - 1 - ran; 3136 /* Set an epoch */ 3137 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) 3138 bbr_set_epoch(bbr, cts, __LINE__); 3139 3140 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 3141 return (ret_val); 3142 } 3143 3144 static void 3145 bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected) 3146 { 3147 uint32_t diff, d_time; 3148 uint64_t del_time, bw, lost, delivered; 3149 3150 if (bbr->r_use_policer == 0) 3151 return; 3152 if (bbr->rc_lt_use_bw) { 3153 /* We are using lt bw do we stop yet? */ 3154 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use; 3155 if (diff > bbr_lt_bw_max_rtts) { 3156 /* Reset it all */ 3157 reset_all: 3158 bbr_reset_lt_bw_sampling(bbr, cts); 3159 if (bbr->rc_filled_pipe) { 3160 bbr_set_epoch(bbr, cts, __LINE__); 3161 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); 3162 bbr_substate_change(bbr, cts, __LINE__, 0); 3163 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; 3164 bbr_log_type_statechange(bbr, cts, __LINE__); 3165 } else { 3166 /* 3167 * This should not happen really 3168 * unless we remove the startup/drain 3169 * restrictions above. 3170 */ 3171 bbr->rc_bbr_state = BBR_STATE_STARTUP; 3172 bbr_set_epoch(bbr, cts, __LINE__); 3173 bbr->r_ctl.rc_bbr_state_time = cts; 3174 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 3175 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; 3176 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; 3177 bbr_set_state_target(bbr, __LINE__); 3178 bbr_log_type_statechange(bbr, cts, __LINE__); 3179 } 3180 /* reason 0 is to stop using lt-bw */ 3181 bbr_log_type_ltbw(bbr, cts, 0, 0, 0, 0, 0); 3182 return; 3183 } 3184 if (bbr_lt_intvl_fp == 0) { 3185 /* Not doing false-postive detection */ 3186 return; 3187 } 3188 /* False positive detection */ 3189 if (diff == bbr_lt_intvl_fp) { 3190 /* At bbr_lt_intvl_fp we record the lost */ 3191 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered; 3192 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 3193 } else if (diff > (bbr_lt_intvl_min_rtts + bbr_lt_intvl_fp)) { 3194 /* Now is our loss rate still high? */ 3195 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost; 3196 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del; 3197 if ((delivered == 0) || 3198 (((lost * 1000)/delivered) < bbr_lt_fd_thresh)) { 3199 /* No still below our threshold */ 3200 bbr_log_type_ltbw(bbr, cts, 7, lost, delivered, 0, 0); 3201 } else { 3202 /* Yikes its still high, it must be a false positive */ 3203 bbr_log_type_ltbw(bbr, cts, 8, lost, delivered, 0, 0); 3204 goto reset_all; 3205 } 3206 } 3207 return; 3208 } 3209 /* 3210 * Wait for the first loss before sampling, to let the policer 3211 * exhaust its tokens and estimate the steady-state rate allowed by 3212 * the policer. Starting samples earlier includes bursts that 3213 * over-estimate the bw. 3214 */ 3215 if (bbr->rc_lt_is_sampling == 0) { 3216 /* reason 1 is to begin doing the sampling */ 3217 if (loss_detected == 0) 3218 return; 3219 bbr_reset_lt_bw_interval(bbr, cts); 3220 bbr->rc_lt_is_sampling = 1; 3221 bbr_log_type_ltbw(bbr, cts, 1, 0, 0, 0, 0); 3222 return; 3223 } 3224 /* Now how long were we delivering long term last> */ 3225 if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time)) 3226 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time; 3227 else 3228 d_time = 0; 3229 3230 /* To avoid underestimates, reset sampling if we run out of data. */ 3231 if (bbr->r_ctl.r_app_limited_until) { 3232 /* Can not measure in app-limited state */ 3233 bbr_reset_lt_bw_sampling(bbr, cts); 3234 /* reason 2 is to reset sampling due to app limits */ 3235 bbr_log_type_ltbw(bbr, cts, 2, 0, 0, 0, d_time); 3236 return; 3237 } 3238 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch; 3239 if (diff < bbr_lt_intvl_min_rtts) { 3240 /* 3241 * need more samples (we don't 3242 * start on a round like linux so 3243 * we need 1 more). 3244 */ 3245 /* 6 is not_enough time or no-loss */ 3246 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time); 3247 return; 3248 } 3249 if (diff > (4 * bbr_lt_intvl_min_rtts)) { 3250 /* 3251 * For now if we wait too long, reset all sampling. We need 3252 * to do some research here, its possible that we should 3253 * base this on how much loss as occurred.. something like 3254 * if its under 10% (or some thresh) reset all otherwise 3255 * don't. Thats for phase II I guess. 3256 */ 3257 bbr_reset_lt_bw_sampling(bbr, cts); 3258 /* reason 3 is to reset sampling due too long of sampling */ 3259 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time); 3260 return; 3261 } 3262 /* 3263 * End sampling interval when a packet is lost, so we estimate the 3264 * policer tokens were exhausted. Stopping the sampling before the 3265 * tokens are exhausted under-estimates the policed rate. 3266 */ 3267 if (loss_detected == 0) { 3268 /* 6 is not_enough time or no-loss */ 3269 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time); 3270 return; 3271 } 3272 /* Calculate packets lost and delivered in sampling interval. */ 3273 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost; 3274 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del; 3275 if ((delivered == 0) || 3276 (((lost * 1000)/delivered) < bbr_lt_loss_thresh)) { 3277 bbr_log_type_ltbw(bbr, cts, 6, lost, delivered, 0, d_time); 3278 return; 3279 } 3280 if (d_time < 1000) { 3281 /* Not enough time. wait */ 3282 /* 6 is not_enough time or no-loss */ 3283 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time); 3284 return; 3285 } 3286 if (d_time >= (0xffffffff / USECS_IN_MSEC)) { 3287 /* Too long */ 3288 bbr_reset_lt_bw_sampling(bbr, cts); 3289 /* reason 3 is to reset sampling due too long of sampling */ 3290 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time); 3291 return; 3292 } 3293 del_time = d_time; 3294 bw = delivered; 3295 bw *= (uint64_t)USECS_IN_SECOND; 3296 bw /= del_time; 3297 bbr_lt_bw_samp_done(bbr, bw, cts, d_time); 3298 } 3299 3300 /* 3301 * Allocate a sendmap from our zone. 3302 */ 3303 static struct bbr_sendmap * 3304 bbr_alloc(struct tcp_bbr *bbr) 3305 { 3306 struct bbr_sendmap *rsm; 3307 3308 BBR_STAT_INC(bbr_to_alloc); 3309 rsm = uma_zalloc(bbr_zone, (M_NOWAIT | M_ZERO)); 3310 if (rsm) { 3311 bbr->r_ctl.rc_num_maps_alloced++; 3312 return (rsm); 3313 } 3314 if (bbr->r_ctl.rc_free_cnt) { 3315 BBR_STAT_INC(bbr_to_alloc_emerg); 3316 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); 3317 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next); 3318 bbr->r_ctl.rc_free_cnt--; 3319 return (rsm); 3320 } 3321 BBR_STAT_INC(bbr_to_alloc_failed); 3322 return (NULL); 3323 } 3324 3325 static struct bbr_sendmap * 3326 bbr_alloc_full_limit(struct tcp_bbr *bbr) 3327 { 3328 if ((V_tcp_map_entries_limit > 0) && 3329 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { 3330 BBR_STAT_INC(bbr_alloc_limited); 3331 if (!bbr->alloc_limit_reported) { 3332 bbr->alloc_limit_reported = 1; 3333 BBR_STAT_INC(bbr_alloc_limited_conns); 3334 } 3335 return (NULL); 3336 } 3337 return (bbr_alloc(bbr)); 3338 } 3339 3340 3341 /* wrapper to allocate a sendmap entry, subject to a specific limit */ 3342 static struct bbr_sendmap * 3343 bbr_alloc_limit(struct tcp_bbr *bbr, uint8_t limit_type) 3344 { 3345 struct bbr_sendmap *rsm; 3346 3347 if (limit_type) { 3348 /* currently there is only one limit type */ 3349 if (V_tcp_map_split_limit > 0 && 3350 bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) { 3351 BBR_STAT_INC(bbr_split_limited); 3352 if (!bbr->alloc_limit_reported) { 3353 bbr->alloc_limit_reported = 1; 3354 BBR_STAT_INC(bbr_alloc_limited_conns); 3355 } 3356 return (NULL); 3357 } 3358 } 3359 3360 /* allocate and mark in the limit type, if set */ 3361 rsm = bbr_alloc(bbr); 3362 if (rsm != NULL && limit_type) { 3363 rsm->r_limit_type = limit_type; 3364 bbr->r_ctl.rc_num_split_allocs++; 3365 } 3366 return (rsm); 3367 } 3368 3369 static void 3370 bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm) 3371 { 3372 if (rsm->r_limit_type) { 3373 /* currently there is only one limit type */ 3374 bbr->r_ctl.rc_num_split_allocs--; 3375 } 3376 if (rsm->r_is_smallmap) 3377 bbr->r_ctl.rc_num_small_maps_alloced--; 3378 if (bbr->r_ctl.rc_tlp_send == rsm) 3379 bbr->r_ctl.rc_tlp_send = NULL; 3380 if (bbr->r_ctl.rc_resend == rsm) { 3381 bbr->r_ctl.rc_resend = NULL; 3382 } 3383 if (bbr->r_ctl.rc_next == rsm) 3384 bbr->r_ctl.rc_next = NULL; 3385 if (bbr->r_ctl.rc_sacklast == rsm) 3386 bbr->r_ctl.rc_sacklast = NULL; 3387 if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) { 3388 memset(rsm, 0, sizeof(struct bbr_sendmap)); 3389 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next); 3390 rsm->r_limit_type = 0; 3391 bbr->r_ctl.rc_free_cnt++; 3392 return; 3393 } 3394 bbr->r_ctl.rc_num_maps_alloced--; 3395 uma_zfree(bbr_zone, rsm); 3396 } 3397 3398 /* 3399 * Returns the BDP. 3400 */ 3401 static uint64_t 3402 bbr_get_bw_delay_prod(uint64_t rtt, uint64_t bw) { 3403 /* 3404 * Calculate the bytes in flight needed given the bw (in bytes per 3405 * second) and the specifyed rtt in useconds. We need to put out the 3406 * returned value per RTT to match that rate. Gain will normaly 3407 * raise it up from there. 3408 * 3409 * This should not overflow as long as the bandwidth is below 1 3410 * TByte per second (bw < 10**12 = 2**40) and the rtt is smaller 3411 * than 1000 seconds (rtt < 10**3 * 10**6 = 10**9 = 2**30). 3412 */ 3413 uint64_t usec_per_sec; 3414 3415 usec_per_sec = USECS_IN_SECOND; 3416 return ((rtt * bw) / usec_per_sec); 3417 } 3418 3419 /* 3420 * Return the initial cwnd. 3421 */ 3422 static uint32_t 3423 bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp) 3424 { 3425 uint32_t i_cwnd; 3426 3427 if (bbr->rc_init_win) { 3428 i_cwnd = bbr->rc_init_win * tp->t_maxseg; 3429 } else if (V_tcp_initcwnd_segments) 3430 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg), 3431 max(2 * tp->t_maxseg, 14600)); 3432 else if (V_tcp_do_rfc3390) 3433 i_cwnd = min(4 * tp->t_maxseg, 3434 max(2 * tp->t_maxseg, 4380)); 3435 else { 3436 /* Per RFC5681 Section 3.1 */ 3437 if (tp->t_maxseg > 2190) 3438 i_cwnd = 2 * tp->t_maxseg; 3439 else if (tp->t_maxseg > 1095) 3440 i_cwnd = 3 * tp->t_maxseg; 3441 else 3442 i_cwnd = 4 * tp->t_maxseg; 3443 } 3444 return (i_cwnd); 3445 } 3446 3447 /* 3448 * Given a specified gain, return the target 3449 * cwnd based on that gain. 3450 */ 3451 static uint32_t 3452 bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw) 3453 { 3454 uint64_t bdp, rtt; 3455 uint32_t cwnd; 3456 3457 if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) || 3458 (bbr_get_full_bw(bbr) == 0)) { 3459 /* No measurements yet */ 3460 return (bbr_initial_cwnd(bbr, bbr->rc_tp)); 3461 } 3462 /* 3463 * Get bytes per RTT needed (rttProp is normally in 3464 * bbr_cwndtarget_rtt_touse) 3465 */ 3466 rtt = bbr_get_rtt(bbr, bbr_cwndtarget_rtt_touse); 3467 /* Get the bdp from the two values */ 3468 bdp = bbr_get_bw_delay_prod(rtt, bw); 3469 /* Now apply the gain */ 3470 cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT)); 3471 3472 return (cwnd); 3473 } 3474 3475 static uint32_t 3476 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain) 3477 { 3478 uint32_t cwnd, mss; 3479 3480 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); 3481 /* Get the base cwnd with gain rounded to a mss */ 3482 cwnd = roundup(bbr_get_raw_target_cwnd(bbr, bw, gain), mss); 3483 /* 3484 * Add in N (2 default since we do not have a 3485 * fq layer to trap packets in) quanta's per the I-D 3486 * section 4.2.3.2 quanta adjust. 3487 */ 3488 cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs); 3489 if (bbr->rc_use_google) { 3490 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && 3491 (bbr_state_val(bbr) == BBR_SUB_GAIN)) { 3492 /* 3493 * The linux implementation adds 3494 * an extra 2 x mss in gain cycle which 3495 * is documented no-where except in the code. 3496 * so we add more for Neal undocumented feature 3497 */ 3498 cwnd += 2 * mss; 3499 } 3500 if ((cwnd / mss) & 0x1) { 3501 /* Round up for odd num mss */ 3502 cwnd += mss; 3503 } 3504 } 3505 /* Are we below the min cwnd? */ 3506 if (cwnd < get_min_cwnd(bbr)) 3507 return (get_min_cwnd(bbr)); 3508 return (cwnd); 3509 } 3510 3511 static uint16_t 3512 bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain) 3513 { 3514 if (gain < 1) 3515 gain = 1; 3516 return (gain); 3517 } 3518 3519 static uint32_t 3520 bbr_get_header_oh(struct tcp_bbr *bbr) 3521 { 3522 int seg_oh; 3523 3524 seg_oh = 0; 3525 if (bbr->r_ctl.rc_inc_tcp_oh) { 3526 /* Do we include TCP overhead? */ 3527 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr)); 3528 } 3529 if (bbr->r_ctl.rc_inc_ip_oh) { 3530 /* Do we include IP overhead? */ 3531 #ifdef INET6 3532 if (bbr->r_is_v6) 3533 seg_oh += sizeof(struct ip6_hdr); 3534 else 3535 #endif 3536 #ifdef INET 3537 seg_oh += sizeof(struct ip); 3538 #endif 3539 } 3540 if (bbr->r_ctl.rc_inc_enet_oh) { 3541 /* Do we include the ethernet overhead? */ 3542 seg_oh += sizeof(struct ether_header); 3543 } 3544 return(seg_oh); 3545 } 3546 3547 3548 static uint32_t 3549 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw) 3550 { 3551 uint64_t divor, res, tim; 3552 3553 if (useconds_time == 0) 3554 return (0); 3555 gain = bbr_gain_adjust(bbr, gain); 3556 divor = (uint64_t)USECS_IN_SECOND * (uint64_t)BBR_UNIT; 3557 tim = useconds_time; 3558 res = (tim * bw * gain) / divor; 3559 if (res == 0) 3560 res = 1; 3561 return ((uint32_t)res); 3562 } 3563 3564 /* 3565 * Given a gain and a length return the delay in useconds that 3566 * should be used to evenly space out packets 3567 * on the connection (based on the gain factor). 3568 */ 3569 static uint32_t 3570 bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog) 3571 { 3572 uint64_t bw, lentim, res; 3573 uint32_t usecs, srtt, over = 0; 3574 uint32_t seg_oh, num_segs, maxseg; 3575 3576 if (len == 0) 3577 return (0); 3578 3579 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; 3580 num_segs = (len + maxseg - 1) / maxseg; 3581 if (bbr->rc_use_google == 0) { 3582 seg_oh = bbr_get_header_oh(bbr); 3583 len += (num_segs * seg_oh); 3584 } 3585 gain = bbr_gain_adjust(bbr, gain); 3586 bw = bbr_get_bw(bbr); 3587 if (bbr->rc_use_google) { 3588 uint64_t cbw; 3589 3590 /* 3591 * Reduce the b/w by the google discount 3592 * factor 10 = 1%. 3593 */ 3594 cbw = bw * (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount); 3595 cbw /= (uint64_t)1000; 3596 /* We don't apply a discount if it results in 0 */ 3597 if (cbw > 0) 3598 bw = cbw; 3599 } 3600 lentim = ((uint64_t)len * 3601 (uint64_t)USECS_IN_SECOND * 3602 (uint64_t)BBR_UNIT); 3603 res = lentim / ((uint64_t)gain * bw); 3604 if (res == 0) 3605 res = 1; 3606 usecs = (uint32_t)res; 3607 srtt = bbr_get_rtt(bbr, BBR_SRTT); 3608 if (bbr_hptsi_max_mul && bbr_hptsi_max_div && 3609 (bbr->rc_use_google == 0) && 3610 (usecs > ((srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div))) { 3611 /* 3612 * We cannot let the delay be more than 1/2 the srtt time. 3613 * Otherwise we cannot pace out or send properly. 3614 */ 3615 over = usecs = (srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div; 3616 BBR_STAT_INC(bbr_hpts_min_time); 3617 } 3618 if (!nolog) 3619 bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1); 3620 return (usecs); 3621 } 3622 3623 static void 3624 bbr_ack_received(struct tcpcb *tp, struct tcp_bbr *bbr, struct tcphdr *th, uint32_t bytes_this_ack, 3625 uint32_t sack_changed, uint32_t prev_acked, int32_t line, uint32_t losses) 3626 { 3627 INP_WLOCK_ASSERT(tp->t_inpcb); 3628 uint64_t bw; 3629 uint32_t cwnd, target_cwnd, saved_bytes, maxseg; 3630 int32_t meth; 3631 3632 #ifdef STATS 3633 if ((tp->t_flags & TF_GPUTINPROG) && 3634 SEQ_GEQ(th->th_ack, tp->gput_ack)) { 3635 /* 3636 * Strech acks and compressed acks will cause this to 3637 * oscillate but we are doing it the same way as the main 3638 * stack so it will be compariable (though possibly not 3639 * ideal). 3640 */ 3641 int32_t cgput; 3642 int64_t gput, time_stamp; 3643 3644 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8; 3645 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000)); 3646 cgput = gput / time_stamp; 3647 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT, 3648 cgput); 3649 if (tp->t_stats_gput_prev > 0) 3650 stats_voi_update_abs_s32(tp->t_stats, 3651 VOI_TCP_GPUT_ND, 3652 ((gput - tp->t_stats_gput_prev) * 100) / 3653 tp->t_stats_gput_prev); 3654 tp->t_flags &= ~TF_GPUTINPROG; 3655 tp->t_stats_gput_prev = cgput; 3656 } 3657 #endif 3658 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && 3659 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) { 3660 /* We don't change anything in probe-rtt */ 3661 return; 3662 } 3663 maxseg = tp->t_maxseg - bbr->rc_last_options; 3664 saved_bytes = bytes_this_ack; 3665 bytes_this_ack += sack_changed; 3666 if (bytes_this_ack > prev_acked) { 3667 bytes_this_ack -= prev_acked; 3668 /* 3669 * A byte ack'd gives us a full mss 3670 * to be like linux i.e. they count packets. 3671 */ 3672 if ((bytes_this_ack < maxseg) && bbr->rc_use_google) 3673 bytes_this_ack = maxseg; 3674 } else { 3675 /* Unlikely */ 3676 bytes_this_ack = 0; 3677 } 3678 cwnd = tp->snd_cwnd; 3679 bw = get_filter_value(&bbr->r_ctl.rc_delrate); 3680 if (bw) 3681 target_cwnd = bbr_get_target_cwnd(bbr, 3682 bw, 3683 (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain); 3684 else 3685 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp); 3686 if (IN_RECOVERY(tp->t_flags) && 3687 (bbr->bbr_prev_in_rec == 0)) { 3688 /* 3689 * We are entering recovery and 3690 * thus packet conservation. 3691 */ 3692 bbr->pkt_conservation = 1; 3693 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime; 3694 cwnd = ctf_flight_size(tp, 3695 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + 3696 bytes_this_ack; 3697 } 3698 if (IN_RECOVERY(tp->t_flags)) { 3699 uint32_t flight; 3700 3701 bbr->bbr_prev_in_rec = 1; 3702 if (cwnd > losses) { 3703 cwnd -= losses; 3704 if (cwnd < maxseg) 3705 cwnd = maxseg; 3706 } else 3707 cwnd = maxseg; 3708 flight = ctf_flight_size(tp, 3709 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 3710 bbr_log_type_cwndupd(bbr, flight, 0, 3711 losses, 10, 0, 0, line); 3712 if (bbr->pkt_conservation) { 3713 uint32_t time_in; 3714 3715 if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start)) 3716 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start; 3717 else 3718 time_in = 0; 3719 3720 if (time_in >= bbr_get_rtt(bbr, BBR_RTT_PROP)) { 3721 /* Clear packet conservation after an rttProp */ 3722 bbr->pkt_conservation = 0; 3723 } else { 3724 if ((flight + bytes_this_ack) > cwnd) 3725 cwnd = flight + bytes_this_ack; 3726 if (cwnd < get_min_cwnd(bbr)) 3727 cwnd = get_min_cwnd(bbr); 3728 tp->snd_cwnd = cwnd; 3729 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, 3730 prev_acked, 1, target_cwnd, th->th_ack, line); 3731 return; 3732 } 3733 } 3734 } else 3735 bbr->bbr_prev_in_rec = 0; 3736 if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) { 3737 bbr->r_ctl.restrict_growth--; 3738 if (bytes_this_ack > maxseg) 3739 bytes_this_ack = maxseg; 3740 } 3741 if (bbr->rc_filled_pipe) { 3742 /* 3743 * Here we have exited startup and filled the pipe. We will 3744 * thus allow the cwnd to shrink to the target. We hit here 3745 * mostly. 3746 */ 3747 uint32_t s_cwnd; 3748 3749 meth = 2; 3750 s_cwnd = min((cwnd + bytes_this_ack), target_cwnd); 3751 if (s_cwnd > cwnd) 3752 cwnd = s_cwnd; 3753 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing) 3754 cwnd = s_cwnd; 3755 } else { 3756 /* 3757 * Here we are still in startup, we increase cwnd by what 3758 * has been acked. 3759 */ 3760 if ((cwnd < target_cwnd) || 3761 (bbr->rc_past_init_win == 0)) { 3762 meth = 3; 3763 cwnd += bytes_this_ack; 3764 } else { 3765 /* 3766 * Method 4 means we are at target so no gain in 3767 * startup and past the initial window. 3768 */ 3769 meth = 4; 3770 } 3771 } 3772 tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr)); 3773 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, line); 3774 } 3775 3776 static void 3777 tcp_bbr_partialack(struct tcpcb *tp) 3778 { 3779 struct tcp_bbr *bbr; 3780 3781 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 3782 INP_WLOCK_ASSERT(tp->t_inpcb); 3783 if (ctf_flight_size(tp, 3784 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= 3785 tp->snd_cwnd) { 3786 bbr->r_wanted_output = 1; 3787 } 3788 } 3789 3790 static void 3791 bbr_post_recovery(struct tcpcb *tp) 3792 { 3793 struct tcp_bbr *bbr; 3794 uint32_t flight; 3795 3796 INP_WLOCK_ASSERT(tp->t_inpcb); 3797 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 3798 /* 3799 * Here we just exit recovery. 3800 */ 3801 EXIT_RECOVERY(tp->t_flags); 3802 /* Lock in our b/w reduction for the specified number of pkt-epochs */ 3803 bbr->r_recovery_bw = 0; 3804 tp->snd_recover = tp->snd_una; 3805 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); 3806 bbr->pkt_conservation = 0; 3807 if (bbr->rc_use_google == 0) { 3808 /* 3809 * For non-google mode lets 3810 * go ahead and make sure we clear 3811 * the recovery state so if we 3812 * bounce back in to recovery we 3813 * will do PC. 3814 */ 3815 bbr->bbr_prev_in_rec = 0; 3816 } 3817 bbr_log_type_exit_rec(bbr); 3818 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { 3819 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent); 3820 bbr_log_type_cwndupd(bbr, 0, 0, 0, 15, 0, 0, __LINE__); 3821 } else { 3822 /* For probe-rtt case lets fix up its saved_cwnd */ 3823 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) { 3824 bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent; 3825 bbr_log_type_cwndupd(bbr, 0, 0, 0, 16, 0, 0, __LINE__); 3826 } 3827 } 3828 flight = ctf_flight_size(tp, 3829 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 3830 if ((bbr->rc_use_google == 0) && 3831 bbr_do_red) { 3832 uint64_t val, lr2use; 3833 uint32_t maxseg, newcwnd, acks_inflight, ratio, cwnd; 3834 uint32_t *cwnd_p; 3835 3836 if (bbr_get_rtt(bbr, BBR_SRTT)) { 3837 val = ((uint64_t)bbr_get_rtt(bbr, BBR_RTT_PROP) * (uint64_t)1000); 3838 val /= bbr_get_rtt(bbr, BBR_SRTT); 3839 ratio = (uint32_t)val; 3840 } else 3841 ratio = 1000; 3842 3843 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div, 3844 bbr->r_ctl.recovery_lr, 21, 3845 ratio, 3846 bbr->r_ctl.rc_red_cwnd_pe, 3847 __LINE__); 3848 if ((ratio < bbr_do_red) || (bbr_do_red == 0)) 3849 goto done; 3850 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && 3851 bbr_prtt_slam_cwnd) || 3852 (bbr_sub_drain_slam_cwnd && 3853 (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && 3854 bbr->rc_hit_state_1 && 3855 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) || 3856 ((bbr->rc_bbr_state == BBR_STATE_DRAIN) && 3857 bbr_slam_cwnd_in_main_drain)) { 3858 /* 3859 * Here we must poke at the saved cwnd 3860 * as well as the cwnd. 3861 */ 3862 cwnd = bbr->r_ctl.rc_saved_cwnd; 3863 cwnd_p = &bbr->r_ctl.rc_saved_cwnd; 3864 } else { 3865 cwnd = tp->snd_cwnd; 3866 cwnd_p = &tp->snd_cwnd; 3867 } 3868 maxseg = tp->t_maxseg - bbr->rc_last_options; 3869 /* Add the overall lr with the recovery lr */ 3870 if (bbr->r_ctl.rc_lost == 0) 3871 lr2use = 0; 3872 else if (bbr->r_ctl.rc_delivered == 0) 3873 lr2use = 1000; 3874 else { 3875 lr2use = bbr->r_ctl.rc_lost * 1000; 3876 lr2use /= bbr->r_ctl.rc_delivered; 3877 } 3878 lr2use += bbr->r_ctl.recovery_lr; 3879 acks_inflight = (flight / (maxseg * 2)); 3880 if (bbr_red_scale) { 3881 lr2use *= bbr_get_rtt(bbr, BBR_SRTT); 3882 lr2use /= bbr_red_scale; 3883 if ((bbr_red_growth_restrict) && 3884 ((bbr_get_rtt(bbr, BBR_SRTT)/bbr_red_scale) > 1)) 3885 bbr->r_ctl.restrict_growth += acks_inflight; 3886 } 3887 if (lr2use) { 3888 val = (uint64_t)cwnd * lr2use; 3889 val /= 1000; 3890 if (cwnd > val) 3891 newcwnd = roundup((cwnd - val), maxseg); 3892 else 3893 newcwnd = maxseg; 3894 } else { 3895 val = (uint64_t)cwnd * (uint64_t)bbr_red_mul; 3896 val /= (uint64_t)bbr_red_div; 3897 newcwnd = roundup((uint32_t)val, maxseg); 3898 } 3899 /* with standard delayed acks how many acks can I expect? */ 3900 if (bbr_drop_limit == 0) { 3901 /* 3902 * Anticpate how much we will 3903 * raise the cwnd based on the acks. 3904 */ 3905 if ((newcwnd + (acks_inflight * maxseg)) < get_min_cwnd(bbr)) { 3906 /* We do enforce the min (with the acks) */ 3907 newcwnd = (get_min_cwnd(bbr) - acks_inflight); 3908 } 3909 } else { 3910 /* 3911 * A strict drop limit of N is is inplace 3912 */ 3913 if (newcwnd < (bbr_drop_limit * maxseg)) { 3914 newcwnd = bbr_drop_limit * maxseg; 3915 } 3916 } 3917 /* For the next N acks do we restrict the growth */ 3918 *cwnd_p = newcwnd; 3919 if (tp->snd_cwnd > newcwnd) 3920 tp->snd_cwnd = newcwnd; 3921 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div, val, 22, 3922 (uint32_t)lr2use, 3923 bbr_get_rtt(bbr, BBR_SRTT), __LINE__); 3924 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch; 3925 } 3926 done: 3927 bbr->r_ctl.recovery_lr = 0; 3928 if (flight <= tp->snd_cwnd) { 3929 bbr->r_wanted_output = 1; 3930 } 3931 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); 3932 } 3933 3934 static void 3935 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts) 3936 { 3937 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate); 3938 /* Limit the drop in b/w to 1/2 our current filter. */ 3939 if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate) 3940 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate; 3941 if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2)) 3942 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2; 3943 tcp_bbr_tso_size_check(bbr, cts); 3944 } 3945 3946 static void 3947 bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_sendmap *rsm) 3948 { 3949 struct tcp_bbr *bbr; 3950 3951 INP_WLOCK_ASSERT(tp->t_inpcb); 3952 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 3953 switch (type) { 3954 case CC_NDUPACK: 3955 if (!IN_RECOVERY(tp->t_flags)) { 3956 tp->snd_recover = tp->snd_max; 3957 /* Start a new epoch */ 3958 bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__); 3959 if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) { 3960 /* 3961 * Move forward the lt epoch 3962 * so it won't count the truncated 3963 * epoch. 3964 */ 3965 bbr->r_ctl.rc_lt_epoch++; 3966 } 3967 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { 3968 /* 3969 * Just like the policer detection code 3970 * if we are in startup we must push 3971 * forward the last startup epoch 3972 * to hide the truncated PE. 3973 */ 3974 bbr->r_ctl.rc_bbr_last_startup_epoch++; 3975 } 3976 bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd; 3977 ENTER_RECOVERY(tp->t_flags); 3978 bbr->rc_tlp_rtx_out = 0; 3979 bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate; 3980 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime); 3981 if (bbr->rc_inp->inp_in_hpts && 3982 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) { 3983 /* 3984 * When we enter recovery, we need to restart 3985 * any timers. This may mean we gain an agg 3986 * early, which will be made up for at the last 3987 * rxt out. 3988 */ 3989 bbr->rc_timer_first = 1; 3990 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); 3991 } 3992 /* 3993 * Calculate a new cwnd based on to the current 3994 * delivery rate with no gain. We get the bdp 3995 * without gaining it up like we normally would and 3996 * we use the last cur_del_rate. 3997 */ 3998 if ((bbr->rc_use_google == 0) && 3999 (bbr->r_ctl.bbr_rttprobe_gain_val || 4000 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) { 4001 tp->snd_cwnd = ctf_flight_size(tp, 4002 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + 4003 (tp->t_maxseg - bbr->rc_last_options); 4004 if (tp->snd_cwnd < get_min_cwnd(bbr)) { 4005 /* We always gate to min cwnd */ 4006 tp->snd_cwnd = get_min_cwnd(bbr); 4007 } 4008 bbr_log_type_cwndupd(bbr, 0, 0, 0, 14, 0, 0, __LINE__); 4009 } 4010 bbr_log_type_enter_rec(bbr, rsm->r_start); 4011 } 4012 break; 4013 case CC_RTO_ERR: 4014 KMOD_TCPSTAT_INC(tcps_sndrexmitbad); 4015 /* RTO was unnecessary, so reset everything. */ 4016 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime); 4017 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { 4018 tp->snd_cwnd = tp->snd_cwnd_prev; 4019 tp->snd_ssthresh = tp->snd_ssthresh_prev; 4020 tp->snd_recover = tp->snd_recover_prev; 4021 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent); 4022 bbr_log_type_cwndupd(bbr, 0, 0, 0, 13, 0, 0, __LINE__); 4023 } 4024 tp->t_badrxtwin = 0; 4025 break; 4026 } 4027 } 4028 4029 /* 4030 * Indicate whether this ack should be delayed. We can delay the ack if 4031 * following conditions are met: 4032 * - There is no delayed ack timer in progress. 4033 * - Our last ack wasn't a 0-sized window. We never want to delay 4034 * the ack that opens up a 0-sized window. 4035 * - LRO wasn't used for this segment. We make sure by checking that the 4036 * segment size is not larger than the MSS. 4037 * - Delayed acks are enabled or this is a half-synchronized T/TCP 4038 * connection. 4039 * - The data being acked is less than a full segment (a stretch ack 4040 * of more than a segment we should ack. 4041 * - nsegs is 1 (if its more than that we received more than 1 ack). 4042 */ 4043 #define DELAY_ACK(tp, bbr, nsegs) \ 4044 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \ 4045 ((tp->t_flags & TF_DELACK) == 0) && \ 4046 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) && \ 4047 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN))) 4048 4049 /* 4050 * Return the lowest RSM in the map of 4051 * packets still in flight that is not acked. 4052 * This should normally find on the first one 4053 * since we remove packets from the send 4054 * map after they are marked ACKED. 4055 */ 4056 static struct bbr_sendmap * 4057 bbr_find_lowest_rsm(struct tcp_bbr *bbr) 4058 { 4059 struct bbr_sendmap *rsm; 4060 4061 /* 4062 * Walk the time-order transmitted list looking for an rsm that is 4063 * not acked. This will be the one that was sent the longest time 4064 * ago that is still outstanding. 4065 */ 4066 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) { 4067 if (rsm->r_flags & BBR_ACKED) { 4068 continue; 4069 } 4070 goto finish; 4071 } 4072 finish: 4073 return (rsm); 4074 } 4075 4076 static struct bbr_sendmap * 4077 bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm) 4078 { 4079 struct bbr_sendmap *prsm; 4080 4081 /* 4082 * Walk the sequence order list backward until we hit and arrive at 4083 * the highest seq not acked. In theory when this is called it 4084 * should be the last segment (which it was not). 4085 */ 4086 prsm = rsm; 4087 TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { 4088 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) { 4089 continue; 4090 } 4091 return (prsm); 4092 } 4093 return (NULL); 4094 } 4095 4096 /* 4097 * Returns to the caller the number of microseconds that 4098 * the packet can be outstanding before we think we 4099 * should have had an ack returned. 4100 */ 4101 static uint32_t 4102 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm) 4103 { 4104 /* 4105 * lro is the flag we use to determine if we have seen reordering. 4106 * If it gets set we have seen reordering. The reorder logic either 4107 * works in one of two ways: 4108 * 4109 * If reorder-fade is configured, then we track the last time we saw 4110 * re-ordering occur. If we reach the point where enough time as 4111 * passed we no longer consider reordering has occuring. 4112 * 4113 * Or if reorder-face is 0, then once we see reordering we consider 4114 * the connection to alway be subject to reordering and just set lro 4115 * to 1. 4116 * 4117 * In the end if lro is non-zero we add the extra time for 4118 * reordering in. 4119 */ 4120 int32_t lro; 4121 uint32_t thresh, t_rxtcur; 4122 4123 if (srtt == 0) 4124 srtt = 1; 4125 if (bbr->r_ctl.rc_reorder_ts) { 4126 if (bbr->r_ctl.rc_reorder_fade) { 4127 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) { 4128 lro = cts - bbr->r_ctl.rc_reorder_ts; 4129 if (lro == 0) { 4130 /* 4131 * No time as passed since the last 4132 * reorder, mark it as reordering. 4133 */ 4134 lro = 1; 4135 } 4136 } else { 4137 /* Negative time? */ 4138 lro = 0; 4139 } 4140 if (lro > bbr->r_ctl.rc_reorder_fade) { 4141 /* Turn off reordering seen too */ 4142 bbr->r_ctl.rc_reorder_ts = 0; 4143 lro = 0; 4144 } 4145 } else { 4146 /* Reodering does not fade */ 4147 lro = 1; 4148 } 4149 } else { 4150 lro = 0; 4151 } 4152 thresh = srtt + bbr->r_ctl.rc_pkt_delay; 4153 if (lro) { 4154 /* It must be set, if not you get 1/4 rtt */ 4155 if (bbr->r_ctl.rc_reorder_shift) 4156 thresh += (srtt >> bbr->r_ctl.rc_reorder_shift); 4157 else 4158 thresh += (srtt >> 2); 4159 } else { 4160 thresh += 1000; 4161 } 4162 /* We don't let the rack timeout be above a RTO */ 4163 if ((bbr->rc_tp)->t_srtt == 0) 4164 t_rxtcur = BBR_INITIAL_RTO; 4165 else 4166 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur); 4167 if (thresh > t_rxtcur) { 4168 thresh = t_rxtcur; 4169 } 4170 /* And we don't want it above the RTO max either */ 4171 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { 4172 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND); 4173 } 4174 bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK); 4175 return (thresh); 4176 } 4177 4178 /* 4179 * Return to the caller the amount of time in mico-seconds 4180 * that should be used for the TLP timer from the last 4181 * send time of this packet. 4182 */ 4183 static uint32_t 4184 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, 4185 struct bbr_sendmap *rsm, uint32_t srtt, 4186 uint32_t cts) 4187 { 4188 uint32_t thresh, len, maxseg, t_rxtcur; 4189 struct bbr_sendmap *prsm; 4190 4191 if (srtt == 0) 4192 srtt = 1; 4193 if (bbr->rc_tlp_threshold) 4194 thresh = srtt + (srtt / bbr->rc_tlp_threshold); 4195 else 4196 thresh = (srtt * 2); 4197 maxseg = tp->t_maxseg - bbr->rc_last_options; 4198 /* Get the previous sent packet, if any */ 4199 len = rsm->r_end - rsm->r_start; 4200 4201 /* 2.1 behavior */ 4202 prsm = TAILQ_PREV(rsm, bbr_head, r_tnext); 4203 if (prsm && (len <= maxseg)) { 4204 /* 4205 * Two packets outstanding, thresh should be (2*srtt) + 4206 * possible inter-packet delay (if any). 4207 */ 4208 uint32_t inter_gap = 0; 4209 int idx, nidx; 4210 4211 idx = rsm->r_rtr_cnt - 1; 4212 nidx = prsm->r_rtr_cnt - 1; 4213 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) { 4214 /* Yes it was sent later (or at the same time) */ 4215 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx]; 4216 } 4217 thresh += inter_gap; 4218 } else if (len <= maxseg) { 4219 /* 4220 * Possibly compensate for delayed-ack. 4221 */ 4222 uint32_t alt_thresh; 4223 4224 alt_thresh = srtt + (srtt / 2) + bbr_delayed_ack_time; 4225 if (alt_thresh > thresh) 4226 thresh = alt_thresh; 4227 } 4228 /* Not above the current RTO */ 4229 if (tp->t_srtt == 0) 4230 t_rxtcur = BBR_INITIAL_RTO; 4231 else 4232 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur); 4233 4234 bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP); 4235 /* Not above an RTO */ 4236 if (thresh > t_rxtcur) { 4237 thresh = t_rxtcur; 4238 } 4239 /* Not above a RTO max */ 4240 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) { 4241 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND); 4242 } 4243 /* And now apply the user TLP min */ 4244 if (thresh < bbr_tlp_min) { 4245 thresh = bbr_tlp_min; 4246 } 4247 return (thresh); 4248 } 4249 4250 /* 4251 * Return one of three RTTs to use (in microseconds). 4252 */ 4253 static __inline uint32_t 4254 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type) 4255 { 4256 uint32_t f_rtt; 4257 uint32_t srtt; 4258 4259 f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop); 4260 if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) { 4261 /* We have no rtt at all */ 4262 if (bbr->rc_tp->t_srtt == 0) 4263 f_rtt = BBR_INITIAL_RTO; 4264 else 4265 f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); 4266 /* 4267 * Since we don't know how good the rtt is apply a 4268 * delayed-ack min 4269 */ 4270 if (f_rtt < bbr_delayed_ack_time) { 4271 f_rtt = bbr_delayed_ack_time; 4272 } 4273 } 4274 /* Take the filter version or last measured pkt-rtt */ 4275 if (rtt_type == BBR_RTT_PROP) { 4276 srtt = f_rtt; 4277 } else if (rtt_type == BBR_RTT_PKTRTT) { 4278 if (bbr->r_ctl.rc_pkt_epoch_rtt) { 4279 srtt = bbr->r_ctl.rc_pkt_epoch_rtt; 4280 } else { 4281 /* No pkt rtt yet */ 4282 srtt = f_rtt; 4283 } 4284 } else if (rtt_type == BBR_RTT_RACK) { 4285 srtt = bbr->r_ctl.rc_last_rtt; 4286 /* We need to add in any internal delay for our timer */ 4287 if (bbr->rc_ack_was_delayed) 4288 srtt += bbr->r_ctl.rc_ack_hdwr_delay; 4289 } else if (rtt_type == BBR_SRTT) { 4290 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); 4291 } else { 4292 /* TSNH */ 4293 srtt = f_rtt; 4294 #ifdef BBR_INVARIANTS 4295 panic("Unknown rtt request type %d", rtt_type); 4296 #endif 4297 } 4298 return (srtt); 4299 } 4300 4301 static int 4302 bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts) 4303 { 4304 uint32_t thresh; 4305 4306 4307 thresh = bbr_calc_thresh_rack(bbr, bbr_get_rtt(bbr, BBR_RTT_RACK), 4308 cts, rsm); 4309 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) { 4310 /* It is lost (past time) */ 4311 return (1); 4312 } 4313 return (0); 4314 } 4315 4316 /* 4317 * Return a sendmap if we need to retransmit something. 4318 */ 4319 static struct bbr_sendmap * 4320 bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4321 { 4322 /* 4323 * Check to see that we don't need to fall into recovery. We will 4324 * need to do so if our oldest transmit is past the time we should 4325 * have had an ack. 4326 */ 4327 4328 struct bbr_sendmap *rsm; 4329 int32_t idx; 4330 4331 if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) { 4332 /* Nothing outstanding that we know of */ 4333 return (NULL); 4334 } 4335 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); 4336 if (rsm == NULL) { 4337 /* Nothing in the transmit map */ 4338 return (NULL); 4339 } 4340 if (tp->t_flags & TF_SENTFIN) { 4341 /* Fin restricted, don't find anything once a fin is sent */ 4342 return (NULL); 4343 } 4344 if (rsm->r_flags & BBR_ACKED) { 4345 /* 4346 * Ok the first one is acked (this really should not happen 4347 * since we remove the from the tmap once they are acked) 4348 */ 4349 rsm = bbr_find_lowest_rsm(bbr); 4350 if (rsm == NULL) 4351 return (NULL); 4352 } 4353 idx = rsm->r_rtr_cnt - 1; 4354 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) { 4355 /* Send timestamp is the same or less? can't be ready */ 4356 return (NULL); 4357 } 4358 /* Get our RTT time */ 4359 if (bbr_is_lost(bbr, rsm, cts) && 4360 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) || 4361 (rsm->r_flags & BBR_SACK_PASSED))) { 4362 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) { 4363 rsm->r_flags |= BBR_MARKED_LOST; 4364 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; 4365 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; 4366 } 4367 bbr_cong_signal(tp, NULL, CC_NDUPACK, rsm); 4368 #ifdef BBR_INVARIANTS 4369 if ((rsm->r_end - rsm->r_start) == 0) 4370 panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm); 4371 #endif 4372 return (rsm); 4373 } 4374 return (NULL); 4375 } 4376 4377 /* 4378 * RACK Timer, here we simply do logging and house keeping. 4379 * the normal bbr_output_wtime() function will call the 4380 * appropriate thing to check if we need to do a RACK retransmit. 4381 * We return 1, saying don't proceed with bbr_output_wtime only 4382 * when all timers have been stopped (destroyed PCB?). 4383 */ 4384 static int 4385 bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4386 { 4387 /* 4388 * This timer simply provides an internal trigger to send out data. 4389 * The check_recovery_mode call will see if there are needed 4390 * retransmissions, if so we will enter fast-recovery. The output 4391 * call may or may not do the same thing depending on sysctl 4392 * settings. 4393 */ 4394 uint32_t lost; 4395 4396 if (bbr->rc_all_timers_stopped) { 4397 return (1); 4398 } 4399 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { 4400 /* Its not time yet */ 4401 return (0); 4402 } 4403 BBR_STAT_INC(bbr_to_tot); 4404 lost = bbr->r_ctl.rc_lost; 4405 if (bbr->r_state && (bbr->r_state != tp->t_state)) 4406 bbr_set_state(tp, bbr, 0); 4407 bbr_log_to_event(bbr, cts, BBR_TO_FRM_RACK); 4408 if (bbr->r_ctl.rc_resend == NULL) { 4409 /* Lets do the check here */ 4410 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); 4411 } 4412 if (bbr_policer_call_from_rack_to) 4413 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); 4414 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK; 4415 return (0); 4416 } 4417 4418 static __inline void 4419 bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start) 4420 { 4421 int idx; 4422 4423 nrsm->r_start = start; 4424 nrsm->r_end = rsm->r_end; 4425 nrsm->r_rtr_cnt = rsm->r_rtr_cnt; 4426 nrsm->r_flags = rsm->r_flags; 4427 /* We don't transfer forward the SYN flag */ 4428 nrsm->r_flags &= ~BBR_HAS_SYN; 4429 /* We move forward the FIN flag, not that this should happen */ 4430 rsm->r_flags &= ~BBR_HAS_FIN; 4431 nrsm->r_dupack = rsm->r_dupack; 4432 nrsm->r_rtr_bytes = 0; 4433 nrsm->r_is_gain = rsm->r_is_gain; 4434 nrsm->r_is_drain = rsm->r_is_drain; 4435 nrsm->r_delivered = rsm->r_delivered; 4436 nrsm->r_ts_valid = rsm->r_ts_valid; 4437 nrsm->r_del_ack_ts = rsm->r_del_ack_ts; 4438 nrsm->r_del_time = rsm->r_del_time; 4439 nrsm->r_app_limited = rsm->r_app_limited; 4440 nrsm->r_first_sent_time = rsm->r_first_sent_time; 4441 nrsm->r_flight_at_send = rsm->r_flight_at_send; 4442 /* We split a piece the lower section looses any just_ret flag. */ 4443 nrsm->r_bbr_state = rsm->r_bbr_state; 4444 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) { 4445 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx]; 4446 } 4447 rsm->r_end = nrsm->r_start; 4448 idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs); 4449 idx /= 8; 4450 /* Check if we got too small */ 4451 if ((rsm->r_is_smallmap == 0) && 4452 ((rsm->r_end - rsm->r_start) <= idx)) { 4453 bbr->r_ctl.rc_num_small_maps_alloced++; 4454 rsm->r_is_smallmap = 1; 4455 } 4456 /* Check the new one as well */ 4457 if ((nrsm->r_end - nrsm->r_start) <= idx) { 4458 bbr->r_ctl.rc_num_small_maps_alloced++; 4459 nrsm->r_is_smallmap = 1; 4460 } 4461 } 4462 4463 static int 4464 bbr_sack_mergable(struct bbr_sendmap *at, 4465 uint32_t start, uint32_t end) 4466 { 4467 /* 4468 * Given a sack block defined by 4469 * start and end, and a current postion 4470 * at. Return 1 if either side of at 4471 * would show that the block is mergable 4472 * to that side. A block to be mergable 4473 * must have overlap with the start/end 4474 * and be in the SACK'd state. 4475 */ 4476 struct bbr_sendmap *l_rsm; 4477 struct bbr_sendmap *r_rsm; 4478 4479 /* first get the either side blocks */ 4480 l_rsm = TAILQ_PREV(at, bbr_head, r_next); 4481 r_rsm = TAILQ_NEXT(at, r_next); 4482 if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) { 4483 /* Potentially mergeable */ 4484 if ((l_rsm->r_end == start) || 4485 (SEQ_LT(start, l_rsm->r_end) && 4486 SEQ_GT(end, l_rsm->r_end))) { 4487 /* 4488 * map blk |------| 4489 * sack blk |------| 4490 * <or> 4491 * map blk |------| 4492 * sack blk |------| 4493 */ 4494 return (1); 4495 } 4496 } 4497 if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) { 4498 /* Potentially mergeable */ 4499 if ((r_rsm->r_start == end) || 4500 (SEQ_LT(start, r_rsm->r_start) && 4501 SEQ_GT(end, r_rsm->r_start))) { 4502 /* 4503 * map blk |---------| 4504 * sack blk |----| 4505 * <or> 4506 * map blk |---------| 4507 * sack blk |-------| 4508 */ 4509 return (1); 4510 } 4511 } 4512 return (0); 4513 } 4514 4515 static struct bbr_sendmap * 4516 bbr_merge_rsm(struct tcp_bbr *bbr, 4517 struct bbr_sendmap *l_rsm, 4518 struct bbr_sendmap *r_rsm) 4519 { 4520 /* 4521 * We are merging two ack'd RSM's, 4522 * the l_rsm is on the left (lower seq 4523 * values) and the r_rsm is on the right 4524 * (higher seq value). The simplest way 4525 * to merge these is to move the right 4526 * one into the left. I don't think there 4527 * is any reason we need to try to find 4528 * the oldest (or last oldest retransmitted). 4529 */ 4530 l_rsm->r_end = r_rsm->r_end; 4531 if (l_rsm->r_dupack < r_rsm->r_dupack) 4532 l_rsm->r_dupack = r_rsm->r_dupack; 4533 if (r_rsm->r_rtr_bytes) 4534 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes; 4535 if (r_rsm->r_in_tmap) { 4536 /* This really should not happen */ 4537 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext); 4538 } 4539 if (r_rsm->r_app_limited) 4540 l_rsm->r_app_limited = r_rsm->r_app_limited; 4541 /* Now the flags */ 4542 if (r_rsm->r_flags & BBR_HAS_FIN) 4543 l_rsm->r_flags |= BBR_HAS_FIN; 4544 if (r_rsm->r_flags & BBR_TLP) 4545 l_rsm->r_flags |= BBR_TLP; 4546 if (r_rsm->r_flags & BBR_RWND_COLLAPSED) 4547 l_rsm->r_flags |= BBR_RWND_COLLAPSED; 4548 if (r_rsm->r_flags & BBR_MARKED_LOST) { 4549 /* This really should not happen */ 4550 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start; 4551 } 4552 TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next); 4553 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) { 4554 /* Transfer the split limit to the map we free */ 4555 r_rsm->r_limit_type = l_rsm->r_limit_type; 4556 l_rsm->r_limit_type = 0; 4557 } 4558 bbr_free(bbr, r_rsm); 4559 return(l_rsm); 4560 } 4561 4562 /* 4563 * TLP Timer, here we simply setup what segment we want to 4564 * have the TLP expire on, the normal bbr_output_wtime() will then 4565 * send it out. 4566 * 4567 * We return 1, saying don't proceed with bbr_output_wtime only 4568 * when all timers have been stopped (destroyed PCB?). 4569 */ 4570 static int 4571 bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4572 { 4573 /* 4574 * Tail Loss Probe. 4575 */ 4576 struct bbr_sendmap *rsm = NULL; 4577 struct socket *so; 4578 uint32_t amm; 4579 uint32_t out, avail; 4580 uint32_t maxseg; 4581 int collapsed_win = 0; 4582 4583 if (bbr->rc_all_timers_stopped) { 4584 return (1); 4585 } 4586 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { 4587 /* Its not time yet */ 4588 return (0); 4589 } 4590 if (ctf_progress_timeout_check(tp, true)) { 4591 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 4592 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT); 4593 return (1); 4594 } 4595 /* Did we somehow get into persists? */ 4596 if (bbr->rc_in_persist) { 4597 return (0); 4598 } 4599 if (bbr->r_state && (bbr->r_state != tp->t_state)) 4600 bbr_set_state(tp, bbr, 0); 4601 BBR_STAT_INC(bbr_tlp_tot); 4602 maxseg = tp->t_maxseg - bbr->rc_last_options; 4603 #ifdef KERN_TLS 4604 if (bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) { 4605 /* 4606 * For hardware TLS we do *not* want to send 4607 * new data. 4608 */ 4609 goto need_retran; 4610 } 4611 #endif 4612 /* 4613 * A TLP timer has expired. We have been idle for 2 rtts. So we now 4614 * need to figure out how to force a full MSS segment out. 4615 */ 4616 so = tp->t_inpcb->inp_socket; 4617 avail = sbavail(&so->so_snd); 4618 out = ctf_outstanding(tp); 4619 if (out > tp->snd_wnd) { 4620 /* special case, we need a retransmission */ 4621 collapsed_win = 1; 4622 goto need_retran; 4623 } 4624 if (avail > out) { 4625 /* New data is available */ 4626 amm = avail - out; 4627 if (amm > maxseg) { 4628 amm = maxseg; 4629 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) { 4630 /* not enough to fill a MTU and no-delay is off */ 4631 goto need_retran; 4632 } 4633 /* Set the send-new override */ 4634 if ((out + amm) <= tp->snd_wnd) { 4635 bbr->rc_tlp_new_data = 1; 4636 } else { 4637 goto need_retran; 4638 } 4639 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; 4640 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max; 4641 bbr->r_ctl.rc_tlp_send = NULL; 4642 /* cap any slots */ 4643 BBR_STAT_INC(bbr_tlp_newdata); 4644 goto send; 4645 } 4646 need_retran: 4647 /* 4648 * Ok we need to arrange the last un-acked segment to be re-sent, or 4649 * optionally the first un-acked segment. 4650 */ 4651 if (collapsed_win == 0) { 4652 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); 4653 if (rsm && (BBR_ACKED | BBR_HAS_FIN)) { 4654 rsm = bbr_find_high_nonack(bbr, rsm); 4655 } 4656 if (rsm == NULL) { 4657 goto restore; 4658 } 4659 } else { 4660 /* 4661 * We must find the last segment 4662 * that was acceptable by the client. 4663 */ 4664 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { 4665 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) { 4666 /* Found one */ 4667 break; 4668 } 4669 } 4670 if (rsm == NULL) { 4671 /* None? if so send the first */ 4672 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 4673 if (rsm == NULL) 4674 goto restore; 4675 } 4676 } 4677 if ((rsm->r_end - rsm->r_start) > maxseg) { 4678 /* 4679 * We need to split this the last segment in two. 4680 */ 4681 struct bbr_sendmap *nrsm; 4682 4683 nrsm = bbr_alloc_full_limit(bbr); 4684 if (nrsm == NULL) { 4685 /* 4686 * We can't get memory to split, we can either just 4687 * not split it. Or retransmit the whole piece, lets 4688 * do the large send (BTLP :-) ). 4689 */ 4690 goto go_for_it; 4691 } 4692 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg)); 4693 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); 4694 if (rsm->r_in_tmap) { 4695 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 4696 nrsm->r_in_tmap = 1; 4697 } 4698 rsm->r_flags &= (~BBR_HAS_FIN); 4699 rsm = nrsm; 4700 } 4701 go_for_it: 4702 bbr->r_ctl.rc_tlp_send = rsm; 4703 bbr->rc_tlp_rtx_out = 1; 4704 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) { 4705 bbr->r_ctl.rc_tlp_seg_send_cnt++; 4706 tp->t_rxtshift++; 4707 } else { 4708 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start; 4709 bbr->r_ctl.rc_tlp_seg_send_cnt = 1; 4710 } 4711 send: 4712 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) { 4713 /* 4714 * Can't [re]/transmit a segment we have retranmitted the 4715 * max times. We need the retransmit timer to take over. 4716 */ 4717 restore: 4718 bbr->rc_tlp_new_data = 0; 4719 bbr->r_ctl.rc_tlp_send = NULL; 4720 if (rsm) 4721 rsm->r_flags &= ~BBR_TLP; 4722 BBR_STAT_INC(bbr_tlp_retran_fail); 4723 return (0); 4724 } else if (rsm) { 4725 rsm->r_flags |= BBR_TLP; 4726 } 4727 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) && 4728 (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) { 4729 /* 4730 * We have retransmitted to many times for TLP. Switch to 4731 * the regular RTO timer 4732 */ 4733 goto restore; 4734 } 4735 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TLP); 4736 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP; 4737 return (0); 4738 } 4739 4740 /* 4741 * Delayed ack Timer, here we simply need to setup the 4742 * ACK_NOW flag and remove the DELACK flag. From there 4743 * the output routine will send the ack out. 4744 * 4745 * We only return 1, saying don't proceed, if all timers 4746 * are stopped (destroyed PCB?). 4747 */ 4748 static int 4749 bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4750 { 4751 if (bbr->rc_all_timers_stopped) { 4752 return (1); 4753 } 4754 bbr_log_to_event(bbr, cts, BBR_TO_FRM_DELACK); 4755 tp->t_flags &= ~TF_DELACK; 4756 tp->t_flags |= TF_ACKNOW; 4757 KMOD_TCPSTAT_INC(tcps_delack); 4758 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK; 4759 return (0); 4760 } 4761 4762 /* 4763 * Here we send a KEEP-ALIVE like probe to the 4764 * peer, we do not send data. 4765 * 4766 * We only return 1, saying don't proceed, if all timers 4767 * are stopped (destroyed PCB?). 4768 */ 4769 static int 4770 bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4771 { 4772 struct tcptemp *t_template; 4773 int32_t retval = 1; 4774 4775 if (bbr->rc_all_timers_stopped) { 4776 return (1); 4777 } 4778 if (bbr->rc_in_persist == 0) 4779 return (0); 4780 KASSERT(tp->t_inpcb != NULL, 4781 ("%s: tp %p tp->t_inpcb == NULL", __func__, tp)); 4782 /* 4783 * Persistence timer into zero window. Force a byte to be output, if 4784 * possible. 4785 */ 4786 bbr_log_to_event(bbr, cts, BBR_TO_FRM_PERSIST); 4787 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT; 4788 KMOD_TCPSTAT_INC(tcps_persisttimeo); 4789 /* 4790 * Have we exceeded the user specified progress time? 4791 */ 4792 if (ctf_progress_timeout_check(tp, true)) { 4793 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 4794 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT); 4795 goto out; 4796 } 4797 /* 4798 * Hack: if the peer is dead/unreachable, we do not time out if the 4799 * window is closed. After a full backoff, drop the connection if 4800 * the idle time (no responses to probes) reaches the maximum 4801 * backoff that we would use if retransmitting. 4802 */ 4803 if (tp->t_rxtshift == TCP_MAXRXTSHIFT && 4804 (ticks - tp->t_rcvtime >= tcp_maxpersistidle || 4805 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) { 4806 KMOD_TCPSTAT_INC(tcps_persistdrop); 4807 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX); 4808 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT); 4809 goto out; 4810 } 4811 if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) && 4812 tp->snd_una == tp->snd_max) { 4813 bbr_exit_persist(tp, bbr, cts, __LINE__); 4814 retval = 0; 4815 goto out; 4816 } 4817 /* 4818 * If the user has closed the socket then drop a persisting 4819 * connection after a much reduced timeout. 4820 */ 4821 if (tp->t_state > TCPS_CLOSE_WAIT && 4822 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) { 4823 KMOD_TCPSTAT_INC(tcps_persistdrop); 4824 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX); 4825 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT); 4826 goto out; 4827 } 4828 t_template = tcpip_maketemplate(bbr->rc_inp); 4829 if (t_template) { 4830 tcp_respond(tp, t_template->tt_ipgen, 4831 &t_template->tt_t, (struct mbuf *)NULL, 4832 tp->rcv_nxt, tp->snd_una - 1, 0); 4833 /* This sends an ack */ 4834 if (tp->t_flags & TF_DELACK) 4835 tp->t_flags &= ~TF_DELACK; 4836 free(t_template, M_TEMP); 4837 } 4838 if (tp->t_rxtshift < TCP_MAXRXTSHIFT) 4839 tp->t_rxtshift++; 4840 bbr_start_hpts_timer(bbr, tp, cts, 3, 0, 0); 4841 out: 4842 return (retval); 4843 } 4844 4845 /* 4846 * If a keepalive goes off, we had no other timers 4847 * happening. We always return 1 here since this 4848 * routine either drops the connection or sends 4849 * out a segment with respond. 4850 */ 4851 static int 4852 bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4853 { 4854 struct tcptemp *t_template; 4855 struct inpcb *inp; 4856 4857 if (bbr->rc_all_timers_stopped) { 4858 return (1); 4859 } 4860 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP; 4861 inp = tp->t_inpcb; 4862 bbr_log_to_event(bbr, cts, BBR_TO_FRM_KEEP); 4863 /* 4864 * Keep-alive timer went off; send something or drop connection if 4865 * idle for too long. 4866 */ 4867 KMOD_TCPSTAT_INC(tcps_keeptimeo); 4868 if (tp->t_state < TCPS_ESTABLISHED) 4869 goto dropit; 4870 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && 4871 tp->t_state <= TCPS_CLOSING) { 4872 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp)) 4873 goto dropit; 4874 /* 4875 * Send a packet designed to force a response if the peer is 4876 * up and reachable: either an ACK if the connection is 4877 * still alive, or an RST if the peer has closed the 4878 * connection due to timeout or reboot. Using sequence 4879 * number tp->snd_una-1 causes the transmitted zero-length 4880 * segment to lie outside the receive window; by the 4881 * protocol spec, this requires the correspondent TCP to 4882 * respond. 4883 */ 4884 KMOD_TCPSTAT_INC(tcps_keepprobe); 4885 t_template = tcpip_maketemplate(inp); 4886 if (t_template) { 4887 tcp_respond(tp, t_template->tt_ipgen, 4888 &t_template->tt_t, (struct mbuf *)NULL, 4889 tp->rcv_nxt, tp->snd_una - 1, 0); 4890 free(t_template, M_TEMP); 4891 } 4892 } 4893 bbr_start_hpts_timer(bbr, tp, cts, 4, 0, 0); 4894 return (1); 4895 dropit: 4896 KMOD_TCPSTAT_INC(tcps_keepdrops); 4897 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX); 4898 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT); 4899 return (1); 4900 } 4901 4902 /* 4903 * Retransmit helper function, clear up all the ack 4904 * flags and take care of important book keeping. 4905 */ 4906 static void 4907 bbr_remxt_tmr(struct tcpcb *tp) 4908 { 4909 /* 4910 * The retransmit timer went off, all sack'd blocks must be 4911 * un-acked. 4912 */ 4913 struct bbr_sendmap *rsm, *trsm = NULL; 4914 struct tcp_bbr *bbr; 4915 uint32_t cts, lost; 4916 4917 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 4918 cts = tcp_get_usecs(&bbr->rc_tv); 4919 lost = bbr->r_ctl.rc_lost; 4920 if (bbr->r_state && (bbr->r_state != tp->t_state)) 4921 bbr_set_state(tp, bbr, 0); 4922 4923 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { 4924 if (rsm->r_flags & BBR_ACKED) { 4925 uint32_t old_flags; 4926 4927 rsm->r_dupack = 0; 4928 if (rsm->r_in_tmap == 0) { 4929 /* We must re-add it back to the tlist */ 4930 if (trsm == NULL) { 4931 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 4932 } else { 4933 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext); 4934 } 4935 rsm->r_in_tmap = 1; 4936 } 4937 old_flags = rsm->r_flags; 4938 rsm->r_flags |= BBR_RXT_CLEARED; 4939 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS); 4940 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__); 4941 } else { 4942 if ((tp->t_state < TCPS_ESTABLISHED) && 4943 (rsm->r_start == tp->snd_una)) { 4944 /* 4945 * Special case for TCP FO. Where 4946 * we sent more data beyond the snd_max. 4947 * We don't mark that as lost and stop here. 4948 */ 4949 break; 4950 } 4951 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) { 4952 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; 4953 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; 4954 } 4955 if (bbr_marks_rxt_sack_passed) { 4956 /* 4957 * With this option, we will rack out 4958 * in 1ms increments the rest of the packets. 4959 */ 4960 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST; 4961 rsm->r_flags &= ~BBR_WAS_SACKPASS; 4962 } else { 4963 /* 4964 * With this option we only mark them lost 4965 * and remove all sack'd markings. We will run 4966 * another RXT or a TLP. This will cause 4967 * us to eventually send more based on what 4968 * ack's come in. 4969 */ 4970 rsm->r_flags |= BBR_MARKED_LOST; 4971 rsm->r_flags &= ~BBR_WAS_SACKPASS; 4972 rsm->r_flags &= ~BBR_SACK_PASSED; 4973 } 4974 } 4975 trsm = rsm; 4976 } 4977 bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map); 4978 /* Clear the count (we just un-acked them) */ 4979 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TMR); 4980 bbr->rc_tlp_new_data = 0; 4981 bbr->r_ctl.rc_tlp_seg_send_cnt = 0; 4982 /* zap the behindness on a rxt */ 4983 bbr->r_ctl.rc_hptsi_agg_delay = 0; 4984 bbr->r_agg_early_set = 0; 4985 bbr->r_ctl.rc_agg_early = 0; 4986 bbr->rc_tlp_rtx_out = 0; 4987 bbr->r_ctl.rc_sacked = 0; 4988 bbr->r_ctl.rc_sacklast = NULL; 4989 bbr->r_timer_override = 1; 4990 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost)); 4991 } 4992 4993 /* 4994 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise 4995 * we will setup to retransmit the lowest seq number outstanding. 4996 */ 4997 static int 4998 bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 4999 { 5000 int32_t rexmt; 5001 int32_t retval = 0; 5002 bool isipv6; 5003 5004 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT; 5005 if (bbr->rc_all_timers_stopped) { 5006 return (1); 5007 } 5008 if (TCPS_HAVEESTABLISHED(tp->t_state) && 5009 (tp->snd_una == tp->snd_max)) { 5010 /* Nothing outstanding .. nothing to do */ 5011 return (0); 5012 } 5013 /* 5014 * Retransmission timer went off. Message has not been acked within 5015 * retransmit interval. Back off to a longer retransmit interval 5016 * and retransmit one segment. 5017 */ 5018 if (ctf_progress_timeout_check(tp, true)) { 5019 retval = 1; 5020 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 5021 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT); 5022 goto out; 5023 } 5024 bbr_remxt_tmr(tp); 5025 if ((bbr->r_ctl.rc_resend == NULL) || 5026 ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) { 5027 /* 5028 * If the rwnd collapsed on 5029 * the one we are retransmitting 5030 * it does not count against the 5031 * rxt count. 5032 */ 5033 tp->t_rxtshift++; 5034 } 5035 if (tp->t_rxtshift > TCP_MAXRXTSHIFT) { 5036 tp->t_rxtshift = TCP_MAXRXTSHIFT; 5037 KMOD_TCPSTAT_INC(tcps_timeoutdrop); 5038 retval = 1; 5039 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN); 5040 tcp_set_inp_to_drop(bbr->rc_inp, 5041 (tp->t_softerror ? (uint16_t) tp->t_softerror : ETIMEDOUT)); 5042 goto out; 5043 } 5044 if (tp->t_state == TCPS_SYN_SENT) { 5045 /* 5046 * If the SYN was retransmitted, indicate CWND to be limited 5047 * to 1 segment in cc_conn_init(). 5048 */ 5049 tp->snd_cwnd = 1; 5050 } else if (tp->t_rxtshift == 1) { 5051 /* 5052 * first retransmit; record ssthresh and cwnd so they can be 5053 * recovered if this turns out to be a "bad" retransmit. A 5054 * retransmit is considered "bad" if an ACK for this segment 5055 * is received within RTT/2 interval; the assumption here is 5056 * that the ACK was already in flight. See "On Estimating 5057 * End-to-End Network Path Properties" by Allman and Paxson 5058 * for more details. 5059 */ 5060 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; 5061 if (!IN_RECOVERY(tp->t_flags)) { 5062 tp->snd_cwnd_prev = tp->snd_cwnd; 5063 tp->snd_ssthresh_prev = tp->snd_ssthresh; 5064 tp->snd_recover_prev = tp->snd_recover; 5065 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1)); 5066 tp->t_flags |= TF_PREVVALID; 5067 } else { 5068 tp->t_flags &= ~TF_PREVVALID; 5069 } 5070 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; 5071 } else { 5072 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options; 5073 tp->t_flags &= ~TF_PREVVALID; 5074 } 5075 KMOD_TCPSTAT_INC(tcps_rexmttimeo); 5076 if ((tp->t_state == TCPS_SYN_SENT) || 5077 (tp->t_state == TCPS_SYN_RECEIVED)) 5078 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift]; 5079 else 5080 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift]; 5081 TCPT_RANGESET(tp->t_rxtcur, rexmt, 5082 MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), 5083 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000)); 5084 /* 5085 * We enter the path for PLMTUD if connection is established or, if 5086 * connection is FIN_WAIT_1 status, reason for the last is that if 5087 * amount of data we send is very small, we could send it in couple 5088 * of packets and process straight to FIN. In that case we won't 5089 * catch ESTABLISHED state. 5090 */ 5091 #ifdef INET6 5092 isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) ? true : false; 5093 #else 5094 isipv6 = false; 5095 #endif 5096 if (((V_tcp_pmtud_blackhole_detect == 1) || 5097 (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) || 5098 (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) && 5099 ((tp->t_state == TCPS_ESTABLISHED) || 5100 (tp->t_state == TCPS_FIN_WAIT_1))) { 5101 5102 /* 5103 * Idea here is that at each stage of mtu probe (usually, 5104 * 1448 -> 1188 -> 524) should be given 2 chances to recover 5105 * before further clamping down. 'tp->t_rxtshift % 2 == 0' 5106 * should take care of that. 5107 */ 5108 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) == 5109 (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) && 5110 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 && 5111 tp->t_rxtshift % 2 == 0)) { 5112 /* 5113 * Enter Path MTU Black-hole Detection mechanism: - 5114 * Disable Path MTU Discovery (IP "DF" bit). - 5115 * Reduce MTU to lower value than what we negotiated 5116 * with peer. 5117 */ 5118 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) { 5119 /* 5120 * Record that we may have found a black 5121 * hole. 5122 */ 5123 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE; 5124 /* Keep track of previous MSS. */ 5125 tp->t_pmtud_saved_maxseg = tp->t_maxseg; 5126 } 5127 /* 5128 * Reduce the MSS to blackhole value or to the 5129 * default in an attempt to retransmit. 5130 */ 5131 #ifdef INET6 5132 isipv6 = bbr->r_is_v6; 5133 if (isipv6 && 5134 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) { 5135 /* Use the sysctl tuneable blackhole MSS. */ 5136 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss; 5137 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated); 5138 } else if (isipv6) { 5139 /* Use the default MSS. */ 5140 tp->t_maxseg = V_tcp_v6mssdflt; 5141 /* 5142 * Disable Path MTU Discovery when we switch 5143 * to minmss. 5144 */ 5145 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 5146 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss); 5147 } 5148 #endif 5149 #if defined(INET6) && defined(INET) 5150 else 5151 #endif 5152 #ifdef INET 5153 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) { 5154 /* Use the sysctl tuneable blackhole MSS. */ 5155 tp->t_maxseg = V_tcp_pmtud_blackhole_mss; 5156 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated); 5157 } else { 5158 /* Use the default MSS. */ 5159 tp->t_maxseg = V_tcp_mssdflt; 5160 /* 5161 * Disable Path MTU Discovery when we switch 5162 * to minmss. 5163 */ 5164 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 5165 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss); 5166 } 5167 #endif 5168 } else { 5169 /* 5170 * If further retransmissions are still unsuccessful 5171 * with a lowered MTU, maybe this isn't a blackhole 5172 * and we restore the previous MSS and blackhole 5173 * detection flags. The limit '6' is determined by 5174 * giving each probe stage (1448, 1188, 524) 2 5175 * chances to recover. 5176 */ 5177 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) && 5178 (tp->t_rxtshift >= 6)) { 5179 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 5180 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE; 5181 tp->t_maxseg = tp->t_pmtud_saved_maxseg; 5182 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed); 5183 } 5184 } 5185 } 5186 /* 5187 * Disable RFC1323 and SACK if we haven't got any response to our 5188 * third SYN to work-around some broken terminal servers (most of 5189 * which have hopefully been retired) that have bad VJ header 5190 * compression code which trashes TCP segments containing 5191 * unknown-to-them TCP options. 5192 */ 5193 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) && 5194 (tp->t_rxtshift == 3)) 5195 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT); 5196 /* 5197 * If we backed off this far, our srtt estimate is probably bogus. 5198 * Clobber it so we'll take the next rtt measurement as our srtt; 5199 * move the current srtt into rttvar to keep the current retransmit 5200 * times until then. 5201 */ 5202 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) { 5203 #ifdef INET6 5204 if (bbr->r_is_v6) 5205 in6_losing(tp->t_inpcb); 5206 else 5207 #endif 5208 in_losing(tp->t_inpcb); 5209 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT); 5210 tp->t_srtt = 0; 5211 } 5212 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); 5213 tp->snd_recover = tp->snd_max; 5214 tp->t_flags |= TF_ACKNOW; 5215 tp->t_rtttime = 0; 5216 out: 5217 return (retval); 5218 } 5219 5220 static int 5221 bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling) 5222 { 5223 int32_t ret = 0; 5224 int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK); 5225 5226 if (timers == 0) { 5227 return (0); 5228 } 5229 if (tp->t_state == TCPS_LISTEN) { 5230 /* no timers on listen sockets */ 5231 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) 5232 return (0); 5233 return (1); 5234 } 5235 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) { 5236 uint32_t left; 5237 5238 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { 5239 ret = -1; 5240 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling); 5241 return (0); 5242 } 5243 if (hpts_calling == 0) { 5244 ret = -2; 5245 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling); 5246 return (0); 5247 } 5248 /* 5249 * Ok our timer went off early and we are not paced false 5250 * alarm, go back to sleep. 5251 */ 5252 left = bbr->r_ctl.rc_timer_exp - cts; 5253 ret = -3; 5254 bbr_log_to_processing(bbr, cts, ret, left, hpts_calling); 5255 tcp_hpts_insert(tp->t_inpcb, HPTS_USEC_TO_SLOTS(left)); 5256 return (1); 5257 } 5258 bbr->rc_tmr_stopped = 0; 5259 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK; 5260 if (timers & PACE_TMR_DELACK) { 5261 ret = bbr_timeout_delack(tp, bbr, cts); 5262 } else if (timers & PACE_TMR_PERSIT) { 5263 ret = bbr_timeout_persist(tp, bbr, cts); 5264 } else if (timers & PACE_TMR_RACK) { 5265 bbr->r_ctl.rc_tlp_rxt_last_time = cts; 5266 ret = bbr_timeout_rack(tp, bbr, cts); 5267 } else if (timers & PACE_TMR_TLP) { 5268 bbr->r_ctl.rc_tlp_rxt_last_time = cts; 5269 ret = bbr_timeout_tlp(tp, bbr, cts); 5270 } else if (timers & PACE_TMR_RXT) { 5271 bbr->r_ctl.rc_tlp_rxt_last_time = cts; 5272 ret = bbr_timeout_rxt(tp, bbr, cts); 5273 } else if (timers & PACE_TMR_KEEP) { 5274 ret = bbr_timeout_keepalive(tp, bbr, cts); 5275 } 5276 bbr_log_to_processing(bbr, cts, ret, timers, hpts_calling); 5277 return (ret); 5278 } 5279 5280 static void 5281 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts) 5282 { 5283 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { 5284 uint8_t hpts_removed = 0; 5285 5286 if (bbr->rc_inp->inp_in_hpts && 5287 (bbr->rc_timer_first == 1)) { 5288 /* 5289 * If we are canceling timer's when we have the 5290 * timer ahead of the output being paced. We also 5291 * must remove ourselves from the hpts. 5292 */ 5293 hpts_removed = 1; 5294 tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); 5295 if (bbr->r_ctl.rc_last_delay_val) { 5296 /* Update the last hptsi delay too */ 5297 uint32_t time_since_send; 5298 5299 if (TSTMP_GT(cts, bbr->rc_pacer_started)) 5300 time_since_send = cts - bbr->rc_pacer_started; 5301 else 5302 time_since_send = 0; 5303 if (bbr->r_ctl.rc_last_delay_val > time_since_send) { 5304 /* Cut down our slot time */ 5305 bbr->r_ctl.rc_last_delay_val -= time_since_send; 5306 } else { 5307 bbr->r_ctl.rc_last_delay_val = 0; 5308 } 5309 bbr->rc_pacer_started = cts; 5310 } 5311 } 5312 bbr->rc_timer_first = 0; 5313 bbr_log_to_cancel(bbr, line, cts, hpts_removed); 5314 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK; 5315 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK); 5316 } 5317 } 5318 5319 static void 5320 bbr_timer_stop(struct tcpcb *tp, uint32_t timer_type) 5321 { 5322 struct tcp_bbr *bbr; 5323 5324 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 5325 bbr->rc_all_timers_stopped = 1; 5326 return; 5327 } 5328 5329 /* 5330 * stop all timers always returning 0. 5331 */ 5332 static int 5333 bbr_stopall(struct tcpcb *tp) 5334 { 5335 return (0); 5336 } 5337 5338 static void 5339 bbr_timer_activate(struct tcpcb *tp, uint32_t timer_type, uint32_t delta) 5340 { 5341 return; 5342 } 5343 5344 /* 5345 * return true if a bbr timer (rack or tlp) is active. 5346 */ 5347 static int 5348 bbr_timer_active(struct tcpcb *tp, uint32_t timer_type) 5349 { 5350 return (0); 5351 } 5352 5353 static uint32_t 5354 bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts) 5355 { 5356 struct bbr_sendmap *rsm; 5357 5358 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); 5359 if ((rsm == NULL) || (u_rsm == rsm)) 5360 return (cts); 5361 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]); 5362 } 5363 5364 static void 5365 bbr_update_rsm(struct tcpcb *tp, struct tcp_bbr *bbr, 5366 struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time) 5367 { 5368 int32_t idx; 5369 5370 rsm->r_rtr_cnt++; 5371 rsm->r_dupack = 0; 5372 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) { 5373 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS; 5374 rsm->r_flags |= BBR_OVERMAX; 5375 } 5376 if (rsm->r_flags & BBR_RWND_COLLAPSED) { 5377 /* Take off the collapsed flag at rxt */ 5378 rsm->r_flags &= ~BBR_RWND_COLLAPSED; 5379 } 5380 if (rsm->r_flags & BBR_MARKED_LOST) { 5381 /* We have retransmitted, its no longer lost */ 5382 rsm->r_flags &= ~BBR_MARKED_LOST; 5383 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; 5384 } 5385 if (rsm->r_flags & BBR_RXT_CLEARED) { 5386 /* 5387 * We hit a RXT timer on it and 5388 * we cleared the "acked" flag. 5389 * We now have it going back into 5390 * flight, we can remove the cleared 5391 * flag and possibly do accounting on 5392 * this piece. 5393 */ 5394 rsm->r_flags &= ~BBR_RXT_CLEARED; 5395 } 5396 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) { 5397 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start); 5398 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start); 5399 } 5400 idx = rsm->r_rtr_cnt - 1; 5401 rsm->r_tim_lastsent[idx] = cts; 5402 rsm->r_pacing_delay = pacing_time; 5403 rsm->r_delivered = bbr->r_ctl.rc_delivered; 5404 rsm->r_ts_valid = bbr->rc_ts_valid; 5405 if (bbr->rc_ts_valid) 5406 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts; 5407 if (bbr->r_ctl.r_app_limited_until) 5408 rsm->r_app_limited = 1; 5409 else 5410 rsm->r_app_limited = 0; 5411 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) 5412 rsm->r_bbr_state = bbr_state_val(bbr); 5413 else 5414 rsm->r_bbr_state = 8; 5415 if (rsm->r_flags & BBR_ACKED) { 5416 /* Problably MTU discovery messing with us */ 5417 uint32_t old_flags; 5418 5419 old_flags = rsm->r_flags; 5420 rsm->r_flags &= ~BBR_ACKED; 5421 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__); 5422 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); 5423 if (bbr->r_ctl.rc_sacked == 0) 5424 bbr->r_ctl.rc_sacklast = NULL; 5425 } 5426 if (rsm->r_in_tmap) { 5427 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 5428 } 5429 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 5430 rsm->r_in_tmap = 1; 5431 if (rsm->r_flags & BBR_SACK_PASSED) { 5432 /* We have retransmitted due to the SACK pass */ 5433 rsm->r_flags &= ~BBR_SACK_PASSED; 5434 rsm->r_flags |= BBR_WAS_SACKPASS; 5435 } 5436 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); 5437 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp, 5438 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 5439 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next); 5440 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) { 5441 rsm->r_is_gain = 1; 5442 rsm->r_is_drain = 0; 5443 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) { 5444 rsm->r_is_drain = 1; 5445 rsm->r_is_gain = 0; 5446 } else { 5447 rsm->r_is_drain = 0; 5448 rsm->r_is_gain = 0; 5449 } 5450 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */ 5451 } 5452 5453 /* 5454 * Returns 0, or the sequence where we stopped 5455 * updating. We also update the lenp to be the amount 5456 * of data left. 5457 */ 5458 5459 static uint32_t 5460 bbr_update_entry(struct tcpcb *tp, struct tcp_bbr *bbr, 5461 struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time) 5462 { 5463 /* 5464 * We (re-)transmitted starting at rsm->r_start for some length 5465 * (possibly less than r_end. 5466 */ 5467 struct bbr_sendmap *nrsm; 5468 uint32_t c_end; 5469 int32_t len; 5470 5471 len = *lenp; 5472 c_end = rsm->r_start + len; 5473 if (SEQ_GEQ(c_end, rsm->r_end)) { 5474 /* 5475 * We retransmitted the whole piece or more than the whole 5476 * slopping into the next rsm. 5477 */ 5478 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time); 5479 if (c_end == rsm->r_end) { 5480 *lenp = 0; 5481 return (0); 5482 } else { 5483 int32_t act_len; 5484 5485 /* Hangs over the end return whats left */ 5486 act_len = rsm->r_end - rsm->r_start; 5487 *lenp = (len - act_len); 5488 return (rsm->r_end); 5489 } 5490 /* We don't get out of this block. */ 5491 } 5492 /* 5493 * Here we retransmitted less than the whole thing which means we 5494 * have to split this into what was transmitted and what was not. 5495 */ 5496 nrsm = bbr_alloc_full_limit(bbr); 5497 if (nrsm == NULL) { 5498 *lenp = 0; 5499 return (0); 5500 } 5501 /* 5502 * So here we are going to take the original rsm and make it what we 5503 * retransmitted. nrsm will be the tail portion we did not 5504 * retransmit. For example say the chunk was 1, 11 (10 bytes). And 5505 * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to 5506 * 1, 6 and the new piece will be 6, 11. 5507 */ 5508 bbr_clone_rsm(bbr, nrsm, rsm, c_end); 5509 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); 5510 nrsm->r_dupack = 0; 5511 if (rsm->r_in_tmap) { 5512 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 5513 nrsm->r_in_tmap = 1; 5514 } 5515 rsm->r_flags &= (~BBR_HAS_FIN); 5516 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time); 5517 *lenp = 0; 5518 return (0); 5519 } 5520 5521 static uint64_t 5522 bbr_get_hardware_rate(struct tcp_bbr *bbr) 5523 { 5524 uint64_t bw; 5525 5526 bw = bbr_get_bw(bbr); 5527 bw *= (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN]; 5528 bw /= (uint64_t)BBR_UNIT; 5529 return(bw); 5530 } 5531 5532 static void 5533 bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts, 5534 uint64_t act_rate, uint64_t rate_wanted) 5535 { 5536 /* 5537 * We could not get a full gains worth 5538 * of rate. 5539 */ 5540 if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) { 5541 /* we can't even get the real rate */ 5542 uint64_t red; 5543 5544 bbr->skip_gain = 1; 5545 bbr->gain_is_limited = 0; 5546 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate; 5547 if (red) 5548 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts); 5549 } else { 5550 /* We can use a lower gain */ 5551 bbr->skip_gain = 0; 5552 bbr->gain_is_limited = 1; 5553 } 5554 } 5555 5556 static void 5557 bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts) 5558 { 5559 const struct tcp_hwrate_limit_table *nrte; 5560 int error, rate = -1; 5561 5562 if (bbr->r_ctl.crte == NULL) 5563 return; 5564 if ((bbr->rc_inp->inp_route.ro_nh == NULL) || 5565 (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) { 5566 /* Lost our routes? */ 5567 /* Clear the way for a re-attempt */ 5568 bbr->bbr_attempt_hdwr_pace = 0; 5569 lost_rate: 5570 bbr->gain_is_limited = 0; 5571 bbr->skip_gain = 0; 5572 bbr->bbr_hdrw_pacing = 0; 5573 counter_u64_add(bbr_flows_whdwr_pacing, -1); 5574 counter_u64_add(bbr_flows_nohdwr_pacing, 1); 5575 tcp_bbr_tso_size_check(bbr, cts); 5576 return; 5577 } 5578 rate = bbr_get_hardware_rate(bbr); 5579 nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte, 5580 bbr->rc_tp, 5581 bbr->rc_inp->inp_route.ro_nh->nh_ifp, 5582 rate, 5583 (RS_PACING_GEQ|RS_PACING_SUB_OK), 5584 &error); 5585 if (nrte == NULL) { 5586 goto lost_rate; 5587 } 5588 if (nrte != bbr->r_ctl.crte) { 5589 bbr->r_ctl.crte = nrte; 5590 if (error == 0) { 5591 BBR_STAT_INC(bbr_hdwr_rl_mod_ok); 5592 if (bbr->r_ctl.crte->rate < rate) { 5593 /* We have a problem */ 5594 bbr_setup_less_of_rate(bbr, cts, 5595 bbr->r_ctl.crte->rate, rate); 5596 } else { 5597 /* We are good */ 5598 bbr->gain_is_limited = 0; 5599 bbr->skip_gain = 0; 5600 } 5601 } else { 5602 /* A failure should release the tag */ 5603 BBR_STAT_INC(bbr_hdwr_rl_mod_fail); 5604 bbr->gain_is_limited = 0; 5605 bbr->skip_gain = 0; 5606 bbr->bbr_hdrw_pacing = 0; 5607 } 5608 bbr_type_log_hdwr_pacing(bbr, 5609 bbr->r_ctl.crte->ptbl->rs_ifp, 5610 rate, 5611 ((bbr->r_ctl.crte == NULL) ? 0 : bbr->r_ctl.crte->rate), 5612 __LINE__, 5613 cts, 5614 error); 5615 } 5616 } 5617 5618 static void 5619 bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts) 5620 { 5621 /* 5622 * If we have hardware pacing support 5623 * we need to factor that in for our 5624 * TSO size. 5625 */ 5626 const struct tcp_hwrate_limit_table *rlp; 5627 uint32_t cur_delay, seg_sz, maxseg, new_tso, delta, hdwr_delay; 5628 5629 if ((bbr->bbr_hdrw_pacing == 0) || 5630 (IN_RECOVERY(bbr->rc_tp->t_flags)) || 5631 (bbr->r_ctl.crte == NULL)) 5632 return; 5633 if (bbr->hw_pacing_set == 0) { 5634 /* Not yet by the hdwr pacing count delay */ 5635 return; 5636 } 5637 if (bbr_hdwr_pace_adjust == 0) { 5638 /* No adjustment */ 5639 return; 5640 } 5641 rlp = bbr->r_ctl.crte; 5642 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) 5643 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; 5644 else 5645 maxseg = BBR_MIN_SEG - bbr->rc_last_options; 5646 /* 5647 * So lets first get the 5648 * time we will take between 5649 * TSO sized sends currently without 5650 * hardware help. 5651 */ 5652 cur_delay = bbr_get_pacing_delay(bbr, BBR_UNIT, 5653 bbr->r_ctl.rc_pace_max_segs, cts, 1); 5654 hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg; 5655 hdwr_delay *= rlp->time_between; 5656 if (cur_delay > hdwr_delay) 5657 delta = cur_delay - hdwr_delay; 5658 else 5659 delta = 0; 5660 bbr_log_type_tsosize(bbr, cts, delta, cur_delay, hdwr_delay, 5661 (bbr->r_ctl.rc_pace_max_segs / maxseg), 5662 1); 5663 if (delta && 5664 (delta < (max(rlp->time_between, 5665 bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) { 5666 /* 5667 * Now lets divide by the pacing 5668 * time between each segment the 5669 * hardware sends rounding up and 5670 * derive a bytes from that. We multiply 5671 * that by bbr_hdwr_pace_adjust to get 5672 * more bang for our buck. 5673 * 5674 * The goal is to have the software pacer 5675 * waiting no more than an additional 5676 * pacing delay if we can (without the 5677 * compensation i.e. x bbr_hdwr_pace_adjust). 5678 */ 5679 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between), 5680 (bbr->r_ctl.rc_pace_max_segs/maxseg)); 5681 seg_sz *= bbr_hdwr_pace_adjust; 5682 if (bbr_hdwr_pace_floor && 5683 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) { 5684 /* Currently hardware paces 5685 * out rs_min_seg segments at a time. 5686 * We need to make sure we always send at least 5687 * a full burst of bbr_hdwr_pace_floor down. 5688 */ 5689 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg; 5690 } 5691 seg_sz *= maxseg; 5692 } else if (delta == 0) { 5693 /* 5694 * The highest pacing rate is 5695 * above our b/w gained. This means 5696 * we probably are going quite fast at 5697 * the hardware highest rate. Lets just multiply 5698 * the calculated TSO size by the 5699 * multiplier factor (its probably 5700 * 4 segments in the default config for 5701 * mlx). 5702 */ 5703 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust; 5704 if (bbr_hdwr_pace_floor && 5705 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) { 5706 /* Currently hardware paces 5707 * out rs_min_seg segments at a time. 5708 * We need to make sure we always send at least 5709 * a full burst of bbr_hdwr_pace_floor down. 5710 */ 5711 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg; 5712 } 5713 } else { 5714 /* 5715 * The pacing time difference is so 5716 * big that the hardware will 5717 * pace out more rapidly then we 5718 * really want and then we 5719 * will have a long delay. Lets just keep 5720 * the same TSO size so its as if 5721 * we were not using hdwr pacing (we 5722 * just gain a bit of spacing from the 5723 * hardware if seg_sz > 1). 5724 */ 5725 seg_sz = bbr->r_ctl.rc_pace_max_segs; 5726 } 5727 if (seg_sz > bbr->r_ctl.rc_pace_max_segs) 5728 new_tso = seg_sz; 5729 else 5730 new_tso = bbr->r_ctl.rc_pace_max_segs; 5731 if (new_tso >= (PACE_MAX_IP_BYTES-maxseg)) 5732 new_tso = PACE_MAX_IP_BYTES - maxseg; 5733 5734 if (new_tso != bbr->r_ctl.rc_pace_max_segs) { 5735 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0); 5736 bbr->r_ctl.rc_pace_max_segs = new_tso; 5737 } 5738 } 5739 5740 static void 5741 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts) 5742 { 5743 uint64_t bw; 5744 uint32_t old_tso = 0, new_tso; 5745 uint32_t maxseg, bytes; 5746 uint32_t tls_seg=0; 5747 /* 5748 * Google/linux uses the following algorithm to determine 5749 * the TSO size based on the b/w of the link (from Neal Cardwell email 9/27/18): 5750 * 5751 * bytes = bw_in_bytes_per_second / 1000 5752 * bytes = min(bytes, 64k) 5753 * tso_segs = bytes / MSS 5754 * if (bw < 1.2Mbs) 5755 * min_tso_segs = 1 5756 * else 5757 * min_tso_segs = 2 5758 * tso_segs = max(tso_segs, min_tso_segs) 5759 * 5760 * * Note apply a device specific limit (we apply this in the 5761 * tcp_m_copym). 5762 * Note that before the initial measurement is made google bursts out 5763 * a full iwnd just like new-reno/cubic. 5764 * 5765 * We do not use this algorithm. Instead we 5766 * use a two phased approach: 5767 * 5768 * if ( bw <= per-tcb-cross-over) 5769 * goal_tso = calculate how much with this bw we 5770 * can send in goal-time seconds. 5771 * if (goal_tso > mss) 5772 * seg = goal_tso / mss 5773 * tso = seg * mss 5774 * else 5775 * tso = mss 5776 * if (tso > per-tcb-max) 5777 * tso = per-tcb-max 5778 * else if ( bw > 512Mbps) 5779 * tso = max-tso (64k/mss) 5780 * else 5781 * goal_tso = bw / per-tcb-divsor 5782 * seg = (goal_tso + mss-1)/mss 5783 * tso = seg * mss 5784 * 5785 * if (tso < per-tcb-floor) 5786 * tso = per-tcb-floor 5787 * if (tso > per-tcb-utter_max) 5788 * tso = per-tcb-utter_max 5789 * 5790 * Note the default per-tcb-divisor is 1000 (same as google). 5791 * the goal cross over is 30Mbps however. To recreate googles 5792 * algorithm you need to set: 5793 * 5794 * cross-over = 23,168,000 bps 5795 * goal-time = 18000 5796 * per-tcb-max = 2 5797 * per-tcb-divisor = 1000 5798 * per-tcb-floor = 1 5799 * 5800 * This will get you "google bbr" behavior with respect to tso size. 5801 * 5802 * Note we do set anything TSO size until we are past the initial 5803 * window. Before that we gnerally use either a single MSS 5804 * or we use the full IW size (so we burst a IW at a time) 5805 * Also note that Hardware-TLS is special and does alternate 5806 * things to minimize PCI Bus Bandwidth use. 5807 */ 5808 5809 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) { 5810 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; 5811 } else { 5812 maxseg = BBR_MIN_SEG - bbr->rc_last_options; 5813 } 5814 #ifdef KERN_TLS 5815 if (bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) { 5816 tls_seg = ctf_get_opt_tls_size(bbr->rc_inp->inp_socket, bbr->rc_tp->snd_wnd); 5817 bbr->r_ctl.rc_pace_min_segs = (tls_seg + bbr->rc_last_options); 5818 } 5819 #endif 5820 old_tso = bbr->r_ctl.rc_pace_max_segs; 5821 if (bbr->rc_past_init_win == 0) { 5822 /* 5823 * Not enough data has been acknowledged to make a 5824 * judgement unless we are hardware TLS. Set up 5825 * the initial TSO based on if we are sending a 5826 * full IW at once or not. 5827 */ 5828 if (bbr->rc_use_google) 5829 bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2); 5830 else if (bbr->bbr_init_win_cheat) 5831 bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp); 5832 else 5833 bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options; 5834 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg) 5835 bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg; 5836 #ifdef KERN_TLS 5837 if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) && tls_seg) { 5838 /* 5839 * For hardware TLS we set our min to the tls_seg size. 5840 */ 5841 bbr->r_ctl.rc_pace_max_segs = tls_seg; 5842 bbr->r_ctl.rc_pace_min_segs = tls_seg + bbr->rc_last_options; 5843 } 5844 #endif 5845 if (bbr->r_ctl.rc_pace_max_segs == 0) { 5846 bbr->r_ctl.rc_pace_max_segs = maxseg; 5847 } 5848 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0); 5849 #ifdef KERN_TLS 5850 if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) == 0) 5851 #endif 5852 bbr_adjust_for_hw_pacing(bbr, cts); 5853 return; 5854 } 5855 /** 5856 * Now lets set the TSO goal based on our delivery rate in 5857 * bytes per second. Note we only do this if 5858 * we have acked at least the initial cwnd worth of data. 5859 */ 5860 bw = bbr_get_bw(bbr); 5861 if (IN_RECOVERY(bbr->rc_tp->t_flags) && 5862 (bbr->rc_use_google == 0)) { 5863 /* We clamp to one MSS in recovery */ 5864 new_tso = maxseg; 5865 } else if (bbr->rc_use_google) { 5866 int min_tso_segs; 5867 5868 /* Google considers the gain too */ 5869 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) { 5870 bw *= bbr->r_ctl.rc_bbr_hptsi_gain; 5871 bw /= BBR_UNIT; 5872 } 5873 bytes = bw / 1024; 5874 if (bytes > (64 * 1024)) 5875 bytes = 64 * 1024; 5876 new_tso = bytes / maxseg; 5877 if (bw < ONE_POINT_TWO_MEG) 5878 min_tso_segs = 1; 5879 else 5880 min_tso_segs = 2; 5881 if (new_tso < min_tso_segs) 5882 new_tso = min_tso_segs; 5883 new_tso *= maxseg; 5884 } else if (bbr->rc_no_pacing) { 5885 new_tso = (PACE_MAX_IP_BYTES / maxseg) * maxseg; 5886 } else if (bw <= bbr->r_ctl.bbr_cross_over) { 5887 /* 5888 * Calculate the worse case b/w TSO if we are inserting no 5889 * more than a delay_target number of TSO's. 5890 */ 5891 uint32_t tso_len, min_tso; 5892 5893 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw); 5894 if (tso_len > maxseg) { 5895 new_tso = tso_len / maxseg; 5896 if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max) 5897 new_tso = bbr->r_ctl.bbr_hptsi_segments_max; 5898 new_tso *= maxseg; 5899 } else { 5900 /* 5901 * less than a full sized frame yikes.. long rtt or 5902 * low bw? 5903 */ 5904 min_tso = bbr_minseg(bbr); 5905 if ((tso_len > min_tso) && (bbr_all_get_min == 0)) 5906 new_tso = rounddown(tso_len, min_tso); 5907 else 5908 new_tso = min_tso; 5909 } 5910 } else if (bw > FIVETWELVE_MBPS) { 5911 /* 5912 * This guy is so fast b/w wise that we can TSO as large as 5913 * possible of segments that the NIC will allow. 5914 */ 5915 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg); 5916 } else { 5917 /* 5918 * This formula is based on attempting to send a segment or 5919 * more every bbr_hptsi_per_second. The default is 1000 5920 * which means you are targeting what you can send every 1ms 5921 * based on the peers bw. 5922 * 5923 * If the number drops to say 500, then you are looking more 5924 * at 2ms and you will raise how much we send in a single 5925 * TSO thus saving CPU (less bbr_output_wtime() calls). The 5926 * trade off of course is you will send more at once and 5927 * thus tend to clump up the sends into larger "bursts" 5928 * building a queue. 5929 */ 5930 bw /= bbr->r_ctl.bbr_hptsi_per_second; 5931 new_tso = roundup(bw, (uint64_t)maxseg); 5932 /* 5933 * Gate the floor to match what our lower than 48Mbps 5934 * algorithm does. The ceiling (bbr_hptsi_segments_max) thus 5935 * becomes the floor for this calculation. 5936 */ 5937 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg)) 5938 new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg); 5939 } 5940 if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor))) 5941 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor; 5942 if (new_tso > PACE_MAX_IP_BYTES) 5943 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg); 5944 /* Enforce an utter maximum if we are not HW-TLS */ 5945 #ifdef KERN_TLS 5946 if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) == 0) 5947 #endif 5948 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) { 5949 new_tso = bbr->r_ctl.bbr_utter_max * maxseg; 5950 } 5951 #ifdef KERN_TLS 5952 if (tls_seg) { 5953 /* 5954 * Lets move the output size 5955 * up to 1 or more TLS record sizes. 5956 */ 5957 uint32_t temp; 5958 5959 temp = roundup(new_tso, tls_seg); 5960 new_tso = temp; 5961 /* Back down if needed to under a full frame */ 5962 while (new_tso > PACE_MAX_IP_BYTES) 5963 new_tso -= tls_seg; 5964 } 5965 #endif 5966 if (old_tso != new_tso) { 5967 /* Only log changes */ 5968 bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0); 5969 bbr->r_ctl.rc_pace_max_segs = new_tso; 5970 } 5971 #ifdef KERN_TLS 5972 if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) && 5973 tls_seg) { 5974 bbr->r_ctl.rc_pace_min_segs = tls_seg + bbr->rc_last_options; 5975 } else 5976 #endif 5977 /* We have hardware pacing and not hardware TLS! */ 5978 bbr_adjust_for_hw_pacing(bbr, cts); 5979 } 5980 5981 static void 5982 bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len, 5983 uint32_t seq_out, uint8_t th_flags, int32_t err, uint32_t cts, 5984 struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc, 5985 struct sockbuf *sb) 5986 { 5987 5988 struct bbr_sendmap *rsm, *nrsm; 5989 register uint32_t snd_max, snd_una; 5990 uint32_t pacing_time; 5991 /* 5992 * Add to the RACK log of packets in flight or retransmitted. If 5993 * there is a TS option we will use the TS echoed, if not we will 5994 * grab a TS. 5995 * 5996 * Retransmissions will increment the count and move the ts to its 5997 * proper place. Note that if options do not include TS's then we 5998 * won't be able to effectively use the ACK for an RTT on a retran. 5999 * 6000 * Notes about r_start and r_end. Lets consider a send starting at 6001 * sequence 1 for 10 bytes. In such an example the r_start would be 6002 * 1 (starting sequence) but the r_end would be r_start+len i.e. 11. 6003 * This means that r_end is actually the first sequence for the next 6004 * slot (11). 6005 * 6006 */ 6007 INP_WLOCK_ASSERT(tp->t_inpcb); 6008 if (err) { 6009 /* 6010 * We don't log errors -- we could but snd_max does not 6011 * advance in this case either. 6012 */ 6013 return; 6014 } 6015 if (th_flags & TH_RST) { 6016 /* 6017 * We don't log resets and we return immediately from 6018 * sending 6019 */ 6020 *abandon = 1; 6021 return; 6022 } 6023 snd_una = tp->snd_una; 6024 if (th_flags & (TH_SYN | TH_FIN) && (hintrsm == NULL)) { 6025 /* 6026 * The call to bbr_log_output is made before bumping 6027 * snd_max. This means we can record one extra byte on a SYN 6028 * or FIN if seq_out is adding more on and a FIN is present 6029 * (and we are not resending). 6030 */ 6031 if (th_flags & TH_SYN) 6032 len++; 6033 if (th_flags & TH_FIN) 6034 len++; 6035 } 6036 if (SEQ_LEQ((seq_out + len), snd_una)) { 6037 /* Are sending an old segment to induce an ack (keep-alive)? */ 6038 return; 6039 } 6040 if (SEQ_LT(seq_out, snd_una)) { 6041 /* huh? should we panic? */ 6042 uint32_t end; 6043 6044 end = seq_out + len; 6045 seq_out = snd_una; 6046 len = end - seq_out; 6047 } 6048 snd_max = tp->snd_max; 6049 if (len == 0) { 6050 /* We don't log zero window probes */ 6051 return; 6052 } 6053 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1); 6054 /* First question is it a retransmission? */ 6055 if (seq_out == snd_max) { 6056 again: 6057 rsm = bbr_alloc(bbr); 6058 if (rsm == NULL) { 6059 return; 6060 } 6061 rsm->r_flags = 0; 6062 if (th_flags & TH_SYN) 6063 rsm->r_flags |= BBR_HAS_SYN; 6064 if (th_flags & TH_FIN) 6065 rsm->r_flags |= BBR_HAS_FIN; 6066 rsm->r_tim_lastsent[0] = cts; 6067 rsm->r_rtr_cnt = 1; 6068 rsm->r_rtr_bytes = 0; 6069 rsm->r_start = seq_out; 6070 rsm->r_end = rsm->r_start + len; 6071 rsm->r_dupack = 0; 6072 rsm->r_delivered = bbr->r_ctl.rc_delivered; 6073 rsm->r_pacing_delay = pacing_time; 6074 rsm->r_ts_valid = bbr->rc_ts_valid; 6075 if (bbr->rc_ts_valid) 6076 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts; 6077 rsm->r_del_time = bbr->r_ctl.rc_del_time; 6078 if (bbr->r_ctl.r_app_limited_until) 6079 rsm->r_app_limited = 1; 6080 else 6081 rsm->r_app_limited = 0; 6082 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts); 6083 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp, 6084 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 6085 /* 6086 * Here we must also add in this rsm since snd_max 6087 * is updated after we return from a new send. 6088 */ 6089 rsm->r_flight_at_send += len; 6090 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next); 6091 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 6092 rsm->r_in_tmap = 1; 6093 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) 6094 rsm->r_bbr_state = bbr_state_val(bbr); 6095 else 6096 rsm->r_bbr_state = 8; 6097 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) { 6098 rsm->r_is_gain = 1; 6099 rsm->r_is_drain = 0; 6100 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) { 6101 rsm->r_is_drain = 1; 6102 rsm->r_is_gain = 0; 6103 } else { 6104 rsm->r_is_drain = 0; 6105 rsm->r_is_gain = 0; 6106 } 6107 return; 6108 } 6109 /* 6110 * If we reach here its a retransmission and we need to find it. 6111 */ 6112 more: 6113 if (hintrsm && (hintrsm->r_start == seq_out)) { 6114 rsm = hintrsm; 6115 hintrsm = NULL; 6116 } else if (bbr->r_ctl.rc_next) { 6117 /* We have a hint from a previous run */ 6118 rsm = bbr->r_ctl.rc_next; 6119 } else { 6120 /* No hints sorry */ 6121 rsm = NULL; 6122 } 6123 if ((rsm) && (rsm->r_start == seq_out)) { 6124 /* 6125 * We used rc_next or hintrsm to retransmit, hopefully the 6126 * likely case. 6127 */ 6128 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time); 6129 if (len == 0) { 6130 return; 6131 } else { 6132 goto more; 6133 } 6134 } 6135 /* Ok it was not the last pointer go through it the hard way. */ 6136 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { 6137 if (rsm->r_start == seq_out) { 6138 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time); 6139 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next); 6140 if (len == 0) { 6141 return; 6142 } else { 6143 continue; 6144 } 6145 } 6146 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) { 6147 /* Transmitted within this piece */ 6148 /* 6149 * Ok we must split off the front and then let the 6150 * update do the rest 6151 */ 6152 nrsm = bbr_alloc_full_limit(bbr); 6153 if (nrsm == NULL) { 6154 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time); 6155 return; 6156 } 6157 /* 6158 * copy rsm to nrsm and then trim the front of rsm 6159 * to not include this part. 6160 */ 6161 bbr_clone_rsm(bbr, nrsm, rsm, seq_out); 6162 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); 6163 if (rsm->r_in_tmap) { 6164 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 6165 nrsm->r_in_tmap = 1; 6166 } 6167 rsm->r_flags &= (~BBR_HAS_FIN); 6168 seq_out = bbr_update_entry(tp, bbr, nrsm, cts, &len, pacing_time); 6169 if (len == 0) { 6170 return; 6171 } 6172 } 6173 } 6174 /* 6175 * Hmm not found in map did they retransmit both old and on into the 6176 * new? 6177 */ 6178 if (seq_out == tp->snd_max) { 6179 goto again; 6180 } else if (SEQ_LT(seq_out, tp->snd_max)) { 6181 #ifdef BBR_INVARIANTS 6182 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n", 6183 seq_out, len, tp->snd_una, tp->snd_max); 6184 printf("Starting Dump of all rack entries\n"); 6185 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { 6186 printf("rsm:%p start:%u end:%u\n", 6187 rsm, rsm->r_start, rsm->r_end); 6188 } 6189 printf("Dump complete\n"); 6190 panic("seq_out not found rack:%p tp:%p", 6191 bbr, tp); 6192 #endif 6193 } else { 6194 #ifdef BBR_INVARIANTS 6195 /* 6196 * Hmm beyond sndmax? (only if we are using the new rtt-pack 6197 * flag) 6198 */ 6199 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p", 6200 seq_out, len, tp->snd_max, tp); 6201 #endif 6202 } 6203 } 6204 6205 static void 6206 bbr_collapse_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, int32_t rtt) 6207 { 6208 /* 6209 * Collapse timeout back the cum-ack moved. 6210 */ 6211 tp->t_rxtshift = 0; 6212 tp->t_softerror = 0; 6213 } 6214 6215 6216 static void 6217 tcp_bbr_xmit_timer(struct tcp_bbr *bbr, uint32_t rtt_usecs, uint32_t rsm_send_time, uint32_t r_start, uint32_t tsin) 6218 { 6219 bbr->rtt_valid = 1; 6220 bbr->r_ctl.cur_rtt = rtt_usecs; 6221 bbr->r_ctl.ts_in = tsin; 6222 if (rsm_send_time) 6223 bbr->r_ctl.cur_rtt_send_time = rsm_send_time; 6224 } 6225 6226 static void 6227 bbr_make_timestamp_determination(struct tcp_bbr *bbr) 6228 { 6229 /** 6230 * We have in our bbr control: 6231 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp). 6232 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts). 6233 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts) 6234 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time) 6235 * 6236 * Now we can calculate the time between the sends by doing: 6237 * 6238 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts 6239 * 6240 * And the peer's time between receiving them by doing: 6241 * 6242 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp 6243 * 6244 * We want to figure out if the timestamp values are in msec, 10msec or usec. 6245 * We also may find that we can't use the timestamps if say we see 6246 * that the peer_delta indicates that though we may have taken 10ms to 6247 * pace out the data, it only saw 1ms between the two packets. This would 6248 * indicate that somewhere on the path is a batching entity that is giving 6249 * out time-slices of the actual b/w. This would mean we could not use 6250 * reliably the peers timestamps. 6251 * 6252 * We expect delta > peer_delta initially. Until we figure out the 6253 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio. 6254 * If we place 1000 there then its a ms vs our usec. If we place 10000 there 6255 * then its 10ms vs our usec. If the peer is running a usec clock we would 6256 * put a 1 there. If the value is faster then ours, we will disable the 6257 * use of timestamps (though we could revist this later if we find it to be not 6258 * just an isolated one or two flows)). 6259 * 6260 * To detect the batching middle boxes we will come up with our compensation and 6261 * if with it in place, we find the peer is drastically off (by some margin) in 6262 * the smaller direction, then we will assume the worst case and disable use of timestamps. 6263 * 6264 */ 6265 uint64_t delta, peer_delta, delta_up; 6266 6267 delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts; 6268 if (delta < bbr_min_usec_delta) { 6269 /* 6270 * Have not seen a min amount of time 6271 * between our send times so we can 6272 * make a determination of the timestamp 6273 * yet. 6274 */ 6275 return; 6276 } 6277 peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp; 6278 if (peer_delta < bbr_min_peer_delta) { 6279 /* 6280 * We may have enough in the form of 6281 * our delta but the peers number 6282 * has not changed that much. It could 6283 * be its clock ratio is such that 6284 * we need more data (10ms tick) or 6285 * there may be other compression scenarios 6286 * going on. In any event we need the 6287 * spread to be larger. 6288 */ 6289 return; 6290 } 6291 /* Ok lets first see which way our delta is going */ 6292 if (peer_delta > delta) { 6293 /* Very unlikely, the peer without 6294 * compensation shows that it saw 6295 * the two sends arrive further apart 6296 * then we saw then in micro-seconds. 6297 */ 6298 if (peer_delta < (delta + ((delta * (uint64_t)1000)/ (uint64_t)bbr_delta_percent))) { 6299 /* well it looks like the peer is a micro-second clock. */ 6300 bbr->rc_ts_clock_set = 1; 6301 bbr->r_ctl.bbr_peer_tsratio = 1; 6302 } else { 6303 bbr->rc_ts_cant_be_used = 1; 6304 bbr->rc_ts_clock_set = 1; 6305 } 6306 return; 6307 } 6308 /* Ok we know that the peer_delta is smaller than our send distance */ 6309 bbr->rc_ts_clock_set = 1; 6310 /* First question is it within the percentage that they are using usec time? */ 6311 delta_up = (peer_delta * 1000) / (uint64_t)bbr_delta_percent; 6312 if ((peer_delta + delta_up) >= delta) { 6313 /* Its a usec clock */ 6314 bbr->r_ctl.bbr_peer_tsratio = 1; 6315 bbr_log_tstmp_validation(bbr, peer_delta, delta); 6316 return; 6317 } 6318 /* Ok if not usec, what about 10usec (though unlikely)? */ 6319 delta_up = (peer_delta * 1000 * 10) / (uint64_t)bbr_delta_percent; 6320 if (((peer_delta * 10) + delta_up) >= delta) { 6321 bbr->r_ctl.bbr_peer_tsratio = 10; 6322 bbr_log_tstmp_validation(bbr, peer_delta, delta); 6323 return; 6324 } 6325 /* And what about 100usec (though again unlikely)? */ 6326 delta_up = (peer_delta * 1000 * 100) / (uint64_t)bbr_delta_percent; 6327 if (((peer_delta * 100) + delta_up) >= delta) { 6328 bbr->r_ctl.bbr_peer_tsratio = 100; 6329 bbr_log_tstmp_validation(bbr, peer_delta, delta); 6330 return; 6331 } 6332 /* And how about 1 msec (the most likely one)? */ 6333 delta_up = (peer_delta * 1000 * 1000) / (uint64_t)bbr_delta_percent; 6334 if (((peer_delta * 1000) + delta_up) >= delta) { 6335 bbr->r_ctl.bbr_peer_tsratio = 1000; 6336 bbr_log_tstmp_validation(bbr, peer_delta, delta); 6337 return; 6338 } 6339 /* Ok if not msec could it be 10 msec? */ 6340 delta_up = (peer_delta * 1000 * 10000) / (uint64_t)bbr_delta_percent; 6341 if (((peer_delta * 10000) + delta_up) >= delta) { 6342 bbr->r_ctl.bbr_peer_tsratio = 10000; 6343 return; 6344 } 6345 /* If we fall down here the clock tick so slowly we can't use it */ 6346 bbr->rc_ts_cant_be_used = 1; 6347 bbr->r_ctl.bbr_peer_tsratio = 0; 6348 bbr_log_tstmp_validation(bbr, peer_delta, delta); 6349 } 6350 6351 /* 6352 * Collect new round-trip time estimate 6353 * and update averages and current timeout. 6354 */ 6355 static void 6356 tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts) 6357 { 6358 int32_t delta; 6359 uint32_t rtt, tsin; 6360 int32_t rtt_ticks; 6361 6362 6363 if (bbr->rtt_valid == 0) 6364 /* No valid sample */ 6365 return; 6366 6367 rtt = bbr->r_ctl.cur_rtt; 6368 tsin = bbr->r_ctl.ts_in; 6369 if (bbr->rc_prtt_set_ts) { 6370 /* 6371 * We are to force feed the rttProp filter due 6372 * to an entry into PROBE_RTT. This assures 6373 * that the times are sync'd between when we 6374 * go into PROBE_RTT and the filter expiration. 6375 * 6376 * Google does not use a true filter, so they do 6377 * this implicitly since they only keep one value 6378 * and when they enter probe-rtt they update the 6379 * value to the newest rtt. 6380 */ 6381 uint32_t rtt_prop; 6382 6383 bbr->rc_prtt_set_ts = 0; 6384 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop); 6385 if (rtt > rtt_prop) 6386 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts); 6387 else 6388 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); 6389 } 6390 if (bbr->rc_ack_was_delayed) 6391 rtt += bbr->r_ctl.rc_ack_hdwr_delay; 6392 6393 if (rtt < bbr->r_ctl.rc_lowest_rtt) 6394 bbr->r_ctl.rc_lowest_rtt = rtt; 6395 bbr_log_rtt_sample(bbr, rtt, tsin); 6396 if (bbr->r_init_rtt) { 6397 /* 6398 * The initial rtt is not-trusted, nuke it and lets get 6399 * our first valid measurement in. 6400 */ 6401 bbr->r_init_rtt = 0; 6402 tp->t_srtt = 0; 6403 } 6404 if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) { 6405 /* 6406 * So we have not yet figured out 6407 * what the peers TSTMP value is 6408 * in (most likely ms). We need a 6409 * series of cum-ack's to determine 6410 * this reliably. 6411 */ 6412 if (bbr->rc_ack_is_cumack) { 6413 if (bbr->rc_ts_data_set) { 6414 /* Lets attempt to determine the timestamp granularity. */ 6415 bbr_make_timestamp_determination(bbr); 6416 } else { 6417 bbr->rc_ts_data_set = 1; 6418 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts; 6419 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time; 6420 } 6421 } else { 6422 /* 6423 * We have to have consecutive acks 6424 * reset any "filled" state to none. 6425 */ 6426 bbr->rc_ts_data_set = 0; 6427 } 6428 } 6429 /* Round it up */ 6430 rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1))); 6431 if (rtt_ticks == 0) 6432 rtt_ticks = 1; 6433 if (tp->t_srtt != 0) { 6434 /* 6435 * srtt is stored as fixed point with 5 bits after the 6436 * binary point (i.e., scaled by 8). The following magic is 6437 * equivalent to the smoothing algorithm in rfc793 with an 6438 * alpha of .875 (srtt = rtt/8 + srtt*7/8 in fixed point). 6439 * Adjust rtt to origin 0. 6440 */ 6441 6442 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT) 6443 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT)); 6444 6445 tp->t_srtt += delta; 6446 if (tp->t_srtt <= 0) 6447 tp->t_srtt = 1; 6448 6449 /* 6450 * We accumulate a smoothed rtt variance (actually, a 6451 * smoothed mean difference), then set the retransmit timer 6452 * to smoothed rtt + 4 times the smoothed variance. rttvar 6453 * is stored as fixed point with 4 bits after the binary 6454 * point (scaled by 16). The following is equivalent to 6455 * rfc793 smoothing with an alpha of .75 (rttvar = 6456 * rttvar*3/4 + |delta| / 4). This replaces rfc793's 6457 * wired-in beta. 6458 */ 6459 if (delta < 0) 6460 delta = -delta; 6461 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT); 6462 tp->t_rttvar += delta; 6463 if (tp->t_rttvar <= 0) 6464 tp->t_rttvar = 1; 6465 if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar) 6466 tp->t_rttbest = tp->t_srtt + tp->t_rttvar; 6467 } else { 6468 /* 6469 * No rtt measurement yet - use the unsmoothed rtt. Set the 6470 * variance to half the rtt (so our first retransmit happens 6471 * at 3*rtt). 6472 */ 6473 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT; 6474 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1); 6475 tp->t_rttbest = tp->t_srtt + tp->t_rttvar; 6476 } 6477 KMOD_TCPSTAT_INC(tcps_rttupdated); 6478 tp->t_rttupdated++; 6479 #ifdef STATS 6480 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks)); 6481 #endif 6482 /* 6483 * the retransmit should happen at rtt + 4 * rttvar. Because of the 6484 * way we do the smoothing, srtt and rttvar will each average +1/2 6485 * tick of bias. When we compute the retransmit timer, we want 1/2 6486 * tick of rounding and 1 extra tick because of +-1/2 tick 6487 * uncertainty in the firing of the timer. The bias will give us 6488 * exactly the 1.5 tick we need. But, because the bias is 6489 * statistical, we have to test that we don't drop below the minimum 6490 * feasible timer (which is 2 ticks). 6491 */ 6492 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp), 6493 max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2), 6494 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000)); 6495 6496 /* 6497 * We received an ack for a packet that wasn't retransmitted; it is 6498 * probably safe to discard any error indications we've received 6499 * recently. This isn't quite right, but close enough for now (a 6500 * route might have failed after we sent a segment, and the return 6501 * path might not be symmetrical). 6502 */ 6503 tp->t_softerror = 0; 6504 rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT); 6505 if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt) 6506 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt; 6507 } 6508 6509 static void 6510 bbr_earlier_retran(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, 6511 uint32_t t, uint32_t cts, int ack_type) 6512 { 6513 /* 6514 * For this RSM, we acknowledged the data from a previous 6515 * transmission, not the last one we made. This means we did a false 6516 * retransmit. 6517 */ 6518 if (rsm->r_flags & BBR_HAS_FIN) { 6519 /* 6520 * The sending of the FIN often is multiple sent when we 6521 * have everything outstanding ack'd. We ignore this case 6522 * since its over now. 6523 */ 6524 return; 6525 } 6526 if (rsm->r_flags & BBR_TLP) { 6527 /* 6528 * We expect TLP's to have this occur often 6529 */ 6530 bbr->rc_tlp_rtx_out = 0; 6531 return; 6532 } 6533 if (ack_type != BBR_CUM_ACKED) { 6534 /* 6535 * If it was not a cum-ack we 6536 * don't really know for sure since 6537 * the timestamp could be from some 6538 * other transmission. 6539 */ 6540 return; 6541 } 6542 6543 if (rsm->r_flags & BBR_WAS_SACKPASS) { 6544 /* 6545 * We retransmitted based on a sack and the earlier 6546 * retransmission ack'd it - re-ordering is occuring. 6547 */ 6548 BBR_STAT_INC(bbr_reorder_seen); 6549 bbr->r_ctl.rc_reorder_ts = cts; 6550 } 6551 /* Back down the loss count */ 6552 if (rsm->r_flags & BBR_MARKED_LOST) { 6553 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; 6554 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; 6555 rsm->r_flags &= ~BBR_MARKED_LOST; 6556 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) 6557 /* LT sampling also needs adjustment */ 6558 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 6559 } 6560 /***** RRS HERE ************************/ 6561 /* Do we need to do this??? */ 6562 /* bbr_reset_lt_bw_sampling(bbr, cts); */ 6563 /***** RRS HERE ************************/ 6564 BBR_STAT_INC(bbr_badfr); 6565 BBR_STAT_ADD(bbr_badfr_bytes, (rsm->r_end - rsm->r_start)); 6566 } 6567 6568 6569 static void 6570 bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line) 6571 { 6572 bbr->r_ctl.rc_rtt_shrinks = cts; 6573 if (bbr_can_force_probertt && 6574 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && 6575 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { 6576 /* 6577 * We should enter probe-rtt its been too long 6578 * since we have been there. 6579 */ 6580 bbr_enter_probe_rtt(bbr, cts, __LINE__); 6581 } else 6582 bbr_check_probe_rtt_limits(bbr, cts); 6583 } 6584 6585 static void 6586 tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts) 6587 { 6588 uint64_t orig_bw; 6589 6590 if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) { 6591 /* We never apply a zero measurment */ 6592 bbr_log_type_bbrupd(bbr, 20, cts, 0, 0, 6593 0, 0, 0, 0, 0, 0); 6594 return; 6595 } 6596 if (bbr->r_ctl.r_measurement_count < 0xffffffff) 6597 bbr->r_ctl.r_measurement_count++; 6598 orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate); 6599 apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch); 6600 bbr_log_type_bbrupd(bbr, 21, cts, (uint32_t)orig_bw, 6601 (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate), 6602 0, 0, 0, 0, 0, 0); 6603 if (orig_bw && 6604 (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) { 6605 if (bbr->bbr_hdrw_pacing) { 6606 /* 6607 * Apply a new rate to the hardware 6608 * possibly. 6609 */ 6610 bbr_update_hardware_pacing_rate(bbr, cts); 6611 } 6612 bbr_set_state_target(bbr, __LINE__); 6613 tcp_bbr_tso_size_check(bbr, cts); 6614 if (bbr->r_recovery_bw) { 6615 bbr_setup_red_bw(bbr, cts); 6616 bbr_log_type_bw_reduce(bbr, BBR_RED_BW_USELRBW); 6617 } 6618 } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate)) 6619 tcp_bbr_tso_size_check(bbr, cts); 6620 } 6621 6622 static void 6623 bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts) 6624 { 6625 if (bbr->rc_in_persist == 0) { 6626 /* We log only when not in persist */ 6627 /* Translate to a Bytes Per Second */ 6628 uint64_t tim, bw, ts_diff, ts_bw; 6629 uint32_t upper, lower, delivered; 6630 6631 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time)) 6632 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time); 6633 else 6634 tim = 1; 6635 /* 6636 * Now that we have processed the tim (skipping the sample 6637 * or possibly updating the time, go ahead and 6638 * calculate the cdr. 6639 */ 6640 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered); 6641 bw = (uint64_t)delivered; 6642 bw *= (uint64_t)USECS_IN_SECOND; 6643 bw /= tim; 6644 if (bw == 0) { 6645 /* We must have a calculatable amount */ 6646 return; 6647 } 6648 upper = (bw >> 32) & 0x00000000ffffffff; 6649 lower = bw & 0x00000000ffffffff; 6650 /* 6651 * If we are using this b/w shove it in now so we 6652 * can see in the trace viewer if it gets over-ridden. 6653 */ 6654 if (rsm->r_ts_valid && 6655 bbr->rc_ts_valid && 6656 bbr->rc_ts_clock_set && 6657 (bbr->rc_ts_cant_be_used == 0) && 6658 bbr->rc_use_ts_limit) { 6659 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1); 6660 ts_diff *= bbr->r_ctl.bbr_peer_tsratio; 6661 if ((delivered == 0) || 6662 (rtt < 1000)) { 6663 /* Can't use the ts */ 6664 bbr_log_type_bbrupd(bbr, 61, cts, 6665 ts_diff, 6666 bbr->r_ctl.last_inbound_ts, 6667 rsm->r_del_ack_ts, 0, 6668 0, 0, 0, delivered); 6669 } else { 6670 ts_bw = (uint64_t)delivered; 6671 ts_bw *= (uint64_t)USECS_IN_SECOND; 6672 ts_bw /= ts_diff; 6673 bbr_log_type_bbrupd(bbr, 62, cts, 6674 (ts_bw >> 32), 6675 (ts_bw & 0xffffffff), 0, 0, 6676 0, 0, ts_diff, delivered); 6677 if ((bbr->ts_can_raise) && 6678 (ts_bw > bw)) { 6679 bbr_log_type_bbrupd(bbr, 8, cts, 6680 delivered, 6681 ts_diff, 6682 (bw >> 32), 6683 (bw & 0x00000000ffffffff), 6684 0, 0, 0, 0); 6685 bw = ts_bw; 6686 } else if (ts_bw && (ts_bw < bw)) { 6687 bbr_log_type_bbrupd(bbr, 7, cts, 6688 delivered, 6689 ts_diff, 6690 (bw >> 32), 6691 (bw & 0x00000000ffffffff), 6692 0, 0, 0, 0); 6693 bw = ts_bw; 6694 } 6695 } 6696 } 6697 if (rsm->r_first_sent_time && 6698 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) { 6699 uint64_t sbw, sti; 6700 /* 6701 * We use what was in flight at the time of our 6702 * send and the size of this send to figure 6703 * out what we have been sending at (amount). 6704 * For the time we take from the time of 6705 * the send of the first send outstanding 6706 * until this send plus this sends pacing 6707 * time. This gives us a good calculation 6708 * as to the rate we have been sending at. 6709 */ 6710 6711 sbw = (uint64_t)(rsm->r_flight_at_send); 6712 sbw *= (uint64_t)USECS_IN_SECOND; 6713 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time; 6714 sti += rsm->r_pacing_delay; 6715 sbw /= sti; 6716 if (sbw < bw) { 6717 bbr_log_type_bbrupd(bbr, 6, cts, 6718 delivered, 6719 (uint32_t)sti, 6720 (bw >> 32), 6721 (uint32_t)bw, 6722 rsm->r_first_sent_time, 0, (sbw >> 32), 6723 (uint32_t)sbw); 6724 bw = sbw; 6725 } 6726 } 6727 /* Use the google algorithm for b/w measurements */ 6728 bbr->r_ctl.rc_bbr_cur_del_rate = bw; 6729 if ((rsm->r_app_limited == 0) || 6730 (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) { 6731 tcp_bbr_commit_bw(bbr, cts); 6732 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered, 6733 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time); 6734 } 6735 } 6736 } 6737 6738 static void 6739 bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts) 6740 { 6741 if (bbr->rc_in_persist == 0) { 6742 /* We log only when not in persist */ 6743 /* Translate to a Bytes Per Second */ 6744 uint64_t tim, bw; 6745 uint32_t upper, lower, delivered; 6746 int no_apply = 0; 6747 6748 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time)) 6749 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time); 6750 else 6751 tim = 1; 6752 /* 6753 * Now that we have processed the tim (skipping the sample 6754 * or possibly updating the time, go ahead and 6755 * calculate the cdr. 6756 */ 6757 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered); 6758 bw = (uint64_t)delivered; 6759 bw *= (uint64_t)USECS_IN_SECOND; 6760 bw /= tim; 6761 if (tim < bbr->r_ctl.rc_lowest_rtt) { 6762 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered, 6763 tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0); 6764 6765 no_apply = 1; 6766 } 6767 upper = (bw >> 32) & 0x00000000ffffffff; 6768 lower = bw & 0x00000000ffffffff; 6769 /* 6770 * If we are using this b/w shove it in now so we 6771 * can see in the trace viewer if it gets over-ridden. 6772 */ 6773 bbr->r_ctl.rc_bbr_cur_del_rate = bw; 6774 /* Gate by the sending rate */ 6775 if (rsm->r_first_sent_time && 6776 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) { 6777 uint64_t sbw, sti; 6778 /* 6779 * We use what was in flight at the time of our 6780 * send and the size of this send to figure 6781 * out what we have been sending at (amount). 6782 * For the time we take from the time of 6783 * the send of the first send outstanding 6784 * until this send plus this sends pacing 6785 * time. This gives us a good calculation 6786 * as to the rate we have been sending at. 6787 */ 6788 6789 sbw = (uint64_t)(rsm->r_flight_at_send); 6790 sbw *= (uint64_t)USECS_IN_SECOND; 6791 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time; 6792 sti += rsm->r_pacing_delay; 6793 sbw /= sti; 6794 if (sbw < bw) { 6795 bbr_log_type_bbrupd(bbr, 6, cts, 6796 delivered, 6797 (uint32_t)sti, 6798 (bw >> 32), 6799 (uint32_t)bw, 6800 rsm->r_first_sent_time, 0, (sbw >> 32), 6801 (uint32_t)sbw); 6802 bw = sbw; 6803 } 6804 if ((sti > tim) && 6805 (sti < bbr->r_ctl.rc_lowest_rtt)) { 6806 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered, 6807 (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0); 6808 no_apply = 1; 6809 } else 6810 no_apply = 0; 6811 } 6812 bbr->r_ctl.rc_bbr_cur_del_rate = bw; 6813 if ((no_apply == 0) && 6814 ((rsm->r_app_limited == 0) || 6815 (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) { 6816 tcp_bbr_commit_bw(bbr, cts); 6817 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered, 6818 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time); 6819 } 6820 } 6821 } 6822 6823 6824 static void 6825 bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin, 6826 uint32_t uts, int32_t match, uint32_t rsm_send_time, int32_t ack_type, struct tcpopt *to) 6827 { 6828 uint64_t old_rttprop; 6829 6830 /* Update our delivery time and amount */ 6831 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start); 6832 bbr->r_ctl.rc_del_time = cts; 6833 if (rtt == 0) { 6834 /* 6835 * 0 means its a retransmit, for now we don't use these for 6836 * the rest of BBR. 6837 */ 6838 return; 6839 } 6840 if ((bbr->rc_use_google == 0) && 6841 (match != BBR_RTT_BY_EXACTMATCH) && 6842 (match != BBR_RTT_BY_TIMESTAMP)){ 6843 /* 6844 * We get a lot of rtt updates, lets not pay attention to 6845 * any that are not an exact match. That way we don't have 6846 * to worry about timestamps and the whole nonsense of 6847 * unsure if its a retransmission etc (if we ever had the 6848 * timestamp fixed to always have the last thing sent this 6849 * would not be a issue). 6850 */ 6851 return; 6852 } 6853 if ((bbr_no_retran && bbr->rc_use_google) && 6854 (match != BBR_RTT_BY_EXACTMATCH) && 6855 (match != BBR_RTT_BY_TIMESTAMP)){ 6856 /* 6857 * We only do measurements in google mode 6858 * with bbr_no_retran on for sure things. 6859 */ 6860 return; 6861 } 6862 /* Only update srtt if we know by exact match */ 6863 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin); 6864 if (ack_type == BBR_CUM_ACKED) 6865 bbr->rc_ack_is_cumack = 1; 6866 else 6867 bbr->rc_ack_is_cumack = 0; 6868 old_rttprop = bbr_get_rtt(bbr, BBR_RTT_PROP); 6869 /* 6870 * Note the following code differs to the original 6871 * BBR spec. It calls for <= not <. However after a 6872 * long discussion in email with Neal, he acknowledged 6873 * that it should be < than so that we will have flows 6874 * going into probe-rtt (we were seeing cases where that 6875 * did not happen and caused ugly things to occur). We 6876 * have added this agreed upon fix to our code base. 6877 */ 6878 if (rtt < old_rttprop) { 6879 /* Update when we last saw a rtt drop */ 6880 bbr_log_rtt_shrinks(bbr, cts, 0, rtt, __LINE__, BBR_RTTS_NEWRTT, 0); 6881 bbr_set_reduced_rtt(bbr, cts, __LINE__); 6882 } 6883 bbr_log_type_bbrrttprop(bbr, rtt, (rsm ? rsm->r_end : 0), uts, cts, 6884 match, rsm->r_start, rsm->r_flags); 6885 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); 6886 if (old_rttprop != bbr_get_rtt(bbr, BBR_RTT_PROP)) { 6887 /* 6888 * The RTT-prop moved, reset the target (may be a 6889 * nop for some states). 6890 */ 6891 bbr_set_state_target(bbr, __LINE__); 6892 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) 6893 bbr_log_rtt_shrinks(bbr, cts, 0, 0, 6894 __LINE__, BBR_RTTS_NEW_TARGET, 0); 6895 else if (old_rttprop < bbr_get_rtt(bbr, BBR_RTT_PROP)) 6896 /* It went up */ 6897 bbr_check_probe_rtt_limits(bbr, cts); 6898 } 6899 if ((bbr->rc_use_google == 0) && 6900 (match == BBR_RTT_BY_TIMESTAMP)) { 6901 /* 6902 * We don't do b/w update with 6903 * these since they are not really 6904 * reliable. 6905 */ 6906 return; 6907 } 6908 if (bbr->r_ctl.r_app_limited_until && 6909 (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) { 6910 /* We are no longer app-limited */ 6911 bbr->r_ctl.r_app_limited_until = 0; 6912 } 6913 if (bbr->rc_use_google) { 6914 bbr_google_measurement(bbr, rsm, rtt, cts); 6915 } else { 6916 bbr_nf_measurement(bbr, rsm, rtt, cts); 6917 } 6918 } 6919 6920 /* 6921 * Convert a timestamp that the main stack 6922 * uses (milliseconds) into one that bbr uses 6923 * (microseconds). Return that converted timestamp. 6924 */ 6925 static uint32_t 6926 bbr_ts_convert(uint32_t cts) { 6927 uint32_t sec, msec; 6928 6929 sec = cts / MS_IN_USEC; 6930 msec = cts - (MS_IN_USEC * sec); 6931 return ((sec * USECS_IN_SECOND) + (msec * MS_IN_USEC)); 6932 } 6933 6934 /* 6935 * Return 0 if we did not update the RTT time, return 6936 * 1 if we did. 6937 */ 6938 static int 6939 bbr_update_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, 6940 struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack) 6941 { 6942 int32_t i; 6943 uint32_t t, uts = 0; 6944 6945 if ((rsm->r_flags & BBR_ACKED) || 6946 (rsm->r_flags & BBR_WAS_RENEGED) || 6947 (rsm->r_flags & BBR_RXT_CLEARED)) { 6948 /* Already done */ 6949 return (0); 6950 } 6951 if (rsm->r_rtr_cnt == 1) { 6952 /* 6953 * Only one transmit. Hopefully the normal case. 6954 */ 6955 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0])) 6956 t = cts - rsm->r_tim_lastsent[0]; 6957 else 6958 t = 1; 6959 if ((int)t <= 0) 6960 t = 1; 6961 bbr->r_ctl.rc_last_rtt = t; 6962 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, 6963 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to); 6964 return (1); 6965 } 6966 /* Convert to usecs */ 6967 if ((bbr_can_use_ts_for_rtt == 1) && 6968 (bbr->rc_use_google == 1) && 6969 (ack_type == BBR_CUM_ACKED) && 6970 (to->to_flags & TOF_TS) && 6971 (to->to_tsecr != 0)) { 6972 6973 t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr; 6974 if (t < 1) 6975 t = 1; 6976 t *= MS_IN_USEC; 6977 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0, 6978 BBR_RTT_BY_TIMESTAMP, 6979 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)], 6980 ack_type, to); 6981 return (1); 6982 } 6983 uts = bbr_ts_convert(to->to_tsecr); 6984 if ((to->to_flags & TOF_TS) && 6985 (to->to_tsecr != 0) && 6986 (ack_type == BBR_CUM_ACKED) && 6987 ((rsm->r_flags & BBR_OVERMAX) == 0)) { 6988 /* 6989 * Now which timestamp does it match? In this block the ACK 6990 * may be coming from a previous transmission. 6991 */ 6992 uint32_t fudge; 6993 6994 fudge = BBR_TIMER_FUDGE; 6995 for (i = 0; i < rsm->r_rtr_cnt; i++) { 6996 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) && 6997 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) { 6998 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) 6999 t = cts - rsm->r_tim_lastsent[i]; 7000 else 7001 t = 1; 7002 if ((int)t <= 0) 7003 t = 1; 7004 bbr->r_ctl.rc_last_rtt = t; 7005 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING, 7006 rsm->r_tim_lastsent[i], ack_type, to); 7007 if ((i + 1) < rsm->r_rtr_cnt) { 7008 /* Likely */ 7009 bbr_earlier_retran(tp, bbr, rsm, t, cts, ack_type); 7010 } else if (rsm->r_flags & BBR_TLP) { 7011 bbr->rc_tlp_rtx_out = 0; 7012 } 7013 return (1); 7014 } 7015 } 7016 /* Fall through if we can't find a matching timestamp */ 7017 } 7018 /* 7019 * Ok its a SACK block that we retransmitted. or a windows 7020 * machine without timestamps. We can tell nothing from the 7021 * time-stamp since its not there or the time the peer last 7022 * recieved a segment that moved forward its cum-ack point. 7023 * 7024 * Lets look at the last retransmit and see what we can tell 7025 * (with BBR for space we only keep 2 note we have to keep 7026 * at least 2 so the map can not be condensed more). 7027 */ 7028 i = rsm->r_rtr_cnt - 1; 7029 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) 7030 t = cts - rsm->r_tim_lastsent[i]; 7031 else 7032 goto not_sure; 7033 if (t < bbr->r_ctl.rc_lowest_rtt) { 7034 /* 7035 * We retransmitted and the ack came back in less 7036 * than the smallest rtt we have observed in the 7037 * windowed rtt. We most likey did an improper 7038 * retransmit as outlined in 4.2 Step 3 point 2 in 7039 * the rack-draft. 7040 * 7041 * Use the prior transmission to update all the 7042 * information as long as there is only one prior 7043 * transmission. 7044 */ 7045 if ((rsm->r_flags & BBR_OVERMAX) == 0) { 7046 #ifdef BBR_INVARIANTS 7047 if (rsm->r_rtr_cnt == 1) 7048 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags); 7049 #endif 7050 i = rsm->r_rtr_cnt - 2; 7051 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i])) 7052 t = cts - rsm->r_tim_lastsent[i]; 7053 else 7054 t = 1; 7055 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET, 7056 rsm->r_tim_lastsent[i], ack_type, to); 7057 bbr_earlier_retran(tp, bbr, rsm, t, cts, ack_type); 7058 } else { 7059 /* 7060 * Too many prior transmissions, just 7061 * updated BBR delivered 7062 */ 7063 not_sure: 7064 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, 7065 BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to); 7066 } 7067 } else { 7068 /* 7069 * We retransmitted it and the retransmit did the 7070 * job. 7071 */ 7072 if (rsm->r_flags & BBR_TLP) 7073 bbr->rc_tlp_rtx_out = 0; 7074 if ((rsm->r_flags & BBR_OVERMAX) == 0) 7075 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, 7076 BBR_RTT_BY_THIS_RETRAN, 0, ack_type, to); 7077 else 7078 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts, 7079 BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to); 7080 return (1); 7081 } 7082 return (0); 7083 } 7084 7085 /* 7086 * Mark the SACK_PASSED flag on all entries prior to rsm send wise. 7087 */ 7088 static void 7089 bbr_log_sack_passed(struct tcpcb *tp, 7090 struct tcp_bbr *bbr, struct bbr_sendmap *rsm) 7091 { 7092 struct bbr_sendmap *nrsm; 7093 7094 nrsm = rsm; 7095 TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap, 7096 bbr_head, r_tnext) { 7097 if (nrsm == rsm) { 7098 /* Skip orginal segment he is acked */ 7099 continue; 7100 } 7101 if (nrsm->r_flags & BBR_ACKED) { 7102 /* Skip ack'd segments */ 7103 continue; 7104 } 7105 if (nrsm->r_flags & BBR_SACK_PASSED) { 7106 /* 7107 * We found one that is already marked 7108 * passed, we have been here before and 7109 * so all others below this are marked. 7110 */ 7111 break; 7112 } 7113 BBR_STAT_INC(bbr_sack_passed); 7114 nrsm->r_flags |= BBR_SACK_PASSED; 7115 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) && 7116 bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) { 7117 bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start; 7118 bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start; 7119 nrsm->r_flags |= BBR_MARKED_LOST; 7120 } 7121 nrsm->r_flags &= ~BBR_WAS_SACKPASS; 7122 } 7123 } 7124 7125 /* 7126 * Returns the number of bytes that were 7127 * newly ack'd by sack blocks. 7128 */ 7129 static uint32_t 7130 bbr_proc_sack_blk(struct tcpcb *tp, struct tcp_bbr *bbr, struct sackblk *sack, 7131 struct tcpopt *to, struct bbr_sendmap **prsm, uint32_t cts) 7132 { 7133 int32_t times = 0; 7134 uint32_t start, end, maxseg, changed = 0; 7135 struct bbr_sendmap *rsm, *nrsm; 7136 int32_t used_ref = 1; 7137 uint8_t went_back = 0, went_fwd = 0; 7138 7139 maxseg = tp->t_maxseg - bbr->rc_last_options; 7140 start = sack->start; 7141 end = sack->end; 7142 rsm = *prsm; 7143 if (rsm == NULL) 7144 used_ref = 0; 7145 7146 /* Do we locate the block behind where we last were? */ 7147 if (rsm && SEQ_LT(start, rsm->r_start)) { 7148 went_back = 1; 7149 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { 7150 if (SEQ_GEQ(start, rsm->r_start) && 7151 SEQ_LT(start, rsm->r_end)) { 7152 goto do_rest_ofb; 7153 } 7154 } 7155 } 7156 start_at_beginning: 7157 went_fwd = 1; 7158 /* 7159 * Ok lets locate the block where this guy is fwd from rsm (if its 7160 * set) 7161 */ 7162 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) { 7163 if (SEQ_GEQ(start, rsm->r_start) && 7164 SEQ_LT(start, rsm->r_end)) { 7165 break; 7166 } 7167 } 7168 do_rest_ofb: 7169 if (rsm == NULL) { 7170 /* 7171 * This happens when we get duplicate sack blocks with the 7172 * same end. For example SACK 4: 100 SACK 3: 100 The sort 7173 * will not change there location so we would just start at 7174 * the end of the first one and get lost. 7175 */ 7176 if (tp->t_flags & TF_SENTFIN) { 7177 /* 7178 * Check to see if we have not logged the FIN that 7179 * went out. 7180 */ 7181 nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); 7182 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) { 7183 /* 7184 * Ok we did not get the FIN logged. 7185 */ 7186 nrsm->r_end++; 7187 rsm = nrsm; 7188 goto do_rest_ofb; 7189 } 7190 } 7191 if (times == 1) { 7192 #ifdef BBR_INVARIANTS 7193 panic("tp:%p bbr:%p sack:%p to:%p prsm:%p", 7194 tp, bbr, sack, to, prsm); 7195 #else 7196 goto out; 7197 #endif 7198 } 7199 times++; 7200 BBR_STAT_INC(bbr_sack_proc_restart); 7201 rsm = NULL; 7202 goto start_at_beginning; 7203 } 7204 /* Ok we have an ACK for some piece of rsm */ 7205 if (rsm->r_start != start) { 7206 /* 7207 * Need to split this in two pieces the before and after. 7208 */ 7209 if (bbr_sack_mergable(rsm, start, end)) 7210 nrsm = bbr_alloc_full_limit(bbr); 7211 else 7212 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT); 7213 if (nrsm == NULL) { 7214 /* We could not allocate ignore the sack */ 7215 struct sackblk blk; 7216 7217 blk.start = start; 7218 blk.end = end; 7219 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk); 7220 goto out; 7221 } 7222 bbr_clone_rsm(bbr, nrsm, rsm, start); 7223 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); 7224 if (rsm->r_in_tmap) { 7225 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 7226 nrsm->r_in_tmap = 1; 7227 } 7228 rsm->r_flags &= (~BBR_HAS_FIN); 7229 rsm = nrsm; 7230 } 7231 if (SEQ_GEQ(end, rsm->r_end)) { 7232 /* 7233 * The end of this block is either beyond this guy or right 7234 * at this guy. 7235 */ 7236 if ((rsm->r_flags & BBR_ACKED) == 0) { 7237 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0); 7238 changed += (rsm->r_end - rsm->r_start); 7239 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); 7240 bbr_log_sack_passed(tp, bbr, rsm); 7241 if (rsm->r_flags & BBR_MARKED_LOST) { 7242 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; 7243 } 7244 /* Is Reordering occuring? */ 7245 if (rsm->r_flags & BBR_SACK_PASSED) { 7246 BBR_STAT_INC(bbr_reorder_seen); 7247 bbr->r_ctl.rc_reorder_ts = cts; 7248 if (rsm->r_flags & BBR_MARKED_LOST) { 7249 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; 7250 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) 7251 /* LT sampling also needs adjustment */ 7252 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 7253 } 7254 } 7255 rsm->r_flags |= BBR_ACKED; 7256 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST); 7257 if (rsm->r_in_tmap) { 7258 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 7259 rsm->r_in_tmap = 0; 7260 } 7261 } 7262 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED); 7263 if (end == rsm->r_end) { 7264 /* This block only - done */ 7265 goto out; 7266 } 7267 /* There is more not coverend by this rsm move on */ 7268 start = rsm->r_end; 7269 nrsm = TAILQ_NEXT(rsm, r_next); 7270 rsm = nrsm; 7271 times = 0; 7272 goto do_rest_ofb; 7273 } 7274 if (rsm->r_flags & BBR_ACKED) { 7275 /* Been here done that */ 7276 goto out; 7277 } 7278 /* Ok we need to split off this one at the tail */ 7279 if (bbr_sack_mergable(rsm, start, end)) 7280 nrsm = bbr_alloc_full_limit(bbr); 7281 else 7282 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT); 7283 if (nrsm == NULL) { 7284 /* failed XXXrrs what can we do but loose the sack info? */ 7285 struct sackblk blk; 7286 7287 blk.start = start; 7288 blk.end = end; 7289 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk); 7290 goto out; 7291 } 7292 /* Clone it */ 7293 bbr_clone_rsm(bbr, nrsm, rsm, end); 7294 /* The sack block does not cover this guy fully */ 7295 rsm->r_flags &= (~BBR_HAS_FIN); 7296 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); 7297 if (rsm->r_in_tmap) { 7298 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 7299 nrsm->r_in_tmap = 1; 7300 } 7301 nrsm->r_dupack = 0; 7302 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0); 7303 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED); 7304 changed += (rsm->r_end - rsm->r_start); 7305 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); 7306 bbr_log_sack_passed(tp, bbr, rsm); 7307 /* Is Reordering occuring? */ 7308 if (rsm->r_flags & BBR_MARKED_LOST) { 7309 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; 7310 } 7311 if (rsm->r_flags & BBR_SACK_PASSED) { 7312 BBR_STAT_INC(bbr_reorder_seen); 7313 bbr->r_ctl.rc_reorder_ts = cts; 7314 if (rsm->r_flags & BBR_MARKED_LOST) { 7315 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; 7316 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) 7317 /* LT sampling also needs adjustment */ 7318 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 7319 } 7320 } 7321 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST); 7322 rsm->r_flags |= BBR_ACKED; 7323 if (rsm->r_in_tmap) { 7324 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 7325 rsm->r_in_tmap = 0; 7326 } 7327 out: 7328 if (rsm && (rsm->r_flags & BBR_ACKED)) { 7329 /* 7330 * Now can we merge this newly acked 7331 * block with either the previous or 7332 * next block? 7333 */ 7334 nrsm = TAILQ_NEXT(rsm, r_next); 7335 if (nrsm && 7336 (nrsm->r_flags & BBR_ACKED)) { 7337 /* yep this and next can be merged */ 7338 rsm = bbr_merge_rsm(bbr, rsm, nrsm); 7339 } 7340 /* Now what about the previous? */ 7341 nrsm = TAILQ_PREV(rsm, bbr_head, r_next); 7342 if (nrsm && 7343 (nrsm->r_flags & BBR_ACKED)) { 7344 /* yep the previous and this can be merged */ 7345 rsm = bbr_merge_rsm(bbr, nrsm, rsm); 7346 } 7347 } 7348 if (used_ref == 0) { 7349 BBR_STAT_INC(bbr_sack_proc_all); 7350 } else { 7351 BBR_STAT_INC(bbr_sack_proc_short); 7352 } 7353 if (went_fwd && went_back) { 7354 BBR_STAT_INC(bbr_sack_search_both); 7355 } else if (went_fwd) { 7356 BBR_STAT_INC(bbr_sack_search_fwd); 7357 } else if (went_back) { 7358 BBR_STAT_INC(bbr_sack_search_back); 7359 } 7360 /* Save off where the next seq is */ 7361 if (rsm) 7362 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next); 7363 else 7364 bbr->r_ctl.rc_sacklast = NULL; 7365 *prsm = rsm; 7366 return (changed); 7367 } 7368 7369 7370 static void inline 7371 bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack) 7372 { 7373 struct bbr_sendmap *tmap; 7374 7375 BBR_STAT_INC(bbr_reneges_seen); 7376 tmap = NULL; 7377 while (rsm && (rsm->r_flags & BBR_ACKED)) { 7378 /* Its no longer sacked, mark it so */ 7379 uint32_t oflags; 7380 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); 7381 #ifdef BBR_INVARIANTS 7382 if (rsm->r_in_tmap) { 7383 panic("bbr:%p rsm:%p flags:0x%x in tmap?", 7384 bbr, rsm, rsm->r_flags); 7385 } 7386 #endif 7387 oflags = rsm->r_flags; 7388 if (rsm->r_flags & BBR_MARKED_LOST) { 7389 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; 7390 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; 7391 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) 7392 /* LT sampling also needs adjustment */ 7393 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 7394 } 7395 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST); 7396 rsm->r_flags |= BBR_WAS_RENEGED; 7397 rsm->r_flags |= BBR_RXT_CLEARED; 7398 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__); 7399 /* Rebuild it into our tmap */ 7400 if (tmap == NULL) { 7401 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 7402 tmap = rsm; 7403 } else { 7404 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext); 7405 tmap = rsm; 7406 } 7407 tmap->r_in_tmap = 1; 7408 /* 7409 * XXXrrs Delivered? Should we do anything here? 7410 * 7411 * Of course we don't on a rxt timeout so maybe its ok that 7412 * we don't? 7413 * 7414 * For now lets not. 7415 */ 7416 rsm = TAILQ_NEXT(rsm, r_next); 7417 } 7418 /* 7419 * Now lets possibly clear the sack filter so we start recognizing 7420 * sacks that cover this area. 7421 */ 7422 sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack); 7423 } 7424 7425 static void 7426 bbr_log_syn(struct tcpcb *tp, struct tcpopt *to) 7427 { 7428 struct tcp_bbr *bbr; 7429 struct bbr_sendmap *rsm; 7430 uint32_t cts; 7431 7432 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 7433 cts = bbr->r_ctl.rc_rcvtime; 7434 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 7435 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) { 7436 if ((rsm->r_end - rsm->r_start) <= 1) { 7437 /* Log out the SYN completely */ 7438 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; 7439 rsm->r_rtr_bytes = 0; 7440 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); 7441 if (rsm->r_in_tmap) { 7442 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 7443 rsm->r_in_tmap = 0; 7444 } 7445 if (bbr->r_ctl.rc_next == rsm) { 7446 /* scoot along the marker */ 7447 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map); 7448 } 7449 if (to != NULL) 7450 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0); 7451 bbr_free(bbr, rsm); 7452 } else { 7453 /* There is more (Fast open)? strip out SYN. */ 7454 rsm->r_flags &= ~BBR_HAS_SYN; 7455 rsm->r_start++; 7456 } 7457 } 7458 } 7459 7460 /* 7461 * Returns the number of bytes that were 7462 * acknowledged by SACK blocks. 7463 */ 7464 7465 static uint32_t 7466 bbr_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, 7467 uint32_t *prev_acked) 7468 { 7469 uint32_t changed, last_seq, entered_recovery = 0; 7470 struct tcp_bbr *bbr; 7471 struct bbr_sendmap *rsm; 7472 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1]; 7473 register uint32_t th_ack; 7474 int32_t i, j, k, new_sb, num_sack_blks = 0; 7475 uint32_t cts, acked, ack_point, sack_changed = 0; 7476 uint32_t p_maxseg, maxseg, p_acked = 0; 7477 7478 INP_WLOCK_ASSERT(tp->t_inpcb); 7479 if (th->th_flags & TH_RST) { 7480 /* We don't log resets */ 7481 return (0); 7482 } 7483 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 7484 cts = bbr->r_ctl.rc_rcvtime; 7485 7486 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 7487 changed = 0; 7488 maxseg = tp->t_maxseg - bbr->rc_last_options; 7489 p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg); 7490 th_ack = th->th_ack; 7491 if (SEQ_GT(th_ack, tp->snd_una)) { 7492 acked = th_ack - tp->snd_una; 7493 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_UPDATE, __LINE__); 7494 bbr->rc_tp->t_acktime = ticks; 7495 } else 7496 acked = 0; 7497 if (SEQ_LEQ(th_ack, tp->snd_una)) { 7498 /* Only sent here for sack processing */ 7499 goto proc_sack; 7500 } 7501 if (rsm && SEQ_GT(th_ack, rsm->r_start)) { 7502 changed = th_ack - rsm->r_start; 7503 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) { 7504 /* 7505 * For the SYN incoming case we will not have called 7506 * tcp_output for the sending of the SYN, so there will be 7507 * no map. All other cases should probably be a panic. 7508 */ 7509 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) { 7510 /* 7511 * We have a timestamp that can be used to generate 7512 * an initial RTT. 7513 */ 7514 uint32_t ts, now, rtt; 7515 7516 ts = bbr_ts_convert(to->to_tsecr); 7517 now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv)); 7518 rtt = now - ts; 7519 if (rtt < 1) 7520 rtt = 1; 7521 bbr_log_type_bbrrttprop(bbr, rtt, 7522 tp->iss, 0, cts, 7523 BBR_RTT_BY_TIMESTAMP, tp->iss, 0); 7524 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); 7525 changed = 1; 7526 bbr->r_wanted_output = 1; 7527 goto out; 7528 } 7529 goto proc_sack; 7530 } else if (rsm == NULL) { 7531 goto out; 7532 } 7533 if (changed) { 7534 /* 7535 * The ACK point is advancing to th_ack, we must drop off 7536 * the packets in the rack log and calculate any eligble 7537 * RTT's. 7538 */ 7539 bbr->r_wanted_output = 1; 7540 more: 7541 if (rsm == NULL) { 7542 7543 if (tp->t_flags & TF_SENTFIN) { 7544 /* if we send a FIN we will not hav a map */ 7545 goto proc_sack; 7546 } 7547 #ifdef BBR_INVARIANTS 7548 panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n", 7549 tp, 7550 th, tp->t_state, bbr, 7551 tp->snd_una, tp->snd_max, changed); 7552 #endif 7553 goto proc_sack; 7554 } 7555 } 7556 if (SEQ_LT(th_ack, rsm->r_start)) { 7557 /* Huh map is missing this */ 7558 #ifdef BBR_INVARIANTS 7559 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d bbr:%p\n", 7560 rsm->r_start, 7561 th_ack, tp->t_state, 7562 bbr->r_state, bbr); 7563 panic("th-ack is bad bbr:%p tp:%p", bbr, tp); 7564 #endif 7565 goto proc_sack; 7566 } else if (th_ack == rsm->r_start) { 7567 /* None here to ack */ 7568 goto proc_sack; 7569 } 7570 /* 7571 * Clear the dup ack counter, it will 7572 * either be freed or if there is some 7573 * remaining we need to start it at zero. 7574 */ 7575 rsm->r_dupack = 0; 7576 /* Now do we consume the whole thing? */ 7577 if (SEQ_GEQ(th_ack, rsm->r_end)) { 7578 /* Its all consumed. */ 7579 uint32_t left; 7580 7581 if (rsm->r_flags & BBR_ACKED) { 7582 /* 7583 * It was acked on the scoreboard -- remove it from 7584 * total 7585 */ 7586 p_acked += (rsm->r_end - rsm->r_start); 7587 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); 7588 if (bbr->r_ctl.rc_sacked == 0) 7589 bbr->r_ctl.rc_sacklast = NULL; 7590 } else { 7591 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack); 7592 if (rsm->r_flags & BBR_MARKED_LOST) { 7593 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start; 7594 } 7595 if (rsm->r_flags & BBR_SACK_PASSED) { 7596 /* 7597 * There are acked segments ACKED on the 7598 * scoreboard further up. We are seeing 7599 * reordering. 7600 */ 7601 BBR_STAT_INC(bbr_reorder_seen); 7602 bbr->r_ctl.rc_reorder_ts = cts; 7603 if (rsm->r_flags & BBR_MARKED_LOST) { 7604 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start; 7605 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost)) 7606 /* LT sampling also needs adjustment */ 7607 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost; 7608 } 7609 } 7610 rsm->r_flags &= ~BBR_MARKED_LOST; 7611 } 7612 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; 7613 rsm->r_rtr_bytes = 0; 7614 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); 7615 if (rsm->r_in_tmap) { 7616 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 7617 rsm->r_in_tmap = 0; 7618 } 7619 if (bbr->r_ctl.rc_next == rsm) { 7620 /* scoot along the marker */ 7621 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map); 7622 } 7623 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED); 7624 /* Adjust the packet counts */ 7625 left = th_ack - rsm->r_end; 7626 /* Free back to zone */ 7627 bbr_free(bbr, rsm); 7628 if (left) { 7629 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 7630 goto more; 7631 } 7632 goto proc_sack; 7633 } 7634 if (rsm->r_flags & BBR_ACKED) { 7635 /* 7636 * It was acked on the scoreboard -- remove it from total 7637 * for the part being cum-acked. 7638 */ 7639 p_acked += (rsm->r_end - rsm->r_start); 7640 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start); 7641 if (bbr->r_ctl.rc_sacked == 0) 7642 bbr->r_ctl.rc_sacklast = NULL; 7643 } else { 7644 /* 7645 * It was acked up to th_ack point for the first time 7646 */ 7647 struct bbr_sendmap lrsm; 7648 7649 memcpy(&lrsm, rsm, sizeof(struct bbr_sendmap)); 7650 lrsm.r_end = th_ack; 7651 bbr_update_rtt(tp, bbr, &lrsm, to, cts, BBR_CUM_ACKED, th_ack); 7652 } 7653 if ((rsm->r_flags & BBR_MARKED_LOST) && 7654 ((rsm->r_flags & BBR_ACKED) == 0)) { 7655 /* 7656 * It was marked lost and partly ack'd now 7657 * for the first time. We lower the rc_lost_bytes 7658 * and still leave it MARKED. 7659 */ 7660 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start; 7661 } 7662 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED); 7663 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; 7664 rsm->r_rtr_bytes = 0; 7665 /* adjust packet count */ 7666 rsm->r_start = th_ack; 7667 proc_sack: 7668 /* Check for reneging */ 7669 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 7670 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) { 7671 /* 7672 * The peer has moved snd_una up to the edge of this send, 7673 * i.e. one that it had previously acked. The only way that 7674 * can be true if the peer threw away data (space issues) 7675 * that it had previously sacked (else it would have given 7676 * us snd_una up to (rsm->r_end). We need to undo the acked 7677 * markings here. 7678 * 7679 * Note we have to look to make sure th_ack is our 7680 * rsm->r_start in case we get an old ack where th_ack is 7681 * behind snd_una. 7682 */ 7683 bbr_peer_reneges(bbr, rsm, th->th_ack); 7684 } 7685 if ((to->to_flags & TOF_SACK) == 0) { 7686 /* We are done nothing left to log */ 7687 goto out; 7688 } 7689 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next); 7690 if (rsm) { 7691 last_seq = rsm->r_end; 7692 } else { 7693 last_seq = tp->snd_max; 7694 } 7695 /* Sack block processing */ 7696 if (SEQ_GT(th_ack, tp->snd_una)) 7697 ack_point = th_ack; 7698 else 7699 ack_point = tp->snd_una; 7700 for (i = 0; i < to->to_nsacks; i++) { 7701 bcopy((to->to_sacks + i * TCPOLEN_SACK), 7702 &sack, sizeof(sack)); 7703 sack.start = ntohl(sack.start); 7704 sack.end = ntohl(sack.end); 7705 if (SEQ_GT(sack.end, sack.start) && 7706 SEQ_GT(sack.start, ack_point) && 7707 SEQ_LT(sack.start, tp->snd_max) && 7708 SEQ_GT(sack.end, ack_point) && 7709 SEQ_LEQ(sack.end, tp->snd_max)) { 7710 if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) && 7711 (SEQ_LT(sack.end, last_seq)) && 7712 ((sack.end - sack.start) < (p_maxseg / 8))) { 7713 /* 7714 * Not the last piece and its smaller than 7715 * 1/8th of a p_maxseg. We ignore this. 7716 */ 7717 BBR_STAT_INC(bbr_runt_sacks); 7718 continue; 7719 } 7720 sack_blocks[num_sack_blks] = sack; 7721 num_sack_blks++; 7722 #ifdef NETFLIX_STATS 7723 } else if (SEQ_LEQ(sack.start, th_ack) && 7724 SEQ_LEQ(sack.end, th_ack)) { 7725 /* 7726 * Its a D-SACK block. 7727 */ 7728 tcp_record_dsack(sack.start, sack.end); 7729 #endif 7730 } 7731 } 7732 if (num_sack_blks == 0) 7733 goto out; 7734 /* 7735 * Sort the SACK blocks so we can update the rack scoreboard with 7736 * just one pass. 7737 */ 7738 new_sb = sack_filter_blks(&bbr->r_ctl.bbr_sf, sack_blocks, 7739 num_sack_blks, th->th_ack); 7740 ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks); 7741 BBR_STAT_ADD(bbr_sack_blocks, num_sack_blks); 7742 BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb)); 7743 num_sack_blks = new_sb; 7744 if (num_sack_blks < 2) { 7745 goto do_sack_work; 7746 } 7747 /* Sort the sacks */ 7748 for (i = 0; i < num_sack_blks; i++) { 7749 for (j = i + 1; j < num_sack_blks; j++) { 7750 if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) { 7751 sack = sack_blocks[i]; 7752 sack_blocks[i] = sack_blocks[j]; 7753 sack_blocks[j] = sack; 7754 } 7755 } 7756 } 7757 /* 7758 * Now are any of the sack block ends the same (yes some 7759 * implememtations send these)? 7760 */ 7761 again: 7762 if (num_sack_blks > 1) { 7763 for (i = 0; i < num_sack_blks; i++) { 7764 for (j = i + 1; j < num_sack_blks; j++) { 7765 if (sack_blocks[i].end == sack_blocks[j].end) { 7766 /* 7767 * Ok these two have the same end we 7768 * want the smallest end and then 7769 * throw away the larger and start 7770 * again. 7771 */ 7772 if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) { 7773 /* 7774 * The second block covers 7775 * more area use that 7776 */ 7777 sack_blocks[i].start = sack_blocks[j].start; 7778 } 7779 /* 7780 * Now collapse out the dup-sack and 7781 * lower the count 7782 */ 7783 for (k = (j + 1); k < num_sack_blks; k++) { 7784 sack_blocks[j].start = sack_blocks[k].start; 7785 sack_blocks[j].end = sack_blocks[k].end; 7786 j++; 7787 } 7788 num_sack_blks--; 7789 goto again; 7790 } 7791 } 7792 } 7793 } 7794 do_sack_work: 7795 rsm = bbr->r_ctl.rc_sacklast; 7796 for (i = 0; i < num_sack_blks; i++) { 7797 acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts); 7798 if (acked) { 7799 bbr->r_wanted_output = 1; 7800 changed += acked; 7801 sack_changed += acked; 7802 } 7803 } 7804 out: 7805 *prev_acked = p_acked; 7806 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) { 7807 /* 7808 * Ok we have a high probability that we need to go in to 7809 * recovery since we have data sack'd 7810 */ 7811 struct bbr_sendmap *rsm; 7812 7813 rsm = bbr_check_recovery_mode(tp, bbr, cts); 7814 if (rsm) { 7815 /* Enter recovery */ 7816 entered_recovery = 1; 7817 bbr->r_wanted_output = 1; 7818 /* 7819 * When we enter recovery we need to assure we send 7820 * one packet. 7821 */ 7822 if (bbr->r_ctl.rc_resend == NULL) { 7823 bbr->r_ctl.rc_resend = rsm; 7824 } 7825 } 7826 } 7827 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) { 7828 /* 7829 * See if we need to rack-retransmit anything if so set it 7830 * up as the thing to resend assuming something else is not 7831 * already in that position. 7832 */ 7833 if (bbr->r_ctl.rc_resend == NULL) { 7834 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); 7835 } 7836 } 7837 /* 7838 * We return the amount that changed via sack, this is used by the 7839 * ack-received code to augment what was changed between th_ack <-> 7840 * snd_una. 7841 */ 7842 return (sack_changed); 7843 } 7844 7845 static void 7846 bbr_strike_dupack(struct tcp_bbr *bbr) 7847 { 7848 struct bbr_sendmap *rsm; 7849 7850 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap); 7851 if (rsm && (rsm->r_dupack < 0xff)) { 7852 rsm->r_dupack++; 7853 if (rsm->r_dupack >= DUP_ACK_THRESHOLD) 7854 bbr->r_wanted_output = 1; 7855 } 7856 } 7857 7858 /* 7859 * Return value of 1, we do not need to call bbr_process_data(). 7860 * return value of 0, bbr_process_data can be called. 7861 * For ret_val if its 0 the TCB is locked and valid, if its non-zero 7862 * its unlocked and probably unsafe to touch the TCB. 7863 */ 7864 static int 7865 bbr_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, 7866 struct tcpcb *tp, struct tcpopt *to, 7867 uint32_t tiwin, int32_t tlen, 7868 int32_t * ofia, int32_t thflags, int32_t * ret_val) 7869 { 7870 int32_t ourfinisacked = 0; 7871 int32_t acked_amount; 7872 uint16_t nsegs; 7873 int32_t acked; 7874 uint32_t lost, sack_changed = 0; 7875 struct mbuf *mfree; 7876 struct tcp_bbr *bbr; 7877 uint32_t prev_acked = 0; 7878 7879 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 7880 lost = bbr->r_ctl.rc_lost; 7881 nsegs = max(1, m->m_pkthdr.lro_nsegs); 7882 if (SEQ_GT(th->th_ack, tp->snd_max)) { 7883 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val); 7884 bbr->r_wanted_output = 1; 7885 return (1); 7886 } 7887 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { 7888 /* Process the ack */ 7889 if (bbr->rc_in_persist) 7890 tp->t_rxtshift = 0; 7891 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd)) 7892 bbr_strike_dupack(bbr); 7893 sack_changed = bbr_log_ack(tp, to, th, &prev_acked); 7894 } 7895 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost)); 7896 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { 7897 /* 7898 * Old ack, behind the last one rcv'd or a duplicate ack 7899 * with SACK info. 7900 */ 7901 if (th->th_ack == tp->snd_una) { 7902 bbr_ack_received(tp, bbr, th, 0, sack_changed, prev_acked, __LINE__, 0); 7903 if (bbr->r_state == TCPS_SYN_SENT) { 7904 /* 7905 * Special case on where we sent SYN. When 7906 * the SYN-ACK is processed in syn_sent 7907 * state it bumps the snd_una. This causes 7908 * us to hit here even though we did ack 1 7909 * byte. 7910 * 7911 * Go through the nothing left case so we 7912 * send data. 7913 */ 7914 goto nothing_left; 7915 } 7916 } 7917 return (0); 7918 } 7919 /* 7920 * If we reach this point, ACK is not a duplicate, i.e., it ACKs 7921 * something we sent. 7922 */ 7923 if (tp->t_flags & TF_NEEDSYN) { 7924 /* 7925 * T/TCP: Connection was half-synchronized, and our SYN has 7926 * been ACK'd (so connection is now fully synchronized). Go 7927 * to non-starred state, increment snd_una for ACK of SYN, 7928 * and check if we can do window scaling. 7929 */ 7930 tp->t_flags &= ~TF_NEEDSYN; 7931 tp->snd_una++; 7932 /* Do window scaling? */ 7933 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 7934 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 7935 tp->rcv_scale = tp->request_r_scale; 7936 /* Send window already scaled. */ 7937 } 7938 } 7939 INP_WLOCK_ASSERT(tp->t_inpcb); 7940 7941 acked = BYTES_THIS_ACK(tp, th); 7942 KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs); 7943 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked); 7944 7945 /* 7946 * If we just performed our first retransmit, and the ACK arrives 7947 * within our recovery window, then it was a mistake to do the 7948 * retransmit in the first place. Recover our original cwnd and 7949 * ssthresh, and proceed to transmit where we left off. 7950 */ 7951 if (tp->t_flags & TF_PREVVALID) { 7952 tp->t_flags &= ~TF_PREVVALID; 7953 if (tp->t_rxtshift == 1 && 7954 (int)(ticks - tp->t_badrxtwin) < 0) 7955 bbr_cong_signal(tp, th, CC_RTO_ERR, NULL); 7956 } 7957 SOCKBUF_LOCK(&so->so_snd); 7958 acked_amount = min(acked, (int)sbavail(&so->so_snd)); 7959 tp->snd_wnd -= acked_amount; 7960 mfree = sbcut_locked(&so->so_snd, acked_amount); 7961 /* NB: sowwakeup_locked() does an implicit unlock. */ 7962 sowwakeup_locked(so); 7963 m_freem(mfree); 7964 if (SEQ_GT(th->th_ack, tp->snd_una)) { 7965 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp)); 7966 } 7967 tp->snd_una = th->th_ack; 7968 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - lost)); 7969 if (IN_RECOVERY(tp->t_flags)) { 7970 if (SEQ_LT(th->th_ack, tp->snd_recover) && 7971 (SEQ_LT(th->th_ack, tp->snd_max))) { 7972 tcp_bbr_partialack(tp); 7973 } else { 7974 bbr_post_recovery(tp); 7975 } 7976 } 7977 if (SEQ_GT(tp->snd_una, tp->snd_recover)) { 7978 tp->snd_recover = tp->snd_una; 7979 } 7980 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { 7981 tp->snd_nxt = tp->snd_max; 7982 } 7983 if (tp->snd_una == tp->snd_max) { 7984 /* Nothing left outstanding */ 7985 nothing_left: 7986 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__); 7987 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0) 7988 bbr->rc_tp->t_acktime = 0; 7989 if ((sbused(&so->so_snd) == 0) && 7990 (tp->t_flags & TF_SENTFIN)) { 7991 ourfinisacked = 1; 7992 } 7993 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); 7994 if (bbr->rc_in_persist == 0) { 7995 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime; 7996 } 7997 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); 7998 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime); 7999 /* 8000 * We invalidate the last ack here since we 8001 * don't want to transfer forward the time 8002 * for our sum's calculations. 8003 */ 8004 if ((tp->t_state >= TCPS_FIN_WAIT_1) && 8005 (sbavail(&so->so_snd) == 0) && 8006 (tp->t_flags2 & TF2_DROP_AF_DATA)) { 8007 /* 8008 * The socket was gone and the peer sent data, time 8009 * to reset him. 8010 */ 8011 *ret_val = 1; 8012 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE); 8013 /* tcp_close will kill the inp pre-log the Reset */ 8014 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 8015 tp = tcp_close(tp); 8016 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen); 8017 BBR_STAT_INC(bbr_dropped_af_data); 8018 return (1); 8019 } 8020 /* Set need output so persist might get set */ 8021 bbr->r_wanted_output = 1; 8022 } 8023 if (ofia) 8024 *ofia = ourfinisacked; 8025 return (0); 8026 } 8027 8028 static void 8029 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line) 8030 { 8031 if (bbr->rc_in_persist == 0) { 8032 bbr_timer_cancel(bbr, __LINE__, cts); 8033 bbr->r_ctl.rc_last_delay_val = 0; 8034 tp->t_rxtshift = 0; 8035 bbr->rc_in_persist = 1; 8036 bbr->r_ctl.rc_went_idle_time = cts; 8037 /* We should be capped when rw went to 0 but just in case */ 8038 bbr_log_type_pesist(bbr, cts, 0, line, 1); 8039 /* Time freezes for the state, so do the accounting now */ 8040 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { 8041 uint32_t time_in; 8042 8043 time_in = cts - bbr->r_ctl.rc_bbr_state_time; 8044 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { 8045 int32_t idx; 8046 8047 idx = bbr_state_val(bbr); 8048 counter_u64_add(bbr_state_time[(idx + 5)], time_in); 8049 } else { 8050 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); 8051 } 8052 } 8053 bbr->r_ctl.rc_bbr_state_time = cts; 8054 } 8055 } 8056 8057 static void 8058 bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time) 8059 { 8060 /* 8061 * Note that if idle time does not exceed our 8062 * threshold, we do nothing continuing the state 8063 * transitions we were last walking through. 8064 */ 8065 if (idle_time >= bbr_idle_restart_threshold) { 8066 if (bbr->rc_use_idle_restart) { 8067 bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT; 8068 /* 8069 * Set our target using BBR_UNIT, so 8070 * we increase at a dramatic rate but 8071 * we stop when we get the pipe 8072 * full again for our current b/w estimate. 8073 */ 8074 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; 8075 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; 8076 bbr_set_state_target(bbr, __LINE__); 8077 /* Now setup our gains to ramp up */ 8078 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; 8079 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; 8080 bbr_log_type_statechange(bbr, cts, __LINE__); 8081 } else { 8082 bbr_substate_change(bbr, cts, __LINE__, 1); 8083 } 8084 } 8085 } 8086 8087 static void 8088 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line) 8089 { 8090 uint32_t idle_time; 8091 8092 if (bbr->rc_in_persist == 0) 8093 return; 8094 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time); 8095 bbr->rc_in_persist = 0; 8096 bbr->rc_hit_state_1 = 0; 8097 bbr->r_ctl.rc_del_time = cts; 8098 /* 8099 * We invalidate the last ack here since we 8100 * don't want to transfer forward the time 8101 * for our sum's calculations. 8102 */ 8103 if (bbr->rc_inp->inp_in_hpts) { 8104 tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); 8105 bbr->rc_timer_first = 0; 8106 bbr->r_ctl.rc_hpts_flags = 0; 8107 bbr->r_ctl.rc_last_delay_val = 0; 8108 bbr->r_ctl.rc_hptsi_agg_delay = 0; 8109 bbr->r_agg_early_set = 0; 8110 bbr->r_ctl.rc_agg_early = 0; 8111 } 8112 bbr_log_type_pesist(bbr, cts, idle_time, line, 0); 8113 if (idle_time >= bbr_rtt_probe_time) { 8114 /* 8115 * This qualifies as a RTT_PROBE session since we drop the 8116 * data outstanding to nothing and waited more than 8117 * bbr_rtt_probe_time. 8118 */ 8119 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_PERSIST, 0); 8120 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts; 8121 } 8122 tp->t_rxtshift = 0; 8123 /* 8124 * If in probeBW and we have persisted more than an RTT lets do 8125 * special handling. 8126 */ 8127 /* Force a time based epoch */ 8128 bbr_set_epoch(bbr, cts, __LINE__); 8129 /* 8130 * Setup the lost so we don't count anything against the guy 8131 * we have been stuck with during persists. 8132 */ 8133 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 8134 /* Time un-freezes for the state */ 8135 bbr->r_ctl.rc_bbr_state_time = cts; 8136 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) || 8137 (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) { 8138 /* 8139 * If we are going back to probe-bw 8140 * or probe_rtt, we may need to possibly 8141 * do a fast restart. 8142 */ 8143 bbr_restart_after_idle(bbr, cts, idle_time); 8144 } 8145 } 8146 8147 static void 8148 bbr_collapsed_window(struct tcp_bbr *bbr) 8149 { 8150 /* 8151 * Now we must walk the 8152 * send map and divide the 8153 * ones left stranded. These 8154 * guys can't cause us to abort 8155 * the connection and are really 8156 * "unsent". However if a buggy 8157 * client actually did keep some 8158 * of the data i.e. collapsed the win 8159 * and refused to ack and then opened 8160 * the win and acked that data. We would 8161 * get into an ack war, the simplier 8162 * method then of just pretending we 8163 * did not send those segments something 8164 * won't work. 8165 */ 8166 struct bbr_sendmap *rsm, *nrsm; 8167 tcp_seq max_seq; 8168 uint32_t maxseg; 8169 int can_split = 0; 8170 int fnd = 0; 8171 8172 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options; 8173 max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd; 8174 bbr_log_type_rwnd_collapse(bbr, max_seq, 1, 0); 8175 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { 8176 /* Find the first seq past or at maxseq */ 8177 if (rsm->r_flags & BBR_RWND_COLLAPSED) 8178 rsm->r_flags &= ~BBR_RWND_COLLAPSED; 8179 if (SEQ_GEQ(max_seq, rsm->r_start) && 8180 SEQ_GEQ(rsm->r_end, max_seq)) { 8181 fnd = 1; 8182 break; 8183 } 8184 } 8185 bbr->rc_has_collapsed = 0; 8186 if (!fnd) { 8187 /* Nothing to do strange */ 8188 return; 8189 } 8190 /* 8191 * Now can we split? 8192 * 8193 * We don't want to split if splitting 8194 * would generate too many small segments 8195 * less we let an attacker fragment our 8196 * send_map and leave us out of memory. 8197 */ 8198 if ((max_seq != rsm->r_start) && 8199 (max_seq != rsm->r_end)){ 8200 /* can we split? */ 8201 int res1, res2; 8202 8203 res1 = max_seq - rsm->r_start; 8204 res2 = rsm->r_end - max_seq; 8205 if ((res1 >= (maxseg/8)) && 8206 (res2 >= (maxseg/8))) { 8207 /* No small pieces here */ 8208 can_split = 1; 8209 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) { 8210 /* We are under the limit */ 8211 can_split = 1; 8212 } 8213 } 8214 /* Ok do we need to split this rsm? */ 8215 if (max_seq == rsm->r_start) { 8216 /* It's this guy no split required */ 8217 nrsm = rsm; 8218 } else if (max_seq == rsm->r_end) { 8219 /* It's the next one no split required. */ 8220 nrsm = TAILQ_NEXT(rsm, r_next); 8221 if (nrsm == NULL) { 8222 /* Huh? */ 8223 return; 8224 } 8225 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) { 8226 /* yep we need to split it */ 8227 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT); 8228 if (nrsm == NULL) { 8229 /* failed XXXrrs what can we do mark the whole? */ 8230 nrsm = rsm; 8231 goto no_split; 8232 } 8233 /* Clone it */ 8234 bbr_log_type_rwnd_collapse(bbr, max_seq, 3, 0); 8235 bbr_clone_rsm(bbr, nrsm, rsm, max_seq); 8236 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next); 8237 if (rsm->r_in_tmap) { 8238 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 8239 nrsm->r_in_tmap = 1; 8240 } 8241 } else { 8242 /* 8243 * Split not allowed just start here just 8244 * use this guy. 8245 */ 8246 nrsm = rsm; 8247 } 8248 no_split: 8249 BBR_STAT_INC(bbr_collapsed_win); 8250 /* reuse fnd as a count */ 8251 fnd = 0; 8252 TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) { 8253 nrsm->r_flags |= BBR_RWND_COLLAPSED; 8254 fnd++; 8255 bbr->rc_has_collapsed = 1; 8256 } 8257 bbr_log_type_rwnd_collapse(bbr, max_seq, 4, fnd); 8258 } 8259 8260 static void 8261 bbr_un_collapse_window(struct tcp_bbr *bbr) 8262 { 8263 struct bbr_sendmap *rsm; 8264 int cleared = 0; 8265 8266 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) { 8267 if (rsm->r_flags & BBR_RWND_COLLAPSED) { 8268 /* Clear the flag */ 8269 rsm->r_flags &= ~BBR_RWND_COLLAPSED; 8270 cleared++; 8271 } else 8272 break; 8273 } 8274 bbr_log_type_rwnd_collapse(bbr, 8275 (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared); 8276 bbr->rc_has_collapsed = 0; 8277 } 8278 8279 /* 8280 * Return value of 1, the TCB is unlocked and most 8281 * likely gone, return value of 0, the TCB is still 8282 * locked. 8283 */ 8284 static int 8285 bbr_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so, 8286 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, 8287 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) 8288 { 8289 /* 8290 * Update window information. Don't look at window if no ACK: TAC's 8291 * send garbage on first SYN. 8292 */ 8293 uint16_t nsegs; 8294 int32_t tfo_syn; 8295 struct tcp_bbr *bbr; 8296 8297 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 8298 INP_WLOCK_ASSERT(tp->t_inpcb); 8299 nsegs = max(1, m->m_pkthdr.lro_nsegs); 8300 if ((thflags & TH_ACK) && 8301 (SEQ_LT(tp->snd_wl1, th->th_seq) || 8302 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || 8303 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { 8304 /* keep track of pure window updates */ 8305 if (tlen == 0 && 8306 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) 8307 KMOD_TCPSTAT_INC(tcps_rcvwinupd); 8308 tp->snd_wnd = tiwin; 8309 tp->snd_wl1 = th->th_seq; 8310 tp->snd_wl2 = th->th_ack; 8311 if (tp->snd_wnd > tp->max_sndwnd) 8312 tp->max_sndwnd = tp->snd_wnd; 8313 bbr->r_wanted_output = 1; 8314 } else if (thflags & TH_ACK) { 8315 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) { 8316 tp->snd_wnd = tiwin; 8317 tp->snd_wl1 = th->th_seq; 8318 tp->snd_wl2 = th->th_ack; 8319 } 8320 } 8321 if (tp->snd_wnd < ctf_outstanding(tp)) 8322 /* The peer collapsed its window on us */ 8323 bbr_collapsed_window(bbr); 8324 else if (bbr->rc_has_collapsed) 8325 bbr_un_collapse_window(bbr); 8326 /* Was persist timer active and now we have window space? */ 8327 if ((bbr->rc_in_persist != 0) && 8328 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2), 8329 bbr_minseg(bbr)))) { 8330 /* 8331 * Make the rate persist at end of persist mode if idle long 8332 * enough 8333 */ 8334 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); 8335 8336 /* Make sure we output to start the timer */ 8337 bbr->r_wanted_output = 1; 8338 } 8339 /* Do we need to enter persist? */ 8340 if ((bbr->rc_in_persist == 0) && 8341 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && 8342 TCPS_HAVEESTABLISHED(tp->t_state) && 8343 (tp->snd_max == tp->snd_una) && 8344 sbavail(&tp->t_inpcb->inp_socket->so_snd) && 8345 (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) { 8346 /* No send window.. we must enter persist */ 8347 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); 8348 } 8349 if (tp->t_flags2 & TF2_DROP_AF_DATA) { 8350 m_freem(m); 8351 return (0); 8352 } 8353 /* 8354 * We don't support urgent data but 8355 * drag along the up just to make sure 8356 * if there is a stack switch no one 8357 * is surprised. 8358 */ 8359 tp->rcv_up = tp->rcv_nxt; 8360 INP_WLOCK_ASSERT(tp->t_inpcb); 8361 8362 /* 8363 * Process the segment text, merging it into the TCP sequencing 8364 * queue, and arranging for acknowledgment of receipt if necessary. 8365 * This process logically involves adjusting tp->rcv_wnd as data is 8366 * presented to the user (this happens in tcp_usrreq.c, case 8367 * PRU_RCVD). If a FIN has already been received on this connection 8368 * then we just ignore the text. 8369 */ 8370 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && 8371 IS_FASTOPEN(tp->t_flags)); 8372 if ((tlen || (thflags & TH_FIN) || tfo_syn) && 8373 TCPS_HAVERCVDFIN(tp->t_state) == 0) { 8374 tcp_seq save_start = th->th_seq; 8375 tcp_seq save_rnxt = tp->rcv_nxt; 8376 int save_tlen = tlen; 8377 8378 m_adj(m, drop_hdrlen); /* delayed header drop */ 8379 /* 8380 * Insert segment which includes th into TCP reassembly 8381 * queue with control block tp. Set thflags to whether 8382 * reassembly now includes a segment with FIN. This handles 8383 * the common case inline (segment is the next to be 8384 * received on an established connection, and the queue is 8385 * empty), avoiding linkage into and removal from the queue 8386 * and repetition of various conversions. Set DELACK for 8387 * segments received in order, but ack immediately when 8388 * segments are out of order (so fast retransmit can work). 8389 */ 8390 if (th->th_seq == tp->rcv_nxt && 8391 SEGQ_EMPTY(tp) && 8392 (TCPS_HAVEESTABLISHED(tp->t_state) || 8393 tfo_syn)) { 8394 #ifdef NETFLIX_SB_LIMITS 8395 u_int mcnt, appended; 8396 8397 if (so->so_rcv.sb_shlim) { 8398 mcnt = m_memcnt(m); 8399 appended = 0; 8400 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, 8401 CFO_NOSLEEP, NULL) == false) { 8402 counter_u64_add(tcp_sb_shlim_fails, 1); 8403 m_freem(m); 8404 return (0); 8405 } 8406 } 8407 8408 #endif 8409 if (DELAY_ACK(tp, bbr, nsegs) || tfo_syn) { 8410 bbr->bbr_segs_rcvd += max(1, nsegs); 8411 tp->t_flags |= TF_DELACK; 8412 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); 8413 } else { 8414 bbr->r_wanted_output = 1; 8415 tp->t_flags |= TF_ACKNOW; 8416 } 8417 tp->rcv_nxt += tlen; 8418 thflags = th->th_flags & TH_FIN; 8419 KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs); 8420 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); 8421 SOCKBUF_LOCK(&so->so_rcv); 8422 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) 8423 m_freem(m); 8424 else 8425 #ifdef NETFLIX_SB_LIMITS 8426 appended = 8427 #endif 8428 sbappendstream_locked(&so->so_rcv, m, 0); 8429 /* NB: sorwakeup_locked() does an implicit unlock. */ 8430 sorwakeup_locked(so); 8431 #ifdef NETFLIX_SB_LIMITS 8432 if (so->so_rcv.sb_shlim && appended != mcnt) 8433 counter_fo_release(so->so_rcv.sb_shlim, 8434 mcnt - appended); 8435 #endif 8436 } else { 8437 /* 8438 * XXX: Due to the header drop above "th" is 8439 * theoretically invalid by now. Fortunately 8440 * m_adj() doesn't actually frees any mbufs when 8441 * trimming from the head. 8442 */ 8443 tcp_seq temp = save_start; 8444 thflags = tcp_reass(tp, th, &temp, &tlen, m); 8445 tp->t_flags |= TF_ACKNOW; 8446 } 8447 if ((tp->t_flags & TF_SACK_PERMIT) && (save_tlen > 0)) { 8448 if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) { 8449 /* 8450 * DSACK actually handled in the fastpath 8451 * above. 8452 */ 8453 tcp_update_sack_list(tp, save_start, 8454 save_start + save_tlen); 8455 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) { 8456 if ((tp->rcv_numsacks >= 1) && 8457 (tp->sackblks[0].end == save_start)) { 8458 /* 8459 * Partial overlap, recorded at todrop 8460 * above. 8461 */ 8462 tcp_update_sack_list(tp, 8463 tp->sackblks[0].start, 8464 tp->sackblks[0].end); 8465 } else { 8466 tcp_update_dsack_list(tp, save_start, 8467 save_start + save_tlen); 8468 } 8469 } else if (tlen >= save_tlen) { 8470 /* Update of sackblks. */ 8471 tcp_update_dsack_list(tp, save_start, 8472 save_start + save_tlen); 8473 } else if (tlen > 0) { 8474 tcp_update_dsack_list(tp, save_start, 8475 save_start + tlen); 8476 } 8477 } 8478 } else { 8479 m_freem(m); 8480 thflags &= ~TH_FIN; 8481 } 8482 8483 /* 8484 * If FIN is received ACK the FIN and let the user know that the 8485 * connection is closing. 8486 */ 8487 if (thflags & TH_FIN) { 8488 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { 8489 socantrcvmore(so); 8490 /* 8491 * If connection is half-synchronized (ie NEEDSYN 8492 * flag on) then delay ACK, so it may be piggybacked 8493 * when SYN is sent. Otherwise, since we received a 8494 * FIN then no more input can be expected, send ACK 8495 * now. 8496 */ 8497 if (tp->t_flags & TF_NEEDSYN) { 8498 tp->t_flags |= TF_DELACK; 8499 bbr_timer_cancel(bbr, 8500 __LINE__, bbr->r_ctl.rc_rcvtime); 8501 } else { 8502 tp->t_flags |= TF_ACKNOW; 8503 } 8504 tp->rcv_nxt++; 8505 } 8506 switch (tp->t_state) { 8507 8508 /* 8509 * In SYN_RECEIVED and ESTABLISHED STATES enter the 8510 * CLOSE_WAIT state. 8511 */ 8512 case TCPS_SYN_RECEIVED: 8513 tp->t_starttime = ticks; 8514 /* FALLTHROUGH */ 8515 case TCPS_ESTABLISHED: 8516 tcp_state_change(tp, TCPS_CLOSE_WAIT); 8517 break; 8518 8519 /* 8520 * If still in FIN_WAIT_1 STATE FIN has not been 8521 * acked so enter the CLOSING state. 8522 */ 8523 case TCPS_FIN_WAIT_1: 8524 tcp_state_change(tp, TCPS_CLOSING); 8525 break; 8526 8527 /* 8528 * In FIN_WAIT_2 state enter the TIME_WAIT state, 8529 * starting the time-wait timer, turning off the 8530 * other standard timers. 8531 */ 8532 case TCPS_FIN_WAIT_2: 8533 bbr->rc_timer_first = 1; 8534 bbr_timer_cancel(bbr, 8535 __LINE__, bbr->r_ctl.rc_rcvtime); 8536 INP_WLOCK_ASSERT(tp->t_inpcb); 8537 tcp_twstart(tp); 8538 return (1); 8539 } 8540 } 8541 /* 8542 * Return any desired output. 8543 */ 8544 if ((tp->t_flags & TF_ACKNOW) || 8545 (sbavail(&so->so_snd) > ctf_outstanding(tp))) { 8546 bbr->r_wanted_output = 1; 8547 } 8548 INP_WLOCK_ASSERT(tp->t_inpcb); 8549 return (0); 8550 } 8551 8552 /* 8553 * Here nothing is really faster, its just that we 8554 * have broken out the fast-data path also just like 8555 * the fast-ack. Return 1 if we processed the packet 8556 * return 0 if you need to take the "slow-path". 8557 */ 8558 static int 8559 bbr_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, 8560 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 8561 uint32_t tiwin, int32_t nxt_pkt) 8562 { 8563 uint16_t nsegs; 8564 int32_t newsize = 0; /* automatic sockbuf scaling */ 8565 struct tcp_bbr *bbr; 8566 #ifdef NETFLIX_SB_LIMITS 8567 u_int mcnt, appended; 8568 #endif 8569 #ifdef TCPDEBUG 8570 /* 8571 * The size of tcp_saveipgen must be the size of the max ip header, 8572 * now IPv6. 8573 */ 8574 u_char tcp_saveipgen[IP6_HDR_LEN]; 8575 struct tcphdr tcp_savetcp; 8576 short ostate = 0; 8577 8578 #endif 8579 /* On the hpts and we would have called output */ 8580 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 8581 8582 /* 8583 * If last ACK falls within this segment's sequence numbers, record 8584 * the timestamp. NOTE that the test is modified according to the 8585 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26). 8586 */ 8587 if (bbr->r_ctl.rc_resend != NULL) { 8588 return (0); 8589 } 8590 if (tiwin && tiwin != tp->snd_wnd) { 8591 return (0); 8592 } 8593 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) { 8594 return (0); 8595 } 8596 if (__predict_false((to->to_flags & TOF_TS) && 8597 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) { 8598 return (0); 8599 } 8600 if (__predict_false((th->th_ack != tp->snd_una))) { 8601 return (0); 8602 } 8603 if (__predict_false(tlen > sbspace(&so->so_rcv))) { 8604 return (0); 8605 } 8606 if ((to->to_flags & TOF_TS) != 0 && 8607 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { 8608 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 8609 tp->ts_recent = to->to_tsval; 8610 } 8611 /* 8612 * This is a pure, in-sequence data packet with nothing on the 8613 * reassembly queue and we have enough buffer space to take it. 8614 */ 8615 nsegs = max(1, m->m_pkthdr.lro_nsegs); 8616 8617 #ifdef NETFLIX_SB_LIMITS 8618 if (so->so_rcv.sb_shlim) { 8619 mcnt = m_memcnt(m); 8620 appended = 0; 8621 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, 8622 CFO_NOSLEEP, NULL) == false) { 8623 counter_u64_add(tcp_sb_shlim_fails, 1); 8624 m_freem(m); 8625 return (1); 8626 } 8627 } 8628 #endif 8629 /* Clean receiver SACK report if present */ 8630 if (tp->rcv_numsacks) 8631 tcp_clean_sackreport(tp); 8632 KMOD_TCPSTAT_INC(tcps_preddat); 8633 tp->rcv_nxt += tlen; 8634 /* 8635 * Pull snd_wl1 up to prevent seq wrap relative to th_seq. 8636 */ 8637 tp->snd_wl1 = th->th_seq; 8638 /* 8639 * Pull rcv_up up to prevent seq wrap relative to rcv_nxt. 8640 */ 8641 tp->rcv_up = tp->rcv_nxt; 8642 KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs); 8643 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); 8644 #ifdef TCPDEBUG 8645 if (so->so_options & SO_DEBUG) 8646 tcp_trace(TA_INPUT, ostate, tp, 8647 (void *)tcp_saveipgen, &tcp_savetcp, 0); 8648 #endif 8649 newsize = tcp_autorcvbuf(m, th, so, tp, tlen); 8650 8651 /* Add data to socket buffer. */ 8652 SOCKBUF_LOCK(&so->so_rcv); 8653 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 8654 m_freem(m); 8655 } else { 8656 /* 8657 * Set new socket buffer size. Give up when limit is 8658 * reached. 8659 */ 8660 if (newsize) 8661 if (!sbreserve_locked(&so->so_rcv, 8662 newsize, so, NULL)) 8663 so->so_rcv.sb_flags &= ~SB_AUTOSIZE; 8664 m_adj(m, drop_hdrlen); /* delayed header drop */ 8665 8666 #ifdef NETFLIX_SB_LIMITS 8667 appended = 8668 #endif 8669 sbappendstream_locked(&so->so_rcv, m, 0); 8670 ctf_calc_rwin(so, tp); 8671 } 8672 /* NB: sorwakeup_locked() does an implicit unlock. */ 8673 sorwakeup_locked(so); 8674 #ifdef NETFLIX_SB_LIMITS 8675 if (so->so_rcv.sb_shlim && mcnt != appended) 8676 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended); 8677 #endif 8678 if (DELAY_ACK(tp, bbr, nsegs)) { 8679 bbr->bbr_segs_rcvd += max(1, nsegs); 8680 tp->t_flags |= TF_DELACK; 8681 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); 8682 } else { 8683 bbr->r_wanted_output = 1; 8684 tp->t_flags |= TF_ACKNOW; 8685 } 8686 return (1); 8687 } 8688 8689 /* 8690 * This subfunction is used to try to highly optimize the 8691 * fast path. We again allow window updates that are 8692 * in sequence to remain in the fast-path. We also add 8693 * in the __predict's to attempt to help the compiler. 8694 * Note that if we return a 0, then we can *not* process 8695 * it and the caller should push the packet into the 8696 * slow-path. If we return 1, then all is well and 8697 * the packet is fully processed. 8698 */ 8699 static int 8700 bbr_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, 8701 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 8702 uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos) 8703 { 8704 int32_t acked; 8705 uint16_t nsegs; 8706 uint32_t sack_changed; 8707 #ifdef TCPDEBUG 8708 /* 8709 * The size of tcp_saveipgen must be the size of the max ip header, 8710 * now IPv6. 8711 */ 8712 u_char tcp_saveipgen[IP6_HDR_LEN]; 8713 struct tcphdr tcp_savetcp; 8714 short ostate = 0; 8715 8716 #endif 8717 uint32_t prev_acked = 0; 8718 struct tcp_bbr *bbr; 8719 8720 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { 8721 /* Old ack, behind (or duplicate to) the last one rcv'd */ 8722 return (0); 8723 } 8724 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) { 8725 /* Above what we have sent? */ 8726 return (0); 8727 } 8728 if (__predict_false(tiwin == 0)) { 8729 /* zero window */ 8730 return (0); 8731 } 8732 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) { 8733 /* We need a SYN or a FIN, unlikely.. */ 8734 return (0); 8735 } 8736 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) { 8737 /* Timestamp is behind .. old ack with seq wrap? */ 8738 return (0); 8739 } 8740 if (__predict_false(IN_RECOVERY(tp->t_flags))) { 8741 /* Still recovering */ 8742 return (0); 8743 } 8744 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 8745 if (__predict_false(bbr->r_ctl.rc_resend != NULL)) { 8746 /* We are retransmitting */ 8747 return (0); 8748 } 8749 if (__predict_false(bbr->rc_in_persist != 0)) { 8750 /* In persist mode */ 8751 return (0); 8752 } 8753 if (bbr->r_ctl.rc_sacked) { 8754 /* We have sack holes on our scoreboard */ 8755 return (0); 8756 } 8757 /* Ok if we reach here, we can process a fast-ack */ 8758 nsegs = max(1, m->m_pkthdr.lro_nsegs); 8759 sack_changed = bbr_log_ack(tp, to, th, &prev_acked); 8760 /* 8761 * We never detect loss in fast ack [we can't 8762 * have a sack and can't be in recovery so 8763 * we always pass 0 (nothing detected)]. 8764 */ 8765 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0); 8766 /* Did the window get updated? */ 8767 if (tiwin != tp->snd_wnd) { 8768 tp->snd_wnd = tiwin; 8769 tp->snd_wl1 = th->th_seq; 8770 if (tp->snd_wnd > tp->max_sndwnd) 8771 tp->max_sndwnd = tp->snd_wnd; 8772 } 8773 /* Do we need to exit persists? */ 8774 if ((bbr->rc_in_persist != 0) && 8775 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2), 8776 bbr_minseg(bbr)))) { 8777 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); 8778 bbr->r_wanted_output = 1; 8779 } 8780 /* Do we need to enter persists? */ 8781 if ((bbr->rc_in_persist == 0) && 8782 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && 8783 TCPS_HAVEESTABLISHED(tp->t_state) && 8784 (tp->snd_max == tp->snd_una) && 8785 sbavail(&tp->t_inpcb->inp_socket->so_snd) && 8786 (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) { 8787 /* No send window.. we must enter persist */ 8788 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); 8789 } 8790 /* 8791 * If last ACK falls within this segment's sequence numbers, record 8792 * the timestamp. NOTE that the test is modified according to the 8793 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26). 8794 */ 8795 if ((to->to_flags & TOF_TS) != 0 && 8796 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { 8797 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime; 8798 tp->ts_recent = to->to_tsval; 8799 } 8800 /* 8801 * This is a pure ack for outstanding data. 8802 */ 8803 KMOD_TCPSTAT_INC(tcps_predack); 8804 8805 /* 8806 * "bad retransmit" recovery. 8807 */ 8808 if (tp->t_flags & TF_PREVVALID) { 8809 tp->t_flags &= ~TF_PREVVALID; 8810 if (tp->t_rxtshift == 1 && 8811 (int)(ticks - tp->t_badrxtwin) < 0) 8812 bbr_cong_signal(tp, th, CC_RTO_ERR, NULL); 8813 } 8814 /* 8815 * Recalculate the transmit timer / rtt. 8816 * 8817 * Some boxes send broken timestamp replies during the SYN+ACK 8818 * phase, ignore timestamps of 0 or we could calculate a huge RTT 8819 * and blow up the retransmit timer. 8820 */ 8821 acked = BYTES_THIS_ACK(tp, th); 8822 8823 #ifdef TCP_HHOOK 8824 /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ 8825 hhook_run_tcp_est_in(tp, th, to); 8826 #endif 8827 8828 KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs); 8829 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked); 8830 sbdrop(&so->so_snd, acked); 8831 8832 if (SEQ_GT(th->th_ack, tp->snd_una)) 8833 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp)); 8834 tp->snd_una = th->th_ack; 8835 if (tp->snd_wnd < ctf_outstanding(tp)) 8836 /* The peer collapsed its window on us */ 8837 bbr_collapsed_window(bbr); 8838 else if (bbr->rc_has_collapsed) 8839 bbr_un_collapse_window(bbr); 8840 8841 if (SEQ_GT(tp->snd_una, tp->snd_recover)) { 8842 tp->snd_recover = tp->snd_una; 8843 } 8844 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, 0); 8845 /* 8846 * Pull snd_wl2 up to prevent seq wrap relative to th_ack. 8847 */ 8848 tp->snd_wl2 = th->th_ack; 8849 m_freem(m); 8850 /* 8851 * If all outstanding data are acked, stop retransmit timer, 8852 * otherwise restart timer using current (possibly backed-off) 8853 * value. If process is waiting for space, wakeup/selwakeup/signal. 8854 * If data are ready to send, let tcp_output decide between more 8855 * output or persist. 8856 */ 8857 #ifdef TCPDEBUG 8858 if (so->so_options & SO_DEBUG) 8859 tcp_trace(TA_INPUT, ostate, tp, 8860 (void *)tcp_saveipgen, 8861 &tcp_savetcp, 0); 8862 #endif 8863 /* Wake up the socket if we have room to write more */ 8864 sowwakeup(so); 8865 if (tp->snd_una == tp->snd_max) { 8866 /* Nothing left outstanding */ 8867 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__); 8868 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0) 8869 bbr->rc_tp->t_acktime = 0; 8870 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); 8871 if (bbr->rc_in_persist == 0) { 8872 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime; 8873 } 8874 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); 8875 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime); 8876 /* 8877 * We invalidate the last ack here since we 8878 * don't want to transfer forward the time 8879 * for our sum's calculations. 8880 */ 8881 bbr->r_wanted_output = 1; 8882 } 8883 if (sbavail(&so->so_snd)) { 8884 bbr->r_wanted_output = 1; 8885 } 8886 return (1); 8887 } 8888 8889 /* 8890 * Return value of 1, the TCB is unlocked and most 8891 * likely gone, return value of 0, the TCB is still 8892 * locked. 8893 */ 8894 static int 8895 bbr_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, 8896 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 8897 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 8898 { 8899 int32_t todrop; 8900 int32_t ourfinisacked = 0; 8901 struct tcp_bbr *bbr; 8902 int32_t ret_val = 0; 8903 8904 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 8905 ctf_calc_rwin(so, tp); 8906 /* 8907 * If the state is SYN_SENT: if seg contains an ACK, but not for our 8908 * SYN, drop the input. if seg contains a RST, then drop the 8909 * connection. if seg does not contain SYN, then drop it. Otherwise 8910 * this is an acceptable SYN segment initialize tp->rcv_nxt and 8911 * tp->irs if seg contains ack then advance tp->snd_una. BRR does 8912 * not support ECN so we will not say we are capable. if SYN has 8913 * been acked change to ESTABLISHED else SYN_RCVD state arrange for 8914 * segment to be acked (eventually) continue processing rest of 8915 * data/controls, beginning with URG 8916 */ 8917 if ((thflags & TH_ACK) && 8918 (SEQ_LEQ(th->th_ack, tp->iss) || 8919 SEQ_GT(th->th_ack, tp->snd_max))) { 8920 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 8921 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 8922 return (1); 8923 } 8924 if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) { 8925 TCP_PROBE5(connect__refused, NULL, tp, 8926 mtod(m, const char *), tp, th); 8927 tp = tcp_drop(tp, ECONNREFUSED); 8928 ctf_do_drop(m, tp); 8929 return (1); 8930 } 8931 if (thflags & TH_RST) { 8932 ctf_do_drop(m, tp); 8933 return (1); 8934 } 8935 if (!(thflags & TH_SYN)) { 8936 ctf_do_drop(m, tp); 8937 return (1); 8938 } 8939 tp->irs = th->th_seq; 8940 tcp_rcvseqinit(tp); 8941 if (thflags & TH_ACK) { 8942 int tfo_partial = 0; 8943 8944 KMOD_TCPSTAT_INC(tcps_connects); 8945 soisconnected(so); 8946 #ifdef MAC 8947 mac_socketpeer_set_from_mbuf(m, so); 8948 #endif 8949 /* Do window scaling on this connection? */ 8950 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 8951 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 8952 tp->rcv_scale = tp->request_r_scale; 8953 } 8954 tp->rcv_adv += min(tp->rcv_wnd, 8955 TCP_MAXWIN << tp->rcv_scale); 8956 /* 8957 * If not all the data that was sent in the TFO SYN 8958 * has been acked, resend the remainder right away. 8959 */ 8960 if (IS_FASTOPEN(tp->t_flags) && 8961 (tp->snd_una != tp->snd_max)) { 8962 tp->snd_nxt = th->th_ack; 8963 tfo_partial = 1; 8964 } 8965 /* 8966 * If there's data, delay ACK; if there's also a FIN ACKNOW 8967 * will be turned on later. 8968 */ 8969 if (DELAY_ACK(tp, bbr, 1) && tlen != 0 && !tfo_partial) { 8970 bbr->bbr_segs_rcvd += 1; 8971 tp->t_flags |= TF_DELACK; 8972 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime); 8973 } else { 8974 bbr->r_wanted_output = 1; 8975 tp->t_flags |= TF_ACKNOW; 8976 } 8977 if (SEQ_GT(th->th_ack, tp->iss)) { 8978 /* 8979 * The SYN is acked 8980 * handle it specially. 8981 */ 8982 bbr_log_syn(tp, to); 8983 } 8984 if (SEQ_GT(th->th_ack, tp->snd_una)) { 8985 /* 8986 * We advance snd_una for the 8987 * fast open case. If th_ack is 8988 * acknowledging data beyond 8989 * snd_una we can't just call 8990 * ack-processing since the 8991 * data stream in our send-map 8992 * will start at snd_una + 1 (one 8993 * beyond the SYN). If its just 8994 * equal we don't need to do that 8995 * and there is no send_map. 8996 */ 8997 tp->snd_una++; 8998 } 8999 /* 9000 * Received <SYN,ACK> in SYN_SENT[*] state. Transitions: 9001 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1 9002 */ 9003 tp->t_starttime = ticks; 9004 if (tp->t_flags & TF_NEEDFIN) { 9005 tcp_state_change(tp, TCPS_FIN_WAIT_1); 9006 tp->t_flags &= ~TF_NEEDFIN; 9007 thflags &= ~TH_SYN; 9008 } else { 9009 tcp_state_change(tp, TCPS_ESTABLISHED); 9010 TCP_PROBE5(connect__established, NULL, tp, 9011 mtod(m, const char *), tp, th); 9012 cc_conn_init(tp); 9013 } 9014 } else { 9015 /* 9016 * Received initial SYN in SYN-SENT[*] state => simultaneous 9017 * open. If segment contains CC option and there is a 9018 * cached CC, apply TAO test. If it succeeds, connection is * 9019 * half-synchronized. Otherwise, do 3-way handshake: 9020 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If 9021 * there was no CC option, clear cached CC value. 9022 */ 9023 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN); 9024 tcp_state_change(tp, TCPS_SYN_RECEIVED); 9025 } 9026 INP_WLOCK_ASSERT(tp->t_inpcb); 9027 /* 9028 * Advance th->th_seq to correspond to first data byte. If data, 9029 * trim to stay within window, dropping FIN if necessary. 9030 */ 9031 th->th_seq++; 9032 if (tlen > tp->rcv_wnd) { 9033 todrop = tlen - tp->rcv_wnd; 9034 m_adj(m, -todrop); 9035 tlen = tp->rcv_wnd; 9036 thflags &= ~TH_FIN; 9037 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin); 9038 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop); 9039 } 9040 tp->snd_wl1 = th->th_seq - 1; 9041 tp->rcv_up = th->th_seq; 9042 /* 9043 * Client side of transaction: already sent SYN and data. If the 9044 * remote host used T/TCP to validate the SYN, our data will be 9045 * ACK'd; if so, enter normal data segment processing in the middle 9046 * of step 5, ack processing. Otherwise, goto step 6. 9047 */ 9048 if (thflags & TH_ACK) { 9049 if ((to->to_flags & TOF_TS) != 0) { 9050 uint32_t t, rtt; 9051 9052 t = tcp_tv_to_mssectick(&bbr->rc_tv); 9053 if (TSTMP_GEQ(t, to->to_tsecr)) { 9054 rtt = t - to->to_tsecr; 9055 if (rtt == 0) { 9056 rtt = 1; 9057 } 9058 rtt *= MS_IN_USEC; 9059 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0); 9060 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, 9061 rtt, bbr->r_ctl.rc_rcvtime); 9062 } 9063 } 9064 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) 9065 return (ret_val); 9066 /* We may have changed to FIN_WAIT_1 above */ 9067 if (tp->t_state == TCPS_FIN_WAIT_1) { 9068 /* 9069 * In FIN_WAIT_1 STATE in addition to the processing 9070 * for the ESTABLISHED state if our FIN is now 9071 * acknowledged then enter FIN_WAIT_2. 9072 */ 9073 if (ourfinisacked) { 9074 /* 9075 * If we can't receive any more data, then 9076 * closing user can proceed. Starting the 9077 * timer is contrary to the specification, 9078 * but if we don't get a FIN we'll hang 9079 * forever. 9080 * 9081 * XXXjl: we should release the tp also, and 9082 * use a compressed state. 9083 */ 9084 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 9085 soisdisconnected(so); 9086 tcp_timer_activate(tp, TT_2MSL, 9087 (tcp_fast_finwait2_recycle ? 9088 tcp_finwait2_timeout : 9089 TP_MAXIDLE(tp))); 9090 } 9091 tcp_state_change(tp, TCPS_FIN_WAIT_2); 9092 } 9093 } 9094 } 9095 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9096 tiwin, thflags, nxt_pkt)); 9097 } 9098 9099 /* 9100 * Return value of 1, the TCB is unlocked and most 9101 * likely gone, return value of 0, the TCB is still 9102 * locked. 9103 */ 9104 static int 9105 bbr_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, 9106 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9107 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9108 { 9109 int32_t ourfinisacked = 0; 9110 int32_t ret_val; 9111 struct tcp_bbr *bbr; 9112 9113 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9114 ctf_calc_rwin(so, tp); 9115 if ((thflags & TH_ACK) && 9116 (SEQ_LEQ(th->th_ack, tp->snd_una) || 9117 SEQ_GT(th->th_ack, tp->snd_max))) { 9118 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 9119 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9120 return (1); 9121 } 9122 if (IS_FASTOPEN(tp->t_flags)) { 9123 /* 9124 * When a TFO connection is in SYN_RECEIVED, the only valid 9125 * packets are the initial SYN, a retransmit/copy of the 9126 * initial SYN (possibly with a subset of the original 9127 * data), a valid ACK, a FIN, or a RST. 9128 */ 9129 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { 9130 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 9131 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9132 return (1); 9133 } else if (thflags & TH_SYN) { 9134 /* non-initial SYN is ignored */ 9135 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) || 9136 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) || 9137 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) { 9138 ctf_do_drop(m, NULL); 9139 return (0); 9140 } 9141 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) { 9142 ctf_do_drop(m, NULL); 9143 return (0); 9144 } 9145 } 9146 if ((thflags & TH_RST) || 9147 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9148 return (ctf_process_rst(m, th, so, tp)); 9149 /* 9150 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9151 * it's less than ts_recent, drop it. 9152 */ 9153 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9154 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9155 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9156 return (ret_val); 9157 } 9158 /* 9159 * In the SYN-RECEIVED state, validate that the packet belongs to 9160 * this connection before trimming the data to fit the receive 9161 * window. Check the sequence number versus IRS since we know the 9162 * sequence numbers haven't wrapped. This is a partial fix for the 9163 * "LAND" DoS attack. 9164 */ 9165 if (SEQ_LT(th->th_seq, tp->irs)) { 9166 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 9167 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9168 return (1); 9169 } 9170 INP_WLOCK_ASSERT(tp->t_inpcb); 9171 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9172 return (ret_val); 9173 } 9174 /* 9175 * If last ACK falls within this segment's sequence numbers, record 9176 * its timestamp. NOTE: 1) That the test incorporates suggestions 9177 * from the latest proposal of the tcplw@cray.com list (Braden 9178 * 1993/04/26). 2) That updating only on newer timestamps interferes 9179 * with our earlier PAWS tests, so this check should be solely 9180 * predicated on the sequence space of this segment. 3) That we 9181 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9182 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9183 * SEG.Len, This modified check allows us to overcome RFC1323's 9184 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9185 * p.869. In such cases, we can still calculate the RTT correctly 9186 * when RCV.NXT == Last.ACK.Sent. 9187 */ 9188 if ((to->to_flags & TOF_TS) != 0 && 9189 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9190 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9191 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9192 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9193 tp->ts_recent = to->to_tsval; 9194 } 9195 tp->snd_wnd = tiwin; 9196 /* 9197 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 9198 * is on (half-synchronized state), then queue data for later 9199 * processing; else drop segment and return. 9200 */ 9201 if ((thflags & TH_ACK) == 0) { 9202 if (IS_FASTOPEN(tp->t_flags)) { 9203 cc_conn_init(tp); 9204 } 9205 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9206 tiwin, thflags, nxt_pkt)); 9207 } 9208 KMOD_TCPSTAT_INC(tcps_connects); 9209 soisconnected(so); 9210 /* Do window scaling? */ 9211 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 9212 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 9213 tp->rcv_scale = tp->request_r_scale; 9214 } 9215 /* 9216 * ok for the first time in lets see if we can use the ts to figure 9217 * out what the initial RTT was. 9218 */ 9219 if ((to->to_flags & TOF_TS) != 0) { 9220 uint32_t t, rtt; 9221 9222 t = tcp_tv_to_mssectick(&bbr->rc_tv); 9223 if (TSTMP_GEQ(t, to->to_tsecr)) { 9224 rtt = t - to->to_tsecr; 9225 if (rtt == 0) { 9226 rtt = 1; 9227 } 9228 rtt *= MS_IN_USEC; 9229 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0); 9230 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime); 9231 } 9232 } 9233 /* Drop off any SYN in the send map (probably not there) */ 9234 if (thflags & TH_ACK) 9235 bbr_log_syn(tp, to); 9236 if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) { 9237 9238 tcp_fastopen_decrement_counter(tp->t_tfo_pending); 9239 tp->t_tfo_pending = NULL; 9240 } 9241 /* 9242 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* -> 9243 * FIN-WAIT-1 9244 */ 9245 tp->t_starttime = ticks; 9246 if (tp->t_flags & TF_NEEDFIN) { 9247 tcp_state_change(tp, TCPS_FIN_WAIT_1); 9248 tp->t_flags &= ~TF_NEEDFIN; 9249 } else { 9250 tcp_state_change(tp, TCPS_ESTABLISHED); 9251 TCP_PROBE5(accept__established, NULL, tp, 9252 mtod(m, const char *), tp, th); 9253 /* 9254 * TFO connections call cc_conn_init() during SYN 9255 * processing. Calling it again here for such connections 9256 * is not harmless as it would undo the snd_cwnd reduction 9257 * that occurs when a TFO SYN|ACK is retransmitted. 9258 */ 9259 if (!IS_FASTOPEN(tp->t_flags)) 9260 cc_conn_init(tp); 9261 } 9262 /* 9263 * Account for the ACK of our SYN prior to 9264 * regular ACK processing below, except for 9265 * simultaneous SYN, which is handled later. 9266 */ 9267 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN)) 9268 tp->snd_una++; 9269 /* 9270 * If segment contains data or ACK, will call tcp_reass() later; if 9271 * not, do so now to pass queued data to user. 9272 */ 9273 if (tlen == 0 && (thflags & TH_FIN) == 0) 9274 (void)tcp_reass(tp, (struct tcphdr *)0, NULL, 0, 9275 (struct mbuf *)0); 9276 tp->snd_wl1 = th->th_seq - 1; 9277 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 9278 return (ret_val); 9279 } 9280 if (tp->t_state == TCPS_FIN_WAIT_1) { 9281 /* We could have went to FIN_WAIT_1 (or EST) above */ 9282 /* 9283 * In FIN_WAIT_1 STATE in addition to the processing for the 9284 * ESTABLISHED state if our FIN is now acknowledged then 9285 * enter FIN_WAIT_2. 9286 */ 9287 if (ourfinisacked) { 9288 /* 9289 * If we can't receive any more data, then closing 9290 * user can proceed. Starting the timer is contrary 9291 * to the specification, but if we don't get a FIN 9292 * we'll hang forever. 9293 * 9294 * XXXjl: we should release the tp also, and use a 9295 * compressed state. 9296 */ 9297 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 9298 soisdisconnected(so); 9299 tcp_timer_activate(tp, TT_2MSL, 9300 (tcp_fast_finwait2_recycle ? 9301 tcp_finwait2_timeout : 9302 TP_MAXIDLE(tp))); 9303 } 9304 tcp_state_change(tp, TCPS_FIN_WAIT_2); 9305 } 9306 } 9307 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9308 tiwin, thflags, nxt_pkt)); 9309 } 9310 9311 /* 9312 * Return value of 1, the TCB is unlocked and most 9313 * likely gone, return value of 0, the TCB is still 9314 * locked. 9315 */ 9316 static int 9317 bbr_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, 9318 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9319 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9320 { 9321 struct tcp_bbr *bbr; 9322 int32_t ret_val; 9323 9324 /* 9325 * Header prediction: check for the two common cases of a 9326 * uni-directional data xfer. If the packet has no control flags, 9327 * is in-sequence, the window didn't change and we're not 9328 * retransmitting, it's a candidate. If the length is zero and the 9329 * ack moved forward, we're the sender side of the xfer. Just free 9330 * the data acked & wake any higher level process that was blocked 9331 * waiting for space. If the length is non-zero and the ack didn't 9332 * move, we're the receiver side. If we're getting packets in-order 9333 * (the reassembly queue is empty), add the data toc The socket 9334 * buffer and note that we need a delayed ack. Make sure that the 9335 * hidden state-flags are also off. Since we check for 9336 * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN. 9337 */ 9338 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9339 if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) { 9340 /* 9341 * If we have delived under 4 segments increase the initial 9342 * window if raised by the peer. We use this to determine 9343 * dynamic and static rwnd's at the end of a connection. 9344 */ 9345 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd); 9346 } 9347 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) && 9348 __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_URG | TH_ACK)) == TH_ACK) && 9349 __predict_true(SEGQ_EMPTY(tp)) && 9350 __predict_true(th->th_seq == tp->rcv_nxt)) { 9351 if (tlen == 0) { 9352 if (bbr_fastack(m, th, so, tp, to, drop_hdrlen, tlen, 9353 tiwin, nxt_pkt, iptos)) { 9354 return (0); 9355 } 9356 } else { 9357 if (bbr_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen, 9358 tiwin, nxt_pkt)) { 9359 return (0); 9360 } 9361 } 9362 } 9363 ctf_calc_rwin(so, tp); 9364 9365 if ((thflags & TH_RST) || 9366 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9367 return (ctf_process_rst(m, th, so, tp)); 9368 /* 9369 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 9370 * synchronized state. 9371 */ 9372 if (thflags & TH_SYN) { 9373 ctf_challenge_ack(m, th, tp, &ret_val); 9374 return (ret_val); 9375 } 9376 /* 9377 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9378 * it's less than ts_recent, drop it. 9379 */ 9380 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9381 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9382 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9383 return (ret_val); 9384 } 9385 INP_WLOCK_ASSERT(tp->t_inpcb); 9386 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9387 return (ret_val); 9388 } 9389 /* 9390 * If last ACK falls within this segment's sequence numbers, record 9391 * its timestamp. NOTE: 1) That the test incorporates suggestions 9392 * from the latest proposal of the tcplw@cray.com list (Braden 9393 * 1993/04/26). 2) That updating only on newer timestamps interferes 9394 * with our earlier PAWS tests, so this check should be solely 9395 * predicated on the sequence space of this segment. 3) That we 9396 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9397 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9398 * SEG.Len, This modified check allows us to overcome RFC1323's 9399 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9400 * p.869. In such cases, we can still calculate the RTT correctly 9401 * when RCV.NXT == Last.ACK.Sent. 9402 */ 9403 if ((to->to_flags & TOF_TS) != 0 && 9404 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9405 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9406 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9407 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9408 tp->ts_recent = to->to_tsval; 9409 } 9410 /* 9411 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 9412 * is on (half-synchronized state), then queue data for later 9413 * processing; else drop segment and return. 9414 */ 9415 if ((thflags & TH_ACK) == 0) { 9416 if (tp->t_flags & TF_NEEDSYN) { 9417 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9418 tiwin, thflags, nxt_pkt)); 9419 } else if (tp->t_flags & TF_ACKNOW) { 9420 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 9421 bbr->r_wanted_output = 1; 9422 return (ret_val); 9423 } else { 9424 ctf_do_drop(m, NULL); 9425 return (0); 9426 } 9427 } 9428 /* 9429 * Ack processing. 9430 */ 9431 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) { 9432 return (ret_val); 9433 } 9434 if (sbavail(&so->so_snd)) { 9435 if (ctf_progress_timeout_check(tp, true)) { 9436 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 9437 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9438 return (1); 9439 } 9440 } 9441 /* State changes only happen in bbr_process_data() */ 9442 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9443 tiwin, thflags, nxt_pkt)); 9444 } 9445 9446 /* 9447 * Return value of 1, the TCB is unlocked and most 9448 * likely gone, return value of 0, the TCB is still 9449 * locked. 9450 */ 9451 static int 9452 bbr_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, 9453 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9454 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9455 { 9456 struct tcp_bbr *bbr; 9457 int32_t ret_val; 9458 9459 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9460 ctf_calc_rwin(so, tp); 9461 if ((thflags & TH_RST) || 9462 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9463 return (ctf_process_rst(m, th, so, tp)); 9464 /* 9465 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 9466 * synchronized state. 9467 */ 9468 if (thflags & TH_SYN) { 9469 ctf_challenge_ack(m, th, tp, &ret_val); 9470 return (ret_val); 9471 } 9472 /* 9473 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9474 * it's less than ts_recent, drop it. 9475 */ 9476 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9477 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9478 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9479 return (ret_val); 9480 } 9481 INP_WLOCK_ASSERT(tp->t_inpcb); 9482 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9483 return (ret_val); 9484 } 9485 /* 9486 * If last ACK falls within this segment's sequence numbers, record 9487 * its timestamp. NOTE: 1) That the test incorporates suggestions 9488 * from the latest proposal of the tcplw@cray.com list (Braden 9489 * 1993/04/26). 2) That updating only on newer timestamps interferes 9490 * with our earlier PAWS tests, so this check should be solely 9491 * predicated on the sequence space of this segment. 3) That we 9492 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9493 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9494 * SEG.Len, This modified check allows us to overcome RFC1323's 9495 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9496 * p.869. In such cases, we can still calculate the RTT correctly 9497 * when RCV.NXT == Last.ACK.Sent. 9498 */ 9499 if ((to->to_flags & TOF_TS) != 0 && 9500 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9501 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9502 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9503 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9504 tp->ts_recent = to->to_tsval; 9505 } 9506 /* 9507 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 9508 * is on (half-synchronized state), then queue data for later 9509 * processing; else drop segment and return. 9510 */ 9511 if ((thflags & TH_ACK) == 0) { 9512 if (tp->t_flags & TF_NEEDSYN) { 9513 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9514 tiwin, thflags, nxt_pkt)); 9515 } else if (tp->t_flags & TF_ACKNOW) { 9516 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 9517 bbr->r_wanted_output = 1; 9518 return (ret_val); 9519 } else { 9520 ctf_do_drop(m, NULL); 9521 return (0); 9522 } 9523 } 9524 /* 9525 * Ack processing. 9526 */ 9527 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) { 9528 return (ret_val); 9529 } 9530 if (sbavail(&so->so_snd)) { 9531 if (ctf_progress_timeout_check(tp, true)) { 9532 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 9533 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9534 return (1); 9535 } 9536 } 9537 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9538 tiwin, thflags, nxt_pkt)); 9539 } 9540 9541 static int 9542 bbr_check_data_after_close(struct mbuf *m, struct tcp_bbr *bbr, 9543 struct tcpcb *tp, int32_t * tlen, struct tcphdr *th, struct socket *so) 9544 { 9545 9546 if (bbr->rc_allow_data_af_clo == 0) { 9547 close_now: 9548 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE); 9549 /* tcp_close will kill the inp pre-log the Reset */ 9550 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 9551 tp = tcp_close(tp); 9552 KMOD_TCPSTAT_INC(tcps_rcvafterclose); 9553 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen)); 9554 return (1); 9555 } 9556 if (sbavail(&so->so_snd) == 0) 9557 goto close_now; 9558 /* Ok we allow data that is ignored and a followup reset */ 9559 tp->rcv_nxt = th->th_seq + *tlen; 9560 tp->t_flags2 |= TF2_DROP_AF_DATA; 9561 bbr->r_wanted_output = 1; 9562 *tlen = 0; 9563 return (0); 9564 } 9565 9566 /* 9567 * Return value of 1, the TCB is unlocked and most 9568 * likely gone, return value of 0, the TCB is still 9569 * locked. 9570 */ 9571 static int 9572 bbr_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, 9573 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9574 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9575 { 9576 int32_t ourfinisacked = 0; 9577 int32_t ret_val; 9578 struct tcp_bbr *bbr; 9579 9580 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9581 ctf_calc_rwin(so, tp); 9582 if ((thflags & TH_RST) || 9583 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9584 return (ctf_process_rst(m, th, so, tp)); 9585 /* 9586 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 9587 * synchronized state. 9588 */ 9589 if (thflags & TH_SYN) { 9590 ctf_challenge_ack(m, th, tp, &ret_val); 9591 return (ret_val); 9592 } 9593 /* 9594 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9595 * it's less than ts_recent, drop it. 9596 */ 9597 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9598 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9599 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9600 return (ret_val); 9601 } 9602 INP_WLOCK_ASSERT(tp->t_inpcb); 9603 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9604 return (ret_val); 9605 } 9606 /* 9607 * If new data are received on a connection after the user processes 9608 * are gone, then RST the other end. 9609 */ 9610 if ((so->so_state & SS_NOFDREF) && tlen) { 9611 /* 9612 * We call a new function now so we might continue and setup 9613 * to reset at all data being ack'd. 9614 */ 9615 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so)) 9616 return (1); 9617 } 9618 /* 9619 * If last ACK falls within this segment's sequence numbers, record 9620 * its timestamp. NOTE: 1) That the test incorporates suggestions 9621 * from the latest proposal of the tcplw@cray.com list (Braden 9622 * 1993/04/26). 2) That updating only on newer timestamps interferes 9623 * with our earlier PAWS tests, so this check should be solely 9624 * predicated on the sequence space of this segment. 3) That we 9625 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9626 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9627 * SEG.Len, This modified check allows us to overcome RFC1323's 9628 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9629 * p.869. In such cases, we can still calculate the RTT correctly 9630 * when RCV.NXT == Last.ACK.Sent. 9631 */ 9632 if ((to->to_flags & TOF_TS) != 0 && 9633 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9634 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9635 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9636 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9637 tp->ts_recent = to->to_tsval; 9638 } 9639 /* 9640 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 9641 * is on (half-synchronized state), then queue data for later 9642 * processing; else drop segment and return. 9643 */ 9644 if ((thflags & TH_ACK) == 0) { 9645 if (tp->t_flags & TF_NEEDSYN) { 9646 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9647 tiwin, thflags, nxt_pkt)); 9648 } else if (tp->t_flags & TF_ACKNOW) { 9649 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 9650 bbr->r_wanted_output = 1; 9651 return (ret_val); 9652 } else { 9653 ctf_do_drop(m, NULL); 9654 return (0); 9655 } 9656 } 9657 /* 9658 * Ack processing. 9659 */ 9660 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 9661 return (ret_val); 9662 } 9663 if (ourfinisacked) { 9664 /* 9665 * If we can't receive any more data, then closing user can 9666 * proceed. Starting the timer is contrary to the 9667 * specification, but if we don't get a FIN we'll hang 9668 * forever. 9669 * 9670 * XXXjl: we should release the tp also, and use a 9671 * compressed state. 9672 */ 9673 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 9674 soisdisconnected(so); 9675 tcp_timer_activate(tp, TT_2MSL, 9676 (tcp_fast_finwait2_recycle ? 9677 tcp_finwait2_timeout : 9678 TP_MAXIDLE(tp))); 9679 } 9680 tcp_state_change(tp, TCPS_FIN_WAIT_2); 9681 } 9682 if (sbavail(&so->so_snd)) { 9683 if (ctf_progress_timeout_check(tp, true)) { 9684 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 9685 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9686 return (1); 9687 } 9688 } 9689 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9690 tiwin, thflags, nxt_pkt)); 9691 } 9692 9693 /* 9694 * Return value of 1, the TCB is unlocked and most 9695 * likely gone, return value of 0, the TCB is still 9696 * locked. 9697 */ 9698 static int 9699 bbr_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, 9700 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9701 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9702 { 9703 int32_t ourfinisacked = 0; 9704 int32_t ret_val; 9705 struct tcp_bbr *bbr; 9706 9707 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9708 ctf_calc_rwin(so, tp); 9709 if ((thflags & TH_RST) || 9710 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9711 return (ctf_process_rst(m, th, so, tp)); 9712 /* 9713 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 9714 * synchronized state. 9715 */ 9716 if (thflags & TH_SYN) { 9717 ctf_challenge_ack(m, th, tp, &ret_val); 9718 return (ret_val); 9719 } 9720 /* 9721 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9722 * it's less than ts_recent, drop it. 9723 */ 9724 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9725 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9726 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9727 return (ret_val); 9728 } 9729 INP_WLOCK_ASSERT(tp->t_inpcb); 9730 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9731 return (ret_val); 9732 } 9733 /* 9734 * If new data are received on a connection after the user processes 9735 * are gone, then RST the other end. 9736 */ 9737 if ((so->so_state & SS_NOFDREF) && tlen) { 9738 /* 9739 * We call a new function now so we might continue and setup 9740 * to reset at all data being ack'd. 9741 */ 9742 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so)) 9743 return (1); 9744 } 9745 /* 9746 * If last ACK falls within this segment's sequence numbers, record 9747 * its timestamp. NOTE: 1) That the test incorporates suggestions 9748 * from the latest proposal of the tcplw@cray.com list (Braden 9749 * 1993/04/26). 2) That updating only on newer timestamps interferes 9750 * with our earlier PAWS tests, so this check should be solely 9751 * predicated on the sequence space of this segment. 3) That we 9752 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9753 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9754 * SEG.Len, This modified check allows us to overcome RFC1323's 9755 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9756 * p.869. In such cases, we can still calculate the RTT correctly 9757 * when RCV.NXT == Last.ACK.Sent. 9758 */ 9759 if ((to->to_flags & TOF_TS) != 0 && 9760 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9761 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9762 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9763 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9764 tp->ts_recent = to->to_tsval; 9765 } 9766 /* 9767 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 9768 * is on (half-synchronized state), then queue data for later 9769 * processing; else drop segment and return. 9770 */ 9771 if ((thflags & TH_ACK) == 0) { 9772 if (tp->t_flags & TF_NEEDSYN) { 9773 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9774 tiwin, thflags, nxt_pkt)); 9775 } else if (tp->t_flags & TF_ACKNOW) { 9776 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 9777 bbr->r_wanted_output = 1; 9778 return (ret_val); 9779 } else { 9780 ctf_do_drop(m, NULL); 9781 return (0); 9782 } 9783 } 9784 /* 9785 * Ack processing. 9786 */ 9787 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 9788 return (ret_val); 9789 } 9790 if (ourfinisacked) { 9791 tcp_twstart(tp); 9792 m_freem(m); 9793 return (1); 9794 } 9795 if (sbavail(&so->so_snd)) { 9796 if (ctf_progress_timeout_check(tp, true)) { 9797 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 9798 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9799 return (1); 9800 } 9801 } 9802 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9803 tiwin, thflags, nxt_pkt)); 9804 } 9805 9806 /* 9807 * Return value of 1, the TCB is unlocked and most 9808 * likely gone, return value of 0, the TCB is still 9809 * locked. 9810 */ 9811 static int 9812 bbr_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, 9813 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9814 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9815 { 9816 int32_t ourfinisacked = 0; 9817 int32_t ret_val; 9818 struct tcp_bbr *bbr; 9819 9820 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9821 ctf_calc_rwin(so, tp); 9822 if ((thflags & TH_RST) || 9823 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9824 return (ctf_process_rst(m, th, so, tp)); 9825 /* 9826 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 9827 * synchronized state. 9828 */ 9829 if (thflags & TH_SYN) { 9830 ctf_challenge_ack(m, th, tp, &ret_val); 9831 return (ret_val); 9832 } 9833 /* 9834 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9835 * it's less than ts_recent, drop it. 9836 */ 9837 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9838 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9839 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9840 return (ret_val); 9841 } 9842 INP_WLOCK_ASSERT(tp->t_inpcb); 9843 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9844 return (ret_val); 9845 } 9846 /* 9847 * If new data are received on a connection after the user processes 9848 * are gone, then RST the other end. 9849 */ 9850 if ((so->so_state & SS_NOFDREF) && tlen) { 9851 /* 9852 * We call a new function now so we might continue and setup 9853 * to reset at all data being ack'd. 9854 */ 9855 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so)) 9856 return (1); 9857 } 9858 /* 9859 * If last ACK falls within this segment's sequence numbers, record 9860 * its timestamp. NOTE: 1) That the test incorporates suggestions 9861 * from the latest proposal of the tcplw@cray.com list (Braden 9862 * 1993/04/26). 2) That updating only on newer timestamps interferes 9863 * with our earlier PAWS tests, so this check should be solely 9864 * predicated on the sequence space of this segment. 3) That we 9865 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9866 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9867 * SEG.Len, This modified check allows us to overcome RFC1323's 9868 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9869 * p.869. In such cases, we can still calculate the RTT correctly 9870 * when RCV.NXT == Last.ACK.Sent. 9871 */ 9872 if ((to->to_flags & TOF_TS) != 0 && 9873 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9874 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9875 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9876 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9877 tp->ts_recent = to->to_tsval; 9878 } 9879 /* 9880 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 9881 * is on (half-synchronized state), then queue data for later 9882 * processing; else drop segment and return. 9883 */ 9884 if ((thflags & TH_ACK) == 0) { 9885 if (tp->t_flags & TF_NEEDSYN) { 9886 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9887 tiwin, thflags, nxt_pkt)); 9888 } else if (tp->t_flags & TF_ACKNOW) { 9889 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 9890 bbr->r_wanted_output = 1; 9891 return (ret_val); 9892 } else { 9893 ctf_do_drop(m, NULL); 9894 return (0); 9895 } 9896 } 9897 /* 9898 * case TCPS_LAST_ACK: Ack processing. 9899 */ 9900 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 9901 return (ret_val); 9902 } 9903 if (ourfinisacked) { 9904 tp = tcp_close(tp); 9905 ctf_do_drop(m, tp); 9906 return (1); 9907 } 9908 if (sbavail(&so->so_snd)) { 9909 if (ctf_progress_timeout_check(tp, true)) { 9910 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 9911 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 9912 return (1); 9913 } 9914 } 9915 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 9916 tiwin, thflags, nxt_pkt)); 9917 } 9918 9919 9920 /* 9921 * Return value of 1, the TCB is unlocked and most 9922 * likely gone, return value of 0, the TCB is still 9923 * locked. 9924 */ 9925 static int 9926 bbr_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, 9927 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 9928 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 9929 { 9930 int32_t ourfinisacked = 0; 9931 int32_t ret_val; 9932 struct tcp_bbr *bbr; 9933 9934 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 9935 ctf_calc_rwin(so, tp); 9936 /* Reset receive buffer auto scaling when not in bulk receive mode. */ 9937 if ((thflags & TH_RST) || 9938 (tp->t_fin_is_rst && (thflags & TH_FIN))) 9939 return (ctf_process_rst(m, th, so, tp)); 9940 9941 /* 9942 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 9943 * synchronized state. 9944 */ 9945 if (thflags & TH_SYN) { 9946 ctf_challenge_ack(m, th, tp, &ret_val); 9947 return (ret_val); 9948 } 9949 INP_WLOCK_ASSERT(tp->t_inpcb); 9950 /* 9951 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 9952 * it's less than ts_recent, drop it. 9953 */ 9954 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 9955 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 9956 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 9957 return (ret_val); 9958 } 9959 INP_WLOCK_ASSERT(tp->t_inpcb); 9960 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) { 9961 return (ret_val); 9962 } 9963 /* 9964 * If new data are received on a connection after the user processes 9965 * are gone, then we may RST the other end depending on the outcome 9966 * of bbr_check_data_after_close. 9967 */ 9968 if ((so->so_state & SS_NOFDREF) && 9969 tlen) { 9970 /* 9971 * We call a new function now so we might continue and setup 9972 * to reset at all data being ack'd. 9973 */ 9974 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so)) 9975 return (1); 9976 } 9977 INP_WLOCK_ASSERT(tp->t_inpcb); 9978 /* 9979 * If last ACK falls within this segment's sequence numbers, record 9980 * its timestamp. NOTE: 1) That the test incorporates suggestions 9981 * from the latest proposal of the tcplw@cray.com list (Braden 9982 * 1993/04/26). 2) That updating only on newer timestamps interferes 9983 * with our earlier PAWS tests, so this check should be solely 9984 * predicated on the sequence space of this segment. 3) That we 9985 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 9986 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 9987 * SEG.Len, This modified check allows us to overcome RFC1323's 9988 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 9989 * p.869. In such cases, we can still calculate the RTT correctly 9990 * when RCV.NXT == Last.ACK.Sent. 9991 */ 9992 INP_WLOCK_ASSERT(tp->t_inpcb); 9993 if ((to->to_flags & TOF_TS) != 0 && 9994 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 9995 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 9996 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 9997 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 9998 tp->ts_recent = to->to_tsval; 9999 } 10000 /* 10001 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 10002 * is on (half-synchronized state), then queue data for later 10003 * processing; else drop segment and return. 10004 */ 10005 if ((thflags & TH_ACK) == 0) { 10006 if (tp->t_flags & TF_NEEDSYN) { 10007 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 10008 tiwin, thflags, nxt_pkt)); 10009 } else if (tp->t_flags & TF_ACKNOW) { 10010 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 10011 bbr->r_wanted_output = 1; 10012 return (ret_val); 10013 } else { 10014 ctf_do_drop(m, NULL); 10015 return (0); 10016 } 10017 } 10018 /* 10019 * Ack processing. 10020 */ 10021 INP_WLOCK_ASSERT(tp->t_inpcb); 10022 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 10023 return (ret_val); 10024 } 10025 if (sbavail(&so->so_snd)) { 10026 if (ctf_progress_timeout_check(tp, true)) { 10027 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__); 10028 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 10029 return (1); 10030 } 10031 } 10032 INP_WLOCK_ASSERT(tp->t_inpcb); 10033 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen, 10034 tiwin, thflags, nxt_pkt)); 10035 } 10036 10037 static void 10038 bbr_stop_all_timers(struct tcpcb *tp) 10039 { 10040 struct tcp_bbr *bbr; 10041 10042 /* 10043 * Assure no timers are running. 10044 */ 10045 if (tcp_timer_active(tp, TT_PERSIST)) { 10046 /* We enter in persists, set the flag appropriately */ 10047 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 10048 bbr->rc_in_persist = 1; 10049 } 10050 tcp_timer_suspend(tp, TT_PERSIST); 10051 tcp_timer_suspend(tp, TT_REXMT); 10052 tcp_timer_suspend(tp, TT_KEEP); 10053 tcp_timer_suspend(tp, TT_DELACK); 10054 } 10055 10056 static void 10057 bbr_google_mode_on(struct tcp_bbr *bbr) 10058 { 10059 bbr->rc_use_google = 1; 10060 bbr->rc_no_pacing = 0; 10061 bbr->r_ctl.bbr_google_discount = bbr_google_discount; 10062 bbr->r_use_policer = bbr_policer_detection_enabled; 10063 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10); 10064 bbr->bbr_use_rack_cheat = 0; 10065 bbr->r_ctl.rc_incr_tmrs = 0; 10066 bbr->r_ctl.rc_inc_tcp_oh = 0; 10067 bbr->r_ctl.rc_inc_ip_oh = 0; 10068 bbr->r_ctl.rc_inc_enet_oh = 0; 10069 reset_time(&bbr->r_ctl.rc_delrate, 10070 BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT); 10071 reset_time_small(&bbr->r_ctl.rc_rttprop, 10072 (11 * USECS_IN_SECOND)); 10073 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv)); 10074 } 10075 10076 static void 10077 bbr_google_mode_off(struct tcp_bbr *bbr) 10078 { 10079 bbr->rc_use_google = 0; 10080 bbr->r_ctl.bbr_google_discount = 0; 10081 bbr->no_pacing_until = bbr_no_pacing_until; 10082 bbr->r_use_policer = 0; 10083 if (bbr->no_pacing_until) 10084 bbr->rc_no_pacing = 1; 10085 else 10086 bbr->rc_no_pacing = 0; 10087 if (bbr_use_rack_resend_cheat) 10088 bbr->bbr_use_rack_cheat = 1; 10089 else 10090 bbr->bbr_use_rack_cheat = 0; 10091 if (bbr_incr_timers) 10092 bbr->r_ctl.rc_incr_tmrs = 1; 10093 else 10094 bbr->r_ctl.rc_incr_tmrs = 0; 10095 if (bbr_include_tcp_oh) 10096 bbr->r_ctl.rc_inc_tcp_oh = 1; 10097 else 10098 bbr->r_ctl.rc_inc_tcp_oh = 0; 10099 if (bbr_include_ip_oh) 10100 bbr->r_ctl.rc_inc_ip_oh = 1; 10101 else 10102 bbr->r_ctl.rc_inc_ip_oh = 0; 10103 if (bbr_include_enet_oh) 10104 bbr->r_ctl.rc_inc_enet_oh = 1; 10105 else 10106 bbr->r_ctl.rc_inc_enet_oh = 0; 10107 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; 10108 reset_time(&bbr->r_ctl.rc_delrate, 10109 bbr_num_pktepo_for_del_limit); 10110 reset_time_small(&bbr->r_ctl.rc_rttprop, 10111 (bbr_filter_len_sec * USECS_IN_SECOND)); 10112 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv)); 10113 } 10114 /* 10115 * Return 0 on success, non-zero on failure 10116 * which indicates the error (usually no memory). 10117 */ 10118 static int 10119 bbr_init(struct tcpcb *tp) 10120 { 10121 struct tcp_bbr *bbr = NULL; 10122 struct inpcb *inp; 10123 uint32_t cts; 10124 10125 tp->t_fb_ptr = uma_zalloc(bbr_pcb_zone, (M_NOWAIT | M_ZERO)); 10126 if (tp->t_fb_ptr == NULL) { 10127 /* 10128 * We need to allocate memory but cant. The INP and INP_INFO 10129 * locks and they are recusive (happens during setup. So a 10130 * scheme to drop the locks fails :( 10131 * 10132 */ 10133 return (ENOMEM); 10134 } 10135 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 10136 bbr->rtt_valid = 0; 10137 inp = tp->t_inpcb; 10138 inp->inp_flags2 |= INP_CANNOT_DO_ECN; 10139 inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 10140 TAILQ_INIT(&bbr->r_ctl.rc_map); 10141 TAILQ_INIT(&bbr->r_ctl.rc_free); 10142 TAILQ_INIT(&bbr->r_ctl.rc_tmap); 10143 bbr->rc_tp = tp; 10144 if (tp->t_inpcb) { 10145 bbr->rc_inp = tp->t_inpcb; 10146 } 10147 cts = tcp_get_usecs(&bbr->rc_tv); 10148 tp->t_acktime = 0; 10149 bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close; 10150 bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade; 10151 bbr->rc_tlp_threshold = bbr_tlp_thresh; 10152 bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh; 10153 bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay; 10154 bbr->r_ctl.rc_min_to = bbr_min_to; 10155 bbr->rc_bbr_state = BBR_STATE_STARTUP; 10156 bbr->r_ctl.bbr_lost_at_state = 0; 10157 bbr->r_ctl.rc_lost_at_startup = 0; 10158 bbr->rc_all_timers_stopped = 0; 10159 bbr->r_ctl.rc_bbr_lastbtlbw = 0; 10160 bbr->r_ctl.rc_pkt_epoch_del = 0; 10161 bbr->r_ctl.rc_pkt_epoch = 0; 10162 bbr->r_ctl.rc_lowest_rtt = 0xffffffff; 10163 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain; 10164 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain; 10165 bbr->r_ctl.rc_went_idle_time = cts; 10166 bbr->rc_pacer_started = cts; 10167 bbr->r_ctl.rc_pkt_epoch_time = cts; 10168 bbr->r_ctl.rc_rcvtime = cts; 10169 bbr->r_ctl.rc_bbr_state_time = cts; 10170 bbr->r_ctl.rc_del_time = cts; 10171 bbr->r_ctl.rc_tlp_rxt_last_time = cts; 10172 bbr->r_ctl.last_in_probertt = cts; 10173 bbr->skip_gain = 0; 10174 bbr->gain_is_limited = 0; 10175 bbr->no_pacing_until = bbr_no_pacing_until; 10176 if (bbr->no_pacing_until) 10177 bbr->rc_no_pacing = 1; 10178 if (bbr_use_google_algo) { 10179 bbr->rc_no_pacing = 0; 10180 bbr->rc_use_google = 1; 10181 bbr->r_ctl.bbr_google_discount = bbr_google_discount; 10182 bbr->r_use_policer = bbr_policer_detection_enabled; 10183 } else { 10184 bbr->rc_use_google = 0; 10185 bbr->r_ctl.bbr_google_discount = 0; 10186 bbr->r_use_policer = 0; 10187 } 10188 if (bbr_ts_limiting) 10189 bbr->rc_use_ts_limit = 1; 10190 else 10191 bbr->rc_use_ts_limit = 0; 10192 if (bbr_ts_can_raise) 10193 bbr->ts_can_raise = 1; 10194 else 10195 bbr->ts_can_raise = 0; 10196 if (V_tcp_delack_enabled == 1) 10197 tp->t_delayed_ack = 2; 10198 else if (V_tcp_delack_enabled == 0) 10199 tp->t_delayed_ack = 0; 10200 else if (V_tcp_delack_enabled < 100) 10201 tp->t_delayed_ack = V_tcp_delack_enabled; 10202 else 10203 tp->t_delayed_ack = 2; 10204 if (bbr->rc_use_google == 0) 10205 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; 10206 else 10207 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10); 10208 bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms; 10209 bbr->rc_max_rto_sec = bbr_rto_max_sec; 10210 bbr->rc_init_win = bbr_def_init_win; 10211 if (tp->t_flags & TF_REQ_TSTMP) 10212 bbr->rc_last_options = TCP_TS_OVERHEAD; 10213 bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options; 10214 bbr->r_ctl.rc_high_rwnd = tp->snd_wnd; 10215 bbr->r_init_rtt = 1; 10216 10217 counter_u64_add(bbr_flows_nohdwr_pacing, 1); 10218 if (bbr_allow_hdwr_pacing) 10219 bbr->bbr_hdw_pace_ena = 1; 10220 else 10221 bbr->bbr_hdw_pace_ena = 0; 10222 if (bbr_sends_full_iwnd) 10223 bbr->bbr_init_win_cheat = 1; 10224 else 10225 bbr->bbr_init_win_cheat = 0; 10226 bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max; 10227 bbr->r_ctl.rc_drain_pg = bbr_drain_gain; 10228 bbr->r_ctl.rc_startup_pg = bbr_high_gain; 10229 bbr->rc_loss_exit = bbr_exit_startup_at_loss; 10230 bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain; 10231 bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second; 10232 bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar; 10233 bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max; 10234 bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor; 10235 bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min; 10236 bbr->r_ctl.bbr_cross_over = bbr_cross_over; 10237 bbr->r_ctl.rc_rtt_shrinks = cts; 10238 if (bbr->rc_use_google) { 10239 setup_time_filter(&bbr->r_ctl.rc_delrate, 10240 FILTER_TYPE_MAX, 10241 BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT); 10242 setup_time_filter_small(&bbr->r_ctl.rc_rttprop, 10243 FILTER_TYPE_MIN, (11 * USECS_IN_SECOND)); 10244 } else { 10245 setup_time_filter(&bbr->r_ctl.rc_delrate, 10246 FILTER_TYPE_MAX, 10247 bbr_num_pktepo_for_del_limit); 10248 setup_time_filter_small(&bbr->r_ctl.rc_rttprop, 10249 FILTER_TYPE_MIN, (bbr_filter_len_sec * USECS_IN_SECOND)); 10250 } 10251 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_INIT, 0); 10252 if (bbr_uses_idle_restart) 10253 bbr->rc_use_idle_restart = 1; 10254 else 10255 bbr->rc_use_idle_restart = 0; 10256 bbr->r_ctl.rc_bbr_cur_del_rate = 0; 10257 bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps; 10258 if (bbr_resends_use_tso) 10259 bbr->rc_resends_use_tso = 1; 10260 #ifdef NETFLIX_PEAKRATE 10261 tp->t_peakrate_thr = tp->t_maxpeakrate; 10262 #endif 10263 if (tp->snd_una != tp->snd_max) { 10264 /* Create a send map for the current outstanding data */ 10265 struct bbr_sendmap *rsm; 10266 10267 rsm = bbr_alloc(bbr); 10268 if (rsm == NULL) { 10269 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr); 10270 tp->t_fb_ptr = NULL; 10271 return (ENOMEM); 10272 } 10273 rsm->r_flags = BBR_OVERMAX; 10274 rsm->r_tim_lastsent[0] = cts; 10275 rsm->r_rtr_cnt = 1; 10276 rsm->r_rtr_bytes = 0; 10277 rsm->r_start = tp->snd_una; 10278 rsm->r_end = tp->snd_max; 10279 rsm->r_dupack = 0; 10280 rsm->r_delivered = bbr->r_ctl.rc_delivered; 10281 rsm->r_ts_valid = 0; 10282 rsm->r_del_ack_ts = tp->ts_recent; 10283 rsm->r_del_time = cts; 10284 if (bbr->r_ctl.r_app_limited_until) 10285 rsm->r_app_limited = 1; 10286 else 10287 rsm->r_app_limited = 0; 10288 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next); 10289 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext); 10290 rsm->r_in_tmap = 1; 10291 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) 10292 rsm->r_bbr_state = bbr_state_val(bbr); 10293 else 10294 rsm->r_bbr_state = 8; 10295 } 10296 if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0)) 10297 bbr->bbr_use_rack_cheat = 1; 10298 if (bbr_incr_timers && (bbr->rc_use_google == 0)) 10299 bbr->r_ctl.rc_incr_tmrs = 1; 10300 if (bbr_include_tcp_oh && (bbr->rc_use_google == 0)) 10301 bbr->r_ctl.rc_inc_tcp_oh = 1; 10302 if (bbr_include_ip_oh && (bbr->rc_use_google == 0)) 10303 bbr->r_ctl.rc_inc_ip_oh = 1; 10304 if (bbr_include_enet_oh && (bbr->rc_use_google == 0)) 10305 bbr->r_ctl.rc_inc_enet_oh = 1; 10306 10307 bbr_log_type_statechange(bbr, cts, __LINE__); 10308 if (TCPS_HAVEESTABLISHED(tp->t_state) && 10309 (tp->t_srtt)) { 10310 uint32_t rtt; 10311 10312 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT); 10313 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts); 10314 } 10315 /* announce the settings and state */ 10316 bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT); 10317 tcp_bbr_tso_size_check(bbr, cts); 10318 /* 10319 * Now call the generic function to start a timer. This will place 10320 * the TCB on the hptsi wheel if a timer is needed with appropriate 10321 * flags. 10322 */ 10323 bbr_stop_all_timers(tp); 10324 bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0); 10325 return (0); 10326 } 10327 10328 /* 10329 * Return 0 if we can accept the connection. Return 10330 * non-zero if we can't handle the connection. A EAGAIN 10331 * means you need to wait until the connection is up. 10332 * a EADDRNOTAVAIL means we can never handle the connection 10333 * (no SACK). 10334 */ 10335 static int 10336 bbr_handoff_ok(struct tcpcb *tp) 10337 { 10338 if ((tp->t_state == TCPS_CLOSED) || 10339 (tp->t_state == TCPS_LISTEN)) { 10340 /* Sure no problem though it may not stick */ 10341 return (0); 10342 } 10343 if ((tp->t_state == TCPS_SYN_SENT) || 10344 (tp->t_state == TCPS_SYN_RECEIVED)) { 10345 /* 10346 * We really don't know you have to get to ESTAB or beyond 10347 * to tell. 10348 */ 10349 return (EAGAIN); 10350 } 10351 if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) { 10352 return (0); 10353 } 10354 /* 10355 * If we reach here we don't do SACK on this connection so we can 10356 * never do rack. 10357 */ 10358 return (EINVAL); 10359 } 10360 10361 static void 10362 bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged) 10363 { 10364 if (tp->t_fb_ptr) { 10365 uint32_t calc; 10366 struct tcp_bbr *bbr; 10367 struct bbr_sendmap *rsm; 10368 10369 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 10370 if (bbr->r_ctl.crte) 10371 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp); 10372 bbr_log_flowend(bbr); 10373 bbr->rc_tp = NULL; 10374 if (tp->t_inpcb) { 10375 /* Backout any flags2 we applied */ 10376 tp->t_inpcb->inp_flags2 &= ~INP_CANNOT_DO_ECN; 10377 tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ; 10378 tp->t_inpcb->inp_flags2 &= ~INP_MBUF_QUEUE_READY; 10379 } 10380 if (bbr->bbr_hdrw_pacing) 10381 counter_u64_add(bbr_flows_whdwr_pacing, -1); 10382 else 10383 counter_u64_add(bbr_flows_nohdwr_pacing, -1); 10384 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 10385 while (rsm) { 10386 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next); 10387 uma_zfree(bbr_zone, rsm); 10388 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 10389 } 10390 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); 10391 while (rsm) { 10392 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next); 10393 uma_zfree(bbr_zone, rsm); 10394 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free); 10395 } 10396 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd; 10397 if (calc > (bbr->r_ctl.rc_init_rwnd / 10)) 10398 BBR_STAT_INC(bbr_dynamic_rwnd); 10399 else 10400 BBR_STAT_INC(bbr_static_rwnd); 10401 bbr->r_ctl.rc_free_cnt = 0; 10402 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr); 10403 tp->t_fb_ptr = NULL; 10404 } 10405 /* Make sure snd_nxt is correctly set */ 10406 tp->snd_nxt = tp->snd_max; 10407 } 10408 10409 static void 10410 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win) 10411 { 10412 switch (tp->t_state) { 10413 case TCPS_SYN_SENT: 10414 bbr->r_state = TCPS_SYN_SENT; 10415 bbr->r_substate = bbr_do_syn_sent; 10416 break; 10417 case TCPS_SYN_RECEIVED: 10418 bbr->r_state = TCPS_SYN_RECEIVED; 10419 bbr->r_substate = bbr_do_syn_recv; 10420 break; 10421 case TCPS_ESTABLISHED: 10422 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd); 10423 bbr->r_state = TCPS_ESTABLISHED; 10424 bbr->r_substate = bbr_do_established; 10425 break; 10426 case TCPS_CLOSE_WAIT: 10427 bbr->r_state = TCPS_CLOSE_WAIT; 10428 bbr->r_substate = bbr_do_close_wait; 10429 break; 10430 case TCPS_FIN_WAIT_1: 10431 bbr->r_state = TCPS_FIN_WAIT_1; 10432 bbr->r_substate = bbr_do_fin_wait_1; 10433 break; 10434 case TCPS_CLOSING: 10435 bbr->r_state = TCPS_CLOSING; 10436 bbr->r_substate = bbr_do_closing; 10437 break; 10438 case TCPS_LAST_ACK: 10439 bbr->r_state = TCPS_LAST_ACK; 10440 bbr->r_substate = bbr_do_lastack; 10441 break; 10442 case TCPS_FIN_WAIT_2: 10443 bbr->r_state = TCPS_FIN_WAIT_2; 10444 bbr->r_substate = bbr_do_fin_wait_2; 10445 break; 10446 case TCPS_LISTEN: 10447 case TCPS_CLOSED: 10448 case TCPS_TIME_WAIT: 10449 default: 10450 break; 10451 }; 10452 } 10453 10454 static void 10455 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int32_t line, int dolog) 10456 { 10457 /* 10458 * Now what state are we going into now? Is there adjustments 10459 * needed? 10460 */ 10461 int32_t old_state, old_gain; 10462 10463 10464 old_state = bbr_state_val(bbr); 10465 old_gain = bbr->r_ctl.rc_bbr_hptsi_gain; 10466 if (bbr_state_val(bbr) == BBR_SUB_LEVEL1) { 10467 /* Save the lowest srtt we saw in our end of the sub-state */ 10468 bbr->rc_hit_state_1 = 0; 10469 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff) 10470 bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state; 10471 } 10472 bbr->rc_bbr_substate++; 10473 if (bbr->rc_bbr_substate >= BBR_SUBSTATE_COUNT) { 10474 /* Cycle back to first state-> gain */ 10475 bbr->rc_bbr_substate = 0; 10476 } 10477 if (bbr_state_val(bbr) == BBR_SUB_GAIN) { 10478 /* 10479 * We enter the gain(5/4) cycle (possibly less if 10480 * shallow buffer detection is enabled) 10481 */ 10482 if (bbr->skip_gain) { 10483 /* 10484 * Hardware pacing has set our rate to 10485 * the max and limited our b/w just 10486 * do level i.e. no gain. 10487 */ 10488 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1]; 10489 } else if (bbr->gain_is_limited && 10490 bbr->bbr_hdrw_pacing && 10491 bbr->r_ctl.crte) { 10492 /* 10493 * We can't gain above the hardware pacing 10494 * rate which is less than our rate + the gain 10495 * calculate the gain needed to reach the hardware 10496 * pacing rate.. 10497 */ 10498 uint64_t bw, rate, gain_calc; 10499 10500 bw = bbr_get_bw(bbr); 10501 rate = bbr->r_ctl.crte->rate; 10502 if ((rate > bw) && 10503 (((bw * (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN]) / (uint64_t)BBR_UNIT) > rate)) { 10504 gain_calc = (rate * BBR_UNIT) / bw; 10505 if (gain_calc < BBR_UNIT) 10506 gain_calc = BBR_UNIT; 10507 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc; 10508 } else { 10509 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN]; 10510 } 10511 } else 10512 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN]; 10513 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) { 10514 bbr->r_ctl.rc_bbr_state_atflight = cts; 10515 } else 10516 bbr->r_ctl.rc_bbr_state_atflight = 0; 10517 } else if (bbr_state_val(bbr) == BBR_SUB_DRAIN) { 10518 bbr->rc_hit_state_1 = 1; 10519 bbr->r_ctl.rc_exta_time_gd = 0; 10520 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp, 10521 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 10522 if (bbr_state_drain_2_tar) { 10523 bbr->r_ctl.rc_bbr_state_atflight = 0; 10524 } else 10525 bbr->r_ctl.rc_bbr_state_atflight = cts; 10526 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN]; 10527 } else { 10528 /* All other cycles hit here 2-7 */ 10529 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) { 10530 if (bbr_sub_drain_slam_cwnd && 10531 (bbr->rc_use_google == 0) && 10532 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { 10533 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; 10534 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10535 } 10536 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP)) 10537 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) - 10538 bbr_get_rtt(bbr, BBR_RTT_PROP)); 10539 else 10540 bbr->r_ctl.rc_exta_time_gd = 0; 10541 if (bbr->r_ctl.rc_exta_time_gd) { 10542 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd; 10543 /* Now chop up the time for each state (div by 7) */ 10544 bbr->r_ctl.rc_level_state_extra /= 7; 10545 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) { 10546 /* Add a randomization */ 10547 bbr_randomize_extra_state_time(bbr); 10548 } 10549 } 10550 } 10551 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); 10552 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)]; 10553 } 10554 if (bbr->rc_use_google) { 10555 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts); 10556 } 10557 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 10558 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain; 10559 if (dolog) 10560 bbr_log_type_statechange(bbr, cts, line); 10561 10562 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { 10563 uint32_t time_in; 10564 10565 time_in = cts - bbr->r_ctl.rc_bbr_state_time; 10566 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { 10567 counter_u64_add(bbr_state_time[(old_state + 5)], time_in); 10568 } else { 10569 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); 10570 } 10571 } 10572 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff; 10573 bbr_set_state_target(bbr, __LINE__); 10574 if (bbr_sub_drain_slam_cwnd && 10575 (bbr->rc_use_google == 0) && 10576 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) { 10577 /* Slam down the cwnd */ 10578 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; 10579 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; 10580 if (bbr_sub_drain_app_limit) { 10581 /* Go app limited if we are on a long drain */ 10582 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + 10583 ctf_flight_size(bbr->rc_tp, 10584 (bbr->r_ctl.rc_sacked + 10585 bbr->r_ctl.rc_lost_bytes))); 10586 } 10587 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10588 } 10589 if (bbr->rc_lt_use_bw) { 10590 /* In policed mode we clamp pacing_gain to BBR_UNIT */ 10591 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; 10592 } 10593 /* Google changes TSO size every cycle */ 10594 if (bbr->rc_use_google) 10595 tcp_bbr_tso_size_check(bbr, cts); 10596 bbr->r_ctl.gain_epoch = cts; 10597 bbr->r_ctl.rc_bbr_state_time = cts; 10598 bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch; 10599 } 10600 10601 static void 10602 bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses) 10603 { 10604 if ((bbr_state_val(bbr) == BBR_SUB_DRAIN) && 10605 (google_allow_early_out == 1) && 10606 (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) { 10607 /* We have reached out target flight size possibly early */ 10608 goto change_state; 10609 } 10610 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) { 10611 return; 10612 } 10613 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) { 10614 /* 10615 * Must be a rttProp movement forward before 10616 * we can change states. 10617 */ 10618 return; 10619 } 10620 if (bbr_state_val(bbr) == BBR_SUB_GAIN) { 10621 /* 10622 * The needed time has passed but for 10623 * the gain cycle extra rules apply: 10624 * 1) If we have seen loss, we exit 10625 * 2) If we have not reached the target 10626 * we stay in GAIN (gain-to-target). 10627 */ 10628 if (google_consider_lost && losses) 10629 goto change_state; 10630 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) { 10631 return; 10632 } 10633 } 10634 change_state: 10635 /* For gain we must reach our target, all others last 1 rttProp */ 10636 bbr_substate_change(bbr, cts, __LINE__, 1); 10637 } 10638 10639 static void 10640 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses) 10641 { 10642 uint32_t flight, bbr_cur_cycle_time; 10643 10644 if (bbr->rc_use_google) { 10645 bbr_set_probebw_google_gains(bbr, cts, losses); 10646 return; 10647 } 10648 if (cts == 0) { 10649 /* 10650 * Never alow cts to be 0 we 10651 * do this so we can judge if 10652 * we have set a timestamp. 10653 */ 10654 cts = 1; 10655 } 10656 if (bbr_state_is_pkt_epoch) 10657 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PKTRTT); 10658 else 10659 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PROP); 10660 10661 if (bbr->r_ctl.rc_bbr_state_atflight == 0) { 10662 if (bbr_state_val(bbr) == BBR_SUB_DRAIN) { 10663 flight = ctf_flight_size(bbr->rc_tp, 10664 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 10665 if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) { 10666 /* Keep it slam down */ 10667 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) { 10668 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; 10669 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10670 } 10671 if (bbr_sub_drain_app_limit) { 10672 /* Go app limited if we are on a long drain */ 10673 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight); 10674 } 10675 } 10676 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) && 10677 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) || 10678 (flight >= bbr->r_ctl.flightsize_at_drain))) { 10679 /* 10680 * Still here after the same time as 10681 * the gain. We need to drain harder 10682 * for the next srtt. Reduce by a set amount 10683 * the gain drop is capped at DRAIN states 10684 * value (88). 10685 */ 10686 bbr->r_ctl.flightsize_at_drain = flight; 10687 if (bbr_drain_drop_mul && 10688 bbr_drain_drop_div && 10689 (bbr_drain_drop_mul < bbr_drain_drop_div)) { 10690 /* Use your specific drop value (def 4/5 = 20%) */ 10691 bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul; 10692 bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div; 10693 } else { 10694 /* You get drop of 20% */ 10695 bbr->r_ctl.rc_bbr_hptsi_gain *= 4; 10696 bbr->r_ctl.rc_bbr_hptsi_gain /= 5; 10697 } 10698 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) { 10699 /* Reduce our gain again to the bottom */ 10700 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1); 10701 } 10702 bbr_log_exit_gain(bbr, cts, 4); 10703 /* 10704 * Extend out so we wait another 10705 * epoch before dropping again. 10706 */ 10707 bbr->r_ctl.gain_epoch = cts; 10708 } 10709 if (flight <= bbr->r_ctl.rc_target_at_state) { 10710 if (bbr_sub_drain_slam_cwnd && 10711 (bbr->rc_use_google == 0) && 10712 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { 10713 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; 10714 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10715 } 10716 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); 10717 bbr_log_exit_gain(bbr, cts, 3); 10718 } 10719 } else { 10720 /* Its a gain */ 10721 if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) { 10722 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); 10723 goto change_state; 10724 } 10725 if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) || 10726 ((ctf_outstanding(bbr->rc_tp) + bbr->rc_tp->t_maxseg - 1) >= 10727 bbr->rc_tp->snd_wnd)) { 10728 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1); 10729 bbr_log_exit_gain(bbr, cts, 2); 10730 } 10731 } 10732 /** 10733 * We fall through and return always one of two things has 10734 * occured. 10735 * 1) We are still not at target 10736 * <or> 10737 * 2) We reached the target and set rc_bbr_state_atflight 10738 * which means we no longer hit this block 10739 * next time we are called. 10740 */ 10741 return; 10742 } 10743 change_state: 10744 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) 10745 return; 10746 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) { 10747 /* Less than a full time-period has passed */ 10748 return; 10749 } 10750 if (bbr->r_ctl.rc_level_state_extra && 10751 (bbr_state_val(bbr) > BBR_SUB_DRAIN) && 10752 ((cts - bbr->r_ctl.rc_bbr_state_time) < 10753 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) { 10754 /* Less than a full time-period + extra has passed */ 10755 return; 10756 } 10757 if (bbr_gain_gets_extra_too && 10758 bbr->r_ctl.rc_level_state_extra && 10759 (bbr_state_val(bbr) == BBR_SUB_GAIN) && 10760 ((cts - bbr->r_ctl.rc_bbr_state_time) < 10761 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) { 10762 /* Less than a full time-period + extra has passed */ 10763 return; 10764 } 10765 bbr_substate_change(bbr, cts, __LINE__, 1); 10766 } 10767 10768 static uint32_t 10769 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain) 10770 { 10771 uint32_t mss, tar; 10772 10773 if (bbr->rc_use_google) { 10774 /* Google just uses the cwnd target */ 10775 tar = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), gain); 10776 } else { 10777 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), 10778 bbr->r_ctl.rc_pace_max_segs); 10779 /* Get the base cwnd with gain rounded to a mss */ 10780 tar = roundup(bbr_get_raw_target_cwnd(bbr, bbr_get_bw(bbr), 10781 gain), mss); 10782 /* Make sure it is within our min */ 10783 if (tar < get_min_cwnd(bbr)) 10784 return (get_min_cwnd(bbr)); 10785 } 10786 return (tar); 10787 } 10788 10789 static void 10790 bbr_set_state_target(struct tcp_bbr *bbr, int line) 10791 { 10792 uint32_t tar, meth; 10793 10794 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) && 10795 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) { 10796 /* Special case using old probe-rtt method */ 10797 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); 10798 meth = 1; 10799 } else { 10800 /* Non-probe-rtt case and reduced probe-rtt */ 10801 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) && 10802 (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) { 10803 /* For gain cycle we use the hptsi gain */ 10804 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain); 10805 meth = 2; 10806 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) { 10807 /* 10808 * If configured, or for google all other states 10809 * get BBR_UNIT. 10810 */ 10811 tar = bbr_get_a_state_target(bbr, BBR_UNIT); 10812 meth = 3; 10813 } else { 10814 /* 10815 * Or we set a target based on the pacing gain 10816 * for non-google mode and default (non-configured). 10817 * Note we don't set a target goal below drain (192). 10818 */ 10819 if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN]) { 10820 tar = bbr_get_a_state_target(bbr, bbr_hptsi_gain[BBR_SUB_DRAIN]); 10821 meth = 4; 10822 } else { 10823 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain); 10824 meth = 5; 10825 } 10826 } 10827 } 10828 bbr_log_set_of_state_target(bbr, tar, line, meth); 10829 bbr->r_ctl.rc_target_at_state = tar; 10830 } 10831 10832 static void 10833 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line) 10834 { 10835 /* Change to probe_rtt */ 10836 uint32_t time_in; 10837 10838 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 10839 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp, 10840 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 10841 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain 10842 + bbr->r_ctl.rc_delivered); 10843 /* Setup so we force feed the filter */ 10844 if (bbr->rc_use_google || bbr_probertt_sets_rtt) 10845 bbr->rc_prtt_set_ts = 1; 10846 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { 10847 time_in = cts - bbr->r_ctl.rc_bbr_state_time; 10848 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); 10849 } 10850 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_ENTERPROBE, 0); 10851 bbr->r_ctl.rc_rtt_shrinks = cts; 10852 bbr->r_ctl.last_in_probertt = cts; 10853 bbr->r_ctl.rc_probertt_srttchktim = cts; 10854 bbr->r_ctl.rc_bbr_state_time = cts; 10855 bbr->rc_bbr_state = BBR_STATE_PROBE_RTT; 10856 /* We need to force the filter to update */ 10857 10858 if ((bbr_sub_drain_slam_cwnd) && 10859 bbr->rc_hit_state_1 && 10860 (bbr->rc_use_google == 0) && 10861 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) { 10862 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd) 10863 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; 10864 } else 10865 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; 10866 /* Update the lost */ 10867 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 10868 if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){ 10869 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN) */ 10870 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); 10871 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10872 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; 10873 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; 10874 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6); 10875 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd; 10876 } else { 10877 /* 10878 * We bring it down slowly by using a hptsi gain that is 10879 * probably 75%. This will slowly float down our outstanding 10880 * without tampering with the cwnd. 10881 */ 10882 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val; 10883 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; 10884 bbr_set_state_target(bbr, __LINE__); 10885 if (bbr_prtt_slam_cwnd && 10886 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { 10887 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; 10888 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10889 } 10890 } 10891 if (ctf_flight_size(bbr->rc_tp, 10892 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= 10893 bbr->r_ctl.rc_target_at_state) { 10894 /* We are at target */ 10895 bbr->r_ctl.rc_bbr_enters_probertt = cts; 10896 } else { 10897 /* We need to come down to reach target before our time begins */ 10898 bbr->r_ctl.rc_bbr_enters_probertt = 0; 10899 } 10900 bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch; 10901 BBR_STAT_INC(bbr_enter_probertt); 10902 bbr_log_exit_gain(bbr, cts, 0); 10903 bbr_log_type_statechange(bbr, cts, line); 10904 } 10905 10906 static void 10907 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts) 10908 { 10909 /* 10910 * Sanity check on probe-rtt intervals. 10911 * In crazy situations where we are competing 10912 * against new-reno flows with huge buffers 10913 * our rtt-prop interval could come to dominate 10914 * things if we can't get through a full set 10915 * of cycles, we need to adjust it. 10916 */ 10917 if (bbr_can_adjust_probertt && 10918 (bbr->rc_use_google == 0)) { 10919 uint16_t val = 0; 10920 uint32_t cur_rttp, fval, newval, baseval; 10921 10922 /* Are we to small and go into probe-rtt to often? */ 10923 baseval = (bbr_get_rtt(bbr, BBR_RTT_PROP) * (BBR_SUBSTATE_COUNT + 1)); 10924 cur_rttp = roundup(baseval, USECS_IN_SECOND); 10925 fval = bbr_filter_len_sec * USECS_IN_SECOND; 10926 if (bbr_is_ratio == 0) { 10927 if (fval > bbr_rtt_probe_limit) 10928 newval = cur_rttp + (fval - bbr_rtt_probe_limit); 10929 else 10930 newval = cur_rttp; 10931 } else { 10932 int mul; 10933 10934 mul = fval / bbr_rtt_probe_limit; 10935 newval = cur_rttp * mul; 10936 } 10937 if (cur_rttp > bbr->r_ctl.rc_probertt_int) { 10938 bbr->r_ctl.rc_probertt_int = cur_rttp; 10939 reset_time_small(&bbr->r_ctl.rc_rttprop, newval); 10940 val = 1; 10941 } else { 10942 /* 10943 * No adjustments were made 10944 * do we need to shrink it? 10945 */ 10946 if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) { 10947 if (cur_rttp <= bbr_rtt_probe_limit) { 10948 /* 10949 * Things have calmed down lets 10950 * shrink all the way to default 10951 */ 10952 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit; 10953 reset_time_small(&bbr->r_ctl.rc_rttprop, 10954 (bbr_filter_len_sec * USECS_IN_SECOND)); 10955 cur_rttp = bbr_rtt_probe_limit; 10956 newval = (bbr_filter_len_sec * USECS_IN_SECOND); 10957 val = 2; 10958 } else { 10959 /* 10960 * Well does some adjustment make sense? 10961 */ 10962 if (cur_rttp < bbr->r_ctl.rc_probertt_int) { 10963 /* We can reduce interval time some */ 10964 bbr->r_ctl.rc_probertt_int = cur_rttp; 10965 reset_time_small(&bbr->r_ctl.rc_rttprop, newval); 10966 val = 3; 10967 } 10968 } 10969 } 10970 } 10971 if (val) 10972 bbr_log_rtt_shrinks(bbr, cts, cur_rttp, newval, __LINE__, BBR_RTTS_RESETS_VALUES, val); 10973 } 10974 } 10975 10976 static void 10977 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts) 10978 { 10979 /* Exit probe-rtt */ 10980 10981 if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) { 10982 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; 10983 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 10984 } 10985 bbr_log_exit_gain(bbr, cts, 1); 10986 bbr->rc_hit_state_1 = 0; 10987 bbr->r_ctl.rc_rtt_shrinks = cts; 10988 bbr->r_ctl.last_in_probertt = cts; 10989 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_RTTPROBE, 0); 10990 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 10991 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, 10992 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + 10993 bbr->r_ctl.rc_delivered); 10994 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { 10995 uint32_t time_in; 10996 10997 time_in = cts - bbr->r_ctl.rc_bbr_state_time; 10998 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); 10999 } 11000 if (bbr->rc_filled_pipe) { 11001 /* Switch to probe_bw */ 11002 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; 11003 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); 11004 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain; 11005 bbr_substate_change(bbr, cts, __LINE__, 0); 11006 bbr_log_type_statechange(bbr, cts, __LINE__); 11007 } else { 11008 /* Back to startup */ 11009 bbr->rc_bbr_state = BBR_STATE_STARTUP; 11010 bbr->r_ctl.rc_bbr_state_time = cts; 11011 /* 11012 * We don't want to give a complete free 3 11013 * measurements until we exit, so we use 11014 * the number of pe's we were in probe-rtt 11015 * to add to the startup_epoch. That way 11016 * we will still retain the old state. 11017 */ 11018 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt); 11019 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 11020 /* Make sure to use the lower pg when shifting back in */ 11021 if (bbr->r_ctl.rc_lost && 11022 bbr_use_lower_gain_in_startup && 11023 (bbr->rc_use_google == 0)) 11024 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower; 11025 else 11026 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg; 11027 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg; 11028 /* Probably not needed but set it anyway */ 11029 bbr_set_state_target(bbr, __LINE__); 11030 bbr_log_type_statechange(bbr, cts, __LINE__); 11031 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11032 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0); 11033 } 11034 bbr_check_probe_rtt_limits(bbr, cts); 11035 } 11036 11037 static int32_t inline 11038 bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts) 11039 { 11040 if ((bbr->rc_past_init_win == 1) && 11041 (bbr->rc_in_persist == 0) && 11042 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) { 11043 return (1); 11044 } 11045 if (bbr_can_force_probertt && 11046 (bbr->rc_in_persist == 0) && 11047 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) && 11048 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) { 11049 return (1); 11050 } 11051 return (0); 11052 } 11053 11054 11055 static int32_t 11056 bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t pkt_epoch) 11057 { 11058 uint64_t btlbw, gain; 11059 if (pkt_epoch == 0) { 11060 /* 11061 * Need to be on a pkt-epoch to continue. 11062 */ 11063 return (0); 11064 } 11065 btlbw = bbr_get_full_bw(bbr); 11066 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * 11067 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; 11068 if (btlbw >= gain) { 11069 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch; 11070 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11071 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3); 11072 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw; 11073 } 11074 if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) 11075 return (1); 11076 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11077 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8); 11078 return(0); 11079 } 11080 11081 static int32_t inline 11082 bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch) 11083 { 11084 /* Have we gained 25% in the last 3 packet based epoch's? */ 11085 uint64_t btlbw, gain; 11086 int do_exit; 11087 int delta, rtt_gain; 11088 11089 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) && 11090 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { 11091 /* 11092 * This qualifies as a RTT_PROBE session since we drop the 11093 * data outstanding to nothing and waited more than 11094 * bbr_rtt_probe_time. 11095 */ 11096 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0); 11097 bbr_set_reduced_rtt(bbr, cts, __LINE__); 11098 } 11099 if (bbr_should_enter_probe_rtt(bbr, cts)) { 11100 bbr_enter_probe_rtt(bbr, cts, __LINE__); 11101 return (0); 11102 } 11103 if (bbr->rc_use_google) 11104 return (bbr_google_startup(bbr, cts, pkt_epoch)); 11105 11106 if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) && 11107 (bbr_use_lower_gain_in_startup)) { 11108 /* Drop to a lower gain 1.5 x since we saw loss */ 11109 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower; 11110 } 11111 if (pkt_epoch == 0) { 11112 /* 11113 * Need to be on a pkt-epoch to continue. 11114 */ 11115 return (0); 11116 } 11117 if (bbr_rtt_gain_thresh) { 11118 /* 11119 * Do we allow a flow to stay 11120 * in startup with no loss and no 11121 * gain in rtt over a set threshold? 11122 */ 11123 if (bbr->r_ctl.rc_pkt_epoch_rtt && 11124 bbr->r_ctl.startup_last_srtt && 11125 (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) { 11126 delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt; 11127 rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt; 11128 } else 11129 rtt_gain = 0; 11130 if ((bbr->r_ctl.startup_last_srtt == 0) || 11131 (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt)) 11132 /* First time or new lower value */ 11133 bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt; 11134 11135 if ((bbr->r_ctl.rc_lost == 0) && 11136 (rtt_gain < bbr_rtt_gain_thresh)) { 11137 /* 11138 * No loss, and we are under 11139 * our gain threhold for 11140 * increasing RTT. 11141 */ 11142 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch) 11143 bbr->r_ctl.rc_bbr_last_startup_epoch++; 11144 bbr_log_startup_event(bbr, cts, rtt_gain, 11145 delta, bbr->r_ctl.startup_last_srtt, 10); 11146 return (0); 11147 } 11148 } 11149 if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) && 11150 (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) && 11151 (!IN_RECOVERY(bbr->rc_tp->t_flags))) { 11152 /* 11153 * We only assess if we have a new measurment when 11154 * we have no loss and are not in recovery. 11155 * Drag up by one our last_startup epoch so we will hold 11156 * the number of non-gain we have already accumulated. 11157 */ 11158 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch) 11159 bbr->r_ctl.rc_bbr_last_startup_epoch++; 11160 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11161 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9); 11162 return (0); 11163 } 11164 /* Case where we reduced the lost (bad retransmit) */ 11165 if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost) 11166 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 11167 bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count; 11168 btlbw = bbr_get_full_bw(bbr); 11169 if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower) 11170 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * 11171 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; 11172 else 11173 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw * 11174 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw; 11175 do_exit = 0; 11176 if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw) 11177 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw; 11178 if (btlbw >= gain) { 11179 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch; 11180 /* Update the lost so we won't exit in next set of tests */ 11181 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 11182 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11183 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3); 11184 } 11185 if ((bbr->rc_loss_exit && 11186 (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) && 11187 (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) && 11188 ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) { 11189 /* 11190 * If we had no gain, we had loss and that loss was above 11191 * our threshould, the rwnd is not constrained, and we have 11192 * had at least 3 packet epochs exit. Note that this is 11193 * switched off by sysctl. Google does not do this by the 11194 * way. 11195 */ 11196 if ((ctf_flight_size(bbr->rc_tp, 11197 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) + 11198 (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) { 11199 do_exit = 1; 11200 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11201 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4); 11202 } else { 11203 /* Just record an updated loss value */ 11204 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 11205 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11206 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5); 11207 } 11208 } else 11209 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost; 11210 if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) || 11211 do_exit) { 11212 /* Return 1 to exit the startup state. */ 11213 return (1); 11214 } 11215 /* Stay in startup */ 11216 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11217 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8); 11218 return (0); 11219 } 11220 11221 static void 11222 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t losses) 11223 { 11224 /* 11225 * A tick occured in the rtt epoch do we need to do anything? 11226 */ 11227 #ifdef BBR_INVARIANTS 11228 if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) && 11229 (bbr->rc_bbr_state != BBR_STATE_DRAIN) && 11230 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) && 11231 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) && 11232 (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) { 11233 /* Debug code? */ 11234 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state); 11235 } 11236 #endif 11237 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { 11238 /* Do we exit the startup state? */ 11239 if (bbr_state_startup(bbr, cts, epoch, pkt_epoch)) { 11240 uint32_t time_in; 11241 11242 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch, 11243 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6); 11244 bbr->rc_filled_pipe = 1; 11245 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 11246 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { 11247 11248 time_in = cts - bbr->r_ctl.rc_bbr_state_time; 11249 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); 11250 } else 11251 time_in = 0; 11252 if (bbr->rc_no_pacing) 11253 bbr->rc_no_pacing = 0; 11254 bbr->r_ctl.rc_bbr_state_time = cts; 11255 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg; 11256 bbr->rc_bbr_state = BBR_STATE_DRAIN; 11257 bbr_set_state_target(bbr, __LINE__); 11258 if ((bbr->rc_use_google == 0) && 11259 bbr_slam_cwnd_in_main_drain) { 11260 /* Here we don't have to worry about probe-rtt */ 11261 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd; 11262 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; 11263 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 11264 } 11265 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain; 11266 bbr_log_type_statechange(bbr, cts, __LINE__); 11267 if (ctf_flight_size(bbr->rc_tp, 11268 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <= 11269 bbr->r_ctl.rc_target_at_state) { 11270 /* 11271 * Switch to probe_bw if we are already 11272 * there 11273 */ 11274 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); 11275 bbr_substate_change(bbr, cts, __LINE__, 0); 11276 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; 11277 bbr_log_type_statechange(bbr, cts, __LINE__); 11278 } 11279 } 11280 } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) { 11281 uint32_t inflight; 11282 struct tcpcb *tp; 11283 11284 tp = bbr->rc_tp; 11285 inflight = ctf_flight_size(tp, 11286 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 11287 if (inflight >= bbr->r_ctl.rc_target_at_state) { 11288 /* We have reached a flight of the cwnd target */ 11289 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; 11290 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; 11291 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT; 11292 bbr_set_state_target(bbr, __LINE__); 11293 /* 11294 * Rig it so we don't do anything crazy and 11295 * start fresh with a new randomization. 11296 */ 11297 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff; 11298 bbr->rc_bbr_substate = BBR_SUB_LEVEL6; 11299 bbr_substate_change(bbr, cts, __LINE__, 1); 11300 } 11301 } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) { 11302 /* Has in-flight reached the bdp (or less)? */ 11303 uint32_t inflight; 11304 struct tcpcb *tp; 11305 11306 tp = bbr->rc_tp; 11307 inflight = ctf_flight_size(tp, 11308 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 11309 if ((bbr->rc_use_google == 0) && 11310 bbr_slam_cwnd_in_main_drain && 11311 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { 11312 /* 11313 * Here we don't have to worry about probe-rtt 11314 * re-slam it, but keep it slammed down. 11315 */ 11316 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; 11317 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 11318 } 11319 if (inflight <= bbr->r_ctl.rc_target_at_state) { 11320 /* We have drained */ 11321 bbr->rc_bbr_state = BBR_STATE_PROBE_BW; 11322 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost; 11323 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) { 11324 uint32_t time_in; 11325 11326 time_in = cts - bbr->r_ctl.rc_bbr_state_time; 11327 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in); 11328 } 11329 if ((bbr->rc_use_google == 0) && 11330 bbr_slam_cwnd_in_main_drain && 11331 (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) { 11332 /* Restore the cwnd */ 11333 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd; 11334 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 11335 } 11336 /* Setup probe-rtt has being done now RRS-HERE */ 11337 bbr->r_ctl.rc_rtt_shrinks = cts; 11338 bbr->r_ctl.last_in_probertt = cts; 11339 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_LEAVE_DRAIN, 0); 11340 /* Randomly pick a sub-state */ 11341 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts); 11342 bbr_substate_change(bbr, cts, __LINE__, 0); 11343 bbr_log_type_statechange(bbr, cts, __LINE__); 11344 } 11345 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) { 11346 uint32_t flight; 11347 11348 flight = ctf_flight_size(bbr->rc_tp, 11349 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 11350 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered); 11351 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) && 11352 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { 11353 /* 11354 * We must keep cwnd at the desired MSS. 11355 */ 11356 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options); 11357 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 11358 } else if ((bbr_prtt_slam_cwnd) && 11359 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) { 11360 /* Re-slam it */ 11361 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state; 11362 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__); 11363 } 11364 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) { 11365 /* Has outstanding reached our target? */ 11366 if (flight <= bbr->r_ctl.rc_target_at_state) { 11367 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_REACHTAR, 0); 11368 bbr->r_ctl.rc_bbr_enters_probertt = cts; 11369 /* If time is exactly 0, be 1usec off */ 11370 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) 11371 bbr->r_ctl.rc_bbr_enters_probertt = 1; 11372 if (bbr->rc_use_google == 0) { 11373 /* 11374 * Restore any lowering that as occured to 11375 * reach here 11376 */ 11377 if (bbr->r_ctl.bbr_rttprobe_gain_val) 11378 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val; 11379 else 11380 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT; 11381 } 11382 } 11383 if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) && 11384 (bbr->rc_use_google == 0) && 11385 bbr->r_ctl.bbr_rttprobe_gain_val && 11386 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) || 11387 (flight >= bbr->r_ctl.flightsize_at_drain))) { 11388 /* 11389 * We have doddled with our current hptsi 11390 * gain an srtt and have still not made it 11391 * to target, or we have increased our flight. 11392 * Lets reduce the gain by xx% 11393 * flooring the reduce at DRAIN (based on 11394 * mul/div) 11395 */ 11396 int red; 11397 11398 bbr->r_ctl.flightsize_at_drain = flight; 11399 bbr->r_ctl.rc_probertt_srttchktim = cts; 11400 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1); 11401 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) { 11402 /* Reduce our gain again */ 11403 bbr->r_ctl.rc_bbr_hptsi_gain -= red; 11404 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG, 0); 11405 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) { 11406 /* one more chance before we give up */ 11407 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1); 11408 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG_FINAL, 0); 11409 } else { 11410 /* At the very bottom */ 11411 bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1); 11412 } 11413 } 11414 } 11415 if (bbr->r_ctl.rc_bbr_enters_probertt && 11416 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) && 11417 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) { 11418 /* Time to exit probe RTT normally */ 11419 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts); 11420 } 11421 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) { 11422 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) && 11423 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { 11424 /* 11425 * This qualifies as a RTT_PROBE session since we 11426 * drop the data outstanding to nothing and waited 11427 * more than bbr_rtt_probe_time. 11428 */ 11429 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0); 11430 bbr_set_reduced_rtt(bbr, cts, __LINE__); 11431 } 11432 if (bbr_should_enter_probe_rtt(bbr, cts)) { 11433 bbr_enter_probe_rtt(bbr, cts, __LINE__); 11434 } else { 11435 bbr_set_probebw_gains(bbr, cts, losses); 11436 } 11437 } 11438 } 11439 11440 static void 11441 bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses) 11442 { 11443 int32_t epoch = 0; 11444 11445 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) { 11446 bbr_set_epoch(bbr, cts, line); 11447 /* At each epoch doe lt bw sampling */ 11448 epoch = 1; 11449 } 11450 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses); 11451 } 11452 11453 static int 11454 bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, 11455 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, 11456 int32_t nxt_pkt, struct timeval *tv) 11457 { 11458 int32_t thflags, retval; 11459 uint32_t cts, lcts; 11460 uint32_t tiwin; 11461 struct tcpopt to; 11462 struct tcp_bbr *bbr; 11463 struct bbr_sendmap *rsm; 11464 struct timeval ltv; 11465 int32_t did_out = 0; 11466 int32_t in_recovery; 11467 uint16_t nsegs; 11468 int32_t prev_state; 11469 uint32_t lost; 11470 11471 nsegs = max(1, m->m_pkthdr.lro_nsegs); 11472 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 11473 /* add in our stats */ 11474 kern_prefetch(bbr, &prev_state); 11475 prev_state = 0; 11476 thflags = th->th_flags; 11477 /* 11478 * If this is either a state-changing packet or current state isn't 11479 * established, we require a write lock on tcbinfo. Otherwise, we 11480 * allow the tcbinfo to be in either alocked or unlocked, as the 11481 * caller may have unnecessarily acquired a write lock due to a 11482 * race. 11483 */ 11484 INP_WLOCK_ASSERT(tp->t_inpcb); 11485 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", 11486 __func__)); 11487 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", 11488 __func__)); 11489 11490 tp->t_rcvtime = ticks; 11491 /* 11492 * Unscale the window into a 32-bit value. For the SYN_SENT state 11493 * the scale is zero. 11494 */ 11495 tiwin = th->th_win << tp->snd_scale; 11496 #ifdef STATS 11497 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin); 11498 #endif 11499 /* 11500 * Parse options on any incoming segment. 11501 */ 11502 tcp_dooptions(&to, (u_char *)(th + 1), 11503 (th->th_off << 2) - sizeof(struct tcphdr), 11504 (thflags & TH_SYN) ? TO_SYN : 0); 11505 11506 if (m->m_flags & M_TSTMP) { 11507 /* Prefer the hardware timestamp if present */ 11508 struct timespec ts; 11509 11510 mbuf_tstmp2timespec(m, &ts); 11511 bbr->rc_tv.tv_sec = ts.tv_sec; 11512 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000; 11513 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv); 11514 } else if (m->m_flags & M_TSTMP_LRO) { 11515 /* Next the arrival timestamp */ 11516 struct timespec ts; 11517 11518 mbuf_tstmp2timespec(m, &ts); 11519 bbr->rc_tv.tv_sec = ts.tv_sec; 11520 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000; 11521 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv); 11522 } else { 11523 /* 11524 * Ok just get the current time. 11525 */ 11526 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv); 11527 } 11528 /* 11529 * If echoed timestamp is later than the current time, fall back to 11530 * non RFC1323 RTT calculation. Normalize timestamp if syncookies 11531 * were used when this connection was established. 11532 */ 11533 if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) { 11534 to.to_tsecr -= tp->ts_offset; 11535 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv))) 11536 to.to_tsecr = 0; 11537 } 11538 /* 11539 * If its the first time in we need to take care of options and 11540 * verify we can do SACK for rack! 11541 */ 11542 if (bbr->r_state == 0) { 11543 /* 11544 * Process options only when we get SYN/ACK back. The SYN 11545 * case for incoming connections is handled in tcp_syncache. 11546 * According to RFC1323 the window field in a SYN (i.e., a 11547 * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX 11548 * this is traditional behavior, may need to be cleaned up. 11549 */ 11550 if (bbr->rc_inp == NULL) { 11551 bbr->rc_inp = tp->t_inpcb; 11552 } 11553 /* 11554 * We need to init rc_inp here since its not init'd when 11555 * bbr_init is called 11556 */ 11557 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { 11558 if ((to.to_flags & TOF_SCALE) && 11559 (tp->t_flags & TF_REQ_SCALE)) { 11560 tp->t_flags |= TF_RCVD_SCALE; 11561 tp->snd_scale = to.to_wscale; 11562 } else 11563 tp->t_flags &= ~TF_REQ_SCALE; 11564 /* 11565 * Initial send window. It will be updated with the 11566 * next incoming segment to the scaled value. 11567 */ 11568 tp->snd_wnd = th->th_win; 11569 if ((to.to_flags & TOF_TS) && 11570 (tp->t_flags & TF_REQ_TSTMP)) { 11571 tp->t_flags |= TF_RCVD_TSTMP; 11572 tp->ts_recent = to.to_tsval; 11573 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv); 11574 } else 11575 tp->t_flags &= ~TF_REQ_TSTMP; 11576 if (to.to_flags & TOF_MSS) 11577 tcp_mss(tp, to.to_mss); 11578 if ((tp->t_flags & TF_SACK_PERMIT) && 11579 (to.to_flags & TOF_SACKPERM) == 0) 11580 tp->t_flags &= ~TF_SACK_PERMIT; 11581 if (IS_FASTOPEN(tp->t_flags)) { 11582 if (to.to_flags & TOF_FASTOPEN) { 11583 uint16_t mss; 11584 11585 if (to.to_flags & TOF_MSS) 11586 mss = to.to_mss; 11587 else 11588 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0) 11589 mss = TCP6_MSS; 11590 else 11591 mss = TCP_MSS; 11592 tcp_fastopen_update_cache(tp, mss, 11593 to.to_tfo_len, to.to_tfo_cookie); 11594 } else 11595 tcp_fastopen_disable_path(tp); 11596 } 11597 } 11598 /* 11599 * At this point we are at the initial call. Here we decide 11600 * if we are doing RACK or not. We do this by seeing if 11601 * TF_SACK_PERMIT is set, if not rack is *not* possible and 11602 * we switch to the default code. 11603 */ 11604 if ((tp->t_flags & TF_SACK_PERMIT) == 0) { 11605 /* Bail */ 11606 tcp_switch_back_to_default(tp); 11607 (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen, 11608 tlen, iptos); 11609 return (1); 11610 } 11611 /* Set the flag */ 11612 bbr->r_is_v6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0; 11613 tcp_set_hpts(tp->t_inpcb); 11614 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack); 11615 } 11616 if (thflags & TH_ACK) { 11617 /* Track ack types */ 11618 if (to.to_flags & TOF_SACK) 11619 BBR_STAT_INC(bbr_acks_with_sacks); 11620 else 11621 BBR_STAT_INC(bbr_plain_acks); 11622 } 11623 /* 11624 * This is the one exception case where we set the rack state 11625 * always. All other times (timers etc) we must have a rack-state 11626 * set (so we assure we have done the checks above for SACK). 11627 */ 11628 if (thflags & TH_FIN) 11629 tcp_log_end_status(tp, TCP_EI_STATUS_CLIENT_FIN); 11630 if (bbr->r_state != tp->t_state) 11631 bbr_set_state(tp, bbr, tiwin); 11632 11633 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL) 11634 kern_prefetch(rsm, &prev_state); 11635 prev_state = bbr->r_state; 11636 bbr->rc_ack_was_delayed = 0; 11637 lost = bbr->r_ctl.rc_lost; 11638 bbr->rc_is_pkt_epoch_now = 0; 11639 if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) { 11640 /* Get the real time into lcts and figure the real delay */ 11641 lcts = tcp_get_usecs(<v); 11642 if (TSTMP_GT(lcts, cts)) { 11643 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts; 11644 bbr->rc_ack_was_delayed = 1; 11645 if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay, 11646 bbr->r_ctl.highest_hdwr_delay)) 11647 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay; 11648 } else { 11649 bbr->r_ctl.rc_ack_hdwr_delay = 0; 11650 bbr->rc_ack_was_delayed = 0; 11651 } 11652 } else { 11653 bbr->r_ctl.rc_ack_hdwr_delay = 0; 11654 bbr->rc_ack_was_delayed = 0; 11655 } 11656 bbr_log_ack_event(bbr, th, &to, tlen, nsegs, cts, nxt_pkt, m); 11657 if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) { 11658 retval = 0; 11659 m_freem(m); 11660 goto done_with_input; 11661 } 11662 /* 11663 * If a segment with the ACK-bit set arrives in the SYN-SENT state 11664 * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9. 11665 */ 11666 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && 11667 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { 11668 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 11669 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11670 return (1); 11671 } 11672 in_recovery = IN_RECOVERY(tp->t_flags); 11673 if (tiwin > bbr->r_ctl.rc_high_rwnd) 11674 bbr->r_ctl.rc_high_rwnd = tiwin; 11675 #ifdef BBR_INVARIANTS 11676 if ((tp->t_inpcb->inp_flags & INP_DROPPED) || 11677 (tp->t_inpcb->inp_flags2 & INP_FREED)) { 11678 panic("tp:%p bbr:%p given a dropped inp:%p", 11679 tp, bbr, tp->t_inpcb); 11680 } 11681 #endif 11682 bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp, 11683 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 11684 bbr->rtt_valid = 0; 11685 if (to.to_flags & TOF_TS) { 11686 bbr->rc_ts_valid = 1; 11687 bbr->r_ctl.last_inbound_ts = to.to_tsval; 11688 } else { 11689 bbr->rc_ts_valid = 0; 11690 bbr->r_ctl.last_inbound_ts = 0; 11691 } 11692 retval = (*bbr->r_substate) (m, th, so, 11693 tp, &to, drop_hdrlen, 11694 tlen, tiwin, thflags, nxt_pkt, iptos); 11695 #ifdef BBR_INVARIANTS 11696 if ((retval == 0) && 11697 (tp->t_inpcb == NULL)) { 11698 panic("retval:%d tp:%p t_inpcb:NULL state:%d", 11699 retval, tp, prev_state); 11700 } 11701 #endif 11702 if (nxt_pkt == 0) 11703 BBR_STAT_INC(bbr_rlock_left_ret0); 11704 else 11705 BBR_STAT_INC(bbr_rlock_left_ret1); 11706 if (retval == 0) { 11707 /* 11708 * If retval is 1 the tcb is unlocked and most likely the tp 11709 * is gone. 11710 */ 11711 INP_WLOCK_ASSERT(tp->t_inpcb); 11712 tcp_bbr_xmit_timer_commit(bbr, tp, cts); 11713 if (bbr->rc_is_pkt_epoch_now) 11714 bbr_set_pktepoch(bbr, cts, __LINE__); 11715 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost)); 11716 if (nxt_pkt == 0) { 11717 if (bbr->r_wanted_output != 0) { 11718 bbr->rc_output_starts_timer = 0; 11719 did_out = 1; 11720 (void)tp->t_fb->tfb_tcp_output(tp); 11721 } else 11722 bbr_start_hpts_timer(bbr, tp, cts, 6, 0, 0); 11723 } 11724 if ((nxt_pkt == 0) && 11725 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) && 11726 (SEQ_GT(tp->snd_max, tp->snd_una) || 11727 (tp->t_flags & TF_DELACK) || 11728 ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) && 11729 (tp->t_state <= TCPS_CLOSING)))) { 11730 /* 11731 * We could not send (probably in the hpts but 11732 * stopped the timer)? 11733 */ 11734 if ((tp->snd_max == tp->snd_una) && 11735 ((tp->t_flags & TF_DELACK) == 0) && 11736 (bbr->rc_inp->inp_in_hpts) && 11737 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { 11738 /* 11739 * keep alive not needed if we are hptsi 11740 * output yet 11741 */ 11742 ; 11743 } else { 11744 if (bbr->rc_inp->inp_in_hpts) { 11745 tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT); 11746 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && 11747 (TSTMP_GT(lcts, bbr->rc_pacer_started))) { 11748 uint32_t del; 11749 11750 del = lcts - bbr->rc_pacer_started; 11751 if (bbr->r_ctl.rc_last_delay_val > del) { 11752 BBR_STAT_INC(bbr_force_timer_start); 11753 bbr->r_ctl.rc_last_delay_val -= del; 11754 bbr->rc_pacer_started = lcts; 11755 } else { 11756 /* We are late */ 11757 bbr->r_ctl.rc_last_delay_val = 0; 11758 BBR_STAT_INC(bbr_force_output); 11759 (void)tp->t_fb->tfb_tcp_output(tp); 11760 } 11761 } 11762 } 11763 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val, 11764 0); 11765 } 11766 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) { 11767 /* Do we have the correct timer running? */ 11768 bbr_timer_audit(tp, bbr, lcts, &so->so_snd); 11769 } 11770 /* Do we have a new state */ 11771 if (bbr->r_state != tp->t_state) 11772 bbr_set_state(tp, bbr, tiwin); 11773 done_with_input: 11774 bbr_log_doseg_done(bbr, cts, nxt_pkt, did_out); 11775 if (did_out) 11776 bbr->r_wanted_output = 0; 11777 #ifdef BBR_INVARIANTS 11778 if (tp->t_inpcb == NULL) { 11779 panic("OP:%d retval:%d tp:%p t_inpcb:NULL state:%d", 11780 did_out, 11781 retval, tp, prev_state); 11782 } 11783 #endif 11784 } 11785 return (retval); 11786 } 11787 11788 static void 11789 bbr_log_type_hrdwtso(struct tcpcb *tp, struct tcp_bbr *bbr, int len, int mod, int what_we_can_send) 11790 { 11791 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 11792 union tcp_log_stackspecific log; 11793 struct timeval tv; 11794 uint32_t cts; 11795 11796 cts = tcp_get_usecs(&tv); 11797 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 11798 log.u_bbr.flex1 = bbr->r_ctl.rc_pace_min_segs; 11799 log.u_bbr.flex2 = what_we_can_send; 11800 log.u_bbr.flex3 = bbr->r_ctl.rc_pace_max_segs; 11801 log.u_bbr.flex4 = len; 11802 log.u_bbr.flex5 = 0; 11803 log.u_bbr.flex7 = mod; 11804 log.u_bbr.flex8 = 1; 11805 TCP_LOG_EVENTP(tp, NULL, 11806 &tp->t_inpcb->inp_socket->so_rcv, 11807 &tp->t_inpcb->inp_socket->so_snd, 11808 TCP_HDWR_TLS, 0, 11809 0, &log, false, &tv); 11810 } 11811 } 11812 11813 static void 11814 bbr_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, 11815 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos) 11816 { 11817 struct timeval tv; 11818 int retval; 11819 11820 /* First lets see if we have old packets */ 11821 if (tp->t_in_pkt) { 11822 if (ctf_do_queued_segments(so, tp, 1)) { 11823 m_freem(m); 11824 return; 11825 } 11826 } 11827 if (m->m_flags & M_TSTMP_LRO) { 11828 tv.tv_sec = m->m_pkthdr.rcv_tstmp /1000000000; 11829 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000)/1000; 11830 } else { 11831 /* Should not be should we kassert instead? */ 11832 tcp_get_usecs(&tv); 11833 } 11834 retval = bbr_do_segment_nounlock(m, th, so, tp, 11835 drop_hdrlen, tlen, iptos, 0, &tv); 11836 if (retval == 0) 11837 INP_WUNLOCK(tp->t_inpcb); 11838 } 11839 11840 /* 11841 * Return how much data can be sent without violating the 11842 * cwnd or rwnd. 11843 */ 11844 11845 static inline uint32_t 11846 bbr_what_can_we_send(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t sendwin, 11847 uint32_t avail, int32_t sb_offset, uint32_t cts) 11848 { 11849 uint32_t len; 11850 11851 if (ctf_outstanding(tp) >= tp->snd_wnd) { 11852 /* We never want to go over our peers rcv-window */ 11853 len = 0; 11854 } else { 11855 uint32_t flight; 11856 11857 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)); 11858 if (flight >= sendwin) { 11859 /* 11860 * We have in flight what we are allowed by cwnd (if 11861 * it was rwnd blocking it would have hit above out 11862 * >= tp->snd_wnd). 11863 */ 11864 return (0); 11865 } 11866 len = sendwin - flight; 11867 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) { 11868 /* We would send too much (beyond the rwnd) */ 11869 len = tp->snd_wnd - ctf_outstanding(tp); 11870 } 11871 if ((len + sb_offset) > avail) { 11872 /* 11873 * We don't have that much in the SB, how much is 11874 * there? 11875 */ 11876 len = avail - sb_offset; 11877 } 11878 } 11879 return (len); 11880 } 11881 11882 static inline void 11883 bbr_do_error_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error) 11884 { 11885 #ifdef NETFLIX_STATS 11886 KMOD_TCPSTAT_INC(tcps_sndpack_error); 11887 KMOD_TCPSTAT_ADD(tcps_sndbyte_error, len); 11888 #endif 11889 } 11890 11891 static inline void 11892 bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error) 11893 { 11894 if (error) { 11895 bbr_do_error_accounting(tp, bbr, rsm, len, error); 11896 return; 11897 } 11898 if (rsm) { 11899 if (rsm->r_flags & BBR_TLP) { 11900 /* 11901 * TLP should not count in retran count, but in its 11902 * own bin 11903 */ 11904 #ifdef NETFLIX_STATS 11905 tp->t_sndtlppack++; 11906 tp->t_sndtlpbyte += len; 11907 KMOD_TCPSTAT_INC(tcps_tlpresends); 11908 KMOD_TCPSTAT_ADD(tcps_tlpresend_bytes, len); 11909 #endif 11910 } else { 11911 /* Retransmit */ 11912 tp->t_sndrexmitpack++; 11913 KMOD_TCPSTAT_INC(tcps_sndrexmitpack); 11914 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len); 11915 #ifdef STATS 11916 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB, 11917 len); 11918 #endif 11919 } 11920 /* 11921 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is 11922 * sub-state 11923 */ 11924 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len); 11925 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) { 11926 /* Non probe_bw log in 1, 2, or 4. */ 11927 counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len); 11928 } else { 11929 /* 11930 * Log our probe state 3, and log also 5-13 to show 11931 * us the recovery sub-state for the send. This 11932 * means that 3 == (5+6+7+8+9+10+11+12+13) 11933 */ 11934 counter_u64_add(bbr_state_resend[BBR_STATE_PROBE_BW], len); 11935 counter_u64_add(bbr_state_resend[(bbr_state_val(bbr) + 5)], len); 11936 } 11937 /* Place in both 16's the totals of retransmitted */ 11938 counter_u64_add(bbr_state_lost[16], len); 11939 counter_u64_add(bbr_state_resend[16], len); 11940 /* Place in 17's the total sent */ 11941 counter_u64_add(bbr_state_resend[17], len); 11942 counter_u64_add(bbr_state_lost[17], len); 11943 11944 } else { 11945 /* New sends */ 11946 KMOD_TCPSTAT_INC(tcps_sndpack); 11947 KMOD_TCPSTAT_ADD(tcps_sndbyte, len); 11948 /* Place in 17's the total sent */ 11949 counter_u64_add(bbr_state_resend[17], len); 11950 counter_u64_add(bbr_state_lost[17], len); 11951 #ifdef STATS 11952 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB, 11953 len); 11954 #endif 11955 } 11956 } 11957 11958 static void 11959 bbr_cwnd_limiting(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t in_level) 11960 { 11961 if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) { 11962 /* 11963 * Limit the cwnd to not be above N x the target plus whats 11964 * is outstanding. The target is based on the current b/w 11965 * estimate. 11966 */ 11967 uint32_t target; 11968 11969 target = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), BBR_UNIT); 11970 target += ctf_outstanding(tp); 11971 target *= bbr_target_cwnd_mult_limit; 11972 if (tp->snd_cwnd > target) 11973 tp->snd_cwnd = target; 11974 bbr_log_type_cwndupd(bbr, 0, 0, 0, 10, 0, 0, __LINE__); 11975 } 11976 } 11977 11978 static int 11979 bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, int32_t maxseg) 11980 { 11981 /* 11982 * "adv" is the amount we could increase the window, taking into 11983 * account that we are limited by TCP_MAXWIN << tp->rcv_scale. 11984 */ 11985 uint32_t adv; 11986 int32_t oldwin; 11987 11988 adv = min(recwin, TCP_MAXWIN << tp->rcv_scale); 11989 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) { 11990 oldwin = (tp->rcv_adv - tp->rcv_nxt); 11991 adv -= oldwin; 11992 } else 11993 oldwin = 0; 11994 11995 /* 11996 * If the new window size ends up being the same as the old size 11997 * when it is scaled, then don't force a window update. 11998 */ 11999 if (oldwin >> tp->rcv_scale == (adv + oldwin) >> tp->rcv_scale) 12000 return (0); 12001 12002 if (adv >= (2 * maxseg) && 12003 (adv >= (so->so_rcv.sb_hiwat / 4) || 12004 recwin <= (so->so_rcv.sb_hiwat / 8) || 12005 so->so_rcv.sb_hiwat <= 8 * maxseg)) { 12006 return (1); 12007 } 12008 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat) 12009 return (1); 12010 return (0); 12011 } 12012 12013 /* 12014 * Return 0 on success and a errno on failure to send. 12015 * Note that a 0 return may not mean we sent anything 12016 * if the TCB was on the hpts. A non-zero return 12017 * does indicate the error we got from ip[6]_output. 12018 */ 12019 static int 12020 bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) 12021 { 12022 struct socket *so; 12023 int32_t len; 12024 uint32_t cts; 12025 uint32_t recwin, sendwin; 12026 int32_t sb_offset; 12027 int32_t flags, abandon, error = 0; 12028 struct tcp_log_buffer *lgb = NULL; 12029 struct mbuf *m; 12030 struct mbuf *mb; 12031 uint32_t if_hw_tsomaxsegcount = 0; 12032 uint32_t if_hw_tsomaxsegsize = 0; 12033 uint32_t if_hw_tsomax = 0; 12034 struct ip *ip = NULL; 12035 #ifdef TCPDEBUG 12036 struct ipovly *ipov = NULL; 12037 #endif 12038 struct tcp_bbr *bbr; 12039 struct tcphdr *th; 12040 #ifdef NETFLIX_TCPOUDP 12041 struct udphdr *udp = NULL; 12042 #endif 12043 u_char opt[TCP_MAXOLEN]; 12044 unsigned ipoptlen, optlen, hdrlen; 12045 #ifdef NETFLIX_TCPOUDP 12046 unsigned ulen; 12047 #endif 12048 uint32_t bbr_seq; 12049 uint32_t delay_calc=0; 12050 uint8_t doing_tlp = 0; 12051 uint8_t local_options; 12052 #ifdef BBR_INVARIANTS 12053 uint8_t doing_retran_from = 0; 12054 uint8_t picked_up_retran = 0; 12055 #endif 12056 uint8_t wanted_cookie = 0; 12057 uint8_t more_to_rxt=0; 12058 int32_t prefetch_so_done = 0; 12059 int32_t prefetch_rsm = 0; 12060 uint32_t what_we_can = 0; 12061 uint32_t tot_len = 0; 12062 uint32_t rtr_cnt = 0; 12063 uint32_t maxseg, pace_max_segs, p_maxseg; 12064 int32_t csum_flags; 12065 int32_t hw_tls; 12066 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 12067 unsigned ipsec_optlen = 0; 12068 12069 #endif 12070 volatile int32_t sack_rxmit; 12071 struct bbr_sendmap *rsm = NULL; 12072 int32_t tso, mtu; 12073 int force_tso = 0; 12074 struct tcpopt to; 12075 int32_t slot = 0; 12076 struct inpcb *inp; 12077 struct sockbuf *sb; 12078 uint32_t hpts_calling; 12079 #ifdef INET6 12080 struct ip6_hdr *ip6 = NULL; 12081 int32_t isipv6; 12082 #endif 12083 uint8_t app_limited = BBR_JR_SENT_DATA; 12084 uint8_t filled_all = 0; 12085 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 12086 /* We take a cache hit here */ 12087 memcpy(&bbr->rc_tv, tv, sizeof(struct timeval)); 12088 cts = tcp_tv_to_usectick(&bbr->rc_tv); 12089 inp = bbr->rc_inp; 12090 so = inp->inp_socket; 12091 sb = &so->so_snd; 12092 #ifdef KERN_TLS 12093 if (sb->sb_flags & SB_TLS_IFNET) 12094 hw_tls = 1; 12095 else 12096 #endif 12097 hw_tls = 0; 12098 kern_prefetch(sb, &maxseg); 12099 maxseg = tp->t_maxseg - bbr->rc_last_options; 12100 if (bbr_minseg(bbr) < maxseg) { 12101 tcp_bbr_tso_size_check(bbr, cts); 12102 } 12103 /* Remove any flags that indicate we are pacing on the inp */ 12104 pace_max_segs = bbr->r_ctl.rc_pace_max_segs; 12105 p_maxseg = min(maxseg, pace_max_segs); 12106 INP_WLOCK_ASSERT(inp); 12107 #ifdef TCP_OFFLOAD 12108 if (tp->t_flags & TF_TOE) 12109 return (tcp_offload_output(tp)); 12110 #endif 12111 12112 #ifdef INET6 12113 if (bbr->r_state) { 12114 /* Use the cache line loaded if possible */ 12115 isipv6 = bbr->r_is_v6; 12116 } else { 12117 isipv6 = (inp->inp_vflag & INP_IPV6) != 0; 12118 } 12119 #endif 12120 if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && 12121 inp->inp_in_hpts) { 12122 /* 12123 * We are on the hpts for some timer but not hptsi output. 12124 * Possibly remove from the hpts so we can send/recv etc. 12125 */ 12126 if ((tp->t_flags & TF_ACKNOW) == 0) { 12127 /* 12128 * No immediate demand right now to send an ack, but 12129 * the user may have read, making room for new data 12130 * (a window update). If so we may want to cancel 12131 * whatever timer is running (KEEP/DEL-ACK?) and 12132 * continue to send out a window update. Or we may 12133 * have gotten more data into the socket buffer to 12134 * send. 12135 */ 12136 recwin = min(max(sbspace(&so->so_rcv), 0), 12137 TCP_MAXWIN << tp->rcv_scale); 12138 if ((bbr_window_update_needed(tp, so, recwin, maxseg) == 0) && 12139 ((tcp_outflags[tp->t_state] & TH_RST) == 0) && 12140 ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <= 12141 (tp->snd_max - tp->snd_una))) { 12142 /* 12143 * Nothing new to send and no window update 12144 * is needed to send. Lets just return and 12145 * let the timer-run off. 12146 */ 12147 return (0); 12148 } 12149 } 12150 tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); 12151 bbr_timer_cancel(bbr, __LINE__, cts); 12152 } 12153 if (bbr->r_ctl.rc_last_delay_val) { 12154 /* Calculate a rough delay for early escape to sending */ 12155 if (SEQ_GT(cts, bbr->rc_pacer_started)) 12156 delay_calc = cts - bbr->rc_pacer_started; 12157 if (delay_calc >= bbr->r_ctl.rc_last_delay_val) 12158 delay_calc -= bbr->r_ctl.rc_last_delay_val; 12159 else 12160 delay_calc = 0; 12161 } 12162 /* Mark that we have called bbr_output(). */ 12163 if ((bbr->r_timer_override) || 12164 (tp->t_state < TCPS_ESTABLISHED)) { 12165 /* Timeouts or early states are exempt */ 12166 if (inp->inp_in_hpts) 12167 tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); 12168 } else if (inp->inp_in_hpts) { 12169 if ((bbr->r_ctl.rc_last_delay_val) && 12170 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && 12171 delay_calc) { 12172 /* 12173 * We were being paced for output and the delay has 12174 * already exceeded when we were supposed to be 12175 * called, lets go ahead and pull out of the hpts 12176 * and call output. 12177 */ 12178 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_LATE], 1); 12179 bbr->r_ctl.rc_last_delay_val = 0; 12180 tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); 12181 } else if (tp->t_state == TCPS_CLOSED) { 12182 bbr->r_ctl.rc_last_delay_val = 0; 12183 tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT); 12184 } else { 12185 /* 12186 * On the hpts, you shall not pass! even if ACKNOW 12187 * is on, we will when the hpts fires, unless of 12188 * course we are overdue. 12189 */ 12190 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_INPACE], 1); 12191 return (0); 12192 } 12193 } 12194 bbr->rc_cwnd_limited = 0; 12195 if (bbr->r_ctl.rc_last_delay_val) { 12196 /* recalculate the real delay and deal with over/under */ 12197 if (SEQ_GT(cts, bbr->rc_pacer_started)) 12198 delay_calc = cts - bbr->rc_pacer_started; 12199 else 12200 delay_calc = 0; 12201 if (delay_calc >= bbr->r_ctl.rc_last_delay_val) 12202 /* Setup the delay which will be added in */ 12203 delay_calc -= bbr->r_ctl.rc_last_delay_val; 12204 else { 12205 /* 12206 * We are early setup to adjust 12207 * our slot time. 12208 */ 12209 uint64_t merged_val; 12210 12211 bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc); 12212 bbr->r_agg_early_set = 1; 12213 if (bbr->r_ctl.rc_hptsi_agg_delay) { 12214 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) { 12215 /* Nope our previous late cancels out the early */ 12216 bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early; 12217 bbr->r_agg_early_set = 0; 12218 bbr->r_ctl.rc_agg_early = 0; 12219 } else { 12220 bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay; 12221 bbr->r_ctl.rc_hptsi_agg_delay = 0; 12222 } 12223 } 12224 merged_val = bbr->rc_pacer_started; 12225 merged_val <<= 32; 12226 merged_val |= bbr->r_ctl.rc_last_delay_val; 12227 bbr_log_pacing_delay_calc(bbr, inp->inp_hpts_calls, 12228 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val, 12229 bbr->r_agg_early_set, 3); 12230 bbr->r_ctl.rc_last_delay_val = 0; 12231 BBR_STAT_INC(bbr_early); 12232 delay_calc = 0; 12233 } 12234 } else { 12235 /* We were not delayed due to hptsi */ 12236 if (bbr->r_agg_early_set) 12237 bbr->r_ctl.rc_agg_early = 0; 12238 bbr->r_agg_early_set = 0; 12239 delay_calc = 0; 12240 } 12241 if (delay_calc) { 12242 /* 12243 * We had a hptsi delay which means we are falling behind on 12244 * sending at the expected rate. Calculate an extra amount 12245 * of data we can send, if any, to put us back on track. 12246 */ 12247 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay) 12248 bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff; 12249 else 12250 bbr->r_ctl.rc_hptsi_agg_delay += delay_calc; 12251 } 12252 sendwin = min(tp->snd_wnd, tp->snd_cwnd); 12253 if ((tp->snd_una == tp->snd_max) && 12254 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) && 12255 (sbavail(sb))) { 12256 /* 12257 * Ok we have been idle with nothing outstanding 12258 * we possibly need to start fresh with either a new 12259 * suite of states or a fast-ramp up. 12260 */ 12261 bbr_restart_after_idle(bbr, 12262 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time)); 12263 } 12264 /* 12265 * Now was there a hptsi delay where we are behind? We only count 12266 * being behind if: a) We are not in recovery. b) There was a delay. 12267 * <and> c) We had room to send something. 12268 * 12269 */ 12270 hpts_calling = inp->inp_hpts_calls; 12271 inp->inp_hpts_calls = 0; 12272 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { 12273 if (bbr_process_timers(tp, bbr, cts, hpts_calling)) { 12274 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_ATIMER], 1); 12275 return (0); 12276 } 12277 } 12278 bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY; 12279 if (hpts_calling && 12280 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { 12281 bbr->r_ctl.rc_last_delay_val = 0; 12282 } 12283 bbr->r_timer_override = 0; 12284 bbr->r_wanted_output = 0; 12285 /* 12286 * For TFO connections in SYN_RECEIVED, only allow the initial 12287 * SYN|ACK and those sent by the retransmit timer. 12288 */ 12289 if (IS_FASTOPEN(tp->t_flags) && 12290 ((tp->t_state == TCPS_SYN_RECEIVED) || 12291 (tp->t_state == TCPS_SYN_SENT)) && 12292 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */ 12293 (tp->t_rxtshift == 0)) { /* not a retransmit */ 12294 len = 0; 12295 goto just_return_nolock; 12296 } 12297 /* 12298 * Before sending anything check for a state update. For hpts 12299 * calling without input this is important. If its input calling 12300 * then this was already done. 12301 */ 12302 if (bbr->rc_use_google == 0) 12303 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0); 12304 again: 12305 /* 12306 * If we've recently taken a timeout, snd_max will be greater than 12307 * snd_max. BBR in general does not pay much attention to snd_nxt 12308 * for historic reasons the persist timer still uses it. This means 12309 * we have to look at it. All retransmissions that are not persits 12310 * use the rsm that needs to be sent so snd_nxt is ignored. At the 12311 * end of this routine we pull snd_nxt always up to snd_max. 12312 */ 12313 doing_tlp = 0; 12314 #ifdef BBR_INVARIANTS 12315 doing_retran_from = picked_up_retran = 0; 12316 #endif 12317 error = 0; 12318 tso = 0; 12319 slot = 0; 12320 mtu = 0; 12321 sendwin = min(tp->snd_wnd, tp->snd_cwnd); 12322 sb_offset = tp->snd_max - tp->snd_una; 12323 flags = tcp_outflags[tp->t_state]; 12324 sack_rxmit = 0; 12325 len = 0; 12326 rsm = NULL; 12327 if (flags & TH_RST) { 12328 SOCKBUF_LOCK(sb); 12329 goto send; 12330 } 12331 recheck_resend: 12332 while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) { 12333 /* We need to always have one in reserve */ 12334 rsm = bbr_alloc(bbr); 12335 if (rsm == NULL) { 12336 error = ENOMEM; 12337 /* Lie to get on the hpts */ 12338 tot_len = tp->t_maxseg; 12339 if (hpts_calling) 12340 /* Retry in a ms */ 12341 slot = 1001; 12342 goto just_return_nolock; 12343 } 12344 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next); 12345 bbr->r_ctl.rc_free_cnt++; 12346 rsm = NULL; 12347 } 12348 /* What do we send, a resend? */ 12349 if (bbr->r_ctl.rc_resend == NULL) { 12350 /* Check for rack timeout */ 12351 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts); 12352 if (bbr->r_ctl.rc_resend) { 12353 #ifdef BBR_INVARIANTS 12354 picked_up_retran = 1; 12355 #endif 12356 bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend); 12357 } 12358 } 12359 if (bbr->r_ctl.rc_resend) { 12360 rsm = bbr->r_ctl.rc_resend; 12361 #ifdef BBR_INVARIANTS 12362 doing_retran_from = 1; 12363 #endif 12364 /* Remove any TLP flags its a RACK or T-O */ 12365 rsm->r_flags &= ~BBR_TLP; 12366 bbr->r_ctl.rc_resend = NULL; 12367 if (SEQ_LT(rsm->r_start, tp->snd_una)) { 12368 #ifdef BBR_INVARIANTS 12369 panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n", 12370 tp, bbr, rsm, rsm->r_start, tp->snd_una); 12371 goto recheck_resend; 12372 #else 12373 /* TSNH */ 12374 rsm = NULL; 12375 goto recheck_resend; 12376 #endif 12377 } 12378 rtr_cnt++; 12379 if (rsm->r_flags & BBR_HAS_SYN) { 12380 /* Only retransmit a SYN by itself */ 12381 len = 0; 12382 if ((flags & TH_SYN) == 0) { 12383 /* Huh something is wrong */ 12384 rsm->r_start++; 12385 if (rsm->r_start == rsm->r_end) { 12386 /* Clean it up, somehow we missed the ack? */ 12387 bbr_log_syn(tp, NULL); 12388 } else { 12389 /* TFO with data? */ 12390 rsm->r_flags &= ~BBR_HAS_SYN; 12391 len = rsm->r_end - rsm->r_start; 12392 } 12393 } else { 12394 /* Retransmitting SYN */ 12395 rsm = NULL; 12396 SOCKBUF_LOCK(sb); 12397 goto send; 12398 } 12399 } else 12400 len = rsm->r_end - rsm->r_start; 12401 if ((bbr->rc_resends_use_tso == 0) && 12402 #ifdef KERN_TLS 12403 ((sb->sb_flags & SB_TLS_IFNET) == 0) && 12404 #endif 12405 (len > maxseg)) { 12406 len = maxseg; 12407 more_to_rxt = 1; 12408 } 12409 sb_offset = rsm->r_start - tp->snd_una; 12410 if (len > 0) { 12411 sack_rxmit = 1; 12412 KMOD_TCPSTAT_INC(tcps_sack_rexmits); 12413 KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes, 12414 min(len, maxseg)); 12415 } else { 12416 /* I dont think this can happen */ 12417 rsm = NULL; 12418 goto recheck_resend; 12419 } 12420 BBR_STAT_INC(bbr_resends_set); 12421 } else if (bbr->r_ctl.rc_tlp_send) { 12422 /* 12423 * Tail loss probe 12424 */ 12425 doing_tlp = 1; 12426 rsm = bbr->r_ctl.rc_tlp_send; 12427 bbr->r_ctl.rc_tlp_send = NULL; 12428 sack_rxmit = 1; 12429 len = rsm->r_end - rsm->r_start; 12430 rtr_cnt++; 12431 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg)) 12432 len = maxseg; 12433 12434 if (SEQ_GT(tp->snd_una, rsm->r_start)) { 12435 #ifdef BBR_INVARIANTS 12436 panic("tp:%p bbc:%p snd_una:%u rsm:%p r_start:%u", 12437 tp, bbr, tp->snd_una, rsm, rsm->r_start); 12438 #else 12439 /* TSNH */ 12440 rsm = NULL; 12441 goto recheck_resend; 12442 #endif 12443 } 12444 sb_offset = rsm->r_start - tp->snd_una; 12445 BBR_STAT_INC(bbr_tlp_set); 12446 } 12447 /* 12448 * Enforce a connection sendmap count limit if set 12449 * as long as we are not retransmiting. 12450 */ 12451 if ((rsm == NULL) && 12452 (V_tcp_map_entries_limit > 0) && 12453 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { 12454 BBR_STAT_INC(bbr_alloc_limited); 12455 if (!bbr->alloc_limit_reported) { 12456 bbr->alloc_limit_reported = 1; 12457 BBR_STAT_INC(bbr_alloc_limited_conns); 12458 } 12459 goto just_return_nolock; 12460 } 12461 #ifdef BBR_INVARIANTS 12462 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) { 12463 panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u", 12464 tp, bbr, rsm, sb_offset, len); 12465 } 12466 #endif 12467 /* 12468 * Get standard flags, and add SYN or FIN if requested by 'hidden' 12469 * state flags. 12470 */ 12471 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL)) 12472 flags |= TH_FIN; 12473 if (tp->t_flags & TF_NEEDSYN) 12474 flags |= TH_SYN; 12475 12476 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) { 12477 /* we are retransmitting the fin */ 12478 len--; 12479 if (len) { 12480 /* 12481 * When retransmitting data do *not* include the 12482 * FIN. This could happen from a TLP probe if we 12483 * allowed data with a FIN. 12484 */ 12485 flags &= ~TH_FIN; 12486 } 12487 } else if (rsm) { 12488 if (flags & TH_FIN) 12489 flags &= ~TH_FIN; 12490 } 12491 if ((sack_rxmit == 0) && (prefetch_rsm == 0)) { 12492 void *end_rsm; 12493 12494 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext); 12495 if (end_rsm) 12496 kern_prefetch(end_rsm, &prefetch_rsm); 12497 prefetch_rsm = 1; 12498 } 12499 SOCKBUF_LOCK(sb); 12500 /* 12501 * If snd_nxt == snd_max and we have transmitted a FIN, the 12502 * sb_offset will be > 0 even if so_snd.sb_cc is 0, resulting in a 12503 * negative length. This can also occur when TCP opens up its 12504 * congestion window while receiving additional duplicate acks after 12505 * fast-retransmit because TCP will reset snd_nxt to snd_max after 12506 * the fast-retransmit. 12507 * 12508 * In the normal retransmit-FIN-only case, however, snd_nxt will be 12509 * set to snd_una, the sb_offset will be 0, and the length may wind 12510 * up 0. 12511 * 12512 * If sack_rxmit is true we are retransmitting from the scoreboard 12513 * in which case len is already set. 12514 */ 12515 if (sack_rxmit == 0) { 12516 uint32_t avail; 12517 12518 avail = sbavail(sb); 12519 if (SEQ_GT(tp->snd_max, tp->snd_una)) 12520 sb_offset = tp->snd_max - tp->snd_una; 12521 else 12522 sb_offset = 0; 12523 if (bbr->rc_tlp_new_data) { 12524 /* TLP is forcing out new data */ 12525 uint32_t tlplen; 12526 12527 doing_tlp = 1; 12528 tlplen = maxseg; 12529 12530 if (tlplen > (uint32_t)(avail - sb_offset)) { 12531 tlplen = (uint32_t)(avail - sb_offset); 12532 } 12533 if (tlplen > tp->snd_wnd) { 12534 len = tp->snd_wnd; 12535 } else { 12536 len = tlplen; 12537 } 12538 bbr->rc_tlp_new_data = 0; 12539 } else { 12540 what_we_can = len = bbr_what_can_we_send(tp, bbr, sendwin, avail, sb_offset, cts); 12541 if ((len < p_maxseg) && 12542 (bbr->rc_in_persist == 0) && 12543 (ctf_outstanding(tp) >= (2 * p_maxseg)) && 12544 ((avail - sb_offset) >= p_maxseg)) { 12545 /* 12546 * We are not completing whats in the socket 12547 * buffer (i.e. there is at least a segment 12548 * waiting to send) and we have 2 or more 12549 * segments outstanding. There is no sense 12550 * of sending a little piece. Lets defer and 12551 * and wait until we can send a whole 12552 * segment. 12553 */ 12554 len = 0; 12555 } 12556 if (bbr->rc_in_persist) { 12557 /* 12558 * We are in persists, figure out if 12559 * a retransmit is available (maybe the previous 12560 * persists we sent) or if we have to send new 12561 * data. 12562 */ 12563 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 12564 if (rsm) { 12565 len = rsm->r_end - rsm->r_start; 12566 if (rsm->r_flags & BBR_HAS_FIN) 12567 len--; 12568 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg)) 12569 len = maxseg; 12570 if (len > 1) 12571 BBR_STAT_INC(bbr_persist_reneg); 12572 /* 12573 * XXXrrs we could force the len to 12574 * 1 byte here to cause the chunk to 12575 * split apart.. but that would then 12576 * mean we always retransmit it as 12577 * one byte even after the window 12578 * opens. 12579 */ 12580 sack_rxmit = 1; 12581 sb_offset = rsm->r_start - tp->snd_una; 12582 } else { 12583 /* 12584 * First time through in persists or peer 12585 * acked our one byte. Though we do have 12586 * to have something in the sb. 12587 */ 12588 len = 1; 12589 sb_offset = 0; 12590 if (avail == 0) 12591 len = 0; 12592 } 12593 } 12594 } 12595 } 12596 if (prefetch_so_done == 0) { 12597 kern_prefetch(so, &prefetch_so_done); 12598 prefetch_so_done = 1; 12599 } 12600 /* 12601 * Lop off SYN bit if it has already been sent. However, if this is 12602 * SYN-SENT state and if segment contains data and if we don't know 12603 * that foreign host supports TAO, suppress sending segment. 12604 */ 12605 if ((flags & TH_SYN) && (rsm == NULL) && 12606 SEQ_GT(tp->snd_max, tp->snd_una)) { 12607 if (tp->t_state != TCPS_SYN_RECEIVED) 12608 flags &= ~TH_SYN; 12609 /* 12610 * When sending additional segments following a TFO SYN|ACK, 12611 * do not include the SYN bit. 12612 */ 12613 if (IS_FASTOPEN(tp->t_flags) && 12614 (tp->t_state == TCPS_SYN_RECEIVED)) 12615 flags &= ~TH_SYN; 12616 sb_offset--, len++; 12617 if (sbavail(sb) == 0) 12618 len = 0; 12619 } else if ((flags & TH_SYN) && rsm) { 12620 /* 12621 * Subtract one from the len for the SYN being 12622 * retransmitted. 12623 */ 12624 len--; 12625 } 12626 /* 12627 * Be careful not to send data and/or FIN on SYN segments. This 12628 * measure is needed to prevent interoperability problems with not 12629 * fully conformant TCP implementations. 12630 */ 12631 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) { 12632 len = 0; 12633 flags &= ~TH_FIN; 12634 } 12635 /* 12636 * On TFO sockets, ensure no data is sent in the following cases: 12637 * 12638 * - When retransmitting SYN|ACK on a passively-created socket 12639 * - When retransmitting SYN on an actively created socket 12640 * - When sending a zero-length cookie (cookie request) on an 12641 * actively created socket 12642 * - When the socket is in the CLOSED state (RST is being sent) 12643 */ 12644 if (IS_FASTOPEN(tp->t_flags) && 12645 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) || 12646 ((tp->t_state == TCPS_SYN_SENT) && 12647 (tp->t_tfo_client_cookie_len == 0)) || 12648 (flags & TH_RST))) { 12649 len = 0; 12650 sack_rxmit = 0; 12651 rsm = NULL; 12652 } 12653 /* Without fast-open there should never be data sent on a SYN */ 12654 if ((flags & TH_SYN) && (!IS_FASTOPEN(tp->t_flags))) 12655 len = 0; 12656 if (len <= 0) { 12657 /* 12658 * If FIN has been sent but not acked, but we haven't been 12659 * called to retransmit, len will be < 0. Otherwise, window 12660 * shrank after we sent into it. If window shrank to 0, 12661 * cancel pending retransmit, pull snd_nxt back to (closed) 12662 * window, and set the persist timer if it isn't already 12663 * going. If the window didn't close completely, just wait 12664 * for an ACK. 12665 * 12666 * We also do a general check here to ensure that we will 12667 * set the persist timer when we have data to send, but a 12668 * 0-byte window. This makes sure the persist timer is set 12669 * even if the packet hits one of the "goto send" lines 12670 * below. 12671 */ 12672 len = 0; 12673 if ((tp->snd_wnd == 0) && 12674 (TCPS_HAVEESTABLISHED(tp->t_state)) && 12675 (tp->snd_una == tp->snd_max) && 12676 (sb_offset < (int)sbavail(sb))) { 12677 /* 12678 * Not enough room in the rwnd to send 12679 * a paced segment out. 12680 */ 12681 bbr_enter_persist(tp, bbr, cts, __LINE__); 12682 } 12683 } else if ((rsm == NULL) && 12684 (doing_tlp == 0) && 12685 (len < bbr->r_ctl.rc_pace_max_segs)) { 12686 /* 12687 * We are not sending a full segment for 12688 * some reason. Should we not send anything (think 12689 * sws or persists)? 12690 */ 12691 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && 12692 (TCPS_HAVEESTABLISHED(tp->t_state)) && 12693 (len < (int)(sbavail(sb) - sb_offset))) { 12694 /* 12695 * Here the rwnd is less than 12696 * the pacing size, this is not a retransmit, 12697 * we are established and 12698 * the send is not the last in the socket buffer 12699 * lets not send, and possibly enter persists. 12700 */ 12701 len = 0; 12702 if (tp->snd_max == tp->snd_una) 12703 bbr_enter_persist(tp, bbr, cts, __LINE__); 12704 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) && 12705 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 12706 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) && 12707 (len < (int)(sbavail(sb) - sb_offset)) && 12708 (len < bbr_minseg(bbr))) { 12709 /* 12710 * Here we are not retransmitting, and 12711 * the cwnd is not so small that we could 12712 * not send at least a min size (rxt timer 12713 * not having gone off), We have 2 segments or 12714 * more already in flight, its not the tail end 12715 * of the socket buffer and the cwnd is blocking 12716 * us from sending out minimum pacing segment size. 12717 * Lets not send anything. 12718 */ 12719 bbr->rc_cwnd_limited = 1; 12720 len = 0; 12721 } else if (((tp->snd_wnd - ctf_outstanding(tp)) < 12722 min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) && 12723 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 12724 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) && 12725 (len < (int)(sbavail(sb) - sb_offset)) && 12726 (TCPS_HAVEESTABLISHED(tp->t_state))) { 12727 /* 12728 * Here we have a send window but we have 12729 * filled it up and we can't send another pacing segment. 12730 * We also have in flight more than 2 segments 12731 * and we are not completing the sb i.e. we allow 12732 * the last bytes of the sb to go out even if 12733 * its not a full pacing segment. 12734 */ 12735 len = 0; 12736 } 12737 } 12738 /* len will be >= 0 after this point. */ 12739 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__)); 12740 tcp_sndbuf_autoscale(tp, so, sendwin); 12741 /* 12742 * 12743 */ 12744 if (bbr->rc_in_persist && 12745 len && 12746 (rsm == NULL) && 12747 (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) { 12748 /* 12749 * We are in persist, not doing a retransmit and don't have enough space 12750 * yet to send a full TSO. So is it at the end of the sb 12751 * if so we need to send else nuke to 0 and don't send. 12752 */ 12753 int sbleft; 12754 if (sbavail(sb) > sb_offset) 12755 sbleft = sbavail(sb) - sb_offset; 12756 else 12757 sbleft = 0; 12758 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) { 12759 /* not at end of sb lets not send */ 12760 len = 0; 12761 } 12762 } 12763 /* 12764 * Decide if we can use TCP Segmentation Offloading (if supported by 12765 * hardware). 12766 * 12767 * TSO may only be used if we are in a pure bulk sending state. The 12768 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP 12769 * options prevent using TSO. With TSO the TCP header is the same 12770 * (except for the sequence number) for all generated packets. This 12771 * makes it impossible to transmit any options which vary per 12772 * generated segment or packet. 12773 * 12774 * IPv4 handling has a clear separation of ip options and ip header 12775 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen() 12776 * does the right thing below to provide length of just ip options 12777 * and thus checking for ipoptlen is enough to decide if ip options 12778 * are present. 12779 */ 12780 #ifdef INET6 12781 if (isipv6) 12782 ipoptlen = ip6_optlen(inp); 12783 else 12784 #endif 12785 if (inp->inp_options) 12786 ipoptlen = inp->inp_options->m_len - 12787 offsetof(struct ipoption, ipopt_list); 12788 else 12789 ipoptlen = 0; 12790 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 12791 /* 12792 * Pre-calculate here as we save another lookup into the darknesses 12793 * of IPsec that way and can actually decide if TSO is ok. 12794 */ 12795 #ifdef INET6 12796 if (isipv6 && IPSEC_ENABLED(ipv6)) 12797 ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp); 12798 #ifdef INET 12799 else 12800 #endif 12801 #endif /* INET6 */ 12802 #ifdef INET 12803 if (IPSEC_ENABLED(ipv4)) 12804 ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp); 12805 #endif /* INET */ 12806 #endif /* IPSEC */ 12807 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 12808 ipoptlen += ipsec_optlen; 12809 #endif 12810 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && 12811 (len > maxseg) && 12812 (tp->t_port == 0) && 12813 ((tp->t_flags & TF_SIGNATURE) == 0) && 12814 tp->rcv_numsacks == 0 && 12815 ipoptlen == 0) 12816 tso = 1; 12817 12818 recwin = min(max(sbspace(&so->so_rcv), 0), 12819 TCP_MAXWIN << tp->rcv_scale); 12820 /* 12821 * Sender silly window avoidance. We transmit under the following 12822 * conditions when len is non-zero: 12823 * 12824 * - We have a full segment (or more with TSO) - This is the last 12825 * buffer in a write()/send() and we are either idle or running 12826 * NODELAY - we've timed out (e.g. persist timer) - we have more 12827 * then 1/2 the maximum send window's worth of data (receiver may be 12828 * limited the window size) - we need to retransmit 12829 */ 12830 if (rsm) 12831 goto send; 12832 if (len) { 12833 if (sack_rxmit) 12834 goto send; 12835 if (len >= p_maxseg) 12836 goto send; 12837 /* 12838 * NOTE! on localhost connections an 'ack' from the remote 12839 * end may occur synchronously with the output and cause us 12840 * to flush a buffer queued with moretocome. XXX 12841 * 12842 */ 12843 if (((tp->t_flags & TF_MORETOCOME) == 0) && /* normal case */ 12844 ((tp->t_flags & TF_NODELAY) || 12845 ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) && 12846 (tp->t_flags & TF_NOPUSH) == 0) { 12847 goto send; 12848 } 12849 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */ 12850 goto send; 12851 } 12852 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) { 12853 goto send; 12854 } 12855 } 12856 /* 12857 * Sending of standalone window updates. 12858 * 12859 * Window updates are important when we close our window due to a 12860 * full socket buffer and are opening it again after the application 12861 * reads data from it. Once the window has opened again and the 12862 * remote end starts to send again the ACK clock takes over and 12863 * provides the most current window information. 12864 * 12865 * We must avoid the silly window syndrome whereas every read from 12866 * the receive buffer, no matter how small, causes a window update 12867 * to be sent. We also should avoid sending a flurry of window 12868 * updates when the socket buffer had queued a lot of data and the 12869 * application is doing small reads. 12870 * 12871 * Prevent a flurry of pointless window updates by only sending an 12872 * update when we can increase the advertized window by more than 12873 * 1/4th of the socket buffer capacity. When the buffer is getting 12874 * full or is very small be more aggressive and send an update 12875 * whenever we can increase by two mss sized segments. In all other 12876 * situations the ACK's to new incoming data will carry further 12877 * window increases. 12878 * 12879 * Don't send an independent window update if a delayed ACK is 12880 * pending (it will get piggy-backed on it) or the remote side 12881 * already has done a half-close and won't send more data. Skip 12882 * this if the connection is in T/TCP half-open state. 12883 */ 12884 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) && 12885 !(tp->t_flags & TF_DELACK) && 12886 !TCPS_HAVERCVDFIN(tp->t_state)) { 12887 /* Check to see if we should do a window update */ 12888 if (bbr_window_update_needed(tp, so, recwin, maxseg)) 12889 goto send; 12890 } 12891 /* 12892 * Send if we owe the peer an ACK, RST, SYN. ACKNOW 12893 * is also a catch-all for the retransmit timer timeout case. 12894 */ 12895 if (tp->t_flags & TF_ACKNOW) { 12896 goto send; 12897 } 12898 if (flags & TH_RST) { 12899 /* Always send a RST if one is due */ 12900 goto send; 12901 } 12902 if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) { 12903 goto send; 12904 } 12905 /* 12906 * If our state indicates that FIN should be sent and we have not 12907 * yet done so, then we need to send. 12908 */ 12909 if (flags & TH_FIN && 12910 ((tp->t_flags & TF_SENTFIN) == 0)) { 12911 goto send; 12912 } 12913 /* 12914 * No reason to send a segment, just return. 12915 */ 12916 just_return: 12917 SOCKBUF_UNLOCK(sb); 12918 just_return_nolock: 12919 if (tot_len) 12920 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); 12921 if (bbr->rc_no_pacing) 12922 slot = 0; 12923 if (tot_len == 0) { 12924 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >= 12925 tp->snd_wnd) { 12926 BBR_STAT_INC(bbr_rwnd_limited); 12927 app_limited = BBR_JR_RWND_LIMITED; 12928 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp)); 12929 if ((bbr->rc_in_persist == 0) && 12930 TCPS_HAVEESTABLISHED(tp->t_state) && 12931 (tp->snd_max == tp->snd_una) && 12932 sbavail(&tp->t_inpcb->inp_socket->so_snd)) { 12933 /* No send window.. we must enter persist */ 12934 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__); 12935 } 12936 } else if (ctf_outstanding(tp) >= sbavail(sb)) { 12937 BBR_STAT_INC(bbr_app_limited); 12938 app_limited = BBR_JR_APP_LIMITED; 12939 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp)); 12940 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 12941 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) { 12942 BBR_STAT_INC(bbr_cwnd_limited); 12943 app_limited = BBR_JR_CWND_LIMITED; 12944 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 12945 bbr->r_ctl.rc_lost_bytes))); 12946 bbr->rc_cwnd_limited = 1; 12947 } else { 12948 BBR_STAT_INC(bbr_app_limited); 12949 app_limited = BBR_JR_APP_LIMITED; 12950 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp)); 12951 } 12952 bbr->r_ctl.rc_hptsi_agg_delay = 0; 12953 bbr->r_agg_early_set = 0; 12954 bbr->r_ctl.rc_agg_early = 0; 12955 bbr->r_ctl.rc_last_delay_val = 0; 12956 } else if (bbr->rc_use_google == 0) 12957 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0); 12958 /* Are we app limited? */ 12959 if ((app_limited == BBR_JR_APP_LIMITED) || 12960 (app_limited == BBR_JR_RWND_LIMITED)) { 12961 /** 12962 * We are application limited. 12963 */ 12964 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 12965 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered); 12966 } 12967 if (tot_len == 0) 12968 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_JUSTRET], 1); 12969 /* Dont update the time if we did not send */ 12970 bbr->r_ctl.rc_last_delay_val = 0; 12971 bbr->rc_output_starts_timer = 1; 12972 bbr_start_hpts_timer(bbr, tp, cts, 9, slot, tot_len); 12973 bbr_log_type_just_return(bbr, cts, tot_len, hpts_calling, app_limited, p_maxseg, len); 12974 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { 12975 /* Make sure snd_nxt is drug up */ 12976 tp->snd_nxt = tp->snd_max; 12977 } 12978 return (error); 12979 12980 send: 12981 if (doing_tlp == 0) { 12982 /* 12983 * Data not a TLP, and its not the rxt firing. If it is the 12984 * rxt firing, we want to leave the tlp_in_progress flag on 12985 * so we don't send another TLP. It has to be a rack timer 12986 * or normal send (response to acked data) to clear the tlp 12987 * in progress flag. 12988 */ 12989 bbr->rc_tlp_in_progress = 0; 12990 bbr->rc_tlp_rtx_out = 0; 12991 } else { 12992 /* 12993 * Its a TLP. 12994 */ 12995 bbr->rc_tlp_in_progress = 1; 12996 } 12997 bbr_timer_cancel(bbr, __LINE__, cts); 12998 if (rsm == NULL) { 12999 if (sbused(sb) > 0) { 13000 /* 13001 * This is sub-optimal. We only send a stand alone 13002 * FIN on its own segment. 13003 */ 13004 if (flags & TH_FIN) { 13005 flags &= ~TH_FIN; 13006 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) { 13007 /* Lets not send this */ 13008 slot = 0; 13009 goto just_return; 13010 } 13011 } 13012 } 13013 } else { 13014 /* 13015 * We do *not* send a FIN on a retransmit if it has data. 13016 * The if clause here where len > 1 should never come true. 13017 */ 13018 if ((len > 0) && 13019 (((rsm->r_flags & BBR_HAS_FIN) == 0) && 13020 (flags & TH_FIN))) { 13021 flags &= ~TH_FIN; 13022 len--; 13023 } 13024 } 13025 SOCKBUF_LOCK_ASSERT(sb); 13026 if (len > 0) { 13027 if ((tp->snd_una == tp->snd_max) && 13028 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) { 13029 /* 13030 * This qualifies as a RTT_PROBE session since we 13031 * drop the data outstanding to nothing and waited 13032 * more than bbr_rtt_probe_time. 13033 */ 13034 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0); 13035 bbr_set_reduced_rtt(bbr, cts, __LINE__); 13036 } 13037 if (len >= maxseg) 13038 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT; 13039 else 13040 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT; 13041 } 13042 /* 13043 * Before ESTABLISHED, force sending of initial options unless TCP 13044 * set not to do any options. NOTE: we assume that the IP/TCP header 13045 * plus TCP options always fit in a single mbuf, leaving room for a 13046 * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr) 13047 * + optlen <= MCLBYTES 13048 */ 13049 optlen = 0; 13050 #ifdef INET6 13051 if (isipv6) 13052 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 13053 else 13054 #endif 13055 hdrlen = sizeof(struct tcpiphdr); 13056 13057 /* 13058 * Compute options for segment. We only have to care about SYN and 13059 * established connection segments. Options for SYN-ACK segments 13060 * are handled in TCP syncache. 13061 */ 13062 to.to_flags = 0; 13063 local_options = 0; 13064 if ((tp->t_flags & TF_NOOPT) == 0) { 13065 /* Maximum segment size. */ 13066 if (flags & TH_SYN) { 13067 to.to_mss = tcp_mssopt(&inp->inp_inc); 13068 #ifdef NETFLIX_TCPOUDP 13069 if (tp->t_port) 13070 to.to_mss -= V_tcp_udp_tunneling_overhead; 13071 #endif 13072 to.to_flags |= TOF_MSS; 13073 /* 13074 * On SYN or SYN|ACK transmits on TFO connections, 13075 * only include the TFO option if it is not a 13076 * retransmit, as the presence of the TFO option may 13077 * have caused the original SYN or SYN|ACK to have 13078 * been dropped by a middlebox. 13079 */ 13080 if (IS_FASTOPEN(tp->t_flags) && 13081 (tp->t_rxtshift == 0)) { 13082 if (tp->t_state == TCPS_SYN_RECEIVED) { 13083 to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN; 13084 to.to_tfo_cookie = 13085 (u_int8_t *)&tp->t_tfo_cookie.server; 13086 to.to_flags |= TOF_FASTOPEN; 13087 wanted_cookie = 1; 13088 } else if (tp->t_state == TCPS_SYN_SENT) { 13089 to.to_tfo_len = 13090 tp->t_tfo_client_cookie_len; 13091 to.to_tfo_cookie = 13092 tp->t_tfo_cookie.client; 13093 to.to_flags |= TOF_FASTOPEN; 13094 wanted_cookie = 1; 13095 } 13096 } 13097 } 13098 /* Window scaling. */ 13099 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) { 13100 to.to_wscale = tp->request_r_scale; 13101 to.to_flags |= TOF_SCALE; 13102 } 13103 /* Timestamps. */ 13104 if ((tp->t_flags & TF_RCVD_TSTMP) || 13105 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) { 13106 to.to_tsval = tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset; 13107 to.to_tsecr = tp->ts_recent; 13108 to.to_flags |= TOF_TS; 13109 local_options += TCPOLEN_TIMESTAMP + 2; 13110 } 13111 /* Set receive buffer autosizing timestamp. */ 13112 if (tp->rfbuf_ts == 0 && 13113 (so->so_rcv.sb_flags & SB_AUTOSIZE)) 13114 tp->rfbuf_ts = tcp_tv_to_mssectick(&bbr->rc_tv); 13115 /* Selective ACK's. */ 13116 if (flags & TH_SYN) 13117 to.to_flags |= TOF_SACKPERM; 13118 else if (TCPS_HAVEESTABLISHED(tp->t_state) && 13119 tp->rcv_numsacks > 0) { 13120 to.to_flags |= TOF_SACK; 13121 to.to_nsacks = tp->rcv_numsacks; 13122 to.to_sacks = (u_char *)tp->sackblks; 13123 } 13124 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) 13125 /* TCP-MD5 (RFC2385). */ 13126 if (tp->t_flags & TF_SIGNATURE) 13127 to.to_flags |= TOF_SIGNATURE; 13128 #endif /* TCP_SIGNATURE */ 13129 13130 /* Processing the options. */ 13131 hdrlen += (optlen = tcp_addoptions(&to, opt)); 13132 /* 13133 * If we wanted a TFO option to be added, but it was unable 13134 * to fit, ensure no data is sent. 13135 */ 13136 if (IS_FASTOPEN(tp->t_flags) && wanted_cookie && 13137 !(to.to_flags & TOF_FASTOPEN)) 13138 len = 0; 13139 } 13140 #ifdef NETFLIX_TCPOUDP 13141 if (tp->t_port) { 13142 if (V_tcp_udp_tunneling_port == 0) { 13143 /* The port was removed?? */ 13144 SOCKBUF_UNLOCK(&so->so_snd); 13145 return (EHOSTUNREACH); 13146 } 13147 hdrlen += sizeof(struct udphdr); 13148 } 13149 #endif 13150 #ifdef INET6 13151 if (isipv6) 13152 ipoptlen = ip6_optlen(tp->t_inpcb); 13153 else 13154 #endif 13155 if (tp->t_inpcb->inp_options) 13156 ipoptlen = tp->t_inpcb->inp_options->m_len - 13157 offsetof(struct ipoption, ipopt_list); 13158 else 13159 ipoptlen = 0; 13160 ipoptlen = 0; 13161 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 13162 ipoptlen += ipsec_optlen; 13163 #endif 13164 if (bbr->rc_last_options != local_options) { 13165 /* 13166 * Cache the options length this generally does not change 13167 * on a connection. We use this to calculate TSO. 13168 */ 13169 bbr->rc_last_options = local_options; 13170 } 13171 maxseg = tp->t_maxseg - (ipoptlen + optlen); 13172 p_maxseg = min(maxseg, pace_max_segs); 13173 /* 13174 * Adjust data length if insertion of options will bump the packet 13175 * length beyond the t_maxseg length. Clear the FIN bit because we 13176 * cut off the tail of the segment. 13177 */ 13178 #ifdef KERN_TLS 13179 /* force TSO for so TLS offload can get mss */ 13180 if (sb->sb_flags & SB_TLS_IFNET) { 13181 force_tso = 1; 13182 } 13183 #endif 13184 13185 if (len > maxseg) { 13186 if (len != 0 && (flags & TH_FIN)) { 13187 flags &= ~TH_FIN; 13188 } 13189 if (tso) { 13190 uint32_t moff; 13191 int32_t max_len; 13192 13193 /* extract TSO information */ 13194 if_hw_tsomax = tp->t_tsomax; 13195 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; 13196 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; 13197 KASSERT(ipoptlen == 0, 13198 ("%s: TSO can't do IP options", __func__)); 13199 13200 /* 13201 * Check if we should limit by maximum payload 13202 * length: 13203 */ 13204 if (if_hw_tsomax != 0) { 13205 /* compute maximum TSO length */ 13206 max_len = (if_hw_tsomax - hdrlen - 13207 max_linkhdr); 13208 if (max_len <= 0) { 13209 len = 0; 13210 } else if (len > max_len) { 13211 len = max_len; 13212 } 13213 } 13214 /* 13215 * Prevent the last segment from being fractional 13216 * unless the send sockbuf can be emptied: 13217 */ 13218 if (((sb_offset + len) < sbavail(sb)) && 13219 (hw_tls == 0)) { 13220 moff = len % (uint32_t)maxseg; 13221 if (moff != 0) { 13222 len -= moff; 13223 } 13224 } 13225 /* 13226 * In case there are too many small fragments don't 13227 * use TSO: 13228 */ 13229 if (len <= maxseg) { 13230 len = maxseg; 13231 tso = 0; 13232 } 13233 } else { 13234 /* Not doing TSO */ 13235 if (optlen + ipoptlen >= tp->t_maxseg) { 13236 /* 13237 * Since we don't have enough space to put 13238 * the IP header chain and the TCP header in 13239 * one packet as required by RFC 7112, don't 13240 * send it. Also ensure that at least one 13241 * byte of the payload can be put into the 13242 * TCP segment. 13243 */ 13244 SOCKBUF_UNLOCK(&so->so_snd); 13245 error = EMSGSIZE; 13246 sack_rxmit = 0; 13247 goto out; 13248 } 13249 len = maxseg; 13250 } 13251 } else { 13252 /* Not doing TSO */ 13253 if_hw_tsomaxsegcount = 0; 13254 tso = 0; 13255 } 13256 KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET, 13257 ("%s: len > IP_MAXPACKET", __func__)); 13258 #ifdef DIAGNOSTIC 13259 #ifdef INET6 13260 if (max_linkhdr + hdrlen > MCLBYTES) 13261 #else 13262 if (max_linkhdr + hdrlen > MHLEN) 13263 #endif 13264 panic("tcphdr too big"); 13265 #endif 13266 /* 13267 * This KASSERT is here to catch edge cases at a well defined place. 13268 * Before, those had triggered (random) panic conditions further 13269 * down. 13270 */ 13271 #ifdef BBR_INVARIANTS 13272 if (sack_rxmit) { 13273 if (SEQ_LT(rsm->r_start, tp->snd_una)) { 13274 panic("RSM:%p TP:%p bbr:%p start:%u is < snd_una:%u", 13275 rsm, tp, bbr, rsm->r_start, tp->snd_una); 13276 } 13277 } 13278 #endif 13279 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__)); 13280 if ((len == 0) && 13281 (flags & TH_FIN) && 13282 (sbused(sb))) { 13283 /* 13284 * We have outstanding data, don't send a fin by itself!. 13285 */ 13286 slot = 0; 13287 goto just_return; 13288 } 13289 /* 13290 * Grab a header mbuf, attaching a copy of data to be transmitted, 13291 * and initialize the header from the template for sends on this 13292 * connection. 13293 */ 13294 if (len) { 13295 uint32_t moff; 13296 uint32_t orig_len; 13297 13298 /* 13299 * We place a limit on sending with hptsi. 13300 */ 13301 if ((rsm == NULL) && len > pace_max_segs) 13302 len = pace_max_segs; 13303 if (len <= maxseg) 13304 tso = 0; 13305 #ifdef INET6 13306 if (MHLEN < hdrlen + max_linkhdr) 13307 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 13308 else 13309 #endif 13310 m = m_gethdr(M_NOWAIT, MT_DATA); 13311 13312 if (m == NULL) { 13313 BBR_STAT_INC(bbr_failed_mbuf_aloc); 13314 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0); 13315 SOCKBUF_UNLOCK(sb); 13316 error = ENOBUFS; 13317 sack_rxmit = 0; 13318 goto out; 13319 } 13320 m->m_data += max_linkhdr; 13321 m->m_len = hdrlen; 13322 /* 13323 * Start the m_copy functions from the closest mbuf to the 13324 * sb_offset in the socket buffer chain. 13325 */ 13326 if ((sb_offset > sbavail(sb)) || ((len + sb_offset) > sbavail(sb))) { 13327 #ifdef BBR_INVARIANTS 13328 if ((len + sb_offset) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0))) 13329 panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u", 13330 tp, bbr, len, sb_offset, sbavail(sb), rsm, 13331 doing_retran_from, 13332 picked_up_retran, 13333 doing_tlp); 13334 13335 #endif 13336 /* 13337 * In this messed up situation we have two choices, 13338 * a) pretend the send worked, and just start timers 13339 * and what not (not good since that may lead us 13340 * back here a lot). <or> b) Send the lowest segment 13341 * in the map. <or> c) Drop the connection. Lets do 13342 * <b> which if it continues to happen will lead to 13343 * <c> via timeouts. 13344 */ 13345 BBR_STAT_INC(bbr_offset_recovery); 13346 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map); 13347 sb_offset = 0; 13348 if (rsm == NULL) { 13349 sack_rxmit = 0; 13350 len = sbavail(sb); 13351 } else { 13352 sack_rxmit = 1; 13353 if (rsm->r_start != tp->snd_una) { 13354 /* 13355 * Things are really messed up, <c> 13356 * is the only thing to do. 13357 */ 13358 BBR_STAT_INC(bbr_offset_drop); 13359 tcp_set_inp_to_drop(inp, EFAULT); 13360 return (0); 13361 } 13362 len = rsm->r_end - rsm->r_start; 13363 } 13364 if (len > sbavail(sb)) 13365 len = sbavail(sb); 13366 if (len > maxseg) 13367 len = maxseg; 13368 } 13369 mb = sbsndptr_noadv(sb, sb_offset, &moff); 13370 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) { 13371 m_copydata(mb, moff, (int)len, 13372 mtod(m, caddr_t)+hdrlen); 13373 if (rsm == NULL) 13374 sbsndptr_adv(sb, mb, len); 13375 m->m_len += len; 13376 } else { 13377 struct sockbuf *msb; 13378 13379 if (rsm) 13380 msb = NULL; 13381 else 13382 msb = sb; 13383 #ifdef BBR_INVARIANTS 13384 if ((len + moff) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0))) { 13385 if (rsm) { 13386 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d ", 13387 tp, bbr, len, moff, 13388 sbavail(sb), rsm, 13389 tp->snd_una, rsm->r_flags, rsm->r_start, 13390 doing_retran_from, 13391 picked_up_retran, 13392 doing_tlp, sack_rxmit); 13393 } else { 13394 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u sb_offset:%u snd_una:%u", 13395 tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una); 13396 } 13397 } 13398 #endif 13399 orig_len = len; 13400 m->m_next = tcp_m_copym( 13401 mb, moff, &len, 13402 if_hw_tsomaxsegcount, 13403 if_hw_tsomaxsegsize, msb, 13404 ((rsm == NULL) ? hw_tls : 0) 13405 #ifdef NETFLIX_COPY_ARGS 13406 , &filled_all 13407 #endif 13408 ); 13409 if (len <= maxseg && !force_tso) { 13410 /* 13411 * Must have ran out of mbufs for the copy 13412 * shorten it to no longer need tso. Lets 13413 * not put on sendalot since we are low on 13414 * mbufs. 13415 */ 13416 tso = 0; 13417 } 13418 if (m->m_next == NULL) { 13419 SOCKBUF_UNLOCK(sb); 13420 (void)m_free(m); 13421 error = ENOBUFS; 13422 sack_rxmit = 0; 13423 goto out; 13424 } 13425 } 13426 #ifdef BBR_INVARIANTS 13427 if (tso && len < maxseg) { 13428 panic("tp:%p tso on, but len:%d < maxseg:%d", 13429 tp, len, maxseg); 13430 } 13431 if (tso && if_hw_tsomaxsegcount) { 13432 int32_t seg_cnt = 0; 13433 struct mbuf *foo; 13434 13435 foo = m; 13436 while (foo) { 13437 seg_cnt++; 13438 foo = foo->m_next; 13439 } 13440 if (seg_cnt > if_hw_tsomaxsegcount) { 13441 panic("seg_cnt:%d > max:%d", seg_cnt, if_hw_tsomaxsegcount); 13442 } 13443 } 13444 #endif 13445 /* 13446 * If we're sending everything we've got, set PUSH. (This 13447 * will keep happy those implementations which only give 13448 * data to the user when a buffer fills or a PUSH comes in.) 13449 */ 13450 if (sb_offset + len == sbused(sb) && 13451 sbused(sb) && 13452 !(flags & TH_SYN)) { 13453 flags |= TH_PUSH; 13454 } 13455 SOCKBUF_UNLOCK(sb); 13456 } else { 13457 SOCKBUF_UNLOCK(sb); 13458 if (tp->t_flags & TF_ACKNOW) 13459 KMOD_TCPSTAT_INC(tcps_sndacks); 13460 else if (flags & (TH_SYN | TH_FIN | TH_RST)) 13461 KMOD_TCPSTAT_INC(tcps_sndctrl); 13462 else 13463 KMOD_TCPSTAT_INC(tcps_sndwinup); 13464 13465 m = m_gethdr(M_NOWAIT, MT_DATA); 13466 if (m == NULL) { 13467 BBR_STAT_INC(bbr_failed_mbuf_aloc); 13468 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0); 13469 error = ENOBUFS; 13470 /* Fudge the send time since we could not send */ 13471 sack_rxmit = 0; 13472 goto out; 13473 } 13474 #ifdef INET6 13475 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) && 13476 MHLEN >= hdrlen) { 13477 M_ALIGN(m, hdrlen); 13478 } else 13479 #endif 13480 m->m_data += max_linkhdr; 13481 m->m_len = hdrlen; 13482 } 13483 SOCKBUF_UNLOCK_ASSERT(sb); 13484 m->m_pkthdr.rcvif = (struct ifnet *)0; 13485 #ifdef MAC 13486 mac_inpcb_create_mbuf(inp, m); 13487 #endif 13488 #ifdef INET6 13489 if (isipv6) { 13490 ip6 = mtod(m, struct ip6_hdr *); 13491 #ifdef NETFLIX_TCPOUDP 13492 if (tp->t_port) { 13493 udp = (struct udphdr *)((caddr_t)ip6 + ipoptlen + sizeof(struct ip6_hdr)); 13494 udp->uh_sport = htons(V_tcp_udp_tunneling_port); 13495 udp->uh_dport = tp->t_port; 13496 ulen = hdrlen + len - sizeof(struct ip6_hdr); 13497 udp->uh_ulen = htons(ulen); 13498 th = (struct tcphdr *)(udp + 1); 13499 } else { 13500 #endif 13501 th = (struct tcphdr *)(ip6 + 1); 13502 13503 #ifdef NETFLIX_TCPOUDP 13504 } 13505 #endif 13506 tcpip_fillheaders(inp, 13507 #ifdef NETFLIX_TCPOUDP 13508 tp->t_port, 13509 #endif 13510 ip6, th); 13511 } else 13512 #endif /* INET6 */ 13513 { 13514 ip = mtod(m, struct ip *); 13515 #ifdef TCPDEBUG 13516 ipov = (struct ipovly *)ip; 13517 #endif 13518 #ifdef NETFLIX_TCPOUDP 13519 if (tp->t_port) { 13520 udp = (struct udphdr *)((caddr_t)ip + ipoptlen + sizeof(struct ip)); 13521 udp->uh_sport = htons(V_tcp_udp_tunneling_port); 13522 udp->uh_dport = tp->t_port; 13523 ulen = hdrlen + len - sizeof(struct ip); 13524 udp->uh_ulen = htons(ulen); 13525 th = (struct tcphdr *)(udp + 1); 13526 } else 13527 #endif 13528 th = (struct tcphdr *)(ip + 1); 13529 tcpip_fillheaders(inp, 13530 #ifdef NETFLIX_TCPOUDP 13531 tp->t_port, 13532 #endif 13533 ip, th); 13534 } 13535 /* 13536 * If we are doing retransmissions, then snd_nxt will not reflect 13537 * the first unsent octet. For ACK only packets, we do not want the 13538 * sequence number of the retransmitted packet, we want the sequence 13539 * number of the next unsent octet. So, if there is no data (and no 13540 * SYN or FIN), use snd_max instead of snd_nxt when filling in 13541 * ti_seq. But if we are in persist state, snd_max might reflect 13542 * one byte beyond the right edge of the window, so use snd_nxt in 13543 * that case, since we know we aren't doing a retransmission. 13544 * (retransmit and persist are mutually exclusive...) 13545 */ 13546 if (sack_rxmit == 0) { 13547 if (len && ((flags & (TH_FIN | TH_SYN | TH_RST)) == 0)) { 13548 /* New data (including new persists) */ 13549 th->th_seq = htonl(tp->snd_max); 13550 bbr_seq = tp->snd_max; 13551 } else if (flags & TH_SYN) { 13552 /* Syn's always send from iss */ 13553 th->th_seq = htonl(tp->iss); 13554 bbr_seq = tp->iss; 13555 } else if (flags & TH_FIN) { 13556 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) { 13557 /* 13558 * If we sent the fin already its 1 minus 13559 * snd_max 13560 */ 13561 th->th_seq = (htonl(tp->snd_max - 1)); 13562 bbr_seq = (tp->snd_max - 1); 13563 } else { 13564 /* First time FIN use snd_max */ 13565 th->th_seq = htonl(tp->snd_max); 13566 bbr_seq = tp->snd_max; 13567 } 13568 } else if (flags & TH_RST) { 13569 /* 13570 * For a Reset send the last cum ack in sequence 13571 * (this like any other choice may still generate a 13572 * challenge ack, if a ack-update packet is in 13573 * flight). 13574 */ 13575 th->th_seq = htonl(tp->snd_una); 13576 bbr_seq = tp->snd_una; 13577 } else { 13578 /* 13579 * len == 0 and not persist we use snd_max, sending 13580 * an ack unless we have sent the fin then its 1 13581 * minus. 13582 */ 13583 /* 13584 * XXXRRS Question if we are in persists and we have 13585 * nothing outstanding to send and we have not sent 13586 * a FIN, we will send an ACK. In such a case it 13587 * might be better to send (tp->snd_una - 1) which 13588 * would force the peer to ack. 13589 */ 13590 if (tp->t_flags & TF_SENTFIN) { 13591 th->th_seq = htonl(tp->snd_max - 1); 13592 bbr_seq = (tp->snd_max - 1); 13593 } else { 13594 th->th_seq = htonl(tp->snd_max); 13595 bbr_seq = tp->snd_max; 13596 } 13597 } 13598 } else { 13599 /* All retransmits use the rsm to guide the send */ 13600 th->th_seq = htonl(rsm->r_start); 13601 bbr_seq = rsm->r_start; 13602 } 13603 th->th_ack = htonl(tp->rcv_nxt); 13604 if (optlen) { 13605 bcopy(opt, th + 1, optlen); 13606 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; 13607 } 13608 th->th_flags = flags; 13609 /* 13610 * Calculate receive window. Don't shrink window, but avoid silly 13611 * window syndrome. 13612 */ 13613 if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) && 13614 recwin < maxseg))) 13615 recwin = 0; 13616 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && 13617 recwin < (tp->rcv_adv - tp->rcv_nxt)) 13618 recwin = (tp->rcv_adv - tp->rcv_nxt); 13619 if (recwin > TCP_MAXWIN << tp->rcv_scale) 13620 recwin = TCP_MAXWIN << tp->rcv_scale; 13621 13622 /* 13623 * According to RFC1323 the window field in a SYN (i.e., a <SYN> or 13624 * <SYN,ACK>) segment itself is never scaled. The <SYN,ACK> case is 13625 * handled in syncache. 13626 */ 13627 if (flags & TH_SYN) 13628 th->th_win = htons((u_short) 13629 (min(sbspace(&so->so_rcv), TCP_MAXWIN))); 13630 else { 13631 /* Avoid shrinking window with window scaling. */ 13632 recwin = roundup2(recwin, 1 << tp->rcv_scale); 13633 th->th_win = htons((u_short)(recwin >> tp->rcv_scale)); 13634 } 13635 /* 13636 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0 13637 * window. This may cause the remote transmitter to stall. This 13638 * flag tells soreceive() to disable delayed acknowledgements when 13639 * draining the buffer. This can occur if the receiver is 13640 * attempting to read more data than can be buffered prior to 13641 * transmitting on the connection. 13642 */ 13643 if (th->th_win == 0) { 13644 tp->t_sndzerowin++; 13645 tp->t_flags |= TF_RXWIN0SENT; 13646 } else 13647 tp->t_flags &= ~TF_RXWIN0SENT; 13648 /* 13649 * We don't support urgent data, but drag along 13650 * the pointer in case of a stack switch. 13651 */ 13652 tp->snd_up = tp->snd_una; 13653 13654 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) 13655 if (to.to_flags & TOF_SIGNATURE) { 13656 /* 13657 * Calculate MD5 signature and put it into the place 13658 * determined before. NOTE: since TCP options buffer doesn't 13659 * point into mbuf's data, calculate offset and use it. 13660 */ 13661 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th, 13662 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) { 13663 /* 13664 * Do not send segment if the calculation of MD5 13665 * digest has failed. 13666 */ 13667 goto out; 13668 } 13669 } 13670 #endif 13671 13672 /* 13673 * Put TCP length in extended header, and then checksum extended 13674 * header and data. 13675 */ 13676 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ 13677 #ifdef INET6 13678 if (isipv6) { 13679 /* 13680 * ip6_plen is not need to be filled now, and will be filled 13681 * in ip6_output. 13682 */ 13683 #ifdef NETFLIX_TCPOUDP 13684 if (tp->t_port) { 13685 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; 13686 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 13687 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); 13688 th->th_sum = htons(0); 13689 UDPSTAT_INC(udps_opackets); 13690 } else { 13691 #endif 13692 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; 13693 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 13694 th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) + 13695 optlen + len, IPPROTO_TCP, 0); 13696 #ifdef NETFLIX_TCPOUDP 13697 } 13698 #endif 13699 } 13700 #endif 13701 #if defined(INET6) && defined(INET) 13702 else 13703 #endif 13704 #ifdef INET 13705 { 13706 #ifdef NETFLIX_TCPOUDP 13707 if (tp->t_port) { 13708 m->m_pkthdr.csum_flags = CSUM_UDP; 13709 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 13710 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, 13711 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); 13712 th->th_sum = htons(0); 13713 UDPSTAT_INC(udps_opackets); 13714 } else { 13715 #endif 13716 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP; 13717 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 13718 th->th_sum = in_pseudo(ip->ip_src.s_addr, 13719 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + 13720 IPPROTO_TCP + len + optlen)); 13721 #ifdef NETFLIX_TCPOUDP 13722 } 13723 #endif 13724 /* IP version must be set here for ipv4/ipv6 checking later */ 13725 KASSERT(ip->ip_v == IPVERSION, 13726 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); 13727 } 13728 #endif 13729 13730 /* 13731 * Enable TSO and specify the size of the segments. The TCP pseudo 13732 * header checksum is always provided. XXX: Fixme: This is currently 13733 * not the case for IPv6. 13734 */ 13735 if (tso || force_tso) { 13736 KASSERT(force_tso || len > maxseg, 13737 ("%s: len:%d <= tso_segsz:%d", __func__, len, maxseg)); 13738 m->m_pkthdr.csum_flags |= CSUM_TSO; 13739 csum_flags |= CSUM_TSO; 13740 m->m_pkthdr.tso_segsz = maxseg; 13741 } 13742 KASSERT(len + hdrlen == m_length(m, NULL), 13743 ("%s: mbuf chain different than expected: %d + %u != %u", 13744 __func__, len, hdrlen, m_length(m, NULL))); 13745 13746 #ifdef TCP_HHOOK 13747 /* Run HHOOK_TC_ESTABLISHED_OUT helper hooks. */ 13748 hhook_run_tcp_est_out(tp, th, &to, len, tso); 13749 #endif 13750 #ifdef TCPDEBUG 13751 /* 13752 * Trace. 13753 */ 13754 if (so->so_options & SO_DEBUG) { 13755 u_short save = 0; 13756 13757 #ifdef INET6 13758 if (!isipv6) 13759 #endif 13760 { 13761 save = ipov->ih_len; 13762 ipov->ih_len = htons(m->m_pkthdr.len /* - hdrlen + 13763 * (th->th_off << 2) */ ); 13764 } 13765 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0); 13766 #ifdef INET6 13767 if (!isipv6) 13768 #endif 13769 ipov->ih_len = save; 13770 } 13771 #endif /* TCPDEBUG */ 13772 13773 /* Log to the black box */ 13774 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 13775 union tcp_log_stackspecific log; 13776 13777 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts); 13778 /* Record info on type of transmission */ 13779 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay; 13780 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3); 13781 log.u_bbr.flex3 = maxseg; 13782 log.u_bbr.flex4 = delay_calc; 13783 /* Encode filled_all into the upper flex5 bit */ 13784 log.u_bbr.flex5 = bbr->rc_past_init_win; 13785 log.u_bbr.flex5 <<= 1; 13786 log.u_bbr.flex5 |= bbr->rc_no_pacing; 13787 log.u_bbr.flex5 <<= 29; 13788 if (filled_all) 13789 log.u_bbr.flex5 |= 0x80000000; 13790 log.u_bbr.flex5 |= tp->t_maxseg; 13791 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs; 13792 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr); 13793 /* lets poke in the low and the high here for debugging */ 13794 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg; 13795 if (rsm || sack_rxmit) { 13796 if (doing_tlp) 13797 log.u_bbr.flex8 = 2; 13798 else 13799 log.u_bbr.flex8 = 1; 13800 } else { 13801 log.u_bbr.flex8 = 0; 13802 } 13803 lgb = tcp_log_event_(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, 13804 len, &log, false, NULL, NULL, 0, tv); 13805 } else { 13806 lgb = NULL; 13807 } 13808 /* 13809 * Fill in IP length and desired time to live and send to IP level. 13810 * There should be a better way to handle ttl and tos; we could keep 13811 * them in the template, but need a way to checksum without them. 13812 */ 13813 /* 13814 * m->m_pkthdr.len should have been set before cksum calcuration, 13815 * because in6_cksum() need it. 13816 */ 13817 #ifdef INET6 13818 if (isipv6) { 13819 /* 13820 * we separately set hoplimit for every segment, since the 13821 * user might want to change the value via setsockopt. Also, 13822 * desired default hop limit might be changed via Neighbor 13823 * Discovery. 13824 */ 13825 ip6->ip6_hlim = in6_selecthlim(inp, NULL); 13826 13827 /* 13828 * Set the packet size here for the benefit of DTrace 13829 * probes. ip6_output() will set it properly; it's supposed 13830 * to include the option header lengths as well. 13831 */ 13832 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); 13833 13834 if (V_path_mtu_discovery && maxseg > V_tcp_minmss) 13835 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 13836 else 13837 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 13838 13839 if (tp->t_state == TCPS_SYN_SENT) 13840 TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th); 13841 13842 TCP_PROBE5(send, NULL, tp, ip6, tp, th); 13843 /* TODO: IPv6 IP6TOS_ECT bit on */ 13844 error = ip6_output(m, inp->in6p_outputopts, 13845 &inp->inp_route6, 13846 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 13847 NULL, NULL, inp); 13848 13849 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL) 13850 mtu = inp->inp_route6.ro_nh->nh_mtu; 13851 } 13852 #endif /* INET6 */ 13853 #if defined(INET) && defined(INET6) 13854 else 13855 #endif 13856 #ifdef INET 13857 { 13858 ip->ip_len = htons(m->m_pkthdr.len); 13859 #ifdef INET6 13860 if (isipv6) 13861 ip->ip_ttl = in6_selecthlim(inp, NULL); 13862 #endif /* INET6 */ 13863 /* 13864 * If we do path MTU discovery, then we set DF on every 13865 * packet. This might not be the best thing to do according 13866 * to RFC3390 Section 2. However the tcp hostcache migitates 13867 * the problem so it affects only the first tcp connection 13868 * with a host. 13869 * 13870 * NB: Don't set DF on small MTU/MSS to have a safe 13871 * fallback. 13872 */ 13873 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { 13874 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 13875 if (tp->t_port == 0 || len < V_tcp_minmss) { 13876 ip->ip_off |= htons(IP_DF); 13877 } 13878 } else { 13879 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 13880 } 13881 13882 if (tp->t_state == TCPS_SYN_SENT) 13883 TCP_PROBE5(connect__request, NULL, tp, ip, tp, th); 13884 13885 TCP_PROBE5(send, NULL, tp, ip, tp, th); 13886 13887 error = ip_output(m, inp->inp_options, &inp->inp_route, 13888 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0, 13889 inp); 13890 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL) 13891 mtu = inp->inp_route.ro_nh->nh_mtu; 13892 } 13893 #endif /* INET */ 13894 out: 13895 13896 if (lgb) { 13897 lgb->tlb_errno = error; 13898 lgb = NULL; 13899 } 13900 /* 13901 * In transmit state, time the transmission and arrange for the 13902 * retransmit. In persist state, just set snd_max. 13903 */ 13904 if (error == 0) { 13905 if (TCPS_HAVEESTABLISHED(tp->t_state) && 13906 (tp->t_flags & TF_SACK_PERMIT) && 13907 tp->rcv_numsacks > 0) 13908 tcp_clean_dsack_blocks(tp); 13909 /* We sent an ack clear the bbr_segs_rcvd count */ 13910 bbr->output_error_seen = 0; 13911 bbr->oerror_cnt = 0; 13912 bbr->bbr_segs_rcvd = 0; 13913 if (len == 0) 13914 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_SNDACK], 1); 13915 else if (hw_tls) { 13916 if (filled_all || 13917 (len >= bbr->r_ctl.rc_pace_max_segs)) 13918 BBR_STAT_INC(bbr_meets_tso_thresh); 13919 else { 13920 if (doing_tlp) { 13921 BBR_STAT_INC(bbr_miss_tlp); 13922 bbr_log_type_hrdwtso(tp, bbr, len, 1, what_we_can); 13923 13924 13925 } else if (rsm) { 13926 BBR_STAT_INC(bbr_miss_retran); 13927 bbr_log_type_hrdwtso(tp, bbr, len, 2, what_we_can); 13928 } else if ((ctf_outstanding(tp) + bbr->r_ctl.rc_pace_max_segs) > sbavail(sb)) { 13929 BBR_STAT_INC(bbr_miss_tso_app); 13930 bbr_log_type_hrdwtso(tp, bbr, len, 3, what_we_can); 13931 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 13932 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_pace_max_segs) > tp->snd_cwnd) { 13933 BBR_STAT_INC(bbr_miss_tso_cwnd); 13934 bbr_log_type_hrdwtso(tp, bbr, len, 4, what_we_can); 13935 } else if ((ctf_outstanding(tp) + bbr->r_ctl.rc_pace_max_segs) > tp->snd_wnd) { 13936 BBR_STAT_INC(bbr_miss_tso_rwnd); 13937 bbr_log_type_hrdwtso(tp, bbr, len, 5, what_we_can); 13938 } else { 13939 BBR_STAT_INC(bbr_miss_unknown); 13940 bbr_log_type_hrdwtso(tp, bbr, len, 6, what_we_can); 13941 } 13942 } 13943 } 13944 /* Do accounting for new sends */ 13945 if ((len > 0) && (rsm == NULL)) { 13946 int idx; 13947 if (tp->snd_una == tp->snd_max) { 13948 /* 13949 * Special case to match google, when 13950 * nothing is in flight the delivered 13951 * time does get updated to the current 13952 * time (see tcp_rate_bsd.c). 13953 */ 13954 bbr->r_ctl.rc_del_time = cts; 13955 } 13956 if (len >= maxseg) { 13957 idx = (len / maxseg) + 3; 13958 if (idx >= TCP_MSS_ACCT_ATIMER) 13959 counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1); 13960 else 13961 counter_u64_add(bbr_out_size[idx], 1); 13962 } else { 13963 /* smaller than a MSS */ 13964 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options); 13965 if (idx >= TCP_MSS_SMALL_MAX_SIZE_DIV) 13966 idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1); 13967 counter_u64_add(bbr_out_size[(idx + TCP_MSS_SMALL_SIZE_OFF)], 1); 13968 } 13969 } 13970 } 13971 abandon = 0; 13972 /* 13973 * We must do the send accounting before we log the output, 13974 * otherwise the state of the rsm could change and we account to the 13975 * wrong bucket. 13976 */ 13977 if (len > 0) { 13978 bbr_do_send_accounting(tp, bbr, rsm, len, error); 13979 if (error == 0) { 13980 if (tp->snd_una == tp->snd_max) 13981 bbr->r_ctl.rc_tlp_rxt_last_time = cts; 13982 } 13983 } 13984 bbr_log_output(bbr, tp, &to, len, bbr_seq, (uint8_t) flags, error, 13985 cts, mb, &abandon, rsm, 0, sb); 13986 if (abandon) { 13987 /* 13988 * If bbr_log_output destroys the TCB or sees a TH_RST being 13989 * sent we should hit this condition. 13990 */ 13991 return (0); 13992 } 13993 if (bbr->rc_in_persist == 0) { 13994 /* 13995 * Advance snd_nxt over sequence space of this segment. 13996 */ 13997 if (error) 13998 /* We don't log or do anything with errors */ 13999 goto skip_upd; 14000 14001 if (tp->snd_una == tp->snd_max && 14002 (len || (flags & (TH_SYN | TH_FIN)))) { 14003 /* 14004 * Update the time we just added data since none was 14005 * outstanding. 14006 */ 14007 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__); 14008 bbr->rc_tp->t_acktime = ticks; 14009 } 14010 if (flags & (TH_SYN | TH_FIN) && (rsm == NULL)) { 14011 if (flags & TH_SYN) { 14012 /* 14013 * Smack the snd_max to iss + 1 14014 * if its a FO we will add len below. 14015 */ 14016 tp->snd_max = tp->iss + 1; 14017 } 14018 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) { 14019 tp->snd_max++; 14020 tp->t_flags |= TF_SENTFIN; 14021 } 14022 } 14023 if (sack_rxmit == 0) 14024 tp->snd_max += len; 14025 skip_upd: 14026 if ((error == 0) && len) 14027 tot_len += len; 14028 } else { 14029 /* Persists case */ 14030 int32_t xlen = len; 14031 14032 if (error) 14033 goto nomore; 14034 14035 if (flags & TH_SYN) 14036 ++xlen; 14037 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) { 14038 ++xlen; 14039 tp->t_flags |= TF_SENTFIN; 14040 } 14041 if (xlen && (tp->snd_una == tp->snd_max)) { 14042 /* 14043 * Update the time we just added data since none was 14044 * outstanding. 14045 */ 14046 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__); 14047 bbr->rc_tp->t_acktime = ticks; 14048 } 14049 if (sack_rxmit == 0) 14050 tp->snd_max += xlen; 14051 tot_len += (len + optlen + ipoptlen); 14052 } 14053 nomore: 14054 if (error) { 14055 /* 14056 * Failures do not advance the seq counter above. For the 14057 * case of ENOBUFS we will fall out and become ack-clocked. 14058 * capping the cwnd at the current flight. 14059 * Everything else will just have to retransmit with the timer 14060 * (no pacer). 14061 */ 14062 SOCKBUF_UNLOCK_ASSERT(sb); 14063 BBR_STAT_INC(bbr_saw_oerr); 14064 /* Clear all delay/early tracks */ 14065 bbr->r_ctl.rc_hptsi_agg_delay = 0; 14066 bbr->r_ctl.rc_agg_early = 0; 14067 bbr->r_agg_early_set = 0; 14068 bbr->output_error_seen = 1; 14069 if (bbr->oerror_cnt < 0xf) 14070 bbr->oerror_cnt++; 14071 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) { 14072 /* drop the session */ 14073 tcp_set_inp_to_drop(inp, ENETDOWN); 14074 } 14075 switch (error) { 14076 case ENOBUFS: 14077 /* 14078 * Make this guy have to get ack's to send 14079 * more but lets make sure we don't 14080 * slam him below a T-O (1MSS). 14081 */ 14082 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) { 14083 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 14084 bbr->r_ctl.rc_lost_bytes)) - maxseg; 14085 if (tp->snd_cwnd < maxseg) 14086 tp->snd_cwnd = maxseg; 14087 } 14088 slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt; 14089 BBR_STAT_INC(bbr_saw_enobuf); 14090 if (bbr->bbr_hdrw_pacing) 14091 counter_u64_add(bbr_hdwr_pacing_enobuf, 1); 14092 else 14093 counter_u64_add(bbr_nohdwr_pacing_enobuf, 1); 14094 /* 14095 * Here even in the enobuf's case we want to do our 14096 * state update. The reason being we may have been 14097 * called by the input function. If so we have had 14098 * things change. 14099 */ 14100 error = 0; 14101 goto enobufs; 14102 case EMSGSIZE: 14103 /* 14104 * For some reason the interface we used initially 14105 * to send segments changed to another or lowered 14106 * its MTU. If TSO was active we either got an 14107 * interface without TSO capabilits or TSO was 14108 * turned off. If we obtained mtu from ip_output() 14109 * then update it and try again. 14110 */ 14111 /* Turn on tracing (or try to) */ 14112 { 14113 int old_maxseg; 14114 14115 old_maxseg = tp->t_maxseg; 14116 BBR_STAT_INC(bbr_saw_emsgsiz); 14117 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, csum_flags, tso, cts); 14118 if (mtu != 0) 14119 tcp_mss_update(tp, -1, mtu, NULL, NULL); 14120 if (old_maxseg <= tp->t_maxseg) { 14121 /* Huh it did not shrink? */ 14122 tp->t_maxseg = old_maxseg - 40; 14123 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, 0, tso, cts); 14124 } 14125 /* 14126 * Nuke all other things that can interfere 14127 * with slot 14128 */ 14129 if ((tot_len + len) && (len >= tp->t_maxseg)) { 14130 slot = bbr_get_pacing_delay(bbr, 14131 bbr->r_ctl.rc_bbr_hptsi_gain, 14132 (tot_len + len), cts, 0); 14133 if (slot < bbr_error_base_paceout) 14134 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt; 14135 } else 14136 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt; 14137 bbr->rc_output_starts_timer = 1; 14138 bbr_start_hpts_timer(bbr, tp, cts, 10, slot, 14139 tot_len); 14140 return (error); 14141 } 14142 case EPERM: 14143 tp->t_softerror = error; 14144 /* Fall through */ 14145 case EHOSTDOWN: 14146 case EHOSTUNREACH: 14147 case ENETDOWN: 14148 case ENETUNREACH: 14149 if (TCPS_HAVERCVDSYN(tp->t_state)) { 14150 tp->t_softerror = error; 14151 } 14152 /* FALLTHROUGH */ 14153 default: 14154 slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt; 14155 bbr->rc_output_starts_timer = 1; 14156 bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0); 14157 return (error); 14158 } 14159 #ifdef STATS 14160 } else if (((tp->t_flags & TF_GPUTINPROG) == 0) && 14161 len && 14162 (rsm == NULL) && 14163 (bbr->rc_in_persist == 0)) { 14164 tp->gput_seq = bbr_seq; 14165 tp->gput_ack = bbr_seq + 14166 min(sbavail(&so->so_snd) - sb_offset, sendwin); 14167 tp->gput_ts = cts; 14168 tp->t_flags |= TF_GPUTINPROG; 14169 #endif 14170 } 14171 KMOD_TCPSTAT_INC(tcps_sndtotal); 14172 if ((bbr->bbr_hdw_pace_ena) && 14173 (bbr->bbr_attempt_hdwr_pace == 0) && 14174 (bbr->rc_past_init_win) && 14175 (bbr->rc_bbr_state != BBR_STATE_STARTUP) && 14176 (get_filter_value(&bbr->r_ctl.rc_delrate)) && 14177 (inp->inp_route.ro_nh && 14178 inp->inp_route.ro_nh->nh_ifp)) { 14179 /* 14180 * We are past the initial window and 14181 * have at least one measurement so we 14182 * could use hardware pacing if its available. 14183 * We have an interface and we have not attempted 14184 * to setup hardware pacing, lets try to now. 14185 */ 14186 uint64_t rate_wanted; 14187 int err = 0; 14188 14189 rate_wanted = bbr_get_hardware_rate(bbr); 14190 bbr->bbr_attempt_hdwr_pace = 1; 14191 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp, 14192 inp->inp_route.ro_nh->nh_ifp, 14193 rate_wanted, 14194 (RS_PACING_GEQ|RS_PACING_SUB_OK), 14195 &err); 14196 if (bbr->r_ctl.crte) { 14197 bbr_type_log_hdwr_pacing(bbr, 14198 bbr->r_ctl.crte->ptbl->rs_ifp, 14199 rate_wanted, 14200 bbr->r_ctl.crte->rate, 14201 __LINE__, cts, err); 14202 BBR_STAT_INC(bbr_hdwr_rl_add_ok); 14203 counter_u64_add(bbr_flows_nohdwr_pacing, -1); 14204 counter_u64_add(bbr_flows_whdwr_pacing, 1); 14205 bbr->bbr_hdrw_pacing = 1; 14206 /* Now what is our gain status? */ 14207 if (bbr->r_ctl.crte->rate < rate_wanted) { 14208 /* We have a problem */ 14209 bbr_setup_less_of_rate(bbr, cts, 14210 bbr->r_ctl.crte->rate, rate_wanted); 14211 } else { 14212 /* We are good */ 14213 bbr->gain_is_limited = 0; 14214 bbr->skip_gain = 0; 14215 } 14216 tcp_bbr_tso_size_check(bbr, cts); 14217 } else { 14218 bbr_type_log_hdwr_pacing(bbr, 14219 inp->inp_route.ro_nh->nh_ifp, 14220 rate_wanted, 14221 0, 14222 __LINE__, cts, err); 14223 BBR_STAT_INC(bbr_hdwr_rl_add_fail); 14224 } 14225 } 14226 if (bbr->bbr_hdrw_pacing) { 14227 /* 14228 * Worry about cases where the route 14229 * changes or something happened that we 14230 * lost our hardware pacing possibly during 14231 * the last ip_output call. 14232 */ 14233 if (inp->inp_snd_tag == NULL) { 14234 /* A change during ip output disabled hw pacing? */ 14235 bbr->bbr_hdrw_pacing = 0; 14236 } else if ((inp->inp_route.ro_nh == NULL) || 14237 (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) { 14238 /* 14239 * We had an interface or route change, 14240 * detach from the current hdwr pacing 14241 * and setup to re-attempt next go 14242 * round. 14243 */ 14244 bbr->bbr_hdrw_pacing = 0; 14245 bbr->bbr_attempt_hdwr_pace = 0; 14246 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp); 14247 tcp_bbr_tso_size_check(bbr, cts); 14248 } 14249 } 14250 /* 14251 * Data sent (as far as we can tell). If this advertises a larger 14252 * window than any other segment, then remember the size of the 14253 * advertised window. Any pending ACK has now been sent. 14254 */ 14255 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) 14256 tp->rcv_adv = tp->rcv_nxt + recwin; 14257 14258 tp->last_ack_sent = tp->rcv_nxt; 14259 if ((error == 0) && 14260 (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) && 14261 (doing_tlp == 0) && 14262 (tso == 0) && 14263 (hw_tls == 0) && 14264 (len > 0) && 14265 ((flags & TH_RST) == 0) && 14266 ((flags & TH_SYN) == 0) && 14267 (IN_RECOVERY(tp->t_flags) == 0) && 14268 (bbr->rc_in_persist == 0) && 14269 (tot_len < bbr->r_ctl.rc_pace_max_segs)) { 14270 /* 14271 * For non-tso we need to goto again until we have sent out 14272 * enough data to match what we are hptsi out every hptsi 14273 * interval. 14274 */ 14275 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { 14276 /* Make sure snd_nxt is drug up */ 14277 tp->snd_nxt = tp->snd_max; 14278 } 14279 if (rsm != NULL) { 14280 rsm = NULL; 14281 goto skip_again; 14282 } 14283 rsm = NULL; 14284 sack_rxmit = 0; 14285 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); 14286 goto again; 14287 } 14288 skip_again: 14289 if ((error == 0) && (flags & TH_FIN)) 14290 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_FIN); 14291 if ((error == 0) && (flags & TH_RST)) 14292 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 14293 if (((flags & (TH_RST | TH_SYN | TH_FIN)) == 0) && tot_len) { 14294 /* 14295 * Calculate/Re-Calculate the hptsi slot in usecs based on 14296 * what we have sent so far 14297 */ 14298 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0); 14299 if (bbr->rc_no_pacing) 14300 slot = 0; 14301 } 14302 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); 14303 enobufs: 14304 if (bbr->rc_use_google == 0) 14305 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0); 14306 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + 14307 bbr->r_ctl.rc_lost_bytes))); 14308 bbr->rc_output_starts_timer = 1; 14309 if (bbr->bbr_use_rack_cheat && 14310 (more_to_rxt || 14311 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) { 14312 /* Rack cheats and shotguns out all rxt's 1ms apart */ 14313 if (slot > 1000) 14314 slot = 1000; 14315 } 14316 if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) { 14317 /* 14318 * We don't change the tso size until some number of sends 14319 * to give the hardware commands time to get down 14320 * to the interface. 14321 */ 14322 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++; 14323 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) { 14324 bbr->hw_pacing_set = 1; 14325 tcp_bbr_tso_size_check(bbr, cts); 14326 } 14327 } 14328 bbr_start_hpts_timer(bbr, tp, cts, 12, slot, tot_len); 14329 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { 14330 /* Make sure snd_nxt is drug up */ 14331 tp->snd_nxt = tp->snd_max; 14332 } 14333 return (error); 14334 14335 } 14336 14337 /* 14338 * See bbr_output_wtime() for return values. 14339 */ 14340 static int 14341 bbr_output(struct tcpcb *tp) 14342 { 14343 int32_t ret; 14344 struct timeval tv; 14345 struct tcp_bbr *bbr; 14346 14347 NET_EPOCH_ASSERT(); 14348 14349 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 14350 INP_WLOCK_ASSERT(tp->t_inpcb); 14351 (void)tcp_get_usecs(&tv); 14352 ret = bbr_output_wtime(tp, &tv); 14353 return (ret); 14354 } 14355 14356 static void 14357 bbr_mtu_chg(struct tcpcb *tp) 14358 { 14359 struct tcp_bbr *bbr; 14360 struct bbr_sendmap *rsm, *frsm = NULL; 14361 uint32_t maxseg; 14362 14363 /* 14364 * The MTU has changed. a) Clear the sack filter. b) Mark everything 14365 * over the current size as SACK_PASS so a retransmit will occur. 14366 */ 14367 14368 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 14369 maxseg = tp->t_maxseg - bbr->rc_last_options; 14370 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una); 14371 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) { 14372 /* Don't mess with ones acked (by sack?) */ 14373 if (rsm->r_flags & BBR_ACKED) 14374 continue; 14375 if ((rsm->r_end - rsm->r_start) > maxseg) { 14376 /* 14377 * We mark sack-passed on all the previous large 14378 * sends we did. This will force them to retransmit. 14379 */ 14380 rsm->r_flags |= BBR_SACK_PASSED; 14381 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) && 14382 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) { 14383 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start; 14384 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start; 14385 rsm->r_flags |= BBR_MARKED_LOST; 14386 } 14387 if (frsm == NULL) 14388 frsm = rsm; 14389 } 14390 } 14391 if (frsm) { 14392 bbr->r_ctl.rc_resend = frsm; 14393 } 14394 } 14395 14396 /* 14397 * bbr_ctloutput() must drop the inpcb lock before performing copyin on 14398 * socket option arguments. When it re-acquires the lock after the copy, it 14399 * has to revalidate that the connection is still valid for the socket 14400 * option. 14401 */ 14402 static int 14403 bbr_set_sockopt(struct socket *so, struct sockopt *sopt, 14404 struct inpcb *inp, struct tcpcb *tp, struct tcp_bbr *bbr) 14405 { 14406 struct epoch_tracker et; 14407 int32_t error = 0, optval; 14408 14409 switch (sopt->sopt_name) { 14410 case TCP_RACK_PACE_MAX_SEG: 14411 case TCP_RACK_MIN_TO: 14412 case TCP_RACK_REORD_THRESH: 14413 case TCP_RACK_REORD_FADE: 14414 case TCP_RACK_TLP_THRESH: 14415 case TCP_RACK_PKT_DELAY: 14416 case TCP_BBR_ALGORITHM: 14417 case TCP_BBR_TSLIMITS: 14418 case TCP_BBR_IWINTSO: 14419 case TCP_BBR_RECFORCE: 14420 case TCP_BBR_STARTUP_PG: 14421 case TCP_BBR_DRAIN_PG: 14422 case TCP_BBR_RWND_IS_APP: 14423 case TCP_BBR_PROBE_RTT_INT: 14424 case TCP_BBR_PROBE_RTT_GAIN: 14425 case TCP_BBR_PROBE_RTT_LEN: 14426 case TCP_BBR_STARTUP_LOSS_EXIT: 14427 case TCP_BBR_USEDEL_RATE: 14428 case TCP_BBR_MIN_RTO: 14429 case TCP_BBR_MAX_RTO: 14430 case TCP_BBR_PACE_PER_SEC: 14431 case TCP_DELACK: 14432 case TCP_BBR_PACE_DEL_TAR: 14433 case TCP_BBR_SEND_IWND_IN_TSO: 14434 case TCP_BBR_EXTRA_STATE: 14435 case TCP_BBR_UTTER_MAX_TSO: 14436 case TCP_BBR_MIN_TOPACEOUT: 14437 case TCP_BBR_FLOOR_MIN_TSO: 14438 case TCP_BBR_TSTMP_RAISES: 14439 case TCP_BBR_POLICER_DETECT: 14440 case TCP_BBR_USE_RACK_CHEAT: 14441 case TCP_DATA_AFTER_CLOSE: 14442 case TCP_BBR_HDWR_PACE: 14443 case TCP_BBR_PACE_SEG_MAX: 14444 case TCP_BBR_PACE_SEG_MIN: 14445 case TCP_BBR_PACE_CROSS: 14446 case TCP_BBR_PACE_OH: 14447 #ifdef NETFLIX_PEAKRATE 14448 case TCP_MAXPEAKRATE: 14449 #endif 14450 case TCP_BBR_TMR_PACE_OH: 14451 case TCP_BBR_RACK_RTT_USE: 14452 case TCP_BBR_RETRAN_WTSO: 14453 break; 14454 default: 14455 return (tcp_default_ctloutput(so, sopt, inp, tp)); 14456 break; 14457 } 14458 INP_WUNLOCK(inp); 14459 error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval)); 14460 if (error) 14461 return (error); 14462 INP_WLOCK(inp); 14463 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 14464 INP_WUNLOCK(inp); 14465 return (ECONNRESET); 14466 } 14467 tp = intotcpcb(inp); 14468 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 14469 switch (sopt->sopt_name) { 14470 case TCP_BBR_PACE_PER_SEC: 14471 BBR_OPTS_INC(tcp_bbr_pace_per_sec); 14472 bbr->r_ctl.bbr_hptsi_per_second = optval; 14473 break; 14474 case TCP_BBR_PACE_DEL_TAR: 14475 BBR_OPTS_INC(tcp_bbr_pace_del_tar); 14476 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval; 14477 break; 14478 case TCP_BBR_PACE_SEG_MAX: 14479 BBR_OPTS_INC(tcp_bbr_pace_seg_max); 14480 bbr->r_ctl.bbr_hptsi_segments_max = optval; 14481 break; 14482 case TCP_BBR_PACE_SEG_MIN: 14483 BBR_OPTS_INC(tcp_bbr_pace_seg_min); 14484 bbr->r_ctl.bbr_hptsi_bytes_min = optval; 14485 break; 14486 case TCP_BBR_PACE_CROSS: 14487 BBR_OPTS_INC(tcp_bbr_pace_cross); 14488 bbr->r_ctl.bbr_cross_over = optval; 14489 break; 14490 case TCP_BBR_ALGORITHM: 14491 BBR_OPTS_INC(tcp_bbr_algorithm); 14492 if (optval && (bbr->rc_use_google == 0)) { 14493 /* Turn on the google mode */ 14494 bbr_google_mode_on(bbr); 14495 if ((optval > 3) && (optval < 500)) { 14496 /* 14497 * Must be at least greater than .3% 14498 * and must be less than 50.0%. 14499 */ 14500 bbr->r_ctl.bbr_google_discount = optval; 14501 } 14502 } else if ((optval == 0) && (bbr->rc_use_google == 1)) { 14503 /* Turn off the google mode */ 14504 bbr_google_mode_off(bbr); 14505 } 14506 break; 14507 case TCP_BBR_TSLIMITS: 14508 BBR_OPTS_INC(tcp_bbr_tslimits); 14509 if (optval == 1) 14510 bbr->rc_use_ts_limit = 1; 14511 else if (optval == 0) 14512 bbr->rc_use_ts_limit = 0; 14513 else 14514 error = EINVAL; 14515 break; 14516 14517 case TCP_BBR_IWINTSO: 14518 BBR_OPTS_INC(tcp_bbr_iwintso); 14519 if ((optval >= 0) && (optval < 128)) { 14520 uint32_t twin; 14521 14522 bbr->rc_init_win = optval; 14523 twin = bbr_initial_cwnd(bbr, tp); 14524 if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd)) 14525 tp->snd_cwnd = twin; 14526 else 14527 error = EBUSY; 14528 } else 14529 error = EINVAL; 14530 break; 14531 case TCP_BBR_STARTUP_PG: 14532 BBR_OPTS_INC(tcp_bbr_startup_pg); 14533 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE)) { 14534 bbr->r_ctl.rc_startup_pg = optval; 14535 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) { 14536 bbr->r_ctl.rc_bbr_hptsi_gain = optval; 14537 } 14538 } else 14539 error = EINVAL; 14540 break; 14541 case TCP_BBR_DRAIN_PG: 14542 BBR_OPTS_INC(tcp_bbr_drain_pg); 14543 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE)) 14544 bbr->r_ctl.rc_drain_pg = optval; 14545 else 14546 error = EINVAL; 14547 break; 14548 case TCP_BBR_PROBE_RTT_LEN: 14549 BBR_OPTS_INC(tcp_bbr_probertt_len); 14550 if (optval <= 1) 14551 reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND)); 14552 else 14553 error = EINVAL; 14554 break; 14555 case TCP_BBR_PROBE_RTT_GAIN: 14556 BBR_OPTS_INC(tcp_bbr_probertt_gain); 14557 if (optval <= BBR_UNIT) 14558 bbr->r_ctl.bbr_rttprobe_gain_val = optval; 14559 else 14560 error = EINVAL; 14561 break; 14562 case TCP_BBR_PROBE_RTT_INT: 14563 BBR_OPTS_INC(tcp_bbr_probe_rtt_int); 14564 if (optval > 1000) 14565 bbr->r_ctl.rc_probertt_int = optval; 14566 else 14567 error = EINVAL; 14568 break; 14569 case TCP_BBR_MIN_TOPACEOUT: 14570 BBR_OPTS_INC(tcp_bbr_topaceout); 14571 if (optval == 0) { 14572 bbr->no_pacing_until = 0; 14573 bbr->rc_no_pacing = 0; 14574 } else if (optval <= 0x00ff) { 14575 bbr->no_pacing_until = optval; 14576 if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) && 14577 (bbr->rc_bbr_state == BBR_STATE_STARTUP)){ 14578 /* Turn on no pacing */ 14579 bbr->rc_no_pacing = 1; 14580 } 14581 } else 14582 error = EINVAL; 14583 break; 14584 case TCP_BBR_STARTUP_LOSS_EXIT: 14585 BBR_OPTS_INC(tcp_bbr_startup_loss_exit); 14586 bbr->rc_loss_exit = optval; 14587 break; 14588 case TCP_BBR_USEDEL_RATE: 14589 error = EINVAL; 14590 break; 14591 case TCP_BBR_MIN_RTO: 14592 BBR_OPTS_INC(tcp_bbr_min_rto); 14593 bbr->r_ctl.rc_min_rto_ms = optval; 14594 break; 14595 case TCP_BBR_MAX_RTO: 14596 BBR_OPTS_INC(tcp_bbr_max_rto); 14597 bbr->rc_max_rto_sec = optval; 14598 break; 14599 case TCP_RACK_MIN_TO: 14600 /* Minimum time between rack t-o's in ms */ 14601 BBR_OPTS_INC(tcp_rack_min_to); 14602 bbr->r_ctl.rc_min_to = optval; 14603 break; 14604 case TCP_RACK_REORD_THRESH: 14605 /* RACK reorder threshold (shift amount) */ 14606 BBR_OPTS_INC(tcp_rack_reord_thresh); 14607 if ((optval > 0) && (optval < 31)) 14608 bbr->r_ctl.rc_reorder_shift = optval; 14609 else 14610 error = EINVAL; 14611 break; 14612 case TCP_RACK_REORD_FADE: 14613 /* Does reordering fade after ms time */ 14614 BBR_OPTS_INC(tcp_rack_reord_fade); 14615 bbr->r_ctl.rc_reorder_fade = optval; 14616 break; 14617 case TCP_RACK_TLP_THRESH: 14618 /* RACK TLP theshold i.e. srtt+(srtt/N) */ 14619 BBR_OPTS_INC(tcp_rack_tlp_thresh); 14620 if (optval) 14621 bbr->rc_tlp_threshold = optval; 14622 else 14623 error = EINVAL; 14624 break; 14625 case TCP_BBR_USE_RACK_CHEAT: 14626 BBR_OPTS_INC(tcp_use_rackcheat); 14627 if (bbr->rc_use_google) { 14628 error = EINVAL; 14629 break; 14630 } 14631 BBR_OPTS_INC(tcp_rack_cheat); 14632 if (optval) 14633 bbr->bbr_use_rack_cheat = 1; 14634 else 14635 bbr->bbr_use_rack_cheat = 0; 14636 break; 14637 case TCP_BBR_FLOOR_MIN_TSO: 14638 BBR_OPTS_INC(tcp_utter_max_tso); 14639 if ((optval >= 0) && (optval < 40)) 14640 bbr->r_ctl.bbr_hptsi_segments_floor = optval; 14641 else 14642 error = EINVAL; 14643 break; 14644 case TCP_BBR_UTTER_MAX_TSO: 14645 BBR_OPTS_INC(tcp_utter_max_tso); 14646 if ((optval >= 0) && (optval < 0xffff)) 14647 bbr->r_ctl.bbr_utter_max = optval; 14648 else 14649 error = EINVAL; 14650 break; 14651 14652 case TCP_BBR_EXTRA_STATE: 14653 BBR_OPTS_INC(tcp_extra_state); 14654 if (optval) 14655 bbr->rc_use_idle_restart = 1; 14656 else 14657 bbr->rc_use_idle_restart = 0; 14658 break; 14659 case TCP_BBR_SEND_IWND_IN_TSO: 14660 BBR_OPTS_INC(tcp_iwnd_tso); 14661 if (optval) { 14662 bbr->bbr_init_win_cheat = 1; 14663 if (bbr->rc_past_init_win == 0) { 14664 uint32_t cts; 14665 cts = tcp_get_usecs(&bbr->rc_tv); 14666 tcp_bbr_tso_size_check(bbr, cts); 14667 } 14668 } else 14669 bbr->bbr_init_win_cheat = 0; 14670 break; 14671 case TCP_BBR_HDWR_PACE: 14672 BBR_OPTS_INC(tcp_hdwr_pacing); 14673 if (optval){ 14674 bbr->bbr_hdw_pace_ena = 1; 14675 bbr->bbr_attempt_hdwr_pace = 0; 14676 } else { 14677 bbr->bbr_hdw_pace_ena = 0; 14678 #ifdef RATELIMIT 14679 if (bbr->bbr_hdrw_pacing) { 14680 bbr->bbr_hdrw_pacing = 0; 14681 in_pcbdetach_txrtlmt(bbr->rc_inp); 14682 } 14683 #endif 14684 } 14685 break; 14686 14687 case TCP_DELACK: 14688 BBR_OPTS_INC(tcp_delack); 14689 if (optval < 100) { 14690 if (optval == 0) /* off */ 14691 tp->t_delayed_ack = 0; 14692 else if (optval == 1) /* on which is 2 */ 14693 tp->t_delayed_ack = 2; 14694 else /* higher than 2 and less than 100 */ 14695 tp->t_delayed_ack = optval; 14696 if (tp->t_flags & TF_DELACK) { 14697 tp->t_flags &= ~TF_DELACK; 14698 tp->t_flags |= TF_ACKNOW; 14699 NET_EPOCH_ENTER(et); 14700 bbr_output(tp); 14701 NET_EPOCH_EXIT(et); 14702 } 14703 } else 14704 error = EINVAL; 14705 break; 14706 case TCP_RACK_PKT_DELAY: 14707 /* RACK added ms i.e. rack-rtt + reord + N */ 14708 BBR_OPTS_INC(tcp_rack_pkt_delay); 14709 bbr->r_ctl.rc_pkt_delay = optval; 14710 break; 14711 #ifdef NETFLIX_PEAKRATE 14712 case TCP_MAXPEAKRATE: 14713 BBR_OPTS_INC(tcp_maxpeak); 14714 error = tcp_set_maxpeakrate(tp, optval); 14715 if (!error) 14716 tp->t_peakrate_thr = tp->t_maxpeakrate; 14717 break; 14718 #endif 14719 case TCP_BBR_RETRAN_WTSO: 14720 BBR_OPTS_INC(tcp_retran_wtso); 14721 if (optval) 14722 bbr->rc_resends_use_tso = 1; 14723 else 14724 bbr->rc_resends_use_tso = 0; 14725 break; 14726 case TCP_DATA_AFTER_CLOSE: 14727 BBR_OPTS_INC(tcp_data_ac); 14728 if (optval) 14729 bbr->rc_allow_data_af_clo = 1; 14730 else 14731 bbr->rc_allow_data_af_clo = 0; 14732 break; 14733 case TCP_BBR_POLICER_DETECT: 14734 BBR_OPTS_INC(tcp_policer_det); 14735 if (bbr->rc_use_google == 0) 14736 error = EINVAL; 14737 else if (optval) 14738 bbr->r_use_policer = 1; 14739 else 14740 bbr->r_use_policer = 0; 14741 break; 14742 14743 case TCP_BBR_TSTMP_RAISES: 14744 BBR_OPTS_INC(tcp_ts_raises); 14745 if (optval) 14746 bbr->ts_can_raise = 1; 14747 else 14748 bbr->ts_can_raise = 0; 14749 break; 14750 case TCP_BBR_TMR_PACE_OH: 14751 BBR_OPTS_INC(tcp_pacing_oh_tmr); 14752 if (bbr->rc_use_google) { 14753 error = EINVAL; 14754 } else { 14755 if (optval) 14756 bbr->r_ctl.rc_incr_tmrs = 1; 14757 else 14758 bbr->r_ctl.rc_incr_tmrs = 0; 14759 } 14760 break; 14761 case TCP_BBR_PACE_OH: 14762 BBR_OPTS_INC(tcp_pacing_oh); 14763 if (bbr->rc_use_google) { 14764 error = EINVAL; 14765 } else { 14766 if (optval > (BBR_INCL_TCP_OH| 14767 BBR_INCL_IP_OH| 14768 BBR_INCL_ENET_OH)) { 14769 error = EINVAL; 14770 break; 14771 } 14772 if (optval & BBR_INCL_TCP_OH) 14773 bbr->r_ctl.rc_inc_tcp_oh = 1; 14774 else 14775 bbr->r_ctl.rc_inc_tcp_oh = 0; 14776 if (optval & BBR_INCL_IP_OH) 14777 bbr->r_ctl.rc_inc_ip_oh = 1; 14778 else 14779 bbr->r_ctl.rc_inc_ip_oh = 0; 14780 if (optval & BBR_INCL_ENET_OH) 14781 bbr->r_ctl.rc_inc_enet_oh = 1; 14782 else 14783 bbr->r_ctl.rc_inc_enet_oh = 0; 14784 } 14785 break; 14786 default: 14787 return (tcp_default_ctloutput(so, sopt, inp, tp)); 14788 break; 14789 } 14790 #ifdef NETFLIX_STATS 14791 tcp_log_socket_option(tp, sopt->sopt_name, optval, error); 14792 #endif 14793 INP_WUNLOCK(inp); 14794 return (error); 14795 } 14796 14797 /* 14798 * return 0 on success, error-num on failure 14799 */ 14800 static int 14801 bbr_get_sockopt(struct socket *so, struct sockopt *sopt, 14802 struct inpcb *inp, struct tcpcb *tp, struct tcp_bbr *bbr) 14803 { 14804 int32_t error, optval; 14805 14806 /* 14807 * Because all our options are either boolean or an int, we can just 14808 * pull everything into optval and then unlock and copy. If we ever 14809 * add a option that is not a int, then this will have quite an 14810 * impact to this routine. 14811 */ 14812 switch (sopt->sopt_name) { 14813 case TCP_BBR_PACE_PER_SEC: 14814 optval = bbr->r_ctl.bbr_hptsi_per_second; 14815 break; 14816 case TCP_BBR_PACE_DEL_TAR: 14817 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar; 14818 break; 14819 case TCP_BBR_PACE_SEG_MAX: 14820 optval = bbr->r_ctl.bbr_hptsi_segments_max; 14821 break; 14822 case TCP_BBR_MIN_TOPACEOUT: 14823 optval = bbr->no_pacing_until; 14824 break; 14825 case TCP_BBR_PACE_SEG_MIN: 14826 optval = bbr->r_ctl.bbr_hptsi_bytes_min; 14827 break; 14828 case TCP_BBR_PACE_CROSS: 14829 optval = bbr->r_ctl.bbr_cross_over; 14830 break; 14831 case TCP_BBR_ALGORITHM: 14832 optval = bbr->rc_use_google; 14833 break; 14834 case TCP_BBR_TSLIMITS: 14835 optval = bbr->rc_use_ts_limit; 14836 break; 14837 case TCP_BBR_IWINTSO: 14838 optval = bbr->rc_init_win; 14839 break; 14840 case TCP_BBR_STARTUP_PG: 14841 optval = bbr->r_ctl.rc_startup_pg; 14842 break; 14843 case TCP_BBR_DRAIN_PG: 14844 optval = bbr->r_ctl.rc_drain_pg; 14845 break; 14846 case TCP_BBR_PROBE_RTT_INT: 14847 optval = bbr->r_ctl.rc_probertt_int; 14848 break; 14849 case TCP_BBR_PROBE_RTT_LEN: 14850 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND); 14851 break; 14852 case TCP_BBR_PROBE_RTT_GAIN: 14853 optval = bbr->r_ctl.bbr_rttprobe_gain_val; 14854 break; 14855 case TCP_BBR_STARTUP_LOSS_EXIT: 14856 optval = bbr->rc_loss_exit; 14857 break; 14858 case TCP_BBR_USEDEL_RATE: 14859 error = EINVAL; 14860 break; 14861 case TCP_BBR_MIN_RTO: 14862 optval = bbr->r_ctl.rc_min_rto_ms; 14863 break; 14864 case TCP_BBR_MAX_RTO: 14865 optval = bbr->rc_max_rto_sec; 14866 break; 14867 case TCP_RACK_PACE_MAX_SEG: 14868 /* Max segments in a pace */ 14869 optval = bbr->r_ctl.rc_pace_max_segs; 14870 break; 14871 case TCP_RACK_MIN_TO: 14872 /* Minimum time between rack t-o's in ms */ 14873 optval = bbr->r_ctl.rc_min_to; 14874 break; 14875 case TCP_RACK_REORD_THRESH: 14876 /* RACK reorder threshold (shift amount) */ 14877 optval = bbr->r_ctl.rc_reorder_shift; 14878 break; 14879 case TCP_RACK_REORD_FADE: 14880 /* Does reordering fade after ms time */ 14881 optval = bbr->r_ctl.rc_reorder_fade; 14882 break; 14883 case TCP_BBR_USE_RACK_CHEAT: 14884 /* Do we use the rack cheat for rxt */ 14885 optval = bbr->bbr_use_rack_cheat; 14886 break; 14887 case TCP_BBR_FLOOR_MIN_TSO: 14888 optval = bbr->r_ctl.bbr_hptsi_segments_floor; 14889 break; 14890 case TCP_BBR_UTTER_MAX_TSO: 14891 optval = bbr->r_ctl.bbr_utter_max; 14892 break; 14893 case TCP_BBR_SEND_IWND_IN_TSO: 14894 /* Do we send TSO size segments initially */ 14895 optval = bbr->bbr_init_win_cheat; 14896 break; 14897 case TCP_BBR_EXTRA_STATE: 14898 optval = bbr->rc_use_idle_restart; 14899 break; 14900 case TCP_RACK_TLP_THRESH: 14901 /* RACK TLP theshold i.e. srtt+(srtt/N) */ 14902 optval = bbr->rc_tlp_threshold; 14903 break; 14904 case TCP_RACK_PKT_DELAY: 14905 /* RACK added ms i.e. rack-rtt + reord + N */ 14906 optval = bbr->r_ctl.rc_pkt_delay; 14907 break; 14908 case TCP_BBR_RETRAN_WTSO: 14909 optval = bbr->rc_resends_use_tso; 14910 break; 14911 case TCP_DATA_AFTER_CLOSE: 14912 optval = bbr->rc_allow_data_af_clo; 14913 break; 14914 case TCP_DELACK: 14915 optval = tp->t_delayed_ack; 14916 break; 14917 case TCP_BBR_HDWR_PACE: 14918 optval = bbr->bbr_hdw_pace_ena; 14919 break; 14920 case TCP_BBR_POLICER_DETECT: 14921 optval = bbr->r_use_policer; 14922 break; 14923 case TCP_BBR_TSTMP_RAISES: 14924 optval = bbr->ts_can_raise; 14925 break; 14926 case TCP_BBR_TMR_PACE_OH: 14927 optval = bbr->r_ctl.rc_incr_tmrs; 14928 break; 14929 case TCP_BBR_PACE_OH: 14930 optval = 0; 14931 if (bbr->r_ctl.rc_inc_tcp_oh) 14932 optval |= BBR_INCL_TCP_OH; 14933 if (bbr->r_ctl.rc_inc_ip_oh) 14934 optval |= BBR_INCL_IP_OH; 14935 if (bbr->r_ctl.rc_inc_enet_oh) 14936 optval |= BBR_INCL_ENET_OH; 14937 break; 14938 default: 14939 return (tcp_default_ctloutput(so, sopt, inp, tp)); 14940 break; 14941 } 14942 INP_WUNLOCK(inp); 14943 error = sooptcopyout(sopt, &optval, sizeof optval); 14944 return (error); 14945 } 14946 14947 /* 14948 * return 0 on success, error-num on failure 14949 */ 14950 static int 14951 bbr_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp) 14952 { 14953 int32_t error = EINVAL; 14954 struct tcp_bbr *bbr; 14955 14956 bbr = (struct tcp_bbr *)tp->t_fb_ptr; 14957 if (bbr == NULL) { 14958 /* Huh? */ 14959 goto out; 14960 } 14961 if (sopt->sopt_dir == SOPT_SET) { 14962 return (bbr_set_sockopt(so, sopt, inp, tp, bbr)); 14963 } else if (sopt->sopt_dir == SOPT_GET) { 14964 return (bbr_get_sockopt(so, sopt, inp, tp, bbr)); 14965 } 14966 out: 14967 INP_WUNLOCK(inp); 14968 return (error); 14969 } 14970 14971 static int 14972 bbr_pru_options(struct tcpcb *tp, int flags) 14973 { 14974 if (flags & PRUS_OOB) 14975 return (EOPNOTSUPP); 14976 return (0); 14977 } 14978 14979 struct tcp_function_block __tcp_bbr = { 14980 .tfb_tcp_block_name = __XSTRING(STACKNAME), 14981 .tfb_tcp_output = bbr_output, 14982 .tfb_do_queued_segments = ctf_do_queued_segments, 14983 .tfb_do_segment_nounlock = bbr_do_segment_nounlock, 14984 .tfb_tcp_do_segment = bbr_do_segment, 14985 .tfb_tcp_ctloutput = bbr_ctloutput, 14986 .tfb_tcp_fb_init = bbr_init, 14987 .tfb_tcp_fb_fini = bbr_fini, 14988 .tfb_tcp_timer_stop_all = bbr_stopall, 14989 .tfb_tcp_timer_activate = bbr_timer_activate, 14990 .tfb_tcp_timer_active = bbr_timer_active, 14991 .tfb_tcp_timer_stop = bbr_timer_stop, 14992 .tfb_tcp_rexmit_tmr = bbr_remxt_tmr, 14993 .tfb_tcp_handoff_ok = bbr_handoff_ok, 14994 .tfb_tcp_mtu_chg = bbr_mtu_chg, 14995 .tfb_pru_options = bbr_pru_options, 14996 }; 14997 14998 static const char *bbr_stack_names[] = { 14999 __XSTRING(STACKNAME), 15000 #ifdef STACKALIAS 15001 __XSTRING(STACKALIAS), 15002 #endif 15003 }; 15004 15005 static bool bbr_mod_inited = false; 15006 15007 static int 15008 tcp_addbbr(module_t mod, int32_t type, void *data) 15009 { 15010 int32_t err = 0; 15011 int num_stacks; 15012 15013 switch (type) { 15014 case MOD_LOAD: 15015 printf("Attempting to load " __XSTRING(MODNAME) "\n"); 15016 bbr_zone = uma_zcreate(__XSTRING(MODNAME) "_map", 15017 sizeof(struct bbr_sendmap), 15018 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 15019 bbr_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb", 15020 sizeof(struct tcp_bbr), 15021 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 15022 sysctl_ctx_init(&bbr_sysctl_ctx); 15023 bbr_sysctl_root = SYSCTL_ADD_NODE(&bbr_sysctl_ctx, 15024 SYSCTL_STATIC_CHILDREN(_net_inet_tcp), 15025 OID_AUTO, 15026 #ifdef STACKALIAS 15027 __XSTRING(STACKALIAS), 15028 #else 15029 __XSTRING(STACKNAME), 15030 #endif 15031 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 15032 ""); 15033 if (bbr_sysctl_root == NULL) { 15034 printf("Failed to add sysctl node\n"); 15035 err = EFAULT; 15036 goto free_uma; 15037 } 15038 bbr_init_sysctls(); 15039 num_stacks = nitems(bbr_stack_names); 15040 err = register_tcp_functions_as_names(&__tcp_bbr, M_WAITOK, 15041 bbr_stack_names, &num_stacks); 15042 if (err) { 15043 printf("Failed to register %s stack name for " 15044 "%s module\n", bbr_stack_names[num_stacks], 15045 __XSTRING(MODNAME)); 15046 sysctl_ctx_free(&bbr_sysctl_ctx); 15047 free_uma: 15048 uma_zdestroy(bbr_zone); 15049 uma_zdestroy(bbr_pcb_zone); 15050 bbr_counter_destroy(); 15051 printf("Failed to register " __XSTRING(MODNAME) 15052 " module err:%d\n", err); 15053 return (err); 15054 } 15055 tcp_lro_reg_mbufq(); 15056 bbr_mod_inited = true; 15057 printf(__XSTRING(MODNAME) " is now available\n"); 15058 break; 15059 case MOD_QUIESCE: 15060 err = deregister_tcp_functions(&__tcp_bbr, true, false); 15061 break; 15062 case MOD_UNLOAD: 15063 err = deregister_tcp_functions(&__tcp_bbr, false, true); 15064 if (err == EBUSY) 15065 break; 15066 if (bbr_mod_inited) { 15067 uma_zdestroy(bbr_zone); 15068 uma_zdestroy(bbr_pcb_zone); 15069 sysctl_ctx_free(&bbr_sysctl_ctx); 15070 bbr_counter_destroy(); 15071 printf(__XSTRING(MODNAME) 15072 " is now no longer available\n"); 15073 bbr_mod_inited = false; 15074 } 15075 tcp_lro_dereg_mbufq(); 15076 err = 0; 15077 break; 15078 default: 15079 return (EOPNOTSUPP); 15080 } 15081 return (err); 15082 } 15083 15084 static moduledata_t tcp_bbr = { 15085 .name = __XSTRING(MODNAME), 15086 .evhand = tcp_addbbr, 15087 .priv = 0 15088 }; 15089 15090 MODULE_VERSION(MODNAME, 1); 15091 DECLARE_MODULE(MODNAME, tcp_bbr, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY); 15092 MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1); 15093