1 /*- 2 * Copyright (c) 2016-2020 Netflix, Inc. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_inet.h" 31 #include "opt_inet6.h" 32 #include "opt_ipsec.h" 33 #include "opt_ratelimit.h" 34 #include "opt_kern_tls.h" 35 #if defined(INET) || defined(INET6) 36 #include <sys/param.h> 37 #include <sys/arb.h> 38 #include <sys/module.h> 39 #include <sys/kernel.h> 40 #ifdef TCP_HHOOK 41 #include <sys/hhook.h> 42 #endif 43 #include <sys/lock.h> 44 #include <sys/malloc.h> 45 #include <sys/lock.h> 46 #include <sys/mutex.h> 47 #include <sys/mbuf.h> 48 #include <sys/proc.h> /* for proc0 declaration */ 49 #include <sys/socket.h> 50 #include <sys/socketvar.h> 51 #include <sys/sysctl.h> 52 #include <sys/systm.h> 53 #ifdef STATS 54 #include <sys/qmath.h> 55 #include <sys/tree.h> 56 #include <sys/stats.h> /* Must come after qmath.h and tree.h */ 57 #else 58 #include <sys/tree.h> 59 #endif 60 #include <sys/refcount.h> 61 #include <sys/queue.h> 62 #include <sys/tim_filter.h> 63 #include <sys/smp.h> 64 #include <sys/kthread.h> 65 #include <sys/kern_prefetch.h> 66 #include <sys/protosw.h> 67 #ifdef TCP_ACCOUNTING 68 #include <sys/sched.h> 69 #include <machine/cpu.h> 70 #endif 71 #include <vm/uma.h> 72 73 #include <net/route.h> 74 #include <net/route/nhop.h> 75 #include <net/vnet.h> 76 77 #define TCPSTATES /* for logging */ 78 79 #include <netinet/in.h> 80 #include <netinet/in_kdtrace.h> 81 #include <netinet/in_pcb.h> 82 #include <netinet/ip.h> 83 #include <netinet/ip_icmp.h> /* required for icmp_var.h */ 84 #include <netinet/icmp_var.h> /* for ICMP_BANDLIM */ 85 #include <netinet/ip_var.h> 86 #include <netinet/ip6.h> 87 #include <netinet6/in6_pcb.h> 88 #include <netinet6/ip6_var.h> 89 #include <netinet/tcp.h> 90 #define TCPOUTFLAGS 91 #include <netinet/tcp_fsm.h> 92 #include <netinet/tcp_log_buf.h> 93 #include <netinet/tcp_seq.h> 94 #include <netinet/tcp_timer.h> 95 #include <netinet/tcp_var.h> 96 #include <netinet/tcp_syncache.h> 97 #include <netinet/tcp_hpts.h> 98 #include <netinet/tcp_ratelimit.h> 99 #include <netinet/tcp_accounting.h> 100 #include <netinet/tcpip.h> 101 #include <netinet/cc/cc.h> 102 #include <netinet/cc/cc_newreno.h> 103 #include <netinet/tcp_fastopen.h> 104 #include <netinet/tcp_lro.h> 105 #ifdef NETFLIX_SHARED_CWND 106 #include <netinet/tcp_shared_cwnd.h> 107 #endif 108 #ifdef TCP_OFFLOAD 109 #include <netinet/tcp_offload.h> 110 #endif 111 #ifdef INET6 112 #include <netinet6/tcp6_var.h> 113 #endif 114 #include <netinet/tcp_ecn.h> 115 116 #include <netipsec/ipsec_support.h> 117 118 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 119 #include <netipsec/ipsec.h> 120 #include <netipsec/ipsec6.h> 121 #endif /* IPSEC */ 122 123 #include <netinet/udp.h> 124 #include <netinet/udp_var.h> 125 #include <machine/in_cksum.h> 126 127 #ifdef MAC 128 #include <security/mac/mac_framework.h> 129 #endif 130 #include "sack_filter.h" 131 #include "tcp_rack.h" 132 #include "rack_bbr_common.h" 133 134 uma_zone_t rack_zone; 135 uma_zone_t rack_pcb_zone; 136 137 #ifndef TICKS2SBT 138 #define TICKS2SBT(__t) (tick_sbt * ((sbintime_t)(__t))) 139 #endif 140 141 VNET_DECLARE(uint32_t, newreno_beta); 142 VNET_DECLARE(uint32_t, newreno_beta_ecn); 143 #define V_newreno_beta VNET(newreno_beta) 144 #define V_newreno_beta_ecn VNET(newreno_beta_ecn) 145 146 147 MALLOC_DEFINE(M_TCPFSB, "tcp_fsb", "TCP fast send block"); 148 MALLOC_DEFINE(M_TCPDO, "tcp_do", "TCP deferred options"); 149 150 struct sysctl_ctx_list rack_sysctl_ctx; 151 struct sysctl_oid *rack_sysctl_root; 152 153 #define CUM_ACKED 1 154 #define SACKED 2 155 156 /* 157 * The RACK module incorporates a number of 158 * TCP ideas that have been put out into the IETF 159 * over the last few years: 160 * - Matt Mathis's Rate Halving which slowly drops 161 * the congestion window so that the ack clock can 162 * be maintained during a recovery. 163 * - Yuchung Cheng's RACK TCP (for which its named) that 164 * will stop us using the number of dup acks and instead 165 * use time as the gage of when we retransmit. 166 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft 167 * of Dukkipati et.al. 168 * RACK depends on SACK, so if an endpoint arrives that 169 * cannot do SACK the state machine below will shuttle the 170 * connection back to using the "default" TCP stack that is 171 * in FreeBSD. 172 * 173 * To implement RACK the original TCP stack was first decomposed 174 * into a functional state machine with individual states 175 * for each of the possible TCP connection states. The do_segment 176 * functions role in life is to mandate the connection supports SACK 177 * initially and then assure that the RACK state matches the conenction 178 * state before calling the states do_segment function. Each 179 * state is simplified due to the fact that the original do_segment 180 * has been decomposed and we *know* what state we are in (no 181 * switches on the state) and all tests for SACK are gone. This 182 * greatly simplifies what each state does. 183 * 184 * TCP output is also over-written with a new version since it 185 * must maintain the new rack scoreboard. 186 * 187 */ 188 static int32_t rack_tlp_thresh = 1; 189 static int32_t rack_tlp_limit = 2; /* No more than 2 TLPs w-out new data */ 190 static int32_t rack_tlp_use_greater = 1; 191 static int32_t rack_reorder_thresh = 2; 192 static int32_t rack_reorder_fade = 60000000; /* 0 - never fade, def 60,000,000 193 * - 60 seconds */ 194 static uint8_t rack_req_measurements = 1; 195 /* Attack threshold detections */ 196 static uint32_t rack_highest_sack_thresh_seen = 0; 197 static uint32_t rack_highest_move_thresh_seen = 0; 198 static int32_t rack_enable_hw_pacing = 0; /* Due to CCSP keep it off by default */ 199 static int32_t rack_hw_pace_extra_slots = 2; /* 2 extra MSS time betweens */ 200 static int32_t rack_hw_rate_caps = 1; /* 1; */ 201 static int32_t rack_hw_rate_min = 0; /* 1500000;*/ 202 static int32_t rack_hw_rate_to_low = 0; /* 1200000; */ 203 static int32_t rack_hw_up_only = 1; 204 static int32_t rack_stats_gets_ms_rtt = 1; 205 static int32_t rack_prr_addbackmax = 2; 206 static int32_t rack_do_hystart = 0; 207 static int32_t rack_apply_rtt_with_reduced_conf = 0; 208 209 static int32_t rack_pkt_delay = 1000; 210 static int32_t rack_send_a_lot_in_prr = 1; 211 static int32_t rack_min_to = 1000; /* Number of microsecond min timeout */ 212 static int32_t rack_verbose_logging = 0; 213 static int32_t rack_ignore_data_after_close = 1; 214 static int32_t rack_enable_shared_cwnd = 1; 215 static int32_t rack_use_cmp_acks = 1; 216 static int32_t rack_use_fsb = 1; 217 static int32_t rack_use_rfo = 1; 218 static int32_t rack_use_rsm_rfo = 1; 219 static int32_t rack_max_abc_post_recovery = 2; 220 static int32_t rack_client_low_buf = 0; 221 static int32_t rack_dsack_std_based = 0x3; /* bit field bit 1 sets rc_rack_tmr_std_based and bit 2 sets rc_rack_use_dsack */ 222 #ifdef TCP_ACCOUNTING 223 static int32_t rack_tcp_accounting = 0; 224 #endif 225 static int32_t rack_limits_scwnd = 1; 226 static int32_t rack_enable_mqueue_for_nonpaced = 0; 227 static int32_t rack_disable_prr = 0; 228 static int32_t use_rack_rr = 1; 229 static int32_t rack_non_rxt_use_cr = 0; /* does a non-rxt in recovery use the configured rate (ss/ca)? */ 230 static int32_t rack_persist_min = 250000; /* 250usec */ 231 static int32_t rack_persist_max = 2000000; /* 2 Second in usec's */ 232 static int32_t rack_sack_not_required = 1; /* set to one to allow non-sack to use rack */ 233 static int32_t rack_default_init_window = 0; /* Use system default */ 234 static int32_t rack_limit_time_with_srtt = 0; 235 static int32_t rack_autosndbuf_inc = 20; /* In percentage form */ 236 static int32_t rack_enobuf_hw_boost_mult = 2; /* How many times the hw rate we boost slot using time_between */ 237 static int32_t rack_enobuf_hw_max = 12000; /* 12 ms in usecs */ 238 static int32_t rack_enobuf_hw_min = 10000; /* 10 ms in usecs */ 239 static int32_t rack_hw_rwnd_factor = 2; /* How many max_segs the rwnd must be before we hold off sending */ 240 241 /* 242 * Currently regular tcp has a rto_min of 30ms 243 * the backoff goes 12 times so that ends up 244 * being a total of 122.850 seconds before a 245 * connection is killed. 246 */ 247 static uint32_t rack_def_data_window = 20; 248 static uint32_t rack_goal_bdp = 2; 249 static uint32_t rack_min_srtts = 1; 250 static uint32_t rack_min_measure_usec = 0; 251 static int32_t rack_tlp_min = 10000; /* 10ms */ 252 static int32_t rack_rto_min = 30000; /* 30,000 usec same as main freebsd */ 253 static int32_t rack_rto_max = 4000000; /* 4 seconds in usec's */ 254 static const int32_t rack_free_cache = 2; 255 static int32_t rack_hptsi_segments = 40; 256 static int32_t rack_rate_sample_method = USE_RTT_LOW; 257 static int32_t rack_pace_every_seg = 0; 258 static int32_t rack_delayed_ack_time = 40000; /* 40ms in usecs */ 259 static int32_t rack_slot_reduction = 4; 260 static int32_t rack_wma_divisor = 8; /* For WMA calculation */ 261 static int32_t rack_cwnd_block_ends_measure = 0; 262 static int32_t rack_rwnd_block_ends_measure = 0; 263 static int32_t rack_def_profile = 0; 264 265 static int32_t rack_lower_cwnd_at_tlp = 0; 266 static int32_t rack_limited_retran = 0; 267 static int32_t rack_always_send_oldest = 0; 268 static int32_t rack_tlp_threshold_use = TLP_USE_TWO_ONE; 269 270 static uint16_t rack_per_of_gp_ss = 250; /* 250 % slow-start */ 271 static uint16_t rack_per_of_gp_ca = 200; /* 200 % congestion-avoidance */ 272 static uint16_t rack_per_of_gp_rec = 200; /* 200 % of bw */ 273 274 /* Probertt */ 275 static uint16_t rack_per_of_gp_probertt = 60; /* 60% of bw */ 276 static uint16_t rack_per_of_gp_lowthresh = 40; /* 40% is bottom */ 277 static uint16_t rack_per_of_gp_probertt_reduce = 10; /* 10% reduction */ 278 static uint16_t rack_atexit_prtt_hbp = 130; /* Clamp to 130% on exit prtt if highly buffered path */ 279 static uint16_t rack_atexit_prtt = 130; /* Clamp to 100% on exit prtt if non highly buffered path */ 280 281 static uint32_t rack_max_drain_wait = 2; /* How man gp srtt's before we give up draining */ 282 static uint32_t rack_must_drain = 1; /* How many GP srtt's we *must* wait */ 283 static uint32_t rack_probertt_use_min_rtt_entry = 1; /* Use the min to calculate the goal else gp_srtt */ 284 static uint32_t rack_probertt_use_min_rtt_exit = 0; 285 static uint32_t rack_probe_rtt_sets_cwnd = 0; 286 static uint32_t rack_probe_rtt_safety_val = 2000000; /* No more than 2 sec in probe-rtt */ 287 static uint32_t rack_time_between_probertt = 9600000; /* 9.6 sec in usecs */ 288 static uint32_t rack_probertt_gpsrtt_cnt_mul = 0; /* How many srtt periods does probe-rtt last top fraction */ 289 static uint32_t rack_probertt_gpsrtt_cnt_div = 0; /* How many srtt periods does probe-rtt last bottom fraction */ 290 static uint32_t rack_min_probertt_hold = 40000; /* Equal to delayed ack time */ 291 static uint32_t rack_probertt_filter_life = 10000000; 292 static uint32_t rack_probertt_lower_within = 10; 293 static uint32_t rack_min_rtt_movement = 250000; /* Must move at least 250ms (in microseconds) to count as a lowering */ 294 static int32_t rack_pace_one_seg = 0; /* Shall we pace for less than 1.4Meg 1MSS at a time */ 295 static int32_t rack_probertt_clear_is = 1; 296 static int32_t rack_max_drain_hbp = 1; /* Extra drain times gpsrtt for highly buffered paths */ 297 static int32_t rack_hbp_thresh = 3; /* what is the divisor max_rtt/min_rtt to decided a hbp */ 298 299 /* Part of pacing */ 300 static int32_t rack_max_per_above = 30; /* When we go to increment stop if above 100+this% */ 301 302 /* Timely information */ 303 /* Combine these two gives the range of 'no change' to bw */ 304 /* ie the up/down provide the upper and lower bound */ 305 static int32_t rack_gp_per_bw_mul_up = 2; /* 2% */ 306 static int32_t rack_gp_per_bw_mul_down = 4; /* 4% */ 307 static int32_t rack_gp_rtt_maxmul = 3; /* 3 x maxmin */ 308 static int32_t rack_gp_rtt_minmul = 1; /* minrtt + (minrtt/mindiv) is lower rtt */ 309 static int32_t rack_gp_rtt_mindiv = 4; /* minrtt + (minrtt * minmul/mindiv) is lower rtt */ 310 static int32_t rack_gp_decrease_per = 20; /* 20% decrease in multiplier */ 311 static int32_t rack_gp_increase_per = 2; /* 2% increase in multiplier */ 312 static int32_t rack_per_lower_bound = 50; /* Don't allow to drop below this multiplier */ 313 static int32_t rack_per_upper_bound_ss = 0; /* Don't allow SS to grow above this */ 314 static int32_t rack_per_upper_bound_ca = 0; /* Don't allow CA to grow above this */ 315 static int32_t rack_do_dyn_mul = 0; /* Are the rack gp multipliers dynamic */ 316 static int32_t rack_gp_no_rec_chg = 1; /* Prohibit recovery from reducing it's multiplier */ 317 static int32_t rack_timely_dec_clear = 6; /* Do we clear decrement count at a value (6)? */ 318 static int32_t rack_timely_max_push_rise = 3; /* One round of pushing */ 319 static int32_t rack_timely_max_push_drop = 3; /* Three round of pushing */ 320 static int32_t rack_timely_min_segs = 4; /* 4 segment minimum */ 321 static int32_t rack_use_max_for_nobackoff = 0; 322 static int32_t rack_timely_int_timely_only = 0; /* do interim timely's only use the timely algo (no b/w changes)? */ 323 static int32_t rack_timely_no_stopping = 0; 324 static int32_t rack_down_raise_thresh = 100; 325 static int32_t rack_req_segs = 1; 326 static uint64_t rack_bw_rate_cap = 0; 327 static uint32_t rack_trace_point_config = 0; 328 static uint32_t rack_trace_point_bb_mode = 4; 329 static int32_t rack_trace_point_count = 0; 330 331 332 /* Weird delayed ack mode */ 333 static int32_t rack_use_imac_dack = 0; 334 /* Rack specific counters */ 335 counter_u64_t rack_saw_enobuf; 336 counter_u64_t rack_saw_enobuf_hw; 337 counter_u64_t rack_saw_enetunreach; 338 counter_u64_t rack_persists_sends; 339 counter_u64_t rack_persists_acks; 340 counter_u64_t rack_persists_loss; 341 counter_u64_t rack_persists_lost_ends; 342 #ifdef INVARIANTS 343 counter_u64_t rack_adjust_map_bw; 344 #endif 345 /* Tail loss probe counters */ 346 counter_u64_t rack_tlp_tot; 347 counter_u64_t rack_tlp_newdata; 348 counter_u64_t rack_tlp_retran; 349 counter_u64_t rack_tlp_retran_bytes; 350 counter_u64_t rack_to_tot; 351 counter_u64_t rack_hot_alloc; 352 counter_u64_t rack_to_alloc; 353 counter_u64_t rack_to_alloc_hard; 354 counter_u64_t rack_to_alloc_emerg; 355 counter_u64_t rack_to_alloc_limited; 356 counter_u64_t rack_alloc_limited_conns; 357 counter_u64_t rack_split_limited; 358 359 counter_u64_t rack_multi_single_eq; 360 counter_u64_t rack_proc_non_comp_ack; 361 362 counter_u64_t rack_fto_send; 363 counter_u64_t rack_fto_rsm_send; 364 counter_u64_t rack_nfto_resend; 365 counter_u64_t rack_non_fto_send; 366 counter_u64_t rack_extended_rfo; 367 368 counter_u64_t rack_sack_proc_all; 369 counter_u64_t rack_sack_proc_short; 370 counter_u64_t rack_sack_proc_restart; 371 counter_u64_t rack_sack_attacks_detected; 372 counter_u64_t rack_sack_attacks_reversed; 373 counter_u64_t rack_sack_used_next_merge; 374 counter_u64_t rack_sack_splits; 375 counter_u64_t rack_sack_used_prev_merge; 376 counter_u64_t rack_sack_skipped_acked; 377 counter_u64_t rack_ack_total; 378 counter_u64_t rack_express_sack; 379 counter_u64_t rack_sack_total; 380 counter_u64_t rack_move_none; 381 counter_u64_t rack_move_some; 382 383 counter_u64_t rack_input_idle_reduces; 384 counter_u64_t rack_collapsed_win; 385 counter_u64_t rack_collapsed_win_seen; 386 counter_u64_t rack_collapsed_win_rxt; 387 counter_u64_t rack_collapsed_win_rxt_bytes; 388 counter_u64_t rack_try_scwnd; 389 counter_u64_t rack_hw_pace_init_fail; 390 counter_u64_t rack_hw_pace_lost; 391 392 counter_u64_t rack_out_size[TCP_MSS_ACCT_SIZE]; 393 counter_u64_t rack_opts_arry[RACK_OPTS_SIZE]; 394 395 396 #define RACK_REXMTVAL(tp) max(rack_rto_min, ((tp)->t_srtt + ((tp)->t_rttvar << 2))) 397 398 #define RACK_TCPT_RANGESET(tv, value, tvmin, tvmax, slop) do { \ 399 (tv) = (value) + slop; \ 400 if ((u_long)(tv) < (u_long)(tvmin)) \ 401 (tv) = (tvmin); \ 402 if ((u_long)(tv) > (u_long)(tvmax)) \ 403 (tv) = (tvmax); \ 404 } while (0) 405 406 static void 407 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line); 408 409 static int 410 rack_process_ack(struct mbuf *m, struct tcphdr *th, 411 struct socket *so, struct tcpcb *tp, struct tcpopt *to, 412 uint32_t tiwin, int32_t tlen, int32_t * ofia, int32_t thflags, int32_t * ret_val); 413 static int 414 rack_process_data(struct mbuf *m, struct tcphdr *th, 415 struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, 416 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt); 417 static void 418 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack, 419 uint32_t th_ack, uint16_t nsegs, uint16_t type, int32_t recovery); 420 static struct rack_sendmap *rack_alloc(struct tcp_rack *rack); 421 static struct rack_sendmap *rack_alloc_limit(struct tcp_rack *rack, 422 uint8_t limit_type); 423 static struct rack_sendmap * 424 rack_check_recovery_mode(struct tcpcb *tp, 425 uint32_t tsused); 426 static void 427 rack_cong_signal(struct tcpcb *tp, 428 uint32_t type, uint32_t ack, int ); 429 static void rack_counter_destroy(void); 430 static int 431 rack_ctloutput(struct inpcb *inp, struct sockopt *sopt); 432 static int32_t rack_ctor(void *mem, int32_t size, void *arg, int32_t how); 433 static void 434 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override); 435 static void 436 rack_do_segment(struct mbuf *m, struct tcphdr *th, 437 struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, 438 uint8_t iptos); 439 static void rack_dtor(void *mem, int32_t size, void *arg); 440 static void 441 rack_log_alt_to_to_cancel(struct tcp_rack *rack, 442 uint32_t flex1, uint32_t flex2, 443 uint32_t flex3, uint32_t flex4, 444 uint32_t flex5, uint32_t flex6, 445 uint16_t flex7, uint8_t mod); 446 447 static void 448 rack_log_pacing_delay_calc(struct tcp_rack *rack, uint32_t len, uint32_t slot, 449 uint64_t bw_est, uint64_t bw, uint64_t len_time, int method, int line, 450 struct rack_sendmap *rsm, uint8_t quality); 451 static struct rack_sendmap * 452 rack_find_high_nonack(struct tcp_rack *rack, 453 struct rack_sendmap *rsm); 454 static struct rack_sendmap *rack_find_lowest_rsm(struct tcp_rack *rack); 455 static void rack_free(struct tcp_rack *rack, struct rack_sendmap *rsm); 456 static void rack_fini(struct tcpcb *tp, int32_t tcb_is_purged); 457 static int rack_get_sockopt(struct inpcb *inp, struct sockopt *sopt); 458 static void 459 rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack, 460 tcp_seq th_ack, int line, uint8_t quality); 461 static uint32_t 462 rack_get_pacing_len(struct tcp_rack *rack, uint64_t bw, uint32_t mss); 463 static int32_t rack_handoff_ok(struct tcpcb *tp); 464 static int32_t rack_init(struct tcpcb *tp); 465 static void rack_init_sysctls(void); 466 static void 467 rack_log_ack(struct tcpcb *tp, struct tcpopt *to, 468 struct tcphdr *th, int entered_rec, int dup_ack_struck); 469 static void 470 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len, 471 uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t ts, 472 struct rack_sendmap *hintrsm, uint16_t add_flags, struct mbuf *s_mb, uint32_t s_moff, int hw_tls); 473 474 static void 475 rack_log_sack_passed(struct tcpcb *tp, struct tcp_rack *rack, 476 struct rack_sendmap *rsm); 477 static void rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rsm); 478 static int32_t rack_output(struct tcpcb *tp); 479 480 static uint32_t 481 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack, 482 struct sackblk *sack, struct tcpopt *to, struct rack_sendmap **prsm, 483 uint32_t cts, int *moved_two); 484 static void rack_post_recovery(struct tcpcb *tp, uint32_t th_seq); 485 static void rack_remxt_tmr(struct tcpcb *tp); 486 static int rack_set_sockopt(struct inpcb *inp, struct sockopt *sopt); 487 static void rack_set_state(struct tcpcb *tp, struct tcp_rack *rack); 488 static int32_t rack_stopall(struct tcpcb *tp); 489 static void rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line); 490 static uint32_t 491 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack, 492 struct rack_sendmap *rsm, uint64_t ts, int32_t * lenp, uint16_t add_flag); 493 static void 494 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack, 495 struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag); 496 static int 497 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack, 498 struct rack_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, tcp_seq th_ack); 499 static int32_t tcp_addrack(module_t mod, int32_t type, void *data); 500 static int 501 rack_do_close_wait(struct mbuf *m, struct tcphdr *th, 502 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 503 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 504 static int 505 rack_do_closing(struct mbuf *m, struct tcphdr *th, 506 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 507 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 508 static int 509 rack_do_established(struct mbuf *m, struct tcphdr *th, 510 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 511 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 512 static int 513 rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, 514 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 515 int32_t tlen, uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos); 516 static int 517 rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, 518 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 519 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 520 static int 521 rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, 522 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 523 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 524 static int 525 rack_do_lastack(struct mbuf *m, struct tcphdr *th, 526 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 527 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 528 static int 529 rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, 530 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 531 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 532 static int 533 rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, 534 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, 535 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos); 536 struct rack_sendmap * 537 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack, 538 uint32_t tsused); 539 static void tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt, 540 uint32_t len, uint32_t us_tim, int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt); 541 static void 542 tcp_rack_partialack(struct tcpcb *tp); 543 static int 544 rack_set_profile(struct tcp_rack *rack, int prof); 545 static void 546 rack_apply_deferred_options(struct tcp_rack *rack); 547 548 int32_t rack_clear_counter=0; 549 550 static inline void 551 rack_trace_point(struct tcp_rack *rack, int num) 552 { 553 if (((rack_trace_point_config == num) || 554 (rack_trace_point_config = 0xffffffff)) && 555 (rack_trace_point_bb_mode != 0) && 556 (rack_trace_point_count > 0) && 557 (rack->rc_tp->t_logstate == 0)) { 558 int res; 559 res = atomic_fetchadd_int(&rack_trace_point_count, -1); 560 if (res > 0) { 561 rack->rc_tp->t_logstate = rack_trace_point_bb_mode; 562 } else { 563 /* Loss a race assure its zero now */ 564 rack_trace_point_count = 0; 565 } 566 } 567 } 568 569 static void 570 rack_swap_beta_values(struct tcp_rack *rack, uint8_t flex8) 571 { 572 struct sockopt sopt; 573 struct cc_newreno_opts opt; 574 struct newreno old; 575 struct tcpcb *tp; 576 int error, failed = 0; 577 578 tp = rack->rc_tp; 579 if (tp->t_cc == NULL) { 580 /* Tcb is leaving */ 581 return; 582 } 583 rack->rc_pacing_cc_set = 1; 584 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) { 585 /* Not new-reno we can't play games with beta! */ 586 failed = 1; 587 goto out; 588 589 } 590 if (CC_ALGO(tp)->ctl_output == NULL) { 591 /* Huh, not using new-reno so no swaps.? */ 592 failed = 2; 593 goto out; 594 } 595 /* Get the current values out */ 596 sopt.sopt_valsize = sizeof(struct cc_newreno_opts); 597 sopt.sopt_dir = SOPT_GET; 598 opt.name = CC_NEWRENO_BETA; 599 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt); 600 if (error) { 601 failed = 3; 602 goto out; 603 } 604 old.beta = opt.val; 605 opt.name = CC_NEWRENO_BETA_ECN; 606 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt); 607 if (error) { 608 failed = 4; 609 goto out; 610 } 611 old.beta_ecn = opt.val; 612 613 /* Now lets set in the values we have stored */ 614 sopt.sopt_dir = SOPT_SET; 615 opt.name = CC_NEWRENO_BETA; 616 opt.val = rack->r_ctl.rc_saved_beta.beta; 617 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt); 618 if (error) { 619 failed = 5; 620 goto out; 621 } 622 opt.name = CC_NEWRENO_BETA_ECN; 623 opt.val = rack->r_ctl.rc_saved_beta.beta_ecn; 624 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt); 625 if (error) { 626 failed = 6; 627 goto out; 628 } 629 /* Save off the values for restoral */ 630 memcpy(&rack->r_ctl.rc_saved_beta, &old, sizeof(struct newreno)); 631 out: 632 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 633 union tcp_log_stackspecific log; 634 struct timeval tv; 635 struct newreno *ptr; 636 637 ptr = ((struct newreno *)tp->t_ccv.cc_data); 638 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 639 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 640 log.u_bbr.flex1 = ptr->beta; 641 log.u_bbr.flex2 = ptr->beta_ecn; 642 log.u_bbr.flex3 = ptr->newreno_flags; 643 log.u_bbr.flex4 = rack->r_ctl.rc_saved_beta.beta; 644 log.u_bbr.flex5 = rack->r_ctl.rc_saved_beta.beta_ecn; 645 log.u_bbr.flex6 = failed; 646 log.u_bbr.flex7 = rack->gp_ready; 647 log.u_bbr.flex7 <<= 1; 648 log.u_bbr.flex7 |= rack->use_fixed_rate; 649 log.u_bbr.flex7 <<= 1; 650 log.u_bbr.flex7 |= rack->rc_pacing_cc_set; 651 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt; 652 log.u_bbr.flex8 = flex8; 653 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, error, 654 0, &log, false, NULL, NULL, 0, &tv); 655 } 656 } 657 658 static void 659 rack_set_cc_pacing(struct tcp_rack *rack) 660 { 661 if (rack->rc_pacing_cc_set) 662 return; 663 /* 664 * Use the swap utility placing in 3 for flex8 to id a 665 * set of a new set of values. 666 */ 667 rack->rc_pacing_cc_set = 1; 668 rack_swap_beta_values(rack, 3); 669 } 670 671 static void 672 rack_undo_cc_pacing(struct tcp_rack *rack) 673 { 674 if (rack->rc_pacing_cc_set == 0) 675 return; 676 /* 677 * Use the swap utility placing in 4 for flex8 to id a 678 * restoral of the old values. 679 */ 680 rack->rc_pacing_cc_set = 0; 681 rack_swap_beta_values(rack, 4); 682 } 683 684 #ifdef NETFLIX_PEAKRATE 685 static inline void 686 rack_update_peakrate_thr(struct tcpcb *tp) 687 { 688 /* Keep in mind that t_maxpeakrate is in B/s. */ 689 uint64_t peak; 690 peak = uqmax((tp->t_maxseg * 2), 691 (((uint64_t)tp->t_maxpeakrate * (uint64_t)(tp->t_srtt)) / (uint64_t)HPTS_USEC_IN_SEC)); 692 tp->t_peakrate_thr = (uint32_t)uqmin(peak, UINT32_MAX); 693 } 694 #endif 695 696 static int 697 sysctl_rack_clear(SYSCTL_HANDLER_ARGS) 698 { 699 uint32_t stat; 700 int32_t error; 701 702 error = SYSCTL_OUT(req, &rack_clear_counter, sizeof(uint32_t)); 703 if (error || req->newptr == NULL) 704 return error; 705 706 error = SYSCTL_IN(req, &stat, sizeof(uint32_t)); 707 if (error) 708 return (error); 709 if (stat == 1) { 710 #ifdef INVARIANTS 711 printf("Clearing RACK counters\n"); 712 #endif 713 counter_u64_zero(rack_tlp_tot); 714 counter_u64_zero(rack_tlp_newdata); 715 counter_u64_zero(rack_tlp_retran); 716 counter_u64_zero(rack_tlp_retran_bytes); 717 counter_u64_zero(rack_to_tot); 718 counter_u64_zero(rack_saw_enobuf); 719 counter_u64_zero(rack_saw_enobuf_hw); 720 counter_u64_zero(rack_saw_enetunreach); 721 counter_u64_zero(rack_persists_sends); 722 counter_u64_zero(rack_persists_acks); 723 counter_u64_zero(rack_persists_loss); 724 counter_u64_zero(rack_persists_lost_ends); 725 #ifdef INVARIANTS 726 counter_u64_zero(rack_adjust_map_bw); 727 #endif 728 counter_u64_zero(rack_to_alloc_hard); 729 counter_u64_zero(rack_to_alloc_emerg); 730 counter_u64_zero(rack_sack_proc_all); 731 counter_u64_zero(rack_fto_send); 732 counter_u64_zero(rack_fto_rsm_send); 733 counter_u64_zero(rack_extended_rfo); 734 counter_u64_zero(rack_hw_pace_init_fail); 735 counter_u64_zero(rack_hw_pace_lost); 736 counter_u64_zero(rack_non_fto_send); 737 counter_u64_zero(rack_nfto_resend); 738 counter_u64_zero(rack_sack_proc_short); 739 counter_u64_zero(rack_sack_proc_restart); 740 counter_u64_zero(rack_to_alloc); 741 counter_u64_zero(rack_to_alloc_limited); 742 counter_u64_zero(rack_alloc_limited_conns); 743 counter_u64_zero(rack_split_limited); 744 counter_u64_zero(rack_multi_single_eq); 745 counter_u64_zero(rack_proc_non_comp_ack); 746 counter_u64_zero(rack_sack_attacks_detected); 747 counter_u64_zero(rack_sack_attacks_reversed); 748 counter_u64_zero(rack_sack_used_next_merge); 749 counter_u64_zero(rack_sack_used_prev_merge); 750 counter_u64_zero(rack_sack_splits); 751 counter_u64_zero(rack_sack_skipped_acked); 752 counter_u64_zero(rack_ack_total); 753 counter_u64_zero(rack_express_sack); 754 counter_u64_zero(rack_sack_total); 755 counter_u64_zero(rack_move_none); 756 counter_u64_zero(rack_move_some); 757 counter_u64_zero(rack_try_scwnd); 758 counter_u64_zero(rack_collapsed_win); 759 counter_u64_zero(rack_collapsed_win_rxt); 760 counter_u64_zero(rack_collapsed_win_seen); 761 counter_u64_zero(rack_collapsed_win_rxt_bytes); 762 } 763 rack_clear_counter = 0; 764 return (0); 765 } 766 767 static void 768 rack_init_sysctls(void) 769 { 770 struct sysctl_oid *rack_counters; 771 struct sysctl_oid *rack_attack; 772 struct sysctl_oid *rack_pacing; 773 struct sysctl_oid *rack_timely; 774 struct sysctl_oid *rack_timers; 775 struct sysctl_oid *rack_tlp; 776 struct sysctl_oid *rack_misc; 777 struct sysctl_oid *rack_features; 778 struct sysctl_oid *rack_measure; 779 struct sysctl_oid *rack_probertt; 780 struct sysctl_oid *rack_hw_pacing; 781 struct sysctl_oid *rack_tracepoint; 782 783 rack_attack = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 784 SYSCTL_CHILDREN(rack_sysctl_root), 785 OID_AUTO, 786 "sack_attack", 787 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 788 "Rack Sack Attack Counters and Controls"); 789 rack_counters = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 790 SYSCTL_CHILDREN(rack_sysctl_root), 791 OID_AUTO, 792 "stats", 793 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 794 "Rack Counters"); 795 SYSCTL_ADD_S32(&rack_sysctl_ctx, 796 SYSCTL_CHILDREN(rack_sysctl_root), 797 OID_AUTO, "rate_sample_method", CTLFLAG_RW, 798 &rack_rate_sample_method , USE_RTT_LOW, 799 "What method should we use for rate sampling 0=high, 1=low "); 800 /* Probe rtt related controls */ 801 rack_probertt = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 802 SYSCTL_CHILDREN(rack_sysctl_root), 803 OID_AUTO, 804 "probertt", 805 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 806 "ProbeRTT related Controls"); 807 SYSCTL_ADD_U16(&rack_sysctl_ctx, 808 SYSCTL_CHILDREN(rack_probertt), 809 OID_AUTO, "exit_per_hpb", CTLFLAG_RW, 810 &rack_atexit_prtt_hbp, 130, 811 "What percentage above goodput do we clamp CA/SS to at exit on high-BDP path 110%"); 812 SYSCTL_ADD_U16(&rack_sysctl_ctx, 813 SYSCTL_CHILDREN(rack_probertt), 814 OID_AUTO, "exit_per_nonhpb", CTLFLAG_RW, 815 &rack_atexit_prtt, 130, 816 "What percentage above goodput do we clamp CA/SS to at exit on a non high-BDP path 100%"); 817 SYSCTL_ADD_U16(&rack_sysctl_ctx, 818 SYSCTL_CHILDREN(rack_probertt), 819 OID_AUTO, "gp_per_mul", CTLFLAG_RW, 820 &rack_per_of_gp_probertt, 60, 821 "What percentage of goodput do we pace at in probertt"); 822 SYSCTL_ADD_U16(&rack_sysctl_ctx, 823 SYSCTL_CHILDREN(rack_probertt), 824 OID_AUTO, "gp_per_reduce", CTLFLAG_RW, 825 &rack_per_of_gp_probertt_reduce, 10, 826 "What percentage of goodput do we reduce every gp_srtt"); 827 SYSCTL_ADD_U16(&rack_sysctl_ctx, 828 SYSCTL_CHILDREN(rack_probertt), 829 OID_AUTO, "gp_per_low", CTLFLAG_RW, 830 &rack_per_of_gp_lowthresh, 40, 831 "What percentage of goodput do we allow the multiplier to fall to"); 832 SYSCTL_ADD_U32(&rack_sysctl_ctx, 833 SYSCTL_CHILDREN(rack_probertt), 834 OID_AUTO, "time_between", CTLFLAG_RW, 835 & rack_time_between_probertt, 96000000, 836 "How many useconds between the lowest rtt falling must past before we enter probertt"); 837 SYSCTL_ADD_U32(&rack_sysctl_ctx, 838 SYSCTL_CHILDREN(rack_probertt), 839 OID_AUTO, "safety", CTLFLAG_RW, 840 &rack_probe_rtt_safety_val, 2000000, 841 "If not zero, provides a maximum usecond that you can stay in probertt (2sec = 2000000)"); 842 SYSCTL_ADD_U32(&rack_sysctl_ctx, 843 SYSCTL_CHILDREN(rack_probertt), 844 OID_AUTO, "sets_cwnd", CTLFLAG_RW, 845 &rack_probe_rtt_sets_cwnd, 0, 846 "Do we set the cwnd too (if always_lower is on)"); 847 SYSCTL_ADD_U32(&rack_sysctl_ctx, 848 SYSCTL_CHILDREN(rack_probertt), 849 OID_AUTO, "maxdrainsrtts", CTLFLAG_RW, 850 &rack_max_drain_wait, 2, 851 "Maximum number of gp_srtt's to hold in drain waiting for flight to reach goal"); 852 SYSCTL_ADD_U32(&rack_sysctl_ctx, 853 SYSCTL_CHILDREN(rack_probertt), 854 OID_AUTO, "mustdrainsrtts", CTLFLAG_RW, 855 &rack_must_drain, 1, 856 "We must drain this many gp_srtt's waiting for flight to reach goal"); 857 SYSCTL_ADD_U32(&rack_sysctl_ctx, 858 SYSCTL_CHILDREN(rack_probertt), 859 OID_AUTO, "goal_use_min_entry", CTLFLAG_RW, 860 &rack_probertt_use_min_rtt_entry, 1, 861 "Should we use the min-rtt to calculate the goal rtt (else gp_srtt) at entry"); 862 SYSCTL_ADD_U32(&rack_sysctl_ctx, 863 SYSCTL_CHILDREN(rack_probertt), 864 OID_AUTO, "goal_use_min_exit", CTLFLAG_RW, 865 &rack_probertt_use_min_rtt_exit, 0, 866 "How to set cwnd at exit, 0 - dynamic, 1 - use min-rtt, 2 - use curgprtt, 3 - entry gp-rtt"); 867 SYSCTL_ADD_U32(&rack_sysctl_ctx, 868 SYSCTL_CHILDREN(rack_probertt), 869 OID_AUTO, "length_div", CTLFLAG_RW, 870 &rack_probertt_gpsrtt_cnt_div, 0, 871 "How many recent goodput srtt periods plus hold tim does probertt last (bottom of fraction)"); 872 SYSCTL_ADD_U32(&rack_sysctl_ctx, 873 SYSCTL_CHILDREN(rack_probertt), 874 OID_AUTO, "length_mul", CTLFLAG_RW, 875 &rack_probertt_gpsrtt_cnt_mul, 0, 876 "How many recent goodput srtt periods plus hold tim does probertt last (top of fraction)"); 877 SYSCTL_ADD_U32(&rack_sysctl_ctx, 878 SYSCTL_CHILDREN(rack_probertt), 879 OID_AUTO, "holdtim_at_target", CTLFLAG_RW, 880 &rack_min_probertt_hold, 200000, 881 "What is the minimum time we hold probertt at target"); 882 SYSCTL_ADD_U32(&rack_sysctl_ctx, 883 SYSCTL_CHILDREN(rack_probertt), 884 OID_AUTO, "filter_life", CTLFLAG_RW, 885 &rack_probertt_filter_life, 10000000, 886 "What is the time for the filters life in useconds"); 887 SYSCTL_ADD_U32(&rack_sysctl_ctx, 888 SYSCTL_CHILDREN(rack_probertt), 889 OID_AUTO, "lower_within", CTLFLAG_RW, 890 &rack_probertt_lower_within, 10, 891 "If the rtt goes lower within this percentage of the time, go into probe-rtt"); 892 SYSCTL_ADD_U32(&rack_sysctl_ctx, 893 SYSCTL_CHILDREN(rack_probertt), 894 OID_AUTO, "must_move", CTLFLAG_RW, 895 &rack_min_rtt_movement, 250, 896 "How much is the minimum movement in rtt to count as a drop for probertt purposes"); 897 SYSCTL_ADD_U32(&rack_sysctl_ctx, 898 SYSCTL_CHILDREN(rack_probertt), 899 OID_AUTO, "clear_is_cnts", CTLFLAG_RW, 900 &rack_probertt_clear_is, 1, 901 "Do we clear I/S counts on exiting probe-rtt"); 902 SYSCTL_ADD_S32(&rack_sysctl_ctx, 903 SYSCTL_CHILDREN(rack_probertt), 904 OID_AUTO, "hbp_extra_drain", CTLFLAG_RW, 905 &rack_max_drain_hbp, 1, 906 "How many extra drain gpsrtt's do we get in highly buffered paths"); 907 SYSCTL_ADD_S32(&rack_sysctl_ctx, 908 SYSCTL_CHILDREN(rack_probertt), 909 OID_AUTO, "hbp_threshold", CTLFLAG_RW, 910 &rack_hbp_thresh, 3, 911 "We are highly buffered if min_rtt_seen / max_rtt_seen > this-threshold"); 912 913 rack_tracepoint = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 914 SYSCTL_CHILDREN(rack_sysctl_root), 915 OID_AUTO, 916 "tp", 917 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 918 "Rack tracepoint facility"); 919 SYSCTL_ADD_U32(&rack_sysctl_ctx, 920 SYSCTL_CHILDREN(rack_tracepoint), 921 OID_AUTO, "number", CTLFLAG_RW, 922 &rack_trace_point_config, 0, 923 "What is the trace point number to activate (0=none, 0xffffffff = all)?"); 924 SYSCTL_ADD_U32(&rack_sysctl_ctx, 925 SYSCTL_CHILDREN(rack_tracepoint), 926 OID_AUTO, "bbmode", CTLFLAG_RW, 927 &rack_trace_point_bb_mode, 4, 928 "What is BB logging mode that is activated?"); 929 SYSCTL_ADD_S32(&rack_sysctl_ctx, 930 SYSCTL_CHILDREN(rack_tracepoint), 931 OID_AUTO, "count", CTLFLAG_RW, 932 &rack_trace_point_count, 0, 933 "How many connections will have BB logging turned on that hit the tracepoint?"); 934 /* Pacing related sysctls */ 935 rack_pacing = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 936 SYSCTL_CHILDREN(rack_sysctl_root), 937 OID_AUTO, 938 "pacing", 939 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 940 "Pacing related Controls"); 941 SYSCTL_ADD_S32(&rack_sysctl_ctx, 942 SYSCTL_CHILDREN(rack_pacing), 943 OID_AUTO, "max_pace_over", CTLFLAG_RW, 944 &rack_max_per_above, 30, 945 "What is the maximum allowable percentage that we can pace above (so 30 = 130% of our goal)"); 946 SYSCTL_ADD_S32(&rack_sysctl_ctx, 947 SYSCTL_CHILDREN(rack_pacing), 948 OID_AUTO, "pace_to_one", CTLFLAG_RW, 949 &rack_pace_one_seg, 0, 950 "Do we allow low b/w pacing of 1MSS instead of two"); 951 SYSCTL_ADD_S32(&rack_sysctl_ctx, 952 SYSCTL_CHILDREN(rack_pacing), 953 OID_AUTO, "limit_wsrtt", CTLFLAG_RW, 954 &rack_limit_time_with_srtt, 0, 955 "Do we limit pacing time based on srtt"); 956 SYSCTL_ADD_S32(&rack_sysctl_ctx, 957 SYSCTL_CHILDREN(rack_pacing), 958 OID_AUTO, "init_win", CTLFLAG_RW, 959 &rack_default_init_window, 0, 960 "Do we have a rack initial window 0 = system default"); 961 SYSCTL_ADD_U16(&rack_sysctl_ctx, 962 SYSCTL_CHILDREN(rack_pacing), 963 OID_AUTO, "gp_per_ss", CTLFLAG_RW, 964 &rack_per_of_gp_ss, 250, 965 "If non zero, what percentage of goodput to pace at in slow start"); 966 SYSCTL_ADD_U16(&rack_sysctl_ctx, 967 SYSCTL_CHILDREN(rack_pacing), 968 OID_AUTO, "gp_per_ca", CTLFLAG_RW, 969 &rack_per_of_gp_ca, 150, 970 "If non zero, what percentage of goodput to pace at in congestion avoidance"); 971 SYSCTL_ADD_U16(&rack_sysctl_ctx, 972 SYSCTL_CHILDREN(rack_pacing), 973 OID_AUTO, "gp_per_rec", CTLFLAG_RW, 974 &rack_per_of_gp_rec, 200, 975 "If non zero, what percentage of goodput to pace at in recovery"); 976 SYSCTL_ADD_S32(&rack_sysctl_ctx, 977 SYSCTL_CHILDREN(rack_pacing), 978 OID_AUTO, "pace_max_seg", CTLFLAG_RW, 979 &rack_hptsi_segments, 40, 980 "What size is the max for TSO segments in pacing and burst mitigation"); 981 SYSCTL_ADD_S32(&rack_sysctl_ctx, 982 SYSCTL_CHILDREN(rack_pacing), 983 OID_AUTO, "burst_reduces", CTLFLAG_RW, 984 &rack_slot_reduction, 4, 985 "When doing only burst mitigation what is the reduce divisor"); 986 SYSCTL_ADD_S32(&rack_sysctl_ctx, 987 SYSCTL_CHILDREN(rack_sysctl_root), 988 OID_AUTO, "use_pacing", CTLFLAG_RW, 989 &rack_pace_every_seg, 0, 990 "If set we use pacing, if clear we use only the original burst mitigation"); 991 SYSCTL_ADD_U64(&rack_sysctl_ctx, 992 SYSCTL_CHILDREN(rack_pacing), 993 OID_AUTO, "rate_cap", CTLFLAG_RW, 994 &rack_bw_rate_cap, 0, 995 "If set we apply this value to the absolute rate cap used by pacing"); 996 SYSCTL_ADD_U8(&rack_sysctl_ctx, 997 SYSCTL_CHILDREN(rack_sysctl_root), 998 OID_AUTO, "req_measure_cnt", CTLFLAG_RW, 999 &rack_req_measurements, 1, 1000 "If doing dynamic pacing, how many measurements must be in before we start pacing?"); 1001 /* Hardware pacing */ 1002 rack_hw_pacing = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1003 SYSCTL_CHILDREN(rack_sysctl_root), 1004 OID_AUTO, 1005 "hdwr_pacing", 1006 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1007 "Pacing related Controls"); 1008 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1009 SYSCTL_CHILDREN(rack_hw_pacing), 1010 OID_AUTO, "rwnd_factor", CTLFLAG_RW, 1011 &rack_hw_rwnd_factor, 2, 1012 "How many times does snd_wnd need to be bigger than pace_max_seg so we will hold off and get more acks?"); 1013 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1014 SYSCTL_CHILDREN(rack_hw_pacing), 1015 OID_AUTO, "pace_enobuf_mult", CTLFLAG_RW, 1016 &rack_enobuf_hw_boost_mult, 2, 1017 "By how many time_betweens should we boost the pacing time if we see a ENOBUFS?"); 1018 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1019 SYSCTL_CHILDREN(rack_hw_pacing), 1020 OID_AUTO, "pace_enobuf_max", CTLFLAG_RW, 1021 &rack_enobuf_hw_max, 2, 1022 "What is the max boost the pacing time if we see a ENOBUFS?"); 1023 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1024 SYSCTL_CHILDREN(rack_hw_pacing), 1025 OID_AUTO, "pace_enobuf_min", CTLFLAG_RW, 1026 &rack_enobuf_hw_min, 2, 1027 "What is the min boost the pacing time if we see a ENOBUFS?"); 1028 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1029 SYSCTL_CHILDREN(rack_hw_pacing), 1030 OID_AUTO, "enable", CTLFLAG_RW, 1031 &rack_enable_hw_pacing, 0, 1032 "Should RACK attempt to use hw pacing?"); 1033 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1034 SYSCTL_CHILDREN(rack_hw_pacing), 1035 OID_AUTO, "rate_cap", CTLFLAG_RW, 1036 &rack_hw_rate_caps, 1, 1037 "Does the highest hardware pacing rate cap the rate we will send at??"); 1038 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1039 SYSCTL_CHILDREN(rack_hw_pacing), 1040 OID_AUTO, "rate_min", CTLFLAG_RW, 1041 &rack_hw_rate_min, 0, 1042 "Do we need a minimum estimate of this many bytes per second in order to engage hw pacing?"); 1043 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1044 SYSCTL_CHILDREN(rack_hw_pacing), 1045 OID_AUTO, "rate_to_low", CTLFLAG_RW, 1046 &rack_hw_rate_to_low, 0, 1047 "If we fall below this rate, dis-engage hw pacing?"); 1048 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1049 SYSCTL_CHILDREN(rack_hw_pacing), 1050 OID_AUTO, "up_only", CTLFLAG_RW, 1051 &rack_hw_up_only, 1, 1052 "Do we allow hw pacing to lower the rate selected?"); 1053 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1054 SYSCTL_CHILDREN(rack_hw_pacing), 1055 OID_AUTO, "extra_mss_precise", CTLFLAG_RW, 1056 &rack_hw_pace_extra_slots, 2, 1057 "If the rates between software and hardware match precisely how many extra time_betweens do we get?"); 1058 rack_timely = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1059 SYSCTL_CHILDREN(rack_sysctl_root), 1060 OID_AUTO, 1061 "timely", 1062 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1063 "Rack Timely RTT Controls"); 1064 /* Timely based GP dynmics */ 1065 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1066 SYSCTL_CHILDREN(rack_timely), 1067 OID_AUTO, "upper", CTLFLAG_RW, 1068 &rack_gp_per_bw_mul_up, 2, 1069 "Rack timely upper range for equal b/w (in percentage)"); 1070 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1071 SYSCTL_CHILDREN(rack_timely), 1072 OID_AUTO, "lower", CTLFLAG_RW, 1073 &rack_gp_per_bw_mul_down, 4, 1074 "Rack timely lower range for equal b/w (in percentage)"); 1075 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1076 SYSCTL_CHILDREN(rack_timely), 1077 OID_AUTO, "rtt_max_mul", CTLFLAG_RW, 1078 &rack_gp_rtt_maxmul, 3, 1079 "Rack timely multiplier of lowest rtt for rtt_max"); 1080 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1081 SYSCTL_CHILDREN(rack_timely), 1082 OID_AUTO, "rtt_min_div", CTLFLAG_RW, 1083 &rack_gp_rtt_mindiv, 4, 1084 "Rack timely divisor used for rtt + (rtt * mul/divisor) for check for lower rtt"); 1085 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1086 SYSCTL_CHILDREN(rack_timely), 1087 OID_AUTO, "rtt_min_mul", CTLFLAG_RW, 1088 &rack_gp_rtt_minmul, 1, 1089 "Rack timely multiplier used for rtt + (rtt * mul/divisor) for check for lower rtt"); 1090 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1091 SYSCTL_CHILDREN(rack_timely), 1092 OID_AUTO, "decrease", CTLFLAG_RW, 1093 &rack_gp_decrease_per, 20, 1094 "Rack timely decrease percentage of our GP multiplication factor"); 1095 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1096 SYSCTL_CHILDREN(rack_timely), 1097 OID_AUTO, "increase", CTLFLAG_RW, 1098 &rack_gp_increase_per, 2, 1099 "Rack timely increase perentage of our GP multiplication factor"); 1100 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1101 SYSCTL_CHILDREN(rack_timely), 1102 OID_AUTO, "lowerbound", CTLFLAG_RW, 1103 &rack_per_lower_bound, 50, 1104 "Rack timely lowest percentage we allow GP multiplier to fall to"); 1105 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1106 SYSCTL_CHILDREN(rack_timely), 1107 OID_AUTO, "upperboundss", CTLFLAG_RW, 1108 &rack_per_upper_bound_ss, 0, 1109 "Rack timely highest percentage we allow GP multiplier in SS to raise to (0 is no upperbound)"); 1110 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1111 SYSCTL_CHILDREN(rack_timely), 1112 OID_AUTO, "upperboundca", CTLFLAG_RW, 1113 &rack_per_upper_bound_ca, 0, 1114 "Rack timely highest percentage we allow GP multiplier to CA raise to (0 is no upperbound)"); 1115 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1116 SYSCTL_CHILDREN(rack_timely), 1117 OID_AUTO, "dynamicgp", CTLFLAG_RW, 1118 &rack_do_dyn_mul, 0, 1119 "Rack timely do we enable dynmaic timely goodput by default"); 1120 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1121 SYSCTL_CHILDREN(rack_timely), 1122 OID_AUTO, "no_rec_red", CTLFLAG_RW, 1123 &rack_gp_no_rec_chg, 1, 1124 "Rack timely do we prohibit the recovery multiplier from being lowered"); 1125 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1126 SYSCTL_CHILDREN(rack_timely), 1127 OID_AUTO, "red_clear_cnt", CTLFLAG_RW, 1128 &rack_timely_dec_clear, 6, 1129 "Rack timely what threshold do we count to before another boost during b/w decent"); 1130 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1131 SYSCTL_CHILDREN(rack_timely), 1132 OID_AUTO, "max_push_rise", CTLFLAG_RW, 1133 &rack_timely_max_push_rise, 3, 1134 "Rack timely how many times do we push up with b/w increase"); 1135 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1136 SYSCTL_CHILDREN(rack_timely), 1137 OID_AUTO, "max_push_drop", CTLFLAG_RW, 1138 &rack_timely_max_push_drop, 3, 1139 "Rack timely how many times do we push back on b/w decent"); 1140 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1141 SYSCTL_CHILDREN(rack_timely), 1142 OID_AUTO, "min_segs", CTLFLAG_RW, 1143 &rack_timely_min_segs, 4, 1144 "Rack timely when setting the cwnd what is the min num segments"); 1145 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1146 SYSCTL_CHILDREN(rack_timely), 1147 OID_AUTO, "noback_max", CTLFLAG_RW, 1148 &rack_use_max_for_nobackoff, 0, 1149 "Rack timely when deciding if to backoff on a loss, do we use under max rtt else min"); 1150 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1151 SYSCTL_CHILDREN(rack_timely), 1152 OID_AUTO, "interim_timely_only", CTLFLAG_RW, 1153 &rack_timely_int_timely_only, 0, 1154 "Rack timely when doing interim timely's do we only do timely (no b/w consideration)"); 1155 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1156 SYSCTL_CHILDREN(rack_timely), 1157 OID_AUTO, "nonstop", CTLFLAG_RW, 1158 &rack_timely_no_stopping, 0, 1159 "Rack timely don't stop increase"); 1160 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1161 SYSCTL_CHILDREN(rack_timely), 1162 OID_AUTO, "dec_raise_thresh", CTLFLAG_RW, 1163 &rack_down_raise_thresh, 100, 1164 "If the CA or SS is below this threshold raise on the first 3 b/w lowers (0=always)"); 1165 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1166 SYSCTL_CHILDREN(rack_timely), 1167 OID_AUTO, "bottom_drag_segs", CTLFLAG_RW, 1168 &rack_req_segs, 1, 1169 "Bottom dragging if not these many segments outstanding and room"); 1170 1171 /* TLP and Rack related parameters */ 1172 rack_tlp = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1173 SYSCTL_CHILDREN(rack_sysctl_root), 1174 OID_AUTO, 1175 "tlp", 1176 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1177 "TLP and Rack related Controls"); 1178 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1179 SYSCTL_CHILDREN(rack_tlp), 1180 OID_AUTO, "use_rrr", CTLFLAG_RW, 1181 &use_rack_rr, 1, 1182 "Do we use Rack Rapid Recovery"); 1183 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1184 SYSCTL_CHILDREN(rack_tlp), 1185 OID_AUTO, "post_rec_labc", CTLFLAG_RW, 1186 &rack_max_abc_post_recovery, 2, 1187 "Since we do early recovery, do we override the l_abc to a value, if so what?"); 1188 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1189 SYSCTL_CHILDREN(rack_tlp), 1190 OID_AUTO, "nonrxt_use_cr", CTLFLAG_RW, 1191 &rack_non_rxt_use_cr, 0, 1192 "Do we use ss/ca rate if in recovery we are transmitting a new data chunk"); 1193 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1194 SYSCTL_CHILDREN(rack_tlp), 1195 OID_AUTO, "tlpmethod", CTLFLAG_RW, 1196 &rack_tlp_threshold_use, TLP_USE_TWO_ONE, 1197 "What method do we do for TLP time calc 0=no-de-ack-comp, 1=ID, 2=2.1, 3=2.2"); 1198 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1199 SYSCTL_CHILDREN(rack_tlp), 1200 OID_AUTO, "limit", CTLFLAG_RW, 1201 &rack_tlp_limit, 2, 1202 "How many TLP's can be sent without sending new data"); 1203 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1204 SYSCTL_CHILDREN(rack_tlp), 1205 OID_AUTO, "use_greater", CTLFLAG_RW, 1206 &rack_tlp_use_greater, 1, 1207 "Should we use the rack_rtt time if its greater than srtt"); 1208 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1209 SYSCTL_CHILDREN(rack_tlp), 1210 OID_AUTO, "tlpminto", CTLFLAG_RW, 1211 &rack_tlp_min, 10000, 1212 "TLP minimum timeout per the specification (in microseconds)"); 1213 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1214 SYSCTL_CHILDREN(rack_tlp), 1215 OID_AUTO, "send_oldest", CTLFLAG_RW, 1216 &rack_always_send_oldest, 0, 1217 "Should we always send the oldest TLP and RACK-TLP"); 1218 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1219 SYSCTL_CHILDREN(rack_tlp), 1220 OID_AUTO, "rack_tlimit", CTLFLAG_RW, 1221 &rack_limited_retran, 0, 1222 "How many times can a rack timeout drive out sends"); 1223 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1224 SYSCTL_CHILDREN(rack_tlp), 1225 OID_AUTO, "tlp_cwnd_flag", CTLFLAG_RW, 1226 &rack_lower_cwnd_at_tlp, 0, 1227 "When a TLP completes a retran should we enter recovery"); 1228 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1229 SYSCTL_CHILDREN(rack_tlp), 1230 OID_AUTO, "reorder_thresh", CTLFLAG_RW, 1231 &rack_reorder_thresh, 2, 1232 "What factor for rack will be added when seeing reordering (shift right)"); 1233 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1234 SYSCTL_CHILDREN(rack_tlp), 1235 OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW, 1236 &rack_tlp_thresh, 1, 1237 "What divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)"); 1238 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1239 SYSCTL_CHILDREN(rack_tlp), 1240 OID_AUTO, "reorder_fade", CTLFLAG_RW, 1241 &rack_reorder_fade, 60000000, 1242 "Does reorder detection fade, if so how many microseconds (0 means never)"); 1243 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1244 SYSCTL_CHILDREN(rack_tlp), 1245 OID_AUTO, "pktdelay", CTLFLAG_RW, 1246 &rack_pkt_delay, 1000, 1247 "Extra RACK time (in microseconds) besides reordering thresh"); 1248 1249 /* Timer related controls */ 1250 rack_timers = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1251 SYSCTL_CHILDREN(rack_sysctl_root), 1252 OID_AUTO, 1253 "timers", 1254 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1255 "Timer related controls"); 1256 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1257 SYSCTL_CHILDREN(rack_timers), 1258 OID_AUTO, "persmin", CTLFLAG_RW, 1259 &rack_persist_min, 250000, 1260 "What is the minimum time in microseconds between persists"); 1261 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1262 SYSCTL_CHILDREN(rack_timers), 1263 OID_AUTO, "persmax", CTLFLAG_RW, 1264 &rack_persist_max, 2000000, 1265 "What is the largest delay in microseconds between persists"); 1266 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1267 SYSCTL_CHILDREN(rack_timers), 1268 OID_AUTO, "delayed_ack", CTLFLAG_RW, 1269 &rack_delayed_ack_time, 40000, 1270 "Delayed ack time (40ms in microseconds)"); 1271 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1272 SYSCTL_CHILDREN(rack_timers), 1273 OID_AUTO, "minrto", CTLFLAG_RW, 1274 &rack_rto_min, 30000, 1275 "Minimum RTO in microseconds -- set with caution below 1000 due to TLP"); 1276 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1277 SYSCTL_CHILDREN(rack_timers), 1278 OID_AUTO, "maxrto", CTLFLAG_RW, 1279 &rack_rto_max, 4000000, 1280 "Maximum RTO in microseconds -- should be at least as large as min_rto"); 1281 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1282 SYSCTL_CHILDREN(rack_timers), 1283 OID_AUTO, "minto", CTLFLAG_RW, 1284 &rack_min_to, 1000, 1285 "Minimum rack timeout in microseconds"); 1286 /* Measure controls */ 1287 rack_measure = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1288 SYSCTL_CHILDREN(rack_sysctl_root), 1289 OID_AUTO, 1290 "measure", 1291 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1292 "Measure related controls"); 1293 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1294 SYSCTL_CHILDREN(rack_measure), 1295 OID_AUTO, "wma_divisor", CTLFLAG_RW, 1296 &rack_wma_divisor, 8, 1297 "When doing b/w calculation what is the divisor for the WMA"); 1298 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1299 SYSCTL_CHILDREN(rack_measure), 1300 OID_AUTO, "end_cwnd", CTLFLAG_RW, 1301 &rack_cwnd_block_ends_measure, 0, 1302 "Does a cwnd just-return end the measurement window (app limited)"); 1303 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1304 SYSCTL_CHILDREN(rack_measure), 1305 OID_AUTO, "end_rwnd", CTLFLAG_RW, 1306 &rack_rwnd_block_ends_measure, 0, 1307 "Does an rwnd just-return end the measurement window (app limited -- not persists)"); 1308 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1309 SYSCTL_CHILDREN(rack_measure), 1310 OID_AUTO, "min_target", CTLFLAG_RW, 1311 &rack_def_data_window, 20, 1312 "What is the minimum target window (in mss) for a GP measurements"); 1313 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1314 SYSCTL_CHILDREN(rack_measure), 1315 OID_AUTO, "goal_bdp", CTLFLAG_RW, 1316 &rack_goal_bdp, 2, 1317 "What is the goal BDP to measure"); 1318 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1319 SYSCTL_CHILDREN(rack_measure), 1320 OID_AUTO, "min_srtts", CTLFLAG_RW, 1321 &rack_min_srtts, 1, 1322 "What is the goal BDP to measure"); 1323 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1324 SYSCTL_CHILDREN(rack_measure), 1325 OID_AUTO, "min_measure_tim", CTLFLAG_RW, 1326 &rack_min_measure_usec, 0, 1327 "What is the Minimum time time for a measurement if 0, this is off"); 1328 /* Features */ 1329 rack_features = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1330 SYSCTL_CHILDREN(rack_sysctl_root), 1331 OID_AUTO, 1332 "features", 1333 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1334 "Feature controls"); 1335 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1336 SYSCTL_CHILDREN(rack_features), 1337 OID_AUTO, "cmpack", CTLFLAG_RW, 1338 &rack_use_cmp_acks, 1, 1339 "Should RACK have LRO send compressed acks"); 1340 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1341 SYSCTL_CHILDREN(rack_features), 1342 OID_AUTO, "fsb", CTLFLAG_RW, 1343 &rack_use_fsb, 1, 1344 "Should RACK use the fast send block?"); 1345 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1346 SYSCTL_CHILDREN(rack_features), 1347 OID_AUTO, "rfo", CTLFLAG_RW, 1348 &rack_use_rfo, 1, 1349 "Should RACK use rack_fast_output()?"); 1350 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1351 SYSCTL_CHILDREN(rack_features), 1352 OID_AUTO, "rsmrfo", CTLFLAG_RW, 1353 &rack_use_rsm_rfo, 1, 1354 "Should RACK use rack_fast_rsm_output()?"); 1355 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1356 SYSCTL_CHILDREN(rack_features), 1357 OID_AUTO, "non_paced_lro_queue", CTLFLAG_RW, 1358 &rack_enable_mqueue_for_nonpaced, 0, 1359 "Should RACK use mbuf queuing for non-paced connections"); 1360 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1361 SYSCTL_CHILDREN(rack_features), 1362 OID_AUTO, "hystartplusplus", CTLFLAG_RW, 1363 &rack_do_hystart, 0, 1364 "Should RACK enable HyStart++ on connections?"); 1365 /* Misc rack controls */ 1366 rack_misc = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 1367 SYSCTL_CHILDREN(rack_sysctl_root), 1368 OID_AUTO, 1369 "misc", 1370 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 1371 "Misc related controls"); 1372 #ifdef TCP_ACCOUNTING 1373 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1374 SYSCTL_CHILDREN(rack_misc), 1375 OID_AUTO, "tcp_acct", CTLFLAG_RW, 1376 &rack_tcp_accounting, 0, 1377 "Should we turn on TCP accounting for all rack sessions?"); 1378 #endif 1379 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1380 SYSCTL_CHILDREN(rack_misc), 1381 OID_AUTO, "apply_rtt_with_low_conf", CTLFLAG_RW, 1382 &rack_apply_rtt_with_reduced_conf, 0, 1383 "When a persist or keep-alive probe is not answered do we calculate rtt on subsequent answers?"); 1384 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1385 SYSCTL_CHILDREN(rack_misc), 1386 OID_AUTO, "rack_dsack_ctl", CTLFLAG_RW, 1387 &rack_dsack_std_based, 3, 1388 "How do we process dsack with respect to rack timers, bit field, 3 is standards based?"); 1389 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1390 SYSCTL_CHILDREN(rack_misc), 1391 OID_AUTO, "prr_addback_max", CTLFLAG_RW, 1392 &rack_prr_addbackmax, 2, 1393 "What is the maximum number of MSS we allow to be added back if prr can't send all its data?"); 1394 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1395 SYSCTL_CHILDREN(rack_misc), 1396 OID_AUTO, "stats_gets_ms", CTLFLAG_RW, 1397 &rack_stats_gets_ms_rtt, 1, 1398 "What do we feed the stats framework (1 = ms_rtt, 0 = us_rtt, 2 = ms_rtt from hdwr, > 2 usec rtt from hdwr)?"); 1399 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1400 SYSCTL_CHILDREN(rack_misc), 1401 OID_AUTO, "clientlowbuf", CTLFLAG_RW, 1402 &rack_client_low_buf, 0, 1403 "Client low buffer level (below this we are more aggressive in DGP exiting recovery (0 = off)?"); 1404 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1405 SYSCTL_CHILDREN(rack_misc), 1406 OID_AUTO, "defprofile", CTLFLAG_RW, 1407 &rack_def_profile, 0, 1408 "Should RACK use a default profile (0=no, num == profile num)?"); 1409 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1410 SYSCTL_CHILDREN(rack_misc), 1411 OID_AUTO, "shared_cwnd", CTLFLAG_RW, 1412 &rack_enable_shared_cwnd, 1, 1413 "Should RACK try to use the shared cwnd on connections where allowed"); 1414 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1415 SYSCTL_CHILDREN(rack_misc), 1416 OID_AUTO, "limits_on_scwnd", CTLFLAG_RW, 1417 &rack_limits_scwnd, 1, 1418 "Should RACK place low end time limits on the shared cwnd feature"); 1419 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1420 SYSCTL_CHILDREN(rack_misc), 1421 OID_AUTO, "iMac_dack", CTLFLAG_RW, 1422 &rack_use_imac_dack, 0, 1423 "Should RACK try to emulate iMac delayed ack"); 1424 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1425 SYSCTL_CHILDREN(rack_misc), 1426 OID_AUTO, "no_prr", CTLFLAG_RW, 1427 &rack_disable_prr, 0, 1428 "Should RACK not use prr and only pace (must have pacing on)"); 1429 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1430 SYSCTL_CHILDREN(rack_misc), 1431 OID_AUTO, "bb_verbose", CTLFLAG_RW, 1432 &rack_verbose_logging, 0, 1433 "Should RACK black box logging be verbose"); 1434 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1435 SYSCTL_CHILDREN(rack_misc), 1436 OID_AUTO, "data_after_close", CTLFLAG_RW, 1437 &rack_ignore_data_after_close, 1, 1438 "Do we hold off sending a RST until all pending data is ack'd"); 1439 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1440 SYSCTL_CHILDREN(rack_misc), 1441 OID_AUTO, "no_sack_needed", CTLFLAG_RW, 1442 &rack_sack_not_required, 1, 1443 "Do we allow rack to run on connections not supporting SACK"); 1444 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1445 SYSCTL_CHILDREN(rack_misc), 1446 OID_AUTO, "prr_sendalot", CTLFLAG_RW, 1447 &rack_send_a_lot_in_prr, 1, 1448 "Send a lot in prr"); 1449 SYSCTL_ADD_S32(&rack_sysctl_ctx, 1450 SYSCTL_CHILDREN(rack_misc), 1451 OID_AUTO, "autoscale", CTLFLAG_RW, 1452 &rack_autosndbuf_inc, 20, 1453 "What percentage should rack scale up its snd buffer by?"); 1454 /* Sack Attacker detection stuff */ 1455 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1456 SYSCTL_CHILDREN(rack_attack), 1457 OID_AUTO, "detect_highsackratio", CTLFLAG_RW, 1458 &rack_highest_sack_thresh_seen, 0, 1459 "Highest sack to ack ratio seen"); 1460 SYSCTL_ADD_U32(&rack_sysctl_ctx, 1461 SYSCTL_CHILDREN(rack_attack), 1462 OID_AUTO, "detect_highmoveratio", CTLFLAG_RW, 1463 &rack_highest_move_thresh_seen, 0, 1464 "Highest move to non-move ratio seen"); 1465 rack_ack_total = counter_u64_alloc(M_WAITOK); 1466 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1467 SYSCTL_CHILDREN(rack_attack), 1468 OID_AUTO, "acktotal", CTLFLAG_RD, 1469 &rack_ack_total, 1470 "Total number of Ack's"); 1471 rack_express_sack = counter_u64_alloc(M_WAITOK); 1472 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1473 SYSCTL_CHILDREN(rack_attack), 1474 OID_AUTO, "exp_sacktotal", CTLFLAG_RD, 1475 &rack_express_sack, 1476 "Total expresss number of Sack's"); 1477 rack_sack_total = counter_u64_alloc(M_WAITOK); 1478 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1479 SYSCTL_CHILDREN(rack_attack), 1480 OID_AUTO, "sacktotal", CTLFLAG_RD, 1481 &rack_sack_total, 1482 "Total number of SACKs"); 1483 rack_move_none = counter_u64_alloc(M_WAITOK); 1484 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1485 SYSCTL_CHILDREN(rack_attack), 1486 OID_AUTO, "move_none", CTLFLAG_RD, 1487 &rack_move_none, 1488 "Total number of SACK index reuse of positions under threshold"); 1489 rack_move_some = counter_u64_alloc(M_WAITOK); 1490 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1491 SYSCTL_CHILDREN(rack_attack), 1492 OID_AUTO, "move_some", CTLFLAG_RD, 1493 &rack_move_some, 1494 "Total number of SACK index reuse of positions over threshold"); 1495 rack_sack_attacks_detected = counter_u64_alloc(M_WAITOK); 1496 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1497 SYSCTL_CHILDREN(rack_attack), 1498 OID_AUTO, "attacks", CTLFLAG_RD, 1499 &rack_sack_attacks_detected, 1500 "Total number of SACK attackers that had sack disabled"); 1501 rack_sack_attacks_reversed = counter_u64_alloc(M_WAITOK); 1502 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1503 SYSCTL_CHILDREN(rack_attack), 1504 OID_AUTO, "reversed", CTLFLAG_RD, 1505 &rack_sack_attacks_reversed, 1506 "Total number of SACK attackers that were later determined false positive"); 1507 rack_sack_used_next_merge = counter_u64_alloc(M_WAITOK); 1508 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1509 SYSCTL_CHILDREN(rack_attack), 1510 OID_AUTO, "nextmerge", CTLFLAG_RD, 1511 &rack_sack_used_next_merge, 1512 "Total number of times we used the next merge"); 1513 rack_sack_used_prev_merge = counter_u64_alloc(M_WAITOK); 1514 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1515 SYSCTL_CHILDREN(rack_attack), 1516 OID_AUTO, "prevmerge", CTLFLAG_RD, 1517 &rack_sack_used_prev_merge, 1518 "Total number of times we used the prev merge"); 1519 /* Counters */ 1520 rack_fto_send = counter_u64_alloc(M_WAITOK); 1521 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1522 SYSCTL_CHILDREN(rack_counters), 1523 OID_AUTO, "fto_send", CTLFLAG_RD, 1524 &rack_fto_send, "Total number of rack_fast_output sends"); 1525 rack_fto_rsm_send = counter_u64_alloc(M_WAITOK); 1526 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1527 SYSCTL_CHILDREN(rack_counters), 1528 OID_AUTO, "fto_rsm_send", CTLFLAG_RD, 1529 &rack_fto_rsm_send, "Total number of rack_fast_rsm_output sends"); 1530 rack_nfto_resend = counter_u64_alloc(M_WAITOK); 1531 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1532 SYSCTL_CHILDREN(rack_counters), 1533 OID_AUTO, "nfto_resend", CTLFLAG_RD, 1534 &rack_nfto_resend, "Total number of rack_output retransmissions"); 1535 rack_non_fto_send = counter_u64_alloc(M_WAITOK); 1536 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1537 SYSCTL_CHILDREN(rack_counters), 1538 OID_AUTO, "nfto_send", CTLFLAG_RD, 1539 &rack_non_fto_send, "Total number of rack_output first sends"); 1540 rack_extended_rfo = counter_u64_alloc(M_WAITOK); 1541 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1542 SYSCTL_CHILDREN(rack_counters), 1543 OID_AUTO, "rfo_extended", CTLFLAG_RD, 1544 &rack_extended_rfo, "Total number of times we extended rfo"); 1545 1546 rack_hw_pace_init_fail = counter_u64_alloc(M_WAITOK); 1547 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1548 SYSCTL_CHILDREN(rack_counters), 1549 OID_AUTO, "hwpace_init_fail", CTLFLAG_RD, 1550 &rack_hw_pace_init_fail, "Total number of times we failed to initialize hw pacing"); 1551 rack_hw_pace_lost = counter_u64_alloc(M_WAITOK); 1552 1553 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1554 SYSCTL_CHILDREN(rack_counters), 1555 OID_AUTO, "hwpace_lost", CTLFLAG_RD, 1556 &rack_hw_pace_lost, "Total number of times we failed to initialize hw pacing"); 1557 rack_tlp_tot = counter_u64_alloc(M_WAITOK); 1558 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1559 SYSCTL_CHILDREN(rack_counters), 1560 OID_AUTO, "tlp_to_total", CTLFLAG_RD, 1561 &rack_tlp_tot, 1562 "Total number of tail loss probe expirations"); 1563 rack_tlp_newdata = counter_u64_alloc(M_WAITOK); 1564 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1565 SYSCTL_CHILDREN(rack_counters), 1566 OID_AUTO, "tlp_new", CTLFLAG_RD, 1567 &rack_tlp_newdata, 1568 "Total number of tail loss probe sending new data"); 1569 rack_tlp_retran = counter_u64_alloc(M_WAITOK); 1570 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1571 SYSCTL_CHILDREN(rack_counters), 1572 OID_AUTO, "tlp_retran", CTLFLAG_RD, 1573 &rack_tlp_retran, 1574 "Total number of tail loss probe sending retransmitted data"); 1575 rack_tlp_retran_bytes = counter_u64_alloc(M_WAITOK); 1576 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1577 SYSCTL_CHILDREN(rack_counters), 1578 OID_AUTO, "tlp_retran_bytes", CTLFLAG_RD, 1579 &rack_tlp_retran_bytes, 1580 "Total bytes of tail loss probe sending retransmitted data"); 1581 rack_to_tot = counter_u64_alloc(M_WAITOK); 1582 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1583 SYSCTL_CHILDREN(rack_counters), 1584 OID_AUTO, "rack_to_tot", CTLFLAG_RD, 1585 &rack_to_tot, 1586 "Total number of times the rack to expired"); 1587 rack_saw_enobuf = counter_u64_alloc(M_WAITOK); 1588 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1589 SYSCTL_CHILDREN(rack_counters), 1590 OID_AUTO, "saw_enobufs", CTLFLAG_RD, 1591 &rack_saw_enobuf, 1592 "Total number of times a sends returned enobuf for non-hdwr paced connections"); 1593 rack_saw_enobuf_hw = counter_u64_alloc(M_WAITOK); 1594 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1595 SYSCTL_CHILDREN(rack_counters), 1596 OID_AUTO, "saw_enobufs_hw", CTLFLAG_RD, 1597 &rack_saw_enobuf_hw, 1598 "Total number of times a send returned enobuf for hdwr paced connections"); 1599 rack_saw_enetunreach = counter_u64_alloc(M_WAITOK); 1600 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1601 SYSCTL_CHILDREN(rack_counters), 1602 OID_AUTO, "saw_enetunreach", CTLFLAG_RD, 1603 &rack_saw_enetunreach, 1604 "Total number of times a send received a enetunreachable"); 1605 rack_hot_alloc = counter_u64_alloc(M_WAITOK); 1606 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1607 SYSCTL_CHILDREN(rack_counters), 1608 OID_AUTO, "alloc_hot", CTLFLAG_RD, 1609 &rack_hot_alloc, 1610 "Total allocations from the top of our list"); 1611 rack_to_alloc = counter_u64_alloc(M_WAITOK); 1612 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1613 SYSCTL_CHILDREN(rack_counters), 1614 OID_AUTO, "allocs", CTLFLAG_RD, 1615 &rack_to_alloc, 1616 "Total allocations of tracking structures"); 1617 rack_to_alloc_hard = counter_u64_alloc(M_WAITOK); 1618 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1619 SYSCTL_CHILDREN(rack_counters), 1620 OID_AUTO, "allochard", CTLFLAG_RD, 1621 &rack_to_alloc_hard, 1622 "Total allocations done with sleeping the hard way"); 1623 rack_to_alloc_emerg = counter_u64_alloc(M_WAITOK); 1624 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1625 SYSCTL_CHILDREN(rack_counters), 1626 OID_AUTO, "allocemerg", CTLFLAG_RD, 1627 &rack_to_alloc_emerg, 1628 "Total allocations done from emergency cache"); 1629 rack_to_alloc_limited = counter_u64_alloc(M_WAITOK); 1630 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1631 SYSCTL_CHILDREN(rack_counters), 1632 OID_AUTO, "alloc_limited", CTLFLAG_RD, 1633 &rack_to_alloc_limited, 1634 "Total allocations dropped due to limit"); 1635 rack_alloc_limited_conns = counter_u64_alloc(M_WAITOK); 1636 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1637 SYSCTL_CHILDREN(rack_counters), 1638 OID_AUTO, "alloc_limited_conns", CTLFLAG_RD, 1639 &rack_alloc_limited_conns, 1640 "Connections with allocations dropped due to limit"); 1641 rack_split_limited = counter_u64_alloc(M_WAITOK); 1642 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1643 SYSCTL_CHILDREN(rack_counters), 1644 OID_AUTO, "split_limited", CTLFLAG_RD, 1645 &rack_split_limited, 1646 "Split allocations dropped due to limit"); 1647 rack_persists_sends = counter_u64_alloc(M_WAITOK); 1648 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1649 SYSCTL_CHILDREN(rack_counters), 1650 OID_AUTO, "persist_sends", CTLFLAG_RD, 1651 &rack_persists_sends, 1652 "Number of times we sent a persist probe"); 1653 rack_persists_acks = counter_u64_alloc(M_WAITOK); 1654 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1655 SYSCTL_CHILDREN(rack_counters), 1656 OID_AUTO, "persist_acks", CTLFLAG_RD, 1657 &rack_persists_acks, 1658 "Number of times a persist probe was acked"); 1659 rack_persists_loss = counter_u64_alloc(M_WAITOK); 1660 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1661 SYSCTL_CHILDREN(rack_counters), 1662 OID_AUTO, "persist_loss", CTLFLAG_RD, 1663 &rack_persists_loss, 1664 "Number of times we detected a lost persist probe (no ack)"); 1665 rack_persists_lost_ends = counter_u64_alloc(M_WAITOK); 1666 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1667 SYSCTL_CHILDREN(rack_counters), 1668 OID_AUTO, "persist_loss_ends", CTLFLAG_RD, 1669 &rack_persists_lost_ends, 1670 "Number of lost persist probe (no ack) that the run ended with a PERSIST abort"); 1671 #ifdef INVARIANTS 1672 rack_adjust_map_bw = counter_u64_alloc(M_WAITOK); 1673 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1674 SYSCTL_CHILDREN(rack_counters), 1675 OID_AUTO, "map_adjust_req", CTLFLAG_RD, 1676 &rack_adjust_map_bw, 1677 "Number of times we hit the case where the sb went up and down on a sendmap entry"); 1678 #endif 1679 rack_multi_single_eq = counter_u64_alloc(M_WAITOK); 1680 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1681 SYSCTL_CHILDREN(rack_counters), 1682 OID_AUTO, "cmp_ack_equiv", CTLFLAG_RD, 1683 &rack_multi_single_eq, 1684 "Number of compressed acks total represented"); 1685 rack_proc_non_comp_ack = counter_u64_alloc(M_WAITOK); 1686 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1687 SYSCTL_CHILDREN(rack_counters), 1688 OID_AUTO, "cmp_ack_not", CTLFLAG_RD, 1689 &rack_proc_non_comp_ack, 1690 "Number of non compresseds acks that we processed"); 1691 1692 1693 rack_sack_proc_all = counter_u64_alloc(M_WAITOK); 1694 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1695 SYSCTL_CHILDREN(rack_counters), 1696 OID_AUTO, "sack_long", CTLFLAG_RD, 1697 &rack_sack_proc_all, 1698 "Total times we had to walk whole list for sack processing"); 1699 rack_sack_proc_restart = counter_u64_alloc(M_WAITOK); 1700 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1701 SYSCTL_CHILDREN(rack_counters), 1702 OID_AUTO, "sack_restart", CTLFLAG_RD, 1703 &rack_sack_proc_restart, 1704 "Total times we had to walk whole list due to a restart"); 1705 rack_sack_proc_short = counter_u64_alloc(M_WAITOK); 1706 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1707 SYSCTL_CHILDREN(rack_counters), 1708 OID_AUTO, "sack_short", CTLFLAG_RD, 1709 &rack_sack_proc_short, 1710 "Total times we took shortcut for sack processing"); 1711 rack_sack_skipped_acked = counter_u64_alloc(M_WAITOK); 1712 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1713 SYSCTL_CHILDREN(rack_attack), 1714 OID_AUTO, "skipacked", CTLFLAG_RD, 1715 &rack_sack_skipped_acked, 1716 "Total number of times we skipped previously sacked"); 1717 rack_sack_splits = counter_u64_alloc(M_WAITOK); 1718 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1719 SYSCTL_CHILDREN(rack_attack), 1720 OID_AUTO, "ofsplit", CTLFLAG_RD, 1721 &rack_sack_splits, 1722 "Total number of times we did the old fashion tree split"); 1723 rack_input_idle_reduces = counter_u64_alloc(M_WAITOK); 1724 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1725 SYSCTL_CHILDREN(rack_counters), 1726 OID_AUTO, "idle_reduce_oninput", CTLFLAG_RD, 1727 &rack_input_idle_reduces, 1728 "Total number of idle reductions on input"); 1729 rack_collapsed_win_seen = counter_u64_alloc(M_WAITOK); 1730 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1731 SYSCTL_CHILDREN(rack_counters), 1732 OID_AUTO, "collapsed_win_seen", CTLFLAG_RD, 1733 &rack_collapsed_win_seen, 1734 "Total number of collapsed window events seen (where our window shrinks)"); 1735 1736 rack_collapsed_win = counter_u64_alloc(M_WAITOK); 1737 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1738 SYSCTL_CHILDREN(rack_counters), 1739 OID_AUTO, "collapsed_win", CTLFLAG_RD, 1740 &rack_collapsed_win, 1741 "Total number of collapsed window events where we mark packets"); 1742 rack_collapsed_win_rxt = counter_u64_alloc(M_WAITOK); 1743 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1744 SYSCTL_CHILDREN(rack_counters), 1745 OID_AUTO, "collapsed_win_rxt", CTLFLAG_RD, 1746 &rack_collapsed_win_rxt, 1747 "Total number of packets that were retransmitted"); 1748 rack_collapsed_win_rxt_bytes = counter_u64_alloc(M_WAITOK); 1749 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1750 SYSCTL_CHILDREN(rack_counters), 1751 OID_AUTO, "collapsed_win_bytes", CTLFLAG_RD, 1752 &rack_collapsed_win_rxt_bytes, 1753 "Total number of bytes that were retransmitted"); 1754 rack_try_scwnd = counter_u64_alloc(M_WAITOK); 1755 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx, 1756 SYSCTL_CHILDREN(rack_counters), 1757 OID_AUTO, "tried_scwnd", CTLFLAG_RD, 1758 &rack_try_scwnd, 1759 "Total number of scwnd attempts"); 1760 COUNTER_ARRAY_ALLOC(rack_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK); 1761 SYSCTL_ADD_COUNTER_U64_ARRAY(&rack_sysctl_ctx, SYSCTL_CHILDREN(rack_sysctl_root), 1762 OID_AUTO, "outsize", CTLFLAG_RD, 1763 rack_out_size, TCP_MSS_ACCT_SIZE, "MSS send sizes"); 1764 COUNTER_ARRAY_ALLOC(rack_opts_arry, RACK_OPTS_SIZE, M_WAITOK); 1765 SYSCTL_ADD_COUNTER_U64_ARRAY(&rack_sysctl_ctx, SYSCTL_CHILDREN(rack_sysctl_root), 1766 OID_AUTO, "opts", CTLFLAG_RD, 1767 rack_opts_arry, RACK_OPTS_SIZE, "RACK Option Stats"); 1768 SYSCTL_ADD_PROC(&rack_sysctl_ctx, 1769 SYSCTL_CHILDREN(rack_sysctl_root), 1770 OID_AUTO, "clear", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 1771 &rack_clear_counter, 0, sysctl_rack_clear, "IU", "Clear counters"); 1772 } 1773 1774 static __inline int 1775 rb_map_cmp(struct rack_sendmap *b, struct rack_sendmap *a) 1776 { 1777 if (SEQ_GEQ(b->r_start, a->r_start) && 1778 SEQ_LT(b->r_start, a->r_end)) { 1779 /* 1780 * The entry b is within the 1781 * block a. i.e.: 1782 * a -- |-------------| 1783 * b -- |----| 1784 * <or> 1785 * b -- |------| 1786 * <or> 1787 * b -- |-----------| 1788 */ 1789 return (0); 1790 } else if (SEQ_GEQ(b->r_start, a->r_end)) { 1791 /* 1792 * b falls as either the next 1793 * sequence block after a so a 1794 * is said to be smaller than b. 1795 * i.e: 1796 * a -- |------| 1797 * b -- |--------| 1798 * or 1799 * b -- |-----| 1800 */ 1801 return (1); 1802 } 1803 /* 1804 * Whats left is where a is 1805 * larger than b. i.e: 1806 * a -- |-------| 1807 * b -- |---| 1808 * or even possibly 1809 * b -- |--------------| 1810 */ 1811 return (-1); 1812 } 1813 1814 RB_PROTOTYPE(rack_rb_tree_head, rack_sendmap, r_next, rb_map_cmp); 1815 RB_GENERATE(rack_rb_tree_head, rack_sendmap, r_next, rb_map_cmp); 1816 1817 static uint32_t 1818 rc_init_window(struct tcp_rack *rack) 1819 { 1820 uint32_t win; 1821 1822 if (rack->rc_init_win == 0) { 1823 /* 1824 * Nothing set by the user, use the system stack 1825 * default. 1826 */ 1827 return (tcp_compute_initwnd(tcp_maxseg(rack->rc_tp))); 1828 } 1829 win = ctf_fixed_maxseg(rack->rc_tp) * rack->rc_init_win; 1830 return (win); 1831 } 1832 1833 static uint64_t 1834 rack_get_fixed_pacing_bw(struct tcp_rack *rack) 1835 { 1836 if (IN_FASTRECOVERY(rack->rc_tp->t_flags)) 1837 return (rack->r_ctl.rc_fixed_pacing_rate_rec); 1838 else if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh) 1839 return (rack->r_ctl.rc_fixed_pacing_rate_ss); 1840 else 1841 return (rack->r_ctl.rc_fixed_pacing_rate_ca); 1842 } 1843 1844 static uint64_t 1845 rack_get_bw(struct tcp_rack *rack) 1846 { 1847 if (rack->use_fixed_rate) { 1848 /* Return the fixed pacing rate */ 1849 return (rack_get_fixed_pacing_bw(rack)); 1850 } 1851 if (rack->r_ctl.gp_bw == 0) { 1852 /* 1853 * We have yet no b/w measurement, 1854 * if we have a user set initial bw 1855 * return it. If we don't have that and 1856 * we have an srtt, use the tcp IW (10) to 1857 * calculate a fictional b/w over the SRTT 1858 * which is more or less a guess. Note 1859 * we don't use our IW from rack on purpose 1860 * so if we have like IW=30, we are not 1861 * calculating a "huge" b/w. 1862 */ 1863 uint64_t bw, srtt; 1864 if (rack->r_ctl.init_rate) 1865 return (rack->r_ctl.init_rate); 1866 1867 /* Has the user set a max peak rate? */ 1868 #ifdef NETFLIX_PEAKRATE 1869 if (rack->rc_tp->t_maxpeakrate) 1870 return (rack->rc_tp->t_maxpeakrate); 1871 #endif 1872 /* Ok lets come up with the IW guess, if we have a srtt */ 1873 if (rack->rc_tp->t_srtt == 0) { 1874 /* 1875 * Go with old pacing method 1876 * i.e. burst mitigation only. 1877 */ 1878 return (0); 1879 } 1880 /* Ok lets get the initial TCP win (not racks) */ 1881 bw = tcp_compute_initwnd(tcp_maxseg(rack->rc_tp)); 1882 srtt = (uint64_t)rack->rc_tp->t_srtt; 1883 bw *= (uint64_t)USECS_IN_SECOND; 1884 bw /= srtt; 1885 if (rack->r_ctl.bw_rate_cap && (bw > rack->r_ctl.bw_rate_cap)) 1886 bw = rack->r_ctl.bw_rate_cap; 1887 return (bw); 1888 } else { 1889 uint64_t bw; 1890 1891 if (rack->r_ctl.num_measurements >= RACK_REQ_AVG) { 1892 /* Averaging is done, we can return the value */ 1893 bw = rack->r_ctl.gp_bw; 1894 } else { 1895 /* Still doing initial average must calculate */ 1896 bw = rack->r_ctl.gp_bw / rack->r_ctl.num_measurements; 1897 } 1898 #ifdef NETFLIX_PEAKRATE 1899 if ((rack->rc_tp->t_maxpeakrate) && 1900 (bw > rack->rc_tp->t_maxpeakrate)) { 1901 /* The user has set a peak rate to pace at 1902 * don't allow us to pace faster than that. 1903 */ 1904 return (rack->rc_tp->t_maxpeakrate); 1905 } 1906 #endif 1907 if (rack->r_ctl.bw_rate_cap && (bw > rack->r_ctl.bw_rate_cap)) 1908 bw = rack->r_ctl.bw_rate_cap; 1909 return (bw); 1910 } 1911 } 1912 1913 static uint16_t 1914 rack_get_output_gain(struct tcp_rack *rack, struct rack_sendmap *rsm) 1915 { 1916 if (rack->use_fixed_rate) { 1917 return (100); 1918 } else if (rack->in_probe_rtt && (rsm == NULL)) 1919 return (rack->r_ctl.rack_per_of_gp_probertt); 1920 else if ((IN_FASTRECOVERY(rack->rc_tp->t_flags) && 1921 rack->r_ctl.rack_per_of_gp_rec)) { 1922 if (rsm) { 1923 /* a retransmission always use the recovery rate */ 1924 return (rack->r_ctl.rack_per_of_gp_rec); 1925 } else if (rack->rack_rec_nonrxt_use_cr) { 1926 /* Directed to use the configured rate */ 1927 goto configured_rate; 1928 } else if (rack->rack_no_prr && 1929 (rack->r_ctl.rack_per_of_gp_rec > 100)) { 1930 /* No PRR, lets just use the b/w estimate only */ 1931 return (100); 1932 } else { 1933 /* 1934 * Here we may have a non-retransmit but we 1935 * have no overrides, so just use the recovery 1936 * rate (prr is in effect). 1937 */ 1938 return (rack->r_ctl.rack_per_of_gp_rec); 1939 } 1940 } 1941 configured_rate: 1942 /* For the configured rate we look at our cwnd vs the ssthresh */ 1943 if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh) 1944 return (rack->r_ctl.rack_per_of_gp_ss); 1945 else 1946 return (rack->r_ctl.rack_per_of_gp_ca); 1947 } 1948 1949 static void 1950 rack_log_dsack_event(struct tcp_rack *rack, uint8_t mod, uint32_t flex4, uint32_t flex5, uint32_t flex6) 1951 { 1952 /* 1953 * Types of logs (mod value) 1954 * 1 = dsack_persists reduced by 1 via T-O or fast recovery exit. 1955 * 2 = a dsack round begins, persist is reset to 16. 1956 * 3 = a dsack round ends 1957 * 4 = Dsack option increases rack rtt flex5 is the srtt input, flex6 is thresh 1958 * 5 = Socket option set changing the control flags rc_rack_tmr_std_based, rc_rack_use_dsack 1959 * 6 = Final rack rtt, flex4 is srtt and flex6 is final limited thresh. 1960 */ 1961 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 1962 union tcp_log_stackspecific log; 1963 struct timeval tv; 1964 1965 memset(&log, 0, sizeof(log)); 1966 log.u_bbr.flex1 = rack->rc_rack_tmr_std_based; 1967 log.u_bbr.flex1 <<= 1; 1968 log.u_bbr.flex1 |= rack->rc_rack_use_dsack; 1969 log.u_bbr.flex1 <<= 1; 1970 log.u_bbr.flex1 |= rack->rc_dsack_round_seen; 1971 log.u_bbr.flex2 = rack->r_ctl.dsack_round_end; 1972 log.u_bbr.flex3 = rack->r_ctl.num_dsack; 1973 log.u_bbr.flex4 = flex4; 1974 log.u_bbr.flex5 = flex5; 1975 log.u_bbr.flex6 = flex6; 1976 log.u_bbr.flex7 = rack->r_ctl.dsack_persist; 1977 log.u_bbr.flex8 = mod; 1978 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 1979 TCP_LOG_EVENTP(rack->rc_tp, NULL, 1980 &rack->rc_inp->inp_socket->so_rcv, 1981 &rack->rc_inp->inp_socket->so_snd, 1982 RACK_DSACK_HANDLING, 0, 1983 0, &log, false, &tv); 1984 } 1985 } 1986 1987 static void 1988 rack_log_hdwr_pacing(struct tcp_rack *rack, 1989 uint64_t rate, uint64_t hw_rate, int line, 1990 int error, uint16_t mod) 1991 { 1992 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 1993 union tcp_log_stackspecific log; 1994 struct timeval tv; 1995 const struct ifnet *ifp; 1996 1997 memset(&log, 0, sizeof(log)); 1998 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff); 1999 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff); 2000 if (rack->r_ctl.crte) { 2001 ifp = rack->r_ctl.crte->ptbl->rs_ifp; 2002 } else if (rack->rc_inp->inp_route.ro_nh && 2003 rack->rc_inp->inp_route.ro_nh->nh_ifp) { 2004 ifp = rack->rc_inp->inp_route.ro_nh->nh_ifp; 2005 } else 2006 ifp = NULL; 2007 if (ifp) { 2008 log.u_bbr.flex3 = (((uint64_t)ifp >> 32) & 0x00000000ffffffff); 2009 log.u_bbr.flex4 = ((uint64_t)ifp & 0x00000000ffffffff); 2010 } 2011 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2012 log.u_bbr.bw_inuse = rate; 2013 log.u_bbr.flex5 = line; 2014 log.u_bbr.flex6 = error; 2015 log.u_bbr.flex7 = mod; 2016 log.u_bbr.applimited = rack->r_ctl.rc_pace_max_segs; 2017 log.u_bbr.flex8 = rack->use_fixed_rate; 2018 log.u_bbr.flex8 <<= 1; 2019 log.u_bbr.flex8 |= rack->rack_hdrw_pacing; 2020 log.u_bbr.pkts_out = rack->rc_tp->t_maxseg; 2021 log.u_bbr.delRate = rack->r_ctl.crte_prev_rate; 2022 if (rack->r_ctl.crte) 2023 log.u_bbr.cur_del_rate = rack->r_ctl.crte->rate; 2024 else 2025 log.u_bbr.cur_del_rate = 0; 2026 log.u_bbr.rttProp = rack->r_ctl.last_hw_bw_req; 2027 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2028 &rack->rc_inp->inp_socket->so_rcv, 2029 &rack->rc_inp->inp_socket->so_snd, 2030 BBR_LOG_HDWR_PACE, 0, 2031 0, &log, false, &tv); 2032 } 2033 } 2034 2035 static uint64_t 2036 rack_get_output_bw(struct tcp_rack *rack, uint64_t bw, struct rack_sendmap *rsm, int *capped) 2037 { 2038 /* 2039 * We allow rack_per_of_gp_xx to dictate our bw rate we want. 2040 */ 2041 uint64_t bw_est, high_rate; 2042 uint64_t gain; 2043 2044 gain = (uint64_t)rack_get_output_gain(rack, rsm); 2045 bw_est = bw * gain; 2046 bw_est /= (uint64_t)100; 2047 /* Never fall below the minimum (def 64kbps) */ 2048 if (bw_est < RACK_MIN_BW) 2049 bw_est = RACK_MIN_BW; 2050 if (rack->r_rack_hw_rate_caps) { 2051 /* Rate caps are in place */ 2052 if (rack->r_ctl.crte != NULL) { 2053 /* We have a hdwr rate already */ 2054 high_rate = tcp_hw_highest_rate(rack->r_ctl.crte); 2055 if (bw_est >= high_rate) { 2056 /* We are capping bw at the highest rate table entry */ 2057 rack_log_hdwr_pacing(rack, 2058 bw_est, high_rate, __LINE__, 2059 0, 3); 2060 bw_est = high_rate; 2061 if (capped) 2062 *capped = 1; 2063 } 2064 } else if ((rack->rack_hdrw_pacing == 0) && 2065 (rack->rack_hdw_pace_ena) && 2066 (rack->rack_attempt_hdwr_pace == 0) && 2067 (rack->rc_inp->inp_route.ro_nh != NULL) && 2068 (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) { 2069 /* 2070 * Special case, we have not yet attempted hardware 2071 * pacing, and yet we may, when we do, find out if we are 2072 * above the highest rate. We need to know the maxbw for the interface 2073 * in question (if it supports ratelimiting). We get back 2074 * a 0, if the interface is not found in the RL lists. 2075 */ 2076 high_rate = tcp_hw_highest_rate_ifp(rack->rc_inp->inp_route.ro_nh->nh_ifp, rack->rc_inp); 2077 if (high_rate) { 2078 /* Yep, we have a rate is it above this rate? */ 2079 if (bw_est > high_rate) { 2080 bw_est = high_rate; 2081 if (capped) 2082 *capped = 1; 2083 } 2084 } 2085 } 2086 } 2087 return (bw_est); 2088 } 2089 2090 static void 2091 rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t tsused, uint32_t thresh, int mod) 2092 { 2093 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2094 union tcp_log_stackspecific log; 2095 struct timeval tv; 2096 2097 if ((mod != 1) && (rack_verbose_logging == 0)) { 2098 /* 2099 * We get 3 values currently for mod 2100 * 1 - We are retransmitting and this tells the reason. 2101 * 2 - We are clearing a dup-ack count. 2102 * 3 - We are incrementing a dup-ack count. 2103 * 2104 * The clear/increment are only logged 2105 * if you have BBverbose on. 2106 */ 2107 return; 2108 } 2109 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2110 log.u_bbr.flex1 = tsused; 2111 log.u_bbr.flex2 = thresh; 2112 log.u_bbr.flex3 = rsm->r_flags; 2113 log.u_bbr.flex4 = rsm->r_dupack; 2114 log.u_bbr.flex5 = rsm->r_start; 2115 log.u_bbr.flex6 = rsm->r_end; 2116 log.u_bbr.flex8 = mod; 2117 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2118 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2119 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2120 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2121 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2122 log.u_bbr.pacing_gain = rack->r_must_retran; 2123 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2124 &rack->rc_inp->inp_socket->so_rcv, 2125 &rack->rc_inp->inp_socket->so_snd, 2126 BBR_LOG_SETTINGS_CHG, 0, 2127 0, &log, false, &tv); 2128 } 2129 } 2130 2131 static void 2132 rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot, uint8_t which) 2133 { 2134 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2135 union tcp_log_stackspecific log; 2136 struct timeval tv; 2137 2138 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2139 log.u_bbr.flex1 = rack->rc_tp->t_srtt; 2140 log.u_bbr.flex2 = to; 2141 log.u_bbr.flex3 = rack->r_ctl.rc_hpts_flags; 2142 log.u_bbr.flex4 = slot; 2143 log.u_bbr.flex5 = rack->rc_inp->inp_hptsslot; 2144 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur; 2145 log.u_bbr.flex7 = rack->rc_in_persist; 2146 log.u_bbr.flex8 = which; 2147 if (rack->rack_no_prr) 2148 log.u_bbr.pkts_out = 0; 2149 else 2150 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt; 2151 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2152 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2153 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2154 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2155 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2156 log.u_bbr.pacing_gain = rack->r_must_retran; 2157 log.u_bbr.cwnd_gain = rack->rc_has_collapsed; 2158 log.u_bbr.lt_epoch = rack->rc_tp->t_rxtshift; 2159 log.u_bbr.lost = rack_rto_min; 2160 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2161 &rack->rc_inp->inp_socket->so_rcv, 2162 &rack->rc_inp->inp_socket->so_snd, 2163 BBR_LOG_TIMERSTAR, 0, 2164 0, &log, false, &tv); 2165 } 2166 } 2167 2168 static void 2169 rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rsm) 2170 { 2171 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2172 union tcp_log_stackspecific log; 2173 struct timeval tv; 2174 2175 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2176 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2177 log.u_bbr.flex8 = to_num; 2178 log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt; 2179 log.u_bbr.flex2 = rack->rc_rack_rtt; 2180 if (rsm == NULL) 2181 log.u_bbr.flex3 = 0; 2182 else 2183 log.u_bbr.flex3 = rsm->r_end - rsm->r_start; 2184 if (rack->rack_no_prr) 2185 log.u_bbr.flex5 = 0; 2186 else 2187 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt; 2188 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2189 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2190 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2191 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2192 log.u_bbr.pacing_gain = rack->r_must_retran; 2193 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2194 &rack->rc_inp->inp_socket->so_rcv, 2195 &rack->rc_inp->inp_socket->so_snd, 2196 BBR_LOG_RTO, 0, 2197 0, &log, false, &tv); 2198 } 2199 } 2200 2201 static void 2202 rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack, 2203 struct rack_sendmap *prev, 2204 struct rack_sendmap *rsm, 2205 struct rack_sendmap *next, 2206 int flag, uint32_t th_ack, int line) 2207 { 2208 if (rack_verbose_logging && (tp->t_logstate != TCP_LOG_STATE_OFF)) { 2209 union tcp_log_stackspecific log; 2210 struct timeval tv; 2211 2212 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2213 log.u_bbr.flex8 = flag; 2214 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2215 log.u_bbr.cur_del_rate = (uint64_t)prev; 2216 log.u_bbr.delRate = (uint64_t)rsm; 2217 log.u_bbr.rttProp = (uint64_t)next; 2218 log.u_bbr.flex7 = 0; 2219 if (prev) { 2220 log.u_bbr.flex1 = prev->r_start; 2221 log.u_bbr.flex2 = prev->r_end; 2222 log.u_bbr.flex7 |= 0x4; 2223 } 2224 if (rsm) { 2225 log.u_bbr.flex3 = rsm->r_start; 2226 log.u_bbr.flex4 = rsm->r_end; 2227 log.u_bbr.flex7 |= 0x2; 2228 } 2229 if (next) { 2230 log.u_bbr.flex5 = next->r_start; 2231 log.u_bbr.flex6 = next->r_end; 2232 log.u_bbr.flex7 |= 0x1; 2233 } 2234 log.u_bbr.applimited = line; 2235 log.u_bbr.pkts_out = th_ack; 2236 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2237 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2238 if (rack->rack_no_prr) 2239 log.u_bbr.lost = 0; 2240 else 2241 log.u_bbr.lost = rack->r_ctl.rc_prr_sndcnt; 2242 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2243 &rack->rc_inp->inp_socket->so_rcv, 2244 &rack->rc_inp->inp_socket->so_snd, 2245 TCP_LOG_MAPCHG, 0, 2246 0, &log, false, &tv); 2247 } 2248 } 2249 2250 static void 2251 rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t len, 2252 struct rack_sendmap *rsm, int conf) 2253 { 2254 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 2255 union tcp_log_stackspecific log; 2256 struct timeval tv; 2257 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2258 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2259 log.u_bbr.flex1 = t; 2260 log.u_bbr.flex2 = len; 2261 log.u_bbr.flex3 = rack->r_ctl.rc_rack_min_rtt; 2262 log.u_bbr.flex4 = rack->r_ctl.rack_rs.rs_rtt_lowest; 2263 log.u_bbr.flex5 = rack->r_ctl.rack_rs.rs_rtt_highest; 2264 log.u_bbr.flex6 = rack->r_ctl.rack_rs.rs_us_rtrcnt; 2265 log.u_bbr.flex7 = conf; 2266 log.u_bbr.rttProp = (uint64_t)rack->r_ctl.rack_rs.rs_rtt_tot; 2267 log.u_bbr.flex8 = rack->r_ctl.rc_rate_sample_method; 2268 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2269 log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt; 2270 log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags; 2271 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2272 if (rsm) { 2273 log.u_bbr.pkt_epoch = rsm->r_start; 2274 log.u_bbr.lost = rsm->r_end; 2275 log.u_bbr.cwnd_gain = rsm->r_rtr_cnt; 2276 /* We loose any upper of the 24 bits */ 2277 log.u_bbr.pacing_gain = (uint16_t)rsm->r_flags; 2278 } else { 2279 /* Its a SYN */ 2280 log.u_bbr.pkt_epoch = rack->rc_tp->iss; 2281 log.u_bbr.lost = 0; 2282 log.u_bbr.cwnd_gain = 0; 2283 log.u_bbr.pacing_gain = 0; 2284 } 2285 /* Write out general bits of interest rrs here */ 2286 log.u_bbr.use_lt_bw = rack->rc_highly_buffered; 2287 log.u_bbr.use_lt_bw <<= 1; 2288 log.u_bbr.use_lt_bw |= rack->forced_ack; 2289 log.u_bbr.use_lt_bw <<= 1; 2290 log.u_bbr.use_lt_bw |= rack->rc_gp_dyn_mul; 2291 log.u_bbr.use_lt_bw <<= 1; 2292 log.u_bbr.use_lt_bw |= rack->in_probe_rtt; 2293 log.u_bbr.use_lt_bw <<= 1; 2294 log.u_bbr.use_lt_bw |= rack->measure_saw_probe_rtt; 2295 log.u_bbr.use_lt_bw <<= 1; 2296 log.u_bbr.use_lt_bw |= rack->app_limited_needs_set; 2297 log.u_bbr.use_lt_bw <<= 1; 2298 log.u_bbr.use_lt_bw |= rack->rc_gp_filled; 2299 log.u_bbr.use_lt_bw <<= 1; 2300 log.u_bbr.use_lt_bw |= rack->rc_dragged_bottom; 2301 log.u_bbr.applimited = rack->r_ctl.rc_target_probertt_flight; 2302 log.u_bbr.epoch = rack->r_ctl.rc_time_probertt_starts; 2303 log.u_bbr.lt_epoch = rack->r_ctl.rc_time_probertt_entered; 2304 log.u_bbr.cur_del_rate = rack->r_ctl.rc_lower_rtt_us_cts; 2305 log.u_bbr.delRate = rack->r_ctl.rc_gp_srtt; 2306 log.u_bbr.bw_inuse = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 2307 log.u_bbr.bw_inuse <<= 32; 2308 if (rsm) 2309 log.u_bbr.bw_inuse |= ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]); 2310 TCP_LOG_EVENTP(tp, NULL, 2311 &rack->rc_inp->inp_socket->so_rcv, 2312 &rack->rc_inp->inp_socket->so_snd, 2313 BBR_LOG_BBRRTT, 0, 2314 0, &log, false, &tv); 2315 2316 2317 } 2318 } 2319 2320 static void 2321 rack_log_rtt_sample(struct tcp_rack *rack, uint32_t rtt) 2322 { 2323 /* 2324 * Log the rtt sample we are 2325 * applying to the srtt algorithm in 2326 * useconds. 2327 */ 2328 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2329 union tcp_log_stackspecific log; 2330 struct timeval tv; 2331 2332 /* Convert our ms to a microsecond */ 2333 memset(&log, 0, sizeof(log)); 2334 log.u_bbr.flex1 = rtt; 2335 log.u_bbr.flex2 = rack->r_ctl.ack_count; 2336 log.u_bbr.flex3 = rack->r_ctl.sack_count; 2337 log.u_bbr.flex4 = rack->r_ctl.sack_noextra_move; 2338 log.u_bbr.flex5 = rack->r_ctl.sack_moved_extra; 2339 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur; 2340 log.u_bbr.flex7 = 1; 2341 log.u_bbr.flex8 = rack->sack_attack_disable; 2342 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2343 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2344 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2345 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2346 log.u_bbr.pacing_gain = rack->r_must_retran; 2347 /* 2348 * We capture in delRate the upper 32 bits as 2349 * the confidence level we had declared, and the 2350 * lower 32 bits as the actual RTT using the arrival 2351 * timestamp. 2352 */ 2353 log.u_bbr.delRate = rack->r_ctl.rack_rs.confidence; 2354 log.u_bbr.delRate <<= 32; 2355 log.u_bbr.delRate |= rack->r_ctl.rack_rs.rs_us_rtt; 2356 /* Lets capture all the things that make up t_rtxcur */ 2357 log.u_bbr.applimited = rack_rto_min; 2358 log.u_bbr.epoch = rack_rto_max; 2359 log.u_bbr.lt_epoch = rack->r_ctl.timer_slop; 2360 log.u_bbr.lost = rack_rto_min; 2361 log.u_bbr.pkt_epoch = TICKS_2_USEC(tcp_rexmit_slop); 2362 log.u_bbr.rttProp = RACK_REXMTVAL(rack->rc_tp); 2363 log.u_bbr.bw_inuse = rack->r_ctl.act_rcv_time.tv_sec; 2364 log.u_bbr.bw_inuse *= HPTS_USEC_IN_SEC; 2365 log.u_bbr.bw_inuse += rack->r_ctl.act_rcv_time.tv_usec; 2366 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2367 &rack->rc_inp->inp_socket->so_rcv, 2368 &rack->rc_inp->inp_socket->so_snd, 2369 TCP_LOG_RTT, 0, 2370 0, &log, false, &tv); 2371 } 2372 } 2373 2374 static void 2375 rack_log_rtt_sample_calc(struct tcp_rack *rack, uint32_t rtt, uint32_t send_time, uint32_t ack_time, int where) 2376 { 2377 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 2378 union tcp_log_stackspecific log; 2379 struct timeval tv; 2380 2381 /* Convert our ms to a microsecond */ 2382 memset(&log, 0, sizeof(log)); 2383 log.u_bbr.flex1 = rtt; 2384 log.u_bbr.flex2 = send_time; 2385 log.u_bbr.flex3 = ack_time; 2386 log.u_bbr.flex4 = where; 2387 log.u_bbr.flex7 = 2; 2388 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2389 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2390 &rack->rc_inp->inp_socket->so_rcv, 2391 &rack->rc_inp->inp_socket->so_snd, 2392 TCP_LOG_RTT, 0, 2393 0, &log, false, &tv); 2394 } 2395 } 2396 2397 2398 2399 static inline void 2400 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line) 2401 { 2402 if (rack_verbose_logging && (tp->t_logstate != TCP_LOG_STATE_OFF)) { 2403 union tcp_log_stackspecific log; 2404 struct timeval tv; 2405 2406 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2407 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2408 log.u_bbr.flex1 = line; 2409 log.u_bbr.flex2 = tick; 2410 log.u_bbr.flex3 = tp->t_maxunacktime; 2411 log.u_bbr.flex4 = tp->t_acktime; 2412 log.u_bbr.flex8 = event; 2413 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2414 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2415 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2416 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2417 log.u_bbr.pacing_gain = rack->r_must_retran; 2418 TCP_LOG_EVENTP(tp, NULL, 2419 &rack->rc_inp->inp_socket->so_rcv, 2420 &rack->rc_inp->inp_socket->so_snd, 2421 BBR_LOG_PROGRESS, 0, 2422 0, &log, false, &tv); 2423 } 2424 } 2425 2426 static void 2427 rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_t cts, struct timeval *tv) 2428 { 2429 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2430 union tcp_log_stackspecific log; 2431 2432 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2433 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2434 log.u_bbr.flex1 = slot; 2435 if (rack->rack_no_prr) 2436 log.u_bbr.flex2 = 0; 2437 else 2438 log.u_bbr.flex2 = rack->r_ctl.rc_prr_sndcnt; 2439 log.u_bbr.flex7 = (0x0000ffff & rack->r_ctl.rc_hpts_flags); 2440 log.u_bbr.flex8 = rack->rc_in_persist; 2441 log.u_bbr.timeStamp = cts; 2442 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2443 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2444 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2445 log.u_bbr.pacing_gain = rack->r_must_retran; 2446 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2447 &rack->rc_inp->inp_socket->so_rcv, 2448 &rack->rc_inp->inp_socket->so_snd, 2449 BBR_LOG_BBRSND, 0, 2450 0, &log, false, tv); 2451 } 2452 } 2453 2454 static void 2455 rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_t did_out, int way_out, int nsegs) 2456 { 2457 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2458 union tcp_log_stackspecific log; 2459 struct timeval tv; 2460 2461 memset(&log, 0, sizeof(log)); 2462 log.u_bbr.flex1 = did_out; 2463 log.u_bbr.flex2 = nxt_pkt; 2464 log.u_bbr.flex3 = way_out; 2465 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags; 2466 if (rack->rack_no_prr) 2467 log.u_bbr.flex5 = 0; 2468 else 2469 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt; 2470 log.u_bbr.flex6 = nsegs; 2471 log.u_bbr.applimited = rack->r_ctl.rc_pace_min_segs; 2472 log.u_bbr.flex7 = rack->rc_ack_can_sendout_data; /* Do we have ack-can-send set */ 2473 log.u_bbr.flex7 <<= 1; 2474 log.u_bbr.flex7 |= rack->r_fast_output; /* is fast output primed */ 2475 log.u_bbr.flex7 <<= 1; 2476 log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */ 2477 log.u_bbr.flex8 = rack->rc_in_persist; 2478 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2479 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2480 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2481 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; 2482 log.u_bbr.use_lt_bw <<= 1; 2483 log.u_bbr.use_lt_bw |= rack->r_might_revert; 2484 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2485 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2486 log.u_bbr.pacing_gain = rack->r_must_retran; 2487 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2488 &rack->rc_inp->inp_socket->so_rcv, 2489 &rack->rc_inp->inp_socket->so_snd, 2490 BBR_LOG_DOSEG_DONE, 0, 2491 0, &log, false, &tv); 2492 } 2493 } 2494 2495 static void 2496 rack_log_type_pacing_sizes(struct tcpcb *tp, struct tcp_rack *rack, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint8_t frm) 2497 { 2498 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 2499 union tcp_log_stackspecific log; 2500 struct timeval tv; 2501 2502 memset(&log, 0, sizeof(log)); 2503 log.u_bbr.flex1 = rack->r_ctl.rc_pace_min_segs; 2504 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs; 2505 log.u_bbr.flex4 = arg1; 2506 log.u_bbr.flex5 = arg2; 2507 log.u_bbr.flex6 = arg3; 2508 log.u_bbr.flex8 = frm; 2509 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2510 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2511 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2512 log.u_bbr.applimited = rack->r_ctl.rc_sacked; 2513 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2514 log.u_bbr.pacing_gain = rack->r_must_retran; 2515 TCP_LOG_EVENTP(tp, NULL, &tptosocket(tp)->so_rcv, 2516 &tptosocket(tp)->so_snd, 2517 TCP_HDWR_PACE_SIZE, 0, 0, &log, false, &tv); 2518 } 2519 } 2520 2521 static void 2522 rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, uint32_t slot, 2523 uint8_t hpts_calling, int reason, uint32_t cwnd_to_use) 2524 { 2525 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2526 union tcp_log_stackspecific log; 2527 struct timeval tv; 2528 2529 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2530 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2531 log.u_bbr.flex1 = slot; 2532 log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags; 2533 log.u_bbr.flex4 = reason; 2534 if (rack->rack_no_prr) 2535 log.u_bbr.flex5 = 0; 2536 else 2537 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt; 2538 log.u_bbr.flex7 = hpts_calling; 2539 log.u_bbr.flex8 = rack->rc_in_persist; 2540 log.u_bbr.lt_epoch = cwnd_to_use; 2541 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2542 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2543 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2544 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2545 log.u_bbr.pacing_gain = rack->r_must_retran; 2546 log.u_bbr.cwnd_gain = rack->rc_has_collapsed; 2547 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2548 &rack->rc_inp->inp_socket->so_rcv, 2549 &rack->rc_inp->inp_socket->so_snd, 2550 BBR_LOG_JUSTRET, 0, 2551 tlen, &log, false, &tv); 2552 } 2553 } 2554 2555 static void 2556 rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32_t us_cts, 2557 struct timeval *tv, uint32_t flags_on_entry) 2558 { 2559 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2560 union tcp_log_stackspecific log; 2561 2562 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2563 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 2564 log.u_bbr.flex1 = line; 2565 log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to; 2566 log.u_bbr.flex3 = flags_on_entry; 2567 log.u_bbr.flex4 = us_cts; 2568 if (rack->rack_no_prr) 2569 log.u_bbr.flex5 = 0; 2570 else 2571 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt; 2572 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur; 2573 log.u_bbr.flex7 = hpts_removed; 2574 log.u_bbr.flex8 = 1; 2575 log.u_bbr.applimited = rack->r_ctl.rc_hpts_flags; 2576 log.u_bbr.timeStamp = us_cts; 2577 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2578 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2579 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2580 log.u_bbr.pacing_gain = rack->r_must_retran; 2581 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2582 &rack->rc_inp->inp_socket->so_rcv, 2583 &rack->rc_inp->inp_socket->so_snd, 2584 BBR_LOG_TIMERCANC, 0, 2585 0, &log, false, tv); 2586 } 2587 } 2588 2589 static void 2590 rack_log_alt_to_to_cancel(struct tcp_rack *rack, 2591 uint32_t flex1, uint32_t flex2, 2592 uint32_t flex3, uint32_t flex4, 2593 uint32_t flex5, uint32_t flex6, 2594 uint16_t flex7, uint8_t mod) 2595 { 2596 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2597 union tcp_log_stackspecific log; 2598 struct timeval tv; 2599 2600 if (mod == 1) { 2601 /* No you can't use 1, its for the real to cancel */ 2602 return; 2603 } 2604 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2605 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2606 log.u_bbr.flex1 = flex1; 2607 log.u_bbr.flex2 = flex2; 2608 log.u_bbr.flex3 = flex3; 2609 log.u_bbr.flex4 = flex4; 2610 log.u_bbr.flex5 = flex5; 2611 log.u_bbr.flex6 = flex6; 2612 log.u_bbr.flex7 = flex7; 2613 log.u_bbr.flex8 = mod; 2614 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2615 &rack->rc_inp->inp_socket->so_rcv, 2616 &rack->rc_inp->inp_socket->so_snd, 2617 BBR_LOG_TIMERCANC, 0, 2618 0, &log, false, &tv); 2619 } 2620 } 2621 2622 static void 2623 rack_log_to_processing(struct tcp_rack *rack, uint32_t cts, int32_t ret, int32_t timers) 2624 { 2625 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2626 union tcp_log_stackspecific log; 2627 struct timeval tv; 2628 2629 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2630 log.u_bbr.flex1 = timers; 2631 log.u_bbr.flex2 = ret; 2632 log.u_bbr.flex3 = rack->r_ctl.rc_timer_exp; 2633 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags; 2634 log.u_bbr.flex5 = cts; 2635 if (rack->rack_no_prr) 2636 log.u_bbr.flex6 = 0; 2637 else 2638 log.u_bbr.flex6 = rack->r_ctl.rc_prr_sndcnt; 2639 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto; 2640 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto; 2641 log.u_bbr.pacing_gain = rack->r_must_retran; 2642 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2643 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2644 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2645 &rack->rc_inp->inp_socket->so_rcv, 2646 &rack->rc_inp->inp_socket->so_snd, 2647 BBR_LOG_TO_PROCESS, 0, 2648 0, &log, false, &tv); 2649 } 2650 } 2651 2652 static void 2653 rack_log_to_prr(struct tcp_rack *rack, int frm, int orig_cwnd, int line) 2654 { 2655 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2656 union tcp_log_stackspecific log; 2657 struct timeval tv; 2658 2659 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2660 log.u_bbr.flex1 = rack->r_ctl.rc_prr_out; 2661 log.u_bbr.flex2 = rack->r_ctl.rc_prr_recovery_fs; 2662 if (rack->rack_no_prr) 2663 log.u_bbr.flex3 = 0; 2664 else 2665 log.u_bbr.flex3 = rack->r_ctl.rc_prr_sndcnt; 2666 log.u_bbr.flex4 = rack->r_ctl.rc_prr_delivered; 2667 log.u_bbr.flex5 = rack->r_ctl.rc_sacked; 2668 log.u_bbr.flex6 = rack->r_ctl.rc_holes_rxt; 2669 log.u_bbr.flex7 = line; 2670 log.u_bbr.flex8 = frm; 2671 log.u_bbr.pkts_out = orig_cwnd; 2672 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2673 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2674 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; 2675 log.u_bbr.use_lt_bw <<= 1; 2676 log.u_bbr.use_lt_bw |= rack->r_might_revert; 2677 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2678 &rack->rc_inp->inp_socket->so_rcv, 2679 &rack->rc_inp->inp_socket->so_snd, 2680 BBR_LOG_BBRUPD, 0, 2681 0, &log, false, &tv); 2682 } 2683 } 2684 2685 #ifdef NETFLIX_EXP_DETECTION 2686 static void 2687 rack_log_sad(struct tcp_rack *rack, int event) 2688 { 2689 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 2690 union tcp_log_stackspecific log; 2691 struct timeval tv; 2692 2693 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 2694 log.u_bbr.flex1 = rack->r_ctl.sack_count; 2695 log.u_bbr.flex2 = rack->r_ctl.ack_count; 2696 log.u_bbr.flex3 = rack->r_ctl.sack_moved_extra; 2697 log.u_bbr.flex4 = rack->r_ctl.sack_noextra_move; 2698 log.u_bbr.flex5 = rack->r_ctl.rc_num_maps_alloced; 2699 log.u_bbr.flex6 = tcp_sack_to_ack_thresh; 2700 log.u_bbr.pkts_out = tcp_sack_to_move_thresh; 2701 log.u_bbr.lt_epoch = (tcp_force_detection << 8); 2702 log.u_bbr.lt_epoch |= rack->do_detection; 2703 log.u_bbr.applimited = tcp_map_minimum; 2704 log.u_bbr.flex7 = rack->sack_attack_disable; 2705 log.u_bbr.flex8 = event; 2706 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 2707 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 2708 log.u_bbr.delivered = tcp_sad_decay_val; 2709 TCP_LOG_EVENTP(rack->rc_tp, NULL, 2710 &rack->rc_inp->inp_socket->so_rcv, 2711 &rack->rc_inp->inp_socket->so_snd, 2712 TCP_SAD_DETECTION, 0, 2713 0, &log, false, &tv); 2714 } 2715 } 2716 #endif 2717 2718 static void 2719 rack_counter_destroy(void) 2720 { 2721 counter_u64_free(rack_fto_send); 2722 counter_u64_free(rack_fto_rsm_send); 2723 counter_u64_free(rack_nfto_resend); 2724 counter_u64_free(rack_hw_pace_init_fail); 2725 counter_u64_free(rack_hw_pace_lost); 2726 counter_u64_free(rack_non_fto_send); 2727 counter_u64_free(rack_extended_rfo); 2728 counter_u64_free(rack_ack_total); 2729 counter_u64_free(rack_express_sack); 2730 counter_u64_free(rack_sack_total); 2731 counter_u64_free(rack_move_none); 2732 counter_u64_free(rack_move_some); 2733 counter_u64_free(rack_sack_attacks_detected); 2734 counter_u64_free(rack_sack_attacks_reversed); 2735 counter_u64_free(rack_sack_used_next_merge); 2736 counter_u64_free(rack_sack_used_prev_merge); 2737 counter_u64_free(rack_tlp_tot); 2738 counter_u64_free(rack_tlp_newdata); 2739 counter_u64_free(rack_tlp_retran); 2740 counter_u64_free(rack_tlp_retran_bytes); 2741 counter_u64_free(rack_to_tot); 2742 counter_u64_free(rack_saw_enobuf); 2743 counter_u64_free(rack_saw_enobuf_hw); 2744 counter_u64_free(rack_saw_enetunreach); 2745 counter_u64_free(rack_hot_alloc); 2746 counter_u64_free(rack_to_alloc); 2747 counter_u64_free(rack_to_alloc_hard); 2748 counter_u64_free(rack_to_alloc_emerg); 2749 counter_u64_free(rack_to_alloc_limited); 2750 counter_u64_free(rack_alloc_limited_conns); 2751 counter_u64_free(rack_split_limited); 2752 counter_u64_free(rack_multi_single_eq); 2753 counter_u64_free(rack_proc_non_comp_ack); 2754 counter_u64_free(rack_sack_proc_all); 2755 counter_u64_free(rack_sack_proc_restart); 2756 counter_u64_free(rack_sack_proc_short); 2757 counter_u64_free(rack_sack_skipped_acked); 2758 counter_u64_free(rack_sack_splits); 2759 counter_u64_free(rack_input_idle_reduces); 2760 counter_u64_free(rack_collapsed_win); 2761 counter_u64_free(rack_collapsed_win_rxt); 2762 counter_u64_free(rack_collapsed_win_rxt_bytes); 2763 counter_u64_free(rack_collapsed_win_seen); 2764 counter_u64_free(rack_try_scwnd); 2765 counter_u64_free(rack_persists_sends); 2766 counter_u64_free(rack_persists_acks); 2767 counter_u64_free(rack_persists_loss); 2768 counter_u64_free(rack_persists_lost_ends); 2769 #ifdef INVARIANTS 2770 counter_u64_free(rack_adjust_map_bw); 2771 #endif 2772 COUNTER_ARRAY_FREE(rack_out_size, TCP_MSS_ACCT_SIZE); 2773 COUNTER_ARRAY_FREE(rack_opts_arry, RACK_OPTS_SIZE); 2774 } 2775 2776 static struct rack_sendmap * 2777 rack_alloc(struct tcp_rack *rack) 2778 { 2779 struct rack_sendmap *rsm; 2780 2781 /* 2782 * First get the top of the list it in 2783 * theory is the "hottest" rsm we have, 2784 * possibly just freed by ack processing. 2785 */ 2786 if (rack->rc_free_cnt > rack_free_cache) { 2787 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free); 2788 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext); 2789 counter_u64_add(rack_hot_alloc, 1); 2790 rack->rc_free_cnt--; 2791 return (rsm); 2792 } 2793 /* 2794 * Once we get under our free cache we probably 2795 * no longer have a "hot" one available. Lets 2796 * get one from UMA. 2797 */ 2798 rsm = uma_zalloc(rack_zone, M_NOWAIT); 2799 if (rsm) { 2800 rack->r_ctl.rc_num_maps_alloced++; 2801 counter_u64_add(rack_to_alloc, 1); 2802 return (rsm); 2803 } 2804 /* 2805 * Dig in to our aux rsm's (the last two) since 2806 * UMA failed to get us one. 2807 */ 2808 if (rack->rc_free_cnt) { 2809 counter_u64_add(rack_to_alloc_emerg, 1); 2810 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free); 2811 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext); 2812 rack->rc_free_cnt--; 2813 return (rsm); 2814 } 2815 return (NULL); 2816 } 2817 2818 static struct rack_sendmap * 2819 rack_alloc_full_limit(struct tcp_rack *rack) 2820 { 2821 if ((V_tcp_map_entries_limit > 0) && 2822 (rack->do_detection == 0) && 2823 (rack->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { 2824 counter_u64_add(rack_to_alloc_limited, 1); 2825 if (!rack->alloc_limit_reported) { 2826 rack->alloc_limit_reported = 1; 2827 counter_u64_add(rack_alloc_limited_conns, 1); 2828 } 2829 return (NULL); 2830 } 2831 return (rack_alloc(rack)); 2832 } 2833 2834 /* wrapper to allocate a sendmap entry, subject to a specific limit */ 2835 static struct rack_sendmap * 2836 rack_alloc_limit(struct tcp_rack *rack, uint8_t limit_type) 2837 { 2838 struct rack_sendmap *rsm; 2839 2840 if (limit_type) { 2841 /* currently there is only one limit type */ 2842 if (V_tcp_map_split_limit > 0 && 2843 (rack->do_detection == 0) && 2844 rack->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) { 2845 counter_u64_add(rack_split_limited, 1); 2846 if (!rack->alloc_limit_reported) { 2847 rack->alloc_limit_reported = 1; 2848 counter_u64_add(rack_alloc_limited_conns, 1); 2849 } 2850 return (NULL); 2851 } 2852 } 2853 2854 /* allocate and mark in the limit type, if set */ 2855 rsm = rack_alloc(rack); 2856 if (rsm != NULL && limit_type) { 2857 rsm->r_limit_type = limit_type; 2858 rack->r_ctl.rc_num_split_allocs++; 2859 } 2860 return (rsm); 2861 } 2862 2863 static void 2864 rack_free(struct tcp_rack *rack, struct rack_sendmap *rsm) 2865 { 2866 if (rsm->r_flags & RACK_APP_LIMITED) { 2867 if (rack->r_ctl.rc_app_limited_cnt > 0) { 2868 rack->r_ctl.rc_app_limited_cnt--; 2869 } 2870 } 2871 if (rsm->r_limit_type) { 2872 /* currently there is only one limit type */ 2873 rack->r_ctl.rc_num_split_allocs--; 2874 } 2875 if (rsm == rack->r_ctl.rc_first_appl) { 2876 if (rack->r_ctl.rc_app_limited_cnt == 0) 2877 rack->r_ctl.rc_first_appl = NULL; 2878 else { 2879 /* Follow the next one out */ 2880 struct rack_sendmap fe; 2881 2882 fe.r_start = rsm->r_nseq_appl; 2883 rack->r_ctl.rc_first_appl = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 2884 } 2885 } 2886 if (rsm == rack->r_ctl.rc_resend) 2887 rack->r_ctl.rc_resend = NULL; 2888 if (rsm == rack->r_ctl.rc_end_appl) 2889 rack->r_ctl.rc_end_appl = NULL; 2890 if (rack->r_ctl.rc_tlpsend == rsm) 2891 rack->r_ctl.rc_tlpsend = NULL; 2892 if (rack->r_ctl.rc_sacklast == rsm) 2893 rack->r_ctl.rc_sacklast = NULL; 2894 memset(rsm, 0, sizeof(struct rack_sendmap)); 2895 TAILQ_INSERT_HEAD(&rack->r_ctl.rc_free, rsm, r_tnext); 2896 rack->rc_free_cnt++; 2897 } 2898 2899 static void 2900 rack_free_trim(struct tcp_rack *rack) 2901 { 2902 struct rack_sendmap *rsm; 2903 2904 /* 2905 * Free up all the tail entries until 2906 * we get our list down to the limit. 2907 */ 2908 while (rack->rc_free_cnt > rack_free_cache) { 2909 rsm = TAILQ_LAST(&rack->r_ctl.rc_free, rack_head); 2910 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext); 2911 rack->rc_free_cnt--; 2912 uma_zfree(rack_zone, rsm); 2913 } 2914 } 2915 2916 2917 static uint32_t 2918 rack_get_measure_window(struct tcpcb *tp, struct tcp_rack *rack) 2919 { 2920 uint64_t srtt, bw, len, tim; 2921 uint32_t segsiz, def_len, minl; 2922 2923 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 2924 def_len = rack_def_data_window * segsiz; 2925 if (rack->rc_gp_filled == 0) { 2926 /* 2927 * We have no measurement (IW is in flight?) so 2928 * we can only guess using our data_window sysctl 2929 * value (usually 20MSS). 2930 */ 2931 return (def_len); 2932 } 2933 /* 2934 * Now we have a number of factors to consider. 2935 * 2936 * 1) We have a desired BDP which is usually 2937 * at least 2. 2938 * 2) We have a minimum number of rtt's usually 1 SRTT 2939 * but we allow it too to be more. 2940 * 3) We want to make sure a measurement last N useconds (if 2941 * we have set rack_min_measure_usec. 2942 * 2943 * We handle the first concern here by trying to create a data 2944 * window of max(rack_def_data_window, DesiredBDP). The 2945 * second concern we handle in not letting the measurement 2946 * window end normally until at least the required SRTT's 2947 * have gone by which is done further below in 2948 * rack_enough_for_measurement(). Finally the third concern 2949 * we also handle here by calculating how long that time 2950 * would take at the current BW and then return the 2951 * max of our first calculation and that length. Note 2952 * that if rack_min_measure_usec is 0, we don't deal 2953 * with concern 3. Also for both Concern 1 and 3 an 2954 * application limited period could end the measurement 2955 * earlier. 2956 * 2957 * So lets calculate the BDP with the "known" b/w using 2958 * the SRTT has our rtt and then multiply it by the 2959 * goal. 2960 */ 2961 bw = rack_get_bw(rack); 2962 srtt = (uint64_t)tp->t_srtt; 2963 len = bw * srtt; 2964 len /= (uint64_t)HPTS_USEC_IN_SEC; 2965 len *= max(1, rack_goal_bdp); 2966 /* Now we need to round up to the nearest MSS */ 2967 len = roundup(len, segsiz); 2968 if (rack_min_measure_usec) { 2969 /* Now calculate our min length for this b/w */ 2970 tim = rack_min_measure_usec; 2971 minl = (tim * bw) / (uint64_t)HPTS_USEC_IN_SEC; 2972 if (minl == 0) 2973 minl = 1; 2974 minl = roundup(minl, segsiz); 2975 if (len < minl) 2976 len = minl; 2977 } 2978 /* 2979 * Now if we have a very small window we want 2980 * to attempt to get the window that is 2981 * as small as possible. This happens on 2982 * low b/w connections and we don't want to 2983 * span huge numbers of rtt's between measurements. 2984 * 2985 * We basically include 2 over our "MIN window" so 2986 * that the measurement can be shortened (possibly) by 2987 * an ack'ed packet. 2988 */ 2989 if (len < def_len) 2990 return (max((uint32_t)len, ((MIN_GP_WIN+2) * segsiz))); 2991 else 2992 return (max((uint32_t)len, def_len)); 2993 2994 } 2995 2996 static int 2997 rack_enough_for_measurement(struct tcpcb *tp, struct tcp_rack *rack, tcp_seq th_ack, uint8_t *quality) 2998 { 2999 uint32_t tim, srtts, segsiz; 3000 3001 /* 3002 * Has enough time passed for the GP measurement to be valid? 3003 */ 3004 if ((tp->snd_max == tp->snd_una) || 3005 (th_ack == tp->snd_max)){ 3006 /* All is acked */ 3007 *quality = RACK_QUALITY_ALLACKED; 3008 return (1); 3009 } 3010 if (SEQ_LT(th_ack, tp->gput_seq)) { 3011 /* Not enough bytes yet */ 3012 return (0); 3013 } 3014 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 3015 if (SEQ_LT(th_ack, tp->gput_ack) && 3016 ((th_ack - tp->gput_seq) < max(rc_init_window(rack), (MIN_GP_WIN * segsiz)))) { 3017 /* Not enough bytes yet */ 3018 return (0); 3019 } 3020 if (rack->r_ctl.rc_first_appl && 3021 (SEQ_GEQ(th_ack, rack->r_ctl.rc_first_appl->r_end))) { 3022 /* 3023 * We are up to the app limited send point 3024 * we have to measure irrespective of the time.. 3025 */ 3026 *quality = RACK_QUALITY_APPLIMITED; 3027 return (1); 3028 } 3029 /* Now what about time? */ 3030 srtts = (rack->r_ctl.rc_gp_srtt * rack_min_srtts); 3031 tim = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - tp->gput_ts; 3032 if (tim >= srtts) { 3033 *quality = RACK_QUALITY_HIGH; 3034 return (1); 3035 } 3036 /* Nope not even a full SRTT has passed */ 3037 return (0); 3038 } 3039 3040 static void 3041 rack_log_timely(struct tcp_rack *rack, 3042 uint32_t logged, uint64_t cur_bw, uint64_t low_bnd, 3043 uint64_t up_bnd, int line, uint8_t method) 3044 { 3045 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 3046 union tcp_log_stackspecific log; 3047 struct timeval tv; 3048 3049 memset(&log, 0, sizeof(log)); 3050 log.u_bbr.flex1 = logged; 3051 log.u_bbr.flex2 = rack->rc_gp_timely_inc_cnt; 3052 log.u_bbr.flex2 <<= 4; 3053 log.u_bbr.flex2 |= rack->rc_gp_timely_dec_cnt; 3054 log.u_bbr.flex2 <<= 4; 3055 log.u_bbr.flex2 |= rack->rc_gp_incr; 3056 log.u_bbr.flex2 <<= 4; 3057 log.u_bbr.flex2 |= rack->rc_gp_bwred; 3058 log.u_bbr.flex3 = rack->rc_gp_incr; 3059 log.u_bbr.flex4 = rack->r_ctl.rack_per_of_gp_ss; 3060 log.u_bbr.flex5 = rack->r_ctl.rack_per_of_gp_ca; 3061 log.u_bbr.flex6 = rack->r_ctl.rack_per_of_gp_rec; 3062 log.u_bbr.flex7 = rack->rc_gp_bwred; 3063 log.u_bbr.flex8 = method; 3064 log.u_bbr.cur_del_rate = cur_bw; 3065 log.u_bbr.delRate = low_bnd; 3066 log.u_bbr.bw_inuse = up_bnd; 3067 log.u_bbr.rttProp = rack_get_bw(rack); 3068 log.u_bbr.pkt_epoch = line; 3069 log.u_bbr.pkts_out = rack->r_ctl.rc_rtt_diff; 3070 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 3071 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 3072 log.u_bbr.epoch = rack->r_ctl.rc_gp_srtt; 3073 log.u_bbr.lt_epoch = rack->r_ctl.rc_prev_gp_srtt; 3074 log.u_bbr.cwnd_gain = rack->rc_dragged_bottom; 3075 log.u_bbr.cwnd_gain <<= 1; 3076 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_rec; 3077 log.u_bbr.cwnd_gain <<= 1; 3078 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ss; 3079 log.u_bbr.cwnd_gain <<= 1; 3080 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ca; 3081 log.u_bbr.lost = rack->r_ctl.rc_loss_count; 3082 TCP_LOG_EVENTP(rack->rc_tp, NULL, 3083 &rack->rc_inp->inp_socket->so_rcv, 3084 &rack->rc_inp->inp_socket->so_snd, 3085 TCP_TIMELY_WORK, 0, 3086 0, &log, false, &tv); 3087 } 3088 } 3089 3090 static int 3091 rack_bw_can_be_raised(struct tcp_rack *rack, uint64_t cur_bw, uint64_t last_bw_est, uint16_t mult) 3092 { 3093 /* 3094 * Before we increase we need to know if 3095 * the estimate just made was less than 3096 * our pacing goal (i.e. (cur_bw * mult) > last_bw_est) 3097 * 3098 * If we already are pacing at a fast enough 3099 * rate to push us faster there is no sense of 3100 * increasing. 3101 * 3102 * We first caculate our actual pacing rate (ss or ca multiplier 3103 * times our cur_bw). 3104 * 3105 * Then we take the last measured rate and multipy by our 3106 * maximum pacing overage to give us a max allowable rate. 3107 * 3108 * If our act_rate is smaller than our max_allowable rate 3109 * then we should increase. Else we should hold steady. 3110 * 3111 */ 3112 uint64_t act_rate, max_allow_rate; 3113 3114 if (rack_timely_no_stopping) 3115 return (1); 3116 3117 if ((cur_bw == 0) || (last_bw_est == 0)) { 3118 /* 3119 * Initial startup case or 3120 * everything is acked case. 3121 */ 3122 rack_log_timely(rack, mult, cur_bw, 0, 0, 3123 __LINE__, 9); 3124 return (1); 3125 } 3126 if (mult <= 100) { 3127 /* 3128 * We can always pace at or slightly above our rate. 3129 */ 3130 rack_log_timely(rack, mult, cur_bw, 0, 0, 3131 __LINE__, 9); 3132 return (1); 3133 } 3134 act_rate = cur_bw * (uint64_t)mult; 3135 act_rate /= 100; 3136 max_allow_rate = last_bw_est * ((uint64_t)rack_max_per_above + (uint64_t)100); 3137 max_allow_rate /= 100; 3138 if (act_rate < max_allow_rate) { 3139 /* 3140 * Here the rate we are actually pacing at 3141 * is smaller than 10% above our last measurement. 3142 * This means we are pacing below what we would 3143 * like to try to achieve (plus some wiggle room). 3144 */ 3145 rack_log_timely(rack, mult, cur_bw, act_rate, max_allow_rate, 3146 __LINE__, 9); 3147 return (1); 3148 } else { 3149 /* 3150 * Here we are already pacing at least rack_max_per_above(10%) 3151 * what we are getting back. This indicates most likely 3152 * that we are being limited (cwnd/rwnd/app) and can't 3153 * get any more b/w. There is no sense of trying to 3154 * raise up the pacing rate its not speeding us up 3155 * and we already are pacing faster than we are getting. 3156 */ 3157 rack_log_timely(rack, mult, cur_bw, act_rate, max_allow_rate, 3158 __LINE__, 8); 3159 return (0); 3160 } 3161 } 3162 3163 static void 3164 rack_validate_multipliers_at_or_above100(struct tcp_rack *rack) 3165 { 3166 /* 3167 * When we drag bottom, we want to assure 3168 * that no multiplier is below 1.0, if so 3169 * we want to restore it to at least that. 3170 */ 3171 if (rack->r_ctl.rack_per_of_gp_rec < 100) { 3172 /* This is unlikely we usually do not touch recovery */ 3173 rack->r_ctl.rack_per_of_gp_rec = 100; 3174 } 3175 if (rack->r_ctl.rack_per_of_gp_ca < 100) { 3176 rack->r_ctl.rack_per_of_gp_ca = 100; 3177 } 3178 if (rack->r_ctl.rack_per_of_gp_ss < 100) { 3179 rack->r_ctl.rack_per_of_gp_ss = 100; 3180 } 3181 } 3182 3183 static void 3184 rack_validate_multipliers_at_or_below_100(struct tcp_rack *rack) 3185 { 3186 if (rack->r_ctl.rack_per_of_gp_ca > 100) { 3187 rack->r_ctl.rack_per_of_gp_ca = 100; 3188 } 3189 if (rack->r_ctl.rack_per_of_gp_ss > 100) { 3190 rack->r_ctl.rack_per_of_gp_ss = 100; 3191 } 3192 } 3193 3194 static void 3195 rack_increase_bw_mul(struct tcp_rack *rack, int timely_says, uint64_t cur_bw, uint64_t last_bw_est, int override) 3196 { 3197 int32_t calc, logged, plus; 3198 3199 logged = 0; 3200 3201 if (override) { 3202 /* 3203 * override is passed when we are 3204 * loosing b/w and making one last 3205 * gasp at trying to not loose out 3206 * to a new-reno flow. 3207 */ 3208 goto extra_boost; 3209 } 3210 /* In classic timely we boost by 5x if we have 5 increases in a row, lets not */ 3211 if (rack->rc_gp_incr && 3212 ((rack->rc_gp_timely_inc_cnt + 1) >= RACK_TIMELY_CNT_BOOST)) { 3213 /* 3214 * Reset and get 5 strokes more before the boost. Note 3215 * that the count is 0 based so we have to add one. 3216 */ 3217 extra_boost: 3218 plus = (uint32_t)rack_gp_increase_per * RACK_TIMELY_CNT_BOOST; 3219 rack->rc_gp_timely_inc_cnt = 0; 3220 } else 3221 plus = (uint32_t)rack_gp_increase_per; 3222 /* Must be at least 1% increase for true timely increases */ 3223 if ((plus < 1) && 3224 ((rack->r_ctl.rc_rtt_diff <= 0) || (timely_says <= 0))) 3225 plus = 1; 3226 if (rack->rc_gp_saw_rec && 3227 (rack->rc_gp_no_rec_chg == 0) && 3228 rack_bw_can_be_raised(rack, cur_bw, last_bw_est, 3229 rack->r_ctl.rack_per_of_gp_rec)) { 3230 /* We have been in recovery ding it too */ 3231 calc = rack->r_ctl.rack_per_of_gp_rec + plus; 3232 if (calc > 0xffff) 3233 calc = 0xffff; 3234 logged |= 1; 3235 rack->r_ctl.rack_per_of_gp_rec = (uint16_t)calc; 3236 if (rack_per_upper_bound_ss && 3237 (rack->rc_dragged_bottom == 0) && 3238 (rack->r_ctl.rack_per_of_gp_rec > rack_per_upper_bound_ss)) 3239 rack->r_ctl.rack_per_of_gp_rec = rack_per_upper_bound_ss; 3240 } 3241 if (rack->rc_gp_saw_ca && 3242 (rack->rc_gp_saw_ss == 0) && 3243 rack_bw_can_be_raised(rack, cur_bw, last_bw_est, 3244 rack->r_ctl.rack_per_of_gp_ca)) { 3245 /* In CA */ 3246 calc = rack->r_ctl.rack_per_of_gp_ca + plus; 3247 if (calc > 0xffff) 3248 calc = 0xffff; 3249 logged |= 2; 3250 rack->r_ctl.rack_per_of_gp_ca = (uint16_t)calc; 3251 if (rack_per_upper_bound_ca && 3252 (rack->rc_dragged_bottom == 0) && 3253 (rack->r_ctl.rack_per_of_gp_ca > rack_per_upper_bound_ca)) 3254 rack->r_ctl.rack_per_of_gp_ca = rack_per_upper_bound_ca; 3255 } 3256 if (rack->rc_gp_saw_ss && 3257 rack_bw_can_be_raised(rack, cur_bw, last_bw_est, 3258 rack->r_ctl.rack_per_of_gp_ss)) { 3259 /* In SS */ 3260 calc = rack->r_ctl.rack_per_of_gp_ss + plus; 3261 if (calc > 0xffff) 3262 calc = 0xffff; 3263 rack->r_ctl.rack_per_of_gp_ss = (uint16_t)calc; 3264 if (rack_per_upper_bound_ss && 3265 (rack->rc_dragged_bottom == 0) && 3266 (rack->r_ctl.rack_per_of_gp_ss > rack_per_upper_bound_ss)) 3267 rack->r_ctl.rack_per_of_gp_ss = rack_per_upper_bound_ss; 3268 logged |= 4; 3269 } 3270 if (logged && 3271 (rack->rc_gp_incr == 0)){ 3272 /* Go into increment mode */ 3273 rack->rc_gp_incr = 1; 3274 rack->rc_gp_timely_inc_cnt = 0; 3275 } 3276 if (rack->rc_gp_incr && 3277 logged && 3278 (rack->rc_gp_timely_inc_cnt < RACK_TIMELY_CNT_BOOST)) { 3279 rack->rc_gp_timely_inc_cnt++; 3280 } 3281 rack_log_timely(rack, logged, plus, 0, 0, 3282 __LINE__, 1); 3283 } 3284 3285 static uint32_t 3286 rack_get_decrease(struct tcp_rack *rack, uint32_t curper, int32_t rtt_diff) 3287 { 3288 /* 3289 * norm_grad = rtt_diff / minrtt; 3290 * new_per = curper * (1 - B * norm_grad) 3291 * 3292 * B = rack_gp_decrease_per (default 10%) 3293 * rtt_dif = input var current rtt-diff 3294 * curper = input var current percentage 3295 * minrtt = from rack filter 3296 * 3297 */ 3298 uint64_t perf; 3299 3300 perf = (((uint64_t)curper * ((uint64_t)1000000 - 3301 ((uint64_t)rack_gp_decrease_per * (uint64_t)10000 * 3302 (((uint64_t)rtt_diff * (uint64_t)1000000)/ 3303 (uint64_t)get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt)))/ 3304 (uint64_t)1000000)) / 3305 (uint64_t)1000000); 3306 if (perf > curper) { 3307 /* TSNH */ 3308 perf = curper - 1; 3309 } 3310 return ((uint32_t)perf); 3311 } 3312 3313 static uint32_t 3314 rack_decrease_highrtt(struct tcp_rack *rack, uint32_t curper, uint32_t rtt) 3315 { 3316 /* 3317 * highrttthresh 3318 * result = curper * (1 - (B * ( 1 - ------ )) 3319 * gp_srtt 3320 * 3321 * B = rack_gp_decrease_per (default 10%) 3322 * highrttthresh = filter_min * rack_gp_rtt_maxmul 3323 */ 3324 uint64_t perf; 3325 uint32_t highrttthresh; 3326 3327 highrttthresh = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_maxmul; 3328 3329 perf = (((uint64_t)curper * ((uint64_t)1000000 - 3330 ((uint64_t)rack_gp_decrease_per * ((uint64_t)1000000 - 3331 ((uint64_t)highrttthresh * (uint64_t)1000000) / 3332 (uint64_t)rtt)) / 100)) /(uint64_t)1000000); 3333 return (perf); 3334 } 3335 3336 static void 3337 rack_decrease_bw_mul(struct tcp_rack *rack, int timely_says, uint32_t rtt, int32_t rtt_diff) 3338 { 3339 uint64_t logvar, logvar2, logvar3; 3340 uint32_t logged, new_per, ss_red, ca_red, rec_red, alt, val; 3341 3342 if (rack->rc_gp_incr) { 3343 /* Turn off increment counting */ 3344 rack->rc_gp_incr = 0; 3345 rack->rc_gp_timely_inc_cnt = 0; 3346 } 3347 ss_red = ca_red = rec_red = 0; 3348 logged = 0; 3349 /* Calculate the reduction value */ 3350 if (rtt_diff < 0) { 3351 rtt_diff *= -1; 3352 } 3353 /* Must be at least 1% reduction */ 3354 if (rack->rc_gp_saw_rec && (rack->rc_gp_no_rec_chg == 0)) { 3355 /* We have been in recovery ding it too */ 3356 if (timely_says == 2) { 3357 new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_rec, rtt); 3358 alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff); 3359 if (alt < new_per) 3360 val = alt; 3361 else 3362 val = new_per; 3363 } else 3364 val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff); 3365 if (rack->r_ctl.rack_per_of_gp_rec > val) { 3366 rec_red = (rack->r_ctl.rack_per_of_gp_rec - val); 3367 rack->r_ctl.rack_per_of_gp_rec = (uint16_t)val; 3368 } else { 3369 rack->r_ctl.rack_per_of_gp_rec = rack_per_lower_bound; 3370 rec_red = 0; 3371 } 3372 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_rec) 3373 rack->r_ctl.rack_per_of_gp_rec = rack_per_lower_bound; 3374 logged |= 1; 3375 } 3376 if (rack->rc_gp_saw_ss) { 3377 /* Sent in SS */ 3378 if (timely_says == 2) { 3379 new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ss, rtt); 3380 alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff); 3381 if (alt < new_per) 3382 val = alt; 3383 else 3384 val = new_per; 3385 } else 3386 val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ss, rtt_diff); 3387 if (rack->r_ctl.rack_per_of_gp_ss > new_per) { 3388 ss_red = rack->r_ctl.rack_per_of_gp_ss - val; 3389 rack->r_ctl.rack_per_of_gp_ss = (uint16_t)val; 3390 } else { 3391 ss_red = new_per; 3392 rack->r_ctl.rack_per_of_gp_ss = rack_per_lower_bound; 3393 logvar = new_per; 3394 logvar <<= 32; 3395 logvar |= alt; 3396 logvar2 = (uint32_t)rtt; 3397 logvar2 <<= 32; 3398 logvar2 |= (uint32_t)rtt_diff; 3399 logvar3 = rack_gp_rtt_maxmul; 3400 logvar3 <<= 32; 3401 logvar3 |= get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt); 3402 rack_log_timely(rack, timely_says, 3403 logvar2, logvar3, 3404 logvar, __LINE__, 10); 3405 } 3406 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_ss) 3407 rack->r_ctl.rack_per_of_gp_ss = rack_per_lower_bound; 3408 logged |= 4; 3409 } else if (rack->rc_gp_saw_ca) { 3410 /* Sent in CA */ 3411 if (timely_says == 2) { 3412 new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ca, rtt); 3413 alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff); 3414 if (alt < new_per) 3415 val = alt; 3416 else 3417 val = new_per; 3418 } else 3419 val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ca, rtt_diff); 3420 if (rack->r_ctl.rack_per_of_gp_ca > val) { 3421 ca_red = rack->r_ctl.rack_per_of_gp_ca - val; 3422 rack->r_ctl.rack_per_of_gp_ca = (uint16_t)val; 3423 } else { 3424 rack->r_ctl.rack_per_of_gp_ca = rack_per_lower_bound; 3425 ca_red = 0; 3426 logvar = new_per; 3427 logvar <<= 32; 3428 logvar |= alt; 3429 logvar2 = (uint32_t)rtt; 3430 logvar2 <<= 32; 3431 logvar2 |= (uint32_t)rtt_diff; 3432 logvar3 = rack_gp_rtt_maxmul; 3433 logvar3 <<= 32; 3434 logvar3 |= get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt); 3435 rack_log_timely(rack, timely_says, 3436 logvar2, logvar3, 3437 logvar, __LINE__, 10); 3438 } 3439 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_ca) 3440 rack->r_ctl.rack_per_of_gp_ca = rack_per_lower_bound; 3441 logged |= 2; 3442 } 3443 if (rack->rc_gp_timely_dec_cnt < 0x7) { 3444 rack->rc_gp_timely_dec_cnt++; 3445 if (rack_timely_dec_clear && 3446 (rack->rc_gp_timely_dec_cnt == rack_timely_dec_clear)) 3447 rack->rc_gp_timely_dec_cnt = 0; 3448 } 3449 logvar = ss_red; 3450 logvar <<= 32; 3451 logvar |= ca_red; 3452 rack_log_timely(rack, logged, rec_red, rack_per_lower_bound, logvar, 3453 __LINE__, 2); 3454 } 3455 3456 static void 3457 rack_log_rtt_shrinks(struct tcp_rack *rack, uint32_t us_cts, 3458 uint32_t rtt, uint32_t line, uint8_t reas) 3459 { 3460 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 3461 union tcp_log_stackspecific log; 3462 struct timeval tv; 3463 3464 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 3465 log.u_bbr.flex1 = line; 3466 log.u_bbr.flex2 = rack->r_ctl.rc_time_probertt_starts; 3467 log.u_bbr.flex3 = rack->r_ctl.rc_lower_rtt_us_cts; 3468 log.u_bbr.flex4 = rack->r_ctl.rack_per_of_gp_ss; 3469 log.u_bbr.flex5 = rtt; 3470 log.u_bbr.flex6 = rack->rc_highly_buffered; 3471 log.u_bbr.flex6 <<= 1; 3472 log.u_bbr.flex6 |= rack->forced_ack; 3473 log.u_bbr.flex6 <<= 1; 3474 log.u_bbr.flex6 |= rack->rc_gp_dyn_mul; 3475 log.u_bbr.flex6 <<= 1; 3476 log.u_bbr.flex6 |= rack->in_probe_rtt; 3477 log.u_bbr.flex6 <<= 1; 3478 log.u_bbr.flex6 |= rack->measure_saw_probe_rtt; 3479 log.u_bbr.flex7 = rack->r_ctl.rack_per_of_gp_probertt; 3480 log.u_bbr.pacing_gain = rack->r_ctl.rack_per_of_gp_ca; 3481 log.u_bbr.cwnd_gain = rack->r_ctl.rack_per_of_gp_rec; 3482 log.u_bbr.flex8 = reas; 3483 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 3484 log.u_bbr.delRate = rack_get_bw(rack); 3485 log.u_bbr.cur_del_rate = rack->r_ctl.rc_highest_us_rtt; 3486 log.u_bbr.cur_del_rate <<= 32; 3487 log.u_bbr.cur_del_rate |= rack->r_ctl.rc_lowest_us_rtt; 3488 log.u_bbr.applimited = rack->r_ctl.rc_time_probertt_entered; 3489 log.u_bbr.pkts_out = rack->r_ctl.rc_rtt_diff; 3490 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 3491 log.u_bbr.epoch = rack->r_ctl.rc_gp_srtt; 3492 log.u_bbr.lt_epoch = rack->r_ctl.rc_prev_gp_srtt; 3493 log.u_bbr.pkt_epoch = rack->r_ctl.rc_lower_rtt_us_cts; 3494 log.u_bbr.delivered = rack->r_ctl.rc_target_probertt_flight; 3495 log.u_bbr.lost = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt); 3496 log.u_bbr.rttProp = us_cts; 3497 log.u_bbr.rttProp <<= 32; 3498 log.u_bbr.rttProp |= rack->r_ctl.rc_entry_gp_rtt; 3499 TCP_LOG_EVENTP(rack->rc_tp, NULL, 3500 &rack->rc_inp->inp_socket->so_rcv, 3501 &rack->rc_inp->inp_socket->so_snd, 3502 BBR_LOG_RTT_SHRINKS, 0, 3503 0, &log, false, &rack->r_ctl.act_rcv_time); 3504 } 3505 } 3506 3507 static void 3508 rack_set_prtt_target(struct tcp_rack *rack, uint32_t segsiz, uint32_t rtt) 3509 { 3510 uint64_t bwdp; 3511 3512 bwdp = rack_get_bw(rack); 3513 bwdp *= (uint64_t)rtt; 3514 bwdp /= (uint64_t)HPTS_USEC_IN_SEC; 3515 rack->r_ctl.rc_target_probertt_flight = roundup((uint32_t)bwdp, segsiz); 3516 if (rack->r_ctl.rc_target_probertt_flight < (segsiz * rack_timely_min_segs)) { 3517 /* 3518 * A window protocol must be able to have 4 packets 3519 * outstanding as the floor in order to function 3520 * (especially considering delayed ack :D). 3521 */ 3522 rack->r_ctl.rc_target_probertt_flight = (segsiz * rack_timely_min_segs); 3523 } 3524 } 3525 3526 static void 3527 rack_enter_probertt(struct tcp_rack *rack, uint32_t us_cts) 3528 { 3529 /** 3530 * ProbeRTT is a bit different in rack_pacing than in 3531 * BBR. It is like BBR in that it uses the lowering of 3532 * the RTT as a signal that we saw something new and 3533 * counts from there for how long between. But it is 3534 * different in that its quite simple. It does not 3535 * play with the cwnd and wait until we get down 3536 * to N segments outstanding and hold that for 3537 * 200ms. Instead it just sets the pacing reduction 3538 * rate to a set percentage (70 by default) and hold 3539 * that for a number of recent GP Srtt's. 3540 */ 3541 uint32_t segsiz; 3542 3543 if (rack->rc_gp_dyn_mul == 0) 3544 return; 3545 3546 if (rack->rc_tp->snd_max == rack->rc_tp->snd_una) { 3547 /* We are idle */ 3548 return; 3549 } 3550 if ((rack->rc_tp->t_flags & TF_GPUTINPROG) && 3551 SEQ_GT(rack->rc_tp->snd_una, rack->rc_tp->gput_seq)) { 3552 /* 3553 * Stop the goodput now, the idea here is 3554 * that future measurements with in_probe_rtt 3555 * won't register if they are not greater so 3556 * we want to get what info (if any) is available 3557 * now. 3558 */ 3559 rack_do_goodput_measurement(rack->rc_tp, rack, 3560 rack->rc_tp->snd_una, __LINE__, 3561 RACK_QUALITY_PROBERTT); 3562 } 3563 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt; 3564 rack->r_ctl.rc_time_probertt_entered = us_cts; 3565 segsiz = min(ctf_fixed_maxseg(rack->rc_tp), 3566 rack->r_ctl.rc_pace_min_segs); 3567 rack->in_probe_rtt = 1; 3568 rack->measure_saw_probe_rtt = 1; 3569 rack->r_ctl.rc_lower_rtt_us_cts = us_cts; 3570 rack->r_ctl.rc_time_probertt_starts = 0; 3571 rack->r_ctl.rc_entry_gp_rtt = rack->r_ctl.rc_gp_srtt; 3572 if (rack_probertt_use_min_rtt_entry) 3573 rack_set_prtt_target(rack, segsiz, get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt)); 3574 else 3575 rack_set_prtt_target(rack, segsiz, rack->r_ctl.rc_gp_srtt); 3576 rack_log_rtt_shrinks(rack, us_cts, get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), 3577 __LINE__, RACK_RTTS_ENTERPROBE); 3578 } 3579 3580 static void 3581 rack_exit_probertt(struct tcp_rack *rack, uint32_t us_cts) 3582 { 3583 struct rack_sendmap *rsm; 3584 uint32_t segsiz; 3585 3586 segsiz = min(ctf_fixed_maxseg(rack->rc_tp), 3587 rack->r_ctl.rc_pace_min_segs); 3588 rack->in_probe_rtt = 0; 3589 if ((rack->rc_tp->t_flags & TF_GPUTINPROG) && 3590 SEQ_GT(rack->rc_tp->snd_una, rack->rc_tp->gput_seq)) { 3591 /* 3592 * Stop the goodput now, the idea here is 3593 * that future measurements with in_probe_rtt 3594 * won't register if they are not greater so 3595 * we want to get what info (if any) is available 3596 * now. 3597 */ 3598 rack_do_goodput_measurement(rack->rc_tp, rack, 3599 rack->rc_tp->snd_una, __LINE__, 3600 RACK_QUALITY_PROBERTT); 3601 } else if (rack->rc_tp->t_flags & TF_GPUTINPROG) { 3602 /* 3603 * We don't have enough data to make a measurement. 3604 * So lets just stop and start here after exiting 3605 * probe-rtt. We probably are not interested in 3606 * the results anyway. 3607 */ 3608 rack->rc_tp->t_flags &= ~TF_GPUTINPROG; 3609 } 3610 /* 3611 * Measurements through the current snd_max are going 3612 * to be limited by the slower pacing rate. 3613 * 3614 * We need to mark these as app-limited so we 3615 * don't collapse the b/w. 3616 */ 3617 rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 3618 if (rsm && ((rsm->r_flags & RACK_APP_LIMITED) == 0)) { 3619 if (rack->r_ctl.rc_app_limited_cnt == 0) 3620 rack->r_ctl.rc_end_appl = rack->r_ctl.rc_first_appl = rsm; 3621 else { 3622 /* 3623 * Go out to the end app limited and mark 3624 * this new one as next and move the end_appl up 3625 * to this guy. 3626 */ 3627 if (rack->r_ctl.rc_end_appl) 3628 rack->r_ctl.rc_end_appl->r_nseq_appl = rsm->r_start; 3629 rack->r_ctl.rc_end_appl = rsm; 3630 } 3631 rsm->r_flags |= RACK_APP_LIMITED; 3632 rack->r_ctl.rc_app_limited_cnt++; 3633 } 3634 /* 3635 * Now, we need to examine our pacing rate multipliers. 3636 * If its under 100%, we need to kick it back up to 3637 * 100%. We also don't let it be over our "max" above 3638 * the actual rate i.e. 100% + rack_clamp_atexit_prtt. 3639 * Note setting clamp_atexit_prtt to 0 has the effect 3640 * of setting CA/SS to 100% always at exit (which is 3641 * the default behavior). 3642 */ 3643 if (rack_probertt_clear_is) { 3644 rack->rc_gp_incr = 0; 3645 rack->rc_gp_bwred = 0; 3646 rack->rc_gp_timely_inc_cnt = 0; 3647 rack->rc_gp_timely_dec_cnt = 0; 3648 } 3649 /* Do we do any clamping at exit? */ 3650 if (rack->rc_highly_buffered && rack_atexit_prtt_hbp) { 3651 rack->r_ctl.rack_per_of_gp_ca = rack_atexit_prtt_hbp; 3652 rack->r_ctl.rack_per_of_gp_ss = rack_atexit_prtt_hbp; 3653 } 3654 if ((rack->rc_highly_buffered == 0) && rack_atexit_prtt) { 3655 rack->r_ctl.rack_per_of_gp_ca = rack_atexit_prtt; 3656 rack->r_ctl.rack_per_of_gp_ss = rack_atexit_prtt; 3657 } 3658 /* 3659 * Lets set rtt_diff to 0, so that we will get a "boost" 3660 * after exiting. 3661 */ 3662 rack->r_ctl.rc_rtt_diff = 0; 3663 3664 /* Clear all flags so we start fresh */ 3665 rack->rc_tp->t_bytes_acked = 0; 3666 rack->rc_tp->t_ccv.flags &= ~CCF_ABC_SENTAWND; 3667 /* 3668 * If configured to, set the cwnd and ssthresh to 3669 * our targets. 3670 */ 3671 if (rack_probe_rtt_sets_cwnd) { 3672 uint64_t ebdp; 3673 uint32_t setto; 3674 3675 /* Set ssthresh so we get into CA once we hit our target */ 3676 if (rack_probertt_use_min_rtt_exit == 1) { 3677 /* Set to min rtt */ 3678 rack_set_prtt_target(rack, segsiz, 3679 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt)); 3680 } else if (rack_probertt_use_min_rtt_exit == 2) { 3681 /* Set to current gp rtt */ 3682 rack_set_prtt_target(rack, segsiz, 3683 rack->r_ctl.rc_gp_srtt); 3684 } else if (rack_probertt_use_min_rtt_exit == 3) { 3685 /* Set to entry gp rtt */ 3686 rack_set_prtt_target(rack, segsiz, 3687 rack->r_ctl.rc_entry_gp_rtt); 3688 } else { 3689 uint64_t sum; 3690 uint32_t setval; 3691 3692 sum = rack->r_ctl.rc_entry_gp_rtt; 3693 sum *= 10; 3694 sum /= (uint64_t)(max(1, rack->r_ctl.rc_gp_srtt)); 3695 if (sum >= 20) { 3696 /* 3697 * A highly buffered path needs 3698 * cwnd space for timely to work. 3699 * Lets set things up as if 3700 * we are heading back here again. 3701 */ 3702 setval = rack->r_ctl.rc_entry_gp_rtt; 3703 } else if (sum >= 15) { 3704 /* 3705 * Lets take the smaller of the 3706 * two since we are just somewhat 3707 * buffered. 3708 */ 3709 setval = rack->r_ctl.rc_gp_srtt; 3710 if (setval > rack->r_ctl.rc_entry_gp_rtt) 3711 setval = rack->r_ctl.rc_entry_gp_rtt; 3712 } else { 3713 /* 3714 * Here we are not highly buffered 3715 * and should pick the min we can to 3716 * keep from causing loss. 3717 */ 3718 setval = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt); 3719 } 3720 rack_set_prtt_target(rack, segsiz, 3721 setval); 3722 } 3723 if (rack_probe_rtt_sets_cwnd > 1) { 3724 /* There is a percentage here to boost */ 3725 ebdp = rack->r_ctl.rc_target_probertt_flight; 3726 ebdp *= rack_probe_rtt_sets_cwnd; 3727 ebdp /= 100; 3728 setto = rack->r_ctl.rc_target_probertt_flight + ebdp; 3729 } else 3730 setto = rack->r_ctl.rc_target_probertt_flight; 3731 rack->rc_tp->snd_cwnd = roundup(setto, segsiz); 3732 if (rack->rc_tp->snd_cwnd < (segsiz * rack_timely_min_segs)) { 3733 /* Enforce a min */ 3734 rack->rc_tp->snd_cwnd = segsiz * rack_timely_min_segs; 3735 } 3736 /* If we set in the cwnd also set the ssthresh point so we are in CA */ 3737 rack->rc_tp->snd_ssthresh = (rack->rc_tp->snd_cwnd - 1); 3738 } 3739 rack_log_rtt_shrinks(rack, us_cts, 3740 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), 3741 __LINE__, RACK_RTTS_EXITPROBE); 3742 /* Clear times last so log has all the info */ 3743 rack->r_ctl.rc_probertt_sndmax_atexit = rack->rc_tp->snd_max; 3744 rack->r_ctl.rc_time_probertt_entered = us_cts; 3745 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts = us_cts; 3746 rack->r_ctl.rc_time_of_last_probertt = us_cts; 3747 } 3748 3749 static void 3750 rack_check_probe_rtt(struct tcp_rack *rack, uint32_t us_cts) 3751 { 3752 /* Check in on probe-rtt */ 3753 if (rack->rc_gp_filled == 0) { 3754 /* We do not do p-rtt unless we have gp measurements */ 3755 return; 3756 } 3757 if (rack->in_probe_rtt) { 3758 uint64_t no_overflow; 3759 uint32_t endtime, must_stay; 3760 3761 if (rack->r_ctl.rc_went_idle_time && 3762 ((us_cts - rack->r_ctl.rc_went_idle_time) > rack_min_probertt_hold)) { 3763 /* 3764 * We went idle during prtt, just exit now. 3765 */ 3766 rack_exit_probertt(rack, us_cts); 3767 } else if (rack_probe_rtt_safety_val && 3768 TSTMP_GT(us_cts, rack->r_ctl.rc_time_probertt_entered) && 3769 ((us_cts - rack->r_ctl.rc_time_probertt_entered) > rack_probe_rtt_safety_val)) { 3770 /* 3771 * Probe RTT safety value triggered! 3772 */ 3773 rack_log_rtt_shrinks(rack, us_cts, 3774 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), 3775 __LINE__, RACK_RTTS_SAFETY); 3776 rack_exit_probertt(rack, us_cts); 3777 } 3778 /* Calculate the max we will wait */ 3779 endtime = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_max_drain_wait); 3780 if (rack->rc_highly_buffered) 3781 endtime += (rack->r_ctl.rc_gp_srtt * rack_max_drain_hbp); 3782 /* Calculate the min we must wait */ 3783 must_stay = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_must_drain); 3784 if ((ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.rc_target_probertt_flight) && 3785 TSTMP_LT(us_cts, endtime)) { 3786 uint32_t calc; 3787 /* Do we lower more? */ 3788 no_exit: 3789 if (TSTMP_GT(us_cts, rack->r_ctl.rc_time_probertt_entered)) 3790 calc = us_cts - rack->r_ctl.rc_time_probertt_entered; 3791 else 3792 calc = 0; 3793 calc /= max(rack->r_ctl.rc_gp_srtt, 1); 3794 if (calc) { 3795 /* Maybe */ 3796 calc *= rack_per_of_gp_probertt_reduce; 3797 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt - calc; 3798 /* Limit it too */ 3799 if (rack->r_ctl.rack_per_of_gp_probertt < rack_per_of_gp_lowthresh) 3800 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_lowthresh; 3801 } 3802 /* We must reach target or the time set */ 3803 return; 3804 } 3805 if (rack->r_ctl.rc_time_probertt_starts == 0) { 3806 if ((TSTMP_LT(us_cts, must_stay) && 3807 rack->rc_highly_buffered) || 3808 (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > 3809 rack->r_ctl.rc_target_probertt_flight)) { 3810 /* We are not past the must_stay time */ 3811 goto no_exit; 3812 } 3813 rack_log_rtt_shrinks(rack, us_cts, 3814 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), 3815 __LINE__, RACK_RTTS_REACHTARGET); 3816 rack->r_ctl.rc_time_probertt_starts = us_cts; 3817 if (rack->r_ctl.rc_time_probertt_starts == 0) 3818 rack->r_ctl.rc_time_probertt_starts = 1; 3819 /* Restore back to our rate we want to pace at in prtt */ 3820 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt; 3821 } 3822 /* 3823 * Setup our end time, some number of gp_srtts plus 200ms. 3824 */ 3825 no_overflow = ((uint64_t)rack->r_ctl.rc_gp_srtt * 3826 (uint64_t)rack_probertt_gpsrtt_cnt_mul); 3827 if (rack_probertt_gpsrtt_cnt_div) 3828 endtime = (uint32_t)(no_overflow / (uint64_t)rack_probertt_gpsrtt_cnt_div); 3829 else 3830 endtime = 0; 3831 endtime += rack_min_probertt_hold; 3832 endtime += rack->r_ctl.rc_time_probertt_starts; 3833 if (TSTMP_GEQ(us_cts, endtime)) { 3834 /* yes, exit probertt */ 3835 rack_exit_probertt(rack, us_cts); 3836 } 3837 3838 } else if ((us_cts - rack->r_ctl.rc_lower_rtt_us_cts) >= rack_time_between_probertt) { 3839 /* Go into probertt, its been too long since we went lower */ 3840 rack_enter_probertt(rack, us_cts); 3841 } 3842 } 3843 3844 static void 3845 rack_update_multiplier(struct tcp_rack *rack, int32_t timely_says, uint64_t last_bw_est, 3846 uint32_t rtt, int32_t rtt_diff) 3847 { 3848 uint64_t cur_bw, up_bnd, low_bnd, subfr; 3849 uint32_t losses; 3850 3851 if ((rack->rc_gp_dyn_mul == 0) || 3852 (rack->use_fixed_rate) || 3853 (rack->in_probe_rtt) || 3854 (rack->rc_always_pace == 0)) { 3855 /* No dynamic GP multiplier in play */ 3856 return; 3857 } 3858 losses = rack->r_ctl.rc_loss_count - rack->r_ctl.rc_loss_at_start; 3859 cur_bw = rack_get_bw(rack); 3860 /* Calculate our up and down range */ 3861 up_bnd = rack->r_ctl.last_gp_comp_bw * (uint64_t)rack_gp_per_bw_mul_up; 3862 up_bnd /= 100; 3863 up_bnd += rack->r_ctl.last_gp_comp_bw; 3864 3865 subfr = (uint64_t)rack->r_ctl.last_gp_comp_bw * (uint64_t)rack_gp_per_bw_mul_down; 3866 subfr /= 100; 3867 low_bnd = rack->r_ctl.last_gp_comp_bw - subfr; 3868 if ((timely_says == 2) && (rack->r_ctl.rc_no_push_at_mrtt)) { 3869 /* 3870 * This is the case where our RTT is above 3871 * the max target and we have been configured 3872 * to just do timely no bonus up stuff in that case. 3873 * 3874 * There are two configurations, set to 1, and we 3875 * just do timely if we are over our max. If its 3876 * set above 1 then we slam the multipliers down 3877 * to 100 and then decrement per timely. 3878 */ 3879 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd, 3880 __LINE__, 3); 3881 if (rack->r_ctl.rc_no_push_at_mrtt > 1) 3882 rack_validate_multipliers_at_or_below_100(rack); 3883 rack_decrease_bw_mul(rack, timely_says, rtt, rtt_diff); 3884 } else if ((last_bw_est < low_bnd) && !losses) { 3885 /* 3886 * We are decreasing this is a bit complicated this 3887 * means we are loosing ground. This could be 3888 * because another flow entered and we are competing 3889 * for b/w with it. This will push the RTT up which 3890 * makes timely unusable unless we want to get shoved 3891 * into a corner and just be backed off (the age 3892 * old problem with delay based CC). 3893 * 3894 * On the other hand if it was a route change we 3895 * would like to stay somewhat contained and not 3896 * blow out the buffers. 3897 */ 3898 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd, 3899 __LINE__, 3); 3900 rack->r_ctl.last_gp_comp_bw = cur_bw; 3901 if (rack->rc_gp_bwred == 0) { 3902 /* Go into reduction counting */ 3903 rack->rc_gp_bwred = 1; 3904 rack->rc_gp_timely_dec_cnt = 0; 3905 } 3906 if ((rack->rc_gp_timely_dec_cnt < rack_timely_max_push_drop) || 3907 (timely_says == 0)) { 3908 /* 3909 * Push another time with a faster pacing 3910 * to try to gain back (we include override to 3911 * get a full raise factor). 3912 */ 3913 if ((rack->rc_gp_saw_ca && rack->r_ctl.rack_per_of_gp_ca <= rack_down_raise_thresh) || 3914 (rack->rc_gp_saw_ss && rack->r_ctl.rack_per_of_gp_ss <= rack_down_raise_thresh) || 3915 (timely_says == 0) || 3916 (rack_down_raise_thresh == 0)) { 3917 /* 3918 * Do an override up in b/w if we were 3919 * below the threshold or if the threshold 3920 * is zero we always do the raise. 3921 */ 3922 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 1); 3923 } else { 3924 /* Log it stays the same */ 3925 rack_log_timely(rack, 0, last_bw_est, low_bnd, 0, 3926 __LINE__, 11); 3927 } 3928 rack->rc_gp_timely_dec_cnt++; 3929 /* We are not incrementing really no-count */ 3930 rack->rc_gp_incr = 0; 3931 rack->rc_gp_timely_inc_cnt = 0; 3932 } else { 3933 /* 3934 * Lets just use the RTT 3935 * information and give up 3936 * pushing. 3937 */ 3938 goto use_timely; 3939 } 3940 } else if ((timely_says != 2) && 3941 !losses && 3942 (last_bw_est > up_bnd)) { 3943 /* 3944 * We are increasing b/w lets keep going, updating 3945 * our b/w and ignoring any timely input, unless 3946 * of course we are at our max raise (if there is one). 3947 */ 3948 3949 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd, 3950 __LINE__, 3); 3951 rack->r_ctl.last_gp_comp_bw = cur_bw; 3952 if (rack->rc_gp_saw_ss && 3953 rack_per_upper_bound_ss && 3954 (rack->r_ctl.rack_per_of_gp_ss == rack_per_upper_bound_ss)) { 3955 /* 3956 * In cases where we can't go higher 3957 * we should just use timely. 3958 */ 3959 goto use_timely; 3960 } 3961 if (rack->rc_gp_saw_ca && 3962 rack_per_upper_bound_ca && 3963 (rack->r_ctl.rack_per_of_gp_ca == rack_per_upper_bound_ca)) { 3964 /* 3965 * In cases where we can't go higher 3966 * we should just use timely. 3967 */ 3968 goto use_timely; 3969 } 3970 rack->rc_gp_bwred = 0; 3971 rack->rc_gp_timely_dec_cnt = 0; 3972 /* You get a set number of pushes if timely is trying to reduce */ 3973 if ((rack->rc_gp_incr < rack_timely_max_push_rise) || (timely_says == 0)) { 3974 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0); 3975 } else { 3976 /* Log it stays the same */ 3977 rack_log_timely(rack, 0, last_bw_est, up_bnd, 0, 3978 __LINE__, 12); 3979 } 3980 return; 3981 } else { 3982 /* 3983 * We are staying between the lower and upper range bounds 3984 * so use timely to decide. 3985 */ 3986 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd, 3987 __LINE__, 3); 3988 use_timely: 3989 if (timely_says) { 3990 rack->rc_gp_incr = 0; 3991 rack->rc_gp_timely_inc_cnt = 0; 3992 if ((rack->rc_gp_timely_dec_cnt < rack_timely_max_push_drop) && 3993 !losses && 3994 (last_bw_est < low_bnd)) { 3995 /* We are loosing ground */ 3996 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0); 3997 rack->rc_gp_timely_dec_cnt++; 3998 /* We are not incrementing really no-count */ 3999 rack->rc_gp_incr = 0; 4000 rack->rc_gp_timely_inc_cnt = 0; 4001 } else 4002 rack_decrease_bw_mul(rack, timely_says, rtt, rtt_diff); 4003 } else { 4004 rack->rc_gp_bwred = 0; 4005 rack->rc_gp_timely_dec_cnt = 0; 4006 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0); 4007 } 4008 } 4009 } 4010 4011 static int32_t 4012 rack_make_timely_judgement(struct tcp_rack *rack, uint32_t rtt, int32_t rtt_diff, uint32_t prev_rtt) 4013 { 4014 int32_t timely_says; 4015 uint64_t log_mult, log_rtt_a_diff; 4016 4017 log_rtt_a_diff = rtt; 4018 log_rtt_a_diff <<= 32; 4019 log_rtt_a_diff |= (uint32_t)rtt_diff; 4020 if (rtt >= (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * 4021 rack_gp_rtt_maxmul)) { 4022 /* Reduce the b/w multiplier */ 4023 timely_says = 2; 4024 log_mult = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_maxmul; 4025 log_mult <<= 32; 4026 log_mult |= prev_rtt; 4027 rack_log_timely(rack, timely_says, log_mult, 4028 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), 4029 log_rtt_a_diff, __LINE__, 4); 4030 } else if (rtt <= (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) + 4031 ((get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_minmul) / 4032 max(rack_gp_rtt_mindiv , 1)))) { 4033 /* Increase the b/w multiplier */ 4034 log_mult = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) + 4035 ((get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_minmul) / 4036 max(rack_gp_rtt_mindiv , 1)); 4037 log_mult <<= 32; 4038 log_mult |= prev_rtt; 4039 timely_says = 0; 4040 rack_log_timely(rack, timely_says, log_mult , 4041 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), 4042 log_rtt_a_diff, __LINE__, 5); 4043 } else { 4044 /* 4045 * Use a gradient to find it the timely gradient 4046 * is: 4047 * grad = rc_rtt_diff / min_rtt; 4048 * 4049 * anything below or equal to 0 will be 4050 * a increase indication. Anything above 4051 * zero is a decrease. Note we take care 4052 * of the actual gradient calculation 4053 * in the reduction (its not needed for 4054 * increase). 4055 */ 4056 log_mult = prev_rtt; 4057 if (rtt_diff <= 0) { 4058 /* 4059 * Rttdiff is less than zero, increase the 4060 * b/w multiplier (its 0 or negative) 4061 */ 4062 timely_says = 0; 4063 rack_log_timely(rack, timely_says, log_mult, 4064 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), log_rtt_a_diff, __LINE__, 6); 4065 } else { 4066 /* Reduce the b/w multiplier */ 4067 timely_says = 1; 4068 rack_log_timely(rack, timely_says, log_mult, 4069 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), log_rtt_a_diff, __LINE__, 7); 4070 } 4071 } 4072 return (timely_says); 4073 } 4074 4075 static void 4076 rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack, 4077 tcp_seq th_ack, int line, uint8_t quality) 4078 { 4079 uint64_t tim, bytes_ps, ltim, stim, utim; 4080 uint32_t segsiz, bytes, reqbytes, us_cts; 4081 int32_t gput, new_rtt_diff, timely_says; 4082 uint64_t resid_bw, subpart = 0, addpart = 0, srtt; 4083 int did_add = 0; 4084 4085 us_cts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 4086 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 4087 if (TSTMP_GEQ(us_cts, tp->gput_ts)) 4088 tim = us_cts - tp->gput_ts; 4089 else 4090 tim = 0; 4091 if (rack->r_ctl.rc_gp_cumack_ts > rack->r_ctl.rc_gp_output_ts) 4092 stim = rack->r_ctl.rc_gp_cumack_ts - rack->r_ctl.rc_gp_output_ts; 4093 else 4094 stim = 0; 4095 /* 4096 * Use the larger of the send time or ack time. This prevents us 4097 * from being influenced by ack artifacts to come up with too 4098 * high of measurement. Note that since we are spanning over many more 4099 * bytes in most of our measurements hopefully that is less likely to 4100 * occur. 4101 */ 4102 if (tim > stim) 4103 utim = max(tim, 1); 4104 else 4105 utim = max(stim, 1); 4106 /* Lets get a msec time ltim too for the old stuff */ 4107 ltim = max(1, (utim / HPTS_USEC_IN_MSEC)); 4108 gput = (((uint64_t) (th_ack - tp->gput_seq)) << 3) / ltim; 4109 reqbytes = min(rc_init_window(rack), (MIN_GP_WIN * segsiz)); 4110 if ((tim == 0) && (stim == 0)) { 4111 /* 4112 * Invalid measurement time, maybe 4113 * all on one ack/one send? 4114 */ 4115 bytes = 0; 4116 bytes_ps = 0; 4117 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes, 4118 0, 0, 0, 10, __LINE__, NULL, quality); 4119 goto skip_measurement; 4120 } 4121 if (rack->r_ctl.rc_gp_lowrtt == 0xffffffff) { 4122 /* We never made a us_rtt measurement? */ 4123 bytes = 0; 4124 bytes_ps = 0; 4125 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes, 4126 0, 0, 0, 10, __LINE__, NULL, quality); 4127 goto skip_measurement; 4128 } 4129 /* 4130 * Calculate the maximum possible b/w this connection 4131 * could have. We base our calculation on the lowest 4132 * rtt we have seen during the measurement and the 4133 * largest rwnd the client has given us in that time. This 4134 * forms a BDP that is the maximum that we could ever 4135 * get to the client. Anything larger is not valid. 4136 * 4137 * I originally had code here that rejected measurements 4138 * where the time was less than 1/2 the latest us_rtt. 4139 * But after thinking on that I realized its wrong since 4140 * say you had a 150Mbps or even 1Gbps link, and you 4141 * were a long way away.. example I am in Europe (100ms rtt) 4142 * talking to my 1Gbps link in S.C. Now measuring say 150,000 4143 * bytes my time would be 1.2ms, and yet my rtt would say 4144 * the measurement was invalid the time was < 50ms. The 4145 * same thing is true for 150Mb (8ms of time). 4146 * 4147 * A better way I realized is to look at what the maximum 4148 * the connection could possibly do. This is gated on 4149 * the lowest RTT we have seen and the highest rwnd. 4150 * We should in theory never exceed that, if we are 4151 * then something on the path is storing up packets 4152 * and then feeding them all at once to our endpoint 4153 * messing up our measurement. 4154 */ 4155 rack->r_ctl.last_max_bw = rack->r_ctl.rc_gp_high_rwnd; 4156 rack->r_ctl.last_max_bw *= HPTS_USEC_IN_SEC; 4157 rack->r_ctl.last_max_bw /= rack->r_ctl.rc_gp_lowrtt; 4158 if (SEQ_LT(th_ack, tp->gput_seq)) { 4159 /* No measurement can be made */ 4160 bytes = 0; 4161 bytes_ps = 0; 4162 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes, 4163 0, 0, 0, 10, __LINE__, NULL, quality); 4164 goto skip_measurement; 4165 } else 4166 bytes = (th_ack - tp->gput_seq); 4167 bytes_ps = (uint64_t)bytes; 4168 /* 4169 * Don't measure a b/w for pacing unless we have gotten at least 4170 * an initial windows worth of data in this measurement interval. 4171 * 4172 * Small numbers of bytes get badly influenced by delayed ack and 4173 * other artifacts. Note we take the initial window or our 4174 * defined minimum GP (defaulting to 10 which hopefully is the 4175 * IW). 4176 */ 4177 if (rack->rc_gp_filled == 0) { 4178 /* 4179 * The initial estimate is special. We 4180 * have blasted out an IW worth of packets 4181 * without a real valid ack ts results. We 4182 * then setup the app_limited_needs_set flag, 4183 * this should get the first ack in (probably 2 4184 * MSS worth) to be recorded as the timestamp. 4185 * We thus allow a smaller number of bytes i.e. 4186 * IW - 2MSS. 4187 */ 4188 reqbytes -= (2 * segsiz); 4189 /* Also lets fill previous for our first measurement to be neutral */ 4190 rack->r_ctl.rc_prev_gp_srtt = rack->r_ctl.rc_gp_srtt; 4191 } 4192 if ((bytes_ps < reqbytes) || rack->app_limited_needs_set) { 4193 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes, 4194 rack->r_ctl.rc_app_limited_cnt, 4195 0, 0, 10, __LINE__, NULL, quality); 4196 goto skip_measurement; 4197 } 4198 /* 4199 * We now need to calculate the Timely like status so 4200 * we can update (possibly) the b/w multipliers. 4201 */ 4202 new_rtt_diff = (int32_t)rack->r_ctl.rc_gp_srtt - (int32_t)rack->r_ctl.rc_prev_gp_srtt; 4203 if (rack->rc_gp_filled == 0) { 4204 /* No previous reading */ 4205 rack->r_ctl.rc_rtt_diff = new_rtt_diff; 4206 } else { 4207 if (rack->measure_saw_probe_rtt == 0) { 4208 /* 4209 * We don't want a probertt to be counted 4210 * since it will be negative incorrectly. We 4211 * expect to be reducing the RTT when we 4212 * pace at a slower rate. 4213 */ 4214 rack->r_ctl.rc_rtt_diff -= (rack->r_ctl.rc_rtt_diff / 8); 4215 rack->r_ctl.rc_rtt_diff += (new_rtt_diff / 8); 4216 } 4217 } 4218 timely_says = rack_make_timely_judgement(rack, 4219 rack->r_ctl.rc_gp_srtt, 4220 rack->r_ctl.rc_rtt_diff, 4221 rack->r_ctl.rc_prev_gp_srtt 4222 ); 4223 bytes_ps *= HPTS_USEC_IN_SEC; 4224 bytes_ps /= utim; 4225 if (bytes_ps > rack->r_ctl.last_max_bw) { 4226 /* 4227 * Something is on path playing 4228 * since this b/w is not possible based 4229 * on our BDP (highest rwnd and lowest rtt 4230 * we saw in the measurement window). 4231 * 4232 * Another option here would be to 4233 * instead skip the measurement. 4234 */ 4235 rack_log_pacing_delay_calc(rack, bytes, reqbytes, 4236 bytes_ps, rack->r_ctl.last_max_bw, 0, 4237 11, __LINE__, NULL, quality); 4238 bytes_ps = rack->r_ctl.last_max_bw; 4239 } 4240 /* We store gp for b/w in bytes per second */ 4241 if (rack->rc_gp_filled == 0) { 4242 /* Initial measurement */ 4243 if (bytes_ps) { 4244 rack->r_ctl.gp_bw = bytes_ps; 4245 rack->rc_gp_filled = 1; 4246 rack->r_ctl.num_measurements = 1; 4247 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL); 4248 } else { 4249 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes, 4250 rack->r_ctl.rc_app_limited_cnt, 4251 0, 0, 10, __LINE__, NULL, quality); 4252 } 4253 if (tcp_in_hpts(rack->rc_inp) && 4254 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { 4255 /* 4256 * Ok we can't trust the pacer in this case 4257 * where we transition from un-paced to paced. 4258 * Or for that matter when the burst mitigation 4259 * was making a wild guess and got it wrong. 4260 * Stop the pacer and clear up all the aggregate 4261 * delays etc. 4262 */ 4263 tcp_hpts_remove(rack->rc_inp); 4264 rack->r_ctl.rc_hpts_flags = 0; 4265 rack->r_ctl.rc_last_output_to = 0; 4266 } 4267 did_add = 2; 4268 } else if (rack->r_ctl.num_measurements < RACK_REQ_AVG) { 4269 /* Still a small number run an average */ 4270 rack->r_ctl.gp_bw += bytes_ps; 4271 addpart = rack->r_ctl.num_measurements; 4272 rack->r_ctl.num_measurements++; 4273 if (rack->r_ctl.num_measurements >= RACK_REQ_AVG) { 4274 /* We have collected enough to move forward */ 4275 rack->r_ctl.gp_bw /= (uint64_t)rack->r_ctl.num_measurements; 4276 } 4277 did_add = 3; 4278 } else { 4279 /* 4280 * We want to take 1/wma of the goodput and add in to 7/8th 4281 * of the old value weighted by the srtt. So if your measurement 4282 * period is say 2 SRTT's long you would get 1/4 as the 4283 * value, if it was like 1/2 SRTT then you would get 1/16th. 4284 * 4285 * But we must be careful not to take too much i.e. if the 4286 * srtt is say 20ms and the measurement is taken over 4287 * 400ms our weight would be 400/20 i.e. 20. On the 4288 * other hand if we get a measurement over 1ms with a 4289 * 10ms rtt we only want to take a much smaller portion. 4290 */ 4291 if (rack->r_ctl.num_measurements < 0xff) { 4292 rack->r_ctl.num_measurements++; 4293 } 4294 srtt = (uint64_t)tp->t_srtt; 4295 if (srtt == 0) { 4296 /* 4297 * Strange why did t_srtt go back to zero? 4298 */ 4299 if (rack->r_ctl.rc_rack_min_rtt) 4300 srtt = rack->r_ctl.rc_rack_min_rtt; 4301 else 4302 srtt = HPTS_USEC_IN_MSEC; 4303 } 4304 /* 4305 * XXXrrs: Note for reviewers, in playing with 4306 * dynamic pacing I discovered this GP calculation 4307 * as done originally leads to some undesired results. 4308 * Basically you can get longer measurements contributing 4309 * too much to the WMA. Thus I changed it if you are doing 4310 * dynamic adjustments to only do the aportioned adjustment 4311 * if we have a very small (time wise) measurement. Longer 4312 * measurements just get there weight (defaulting to 1/8) 4313 * add to the WMA. We may want to think about changing 4314 * this to always do that for both sides i.e. dynamic 4315 * and non-dynamic... but considering lots of folks 4316 * were playing with this I did not want to change the 4317 * calculation per.se. without your thoughts.. Lawerence? 4318 * Peter?? 4319 */ 4320 if (rack->rc_gp_dyn_mul == 0) { 4321 subpart = rack->r_ctl.gp_bw * utim; 4322 subpart /= (srtt * 8); 4323 if (subpart < (rack->r_ctl.gp_bw / 2)) { 4324 /* 4325 * The b/w update takes no more 4326 * away then 1/2 our running total 4327 * so factor it in. 4328 */ 4329 addpart = bytes_ps * utim; 4330 addpart /= (srtt * 8); 4331 } else { 4332 /* 4333 * Don't allow a single measurement 4334 * to account for more than 1/2 of the 4335 * WMA. This could happen on a retransmission 4336 * where utim becomes huge compared to 4337 * srtt (multiple retransmissions when using 4338 * the sending rate which factors in all the 4339 * transmissions from the first one). 4340 */ 4341 subpart = rack->r_ctl.gp_bw / 2; 4342 addpart = bytes_ps / 2; 4343 } 4344 resid_bw = rack->r_ctl.gp_bw - subpart; 4345 rack->r_ctl.gp_bw = resid_bw + addpart; 4346 did_add = 1; 4347 } else { 4348 if ((utim / srtt) <= 1) { 4349 /* 4350 * The b/w update was over a small period 4351 * of time. The idea here is to prevent a small 4352 * measurement time period from counting 4353 * too much. So we scale it based on the 4354 * time so it attributes less than 1/rack_wma_divisor 4355 * of its measurement. 4356 */ 4357 subpart = rack->r_ctl.gp_bw * utim; 4358 subpart /= (srtt * rack_wma_divisor); 4359 addpart = bytes_ps * utim; 4360 addpart /= (srtt * rack_wma_divisor); 4361 } else { 4362 /* 4363 * The scaled measurement was long 4364 * enough so lets just add in the 4365 * portion of the measurement i.e. 1/rack_wma_divisor 4366 */ 4367 subpart = rack->r_ctl.gp_bw / rack_wma_divisor; 4368 addpart = bytes_ps / rack_wma_divisor; 4369 } 4370 if ((rack->measure_saw_probe_rtt == 0) || 4371 (bytes_ps > rack->r_ctl.gp_bw)) { 4372 /* 4373 * For probe-rtt we only add it in 4374 * if its larger, all others we just 4375 * add in. 4376 */ 4377 did_add = 1; 4378 resid_bw = rack->r_ctl.gp_bw - subpart; 4379 rack->r_ctl.gp_bw = resid_bw + addpart; 4380 } 4381 } 4382 } 4383 if ((rack->gp_ready == 0) && 4384 (rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) { 4385 /* We have enough measurements now */ 4386 rack->gp_ready = 1; 4387 rack_set_cc_pacing(rack); 4388 if (rack->defer_options) 4389 rack_apply_deferred_options(rack); 4390 } 4391 rack_log_pacing_delay_calc(rack, subpart, addpart, bytes_ps, stim, 4392 rack_get_bw(rack), 22, did_add, NULL, quality); 4393 /* We do not update any multipliers if we are in or have seen a probe-rtt */ 4394 if ((rack->measure_saw_probe_rtt == 0) && rack->rc_gp_rtt_set) 4395 rack_update_multiplier(rack, timely_says, bytes_ps, 4396 rack->r_ctl.rc_gp_srtt, 4397 rack->r_ctl.rc_rtt_diff); 4398 rack_log_pacing_delay_calc(rack, bytes, tim, bytes_ps, stim, 4399 rack_get_bw(rack), 3, line, NULL, quality); 4400 /* reset the gp srtt and setup the new prev */ 4401 rack->r_ctl.rc_prev_gp_srtt = rack->r_ctl.rc_gp_srtt; 4402 /* Record the lost count for the next measurement */ 4403 rack->r_ctl.rc_loss_at_start = rack->r_ctl.rc_loss_count; 4404 /* 4405 * We restart our diffs based on the gpsrtt in the 4406 * measurement window. 4407 */ 4408 rack->rc_gp_rtt_set = 0; 4409 rack->rc_gp_saw_rec = 0; 4410 rack->rc_gp_saw_ca = 0; 4411 rack->rc_gp_saw_ss = 0; 4412 rack->rc_dragged_bottom = 0; 4413 skip_measurement: 4414 4415 #ifdef STATS 4416 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT, 4417 gput); 4418 /* 4419 * XXXLAS: This is a temporary hack, and should be 4420 * chained off VOI_TCP_GPUT when stats(9) grows an 4421 * API to deal with chained VOIs. 4422 */ 4423 if (tp->t_stats_gput_prev > 0) 4424 stats_voi_update_abs_s32(tp->t_stats, 4425 VOI_TCP_GPUT_ND, 4426 ((gput - tp->t_stats_gput_prev) * 100) / 4427 tp->t_stats_gput_prev); 4428 #endif 4429 tp->t_flags &= ~TF_GPUTINPROG; 4430 tp->t_stats_gput_prev = gput; 4431 /* 4432 * Now are we app limited now and there is space from where we 4433 * were to where we want to go? 4434 * 4435 * We don't do the other case i.e. non-applimited here since 4436 * the next send will trigger us picking up the missing data. 4437 */ 4438 if (rack->r_ctl.rc_first_appl && 4439 TCPS_HAVEESTABLISHED(tp->t_state) && 4440 rack->r_ctl.rc_app_limited_cnt && 4441 (SEQ_GT(rack->r_ctl.rc_first_appl->r_start, th_ack)) && 4442 ((rack->r_ctl.rc_first_appl->r_end - th_ack) > 4443 max(rc_init_window(rack), (MIN_GP_WIN * segsiz)))) { 4444 /* 4445 * Yep there is enough outstanding to make a measurement here. 4446 */ 4447 struct rack_sendmap *rsm, fe; 4448 4449 rack->r_ctl.rc_gp_lowrtt = 0xffffffff; 4450 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd; 4451 tp->gput_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 4452 rack->app_limited_needs_set = 0; 4453 tp->gput_seq = th_ack; 4454 if (rack->in_probe_rtt) 4455 rack->measure_saw_probe_rtt = 1; 4456 else if ((rack->measure_saw_probe_rtt) && 4457 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit))) 4458 rack->measure_saw_probe_rtt = 0; 4459 if ((rack->r_ctl.rc_first_appl->r_end - th_ack) >= rack_get_measure_window(tp, rack)) { 4460 /* There is a full window to gain info from */ 4461 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack); 4462 } else { 4463 /* We can only measure up to the applimited point */ 4464 tp->gput_ack = tp->gput_seq + (rack->r_ctl.rc_first_appl->r_end - th_ack); 4465 if ((tp->gput_ack - tp->gput_seq) < (MIN_GP_WIN * segsiz)) { 4466 /* 4467 * We don't have enough to make a measurement. 4468 */ 4469 tp->t_flags &= ~TF_GPUTINPROG; 4470 rack_log_pacing_delay_calc(rack, tp->gput_ack, tp->gput_seq, 4471 0, 0, 0, 6, __LINE__, NULL, quality); 4472 return; 4473 } 4474 } 4475 if (tp->t_state >= TCPS_FIN_WAIT_1) { 4476 /* 4477 * We will get no more data into the SB 4478 * this means we need to have the data available 4479 * before we start a measurement. 4480 */ 4481 if (sbavail(&tptosocket(tp)->so_snd) < (tp->gput_ack - tp->gput_seq)) { 4482 /* Nope not enough data. */ 4483 return; 4484 } 4485 } 4486 tp->t_flags |= TF_GPUTINPROG; 4487 /* 4488 * Now we need to find the timestamp of the send at tp->gput_seq 4489 * for the send based measurement. 4490 */ 4491 fe.r_start = tp->gput_seq; 4492 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 4493 if (rsm) { 4494 /* Ok send-based limit is set */ 4495 if (SEQ_LT(rsm->r_start, tp->gput_seq)) { 4496 /* 4497 * Move back to include the earlier part 4498 * so our ack time lines up right (this may 4499 * make an overlapping measurement but thats 4500 * ok). 4501 */ 4502 tp->gput_seq = rsm->r_start; 4503 } 4504 if (rsm->r_flags & RACK_ACKED) 4505 tp->gput_ts = (uint32_t)rsm->r_ack_arrival; 4506 else 4507 rack->app_limited_needs_set = 1; 4508 rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 4509 } else { 4510 /* 4511 * If we don't find the rsm due to some 4512 * send-limit set the current time, which 4513 * basically disables the send-limit. 4514 */ 4515 struct timeval tv; 4516 4517 microuptime(&tv); 4518 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv); 4519 } 4520 rack_log_pacing_delay_calc(rack, 4521 tp->gput_seq, 4522 tp->gput_ack, 4523 (uint64_t)rsm, 4524 tp->gput_ts, 4525 rack->r_ctl.rc_app_limited_cnt, 4526 9, 4527 __LINE__, NULL, quality); 4528 } 4529 } 4530 4531 /* 4532 * CC wrapper hook functions 4533 */ 4534 static void 4535 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack, uint32_t th_ack, uint16_t nsegs, 4536 uint16_t type, int32_t recovery) 4537 { 4538 uint32_t prior_cwnd, acked; 4539 struct tcp_log_buffer *lgb = NULL; 4540 uint8_t labc_to_use, quality; 4541 4542 INP_WLOCK_ASSERT(tptoinpcb(tp)); 4543 tp->t_ccv.nsegs = nsegs; 4544 acked = tp->t_ccv.bytes_this_ack = (th_ack - tp->snd_una); 4545 if ((recovery) && (rack->r_ctl.rc_early_recovery_segs)) { 4546 uint32_t max; 4547 4548 max = rack->r_ctl.rc_early_recovery_segs * ctf_fixed_maxseg(tp); 4549 if (tp->t_ccv.bytes_this_ack > max) { 4550 tp->t_ccv.bytes_this_ack = max; 4551 } 4552 } 4553 #ifdef STATS 4554 stats_voi_update_abs_s32(tp->t_stats, VOI_TCP_CALCFRWINDIFF, 4555 ((int32_t)rack->r_ctl.cwnd_to_use) - tp->snd_wnd); 4556 #endif 4557 quality = RACK_QUALITY_NONE; 4558 if ((tp->t_flags & TF_GPUTINPROG) && 4559 rack_enough_for_measurement(tp, rack, th_ack, &quality)) { 4560 /* Measure the Goodput */ 4561 rack_do_goodput_measurement(tp, rack, th_ack, __LINE__, quality); 4562 #ifdef NETFLIX_PEAKRATE 4563 if ((type == CC_ACK) && 4564 (tp->t_maxpeakrate)) { 4565 /* 4566 * We update t_peakrate_thr. This gives us roughly 4567 * one update per round trip time. Note 4568 * it will only be used if pace_always is off i.e 4569 * we don't do this for paced flows. 4570 */ 4571 rack_update_peakrate_thr(tp); 4572 } 4573 #endif 4574 } 4575 /* Which way our we limited, if not cwnd limited no advance in CA */ 4576 if (tp->snd_cwnd <= tp->snd_wnd) 4577 tp->t_ccv.flags |= CCF_CWND_LIMITED; 4578 else 4579 tp->t_ccv.flags &= ~CCF_CWND_LIMITED; 4580 if (tp->snd_cwnd > tp->snd_ssthresh) { 4581 tp->t_bytes_acked += min(tp->t_ccv.bytes_this_ack, 4582 nsegs * V_tcp_abc_l_var * ctf_fixed_maxseg(tp)); 4583 /* For the setting of a window past use the actual scwnd we are using */ 4584 if (tp->t_bytes_acked >= rack->r_ctl.cwnd_to_use) { 4585 tp->t_bytes_acked -= rack->r_ctl.cwnd_to_use; 4586 tp->t_ccv.flags |= CCF_ABC_SENTAWND; 4587 } 4588 } else { 4589 tp->t_ccv.flags &= ~CCF_ABC_SENTAWND; 4590 tp->t_bytes_acked = 0; 4591 } 4592 prior_cwnd = tp->snd_cwnd; 4593 if ((recovery == 0) || (rack_max_abc_post_recovery == 0) || rack->r_use_labc_for_rec || 4594 (rack_client_low_buf && (rack->client_bufferlvl < rack_client_low_buf))) 4595 labc_to_use = rack->rc_labc; 4596 else 4597 labc_to_use = rack_max_abc_post_recovery; 4598 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 4599 union tcp_log_stackspecific log; 4600 struct timeval tv; 4601 4602 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 4603 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 4604 log.u_bbr.flex1 = th_ack; 4605 log.u_bbr.flex2 = tp->t_ccv.flags; 4606 log.u_bbr.flex3 = tp->t_ccv.bytes_this_ack; 4607 log.u_bbr.flex4 = tp->t_ccv.nsegs; 4608 log.u_bbr.flex5 = labc_to_use; 4609 log.u_bbr.flex6 = prior_cwnd; 4610 log.u_bbr.flex7 = V_tcp_do_newsack; 4611 log.u_bbr.flex8 = 1; 4612 lgb = tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0, 4613 0, &log, false, NULL, NULL, 0, &tv); 4614 } 4615 if (CC_ALGO(tp)->ack_received != NULL) { 4616 /* XXXLAS: Find a way to live without this */ 4617 tp->t_ccv.curack = th_ack; 4618 tp->t_ccv.labc = labc_to_use; 4619 tp->t_ccv.flags |= CCF_USE_LOCAL_ABC; 4620 CC_ALGO(tp)->ack_received(&tp->t_ccv, type); 4621 } 4622 if (lgb) { 4623 lgb->tlb_stackinfo.u_bbr.flex6 = tp->snd_cwnd; 4624 } 4625 if (rack->r_must_retran) { 4626 if (SEQ_GEQ(th_ack, rack->r_ctl.rc_snd_max_at_rto)) { 4627 /* 4628 * We now are beyond the rxt point so lets disable 4629 * the flag. 4630 */ 4631 rack->r_ctl.rc_out_at_rto = 0; 4632 rack->r_must_retran = 0; 4633 } else if ((prior_cwnd + ctf_fixed_maxseg(tp)) <= tp->snd_cwnd) { 4634 /* 4635 * Only decrement the rc_out_at_rto if the cwnd advances 4636 * at least a whole segment. Otherwise next time the peer 4637 * acks, we won't be able to send this generaly happens 4638 * when we are in Congestion Avoidance. 4639 */ 4640 if (acked <= rack->r_ctl.rc_out_at_rto){ 4641 rack->r_ctl.rc_out_at_rto -= acked; 4642 } else { 4643 rack->r_ctl.rc_out_at_rto = 0; 4644 } 4645 } 4646 } 4647 #ifdef STATS 4648 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_LCWIN, rack->r_ctl.cwnd_to_use); 4649 #endif 4650 if (rack->r_ctl.rc_rack_largest_cwnd < rack->r_ctl.cwnd_to_use) { 4651 rack->r_ctl.rc_rack_largest_cwnd = rack->r_ctl.cwnd_to_use; 4652 } 4653 #ifdef NETFLIX_PEAKRATE 4654 /* we enforce max peak rate if it is set and we are not pacing */ 4655 if ((rack->rc_always_pace == 0) && 4656 tp->t_peakrate_thr && 4657 (tp->snd_cwnd > tp->t_peakrate_thr)) { 4658 tp->snd_cwnd = tp->t_peakrate_thr; 4659 } 4660 #endif 4661 } 4662 4663 static void 4664 tcp_rack_partialack(struct tcpcb *tp) 4665 { 4666 struct tcp_rack *rack; 4667 4668 rack = (struct tcp_rack *)tp->t_fb_ptr; 4669 INP_WLOCK_ASSERT(tptoinpcb(tp)); 4670 /* 4671 * If we are doing PRR and have enough 4672 * room to send <or> we are pacing and prr 4673 * is disabled we will want to see if we 4674 * can send data (by setting r_wanted_output to 4675 * true). 4676 */ 4677 if ((rack->r_ctl.rc_prr_sndcnt > 0) || 4678 rack->rack_no_prr) 4679 rack->r_wanted_output = 1; 4680 } 4681 4682 static void 4683 rack_post_recovery(struct tcpcb *tp, uint32_t th_ack) 4684 { 4685 struct tcp_rack *rack; 4686 uint32_t orig_cwnd; 4687 4688 orig_cwnd = tp->snd_cwnd; 4689 INP_WLOCK_ASSERT(tptoinpcb(tp)); 4690 rack = (struct tcp_rack *)tp->t_fb_ptr; 4691 /* only alert CC if we alerted when we entered */ 4692 if (CC_ALGO(tp)->post_recovery != NULL) { 4693 tp->t_ccv.curack = th_ack; 4694 CC_ALGO(tp)->post_recovery(&tp->t_ccv); 4695 if (tp->snd_cwnd < tp->snd_ssthresh) { 4696 /* 4697 * Rack has burst control and pacing 4698 * so lets not set this any lower than 4699 * snd_ssthresh per RFC-6582 (option 2). 4700 */ 4701 tp->snd_cwnd = tp->snd_ssthresh; 4702 } 4703 } 4704 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 4705 union tcp_log_stackspecific log; 4706 struct timeval tv; 4707 4708 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 4709 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 4710 log.u_bbr.flex1 = th_ack; 4711 log.u_bbr.flex2 = tp->t_ccv.flags; 4712 log.u_bbr.flex3 = tp->t_ccv.bytes_this_ack; 4713 log.u_bbr.flex4 = tp->t_ccv.nsegs; 4714 log.u_bbr.flex5 = V_tcp_abc_l_var; 4715 log.u_bbr.flex6 = orig_cwnd; 4716 log.u_bbr.flex7 = V_tcp_do_newsack; 4717 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt; 4718 log.u_bbr.flex8 = 2; 4719 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0, 4720 0, &log, false, NULL, NULL, 0, &tv); 4721 } 4722 if ((rack->rack_no_prr == 0) && 4723 (rack->no_prr_addback == 0) && 4724 (rack->r_ctl.rc_prr_sndcnt > 0)) { 4725 /* 4726 * Suck the next prr cnt back into cwnd, but 4727 * only do that if we are not application limited. 4728 */ 4729 if (ctf_outstanding(tp) <= sbavail(&tptosocket(tp)->so_snd)) { 4730 /* 4731 * We are allowed to add back to the cwnd the amount we did 4732 * not get out if: 4733 * a) no_prr_addback is off. 4734 * b) we are not app limited 4735 * c) we are doing prr 4736 * <and> 4737 * d) it is bounded by rack_prr_addbackmax (if addback is 0, then none). 4738 */ 4739 tp->snd_cwnd += min((ctf_fixed_maxseg(tp) * rack_prr_addbackmax), 4740 rack->r_ctl.rc_prr_sndcnt); 4741 } 4742 rack->r_ctl.rc_prr_sndcnt = 0; 4743 rack_log_to_prr(rack, 1, 0, __LINE__); 4744 } 4745 rack_log_to_prr(rack, 14, orig_cwnd, __LINE__); 4746 tp->snd_recover = tp->snd_una; 4747 if (rack->r_ctl.dsack_persist) { 4748 rack->r_ctl.dsack_persist--; 4749 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) { 4750 rack->r_ctl.num_dsack = 0; 4751 } 4752 rack_log_dsack_event(rack, 1, __LINE__, 0, 0); 4753 } 4754 EXIT_RECOVERY(tp->t_flags); 4755 } 4756 4757 static void 4758 rack_cong_signal(struct tcpcb *tp, uint32_t type, uint32_t ack, int line) 4759 { 4760 struct tcp_rack *rack; 4761 uint32_t ssthresh_enter, cwnd_enter, in_rec_at_entry, orig_cwnd; 4762 4763 INP_WLOCK_ASSERT(tptoinpcb(tp)); 4764 #ifdef STATS 4765 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type); 4766 #endif 4767 if (IN_RECOVERY(tp->t_flags) == 0) { 4768 in_rec_at_entry = 0; 4769 ssthresh_enter = tp->snd_ssthresh; 4770 cwnd_enter = tp->snd_cwnd; 4771 } else 4772 in_rec_at_entry = 1; 4773 rack = (struct tcp_rack *)tp->t_fb_ptr; 4774 switch (type) { 4775 case CC_NDUPACK: 4776 tp->t_flags &= ~TF_WASFRECOVERY; 4777 tp->t_flags &= ~TF_WASCRECOVERY; 4778 if (!IN_FASTRECOVERY(tp->t_flags)) { 4779 rack->r_ctl.rc_prr_delivered = 0; 4780 rack->r_ctl.rc_prr_out = 0; 4781 if (rack->rack_no_prr == 0) { 4782 rack->r_ctl.rc_prr_sndcnt = ctf_fixed_maxseg(tp); 4783 rack_log_to_prr(rack, 2, in_rec_at_entry, line); 4784 } 4785 rack->r_ctl.rc_prr_recovery_fs = tp->snd_max - tp->snd_una; 4786 tp->snd_recover = tp->snd_max; 4787 if (tp->t_flags2 & TF2_ECN_PERMIT) 4788 tp->t_flags2 |= TF2_ECN_SND_CWR; 4789 } 4790 break; 4791 case CC_ECN: 4792 if (!IN_CONGRECOVERY(tp->t_flags) || 4793 /* 4794 * Allow ECN reaction on ACK to CWR, if 4795 * that data segment was also CE marked. 4796 */ 4797 SEQ_GEQ(ack, tp->snd_recover)) { 4798 EXIT_CONGRECOVERY(tp->t_flags); 4799 KMOD_TCPSTAT_INC(tcps_ecn_rcwnd); 4800 tp->snd_recover = tp->snd_max + 1; 4801 if (tp->t_flags2 & TF2_ECN_PERMIT) 4802 tp->t_flags2 |= TF2_ECN_SND_CWR; 4803 } 4804 break; 4805 case CC_RTO: 4806 tp->t_dupacks = 0; 4807 tp->t_bytes_acked = 0; 4808 EXIT_RECOVERY(tp->t_flags); 4809 tp->snd_ssthresh = max(2, min(tp->snd_wnd, rack->r_ctl.cwnd_to_use) / 2 / 4810 ctf_fixed_maxseg(tp)) * ctf_fixed_maxseg(tp); 4811 orig_cwnd = tp->snd_cwnd; 4812 tp->snd_cwnd = ctf_fixed_maxseg(tp); 4813 rack_log_to_prr(rack, 16, orig_cwnd, line); 4814 if (tp->t_flags2 & TF2_ECN_PERMIT) 4815 tp->t_flags2 |= TF2_ECN_SND_CWR; 4816 break; 4817 case CC_RTO_ERR: 4818 KMOD_TCPSTAT_INC(tcps_sndrexmitbad); 4819 /* RTO was unnecessary, so reset everything. */ 4820 tp->snd_cwnd = tp->snd_cwnd_prev; 4821 tp->snd_ssthresh = tp->snd_ssthresh_prev; 4822 tp->snd_recover = tp->snd_recover_prev; 4823 if (tp->t_flags & TF_WASFRECOVERY) { 4824 ENTER_FASTRECOVERY(tp->t_flags); 4825 tp->t_flags &= ~TF_WASFRECOVERY; 4826 } 4827 if (tp->t_flags & TF_WASCRECOVERY) { 4828 ENTER_CONGRECOVERY(tp->t_flags); 4829 tp->t_flags &= ~TF_WASCRECOVERY; 4830 } 4831 tp->snd_nxt = tp->snd_max; 4832 tp->t_badrxtwin = 0; 4833 break; 4834 } 4835 if ((CC_ALGO(tp)->cong_signal != NULL) && 4836 (type != CC_RTO)){ 4837 tp->t_ccv.curack = ack; 4838 CC_ALGO(tp)->cong_signal(&tp->t_ccv, type); 4839 } 4840 if ((in_rec_at_entry == 0) && IN_RECOVERY(tp->t_flags)) { 4841 rack_log_to_prr(rack, 15, cwnd_enter, line); 4842 rack->r_ctl.dsack_byte_cnt = 0; 4843 rack->r_ctl.retran_during_recovery = 0; 4844 rack->r_ctl.rc_cwnd_at_erec = cwnd_enter; 4845 rack->r_ctl.rc_ssthresh_at_erec = ssthresh_enter; 4846 rack->r_ent_rec_ns = 1; 4847 } 4848 } 4849 4850 static inline void 4851 rack_cc_after_idle(struct tcp_rack *rack, struct tcpcb *tp) 4852 { 4853 uint32_t i_cwnd; 4854 4855 INP_WLOCK_ASSERT(tptoinpcb(tp)); 4856 4857 #ifdef NETFLIX_STATS 4858 KMOD_TCPSTAT_INC(tcps_idle_restarts); 4859 if (tp->t_state == TCPS_ESTABLISHED) 4860 KMOD_TCPSTAT_INC(tcps_idle_estrestarts); 4861 #endif 4862 if (CC_ALGO(tp)->after_idle != NULL) 4863 CC_ALGO(tp)->after_idle(&tp->t_ccv); 4864 4865 if (tp->snd_cwnd == 1) 4866 i_cwnd = tp->t_maxseg; /* SYN(-ACK) lost */ 4867 else 4868 i_cwnd = rc_init_window(rack); 4869 4870 /* 4871 * Being idle is no different than the initial window. If the cc 4872 * clamps it down below the initial window raise it to the initial 4873 * window. 4874 */ 4875 if (tp->snd_cwnd < i_cwnd) { 4876 tp->snd_cwnd = i_cwnd; 4877 } 4878 } 4879 4880 /* 4881 * Indicate whether this ack should be delayed. We can delay the ack if 4882 * following conditions are met: 4883 * - There is no delayed ack timer in progress. 4884 * - Our last ack wasn't a 0-sized window. We never want to delay 4885 * the ack that opens up a 0-sized window. 4886 * - LRO wasn't used for this segment. We make sure by checking that the 4887 * segment size is not larger than the MSS. 4888 * - Delayed acks are enabled or this is a half-synchronized T/TCP 4889 * connection. 4890 */ 4891 #define DELAY_ACK(tp, tlen) \ 4892 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \ 4893 ((tp->t_flags & TF_DELACK) == 0) && \ 4894 (tlen <= tp->t_maxseg) && \ 4895 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN))) 4896 4897 static struct rack_sendmap * 4898 rack_find_lowest_rsm(struct tcp_rack *rack) 4899 { 4900 struct rack_sendmap *rsm; 4901 4902 /* 4903 * Walk the time-order transmitted list looking for an rsm that is 4904 * not acked. This will be the one that was sent the longest time 4905 * ago that is still outstanding. 4906 */ 4907 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) { 4908 if (rsm->r_flags & RACK_ACKED) { 4909 continue; 4910 } 4911 goto finish; 4912 } 4913 finish: 4914 return (rsm); 4915 } 4916 4917 static struct rack_sendmap * 4918 rack_find_high_nonack(struct tcp_rack *rack, struct rack_sendmap *rsm) 4919 { 4920 struct rack_sendmap *prsm; 4921 4922 /* 4923 * Walk the sequence order list backward until we hit and arrive at 4924 * the highest seq not acked. In theory when this is called it 4925 * should be the last segment (which it was not). 4926 */ 4927 prsm = rsm; 4928 RB_FOREACH_REVERSE_FROM(prsm, rack_rb_tree_head, rsm) { 4929 if (prsm->r_flags & (RACK_ACKED | RACK_HAS_FIN)) { 4930 continue; 4931 } 4932 return (prsm); 4933 } 4934 return (NULL); 4935 } 4936 4937 static uint32_t 4938 rack_calc_thresh_rack(struct tcp_rack *rack, uint32_t srtt, uint32_t cts) 4939 { 4940 int32_t lro; 4941 uint32_t thresh; 4942 4943 /* 4944 * lro is the flag we use to determine if we have seen reordering. 4945 * If it gets set we have seen reordering. The reorder logic either 4946 * works in one of two ways: 4947 * 4948 * If reorder-fade is configured, then we track the last time we saw 4949 * re-ordering occur. If we reach the point where enough time as 4950 * passed we no longer consider reordering has occuring. 4951 * 4952 * Or if reorder-face is 0, then once we see reordering we consider 4953 * the connection to alway be subject to reordering and just set lro 4954 * to 1. 4955 * 4956 * In the end if lro is non-zero we add the extra time for 4957 * reordering in. 4958 */ 4959 if (srtt == 0) 4960 srtt = 1; 4961 if (rack->r_ctl.rc_reorder_ts) { 4962 if (rack->r_ctl.rc_reorder_fade) { 4963 if (SEQ_GEQ(cts, rack->r_ctl.rc_reorder_ts)) { 4964 lro = cts - rack->r_ctl.rc_reorder_ts; 4965 if (lro == 0) { 4966 /* 4967 * No time as passed since the last 4968 * reorder, mark it as reordering. 4969 */ 4970 lro = 1; 4971 } 4972 } else { 4973 /* Negative time? */ 4974 lro = 0; 4975 } 4976 if (lro > rack->r_ctl.rc_reorder_fade) { 4977 /* Turn off reordering seen too */ 4978 rack->r_ctl.rc_reorder_ts = 0; 4979 lro = 0; 4980 } 4981 } else { 4982 /* Reodering does not fade */ 4983 lro = 1; 4984 } 4985 } else { 4986 lro = 0; 4987 } 4988 if (rack->rc_rack_tmr_std_based == 0) { 4989 thresh = srtt + rack->r_ctl.rc_pkt_delay; 4990 } else { 4991 /* Standards based pkt-delay is 1/4 srtt */ 4992 thresh = srtt + (srtt >> 2); 4993 } 4994 if (lro && (rack->rc_rack_tmr_std_based == 0)) { 4995 /* It must be set, if not you get 1/4 rtt */ 4996 if (rack->r_ctl.rc_reorder_shift) 4997 thresh += (srtt >> rack->r_ctl.rc_reorder_shift); 4998 else 4999 thresh += (srtt >> 2); 5000 } 5001 if (rack->rc_rack_use_dsack && 5002 lro && 5003 (rack->r_ctl.num_dsack > 0)) { 5004 /* 5005 * We only increase the reordering window if we 5006 * have seen reordering <and> we have a DSACK count. 5007 */ 5008 thresh += rack->r_ctl.num_dsack * (srtt >> 2); 5009 rack_log_dsack_event(rack, 4, __LINE__, srtt, thresh); 5010 } 5011 /* SRTT * 2 is the ceiling */ 5012 if (thresh > (srtt * 2)) { 5013 thresh = srtt * 2; 5014 } 5015 /* And we don't want it above the RTO max either */ 5016 if (thresh > rack_rto_max) { 5017 thresh = rack_rto_max; 5018 } 5019 rack_log_dsack_event(rack, 6, __LINE__, srtt, thresh); 5020 return (thresh); 5021 } 5022 5023 static uint32_t 5024 rack_calc_thresh_tlp(struct tcpcb *tp, struct tcp_rack *rack, 5025 struct rack_sendmap *rsm, uint32_t srtt) 5026 { 5027 struct rack_sendmap *prsm; 5028 uint32_t thresh, len; 5029 int segsiz; 5030 5031 if (srtt == 0) 5032 srtt = 1; 5033 if (rack->r_ctl.rc_tlp_threshold) 5034 thresh = srtt + (srtt / rack->r_ctl.rc_tlp_threshold); 5035 else 5036 thresh = (srtt * 2); 5037 5038 /* Get the previous sent packet, if any */ 5039 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 5040 len = rsm->r_end - rsm->r_start; 5041 if (rack->rack_tlp_threshold_use == TLP_USE_ID) { 5042 /* Exactly like the ID */ 5043 if (((tp->snd_max - tp->snd_una) - rack->r_ctl.rc_sacked + rack->r_ctl.rc_holes_rxt) <= segsiz) { 5044 uint32_t alt_thresh; 5045 /* 5046 * Compensate for delayed-ack with the d-ack time. 5047 */ 5048 alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time; 5049 if (alt_thresh > thresh) 5050 thresh = alt_thresh; 5051 } 5052 } else if (rack->rack_tlp_threshold_use == TLP_USE_TWO_ONE) { 5053 /* 2.1 behavior */ 5054 prsm = TAILQ_PREV(rsm, rack_head, r_tnext); 5055 if (prsm && (len <= segsiz)) { 5056 /* 5057 * Two packets outstanding, thresh should be (2*srtt) + 5058 * possible inter-packet delay (if any). 5059 */ 5060 uint32_t inter_gap = 0; 5061 int idx, nidx; 5062 5063 idx = rsm->r_rtr_cnt - 1; 5064 nidx = prsm->r_rtr_cnt - 1; 5065 if (rsm->r_tim_lastsent[nidx] >= prsm->r_tim_lastsent[idx]) { 5066 /* Yes it was sent later (or at the same time) */ 5067 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx]; 5068 } 5069 thresh += inter_gap; 5070 } else if (len <= segsiz) { 5071 /* 5072 * Possibly compensate for delayed-ack. 5073 */ 5074 uint32_t alt_thresh; 5075 5076 alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time; 5077 if (alt_thresh > thresh) 5078 thresh = alt_thresh; 5079 } 5080 } else if (rack->rack_tlp_threshold_use == TLP_USE_TWO_TWO) { 5081 /* 2.2 behavior */ 5082 if (len <= segsiz) { 5083 uint32_t alt_thresh; 5084 /* 5085 * Compensate for delayed-ack with the d-ack time. 5086 */ 5087 alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time; 5088 if (alt_thresh > thresh) 5089 thresh = alt_thresh; 5090 } 5091 } 5092 /* Not above an RTO */ 5093 if (thresh > tp->t_rxtcur) { 5094 thresh = tp->t_rxtcur; 5095 } 5096 /* Not above a RTO max */ 5097 if (thresh > rack_rto_max) { 5098 thresh = rack_rto_max; 5099 } 5100 /* Apply user supplied min TLP */ 5101 if (thresh < rack_tlp_min) { 5102 thresh = rack_tlp_min; 5103 } 5104 return (thresh); 5105 } 5106 5107 static uint32_t 5108 rack_grab_rtt(struct tcpcb *tp, struct tcp_rack *rack) 5109 { 5110 /* 5111 * We want the rack_rtt which is the 5112 * last rtt we measured. However if that 5113 * does not exist we fallback to the srtt (which 5114 * we probably will never do) and then as a last 5115 * resort we use RACK_INITIAL_RTO if no srtt is 5116 * yet set. 5117 */ 5118 if (rack->rc_rack_rtt) 5119 return (rack->rc_rack_rtt); 5120 else if (tp->t_srtt == 0) 5121 return (RACK_INITIAL_RTO); 5122 return (tp->t_srtt); 5123 } 5124 5125 static struct rack_sendmap * 5126 rack_check_recovery_mode(struct tcpcb *tp, uint32_t tsused) 5127 { 5128 /* 5129 * Check to see that we don't need to fall into recovery. We will 5130 * need to do so if our oldest transmit is past the time we should 5131 * have had an ack. 5132 */ 5133 struct tcp_rack *rack; 5134 struct rack_sendmap *rsm; 5135 int32_t idx; 5136 uint32_t srtt, thresh; 5137 5138 rack = (struct tcp_rack *)tp->t_fb_ptr; 5139 if (RB_EMPTY(&rack->r_ctl.rc_mtree)) { 5140 return (NULL); 5141 } 5142 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 5143 if (rsm == NULL) 5144 return (NULL); 5145 5146 5147 if (rsm->r_flags & RACK_ACKED) { 5148 rsm = rack_find_lowest_rsm(rack); 5149 if (rsm == NULL) 5150 return (NULL); 5151 } 5152 idx = rsm->r_rtr_cnt - 1; 5153 srtt = rack_grab_rtt(tp, rack); 5154 thresh = rack_calc_thresh_rack(rack, srtt, tsused); 5155 if (TSTMP_LT(tsused, ((uint32_t)rsm->r_tim_lastsent[idx]))) { 5156 return (NULL); 5157 } 5158 if ((tsused - ((uint32_t)rsm->r_tim_lastsent[idx])) < thresh) { 5159 return (NULL); 5160 } 5161 /* Ok if we reach here we are over-due and this guy can be sent */ 5162 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__); 5163 return (rsm); 5164 } 5165 5166 static uint32_t 5167 rack_get_persists_timer_val(struct tcpcb *tp, struct tcp_rack *rack) 5168 { 5169 int32_t t; 5170 int32_t tt; 5171 uint32_t ret_val; 5172 5173 t = (tp->t_srtt + (tp->t_rttvar << 2)); 5174 RACK_TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift], 5175 rack_persist_min, rack_persist_max, rack->r_ctl.timer_slop); 5176 rack->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT; 5177 ret_val = (uint32_t)tt; 5178 return (ret_val); 5179 } 5180 5181 static uint32_t 5182 rack_timer_start(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int sup_rack) 5183 { 5184 /* 5185 * Start the FR timer, we do this based on getting the first one in 5186 * the rc_tmap. Note that if its NULL we must stop the timer. in all 5187 * events we need to stop the running timer (if its running) before 5188 * starting the new one. 5189 */ 5190 uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse; 5191 uint32_t srtt_cur; 5192 int32_t idx; 5193 int32_t is_tlp_timer = 0; 5194 struct rack_sendmap *rsm; 5195 5196 if (rack->t_timers_stopped) { 5197 /* All timers have been stopped none are to run */ 5198 return (0); 5199 } 5200 if (rack->rc_in_persist) { 5201 /* We can't start any timer in persists */ 5202 return (rack_get_persists_timer_val(tp, rack)); 5203 } 5204 rack->rc_on_min_to = 0; 5205 if ((tp->t_state < TCPS_ESTABLISHED) || 5206 ((tp->t_flags & TF_SACK_PERMIT) == 0)) { 5207 goto activate_rxt; 5208 } 5209 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 5210 if ((rsm == NULL) || sup_rack) { 5211 /* Nothing on the send map or no rack */ 5212 activate_rxt: 5213 time_since_sent = 0; 5214 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 5215 if (rsm) { 5216 /* 5217 * Should we discount the RTX timer any? 5218 * 5219 * We want to discount it the smallest amount. 5220 * If a timer (Rack/TLP or RXT) has gone off more 5221 * recently thats the discount we want to use (now - timer time). 5222 * If the retransmit of the oldest packet was more recent then 5223 * we want to use that (now - oldest-packet-last_transmit_time). 5224 * 5225 */ 5226 idx = rsm->r_rtr_cnt - 1; 5227 if (TSTMP_GEQ(rack->r_ctl.rc_tlp_rxt_last_time, ((uint32_t)rsm->r_tim_lastsent[idx]))) 5228 tstmp_touse = (uint32_t)rack->r_ctl.rc_tlp_rxt_last_time; 5229 else 5230 tstmp_touse = (uint32_t)rsm->r_tim_lastsent[idx]; 5231 if (TSTMP_GT(cts, tstmp_touse)) 5232 time_since_sent = cts - tstmp_touse; 5233 } 5234 if (SEQ_LT(tp->snd_una, tp->snd_max) || 5235 sbavail(&tptosocket(tp)->so_snd)) { 5236 rack->r_ctl.rc_hpts_flags |= PACE_TMR_RXT; 5237 to = tp->t_rxtcur; 5238 if (to > time_since_sent) 5239 to -= time_since_sent; 5240 else 5241 to = rack->r_ctl.rc_min_to; 5242 if (to == 0) 5243 to = 1; 5244 /* Special case for KEEPINIT */ 5245 if ((TCPS_HAVEESTABLISHED(tp->t_state) == 0) && 5246 (TP_KEEPINIT(tp) != 0) && 5247 rsm) { 5248 /* 5249 * We have to put a ceiling on the rxt timer 5250 * of the keep-init timeout. 5251 */ 5252 uint32_t max_time, red; 5253 5254 max_time = TICKS_2_USEC(TP_KEEPINIT(tp)); 5255 if (TSTMP_GT(cts, (uint32_t)rsm->r_tim_lastsent[0])) { 5256 red = (cts - (uint32_t)rsm->r_tim_lastsent[0]); 5257 if (red < max_time) 5258 max_time -= red; 5259 else 5260 max_time = 1; 5261 } 5262 /* Reduce timeout to the keep value if needed */ 5263 if (max_time < to) 5264 to = max_time; 5265 } 5266 return (to); 5267 } 5268 return (0); 5269 } 5270 if (rsm->r_flags & RACK_ACKED) { 5271 rsm = rack_find_lowest_rsm(rack); 5272 if (rsm == NULL) { 5273 /* No lowest? */ 5274 goto activate_rxt; 5275 } 5276 } 5277 if (rack->sack_attack_disable) { 5278 /* 5279 * We don't want to do 5280 * any TLP's if you are an attacker. 5281 * Though if you are doing what 5282 * is expected you may still have 5283 * SACK-PASSED marks. 5284 */ 5285 goto activate_rxt; 5286 } 5287 /* Convert from ms to usecs */ 5288 if ((rsm->r_flags & RACK_SACK_PASSED) || 5289 (rsm->r_flags & RACK_RWND_COLLAPSED) || 5290 (rsm->r_dupack >= DUP_ACK_THRESHOLD)) { 5291 if ((tp->t_flags & TF_SENTFIN) && 5292 ((tp->snd_max - tp->snd_una) == 1) && 5293 (rsm->r_flags & RACK_HAS_FIN)) { 5294 /* 5295 * We don't start a rack timer if all we have is a 5296 * FIN outstanding. 5297 */ 5298 goto activate_rxt; 5299 } 5300 if ((rack->use_rack_rr == 0) && 5301 (IN_FASTRECOVERY(tp->t_flags)) && 5302 (rack->rack_no_prr == 0) && 5303 (rack->r_ctl.rc_prr_sndcnt < ctf_fixed_maxseg(tp))) { 5304 /* 5305 * We are not cheating, in recovery and 5306 * not enough ack's to yet get our next 5307 * retransmission out. 5308 * 5309 * Note that classified attackers do not 5310 * get to use the rack-cheat. 5311 */ 5312 goto activate_tlp; 5313 } 5314 srtt = rack_grab_rtt(tp, rack); 5315 thresh = rack_calc_thresh_rack(rack, srtt, cts); 5316 idx = rsm->r_rtr_cnt - 1; 5317 exp = ((uint32_t)rsm->r_tim_lastsent[idx]) + thresh; 5318 if (SEQ_GEQ(exp, cts)) { 5319 to = exp - cts; 5320 if (to < rack->r_ctl.rc_min_to) { 5321 to = rack->r_ctl.rc_min_to; 5322 if (rack->r_rr_config == 3) 5323 rack->rc_on_min_to = 1; 5324 } 5325 } else { 5326 to = rack->r_ctl.rc_min_to; 5327 if (rack->r_rr_config == 3) 5328 rack->rc_on_min_to = 1; 5329 } 5330 } else { 5331 /* Ok we need to do a TLP not RACK */ 5332 activate_tlp: 5333 if ((rack->rc_tlp_in_progress != 0) && 5334 (rack->r_ctl.rc_tlp_cnt_out >= rack_tlp_limit)) { 5335 /* 5336 * The previous send was a TLP and we have sent 5337 * N TLP's without sending new data. 5338 */ 5339 goto activate_rxt; 5340 } 5341 rsm = TAILQ_LAST_FAST(&rack->r_ctl.rc_tmap, rack_sendmap, r_tnext); 5342 if (rsm == NULL) { 5343 /* We found no rsm to TLP with. */ 5344 goto activate_rxt; 5345 } 5346 if (rsm->r_flags & RACK_HAS_FIN) { 5347 /* If its a FIN we dont do TLP */ 5348 rsm = NULL; 5349 goto activate_rxt; 5350 } 5351 idx = rsm->r_rtr_cnt - 1; 5352 time_since_sent = 0; 5353 if (TSTMP_GEQ(((uint32_t)rsm->r_tim_lastsent[idx]), rack->r_ctl.rc_tlp_rxt_last_time)) 5354 tstmp_touse = (uint32_t)rsm->r_tim_lastsent[idx]; 5355 else 5356 tstmp_touse = (uint32_t)rack->r_ctl.rc_tlp_rxt_last_time; 5357 if (TSTMP_GT(cts, tstmp_touse)) 5358 time_since_sent = cts - tstmp_touse; 5359 is_tlp_timer = 1; 5360 if (tp->t_srtt) { 5361 if ((rack->rc_srtt_measure_made == 0) && 5362 (tp->t_srtt == 1)) { 5363 /* 5364 * If another stack as run and set srtt to 1, 5365 * then the srtt was 0, so lets use the initial. 5366 */ 5367 srtt = RACK_INITIAL_RTO; 5368 } else { 5369 srtt_cur = tp->t_srtt; 5370 srtt = srtt_cur; 5371 } 5372 } else 5373 srtt = RACK_INITIAL_RTO; 5374 /* 5375 * If the SRTT is not keeping up and the 5376 * rack RTT has spiked we want to use 5377 * the last RTT not the smoothed one. 5378 */ 5379 if (rack_tlp_use_greater && 5380 tp->t_srtt && 5381 (srtt < rack_grab_rtt(tp, rack))) { 5382 srtt = rack_grab_rtt(tp, rack); 5383 } 5384 thresh = rack_calc_thresh_tlp(tp, rack, rsm, srtt); 5385 if (thresh > time_since_sent) { 5386 to = thresh - time_since_sent; 5387 } else { 5388 to = rack->r_ctl.rc_min_to; 5389 rack_log_alt_to_to_cancel(rack, 5390 thresh, /* flex1 */ 5391 time_since_sent, /* flex2 */ 5392 tstmp_touse, /* flex3 */ 5393 rack->r_ctl.rc_tlp_rxt_last_time, /* flex4 */ 5394 (uint32_t)rsm->r_tim_lastsent[idx], 5395 srtt, 5396 idx, 99); 5397 } 5398 if (to < rack_tlp_min) { 5399 to = rack_tlp_min; 5400 } 5401 if (to > TICKS_2_USEC(TCPTV_REXMTMAX)) { 5402 /* 5403 * If the TLP time works out to larger than the max 5404 * RTO lets not do TLP.. just RTO. 5405 */ 5406 goto activate_rxt; 5407 } 5408 } 5409 if (is_tlp_timer == 0) { 5410 rack->r_ctl.rc_hpts_flags |= PACE_TMR_RACK; 5411 } else { 5412 rack->r_ctl.rc_hpts_flags |= PACE_TMR_TLP; 5413 } 5414 if (to == 0) 5415 to = 1; 5416 return (to); 5417 } 5418 5419 static void 5420 rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 5421 { 5422 if (rack->rc_in_persist == 0) { 5423 if (tp->t_flags & TF_GPUTINPROG) { 5424 /* 5425 * Stop the goodput now, the calling of the 5426 * measurement function clears the flag. 5427 */ 5428 rack_do_goodput_measurement(tp, rack, tp->snd_una, __LINE__, 5429 RACK_QUALITY_PERSIST); 5430 } 5431 #ifdef NETFLIX_SHARED_CWND 5432 if (rack->r_ctl.rc_scw) { 5433 tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index); 5434 rack->rack_scwnd_is_idle = 1; 5435 } 5436 #endif 5437 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL); 5438 if (rack->r_ctl.rc_went_idle_time == 0) 5439 rack->r_ctl.rc_went_idle_time = 1; 5440 rack_timer_cancel(tp, rack, cts, __LINE__); 5441 rack->r_ctl.persist_lost_ends = 0; 5442 rack->probe_not_answered = 0; 5443 rack->forced_ack = 0; 5444 tp->t_rxtshift = 0; 5445 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 5446 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 5447 rack->rc_in_persist = 1; 5448 } 5449 } 5450 5451 static void 5452 rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 5453 { 5454 if (tcp_in_hpts(rack->rc_inp)) { 5455 tcp_hpts_remove(rack->rc_inp); 5456 rack->r_ctl.rc_hpts_flags = 0; 5457 } 5458 #ifdef NETFLIX_SHARED_CWND 5459 if (rack->r_ctl.rc_scw) { 5460 tcp_shared_cwnd_active(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index); 5461 rack->rack_scwnd_is_idle = 0; 5462 } 5463 #endif 5464 if (rack->rc_gp_dyn_mul && 5465 (rack->use_fixed_rate == 0) && 5466 (rack->rc_always_pace)) { 5467 /* 5468 * Do we count this as if a probe-rtt just 5469 * finished? 5470 */ 5471 uint32_t time_idle, idle_min; 5472 5473 time_idle = tcp_get_usecs(NULL) - rack->r_ctl.rc_went_idle_time; 5474 idle_min = rack_min_probertt_hold; 5475 if (rack_probertt_gpsrtt_cnt_div) { 5476 uint64_t extra; 5477 extra = (uint64_t)rack->r_ctl.rc_gp_srtt * 5478 (uint64_t)rack_probertt_gpsrtt_cnt_mul; 5479 extra /= (uint64_t)rack_probertt_gpsrtt_cnt_div; 5480 idle_min += (uint32_t)extra; 5481 } 5482 if (time_idle >= idle_min) { 5483 /* Yes, we count it as a probe-rtt. */ 5484 uint32_t us_cts; 5485 5486 us_cts = tcp_get_usecs(NULL); 5487 if (rack->in_probe_rtt == 0) { 5488 rack->r_ctl.rc_lower_rtt_us_cts = us_cts; 5489 rack->r_ctl.rc_time_probertt_entered = rack->r_ctl.rc_lower_rtt_us_cts; 5490 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts; 5491 rack->r_ctl.rc_time_of_last_probertt = rack->r_ctl.rc_lower_rtt_us_cts; 5492 } else { 5493 rack_exit_probertt(rack, us_cts); 5494 } 5495 } 5496 } 5497 rack->rc_in_persist = 0; 5498 rack->r_ctl.rc_went_idle_time = 0; 5499 tp->t_rxtshift = 0; 5500 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 5501 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 5502 rack->r_ctl.rc_agg_delayed = 0; 5503 rack->r_early = 0; 5504 rack->r_late = 0; 5505 rack->r_ctl.rc_agg_early = 0; 5506 } 5507 5508 static void 5509 rack_log_hpts_diag(struct tcp_rack *rack, uint32_t cts, 5510 struct hpts_diag *diag, struct timeval *tv) 5511 { 5512 if (rack_verbose_logging && rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 5513 union tcp_log_stackspecific log; 5514 5515 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 5516 log.u_bbr.flex1 = diag->p_nxt_slot; 5517 log.u_bbr.flex2 = diag->p_cur_slot; 5518 log.u_bbr.flex3 = diag->slot_req; 5519 log.u_bbr.flex4 = diag->inp_hptsslot; 5520 log.u_bbr.flex5 = diag->slot_remaining; 5521 log.u_bbr.flex6 = diag->need_new_to; 5522 log.u_bbr.flex7 = diag->p_hpts_active; 5523 log.u_bbr.flex8 = diag->p_on_min_sleep; 5524 /* Hijack other fields as needed */ 5525 log.u_bbr.epoch = diag->have_slept; 5526 log.u_bbr.lt_epoch = diag->yet_to_sleep; 5527 log.u_bbr.pkts_out = diag->co_ret; 5528 log.u_bbr.applimited = diag->hpts_sleep_time; 5529 log.u_bbr.delivered = diag->p_prev_slot; 5530 log.u_bbr.inflight = diag->p_runningslot; 5531 log.u_bbr.bw_inuse = diag->wheel_slot; 5532 log.u_bbr.rttProp = diag->wheel_cts; 5533 log.u_bbr.timeStamp = cts; 5534 log.u_bbr.delRate = diag->maxslots; 5535 log.u_bbr.cur_del_rate = diag->p_curtick; 5536 log.u_bbr.cur_del_rate <<= 32; 5537 log.u_bbr.cur_del_rate |= diag->p_lasttick; 5538 TCP_LOG_EVENTP(rack->rc_tp, NULL, 5539 &rack->rc_inp->inp_socket->so_rcv, 5540 &rack->rc_inp->inp_socket->so_snd, 5541 BBR_LOG_HPTSDIAG, 0, 5542 0, &log, false, tv); 5543 } 5544 5545 } 5546 5547 static void 5548 rack_log_wakeup(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb, uint32_t len, int type) 5549 { 5550 if (rack_verbose_logging && rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 5551 union tcp_log_stackspecific log; 5552 struct timeval tv; 5553 5554 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 5555 log.u_bbr.flex1 = sb->sb_flags; 5556 log.u_bbr.flex2 = len; 5557 log.u_bbr.flex3 = sb->sb_state; 5558 log.u_bbr.flex8 = type; 5559 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 5560 TCP_LOG_EVENTP(rack->rc_tp, NULL, 5561 &rack->rc_inp->inp_socket->so_rcv, 5562 &rack->rc_inp->inp_socket->so_snd, 5563 TCP_LOG_SB_WAKE, 0, 5564 len, &log, false, &tv); 5565 } 5566 } 5567 5568 static void 5569 rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts, 5570 int32_t slot, uint32_t tot_len_this_send, int sup_rack) 5571 { 5572 struct hpts_diag diag; 5573 struct inpcb *inp = tptoinpcb(tp); 5574 struct timeval tv; 5575 uint32_t delayed_ack = 0; 5576 uint32_t hpts_timeout; 5577 uint32_t entry_slot = slot; 5578 uint8_t stopped; 5579 uint32_t left = 0; 5580 uint32_t us_cts; 5581 5582 if ((tp->t_state == TCPS_CLOSED) || 5583 (tp->t_state == TCPS_LISTEN)) { 5584 return; 5585 } 5586 if (tcp_in_hpts(inp)) { 5587 /* Already on the pacer */ 5588 return; 5589 } 5590 stopped = rack->rc_tmr_stopped; 5591 if (stopped && TSTMP_GT(rack->r_ctl.rc_timer_exp, cts)) { 5592 left = rack->r_ctl.rc_timer_exp - cts; 5593 } 5594 rack->r_ctl.rc_timer_exp = 0; 5595 rack->r_ctl.rc_hpts_flags = 0; 5596 us_cts = tcp_get_usecs(&tv); 5597 /* Now early/late accounting */ 5598 rack_log_pacing_delay_calc(rack, entry_slot, slot, 0, 0, 0, 26, __LINE__, NULL, 0); 5599 if (rack->r_early && (rack->rc_ack_can_sendout_data == 0)) { 5600 /* 5601 * We have a early carry over set, 5602 * we can always add more time so we 5603 * can always make this compensation. 5604 * 5605 * Note if ack's are allowed to wake us do not 5606 * penalize the next timer for being awoke 5607 * by an ack aka the rc_agg_early (non-paced mode). 5608 */ 5609 slot += rack->r_ctl.rc_agg_early; 5610 rack->r_early = 0; 5611 rack->r_ctl.rc_agg_early = 0; 5612 } 5613 if (rack->r_late) { 5614 /* 5615 * This is harder, we can 5616 * compensate some but it 5617 * really depends on what 5618 * the current pacing time is. 5619 */ 5620 if (rack->r_ctl.rc_agg_delayed >= slot) { 5621 /* 5622 * We can't compensate for it all. 5623 * And we have to have some time 5624 * on the clock. We always have a min 5625 * 10 slots (10 x 10 i.e. 100 usecs). 5626 */ 5627 if (slot <= HPTS_TICKS_PER_SLOT) { 5628 /* We gain delay */ 5629 rack->r_ctl.rc_agg_delayed += (HPTS_TICKS_PER_SLOT - slot); 5630 slot = HPTS_TICKS_PER_SLOT; 5631 } else { 5632 /* We take off some */ 5633 rack->r_ctl.rc_agg_delayed -= (slot - HPTS_TICKS_PER_SLOT); 5634 slot = HPTS_TICKS_PER_SLOT; 5635 } 5636 } else { 5637 slot -= rack->r_ctl.rc_agg_delayed; 5638 rack->r_ctl.rc_agg_delayed = 0; 5639 /* Make sure we have 100 useconds at minimum */ 5640 if (slot < HPTS_TICKS_PER_SLOT) { 5641 rack->r_ctl.rc_agg_delayed = HPTS_TICKS_PER_SLOT - slot; 5642 slot = HPTS_TICKS_PER_SLOT; 5643 } 5644 if (rack->r_ctl.rc_agg_delayed == 0) 5645 rack->r_late = 0; 5646 } 5647 } 5648 if (slot) { 5649 /* We are pacing too */ 5650 rack->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT; 5651 } 5652 hpts_timeout = rack_timer_start(tp, rack, cts, sup_rack); 5653 #ifdef NETFLIX_EXP_DETECTION 5654 if (rack->sack_attack_disable && 5655 (slot < tcp_sad_pacing_interval)) { 5656 /* 5657 * We have a potential attacker on 5658 * the line. We have possibly some 5659 * (or now) pacing time set. We want to 5660 * slow down the processing of sacks by some 5661 * amount (if it is an attacker). Set the default 5662 * slot for attackers in place (unless the original 5663 * interval is longer). Its stored in 5664 * micro-seconds, so lets convert to msecs. 5665 */ 5666 slot = tcp_sad_pacing_interval; 5667 } 5668 #endif 5669 if (tp->t_flags & TF_DELACK) { 5670 delayed_ack = TICKS_2_USEC(tcp_delacktime); 5671 rack->r_ctl.rc_hpts_flags |= PACE_TMR_DELACK; 5672 } 5673 if (delayed_ack && ((hpts_timeout == 0) || 5674 (delayed_ack < hpts_timeout))) 5675 hpts_timeout = delayed_ack; 5676 else 5677 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK; 5678 /* 5679 * If no timers are going to run and we will fall off the hptsi 5680 * wheel, we resort to a keep-alive timer if its configured. 5681 */ 5682 if ((hpts_timeout == 0) && 5683 (slot == 0)) { 5684 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && 5685 (tp->t_state <= TCPS_CLOSING)) { 5686 /* 5687 * Ok we have no timer (persists, rack, tlp, rxt or 5688 * del-ack), we don't have segments being paced. So 5689 * all that is left is the keepalive timer. 5690 */ 5691 if (TCPS_HAVEESTABLISHED(tp->t_state)) { 5692 /* Get the established keep-alive time */ 5693 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp)); 5694 } else { 5695 /* 5696 * Get the initial setup keep-alive time, 5697 * note that this is probably not going to 5698 * happen, since rack will be running a rxt timer 5699 * if a SYN of some sort is outstanding. It is 5700 * actually handled in rack_timeout_rxt(). 5701 */ 5702 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp)); 5703 } 5704 rack->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP; 5705 if (rack->in_probe_rtt) { 5706 /* 5707 * We want to instead not wake up a long time from 5708 * now but to wake up about the time we would 5709 * exit probe-rtt and initiate a keep-alive ack. 5710 * This will get us out of probe-rtt and update 5711 * our min-rtt. 5712 */ 5713 hpts_timeout = rack_min_probertt_hold; 5714 } 5715 } 5716 } 5717 if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) == 5718 (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) { 5719 /* 5720 * RACK, TLP, persists and RXT timers all are restartable 5721 * based on actions input .. i.e we received a packet (ack 5722 * or sack) and that changes things (rw, or snd_una etc). 5723 * Thus we can restart them with a new value. For 5724 * keep-alive, delayed_ack we keep track of what was left 5725 * and restart the timer with a smaller value. 5726 */ 5727 if (left < hpts_timeout) 5728 hpts_timeout = left; 5729 } 5730 if (hpts_timeout) { 5731 /* 5732 * Hack alert for now we can't time-out over 2,147,483 5733 * seconds (a bit more than 596 hours), which is probably ok 5734 * :). 5735 */ 5736 if (hpts_timeout > 0x7ffffffe) 5737 hpts_timeout = 0x7ffffffe; 5738 rack->r_ctl.rc_timer_exp = cts + hpts_timeout; 5739 } 5740 rack_log_pacing_delay_calc(rack, entry_slot, slot, hpts_timeout, 0, 0, 27, __LINE__, NULL, 0); 5741 if ((rack->gp_ready == 0) && 5742 (rack->use_fixed_rate == 0) && 5743 (hpts_timeout < slot) && 5744 (rack->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) { 5745 /* 5746 * We have no good estimate yet for the 5747 * old clunky burst mitigation or the 5748 * real pacing. And the tlp or rxt is smaller 5749 * than the pacing calculation. Lets not 5750 * pace that long since we know the calculation 5751 * so far is not accurate. 5752 */ 5753 slot = hpts_timeout; 5754 } 5755 /** 5756 * Turn off all the flags for queuing by default. The 5757 * flags have important meanings to what happens when 5758 * LRO interacts with the transport. Most likely (by default now) 5759 * mbuf_queueing and ack compression are on. So the transport 5760 * has a couple of flags that control what happens (if those 5761 * are not on then these flags won't have any effect since it 5762 * won't go through the queuing LRO path). 5763 * 5764 * INP_MBUF_QUEUE_READY - This flags says that I am busy 5765 * pacing output, so don't disturb. But 5766 * it also means LRO can wake me if there 5767 * is a SACK arrival. 5768 * 5769 * INP_DONT_SACK_QUEUE - This flag is used in conjunction 5770 * with the above flag (QUEUE_READY) and 5771 * when present it says don't even wake me 5772 * if a SACK arrives. 5773 * 5774 * The idea behind these flags is that if we are pacing we 5775 * set the MBUF_QUEUE_READY and only get woken up if 5776 * a SACK arrives (which could change things) or if 5777 * our pacing timer expires. If, however, we have a rack 5778 * timer running, then we don't even want a sack to wake 5779 * us since the rack timer has to expire before we can send. 5780 * 5781 * Other cases should usually have none of the flags set 5782 * so LRO can call into us. 5783 */ 5784 inp->inp_flags2 &= ~(INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY); 5785 if (slot) { 5786 rack->r_ctl.rc_last_output_to = us_cts + slot; 5787 /* 5788 * A pacing timer (slot) is being set, in 5789 * such a case we cannot send (we are blocked by 5790 * the timer). So lets tell LRO that it should not 5791 * wake us unless there is a SACK. Note this only 5792 * will be effective if mbuf queueing is on or 5793 * compressed acks are being processed. 5794 */ 5795 inp->inp_flags2 |= INP_MBUF_QUEUE_READY; 5796 /* 5797 * But wait if we have a Rack timer running 5798 * even a SACK should not disturb us (with 5799 * the exception of r_rr_config 3). 5800 */ 5801 if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK) && 5802 (rack->r_rr_config != 3)) 5803 inp->inp_flags2 |= INP_DONT_SACK_QUEUE; 5804 if (rack->rc_ack_can_sendout_data) { 5805 /* 5806 * Ahh but wait, this is that special case 5807 * where the pacing timer can be disturbed 5808 * backout the changes (used for non-paced 5809 * burst limiting). 5810 */ 5811 inp->inp_flags2 &= ~(INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY); 5812 } 5813 if ((rack->use_rack_rr) && 5814 (rack->r_rr_config < 2) && 5815 ((hpts_timeout) && (hpts_timeout < slot))) { 5816 /* 5817 * Arrange for the hpts to kick back in after the 5818 * t-o if the t-o does not cause a send. 5819 */ 5820 (void)tcp_hpts_insert_diag(inp, HPTS_USEC_TO_SLOTS(hpts_timeout), 5821 __LINE__, &diag); 5822 rack_log_hpts_diag(rack, us_cts, &diag, &tv); 5823 rack_log_to_start(rack, cts, hpts_timeout, slot, 0); 5824 } else { 5825 (void)tcp_hpts_insert_diag(inp, HPTS_USEC_TO_SLOTS(slot), 5826 __LINE__, &diag); 5827 rack_log_hpts_diag(rack, us_cts, &diag, &tv); 5828 rack_log_to_start(rack, cts, hpts_timeout, slot, 1); 5829 } 5830 } else if (hpts_timeout) { 5831 /* 5832 * With respect to inp_flags2 here, lets let any new acks wake 5833 * us up here. Since we are not pacing (no pacing timer), output 5834 * can happen so we should let it. If its a Rack timer, then any inbound 5835 * packet probably won't change the sending (we will be blocked) 5836 * but it may change the prr stats so letting it in (the set defaults 5837 * at the start of this block) are good enough. 5838 */ 5839 (void)tcp_hpts_insert_diag(inp, HPTS_USEC_TO_SLOTS(hpts_timeout), 5840 __LINE__, &diag); 5841 rack_log_hpts_diag(rack, us_cts, &diag, &tv); 5842 rack_log_to_start(rack, cts, hpts_timeout, slot, 0); 5843 } else { 5844 /* No timer starting */ 5845 #ifdef INVARIANTS 5846 if (SEQ_GT(tp->snd_max, tp->snd_una)) { 5847 panic("tp:%p rack:%p tlts:%d cts:%u slot:%u pto:%u -- no timer started?", 5848 tp, rack, tot_len_this_send, cts, slot, hpts_timeout); 5849 } 5850 #endif 5851 } 5852 rack->rc_tmr_stopped = 0; 5853 if (slot) 5854 rack_log_type_bbrsnd(rack, tot_len_this_send, slot, us_cts, &tv); 5855 } 5856 5857 /* 5858 * RACK Timer, here we simply do logging and house keeping. 5859 * the normal rack_output() function will call the 5860 * appropriate thing to check if we need to do a RACK retransmit. 5861 * We return 1, saying don't proceed with rack_output only 5862 * when all timers have been stopped (destroyed PCB?). 5863 */ 5864 static int 5865 rack_timeout_rack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 5866 { 5867 /* 5868 * This timer simply provides an internal trigger to send out data. 5869 * The check_recovery_mode call will see if there are needed 5870 * retransmissions, if so we will enter fast-recovery. The output 5871 * call may or may not do the same thing depending on sysctl 5872 * settings. 5873 */ 5874 struct rack_sendmap *rsm; 5875 5876 counter_u64_add(rack_to_tot, 1); 5877 if (rack->r_state && (rack->r_state != tp->t_state)) 5878 rack_set_state(tp, rack); 5879 rack->rc_on_min_to = 0; 5880 rsm = rack_check_recovery_mode(tp, cts); 5881 rack_log_to_event(rack, RACK_TO_FRM_RACK, rsm); 5882 if (rsm) { 5883 rack->r_ctl.rc_resend = rsm; 5884 rack->r_timer_override = 1; 5885 if (rack->use_rack_rr) { 5886 /* 5887 * Don't accumulate extra pacing delay 5888 * we are allowing the rack timer to 5889 * over-ride pacing i.e. rrr takes precedence 5890 * if the pacing interval is longer than the rrr 5891 * time (in other words we get the min pacing 5892 * time versus rrr pacing time). 5893 */ 5894 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT; 5895 } 5896 } 5897 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK; 5898 if (rsm == NULL) { 5899 /* restart a timer and return 1 */ 5900 rack_start_hpts_timer(rack, tp, cts, 5901 0, 0, 0); 5902 return (1); 5903 } 5904 return (0); 5905 } 5906 5907 static void 5908 rack_adjust_orig_mlen(struct rack_sendmap *rsm) 5909 { 5910 if (rsm->m->m_len > rsm->orig_m_len) { 5911 /* 5912 * Mbuf grew, caused by sbcompress, our offset does 5913 * not change. 5914 */ 5915 rsm->orig_m_len = rsm->m->m_len; 5916 } else if (rsm->m->m_len < rsm->orig_m_len) { 5917 /* 5918 * Mbuf shrank, trimmed off the top by an ack, our 5919 * offset changes. 5920 */ 5921 rsm->soff -= (rsm->orig_m_len - rsm->m->m_len); 5922 rsm->orig_m_len = rsm->m->m_len; 5923 } 5924 } 5925 5926 static void 5927 rack_setup_offset_for_rsm(struct rack_sendmap *src_rsm, struct rack_sendmap *rsm) 5928 { 5929 struct mbuf *m; 5930 uint32_t soff; 5931 5932 if (src_rsm->m && (src_rsm->orig_m_len != src_rsm->m->m_len)) { 5933 /* Fix up the orig_m_len and possibly the mbuf offset */ 5934 rack_adjust_orig_mlen(src_rsm); 5935 } 5936 m = src_rsm->m; 5937 soff = src_rsm->soff + (src_rsm->r_end - src_rsm->r_start); 5938 while (soff >= m->m_len) { 5939 /* Move out past this mbuf */ 5940 soff -= m->m_len; 5941 m = m->m_next; 5942 KASSERT((m != NULL), 5943 ("rsm:%p nrsm:%p hit at soff:%u null m", 5944 src_rsm, rsm, soff)); 5945 } 5946 rsm->m = m; 5947 rsm->soff = soff; 5948 rsm->orig_m_len = m->m_len; 5949 } 5950 5951 static __inline void 5952 rack_clone_rsm(struct tcp_rack *rack, struct rack_sendmap *nrsm, 5953 struct rack_sendmap *rsm, uint32_t start) 5954 { 5955 int idx; 5956 5957 nrsm->r_start = start; 5958 nrsm->r_end = rsm->r_end; 5959 nrsm->r_rtr_cnt = rsm->r_rtr_cnt; 5960 nrsm->r_flags = rsm->r_flags; 5961 nrsm->r_dupack = rsm->r_dupack; 5962 nrsm->r_no_rtt_allowed = rsm->r_no_rtt_allowed; 5963 nrsm->r_rtr_bytes = 0; 5964 nrsm->r_fas = rsm->r_fas; 5965 rsm->r_end = nrsm->r_start; 5966 nrsm->r_just_ret = rsm->r_just_ret; 5967 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) { 5968 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx]; 5969 } 5970 /* Now if we have SYN flag we keep it on the left edge */ 5971 if (nrsm->r_flags & RACK_HAS_SYN) 5972 nrsm->r_flags &= ~RACK_HAS_SYN; 5973 /* Now if we have a FIN flag we keep it on the right edge */ 5974 if (rsm->r_flags & RACK_HAS_FIN) 5975 rsm->r_flags &= ~RACK_HAS_FIN; 5976 /* Push bit must go to the right edge as well */ 5977 if (rsm->r_flags & RACK_HAD_PUSH) 5978 rsm->r_flags &= ~RACK_HAD_PUSH; 5979 /* Clone over the state of the hw_tls flag */ 5980 nrsm->r_hw_tls = rsm->r_hw_tls; 5981 /* 5982 * Now we need to find nrsm's new location in the mbuf chain 5983 * we basically calculate a new offset, which is soff + 5984 * how much is left in original rsm. Then we walk out the mbuf 5985 * chain to find the righ position, it may be the same mbuf 5986 * or maybe not. 5987 */ 5988 KASSERT(((rsm->m != NULL) || 5989 (rsm->r_flags & (RACK_HAS_SYN|RACK_HAS_FIN))), 5990 ("rsm:%p nrsm:%p rack:%p -- rsm->m is NULL?", rsm, nrsm, rack)); 5991 if (rsm->m) 5992 rack_setup_offset_for_rsm(rsm, nrsm); 5993 } 5994 5995 static struct rack_sendmap * 5996 rack_merge_rsm(struct tcp_rack *rack, 5997 struct rack_sendmap *l_rsm, 5998 struct rack_sendmap *r_rsm) 5999 { 6000 /* 6001 * We are merging two ack'd RSM's, 6002 * the l_rsm is on the left (lower seq 6003 * values) and the r_rsm is on the right 6004 * (higher seq value). The simplest way 6005 * to merge these is to move the right 6006 * one into the left. I don't think there 6007 * is any reason we need to try to find 6008 * the oldest (or last oldest retransmitted). 6009 */ 6010 #ifdef INVARIANTS 6011 struct rack_sendmap *rm; 6012 #endif 6013 rack_log_map_chg(rack->rc_tp, rack, NULL, 6014 l_rsm, r_rsm, MAP_MERGE, r_rsm->r_end, __LINE__); 6015 l_rsm->r_end = r_rsm->r_end; 6016 if (l_rsm->r_dupack < r_rsm->r_dupack) 6017 l_rsm->r_dupack = r_rsm->r_dupack; 6018 if (r_rsm->r_rtr_bytes) 6019 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes; 6020 if (r_rsm->r_in_tmap) { 6021 /* This really should not happen */ 6022 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, r_rsm, r_tnext); 6023 r_rsm->r_in_tmap = 0; 6024 } 6025 6026 /* Now the flags */ 6027 if (r_rsm->r_flags & RACK_HAS_FIN) 6028 l_rsm->r_flags |= RACK_HAS_FIN; 6029 if (r_rsm->r_flags & RACK_TLP) 6030 l_rsm->r_flags |= RACK_TLP; 6031 if (r_rsm->r_flags & RACK_RWND_COLLAPSED) 6032 l_rsm->r_flags |= RACK_RWND_COLLAPSED; 6033 if ((r_rsm->r_flags & RACK_APP_LIMITED) && 6034 ((l_rsm->r_flags & RACK_APP_LIMITED) == 0)) { 6035 /* 6036 * If both are app-limited then let the 6037 * free lower the count. If right is app 6038 * limited and left is not, transfer. 6039 */ 6040 l_rsm->r_flags |= RACK_APP_LIMITED; 6041 r_rsm->r_flags &= ~RACK_APP_LIMITED; 6042 if (r_rsm == rack->r_ctl.rc_first_appl) 6043 rack->r_ctl.rc_first_appl = l_rsm; 6044 } 6045 #ifndef INVARIANTS 6046 (void)RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, r_rsm); 6047 #else 6048 rm = RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, r_rsm); 6049 if (rm != r_rsm) { 6050 panic("removing head in rack:%p rsm:%p rm:%p", 6051 rack, r_rsm, rm); 6052 } 6053 #endif 6054 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) { 6055 /* Transfer the split limit to the map we free */ 6056 r_rsm->r_limit_type = l_rsm->r_limit_type; 6057 l_rsm->r_limit_type = 0; 6058 } 6059 rack_free(rack, r_rsm); 6060 return (l_rsm); 6061 } 6062 6063 /* 6064 * TLP Timer, here we simply setup what segment we want to 6065 * have the TLP expire on, the normal rack_output() will then 6066 * send it out. 6067 * 6068 * We return 1, saying don't proceed with rack_output only 6069 * when all timers have been stopped (destroyed PCB?). 6070 */ 6071 static int 6072 rack_timeout_tlp(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t *doing_tlp) 6073 { 6074 /* 6075 * Tail Loss Probe. 6076 */ 6077 struct rack_sendmap *rsm = NULL; 6078 #ifdef INVARIANTS 6079 struct rack_sendmap *insret; 6080 #endif 6081 struct socket *so = tptosocket(tp); 6082 uint32_t amm; 6083 uint32_t out, avail; 6084 int collapsed_win = 0; 6085 6086 if (TSTMP_LT(cts, rack->r_ctl.rc_timer_exp)) { 6087 /* Its not time yet */ 6088 return (0); 6089 } 6090 if (ctf_progress_timeout_check(tp, true)) { 6091 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__); 6092 return (-ETIMEDOUT); /* tcp_drop() */ 6093 } 6094 /* 6095 * A TLP timer has expired. We have been idle for 2 rtts. So we now 6096 * need to figure out how to force a full MSS segment out. 6097 */ 6098 rack_log_to_event(rack, RACK_TO_FRM_TLP, NULL); 6099 rack->r_ctl.retran_during_recovery = 0; 6100 rack->r_ctl.dsack_byte_cnt = 0; 6101 counter_u64_add(rack_tlp_tot, 1); 6102 if (rack->r_state && (rack->r_state != tp->t_state)) 6103 rack_set_state(tp, rack); 6104 avail = sbavail(&so->so_snd); 6105 out = tp->snd_max - tp->snd_una; 6106 if ((out > tp->snd_wnd) || rack->rc_has_collapsed) { 6107 /* special case, we need a retransmission */ 6108 collapsed_win = 1; 6109 goto need_retran; 6110 } 6111 if (rack->r_ctl.dsack_persist && (rack->r_ctl.rc_tlp_cnt_out >= 1)) { 6112 rack->r_ctl.dsack_persist--; 6113 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) { 6114 rack->r_ctl.num_dsack = 0; 6115 } 6116 rack_log_dsack_event(rack, 1, __LINE__, 0, 0); 6117 } 6118 if ((tp->t_flags & TF_GPUTINPROG) && 6119 (rack->r_ctl.rc_tlp_cnt_out == 1)) { 6120 /* 6121 * If this is the second in a row 6122 * TLP and we are doing a measurement 6123 * its time to abandon the measurement. 6124 * Something is likely broken on 6125 * the clients network and measuring a 6126 * broken network does us no good. 6127 */ 6128 tp->t_flags &= ~TF_GPUTINPROG; 6129 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, 6130 rack->r_ctl.rc_gp_srtt /*flex1*/, 6131 tp->gput_seq, 6132 0, 0, 18, __LINE__, NULL, 0); 6133 } 6134 /* 6135 * Check our send oldest always settings, and if 6136 * there is an oldest to send jump to the need_retran. 6137 */ 6138 if (rack_always_send_oldest && (TAILQ_EMPTY(&rack->r_ctl.rc_tmap) == 0)) 6139 goto need_retran; 6140 6141 if (avail > out) { 6142 /* New data is available */ 6143 amm = avail - out; 6144 if (amm > ctf_fixed_maxseg(tp)) { 6145 amm = ctf_fixed_maxseg(tp); 6146 if ((amm + out) > tp->snd_wnd) { 6147 /* We are rwnd limited */ 6148 goto need_retran; 6149 } 6150 } else if (amm < ctf_fixed_maxseg(tp)) { 6151 /* not enough to fill a MTU */ 6152 goto need_retran; 6153 } 6154 if (IN_FASTRECOVERY(tp->t_flags)) { 6155 /* Unlikely */ 6156 if (rack->rack_no_prr == 0) { 6157 if (out + amm <= tp->snd_wnd) { 6158 rack->r_ctl.rc_prr_sndcnt = amm; 6159 rack->r_ctl.rc_tlp_new_data = amm; 6160 rack_log_to_prr(rack, 4, 0, __LINE__); 6161 } 6162 } else 6163 goto need_retran; 6164 } else { 6165 /* Set the send-new override */ 6166 if (out + amm <= tp->snd_wnd) 6167 rack->r_ctl.rc_tlp_new_data = amm; 6168 else 6169 goto need_retran; 6170 } 6171 rack->r_ctl.rc_tlpsend = NULL; 6172 counter_u64_add(rack_tlp_newdata, 1); 6173 goto send; 6174 } 6175 need_retran: 6176 /* 6177 * Ok we need to arrange the last un-acked segment to be re-sent, or 6178 * optionally the first un-acked segment. 6179 */ 6180 if (collapsed_win == 0) { 6181 if (rack_always_send_oldest) 6182 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 6183 else { 6184 rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 6185 if (rsm && (rsm->r_flags & (RACK_ACKED | RACK_HAS_FIN))) { 6186 rsm = rack_find_high_nonack(rack, rsm); 6187 } 6188 } 6189 if (rsm == NULL) { 6190 #ifdef TCP_BLACKBOX 6191 tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true); 6192 #endif 6193 goto out; 6194 } 6195 } else { 6196 /* 6197 * We must find the last segment 6198 * that was acceptable by the client. 6199 */ 6200 RB_FOREACH_REVERSE(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) { 6201 if ((rsm->r_flags & RACK_RWND_COLLAPSED) == 0) { 6202 /* Found one */ 6203 break; 6204 } 6205 } 6206 if (rsm == NULL) { 6207 /* None? if so send the first */ 6208 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 6209 if (rsm == NULL) { 6210 #ifdef TCP_BLACKBOX 6211 tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true); 6212 #endif 6213 goto out; 6214 } 6215 } 6216 } 6217 if ((rsm->r_end - rsm->r_start) > ctf_fixed_maxseg(tp)) { 6218 /* 6219 * We need to split this the last segment in two. 6220 */ 6221 struct rack_sendmap *nrsm; 6222 6223 nrsm = rack_alloc_full_limit(rack); 6224 if (nrsm == NULL) { 6225 /* 6226 * No memory to split, we will just exit and punt 6227 * off to the RXT timer. 6228 */ 6229 goto out; 6230 } 6231 rack_clone_rsm(rack, nrsm, rsm, 6232 (rsm->r_end - ctf_fixed_maxseg(tp))); 6233 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__); 6234 #ifndef INVARIANTS 6235 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 6236 #else 6237 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 6238 if (insret != NULL) { 6239 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 6240 nrsm, insret, rack, rsm); 6241 } 6242 #endif 6243 if (rsm->r_in_tmap) { 6244 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 6245 nrsm->r_in_tmap = 1; 6246 } 6247 rsm = nrsm; 6248 } 6249 rack->r_ctl.rc_tlpsend = rsm; 6250 send: 6251 /* Make sure output path knows we are doing a TLP */ 6252 *doing_tlp = 1; 6253 rack->r_timer_override = 1; 6254 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP; 6255 return (0); 6256 out: 6257 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP; 6258 return (0); 6259 } 6260 6261 /* 6262 * Delayed ack Timer, here we simply need to setup the 6263 * ACK_NOW flag and remove the DELACK flag. From there 6264 * the output routine will send the ack out. 6265 * 6266 * We only return 1, saying don't proceed, if all timers 6267 * are stopped (destroyed PCB?). 6268 */ 6269 static int 6270 rack_timeout_delack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 6271 { 6272 6273 rack_log_to_event(rack, RACK_TO_FRM_DELACK, NULL); 6274 tp->t_flags &= ~TF_DELACK; 6275 tp->t_flags |= TF_ACKNOW; 6276 KMOD_TCPSTAT_INC(tcps_delack); 6277 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK; 6278 return (0); 6279 } 6280 6281 /* 6282 * Persists timer, here we simply send the 6283 * same thing as a keepalive will. 6284 * the one byte send. 6285 * 6286 * We only return 1, saying don't proceed, if all timers 6287 * are stopped (destroyed PCB?). 6288 */ 6289 static int 6290 rack_timeout_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 6291 { 6292 struct tcptemp *t_template; 6293 int32_t retval = 1; 6294 6295 if (rack->rc_in_persist == 0) 6296 return (0); 6297 if (ctf_progress_timeout_check(tp, false)) { 6298 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX); 6299 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__); 6300 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends); 6301 return (-ETIMEDOUT); /* tcp_drop() */ 6302 } 6303 /* 6304 * Persistence timer into zero window. Force a byte to be output, if 6305 * possible. 6306 */ 6307 KMOD_TCPSTAT_INC(tcps_persisttimeo); 6308 /* 6309 * Hack: if the peer is dead/unreachable, we do not time out if the 6310 * window is closed. After a full backoff, drop the connection if 6311 * the idle time (no responses to probes) reaches the maximum 6312 * backoff that we would use if retransmitting. 6313 */ 6314 if (tp->t_rxtshift == TCP_MAXRXTSHIFT && 6315 (ticks - tp->t_rcvtime >= tcp_maxpersistidle || 6316 TICKS_2_USEC(ticks - tp->t_rcvtime) >= RACK_REXMTVAL(tp) * tcp_totbackoff)) { 6317 KMOD_TCPSTAT_INC(tcps_persistdrop); 6318 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX); 6319 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends); 6320 retval = -ETIMEDOUT; /* tcp_drop() */ 6321 goto out; 6322 } 6323 if ((sbavail(&rack->rc_inp->inp_socket->so_snd) == 0) && 6324 tp->snd_una == tp->snd_max) 6325 rack_exit_persist(tp, rack, cts); 6326 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT; 6327 /* 6328 * If the user has closed the socket then drop a persisting 6329 * connection after a much reduced timeout. 6330 */ 6331 if (tp->t_state > TCPS_CLOSE_WAIT && 6332 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) { 6333 KMOD_TCPSTAT_INC(tcps_persistdrop); 6334 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX); 6335 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends); 6336 retval = -ETIMEDOUT; /* tcp_drop() */ 6337 goto out; 6338 } 6339 t_template = tcpip_maketemplate(rack->rc_inp); 6340 if (t_template) { 6341 /* only set it if we were answered */ 6342 if (rack->forced_ack == 0) { 6343 rack->forced_ack = 1; 6344 rack->r_ctl.forced_ack_ts = tcp_get_usecs(NULL); 6345 } else { 6346 rack->probe_not_answered = 1; 6347 counter_u64_add(rack_persists_loss, 1); 6348 rack->r_ctl.persist_lost_ends++; 6349 } 6350 counter_u64_add(rack_persists_sends, 1); 6351 tcp_respond(tp, t_template->tt_ipgen, 6352 &t_template->tt_t, (struct mbuf *)NULL, 6353 tp->rcv_nxt, tp->snd_una - 1, 0); 6354 /* This sends an ack */ 6355 if (tp->t_flags & TF_DELACK) 6356 tp->t_flags &= ~TF_DELACK; 6357 free(t_template, M_TEMP); 6358 } 6359 if (tp->t_rxtshift < TCP_MAXRXTSHIFT) 6360 tp->t_rxtshift++; 6361 out: 6362 rack_log_to_event(rack, RACK_TO_FRM_PERSIST, NULL); 6363 rack_start_hpts_timer(rack, tp, cts, 6364 0, 0, 0); 6365 return (retval); 6366 } 6367 6368 /* 6369 * If a keepalive goes off, we had no other timers 6370 * happening. We always return 1 here since this 6371 * routine either drops the connection or sends 6372 * out a segment with respond. 6373 */ 6374 static int 6375 rack_timeout_keepalive(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 6376 { 6377 struct tcptemp *t_template; 6378 struct inpcb *inp = tptoinpcb(tp); 6379 6380 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP; 6381 rack_log_to_event(rack, RACK_TO_FRM_KEEP, NULL); 6382 /* 6383 * Keep-alive timer went off; send something or drop connection if 6384 * idle for too long. 6385 */ 6386 KMOD_TCPSTAT_INC(tcps_keeptimeo); 6387 if (tp->t_state < TCPS_ESTABLISHED) 6388 goto dropit; 6389 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) && 6390 tp->t_state <= TCPS_CLOSING) { 6391 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp)) 6392 goto dropit; 6393 /* 6394 * Send a packet designed to force a response if the peer is 6395 * up and reachable: either an ACK if the connection is 6396 * still alive, or an RST if the peer has closed the 6397 * connection due to timeout or reboot. Using sequence 6398 * number tp->snd_una-1 causes the transmitted zero-length 6399 * segment to lie outside the receive window; by the 6400 * protocol spec, this requires the correspondent TCP to 6401 * respond. 6402 */ 6403 KMOD_TCPSTAT_INC(tcps_keepprobe); 6404 t_template = tcpip_maketemplate(inp); 6405 if (t_template) { 6406 if (rack->forced_ack == 0) { 6407 rack->forced_ack = 1; 6408 rack->r_ctl.forced_ack_ts = tcp_get_usecs(NULL); 6409 } else { 6410 rack->probe_not_answered = 1; 6411 } 6412 tcp_respond(tp, t_template->tt_ipgen, 6413 &t_template->tt_t, (struct mbuf *)NULL, 6414 tp->rcv_nxt, tp->snd_una - 1, 0); 6415 free(t_template, M_TEMP); 6416 } 6417 } 6418 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); 6419 return (1); 6420 dropit: 6421 KMOD_TCPSTAT_INC(tcps_keepdrops); 6422 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX); 6423 return (-ETIMEDOUT); /* tcp_drop() */ 6424 } 6425 6426 /* 6427 * Retransmit helper function, clear up all the ack 6428 * flags and take care of important book keeping. 6429 */ 6430 static void 6431 rack_remxt_tmr(struct tcpcb *tp) 6432 { 6433 /* 6434 * The retransmit timer went off, all sack'd blocks must be 6435 * un-acked. 6436 */ 6437 struct rack_sendmap *rsm, *trsm = NULL; 6438 struct tcp_rack *rack; 6439 6440 rack = (struct tcp_rack *)tp->t_fb_ptr; 6441 rack_timer_cancel(tp, rack, tcp_get_usecs(NULL), __LINE__); 6442 rack_log_to_event(rack, RACK_TO_FRM_TMR, NULL); 6443 if (rack->r_state && (rack->r_state != tp->t_state)) 6444 rack_set_state(tp, rack); 6445 /* 6446 * Ideally we would like to be able to 6447 * mark SACK-PASS on anything not acked here. 6448 * 6449 * However, if we do that we would burst out 6450 * all that data 1ms apart. This would be unwise, 6451 * so for now we will just let the normal rxt timer 6452 * and tlp timer take care of it. 6453 * 6454 * Also we really need to stick them back in sequence 6455 * order. This way we send in the proper order and any 6456 * sacks that come floating in will "re-ack" the data. 6457 * To do this we zap the tmap with an INIT and then 6458 * walk through and place every rsm in the RB tree 6459 * back in its seq ordered place. 6460 */ 6461 TAILQ_INIT(&rack->r_ctl.rc_tmap); 6462 RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) { 6463 rsm->r_dupack = 0; 6464 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); 6465 /* We must re-add it back to the tlist */ 6466 if (trsm == NULL) { 6467 TAILQ_INSERT_HEAD(&rack->r_ctl.rc_tmap, rsm, r_tnext); 6468 } else { 6469 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext); 6470 } 6471 rsm->r_in_tmap = 1; 6472 trsm = rsm; 6473 if (rsm->r_flags & RACK_ACKED) 6474 rsm->r_flags |= RACK_WAS_ACKED; 6475 rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS | RACK_RWND_COLLAPSED); 6476 rsm->r_flags |= RACK_MUST_RXT; 6477 } 6478 /* Clear the count (we just un-acked them) */ 6479 rack->r_ctl.rc_last_timeout_snduna = tp->snd_una; 6480 rack->r_ctl.rc_sacked = 0; 6481 rack->r_ctl.rc_sacklast = NULL; 6482 rack->r_ctl.rc_agg_delayed = 0; 6483 rack->r_early = 0; 6484 rack->r_ctl.rc_agg_early = 0; 6485 rack->r_late = 0; 6486 /* Clear the tlp rtx mark */ 6487 rack->r_ctl.rc_resend = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 6488 if (rack->r_ctl.rc_resend != NULL) 6489 rack->r_ctl.rc_resend->r_flags |= RACK_TO_REXT; 6490 rack->r_ctl.rc_prr_sndcnt = 0; 6491 rack_log_to_prr(rack, 6, 0, __LINE__); 6492 rack->r_timer_override = 1; 6493 if ((((tp->t_flags & TF_SACK_PERMIT) == 0) 6494 #ifdef NETFLIX_EXP_DETECTION 6495 || (rack->sack_attack_disable != 0) 6496 #endif 6497 ) && ((tp->t_flags & TF_SENTFIN) == 0)) { 6498 /* 6499 * For non-sack customers new data 6500 * needs to go out as retransmits until 6501 * we retransmit up to snd_max. 6502 */ 6503 rack->r_must_retran = 1; 6504 rack->r_ctl.rc_out_at_rto = ctf_flight_size(rack->rc_tp, 6505 rack->r_ctl.rc_sacked); 6506 } 6507 rack->r_ctl.rc_snd_max_at_rto = tp->snd_max; 6508 } 6509 6510 static void 6511 rack_convert_rtts(struct tcpcb *tp) 6512 { 6513 if (tp->t_srtt > 1) { 6514 uint32_t val, frac; 6515 6516 val = tp->t_srtt >> TCP_RTT_SHIFT; 6517 frac = tp->t_srtt & 0x1f; 6518 tp->t_srtt = TICKS_2_USEC(val); 6519 /* 6520 * frac is the fractional part of the srtt (if any) 6521 * but its in ticks and every bit represents 6522 * 1/32nd of a hz. 6523 */ 6524 if (frac) { 6525 if (hz == 1000) { 6526 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE); 6527 } else { 6528 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE)); 6529 } 6530 tp->t_srtt += frac; 6531 } 6532 } 6533 if (tp->t_rttvar) { 6534 uint32_t val, frac; 6535 6536 val = tp->t_rttvar >> TCP_RTTVAR_SHIFT; 6537 frac = tp->t_rttvar & 0x1f; 6538 tp->t_rttvar = TICKS_2_USEC(val); 6539 /* 6540 * frac is the fractional part of the srtt (if any) 6541 * but its in ticks and every bit represents 6542 * 1/32nd of a hz. 6543 */ 6544 if (frac) { 6545 if (hz == 1000) { 6546 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE); 6547 } else { 6548 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE)); 6549 } 6550 tp->t_rttvar += frac; 6551 } 6552 } 6553 tp->t_rxtcur = RACK_REXMTVAL(tp); 6554 if (TCPS_HAVEESTABLISHED(tp->t_state)) { 6555 tp->t_rxtcur += TICKS_2_USEC(tcp_rexmit_slop); 6556 } 6557 if (tp->t_rxtcur > rack_rto_max) { 6558 tp->t_rxtcur = rack_rto_max; 6559 } 6560 } 6561 6562 static void 6563 rack_cc_conn_init(struct tcpcb *tp) 6564 { 6565 struct tcp_rack *rack; 6566 uint32_t srtt; 6567 6568 rack = (struct tcp_rack *)tp->t_fb_ptr; 6569 srtt = tp->t_srtt; 6570 cc_conn_init(tp); 6571 /* 6572 * Now convert to rack's internal format, 6573 * if required. 6574 */ 6575 if ((srtt == 0) && (tp->t_srtt != 0)) 6576 rack_convert_rtts(tp); 6577 /* 6578 * We want a chance to stay in slowstart as 6579 * we create a connection. TCP spec says that 6580 * initially ssthresh is infinite. For our 6581 * purposes that is the snd_wnd. 6582 */ 6583 if (tp->snd_ssthresh < tp->snd_wnd) { 6584 tp->snd_ssthresh = tp->snd_wnd; 6585 } 6586 /* 6587 * We also want to assure a IW worth of 6588 * data can get inflight. 6589 */ 6590 if (rc_init_window(rack) < tp->snd_cwnd) 6591 tp->snd_cwnd = rc_init_window(rack); 6592 } 6593 6594 /* 6595 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise 6596 * we will setup to retransmit the lowest seq number outstanding. 6597 */ 6598 static int 6599 rack_timeout_rxt(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts) 6600 { 6601 struct inpcb *inp = tptoinpcb(tp); 6602 int32_t rexmt; 6603 int32_t retval = 0; 6604 bool isipv6; 6605 6606 if ((tp->t_flags & TF_GPUTINPROG) && 6607 (tp->t_rxtshift)) { 6608 /* 6609 * We have had a second timeout 6610 * measurements on successive rxt's are not profitable. 6611 * It is unlikely to be of any use (the network is 6612 * broken or the client went away). 6613 */ 6614 tp->t_flags &= ~TF_GPUTINPROG; 6615 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, 6616 rack->r_ctl.rc_gp_srtt /*flex1*/, 6617 tp->gput_seq, 6618 0, 0, 18, __LINE__, NULL, 0); 6619 } 6620 if (ctf_progress_timeout_check(tp, false)) { 6621 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN); 6622 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__); 6623 return (-ETIMEDOUT); /* tcp_drop() */ 6624 } 6625 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT; 6626 rack->r_ctl.retran_during_recovery = 0; 6627 rack->rc_ack_required = 1; 6628 rack->r_ctl.dsack_byte_cnt = 0; 6629 if (IN_FASTRECOVERY(tp->t_flags)) 6630 tp->t_flags |= TF_WASFRECOVERY; 6631 else 6632 tp->t_flags &= ~TF_WASFRECOVERY; 6633 if (IN_CONGRECOVERY(tp->t_flags)) 6634 tp->t_flags |= TF_WASCRECOVERY; 6635 else 6636 tp->t_flags &= ~TF_WASCRECOVERY; 6637 if (TCPS_HAVEESTABLISHED(tp->t_state) && 6638 (tp->snd_una == tp->snd_max)) { 6639 /* Nothing outstanding .. nothing to do */ 6640 return (0); 6641 } 6642 if (rack->r_ctl.dsack_persist) { 6643 rack->r_ctl.dsack_persist--; 6644 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) { 6645 rack->r_ctl.num_dsack = 0; 6646 } 6647 rack_log_dsack_event(rack, 1, __LINE__, 0, 0); 6648 } 6649 /* 6650 * Rack can only run one timer at a time, so we cannot 6651 * run a KEEPINIT (gating SYN sending) and a retransmit 6652 * timer for the SYN. So if we are in a front state and 6653 * have a KEEPINIT timer we need to check the first transmit 6654 * against now to see if we have exceeded the KEEPINIT time 6655 * (if one is set). 6656 */ 6657 if ((TCPS_HAVEESTABLISHED(tp->t_state) == 0) && 6658 (TP_KEEPINIT(tp) != 0)) { 6659 struct rack_sendmap *rsm; 6660 6661 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 6662 if (rsm) { 6663 /* Ok we have something outstanding to test keepinit with */ 6664 if ((TSTMP_GT(cts, (uint32_t)rsm->r_tim_lastsent[0])) && 6665 ((cts - (uint32_t)rsm->r_tim_lastsent[0]) >= TICKS_2_USEC(TP_KEEPINIT(tp)))) { 6666 /* We have exceeded the KEEPINIT time */ 6667 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX); 6668 goto drop_it; 6669 } 6670 } 6671 } 6672 /* 6673 * Retransmission timer went off. Message has not been acked within 6674 * retransmit interval. Back off to a longer retransmit interval 6675 * and retransmit one segment. 6676 */ 6677 rack_remxt_tmr(tp); 6678 if ((rack->r_ctl.rc_resend == NULL) || 6679 ((rack->r_ctl.rc_resend->r_flags & RACK_RWND_COLLAPSED) == 0)) { 6680 /* 6681 * If the rwnd collapsed on 6682 * the one we are retransmitting 6683 * it does not count against the 6684 * rxt count. 6685 */ 6686 tp->t_rxtshift++; 6687 } 6688 if (tp->t_rxtshift > TCP_MAXRXTSHIFT) { 6689 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN); 6690 drop_it: 6691 tp->t_rxtshift = TCP_MAXRXTSHIFT; 6692 KMOD_TCPSTAT_INC(tcps_timeoutdrop); 6693 /* XXXGL: previously t_softerror was casted to uint16_t */ 6694 MPASS(tp->t_softerror >= 0); 6695 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT; 6696 goto out; /* tcp_drop() */ 6697 } 6698 if (tp->t_state == TCPS_SYN_SENT) { 6699 /* 6700 * If the SYN was retransmitted, indicate CWND to be limited 6701 * to 1 segment in cc_conn_init(). 6702 */ 6703 tp->snd_cwnd = 1; 6704 } else if (tp->t_rxtshift == 1) { 6705 /* 6706 * first retransmit; record ssthresh and cwnd so they can be 6707 * recovered if this turns out to be a "bad" retransmit. A 6708 * retransmit is considered "bad" if an ACK for this segment 6709 * is received within RTT/2 interval; the assumption here is 6710 * that the ACK was already in flight. See "On Estimating 6711 * End-to-End Network Path Properties" by Allman and Paxson 6712 * for more details. 6713 */ 6714 tp->snd_cwnd_prev = tp->snd_cwnd; 6715 tp->snd_ssthresh_prev = tp->snd_ssthresh; 6716 tp->snd_recover_prev = tp->snd_recover; 6717 tp->t_badrxtwin = ticks + (USEC_2_TICKS(tp->t_srtt)/2); 6718 tp->t_flags |= TF_PREVVALID; 6719 } else if ((tp->t_flags & TF_RCVD_TSTMP) == 0) 6720 tp->t_flags &= ~TF_PREVVALID; 6721 KMOD_TCPSTAT_INC(tcps_rexmttimeo); 6722 if ((tp->t_state == TCPS_SYN_SENT) || 6723 (tp->t_state == TCPS_SYN_RECEIVED)) 6724 rexmt = RACK_INITIAL_RTO * tcp_backoff[tp->t_rxtshift]; 6725 else 6726 rexmt = max(rack_rto_min, (tp->t_srtt + (tp->t_rttvar << 2))) * tcp_backoff[tp->t_rxtshift]; 6727 6728 RACK_TCPT_RANGESET(tp->t_rxtcur, rexmt, 6729 max(rack_rto_min, rexmt), rack_rto_max, rack->r_ctl.timer_slop); 6730 /* 6731 * We enter the path for PLMTUD if connection is established or, if 6732 * connection is FIN_WAIT_1 status, reason for the last is that if 6733 * amount of data we send is very small, we could send it in couple 6734 * of packets and process straight to FIN. In that case we won't 6735 * catch ESTABLISHED state. 6736 */ 6737 #ifdef INET6 6738 isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false; 6739 #else 6740 isipv6 = false; 6741 #endif 6742 if (((V_tcp_pmtud_blackhole_detect == 1) || 6743 (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) || 6744 (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) && 6745 ((tp->t_state == TCPS_ESTABLISHED) || 6746 (tp->t_state == TCPS_FIN_WAIT_1))) { 6747 /* 6748 * Idea here is that at each stage of mtu probe (usually, 6749 * 1448 -> 1188 -> 524) should be given 2 chances to recover 6750 * before further clamping down. 'tp->t_rxtshift % 2 == 0' 6751 * should take care of that. 6752 */ 6753 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) == 6754 (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) && 6755 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 && 6756 tp->t_rxtshift % 2 == 0)) { 6757 /* 6758 * Enter Path MTU Black-hole Detection mechanism: - 6759 * Disable Path MTU Discovery (IP "DF" bit). - 6760 * Reduce MTU to lower value than what we negotiated 6761 * with peer. 6762 */ 6763 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) { 6764 /* Record that we may have found a black hole. */ 6765 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE; 6766 /* Keep track of previous MSS. */ 6767 tp->t_pmtud_saved_maxseg = tp->t_maxseg; 6768 } 6769 6770 /* 6771 * Reduce the MSS to blackhole value or to the 6772 * default in an attempt to retransmit. 6773 */ 6774 #ifdef INET6 6775 if (isipv6 && 6776 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) { 6777 /* Use the sysctl tuneable blackhole MSS. */ 6778 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss; 6779 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated); 6780 } else if (isipv6) { 6781 /* Use the default MSS. */ 6782 tp->t_maxseg = V_tcp_v6mssdflt; 6783 /* 6784 * Disable Path MTU Discovery when we switch 6785 * to minmss. 6786 */ 6787 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 6788 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss); 6789 } 6790 #endif 6791 #if defined(INET6) && defined(INET) 6792 else 6793 #endif 6794 #ifdef INET 6795 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) { 6796 /* Use the sysctl tuneable blackhole MSS. */ 6797 tp->t_maxseg = V_tcp_pmtud_blackhole_mss; 6798 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated); 6799 } else { 6800 /* Use the default MSS. */ 6801 tp->t_maxseg = V_tcp_mssdflt; 6802 /* 6803 * Disable Path MTU Discovery when we switch 6804 * to minmss. 6805 */ 6806 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 6807 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss); 6808 } 6809 #endif 6810 } else { 6811 /* 6812 * If further retransmissions are still unsuccessful 6813 * with a lowered MTU, maybe this isn't a blackhole 6814 * and we restore the previous MSS and blackhole 6815 * detection flags. The limit '6' is determined by 6816 * giving each probe stage (1448, 1188, 524) 2 6817 * chances to recover. 6818 */ 6819 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) && 6820 (tp->t_rxtshift >= 6)) { 6821 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 6822 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE; 6823 tp->t_maxseg = tp->t_pmtud_saved_maxseg; 6824 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed); 6825 } 6826 } 6827 } 6828 /* 6829 * Disable RFC1323 and SACK if we haven't got any response to 6830 * our third SYN to work-around some broken terminal servers 6831 * (most of which have hopefully been retired) that have bad VJ 6832 * header compression code which trashes TCP segments containing 6833 * unknown-to-them TCP options. 6834 */ 6835 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) && 6836 (tp->t_rxtshift == 3)) 6837 tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_SACK_PERMIT); 6838 /* 6839 * If we backed off this far, our srtt estimate is probably bogus. 6840 * Clobber it so we'll take the next rtt measurement as our srtt; 6841 * move the current srtt into rttvar to keep the current retransmit 6842 * times until then. 6843 */ 6844 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) { 6845 #ifdef INET6 6846 if ((inp->inp_vflag & INP_IPV6) != 0) 6847 in6_losing(inp); 6848 else 6849 #endif 6850 in_losing(inp); 6851 tp->t_rttvar += tp->t_srtt; 6852 tp->t_srtt = 0; 6853 } 6854 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); 6855 tp->snd_recover = tp->snd_max; 6856 tp->t_flags |= TF_ACKNOW; 6857 tp->t_rtttime = 0; 6858 rack_cong_signal(tp, CC_RTO, tp->snd_una, __LINE__); 6859 out: 6860 return (retval); 6861 } 6862 6863 static int 6864 rack_process_timers(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t hpts_calling, uint8_t *doing_tlp) 6865 { 6866 int32_t ret = 0; 6867 int32_t timers = (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK); 6868 6869 if ((tp->t_state >= TCPS_FIN_WAIT_1) && 6870 (tp->t_flags & TF_GPUTINPROG)) { 6871 /* 6872 * We have a goodput in progress 6873 * and we have entered a late state. 6874 * Do we have enough data in the sb 6875 * to handle the GPUT request? 6876 */ 6877 uint32_t bytes; 6878 6879 bytes = tp->gput_ack - tp->gput_seq; 6880 if (SEQ_GT(tp->gput_seq, tp->snd_una)) 6881 bytes += tp->gput_seq - tp->snd_una; 6882 if (bytes > sbavail(&tptosocket(tp)->so_snd)) { 6883 /* 6884 * There are not enough bytes in the socket 6885 * buffer that have been sent to cover this 6886 * measurement. Cancel it. 6887 */ 6888 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, 6889 rack->r_ctl.rc_gp_srtt /*flex1*/, 6890 tp->gput_seq, 6891 0, 0, 18, __LINE__, NULL, 0); 6892 tp->t_flags &= ~TF_GPUTINPROG; 6893 } 6894 } 6895 if (timers == 0) { 6896 return (0); 6897 } 6898 if (tp->t_state == TCPS_LISTEN) { 6899 /* no timers on listen sockets */ 6900 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) 6901 return (0); 6902 return (1); 6903 } 6904 if ((timers & PACE_TMR_RACK) && 6905 rack->rc_on_min_to) { 6906 /* 6907 * For the rack timer when we 6908 * are on a min-timeout (which means rrr_conf = 3) 6909 * we don't want to check the timer. It may 6910 * be going off for a pace and thats ok we 6911 * want to send the retransmit (if its ready). 6912 * 6913 * If its on a normal rack timer (non-min) then 6914 * we will check if its expired. 6915 */ 6916 goto skip_time_check; 6917 } 6918 if (TSTMP_LT(cts, rack->r_ctl.rc_timer_exp)) { 6919 uint32_t left; 6920 6921 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { 6922 ret = -1; 6923 rack_log_to_processing(rack, cts, ret, 0); 6924 return (0); 6925 } 6926 if (hpts_calling == 0) { 6927 /* 6928 * A user send or queued mbuf (sack) has called us? We 6929 * return 0 and let the pacing guards 6930 * deal with it if they should or 6931 * should not cause a send. 6932 */ 6933 ret = -2; 6934 rack_log_to_processing(rack, cts, ret, 0); 6935 return (0); 6936 } 6937 /* 6938 * Ok our timer went off early and we are not paced false 6939 * alarm, go back to sleep. 6940 */ 6941 ret = -3; 6942 left = rack->r_ctl.rc_timer_exp - cts; 6943 tcp_hpts_insert(tptoinpcb(tp), HPTS_MS_TO_SLOTS(left)); 6944 rack_log_to_processing(rack, cts, ret, left); 6945 return (1); 6946 } 6947 skip_time_check: 6948 rack->rc_tmr_stopped = 0; 6949 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK; 6950 if (timers & PACE_TMR_DELACK) { 6951 ret = rack_timeout_delack(tp, rack, cts); 6952 } else if (timers & PACE_TMR_RACK) { 6953 rack->r_ctl.rc_tlp_rxt_last_time = cts; 6954 rack->r_fast_output = 0; 6955 ret = rack_timeout_rack(tp, rack, cts); 6956 } else if (timers & PACE_TMR_TLP) { 6957 rack->r_ctl.rc_tlp_rxt_last_time = cts; 6958 ret = rack_timeout_tlp(tp, rack, cts, doing_tlp); 6959 } else if (timers & PACE_TMR_RXT) { 6960 rack->r_ctl.rc_tlp_rxt_last_time = cts; 6961 rack->r_fast_output = 0; 6962 ret = rack_timeout_rxt(tp, rack, cts); 6963 } else if (timers & PACE_TMR_PERSIT) { 6964 ret = rack_timeout_persist(tp, rack, cts); 6965 } else if (timers & PACE_TMR_KEEP) { 6966 ret = rack_timeout_keepalive(tp, rack, cts); 6967 } 6968 rack_log_to_processing(rack, cts, ret, timers); 6969 return (ret); 6970 } 6971 6972 static void 6973 rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line) 6974 { 6975 struct timeval tv; 6976 uint32_t us_cts, flags_on_entry; 6977 uint8_t hpts_removed = 0; 6978 6979 flags_on_entry = rack->r_ctl.rc_hpts_flags; 6980 us_cts = tcp_get_usecs(&tv); 6981 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && 6982 ((TSTMP_GEQ(us_cts, rack->r_ctl.rc_last_output_to)) || 6983 ((tp->snd_max - tp->snd_una) == 0))) { 6984 tcp_hpts_remove(rack->rc_inp); 6985 hpts_removed = 1; 6986 /* If we were not delayed cancel out the flag. */ 6987 if ((tp->snd_max - tp->snd_una) == 0) 6988 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT; 6989 rack_log_to_cancel(rack, hpts_removed, line, us_cts, &tv, flags_on_entry); 6990 } 6991 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { 6992 rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK; 6993 if (tcp_in_hpts(rack->rc_inp) && 6994 ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) { 6995 /* 6996 * Canceling timer's when we have no output being 6997 * paced. We also must remove ourselves from the 6998 * hpts. 6999 */ 7000 tcp_hpts_remove(rack->rc_inp); 7001 hpts_removed = 1; 7002 } 7003 rack->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK); 7004 } 7005 if (hpts_removed == 0) 7006 rack_log_to_cancel(rack, hpts_removed, line, us_cts, &tv, flags_on_entry); 7007 } 7008 7009 static int 7010 rack_stopall(struct tcpcb *tp) 7011 { 7012 struct tcp_rack *rack; 7013 rack = (struct tcp_rack *)tp->t_fb_ptr; 7014 rack->t_timers_stopped = 1; 7015 return (0); 7016 } 7017 7018 static void 7019 rack_stop_all_timers(struct tcpcb *tp) 7020 { 7021 struct tcp_rack *rack; 7022 7023 /* 7024 * Assure no timers are running. 7025 */ 7026 if (tcp_timer_active(tp, TT_PERSIST)) { 7027 /* We enter in persists, set the flag appropriately */ 7028 rack = (struct tcp_rack *)tp->t_fb_ptr; 7029 rack->rc_in_persist = 1; 7030 } 7031 } 7032 7033 static void 7034 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack, 7035 struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag) 7036 { 7037 int32_t idx; 7038 7039 rsm->r_rtr_cnt++; 7040 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); 7041 rsm->r_dupack = 0; 7042 if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) { 7043 rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS; 7044 rsm->r_flags |= RACK_OVERMAX; 7045 } 7046 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) { 7047 rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start); 7048 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start); 7049 } 7050 idx = rsm->r_rtr_cnt - 1; 7051 rsm->r_tim_lastsent[idx] = ts; 7052 /* 7053 * Here we don't add in the len of send, since its already 7054 * in snduna <->snd_max. 7055 */ 7056 rsm->r_fas = ctf_flight_size(rack->rc_tp, 7057 rack->r_ctl.rc_sacked); 7058 if (rsm->r_flags & RACK_ACKED) { 7059 /* Problably MTU discovery messing with us */ 7060 rsm->r_flags &= ~RACK_ACKED; 7061 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); 7062 } 7063 if (rsm->r_in_tmap) { 7064 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext); 7065 rsm->r_in_tmap = 0; 7066 } 7067 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext); 7068 rsm->r_in_tmap = 1; 7069 /* Take off the must retransmit flag, if its on */ 7070 if (rsm->r_flags & RACK_MUST_RXT) { 7071 if (rack->r_must_retran) 7072 rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start); 7073 if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) { 7074 /* 7075 * We have retransmitted all we need. Clear 7076 * any must retransmit flags. 7077 */ 7078 rack->r_must_retran = 0; 7079 rack->r_ctl.rc_out_at_rto = 0; 7080 } 7081 rsm->r_flags &= ~RACK_MUST_RXT; 7082 } 7083 if (rsm->r_flags & RACK_SACK_PASSED) { 7084 /* We have retransmitted due to the SACK pass */ 7085 rsm->r_flags &= ~RACK_SACK_PASSED; 7086 rsm->r_flags |= RACK_WAS_SACKPASS; 7087 } 7088 } 7089 7090 static uint32_t 7091 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack, 7092 struct rack_sendmap *rsm, uint64_t ts, int32_t *lenp, uint16_t add_flag) 7093 { 7094 /* 7095 * We (re-)transmitted starting at rsm->r_start for some length 7096 * (possibly less than r_end. 7097 */ 7098 struct rack_sendmap *nrsm; 7099 #ifdef INVARIANTS 7100 struct rack_sendmap *insret; 7101 #endif 7102 uint32_t c_end; 7103 int32_t len; 7104 7105 len = *lenp; 7106 c_end = rsm->r_start + len; 7107 if (SEQ_GEQ(c_end, rsm->r_end)) { 7108 /* 7109 * We retransmitted the whole piece or more than the whole 7110 * slopping into the next rsm. 7111 */ 7112 rack_update_rsm(tp, rack, rsm, ts, add_flag); 7113 if (c_end == rsm->r_end) { 7114 *lenp = 0; 7115 return (0); 7116 } else { 7117 int32_t act_len; 7118 7119 /* Hangs over the end return whats left */ 7120 act_len = rsm->r_end - rsm->r_start; 7121 *lenp = (len - act_len); 7122 return (rsm->r_end); 7123 } 7124 /* We don't get out of this block. */ 7125 } 7126 /* 7127 * Here we retransmitted less than the whole thing which means we 7128 * have to split this into what was transmitted and what was not. 7129 */ 7130 nrsm = rack_alloc_full_limit(rack); 7131 if (nrsm == NULL) { 7132 /* 7133 * We can't get memory, so lets not proceed. 7134 */ 7135 *lenp = 0; 7136 return (0); 7137 } 7138 /* 7139 * So here we are going to take the original rsm and make it what we 7140 * retransmitted. nrsm will be the tail portion we did not 7141 * retransmit. For example say the chunk was 1, 11 (10 bytes). And 7142 * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to 7143 * 1, 6 and the new piece will be 6, 11. 7144 */ 7145 rack_clone_rsm(rack, nrsm, rsm, c_end); 7146 nrsm->r_dupack = 0; 7147 rack_log_retran_reason(rack, nrsm, __LINE__, 0, 2); 7148 #ifndef INVARIANTS 7149 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 7150 #else 7151 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 7152 if (insret != NULL) { 7153 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 7154 nrsm, insret, rack, rsm); 7155 } 7156 #endif 7157 if (rsm->r_in_tmap) { 7158 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 7159 nrsm->r_in_tmap = 1; 7160 } 7161 rsm->r_flags &= (~RACK_HAS_FIN); 7162 rack_update_rsm(tp, rack, rsm, ts, add_flag); 7163 /* Log a split of rsm into rsm and nrsm */ 7164 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__); 7165 *lenp = 0; 7166 return (0); 7167 } 7168 7169 static void 7170 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len, 7171 uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t cts, 7172 struct rack_sendmap *hintrsm, uint16_t add_flag, struct mbuf *s_mb, uint32_t s_moff, int hw_tls) 7173 { 7174 struct tcp_rack *rack; 7175 struct rack_sendmap *rsm, *nrsm, fe; 7176 #ifdef INVARIANTS 7177 struct rack_sendmap *insret; 7178 #endif 7179 register uint32_t snd_max, snd_una; 7180 7181 /* 7182 * Add to the RACK log of packets in flight or retransmitted. If 7183 * there is a TS option we will use the TS echoed, if not we will 7184 * grab a TS. 7185 * 7186 * Retransmissions will increment the count and move the ts to its 7187 * proper place. Note that if options do not include TS's then we 7188 * won't be able to effectively use the ACK for an RTT on a retran. 7189 * 7190 * Notes about r_start and r_end. Lets consider a send starting at 7191 * sequence 1 for 10 bytes. In such an example the r_start would be 7192 * 1 (starting sequence) but the r_end would be r_start+len i.e. 11. 7193 * This means that r_end is actually the first sequence for the next 7194 * slot (11). 7195 * 7196 */ 7197 /* 7198 * If err is set what do we do XXXrrs? should we not add the thing? 7199 * -- i.e. return if err != 0 or should we pretend we sent it? -- 7200 * i.e. proceed with add ** do this for now. 7201 */ 7202 INP_WLOCK_ASSERT(tptoinpcb(tp)); 7203 if (err) 7204 /* 7205 * We don't log errors -- we could but snd_max does not 7206 * advance in this case either. 7207 */ 7208 return; 7209 7210 if (th_flags & TH_RST) { 7211 /* 7212 * We don't log resets and we return immediately from 7213 * sending 7214 */ 7215 return; 7216 } 7217 rack = (struct tcp_rack *)tp->t_fb_ptr; 7218 snd_una = tp->snd_una; 7219 snd_max = tp->snd_max; 7220 if (th_flags & (TH_SYN | TH_FIN)) { 7221 /* 7222 * The call to rack_log_output is made before bumping 7223 * snd_max. This means we can record one extra byte on a SYN 7224 * or FIN if seq_out is adding more on and a FIN is present 7225 * (and we are not resending). 7226 */ 7227 if ((th_flags & TH_SYN) && (seq_out == tp->iss)) 7228 len++; 7229 if (th_flags & TH_FIN) 7230 len++; 7231 if (SEQ_LT(snd_max, tp->snd_nxt)) { 7232 /* 7233 * The add/update as not been done for the FIN/SYN 7234 * yet. 7235 */ 7236 snd_max = tp->snd_nxt; 7237 } 7238 } 7239 if (SEQ_LEQ((seq_out + len), snd_una)) { 7240 /* Are sending an old segment to induce an ack (keep-alive)? */ 7241 return; 7242 } 7243 if (SEQ_LT(seq_out, snd_una)) { 7244 /* huh? should we panic? */ 7245 uint32_t end; 7246 7247 end = seq_out + len; 7248 seq_out = snd_una; 7249 if (SEQ_GEQ(end, seq_out)) 7250 len = end - seq_out; 7251 else 7252 len = 0; 7253 } 7254 if (len == 0) { 7255 /* We don't log zero window probes */ 7256 return; 7257 } 7258 if (IN_FASTRECOVERY(tp->t_flags)) { 7259 rack->r_ctl.rc_prr_out += len; 7260 } 7261 /* First question is it a retransmission or new? */ 7262 if (seq_out == snd_max) { 7263 /* Its new */ 7264 again: 7265 rsm = rack_alloc(rack); 7266 if (rsm == NULL) { 7267 /* 7268 * Hmm out of memory and the tcb got destroyed while 7269 * we tried to wait. 7270 */ 7271 return; 7272 } 7273 if (th_flags & TH_FIN) { 7274 rsm->r_flags = RACK_HAS_FIN|add_flag; 7275 } else { 7276 rsm->r_flags = add_flag; 7277 } 7278 if (hw_tls) 7279 rsm->r_hw_tls = 1; 7280 rsm->r_tim_lastsent[0] = cts; 7281 rsm->r_rtr_cnt = 1; 7282 rsm->r_rtr_bytes = 0; 7283 if (th_flags & TH_SYN) { 7284 /* The data space is one beyond snd_una */ 7285 rsm->r_flags |= RACK_HAS_SYN; 7286 } 7287 rsm->r_start = seq_out; 7288 rsm->r_end = rsm->r_start + len; 7289 rsm->r_dupack = 0; 7290 /* 7291 * save off the mbuf location that 7292 * sndmbuf_noadv returned (which is 7293 * where we started copying from).. 7294 */ 7295 rsm->m = s_mb; 7296 rsm->soff = s_moff; 7297 /* 7298 * Here we do add in the len of send, since its not yet 7299 * reflected in in snduna <->snd_max 7300 */ 7301 rsm->r_fas = (ctf_flight_size(rack->rc_tp, 7302 rack->r_ctl.rc_sacked) + 7303 (rsm->r_end - rsm->r_start)); 7304 /* rsm->m will be NULL if RACK_HAS_SYN or RACK_HAS_FIN is set */ 7305 if (rsm->m) { 7306 if (rsm->m->m_len <= rsm->soff) { 7307 /* 7308 * XXXrrs Question, will this happen? 7309 * 7310 * If sbsndptr is set at the correct place 7311 * then s_moff should always be somewhere 7312 * within rsm->m. But if the sbsndptr was 7313 * off then that won't be true. If it occurs 7314 * we need to walkout to the correct location. 7315 */ 7316 struct mbuf *lm; 7317 7318 lm = rsm->m; 7319 while (lm->m_len <= rsm->soff) { 7320 rsm->soff -= lm->m_len; 7321 lm = lm->m_next; 7322 KASSERT(lm != NULL, ("%s rack:%p lm goes null orig_off:%u origmb:%p rsm->soff:%u", 7323 __func__, rack, s_moff, s_mb, rsm->soff)); 7324 } 7325 rsm->m = lm; 7326 } 7327 rsm->orig_m_len = rsm->m->m_len; 7328 } else 7329 rsm->orig_m_len = 0; 7330 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); 7331 /* Log a new rsm */ 7332 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_NEW, 0, __LINE__); 7333 #ifndef INVARIANTS 7334 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 7335 #else 7336 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 7337 if (insret != NULL) { 7338 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 7339 nrsm, insret, rack, rsm); 7340 } 7341 #endif 7342 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext); 7343 rsm->r_in_tmap = 1; 7344 /* 7345 * Special case detection, is there just a single 7346 * packet outstanding when we are not in recovery? 7347 * 7348 * If this is true mark it so. 7349 */ 7350 if ((IN_FASTRECOVERY(tp->t_flags) == 0) && 7351 (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) == ctf_fixed_maxseg(tp))) { 7352 struct rack_sendmap *prsm; 7353 7354 prsm = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 7355 if (prsm) 7356 prsm->r_one_out_nr = 1; 7357 } 7358 return; 7359 } 7360 /* 7361 * If we reach here its a retransmission and we need to find it. 7362 */ 7363 memset(&fe, 0, sizeof(fe)); 7364 more: 7365 if (hintrsm && (hintrsm->r_start == seq_out)) { 7366 rsm = hintrsm; 7367 hintrsm = NULL; 7368 } else { 7369 /* No hints sorry */ 7370 rsm = NULL; 7371 } 7372 if ((rsm) && (rsm->r_start == seq_out)) { 7373 seq_out = rack_update_entry(tp, rack, rsm, cts, &len, add_flag); 7374 if (len == 0) { 7375 return; 7376 } else { 7377 goto more; 7378 } 7379 } 7380 /* Ok it was not the last pointer go through it the hard way. */ 7381 refind: 7382 fe.r_start = seq_out; 7383 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 7384 if (rsm) { 7385 if (rsm->r_start == seq_out) { 7386 seq_out = rack_update_entry(tp, rack, rsm, cts, &len, add_flag); 7387 if (len == 0) { 7388 return; 7389 } else { 7390 goto refind; 7391 } 7392 } 7393 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) { 7394 /* Transmitted within this piece */ 7395 /* 7396 * Ok we must split off the front and then let the 7397 * update do the rest 7398 */ 7399 nrsm = rack_alloc_full_limit(rack); 7400 if (nrsm == NULL) { 7401 rack_update_rsm(tp, rack, rsm, cts, add_flag); 7402 return; 7403 } 7404 /* 7405 * copy rsm to nrsm and then trim the front of rsm 7406 * to not include this part. 7407 */ 7408 rack_clone_rsm(rack, nrsm, rsm, seq_out); 7409 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__); 7410 #ifndef INVARIANTS 7411 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 7412 #else 7413 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 7414 if (insret != NULL) { 7415 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 7416 nrsm, insret, rack, rsm); 7417 } 7418 #endif 7419 if (rsm->r_in_tmap) { 7420 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 7421 nrsm->r_in_tmap = 1; 7422 } 7423 rsm->r_flags &= (~RACK_HAS_FIN); 7424 seq_out = rack_update_entry(tp, rack, nrsm, cts, &len, add_flag); 7425 if (len == 0) { 7426 return; 7427 } else if (len > 0) 7428 goto refind; 7429 } 7430 } 7431 /* 7432 * Hmm not found in map did they retransmit both old and on into the 7433 * new? 7434 */ 7435 if (seq_out == tp->snd_max) { 7436 goto again; 7437 } else if (SEQ_LT(seq_out, tp->snd_max)) { 7438 #ifdef INVARIANTS 7439 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n", 7440 seq_out, len, tp->snd_una, tp->snd_max); 7441 printf("Starting Dump of all rack entries\n"); 7442 RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) { 7443 printf("rsm:%p start:%u end:%u\n", 7444 rsm, rsm->r_start, rsm->r_end); 7445 } 7446 printf("Dump complete\n"); 7447 panic("seq_out not found rack:%p tp:%p", 7448 rack, tp); 7449 #endif 7450 } else { 7451 #ifdef INVARIANTS 7452 /* 7453 * Hmm beyond sndmax? (only if we are using the new rtt-pack 7454 * flag) 7455 */ 7456 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p", 7457 seq_out, len, tp->snd_max, tp); 7458 #endif 7459 } 7460 } 7461 7462 /* 7463 * Record one of the RTT updates from an ack into 7464 * our sample structure. 7465 */ 7466 7467 static void 7468 tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt, uint32_t len, uint32_t us_rtt, 7469 int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt) 7470 { 7471 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) || 7472 (rack->r_ctl.rack_rs.rs_rtt_lowest > rtt)) { 7473 rack->r_ctl.rack_rs.rs_rtt_lowest = rtt; 7474 } 7475 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) || 7476 (rack->r_ctl.rack_rs.rs_rtt_highest < rtt)) { 7477 rack->r_ctl.rack_rs.rs_rtt_highest = rtt; 7478 } 7479 if (rack->rc_tp->t_flags & TF_GPUTINPROG) { 7480 if (us_rtt < rack->r_ctl.rc_gp_lowrtt) 7481 rack->r_ctl.rc_gp_lowrtt = us_rtt; 7482 if (rack->rc_tp->snd_wnd > rack->r_ctl.rc_gp_high_rwnd) 7483 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd; 7484 } 7485 if ((confidence == 1) && 7486 ((rsm == NULL) || 7487 (rsm->r_just_ret) || 7488 (rsm->r_one_out_nr && 7489 len < (ctf_fixed_maxseg(rack->rc_tp) * 2)))) { 7490 /* 7491 * If the rsm had a just return 7492 * hit it then we can't trust the 7493 * rtt measurement for buffer deterimination 7494 * Note that a confidence of 2, indicates 7495 * SACK'd which overrides the r_just_ret or 7496 * the r_one_out_nr. If it was a CUM-ACK and 7497 * we had only two outstanding, but get an 7498 * ack for only 1. Then that also lowers our 7499 * confidence. 7500 */ 7501 confidence = 0; 7502 } 7503 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) || 7504 (rack->r_ctl.rack_rs.rs_us_rtt > us_rtt)) { 7505 if (rack->r_ctl.rack_rs.confidence == 0) { 7506 /* 7507 * We take anything with no current confidence 7508 * saved. 7509 */ 7510 rack->r_ctl.rack_rs.rs_us_rtt = us_rtt; 7511 rack->r_ctl.rack_rs.confidence = confidence; 7512 rack->r_ctl.rack_rs.rs_us_rtrcnt = rtrcnt; 7513 } else if (confidence || rack->r_ctl.rack_rs.confidence) { 7514 /* 7515 * Once we have a confident number, 7516 * we can update it with a smaller 7517 * value since this confident number 7518 * may include the DSACK time until 7519 * the next segment (the second one) arrived. 7520 */ 7521 rack->r_ctl.rack_rs.rs_us_rtt = us_rtt; 7522 rack->r_ctl.rack_rs.confidence = confidence; 7523 rack->r_ctl.rack_rs.rs_us_rtrcnt = rtrcnt; 7524 } 7525 } 7526 rack_log_rtt_upd(rack->rc_tp, rack, us_rtt, len, rsm, confidence); 7527 rack->r_ctl.rack_rs.rs_flags = RACK_RTT_VALID; 7528 rack->r_ctl.rack_rs.rs_rtt_tot += rtt; 7529 rack->r_ctl.rack_rs.rs_rtt_cnt++; 7530 } 7531 7532 /* 7533 * Collect new round-trip time estimate 7534 * and update averages and current timeout. 7535 */ 7536 static void 7537 tcp_rack_xmit_timer_commit(struct tcp_rack *rack, struct tcpcb *tp) 7538 { 7539 int32_t delta; 7540 int32_t rtt; 7541 7542 if (rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) 7543 /* No valid sample */ 7544 return; 7545 if (rack->r_ctl.rc_rate_sample_method == USE_RTT_LOW) { 7546 /* We are to use the lowest RTT seen in a single ack */ 7547 rtt = rack->r_ctl.rack_rs.rs_rtt_lowest; 7548 } else if (rack->r_ctl.rc_rate_sample_method == USE_RTT_HIGH) { 7549 /* We are to use the highest RTT seen in a single ack */ 7550 rtt = rack->r_ctl.rack_rs.rs_rtt_highest; 7551 } else if (rack->r_ctl.rc_rate_sample_method == USE_RTT_AVG) { 7552 /* We are to use the average RTT seen in a single ack */ 7553 rtt = (int32_t)(rack->r_ctl.rack_rs.rs_rtt_tot / 7554 (uint64_t)rack->r_ctl.rack_rs.rs_rtt_cnt); 7555 } else { 7556 #ifdef INVARIANTS 7557 panic("Unknown rtt variant %d", rack->r_ctl.rc_rate_sample_method); 7558 #endif 7559 return; 7560 } 7561 if (rtt == 0) 7562 rtt = 1; 7563 if (rack->rc_gp_rtt_set == 0) { 7564 /* 7565 * With no RTT we have to accept 7566 * even one we are not confident of. 7567 */ 7568 rack->r_ctl.rc_gp_srtt = rack->r_ctl.rack_rs.rs_us_rtt; 7569 rack->rc_gp_rtt_set = 1; 7570 } else if (rack->r_ctl.rack_rs.confidence) { 7571 /* update the running gp srtt */ 7572 rack->r_ctl.rc_gp_srtt -= (rack->r_ctl.rc_gp_srtt/8); 7573 rack->r_ctl.rc_gp_srtt += rack->r_ctl.rack_rs.rs_us_rtt / 8; 7574 } 7575 if (rack->r_ctl.rack_rs.confidence) { 7576 /* 7577 * record the low and high for highly buffered path computation, 7578 * we only do this if we are confident (not a retransmission). 7579 */ 7580 if (rack->r_ctl.rc_highest_us_rtt < rack->r_ctl.rack_rs.rs_us_rtt) { 7581 rack->r_ctl.rc_highest_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt; 7582 } 7583 if (rack->rc_highly_buffered == 0) { 7584 /* 7585 * Currently once we declare a path has 7586 * highly buffered there is no going 7587 * back, which may be a problem... 7588 */ 7589 if ((rack->r_ctl.rc_highest_us_rtt / rack->r_ctl.rc_lowest_us_rtt) > rack_hbp_thresh) { 7590 rack_log_rtt_shrinks(rack, rack->r_ctl.rack_rs.rs_us_rtt, 7591 rack->r_ctl.rc_highest_us_rtt, 7592 rack->r_ctl.rc_lowest_us_rtt, 7593 RACK_RTTS_SEEHBP); 7594 rack->rc_highly_buffered = 1; 7595 } 7596 } 7597 } 7598 if ((rack->r_ctl.rack_rs.confidence) || 7599 (rack->r_ctl.rack_rs.rs_us_rtrcnt == 1)) { 7600 /* 7601 * If we are highly confident of it <or> it was 7602 * never retransmitted we accept it as the last us_rtt. 7603 */ 7604 rack->r_ctl.rc_last_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt; 7605 /* The lowest rtt can be set if its was not retransmited */ 7606 if (rack->r_ctl.rc_lowest_us_rtt > rack->r_ctl.rack_rs.rs_us_rtt) { 7607 rack->r_ctl.rc_lowest_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt; 7608 if (rack->r_ctl.rc_lowest_us_rtt == 0) 7609 rack->r_ctl.rc_lowest_us_rtt = 1; 7610 } 7611 } 7612 rack = (struct tcp_rack *)tp->t_fb_ptr; 7613 if (tp->t_srtt != 0) { 7614 /* 7615 * We keep a simple srtt in microseconds, like our rtt 7616 * measurement. We don't need to do any tricks with shifting 7617 * etc. Instead we just add in 1/8th of the new measurement 7618 * and subtract out 1/8 of the old srtt. We do the same with 7619 * the variance after finding the absolute value of the 7620 * difference between this sample and the current srtt. 7621 */ 7622 delta = tp->t_srtt - rtt; 7623 /* Take off 1/8th of the current sRTT */ 7624 tp->t_srtt -= (tp->t_srtt >> 3); 7625 /* Add in 1/8th of the new RTT just measured */ 7626 tp->t_srtt += (rtt >> 3); 7627 if (tp->t_srtt <= 0) 7628 tp->t_srtt = 1; 7629 /* Now lets make the absolute value of the variance */ 7630 if (delta < 0) 7631 delta = -delta; 7632 /* Subtract out 1/8th */ 7633 tp->t_rttvar -= (tp->t_rttvar >> 3); 7634 /* Add in 1/8th of the new variance we just saw */ 7635 tp->t_rttvar += (delta >> 3); 7636 if (tp->t_rttvar <= 0) 7637 tp->t_rttvar = 1; 7638 } else { 7639 /* 7640 * No rtt measurement yet - use the unsmoothed rtt. Set the 7641 * variance to half the rtt (so our first retransmit happens 7642 * at 3*rtt). 7643 */ 7644 tp->t_srtt = rtt; 7645 tp->t_rttvar = rtt >> 1; 7646 } 7647 rack->rc_srtt_measure_made = 1; 7648 KMOD_TCPSTAT_INC(tcps_rttupdated); 7649 if (tp->t_rttupdated < UCHAR_MAX) 7650 tp->t_rttupdated++; 7651 #ifdef STATS 7652 if (rack_stats_gets_ms_rtt == 0) { 7653 /* Send in the microsecond rtt used for rxt timeout purposes */ 7654 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt)); 7655 } else if (rack_stats_gets_ms_rtt == 1) { 7656 /* Send in the millisecond rtt used for rxt timeout purposes */ 7657 int32_t ms_rtt; 7658 7659 /* Round up */ 7660 ms_rtt = (rtt + HPTS_USEC_IN_MSEC - 1) / HPTS_USEC_IN_MSEC; 7661 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, ms_rtt)); 7662 } else if (rack_stats_gets_ms_rtt == 2) { 7663 /* Send in the millisecond rtt has close to the path RTT as we can get */ 7664 int32_t ms_rtt; 7665 7666 /* Round up */ 7667 ms_rtt = (rack->r_ctl.rack_rs.rs_us_rtt + HPTS_USEC_IN_MSEC - 1) / HPTS_USEC_IN_MSEC; 7668 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, ms_rtt)); 7669 } else { 7670 /* Send in the microsecond rtt has close to the path RTT as we can get */ 7671 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rack->r_ctl.rack_rs.rs_us_rtt)); 7672 } 7673 7674 #endif 7675 /* 7676 * the retransmit should happen at rtt + 4 * rttvar. Because of the 7677 * way we do the smoothing, srtt and rttvar will each average +1/2 7678 * tick of bias. When we compute the retransmit timer, we want 1/2 7679 * tick of rounding and 1 extra tick because of +-1/2 tick 7680 * uncertainty in the firing of the timer. The bias will give us 7681 * exactly the 1.5 tick we need. But, because the bias is 7682 * statistical, we have to test that we don't drop below the minimum 7683 * feasible timer (which is 2 ticks). 7684 */ 7685 tp->t_rxtshift = 0; 7686 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 7687 max(rack_rto_min, rtt + 2), rack_rto_max, rack->r_ctl.timer_slop); 7688 rack_log_rtt_sample(rack, rtt); 7689 tp->t_softerror = 0; 7690 } 7691 7692 7693 static void 7694 rack_apply_updated_usrtt(struct tcp_rack *rack, uint32_t us_rtt, uint32_t us_cts) 7695 { 7696 /* 7697 * Apply to filter the inbound us-rtt at us_cts. 7698 */ 7699 uint32_t old_rtt; 7700 7701 old_rtt = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt); 7702 apply_filter_min_small(&rack->r_ctl.rc_gp_min_rtt, 7703 us_rtt, us_cts); 7704 if (old_rtt > us_rtt) { 7705 /* We just hit a new lower rtt time */ 7706 rack_log_rtt_shrinks(rack, us_cts, old_rtt, 7707 __LINE__, RACK_RTTS_NEWRTT); 7708 /* 7709 * Only count it if its lower than what we saw within our 7710 * calculated range. 7711 */ 7712 if ((old_rtt - us_rtt) > rack_min_rtt_movement) { 7713 if (rack_probertt_lower_within && 7714 rack->rc_gp_dyn_mul && 7715 (rack->use_fixed_rate == 0) && 7716 (rack->rc_always_pace)) { 7717 /* 7718 * We are seeing a new lower rtt very close 7719 * to the time that we would have entered probe-rtt. 7720 * This is probably due to the fact that a peer flow 7721 * has entered probe-rtt. Lets go in now too. 7722 */ 7723 uint32_t val; 7724 7725 val = rack_probertt_lower_within * rack_time_between_probertt; 7726 val /= 100; 7727 if ((rack->in_probe_rtt == 0) && 7728 ((us_cts - rack->r_ctl.rc_lower_rtt_us_cts) >= (rack_time_between_probertt - val))) { 7729 rack_enter_probertt(rack, us_cts); 7730 } 7731 } 7732 rack->r_ctl.rc_lower_rtt_us_cts = us_cts; 7733 } 7734 } 7735 } 7736 7737 static int 7738 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack, 7739 struct rack_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, tcp_seq th_ack) 7740 { 7741 uint32_t us_rtt; 7742 int32_t i, all; 7743 uint32_t t, len_acked; 7744 7745 if ((rsm->r_flags & RACK_ACKED) || 7746 (rsm->r_flags & RACK_WAS_ACKED)) 7747 /* Already done */ 7748 return (0); 7749 if (rsm->r_no_rtt_allowed) { 7750 /* Not allowed */ 7751 return (0); 7752 } 7753 if (ack_type == CUM_ACKED) { 7754 if (SEQ_GT(th_ack, rsm->r_end)) { 7755 len_acked = rsm->r_end - rsm->r_start; 7756 all = 1; 7757 } else { 7758 len_acked = th_ack - rsm->r_start; 7759 all = 0; 7760 } 7761 } else { 7762 len_acked = rsm->r_end - rsm->r_start; 7763 all = 0; 7764 } 7765 if (rsm->r_rtr_cnt == 1) { 7766 7767 t = cts - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]; 7768 if ((int)t <= 0) 7769 t = 1; 7770 if (!tp->t_rttlow || tp->t_rttlow > t) 7771 tp->t_rttlow = t; 7772 if (!rack->r_ctl.rc_rack_min_rtt || 7773 SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) { 7774 rack->r_ctl.rc_rack_min_rtt = t; 7775 if (rack->r_ctl.rc_rack_min_rtt == 0) { 7776 rack->r_ctl.rc_rack_min_rtt = 1; 7777 } 7778 } 7779 if (TSTMP_GT(tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])) 7780 us_rtt = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 7781 else 7782 us_rtt = tcp_get_usecs(NULL) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 7783 if (us_rtt == 0) 7784 us_rtt = 1; 7785 if (CC_ALGO(tp)->rttsample != NULL) { 7786 /* Kick the RTT to the CC */ 7787 CC_ALGO(tp)->rttsample(&tp->t_ccv, us_rtt, 1, rsm->r_fas); 7788 } 7789 rack_apply_updated_usrtt(rack, us_rtt, tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time)); 7790 if (ack_type == SACKED) { 7791 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 1); 7792 tcp_rack_xmit_timer(rack, t + 1, len_acked, us_rtt, 2 , rsm, rsm->r_rtr_cnt); 7793 } else { 7794 /* 7795 * We need to setup what our confidence 7796 * is in this ack. 7797 * 7798 * If the rsm was app limited and it is 7799 * less than a mss in length (the end 7800 * of the send) then we have a gap. If we 7801 * were app limited but say we were sending 7802 * multiple MSS's then we are more confident 7803 * int it. 7804 * 7805 * When we are not app-limited then we see if 7806 * the rsm is being included in the current 7807 * measurement, we tell this by the app_limited_needs_set 7808 * flag. 7809 * 7810 * Note that being cwnd blocked is not applimited 7811 * as well as the pacing delay between packets which 7812 * are sending only 1 or 2 MSS's also will show up 7813 * in the RTT. We probably need to examine this algorithm 7814 * a bit more and enhance it to account for the delay 7815 * between rsm's. We could do that by saving off the 7816 * pacing delay of each rsm (in an rsm) and then 7817 * factoring that in somehow though for now I am 7818 * not sure how :) 7819 */ 7820 int calc_conf = 0; 7821 7822 if (rsm->r_flags & RACK_APP_LIMITED) { 7823 if (all && (len_acked <= ctf_fixed_maxseg(tp))) 7824 calc_conf = 0; 7825 else 7826 calc_conf = 1; 7827 } else if (rack->app_limited_needs_set == 0) { 7828 calc_conf = 1; 7829 } else { 7830 calc_conf = 0; 7831 } 7832 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 2); 7833 tcp_rack_xmit_timer(rack, t + 1, len_acked, us_rtt, 7834 calc_conf, rsm, rsm->r_rtr_cnt); 7835 } 7836 if ((rsm->r_flags & RACK_TLP) && 7837 (!IN_FASTRECOVERY(tp->t_flags))) { 7838 /* Segment was a TLP and our retrans matched */ 7839 if (rack->r_ctl.rc_tlp_cwnd_reduce) { 7840 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__); 7841 } 7842 } 7843 if (SEQ_LT(rack->r_ctl.rc_rack_tmit_time, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)])) { 7844 /* New more recent rack_tmit_time */ 7845 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]; 7846 rack->rc_rack_rtt = t; 7847 } 7848 return (1); 7849 } 7850 /* 7851 * We clear the soft/rxtshift since we got an ack. 7852 * There is no assurance we will call the commit() function 7853 * so we need to clear these to avoid incorrect handling. 7854 */ 7855 tp->t_rxtshift = 0; 7856 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 7857 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 7858 tp->t_softerror = 0; 7859 if (to && (to->to_flags & TOF_TS) && 7860 (ack_type == CUM_ACKED) && 7861 (to->to_tsecr) && 7862 ((rsm->r_flags & RACK_OVERMAX) == 0)) { 7863 /* 7864 * Now which timestamp does it match? In this block the ACK 7865 * must be coming from a previous transmission. 7866 */ 7867 for (i = 0; i < rsm->r_rtr_cnt; i++) { 7868 if (rack_ts_to_msec(rsm->r_tim_lastsent[i]) == to->to_tsecr) { 7869 t = cts - (uint32_t)rsm->r_tim_lastsent[i]; 7870 if ((int)t <= 0) 7871 t = 1; 7872 if (CC_ALGO(tp)->rttsample != NULL) { 7873 /* 7874 * Kick the RTT to the CC, here 7875 * we lie a bit in that we know the 7876 * retransmission is correct even though 7877 * we retransmitted. This is because 7878 * we match the timestamps. 7879 */ 7880 if (TSTMP_GT(tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[i])) 7881 us_rtt = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[i]; 7882 else 7883 us_rtt = tcp_get_usecs(NULL) - (uint32_t)rsm->r_tim_lastsent[i]; 7884 CC_ALGO(tp)->rttsample(&tp->t_ccv, us_rtt, 1, rsm->r_fas); 7885 } 7886 if ((i + 1) < rsm->r_rtr_cnt) { 7887 /* 7888 * The peer ack'd from our previous 7889 * transmission. We have a spurious 7890 * retransmission and thus we dont 7891 * want to update our rack_rtt. 7892 * 7893 * Hmm should there be a CC revert here? 7894 * 7895 */ 7896 return (0); 7897 } 7898 if (!tp->t_rttlow || tp->t_rttlow > t) 7899 tp->t_rttlow = t; 7900 if (!rack->r_ctl.rc_rack_min_rtt || SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) { 7901 rack->r_ctl.rc_rack_min_rtt = t; 7902 if (rack->r_ctl.rc_rack_min_rtt == 0) { 7903 rack->r_ctl.rc_rack_min_rtt = 1; 7904 } 7905 } 7906 if (SEQ_LT(rack->r_ctl.rc_rack_tmit_time, 7907 (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)])) { 7908 /* New more recent rack_tmit_time */ 7909 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]; 7910 rack->rc_rack_rtt = t; 7911 } 7912 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[i], cts, 3); 7913 tcp_rack_xmit_timer(rack, t + 1, len_acked, t, 0, rsm, 7914 rsm->r_rtr_cnt); 7915 return (1); 7916 } 7917 } 7918 goto ts_not_found; 7919 } else { 7920 /* 7921 * Ok its a SACK block that we retransmitted. or a windows 7922 * machine without timestamps. We can tell nothing from the 7923 * time-stamp since its not there or the time the peer last 7924 * recieved a segment that moved forward its cum-ack point. 7925 */ 7926 ts_not_found: 7927 i = rsm->r_rtr_cnt - 1; 7928 t = cts - (uint32_t)rsm->r_tim_lastsent[i]; 7929 if ((int)t <= 0) 7930 t = 1; 7931 if (rack->r_ctl.rc_rack_min_rtt && SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) { 7932 /* 7933 * We retransmitted and the ack came back in less 7934 * than the smallest rtt we have observed. We most 7935 * likely did an improper retransmit as outlined in 7936 * 6.2 Step 2 point 2 in the rack-draft so we 7937 * don't want to update our rack_rtt. We in 7938 * theory (in future) might want to think about reverting our 7939 * cwnd state but we won't for now. 7940 */ 7941 return (0); 7942 } else if (rack->r_ctl.rc_rack_min_rtt) { 7943 /* 7944 * We retransmitted it and the retransmit did the 7945 * job. 7946 */ 7947 if (!rack->r_ctl.rc_rack_min_rtt || 7948 SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) { 7949 rack->r_ctl.rc_rack_min_rtt = t; 7950 if (rack->r_ctl.rc_rack_min_rtt == 0) { 7951 rack->r_ctl.rc_rack_min_rtt = 1; 7952 } 7953 } 7954 if (SEQ_LT(rack->r_ctl.rc_rack_tmit_time, (uint32_t)rsm->r_tim_lastsent[i])) { 7955 /* New more recent rack_tmit_time */ 7956 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[i]; 7957 rack->rc_rack_rtt = t; 7958 } 7959 return (1); 7960 } 7961 } 7962 return (0); 7963 } 7964 7965 /* 7966 * Mark the SACK_PASSED flag on all entries prior to rsm send wise. 7967 */ 7968 static void 7969 rack_log_sack_passed(struct tcpcb *tp, 7970 struct tcp_rack *rack, struct rack_sendmap *rsm) 7971 { 7972 struct rack_sendmap *nrsm; 7973 7974 nrsm = rsm; 7975 TAILQ_FOREACH_REVERSE_FROM(nrsm, &rack->r_ctl.rc_tmap, 7976 rack_head, r_tnext) { 7977 if (nrsm == rsm) { 7978 /* Skip original segment he is acked */ 7979 continue; 7980 } 7981 if (nrsm->r_flags & RACK_ACKED) { 7982 /* 7983 * Skip ack'd segments, though we 7984 * should not see these, since tmap 7985 * should not have ack'd segments. 7986 */ 7987 continue; 7988 } 7989 if (nrsm->r_flags & RACK_RWND_COLLAPSED) { 7990 /* 7991 * If the peer dropped the rwnd on 7992 * these then we don't worry about them. 7993 */ 7994 continue; 7995 } 7996 if (nrsm->r_flags & RACK_SACK_PASSED) { 7997 /* 7998 * We found one that is already marked 7999 * passed, we have been here before and 8000 * so all others below this are marked. 8001 */ 8002 break; 8003 } 8004 nrsm->r_flags |= RACK_SACK_PASSED; 8005 nrsm->r_flags &= ~RACK_WAS_SACKPASS; 8006 } 8007 } 8008 8009 static void 8010 rack_need_set_test(struct tcpcb *tp, 8011 struct tcp_rack *rack, 8012 struct rack_sendmap *rsm, 8013 tcp_seq th_ack, 8014 int line, 8015 int use_which) 8016 { 8017 8018 if ((tp->t_flags & TF_GPUTINPROG) && 8019 SEQ_GEQ(rsm->r_end, tp->gput_seq)) { 8020 /* 8021 * We were app limited, and this ack 8022 * butts up or goes beyond the point where we want 8023 * to start our next measurement. We need 8024 * to record the new gput_ts as here and 8025 * possibly update the start sequence. 8026 */ 8027 uint32_t seq, ts; 8028 8029 if (rsm->r_rtr_cnt > 1) { 8030 /* 8031 * This is a retransmit, can we 8032 * really make any assessment at this 8033 * point? We are not really sure of 8034 * the timestamp, is it this or the 8035 * previous transmission? 8036 * 8037 * Lets wait for something better that 8038 * is not retransmitted. 8039 */ 8040 return; 8041 } 8042 seq = tp->gput_seq; 8043 ts = tp->gput_ts; 8044 rack->app_limited_needs_set = 0; 8045 tp->gput_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 8046 /* Do we start at a new end? */ 8047 if ((use_which == RACK_USE_BEG) && 8048 SEQ_GEQ(rsm->r_start, tp->gput_seq)) { 8049 /* 8050 * When we get an ACK that just eats 8051 * up some of the rsm, we set RACK_USE_BEG 8052 * since whats at r_start (i.e. th_ack) 8053 * is left unacked and thats where the 8054 * measurement not starts. 8055 */ 8056 tp->gput_seq = rsm->r_start; 8057 rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 8058 } 8059 if ((use_which == RACK_USE_END) && 8060 SEQ_GEQ(rsm->r_end, tp->gput_seq)) { 8061 /* 8062 * We use the end when the cumack 8063 * is moving forward and completely 8064 * deleting the rsm passed so basically 8065 * r_end holds th_ack. 8066 * 8067 * For SACK's we also want to use the end 8068 * since this piece just got sacked and 8069 * we want to target anything after that 8070 * in our measurement. 8071 */ 8072 tp->gput_seq = rsm->r_end; 8073 rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 8074 } 8075 if (use_which == RACK_USE_END_OR_THACK) { 8076 /* 8077 * special case for ack moving forward, 8078 * not a sack, we need to move all the 8079 * way up to where this ack cum-ack moves 8080 * to. 8081 */ 8082 if (SEQ_GT(th_ack, rsm->r_end)) 8083 tp->gput_seq = th_ack; 8084 else 8085 tp->gput_seq = rsm->r_end; 8086 rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 8087 } 8088 if (SEQ_GT(tp->gput_seq, tp->gput_ack)) { 8089 /* 8090 * We moved beyond this guy's range, re-calculate 8091 * the new end point. 8092 */ 8093 if (rack->rc_gp_filled == 0) { 8094 tp->gput_ack = tp->gput_seq + max(rc_init_window(rack), (MIN_GP_WIN * ctf_fixed_maxseg(tp))); 8095 } else { 8096 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack); 8097 } 8098 } 8099 /* 8100 * We are moving the goal post, we may be able to clear the 8101 * measure_saw_probe_rtt flag. 8102 */ 8103 if ((rack->in_probe_rtt == 0) && 8104 (rack->measure_saw_probe_rtt) && 8105 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit))) 8106 rack->measure_saw_probe_rtt = 0; 8107 rack_log_pacing_delay_calc(rack, ts, tp->gput_ts, 8108 seq, tp->gput_seq, 0, 5, line, NULL, 0); 8109 if (rack->rc_gp_filled && 8110 ((tp->gput_ack - tp->gput_seq) < 8111 max(rc_init_window(rack), (MIN_GP_WIN * 8112 ctf_fixed_maxseg(tp))))) { 8113 uint32_t ideal_amount; 8114 8115 ideal_amount = rack_get_measure_window(tp, rack); 8116 if (ideal_amount > sbavail(&tptosocket(tp)->so_snd)) { 8117 /* 8118 * There is no sense of continuing this measurement 8119 * because its too small to gain us anything we 8120 * trust. Skip it and that way we can start a new 8121 * measurement quicker. 8122 */ 8123 tp->t_flags &= ~TF_GPUTINPROG; 8124 rack_log_pacing_delay_calc(rack, tp->gput_ack, tp->gput_seq, 8125 0, 0, 0, 6, __LINE__, NULL, 0); 8126 } else { 8127 /* 8128 * Reset the window further out. 8129 */ 8130 tp->gput_ack = tp->gput_seq + ideal_amount; 8131 } 8132 } 8133 } 8134 } 8135 8136 static inline int 8137 is_rsm_inside_declared_tlp_block(struct tcp_rack *rack, struct rack_sendmap *rsm) 8138 { 8139 if (SEQ_LT(rsm->r_end, rack->r_ctl.last_tlp_acked_start)) { 8140 /* Behind our TLP definition or right at */ 8141 return (0); 8142 } 8143 if (SEQ_GT(rsm->r_start, rack->r_ctl.last_tlp_acked_end)) { 8144 /* The start is beyond or right at our end of TLP definition */ 8145 return (0); 8146 } 8147 /* It has to be a sub-part of the original TLP recorded */ 8148 return (1); 8149 } 8150 8151 8152 static uint32_t 8153 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack, struct sackblk *sack, 8154 struct tcpopt *to, struct rack_sendmap **prsm, uint32_t cts, int *moved_two) 8155 { 8156 uint32_t start, end, changed = 0; 8157 struct rack_sendmap stack_map; 8158 struct rack_sendmap *rsm, *nrsm, fe, *prev, *next; 8159 #ifdef INVARIANTS 8160 struct rack_sendmap *insret; 8161 #endif 8162 int32_t used_ref = 1; 8163 int moved = 0; 8164 8165 start = sack->start; 8166 end = sack->end; 8167 rsm = *prsm; 8168 memset(&fe, 0, sizeof(fe)); 8169 do_rest_ofb: 8170 if ((rsm == NULL) || 8171 (SEQ_LT(end, rsm->r_start)) || 8172 (SEQ_GEQ(start, rsm->r_end)) || 8173 (SEQ_LT(start, rsm->r_start))) { 8174 /* 8175 * We are not in the right spot, 8176 * find the correct spot in the tree. 8177 */ 8178 used_ref = 0; 8179 fe.r_start = start; 8180 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 8181 moved++; 8182 } 8183 if (rsm == NULL) { 8184 /* TSNH */ 8185 goto out; 8186 } 8187 /* Ok we have an ACK for some piece of this rsm */ 8188 if (rsm->r_start != start) { 8189 if ((rsm->r_flags & RACK_ACKED) == 0) { 8190 /* 8191 * Before any splitting or hookery is 8192 * done is it a TLP of interest i.e. rxt? 8193 */ 8194 if ((rsm->r_flags & RACK_TLP) && 8195 (rsm->r_rtr_cnt > 1)) { 8196 /* 8197 * We are splitting a rxt TLP, check 8198 * if we need to save off the start/end 8199 */ 8200 if (rack->rc_last_tlp_acked_set && 8201 (is_rsm_inside_declared_tlp_block(rack, rsm))) { 8202 /* 8203 * We already turned this on since we are inside 8204 * the previous one was a partially sack now we 8205 * are getting another one (maybe all of it). 8206 * 8207 */ 8208 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end); 8209 /* 8210 * Lets make sure we have all of it though. 8211 */ 8212 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) { 8213 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8214 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8215 rack->r_ctl.last_tlp_acked_end); 8216 } 8217 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) { 8218 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8219 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8220 rack->r_ctl.last_tlp_acked_end); 8221 } 8222 } else { 8223 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8224 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8225 rack->rc_last_tlp_past_cumack = 0; 8226 rack->rc_last_tlp_acked_set = 1; 8227 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end); 8228 } 8229 } 8230 /** 8231 * Need to split this in two pieces the before and after, 8232 * the before remains in the map, the after must be 8233 * added. In other words we have: 8234 * rsm |--------------| 8235 * sackblk |-------> 8236 * rsm will become 8237 * rsm |---| 8238 * and nrsm will be the sacked piece 8239 * nrsm |----------| 8240 * 8241 * But before we start down that path lets 8242 * see if the sack spans over on top of 8243 * the next guy and it is already sacked. 8244 * 8245 */ 8246 next = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8247 if (next && (next->r_flags & RACK_ACKED) && 8248 SEQ_GEQ(end, next->r_start)) { 8249 /** 8250 * So the next one is already acked, and 8251 * we can thus by hookery use our stack_map 8252 * to reflect the piece being sacked and 8253 * then adjust the two tree entries moving 8254 * the start and ends around. So we start like: 8255 * rsm |------------| (not-acked) 8256 * next |-----------| (acked) 8257 * sackblk |--------> 8258 * We want to end like so: 8259 * rsm |------| (not-acked) 8260 * next |-----------------| (acked) 8261 * nrsm |-----| 8262 * Where nrsm is a temporary stack piece we 8263 * use to update all the gizmos. 8264 */ 8265 /* Copy up our fudge block */ 8266 nrsm = &stack_map; 8267 memcpy(nrsm, rsm, sizeof(struct rack_sendmap)); 8268 /* Now adjust our tree blocks */ 8269 rsm->r_end = start; 8270 next->r_start = start; 8271 /* Now we must adjust back where next->m is */ 8272 rack_setup_offset_for_rsm(rsm, next); 8273 8274 /* We don't need to adjust rsm, it did not change */ 8275 /* Clear out the dup ack count of the remainder */ 8276 rsm->r_dupack = 0; 8277 rsm->r_just_ret = 0; 8278 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); 8279 /* Now lets make sure our fudge block is right */ 8280 nrsm->r_start = start; 8281 /* Now lets update all the stats and such */ 8282 rack_update_rtt(tp, rack, nrsm, to, cts, SACKED, 0); 8283 if (rack->app_limited_needs_set) 8284 rack_need_set_test(tp, rack, nrsm, tp->snd_una, __LINE__, RACK_USE_END); 8285 changed += (nrsm->r_end - nrsm->r_start); 8286 rack->r_ctl.rc_sacked += (nrsm->r_end - nrsm->r_start); 8287 if (nrsm->r_flags & RACK_SACK_PASSED) { 8288 rack->r_ctl.rc_reorder_ts = cts; 8289 } 8290 /* 8291 * Now we want to go up from rsm (the 8292 * one left un-acked) to the next one 8293 * in the tmap. We do this so when 8294 * we walk backwards we include marking 8295 * sack-passed on rsm (The one passed in 8296 * is skipped since it is generally called 8297 * on something sacked before removing it 8298 * from the tmap). 8299 */ 8300 if (rsm->r_in_tmap) { 8301 nrsm = TAILQ_NEXT(rsm, r_tnext); 8302 /* 8303 * Now that we have the next 8304 * one walk backwards from there. 8305 */ 8306 if (nrsm && nrsm->r_in_tmap) 8307 rack_log_sack_passed(tp, rack, nrsm); 8308 } 8309 /* Now are we done? */ 8310 if (SEQ_LT(end, next->r_end) || 8311 (end == next->r_end)) { 8312 /* Done with block */ 8313 goto out; 8314 } 8315 rack_log_map_chg(tp, rack, &stack_map, rsm, next, MAP_SACK_M1, end, __LINE__); 8316 counter_u64_add(rack_sack_used_next_merge, 1); 8317 /* Postion for the next block */ 8318 start = next->r_end; 8319 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, next); 8320 if (rsm == NULL) 8321 goto out; 8322 } else { 8323 /** 8324 * We can't use any hookery here, so we 8325 * need to split the map. We enter like 8326 * so: 8327 * rsm |--------| 8328 * sackblk |-----> 8329 * We will add the new block nrsm and 8330 * that will be the new portion, and then 8331 * fall through after reseting rsm. So we 8332 * split and look like this: 8333 * rsm |----| 8334 * sackblk |-----> 8335 * nrsm |---| 8336 * We then fall through reseting 8337 * rsm to nrsm, so the next block 8338 * picks it up. 8339 */ 8340 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT); 8341 if (nrsm == NULL) { 8342 /* 8343 * failed XXXrrs what can we do but loose the sack 8344 * info? 8345 */ 8346 goto out; 8347 } 8348 counter_u64_add(rack_sack_splits, 1); 8349 rack_clone_rsm(rack, nrsm, rsm, start); 8350 rsm->r_just_ret = 0; 8351 #ifndef INVARIANTS 8352 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 8353 #else 8354 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 8355 if (insret != NULL) { 8356 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 8357 nrsm, insret, rack, rsm); 8358 } 8359 #endif 8360 if (rsm->r_in_tmap) { 8361 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 8362 nrsm->r_in_tmap = 1; 8363 } 8364 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SACK_M2, end, __LINE__); 8365 rsm->r_flags &= (~RACK_HAS_FIN); 8366 /* Position us to point to the new nrsm that starts the sack blk */ 8367 rsm = nrsm; 8368 } 8369 } else { 8370 /* Already sacked this piece */ 8371 counter_u64_add(rack_sack_skipped_acked, 1); 8372 moved++; 8373 if (end == rsm->r_end) { 8374 /* Done with block */ 8375 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8376 goto out; 8377 } else if (SEQ_LT(end, rsm->r_end)) { 8378 /* A partial sack to a already sacked block */ 8379 moved++; 8380 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8381 goto out; 8382 } else { 8383 /* 8384 * The end goes beyond this guy 8385 * reposition the start to the 8386 * next block. 8387 */ 8388 start = rsm->r_end; 8389 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8390 if (rsm == NULL) 8391 goto out; 8392 } 8393 } 8394 } 8395 if (SEQ_GEQ(end, rsm->r_end)) { 8396 /** 8397 * The end of this block is either beyond this guy or right 8398 * at this guy. I.e.: 8399 * rsm --- |-----| 8400 * end |-----| 8401 * <or> 8402 * end |---------| 8403 */ 8404 if ((rsm->r_flags & RACK_ACKED) == 0) { 8405 /* 8406 * Is it a TLP of interest? 8407 */ 8408 if ((rsm->r_flags & RACK_TLP) && 8409 (rsm->r_rtr_cnt > 1)) { 8410 /* 8411 * We are splitting a rxt TLP, check 8412 * if we need to save off the start/end 8413 */ 8414 if (rack->rc_last_tlp_acked_set && 8415 (is_rsm_inside_declared_tlp_block(rack, rsm))) { 8416 /* 8417 * We already turned this on since we are inside 8418 * the previous one was a partially sack now we 8419 * are getting another one (maybe all of it). 8420 */ 8421 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end); 8422 /* 8423 * Lets make sure we have all of it though. 8424 */ 8425 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) { 8426 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8427 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8428 rack->r_ctl.last_tlp_acked_end); 8429 } 8430 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) { 8431 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8432 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8433 rack->r_ctl.last_tlp_acked_end); 8434 } 8435 } else { 8436 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8437 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8438 rack->rc_last_tlp_past_cumack = 0; 8439 rack->rc_last_tlp_acked_set = 1; 8440 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end); 8441 } 8442 } 8443 rack_update_rtt(tp, rack, rsm, to, cts, SACKED, 0); 8444 changed += (rsm->r_end - rsm->r_start); 8445 rack->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); 8446 if (rsm->r_in_tmap) /* should be true */ 8447 rack_log_sack_passed(tp, rack, rsm); 8448 /* Is Reordering occuring? */ 8449 if (rsm->r_flags & RACK_SACK_PASSED) { 8450 rsm->r_flags &= ~RACK_SACK_PASSED; 8451 rack->r_ctl.rc_reorder_ts = cts; 8452 } 8453 if (rack->app_limited_needs_set) 8454 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_END); 8455 rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time); 8456 rsm->r_flags |= RACK_ACKED; 8457 if (rsm->r_in_tmap) { 8458 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext); 8459 rsm->r_in_tmap = 0; 8460 } 8461 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_SACK_M3, end, __LINE__); 8462 } else { 8463 counter_u64_add(rack_sack_skipped_acked, 1); 8464 moved++; 8465 } 8466 if (end == rsm->r_end) { 8467 /* This block only - done, setup for next */ 8468 goto out; 8469 } 8470 /* 8471 * There is more not coverend by this rsm move on 8472 * to the next block in the RB tree. 8473 */ 8474 nrsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8475 start = rsm->r_end; 8476 rsm = nrsm; 8477 if (rsm == NULL) 8478 goto out; 8479 goto do_rest_ofb; 8480 } 8481 /** 8482 * The end of this sack block is smaller than 8483 * our rsm i.e.: 8484 * rsm --- |-----| 8485 * end |--| 8486 */ 8487 if ((rsm->r_flags & RACK_ACKED) == 0) { 8488 /* 8489 * Is it a TLP of interest? 8490 */ 8491 if ((rsm->r_flags & RACK_TLP) && 8492 (rsm->r_rtr_cnt > 1)) { 8493 /* 8494 * We are splitting a rxt TLP, check 8495 * if we need to save off the start/end 8496 */ 8497 if (rack->rc_last_tlp_acked_set && 8498 (is_rsm_inside_declared_tlp_block(rack, rsm))) { 8499 /* 8500 * We already turned this on since we are inside 8501 * the previous one was a partially sack now we 8502 * are getting another one (maybe all of it). 8503 */ 8504 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end); 8505 /* 8506 * Lets make sure we have all of it though. 8507 */ 8508 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) { 8509 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8510 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8511 rack->r_ctl.last_tlp_acked_end); 8512 } 8513 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) { 8514 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8515 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8516 rack->r_ctl.last_tlp_acked_end); 8517 } 8518 } else { 8519 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8520 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8521 rack->rc_last_tlp_past_cumack = 0; 8522 rack->rc_last_tlp_acked_set = 1; 8523 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end); 8524 } 8525 } 8526 prev = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8527 if (prev && 8528 (prev->r_flags & RACK_ACKED)) { 8529 /** 8530 * Goal, we want the right remainder of rsm to shrink 8531 * in place and span from (rsm->r_start = end) to rsm->r_end. 8532 * We want to expand prev to go all the way 8533 * to prev->r_end <- end. 8534 * so in the tree we have before: 8535 * prev |--------| (acked) 8536 * rsm |-------| (non-acked) 8537 * sackblk |-| 8538 * We churn it so we end up with 8539 * prev |----------| (acked) 8540 * rsm |-----| (non-acked) 8541 * nrsm |-| (temporary) 8542 * 8543 * Note if either prev/rsm is a TLP we don't 8544 * do this. 8545 */ 8546 nrsm = &stack_map; 8547 memcpy(nrsm, rsm, sizeof(struct rack_sendmap)); 8548 prev->r_end = end; 8549 rsm->r_start = end; 8550 /* Now adjust nrsm (stack copy) to be 8551 * the one that is the small 8552 * piece that was "sacked". 8553 */ 8554 nrsm->r_end = end; 8555 rsm->r_dupack = 0; 8556 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); 8557 /* 8558 * Now that the rsm has had its start moved forward 8559 * lets go ahead and get its new place in the world. 8560 */ 8561 rack_setup_offset_for_rsm(prev, rsm); 8562 /* 8563 * Now nrsm is our new little piece 8564 * that is acked (which was merged 8565 * to prev). Update the rtt and changed 8566 * based on that. Also check for reordering. 8567 */ 8568 rack_update_rtt(tp, rack, nrsm, to, cts, SACKED, 0); 8569 if (rack->app_limited_needs_set) 8570 rack_need_set_test(tp, rack, nrsm, tp->snd_una, __LINE__, RACK_USE_END); 8571 changed += (nrsm->r_end - nrsm->r_start); 8572 rack->r_ctl.rc_sacked += (nrsm->r_end - nrsm->r_start); 8573 if (nrsm->r_flags & RACK_SACK_PASSED) { 8574 rack->r_ctl.rc_reorder_ts = cts; 8575 } 8576 rack_log_map_chg(tp, rack, prev, &stack_map, rsm, MAP_SACK_M4, end, __LINE__); 8577 rsm = prev; 8578 counter_u64_add(rack_sack_used_prev_merge, 1); 8579 } else { 8580 /** 8581 * This is the case where our previous 8582 * block is not acked either, so we must 8583 * split the block in two. 8584 */ 8585 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT); 8586 if (nrsm == NULL) { 8587 /* failed rrs what can we do but loose the sack info? */ 8588 goto out; 8589 } 8590 if ((rsm->r_flags & RACK_TLP) && 8591 (rsm->r_rtr_cnt > 1)) { 8592 /* 8593 * We are splitting a rxt TLP, check 8594 * if we need to save off the start/end 8595 */ 8596 if (rack->rc_last_tlp_acked_set && 8597 (is_rsm_inside_declared_tlp_block(rack, rsm))) { 8598 /* 8599 * We already turned this on since this block is inside 8600 * the previous one was a partially sack now we 8601 * are getting another one (maybe all of it). 8602 */ 8603 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end); 8604 /* 8605 * Lets make sure we have all of it though. 8606 */ 8607 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) { 8608 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8609 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8610 rack->r_ctl.last_tlp_acked_end); 8611 } 8612 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) { 8613 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8614 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8615 rack->r_ctl.last_tlp_acked_end); 8616 } 8617 } else { 8618 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8619 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8620 rack->rc_last_tlp_acked_set = 1; 8621 rack->rc_last_tlp_past_cumack = 0; 8622 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end); 8623 } 8624 } 8625 /** 8626 * In this case nrsm becomes 8627 * nrsm->r_start = end; 8628 * nrsm->r_end = rsm->r_end; 8629 * which is un-acked. 8630 * <and> 8631 * rsm->r_end = nrsm->r_start; 8632 * i.e. the remaining un-acked 8633 * piece is left on the left 8634 * hand side. 8635 * 8636 * So we start like this 8637 * rsm |----------| (not acked) 8638 * sackblk |---| 8639 * build it so we have 8640 * rsm |---| (acked) 8641 * nrsm |------| (not acked) 8642 */ 8643 counter_u64_add(rack_sack_splits, 1); 8644 rack_clone_rsm(rack, nrsm, rsm, end); 8645 rsm->r_flags &= (~RACK_HAS_FIN); 8646 rsm->r_just_ret = 0; 8647 #ifndef INVARIANTS 8648 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 8649 #else 8650 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 8651 if (insret != NULL) { 8652 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 8653 nrsm, insret, rack, rsm); 8654 } 8655 #endif 8656 if (rsm->r_in_tmap) { 8657 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 8658 nrsm->r_in_tmap = 1; 8659 } 8660 nrsm->r_dupack = 0; 8661 rack_log_retran_reason(rack, nrsm, __LINE__, 0, 2); 8662 rack_update_rtt(tp, rack, rsm, to, cts, SACKED, 0); 8663 changed += (rsm->r_end - rsm->r_start); 8664 rack->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start); 8665 if (rsm->r_in_tmap) /* should be true */ 8666 rack_log_sack_passed(tp, rack, rsm); 8667 /* Is Reordering occuring? */ 8668 if (rsm->r_flags & RACK_SACK_PASSED) { 8669 rsm->r_flags &= ~RACK_SACK_PASSED; 8670 rack->r_ctl.rc_reorder_ts = cts; 8671 } 8672 if (rack->app_limited_needs_set) 8673 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_END); 8674 rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time); 8675 rsm->r_flags |= RACK_ACKED; 8676 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SACK_M5, end, __LINE__); 8677 if (rsm->r_in_tmap) { 8678 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext); 8679 rsm->r_in_tmap = 0; 8680 } 8681 } 8682 } else if (start != end){ 8683 /* 8684 * The block was already acked. 8685 */ 8686 counter_u64_add(rack_sack_skipped_acked, 1); 8687 moved++; 8688 } 8689 out: 8690 if (rsm && 8691 ((rsm->r_flags & RACK_TLP) == 0) && 8692 (rsm->r_flags & RACK_ACKED)) { 8693 /* 8694 * Now can we merge where we worked 8695 * with either the previous or 8696 * next block? 8697 */ 8698 next = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8699 while (next) { 8700 if (next->r_flags & RACK_TLP) 8701 break; 8702 if (next->r_flags & RACK_ACKED) { 8703 /* yep this and next can be merged */ 8704 rsm = rack_merge_rsm(rack, rsm, next); 8705 next = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8706 } else 8707 break; 8708 } 8709 /* Now what about the previous? */ 8710 prev = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8711 while (prev) { 8712 if (prev->r_flags & RACK_TLP) 8713 break; 8714 if (prev->r_flags & RACK_ACKED) { 8715 /* yep the previous and this can be merged */ 8716 rsm = rack_merge_rsm(rack, prev, rsm); 8717 prev = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8718 } else 8719 break; 8720 } 8721 } 8722 if (used_ref == 0) { 8723 counter_u64_add(rack_sack_proc_all, 1); 8724 } else { 8725 counter_u64_add(rack_sack_proc_short, 1); 8726 } 8727 /* Save off the next one for quick reference. */ 8728 if (rsm) 8729 nrsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8730 else 8731 nrsm = NULL; 8732 *prsm = rack->r_ctl.rc_sacklast = nrsm; 8733 /* Pass back the moved. */ 8734 *moved_two = moved; 8735 return (changed); 8736 } 8737 8738 static void inline 8739 rack_peer_reneges(struct tcp_rack *rack, struct rack_sendmap *rsm, tcp_seq th_ack) 8740 { 8741 struct rack_sendmap *tmap; 8742 8743 tmap = NULL; 8744 while (rsm && (rsm->r_flags & RACK_ACKED)) { 8745 /* Its no longer sacked, mark it so */ 8746 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); 8747 #ifdef INVARIANTS 8748 if (rsm->r_in_tmap) { 8749 panic("rack:%p rsm:%p flags:0x%x in tmap?", 8750 rack, rsm, rsm->r_flags); 8751 } 8752 #endif 8753 rsm->r_flags &= ~(RACK_ACKED|RACK_SACK_PASSED|RACK_WAS_SACKPASS); 8754 /* Rebuild it into our tmap */ 8755 if (tmap == NULL) { 8756 TAILQ_INSERT_HEAD(&rack->r_ctl.rc_tmap, rsm, r_tnext); 8757 tmap = rsm; 8758 } else { 8759 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, tmap, rsm, r_tnext); 8760 tmap = rsm; 8761 } 8762 tmap->r_in_tmap = 1; 8763 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8764 } 8765 /* 8766 * Now lets possibly clear the sack filter so we start 8767 * recognizing sacks that cover this area. 8768 */ 8769 sack_filter_clear(&rack->r_ctl.rack_sf, th_ack); 8770 8771 } 8772 8773 static void 8774 rack_do_decay(struct tcp_rack *rack) 8775 { 8776 struct timeval res; 8777 8778 #define timersub(tvp, uvp, vvp) \ 8779 do { \ 8780 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ 8781 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \ 8782 if ((vvp)->tv_usec < 0) { \ 8783 (vvp)->tv_sec--; \ 8784 (vvp)->tv_usec += 1000000; \ 8785 } \ 8786 } while (0) 8787 8788 timersub(&rack->r_ctl.act_rcv_time, &rack->r_ctl.rc_last_time_decay, &res); 8789 #undef timersub 8790 8791 rack->r_ctl.input_pkt++; 8792 if ((rack->rc_in_persist) || 8793 (res.tv_sec >= 1) || 8794 (rack->rc_tp->snd_max == rack->rc_tp->snd_una)) { 8795 /* 8796 * Check for decay of non-SAD, 8797 * we want all SAD detection metrics to 8798 * decay 1/4 per second (or more) passed. 8799 */ 8800 #ifdef NETFLIX_EXP_DETECTION 8801 uint32_t pkt_delta; 8802 8803 pkt_delta = rack->r_ctl.input_pkt - rack->r_ctl.saved_input_pkt; 8804 #endif 8805 /* Update our saved tracking values */ 8806 rack->r_ctl.saved_input_pkt = rack->r_ctl.input_pkt; 8807 rack->r_ctl.rc_last_time_decay = rack->r_ctl.act_rcv_time; 8808 /* Now do we escape without decay? */ 8809 #ifdef NETFLIX_EXP_DETECTION 8810 if (rack->rc_in_persist || 8811 (rack->rc_tp->snd_max == rack->rc_tp->snd_una) || 8812 (pkt_delta < tcp_sad_low_pps)){ 8813 /* 8814 * We don't decay idle connections 8815 * or ones that have a low input pps. 8816 */ 8817 return; 8818 } 8819 /* Decay the counters */ 8820 rack->r_ctl.ack_count = ctf_decay_count(rack->r_ctl.ack_count, 8821 tcp_sad_decay_val); 8822 rack->r_ctl.sack_count = ctf_decay_count(rack->r_ctl.sack_count, 8823 tcp_sad_decay_val); 8824 rack->r_ctl.sack_moved_extra = ctf_decay_count(rack->r_ctl.sack_moved_extra, 8825 tcp_sad_decay_val); 8826 rack->r_ctl.sack_noextra_move = ctf_decay_count(rack->r_ctl.sack_noextra_move, 8827 tcp_sad_decay_val); 8828 #endif 8829 } 8830 } 8831 8832 static void 8833 rack_process_to_cumack(struct tcpcb *tp, struct tcp_rack *rack, register uint32_t th_ack, uint32_t cts, struct tcpopt *to) 8834 { 8835 struct rack_sendmap *rsm; 8836 #ifdef INVARIANTS 8837 struct rack_sendmap *rm; 8838 #endif 8839 8840 /* 8841 * The ACK point is advancing to th_ack, we must drop off 8842 * the packets in the rack log and calculate any eligble 8843 * RTT's. 8844 */ 8845 rack->r_wanted_output = 1; 8846 8847 /* Tend any TLP that has been marked for 1/2 the seq space (its old) */ 8848 if ((rack->rc_last_tlp_acked_set == 1)&& 8849 (rack->rc_last_tlp_past_cumack == 1) && 8850 (SEQ_GT(rack->r_ctl.last_tlp_acked_start, th_ack))) { 8851 /* 8852 * We have reached the point where our last rack 8853 * tlp retransmit sequence is ahead of the cum-ack. 8854 * This can only happen when the cum-ack moves all 8855 * the way around (its been a full 2^^31+1 bytes 8856 * or more since we sent a retransmitted TLP). Lets 8857 * turn off the valid flag since its not really valid. 8858 * 8859 * Note since sack's also turn on this event we have 8860 * a complication, we have to wait to age it out until 8861 * the cum-ack is by the TLP before checking which is 8862 * what the next else clause does. 8863 */ 8864 rack_log_dsack_event(rack, 9, __LINE__, 8865 rack->r_ctl.last_tlp_acked_start, 8866 rack->r_ctl.last_tlp_acked_end); 8867 rack->rc_last_tlp_acked_set = 0; 8868 rack->rc_last_tlp_past_cumack = 0; 8869 } else if ((rack->rc_last_tlp_acked_set == 1) && 8870 (rack->rc_last_tlp_past_cumack == 0) && 8871 (SEQ_GEQ(th_ack, rack->r_ctl.last_tlp_acked_end))) { 8872 /* 8873 * It is safe to start aging TLP's out. 8874 */ 8875 rack->rc_last_tlp_past_cumack = 1; 8876 } 8877 /* We do the same for the tlp send seq as well */ 8878 if ((rack->rc_last_sent_tlp_seq_valid == 1) && 8879 (rack->rc_last_sent_tlp_past_cumack == 1) && 8880 (SEQ_GT(rack->r_ctl.last_sent_tlp_seq, th_ack))) { 8881 rack_log_dsack_event(rack, 9, __LINE__, 8882 rack->r_ctl.last_sent_tlp_seq, 8883 (rack->r_ctl.last_sent_tlp_seq + 8884 rack->r_ctl.last_sent_tlp_len)); 8885 rack->rc_last_sent_tlp_seq_valid = 0; 8886 rack->rc_last_sent_tlp_past_cumack = 0; 8887 } else if ((rack->rc_last_sent_tlp_seq_valid == 1) && 8888 (rack->rc_last_sent_tlp_past_cumack == 0) && 8889 (SEQ_GEQ(th_ack, rack->r_ctl.last_sent_tlp_seq))) { 8890 /* 8891 * It is safe to start aging TLP's send. 8892 */ 8893 rack->rc_last_sent_tlp_past_cumack = 1; 8894 } 8895 more: 8896 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 8897 if (rsm == NULL) { 8898 if ((th_ack - 1) == tp->iss) { 8899 /* 8900 * For the SYN incoming case we will not 8901 * have called tcp_output for the sending of 8902 * the SYN, so there will be no map. All 8903 * other cases should probably be a panic. 8904 */ 8905 return; 8906 } 8907 if (tp->t_flags & TF_SENTFIN) { 8908 /* if we sent a FIN we often will not have map */ 8909 return; 8910 } 8911 #ifdef INVARIANTS 8912 panic("No rack map tp:%p for state:%d ack:%u rack:%p snd_una:%u snd_max:%u snd_nxt:%u\n", 8913 tp, 8914 tp->t_state, th_ack, rack, 8915 tp->snd_una, tp->snd_max, tp->snd_nxt); 8916 #endif 8917 return; 8918 } 8919 if (SEQ_LT(th_ack, rsm->r_start)) { 8920 /* Huh map is missing this */ 8921 #ifdef INVARIANTS 8922 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d\n", 8923 rsm->r_start, 8924 th_ack, tp->t_state, rack->r_state); 8925 #endif 8926 return; 8927 } 8928 rack_update_rtt(tp, rack, rsm, to, cts, CUM_ACKED, th_ack); 8929 8930 /* Now was it a retransmitted TLP? */ 8931 if ((rsm->r_flags & RACK_TLP) && 8932 (rsm->r_rtr_cnt > 1)) { 8933 /* 8934 * Yes, this rsm was a TLP and retransmitted, remember that 8935 * since if a DSACK comes back on this we don't want 8936 * to think of it as a reordered segment. This may 8937 * get updated again with possibly even other TLPs 8938 * in flight, but thats ok. Only when we don't send 8939 * a retransmitted TLP for 1/2 the sequences space 8940 * will it get turned off (above). 8941 */ 8942 if (rack->rc_last_tlp_acked_set && 8943 (is_rsm_inside_declared_tlp_block(rack, rsm))) { 8944 /* 8945 * We already turned this on since the end matches, 8946 * the previous one was a partially ack now we 8947 * are getting another one (maybe all of it). 8948 */ 8949 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end); 8950 /* 8951 * Lets make sure we have all of it though. 8952 */ 8953 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) { 8954 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8955 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8956 rack->r_ctl.last_tlp_acked_end); 8957 } 8958 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) { 8959 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8960 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start, 8961 rack->r_ctl.last_tlp_acked_end); 8962 } 8963 } else { 8964 rack->rc_last_tlp_past_cumack = 1; 8965 rack->r_ctl.last_tlp_acked_start = rsm->r_start; 8966 rack->r_ctl.last_tlp_acked_end = rsm->r_end; 8967 rack->rc_last_tlp_acked_set = 1; 8968 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end); 8969 } 8970 } 8971 /* Now do we consume the whole thing? */ 8972 if (SEQ_GEQ(th_ack, rsm->r_end)) { 8973 /* Its all consumed. */ 8974 uint32_t left; 8975 uint8_t newly_acked; 8976 8977 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_FREE, rsm->r_end, __LINE__); 8978 rack->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes; 8979 rsm->r_rtr_bytes = 0; 8980 /* Record the time of highest cumack sent */ 8981 rack->r_ctl.rc_gp_cumack_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 8982 #ifndef INVARIANTS 8983 (void)RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8984 #else 8985 rm = RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 8986 if (rm != rsm) { 8987 panic("removing head in rack:%p rsm:%p rm:%p", 8988 rack, rsm, rm); 8989 } 8990 #endif 8991 if (rsm->r_in_tmap) { 8992 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext); 8993 rsm->r_in_tmap = 0; 8994 } 8995 newly_acked = 1; 8996 if (rsm->r_flags & RACK_ACKED) { 8997 /* 8998 * It was acked on the scoreboard -- remove 8999 * it from total 9000 */ 9001 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start); 9002 newly_acked = 0; 9003 } else if (rsm->r_flags & RACK_SACK_PASSED) { 9004 /* 9005 * There are segments ACKED on the 9006 * scoreboard further up. We are seeing 9007 * reordering. 9008 */ 9009 rsm->r_flags &= ~RACK_SACK_PASSED; 9010 rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time); 9011 rsm->r_flags |= RACK_ACKED; 9012 rack->r_ctl.rc_reorder_ts = cts; 9013 if (rack->r_ent_rec_ns) { 9014 /* 9015 * We have sent no more, and we saw an sack 9016 * then ack arrive. 9017 */ 9018 rack->r_might_revert = 1; 9019 } 9020 } 9021 if ((rsm->r_flags & RACK_TO_REXT) && 9022 (tp->t_flags & TF_RCVD_TSTMP) && 9023 (to->to_flags & TOF_TS) && 9024 (to->to_tsecr != 0) && 9025 (tp->t_flags & TF_PREVVALID)) { 9026 /* 9027 * We can use the timestamp to see 9028 * if this retransmission was from the 9029 * first transmit. If so we made a mistake. 9030 */ 9031 tp->t_flags &= ~TF_PREVVALID; 9032 if (to->to_tsecr == rack_ts_to_msec(rsm->r_tim_lastsent[0])) { 9033 /* The first transmit is what this ack is for */ 9034 rack_cong_signal(tp, CC_RTO_ERR, th_ack, __LINE__); 9035 } 9036 } 9037 left = th_ack - rsm->r_end; 9038 if (rack->app_limited_needs_set && newly_acked) 9039 rack_need_set_test(tp, rack, rsm, th_ack, __LINE__, RACK_USE_END_OR_THACK); 9040 /* Free back to zone */ 9041 rack_free(rack, rsm); 9042 if (left) { 9043 goto more; 9044 } 9045 /* Check for reneging */ 9046 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 9047 if (rsm && (rsm->r_flags & RACK_ACKED) && (th_ack == rsm->r_start)) { 9048 /* 9049 * The peer has moved snd_una up to 9050 * the edge of this send, i.e. one 9051 * that it had previously acked. The only 9052 * way that can be true if the peer threw 9053 * away data (space issues) that it had 9054 * previously sacked (else it would have 9055 * given us snd_una up to (rsm->r_end). 9056 * We need to undo the acked markings here. 9057 * 9058 * Note we have to look to make sure th_ack is 9059 * our rsm->r_start in case we get an old ack 9060 * where th_ack is behind snd_una. 9061 */ 9062 rack_peer_reneges(rack, rsm, th_ack); 9063 } 9064 return; 9065 } 9066 if (rsm->r_flags & RACK_ACKED) { 9067 /* 9068 * It was acked on the scoreboard -- remove it from 9069 * total for the part being cum-acked. 9070 */ 9071 rack->r_ctl.rc_sacked -= (th_ack - rsm->r_start); 9072 } 9073 /* 9074 * Clear the dup ack count for 9075 * the piece that remains. 9076 */ 9077 rsm->r_dupack = 0; 9078 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2); 9079 if (rsm->r_rtr_bytes) { 9080 /* 9081 * It was retransmitted adjust the 9082 * sack holes for what was acked. 9083 */ 9084 int ack_am; 9085 9086 ack_am = (th_ack - rsm->r_start); 9087 if (ack_am >= rsm->r_rtr_bytes) { 9088 rack->r_ctl.rc_holes_rxt -= ack_am; 9089 rsm->r_rtr_bytes -= ack_am; 9090 } 9091 } 9092 /* 9093 * Update where the piece starts and record 9094 * the time of send of highest cumack sent. 9095 */ 9096 rack->r_ctl.rc_gp_cumack_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]; 9097 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_TRIM_HEAD, th_ack, __LINE__); 9098 /* Now we need to move our offset forward too */ 9099 if (rsm->m && (rsm->orig_m_len != rsm->m->m_len)) { 9100 /* Fix up the orig_m_len and possibly the mbuf offset */ 9101 rack_adjust_orig_mlen(rsm); 9102 } 9103 rsm->soff += (th_ack - rsm->r_start); 9104 rsm->r_start = th_ack; 9105 /* Now do we need to move the mbuf fwd too? */ 9106 if (rsm->m) { 9107 while (rsm->soff >= rsm->m->m_len) { 9108 rsm->soff -= rsm->m->m_len; 9109 rsm->m = rsm->m->m_next; 9110 KASSERT((rsm->m != NULL), 9111 (" nrsm:%p hit at soff:%u null m", 9112 rsm, rsm->soff)); 9113 } 9114 rsm->orig_m_len = rsm->m->m_len; 9115 } 9116 if (rack->app_limited_needs_set) 9117 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_BEG); 9118 } 9119 9120 static void 9121 rack_handle_might_revert(struct tcpcb *tp, struct tcp_rack *rack) 9122 { 9123 struct rack_sendmap *rsm; 9124 int sack_pass_fnd = 0; 9125 9126 if (rack->r_might_revert) { 9127 /* 9128 * Ok we have reordering, have not sent anything, we 9129 * might want to revert the congestion state if nothing 9130 * further has SACK_PASSED on it. Lets check. 9131 * 9132 * We also get here when we have DSACKs come in for 9133 * all the data that we FR'd. Note that a rxt or tlp 9134 * timer clears this from happening. 9135 */ 9136 9137 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) { 9138 if (rsm->r_flags & RACK_SACK_PASSED) { 9139 sack_pass_fnd = 1; 9140 break; 9141 } 9142 } 9143 if (sack_pass_fnd == 0) { 9144 /* 9145 * We went into recovery 9146 * incorrectly due to reordering! 9147 */ 9148 int orig_cwnd; 9149 9150 rack->r_ent_rec_ns = 0; 9151 orig_cwnd = tp->snd_cwnd; 9152 tp->snd_ssthresh = rack->r_ctl.rc_ssthresh_at_erec; 9153 tp->snd_recover = tp->snd_una; 9154 rack_log_to_prr(rack, 14, orig_cwnd, __LINE__); 9155 EXIT_RECOVERY(tp->t_flags); 9156 } 9157 rack->r_might_revert = 0; 9158 } 9159 } 9160 9161 #ifdef NETFLIX_EXP_DETECTION 9162 static void 9163 rack_do_detection(struct tcpcb *tp, struct tcp_rack *rack, uint32_t bytes_this_ack, uint32_t segsiz) 9164 { 9165 if ((rack->do_detection || tcp_force_detection) && 9166 tcp_sack_to_ack_thresh && 9167 tcp_sack_to_move_thresh && 9168 ((rack->r_ctl.rc_num_maps_alloced > tcp_map_minimum) || rack->sack_attack_disable)) { 9169 /* 9170 * We have thresholds set to find 9171 * possible attackers and disable sack. 9172 * Check them. 9173 */ 9174 uint64_t ackratio, moveratio, movetotal; 9175 9176 /* Log detecting */ 9177 rack_log_sad(rack, 1); 9178 ackratio = (uint64_t)(rack->r_ctl.sack_count); 9179 ackratio *= (uint64_t)(1000); 9180 if (rack->r_ctl.ack_count) 9181 ackratio /= (uint64_t)(rack->r_ctl.ack_count); 9182 else { 9183 /* We really should not hit here */ 9184 ackratio = 1000; 9185 } 9186 if ((rack->sack_attack_disable == 0) && 9187 (ackratio > rack_highest_sack_thresh_seen)) 9188 rack_highest_sack_thresh_seen = (uint32_t)ackratio; 9189 movetotal = rack->r_ctl.sack_moved_extra; 9190 movetotal += rack->r_ctl.sack_noextra_move; 9191 moveratio = rack->r_ctl.sack_moved_extra; 9192 moveratio *= (uint64_t)1000; 9193 if (movetotal) 9194 moveratio /= movetotal; 9195 else { 9196 /* No moves, thats pretty good */ 9197 moveratio = 0; 9198 } 9199 if ((rack->sack_attack_disable == 0) && 9200 (moveratio > rack_highest_move_thresh_seen)) 9201 rack_highest_move_thresh_seen = (uint32_t)moveratio; 9202 if (rack->sack_attack_disable == 0) { 9203 if ((ackratio > tcp_sack_to_ack_thresh) && 9204 (moveratio > tcp_sack_to_move_thresh)) { 9205 /* Disable sack processing */ 9206 rack->sack_attack_disable = 1; 9207 if (rack->r_rep_attack == 0) { 9208 rack->r_rep_attack = 1; 9209 counter_u64_add(rack_sack_attacks_detected, 1); 9210 } 9211 if (tcp_attack_on_turns_on_logging) { 9212 /* 9213 * Turn on logging, used for debugging 9214 * false positives. 9215 */ 9216 rack->rc_tp->t_logstate = tcp_attack_on_turns_on_logging; 9217 } 9218 /* Clamp the cwnd at flight size */ 9219 rack->r_ctl.rc_saved_cwnd = rack->rc_tp->snd_cwnd; 9220 rack->rc_tp->snd_cwnd = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 9221 rack_log_sad(rack, 2); 9222 } 9223 } else { 9224 /* We are sack-disabled check for false positives */ 9225 if ((ackratio <= tcp_restoral_thresh) || 9226 (rack->r_ctl.rc_num_maps_alloced < tcp_map_minimum)) { 9227 rack->sack_attack_disable = 0; 9228 rack_log_sad(rack, 3); 9229 /* Restart counting */ 9230 rack->r_ctl.sack_count = 0; 9231 rack->r_ctl.sack_moved_extra = 0; 9232 rack->r_ctl.sack_noextra_move = 1; 9233 rack->r_ctl.ack_count = max(1, 9234 (bytes_this_ack / segsiz)); 9235 9236 if (rack->r_rep_reverse == 0) { 9237 rack->r_rep_reverse = 1; 9238 counter_u64_add(rack_sack_attacks_reversed, 1); 9239 } 9240 /* Restore the cwnd */ 9241 if (rack->r_ctl.rc_saved_cwnd > rack->rc_tp->snd_cwnd) 9242 rack->rc_tp->snd_cwnd = rack->r_ctl.rc_saved_cwnd; 9243 } 9244 } 9245 } 9246 } 9247 #endif 9248 9249 static int 9250 rack_note_dsack(struct tcp_rack *rack, tcp_seq start, tcp_seq end) 9251 { 9252 9253 uint32_t am, l_end; 9254 int was_tlp = 0; 9255 9256 if (SEQ_GT(end, start)) 9257 am = end - start; 9258 else 9259 am = 0; 9260 if ((rack->rc_last_tlp_acked_set ) && 9261 (SEQ_GEQ(start, rack->r_ctl.last_tlp_acked_start)) && 9262 (SEQ_LEQ(end, rack->r_ctl.last_tlp_acked_end))) { 9263 /* 9264 * The DSACK is because of a TLP which we don't 9265 * do anything with the reordering window over since 9266 * it was not reordering that caused the DSACK but 9267 * our previous retransmit TLP. 9268 */ 9269 rack_log_dsack_event(rack, 7, __LINE__, start, end); 9270 was_tlp = 1; 9271 goto skip_dsack_round; 9272 } 9273 if (rack->rc_last_sent_tlp_seq_valid) { 9274 l_end = rack->r_ctl.last_sent_tlp_seq + rack->r_ctl.last_sent_tlp_len; 9275 if (SEQ_GEQ(start, rack->r_ctl.last_sent_tlp_seq) && 9276 (SEQ_LEQ(end, l_end))) { 9277 /* 9278 * This dsack is from the last sent TLP, ignore it 9279 * for reordering purposes. 9280 */ 9281 rack_log_dsack_event(rack, 7, __LINE__, start, end); 9282 was_tlp = 1; 9283 goto skip_dsack_round; 9284 } 9285 } 9286 if (rack->rc_dsack_round_seen == 0) { 9287 rack->rc_dsack_round_seen = 1; 9288 rack->r_ctl.dsack_round_end = rack->rc_tp->snd_max; 9289 rack->r_ctl.num_dsack++; 9290 rack->r_ctl.dsack_persist = 16; /* 16 is from the standard */ 9291 rack_log_dsack_event(rack, 2, __LINE__, 0, 0); 9292 } 9293 skip_dsack_round: 9294 /* 9295 * We keep track of how many DSACK blocks we get 9296 * after a recovery incident. 9297 */ 9298 rack->r_ctl.dsack_byte_cnt += am; 9299 if (!IN_FASTRECOVERY(rack->rc_tp->t_flags) && 9300 rack->r_ctl.retran_during_recovery && 9301 (rack->r_ctl.dsack_byte_cnt >= rack->r_ctl.retran_during_recovery)) { 9302 /* 9303 * False recovery most likely culprit is reordering. If 9304 * nothing else is missing we need to revert. 9305 */ 9306 rack->r_might_revert = 1; 9307 rack_handle_might_revert(rack->rc_tp, rack); 9308 rack->r_might_revert = 0; 9309 rack->r_ctl.retran_during_recovery = 0; 9310 rack->r_ctl.dsack_byte_cnt = 0; 9311 } 9312 return (was_tlp); 9313 } 9314 9315 static uint32_t 9316 do_rack_compute_pipe(struct tcpcb *tp, struct tcp_rack *rack, uint32_t snd_una) 9317 { 9318 return (((tp->snd_max - snd_una) - rack->r_ctl.rc_sacked) + rack->r_ctl.rc_holes_rxt); 9319 } 9320 9321 static int32_t 9322 rack_compute_pipe(struct tcpcb *tp) 9323 { 9324 return ((int32_t)do_rack_compute_pipe(tp, 9325 (struct tcp_rack *)tp->t_fb_ptr, 9326 tp->snd_una)); 9327 } 9328 9329 static void 9330 rack_update_prr(struct tcpcb *tp, struct tcp_rack *rack, uint32_t changed, tcp_seq th_ack) 9331 { 9332 /* Deal with changed and PRR here (in recovery only) */ 9333 uint32_t pipe, snd_una; 9334 9335 rack->r_ctl.rc_prr_delivered += changed; 9336 9337 if (sbavail(&rack->rc_inp->inp_socket->so_snd) <= (tp->snd_max - tp->snd_una)) { 9338 /* 9339 * It is all outstanding, we are application limited 9340 * and thus we don't need more room to send anything. 9341 * Note we use tp->snd_una here and not th_ack because 9342 * the data as yet not been cut from the sb. 9343 */ 9344 rack->r_ctl.rc_prr_sndcnt = 0; 9345 return; 9346 } 9347 /* Compute prr_sndcnt */ 9348 if (SEQ_GT(tp->snd_una, th_ack)) { 9349 snd_una = tp->snd_una; 9350 } else { 9351 snd_una = th_ack; 9352 } 9353 pipe = do_rack_compute_pipe(tp, rack, snd_una); 9354 if (pipe > tp->snd_ssthresh) { 9355 long sndcnt; 9356 9357 sndcnt = rack->r_ctl.rc_prr_delivered * tp->snd_ssthresh; 9358 if (rack->r_ctl.rc_prr_recovery_fs > 0) 9359 sndcnt /= (long)rack->r_ctl.rc_prr_recovery_fs; 9360 else { 9361 rack->r_ctl.rc_prr_sndcnt = 0; 9362 rack_log_to_prr(rack, 9, 0, __LINE__); 9363 sndcnt = 0; 9364 } 9365 sndcnt++; 9366 if (sndcnt > (long)rack->r_ctl.rc_prr_out) 9367 sndcnt -= rack->r_ctl.rc_prr_out; 9368 else 9369 sndcnt = 0; 9370 rack->r_ctl.rc_prr_sndcnt = sndcnt; 9371 rack_log_to_prr(rack, 10, 0, __LINE__); 9372 } else { 9373 uint32_t limit; 9374 9375 if (rack->r_ctl.rc_prr_delivered > rack->r_ctl.rc_prr_out) 9376 limit = (rack->r_ctl.rc_prr_delivered - rack->r_ctl.rc_prr_out); 9377 else 9378 limit = 0; 9379 if (changed > limit) 9380 limit = changed; 9381 limit += ctf_fixed_maxseg(tp); 9382 if (tp->snd_ssthresh > pipe) { 9383 rack->r_ctl.rc_prr_sndcnt = min((tp->snd_ssthresh - pipe), limit); 9384 rack_log_to_prr(rack, 11, 0, __LINE__); 9385 } else { 9386 rack->r_ctl.rc_prr_sndcnt = min(0, limit); 9387 rack_log_to_prr(rack, 12, 0, __LINE__); 9388 } 9389 } 9390 } 9391 9392 static void 9393 rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered_recovery, int dup_ack_struck) 9394 { 9395 uint32_t changed; 9396 struct tcp_rack *rack; 9397 struct rack_sendmap *rsm; 9398 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1]; 9399 register uint32_t th_ack; 9400 int32_t i, j, k, num_sack_blks = 0; 9401 uint32_t cts, acked, ack_point; 9402 int loop_start = 0, moved_two = 0; 9403 uint32_t tsused; 9404 9405 9406 INP_WLOCK_ASSERT(tptoinpcb(tp)); 9407 if (tcp_get_flags(th) & TH_RST) { 9408 /* We don't log resets */ 9409 return; 9410 } 9411 rack = (struct tcp_rack *)tp->t_fb_ptr; 9412 cts = tcp_get_usecs(NULL); 9413 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 9414 changed = 0; 9415 th_ack = th->th_ack; 9416 if (rack->sack_attack_disable == 0) 9417 rack_do_decay(rack); 9418 if (BYTES_THIS_ACK(tp, th) >= ctf_fixed_maxseg(rack->rc_tp)) { 9419 /* 9420 * You only get credit for 9421 * MSS and greater (and you get extra 9422 * credit for larger cum-ack moves). 9423 */ 9424 int ac; 9425 9426 ac = BYTES_THIS_ACK(tp, th) / ctf_fixed_maxseg(rack->rc_tp); 9427 rack->r_ctl.ack_count += ac; 9428 counter_u64_add(rack_ack_total, ac); 9429 } 9430 if (rack->r_ctl.ack_count > 0xfff00000) { 9431 /* 9432 * reduce the number to keep us under 9433 * a uint32_t. 9434 */ 9435 rack->r_ctl.ack_count /= 2; 9436 rack->r_ctl.sack_count /= 2; 9437 } 9438 if (SEQ_GT(th_ack, tp->snd_una)) { 9439 rack_log_progress_event(rack, tp, ticks, PROGRESS_UPDATE, __LINE__); 9440 tp->t_acktime = ticks; 9441 } 9442 if (rsm && SEQ_GT(th_ack, rsm->r_start)) 9443 changed = th_ack - rsm->r_start; 9444 if (changed) { 9445 rack_process_to_cumack(tp, rack, th_ack, cts, to); 9446 } 9447 if ((to->to_flags & TOF_SACK) == 0) { 9448 /* We are done nothing left and no sack. */ 9449 rack_handle_might_revert(tp, rack); 9450 /* 9451 * For cases where we struck a dup-ack 9452 * with no SACK, add to the changes so 9453 * PRR will work right. 9454 */ 9455 if (dup_ack_struck && (changed == 0)) { 9456 changed += ctf_fixed_maxseg(rack->rc_tp); 9457 } 9458 goto out; 9459 } 9460 /* Sack block processing */ 9461 if (SEQ_GT(th_ack, tp->snd_una)) 9462 ack_point = th_ack; 9463 else 9464 ack_point = tp->snd_una; 9465 for (i = 0; i < to->to_nsacks; i++) { 9466 bcopy((to->to_sacks + i * TCPOLEN_SACK), 9467 &sack, sizeof(sack)); 9468 sack.start = ntohl(sack.start); 9469 sack.end = ntohl(sack.end); 9470 if (SEQ_GT(sack.end, sack.start) && 9471 SEQ_GT(sack.start, ack_point) && 9472 SEQ_LT(sack.start, tp->snd_max) && 9473 SEQ_GT(sack.end, ack_point) && 9474 SEQ_LEQ(sack.end, tp->snd_max)) { 9475 sack_blocks[num_sack_blks] = sack; 9476 num_sack_blks++; 9477 } else if (SEQ_LEQ(sack.start, th_ack) && 9478 SEQ_LEQ(sack.end, th_ack)) { 9479 int was_tlp; 9480 9481 was_tlp = rack_note_dsack(rack, sack.start, sack.end); 9482 /* 9483 * Its a D-SACK block. 9484 */ 9485 tcp_record_dsack(tp, sack.start, sack.end, was_tlp); 9486 } 9487 } 9488 if (rack->rc_dsack_round_seen) { 9489 /* Is the dsack roound over? */ 9490 if (SEQ_GEQ(th_ack, rack->r_ctl.dsack_round_end)) { 9491 /* Yes it is */ 9492 rack->rc_dsack_round_seen = 0; 9493 rack_log_dsack_event(rack, 3, __LINE__, 0, 0); 9494 } 9495 } 9496 /* 9497 * Sort the SACK blocks so we can update the rack scoreboard with 9498 * just one pass. 9499 */ 9500 num_sack_blks = sack_filter_blks(&rack->r_ctl.rack_sf, sack_blocks, 9501 num_sack_blks, th->th_ack); 9502 ctf_log_sack_filter(rack->rc_tp, num_sack_blks, sack_blocks); 9503 if (num_sack_blks == 0) { 9504 /* Nothing to sack (DSACKs?) */ 9505 goto out_with_totals; 9506 } 9507 if (num_sack_blks < 2) { 9508 /* Only one, we don't need to sort */ 9509 goto do_sack_work; 9510 } 9511 /* Sort the sacks */ 9512 for (i = 0; i < num_sack_blks; i++) { 9513 for (j = i + 1; j < num_sack_blks; j++) { 9514 if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) { 9515 sack = sack_blocks[i]; 9516 sack_blocks[i] = sack_blocks[j]; 9517 sack_blocks[j] = sack; 9518 } 9519 } 9520 } 9521 /* 9522 * Now are any of the sack block ends the same (yes some 9523 * implementations send these)? 9524 */ 9525 again: 9526 if (num_sack_blks == 0) 9527 goto out_with_totals; 9528 if (num_sack_blks > 1) { 9529 for (i = 0; i < num_sack_blks; i++) { 9530 for (j = i + 1; j < num_sack_blks; j++) { 9531 if (sack_blocks[i].end == sack_blocks[j].end) { 9532 /* 9533 * Ok these two have the same end we 9534 * want the smallest end and then 9535 * throw away the larger and start 9536 * again. 9537 */ 9538 if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) { 9539 /* 9540 * The second block covers 9541 * more area use that 9542 */ 9543 sack_blocks[i].start = sack_blocks[j].start; 9544 } 9545 /* 9546 * Now collapse out the dup-sack and 9547 * lower the count 9548 */ 9549 for (k = (j + 1); k < num_sack_blks; k++) { 9550 sack_blocks[j].start = sack_blocks[k].start; 9551 sack_blocks[j].end = sack_blocks[k].end; 9552 j++; 9553 } 9554 num_sack_blks--; 9555 goto again; 9556 } 9557 } 9558 } 9559 } 9560 do_sack_work: 9561 /* 9562 * First lets look to see if 9563 * we have retransmitted and 9564 * can use the transmit next? 9565 */ 9566 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 9567 if (rsm && 9568 SEQ_GT(sack_blocks[0].end, rsm->r_start) && 9569 SEQ_LT(sack_blocks[0].start, rsm->r_end)) { 9570 /* 9571 * We probably did the FR and the next 9572 * SACK in continues as we would expect. 9573 */ 9574 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[0], to, &rsm, cts, &moved_two); 9575 if (acked) { 9576 rack->r_wanted_output = 1; 9577 changed += acked; 9578 } 9579 if (num_sack_blks == 1) { 9580 /* 9581 * This is what we would expect from 9582 * a normal implementation to happen 9583 * after we have retransmitted the FR, 9584 * i.e the sack-filter pushes down 9585 * to 1 block and the next to be retransmitted 9586 * is the sequence in the sack block (has more 9587 * are acked). Count this as ACK'd data to boost 9588 * up the chances of recovering any false positives. 9589 */ 9590 rack->r_ctl.ack_count += (acked / ctf_fixed_maxseg(rack->rc_tp)); 9591 counter_u64_add(rack_ack_total, (acked / ctf_fixed_maxseg(rack->rc_tp))); 9592 counter_u64_add(rack_express_sack, 1); 9593 if (rack->r_ctl.ack_count > 0xfff00000) { 9594 /* 9595 * reduce the number to keep us under 9596 * a uint32_t. 9597 */ 9598 rack->r_ctl.ack_count /= 2; 9599 rack->r_ctl.sack_count /= 2; 9600 } 9601 goto out_with_totals; 9602 } else { 9603 /* 9604 * Start the loop through the 9605 * rest of blocks, past the first block. 9606 */ 9607 moved_two = 0; 9608 loop_start = 1; 9609 } 9610 } 9611 /* Its a sack of some sort */ 9612 rack->r_ctl.sack_count++; 9613 if (rack->r_ctl.sack_count > 0xfff00000) { 9614 /* 9615 * reduce the number to keep us under 9616 * a uint32_t. 9617 */ 9618 rack->r_ctl.ack_count /= 2; 9619 rack->r_ctl.sack_count /= 2; 9620 } 9621 counter_u64_add(rack_sack_total, 1); 9622 if (rack->sack_attack_disable) { 9623 /* An attacker disablement is in place */ 9624 if (num_sack_blks > 1) { 9625 rack->r_ctl.sack_count += (num_sack_blks - 1); 9626 rack->r_ctl.sack_moved_extra++; 9627 counter_u64_add(rack_move_some, 1); 9628 if (rack->r_ctl.sack_moved_extra > 0xfff00000) { 9629 rack->r_ctl.sack_moved_extra /= 2; 9630 rack->r_ctl.sack_noextra_move /= 2; 9631 } 9632 } 9633 goto out; 9634 } 9635 rsm = rack->r_ctl.rc_sacklast; 9636 for (i = loop_start; i < num_sack_blks; i++) { 9637 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[i], to, &rsm, cts, &moved_two); 9638 if (acked) { 9639 rack->r_wanted_output = 1; 9640 changed += acked; 9641 } 9642 if (moved_two) { 9643 /* 9644 * If we did not get a SACK for at least a MSS and 9645 * had to move at all, or if we moved more than our 9646 * threshold, it counts against the "extra" move. 9647 */ 9648 rack->r_ctl.sack_moved_extra += moved_two; 9649 counter_u64_add(rack_move_some, 1); 9650 } else { 9651 /* 9652 * else we did not have to move 9653 * any more than we would expect. 9654 */ 9655 rack->r_ctl.sack_noextra_move++; 9656 counter_u64_add(rack_move_none, 1); 9657 } 9658 if (moved_two && (acked < ctf_fixed_maxseg(rack->rc_tp))) { 9659 /* 9660 * If the SACK was not a full MSS then 9661 * we add to sack_count the number of 9662 * MSS's (or possibly more than 9663 * a MSS if its a TSO send) we had to skip by. 9664 */ 9665 rack->r_ctl.sack_count += moved_two; 9666 counter_u64_add(rack_sack_total, moved_two); 9667 } 9668 /* 9669 * Now we need to setup for the next 9670 * round. First we make sure we won't 9671 * exceed the size of our uint32_t on 9672 * the various counts, and then clear out 9673 * moved_two. 9674 */ 9675 if ((rack->r_ctl.sack_moved_extra > 0xfff00000) || 9676 (rack->r_ctl.sack_noextra_move > 0xfff00000)) { 9677 rack->r_ctl.sack_moved_extra /= 2; 9678 rack->r_ctl.sack_noextra_move /= 2; 9679 } 9680 if (rack->r_ctl.sack_count > 0xfff00000) { 9681 rack->r_ctl.ack_count /= 2; 9682 rack->r_ctl.sack_count /= 2; 9683 } 9684 moved_two = 0; 9685 } 9686 out_with_totals: 9687 if (num_sack_blks > 1) { 9688 /* 9689 * You get an extra stroke if 9690 * you have more than one sack-blk, this 9691 * could be where we are skipping forward 9692 * and the sack-filter is still working, or 9693 * it could be an attacker constantly 9694 * moving us. 9695 */ 9696 rack->r_ctl.sack_moved_extra++; 9697 counter_u64_add(rack_move_some, 1); 9698 } 9699 out: 9700 #ifdef NETFLIX_EXP_DETECTION 9701 rack_do_detection(tp, rack, BYTES_THIS_ACK(tp, th), ctf_fixed_maxseg(rack->rc_tp)); 9702 #endif 9703 if (changed) { 9704 /* Something changed cancel the rack timer */ 9705 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 9706 } 9707 tsused = tcp_get_usecs(NULL); 9708 rsm = tcp_rack_output(tp, rack, tsused); 9709 if ((!IN_FASTRECOVERY(tp->t_flags)) && 9710 rsm && 9711 ((rsm->r_flags & RACK_MUST_RXT) == 0)) { 9712 /* Enter recovery */ 9713 entered_recovery = 1; 9714 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__); 9715 /* 9716 * When we enter recovery we need to assure we send 9717 * one packet. 9718 */ 9719 if (rack->rack_no_prr == 0) { 9720 rack->r_ctl.rc_prr_sndcnt = ctf_fixed_maxseg(tp); 9721 rack_log_to_prr(rack, 8, 0, __LINE__); 9722 } 9723 rack->r_timer_override = 1; 9724 rack->r_early = 0; 9725 rack->r_ctl.rc_agg_early = 0; 9726 } else if (IN_FASTRECOVERY(tp->t_flags) && 9727 rsm && 9728 (rack->r_rr_config == 3)) { 9729 /* 9730 * Assure we can output and we get no 9731 * remembered pace time except the retransmit. 9732 */ 9733 rack->r_timer_override = 1; 9734 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT; 9735 rack->r_ctl.rc_resend = rsm; 9736 } 9737 if (IN_FASTRECOVERY(tp->t_flags) && 9738 (rack->rack_no_prr == 0) && 9739 (entered_recovery == 0)) { 9740 rack_update_prr(tp, rack, changed, th_ack); 9741 if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) && 9742 ((tcp_in_hpts(rack->rc_inp) == 0) && 9743 ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) { 9744 /* 9745 * If you are pacing output you don't want 9746 * to override. 9747 */ 9748 rack->r_early = 0; 9749 rack->r_ctl.rc_agg_early = 0; 9750 rack->r_timer_override = 1; 9751 } 9752 } 9753 } 9754 9755 static void 9756 rack_strike_dupack(struct tcp_rack *rack) 9757 { 9758 struct rack_sendmap *rsm; 9759 9760 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 9761 while (rsm && (rsm->r_dupack >= DUP_ACK_THRESHOLD)) { 9762 rsm = TAILQ_NEXT(rsm, r_tnext); 9763 if (rsm->r_flags & RACK_MUST_RXT) { 9764 /* Sendmap entries that are marked to 9765 * be retransmitted do not need dupack's 9766 * struck. We get these marks for a number 9767 * of reasons (rxt timeout with no sack, 9768 * mtu change, or rwnd collapses). When 9769 * these events occur, we know we must retransmit 9770 * them and mark the sendmap entries. Dupack counting 9771 * is not needed since we are already set to retransmit 9772 * it as soon as we can. 9773 */ 9774 continue; 9775 } 9776 } 9777 if (rsm && (rsm->r_dupack < 0xff)) { 9778 rsm->r_dupack++; 9779 if (rsm->r_dupack >= DUP_ACK_THRESHOLD) { 9780 struct timeval tv; 9781 uint32_t cts; 9782 /* 9783 * Here we see if we need to retransmit. For 9784 * a SACK type connection if enough time has passed 9785 * we will get a return of the rsm. For a non-sack 9786 * connection we will get the rsm returned if the 9787 * dupack value is 3 or more. 9788 */ 9789 cts = tcp_get_usecs(&tv); 9790 rack->r_ctl.rc_resend = tcp_rack_output(rack->rc_tp, rack, cts); 9791 if (rack->r_ctl.rc_resend != NULL) { 9792 if (!IN_FASTRECOVERY(rack->rc_tp->t_flags)) { 9793 rack_cong_signal(rack->rc_tp, CC_NDUPACK, 9794 rack->rc_tp->snd_una, __LINE__); 9795 } 9796 rack->r_wanted_output = 1; 9797 rack->r_timer_override = 1; 9798 rack_log_retran_reason(rack, rsm, __LINE__, 1, 3); 9799 } 9800 } else { 9801 rack_log_retran_reason(rack, rsm, __LINE__, 0, 3); 9802 } 9803 } 9804 } 9805 9806 static void 9807 rack_check_bottom_drag(struct tcpcb *tp, 9808 struct tcp_rack *rack, 9809 struct socket *so, int32_t acked) 9810 { 9811 uint32_t segsiz, minseg; 9812 9813 segsiz = ctf_fixed_maxseg(tp); 9814 minseg = segsiz; 9815 9816 if (tp->snd_max == tp->snd_una) { 9817 /* 9818 * We are doing dynamic pacing and we are way 9819 * under. Basically everything got acked while 9820 * we were still waiting on the pacer to expire. 9821 * 9822 * This means we need to boost the b/w in 9823 * addition to any earlier boosting of 9824 * the multiplier. 9825 */ 9826 rack->rc_dragged_bottom = 1; 9827 rack_validate_multipliers_at_or_above100(rack); 9828 /* 9829 * Lets use the segment bytes acked plus 9830 * the lowest RTT seen as the basis to 9831 * form a b/w estimate. This will be off 9832 * due to the fact that the true estimate 9833 * should be around 1/2 the time of the RTT 9834 * but we can settle for that. 9835 */ 9836 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_VALID) && 9837 acked) { 9838 uint64_t bw, calc_bw, rtt; 9839 9840 rtt = rack->r_ctl.rack_rs.rs_us_rtt; 9841 if (rtt == 0) { 9842 /* no us sample is there a ms one? */ 9843 if (rack->r_ctl.rack_rs.rs_rtt_lowest) { 9844 rtt = rack->r_ctl.rack_rs.rs_rtt_lowest; 9845 } else { 9846 goto no_measurement; 9847 } 9848 } 9849 bw = acked; 9850 calc_bw = bw * 1000000; 9851 calc_bw /= rtt; 9852 if (rack->r_ctl.last_max_bw && 9853 (rack->r_ctl.last_max_bw < calc_bw)) { 9854 /* 9855 * If we have a last calculated max bw 9856 * enforce it. 9857 */ 9858 calc_bw = rack->r_ctl.last_max_bw; 9859 } 9860 /* now plop it in */ 9861 if (rack->rc_gp_filled == 0) { 9862 if (calc_bw > ONE_POINT_TWO_MEG) { 9863 /* 9864 * If we have no measurement 9865 * don't let us set in more than 9866 * 1.2Mbps. If we are still too 9867 * low after pacing with this we 9868 * will hopefully have a max b/w 9869 * available to sanity check things. 9870 */ 9871 calc_bw = ONE_POINT_TWO_MEG; 9872 } 9873 rack->r_ctl.rc_rtt_diff = 0; 9874 rack->r_ctl.gp_bw = calc_bw; 9875 rack->rc_gp_filled = 1; 9876 if (rack->r_ctl.num_measurements < RACK_REQ_AVG) 9877 rack->r_ctl.num_measurements = RACK_REQ_AVG; 9878 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL); 9879 } else if (calc_bw > rack->r_ctl.gp_bw) { 9880 rack->r_ctl.rc_rtt_diff = 0; 9881 if (rack->r_ctl.num_measurements < RACK_REQ_AVG) 9882 rack->r_ctl.num_measurements = RACK_REQ_AVG; 9883 rack->r_ctl.gp_bw = calc_bw; 9884 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL); 9885 } else 9886 rack_increase_bw_mul(rack, -1, 0, 0, 1); 9887 if ((rack->gp_ready == 0) && 9888 (rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) { 9889 /* We have enough measurements now */ 9890 rack->gp_ready = 1; 9891 rack_set_cc_pacing(rack); 9892 if (rack->defer_options) 9893 rack_apply_deferred_options(rack); 9894 } 9895 /* 9896 * For acks over 1mss we do a extra boost to simulate 9897 * where we would get 2 acks (we want 110 for the mul). 9898 */ 9899 if (acked > segsiz) 9900 rack_increase_bw_mul(rack, -1, 0, 0, 1); 9901 } else { 9902 /* 9903 * zero rtt possibly?, settle for just an old increase. 9904 */ 9905 no_measurement: 9906 rack_increase_bw_mul(rack, -1, 0, 0, 1); 9907 } 9908 } else if ((IN_FASTRECOVERY(tp->t_flags) == 0) && 9909 (sbavail(&so->so_snd) > max((segsiz * (4 + rack_req_segs)), 9910 minseg)) && 9911 (rack->r_ctl.cwnd_to_use > max((segsiz * (rack_req_segs + 2)), minseg)) && 9912 (tp->snd_wnd > max((segsiz * (rack_req_segs + 2)), minseg)) && 9913 (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) <= 9914 (segsiz * rack_req_segs))) { 9915 /* 9916 * We are doing dynamic GP pacing and 9917 * we have everything except 1MSS or less 9918 * bytes left out. We are still pacing away. 9919 * And there is data that could be sent, This 9920 * means we are inserting delayed ack time in 9921 * our measurements because we are pacing too slow. 9922 */ 9923 rack_validate_multipliers_at_or_above100(rack); 9924 rack->rc_dragged_bottom = 1; 9925 rack_increase_bw_mul(rack, -1, 0, 0, 1); 9926 } 9927 } 9928 9929 9930 9931 static void 9932 rack_gain_for_fastoutput(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t acked_amount) 9933 { 9934 /* 9935 * The fast output path is enabled and we 9936 * have moved the cumack forward. Lets see if 9937 * we can expand forward the fast path length by 9938 * that amount. What we would ideally like to 9939 * do is increase the number of bytes in the 9940 * fast path block (left_to_send) by the 9941 * acked amount. However we have to gate that 9942 * by two factors: 9943 * 1) The amount outstanding and the rwnd of the peer 9944 * (i.e. we don't want to exceed the rwnd of the peer). 9945 * <and> 9946 * 2) The amount of data left in the socket buffer (i.e. 9947 * we can't send beyond what is in the buffer). 9948 * 9949 * Note that this does not take into account any increase 9950 * in the cwnd. We will only extend the fast path by 9951 * what was acked. 9952 */ 9953 uint32_t new_total, gating_val; 9954 9955 new_total = acked_amount + rack->r_ctl.fsb.left_to_send; 9956 gating_val = min((sbavail(&so->so_snd) - (tp->snd_max - tp->snd_una)), 9957 (tp->snd_wnd - (tp->snd_max - tp->snd_una))); 9958 if (new_total <= gating_val) { 9959 /* We can increase left_to_send by the acked amount */ 9960 counter_u64_add(rack_extended_rfo, 1); 9961 rack->r_ctl.fsb.left_to_send = new_total; 9962 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(&rack->rc_inp->inp_socket->so_snd) - (tp->snd_max - tp->snd_una))), 9963 ("rack:%p left_to_send:%u sbavail:%u out:%u", 9964 rack, rack->r_ctl.fsb.left_to_send, 9965 sbavail(&rack->rc_inp->inp_socket->so_snd), 9966 (tp->snd_max - tp->snd_una))); 9967 9968 } 9969 } 9970 9971 static void 9972 rack_adjust_sendmap(struct tcp_rack *rack, struct sockbuf *sb, tcp_seq snd_una) 9973 { 9974 /* 9975 * Here any sendmap entry that points to the 9976 * beginning mbuf must be adjusted to the correct 9977 * offset. This must be called with: 9978 * 1) The socket buffer locked 9979 * 2) snd_una adjusted to its new position. 9980 * 9981 * Note that (2) implies rack_ack_received has also 9982 * been called. 9983 * 9984 * We grab the first mbuf in the socket buffer and 9985 * then go through the front of the sendmap, recalculating 9986 * the stored offset for any sendmap entry that has 9987 * that mbuf. We must use the sb functions to do this 9988 * since its possible an add was done has well as 9989 * the subtraction we may have just completed. This should 9990 * not be a penalty though, since we just referenced the sb 9991 * to go in and trim off the mbufs that we freed (of course 9992 * there will be a penalty for the sendmap references though). 9993 */ 9994 struct mbuf *m; 9995 struct rack_sendmap *rsm; 9996 9997 SOCKBUF_LOCK_ASSERT(sb); 9998 m = sb->sb_mb; 9999 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 10000 if ((rsm == NULL) || (m == NULL)) { 10001 /* Nothing outstanding */ 10002 return; 10003 } 10004 while (rsm->m && (rsm->m == m)) { 10005 /* one to adjust */ 10006 #ifdef INVARIANTS 10007 struct mbuf *tm; 10008 uint32_t soff; 10009 10010 tm = sbsndmbuf(sb, (rsm->r_start - snd_una), &soff); 10011 if (rsm->orig_m_len != m->m_len) { 10012 rack_adjust_orig_mlen(rsm); 10013 } 10014 if (rsm->soff != soff) { 10015 /* 10016 * This is not a fatal error, we anticipate it 10017 * might happen (the else code), so we count it here 10018 * so that under invariant we can see that it really 10019 * does happen. 10020 */ 10021 counter_u64_add(rack_adjust_map_bw, 1); 10022 } 10023 rsm->m = tm; 10024 rsm->soff = soff; 10025 if (tm) 10026 rsm->orig_m_len = rsm->m->m_len; 10027 else 10028 rsm->orig_m_len = 0; 10029 #else 10030 rsm->m = sbsndmbuf(sb, (rsm->r_start - snd_una), &rsm->soff); 10031 if (rsm->m) 10032 rsm->orig_m_len = rsm->m->m_len; 10033 else 10034 rsm->orig_m_len = 0; 10035 #endif 10036 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, 10037 rsm); 10038 if (rsm == NULL) 10039 break; 10040 } 10041 } 10042 10043 /* 10044 * Return value of 1, we do not need to call rack_process_data(). 10045 * return value of 0, rack_process_data can be called. 10046 * For ret_val if its 0 the TCP is locked, if its non-zero 10047 * its unlocked and probably unsafe to touch the TCB. 10048 */ 10049 static int 10050 rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so, 10051 struct tcpcb *tp, struct tcpopt *to, 10052 uint32_t tiwin, int32_t tlen, 10053 int32_t * ofia, int32_t thflags, int32_t *ret_val) 10054 { 10055 int32_t ourfinisacked = 0; 10056 int32_t nsegs, acked_amount; 10057 int32_t acked; 10058 struct mbuf *mfree; 10059 struct tcp_rack *rack; 10060 int32_t under_pacing = 0; 10061 int32_t recovery = 0; 10062 10063 INP_WLOCK_ASSERT(tptoinpcb(tp)); 10064 10065 rack = (struct tcp_rack *)tp->t_fb_ptr; 10066 if (SEQ_GT(th->th_ack, tp->snd_max)) { 10067 __ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val, 10068 &rack->r_ctl.challenge_ack_ts, 10069 &rack->r_ctl.challenge_ack_cnt); 10070 rack->r_wanted_output = 1; 10071 return (1); 10072 } 10073 if (rack->gp_ready && 10074 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { 10075 under_pacing = 1; 10076 } 10077 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) { 10078 int in_rec, dup_ack_struck = 0; 10079 10080 in_rec = IN_FASTRECOVERY(tp->t_flags); 10081 if (rack->rc_in_persist) { 10082 tp->t_rxtshift = 0; 10083 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 10084 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 10085 } 10086 if ((th->th_ack == tp->snd_una) && 10087 (tiwin == tp->snd_wnd) && 10088 ((to->to_flags & TOF_SACK) == 0)) { 10089 rack_strike_dupack(rack); 10090 dup_ack_struck = 1; 10091 } 10092 rack_log_ack(tp, to, th, ((in_rec == 0) && IN_FASTRECOVERY(tp->t_flags)), dup_ack_struck); 10093 } 10094 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { 10095 /* 10096 * Old ack, behind (or duplicate to) the last one rcv'd 10097 * Note: We mark reordering is occuring if its 10098 * less than and we have not closed our window. 10099 */ 10100 if (SEQ_LT(th->th_ack, tp->snd_una) && (sbspace(&so->so_rcv) > ctf_fixed_maxseg(tp))) { 10101 rack->r_ctl.rc_reorder_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 10102 } 10103 return (0); 10104 } 10105 /* 10106 * If we reach this point, ACK is not a duplicate, i.e., it ACKs 10107 * something we sent. 10108 */ 10109 if (tp->t_flags & TF_NEEDSYN) { 10110 /* 10111 * T/TCP: Connection was half-synchronized, and our SYN has 10112 * been ACK'd (so connection is now fully synchronized). Go 10113 * to non-starred state, increment snd_una for ACK of SYN, 10114 * and check if we can do window scaling. 10115 */ 10116 tp->t_flags &= ~TF_NEEDSYN; 10117 tp->snd_una++; 10118 /* Do window scaling? */ 10119 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 10120 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 10121 tp->rcv_scale = tp->request_r_scale; 10122 /* Send window already scaled. */ 10123 } 10124 } 10125 nsegs = max(1, m->m_pkthdr.lro_nsegs); 10126 10127 acked = BYTES_THIS_ACK(tp, th); 10128 if (acked) { 10129 /* 10130 * Any time we move the cum-ack forward clear 10131 * keep-alive tied probe-not-answered. The 10132 * persists clears its own on entry. 10133 */ 10134 rack->probe_not_answered = 0; 10135 } 10136 KMOD_TCPSTAT_ADD(tcps_rcvackpack, nsegs); 10137 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked); 10138 /* 10139 * If we just performed our first retransmit, and the ACK arrives 10140 * within our recovery window, then it was a mistake to do the 10141 * retransmit in the first place. Recover our original cwnd and 10142 * ssthresh, and proceed to transmit where we left off. 10143 */ 10144 if ((tp->t_flags & TF_PREVVALID) && 10145 ((tp->t_flags & TF_RCVD_TSTMP) == 0)) { 10146 tp->t_flags &= ~TF_PREVVALID; 10147 if (tp->t_rxtshift == 1 && 10148 (int)(ticks - tp->t_badrxtwin) < 0) 10149 rack_cong_signal(tp, CC_RTO_ERR, th->th_ack, __LINE__); 10150 } 10151 if (acked) { 10152 /* assure we are not backed off */ 10153 tp->t_rxtshift = 0; 10154 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 10155 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 10156 rack->rc_tlp_in_progress = 0; 10157 rack->r_ctl.rc_tlp_cnt_out = 0; 10158 /* 10159 * If it is the RXT timer we want to 10160 * stop it, so we can restart a TLP. 10161 */ 10162 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) 10163 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 10164 #ifdef NETFLIX_HTTP_LOGGING 10165 tcp_http_check_for_comp(rack->rc_tp, th->th_ack); 10166 #endif 10167 } 10168 /* 10169 * If we have a timestamp reply, update smoothed round trip time. If 10170 * no timestamp is present but transmit timer is running and timed 10171 * sequence number was acked, update smoothed round trip time. Since 10172 * we now have an rtt measurement, cancel the timer backoff (cf., 10173 * Phil Karn's retransmit alg.). Recompute the initial retransmit 10174 * timer. 10175 * 10176 * Some boxes send broken timestamp replies during the SYN+ACK 10177 * phase, ignore timestamps of 0 or we could calculate a huge RTT 10178 * and blow up the retransmit timer. 10179 */ 10180 /* 10181 * If all outstanding data is acked, stop retransmit timer and 10182 * remember to restart (more output or persist). If there is more 10183 * data to be acked, restart retransmit timer, using current 10184 * (possibly backed-off) value. 10185 */ 10186 if (acked == 0) { 10187 if (ofia) 10188 *ofia = ourfinisacked; 10189 return (0); 10190 } 10191 if (IN_RECOVERY(tp->t_flags)) { 10192 if (SEQ_LT(th->th_ack, tp->snd_recover) && 10193 (SEQ_LT(th->th_ack, tp->snd_max))) { 10194 tcp_rack_partialack(tp); 10195 } else { 10196 rack_post_recovery(tp, th->th_ack); 10197 recovery = 1; 10198 } 10199 } 10200 /* 10201 * Let the congestion control algorithm update congestion control 10202 * related information. This typically means increasing the 10203 * congestion window. 10204 */ 10205 rack_ack_received(tp, rack, th->th_ack, nsegs, CC_ACK, recovery); 10206 SOCKBUF_LOCK(&so->so_snd); 10207 acked_amount = min(acked, (int)sbavail(&so->so_snd)); 10208 tp->snd_wnd -= acked_amount; 10209 mfree = sbcut_locked(&so->so_snd, acked_amount); 10210 if ((sbused(&so->so_snd) == 0) && 10211 (acked > acked_amount) && 10212 (tp->t_state >= TCPS_FIN_WAIT_1) && 10213 (tp->t_flags & TF_SENTFIN)) { 10214 /* 10215 * We must be sure our fin 10216 * was sent and acked (we can be 10217 * in FIN_WAIT_1 without having 10218 * sent the fin). 10219 */ 10220 ourfinisacked = 1; 10221 } 10222 tp->snd_una = th->th_ack; 10223 if (acked_amount && sbavail(&so->so_snd)) 10224 rack_adjust_sendmap(rack, &so->so_snd, tp->snd_una); 10225 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2); 10226 /* NB: sowwakeup_locked() does an implicit unlock. */ 10227 sowwakeup_locked(so); 10228 m_freem(mfree); 10229 if (SEQ_GT(tp->snd_una, tp->snd_recover)) 10230 tp->snd_recover = tp->snd_una; 10231 10232 if (SEQ_LT(tp->snd_nxt, tp->snd_una)) { 10233 tp->snd_nxt = tp->snd_una; 10234 } 10235 if (under_pacing && 10236 (rack->use_fixed_rate == 0) && 10237 (rack->in_probe_rtt == 0) && 10238 rack->rc_gp_dyn_mul && 10239 rack->rc_always_pace) { 10240 /* Check if we are dragging bottom */ 10241 rack_check_bottom_drag(tp, rack, so, acked); 10242 } 10243 if (tp->snd_una == tp->snd_max) { 10244 /* Nothing left outstanding */ 10245 tp->t_flags &= ~TF_PREVVALID; 10246 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL); 10247 rack->r_ctl.retran_during_recovery = 0; 10248 rack->r_ctl.dsack_byte_cnt = 0; 10249 if (rack->r_ctl.rc_went_idle_time == 0) 10250 rack->r_ctl.rc_went_idle_time = 1; 10251 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__); 10252 if (sbavail(&tptosocket(tp)->so_snd) == 0) 10253 tp->t_acktime = 0; 10254 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 10255 /* Set need output so persist might get set */ 10256 rack->r_wanted_output = 1; 10257 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); 10258 if ((tp->t_state >= TCPS_FIN_WAIT_1) && 10259 (sbavail(&so->so_snd) == 0) && 10260 (tp->t_flags2 & TF2_DROP_AF_DATA)) { 10261 /* 10262 * The socket was gone and the 10263 * peer sent data (now or in the past), time to 10264 * reset him. 10265 */ 10266 *ret_val = 1; 10267 /* tcp_close will kill the inp pre-log the Reset */ 10268 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 10269 tp = tcp_close(tp); 10270 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen); 10271 return (1); 10272 } 10273 } 10274 if (ofia) 10275 *ofia = ourfinisacked; 10276 return (0); 10277 } 10278 10279 10280 static void 10281 rack_log_collapse(struct tcp_rack *rack, uint32_t cnt, uint32_t split, uint32_t out, int line, 10282 int dir, uint32_t flags, struct rack_sendmap *rsm) 10283 { 10284 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 10285 union tcp_log_stackspecific log; 10286 struct timeval tv; 10287 10288 memset(&log, 0, sizeof(log)); 10289 log.u_bbr.flex1 = cnt; 10290 log.u_bbr.flex2 = split; 10291 log.u_bbr.flex3 = out; 10292 log.u_bbr.flex4 = line; 10293 log.u_bbr.flex5 = rack->r_must_retran; 10294 log.u_bbr.flex6 = flags; 10295 log.u_bbr.flex7 = rack->rc_has_collapsed; 10296 log.u_bbr.flex8 = dir; /* 10297 * 1 is collapsed, 0 is uncollapsed, 10298 * 2 is log of a rsm being marked, 3 is a split. 10299 */ 10300 if (rsm == NULL) 10301 log.u_bbr.rttProp = 0; 10302 else 10303 log.u_bbr.rttProp = (uint64_t)rsm; 10304 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 10305 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 10306 TCP_LOG_EVENTP(rack->rc_tp, NULL, 10307 &rack->rc_inp->inp_socket->so_rcv, 10308 &rack->rc_inp->inp_socket->so_snd, 10309 TCP_RACK_LOG_COLLAPSE, 0, 10310 0, &log, false, &tv); 10311 } 10312 } 10313 10314 static void 10315 rack_collapsed_window(struct tcp_rack *rack, uint32_t out, int line) 10316 { 10317 /* 10318 * Here all we do is mark the collapsed point and set the flag. 10319 * This may happen again and again, but there is no 10320 * sense splitting our map until we know where the 10321 * peer finally lands in the collapse. 10322 */ 10323 rack_trace_point(rack, RACK_TP_COLLAPSED_WND); 10324 if ((rack->rc_has_collapsed == 0) || 10325 (rack->r_ctl.last_collapse_point != (rack->rc_tp->snd_una + rack->rc_tp->snd_wnd))) 10326 counter_u64_add(rack_collapsed_win_seen, 1); 10327 rack->r_ctl.last_collapse_point = rack->rc_tp->snd_una + rack->rc_tp->snd_wnd; 10328 rack->r_ctl.high_collapse_point = rack->rc_tp->snd_max; 10329 rack->rc_has_collapsed = 1; 10330 rack->r_collapse_point_valid = 1; 10331 rack_log_collapse(rack, 0, 0, rack->r_ctl.last_collapse_point, line, 1, 0, NULL); 10332 } 10333 10334 static void 10335 rack_un_collapse_window(struct tcp_rack *rack, int line) 10336 { 10337 struct rack_sendmap *nrsm, *rsm, fe; 10338 int cnt = 0, split = 0; 10339 #ifdef INVARIANTS 10340 struct rack_sendmap *insret; 10341 #endif 10342 10343 memset(&fe, 0, sizeof(fe)); 10344 rack->rc_has_collapsed = 0; 10345 fe.r_start = rack->r_ctl.last_collapse_point; 10346 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 10347 if (rsm == NULL) { 10348 /* Nothing to do maybe the peer ack'ed it all */ 10349 rack_log_collapse(rack, 0, 0, ctf_outstanding(rack->rc_tp), line, 0, 0, NULL); 10350 return; 10351 } 10352 /* Now do we need to split this one? */ 10353 if (SEQ_GT(rack->r_ctl.last_collapse_point, rsm->r_start)) { 10354 rack_log_collapse(rack, rsm->r_start, rsm->r_end, 10355 rack->r_ctl.last_collapse_point, line, 3, rsm->r_flags, rsm); 10356 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT); 10357 if (nrsm == NULL) { 10358 /* We can't get a rsm, mark all? */ 10359 nrsm = rsm; 10360 goto no_split; 10361 } 10362 /* Clone it */ 10363 split = 1; 10364 rack_clone_rsm(rack, nrsm, rsm, rack->r_ctl.last_collapse_point); 10365 #ifndef INVARIANTS 10366 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 10367 #else 10368 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm); 10369 if (insret != NULL) { 10370 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p", 10371 nrsm, insret, rack, rsm); 10372 } 10373 #endif 10374 rack_log_map_chg(rack->rc_tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 10375 rack->r_ctl.last_collapse_point, __LINE__); 10376 if (rsm->r_in_tmap) { 10377 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext); 10378 nrsm->r_in_tmap = 1; 10379 } 10380 /* 10381 * Set in the new RSM as the 10382 * collapsed starting point 10383 */ 10384 rsm = nrsm; 10385 } 10386 no_split: 10387 RB_FOREACH_FROM(nrsm, rack_rb_tree_head, rsm) { 10388 nrsm->r_flags |= RACK_RWND_COLLAPSED; 10389 rack_log_collapse(rack, nrsm->r_start, nrsm->r_end, 0, line, 4, nrsm->r_flags, nrsm); 10390 cnt++; 10391 } 10392 if (cnt) { 10393 counter_u64_add(rack_collapsed_win, 1); 10394 } 10395 rack_log_collapse(rack, cnt, split, ctf_outstanding(rack->rc_tp), line, 0, 0, NULL); 10396 } 10397 10398 static void 10399 rack_handle_delayed_ack(struct tcpcb *tp, struct tcp_rack *rack, 10400 int32_t tlen, int32_t tfo_syn) 10401 { 10402 if (DELAY_ACK(tp, tlen) || tfo_syn) { 10403 if (rack->rc_dack_mode && 10404 (tlen > 500) && 10405 (rack->rc_dack_toggle == 1)) { 10406 goto no_delayed_ack; 10407 } 10408 rack_timer_cancel(tp, rack, 10409 rack->r_ctl.rc_rcvtime, __LINE__); 10410 tp->t_flags |= TF_DELACK; 10411 } else { 10412 no_delayed_ack: 10413 rack->r_wanted_output = 1; 10414 tp->t_flags |= TF_ACKNOW; 10415 if (rack->rc_dack_mode) { 10416 if (tp->t_flags & TF_DELACK) 10417 rack->rc_dack_toggle = 1; 10418 else 10419 rack->rc_dack_toggle = 0; 10420 } 10421 } 10422 } 10423 10424 static void 10425 rack_validate_fo_sendwin_up(struct tcpcb *tp, struct tcp_rack *rack) 10426 { 10427 /* 10428 * If fast output is in progress, lets validate that 10429 * the new window did not shrink on us and make it 10430 * so fast output should end. 10431 */ 10432 if (rack->r_fast_output) { 10433 uint32_t out; 10434 10435 /* 10436 * Calculate what we will send if left as is 10437 * and compare that to our send window. 10438 */ 10439 out = ctf_outstanding(tp); 10440 if ((out + rack->r_ctl.fsb.left_to_send) > tp->snd_wnd) { 10441 /* ok we have an issue */ 10442 if (out >= tp->snd_wnd) { 10443 /* Turn off fast output the window is met or collapsed */ 10444 rack->r_fast_output = 0; 10445 } else { 10446 /* we have some room left */ 10447 rack->r_ctl.fsb.left_to_send = tp->snd_wnd - out; 10448 if (rack->r_ctl.fsb.left_to_send < ctf_fixed_maxseg(tp)) { 10449 /* If not at least 1 full segment never mind */ 10450 rack->r_fast_output = 0; 10451 } 10452 } 10453 } 10454 } 10455 } 10456 10457 10458 /* 10459 * Return value of 1, the TCB is unlocked and most 10460 * likely gone, return value of 0, the TCP is still 10461 * locked. 10462 */ 10463 static int 10464 rack_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so, 10465 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, 10466 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt) 10467 { 10468 /* 10469 * Update window information. Don't look at window if no ACK: TAC's 10470 * send garbage on first SYN. 10471 */ 10472 int32_t nsegs; 10473 int32_t tfo_syn; 10474 struct tcp_rack *rack; 10475 10476 INP_WLOCK_ASSERT(tptoinpcb(tp)); 10477 10478 rack = (struct tcp_rack *)tp->t_fb_ptr; 10479 nsegs = max(1, m->m_pkthdr.lro_nsegs); 10480 if ((thflags & TH_ACK) && 10481 (SEQ_LT(tp->snd_wl1, th->th_seq) || 10482 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) || 10483 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) { 10484 /* keep track of pure window updates */ 10485 if (tlen == 0 && 10486 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd) 10487 KMOD_TCPSTAT_INC(tcps_rcvwinupd); 10488 tp->snd_wnd = tiwin; 10489 rack_validate_fo_sendwin_up(tp, rack); 10490 tp->snd_wl1 = th->th_seq; 10491 tp->snd_wl2 = th->th_ack; 10492 if (tp->snd_wnd > tp->max_sndwnd) 10493 tp->max_sndwnd = tp->snd_wnd; 10494 rack->r_wanted_output = 1; 10495 } else if (thflags & TH_ACK) { 10496 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) { 10497 tp->snd_wnd = tiwin; 10498 rack_validate_fo_sendwin_up(tp, rack); 10499 tp->snd_wl1 = th->th_seq; 10500 tp->snd_wl2 = th->th_ack; 10501 } 10502 } 10503 if (tp->snd_wnd < ctf_outstanding(tp)) 10504 /* The peer collapsed the window */ 10505 rack_collapsed_window(rack, ctf_outstanding(tp), __LINE__); 10506 else if (rack->rc_has_collapsed) 10507 rack_un_collapse_window(rack, __LINE__); 10508 if ((rack->r_collapse_point_valid) && 10509 (SEQ_GT(th->th_ack, rack->r_ctl.high_collapse_point))) 10510 rack->r_collapse_point_valid = 0; 10511 /* Was persist timer active and now we have window space? */ 10512 if ((rack->rc_in_persist != 0) && 10513 (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2), 10514 rack->r_ctl.rc_pace_min_segs))) { 10515 rack_exit_persist(tp, rack, rack->r_ctl.rc_rcvtime); 10516 tp->snd_nxt = tp->snd_max; 10517 /* Make sure we output to start the timer */ 10518 rack->r_wanted_output = 1; 10519 } 10520 /* Do we enter persists? */ 10521 if ((rack->rc_in_persist == 0) && 10522 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) && 10523 TCPS_HAVEESTABLISHED(tp->t_state) && 10524 ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) && 10525 sbavail(&tptosocket(tp)->so_snd) && 10526 (sbavail(&tptosocket(tp)->so_snd) > tp->snd_wnd)) { 10527 /* 10528 * Here the rwnd is less than 10529 * the pacing size, we are established, 10530 * nothing is outstanding, and there is 10531 * data to send. Enter persists. 10532 */ 10533 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime); 10534 } 10535 if (tp->t_flags2 & TF2_DROP_AF_DATA) { 10536 m_freem(m); 10537 return (0); 10538 } 10539 /* 10540 * don't process the URG bit, ignore them drag 10541 * along the up. 10542 */ 10543 tp->rcv_up = tp->rcv_nxt; 10544 10545 /* 10546 * Process the segment text, merging it into the TCP sequencing 10547 * queue, and arranging for acknowledgment of receipt if necessary. 10548 * This process logically involves adjusting tp->rcv_wnd as data is 10549 * presented to the user (this happens in tcp_usrreq.c, case 10550 * PRU_RCVD). If a FIN has already been received on this connection 10551 * then we just ignore the text. 10552 */ 10553 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) && 10554 IS_FASTOPEN(tp->t_flags)); 10555 if ((tlen || (thflags & TH_FIN) || (tfo_syn && tlen > 0)) && 10556 TCPS_HAVERCVDFIN(tp->t_state) == 0) { 10557 tcp_seq save_start = th->th_seq; 10558 tcp_seq save_rnxt = tp->rcv_nxt; 10559 int save_tlen = tlen; 10560 10561 m_adj(m, drop_hdrlen); /* delayed header drop */ 10562 /* 10563 * Insert segment which includes th into TCP reassembly 10564 * queue with control block tp. Set thflags to whether 10565 * reassembly now includes a segment with FIN. This handles 10566 * the common case inline (segment is the next to be 10567 * received on an established connection, and the queue is 10568 * empty), avoiding linkage into and removal from the queue 10569 * and repetition of various conversions. Set DELACK for 10570 * segments received in order, but ack immediately when 10571 * segments are out of order (so fast retransmit can work). 10572 */ 10573 if (th->th_seq == tp->rcv_nxt && 10574 SEGQ_EMPTY(tp) && 10575 (TCPS_HAVEESTABLISHED(tp->t_state) || 10576 tfo_syn)) { 10577 #ifdef NETFLIX_SB_LIMITS 10578 u_int mcnt, appended; 10579 10580 if (so->so_rcv.sb_shlim) { 10581 mcnt = m_memcnt(m); 10582 appended = 0; 10583 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, 10584 CFO_NOSLEEP, NULL) == false) { 10585 counter_u64_add(tcp_sb_shlim_fails, 1); 10586 m_freem(m); 10587 return (0); 10588 } 10589 } 10590 #endif 10591 rack_handle_delayed_ack(tp, rack, tlen, tfo_syn); 10592 tp->rcv_nxt += tlen; 10593 if (tlen && 10594 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && 10595 (tp->t_fbyte_in == 0)) { 10596 tp->t_fbyte_in = ticks; 10597 if (tp->t_fbyte_in == 0) 10598 tp->t_fbyte_in = 1; 10599 if (tp->t_fbyte_out && tp->t_fbyte_in) 10600 tp->t_flags2 |= TF2_FBYTES_COMPLETE; 10601 } 10602 thflags = tcp_get_flags(th) & TH_FIN; 10603 KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs); 10604 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); 10605 SOCKBUF_LOCK(&so->so_rcv); 10606 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 10607 m_freem(m); 10608 } else 10609 #ifdef NETFLIX_SB_LIMITS 10610 appended = 10611 #endif 10612 sbappendstream_locked(&so->so_rcv, m, 0); 10613 10614 rack_log_wakeup(tp,rack, &so->so_rcv, tlen, 1); 10615 /* NB: sorwakeup_locked() does an implicit unlock. */ 10616 sorwakeup_locked(so); 10617 #ifdef NETFLIX_SB_LIMITS 10618 if (so->so_rcv.sb_shlim && appended != mcnt) 10619 counter_fo_release(so->so_rcv.sb_shlim, 10620 mcnt - appended); 10621 #endif 10622 } else { 10623 /* 10624 * XXX: Due to the header drop above "th" is 10625 * theoretically invalid by now. Fortunately 10626 * m_adj() doesn't actually frees any mbufs when 10627 * trimming from the head. 10628 */ 10629 tcp_seq temp = save_start; 10630 10631 thflags = tcp_reass(tp, th, &temp, &tlen, m); 10632 tp->t_flags |= TF_ACKNOW; 10633 if (tp->t_flags & TF_WAKESOR) { 10634 tp->t_flags &= ~TF_WAKESOR; 10635 /* NB: sorwakeup_locked() does an implicit unlock. */ 10636 sorwakeup_locked(so); 10637 } 10638 } 10639 if ((tp->t_flags & TF_SACK_PERMIT) && 10640 (save_tlen > 0) && 10641 TCPS_HAVEESTABLISHED(tp->t_state)) { 10642 if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) { 10643 /* 10644 * DSACK actually handled in the fastpath 10645 * above. 10646 */ 10647 RACK_OPTS_INC(tcp_sack_path_1); 10648 tcp_update_sack_list(tp, save_start, 10649 save_start + save_tlen); 10650 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) { 10651 if ((tp->rcv_numsacks >= 1) && 10652 (tp->sackblks[0].end == save_start)) { 10653 /* 10654 * Partial overlap, recorded at todrop 10655 * above. 10656 */ 10657 RACK_OPTS_INC(tcp_sack_path_2a); 10658 tcp_update_sack_list(tp, 10659 tp->sackblks[0].start, 10660 tp->sackblks[0].end); 10661 } else { 10662 RACK_OPTS_INC(tcp_sack_path_2b); 10663 tcp_update_dsack_list(tp, save_start, 10664 save_start + save_tlen); 10665 } 10666 } else if (tlen >= save_tlen) { 10667 /* Update of sackblks. */ 10668 RACK_OPTS_INC(tcp_sack_path_3); 10669 tcp_update_dsack_list(tp, save_start, 10670 save_start + save_tlen); 10671 } else if (tlen > 0) { 10672 RACK_OPTS_INC(tcp_sack_path_4); 10673 tcp_update_dsack_list(tp, save_start, 10674 save_start + tlen); 10675 } 10676 } 10677 } else { 10678 m_freem(m); 10679 thflags &= ~TH_FIN; 10680 } 10681 10682 /* 10683 * If FIN is received ACK the FIN and let the user know that the 10684 * connection is closing. 10685 */ 10686 if (thflags & TH_FIN) { 10687 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { 10688 /* The socket upcall is handled by socantrcvmore. */ 10689 socantrcvmore(so); 10690 /* 10691 * If connection is half-synchronized (ie NEEDSYN 10692 * flag on) then delay ACK, so it may be piggybacked 10693 * when SYN is sent. Otherwise, since we received a 10694 * FIN then no more input can be expected, send ACK 10695 * now. 10696 */ 10697 if (tp->t_flags & TF_NEEDSYN) { 10698 rack_timer_cancel(tp, rack, 10699 rack->r_ctl.rc_rcvtime, __LINE__); 10700 tp->t_flags |= TF_DELACK; 10701 } else { 10702 tp->t_flags |= TF_ACKNOW; 10703 } 10704 tp->rcv_nxt++; 10705 } 10706 switch (tp->t_state) { 10707 /* 10708 * In SYN_RECEIVED and ESTABLISHED STATES enter the 10709 * CLOSE_WAIT state. 10710 */ 10711 case TCPS_SYN_RECEIVED: 10712 tp->t_starttime = ticks; 10713 /* FALLTHROUGH */ 10714 case TCPS_ESTABLISHED: 10715 rack_timer_cancel(tp, rack, 10716 rack->r_ctl.rc_rcvtime, __LINE__); 10717 tcp_state_change(tp, TCPS_CLOSE_WAIT); 10718 break; 10719 10720 /* 10721 * If still in FIN_WAIT_1 STATE FIN has not been 10722 * acked so enter the CLOSING state. 10723 */ 10724 case TCPS_FIN_WAIT_1: 10725 rack_timer_cancel(tp, rack, 10726 rack->r_ctl.rc_rcvtime, __LINE__); 10727 tcp_state_change(tp, TCPS_CLOSING); 10728 break; 10729 10730 /* 10731 * In FIN_WAIT_2 state enter the TIME_WAIT state, 10732 * starting the time-wait timer, turning off the 10733 * other standard timers. 10734 */ 10735 case TCPS_FIN_WAIT_2: 10736 rack_timer_cancel(tp, rack, 10737 rack->r_ctl.rc_rcvtime, __LINE__); 10738 tcp_twstart(tp); 10739 return (1); 10740 } 10741 } 10742 /* 10743 * Return any desired output. 10744 */ 10745 if ((tp->t_flags & TF_ACKNOW) || 10746 (sbavail(&so->so_snd) > (tp->snd_max - tp->snd_una))) { 10747 rack->r_wanted_output = 1; 10748 } 10749 return (0); 10750 } 10751 10752 /* 10753 * Here nothing is really faster, its just that we 10754 * have broken out the fast-data path also just like 10755 * the fast-ack. 10756 */ 10757 static int 10758 rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so, 10759 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 10760 uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos) 10761 { 10762 int32_t nsegs; 10763 int32_t newsize = 0; /* automatic sockbuf scaling */ 10764 struct tcp_rack *rack; 10765 #ifdef NETFLIX_SB_LIMITS 10766 u_int mcnt, appended; 10767 #endif 10768 10769 /* 10770 * If last ACK falls within this segment's sequence numbers, record 10771 * the timestamp. NOTE that the test is modified according to the 10772 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26). 10773 */ 10774 if (__predict_false(th->th_seq != tp->rcv_nxt)) { 10775 return (0); 10776 } 10777 if (__predict_false(tp->snd_nxt != tp->snd_max)) { 10778 return (0); 10779 } 10780 if (tiwin && tiwin != tp->snd_wnd) { 10781 return (0); 10782 } 10783 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) { 10784 return (0); 10785 } 10786 if (__predict_false((to->to_flags & TOF_TS) && 10787 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) { 10788 return (0); 10789 } 10790 if (__predict_false((th->th_ack != tp->snd_una))) { 10791 return (0); 10792 } 10793 if (__predict_false(tlen > sbspace(&so->so_rcv))) { 10794 return (0); 10795 } 10796 if ((to->to_flags & TOF_TS) != 0 && 10797 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { 10798 tp->ts_recent_age = tcp_ts_getticks(); 10799 tp->ts_recent = to->to_tsval; 10800 } 10801 rack = (struct tcp_rack *)tp->t_fb_ptr; 10802 /* 10803 * This is a pure, in-sequence data packet with nothing on the 10804 * reassembly queue and we have enough buffer space to take it. 10805 */ 10806 nsegs = max(1, m->m_pkthdr.lro_nsegs); 10807 10808 #ifdef NETFLIX_SB_LIMITS 10809 if (so->so_rcv.sb_shlim) { 10810 mcnt = m_memcnt(m); 10811 appended = 0; 10812 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt, 10813 CFO_NOSLEEP, NULL) == false) { 10814 counter_u64_add(tcp_sb_shlim_fails, 1); 10815 m_freem(m); 10816 return (1); 10817 } 10818 } 10819 #endif 10820 /* Clean receiver SACK report if present */ 10821 if (tp->rcv_numsacks) 10822 tcp_clean_sackreport(tp); 10823 KMOD_TCPSTAT_INC(tcps_preddat); 10824 tp->rcv_nxt += tlen; 10825 if (tlen && 10826 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) && 10827 (tp->t_fbyte_in == 0)) { 10828 tp->t_fbyte_in = ticks; 10829 if (tp->t_fbyte_in == 0) 10830 tp->t_fbyte_in = 1; 10831 if (tp->t_fbyte_out && tp->t_fbyte_in) 10832 tp->t_flags2 |= TF2_FBYTES_COMPLETE; 10833 } 10834 /* 10835 * Pull snd_wl1 up to prevent seq wrap relative to th_seq. 10836 */ 10837 tp->snd_wl1 = th->th_seq; 10838 /* 10839 * Pull rcv_up up to prevent seq wrap relative to rcv_nxt. 10840 */ 10841 tp->rcv_up = tp->rcv_nxt; 10842 KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs); 10843 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen); 10844 newsize = tcp_autorcvbuf(m, th, so, tp, tlen); 10845 10846 /* Add data to socket buffer. */ 10847 SOCKBUF_LOCK(&so->so_rcv); 10848 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 10849 m_freem(m); 10850 } else { 10851 /* 10852 * Set new socket buffer size. Give up when limit is 10853 * reached. 10854 */ 10855 if (newsize) 10856 if (!sbreserve_locked(so, SO_RCV, newsize, NULL)) 10857 so->so_rcv.sb_flags &= ~SB_AUTOSIZE; 10858 m_adj(m, drop_hdrlen); /* delayed header drop */ 10859 #ifdef NETFLIX_SB_LIMITS 10860 appended = 10861 #endif 10862 sbappendstream_locked(&so->so_rcv, m, 0); 10863 ctf_calc_rwin(so, tp); 10864 } 10865 rack_log_wakeup(tp,rack, &so->so_rcv, tlen, 1); 10866 /* NB: sorwakeup_locked() does an implicit unlock. */ 10867 sorwakeup_locked(so); 10868 #ifdef NETFLIX_SB_LIMITS 10869 if (so->so_rcv.sb_shlim && mcnt != appended) 10870 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended); 10871 #endif 10872 rack_handle_delayed_ack(tp, rack, tlen, 0); 10873 if (tp->snd_una == tp->snd_max) 10874 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); 10875 return (1); 10876 } 10877 10878 /* 10879 * This subfunction is used to try to highly optimize the 10880 * fast path. We again allow window updates that are 10881 * in sequence to remain in the fast-path. We also add 10882 * in the __predict's to attempt to help the compiler. 10883 * Note that if we return a 0, then we can *not* process 10884 * it and the caller should push the packet into the 10885 * slow-path. 10886 */ 10887 static int 10888 rack_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so, 10889 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 10890 uint32_t tiwin, int32_t nxt_pkt, uint32_t cts) 10891 { 10892 int32_t acked; 10893 int32_t nsegs; 10894 int32_t under_pacing = 0; 10895 struct tcp_rack *rack; 10896 10897 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) { 10898 /* Old ack, behind (or duplicate to) the last one rcv'd */ 10899 return (0); 10900 } 10901 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) { 10902 /* Above what we have sent? */ 10903 return (0); 10904 } 10905 if (__predict_false(tp->snd_nxt != tp->snd_max)) { 10906 /* We are retransmitting */ 10907 return (0); 10908 } 10909 if (__predict_false(tiwin == 0)) { 10910 /* zero window */ 10911 return (0); 10912 } 10913 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) { 10914 /* We need a SYN or a FIN, unlikely.. */ 10915 return (0); 10916 } 10917 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) { 10918 /* Timestamp is behind .. old ack with seq wrap? */ 10919 return (0); 10920 } 10921 if (__predict_false(IN_RECOVERY(tp->t_flags))) { 10922 /* Still recovering */ 10923 return (0); 10924 } 10925 rack = (struct tcp_rack *)tp->t_fb_ptr; 10926 if (rack->r_ctl.rc_sacked) { 10927 /* We have sack holes on our scoreboard */ 10928 return (0); 10929 } 10930 /* Ok if we reach here, we can process a fast-ack */ 10931 if (rack->gp_ready && 10932 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { 10933 under_pacing = 1; 10934 } 10935 nsegs = max(1, m->m_pkthdr.lro_nsegs); 10936 rack_log_ack(tp, to, th, 0, 0); 10937 /* Did the window get updated? */ 10938 if (tiwin != tp->snd_wnd) { 10939 tp->snd_wnd = tiwin; 10940 rack_validate_fo_sendwin_up(tp, rack); 10941 tp->snd_wl1 = th->th_seq; 10942 if (tp->snd_wnd > tp->max_sndwnd) 10943 tp->max_sndwnd = tp->snd_wnd; 10944 } 10945 /* Do we exit persists? */ 10946 if ((rack->rc_in_persist != 0) && 10947 (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2), 10948 rack->r_ctl.rc_pace_min_segs))) { 10949 rack_exit_persist(tp, rack, cts); 10950 } 10951 /* Do we enter persists? */ 10952 if ((rack->rc_in_persist == 0) && 10953 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) && 10954 TCPS_HAVEESTABLISHED(tp->t_state) && 10955 ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) && 10956 sbavail(&tptosocket(tp)->so_snd) && 10957 (sbavail(&tptosocket(tp)->so_snd) > tp->snd_wnd)) { 10958 /* 10959 * Here the rwnd is less than 10960 * the pacing size, we are established, 10961 * nothing is outstanding, and there is 10962 * data to send. Enter persists. 10963 */ 10964 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime); 10965 } 10966 /* 10967 * If last ACK falls within this segment's sequence numbers, record 10968 * the timestamp. NOTE that the test is modified according to the 10969 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26). 10970 */ 10971 if ((to->to_flags & TOF_TS) != 0 && 10972 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) { 10973 tp->ts_recent_age = tcp_ts_getticks(); 10974 tp->ts_recent = to->to_tsval; 10975 } 10976 /* 10977 * This is a pure ack for outstanding data. 10978 */ 10979 KMOD_TCPSTAT_INC(tcps_predack); 10980 10981 /* 10982 * "bad retransmit" recovery. 10983 */ 10984 if ((tp->t_flags & TF_PREVVALID) && 10985 ((tp->t_flags & TF_RCVD_TSTMP) == 0)) { 10986 tp->t_flags &= ~TF_PREVVALID; 10987 if (tp->t_rxtshift == 1 && 10988 (int)(ticks - tp->t_badrxtwin) < 0) 10989 rack_cong_signal(tp, CC_RTO_ERR, th->th_ack, __LINE__); 10990 } 10991 /* 10992 * Recalculate the transmit timer / rtt. 10993 * 10994 * Some boxes send broken timestamp replies during the SYN+ACK 10995 * phase, ignore timestamps of 0 or we could calculate a huge RTT 10996 * and blow up the retransmit timer. 10997 */ 10998 acked = BYTES_THIS_ACK(tp, th); 10999 11000 #ifdef TCP_HHOOK 11001 /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */ 11002 hhook_run_tcp_est_in(tp, th, to); 11003 #endif 11004 KMOD_TCPSTAT_ADD(tcps_rcvackpack, nsegs); 11005 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked); 11006 if (acked) { 11007 struct mbuf *mfree; 11008 11009 rack_ack_received(tp, rack, th->th_ack, nsegs, CC_ACK, 0); 11010 SOCKBUF_LOCK(&so->so_snd); 11011 mfree = sbcut_locked(&so->so_snd, acked); 11012 tp->snd_una = th->th_ack; 11013 /* Note we want to hold the sb lock through the sendmap adjust */ 11014 rack_adjust_sendmap(rack, &so->so_snd, tp->snd_una); 11015 /* Wake up the socket if we have room to write more */ 11016 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2); 11017 sowwakeup_locked(so); 11018 m_freem(mfree); 11019 tp->t_rxtshift = 0; 11020 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 11021 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 11022 rack->rc_tlp_in_progress = 0; 11023 rack->r_ctl.rc_tlp_cnt_out = 0; 11024 /* 11025 * If it is the RXT timer we want to 11026 * stop it, so we can restart a TLP. 11027 */ 11028 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) 11029 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 11030 #ifdef NETFLIX_HTTP_LOGGING 11031 tcp_http_check_for_comp(rack->rc_tp, th->th_ack); 11032 #endif 11033 } 11034 /* 11035 * Let the congestion control algorithm update congestion control 11036 * related information. This typically means increasing the 11037 * congestion window. 11038 */ 11039 if (tp->snd_wnd < ctf_outstanding(tp)) { 11040 /* The peer collapsed the window */ 11041 rack_collapsed_window(rack, ctf_outstanding(tp), __LINE__); 11042 } else if (rack->rc_has_collapsed) 11043 rack_un_collapse_window(rack, __LINE__); 11044 if ((rack->r_collapse_point_valid) && 11045 (SEQ_GT(tp->snd_una, rack->r_ctl.high_collapse_point))) 11046 rack->r_collapse_point_valid = 0; 11047 /* 11048 * Pull snd_wl2 up to prevent seq wrap relative to th_ack. 11049 */ 11050 tp->snd_wl2 = th->th_ack; 11051 tp->t_dupacks = 0; 11052 m_freem(m); 11053 /* ND6_HINT(tp); *//* Some progress has been made. */ 11054 11055 /* 11056 * If all outstanding data are acked, stop retransmit timer, 11057 * otherwise restart timer using current (possibly backed-off) 11058 * value. If process is waiting for space, wakeup/selwakeup/signal. 11059 * If data are ready to send, let tcp_output decide between more 11060 * output or persist. 11061 */ 11062 if (under_pacing && 11063 (rack->use_fixed_rate == 0) && 11064 (rack->in_probe_rtt == 0) && 11065 rack->rc_gp_dyn_mul && 11066 rack->rc_always_pace) { 11067 /* Check if we are dragging bottom */ 11068 rack_check_bottom_drag(tp, rack, so, acked); 11069 } 11070 if (tp->snd_una == tp->snd_max) { 11071 tp->t_flags &= ~TF_PREVVALID; 11072 rack->r_ctl.retran_during_recovery = 0; 11073 rack->r_ctl.dsack_byte_cnt = 0; 11074 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL); 11075 if (rack->r_ctl.rc_went_idle_time == 0) 11076 rack->r_ctl.rc_went_idle_time = 1; 11077 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__); 11078 if (sbavail(&tptosocket(tp)->so_snd) == 0) 11079 tp->t_acktime = 0; 11080 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 11081 } 11082 if (acked && rack->r_fast_output) 11083 rack_gain_for_fastoutput(rack, tp, so, (uint32_t)acked); 11084 if (sbavail(&so->so_snd)) { 11085 rack->r_wanted_output = 1; 11086 } 11087 return (1); 11088 } 11089 11090 /* 11091 * Return value of 1, the TCB is unlocked and most 11092 * likely gone, return value of 0, the TCP is still 11093 * locked. 11094 */ 11095 static int 11096 rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so, 11097 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 11098 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 11099 { 11100 int32_t ret_val = 0; 11101 int32_t todrop; 11102 int32_t ourfinisacked = 0; 11103 struct tcp_rack *rack; 11104 11105 INP_WLOCK_ASSERT(tptoinpcb(tp)); 11106 11107 ctf_calc_rwin(so, tp); 11108 /* 11109 * If the state is SYN_SENT: if seg contains an ACK, but not for our 11110 * SYN, drop the input. if seg contains a RST, then drop the 11111 * connection. if seg does not contain SYN, then drop it. Otherwise 11112 * this is an acceptable SYN segment initialize tp->rcv_nxt and 11113 * tp->irs if seg contains ack then advance tp->snd_una if seg 11114 * contains an ECE and ECN support is enabled, the stream is ECN 11115 * capable. if SYN has been acked change to ESTABLISHED else 11116 * SYN_RCVD state arrange for segment to be acked (eventually) 11117 * continue processing rest of data/controls. 11118 */ 11119 if ((thflags & TH_ACK) && 11120 (SEQ_LEQ(th->th_ack, tp->iss) || 11121 SEQ_GT(th->th_ack, tp->snd_max))) { 11122 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 11123 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11124 return (1); 11125 } 11126 if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) { 11127 TCP_PROBE5(connect__refused, NULL, tp, 11128 mtod(m, const char *), tp, th); 11129 tp = tcp_drop(tp, ECONNREFUSED); 11130 ctf_do_drop(m, tp); 11131 return (1); 11132 } 11133 if (thflags & TH_RST) { 11134 ctf_do_drop(m, tp); 11135 return (1); 11136 } 11137 if (!(thflags & TH_SYN)) { 11138 ctf_do_drop(m, tp); 11139 return (1); 11140 } 11141 tp->irs = th->th_seq; 11142 tcp_rcvseqinit(tp); 11143 rack = (struct tcp_rack *)tp->t_fb_ptr; 11144 if (thflags & TH_ACK) { 11145 int tfo_partial = 0; 11146 11147 KMOD_TCPSTAT_INC(tcps_connects); 11148 soisconnected(so); 11149 #ifdef MAC 11150 mac_socketpeer_set_from_mbuf(m, so); 11151 #endif 11152 /* Do window scaling on this connection? */ 11153 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 11154 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 11155 tp->rcv_scale = tp->request_r_scale; 11156 } 11157 tp->rcv_adv += min(tp->rcv_wnd, 11158 TCP_MAXWIN << tp->rcv_scale); 11159 /* 11160 * If not all the data that was sent in the TFO SYN 11161 * has been acked, resend the remainder right away. 11162 */ 11163 if (IS_FASTOPEN(tp->t_flags) && 11164 (tp->snd_una != tp->snd_max)) { 11165 tp->snd_nxt = th->th_ack; 11166 tfo_partial = 1; 11167 } 11168 /* 11169 * If there's data, delay ACK; if there's also a FIN ACKNOW 11170 * will be turned on later. 11171 */ 11172 if (DELAY_ACK(tp, tlen) && tlen != 0 && !tfo_partial) { 11173 rack_timer_cancel(tp, rack, 11174 rack->r_ctl.rc_rcvtime, __LINE__); 11175 tp->t_flags |= TF_DELACK; 11176 } else { 11177 rack->r_wanted_output = 1; 11178 tp->t_flags |= TF_ACKNOW; 11179 rack->rc_dack_toggle = 0; 11180 } 11181 11182 tcp_ecn_input_syn_sent(tp, thflags, iptos); 11183 11184 if (SEQ_GT(th->th_ack, tp->snd_una)) { 11185 /* 11186 * We advance snd_una for the 11187 * fast open case. If th_ack is 11188 * acknowledging data beyond 11189 * snd_una we can't just call 11190 * ack-processing since the 11191 * data stream in our send-map 11192 * will start at snd_una + 1 (one 11193 * beyond the SYN). If its just 11194 * equal we don't need to do that 11195 * and there is no send_map. 11196 */ 11197 tp->snd_una++; 11198 } 11199 /* 11200 * Received <SYN,ACK> in SYN_SENT[*] state. Transitions: 11201 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1 11202 */ 11203 tp->t_starttime = ticks; 11204 if (tp->t_flags & TF_NEEDFIN) { 11205 tcp_state_change(tp, TCPS_FIN_WAIT_1); 11206 tp->t_flags &= ~TF_NEEDFIN; 11207 thflags &= ~TH_SYN; 11208 } else { 11209 tcp_state_change(tp, TCPS_ESTABLISHED); 11210 TCP_PROBE5(connect__established, NULL, tp, 11211 mtod(m, const char *), tp, th); 11212 rack_cc_conn_init(tp); 11213 } 11214 } else { 11215 /* 11216 * Received initial SYN in SYN-SENT[*] state => simultaneous 11217 * open. If segment contains CC option and there is a 11218 * cached CC, apply TAO test. If it succeeds, connection is * 11219 * half-synchronized. Otherwise, do 3-way handshake: 11220 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If 11221 * there was no CC option, clear cached CC value. 11222 */ 11223 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN); 11224 tcp_state_change(tp, TCPS_SYN_RECEIVED); 11225 } 11226 /* 11227 * Advance th->th_seq to correspond to first data byte. If data, 11228 * trim to stay within window, dropping FIN if necessary. 11229 */ 11230 th->th_seq++; 11231 if (tlen > tp->rcv_wnd) { 11232 todrop = tlen - tp->rcv_wnd; 11233 m_adj(m, -todrop); 11234 tlen = tp->rcv_wnd; 11235 thflags &= ~TH_FIN; 11236 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin); 11237 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop); 11238 } 11239 tp->snd_wl1 = th->th_seq - 1; 11240 tp->rcv_up = th->th_seq; 11241 /* 11242 * Client side of transaction: already sent SYN and data. If the 11243 * remote host used T/TCP to validate the SYN, our data will be 11244 * ACK'd; if so, enter normal data segment processing in the middle 11245 * of step 5, ack processing. Otherwise, goto step 6. 11246 */ 11247 if (thflags & TH_ACK) { 11248 /* For syn-sent we need to possibly update the rtt */ 11249 if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) { 11250 uint32_t t, mcts; 11251 11252 mcts = tcp_ts_getticks(); 11253 t = (mcts - to->to_tsecr) * HPTS_USEC_IN_MSEC; 11254 if (!tp->t_rttlow || tp->t_rttlow > t) 11255 tp->t_rttlow = t; 11256 rack_log_rtt_sample_calc(rack, t, (to->to_tsecr * 1000), (mcts * 1000), 4); 11257 tcp_rack_xmit_timer(rack, t + 1, 1, t, 0, NULL, 2); 11258 tcp_rack_xmit_timer_commit(rack, tp); 11259 } 11260 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) 11261 return (ret_val); 11262 /* We may have changed to FIN_WAIT_1 above */ 11263 if (tp->t_state == TCPS_FIN_WAIT_1) { 11264 /* 11265 * In FIN_WAIT_1 STATE in addition to the processing 11266 * for the ESTABLISHED state if our FIN is now 11267 * acknowledged then enter FIN_WAIT_2. 11268 */ 11269 if (ourfinisacked) { 11270 /* 11271 * If we can't receive any more data, then 11272 * closing user can proceed. Starting the 11273 * timer is contrary to the specification, 11274 * but if we don't get a FIN we'll hang 11275 * forever. 11276 * 11277 * XXXjl: we should release the tp also, and 11278 * use a compressed state. 11279 */ 11280 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 11281 soisdisconnected(so); 11282 tcp_timer_activate(tp, TT_2MSL, 11283 (tcp_fast_finwait2_recycle ? 11284 tcp_finwait2_timeout : 11285 TP_MAXIDLE(tp))); 11286 } 11287 tcp_state_change(tp, TCPS_FIN_WAIT_2); 11288 } 11289 } 11290 } 11291 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11292 tiwin, thflags, nxt_pkt)); 11293 } 11294 11295 /* 11296 * Return value of 1, the TCB is unlocked and most 11297 * likely gone, return value of 0, the TCP is still 11298 * locked. 11299 */ 11300 static int 11301 rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so, 11302 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 11303 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 11304 { 11305 struct tcp_rack *rack; 11306 int32_t ret_val = 0; 11307 int32_t ourfinisacked = 0; 11308 11309 ctf_calc_rwin(so, tp); 11310 if ((thflags & TH_ACK) && 11311 (SEQ_LEQ(th->th_ack, tp->snd_una) || 11312 SEQ_GT(th->th_ack, tp->snd_max))) { 11313 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 11314 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11315 return (1); 11316 } 11317 rack = (struct tcp_rack *)tp->t_fb_ptr; 11318 if (IS_FASTOPEN(tp->t_flags)) { 11319 /* 11320 * When a TFO connection is in SYN_RECEIVED, the 11321 * only valid packets are the initial SYN, a 11322 * retransmit/copy of the initial SYN (possibly with 11323 * a subset of the original data), a valid ACK, a 11324 * FIN, or a RST. 11325 */ 11326 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) { 11327 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 11328 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11329 return (1); 11330 } else if (thflags & TH_SYN) { 11331 /* non-initial SYN is ignored */ 11332 if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) || 11333 (rack->r_ctl.rc_hpts_flags & PACE_TMR_TLP) || 11334 (rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) { 11335 ctf_do_drop(m, NULL); 11336 return (0); 11337 } 11338 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) { 11339 ctf_do_drop(m, NULL); 11340 return (0); 11341 } 11342 } 11343 11344 if ((thflags & TH_RST) || 11345 (tp->t_fin_is_rst && (thflags & TH_FIN))) 11346 return (__ctf_process_rst(m, th, so, tp, 11347 &rack->r_ctl.challenge_ack_ts, 11348 &rack->r_ctl.challenge_ack_cnt)); 11349 /* 11350 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 11351 * it's less than ts_recent, drop it. 11352 */ 11353 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 11354 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 11355 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 11356 return (ret_val); 11357 } 11358 /* 11359 * In the SYN-RECEIVED state, validate that the packet belongs to 11360 * this connection before trimming the data to fit the receive 11361 * window. Check the sequence number versus IRS since we know the 11362 * sequence numbers haven't wrapped. This is a partial fix for the 11363 * "LAND" DoS attack. 11364 */ 11365 if (SEQ_LT(th->th_seq, tp->irs)) { 11366 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 11367 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11368 return (1); 11369 } 11370 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 11371 &rack->r_ctl.challenge_ack_ts, 11372 &rack->r_ctl.challenge_ack_cnt)) { 11373 return (ret_val); 11374 } 11375 /* 11376 * If last ACK falls within this segment's sequence numbers, record 11377 * its timestamp. NOTE: 1) That the test incorporates suggestions 11378 * from the latest proposal of the tcplw@cray.com list (Braden 11379 * 1993/04/26). 2) That updating only on newer timestamps interferes 11380 * with our earlier PAWS tests, so this check should be solely 11381 * predicated on the sequence space of this segment. 3) That we 11382 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 11383 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 11384 * SEG.Len, This modified check allows us to overcome RFC1323's 11385 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 11386 * p.869. In such cases, we can still calculate the RTT correctly 11387 * when RCV.NXT == Last.ACK.Sent. 11388 */ 11389 if ((to->to_flags & TOF_TS) != 0 && 11390 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 11391 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 11392 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 11393 tp->ts_recent_age = tcp_ts_getticks(); 11394 tp->ts_recent = to->to_tsval; 11395 } 11396 tp->snd_wnd = tiwin; 11397 rack_validate_fo_sendwin_up(tp, rack); 11398 /* 11399 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 11400 * is on (half-synchronized state), then queue data for later 11401 * processing; else drop segment and return. 11402 */ 11403 if ((thflags & TH_ACK) == 0) { 11404 if (IS_FASTOPEN(tp->t_flags)) { 11405 rack_cc_conn_init(tp); 11406 } 11407 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11408 tiwin, thflags, nxt_pkt)); 11409 } 11410 KMOD_TCPSTAT_INC(tcps_connects); 11411 if (tp->t_flags & TF_SONOTCONN) { 11412 tp->t_flags &= ~TF_SONOTCONN; 11413 soisconnected(so); 11414 } 11415 /* Do window scaling? */ 11416 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) == 11417 (TF_RCVD_SCALE | TF_REQ_SCALE)) { 11418 tp->rcv_scale = tp->request_r_scale; 11419 } 11420 /* 11421 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* -> 11422 * FIN-WAIT-1 11423 */ 11424 tp->t_starttime = ticks; 11425 if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) { 11426 tcp_fastopen_decrement_counter(tp->t_tfo_pending); 11427 tp->t_tfo_pending = NULL; 11428 } 11429 if (tp->t_flags & TF_NEEDFIN) { 11430 tcp_state_change(tp, TCPS_FIN_WAIT_1); 11431 tp->t_flags &= ~TF_NEEDFIN; 11432 } else { 11433 tcp_state_change(tp, TCPS_ESTABLISHED); 11434 TCP_PROBE5(accept__established, NULL, tp, 11435 mtod(m, const char *), tp, th); 11436 /* 11437 * TFO connections call cc_conn_init() during SYN 11438 * processing. Calling it again here for such connections 11439 * is not harmless as it would undo the snd_cwnd reduction 11440 * that occurs when a TFO SYN|ACK is retransmitted. 11441 */ 11442 if (!IS_FASTOPEN(tp->t_flags)) 11443 rack_cc_conn_init(tp); 11444 } 11445 /* 11446 * Account for the ACK of our SYN prior to 11447 * regular ACK processing below, except for 11448 * simultaneous SYN, which is handled later. 11449 */ 11450 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN)) 11451 tp->snd_una++; 11452 /* 11453 * If segment contains data or ACK, will call tcp_reass() later; if 11454 * not, do so now to pass queued data to user. 11455 */ 11456 if (tlen == 0 && (thflags & TH_FIN) == 0) { 11457 (void) tcp_reass(tp, (struct tcphdr *)0, NULL, 0, 11458 (struct mbuf *)0); 11459 if (tp->t_flags & TF_WAKESOR) { 11460 tp->t_flags &= ~TF_WAKESOR; 11461 /* NB: sorwakeup_locked() does an implicit unlock. */ 11462 sorwakeup_locked(so); 11463 } 11464 } 11465 tp->snd_wl1 = th->th_seq - 1; 11466 /* For syn-recv we need to possibly update the rtt */ 11467 if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) { 11468 uint32_t t, mcts; 11469 11470 mcts = tcp_ts_getticks(); 11471 t = (mcts - to->to_tsecr) * HPTS_USEC_IN_MSEC; 11472 if (!tp->t_rttlow || tp->t_rttlow > t) 11473 tp->t_rttlow = t; 11474 rack_log_rtt_sample_calc(rack, t, (to->to_tsecr * 1000), (mcts * 1000), 5); 11475 tcp_rack_xmit_timer(rack, t + 1, 1, t, 0, NULL, 2); 11476 tcp_rack_xmit_timer_commit(rack, tp); 11477 } 11478 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 11479 return (ret_val); 11480 } 11481 if (tp->t_state == TCPS_FIN_WAIT_1) { 11482 /* We could have went to FIN_WAIT_1 (or EST) above */ 11483 /* 11484 * In FIN_WAIT_1 STATE in addition to the processing for the 11485 * ESTABLISHED state if our FIN is now acknowledged then 11486 * enter FIN_WAIT_2. 11487 */ 11488 if (ourfinisacked) { 11489 /* 11490 * If we can't receive any more data, then closing 11491 * user can proceed. Starting the timer is contrary 11492 * to the specification, but if we don't get a FIN 11493 * we'll hang forever. 11494 * 11495 * XXXjl: we should release the tp also, and use a 11496 * compressed state. 11497 */ 11498 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 11499 soisdisconnected(so); 11500 tcp_timer_activate(tp, TT_2MSL, 11501 (tcp_fast_finwait2_recycle ? 11502 tcp_finwait2_timeout : 11503 TP_MAXIDLE(tp))); 11504 } 11505 tcp_state_change(tp, TCPS_FIN_WAIT_2); 11506 } 11507 } 11508 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11509 tiwin, thflags, nxt_pkt)); 11510 } 11511 11512 /* 11513 * Return value of 1, the TCB is unlocked and most 11514 * likely gone, return value of 0, the TCP is still 11515 * locked. 11516 */ 11517 static int 11518 rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so, 11519 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 11520 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 11521 { 11522 int32_t ret_val = 0; 11523 struct tcp_rack *rack; 11524 11525 /* 11526 * Header prediction: check for the two common cases of a 11527 * uni-directional data xfer. If the packet has no control flags, 11528 * is in-sequence, the window didn't change and we're not 11529 * retransmitting, it's a candidate. If the length is zero and the 11530 * ack moved forward, we're the sender side of the xfer. Just free 11531 * the data acked & wake any higher level process that was blocked 11532 * waiting for space. If the length is non-zero and the ack didn't 11533 * move, we're the receiver side. If we're getting packets in-order 11534 * (the reassembly queue is empty), add the data toc The socket 11535 * buffer and note that we need a delayed ack. Make sure that the 11536 * hidden state-flags are also off. Since we check for 11537 * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN. 11538 */ 11539 rack = (struct tcp_rack *)tp->t_fb_ptr; 11540 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) && 11541 __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_ACK)) == TH_ACK) && 11542 __predict_true(SEGQ_EMPTY(tp)) && 11543 __predict_true(th->th_seq == tp->rcv_nxt)) { 11544 if (tlen == 0) { 11545 if (rack_fastack(m, th, so, tp, to, drop_hdrlen, tlen, 11546 tiwin, nxt_pkt, rack->r_ctl.rc_rcvtime)) { 11547 return (0); 11548 } 11549 } else { 11550 if (rack_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen, 11551 tiwin, nxt_pkt, iptos)) { 11552 return (0); 11553 } 11554 } 11555 } 11556 ctf_calc_rwin(so, tp); 11557 11558 if ((thflags & TH_RST) || 11559 (tp->t_fin_is_rst && (thflags & TH_FIN))) 11560 return (__ctf_process_rst(m, th, so, tp, 11561 &rack->r_ctl.challenge_ack_ts, 11562 &rack->r_ctl.challenge_ack_cnt)); 11563 11564 /* 11565 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 11566 * synchronized state. 11567 */ 11568 if (thflags & TH_SYN) { 11569 ctf_challenge_ack(m, th, tp, iptos, &ret_val); 11570 return (ret_val); 11571 } 11572 /* 11573 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 11574 * it's less than ts_recent, drop it. 11575 */ 11576 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 11577 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 11578 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 11579 return (ret_val); 11580 } 11581 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 11582 &rack->r_ctl.challenge_ack_ts, 11583 &rack->r_ctl.challenge_ack_cnt)) { 11584 return (ret_val); 11585 } 11586 /* 11587 * If last ACK falls within this segment's sequence numbers, record 11588 * its timestamp. NOTE: 1) That the test incorporates suggestions 11589 * from the latest proposal of the tcplw@cray.com list (Braden 11590 * 1993/04/26). 2) That updating only on newer timestamps interferes 11591 * with our earlier PAWS tests, so this check should be solely 11592 * predicated on the sequence space of this segment. 3) That we 11593 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 11594 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 11595 * SEG.Len, This modified check allows us to overcome RFC1323's 11596 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 11597 * p.869. In such cases, we can still calculate the RTT correctly 11598 * when RCV.NXT == Last.ACK.Sent. 11599 */ 11600 if ((to->to_flags & TOF_TS) != 0 && 11601 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 11602 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 11603 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 11604 tp->ts_recent_age = tcp_ts_getticks(); 11605 tp->ts_recent = to->to_tsval; 11606 } 11607 /* 11608 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 11609 * is on (half-synchronized state), then queue data for later 11610 * processing; else drop segment and return. 11611 */ 11612 if ((thflags & TH_ACK) == 0) { 11613 if (tp->t_flags & TF_NEEDSYN) { 11614 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11615 tiwin, thflags, nxt_pkt)); 11616 11617 } else if (tp->t_flags & TF_ACKNOW) { 11618 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 11619 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1; 11620 return (ret_val); 11621 } else { 11622 ctf_do_drop(m, NULL); 11623 return (0); 11624 } 11625 } 11626 /* 11627 * Ack processing. 11628 */ 11629 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) { 11630 return (ret_val); 11631 } 11632 if (sbavail(&so->so_snd)) { 11633 if (ctf_progress_timeout_check(tp, true)) { 11634 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__); 11635 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11636 return (1); 11637 } 11638 } 11639 /* State changes only happen in rack_process_data() */ 11640 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11641 tiwin, thflags, nxt_pkt)); 11642 } 11643 11644 /* 11645 * Return value of 1, the TCB is unlocked and most 11646 * likely gone, return value of 0, the TCP is still 11647 * locked. 11648 */ 11649 static int 11650 rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so, 11651 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 11652 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 11653 { 11654 int32_t ret_val = 0; 11655 struct tcp_rack *rack; 11656 11657 rack = (struct tcp_rack *)tp->t_fb_ptr; 11658 ctf_calc_rwin(so, tp); 11659 if ((thflags & TH_RST) || 11660 (tp->t_fin_is_rst && (thflags & TH_FIN))) 11661 return (__ctf_process_rst(m, th, so, tp, 11662 &rack->r_ctl.challenge_ack_ts, 11663 &rack->r_ctl.challenge_ack_cnt)); 11664 /* 11665 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 11666 * synchronized state. 11667 */ 11668 if (thflags & TH_SYN) { 11669 ctf_challenge_ack(m, th, tp, iptos, &ret_val); 11670 return (ret_val); 11671 } 11672 /* 11673 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 11674 * it's less than ts_recent, drop it. 11675 */ 11676 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 11677 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 11678 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 11679 return (ret_val); 11680 } 11681 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 11682 &rack->r_ctl.challenge_ack_ts, 11683 &rack->r_ctl.challenge_ack_cnt)) { 11684 return (ret_val); 11685 } 11686 /* 11687 * If last ACK falls within this segment's sequence numbers, record 11688 * its timestamp. NOTE: 1) That the test incorporates suggestions 11689 * from the latest proposal of the tcplw@cray.com list (Braden 11690 * 1993/04/26). 2) That updating only on newer timestamps interferes 11691 * with our earlier PAWS tests, so this check should be solely 11692 * predicated on the sequence space of this segment. 3) That we 11693 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 11694 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 11695 * SEG.Len, This modified check allows us to overcome RFC1323's 11696 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 11697 * p.869. In such cases, we can still calculate the RTT correctly 11698 * when RCV.NXT == Last.ACK.Sent. 11699 */ 11700 if ((to->to_flags & TOF_TS) != 0 && 11701 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 11702 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 11703 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 11704 tp->ts_recent_age = tcp_ts_getticks(); 11705 tp->ts_recent = to->to_tsval; 11706 } 11707 /* 11708 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 11709 * is on (half-synchronized state), then queue data for later 11710 * processing; else drop segment and return. 11711 */ 11712 if ((thflags & TH_ACK) == 0) { 11713 if (tp->t_flags & TF_NEEDSYN) { 11714 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11715 tiwin, thflags, nxt_pkt)); 11716 11717 } else if (tp->t_flags & TF_ACKNOW) { 11718 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 11719 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1; 11720 return (ret_val); 11721 } else { 11722 ctf_do_drop(m, NULL); 11723 return (0); 11724 } 11725 } 11726 /* 11727 * Ack processing. 11728 */ 11729 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) { 11730 return (ret_val); 11731 } 11732 if (sbavail(&so->so_snd)) { 11733 if (ctf_progress_timeout_check(tp, true)) { 11734 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr, 11735 tp, tick, PROGRESS_DROP, __LINE__); 11736 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11737 return (1); 11738 } 11739 } 11740 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11741 tiwin, thflags, nxt_pkt)); 11742 } 11743 11744 static int 11745 rack_check_data_after_close(struct mbuf *m, 11746 struct tcpcb *tp, int32_t *tlen, struct tcphdr *th, struct socket *so) 11747 { 11748 struct tcp_rack *rack; 11749 11750 rack = (struct tcp_rack *)tp->t_fb_ptr; 11751 if (rack->rc_allow_data_af_clo == 0) { 11752 close_now: 11753 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE); 11754 /* tcp_close will kill the inp pre-log the Reset */ 11755 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 11756 tp = tcp_close(tp); 11757 KMOD_TCPSTAT_INC(tcps_rcvafterclose); 11758 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen)); 11759 return (1); 11760 } 11761 if (sbavail(&so->so_snd) == 0) 11762 goto close_now; 11763 /* Ok we allow data that is ignored and a followup reset */ 11764 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE); 11765 tp->rcv_nxt = th->th_seq + *tlen; 11766 tp->t_flags2 |= TF2_DROP_AF_DATA; 11767 rack->r_wanted_output = 1; 11768 *tlen = 0; 11769 return (0); 11770 } 11771 11772 /* 11773 * Return value of 1, the TCB is unlocked and most 11774 * likely gone, return value of 0, the TCP is still 11775 * locked. 11776 */ 11777 static int 11778 rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so, 11779 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 11780 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 11781 { 11782 int32_t ret_val = 0; 11783 int32_t ourfinisacked = 0; 11784 struct tcp_rack *rack; 11785 11786 rack = (struct tcp_rack *)tp->t_fb_ptr; 11787 ctf_calc_rwin(so, tp); 11788 11789 if ((thflags & TH_RST) || 11790 (tp->t_fin_is_rst && (thflags & TH_FIN))) 11791 return (__ctf_process_rst(m, th, so, tp, 11792 &rack->r_ctl.challenge_ack_ts, 11793 &rack->r_ctl.challenge_ack_cnt)); 11794 /* 11795 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 11796 * synchronized state. 11797 */ 11798 if (thflags & TH_SYN) { 11799 ctf_challenge_ack(m, th, tp, iptos, &ret_val); 11800 return (ret_val); 11801 } 11802 /* 11803 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 11804 * it's less than ts_recent, drop it. 11805 */ 11806 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 11807 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 11808 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 11809 return (ret_val); 11810 } 11811 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 11812 &rack->r_ctl.challenge_ack_ts, 11813 &rack->r_ctl.challenge_ack_cnt)) { 11814 return (ret_val); 11815 } 11816 /* 11817 * If new data are received on a connection after the user processes 11818 * are gone, then RST the other end. 11819 */ 11820 if ((tp->t_flags & TF_CLOSED) && tlen && 11821 rack_check_data_after_close(m, tp, &tlen, th, so)) 11822 return (1); 11823 /* 11824 * If last ACK falls within this segment's sequence numbers, record 11825 * its timestamp. NOTE: 1) That the test incorporates suggestions 11826 * from the latest proposal of the tcplw@cray.com list (Braden 11827 * 1993/04/26). 2) That updating only on newer timestamps interferes 11828 * with our earlier PAWS tests, so this check should be solely 11829 * predicated on the sequence space of this segment. 3) That we 11830 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 11831 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 11832 * SEG.Len, This modified check allows us to overcome RFC1323's 11833 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 11834 * p.869. In such cases, we can still calculate the RTT correctly 11835 * when RCV.NXT == Last.ACK.Sent. 11836 */ 11837 if ((to->to_flags & TOF_TS) != 0 && 11838 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 11839 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 11840 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 11841 tp->ts_recent_age = tcp_ts_getticks(); 11842 tp->ts_recent = to->to_tsval; 11843 } 11844 /* 11845 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 11846 * is on (half-synchronized state), then queue data for later 11847 * processing; else drop segment and return. 11848 */ 11849 if ((thflags & TH_ACK) == 0) { 11850 if (tp->t_flags & TF_NEEDSYN) { 11851 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11852 tiwin, thflags, nxt_pkt)); 11853 } else if (tp->t_flags & TF_ACKNOW) { 11854 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 11855 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1; 11856 return (ret_val); 11857 } else { 11858 ctf_do_drop(m, NULL); 11859 return (0); 11860 } 11861 } 11862 /* 11863 * Ack processing. 11864 */ 11865 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 11866 return (ret_val); 11867 } 11868 if (ourfinisacked) { 11869 /* 11870 * If we can't receive any more data, then closing user can 11871 * proceed. Starting the timer is contrary to the 11872 * specification, but if we don't get a FIN we'll hang 11873 * forever. 11874 * 11875 * XXXjl: we should release the tp also, and use a 11876 * compressed state. 11877 */ 11878 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 11879 soisdisconnected(so); 11880 tcp_timer_activate(tp, TT_2MSL, 11881 (tcp_fast_finwait2_recycle ? 11882 tcp_finwait2_timeout : 11883 TP_MAXIDLE(tp))); 11884 } 11885 tcp_state_change(tp, TCPS_FIN_WAIT_2); 11886 } 11887 if (sbavail(&so->so_snd)) { 11888 if (ctf_progress_timeout_check(tp, true)) { 11889 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr, 11890 tp, tick, PROGRESS_DROP, __LINE__); 11891 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 11892 return (1); 11893 } 11894 } 11895 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11896 tiwin, thflags, nxt_pkt)); 11897 } 11898 11899 /* 11900 * Return value of 1, the TCB is unlocked and most 11901 * likely gone, return value of 0, the TCP is still 11902 * locked. 11903 */ 11904 static int 11905 rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so, 11906 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 11907 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 11908 { 11909 int32_t ret_val = 0; 11910 int32_t ourfinisacked = 0; 11911 struct tcp_rack *rack; 11912 11913 rack = (struct tcp_rack *)tp->t_fb_ptr; 11914 ctf_calc_rwin(so, tp); 11915 11916 if ((thflags & TH_RST) || 11917 (tp->t_fin_is_rst && (thflags & TH_FIN))) 11918 return (__ctf_process_rst(m, th, so, tp, 11919 &rack->r_ctl.challenge_ack_ts, 11920 &rack->r_ctl.challenge_ack_cnt)); 11921 /* 11922 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 11923 * synchronized state. 11924 */ 11925 if (thflags & TH_SYN) { 11926 ctf_challenge_ack(m, th, tp, iptos, &ret_val); 11927 return (ret_val); 11928 } 11929 /* 11930 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 11931 * it's less than ts_recent, drop it. 11932 */ 11933 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 11934 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 11935 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 11936 return (ret_val); 11937 } 11938 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 11939 &rack->r_ctl.challenge_ack_ts, 11940 &rack->r_ctl.challenge_ack_cnt)) { 11941 return (ret_val); 11942 } 11943 /* 11944 * If new data are received on a connection after the user processes 11945 * are gone, then RST the other end. 11946 */ 11947 if ((tp->t_flags & TF_CLOSED) && tlen && 11948 rack_check_data_after_close(m, tp, &tlen, th, so)) 11949 return (1); 11950 /* 11951 * If last ACK falls within this segment's sequence numbers, record 11952 * its timestamp. NOTE: 1) That the test incorporates suggestions 11953 * from the latest proposal of the tcplw@cray.com list (Braden 11954 * 1993/04/26). 2) That updating only on newer timestamps interferes 11955 * with our earlier PAWS tests, so this check should be solely 11956 * predicated on the sequence space of this segment. 3) That we 11957 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 11958 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 11959 * SEG.Len, This modified check allows us to overcome RFC1323's 11960 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 11961 * p.869. In such cases, we can still calculate the RTT correctly 11962 * when RCV.NXT == Last.ACK.Sent. 11963 */ 11964 if ((to->to_flags & TOF_TS) != 0 && 11965 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 11966 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 11967 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 11968 tp->ts_recent_age = tcp_ts_getticks(); 11969 tp->ts_recent = to->to_tsval; 11970 } 11971 /* 11972 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 11973 * is on (half-synchronized state), then queue data for later 11974 * processing; else drop segment and return. 11975 */ 11976 if ((thflags & TH_ACK) == 0) { 11977 if (tp->t_flags & TF_NEEDSYN) { 11978 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 11979 tiwin, thflags, nxt_pkt)); 11980 } else if (tp->t_flags & TF_ACKNOW) { 11981 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 11982 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1; 11983 return (ret_val); 11984 } else { 11985 ctf_do_drop(m, NULL); 11986 return (0); 11987 } 11988 } 11989 /* 11990 * Ack processing. 11991 */ 11992 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 11993 return (ret_val); 11994 } 11995 if (ourfinisacked) { 11996 tcp_twstart(tp); 11997 m_freem(m); 11998 return (1); 11999 } 12000 if (sbavail(&so->so_snd)) { 12001 if (ctf_progress_timeout_check(tp, true)) { 12002 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr, 12003 tp, tick, PROGRESS_DROP, __LINE__); 12004 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 12005 return (1); 12006 } 12007 } 12008 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 12009 tiwin, thflags, nxt_pkt)); 12010 } 12011 12012 /* 12013 * Return value of 1, the TCB is unlocked and most 12014 * likely gone, return value of 0, the TCP is still 12015 * locked. 12016 */ 12017 static int 12018 rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so, 12019 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 12020 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 12021 { 12022 int32_t ret_val = 0; 12023 int32_t ourfinisacked = 0; 12024 struct tcp_rack *rack; 12025 12026 rack = (struct tcp_rack *)tp->t_fb_ptr; 12027 ctf_calc_rwin(so, tp); 12028 12029 if ((thflags & TH_RST) || 12030 (tp->t_fin_is_rst && (thflags & TH_FIN))) 12031 return (__ctf_process_rst(m, th, so, tp, 12032 &rack->r_ctl.challenge_ack_ts, 12033 &rack->r_ctl.challenge_ack_cnt)); 12034 /* 12035 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 12036 * synchronized state. 12037 */ 12038 if (thflags & TH_SYN) { 12039 ctf_challenge_ack(m, th, tp, iptos, &ret_val); 12040 return (ret_val); 12041 } 12042 /* 12043 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 12044 * it's less than ts_recent, drop it. 12045 */ 12046 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 12047 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 12048 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 12049 return (ret_val); 12050 } 12051 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 12052 &rack->r_ctl.challenge_ack_ts, 12053 &rack->r_ctl.challenge_ack_cnt)) { 12054 return (ret_val); 12055 } 12056 /* 12057 * If new data are received on a connection after the user processes 12058 * are gone, then RST the other end. 12059 */ 12060 if ((tp->t_flags & TF_CLOSED) && tlen && 12061 rack_check_data_after_close(m, tp, &tlen, th, so)) 12062 return (1); 12063 /* 12064 * If last ACK falls within this segment's sequence numbers, record 12065 * its timestamp. NOTE: 1) That the test incorporates suggestions 12066 * from the latest proposal of the tcplw@cray.com list (Braden 12067 * 1993/04/26). 2) That updating only on newer timestamps interferes 12068 * with our earlier PAWS tests, so this check should be solely 12069 * predicated on the sequence space of this segment. 3) That we 12070 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 12071 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 12072 * SEG.Len, This modified check allows us to overcome RFC1323's 12073 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 12074 * p.869. In such cases, we can still calculate the RTT correctly 12075 * when RCV.NXT == Last.ACK.Sent. 12076 */ 12077 if ((to->to_flags & TOF_TS) != 0 && 12078 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 12079 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 12080 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 12081 tp->ts_recent_age = tcp_ts_getticks(); 12082 tp->ts_recent = to->to_tsval; 12083 } 12084 /* 12085 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 12086 * is on (half-synchronized state), then queue data for later 12087 * processing; else drop segment and return. 12088 */ 12089 if ((thflags & TH_ACK) == 0) { 12090 if (tp->t_flags & TF_NEEDSYN) { 12091 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 12092 tiwin, thflags, nxt_pkt)); 12093 } else if (tp->t_flags & TF_ACKNOW) { 12094 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 12095 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1; 12096 return (ret_val); 12097 } else { 12098 ctf_do_drop(m, NULL); 12099 return (0); 12100 } 12101 } 12102 /* 12103 * case TCPS_LAST_ACK: Ack processing. 12104 */ 12105 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 12106 return (ret_val); 12107 } 12108 if (ourfinisacked) { 12109 tp = tcp_close(tp); 12110 ctf_do_drop(m, tp); 12111 return (1); 12112 } 12113 if (sbavail(&so->so_snd)) { 12114 if (ctf_progress_timeout_check(tp, true)) { 12115 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr, 12116 tp, tick, PROGRESS_DROP, __LINE__); 12117 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 12118 return (1); 12119 } 12120 } 12121 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 12122 tiwin, thflags, nxt_pkt)); 12123 } 12124 12125 /* 12126 * Return value of 1, the TCB is unlocked and most 12127 * likely gone, return value of 0, the TCP is still 12128 * locked. 12129 */ 12130 static int 12131 rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so, 12132 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen, 12133 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos) 12134 { 12135 int32_t ret_val = 0; 12136 int32_t ourfinisacked = 0; 12137 struct tcp_rack *rack; 12138 12139 rack = (struct tcp_rack *)tp->t_fb_ptr; 12140 ctf_calc_rwin(so, tp); 12141 12142 /* Reset receive buffer auto scaling when not in bulk receive mode. */ 12143 if ((thflags & TH_RST) || 12144 (tp->t_fin_is_rst && (thflags & TH_FIN))) 12145 return (__ctf_process_rst(m, th, so, tp, 12146 &rack->r_ctl.challenge_ack_ts, 12147 &rack->r_ctl.challenge_ack_cnt)); 12148 /* 12149 * RFC5961 Section 4.2 Send challenge ACK for any SYN in 12150 * synchronized state. 12151 */ 12152 if (thflags & TH_SYN) { 12153 ctf_challenge_ack(m, th, tp, iptos, &ret_val); 12154 return (ret_val); 12155 } 12156 /* 12157 * RFC 1323 PAWS: If we have a timestamp reply on this segment and 12158 * it's less than ts_recent, drop it. 12159 */ 12160 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent && 12161 TSTMP_LT(to->to_tsval, tp->ts_recent)) { 12162 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val)) 12163 return (ret_val); 12164 } 12165 if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val, 12166 &rack->r_ctl.challenge_ack_ts, 12167 &rack->r_ctl.challenge_ack_cnt)) { 12168 return (ret_val); 12169 } 12170 /* 12171 * If new data are received on a connection after the user processes 12172 * are gone, then RST the other end. 12173 */ 12174 if ((tp->t_flags & TF_CLOSED) && tlen && 12175 rack_check_data_after_close(m, tp, &tlen, th, so)) 12176 return (1); 12177 /* 12178 * If last ACK falls within this segment's sequence numbers, record 12179 * its timestamp. NOTE: 1) That the test incorporates suggestions 12180 * from the latest proposal of the tcplw@cray.com list (Braden 12181 * 1993/04/26). 2) That updating only on newer timestamps interferes 12182 * with our earlier PAWS tests, so this check should be solely 12183 * predicated on the sequence space of this segment. 3) That we 12184 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ 12185 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ + 12186 * SEG.Len, This modified check allows us to overcome RFC1323's 12187 * limitations as described in Stevens TCP/IP Illustrated Vol. 2 12188 * p.869. In such cases, we can still calculate the RTT correctly 12189 * when RCV.NXT == Last.ACK.Sent. 12190 */ 12191 if ((to->to_flags & TOF_TS) != 0 && 12192 SEQ_LEQ(th->th_seq, tp->last_ack_sent) && 12193 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen + 12194 ((thflags & (TH_SYN | TH_FIN)) != 0))) { 12195 tp->ts_recent_age = tcp_ts_getticks(); 12196 tp->ts_recent = to->to_tsval; 12197 } 12198 /* 12199 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag 12200 * is on (half-synchronized state), then queue data for later 12201 * processing; else drop segment and return. 12202 */ 12203 if ((thflags & TH_ACK) == 0) { 12204 if (tp->t_flags & TF_NEEDSYN) { 12205 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 12206 tiwin, thflags, nxt_pkt)); 12207 } else if (tp->t_flags & TF_ACKNOW) { 12208 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val); 12209 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1; 12210 return (ret_val); 12211 } else { 12212 ctf_do_drop(m, NULL); 12213 return (0); 12214 } 12215 } 12216 /* 12217 * Ack processing. 12218 */ 12219 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) { 12220 return (ret_val); 12221 } 12222 if (sbavail(&so->so_snd)) { 12223 if (ctf_progress_timeout_check(tp, true)) { 12224 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr, 12225 tp, tick, PROGRESS_DROP, __LINE__); 12226 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 12227 return (1); 12228 } 12229 } 12230 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen, 12231 tiwin, thflags, nxt_pkt)); 12232 } 12233 12234 static void inline 12235 rack_clear_rate_sample(struct tcp_rack *rack) 12236 { 12237 rack->r_ctl.rack_rs.rs_flags = RACK_RTT_EMPTY; 12238 rack->r_ctl.rack_rs.rs_rtt_cnt = 0; 12239 rack->r_ctl.rack_rs.rs_rtt_tot = 0; 12240 } 12241 12242 static void 12243 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override) 12244 { 12245 uint64_t bw_est, rate_wanted; 12246 int chged = 0; 12247 uint32_t user_max, orig_min, orig_max; 12248 12249 orig_min = rack->r_ctl.rc_pace_min_segs; 12250 orig_max = rack->r_ctl.rc_pace_max_segs; 12251 user_max = ctf_fixed_maxseg(tp) * rack->rc_user_set_max_segs; 12252 if (ctf_fixed_maxseg(tp) != rack->r_ctl.rc_pace_min_segs) 12253 chged = 1; 12254 rack->r_ctl.rc_pace_min_segs = ctf_fixed_maxseg(tp); 12255 if (rack->use_fixed_rate || rack->rc_force_max_seg) { 12256 if (user_max != rack->r_ctl.rc_pace_max_segs) 12257 chged = 1; 12258 } 12259 if (rack->rc_force_max_seg) { 12260 rack->r_ctl.rc_pace_max_segs = user_max; 12261 } else if (rack->use_fixed_rate) { 12262 bw_est = rack_get_bw(rack); 12263 if ((rack->r_ctl.crte == NULL) || 12264 (bw_est != rack->r_ctl.crte->rate)) { 12265 rack->r_ctl.rc_pace_max_segs = user_max; 12266 } else { 12267 /* We are pacing right at the hardware rate */ 12268 uint32_t segsiz; 12269 12270 segsiz = min(ctf_fixed_maxseg(tp), 12271 rack->r_ctl.rc_pace_min_segs); 12272 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size( 12273 tp, bw_est, segsiz, 0, 12274 rack->r_ctl.crte, NULL); 12275 } 12276 } else if (rack->rc_always_pace) { 12277 if (rack->r_ctl.gp_bw || 12278 #ifdef NETFLIX_PEAKRATE 12279 rack->rc_tp->t_maxpeakrate || 12280 #endif 12281 rack->r_ctl.init_rate) { 12282 /* We have a rate of some sort set */ 12283 uint32_t orig; 12284 12285 bw_est = rack_get_bw(rack); 12286 orig = rack->r_ctl.rc_pace_max_segs; 12287 if (fill_override) 12288 rate_wanted = *fill_override; 12289 else 12290 rate_wanted = rack_get_output_bw(rack, bw_est, NULL, NULL); 12291 if (rate_wanted) { 12292 /* We have something */ 12293 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack, 12294 rate_wanted, 12295 ctf_fixed_maxseg(rack->rc_tp)); 12296 } else 12297 rack->r_ctl.rc_pace_max_segs = rack->r_ctl.rc_pace_min_segs; 12298 if (orig != rack->r_ctl.rc_pace_max_segs) 12299 chged = 1; 12300 } else if ((rack->r_ctl.gp_bw == 0) && 12301 (rack->r_ctl.rc_pace_max_segs == 0)) { 12302 /* 12303 * If we have nothing limit us to bursting 12304 * out IW sized pieces. 12305 */ 12306 chged = 1; 12307 rack->r_ctl.rc_pace_max_segs = rc_init_window(rack); 12308 } 12309 } 12310 if (rack->r_ctl.rc_pace_max_segs > PACE_MAX_IP_BYTES) { 12311 chged = 1; 12312 rack->r_ctl.rc_pace_max_segs = PACE_MAX_IP_BYTES; 12313 } 12314 if (chged) 12315 rack_log_type_pacing_sizes(tp, rack, orig_min, orig_max, line, 2); 12316 } 12317 12318 12319 static void 12320 rack_init_fsb_block(struct tcpcb *tp, struct tcp_rack *rack) 12321 { 12322 #ifdef INET6 12323 struct ip6_hdr *ip6 = NULL; 12324 #endif 12325 #ifdef INET 12326 struct ip *ip = NULL; 12327 #endif 12328 struct udphdr *udp = NULL; 12329 12330 /* Ok lets fill in the fast block, it can only be used with no IP options! */ 12331 #ifdef INET6 12332 if (rack->r_is_v6) { 12333 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 12334 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr; 12335 if (tp->t_port) { 12336 rack->r_ctl.fsb.tcp_ip_hdr_len += sizeof(struct udphdr); 12337 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 12338 udp->uh_sport = htons(V_tcp_udp_tunneling_port); 12339 udp->uh_dport = tp->t_port; 12340 rack->r_ctl.fsb.udp = udp; 12341 rack->r_ctl.fsb.th = (struct tcphdr *)(udp + 1); 12342 } else 12343 { 12344 rack->r_ctl.fsb.th = (struct tcphdr *)(ip6 + 1); 12345 rack->r_ctl.fsb.udp = NULL; 12346 } 12347 tcpip_fillheaders(rack->rc_inp, 12348 tp->t_port, 12349 ip6, rack->r_ctl.fsb.th); 12350 } else 12351 #endif /* INET6 */ 12352 #ifdef INET 12353 { 12354 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct tcpiphdr); 12355 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr; 12356 if (tp->t_port) { 12357 rack->r_ctl.fsb.tcp_ip_hdr_len += sizeof(struct udphdr); 12358 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip)); 12359 udp->uh_sport = htons(V_tcp_udp_tunneling_port); 12360 udp->uh_dport = tp->t_port; 12361 rack->r_ctl.fsb.udp = udp; 12362 rack->r_ctl.fsb.th = (struct tcphdr *)(udp + 1); 12363 } else 12364 { 12365 rack->r_ctl.fsb.udp = NULL; 12366 rack->r_ctl.fsb.th = (struct tcphdr *)(ip + 1); 12367 } 12368 tcpip_fillheaders(rack->rc_inp, 12369 tp->t_port, 12370 ip, rack->r_ctl.fsb.th); 12371 } 12372 #endif 12373 rack->r_fsb_inited = 1; 12374 } 12375 12376 static int 12377 rack_init_fsb(struct tcpcb *tp, struct tcp_rack *rack) 12378 { 12379 /* 12380 * Allocate the larger of spaces V6 if available else just 12381 * V4 and include udphdr (overbook) 12382 */ 12383 #ifdef INET6 12384 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + sizeof(struct udphdr); 12385 #else 12386 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct tcpiphdr) + sizeof(struct udphdr); 12387 #endif 12388 rack->r_ctl.fsb.tcp_ip_hdr = malloc(rack->r_ctl.fsb.tcp_ip_hdr_len, 12389 M_TCPFSB, M_NOWAIT|M_ZERO); 12390 if (rack->r_ctl.fsb.tcp_ip_hdr == NULL) { 12391 return (ENOMEM); 12392 } 12393 rack->r_fsb_inited = 0; 12394 return (0); 12395 } 12396 12397 static int 12398 rack_init(struct tcpcb *tp) 12399 { 12400 struct inpcb *inp = tptoinpcb(tp); 12401 struct tcp_rack *rack = NULL; 12402 #ifdef INVARIANTS 12403 struct rack_sendmap *insret; 12404 #endif 12405 uint32_t iwin, snt, us_cts; 12406 int err; 12407 12408 tp->t_fb_ptr = uma_zalloc(rack_pcb_zone, M_NOWAIT); 12409 if (tp->t_fb_ptr == NULL) { 12410 /* 12411 * We need to allocate memory but cant. The INP and INP_INFO 12412 * locks and they are recursive (happens during setup. So a 12413 * scheme to drop the locks fails :( 12414 * 12415 */ 12416 return (ENOMEM); 12417 } 12418 memset(tp->t_fb_ptr, 0, sizeof(struct tcp_rack)); 12419 12420 rack = (struct tcp_rack *)tp->t_fb_ptr; 12421 RB_INIT(&rack->r_ctl.rc_mtree); 12422 TAILQ_INIT(&rack->r_ctl.rc_free); 12423 TAILQ_INIT(&rack->r_ctl.rc_tmap); 12424 rack->rc_tp = tp; 12425 rack->rc_inp = inp; 12426 /* Set the flag */ 12427 rack->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0; 12428 /* Probably not needed but lets be sure */ 12429 rack_clear_rate_sample(rack); 12430 /* 12431 * Save off the default values, socket options will poke 12432 * at these if pacing is not on or we have not yet 12433 * reached where pacing is on (gp_ready/fixed enabled). 12434 * When they get set into the CC module (when gp_ready 12435 * is enabled or we enable fixed) then we will set these 12436 * values into the CC and place in here the old values 12437 * so we have a restoral. Then we will set the flag 12438 * rc_pacing_cc_set. That way whenever we turn off pacing 12439 * or switch off this stack, we will know to go restore 12440 * the saved values. 12441 */ 12442 rack->r_ctl.rc_saved_beta.beta = V_newreno_beta_ecn; 12443 rack->r_ctl.rc_saved_beta.beta_ecn = V_newreno_beta_ecn; 12444 /* We want abe like behavior as well */ 12445 rack->r_ctl.rc_saved_beta.newreno_flags |= CC_NEWRENO_BETA_ECN_ENABLED; 12446 rack->r_ctl.rc_reorder_fade = rack_reorder_fade; 12447 rack->rc_allow_data_af_clo = rack_ignore_data_after_close; 12448 rack->r_ctl.rc_tlp_threshold = rack_tlp_thresh; 12449 rack->r_ctl.roundends = tp->snd_max; 12450 if (use_rack_rr) 12451 rack->use_rack_rr = 1; 12452 if (V_tcp_delack_enabled) 12453 tp->t_delayed_ack = 1; 12454 else 12455 tp->t_delayed_ack = 0; 12456 #ifdef TCP_ACCOUNTING 12457 if (rack_tcp_accounting) { 12458 tp->t_flags2 |= TF2_TCP_ACCOUNTING; 12459 } 12460 #endif 12461 if (rack_enable_shared_cwnd) 12462 rack->rack_enable_scwnd = 1; 12463 rack->rc_user_set_max_segs = rack_hptsi_segments; 12464 rack->rc_force_max_seg = 0; 12465 if (rack_use_imac_dack) 12466 rack->rc_dack_mode = 1; 12467 TAILQ_INIT(&rack->r_ctl.opt_list); 12468 rack->r_ctl.rc_reorder_shift = rack_reorder_thresh; 12469 rack->r_ctl.rc_pkt_delay = rack_pkt_delay; 12470 rack->r_ctl.rc_tlp_cwnd_reduce = rack_lower_cwnd_at_tlp; 12471 rack->r_ctl.rc_lowest_us_rtt = 0xffffffff; 12472 rack->r_ctl.rc_highest_us_rtt = 0; 12473 rack->r_ctl.bw_rate_cap = rack_bw_rate_cap; 12474 rack->r_ctl.timer_slop = TICKS_2_USEC(tcp_rexmit_slop); 12475 if (rack_use_cmp_acks) 12476 rack->r_use_cmp_ack = 1; 12477 if (rack_disable_prr) 12478 rack->rack_no_prr = 1; 12479 if (rack_gp_no_rec_chg) 12480 rack->rc_gp_no_rec_chg = 1; 12481 if (rack_pace_every_seg && tcp_can_enable_pacing()) { 12482 rack->rc_always_pace = 1; 12483 if (rack->use_fixed_rate || rack->gp_ready) 12484 rack_set_cc_pacing(rack); 12485 } else 12486 rack->rc_always_pace = 0; 12487 if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack) 12488 rack->r_mbuf_queue = 1; 12489 else 12490 rack->r_mbuf_queue = 0; 12491 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack) 12492 inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 12493 else 12494 inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ; 12495 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12496 if (rack_limits_scwnd) 12497 rack->r_limit_scw = 1; 12498 else 12499 rack->r_limit_scw = 0; 12500 rack->rc_labc = V_tcp_abc_l_var; 12501 rack->r_ctl.rc_high_rwnd = tp->snd_wnd; 12502 rack->r_ctl.cwnd_to_use = tp->snd_cwnd; 12503 rack->r_ctl.rc_rate_sample_method = rack_rate_sample_method; 12504 rack->rack_tlp_threshold_use = rack_tlp_threshold_use; 12505 rack->r_ctl.rc_prr_sendalot = rack_send_a_lot_in_prr; 12506 rack->r_ctl.rc_min_to = rack_min_to; 12507 microuptime(&rack->r_ctl.act_rcv_time); 12508 rack->r_ctl.rc_last_time_decay = rack->r_ctl.act_rcv_time; 12509 rack->rc_init_win = rack_default_init_window; 12510 rack->r_ctl.rack_per_of_gp_ss = rack_per_of_gp_ss; 12511 if (rack_hw_up_only) 12512 rack->r_up_only = 1; 12513 if (rack_do_dyn_mul) { 12514 /* When dynamic adjustment is on CA needs to start at 100% */ 12515 rack->rc_gp_dyn_mul = 1; 12516 if (rack_do_dyn_mul >= 100) 12517 rack->r_ctl.rack_per_of_gp_ca = rack_do_dyn_mul; 12518 } else 12519 rack->r_ctl.rack_per_of_gp_ca = rack_per_of_gp_ca; 12520 rack->r_ctl.rack_per_of_gp_rec = rack_per_of_gp_rec; 12521 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt; 12522 rack->r_ctl.rc_tlp_rxt_last_time = tcp_tv_to_mssectick(&rack->r_ctl.act_rcv_time); 12523 setup_time_filter_small(&rack->r_ctl.rc_gp_min_rtt, FILTER_TYPE_MIN, 12524 rack_probertt_filter_life); 12525 us_cts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 12526 rack->r_ctl.rc_lower_rtt_us_cts = us_cts; 12527 rack->r_ctl.rc_time_of_last_probertt = us_cts; 12528 rack->r_ctl.challenge_ack_ts = tcp_ts_getticks(); 12529 rack->r_ctl.rc_time_probertt_starts = 0; 12530 if (rack_dsack_std_based & 0x1) { 12531 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */ 12532 rack->rc_rack_tmr_std_based = 1; 12533 } 12534 if (rack_dsack_std_based & 0x2) { 12535 /* Basically this means rack timers are extended based on dsack by up to (2 * srtt) */ 12536 rack->rc_rack_use_dsack = 1; 12537 } 12538 /* We require at least one measurement, even if the sysctl is 0 */ 12539 if (rack_req_measurements) 12540 rack->r_ctl.req_measurements = rack_req_measurements; 12541 else 12542 rack->r_ctl.req_measurements = 1; 12543 if (rack_enable_hw_pacing) 12544 rack->rack_hdw_pace_ena = 1; 12545 if (rack_hw_rate_caps) 12546 rack->r_rack_hw_rate_caps = 1; 12547 /* Do we force on detection? */ 12548 #ifdef NETFLIX_EXP_DETECTION 12549 if (tcp_force_detection) 12550 rack->do_detection = 1; 12551 else 12552 #endif 12553 rack->do_detection = 0; 12554 if (rack_non_rxt_use_cr) 12555 rack->rack_rec_nonrxt_use_cr = 1; 12556 err = rack_init_fsb(tp, rack); 12557 if (err) { 12558 uma_zfree(rack_pcb_zone, tp->t_fb_ptr); 12559 tp->t_fb_ptr = NULL; 12560 return (err); 12561 } 12562 if (tp->snd_una != tp->snd_max) { 12563 /* Create a send map for the current outstanding data */ 12564 struct rack_sendmap *rsm; 12565 12566 rsm = rack_alloc(rack); 12567 if (rsm == NULL) { 12568 uma_zfree(rack_pcb_zone, tp->t_fb_ptr); 12569 tp->t_fb_ptr = NULL; 12570 return (ENOMEM); 12571 } 12572 rsm->r_no_rtt_allowed = 1; 12573 rsm->r_tim_lastsent[0] = rack_to_usec_ts(&rack->r_ctl.act_rcv_time); 12574 rsm->r_rtr_cnt = 1; 12575 rsm->r_rtr_bytes = 0; 12576 if (tp->t_flags & TF_SENTFIN) 12577 rsm->r_flags |= RACK_HAS_FIN; 12578 if ((tp->snd_una == tp->iss) && 12579 !TCPS_HAVEESTABLISHED(tp->t_state)) 12580 rsm->r_flags |= RACK_HAS_SYN; 12581 rsm->r_start = tp->snd_una; 12582 rsm->r_end = tp->snd_max; 12583 rsm->r_dupack = 0; 12584 if (rack->rc_inp->inp_socket->so_snd.sb_mb != NULL) { 12585 rsm->m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd, 0, &rsm->soff); 12586 if (rsm->m) 12587 rsm->orig_m_len = rsm->m->m_len; 12588 else 12589 rsm->orig_m_len = 0; 12590 } else { 12591 /* 12592 * This can happen if we have a stand-alone FIN or 12593 * SYN. 12594 */ 12595 rsm->m = NULL; 12596 rsm->orig_m_len = 0; 12597 rsm->soff = 0; 12598 } 12599 #ifndef INVARIANTS 12600 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 12601 #else 12602 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 12603 if (insret != NULL) { 12604 panic("Insert in rb tree fails ret:%p rack:%p rsm:%p", 12605 insret, rack, rsm); 12606 } 12607 #endif 12608 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext); 12609 rsm->r_in_tmap = 1; 12610 } 12611 /* 12612 * Timers in Rack are kept in microseconds so lets 12613 * convert any initial incoming variables 12614 * from ticks into usecs. Note that we 12615 * also change the values of t_srtt and t_rttvar, if 12616 * they are non-zero. They are kept with a 5 12617 * bit decimal so we have to carefully convert 12618 * these to get the full precision. 12619 */ 12620 rack_convert_rtts(tp); 12621 tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow); 12622 if (rack_do_hystart) { 12623 tp->t_ccv.flags |= CCF_HYSTART_ALLOWED; 12624 if (rack_do_hystart > 1) 12625 tp->t_ccv.flags |= CCF_HYSTART_CAN_SH_CWND; 12626 if (rack_do_hystart > 2) 12627 tp->t_ccv.flags |= CCF_HYSTART_CONS_SSTH; 12628 } 12629 if (rack_def_profile) 12630 rack_set_profile(rack, rack_def_profile); 12631 /* Cancel the GP measurement in progress */ 12632 tp->t_flags &= ~TF_GPUTINPROG; 12633 if (SEQ_GT(tp->snd_max, tp->iss)) 12634 snt = tp->snd_max - tp->iss; 12635 else 12636 snt = 0; 12637 iwin = rc_init_window(rack); 12638 if (snt < iwin) { 12639 /* We are not past the initial window 12640 * so we need to make sure cwnd is 12641 * correct. 12642 */ 12643 if (tp->snd_cwnd < iwin) 12644 tp->snd_cwnd = iwin; 12645 /* 12646 * If we are within the initial window 12647 * we want ssthresh to be unlimited. Setting 12648 * it to the rwnd (which the default stack does 12649 * and older racks) is not really a good idea 12650 * since we want to be in SS and grow both the 12651 * cwnd and the rwnd (via dynamic rwnd growth). If 12652 * we set it to the rwnd then as the peer grows its 12653 * rwnd we will be stuck in CA and never hit SS. 12654 * 12655 * Its far better to raise it up high (this takes the 12656 * risk that there as been a loss already, probably 12657 * we should have an indicator in all stacks of loss 12658 * but we don't), but considering the normal use this 12659 * is a risk worth taking. The consequences of not 12660 * hitting SS are far worse than going one more time 12661 * into it early on (before we have sent even a IW). 12662 * It is highly unlikely that we will have had a loss 12663 * before getting the IW out. 12664 */ 12665 tp->snd_ssthresh = 0xffffffff; 12666 } 12667 rack_stop_all_timers(tp); 12668 /* Lets setup the fsb block */ 12669 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0); 12670 rack_log_rtt_shrinks(rack, us_cts, tp->t_rxtcur, 12671 __LINE__, RACK_RTTS_INIT); 12672 return (0); 12673 } 12674 12675 static int 12676 rack_handoff_ok(struct tcpcb *tp) 12677 { 12678 if ((tp->t_state == TCPS_CLOSED) || 12679 (tp->t_state == TCPS_LISTEN)) { 12680 /* Sure no problem though it may not stick */ 12681 return (0); 12682 } 12683 if ((tp->t_state == TCPS_SYN_SENT) || 12684 (tp->t_state == TCPS_SYN_RECEIVED)) { 12685 /* 12686 * We really don't know if you support sack, 12687 * you have to get to ESTAB or beyond to tell. 12688 */ 12689 return (EAGAIN); 12690 } 12691 if ((tp->t_flags & TF_SENTFIN) && ((tp->snd_max - tp->snd_una) > 1)) { 12692 /* 12693 * Rack will only send a FIN after all data is acknowledged. 12694 * So in this case we have more data outstanding. We can't 12695 * switch stacks until either all data and only the FIN 12696 * is left (in which case rack_init() now knows how 12697 * to deal with that) <or> all is acknowledged and we 12698 * are only left with incoming data, though why you 12699 * would want to switch to rack after all data is acknowledged 12700 * I have no idea (rrs)! 12701 */ 12702 return (EAGAIN); 12703 } 12704 if ((tp->t_flags & TF_SACK_PERMIT) || rack_sack_not_required){ 12705 return (0); 12706 } 12707 /* 12708 * If we reach here we don't do SACK on this connection so we can 12709 * never do rack. 12710 */ 12711 return (EINVAL); 12712 } 12713 12714 12715 static void 12716 rack_fini(struct tcpcb *tp, int32_t tcb_is_purged) 12717 { 12718 struct inpcb *inp = tptoinpcb(tp); 12719 12720 if (tp->t_fb_ptr) { 12721 struct tcp_rack *rack; 12722 struct rack_sendmap *rsm, *nrsm; 12723 #ifdef INVARIANTS 12724 struct rack_sendmap *rm; 12725 #endif 12726 12727 rack = (struct tcp_rack *)tp->t_fb_ptr; 12728 if (tp->t_in_pkt) { 12729 /* 12730 * It is unsafe to process the packets since a 12731 * reset may be lurking in them (its rare but it 12732 * can occur). If we were to find a RST, then we 12733 * would end up dropping the connection and the 12734 * INP lock, so when we return the caller (tcp_usrreq) 12735 * will blow up when it trys to unlock the inp. 12736 */ 12737 struct mbuf *save, *m; 12738 12739 m = tp->t_in_pkt; 12740 tp->t_in_pkt = NULL; 12741 tp->t_tail_pkt = NULL; 12742 while (m) { 12743 save = m->m_nextpkt; 12744 m->m_nextpkt = NULL; 12745 m_freem(m); 12746 m = save; 12747 } 12748 } 12749 tp->t_flags &= ~TF_FORCEDATA; 12750 #ifdef NETFLIX_SHARED_CWND 12751 if (rack->r_ctl.rc_scw) { 12752 uint32_t limit; 12753 12754 if (rack->r_limit_scw) 12755 limit = max(1, rack->r_ctl.rc_lowest_us_rtt); 12756 else 12757 limit = 0; 12758 tcp_shared_cwnd_free_full(tp, rack->r_ctl.rc_scw, 12759 rack->r_ctl.rc_scw_index, 12760 limit); 12761 rack->r_ctl.rc_scw = NULL; 12762 } 12763 #endif 12764 if (rack->r_ctl.fsb.tcp_ip_hdr) { 12765 free(rack->r_ctl.fsb.tcp_ip_hdr, M_TCPFSB); 12766 rack->r_ctl.fsb.tcp_ip_hdr = NULL; 12767 rack->r_ctl.fsb.th = NULL; 12768 } 12769 /* Convert back to ticks, with */ 12770 if (tp->t_srtt > 1) { 12771 uint32_t val, frac; 12772 12773 val = USEC_2_TICKS(tp->t_srtt); 12774 frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz); 12775 tp->t_srtt = val << TCP_RTT_SHIFT; 12776 /* 12777 * frac is the fractional part here is left 12778 * over from converting to hz and shifting. 12779 * We need to convert this to the 5 bit 12780 * remainder. 12781 */ 12782 if (frac) { 12783 if (hz == 1000) { 12784 frac = (((uint64_t)frac * (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC); 12785 } else { 12786 frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC); 12787 } 12788 tp->t_srtt += frac; 12789 } 12790 } 12791 if (tp->t_rttvar) { 12792 uint32_t val, frac; 12793 12794 val = USEC_2_TICKS(tp->t_rttvar); 12795 frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz); 12796 tp->t_rttvar = val << TCP_RTTVAR_SHIFT; 12797 /* 12798 * frac is the fractional part here is left 12799 * over from converting to hz and shifting. 12800 * We need to convert this to the 5 bit 12801 * remainder. 12802 */ 12803 if (frac) { 12804 if (hz == 1000) { 12805 frac = (((uint64_t)frac * (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC); 12806 } else { 12807 frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC); 12808 } 12809 tp->t_rttvar += frac; 12810 } 12811 } 12812 tp->t_rxtcur = USEC_2_TICKS(tp->t_rxtcur); 12813 tp->t_rttlow = USEC_2_TICKS(tp->t_rttlow); 12814 if (rack->rc_always_pace) { 12815 tcp_decrement_paced_conn(); 12816 rack_undo_cc_pacing(rack); 12817 rack->rc_always_pace = 0; 12818 } 12819 /* Clean up any options if they were not applied */ 12820 while (!TAILQ_EMPTY(&rack->r_ctl.opt_list)) { 12821 struct deferred_opt_list *dol; 12822 12823 dol = TAILQ_FIRST(&rack->r_ctl.opt_list); 12824 TAILQ_REMOVE(&rack->r_ctl.opt_list, dol, next); 12825 free(dol, M_TCPDO); 12826 } 12827 /* rack does not use force data but other stacks may clear it */ 12828 if (rack->r_ctl.crte != NULL) { 12829 tcp_rel_pacing_rate(rack->r_ctl.crte, tp); 12830 rack->rack_hdrw_pacing = 0; 12831 rack->r_ctl.crte = NULL; 12832 } 12833 #ifdef TCP_BLACKBOX 12834 tcp_log_flowend(tp); 12835 #endif 12836 RB_FOREACH_SAFE(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm) { 12837 #ifndef INVARIANTS 12838 (void)RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 12839 #else 12840 rm = RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm); 12841 if (rm != rsm) { 12842 panic("At fini, rack:%p rsm:%p rm:%p", 12843 rack, rsm, rm); 12844 } 12845 #endif 12846 uma_zfree(rack_zone, rsm); 12847 } 12848 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free); 12849 while (rsm) { 12850 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext); 12851 uma_zfree(rack_zone, rsm); 12852 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free); 12853 } 12854 rack->rc_free_cnt = 0; 12855 uma_zfree(rack_pcb_zone, tp->t_fb_ptr); 12856 tp->t_fb_ptr = NULL; 12857 } 12858 inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ; 12859 inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY; 12860 inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE; 12861 inp->inp_flags2 &= ~INP_MBUF_ACKCMP; 12862 /* Cancel the GP measurement in progress */ 12863 tp->t_flags &= ~TF_GPUTINPROG; 12864 inp->inp_flags2 &= ~INP_MBUF_L_ACKS; 12865 /* Make sure snd_nxt is correctly set */ 12866 tp->snd_nxt = tp->snd_max; 12867 } 12868 12869 static void 12870 rack_set_state(struct tcpcb *tp, struct tcp_rack *rack) 12871 { 12872 if ((rack->r_state == TCPS_CLOSED) && (tp->t_state != TCPS_CLOSED)) { 12873 rack->r_is_v6 = (tptoinpcb(tp)->inp_vflag & INP_IPV6) != 0; 12874 } 12875 switch (tp->t_state) { 12876 case TCPS_SYN_SENT: 12877 rack->r_state = TCPS_SYN_SENT; 12878 rack->r_substate = rack_do_syn_sent; 12879 break; 12880 case TCPS_SYN_RECEIVED: 12881 rack->r_state = TCPS_SYN_RECEIVED; 12882 rack->r_substate = rack_do_syn_recv; 12883 break; 12884 case TCPS_ESTABLISHED: 12885 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12886 rack->r_state = TCPS_ESTABLISHED; 12887 rack->r_substate = rack_do_established; 12888 break; 12889 case TCPS_CLOSE_WAIT: 12890 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12891 rack->r_state = TCPS_CLOSE_WAIT; 12892 rack->r_substate = rack_do_close_wait; 12893 break; 12894 case TCPS_FIN_WAIT_1: 12895 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12896 rack->r_state = TCPS_FIN_WAIT_1; 12897 rack->r_substate = rack_do_fin_wait_1; 12898 break; 12899 case TCPS_CLOSING: 12900 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12901 rack->r_state = TCPS_CLOSING; 12902 rack->r_substate = rack_do_closing; 12903 break; 12904 case TCPS_LAST_ACK: 12905 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12906 rack->r_state = TCPS_LAST_ACK; 12907 rack->r_substate = rack_do_lastack; 12908 break; 12909 case TCPS_FIN_WAIT_2: 12910 rack_set_pace_segments(tp, rack, __LINE__, NULL); 12911 rack->r_state = TCPS_FIN_WAIT_2; 12912 rack->r_substate = rack_do_fin_wait_2; 12913 break; 12914 case TCPS_LISTEN: 12915 case TCPS_CLOSED: 12916 case TCPS_TIME_WAIT: 12917 default: 12918 break; 12919 }; 12920 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state)) 12921 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP; 12922 12923 } 12924 12925 static void 12926 rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb) 12927 { 12928 /* 12929 * We received an ack, and then did not 12930 * call send or were bounced out due to the 12931 * hpts was running. Now a timer is up as well, is 12932 * it the right timer? 12933 */ 12934 struct rack_sendmap *rsm; 12935 int tmr_up; 12936 12937 tmr_up = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK; 12938 if (rack->rc_in_persist && (tmr_up == PACE_TMR_PERSIT)) 12939 return; 12940 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 12941 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) && 12942 (tmr_up == PACE_TMR_RXT)) { 12943 /* Should be an RXT */ 12944 return; 12945 } 12946 if (rsm == NULL) { 12947 /* Nothing outstanding? */ 12948 if (tp->t_flags & TF_DELACK) { 12949 if (tmr_up == PACE_TMR_DELACK) 12950 /* We are supposed to have delayed ack up and we do */ 12951 return; 12952 } else if (sbavail(&tptosocket(tp)->so_snd) && (tmr_up == PACE_TMR_RXT)) { 12953 /* 12954 * if we hit enobufs then we would expect the possibility 12955 * of nothing outstanding and the RXT up (and the hptsi timer). 12956 */ 12957 return; 12958 } else if (((V_tcp_always_keepalive || 12959 rack->rc_inp->inp_socket->so_options & SO_KEEPALIVE) && 12960 (tp->t_state <= TCPS_CLOSING)) && 12961 (tmr_up == PACE_TMR_KEEP) && 12962 (tp->snd_max == tp->snd_una)) { 12963 /* We should have keep alive up and we do */ 12964 return; 12965 } 12966 } 12967 if (SEQ_GT(tp->snd_max, tp->snd_una) && 12968 ((tmr_up == PACE_TMR_TLP) || 12969 (tmr_up == PACE_TMR_RACK) || 12970 (tmr_up == PACE_TMR_RXT))) { 12971 /* 12972 * Either a Rack, TLP or RXT is fine if we 12973 * have outstanding data. 12974 */ 12975 return; 12976 } else if (tmr_up == PACE_TMR_DELACK) { 12977 /* 12978 * If the delayed ack was going to go off 12979 * before the rtx/tlp/rack timer were going to 12980 * expire, then that would be the timer in control. 12981 * Note we don't check the time here trusting the 12982 * code is correct. 12983 */ 12984 return; 12985 } 12986 /* 12987 * Ok the timer originally started is not what we want now. 12988 * We will force the hpts to be stopped if any, and restart 12989 * with the slot set to what was in the saved slot. 12990 */ 12991 if (tcp_in_hpts(rack->rc_inp)) { 12992 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { 12993 uint32_t us_cts; 12994 12995 us_cts = tcp_get_usecs(NULL); 12996 if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) { 12997 rack->r_early = 1; 12998 rack->r_ctl.rc_agg_early += (rack->r_ctl.rc_last_output_to - us_cts); 12999 } 13000 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT; 13001 } 13002 tcp_hpts_remove(rack->rc_inp); 13003 } 13004 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 13005 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0); 13006 } 13007 13008 13009 static void 13010 rack_do_win_updates(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tiwin, uint32_t seq, uint32_t ack, uint32_t cts, uint32_t high_seq) 13011 { 13012 if ((SEQ_LT(tp->snd_wl1, seq) || 13013 (tp->snd_wl1 == seq && (SEQ_LT(tp->snd_wl2, ack) || 13014 (tp->snd_wl2 == ack && tiwin > tp->snd_wnd))))) { 13015 /* keep track of pure window updates */ 13016 if ((tp->snd_wl2 == ack) && (tiwin > tp->snd_wnd)) 13017 KMOD_TCPSTAT_INC(tcps_rcvwinupd); 13018 tp->snd_wnd = tiwin; 13019 rack_validate_fo_sendwin_up(tp, rack); 13020 tp->snd_wl1 = seq; 13021 tp->snd_wl2 = ack; 13022 if (tp->snd_wnd > tp->max_sndwnd) 13023 tp->max_sndwnd = tp->snd_wnd; 13024 rack->r_wanted_output = 1; 13025 } else if ((tp->snd_wl2 == ack) && (tiwin < tp->snd_wnd)) { 13026 tp->snd_wnd = tiwin; 13027 rack_validate_fo_sendwin_up(tp, rack); 13028 tp->snd_wl1 = seq; 13029 tp->snd_wl2 = ack; 13030 } else { 13031 /* Not a valid win update */ 13032 return; 13033 } 13034 /* Do we exit persists? */ 13035 if ((rack->rc_in_persist != 0) && 13036 (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2), 13037 rack->r_ctl.rc_pace_min_segs))) { 13038 rack_exit_persist(tp, rack, cts); 13039 } 13040 /* Do we enter persists? */ 13041 if ((rack->rc_in_persist == 0) && 13042 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) && 13043 TCPS_HAVEESTABLISHED(tp->t_state) && 13044 ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) && 13045 sbavail(&tptosocket(tp)->so_snd) && 13046 (sbavail(&tptosocket(tp)->so_snd) > tp->snd_wnd)) { 13047 /* 13048 * Here the rwnd is less than 13049 * the pacing size, we are established, 13050 * nothing is outstanding, and there is 13051 * data to send. Enter persists. 13052 */ 13053 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime); 13054 } 13055 } 13056 13057 static void 13058 rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent *ae, int ackval, uint32_t high_seq) 13059 { 13060 13061 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 13062 struct inpcb *inp = tptoinpcb(tp); 13063 union tcp_log_stackspecific log; 13064 struct timeval ltv; 13065 char tcp_hdr_buf[60]; 13066 struct tcphdr *th; 13067 struct timespec ts; 13068 uint32_t orig_snd_una; 13069 uint8_t xx = 0; 13070 13071 #ifdef NETFLIX_HTTP_LOGGING 13072 struct http_sendfile_track *http_req; 13073 13074 if (SEQ_GT(ae->ack, tp->snd_una)) { 13075 http_req = tcp_http_find_req_for_seq(tp, (ae->ack-1)); 13076 } else { 13077 http_req = tcp_http_find_req_for_seq(tp, ae->ack); 13078 } 13079 #endif 13080 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 13081 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 13082 if (rack->rack_no_prr == 0) 13083 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; 13084 else 13085 log.u_bbr.flex1 = 0; 13086 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; 13087 log.u_bbr.use_lt_bw <<= 1; 13088 log.u_bbr.use_lt_bw |= rack->r_might_revert; 13089 log.u_bbr.flex2 = rack->r_ctl.rc_num_maps_alloced; 13090 log.u_bbr.inflight = ctf_flight_size(tp, rack->r_ctl.rc_sacked); 13091 log.u_bbr.pkts_out = tp->t_maxseg; 13092 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags; 13093 log.u_bbr.flex7 = 1; 13094 log.u_bbr.lost = ae->flags; 13095 log.u_bbr.cwnd_gain = ackval; 13096 log.u_bbr.pacing_gain = 0x2; 13097 if (ae->flags & TSTMP_HDWR) { 13098 /* Record the hardware timestamp if present */ 13099 log.u_bbr.flex3 = M_TSTMP; 13100 ts.tv_sec = ae->timestamp / 1000000000; 13101 ts.tv_nsec = ae->timestamp % 1000000000; 13102 ltv.tv_sec = ts.tv_sec; 13103 ltv.tv_usec = ts.tv_nsec / 1000; 13104 log.u_bbr.lt_epoch = tcp_tv_to_usectick(<v); 13105 } else if (ae->flags & TSTMP_LRO) { 13106 /* Record the LRO the arrival timestamp */ 13107 log.u_bbr.flex3 = M_TSTMP_LRO; 13108 ts.tv_sec = ae->timestamp / 1000000000; 13109 ts.tv_nsec = ae->timestamp % 1000000000; 13110 ltv.tv_sec = ts.tv_sec; 13111 ltv.tv_usec = ts.tv_nsec / 1000; 13112 log.u_bbr.flex5 = tcp_tv_to_usectick(<v); 13113 } 13114 log.u_bbr.timeStamp = tcp_get_usecs(<v); 13115 /* Log the rcv time */ 13116 log.u_bbr.delRate = ae->timestamp; 13117 #ifdef NETFLIX_HTTP_LOGGING 13118 log.u_bbr.applimited = tp->t_http_closed; 13119 log.u_bbr.applimited <<= 8; 13120 log.u_bbr.applimited |= tp->t_http_open; 13121 log.u_bbr.applimited <<= 8; 13122 log.u_bbr.applimited |= tp->t_http_req; 13123 if (http_req) { 13124 /* Copy out any client req info */ 13125 /* seconds */ 13126 log.u_bbr.pkt_epoch = (http_req->localtime / HPTS_USEC_IN_SEC); 13127 /* useconds */ 13128 log.u_bbr.delivered = (http_req->localtime % HPTS_USEC_IN_SEC); 13129 log.u_bbr.rttProp = http_req->timestamp; 13130 log.u_bbr.cur_del_rate = http_req->start; 13131 if (http_req->flags & TCP_HTTP_TRACK_FLG_OPEN) { 13132 log.u_bbr.flex8 |= 1; 13133 } else { 13134 log.u_bbr.flex8 |= 2; 13135 log.u_bbr.bw_inuse = http_req->end; 13136 } 13137 log.u_bbr.flex6 = http_req->start_seq; 13138 if (http_req->flags & TCP_HTTP_TRACK_FLG_COMP) { 13139 log.u_bbr.flex8 |= 4; 13140 log.u_bbr.epoch = http_req->end_seq; 13141 } 13142 } 13143 #endif 13144 memset(tcp_hdr_buf, 0, sizeof(tcp_hdr_buf)); 13145 th = (struct tcphdr *)tcp_hdr_buf; 13146 th->th_seq = ae->seq; 13147 th->th_ack = ae->ack; 13148 th->th_win = ae->win; 13149 /* Now fill in the ports */ 13150 th->th_sport = inp->inp_fport; 13151 th->th_dport = inp->inp_lport; 13152 tcp_set_flags(th, ae->flags); 13153 /* Now do we have a timestamp option? */ 13154 if (ae->flags & HAS_TSTMP) { 13155 u_char *cp; 13156 uint32_t val; 13157 13158 th->th_off = ((sizeof(struct tcphdr) + TCPOLEN_TSTAMP_APPA) >> 2); 13159 cp = (u_char *)(th + 1); 13160 *cp = TCPOPT_NOP; 13161 cp++; 13162 *cp = TCPOPT_NOP; 13163 cp++; 13164 *cp = TCPOPT_TIMESTAMP; 13165 cp++; 13166 *cp = TCPOLEN_TIMESTAMP; 13167 cp++; 13168 val = htonl(ae->ts_value); 13169 bcopy((char *)&val, 13170 (char *)cp, sizeof(uint32_t)); 13171 val = htonl(ae->ts_echo); 13172 bcopy((char *)&val, 13173 (char *)(cp + 4), sizeof(uint32_t)); 13174 } else 13175 th->th_off = (sizeof(struct tcphdr) >> 2); 13176 13177 /* 13178 * For sane logging we need to play a little trick. 13179 * If the ack were fully processed we would have moved 13180 * snd_una to high_seq, but since compressed acks are 13181 * processed in two phases, at this point (logging) snd_una 13182 * won't be advanced. So we would see multiple acks showing 13183 * the advancement. We can prevent that by "pretending" that 13184 * snd_una was advanced and then un-advancing it so that the 13185 * logging code has the right value for tlb_snd_una. 13186 */ 13187 if (tp->snd_una != high_seq) { 13188 orig_snd_una = tp->snd_una; 13189 tp->snd_una = high_seq; 13190 xx = 1; 13191 } else 13192 xx = 0; 13193 TCP_LOG_EVENTP(tp, th, 13194 &tptosocket(tp)->so_rcv, 13195 &tptosocket(tp)->so_snd, TCP_LOG_IN, 0, 13196 0, &log, true, <v); 13197 if (xx) { 13198 tp->snd_una = orig_snd_una; 13199 } 13200 } 13201 13202 } 13203 13204 static void 13205 rack_handle_probe_response(struct tcp_rack *rack, uint32_t tiwin, uint32_t us_cts) 13206 { 13207 uint32_t us_rtt; 13208 /* 13209 * A persist or keep-alive was forced out, update our 13210 * min rtt time. Note now worry about lost responses. 13211 * When a subsequent keep-alive or persist times out 13212 * and forced_ack is still on, then the last probe 13213 * was not responded to. In such cases we have a 13214 * sysctl that controls the behavior. Either we apply 13215 * the rtt but with reduced confidence (0). Or we just 13216 * plain don't apply the rtt estimate. Having data flow 13217 * will clear the probe_not_answered flag i.e. cum-ack 13218 * move forward <or> exiting and reentering persists. 13219 */ 13220 13221 rack->forced_ack = 0; 13222 rack->rc_tp->t_rxtshift = 0; 13223 if ((rack->rc_in_persist && 13224 (tiwin == rack->rc_tp->snd_wnd)) || 13225 (rack->rc_in_persist == 0)) { 13226 /* 13227 * In persists only apply the RTT update if this is 13228 * a response to our window probe. And that 13229 * means the rwnd sent must match the current 13230 * snd_wnd. If it does not, then we got a 13231 * window update ack instead. For keepalive 13232 * we allow the answer no matter what the window. 13233 * 13234 * Note that if the probe_not_answered is set then 13235 * the forced_ack_ts is the oldest one i.e. the first 13236 * probe sent that might have been lost. This assures 13237 * us that if we do calculate an RTT it is longer not 13238 * some short thing. 13239 */ 13240 if (rack->rc_in_persist) 13241 counter_u64_add(rack_persists_acks, 1); 13242 us_rtt = us_cts - rack->r_ctl.forced_ack_ts; 13243 if (us_rtt == 0) 13244 us_rtt = 1; 13245 if (rack->probe_not_answered == 0) { 13246 rack_apply_updated_usrtt(rack, us_rtt, us_cts); 13247 tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 3, NULL, 1); 13248 } else { 13249 /* We have a retransmitted probe here too */ 13250 if (rack_apply_rtt_with_reduced_conf) { 13251 rack_apply_updated_usrtt(rack, us_rtt, us_cts); 13252 tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1); 13253 } 13254 } 13255 } 13256 } 13257 13258 static int 13259 rack_do_compressed_ack_processing(struct tcpcb *tp, struct socket *so, struct mbuf *m, int nxt_pkt, struct timeval *tv) 13260 { 13261 /* 13262 * Handle a "special" compressed ack mbuf. Each incoming 13263 * ack has only four possible dispositions: 13264 * 13265 * A) It moves the cum-ack forward 13266 * B) It is behind the cum-ack. 13267 * C) It is a window-update ack. 13268 * D) It is a dup-ack. 13269 * 13270 * Note that we can have between 1 -> TCP_COMP_ACK_ENTRIES 13271 * in the incoming mbuf. We also need to still pay attention 13272 * to nxt_pkt since there may be another packet after this 13273 * one. 13274 */ 13275 #ifdef TCP_ACCOUNTING 13276 uint64_t ts_val; 13277 uint64_t rdstc; 13278 #endif 13279 int segsiz; 13280 struct timespec ts; 13281 struct tcp_rack *rack; 13282 struct tcp_ackent *ae; 13283 uint32_t tiwin, ms_cts, cts, acked, acked_amount, high_seq, win_seq, the_win, win_upd_ack; 13284 int cnt, i, did_out, ourfinisacked = 0; 13285 struct tcpopt to_holder, *to = NULL; 13286 #ifdef TCP_ACCOUNTING 13287 int win_up_req = 0; 13288 #endif 13289 int nsegs = 0; 13290 int under_pacing = 1; 13291 int recovery = 0; 13292 #ifdef TCP_ACCOUNTING 13293 sched_pin(); 13294 #endif 13295 rack = (struct tcp_rack *)tp->t_fb_ptr; 13296 if (rack->gp_ready && 13297 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) 13298 under_pacing = 0; 13299 else 13300 under_pacing = 1; 13301 13302 if (rack->r_state != tp->t_state) 13303 rack_set_state(tp, rack); 13304 if ((tp->t_state >= TCPS_FIN_WAIT_1) && 13305 (tp->t_flags & TF_GPUTINPROG)) { 13306 /* 13307 * We have a goodput in progress 13308 * and we have entered a late state. 13309 * Do we have enough data in the sb 13310 * to handle the GPUT request? 13311 */ 13312 uint32_t bytes; 13313 13314 bytes = tp->gput_ack - tp->gput_seq; 13315 if (SEQ_GT(tp->gput_seq, tp->snd_una)) 13316 bytes += tp->gput_seq - tp->snd_una; 13317 if (bytes > sbavail(&tptosocket(tp)->so_snd)) { 13318 /* 13319 * There are not enough bytes in the socket 13320 * buffer that have been sent to cover this 13321 * measurement. Cancel it. 13322 */ 13323 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, 13324 rack->r_ctl.rc_gp_srtt /*flex1*/, 13325 tp->gput_seq, 13326 0, 0, 18, __LINE__, NULL, 0); 13327 tp->t_flags &= ~TF_GPUTINPROG; 13328 } 13329 } 13330 to = &to_holder; 13331 to->to_flags = 0; 13332 KASSERT((m->m_len >= sizeof(struct tcp_ackent)), 13333 ("tp:%p m_cmpack:%p with invalid len:%u", tp, m, m->m_len)); 13334 cnt = m->m_len / sizeof(struct tcp_ackent); 13335 counter_u64_add(rack_multi_single_eq, cnt); 13336 high_seq = tp->snd_una; 13337 the_win = tp->snd_wnd; 13338 win_seq = tp->snd_wl1; 13339 win_upd_ack = tp->snd_wl2; 13340 cts = tcp_tv_to_usectick(tv); 13341 ms_cts = tcp_tv_to_mssectick(tv); 13342 rack->r_ctl.rc_rcvtime = cts; 13343 segsiz = ctf_fixed_maxseg(tp); 13344 if ((rack->rc_gp_dyn_mul) && 13345 (rack->use_fixed_rate == 0) && 13346 (rack->rc_always_pace)) { 13347 /* Check in on probertt */ 13348 rack_check_probe_rtt(rack, cts); 13349 } 13350 for (i = 0; i < cnt; i++) { 13351 #ifdef TCP_ACCOUNTING 13352 ts_val = get_cyclecount(); 13353 #endif 13354 rack_clear_rate_sample(rack); 13355 ae = ((mtod(m, struct tcp_ackent *)) + i); 13356 /* Setup the window */ 13357 tiwin = ae->win << tp->snd_scale; 13358 if (tiwin > rack->r_ctl.rc_high_rwnd) 13359 rack->r_ctl.rc_high_rwnd = tiwin; 13360 /* figure out the type of ack */ 13361 if (SEQ_LT(ae->ack, high_seq)) { 13362 /* Case B*/ 13363 ae->ack_val_set = ACK_BEHIND; 13364 } else if (SEQ_GT(ae->ack, high_seq)) { 13365 /* Case A */ 13366 ae->ack_val_set = ACK_CUMACK; 13367 } else if ((tiwin == the_win) && (rack->rc_in_persist == 0)){ 13368 /* Case D */ 13369 ae->ack_val_set = ACK_DUPACK; 13370 } else { 13371 /* Case C */ 13372 ae->ack_val_set = ACK_RWND; 13373 } 13374 rack_log_input_packet(tp, rack, ae, ae->ack_val_set, high_seq); 13375 /* Validate timestamp */ 13376 if (ae->flags & HAS_TSTMP) { 13377 /* Setup for a timestamp */ 13378 to->to_flags = TOF_TS; 13379 ae->ts_echo -= tp->ts_offset; 13380 to->to_tsecr = ae->ts_echo; 13381 to->to_tsval = ae->ts_value; 13382 /* 13383 * If echoed timestamp is later than the current time, fall back to 13384 * non RFC1323 RTT calculation. Normalize timestamp if syncookies 13385 * were used when this connection was established. 13386 */ 13387 if (TSTMP_GT(ae->ts_echo, ms_cts)) 13388 to->to_tsecr = 0; 13389 if (tp->ts_recent && 13390 TSTMP_LT(ae->ts_value, tp->ts_recent)) { 13391 if (ctf_ts_check_ac(tp, (ae->flags & 0xff))) { 13392 #ifdef TCP_ACCOUNTING 13393 rdstc = get_cyclecount(); 13394 if (rdstc > ts_val) { 13395 counter_u64_add(tcp_proc_time[ae->ack_val_set] , 13396 (rdstc - ts_val)); 13397 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13398 tp->tcp_proc_time[ae->ack_val_set] += (rdstc - ts_val); 13399 } 13400 } 13401 #endif 13402 continue; 13403 } 13404 } 13405 if (SEQ_LEQ(ae->seq, tp->last_ack_sent) && 13406 SEQ_LEQ(tp->last_ack_sent, ae->seq)) { 13407 tp->ts_recent_age = tcp_ts_getticks(); 13408 tp->ts_recent = ae->ts_value; 13409 } 13410 } else { 13411 /* Setup for a no options */ 13412 to->to_flags = 0; 13413 } 13414 /* Update the rcv time and perform idle reduction possibly */ 13415 if (tp->t_idle_reduce && 13416 (tp->snd_max == tp->snd_una) && 13417 (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) { 13418 counter_u64_add(rack_input_idle_reduces, 1); 13419 rack_cc_after_idle(rack, tp); 13420 } 13421 tp->t_rcvtime = ticks; 13422 /* Now what about ECN of a chain of pure ACKs? */ 13423 if (tcp_ecn_input_segment(tp, ae->flags, 0, 13424 tcp_packets_this_ack(tp, ae->ack), 13425 ae->codepoint)) 13426 rack_cong_signal(tp, CC_ECN, ae->ack, __LINE__); 13427 #ifdef TCP_ACCOUNTING 13428 /* Count for the specific type of ack in */ 13429 counter_u64_add(tcp_cnt_counters[ae->ack_val_set], 1); 13430 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13431 tp->tcp_cnt_counters[ae->ack_val_set]++; 13432 } 13433 #endif 13434 /* 13435 * Note how we could move up these in the determination 13436 * above, but we don't so that way the timestamp checks (and ECN) 13437 * is done first before we do any processing on the ACK. 13438 * The non-compressed path through the code has this 13439 * weakness (noted by @jtl) that it actually does some 13440 * processing before verifying the timestamp information. 13441 * We don't take that path here which is why we set 13442 * the ack_val_set first, do the timestamp and ecn 13443 * processing, and then look at what we have setup. 13444 */ 13445 if (ae->ack_val_set == ACK_BEHIND) { 13446 /* 13447 * Case B flag reordering, if window is not closed 13448 * or it could be a keep-alive or persists 13449 */ 13450 if (SEQ_LT(ae->ack, tp->snd_una) && (sbspace(&so->so_rcv) > segsiz)) { 13451 rack->r_ctl.rc_reorder_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 13452 } 13453 } else if (ae->ack_val_set == ACK_DUPACK) { 13454 /* Case D */ 13455 rack_strike_dupack(rack); 13456 } else if (ae->ack_val_set == ACK_RWND) { 13457 /* Case C */ 13458 if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) { 13459 ts.tv_sec = ae->timestamp / 1000000000; 13460 ts.tv_nsec = ae->timestamp % 1000000000; 13461 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec; 13462 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000; 13463 } else { 13464 rack->r_ctl.act_rcv_time = *tv; 13465 } 13466 if (rack->forced_ack) { 13467 rack_handle_probe_response(rack, tiwin, 13468 tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time)); 13469 } 13470 #ifdef TCP_ACCOUNTING 13471 win_up_req = 1; 13472 #endif 13473 win_upd_ack = ae->ack; 13474 win_seq = ae->seq; 13475 the_win = tiwin; 13476 rack_do_win_updates(tp, rack, the_win, win_seq, win_upd_ack, cts, high_seq); 13477 } else { 13478 /* Case A */ 13479 if (SEQ_GT(ae->ack, tp->snd_max)) { 13480 /* 13481 * We just send an ack since the incoming 13482 * ack is beyond the largest seq we sent. 13483 */ 13484 if ((tp->t_flags & TF_ACKNOW) == 0) { 13485 ctf_ack_war_checks(tp, &rack->r_ctl.challenge_ack_ts, &rack->r_ctl.challenge_ack_cnt); 13486 if (tp->t_flags && TF_ACKNOW) 13487 rack->r_wanted_output = 1; 13488 } 13489 } else { 13490 nsegs++; 13491 /* If the window changed setup to update */ 13492 if (tiwin != tp->snd_wnd) { 13493 win_upd_ack = ae->ack; 13494 win_seq = ae->seq; 13495 the_win = tiwin; 13496 rack_do_win_updates(tp, rack, the_win, win_seq, win_upd_ack, cts, high_seq); 13497 } 13498 #ifdef TCP_ACCOUNTING 13499 /* Account for the acks */ 13500 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13501 tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz); 13502 } 13503 counter_u64_add(tcp_cnt_counters[CNT_OF_ACKS_IN], 13504 (((ae->ack - high_seq) + segsiz - 1) / segsiz)); 13505 #endif 13506 high_seq = ae->ack; 13507 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 13508 union tcp_log_stackspecific log; 13509 struct timeval tv; 13510 13511 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 13512 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 13513 log.u_bbr.flex1 = high_seq; 13514 log.u_bbr.flex2 = rack->r_ctl.roundends; 13515 log.u_bbr.flex3 = rack->r_ctl.current_round; 13516 log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround; 13517 log.u_bbr.flex8 = 8; 13518 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0, 13519 0, &log, false, NULL, NULL, 0, &tv); 13520 } 13521 /* 13522 * The draft (v3) calls for us to use SEQ_GEQ, but that 13523 * causes issues when we are just going app limited. Lets 13524 * instead use SEQ_GT <or> where its equal but more data 13525 * is outstanding. 13526 */ 13527 if ((SEQ_GT(high_seq, rack->r_ctl.roundends)) || 13528 ((high_seq == rack->r_ctl.roundends) && 13529 SEQ_GT(tp->snd_max, tp->snd_una))) { 13530 rack->r_ctl.current_round++; 13531 rack->r_ctl.roundends = tp->snd_max; 13532 if (CC_ALGO(tp)->newround != NULL) { 13533 CC_ALGO(tp)->newround(&tp->t_ccv, rack->r_ctl.current_round); 13534 } 13535 } 13536 /* Setup our act_rcv_time */ 13537 if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) { 13538 ts.tv_sec = ae->timestamp / 1000000000; 13539 ts.tv_nsec = ae->timestamp % 1000000000; 13540 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec; 13541 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000; 13542 } else { 13543 rack->r_ctl.act_rcv_time = *tv; 13544 } 13545 rack_process_to_cumack(tp, rack, ae->ack, cts, to); 13546 if (rack->rc_dsack_round_seen) { 13547 /* Is the dsack round over? */ 13548 if (SEQ_GEQ(ae->ack, rack->r_ctl.dsack_round_end)) { 13549 /* Yes it is */ 13550 rack->rc_dsack_round_seen = 0; 13551 rack_log_dsack_event(rack, 3, __LINE__, 0, 0); 13552 } 13553 } 13554 } 13555 } 13556 /* And lets be sure to commit the rtt measurements for this ack */ 13557 tcp_rack_xmit_timer_commit(rack, tp); 13558 #ifdef TCP_ACCOUNTING 13559 rdstc = get_cyclecount(); 13560 if (rdstc > ts_val) { 13561 counter_u64_add(tcp_proc_time[ae->ack_val_set] , (rdstc - ts_val)); 13562 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13563 tp->tcp_proc_time[ae->ack_val_set] += (rdstc - ts_val); 13564 if (ae->ack_val_set == ACK_CUMACK) 13565 tp->tcp_proc_time[CYC_HANDLE_MAP] += (rdstc - ts_val); 13566 } 13567 } 13568 #endif 13569 } 13570 #ifdef TCP_ACCOUNTING 13571 ts_val = get_cyclecount(); 13572 #endif 13573 /* Tend to any collapsed window */ 13574 if (SEQ_GT(tp->snd_max, high_seq) && (tp->snd_wnd < (tp->snd_max - high_seq))) { 13575 /* The peer collapsed the window */ 13576 rack_collapsed_window(rack, (tp->snd_max - high_seq), __LINE__); 13577 } else if (rack->rc_has_collapsed) 13578 rack_un_collapse_window(rack, __LINE__); 13579 if ((rack->r_collapse_point_valid) && 13580 (SEQ_GT(high_seq, rack->r_ctl.high_collapse_point))) 13581 rack->r_collapse_point_valid = 0; 13582 acked_amount = acked = (high_seq - tp->snd_una); 13583 if (acked) { 13584 /* 13585 * Clear the probe not answered flag 13586 * since cum-ack moved forward. 13587 */ 13588 rack->probe_not_answered = 0; 13589 if (rack->sack_attack_disable == 0) 13590 rack_do_decay(rack); 13591 if (acked >= segsiz) { 13592 /* 13593 * You only get credit for 13594 * MSS and greater (and you get extra 13595 * credit for larger cum-ack moves). 13596 */ 13597 int ac; 13598 13599 ac = acked / segsiz; 13600 rack->r_ctl.ack_count += ac; 13601 counter_u64_add(rack_ack_total, ac); 13602 } 13603 if (rack->r_ctl.ack_count > 0xfff00000) { 13604 /* 13605 * reduce the number to keep us under 13606 * a uint32_t. 13607 */ 13608 rack->r_ctl.ack_count /= 2; 13609 rack->r_ctl.sack_count /= 2; 13610 } 13611 if (tp->t_flags & TF_NEEDSYN) { 13612 /* 13613 * T/TCP: Connection was half-synchronized, and our SYN has 13614 * been ACK'd (so connection is now fully synchronized). Go 13615 * to non-starred state, increment snd_una for ACK of SYN, 13616 * and check if we can do window scaling. 13617 */ 13618 tp->t_flags &= ~TF_NEEDSYN; 13619 tp->snd_una++; 13620 acked_amount = acked = (high_seq - tp->snd_una); 13621 } 13622 if (acked > sbavail(&so->so_snd)) 13623 acked_amount = sbavail(&so->so_snd); 13624 #ifdef NETFLIX_EXP_DETECTION 13625 /* 13626 * We only care on a cum-ack move if we are in a sack-disabled 13627 * state. We have already added in to the ack_count, and we never 13628 * would disable on a cum-ack move, so we only care to do the 13629 * detection if it may "undo" it, i.e. we were in disabled already. 13630 */ 13631 if (rack->sack_attack_disable) 13632 rack_do_detection(tp, rack, acked_amount, segsiz); 13633 #endif 13634 if (IN_FASTRECOVERY(tp->t_flags) && 13635 (rack->rack_no_prr == 0)) 13636 rack_update_prr(tp, rack, acked_amount, high_seq); 13637 if (IN_RECOVERY(tp->t_flags)) { 13638 if (SEQ_LT(high_seq, tp->snd_recover) && 13639 (SEQ_LT(high_seq, tp->snd_max))) { 13640 tcp_rack_partialack(tp); 13641 } else { 13642 rack_post_recovery(tp, high_seq); 13643 recovery = 1; 13644 } 13645 } 13646 /* Handle the rack-log-ack part (sendmap) */ 13647 if ((sbused(&so->so_snd) == 0) && 13648 (acked > acked_amount) && 13649 (tp->t_state >= TCPS_FIN_WAIT_1) && 13650 (tp->t_flags & TF_SENTFIN)) { 13651 /* 13652 * We must be sure our fin 13653 * was sent and acked (we can be 13654 * in FIN_WAIT_1 without having 13655 * sent the fin). 13656 */ 13657 ourfinisacked = 1; 13658 /* 13659 * Lets make sure snd_una is updated 13660 * since most likely acked_amount = 0 (it 13661 * should be). 13662 */ 13663 tp->snd_una = high_seq; 13664 } 13665 /* Did we make a RTO error? */ 13666 if ((tp->t_flags & TF_PREVVALID) && 13667 ((tp->t_flags & TF_RCVD_TSTMP) == 0)) { 13668 tp->t_flags &= ~TF_PREVVALID; 13669 if (tp->t_rxtshift == 1 && 13670 (int)(ticks - tp->t_badrxtwin) < 0) 13671 rack_cong_signal(tp, CC_RTO_ERR, high_seq, __LINE__); 13672 } 13673 /* Handle the data in the socket buffer */ 13674 KMOD_TCPSTAT_ADD(tcps_rcvackpack, 1); 13675 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked); 13676 if (acked_amount > 0) { 13677 struct mbuf *mfree; 13678 13679 rack_ack_received(tp, rack, high_seq, nsegs, CC_ACK, recovery); 13680 SOCKBUF_LOCK(&so->so_snd); 13681 mfree = sbcut_locked(&so->so_snd, acked_amount); 13682 tp->snd_una = high_seq; 13683 /* Note we want to hold the sb lock through the sendmap adjust */ 13684 rack_adjust_sendmap(rack, &so->so_snd, tp->snd_una); 13685 /* Wake up the socket if we have room to write more */ 13686 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2); 13687 sowwakeup_locked(so); 13688 m_freem(mfree); 13689 } 13690 /* update progress */ 13691 tp->t_acktime = ticks; 13692 rack_log_progress_event(rack, tp, tp->t_acktime, 13693 PROGRESS_UPDATE, __LINE__); 13694 /* Clear out shifts and such */ 13695 tp->t_rxtshift = 0; 13696 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 13697 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop); 13698 rack->rc_tlp_in_progress = 0; 13699 rack->r_ctl.rc_tlp_cnt_out = 0; 13700 /* Send recover and snd_nxt must be dragged along */ 13701 if (SEQ_GT(tp->snd_una, tp->snd_recover)) 13702 tp->snd_recover = tp->snd_una; 13703 if (SEQ_LT(tp->snd_nxt, tp->snd_una)) 13704 tp->snd_nxt = tp->snd_una; 13705 /* 13706 * If the RXT timer is running we want to 13707 * stop it, so we can restart a TLP (or new RXT). 13708 */ 13709 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) 13710 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 13711 #ifdef NETFLIX_HTTP_LOGGING 13712 tcp_http_check_for_comp(rack->rc_tp, high_seq); 13713 #endif 13714 tp->snd_wl2 = high_seq; 13715 tp->t_dupacks = 0; 13716 if (under_pacing && 13717 (rack->use_fixed_rate == 0) && 13718 (rack->in_probe_rtt == 0) && 13719 rack->rc_gp_dyn_mul && 13720 rack->rc_always_pace) { 13721 /* Check if we are dragging bottom */ 13722 rack_check_bottom_drag(tp, rack, so, acked); 13723 } 13724 if (tp->snd_una == tp->snd_max) { 13725 tp->t_flags &= ~TF_PREVVALID; 13726 rack->r_ctl.retran_during_recovery = 0; 13727 rack->r_ctl.dsack_byte_cnt = 0; 13728 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL); 13729 if (rack->r_ctl.rc_went_idle_time == 0) 13730 rack->r_ctl.rc_went_idle_time = 1; 13731 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__); 13732 if (sbavail(&tptosocket(tp)->so_snd) == 0) 13733 tp->t_acktime = 0; 13734 /* Set so we might enter persists... */ 13735 rack->r_wanted_output = 1; 13736 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 13737 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); 13738 if ((tp->t_state >= TCPS_FIN_WAIT_1) && 13739 (sbavail(&so->so_snd) == 0) && 13740 (tp->t_flags2 & TF2_DROP_AF_DATA)) { 13741 /* 13742 * The socket was gone and the 13743 * peer sent data (not now in the past), time to 13744 * reset him. 13745 */ 13746 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__); 13747 /* tcp_close will kill the inp pre-log the Reset */ 13748 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 13749 #ifdef TCP_ACCOUNTING 13750 rdstc = get_cyclecount(); 13751 if (rdstc > ts_val) { 13752 counter_u64_add(tcp_proc_time[ACK_CUMACK] , (rdstc - ts_val)); 13753 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13754 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val); 13755 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val); 13756 } 13757 } 13758 #endif 13759 m_freem(m); 13760 tp = tcp_close(tp); 13761 if (tp == NULL) { 13762 #ifdef TCP_ACCOUNTING 13763 sched_unpin(); 13764 #endif 13765 return (1); 13766 } 13767 /* 13768 * We would normally do drop-with-reset which would 13769 * send back a reset. We can't since we don't have 13770 * all the needed bits. Instead lets arrange for 13771 * a call to tcp_output(). That way since we 13772 * are in the closed state we will generate a reset. 13773 * 13774 * Note if tcp_accounting is on we don't unpin since 13775 * we do that after the goto label. 13776 */ 13777 goto send_out_a_rst; 13778 } 13779 if ((sbused(&so->so_snd) == 0) && 13780 (tp->t_state >= TCPS_FIN_WAIT_1) && 13781 (tp->t_flags & TF_SENTFIN)) { 13782 /* 13783 * If we can't receive any more data, then closing user can 13784 * proceed. Starting the timer is contrary to the 13785 * specification, but if we don't get a FIN we'll hang 13786 * forever. 13787 * 13788 */ 13789 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 13790 soisdisconnected(so); 13791 tcp_timer_activate(tp, TT_2MSL, 13792 (tcp_fast_finwait2_recycle ? 13793 tcp_finwait2_timeout : 13794 TP_MAXIDLE(tp))); 13795 } 13796 if (ourfinisacked == 0) { 13797 /* 13798 * We don't change to fin-wait-2 if we have our fin acked 13799 * which means we are probably in TCPS_CLOSING. 13800 */ 13801 tcp_state_change(tp, TCPS_FIN_WAIT_2); 13802 } 13803 } 13804 } 13805 /* Wake up the socket if we have room to write more */ 13806 if (sbavail(&so->so_snd)) { 13807 rack->r_wanted_output = 1; 13808 if (ctf_progress_timeout_check(tp, true)) { 13809 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr, 13810 tp, tick, PROGRESS_DROP, __LINE__); 13811 /* 13812 * We cheat here and don't send a RST, we should send one 13813 * when the pacer drops the connection. 13814 */ 13815 #ifdef TCP_ACCOUNTING 13816 rdstc = get_cyclecount(); 13817 if (rdstc > ts_val) { 13818 counter_u64_add(tcp_proc_time[ACK_CUMACK] , (rdstc - ts_val)); 13819 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13820 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val); 13821 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val); 13822 } 13823 } 13824 sched_unpin(); 13825 #endif 13826 (void)tcp_drop(tp, ETIMEDOUT); 13827 m_freem(m); 13828 return (1); 13829 } 13830 } 13831 if (ourfinisacked) { 13832 switch(tp->t_state) { 13833 case TCPS_CLOSING: 13834 #ifdef TCP_ACCOUNTING 13835 rdstc = get_cyclecount(); 13836 if (rdstc > ts_val) { 13837 counter_u64_add(tcp_proc_time[ACK_CUMACK] , 13838 (rdstc - ts_val)); 13839 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13840 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val); 13841 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val); 13842 } 13843 } 13844 sched_unpin(); 13845 #endif 13846 tcp_twstart(tp); 13847 m_freem(m); 13848 return (1); 13849 break; 13850 case TCPS_LAST_ACK: 13851 #ifdef TCP_ACCOUNTING 13852 rdstc = get_cyclecount(); 13853 if (rdstc > ts_val) { 13854 counter_u64_add(tcp_proc_time[ACK_CUMACK] , 13855 (rdstc - ts_val)); 13856 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13857 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val); 13858 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val); 13859 } 13860 } 13861 sched_unpin(); 13862 #endif 13863 tp = tcp_close(tp); 13864 ctf_do_drop(m, tp); 13865 return (1); 13866 break; 13867 case TCPS_FIN_WAIT_1: 13868 #ifdef TCP_ACCOUNTING 13869 rdstc = get_cyclecount(); 13870 if (rdstc > ts_val) { 13871 counter_u64_add(tcp_proc_time[ACK_CUMACK] , 13872 (rdstc - ts_val)); 13873 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13874 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val); 13875 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val); 13876 } 13877 } 13878 #endif 13879 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 13880 soisdisconnected(so); 13881 tcp_timer_activate(tp, TT_2MSL, 13882 (tcp_fast_finwait2_recycle ? 13883 tcp_finwait2_timeout : 13884 TP_MAXIDLE(tp))); 13885 } 13886 tcp_state_change(tp, TCPS_FIN_WAIT_2); 13887 break; 13888 default: 13889 break; 13890 } 13891 } 13892 if (rack->r_fast_output) { 13893 /* 13894 * We re doing fast output.. can we expand that? 13895 */ 13896 rack_gain_for_fastoutput(rack, tp, so, acked_amount); 13897 } 13898 #ifdef TCP_ACCOUNTING 13899 rdstc = get_cyclecount(); 13900 if (rdstc > ts_val) { 13901 counter_u64_add(tcp_proc_time[ACK_CUMACK] , (rdstc - ts_val)); 13902 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13903 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val); 13904 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val); 13905 } 13906 } 13907 13908 } else if (win_up_req) { 13909 rdstc = get_cyclecount(); 13910 if (rdstc > ts_val) { 13911 counter_u64_add(tcp_proc_time[ACK_RWND] , (rdstc - ts_val)); 13912 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 13913 tp->tcp_proc_time[ACK_RWND] += (rdstc - ts_val); 13914 } 13915 } 13916 #endif 13917 } 13918 /* Now is there a next packet, if so we are done */ 13919 m_freem(m); 13920 did_out = 0; 13921 if (nxt_pkt) { 13922 #ifdef TCP_ACCOUNTING 13923 sched_unpin(); 13924 #endif 13925 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, 5, nsegs); 13926 return (0); 13927 } 13928 rack_handle_might_revert(tp, rack); 13929 ctf_calc_rwin(so, tp); 13930 if ((rack->r_wanted_output != 0) || (rack->r_fast_output != 0)) { 13931 send_out_a_rst: 13932 if (tcp_output(tp) < 0) { 13933 #ifdef TCP_ACCOUNTING 13934 sched_unpin(); 13935 #endif 13936 return (1); 13937 } 13938 did_out = 1; 13939 } 13940 rack_free_trim(rack); 13941 #ifdef TCP_ACCOUNTING 13942 sched_unpin(); 13943 #endif 13944 rack_timer_audit(tp, rack, &so->so_snd); 13945 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, 6, nsegs); 13946 return (0); 13947 } 13948 13949 13950 static int 13951 rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so, 13952 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, 13953 int32_t nxt_pkt, struct timeval *tv) 13954 { 13955 struct inpcb *inp = tptoinpcb(tp); 13956 #ifdef TCP_ACCOUNTING 13957 uint64_t ts_val; 13958 #endif 13959 int32_t thflags, retval, did_out = 0; 13960 int32_t way_out = 0; 13961 /* 13962 * cts - is the current time from tv (caller gets ts) in microseconds. 13963 * ms_cts - is the current time from tv in milliseconds. 13964 * us_cts - is the time that LRO or hardware actually got the packet in microseconds. 13965 */ 13966 uint32_t cts, us_cts, ms_cts; 13967 uint32_t tiwin, high_seq; 13968 struct timespec ts; 13969 struct tcpopt to; 13970 struct tcp_rack *rack; 13971 struct rack_sendmap *rsm; 13972 int32_t prev_state = 0; 13973 #ifdef TCP_ACCOUNTING 13974 int ack_val_set = 0xf; 13975 #endif 13976 int nsegs; 13977 13978 NET_EPOCH_ASSERT(); 13979 INP_WLOCK_ASSERT(inp); 13980 13981 /* 13982 * tv passed from common code is from either M_TSTMP_LRO or 13983 * tcp_get_usecs() if no LRO m_pkthdr timestamp is present. 13984 */ 13985 rack = (struct tcp_rack *)tp->t_fb_ptr; 13986 if (m->m_flags & M_ACKCMP) { 13987 /* 13988 * All compressed ack's are ack's by definition so 13989 * remove any ack required flag and then do the processing. 13990 */ 13991 rack->rc_ack_required = 0; 13992 return (rack_do_compressed_ack_processing(tp, so, m, nxt_pkt, tv)); 13993 } 13994 if (m->m_flags & M_ACKCMP) { 13995 panic("Impossible reach m has ackcmp? m:%p tp:%p", m, tp); 13996 } 13997 cts = tcp_tv_to_usectick(tv); 13998 ms_cts = tcp_tv_to_mssectick(tv); 13999 nsegs = m->m_pkthdr.lro_nsegs; 14000 counter_u64_add(rack_proc_non_comp_ack, 1); 14001 thflags = tcp_get_flags(th); 14002 #ifdef TCP_ACCOUNTING 14003 sched_pin(); 14004 if (thflags & TH_ACK) 14005 ts_val = get_cyclecount(); 14006 #endif 14007 if ((m->m_flags & M_TSTMP) || 14008 (m->m_flags & M_TSTMP_LRO)) { 14009 mbuf_tstmp2timespec(m, &ts); 14010 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec; 14011 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000; 14012 } else 14013 rack->r_ctl.act_rcv_time = *tv; 14014 kern_prefetch(rack, &prev_state); 14015 prev_state = 0; 14016 /* 14017 * Unscale the window into a 32-bit value. For the SYN_SENT state 14018 * the scale is zero. 14019 */ 14020 tiwin = th->th_win << tp->snd_scale; 14021 #ifdef TCP_ACCOUNTING 14022 if (thflags & TH_ACK) { 14023 /* 14024 * We have a tradeoff here. We can either do what we are 14025 * doing i.e. pinning to this CPU and then doing the accounting 14026 * <or> we could do a critical enter, setup the rdtsc and cpu 14027 * as in below, and then validate we are on the same CPU on 14028 * exit. I have choosen to not do the critical enter since 14029 * that often will gain you a context switch, and instead lock 14030 * us (line above this if) to the same CPU with sched_pin(). This 14031 * means we may be context switched out for a higher priority 14032 * interupt but we won't be moved to another CPU. 14033 * 14034 * If this occurs (which it won't very often since we most likely 14035 * are running this code in interupt context and only a higher 14036 * priority will bump us ... clock?) we will falsely add in 14037 * to the time the interupt processing time plus the ack processing 14038 * time. This is ok since its a rare event. 14039 */ 14040 ack_val_set = tcp_do_ack_accounting(tp, th, &to, tiwin, 14041 ctf_fixed_maxseg(tp)); 14042 } 14043 #endif 14044 /* 14045 * Parse options on any incoming segment. 14046 */ 14047 memset(&to, 0, sizeof(to)); 14048 tcp_dooptions(&to, (u_char *)(th + 1), 14049 (th->th_off << 2) - sizeof(struct tcphdr), 14050 (thflags & TH_SYN) ? TO_SYN : 0); 14051 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN", 14052 __func__)); 14053 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT", 14054 __func__)); 14055 14056 if ((tp->t_state >= TCPS_FIN_WAIT_1) && 14057 (tp->t_flags & TF_GPUTINPROG)) { 14058 /* 14059 * We have a goodput in progress 14060 * and we have entered a late state. 14061 * Do we have enough data in the sb 14062 * to handle the GPUT request? 14063 */ 14064 uint32_t bytes; 14065 14066 bytes = tp->gput_ack - tp->gput_seq; 14067 if (SEQ_GT(tp->gput_seq, tp->snd_una)) 14068 bytes += tp->gput_seq - tp->snd_una; 14069 if (bytes > sbavail(&tptosocket(tp)->so_snd)) { 14070 /* 14071 * There are not enough bytes in the socket 14072 * buffer that have been sent to cover this 14073 * measurement. Cancel it. 14074 */ 14075 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, 14076 rack->r_ctl.rc_gp_srtt /*flex1*/, 14077 tp->gput_seq, 14078 0, 0, 18, __LINE__, NULL, 0); 14079 tp->t_flags &= ~TF_GPUTINPROG; 14080 } 14081 } 14082 high_seq = th->th_ack; 14083 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 14084 union tcp_log_stackspecific log; 14085 struct timeval ltv; 14086 #ifdef NETFLIX_HTTP_LOGGING 14087 struct http_sendfile_track *http_req; 14088 14089 if (SEQ_GT(th->th_ack, tp->snd_una)) { 14090 http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1)); 14091 } else { 14092 http_req = tcp_http_find_req_for_seq(tp, th->th_ack); 14093 } 14094 #endif 14095 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 14096 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 14097 if (rack->rack_no_prr == 0) 14098 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; 14099 else 14100 log.u_bbr.flex1 = 0; 14101 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns; 14102 log.u_bbr.use_lt_bw <<= 1; 14103 log.u_bbr.use_lt_bw |= rack->r_might_revert; 14104 log.u_bbr.flex2 = rack->r_ctl.rc_num_maps_alloced; 14105 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 14106 log.u_bbr.pkts_out = rack->rc_tp->t_maxseg; 14107 log.u_bbr.flex3 = m->m_flags; 14108 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags; 14109 log.u_bbr.lost = thflags; 14110 log.u_bbr.pacing_gain = 0x1; 14111 #ifdef TCP_ACCOUNTING 14112 log.u_bbr.cwnd_gain = ack_val_set; 14113 #endif 14114 log.u_bbr.flex7 = 2; 14115 if (m->m_flags & M_TSTMP) { 14116 /* Record the hardware timestamp if present */ 14117 mbuf_tstmp2timespec(m, &ts); 14118 ltv.tv_sec = ts.tv_sec; 14119 ltv.tv_usec = ts.tv_nsec / 1000; 14120 log.u_bbr.lt_epoch = tcp_tv_to_usectick(<v); 14121 } else if (m->m_flags & M_TSTMP_LRO) { 14122 /* Record the LRO the arrival timestamp */ 14123 mbuf_tstmp2timespec(m, &ts); 14124 ltv.tv_sec = ts.tv_sec; 14125 ltv.tv_usec = ts.tv_nsec / 1000; 14126 log.u_bbr.flex5 = tcp_tv_to_usectick(<v); 14127 } 14128 log.u_bbr.timeStamp = tcp_get_usecs(<v); 14129 /* Log the rcv time */ 14130 log.u_bbr.delRate = m->m_pkthdr.rcv_tstmp; 14131 #ifdef NETFLIX_HTTP_LOGGING 14132 log.u_bbr.applimited = tp->t_http_closed; 14133 log.u_bbr.applimited <<= 8; 14134 log.u_bbr.applimited |= tp->t_http_open; 14135 log.u_bbr.applimited <<= 8; 14136 log.u_bbr.applimited |= tp->t_http_req; 14137 if (http_req) { 14138 /* Copy out any client req info */ 14139 /* seconds */ 14140 log.u_bbr.pkt_epoch = (http_req->localtime / HPTS_USEC_IN_SEC); 14141 /* useconds */ 14142 log.u_bbr.delivered = (http_req->localtime % HPTS_USEC_IN_SEC); 14143 log.u_bbr.rttProp = http_req->timestamp; 14144 log.u_bbr.cur_del_rate = http_req->start; 14145 if (http_req->flags & TCP_HTTP_TRACK_FLG_OPEN) { 14146 log.u_bbr.flex8 |= 1; 14147 } else { 14148 log.u_bbr.flex8 |= 2; 14149 log.u_bbr.bw_inuse = http_req->end; 14150 } 14151 log.u_bbr.flex6 = http_req->start_seq; 14152 if (http_req->flags & TCP_HTTP_TRACK_FLG_COMP) { 14153 log.u_bbr.flex8 |= 4; 14154 log.u_bbr.epoch = http_req->end_seq; 14155 } 14156 } 14157 #endif 14158 TCP_LOG_EVENTP(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0, 14159 tlen, &log, true, <v); 14160 } 14161 /* Remove ack required flag if set, we have one */ 14162 if (thflags & TH_ACK) 14163 rack->rc_ack_required = 0; 14164 if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) { 14165 way_out = 4; 14166 retval = 0; 14167 m_freem(m); 14168 goto done_with_input; 14169 } 14170 /* 14171 * If a segment with the ACK-bit set arrives in the SYN-SENT state 14172 * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9. 14173 */ 14174 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) && 14175 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) { 14176 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT); 14177 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen); 14178 #ifdef TCP_ACCOUNTING 14179 sched_unpin(); 14180 #endif 14181 return (1); 14182 } 14183 /* 14184 * If timestamps were negotiated during SYN/ACK and a 14185 * segment without a timestamp is received, silently drop 14186 * the segment, unless it is a RST segment or missing timestamps are 14187 * tolerated. 14188 * See section 3.2 of RFC 7323. 14189 */ 14190 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) && 14191 ((thflags & TH_RST) == 0) && (V_tcp_tolerate_missing_ts == 0)) { 14192 way_out = 5; 14193 retval = 0; 14194 m_freem(m); 14195 goto done_with_input; 14196 } 14197 14198 /* 14199 * Segment received on connection. Reset idle time and keep-alive 14200 * timer. XXX: This should be done after segment validation to 14201 * ignore broken/spoofed segs. 14202 */ 14203 if (tp->t_idle_reduce && 14204 (tp->snd_max == tp->snd_una) && 14205 (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) { 14206 counter_u64_add(rack_input_idle_reduces, 1); 14207 rack_cc_after_idle(rack, tp); 14208 } 14209 tp->t_rcvtime = ticks; 14210 #ifdef STATS 14211 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin); 14212 #endif 14213 if (tiwin > rack->r_ctl.rc_high_rwnd) 14214 rack->r_ctl.rc_high_rwnd = tiwin; 14215 /* 14216 * TCP ECN processing. XXXJTL: If we ever use ECN, we need to move 14217 * this to occur after we've validated the segment. 14218 */ 14219 if (tcp_ecn_input_segment(tp, thflags, tlen, 14220 tcp_packets_this_ack(tp, th->th_ack), 14221 iptos)) 14222 rack_cong_signal(tp, CC_ECN, th->th_ack, __LINE__); 14223 14224 /* 14225 * If echoed timestamp is later than the current time, fall back to 14226 * non RFC1323 RTT calculation. Normalize timestamp if syncookies 14227 * were used when this connection was established. 14228 */ 14229 if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) { 14230 to.to_tsecr -= tp->ts_offset; 14231 if (TSTMP_GT(to.to_tsecr, ms_cts)) 14232 to.to_tsecr = 0; 14233 } 14234 14235 /* 14236 * If its the first time in we need to take care of options and 14237 * verify we can do SACK for rack! 14238 */ 14239 if (rack->r_state == 0) { 14240 /* Should be init'd by rack_init() */ 14241 KASSERT(rack->rc_inp != NULL, 14242 ("%s: rack->rc_inp unexpectedly NULL", __func__)); 14243 if (rack->rc_inp == NULL) { 14244 rack->rc_inp = inp; 14245 } 14246 14247 /* 14248 * Process options only when we get SYN/ACK back. The SYN 14249 * case for incoming connections is handled in tcp_syncache. 14250 * According to RFC1323 the window field in a SYN (i.e., a 14251 * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX 14252 * this is traditional behavior, may need to be cleaned up. 14253 */ 14254 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) { 14255 /* Handle parallel SYN for ECN */ 14256 tcp_ecn_input_parallel_syn(tp, thflags, iptos); 14257 if ((to.to_flags & TOF_SCALE) && 14258 (tp->t_flags & TF_REQ_SCALE)) { 14259 tp->t_flags |= TF_RCVD_SCALE; 14260 tp->snd_scale = to.to_wscale; 14261 } else 14262 tp->t_flags &= ~TF_REQ_SCALE; 14263 /* 14264 * Initial send window. It will be updated with the 14265 * next incoming segment to the scaled value. 14266 */ 14267 tp->snd_wnd = th->th_win; 14268 rack_validate_fo_sendwin_up(tp, rack); 14269 if ((to.to_flags & TOF_TS) && 14270 (tp->t_flags & TF_REQ_TSTMP)) { 14271 tp->t_flags |= TF_RCVD_TSTMP; 14272 tp->ts_recent = to.to_tsval; 14273 tp->ts_recent_age = cts; 14274 } else 14275 tp->t_flags &= ~TF_REQ_TSTMP; 14276 if (to.to_flags & TOF_MSS) { 14277 tcp_mss(tp, to.to_mss); 14278 } 14279 if ((tp->t_flags & TF_SACK_PERMIT) && 14280 (to.to_flags & TOF_SACKPERM) == 0) 14281 tp->t_flags &= ~TF_SACK_PERMIT; 14282 if (IS_FASTOPEN(tp->t_flags)) { 14283 if (to.to_flags & TOF_FASTOPEN) { 14284 uint16_t mss; 14285 14286 if (to.to_flags & TOF_MSS) 14287 mss = to.to_mss; 14288 else 14289 if ((inp->inp_vflag & INP_IPV6) != 0) 14290 mss = TCP6_MSS; 14291 else 14292 mss = TCP_MSS; 14293 tcp_fastopen_update_cache(tp, mss, 14294 to.to_tfo_len, to.to_tfo_cookie); 14295 } else 14296 tcp_fastopen_disable_path(tp); 14297 } 14298 } 14299 /* 14300 * At this point we are at the initial call. Here we decide 14301 * if we are doing RACK or not. We do this by seeing if 14302 * TF_SACK_PERMIT is set and the sack-not-required is clear. 14303 * The code now does do dup-ack counting so if you don't 14304 * switch back you won't get rack & TLP, but you will still 14305 * get this stack. 14306 */ 14307 14308 if ((rack_sack_not_required == 0) && 14309 ((tp->t_flags & TF_SACK_PERMIT) == 0)) { 14310 tcp_switch_back_to_default(tp); 14311 (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen, 14312 tlen, iptos); 14313 #ifdef TCP_ACCOUNTING 14314 sched_unpin(); 14315 #endif 14316 return (1); 14317 } 14318 tcp_set_hpts(inp); 14319 sack_filter_clear(&rack->r_ctl.rack_sf, th->th_ack); 14320 } 14321 if (thflags & TH_FIN) 14322 tcp_log_end_status(tp, TCP_EI_STATUS_CLIENT_FIN); 14323 us_cts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 14324 if ((rack->rc_gp_dyn_mul) && 14325 (rack->use_fixed_rate == 0) && 14326 (rack->rc_always_pace)) { 14327 /* Check in on probertt */ 14328 rack_check_probe_rtt(rack, us_cts); 14329 } 14330 rack_clear_rate_sample(rack); 14331 if ((rack->forced_ack) && 14332 ((tcp_get_flags(th) & TH_RST) == 0)) { 14333 rack_handle_probe_response(rack, tiwin, us_cts); 14334 } 14335 /* 14336 * This is the one exception case where we set the rack state 14337 * always. All other times (timers etc) we must have a rack-state 14338 * set (so we assure we have done the checks above for SACK). 14339 */ 14340 rack->r_ctl.rc_rcvtime = cts; 14341 if (rack->r_state != tp->t_state) 14342 rack_set_state(tp, rack); 14343 if (SEQ_GT(th->th_ack, tp->snd_una) && 14344 (rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree)) != NULL) 14345 kern_prefetch(rsm, &prev_state); 14346 prev_state = rack->r_state; 14347 retval = (*rack->r_substate) (m, th, so, 14348 tp, &to, drop_hdrlen, 14349 tlen, tiwin, thflags, nxt_pkt, iptos); 14350 if (retval == 0) { 14351 /* 14352 * If retval is 1 the tcb is unlocked and most likely the tp 14353 * is gone. 14354 */ 14355 INP_WLOCK_ASSERT(inp); 14356 if ((rack->rc_gp_dyn_mul) && 14357 (rack->rc_always_pace) && 14358 (rack->use_fixed_rate == 0) && 14359 rack->in_probe_rtt && 14360 (rack->r_ctl.rc_time_probertt_starts == 0)) { 14361 /* 14362 * If we are going for target, lets recheck before 14363 * we output. 14364 */ 14365 rack_check_probe_rtt(rack, us_cts); 14366 } 14367 if (rack->set_pacing_done_a_iw == 0) { 14368 /* How much has been acked? */ 14369 if ((tp->snd_una - tp->iss) > (ctf_fixed_maxseg(tp) * 10)) { 14370 /* We have enough to set in the pacing segment size */ 14371 rack->set_pacing_done_a_iw = 1; 14372 rack_set_pace_segments(tp, rack, __LINE__, NULL); 14373 } 14374 } 14375 tcp_rack_xmit_timer_commit(rack, tp); 14376 #ifdef TCP_ACCOUNTING 14377 /* 14378 * If we set the ack_val_se to what ack processing we are doing 14379 * we also want to track how many cycles we burned. Note 14380 * the bits after tcp_output we let be "free". This is because 14381 * we are also tracking the tcp_output times as well. Note the 14382 * use of 0xf here since we only have 11 counter (0 - 0xa) and 14383 * 0xf cannot be returned and is what we initialize it too to 14384 * indicate we are not doing the tabulations. 14385 */ 14386 if (ack_val_set != 0xf) { 14387 uint64_t crtsc; 14388 14389 crtsc = get_cyclecount(); 14390 counter_u64_add(tcp_proc_time[ack_val_set] , (crtsc - ts_val)); 14391 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 14392 tp->tcp_proc_time[ack_val_set] += (crtsc - ts_val); 14393 } 14394 } 14395 #endif 14396 if (nxt_pkt == 0) { 14397 if ((rack->r_wanted_output != 0) || (rack->r_fast_output != 0)) { 14398 do_output_now: 14399 if (tcp_output(tp) < 0) 14400 return (1); 14401 did_out = 1; 14402 } 14403 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); 14404 rack_free_trim(rack); 14405 } 14406 /* Update any rounds needed */ 14407 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) { 14408 union tcp_log_stackspecific log; 14409 struct timeval tv; 14410 14411 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 14412 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 14413 log.u_bbr.flex1 = high_seq; 14414 log.u_bbr.flex2 = rack->r_ctl.roundends; 14415 log.u_bbr.flex3 = rack->r_ctl.current_round; 14416 log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround; 14417 log.u_bbr.flex8 = 9; 14418 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0, 14419 0, &log, false, NULL, NULL, 0, &tv); 14420 } 14421 /* 14422 * The draft (v3) calls for us to use SEQ_GEQ, but that 14423 * causes issues when we are just going app limited. Lets 14424 * instead use SEQ_GT <or> where its equal but more data 14425 * is outstanding. 14426 */ 14427 if ((SEQ_GT(tp->snd_una, rack->r_ctl.roundends)) || 14428 ((tp->snd_una == rack->r_ctl.roundends) && SEQ_GT(tp->snd_max, tp->snd_una))) { 14429 rack->r_ctl.current_round++; 14430 rack->r_ctl.roundends = tp->snd_max; 14431 if (CC_ALGO(tp)->newround != NULL) { 14432 CC_ALGO(tp)->newround(&tp->t_ccv, rack->r_ctl.current_round); 14433 } 14434 } 14435 if ((nxt_pkt == 0) && 14436 ((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) && 14437 (SEQ_GT(tp->snd_max, tp->snd_una) || 14438 (tp->t_flags & TF_DELACK) || 14439 ((V_tcp_always_keepalive || rack->rc_inp->inp_socket->so_options & SO_KEEPALIVE) && 14440 (tp->t_state <= TCPS_CLOSING)))) { 14441 /* We could not send (probably in the hpts but stopped the timer earlier)? */ 14442 if ((tp->snd_max == tp->snd_una) && 14443 ((tp->t_flags & TF_DELACK) == 0) && 14444 (tcp_in_hpts(rack->rc_inp)) && 14445 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) { 14446 /* keep alive not needed if we are hptsi output yet */ 14447 ; 14448 } else { 14449 int late = 0; 14450 if (tcp_in_hpts(inp)) { 14451 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) { 14452 us_cts = tcp_get_usecs(NULL); 14453 if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) { 14454 rack->r_early = 1; 14455 rack->r_ctl.rc_agg_early += (rack->r_ctl.rc_last_output_to - us_cts); 14456 } else 14457 late = 1; 14458 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT; 14459 } 14460 tcp_hpts_remove(inp); 14461 } 14462 if (late && (did_out == 0)) { 14463 /* 14464 * We are late in the sending 14465 * and we did not call the output 14466 * (this probably should not happen). 14467 */ 14468 goto do_output_now; 14469 } 14470 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0); 14471 } 14472 way_out = 1; 14473 } else if (nxt_pkt == 0) { 14474 /* Do we have the correct timer running? */ 14475 rack_timer_audit(tp, rack, &so->so_snd); 14476 way_out = 2; 14477 } 14478 done_with_input: 14479 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, way_out, max(1, nsegs)); 14480 if (did_out) 14481 rack->r_wanted_output = 0; 14482 #ifdef TCP_ACCOUNTING 14483 } else { 14484 /* 14485 * Track the time (see above). 14486 */ 14487 if (ack_val_set != 0xf) { 14488 uint64_t crtsc; 14489 14490 crtsc = get_cyclecount(); 14491 counter_u64_add(tcp_proc_time[ack_val_set] , (crtsc - ts_val)); 14492 /* 14493 * Note we *DO NOT* increment the per-tcb counters since 14494 * in the else the TP may be gone!! 14495 */ 14496 } 14497 #endif 14498 } 14499 #ifdef TCP_ACCOUNTING 14500 sched_unpin(); 14501 #endif 14502 return (retval); 14503 } 14504 14505 void 14506 rack_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so, 14507 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos) 14508 { 14509 struct timeval tv; 14510 14511 /* First lets see if we have old packets */ 14512 if (tp->t_in_pkt) { 14513 if (ctf_do_queued_segments(so, tp, 1)) { 14514 m_freem(m); 14515 return; 14516 } 14517 } 14518 if (m->m_flags & M_TSTMP_LRO) { 14519 mbuf_tstmp2timeval(m, &tv); 14520 } else { 14521 /* Should not be should we kassert instead? */ 14522 tcp_get_usecs(&tv); 14523 } 14524 if (rack_do_segment_nounlock(m, th, so, tp, 14525 drop_hdrlen, tlen, iptos, 0, &tv) == 0) { 14526 INP_WUNLOCK(tptoinpcb(tp)); 14527 } 14528 } 14529 14530 struct rack_sendmap * 14531 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tsused) 14532 { 14533 struct rack_sendmap *rsm = NULL; 14534 int32_t idx; 14535 uint32_t srtt = 0, thresh = 0, ts_low = 0; 14536 14537 /* Return the next guy to be re-transmitted */ 14538 if (RB_EMPTY(&rack->r_ctl.rc_mtree)) { 14539 return (NULL); 14540 } 14541 if (tp->t_flags & TF_SENTFIN) { 14542 /* retran the end FIN? */ 14543 return (NULL); 14544 } 14545 /* ok lets look at this one */ 14546 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 14547 if (rack->r_must_retran && rsm && (rsm->r_flags & RACK_MUST_RXT)) { 14548 return (rsm); 14549 } 14550 if (rsm && ((rsm->r_flags & RACK_ACKED) == 0)) { 14551 goto check_it; 14552 } 14553 rsm = rack_find_lowest_rsm(rack); 14554 if (rsm == NULL) { 14555 return (NULL); 14556 } 14557 check_it: 14558 if (((rack->rc_tp->t_flags & TF_SACK_PERMIT) == 0) && 14559 (rsm->r_dupack >= DUP_ACK_THRESHOLD)) { 14560 /* 14561 * No sack so we automatically do the 3 strikes and 14562 * retransmit (no rack timer would be started). 14563 */ 14564 14565 return (rsm); 14566 } 14567 if (rsm->r_flags & RACK_ACKED) { 14568 return (NULL); 14569 } 14570 if (((rsm->r_flags & RACK_SACK_PASSED) == 0) && 14571 (rsm->r_dupack < DUP_ACK_THRESHOLD)) { 14572 /* Its not yet ready */ 14573 return (NULL); 14574 } 14575 srtt = rack_grab_rtt(tp, rack); 14576 idx = rsm->r_rtr_cnt - 1; 14577 ts_low = (uint32_t)rsm->r_tim_lastsent[idx]; 14578 thresh = rack_calc_thresh_rack(rack, srtt, tsused); 14579 if ((tsused == ts_low) || 14580 (TSTMP_LT(tsused, ts_low))) { 14581 /* No time since sending */ 14582 return (NULL); 14583 } 14584 if ((tsused - ts_low) < thresh) { 14585 /* It has not been long enough yet */ 14586 return (NULL); 14587 } 14588 if ((rsm->r_dupack >= DUP_ACK_THRESHOLD) || 14589 ((rsm->r_flags & RACK_SACK_PASSED) && 14590 (rack->sack_attack_disable == 0))) { 14591 /* 14592 * We have passed the dup-ack threshold <or> 14593 * a SACK has indicated this is missing. 14594 * Note that if you are a declared attacker 14595 * it is only the dup-ack threshold that 14596 * will cause retransmits. 14597 */ 14598 /* log retransmit reason */ 14599 rack_log_retran_reason(rack, rsm, (tsused - ts_low), thresh, 1); 14600 rack->r_fast_output = 0; 14601 return (rsm); 14602 } 14603 return (NULL); 14604 } 14605 14606 static void 14607 rack_log_pacing_delay_calc(struct tcp_rack *rack, uint32_t len, uint32_t slot, 14608 uint64_t bw_est, uint64_t bw, uint64_t len_time, int method, 14609 int line, struct rack_sendmap *rsm, uint8_t quality) 14610 { 14611 if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) { 14612 union tcp_log_stackspecific log; 14613 struct timeval tv; 14614 14615 memset(&log, 0, sizeof(log)); 14616 log.u_bbr.flex1 = slot; 14617 log.u_bbr.flex2 = len; 14618 log.u_bbr.flex3 = rack->r_ctl.rc_pace_min_segs; 14619 log.u_bbr.flex4 = rack->r_ctl.rc_pace_max_segs; 14620 log.u_bbr.flex5 = rack->r_ctl.rack_per_of_gp_ss; 14621 log.u_bbr.flex6 = rack->r_ctl.rack_per_of_gp_ca; 14622 log.u_bbr.use_lt_bw = rack->rc_ack_can_sendout_data; 14623 log.u_bbr.use_lt_bw <<= 1; 14624 log.u_bbr.use_lt_bw |= rack->r_late; 14625 log.u_bbr.use_lt_bw <<= 1; 14626 log.u_bbr.use_lt_bw |= rack->r_early; 14627 log.u_bbr.use_lt_bw <<= 1; 14628 log.u_bbr.use_lt_bw |= rack->app_limited_needs_set; 14629 log.u_bbr.use_lt_bw <<= 1; 14630 log.u_bbr.use_lt_bw |= rack->rc_gp_filled; 14631 log.u_bbr.use_lt_bw <<= 1; 14632 log.u_bbr.use_lt_bw |= rack->measure_saw_probe_rtt; 14633 log.u_bbr.use_lt_bw <<= 1; 14634 log.u_bbr.use_lt_bw |= rack->in_probe_rtt; 14635 log.u_bbr.use_lt_bw <<= 1; 14636 log.u_bbr.use_lt_bw |= rack->gp_ready; 14637 log.u_bbr.pkt_epoch = line; 14638 log.u_bbr.epoch = rack->r_ctl.rc_agg_delayed; 14639 log.u_bbr.lt_epoch = rack->r_ctl.rc_agg_early; 14640 log.u_bbr.applimited = rack->r_ctl.rack_per_of_gp_rec; 14641 log.u_bbr.bw_inuse = bw_est; 14642 log.u_bbr.delRate = bw; 14643 if (rack->r_ctl.gp_bw == 0) 14644 log.u_bbr.cur_del_rate = 0; 14645 else 14646 log.u_bbr.cur_del_rate = rack_get_bw(rack); 14647 log.u_bbr.rttProp = len_time; 14648 log.u_bbr.pkts_out = rack->r_ctl.rc_rack_min_rtt; 14649 log.u_bbr.lost = rack->r_ctl.rc_probertt_sndmax_atexit; 14650 log.u_bbr.pacing_gain = rack_get_output_gain(rack, rsm); 14651 if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh) { 14652 /* We are in slow start */ 14653 log.u_bbr.flex7 = 1; 14654 } else { 14655 /* we are on congestion avoidance */ 14656 log.u_bbr.flex7 = 0; 14657 } 14658 log.u_bbr.flex8 = method; 14659 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 14660 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 14661 log.u_bbr.cwnd_gain = rack->rc_gp_saw_rec; 14662 log.u_bbr.cwnd_gain <<= 1; 14663 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ss; 14664 log.u_bbr.cwnd_gain <<= 1; 14665 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ca; 14666 log.u_bbr.bbr_substate = quality; 14667 TCP_LOG_EVENTP(rack->rc_tp, NULL, 14668 &rack->rc_inp->inp_socket->so_rcv, 14669 &rack->rc_inp->inp_socket->so_snd, 14670 BBR_LOG_HPTSI_CALC, 0, 14671 0, &log, false, &tv); 14672 } 14673 } 14674 14675 static uint32_t 14676 rack_get_pacing_len(struct tcp_rack *rack, uint64_t bw, uint32_t mss) 14677 { 14678 uint32_t new_tso, user_max; 14679 14680 user_max = rack->rc_user_set_max_segs * mss; 14681 if (rack->rc_force_max_seg) { 14682 return (user_max); 14683 } 14684 if (rack->use_fixed_rate && 14685 ((rack->r_ctl.crte == NULL) || 14686 (bw != rack->r_ctl.crte->rate))) { 14687 /* Use the user mss since we are not exactly matched */ 14688 return (user_max); 14689 } 14690 new_tso = tcp_get_pacing_burst_size(rack->rc_tp, bw, mss, rack_pace_one_seg, rack->r_ctl.crte, NULL); 14691 if (new_tso > user_max) 14692 new_tso = user_max; 14693 return (new_tso); 14694 } 14695 14696 static int32_t 14697 pace_to_fill_cwnd(struct tcp_rack *rack, int32_t slot, uint32_t len, uint32_t segsiz, int *capped, uint64_t *rate_wanted, uint8_t non_paced) 14698 { 14699 uint64_t lentim, fill_bw; 14700 14701 /* Lets first see if we are full, if so continue with normal rate */ 14702 rack->r_via_fill_cw = 0; 14703 if (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.cwnd_to_use) 14704 return (slot); 14705 if ((ctf_outstanding(rack->rc_tp) + (segsiz-1)) > rack->rc_tp->snd_wnd) 14706 return (slot); 14707 if (rack->r_ctl.rc_last_us_rtt == 0) 14708 return (slot); 14709 if (rack->rc_pace_fill_if_rttin_range && 14710 (rack->r_ctl.rc_last_us_rtt >= 14711 (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack->rtt_limit_mul))) { 14712 /* The rtt is huge, N * smallest, lets not fill */ 14713 return (slot); 14714 } 14715 /* 14716 * first lets calculate the b/w based on the last us-rtt 14717 * and the sndwnd. 14718 */ 14719 fill_bw = rack->r_ctl.cwnd_to_use; 14720 /* Take the rwnd if its smaller */ 14721 if (fill_bw > rack->rc_tp->snd_wnd) 14722 fill_bw = rack->rc_tp->snd_wnd; 14723 if (rack->r_fill_less_agg) { 14724 /* 14725 * Now take away the inflight (this will reduce our 14726 * aggressiveness and yeah, if we get that much out in 1RTT 14727 * we will have had acks come back and still be behind). 14728 */ 14729 fill_bw -= ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 14730 } 14731 /* Now lets make it into a b/w */ 14732 fill_bw *= (uint64_t)HPTS_USEC_IN_SEC; 14733 fill_bw /= (uint64_t)rack->r_ctl.rc_last_us_rtt; 14734 /* We are below the min b/w */ 14735 if (non_paced) 14736 *rate_wanted = fill_bw; 14737 if ((fill_bw < RACK_MIN_BW) || (fill_bw < *rate_wanted)) 14738 return (slot); 14739 if (rack->r_ctl.bw_rate_cap && (fill_bw > rack->r_ctl.bw_rate_cap)) 14740 fill_bw = rack->r_ctl.bw_rate_cap; 14741 rack->r_via_fill_cw = 1; 14742 if (rack->r_rack_hw_rate_caps && 14743 (rack->r_ctl.crte != NULL)) { 14744 uint64_t high_rate; 14745 14746 high_rate = tcp_hw_highest_rate(rack->r_ctl.crte); 14747 if (fill_bw > high_rate) { 14748 /* We are capping bw at the highest rate table entry */ 14749 if (*rate_wanted > high_rate) { 14750 /* The original rate was also capped */ 14751 rack->r_via_fill_cw = 0; 14752 } 14753 rack_log_hdwr_pacing(rack, 14754 fill_bw, high_rate, __LINE__, 14755 0, 3); 14756 fill_bw = high_rate; 14757 if (capped) 14758 *capped = 1; 14759 } 14760 } else if ((rack->r_ctl.crte == NULL) && 14761 (rack->rack_hdrw_pacing == 0) && 14762 (rack->rack_hdw_pace_ena) && 14763 rack->r_rack_hw_rate_caps && 14764 (rack->rack_attempt_hdwr_pace == 0) && 14765 (rack->rc_inp->inp_route.ro_nh != NULL) && 14766 (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) { 14767 /* 14768 * Ok we may have a first attempt that is greater than our top rate 14769 * lets check. 14770 */ 14771 uint64_t high_rate; 14772 14773 high_rate = tcp_hw_highest_rate_ifp(rack->rc_inp->inp_route.ro_nh->nh_ifp, rack->rc_inp); 14774 if (high_rate) { 14775 if (fill_bw > high_rate) { 14776 fill_bw = high_rate; 14777 if (capped) 14778 *capped = 1; 14779 } 14780 } 14781 } 14782 /* 14783 * Ok fill_bw holds our mythical b/w to fill the cwnd 14784 * in a rtt, what does that time wise equate too? 14785 */ 14786 lentim = (uint64_t)(len) * (uint64_t)HPTS_USEC_IN_SEC; 14787 lentim /= fill_bw; 14788 *rate_wanted = fill_bw; 14789 if (non_paced || (lentim < slot)) { 14790 rack_log_pacing_delay_calc(rack, len, slot, fill_bw, 14791 0, lentim, 12, __LINE__, NULL, 0); 14792 return ((int32_t)lentim); 14793 } else 14794 return (slot); 14795 } 14796 14797 static int32_t 14798 rack_get_pacing_delay(struct tcp_rack *rack, struct tcpcb *tp, uint32_t len, struct rack_sendmap *rsm, uint32_t segsiz) 14799 { 14800 uint64_t srtt; 14801 int32_t slot = 0; 14802 int can_start_hw_pacing = 1; 14803 int err; 14804 14805 if (rack->rc_always_pace == 0) { 14806 /* 14807 * We use the most optimistic possible cwnd/srtt for 14808 * sending calculations. This will make our 14809 * calculation anticipate getting more through 14810 * quicker then possible. But thats ok we don't want 14811 * the peer to have a gap in data sending. 14812 */ 14813 uint64_t cwnd, tr_perms = 0; 14814 int32_t reduce = 0; 14815 14816 old_method: 14817 /* 14818 * We keep no precise pacing with the old method 14819 * instead we use the pacer to mitigate bursts. 14820 */ 14821 if (rack->r_ctl.rc_rack_min_rtt) 14822 srtt = rack->r_ctl.rc_rack_min_rtt; 14823 else 14824 srtt = max(tp->t_srtt, 1); 14825 if (rack->r_ctl.rc_rack_largest_cwnd) 14826 cwnd = rack->r_ctl.rc_rack_largest_cwnd; 14827 else 14828 cwnd = rack->r_ctl.cwnd_to_use; 14829 /* Inflate cwnd by 1000 so srtt of usecs is in ms */ 14830 tr_perms = (cwnd * 1000) / srtt; 14831 if (tr_perms == 0) { 14832 tr_perms = ctf_fixed_maxseg(tp); 14833 } 14834 /* 14835 * Calculate how long this will take to drain, if 14836 * the calculation comes out to zero, thats ok we 14837 * will use send_a_lot to possibly spin around for 14838 * more increasing tot_len_this_send to the point 14839 * that its going to require a pace, or we hit the 14840 * cwnd. Which in that case we are just waiting for 14841 * a ACK. 14842 */ 14843 slot = len / tr_perms; 14844 /* Now do we reduce the time so we don't run dry? */ 14845 if (slot && rack_slot_reduction) { 14846 reduce = (slot / rack_slot_reduction); 14847 if (reduce < slot) { 14848 slot -= reduce; 14849 } else 14850 slot = 0; 14851 } 14852 slot *= HPTS_USEC_IN_MSEC; 14853 if (rack->rc_pace_to_cwnd) { 14854 uint64_t rate_wanted = 0; 14855 14856 slot = pace_to_fill_cwnd(rack, slot, len, segsiz, NULL, &rate_wanted, 1); 14857 rack->rc_ack_can_sendout_data = 1; 14858 rack_log_pacing_delay_calc(rack, len, slot, rate_wanted, 0, 0, 14, __LINE__, NULL, 0); 14859 } else 14860 rack_log_pacing_delay_calc(rack, len, slot, tr_perms, reduce, 0, 7, __LINE__, NULL, 0); 14861 } else { 14862 uint64_t bw_est, res, lentim, rate_wanted; 14863 uint32_t orig_val, segs, oh; 14864 int capped = 0; 14865 int prev_fill; 14866 14867 if ((rack->r_rr_config == 1) && rsm) { 14868 return (rack->r_ctl.rc_min_to); 14869 } 14870 if (rack->use_fixed_rate) { 14871 rate_wanted = bw_est = rack_get_fixed_pacing_bw(rack); 14872 } else if ((rack->r_ctl.init_rate == 0) && 14873 #ifdef NETFLIX_PEAKRATE 14874 (rack->rc_tp->t_maxpeakrate == 0) && 14875 #endif 14876 (rack->r_ctl.gp_bw == 0)) { 14877 /* no way to yet do an estimate */ 14878 bw_est = rate_wanted = 0; 14879 } else { 14880 bw_est = rack_get_bw(rack); 14881 rate_wanted = rack_get_output_bw(rack, bw_est, rsm, &capped); 14882 } 14883 if ((bw_est == 0) || (rate_wanted == 0) || 14884 ((rack->gp_ready == 0) && (rack->use_fixed_rate == 0))) { 14885 /* 14886 * No way yet to make a b/w estimate or 14887 * our raise is set incorrectly. 14888 */ 14889 goto old_method; 14890 } 14891 /* We need to account for all the overheads */ 14892 segs = (len + segsiz - 1) / segsiz; 14893 /* 14894 * We need the diff between 1514 bytes (e-mtu with e-hdr) 14895 * and how much data we put in each packet. Yes this 14896 * means we may be off if we are larger than 1500 bytes 14897 * or smaller. But this just makes us more conservative. 14898 */ 14899 if (rack_hw_rate_min && 14900 (bw_est < rack_hw_rate_min)) 14901 can_start_hw_pacing = 0; 14902 if (ETHERNET_SEGMENT_SIZE > segsiz) 14903 oh = ETHERNET_SEGMENT_SIZE - segsiz; 14904 else 14905 oh = 0; 14906 segs *= oh; 14907 lentim = (uint64_t)(len + segs) * (uint64_t)HPTS_USEC_IN_SEC; 14908 res = lentim / rate_wanted; 14909 slot = (uint32_t)res; 14910 orig_val = rack->r_ctl.rc_pace_max_segs; 14911 if (rack->r_ctl.crte == NULL) { 14912 /* 14913 * Only do this if we are not hardware pacing 14914 * since if we are doing hw-pacing below we will 14915 * set make a call after setting up or changing 14916 * the rate. 14917 */ 14918 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL); 14919 } else if (rack->rc_inp->inp_snd_tag == NULL) { 14920 /* 14921 * We lost our rate somehow, this can happen 14922 * if the interface changed underneath us. 14923 */ 14924 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp); 14925 rack->r_ctl.crte = NULL; 14926 /* Lets re-allow attempting to setup pacing */ 14927 rack->rack_hdrw_pacing = 0; 14928 rack->rack_attempt_hdwr_pace = 0; 14929 rack_log_hdwr_pacing(rack, 14930 rate_wanted, bw_est, __LINE__, 14931 0, 6); 14932 } 14933 /* Did we change the TSO size, if so log it */ 14934 if (rack->r_ctl.rc_pace_max_segs != orig_val) 14935 rack_log_pacing_delay_calc(rack, len, slot, orig_val, 0, 0, 15, __LINE__, NULL, 0); 14936 prev_fill = rack->r_via_fill_cw; 14937 if ((rack->rc_pace_to_cwnd) && 14938 (capped == 0) && 14939 (rack->use_fixed_rate == 0) && 14940 (rack->in_probe_rtt == 0) && 14941 (IN_FASTRECOVERY(rack->rc_tp->t_flags) == 0)) { 14942 /* 14943 * We want to pace at our rate *or* faster to 14944 * fill the cwnd to the max if its not full. 14945 */ 14946 slot = pace_to_fill_cwnd(rack, slot, (len+segs), segsiz, &capped, &rate_wanted, 0); 14947 } 14948 if ((rack->rc_inp->inp_route.ro_nh != NULL) && 14949 (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) { 14950 if ((rack->rack_hdw_pace_ena) && 14951 (can_start_hw_pacing > 0) && 14952 (rack->rack_hdrw_pacing == 0) && 14953 (rack->rack_attempt_hdwr_pace == 0)) { 14954 /* 14955 * Lets attempt to turn on hardware pacing 14956 * if we can. 14957 */ 14958 rack->rack_attempt_hdwr_pace = 1; 14959 rack->r_ctl.crte = tcp_set_pacing_rate(rack->rc_tp, 14960 rack->rc_inp->inp_route.ro_nh->nh_ifp, 14961 rate_wanted, 14962 RS_PACING_GEQ, 14963 &err, &rack->r_ctl.crte_prev_rate); 14964 if (rack->r_ctl.crte) { 14965 rack->rack_hdrw_pacing = 1; 14966 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size(tp, rate_wanted, segsiz, 14967 0, rack->r_ctl.crte, 14968 NULL); 14969 rack_log_hdwr_pacing(rack, 14970 rate_wanted, rack->r_ctl.crte->rate, __LINE__, 14971 err, 0); 14972 rack->r_ctl.last_hw_bw_req = rate_wanted; 14973 } else { 14974 counter_u64_add(rack_hw_pace_init_fail, 1); 14975 } 14976 } else if (rack->rack_hdrw_pacing && 14977 (rack->r_ctl.last_hw_bw_req != rate_wanted)) { 14978 /* Do we need to adjust our rate? */ 14979 const struct tcp_hwrate_limit_table *nrte; 14980 14981 if (rack->r_up_only && 14982 (rate_wanted < rack->r_ctl.crte->rate)) { 14983 /** 14984 * We have four possible states here 14985 * having to do with the previous time 14986 * and this time. 14987 * previous | this-time 14988 * A) 0 | 0 -- fill_cw not in the picture 14989 * B) 1 | 0 -- we were doing a fill-cw but now are not 14990 * C) 1 | 1 -- all rates from fill_cw 14991 * D) 0 | 1 -- we were doing non-fill and now we are filling 14992 * 14993 * For case A, C and D we don't allow a drop. But for 14994 * case B where we now our on our steady rate we do 14995 * allow a drop. 14996 * 14997 */ 14998 if (!((prev_fill == 1) && (rack->r_via_fill_cw == 0))) 14999 goto done_w_hdwr; 15000 } 15001 if ((rate_wanted > rack->r_ctl.crte->rate) || 15002 (rate_wanted <= rack->r_ctl.crte_prev_rate)) { 15003 if (rack_hw_rate_to_low && 15004 (bw_est < rack_hw_rate_to_low)) { 15005 /* 15006 * The pacing rate is too low for hardware, but 15007 * do allow hardware pacing to be restarted. 15008 */ 15009 rack_log_hdwr_pacing(rack, 15010 bw_est, rack->r_ctl.crte->rate, __LINE__, 15011 0, 5); 15012 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp); 15013 rack->r_ctl.crte = NULL; 15014 rack->rack_attempt_hdwr_pace = 0; 15015 rack->rack_hdrw_pacing = 0; 15016 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted); 15017 goto done_w_hdwr; 15018 } 15019 nrte = tcp_chg_pacing_rate(rack->r_ctl.crte, 15020 rack->rc_tp, 15021 rack->rc_inp->inp_route.ro_nh->nh_ifp, 15022 rate_wanted, 15023 RS_PACING_GEQ, 15024 &err, &rack->r_ctl.crte_prev_rate); 15025 if (nrte == NULL) { 15026 /* Lost the rate */ 15027 rack->rack_hdrw_pacing = 0; 15028 rack->r_ctl.crte = NULL; 15029 rack_log_hdwr_pacing(rack, 15030 rate_wanted, 0, __LINE__, 15031 err, 1); 15032 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted); 15033 counter_u64_add(rack_hw_pace_lost, 1); 15034 } else if (nrte != rack->r_ctl.crte) { 15035 rack->r_ctl.crte = nrte; 15036 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size(tp, rate_wanted, 15037 segsiz, 0, 15038 rack->r_ctl.crte, 15039 NULL); 15040 rack_log_hdwr_pacing(rack, 15041 rate_wanted, rack->r_ctl.crte->rate, __LINE__, 15042 err, 2); 15043 rack->r_ctl.last_hw_bw_req = rate_wanted; 15044 } 15045 } else { 15046 /* We just need to adjust the segment size */ 15047 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted); 15048 rack_log_hdwr_pacing(rack, 15049 rate_wanted, rack->r_ctl.crte->rate, __LINE__, 15050 0, 4); 15051 rack->r_ctl.last_hw_bw_req = rate_wanted; 15052 } 15053 } 15054 } 15055 if ((rack->r_ctl.crte != NULL) && 15056 (rack->r_ctl.crte->rate == rate_wanted)) { 15057 /* 15058 * We need to add a extra if the rates 15059 * are exactly matched. The idea is 15060 * we want the software to make sure the 15061 * queue is empty before adding more, this 15062 * gives us N MSS extra pace times where 15063 * N is our sysctl 15064 */ 15065 slot += (rack->r_ctl.crte->time_between * rack_hw_pace_extra_slots); 15066 } 15067 done_w_hdwr: 15068 if (rack_limit_time_with_srtt && 15069 (rack->use_fixed_rate == 0) && 15070 #ifdef NETFLIX_PEAKRATE 15071 (rack->rc_tp->t_maxpeakrate == 0) && 15072 #endif 15073 (rack->rack_hdrw_pacing == 0)) { 15074 /* 15075 * Sanity check, we do not allow the pacing delay 15076 * to be longer than the SRTT of the path. If it is 15077 * a slow path, then adding a packet should increase 15078 * the RTT and compensate for this i.e. the srtt will 15079 * be greater so the allowed pacing time will be greater. 15080 * 15081 * Note this restriction is not for where a peak rate 15082 * is set, we are doing fixed pacing or hardware pacing. 15083 */ 15084 if (rack->rc_tp->t_srtt) 15085 srtt = rack->rc_tp->t_srtt; 15086 else 15087 srtt = RACK_INITIAL_RTO * HPTS_USEC_IN_MSEC; /* its in ms convert */ 15088 if (srtt < (uint64_t)slot) { 15089 rack_log_pacing_delay_calc(rack, srtt, slot, rate_wanted, bw_est, lentim, 99, __LINE__, NULL, 0); 15090 slot = srtt; 15091 } 15092 } 15093 rack_log_pacing_delay_calc(rack, len, slot, rate_wanted, bw_est, lentim, 2, __LINE__, rsm, 0); 15094 } 15095 if (rack->r_ctl.crte && (rack->r_ctl.crte->rs_num_enobufs > 0)) { 15096 /* 15097 * If this rate is seeing enobufs when it 15098 * goes to send then either the nic is out 15099 * of gas or we are mis-estimating the time 15100 * somehow and not letting the queue empty 15101 * completely. Lets add to the pacing time. 15102 */ 15103 int hw_boost_delay; 15104 15105 hw_boost_delay = rack->r_ctl.crte->time_between * rack_enobuf_hw_boost_mult; 15106 if (hw_boost_delay > rack_enobuf_hw_max) 15107 hw_boost_delay = rack_enobuf_hw_max; 15108 else if (hw_boost_delay < rack_enobuf_hw_min) 15109 hw_boost_delay = rack_enobuf_hw_min; 15110 slot += hw_boost_delay; 15111 } 15112 return (slot); 15113 } 15114 15115 static void 15116 rack_start_gp_measurement(struct tcpcb *tp, struct tcp_rack *rack, 15117 tcp_seq startseq, uint32_t sb_offset) 15118 { 15119 struct rack_sendmap *my_rsm = NULL; 15120 struct rack_sendmap fe; 15121 15122 if (tp->t_state < TCPS_ESTABLISHED) { 15123 /* 15124 * We don't start any measurements if we are 15125 * not at least established. 15126 */ 15127 return; 15128 } 15129 if (tp->t_state >= TCPS_FIN_WAIT_1) { 15130 /* 15131 * We will get no more data into the SB 15132 * this means we need to have the data available 15133 * before we start a measurement. 15134 */ 15135 15136 if (sbavail(&tptosocket(tp)->so_snd) < 15137 max(rc_init_window(rack), 15138 (MIN_GP_WIN * ctf_fixed_maxseg(tp)))) { 15139 /* Nope not enough data */ 15140 return; 15141 } 15142 } 15143 tp->t_flags |= TF_GPUTINPROG; 15144 rack->r_ctl.rc_gp_lowrtt = 0xffffffff; 15145 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd; 15146 tp->gput_seq = startseq; 15147 rack->app_limited_needs_set = 0; 15148 if (rack->in_probe_rtt) 15149 rack->measure_saw_probe_rtt = 1; 15150 else if ((rack->measure_saw_probe_rtt) && 15151 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit))) 15152 rack->measure_saw_probe_rtt = 0; 15153 if (rack->rc_gp_filled) 15154 tp->gput_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time); 15155 else { 15156 /* Special case initial measurement */ 15157 struct timeval tv; 15158 15159 tp->gput_ts = tcp_get_usecs(&tv); 15160 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv); 15161 } 15162 /* 15163 * We take a guess out into the future, 15164 * if we have no measurement and no 15165 * initial rate, we measure the first 15166 * initial-windows worth of data to 15167 * speed up getting some GP measurement and 15168 * thus start pacing. 15169 */ 15170 if ((rack->rc_gp_filled == 0) && (rack->r_ctl.init_rate == 0)) { 15171 rack->app_limited_needs_set = 1; 15172 tp->gput_ack = startseq + max(rc_init_window(rack), 15173 (MIN_GP_WIN * ctf_fixed_maxseg(tp))); 15174 rack_log_pacing_delay_calc(rack, 15175 tp->gput_seq, 15176 tp->gput_ack, 15177 0, 15178 tp->gput_ts, 15179 rack->r_ctl.rc_app_limited_cnt, 15180 9, 15181 __LINE__, NULL, 0); 15182 return; 15183 } 15184 if (sb_offset) { 15185 /* 15186 * We are out somewhere in the sb 15187 * can we use the already outstanding data? 15188 */ 15189 if (rack->r_ctl.rc_app_limited_cnt == 0) { 15190 /* 15191 * Yes first one is good and in this case 15192 * the tp->gput_ts is correctly set based on 15193 * the last ack that arrived (no need to 15194 * set things up when an ack comes in). 15195 */ 15196 my_rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 15197 if ((my_rsm == NULL) || 15198 (my_rsm->r_rtr_cnt != 1)) { 15199 /* retransmission? */ 15200 goto use_latest; 15201 } 15202 } else { 15203 if (rack->r_ctl.rc_first_appl == NULL) { 15204 /* 15205 * If rc_first_appl is NULL 15206 * then the cnt should be 0. 15207 * This is probably an error, maybe 15208 * a KASSERT would be approprate. 15209 */ 15210 goto use_latest; 15211 } 15212 /* 15213 * If we have a marker pointer to the last one that is 15214 * app limited we can use that, but we need to set 15215 * things up so that when it gets ack'ed we record 15216 * the ack time (if its not already acked). 15217 */ 15218 rack->app_limited_needs_set = 1; 15219 /* 15220 * We want to get to the rsm that is either 15221 * next with space i.e. over 1 MSS or the one 15222 * after that (after the app-limited). 15223 */ 15224 my_rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, 15225 rack->r_ctl.rc_first_appl); 15226 if (my_rsm) { 15227 if ((my_rsm->r_end - my_rsm->r_start) <= ctf_fixed_maxseg(tp)) 15228 /* Have to use the next one */ 15229 my_rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, 15230 my_rsm); 15231 else { 15232 /* Use after the first MSS of it is acked */ 15233 tp->gput_seq = my_rsm->r_start + ctf_fixed_maxseg(tp); 15234 goto start_set; 15235 } 15236 } 15237 if ((my_rsm == NULL) || 15238 (my_rsm->r_rtr_cnt != 1)) { 15239 /* 15240 * Either its a retransmit or 15241 * the last is the app-limited one. 15242 */ 15243 goto use_latest; 15244 } 15245 } 15246 tp->gput_seq = my_rsm->r_start; 15247 start_set: 15248 if (my_rsm->r_flags & RACK_ACKED) { 15249 /* 15250 * This one has been acked use the arrival ack time 15251 */ 15252 tp->gput_ts = (uint32_t)my_rsm->r_ack_arrival; 15253 rack->app_limited_needs_set = 0; 15254 } 15255 rack->r_ctl.rc_gp_output_ts = my_rsm->r_tim_lastsent[(my_rsm->r_rtr_cnt-1)]; 15256 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack); 15257 rack_log_pacing_delay_calc(rack, 15258 tp->gput_seq, 15259 tp->gput_ack, 15260 (uint64_t)my_rsm, 15261 tp->gput_ts, 15262 rack->r_ctl.rc_app_limited_cnt, 15263 9, 15264 __LINE__, NULL, 0); 15265 return; 15266 } 15267 15268 use_latest: 15269 /* 15270 * We don't know how long we may have been 15271 * idle or if this is the first-send. Lets 15272 * setup the flag so we will trim off 15273 * the first ack'd data so we get a true 15274 * measurement. 15275 */ 15276 rack->app_limited_needs_set = 1; 15277 tp->gput_ack = startseq + rack_get_measure_window(tp, rack); 15278 /* Find this guy so we can pull the send time */ 15279 fe.r_start = startseq; 15280 my_rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 15281 if (my_rsm) { 15282 rack->r_ctl.rc_gp_output_ts = my_rsm->r_tim_lastsent[(my_rsm->r_rtr_cnt-1)]; 15283 if (my_rsm->r_flags & RACK_ACKED) { 15284 /* 15285 * Unlikely since its probably what was 15286 * just transmitted (but I am paranoid). 15287 */ 15288 tp->gput_ts = (uint32_t)my_rsm->r_ack_arrival; 15289 rack->app_limited_needs_set = 0; 15290 } 15291 if (SEQ_LT(my_rsm->r_start, tp->gput_seq)) { 15292 /* This also is unlikely */ 15293 tp->gput_seq = my_rsm->r_start; 15294 } 15295 } else { 15296 /* 15297 * TSNH unless we have some send-map limit, 15298 * and even at that it should not be hitting 15299 * that limit (we should have stopped sending). 15300 */ 15301 struct timeval tv; 15302 15303 microuptime(&tv); 15304 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv); 15305 } 15306 rack_log_pacing_delay_calc(rack, 15307 tp->gput_seq, 15308 tp->gput_ack, 15309 (uint64_t)my_rsm, 15310 tp->gput_ts, 15311 rack->r_ctl.rc_app_limited_cnt, 15312 9, __LINE__, NULL, 0); 15313 } 15314 15315 static inline uint32_t 15316 rack_what_can_we_send(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cwnd_to_use, 15317 uint32_t avail, int32_t sb_offset) 15318 { 15319 uint32_t len; 15320 uint32_t sendwin; 15321 15322 if (tp->snd_wnd > cwnd_to_use) 15323 sendwin = cwnd_to_use; 15324 else 15325 sendwin = tp->snd_wnd; 15326 if (ctf_outstanding(tp) >= tp->snd_wnd) { 15327 /* We never want to go over our peers rcv-window */ 15328 len = 0; 15329 } else { 15330 uint32_t flight; 15331 15332 flight = ctf_flight_size(tp, rack->r_ctl.rc_sacked); 15333 if (flight >= sendwin) { 15334 /* 15335 * We have in flight what we are allowed by cwnd (if 15336 * it was rwnd blocking it would have hit above out 15337 * >= tp->snd_wnd). 15338 */ 15339 return (0); 15340 } 15341 len = sendwin - flight; 15342 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) { 15343 /* We would send too much (beyond the rwnd) */ 15344 len = tp->snd_wnd - ctf_outstanding(tp); 15345 } 15346 if ((len + sb_offset) > avail) { 15347 /* 15348 * We don't have that much in the SB, how much is 15349 * there? 15350 */ 15351 len = avail - sb_offset; 15352 } 15353 } 15354 return (len); 15355 } 15356 15357 static void 15358 rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t flags, 15359 unsigned ipoptlen, int32_t orig_len, int32_t len, int error, 15360 int rsm_is_null, int optlen, int line, uint16_t mode) 15361 { 15362 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 15363 union tcp_log_stackspecific log; 15364 struct timeval tv; 15365 15366 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 15367 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 15368 log.u_bbr.flex1 = error; 15369 log.u_bbr.flex2 = flags; 15370 log.u_bbr.flex3 = rsm_is_null; 15371 log.u_bbr.flex4 = ipoptlen; 15372 log.u_bbr.flex5 = tp->rcv_numsacks; 15373 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early; 15374 log.u_bbr.flex7 = optlen; 15375 log.u_bbr.flex8 = rack->r_fsb_inited; 15376 log.u_bbr.applimited = rack->r_fast_output; 15377 log.u_bbr.bw_inuse = rack_get_bw(rack); 15378 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL); 15379 log.u_bbr.cwnd_gain = mode; 15380 log.u_bbr.pkts_out = orig_len; 15381 log.u_bbr.lt_epoch = len; 15382 log.u_bbr.delivered = line; 15383 log.u_bbr.timeStamp = tcp_get_usecs(&tv); 15384 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 15385 tcp_log_event_(tp, NULL, &so->so_rcv, &so->so_snd, TCP_LOG_FSB, 0, 15386 len, &log, false, NULL, NULL, 0, &tv); 15387 } 15388 } 15389 15390 15391 static struct mbuf * 15392 rack_fo_base_copym(struct mbuf *the_m, uint32_t the_off, int32_t *plen, 15393 struct rack_fast_send_blk *fsb, 15394 int32_t seglimit, int32_t segsize, int hw_tls) 15395 { 15396 #ifdef KERN_TLS 15397 struct ktls_session *tls, *ntls; 15398 #ifdef INVARIANTS 15399 struct mbuf *start; 15400 #endif 15401 #endif 15402 struct mbuf *m, *n, **np, *smb; 15403 struct mbuf *top; 15404 int32_t off, soff; 15405 int32_t len = *plen; 15406 int32_t fragsize; 15407 int32_t len_cp = 0; 15408 uint32_t mlen, frags; 15409 15410 soff = off = the_off; 15411 smb = m = the_m; 15412 np = ⊤ 15413 top = NULL; 15414 #ifdef KERN_TLS 15415 if (hw_tls && (m->m_flags & M_EXTPG)) 15416 tls = m->m_epg_tls; 15417 else 15418 tls = NULL; 15419 #ifdef INVARIANTS 15420 start = m; 15421 #endif 15422 #endif 15423 while (len > 0) { 15424 if (m == NULL) { 15425 *plen = len_cp; 15426 break; 15427 } 15428 #ifdef KERN_TLS 15429 if (hw_tls) { 15430 if (m->m_flags & M_EXTPG) 15431 ntls = m->m_epg_tls; 15432 else 15433 ntls = NULL; 15434 15435 /* 15436 * Avoid mixing TLS records with handshake 15437 * data or TLS records from different 15438 * sessions. 15439 */ 15440 if (tls != ntls) { 15441 MPASS(m != start); 15442 *plen = len_cp; 15443 break; 15444 } 15445 } 15446 #endif 15447 mlen = min(len, m->m_len - off); 15448 if (seglimit) { 15449 /* 15450 * For M_EXTPG mbufs, add 3 segments 15451 * + 1 in case we are crossing page boundaries 15452 * + 2 in case the TLS hdr/trailer are used 15453 * It is cheaper to just add the segments 15454 * than it is to take the cache miss to look 15455 * at the mbuf ext_pgs state in detail. 15456 */ 15457 if (m->m_flags & M_EXTPG) { 15458 fragsize = min(segsize, PAGE_SIZE); 15459 frags = 3; 15460 } else { 15461 fragsize = segsize; 15462 frags = 0; 15463 } 15464 15465 /* Break if we really can't fit anymore. */ 15466 if ((frags + 1) >= seglimit) { 15467 *plen = len_cp; 15468 break; 15469 } 15470 15471 /* 15472 * Reduce size if you can't copy the whole 15473 * mbuf. If we can't copy the whole mbuf, also 15474 * adjust len so the loop will end after this 15475 * mbuf. 15476 */ 15477 if ((frags + howmany(mlen, fragsize)) >= seglimit) { 15478 mlen = (seglimit - frags - 1) * fragsize; 15479 len = mlen; 15480 *plen = len_cp + len; 15481 } 15482 frags += howmany(mlen, fragsize); 15483 if (frags == 0) 15484 frags++; 15485 seglimit -= frags; 15486 KASSERT(seglimit > 0, 15487 ("%s: seglimit went too low", __func__)); 15488 } 15489 n = m_get(M_NOWAIT, m->m_type); 15490 *np = n; 15491 if (n == NULL) 15492 goto nospace; 15493 n->m_len = mlen; 15494 soff += mlen; 15495 len_cp += n->m_len; 15496 if (m->m_flags & (M_EXT|M_EXTPG)) { 15497 n->m_data = m->m_data + off; 15498 mb_dupcl(n, m); 15499 } else { 15500 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 15501 (u_int)n->m_len); 15502 } 15503 len -= n->m_len; 15504 off = 0; 15505 m = m->m_next; 15506 np = &n->m_next; 15507 if (len || (soff == smb->m_len)) { 15508 /* 15509 * We have more so we move forward or 15510 * we have consumed the entire mbuf and 15511 * len has fell to 0. 15512 */ 15513 soff = 0; 15514 smb = m; 15515 } 15516 15517 } 15518 if (fsb != NULL) { 15519 fsb->m = smb; 15520 fsb->off = soff; 15521 if (smb) { 15522 /* 15523 * Save off the size of the mbuf. We do 15524 * this so that we can recognize when it 15525 * has been trimmed by sbcut() as acks 15526 * come in. 15527 */ 15528 fsb->o_m_len = smb->m_len; 15529 } else { 15530 /* 15531 * This is the case where the next mbuf went to NULL. This 15532 * means with this copy we have sent everything in the sb. 15533 * In theory we could clear the fast_output flag, but lets 15534 * not since its possible that we could get more added 15535 * and acks that call the extend function which would let 15536 * us send more. 15537 */ 15538 fsb->o_m_len = 0; 15539 } 15540 } 15541 return (top); 15542 nospace: 15543 if (top) 15544 m_freem(top); 15545 return (NULL); 15546 15547 } 15548 15549 /* 15550 * This is a copy of m_copym(), taking the TSO segment size/limit 15551 * constraints into account, and advancing the sndptr as it goes. 15552 */ 15553 static struct mbuf * 15554 rack_fo_m_copym(struct tcp_rack *rack, int32_t *plen, 15555 int32_t seglimit, int32_t segsize, struct mbuf **s_mb, int *s_soff) 15556 { 15557 struct mbuf *m, *n; 15558 int32_t soff; 15559 15560 soff = rack->r_ctl.fsb.off; 15561 m = rack->r_ctl.fsb.m; 15562 if (rack->r_ctl.fsb.o_m_len > m->m_len) { 15563 /* 15564 * The mbuf had the front of it chopped off by an ack 15565 * we need to adjust the soff/off by that difference. 15566 */ 15567 uint32_t delta; 15568 15569 delta = rack->r_ctl.fsb.o_m_len - m->m_len; 15570 soff -= delta; 15571 } else if (rack->r_ctl.fsb.o_m_len < m->m_len) { 15572 /* 15573 * The mbuf was expanded probably by 15574 * a m_compress. Just update o_m_len. 15575 */ 15576 rack->r_ctl.fsb.o_m_len = m->m_len; 15577 } 15578 KASSERT(soff >= 0, ("%s, negative off %d", __FUNCTION__, soff)); 15579 KASSERT(*plen >= 0, ("%s, negative len %d", __FUNCTION__, *plen)); 15580 KASSERT(soff < m->m_len, ("%s rack:%p len:%u m:%p m->m_len:%u < off?", 15581 __FUNCTION__, 15582 rack, *plen, m, m->m_len)); 15583 /* Save off the right location before we copy and advance */ 15584 *s_soff = soff; 15585 *s_mb = rack->r_ctl.fsb.m; 15586 n = rack_fo_base_copym(m, soff, plen, 15587 &rack->r_ctl.fsb, 15588 seglimit, segsize, rack->r_ctl.fsb.hw_tls); 15589 return (n); 15590 } 15591 15592 static int 15593 rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendmap *rsm, 15594 uint64_t ts_val, uint32_t cts, uint32_t ms_cts, struct timeval *tv, int len, uint8_t doing_tlp) 15595 { 15596 /* 15597 * Enter the fast retransmit path. We are given that a sched_pin is 15598 * in place (if accounting is compliled in) and the cycle count taken 15599 * at the entry is in the ts_val. The concept her is that the rsm 15600 * now holds the mbuf offsets and such so we can directly transmit 15601 * without a lot of overhead, the len field is already set for 15602 * us to prohibit us from sending too much (usually its 1MSS). 15603 */ 15604 struct ip *ip = NULL; 15605 struct udphdr *udp = NULL; 15606 struct tcphdr *th = NULL; 15607 struct mbuf *m = NULL; 15608 struct inpcb *inp; 15609 uint8_t *cpto; 15610 struct tcp_log_buffer *lgb; 15611 #ifdef TCP_ACCOUNTING 15612 uint64_t crtsc; 15613 int cnt_thru = 1; 15614 #endif 15615 struct tcpopt to; 15616 u_char opt[TCP_MAXOLEN]; 15617 uint32_t hdrlen, optlen; 15618 int32_t slot, segsiz, max_val, tso = 0, error = 0, ulen = 0; 15619 uint16_t flags; 15620 uint32_t if_hw_tsomaxsegcount = 0, startseq; 15621 uint32_t if_hw_tsomaxsegsize; 15622 15623 #ifdef INET6 15624 struct ip6_hdr *ip6 = NULL; 15625 15626 if (rack->r_is_v6) { 15627 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr; 15628 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 15629 } else 15630 #endif /* INET6 */ 15631 { 15632 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr; 15633 hdrlen = sizeof(struct tcpiphdr); 15634 } 15635 if (tp->t_port && (V_tcp_udp_tunneling_port == 0)) { 15636 goto failed; 15637 } 15638 if (doing_tlp) { 15639 /* Its a TLP add the flag, it may already be there but be sure */ 15640 rsm->r_flags |= RACK_TLP; 15641 } else { 15642 /* If it was a TLP it is not not on this retransmit */ 15643 rsm->r_flags &= ~RACK_TLP; 15644 } 15645 startseq = rsm->r_start; 15646 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 15647 inp = rack->rc_inp; 15648 to.to_flags = 0; 15649 flags = tcp_outflags[tp->t_state]; 15650 if (flags & (TH_SYN|TH_RST)) { 15651 goto failed; 15652 } 15653 if (rsm->r_flags & RACK_HAS_FIN) { 15654 /* We can't send a FIN here */ 15655 goto failed; 15656 } 15657 if (flags & TH_FIN) { 15658 /* We never send a FIN */ 15659 flags &= ~TH_FIN; 15660 } 15661 if (tp->t_flags & TF_RCVD_TSTMP) { 15662 to.to_tsval = ms_cts + tp->ts_offset; 15663 to.to_tsecr = tp->ts_recent; 15664 to.to_flags = TOF_TS; 15665 } 15666 optlen = tcp_addoptions(&to, opt); 15667 hdrlen += optlen; 15668 udp = rack->r_ctl.fsb.udp; 15669 if (udp) 15670 hdrlen += sizeof(struct udphdr); 15671 if (rack->r_ctl.rc_pace_max_segs) 15672 max_val = rack->r_ctl.rc_pace_max_segs; 15673 else if (rack->rc_user_set_max_segs) 15674 max_val = rack->rc_user_set_max_segs * segsiz; 15675 else 15676 max_val = len; 15677 if ((tp->t_flags & TF_TSO) && 15678 V_tcp_do_tso && 15679 (len > segsiz) && 15680 (tp->t_port == 0)) 15681 tso = 1; 15682 #ifdef INET6 15683 if (MHLEN < hdrlen + max_linkhdr) 15684 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 15685 else 15686 #endif 15687 m = m_gethdr(M_NOWAIT, MT_DATA); 15688 if (m == NULL) 15689 goto failed; 15690 m->m_data += max_linkhdr; 15691 m->m_len = hdrlen; 15692 th = rack->r_ctl.fsb.th; 15693 /* Establish the len to send */ 15694 if (len > max_val) 15695 len = max_val; 15696 if ((tso) && (len + optlen > tp->t_maxseg)) { 15697 uint32_t if_hw_tsomax; 15698 int32_t max_len; 15699 15700 /* extract TSO information */ 15701 if_hw_tsomax = tp->t_tsomax; 15702 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; 15703 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; 15704 /* 15705 * Check if we should limit by maximum payload 15706 * length: 15707 */ 15708 if (if_hw_tsomax != 0) { 15709 /* compute maximum TSO length */ 15710 max_len = (if_hw_tsomax - hdrlen - 15711 max_linkhdr); 15712 if (max_len <= 0) { 15713 goto failed; 15714 } else if (len > max_len) { 15715 len = max_len; 15716 } 15717 } 15718 if (len <= segsiz) { 15719 /* 15720 * In case there are too many small fragments don't 15721 * use TSO: 15722 */ 15723 tso = 0; 15724 } 15725 } else { 15726 tso = 0; 15727 } 15728 if ((tso == 0) && (len > segsiz)) 15729 len = segsiz; 15730 if ((len == 0) || 15731 (len <= MHLEN - hdrlen - max_linkhdr)) { 15732 goto failed; 15733 } 15734 th->th_seq = htonl(rsm->r_start); 15735 th->th_ack = htonl(tp->rcv_nxt); 15736 /* 15737 * The PUSH bit should only be applied 15738 * if the full retransmission is made. If 15739 * we are sending less than this is the 15740 * left hand edge and should not have 15741 * the PUSH bit. 15742 */ 15743 if ((rsm->r_flags & RACK_HAD_PUSH) && 15744 (len == (rsm->r_end - rsm->r_start))) 15745 flags |= TH_PUSH; 15746 th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale)); 15747 if (th->th_win == 0) { 15748 tp->t_sndzerowin++; 15749 tp->t_flags |= TF_RXWIN0SENT; 15750 } else 15751 tp->t_flags &= ~TF_RXWIN0SENT; 15752 if (rsm->r_flags & RACK_TLP) { 15753 /* 15754 * TLP should not count in retran count, but 15755 * in its own bin 15756 */ 15757 counter_u64_add(rack_tlp_retran, 1); 15758 counter_u64_add(rack_tlp_retran_bytes, len); 15759 } else { 15760 tp->t_sndrexmitpack++; 15761 KMOD_TCPSTAT_INC(tcps_sndrexmitpack); 15762 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len); 15763 } 15764 #ifdef STATS 15765 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB, 15766 len); 15767 #endif 15768 if (rsm->m == NULL) 15769 goto failed; 15770 if (rsm->orig_m_len != rsm->m->m_len) { 15771 /* Fix up the orig_m_len and possibly the mbuf offset */ 15772 rack_adjust_orig_mlen(rsm); 15773 } 15774 m->m_next = rack_fo_base_copym(rsm->m, rsm->soff, &len, NULL, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, rsm->r_hw_tls); 15775 if (len <= segsiz) { 15776 /* 15777 * Must have ran out of mbufs for the copy 15778 * shorten it to no longer need tso. Lets 15779 * not put on sendalot since we are low on 15780 * mbufs. 15781 */ 15782 tso = 0; 15783 } 15784 if ((m->m_next == NULL) || (len <= 0)){ 15785 goto failed; 15786 } 15787 if (udp) { 15788 if (rack->r_is_v6) 15789 ulen = hdrlen + len - sizeof(struct ip6_hdr); 15790 else 15791 ulen = hdrlen + len - sizeof(struct ip); 15792 udp->uh_ulen = htons(ulen); 15793 } 15794 m->m_pkthdr.rcvif = (struct ifnet *)0; 15795 if (TCPS_HAVERCVDSYN(tp->t_state) && 15796 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) { 15797 int ect = tcp_ecn_output_established(tp, &flags, len, true); 15798 if ((tp->t_state == TCPS_SYN_RECEIVED) && 15799 (tp->t_flags2 & TF2_ECN_SND_ECE)) 15800 tp->t_flags2 &= ~TF2_ECN_SND_ECE; 15801 #ifdef INET6 15802 if (rack->r_is_v6) { 15803 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20); 15804 ip6->ip6_flow |= htonl(ect << 20); 15805 } 15806 else 15807 #endif 15808 { 15809 ip->ip_tos &= ~IPTOS_ECN_MASK; 15810 ip->ip_tos |= ect; 15811 } 15812 } 15813 tcp_set_flags(th, flags); 15814 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ 15815 #ifdef INET6 15816 if (rack->r_is_v6) { 15817 if (tp->t_port) { 15818 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; 15819 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 15820 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); 15821 th->th_sum = htons(0); 15822 UDPSTAT_INC(udps_opackets); 15823 } else { 15824 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; 15825 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 15826 th->th_sum = in6_cksum_pseudo(ip6, 15827 sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP, 15828 0); 15829 } 15830 } 15831 #endif 15832 #if defined(INET6) && defined(INET) 15833 else 15834 #endif 15835 #ifdef INET 15836 { 15837 if (tp->t_port) { 15838 m->m_pkthdr.csum_flags = CSUM_UDP; 15839 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 15840 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, 15841 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); 15842 th->th_sum = htons(0); 15843 UDPSTAT_INC(udps_opackets); 15844 } else { 15845 m->m_pkthdr.csum_flags = CSUM_TCP; 15846 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 15847 th->th_sum = in_pseudo(ip->ip_src.s_addr, 15848 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + 15849 IPPROTO_TCP + len + optlen)); 15850 } 15851 /* IP version must be set here for ipv4/ipv6 checking later */ 15852 KASSERT(ip->ip_v == IPVERSION, 15853 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); 15854 } 15855 #endif 15856 if (tso) { 15857 KASSERT(len > tp->t_maxseg - optlen, 15858 ("%s: len <= tso_segsz tp:%p", __func__, tp)); 15859 m->m_pkthdr.csum_flags |= CSUM_TSO; 15860 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen; 15861 } 15862 #ifdef INET6 15863 if (rack->r_is_v6) { 15864 ip6->ip6_hlim = rack->r_ctl.fsb.hoplimit; 15865 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); 15866 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) 15867 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 15868 else 15869 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 15870 } 15871 #endif 15872 #if defined(INET) && defined(INET6) 15873 else 15874 #endif 15875 #ifdef INET 15876 { 15877 ip->ip_len = htons(m->m_pkthdr.len); 15878 ip->ip_ttl = rack->r_ctl.fsb.hoplimit; 15879 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { 15880 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 15881 if (tp->t_port == 0 || len < V_tcp_minmss) { 15882 ip->ip_off |= htons(IP_DF); 15883 } 15884 } else { 15885 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 15886 } 15887 } 15888 #endif 15889 /* Time to copy in our header */ 15890 cpto = mtod(m, uint8_t *); 15891 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len); 15892 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr)); 15893 if (optlen) { 15894 bcopy(opt, th + 1, optlen); 15895 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; 15896 } else { 15897 th->th_off = sizeof(struct tcphdr) >> 2; 15898 } 15899 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 15900 union tcp_log_stackspecific log; 15901 15902 if (rsm->r_flags & RACK_RWND_COLLAPSED) { 15903 rack_log_collapse(rack, rsm->r_start, rsm->r_end, 0, __LINE__, 5, rsm->r_flags, rsm); 15904 counter_u64_add(rack_collapsed_win_rxt, 1); 15905 counter_u64_add(rack_collapsed_win_rxt_bytes, (rsm->r_end - rsm->r_start)); 15906 } 15907 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 15908 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 15909 if (rack->rack_no_prr) 15910 log.u_bbr.flex1 = 0; 15911 else 15912 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; 15913 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs; 15914 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs; 15915 log.u_bbr.flex4 = max_val; 15916 log.u_bbr.flex5 = 0; 15917 /* Save off the early/late values */ 15918 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early; 15919 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed; 15920 log.u_bbr.bw_inuse = rack_get_bw(rack); 15921 if (doing_tlp == 0) 15922 log.u_bbr.flex8 = 1; 15923 else 15924 log.u_bbr.flex8 = 2; 15925 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL); 15926 log.u_bbr.flex7 = 55; 15927 log.u_bbr.pkts_out = tp->t_maxseg; 15928 log.u_bbr.timeStamp = cts; 15929 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 15930 log.u_bbr.lt_epoch = rack->r_ctl.cwnd_to_use; 15931 log.u_bbr.delivered = 0; 15932 lgb = tcp_log_event_(tp, th, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK, 15933 len, &log, false, NULL, NULL, 0, tv); 15934 } else 15935 lgb = NULL; 15936 #ifdef INET6 15937 if (rack->r_is_v6) { 15938 error = ip6_output(m, NULL, 15939 &inp->inp_route6, 15940 0, NULL, NULL, inp); 15941 } 15942 else 15943 #endif 15944 #ifdef INET 15945 { 15946 error = ip_output(m, NULL, 15947 &inp->inp_route, 15948 0, 0, inp); 15949 } 15950 #endif 15951 m = NULL; 15952 if (lgb) { 15953 lgb->tlb_errno = error; 15954 lgb = NULL; 15955 } 15956 if (error) { 15957 goto failed; 15958 } 15959 rack_log_output(tp, &to, len, rsm->r_start, flags, error, rack_to_usec_ts(tv), 15960 rsm, RACK_SENT_FP, rsm->m, rsm->soff, rsm->r_hw_tls); 15961 if (doing_tlp && (rack->fast_rsm_hack == 0)) { 15962 rack->rc_tlp_in_progress = 1; 15963 rack->r_ctl.rc_tlp_cnt_out++; 15964 } 15965 if (error == 0) { 15966 tcp_account_for_send(tp, len, 1, doing_tlp, rsm->r_hw_tls); 15967 if (doing_tlp) { 15968 rack->rc_last_sent_tlp_past_cumack = 0; 15969 rack->rc_last_sent_tlp_seq_valid = 1; 15970 rack->r_ctl.last_sent_tlp_seq = rsm->r_start; 15971 rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start; 15972 } 15973 } 15974 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); 15975 rack->forced_ack = 0; /* If we send something zap the FA flag */ 15976 if (IN_FASTRECOVERY(tp->t_flags) && rsm) 15977 rack->r_ctl.retran_during_recovery += len; 15978 { 15979 int idx; 15980 15981 idx = (len / segsiz) + 3; 15982 if (idx >= TCP_MSS_ACCT_ATIMER) 15983 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1); 15984 else 15985 counter_u64_add(rack_out_size[idx], 1); 15986 } 15987 if (tp->t_rtttime == 0) { 15988 tp->t_rtttime = ticks; 15989 tp->t_rtseq = startseq; 15990 KMOD_TCPSTAT_INC(tcps_segstimed); 15991 } 15992 counter_u64_add(rack_fto_rsm_send, 1); 15993 if (error && (error == ENOBUFS)) { 15994 if (rack->r_ctl.crte != NULL) { 15995 rack_trace_point(rack, RACK_TP_HWENOBUF); 15996 } else 15997 rack_trace_point(rack, RACK_TP_ENOBUF); 15998 slot = ((1 + rack->rc_enobuf) * HPTS_USEC_IN_MSEC); 15999 if (rack->rc_enobuf < 0x7f) 16000 rack->rc_enobuf++; 16001 if (slot < (10 * HPTS_USEC_IN_MSEC)) 16002 slot = 10 * HPTS_USEC_IN_MSEC; 16003 } else 16004 slot = rack_get_pacing_delay(rack, tp, len, NULL, segsiz); 16005 if ((slot == 0) || 16006 (rack->rc_always_pace == 0) || 16007 (rack->r_rr_config == 1)) { 16008 /* 16009 * We have no pacing set or we 16010 * are using old-style rack or 16011 * we are overridden to use the old 1ms pacing. 16012 */ 16013 slot = rack->r_ctl.rc_min_to; 16014 } 16015 rack_start_hpts_timer(rack, tp, cts, slot, len, 0); 16016 #ifdef TCP_ACCOUNTING 16017 crtsc = get_cyclecount(); 16018 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16019 tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru; 16020 } 16021 counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru); 16022 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16023 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val); 16024 } 16025 counter_u64_add(tcp_proc_time[SND_OUT_DATA], (crtsc - ts_val)); 16026 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16027 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((len + segsiz - 1) / segsiz); 16028 } 16029 counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((len + segsiz - 1) / segsiz)); 16030 sched_unpin(); 16031 #endif 16032 return (0); 16033 failed: 16034 if (m) 16035 m_free(m); 16036 return (-1); 16037 } 16038 16039 static void 16040 rack_sndbuf_autoscale(struct tcp_rack *rack) 16041 { 16042 /* 16043 * Automatic sizing of send socket buffer. Often the send buffer 16044 * size is not optimally adjusted to the actual network conditions 16045 * at hand (delay bandwidth product). Setting the buffer size too 16046 * small limits throughput on links with high bandwidth and high 16047 * delay (eg. trans-continental/oceanic links). Setting the 16048 * buffer size too big consumes too much real kernel memory, 16049 * especially with many connections on busy servers. 16050 * 16051 * The criteria to step up the send buffer one notch are: 16052 * 1. receive window of remote host is larger than send buffer 16053 * (with a fudge factor of 5/4th); 16054 * 2. send buffer is filled to 7/8th with data (so we actually 16055 * have data to make use of it); 16056 * 3. send buffer fill has not hit maximal automatic size; 16057 * 4. our send window (slow start and cogestion controlled) is 16058 * larger than sent but unacknowledged data in send buffer. 16059 * 16060 * Note that the rack version moves things much faster since 16061 * we want to avoid hitting cache lines in the rack_fast_output() 16062 * path so this is called much less often and thus moves 16063 * the SB forward by a percentage. 16064 */ 16065 struct socket *so; 16066 struct tcpcb *tp; 16067 uint32_t sendwin, scaleup; 16068 16069 tp = rack->rc_tp; 16070 so = rack->rc_inp->inp_socket; 16071 sendwin = min(rack->r_ctl.cwnd_to_use, tp->snd_wnd); 16072 if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) { 16073 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat && 16074 sbused(&so->so_snd) >= 16075 (so->so_snd.sb_hiwat / 8 * 7) && 16076 sbused(&so->so_snd) < V_tcp_autosndbuf_max && 16077 sendwin >= (sbused(&so->so_snd) - 16078 (tp->snd_nxt - tp->snd_una))) { 16079 if (rack_autosndbuf_inc) 16080 scaleup = (rack_autosndbuf_inc * so->so_snd.sb_hiwat) / 100; 16081 else 16082 scaleup = V_tcp_autosndbuf_inc; 16083 if (scaleup < V_tcp_autosndbuf_inc) 16084 scaleup = V_tcp_autosndbuf_inc; 16085 scaleup += so->so_snd.sb_hiwat; 16086 if (scaleup > V_tcp_autosndbuf_max) 16087 scaleup = V_tcp_autosndbuf_max; 16088 if (!sbreserve_locked(so, SO_SND, scaleup, curthread)) 16089 so->so_snd.sb_flags &= ~SB_AUTOSIZE; 16090 } 16091 } 16092 } 16093 16094 static int 16095 rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val, 16096 uint32_t cts, uint32_t ms_cts, struct timeval *tv, long tot_len, int *send_err) 16097 { 16098 /* 16099 * Enter to do fast output. We are given that the sched_pin is 16100 * in place (if accounting is compiled in) and the cycle count taken 16101 * at entry is in place in ts_val. The idea here is that 16102 * we know how many more bytes needs to be sent (presumably either 16103 * during pacing or to fill the cwnd and that was greater than 16104 * the max-burst). We have how much to send and all the info we 16105 * need to just send. 16106 */ 16107 #ifdef INET 16108 struct ip *ip = NULL; 16109 #endif 16110 struct udphdr *udp = NULL; 16111 struct tcphdr *th = NULL; 16112 struct mbuf *m, *s_mb; 16113 struct inpcb *inp; 16114 uint8_t *cpto; 16115 struct tcp_log_buffer *lgb; 16116 #ifdef TCP_ACCOUNTING 16117 uint64_t crtsc; 16118 #endif 16119 struct tcpopt to; 16120 u_char opt[TCP_MAXOLEN]; 16121 uint32_t hdrlen, optlen; 16122 #ifdef TCP_ACCOUNTING 16123 int cnt_thru = 1; 16124 #endif 16125 int32_t slot, segsiz, len, max_val, tso = 0, sb_offset, error, ulen = 0; 16126 uint16_t flags; 16127 uint32_t s_soff; 16128 uint32_t if_hw_tsomaxsegcount = 0, startseq; 16129 uint32_t if_hw_tsomaxsegsize; 16130 uint16_t add_flag = RACK_SENT_FP; 16131 #ifdef INET6 16132 struct ip6_hdr *ip6 = NULL; 16133 16134 if (rack->r_is_v6) { 16135 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr; 16136 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 16137 } else 16138 #endif /* INET6 */ 16139 { 16140 #ifdef INET 16141 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr; 16142 hdrlen = sizeof(struct tcpiphdr); 16143 #endif 16144 } 16145 if (tp->t_port && (V_tcp_udp_tunneling_port == 0)) { 16146 m = NULL; 16147 goto failed; 16148 } 16149 startseq = tp->snd_max; 16150 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 16151 inp = rack->rc_inp; 16152 len = rack->r_ctl.fsb.left_to_send; 16153 to.to_flags = 0; 16154 flags = rack->r_ctl.fsb.tcp_flags; 16155 if (tp->t_flags & TF_RCVD_TSTMP) { 16156 to.to_tsval = ms_cts + tp->ts_offset; 16157 to.to_tsecr = tp->ts_recent; 16158 to.to_flags = TOF_TS; 16159 } 16160 optlen = tcp_addoptions(&to, opt); 16161 hdrlen += optlen; 16162 udp = rack->r_ctl.fsb.udp; 16163 if (udp) 16164 hdrlen += sizeof(struct udphdr); 16165 if (rack->r_ctl.rc_pace_max_segs) 16166 max_val = rack->r_ctl.rc_pace_max_segs; 16167 else if (rack->rc_user_set_max_segs) 16168 max_val = rack->rc_user_set_max_segs * segsiz; 16169 else 16170 max_val = len; 16171 if ((tp->t_flags & TF_TSO) && 16172 V_tcp_do_tso && 16173 (len > segsiz) && 16174 (tp->t_port == 0)) 16175 tso = 1; 16176 again: 16177 #ifdef INET6 16178 if (MHLEN < hdrlen + max_linkhdr) 16179 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 16180 else 16181 #endif 16182 m = m_gethdr(M_NOWAIT, MT_DATA); 16183 if (m == NULL) 16184 goto failed; 16185 m->m_data += max_linkhdr; 16186 m->m_len = hdrlen; 16187 th = rack->r_ctl.fsb.th; 16188 /* Establish the len to send */ 16189 if (len > max_val) 16190 len = max_val; 16191 if ((tso) && (len + optlen > tp->t_maxseg)) { 16192 uint32_t if_hw_tsomax; 16193 int32_t max_len; 16194 16195 /* extract TSO information */ 16196 if_hw_tsomax = tp->t_tsomax; 16197 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; 16198 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; 16199 /* 16200 * Check if we should limit by maximum payload 16201 * length: 16202 */ 16203 if (if_hw_tsomax != 0) { 16204 /* compute maximum TSO length */ 16205 max_len = (if_hw_tsomax - hdrlen - 16206 max_linkhdr); 16207 if (max_len <= 0) { 16208 goto failed; 16209 } else if (len > max_len) { 16210 len = max_len; 16211 } 16212 } 16213 if (len <= segsiz) { 16214 /* 16215 * In case there are too many small fragments don't 16216 * use TSO: 16217 */ 16218 tso = 0; 16219 } 16220 } else { 16221 tso = 0; 16222 } 16223 if ((tso == 0) && (len > segsiz)) 16224 len = segsiz; 16225 if ((len == 0) || 16226 (len <= MHLEN - hdrlen - max_linkhdr)) { 16227 goto failed; 16228 } 16229 sb_offset = tp->snd_max - tp->snd_una; 16230 th->th_seq = htonl(tp->snd_max); 16231 th->th_ack = htonl(tp->rcv_nxt); 16232 th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale)); 16233 if (th->th_win == 0) { 16234 tp->t_sndzerowin++; 16235 tp->t_flags |= TF_RXWIN0SENT; 16236 } else 16237 tp->t_flags &= ~TF_RXWIN0SENT; 16238 tp->snd_up = tp->snd_una; /* drag it along, its deprecated */ 16239 KMOD_TCPSTAT_INC(tcps_sndpack); 16240 KMOD_TCPSTAT_ADD(tcps_sndbyte, len); 16241 #ifdef STATS 16242 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB, 16243 len); 16244 #endif 16245 if (rack->r_ctl.fsb.m == NULL) 16246 goto failed; 16247 16248 /* s_mb and s_soff are saved for rack_log_output */ 16249 m->m_next = rack_fo_m_copym(rack, &len, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, 16250 &s_mb, &s_soff); 16251 if (len <= segsiz) { 16252 /* 16253 * Must have ran out of mbufs for the copy 16254 * shorten it to no longer need tso. Lets 16255 * not put on sendalot since we are low on 16256 * mbufs. 16257 */ 16258 tso = 0; 16259 } 16260 if (rack->r_ctl.fsb.rfo_apply_push && 16261 (len == rack->r_ctl.fsb.left_to_send)) { 16262 flags |= TH_PUSH; 16263 add_flag |= RACK_HAD_PUSH; 16264 } 16265 if ((m->m_next == NULL) || (len <= 0)){ 16266 goto failed; 16267 } 16268 if (udp) { 16269 if (rack->r_is_v6) 16270 ulen = hdrlen + len - sizeof(struct ip6_hdr); 16271 else 16272 ulen = hdrlen + len - sizeof(struct ip); 16273 udp->uh_ulen = htons(ulen); 16274 } 16275 m->m_pkthdr.rcvif = (struct ifnet *)0; 16276 if (TCPS_HAVERCVDSYN(tp->t_state) && 16277 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) { 16278 int ect = tcp_ecn_output_established(tp, &flags, len, false); 16279 if ((tp->t_state == TCPS_SYN_RECEIVED) && 16280 (tp->t_flags2 & TF2_ECN_SND_ECE)) 16281 tp->t_flags2 &= ~TF2_ECN_SND_ECE; 16282 #ifdef INET6 16283 if (rack->r_is_v6) { 16284 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20); 16285 ip6->ip6_flow |= htonl(ect << 20); 16286 } 16287 else 16288 #endif 16289 { 16290 #ifdef INET 16291 ip->ip_tos &= ~IPTOS_ECN_MASK; 16292 ip->ip_tos |= ect; 16293 #endif 16294 } 16295 } 16296 tcp_set_flags(th, flags); 16297 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ 16298 #ifdef INET6 16299 if (rack->r_is_v6) { 16300 if (tp->t_port) { 16301 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; 16302 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 16303 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); 16304 th->th_sum = htons(0); 16305 UDPSTAT_INC(udps_opackets); 16306 } else { 16307 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; 16308 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 16309 th->th_sum = in6_cksum_pseudo(ip6, 16310 sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP, 16311 0); 16312 } 16313 } 16314 #endif 16315 #if defined(INET6) && defined(INET) 16316 else 16317 #endif 16318 #ifdef INET 16319 { 16320 if (tp->t_port) { 16321 m->m_pkthdr.csum_flags = CSUM_UDP; 16322 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 16323 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, 16324 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); 16325 th->th_sum = htons(0); 16326 UDPSTAT_INC(udps_opackets); 16327 } else { 16328 m->m_pkthdr.csum_flags = CSUM_TCP; 16329 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 16330 th->th_sum = in_pseudo(ip->ip_src.s_addr, 16331 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + 16332 IPPROTO_TCP + len + optlen)); 16333 } 16334 /* IP version must be set here for ipv4/ipv6 checking later */ 16335 KASSERT(ip->ip_v == IPVERSION, 16336 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); 16337 } 16338 #endif 16339 if (tso) { 16340 KASSERT(len > tp->t_maxseg - optlen, 16341 ("%s: len <= tso_segsz tp:%p", __func__, tp)); 16342 m->m_pkthdr.csum_flags |= CSUM_TSO; 16343 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen; 16344 } 16345 #ifdef INET6 16346 if (rack->r_is_v6) { 16347 ip6->ip6_hlim = rack->r_ctl.fsb.hoplimit; 16348 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); 16349 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) 16350 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 16351 else 16352 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 16353 } 16354 #endif 16355 #if defined(INET) && defined(INET6) 16356 else 16357 #endif 16358 #ifdef INET 16359 { 16360 ip->ip_len = htons(m->m_pkthdr.len); 16361 ip->ip_ttl = rack->r_ctl.fsb.hoplimit; 16362 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { 16363 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 16364 if (tp->t_port == 0 || len < V_tcp_minmss) { 16365 ip->ip_off |= htons(IP_DF); 16366 } 16367 } else { 16368 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 16369 } 16370 } 16371 #endif 16372 /* Time to copy in our header */ 16373 cpto = mtod(m, uint8_t *); 16374 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len); 16375 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr)); 16376 if (optlen) { 16377 bcopy(opt, th + 1, optlen); 16378 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; 16379 } else { 16380 th->th_off = sizeof(struct tcphdr) >> 2; 16381 } 16382 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 16383 union tcp_log_stackspecific log; 16384 16385 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 16386 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 16387 if (rack->rack_no_prr) 16388 log.u_bbr.flex1 = 0; 16389 else 16390 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; 16391 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs; 16392 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs; 16393 log.u_bbr.flex4 = max_val; 16394 log.u_bbr.flex5 = 0; 16395 /* Save off the early/late values */ 16396 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early; 16397 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed; 16398 log.u_bbr.bw_inuse = rack_get_bw(rack); 16399 log.u_bbr.flex8 = 0; 16400 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL); 16401 log.u_bbr.flex7 = 44; 16402 log.u_bbr.pkts_out = tp->t_maxseg; 16403 log.u_bbr.timeStamp = cts; 16404 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 16405 log.u_bbr.lt_epoch = rack->r_ctl.cwnd_to_use; 16406 log.u_bbr.delivered = 0; 16407 lgb = tcp_log_event_(tp, th, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK, 16408 len, &log, false, NULL, NULL, 0, tv); 16409 } else 16410 lgb = NULL; 16411 #ifdef INET6 16412 if (rack->r_is_v6) { 16413 error = ip6_output(m, NULL, 16414 &inp->inp_route6, 16415 0, NULL, NULL, inp); 16416 } 16417 #endif 16418 #if defined(INET) && defined(INET6) 16419 else 16420 #endif 16421 #ifdef INET 16422 { 16423 error = ip_output(m, NULL, 16424 &inp->inp_route, 16425 0, 0, inp); 16426 } 16427 #endif 16428 if (lgb) { 16429 lgb->tlb_errno = error; 16430 lgb = NULL; 16431 } 16432 if (error) { 16433 *send_err = error; 16434 m = NULL; 16435 goto failed; 16436 } 16437 rack_log_output(tp, &to, len, tp->snd_max, flags, error, rack_to_usec_ts(tv), 16438 NULL, add_flag, s_mb, s_soff, rack->r_ctl.fsb.hw_tls); 16439 m = NULL; 16440 if (tp->snd_una == tp->snd_max) { 16441 rack->r_ctl.rc_tlp_rxt_last_time = cts; 16442 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__); 16443 tp->t_acktime = ticks; 16444 } 16445 if (error == 0) 16446 tcp_account_for_send(tp, len, 0, 0, rack->r_ctl.fsb.hw_tls); 16447 16448 rack->forced_ack = 0; /* If we send something zap the FA flag */ 16449 tot_len += len; 16450 if ((tp->t_flags & TF_GPUTINPROG) == 0) 16451 rack_start_gp_measurement(tp, rack, tp->snd_max, sb_offset); 16452 tp->snd_max += len; 16453 tp->snd_nxt = tp->snd_max; 16454 { 16455 int idx; 16456 16457 idx = (len / segsiz) + 3; 16458 if (idx >= TCP_MSS_ACCT_ATIMER) 16459 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1); 16460 else 16461 counter_u64_add(rack_out_size[idx], 1); 16462 } 16463 if (len <= rack->r_ctl.fsb.left_to_send) 16464 rack->r_ctl.fsb.left_to_send -= len; 16465 else 16466 rack->r_ctl.fsb.left_to_send = 0; 16467 if (rack->r_ctl.fsb.left_to_send < segsiz) { 16468 rack->r_fast_output = 0; 16469 rack->r_ctl.fsb.left_to_send = 0; 16470 /* At the end of fast_output scale up the sb */ 16471 SOCKBUF_LOCK(&rack->rc_inp->inp_socket->so_snd); 16472 rack_sndbuf_autoscale(rack); 16473 SOCKBUF_UNLOCK(&rack->rc_inp->inp_socket->so_snd); 16474 } 16475 if (tp->t_rtttime == 0) { 16476 tp->t_rtttime = ticks; 16477 tp->t_rtseq = startseq; 16478 KMOD_TCPSTAT_INC(tcps_segstimed); 16479 } 16480 if ((rack->r_ctl.fsb.left_to_send >= segsiz) && 16481 (max_val > len) && 16482 (tso == 0)) { 16483 max_val -= len; 16484 len = segsiz; 16485 th = rack->r_ctl.fsb.th; 16486 #ifdef TCP_ACCOUNTING 16487 cnt_thru++; 16488 #endif 16489 goto again; 16490 } 16491 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); 16492 counter_u64_add(rack_fto_send, 1); 16493 slot = rack_get_pacing_delay(rack, tp, tot_len, NULL, segsiz); 16494 rack_start_hpts_timer(rack, tp, cts, slot, tot_len, 0); 16495 #ifdef TCP_ACCOUNTING 16496 crtsc = get_cyclecount(); 16497 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16498 tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru; 16499 } 16500 counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru); 16501 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16502 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val); 16503 } 16504 counter_u64_add(tcp_proc_time[SND_OUT_DATA], (crtsc - ts_val)); 16505 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16506 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len + segsiz - 1) / segsiz); 16507 } 16508 counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((tot_len + segsiz - 1) / segsiz)); 16509 sched_unpin(); 16510 #endif 16511 return (0); 16512 failed: 16513 if (m) 16514 m_free(m); 16515 rack->r_fast_output = 0; 16516 return (-1); 16517 } 16518 16519 static struct rack_sendmap * 16520 rack_check_collapsed(struct tcp_rack *rack, uint32_t cts) 16521 { 16522 struct rack_sendmap *rsm = NULL; 16523 struct rack_sendmap fe; 16524 int thresh; 16525 16526 restart: 16527 fe.r_start = rack->r_ctl.last_collapse_point; 16528 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe); 16529 if ((rsm == NULL) || ((rsm->r_flags & RACK_RWND_COLLAPSED) == 0)) { 16530 /* Nothing, strange turn off validity */ 16531 rack->r_collapse_point_valid = 0; 16532 return (NULL); 16533 } 16534 /* Can we send it yet? */ 16535 if (rsm->r_end > (rack->rc_tp->snd_una + rack->rc_tp->snd_wnd)) { 16536 /* 16537 * Receiver window has not grown enough for 16538 * the segment to be put on the wire. 16539 */ 16540 return (NULL); 16541 } 16542 if (rsm->r_flags & RACK_ACKED) { 16543 /* 16544 * It has been sacked, lets move to the 16545 * next one if possible. 16546 */ 16547 rack->r_ctl.last_collapse_point = rsm->r_end; 16548 /* Are we done? */ 16549 if (SEQ_GEQ(rack->r_ctl.last_collapse_point, 16550 rack->r_ctl.high_collapse_point)) { 16551 rack->r_collapse_point_valid = 0; 16552 return (NULL); 16553 } 16554 goto restart; 16555 } 16556 /* Now has it been long enough ? */ 16557 thresh = rack_calc_thresh_rack(rack, rack_grab_rtt(rack->rc_tp, rack), cts); 16558 if ((cts - ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])) > thresh) { 16559 rack_log_collapse(rack, rsm->r_start, 16560 (cts - ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])), 16561 thresh, __LINE__, 6, rsm->r_flags, rsm); 16562 return (rsm); 16563 } 16564 /* Not enough time */ 16565 rack_log_collapse(rack, rsm->r_start, 16566 (cts - ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])), 16567 thresh, __LINE__, 7, rsm->r_flags, rsm); 16568 return (NULL); 16569 } 16570 16571 static int 16572 rack_output(struct tcpcb *tp) 16573 { 16574 struct socket *so; 16575 uint32_t recwin; 16576 uint32_t sb_offset, s_moff = 0; 16577 int32_t len, error = 0; 16578 uint16_t flags; 16579 struct mbuf *m, *s_mb = NULL; 16580 struct mbuf *mb; 16581 uint32_t if_hw_tsomaxsegcount = 0; 16582 uint32_t if_hw_tsomaxsegsize; 16583 int32_t segsiz, minseg; 16584 long tot_len_this_send = 0; 16585 #ifdef INET 16586 struct ip *ip = NULL; 16587 #endif 16588 struct udphdr *udp = NULL; 16589 struct tcp_rack *rack; 16590 struct tcphdr *th; 16591 uint8_t pass = 0; 16592 uint8_t mark = 0; 16593 uint8_t wanted_cookie = 0; 16594 u_char opt[TCP_MAXOLEN]; 16595 unsigned ipoptlen, optlen, hdrlen, ulen=0; 16596 uint32_t rack_seq; 16597 16598 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 16599 unsigned ipsec_optlen = 0; 16600 16601 #endif 16602 int32_t idle, sendalot; 16603 int32_t sub_from_prr = 0; 16604 volatile int32_t sack_rxmit; 16605 struct rack_sendmap *rsm = NULL; 16606 int32_t tso, mtu; 16607 struct tcpopt to; 16608 int32_t slot = 0; 16609 int32_t sup_rack = 0; 16610 uint32_t cts, ms_cts, delayed, early; 16611 uint16_t add_flag = RACK_SENT_SP; 16612 /* The doing_tlp flag will be set by the actual rack_timeout_tlp() */ 16613 uint8_t hpts_calling, doing_tlp = 0; 16614 uint32_t cwnd_to_use, pace_max_seg; 16615 int32_t do_a_prefetch = 0; 16616 int32_t prefetch_rsm = 0; 16617 int32_t orig_len = 0; 16618 struct timeval tv; 16619 int32_t prefetch_so_done = 0; 16620 struct tcp_log_buffer *lgb; 16621 struct inpcb *inp = tptoinpcb(tp); 16622 struct sockbuf *sb; 16623 uint64_t ts_val = 0; 16624 #ifdef TCP_ACCOUNTING 16625 uint64_t crtsc; 16626 #endif 16627 #ifdef INET6 16628 struct ip6_hdr *ip6 = NULL; 16629 int32_t isipv6; 16630 #endif 16631 bool hw_tls = false; 16632 16633 NET_EPOCH_ASSERT(); 16634 INP_WLOCK_ASSERT(inp); 16635 16636 /* setup and take the cache hits here */ 16637 rack = (struct tcp_rack *)tp->t_fb_ptr; 16638 #ifdef TCP_ACCOUNTING 16639 sched_pin(); 16640 ts_val = get_cyclecount(); 16641 #endif 16642 hpts_calling = inp->inp_hpts_calls; 16643 #ifdef TCP_OFFLOAD 16644 if (tp->t_flags & TF_TOE) { 16645 #ifdef TCP_ACCOUNTING 16646 sched_unpin(); 16647 #endif 16648 return (tcp_offload_output(tp)); 16649 } 16650 #endif 16651 /* 16652 * For TFO connections in SYN_RECEIVED, only allow the initial 16653 * SYN|ACK and those sent by the retransmit timer. 16654 */ 16655 if (IS_FASTOPEN(tp->t_flags) && 16656 (tp->t_state == TCPS_SYN_RECEIVED) && 16657 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN|ACK sent */ 16658 (rack->r_ctl.rc_resend == NULL)) { /* not a retransmit */ 16659 #ifdef TCP_ACCOUNTING 16660 sched_unpin(); 16661 #endif 16662 return (0); 16663 } 16664 #ifdef INET6 16665 if (rack->r_state) { 16666 /* Use the cache line loaded if possible */ 16667 isipv6 = rack->r_is_v6; 16668 } else { 16669 isipv6 = (rack->rc_inp->inp_vflag & INP_IPV6) != 0; 16670 } 16671 #endif 16672 early = 0; 16673 cts = tcp_get_usecs(&tv); 16674 ms_cts = tcp_tv_to_mssectick(&tv); 16675 if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) && 16676 tcp_in_hpts(rack->rc_inp)) { 16677 /* 16678 * We are on the hpts for some timer but not hptsi output. 16679 * Remove from the hpts unconditionally. 16680 */ 16681 rack_timer_cancel(tp, rack, cts, __LINE__); 16682 } 16683 /* Are we pacing and late? */ 16684 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && 16685 TSTMP_GEQ(cts, rack->r_ctl.rc_last_output_to)) { 16686 /* We are delayed */ 16687 delayed = cts - rack->r_ctl.rc_last_output_to; 16688 } else { 16689 delayed = 0; 16690 } 16691 /* Do the timers, which may override the pacer */ 16692 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) { 16693 int retval; 16694 16695 retval = rack_process_timers(tp, rack, cts, hpts_calling, 16696 &doing_tlp); 16697 if (retval != 0) { 16698 counter_u64_add(rack_out_size[TCP_MSS_ACCT_ATIMER], 1); 16699 #ifdef TCP_ACCOUNTING 16700 sched_unpin(); 16701 #endif 16702 /* 16703 * If timers want tcp_drop(), then pass error out, 16704 * otherwise suppress it. 16705 */ 16706 return (retval < 0 ? retval : 0); 16707 } 16708 } 16709 if (rack->rc_in_persist) { 16710 if (tcp_in_hpts(rack->rc_inp) == 0) { 16711 /* Timer is not running */ 16712 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); 16713 } 16714 #ifdef TCP_ACCOUNTING 16715 sched_unpin(); 16716 #endif 16717 return (0); 16718 } 16719 if ((rack->rc_ack_required == 1) && 16720 (rack->r_timer_override == 0)){ 16721 /* A timeout occurred and no ack has arrived */ 16722 if (tcp_in_hpts(rack->rc_inp) == 0) { 16723 /* Timer is not running */ 16724 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0); 16725 } 16726 #ifdef TCP_ACCOUNTING 16727 sched_unpin(); 16728 #endif 16729 return (0); 16730 } 16731 if ((rack->r_timer_override) || 16732 (rack->rc_ack_can_sendout_data) || 16733 (delayed) || 16734 (tp->t_state < TCPS_ESTABLISHED)) { 16735 rack->rc_ack_can_sendout_data = 0; 16736 if (tcp_in_hpts(rack->rc_inp)) 16737 tcp_hpts_remove(rack->rc_inp); 16738 } else if (tcp_in_hpts(rack->rc_inp)) { 16739 /* 16740 * On the hpts you can't pass even if ACKNOW is on, we will 16741 * when the hpts fires. 16742 */ 16743 #ifdef TCP_ACCOUNTING 16744 crtsc = get_cyclecount(); 16745 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16746 tp->tcp_proc_time[SND_BLOCKED] += (crtsc - ts_val); 16747 } 16748 counter_u64_add(tcp_proc_time[SND_BLOCKED], (crtsc - ts_val)); 16749 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 16750 tp->tcp_cnt_counters[SND_BLOCKED]++; 16751 } 16752 counter_u64_add(tcp_cnt_counters[SND_BLOCKED], 1); 16753 sched_unpin(); 16754 #endif 16755 counter_u64_add(rack_out_size[TCP_MSS_ACCT_INPACE], 1); 16756 return (0); 16757 } 16758 rack->rc_inp->inp_hpts_calls = 0; 16759 /* Finish out both pacing early and late accounting */ 16760 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) && 16761 TSTMP_GT(rack->r_ctl.rc_last_output_to, cts)) { 16762 early = rack->r_ctl.rc_last_output_to - cts; 16763 } else 16764 early = 0; 16765 if (delayed) { 16766 rack->r_ctl.rc_agg_delayed += delayed; 16767 rack->r_late = 1; 16768 } else if (early) { 16769 rack->r_ctl.rc_agg_early += early; 16770 rack->r_early = 1; 16771 } 16772 /* Now that early/late accounting is done turn off the flag */ 16773 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT; 16774 rack->r_wanted_output = 0; 16775 rack->r_timer_override = 0; 16776 if ((tp->t_state != rack->r_state) && 16777 TCPS_HAVEESTABLISHED(tp->t_state)) { 16778 rack_set_state(tp, rack); 16779 } 16780 if ((rack->r_fast_output) && 16781 (doing_tlp == 0) && 16782 (tp->rcv_numsacks == 0)) { 16783 int ret; 16784 16785 error = 0; 16786 ret = rack_fast_output(tp, rack, ts_val, cts, ms_cts, &tv, tot_len_this_send, &error); 16787 if (ret >= 0) 16788 return(ret); 16789 else if (error) { 16790 inp = rack->rc_inp; 16791 so = inp->inp_socket; 16792 sb = &so->so_snd; 16793 goto nomore; 16794 } 16795 } 16796 inp = rack->rc_inp; 16797 /* 16798 * For TFO connections in SYN_SENT or SYN_RECEIVED, 16799 * only allow the initial SYN or SYN|ACK and those sent 16800 * by the retransmit timer. 16801 */ 16802 if (IS_FASTOPEN(tp->t_flags) && 16803 ((tp->t_state == TCPS_SYN_RECEIVED) || 16804 (tp->t_state == TCPS_SYN_SENT)) && 16805 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */ 16806 (tp->t_rxtshift == 0)) { /* not a retransmit */ 16807 cwnd_to_use = rack->r_ctl.cwnd_to_use = tp->snd_cwnd; 16808 so = inp->inp_socket; 16809 sb = &so->so_snd; 16810 goto just_return_nolock; 16811 } 16812 /* 16813 * Determine length of data that should be transmitted, and flags 16814 * that will be used. If there is some data or critical controls 16815 * (SYN, RST) to send, then transmit; otherwise, investigate 16816 * further. 16817 */ 16818 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una); 16819 if (tp->t_idle_reduce) { 16820 if (idle && (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) 16821 rack_cc_after_idle(rack, tp); 16822 } 16823 tp->t_flags &= ~TF_LASTIDLE; 16824 if (idle) { 16825 if (tp->t_flags & TF_MORETOCOME) { 16826 tp->t_flags |= TF_LASTIDLE; 16827 idle = 0; 16828 } 16829 } 16830 if ((tp->snd_una == tp->snd_max) && 16831 rack->r_ctl.rc_went_idle_time && 16832 TSTMP_GT(cts, rack->r_ctl.rc_went_idle_time)) { 16833 idle = cts - rack->r_ctl.rc_went_idle_time; 16834 if (idle > rack_min_probertt_hold) { 16835 /* Count as a probe rtt */ 16836 if (rack->in_probe_rtt == 0) { 16837 rack->r_ctl.rc_lower_rtt_us_cts = cts; 16838 rack->r_ctl.rc_time_probertt_entered = rack->r_ctl.rc_lower_rtt_us_cts; 16839 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts; 16840 rack->r_ctl.rc_time_of_last_probertt = rack->r_ctl.rc_lower_rtt_us_cts; 16841 } else { 16842 rack_exit_probertt(rack, cts); 16843 } 16844 } 16845 idle = 0; 16846 } 16847 if (rack_use_fsb && (rack->r_fsb_inited == 0) && (rack->r_state != TCPS_CLOSED)) 16848 rack_init_fsb_block(tp, rack); 16849 again: 16850 /* 16851 * If we've recently taken a timeout, snd_max will be greater than 16852 * snd_nxt. There may be SACK information that allows us to avoid 16853 * resending already delivered data. Adjust snd_nxt accordingly. 16854 */ 16855 sendalot = 0; 16856 cts = tcp_get_usecs(&tv); 16857 ms_cts = tcp_tv_to_mssectick(&tv); 16858 tso = 0; 16859 mtu = 0; 16860 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs); 16861 minseg = segsiz; 16862 if (rack->r_ctl.rc_pace_max_segs == 0) 16863 pace_max_seg = rack->rc_user_set_max_segs * segsiz; 16864 else 16865 pace_max_seg = rack->r_ctl.rc_pace_max_segs; 16866 sb_offset = tp->snd_max - tp->snd_una; 16867 cwnd_to_use = rack->r_ctl.cwnd_to_use = tp->snd_cwnd; 16868 flags = tcp_outflags[tp->t_state]; 16869 while (rack->rc_free_cnt < rack_free_cache) { 16870 rsm = rack_alloc(rack); 16871 if (rsm == NULL) { 16872 if (inp->inp_hpts_calls) 16873 /* Retry in a ms */ 16874 slot = (1 * HPTS_USEC_IN_MSEC); 16875 so = inp->inp_socket; 16876 sb = &so->so_snd; 16877 goto just_return_nolock; 16878 } 16879 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_free, rsm, r_tnext); 16880 rack->rc_free_cnt++; 16881 rsm = NULL; 16882 } 16883 if (inp->inp_hpts_calls) 16884 inp->inp_hpts_calls = 0; 16885 sack_rxmit = 0; 16886 len = 0; 16887 rsm = NULL; 16888 if (flags & TH_RST) { 16889 SOCKBUF_LOCK(&inp->inp_socket->so_snd); 16890 so = inp->inp_socket; 16891 sb = &so->so_snd; 16892 goto send; 16893 } 16894 if (rack->r_ctl.rc_resend) { 16895 /* Retransmit timer */ 16896 rsm = rack->r_ctl.rc_resend; 16897 rack->r_ctl.rc_resend = NULL; 16898 len = rsm->r_end - rsm->r_start; 16899 sack_rxmit = 1; 16900 sendalot = 0; 16901 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start), 16902 ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p", 16903 __func__, __LINE__, 16904 rsm->r_start, tp->snd_una, tp, rack, rsm)); 16905 sb_offset = rsm->r_start - tp->snd_una; 16906 if (len >= segsiz) 16907 len = segsiz; 16908 } else if (rack->r_collapse_point_valid && 16909 ((rsm = rack_check_collapsed(rack, cts)) != NULL)) { 16910 /* 16911 * If an RSM is returned then enough time has passed 16912 * for us to retransmit it. Move up the collapse point, 16913 * since this rsm has its chance to retransmit now. 16914 */ 16915 rack_trace_point(rack, RACK_TP_COLLAPSED_RXT); 16916 rack->r_ctl.last_collapse_point = rsm->r_end; 16917 /* Are we done? */ 16918 if (SEQ_GEQ(rack->r_ctl.last_collapse_point, 16919 rack->r_ctl.high_collapse_point)) 16920 rack->r_collapse_point_valid = 0; 16921 sack_rxmit = 1; 16922 /* We are not doing a TLP */ 16923 doing_tlp = 0; 16924 len = rsm->r_end - rsm->r_start; 16925 sb_offset = rsm->r_start - tp->snd_una; 16926 sendalot = 0; 16927 if ((rack->full_size_rxt == 0) && 16928 (rack->shape_rxt_to_pacing_min == 0) && 16929 (len >= segsiz)) 16930 len = segsiz; 16931 } else if ((rsm = tcp_rack_output(tp, rack, cts)) != NULL) { 16932 /* We have a retransmit that takes precedence */ 16933 if ((!IN_FASTRECOVERY(tp->t_flags)) && 16934 ((rsm->r_flags & RACK_MUST_RXT) == 0) && 16935 ((tp->t_flags & TF_WASFRECOVERY) == 0)) { 16936 /* Enter recovery if not induced by a time-out */ 16937 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__); 16938 } 16939 #ifdef INVARIANTS 16940 if (SEQ_LT(rsm->r_start, tp->snd_una)) { 16941 panic("Huh, tp:%p rack:%p rsm:%p start:%u < snd_una:%u\n", 16942 tp, rack, rsm, rsm->r_start, tp->snd_una); 16943 } 16944 #endif 16945 len = rsm->r_end - rsm->r_start; 16946 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start), 16947 ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p", 16948 __func__, __LINE__, 16949 rsm->r_start, tp->snd_una, tp, rack, rsm)); 16950 sb_offset = rsm->r_start - tp->snd_una; 16951 sendalot = 0; 16952 if (len >= segsiz) 16953 len = segsiz; 16954 if (len > 0) { 16955 sack_rxmit = 1; 16956 KMOD_TCPSTAT_INC(tcps_sack_rexmits); 16957 KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes, 16958 min(len, segsiz)); 16959 } 16960 } else if (rack->r_ctl.rc_tlpsend) { 16961 /* Tail loss probe */ 16962 long cwin; 16963 long tlen; 16964 16965 /* 16966 * Check if we can do a TLP with a RACK'd packet 16967 * this can happen if we are not doing the rack 16968 * cheat and we skipped to a TLP and it 16969 * went off. 16970 */ 16971 rsm = rack->r_ctl.rc_tlpsend; 16972 /* We are doing a TLP make sure the flag is preent */ 16973 rsm->r_flags |= RACK_TLP; 16974 rack->r_ctl.rc_tlpsend = NULL; 16975 sack_rxmit = 1; 16976 tlen = rsm->r_end - rsm->r_start; 16977 if (tlen > segsiz) 16978 tlen = segsiz; 16979 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start), 16980 ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p", 16981 __func__, __LINE__, 16982 rsm->r_start, tp->snd_una, tp, rack, rsm)); 16983 sb_offset = rsm->r_start - tp->snd_una; 16984 cwin = min(tp->snd_wnd, tlen); 16985 len = cwin; 16986 } 16987 if (rack->r_must_retran && 16988 (doing_tlp == 0) && 16989 (SEQ_GT(tp->snd_max, tp->snd_una)) && 16990 (rsm == NULL)) { 16991 /* 16992 * There are two different ways that we 16993 * can get into this block: 16994 * a) This is a non-sack connection, we had a time-out 16995 * and thus r_must_retran was set and everything 16996 * left outstanding as been marked for retransmit. 16997 * b) The MTU of the path shrank, so that everything 16998 * was marked to be retransmitted with the smaller 16999 * mtu and r_must_retran was set. 17000 * 17001 * This means that we expect the sendmap (outstanding) 17002 * to all be marked must. We can use the tmap to 17003 * look at them. 17004 * 17005 */ 17006 int sendwin, flight; 17007 17008 sendwin = min(tp->snd_wnd, tp->snd_cwnd); 17009 flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto); 17010 if (flight >= sendwin) { 17011 /* 17012 * We can't send yet. 17013 */ 17014 so = inp->inp_socket; 17015 sb = &so->so_snd; 17016 goto just_return_nolock; 17017 } 17018 /* 17019 * This is the case a/b mentioned above. All 17020 * outstanding/not-acked should be marked. 17021 * We can use the tmap to find them. 17022 */ 17023 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap); 17024 if (rsm == NULL) { 17025 /* TSNH */ 17026 rack->r_must_retran = 0; 17027 rack->r_ctl.rc_out_at_rto = 0; 17028 so = inp->inp_socket; 17029 sb = &so->so_snd; 17030 goto just_return_nolock; 17031 } 17032 if ((rsm->r_flags & RACK_MUST_RXT) == 0) { 17033 /* 17034 * The first one does not have the flag, did we collapse 17035 * further up in our list? 17036 */ 17037 rack->r_must_retran = 0; 17038 rack->r_ctl.rc_out_at_rto = 0; 17039 rsm = NULL; 17040 sack_rxmit = 0; 17041 } else { 17042 sack_rxmit = 1; 17043 len = rsm->r_end - rsm->r_start; 17044 sb_offset = rsm->r_start - tp->snd_una; 17045 sendalot = 0; 17046 if ((rack->full_size_rxt == 0) && 17047 (rack->shape_rxt_to_pacing_min == 0) && 17048 (len >= segsiz)) 17049 len = segsiz; 17050 /* 17051 * Delay removing the flag RACK_MUST_RXT so 17052 * that the fastpath for retransmit will 17053 * work with this rsm. 17054 */ 17055 } 17056 } 17057 /* 17058 * Enforce a connection sendmap count limit if set 17059 * as long as we are not retransmiting. 17060 */ 17061 if ((rsm == NULL) && 17062 (rack->do_detection == 0) && 17063 (V_tcp_map_entries_limit > 0) && 17064 (rack->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) { 17065 counter_u64_add(rack_to_alloc_limited, 1); 17066 if (!rack->alloc_limit_reported) { 17067 rack->alloc_limit_reported = 1; 17068 counter_u64_add(rack_alloc_limited_conns, 1); 17069 } 17070 so = inp->inp_socket; 17071 sb = &so->so_snd; 17072 goto just_return_nolock; 17073 } 17074 if (rsm && (rsm->r_flags & RACK_HAS_FIN)) { 17075 /* we are retransmitting the fin */ 17076 len--; 17077 if (len) { 17078 /* 17079 * When retransmitting data do *not* include the 17080 * FIN. This could happen from a TLP probe. 17081 */ 17082 flags &= ~TH_FIN; 17083 } 17084 } 17085 if (rsm && rack->r_fsb_inited && rack_use_rsm_rfo && 17086 ((rsm->r_flags & RACK_HAS_FIN) == 0)) { 17087 int ret; 17088 17089 ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp); 17090 if (ret == 0) 17091 return (0); 17092 } 17093 so = inp->inp_socket; 17094 sb = &so->so_snd; 17095 if (do_a_prefetch == 0) { 17096 kern_prefetch(sb, &do_a_prefetch); 17097 do_a_prefetch = 1; 17098 } 17099 #ifdef NETFLIX_SHARED_CWND 17100 if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) && 17101 rack->rack_enable_scwnd) { 17102 /* We are doing cwnd sharing */ 17103 if (rack->gp_ready && 17104 (rack->rack_attempted_scwnd == 0) && 17105 (rack->r_ctl.rc_scw == NULL) && 17106 tp->t_lib) { 17107 /* The pcbid is in, lets make an attempt */ 17108 counter_u64_add(rack_try_scwnd, 1); 17109 rack->rack_attempted_scwnd = 1; 17110 rack->r_ctl.rc_scw = tcp_shared_cwnd_alloc(tp, 17111 &rack->r_ctl.rc_scw_index, 17112 segsiz); 17113 } 17114 if (rack->r_ctl.rc_scw && 17115 (rack->rack_scwnd_is_idle == 1) && 17116 sbavail(&so->so_snd)) { 17117 /* we are no longer out of data */ 17118 tcp_shared_cwnd_active(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index); 17119 rack->rack_scwnd_is_idle = 0; 17120 } 17121 if (rack->r_ctl.rc_scw) { 17122 /* First lets update and get the cwnd */ 17123 rack->r_ctl.cwnd_to_use = cwnd_to_use = tcp_shared_cwnd_update(rack->r_ctl.rc_scw, 17124 rack->r_ctl.rc_scw_index, 17125 tp->snd_cwnd, tp->snd_wnd, segsiz); 17126 } 17127 } 17128 #endif 17129 /* 17130 * Get standard flags, and add SYN or FIN if requested by 'hidden' 17131 * state flags. 17132 */ 17133 if (tp->t_flags & TF_NEEDFIN) 17134 flags |= TH_FIN; 17135 if (tp->t_flags & TF_NEEDSYN) 17136 flags |= TH_SYN; 17137 if ((sack_rxmit == 0) && (prefetch_rsm == 0)) { 17138 void *end_rsm; 17139 end_rsm = TAILQ_LAST_FAST(&rack->r_ctl.rc_tmap, rack_sendmap, r_tnext); 17140 if (end_rsm) 17141 kern_prefetch(end_rsm, &prefetch_rsm); 17142 prefetch_rsm = 1; 17143 } 17144 SOCKBUF_LOCK(sb); 17145 /* 17146 * If snd_nxt == snd_max and we have transmitted a FIN, the 17147 * sb_offset will be > 0 even if so_snd.sb_cc is 0, resulting in a 17148 * negative length. This can also occur when TCP opens up its 17149 * congestion window while receiving additional duplicate acks after 17150 * fast-retransmit because TCP will reset snd_nxt to snd_max after 17151 * the fast-retransmit. 17152 * 17153 * In the normal retransmit-FIN-only case, however, snd_nxt will be 17154 * set to snd_una, the sb_offset will be 0, and the length may wind 17155 * up 0. 17156 * 17157 * If sack_rxmit is true we are retransmitting from the scoreboard 17158 * in which case len is already set. 17159 */ 17160 if ((sack_rxmit == 0) && 17161 (TCPS_HAVEESTABLISHED(tp->t_state) || IS_FASTOPEN(tp->t_flags))) { 17162 uint32_t avail; 17163 17164 avail = sbavail(sb); 17165 if (SEQ_GT(tp->snd_nxt, tp->snd_una) && avail) 17166 sb_offset = tp->snd_nxt - tp->snd_una; 17167 else 17168 sb_offset = 0; 17169 if ((IN_FASTRECOVERY(tp->t_flags) == 0) || rack->rack_no_prr) { 17170 if (rack->r_ctl.rc_tlp_new_data) { 17171 /* TLP is forcing out new data */ 17172 if (rack->r_ctl.rc_tlp_new_data > (uint32_t) (avail - sb_offset)) { 17173 rack->r_ctl.rc_tlp_new_data = (uint32_t) (avail - sb_offset); 17174 } 17175 if ((rack->r_ctl.rc_tlp_new_data + sb_offset) > tp->snd_wnd) { 17176 if (tp->snd_wnd > sb_offset) 17177 len = tp->snd_wnd - sb_offset; 17178 else 17179 len = 0; 17180 } else { 17181 len = rack->r_ctl.rc_tlp_new_data; 17182 } 17183 rack->r_ctl.rc_tlp_new_data = 0; 17184 } else { 17185 len = rack_what_can_we_send(tp, rack, cwnd_to_use, avail, sb_offset); 17186 } 17187 if ((rack->r_ctl.crte == NULL) && IN_FASTRECOVERY(tp->t_flags) && (len > segsiz)) { 17188 /* 17189 * For prr=off, we need to send only 1 MSS 17190 * at a time. We do this because another sack could 17191 * be arriving that causes us to send retransmits and 17192 * we don't want to be on a long pace due to a larger send 17193 * that keeps us from sending out the retransmit. 17194 */ 17195 len = segsiz; 17196 } 17197 } else { 17198 uint32_t outstanding; 17199 /* 17200 * We are inside of a Fast recovery episode, this 17201 * is caused by a SACK or 3 dup acks. At this point 17202 * we have sent all the retransmissions and we rely 17203 * on PRR to dictate what we will send in the form of 17204 * new data. 17205 */ 17206 17207 outstanding = tp->snd_max - tp->snd_una; 17208 if ((rack->r_ctl.rc_prr_sndcnt + outstanding) > tp->snd_wnd) { 17209 if (tp->snd_wnd > outstanding) { 17210 len = tp->snd_wnd - outstanding; 17211 /* Check to see if we have the data */ 17212 if ((sb_offset + len) > avail) { 17213 /* It does not all fit */ 17214 if (avail > sb_offset) 17215 len = avail - sb_offset; 17216 else 17217 len = 0; 17218 } 17219 } else { 17220 len = 0; 17221 } 17222 } else if (avail > sb_offset) { 17223 len = avail - sb_offset; 17224 } else { 17225 len = 0; 17226 } 17227 if (len > 0) { 17228 if (len > rack->r_ctl.rc_prr_sndcnt) { 17229 len = rack->r_ctl.rc_prr_sndcnt; 17230 } 17231 if (len > 0) { 17232 sub_from_prr = 1; 17233 } 17234 } 17235 if (len > segsiz) { 17236 /* 17237 * We should never send more than a MSS when 17238 * retransmitting or sending new data in prr 17239 * mode unless the override flag is on. Most 17240 * likely the PRR algorithm is not going to 17241 * let us send a lot as well :-) 17242 */ 17243 if (rack->r_ctl.rc_prr_sendalot == 0) { 17244 len = segsiz; 17245 } 17246 } else if (len < segsiz) { 17247 /* 17248 * Do we send any? The idea here is if the 17249 * send empty's the socket buffer we want to 17250 * do it. However if not then lets just wait 17251 * for our prr_sndcnt to get bigger. 17252 */ 17253 long leftinsb; 17254 17255 leftinsb = sbavail(sb) - sb_offset; 17256 if (leftinsb > len) { 17257 /* This send does not empty the sb */ 17258 len = 0; 17259 } 17260 } 17261 } 17262 } else if (!TCPS_HAVEESTABLISHED(tp->t_state)) { 17263 /* 17264 * If you have not established 17265 * and are not doing FAST OPEN 17266 * no data please. 17267 */ 17268 if ((sack_rxmit == 0) && 17269 (!IS_FASTOPEN(tp->t_flags))){ 17270 len = 0; 17271 sb_offset = 0; 17272 } 17273 } 17274 if (prefetch_so_done == 0) { 17275 kern_prefetch(so, &prefetch_so_done); 17276 prefetch_so_done = 1; 17277 } 17278 /* 17279 * Lop off SYN bit if it has already been sent. However, if this is 17280 * SYN-SENT state and if segment contains data and if we don't know 17281 * that foreign host supports TAO, suppress sending segment. 17282 */ 17283 if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una) && 17284 ((sack_rxmit == 0) && (tp->t_rxtshift == 0))) { 17285 /* 17286 * When sending additional segments following a TFO SYN|ACK, 17287 * do not include the SYN bit. 17288 */ 17289 if (IS_FASTOPEN(tp->t_flags) && 17290 (tp->t_state == TCPS_SYN_RECEIVED)) 17291 flags &= ~TH_SYN; 17292 } 17293 /* 17294 * Be careful not to send data and/or FIN on SYN segments. This 17295 * measure is needed to prevent interoperability problems with not 17296 * fully conformant TCP implementations. 17297 */ 17298 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) { 17299 len = 0; 17300 flags &= ~TH_FIN; 17301 } 17302 /* 17303 * On TFO sockets, ensure no data is sent in the following cases: 17304 * 17305 * - When retransmitting SYN|ACK on a passively-created socket 17306 * 17307 * - When retransmitting SYN on an actively created socket 17308 * 17309 * - When sending a zero-length cookie (cookie request) on an 17310 * actively created socket 17311 * 17312 * - When the socket is in the CLOSED state (RST is being sent) 17313 */ 17314 if (IS_FASTOPEN(tp->t_flags) && 17315 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) || 17316 ((tp->t_state == TCPS_SYN_SENT) && 17317 (tp->t_tfo_client_cookie_len == 0)) || 17318 (flags & TH_RST))) { 17319 sack_rxmit = 0; 17320 len = 0; 17321 } 17322 /* Without fast-open there should never be data sent on a SYN */ 17323 if ((flags & TH_SYN) && (!IS_FASTOPEN(tp->t_flags))) { 17324 tp->snd_nxt = tp->iss; 17325 len = 0; 17326 } 17327 if ((len > segsiz) && (tcp_dsack_block_exists(tp))) { 17328 /* We only send 1 MSS if we have a DSACK block */ 17329 add_flag |= RACK_SENT_W_DSACK; 17330 len = segsiz; 17331 } 17332 orig_len = len; 17333 if (len <= 0) { 17334 /* 17335 * If FIN has been sent but not acked, but we haven't been 17336 * called to retransmit, len will be < 0. Otherwise, window 17337 * shrank after we sent into it. If window shrank to 0, 17338 * cancel pending retransmit, pull snd_nxt back to (closed) 17339 * window, and set the persist timer if it isn't already 17340 * going. If the window didn't close completely, just wait 17341 * for an ACK. 17342 * 17343 * We also do a general check here to ensure that we will 17344 * set the persist timer when we have data to send, but a 17345 * 0-byte window. This makes sure the persist timer is set 17346 * even if the packet hits one of the "goto send" lines 17347 * below. 17348 */ 17349 len = 0; 17350 if ((tp->snd_wnd == 0) && 17351 (TCPS_HAVEESTABLISHED(tp->t_state)) && 17352 (tp->snd_una == tp->snd_max) && 17353 (sb_offset < (int)sbavail(sb))) { 17354 rack_enter_persist(tp, rack, cts); 17355 } 17356 } else if ((rsm == NULL) && 17357 (doing_tlp == 0) && 17358 (len < pace_max_seg)) { 17359 /* 17360 * We are not sending a maximum sized segment for 17361 * some reason. Should we not send anything (think 17362 * sws or persists)? 17363 */ 17364 if ((tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), minseg)) && 17365 (TCPS_HAVEESTABLISHED(tp->t_state)) && 17366 (len < minseg) && 17367 (len < (int)(sbavail(sb) - sb_offset))) { 17368 /* 17369 * Here the rwnd is less than 17370 * the minimum pacing size, this is not a retransmit, 17371 * we are established and 17372 * the send is not the last in the socket buffer 17373 * we send nothing, and we may enter persists 17374 * if nothing is outstanding. 17375 */ 17376 len = 0; 17377 if (tp->snd_max == tp->snd_una) { 17378 /* 17379 * Nothing out we can 17380 * go into persists. 17381 */ 17382 rack_enter_persist(tp, rack, cts); 17383 } 17384 } else if ((cwnd_to_use >= max(minseg, (segsiz * 4))) && 17385 (ctf_flight_size(tp, rack->r_ctl.rc_sacked) > (2 * segsiz)) && 17386 (len < (int)(sbavail(sb) - sb_offset)) && 17387 (len < minseg)) { 17388 /* 17389 * Here we are not retransmitting, and 17390 * the cwnd is not so small that we could 17391 * not send at least a min size (rxt timer 17392 * not having gone off), We have 2 segments or 17393 * more already in flight, its not the tail end 17394 * of the socket buffer and the cwnd is blocking 17395 * us from sending out a minimum pacing segment size. 17396 * Lets not send anything. 17397 */ 17398 len = 0; 17399 } else if (((tp->snd_wnd - ctf_outstanding(tp)) < 17400 min((rack->r_ctl.rc_high_rwnd/2), minseg)) && 17401 (ctf_flight_size(tp, rack->r_ctl.rc_sacked) > (2 * segsiz)) && 17402 (len < (int)(sbavail(sb) - sb_offset)) && 17403 (TCPS_HAVEESTABLISHED(tp->t_state))) { 17404 /* 17405 * Here we have a send window but we have 17406 * filled it up and we can't send another pacing segment. 17407 * We also have in flight more than 2 segments 17408 * and we are not completing the sb i.e. we allow 17409 * the last bytes of the sb to go out even if 17410 * its not a full pacing segment. 17411 */ 17412 len = 0; 17413 } else if ((rack->r_ctl.crte != NULL) && 17414 (tp->snd_wnd >= (pace_max_seg * max(1, rack_hw_rwnd_factor))) && 17415 (cwnd_to_use >= (pace_max_seg + (4 * segsiz))) && 17416 (ctf_flight_size(tp, rack->r_ctl.rc_sacked) >= (2 * segsiz)) && 17417 (len < (int)(sbavail(sb) - sb_offset))) { 17418 /* 17419 * Here we are doing hardware pacing, this is not a TLP, 17420 * we are not sending a pace max segment size, there is rwnd 17421 * room to send at least N pace_max_seg, the cwnd is greater 17422 * than or equal to a full pacing segments plus 4 mss and we have 2 or 17423 * more segments in flight and its not the tail of the socket buffer. 17424 * 17425 * We don't want to send instead we need to get more ack's in to 17426 * allow us to send a full pacing segment. Normally, if we are pacing 17427 * about the right speed, we should have finished our pacing 17428 * send as most of the acks have come back if we are at the 17429 * right rate. This is a bit fuzzy since return path delay 17430 * can delay the acks, which is why we want to make sure we 17431 * have cwnd space to have a bit more than a max pace segments in flight. 17432 * 17433 * If we have not gotten our acks back we are pacing at too high a 17434 * rate delaying will not hurt and will bring our GP estimate down by 17435 * injecting the delay. If we don't do this we will send 17436 * 2 MSS out in response to the acks being clocked in which 17437 * defeats the point of hw-pacing (i.e. to help us get 17438 * larger TSO's out). 17439 */ 17440 len = 0; 17441 17442 } 17443 17444 } 17445 /* len will be >= 0 after this point. */ 17446 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__)); 17447 rack_sndbuf_autoscale(rack); 17448 /* 17449 * Decide if we can use TCP Segmentation Offloading (if supported by 17450 * hardware). 17451 * 17452 * TSO may only be used if we are in a pure bulk sending state. The 17453 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP 17454 * options prevent using TSO. With TSO the TCP header is the same 17455 * (except for the sequence number) for all generated packets. This 17456 * makes it impossible to transmit any options which vary per 17457 * generated segment or packet. 17458 * 17459 * IPv4 handling has a clear separation of ip options and ip header 17460 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen() does 17461 * the right thing below to provide length of just ip options and thus 17462 * checking for ipoptlen is enough to decide if ip options are present. 17463 */ 17464 ipoptlen = 0; 17465 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 17466 /* 17467 * Pre-calculate here as we save another lookup into the darknesses 17468 * of IPsec that way and can actually decide if TSO is ok. 17469 */ 17470 #ifdef INET6 17471 if (isipv6 && IPSEC_ENABLED(ipv6)) 17472 ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp); 17473 #ifdef INET 17474 else 17475 #endif 17476 #endif /* INET6 */ 17477 #ifdef INET 17478 if (IPSEC_ENABLED(ipv4)) 17479 ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp); 17480 #endif /* INET */ 17481 #endif 17482 17483 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 17484 ipoptlen += ipsec_optlen; 17485 #endif 17486 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > segsiz && 17487 (tp->t_port == 0) && 17488 ((tp->t_flags & TF_SIGNATURE) == 0) && 17489 tp->rcv_numsacks == 0 && sack_rxmit == 0 && 17490 ipoptlen == 0) 17491 tso = 1; 17492 { 17493 uint32_t outstanding __unused; 17494 17495 outstanding = tp->snd_max - tp->snd_una; 17496 if (tp->t_flags & TF_SENTFIN) { 17497 /* 17498 * If we sent a fin, snd_max is 1 higher than 17499 * snd_una 17500 */ 17501 outstanding--; 17502 } 17503 if (sack_rxmit) { 17504 if ((rsm->r_flags & RACK_HAS_FIN) == 0) 17505 flags &= ~TH_FIN; 17506 } else { 17507 if (SEQ_LT(tp->snd_nxt + len, tp->snd_una + 17508 sbused(sb))) 17509 flags &= ~TH_FIN; 17510 } 17511 } 17512 recwin = lmin(lmax(sbspace(&so->so_rcv), 0), 17513 (long)TCP_MAXWIN << tp->rcv_scale); 17514 17515 /* 17516 * Sender silly window avoidance. We transmit under the following 17517 * conditions when len is non-zero: 17518 * 17519 * - We have a full segment (or more with TSO) - This is the last 17520 * buffer in a write()/send() and we are either idle or running 17521 * NODELAY - we've timed out (e.g. persist timer) - we have more 17522 * then 1/2 the maximum send window's worth of data (receiver may be 17523 * limited the window size) - we need to retransmit 17524 */ 17525 if (len) { 17526 if (len >= segsiz) { 17527 goto send; 17528 } 17529 /* 17530 * NOTE! on localhost connections an 'ack' from the remote 17531 * end may occur synchronously with the output and cause us 17532 * to flush a buffer queued with moretocome. XXX 17533 * 17534 */ 17535 if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */ 17536 (idle || (tp->t_flags & TF_NODELAY)) && 17537 ((uint32_t)len + (uint32_t)sb_offset >= sbavail(sb)) && 17538 (tp->t_flags & TF_NOPUSH) == 0) { 17539 pass = 2; 17540 goto send; 17541 } 17542 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */ 17543 pass = 22; 17544 goto send; 17545 } 17546 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) { 17547 pass = 4; 17548 goto send; 17549 } 17550 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { /* retransmit case */ 17551 pass = 5; 17552 goto send; 17553 } 17554 if (sack_rxmit) { 17555 pass = 6; 17556 goto send; 17557 } 17558 if (((tp->snd_wnd - ctf_outstanding(tp)) < segsiz) && 17559 (ctf_outstanding(tp) < (segsiz * 2))) { 17560 /* 17561 * We have less than two MSS outstanding (delayed ack) 17562 * and our rwnd will not let us send a full sized 17563 * MSS. Lets go ahead and let this small segment 17564 * out because we want to try to have at least two 17565 * packets inflight to not be caught by delayed ack. 17566 */ 17567 pass = 12; 17568 goto send; 17569 } 17570 } 17571 /* 17572 * Sending of standalone window updates. 17573 * 17574 * Window updates are important when we close our window due to a 17575 * full socket buffer and are opening it again after the application 17576 * reads data from it. Once the window has opened again and the 17577 * remote end starts to send again the ACK clock takes over and 17578 * provides the most current window information. 17579 * 17580 * We must avoid the silly window syndrome whereas every read from 17581 * the receive buffer, no matter how small, causes a window update 17582 * to be sent. We also should avoid sending a flurry of window 17583 * updates when the socket buffer had queued a lot of data and the 17584 * application is doing small reads. 17585 * 17586 * Prevent a flurry of pointless window updates by only sending an 17587 * update when we can increase the advertized window by more than 17588 * 1/4th of the socket buffer capacity. When the buffer is getting 17589 * full or is very small be more aggressive and send an update 17590 * whenever we can increase by two mss sized segments. In all other 17591 * situations the ACK's to new incoming data will carry further 17592 * window increases. 17593 * 17594 * Don't send an independent window update if a delayed ACK is 17595 * pending (it will get piggy-backed on it) or the remote side 17596 * already has done a half-close and won't send more data. Skip 17597 * this if the connection is in T/TCP half-open state. 17598 */ 17599 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) && 17600 !(tp->t_flags & TF_DELACK) && 17601 !TCPS_HAVERCVDFIN(tp->t_state)) { 17602 /* 17603 * "adv" is the amount we could increase the window, taking 17604 * into account that we are limited by TCP_MAXWIN << 17605 * tp->rcv_scale. 17606 */ 17607 int32_t adv; 17608 int oldwin; 17609 17610 adv = recwin; 17611 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) { 17612 oldwin = (tp->rcv_adv - tp->rcv_nxt); 17613 if (adv > oldwin) 17614 adv -= oldwin; 17615 else { 17616 /* We can't increase the window */ 17617 adv = 0; 17618 } 17619 } else 17620 oldwin = 0; 17621 17622 /* 17623 * If the new window size ends up being the same as or less 17624 * than the old size when it is scaled, then don't force 17625 * a window update. 17626 */ 17627 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale) 17628 goto dontupdate; 17629 17630 if (adv >= (int32_t)(2 * segsiz) && 17631 (adv >= (int32_t)(so->so_rcv.sb_hiwat / 4) || 17632 recwin <= (int32_t)(so->so_rcv.sb_hiwat / 8) || 17633 so->so_rcv.sb_hiwat <= 8 * segsiz)) { 17634 pass = 7; 17635 goto send; 17636 } 17637 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat) { 17638 pass = 23; 17639 goto send; 17640 } 17641 } 17642 dontupdate: 17643 17644 /* 17645 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW 17646 * is also a catch-all for the retransmit timer timeout case. 17647 */ 17648 if (tp->t_flags & TF_ACKNOW) { 17649 pass = 8; 17650 goto send; 17651 } 17652 if (((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0)) { 17653 pass = 9; 17654 goto send; 17655 } 17656 /* 17657 * If our state indicates that FIN should be sent and we have not 17658 * yet done so, then we need to send. 17659 */ 17660 if ((flags & TH_FIN) && 17661 (tp->snd_nxt == tp->snd_una)) { 17662 pass = 11; 17663 goto send; 17664 } 17665 /* 17666 * No reason to send a segment, just return. 17667 */ 17668 just_return: 17669 SOCKBUF_UNLOCK(sb); 17670 just_return_nolock: 17671 { 17672 int app_limited = CTF_JR_SENT_DATA; 17673 17674 if (tot_len_this_send > 0) { 17675 /* Make sure snd_nxt is up to max */ 17676 rack->r_ctl.fsb.recwin = recwin; 17677 slot = rack_get_pacing_delay(rack, tp, tot_len_this_send, NULL, segsiz); 17678 if ((error == 0) && 17679 rack_use_rfo && 17680 ((flags & (TH_SYN|TH_FIN)) == 0) && 17681 (ipoptlen == 0) && 17682 (tp->snd_nxt == tp->snd_max) && 17683 (tp->rcv_numsacks == 0) && 17684 rack->r_fsb_inited && 17685 TCPS_HAVEESTABLISHED(tp->t_state) && 17686 (rack->r_must_retran == 0) && 17687 ((tp->t_flags & TF_NEEDFIN) == 0) && 17688 (len > 0) && (orig_len > 0) && 17689 (orig_len > len) && 17690 ((orig_len - len) >= segsiz) && 17691 ((optlen == 0) || 17692 ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) { 17693 /* We can send at least one more MSS using our fsb */ 17694 17695 rack->r_fast_output = 1; 17696 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off); 17697 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; 17698 rack->r_ctl.fsb.tcp_flags = flags; 17699 rack->r_ctl.fsb.left_to_send = orig_len - len; 17700 if (hw_tls) 17701 rack->r_ctl.fsb.hw_tls = 1; 17702 else 17703 rack->r_ctl.fsb.hw_tls = 0; 17704 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))), 17705 ("rack:%p left_to_send:%u sbavail:%u out:%u", 17706 rack, rack->r_ctl.fsb.left_to_send, sbavail(sb), 17707 (tp->snd_max - tp->snd_una))); 17708 if (rack->r_ctl.fsb.left_to_send < segsiz) 17709 rack->r_fast_output = 0; 17710 else { 17711 if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una))) 17712 rack->r_ctl.fsb.rfo_apply_push = 1; 17713 else 17714 rack->r_ctl.fsb.rfo_apply_push = 0; 17715 } 17716 } else 17717 rack->r_fast_output = 0; 17718 17719 17720 rack_log_fsb(rack, tp, so, flags, 17721 ipoptlen, orig_len, len, 0, 17722 1, optlen, __LINE__, 1); 17723 if (SEQ_GT(tp->snd_max, tp->snd_nxt)) 17724 tp->snd_nxt = tp->snd_max; 17725 } else { 17726 int end_window = 0; 17727 uint32_t seq = tp->gput_ack; 17728 17729 rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 17730 if (rsm) { 17731 /* 17732 * Mark the last sent that we just-returned (hinting 17733 * that delayed ack may play a role in any rtt measurement). 17734 */ 17735 rsm->r_just_ret = 1; 17736 } 17737 counter_u64_add(rack_out_size[TCP_MSS_ACCT_JUSTRET], 1); 17738 rack->r_ctl.rc_agg_delayed = 0; 17739 rack->r_early = 0; 17740 rack->r_late = 0; 17741 rack->r_ctl.rc_agg_early = 0; 17742 if ((ctf_outstanding(tp) + 17743 min(max(segsiz, (rack->r_ctl.rc_high_rwnd/2)), 17744 minseg)) >= tp->snd_wnd) { 17745 /* We are limited by the rwnd */ 17746 app_limited = CTF_JR_RWND_LIMITED; 17747 if (IN_FASTRECOVERY(tp->t_flags)) 17748 rack->r_ctl.rc_prr_sndcnt = 0; 17749 } else if (ctf_outstanding(tp) >= sbavail(sb)) { 17750 /* We are limited by whats available -- app limited */ 17751 app_limited = CTF_JR_APP_LIMITED; 17752 if (IN_FASTRECOVERY(tp->t_flags)) 17753 rack->r_ctl.rc_prr_sndcnt = 0; 17754 } else if ((idle == 0) && 17755 ((tp->t_flags & TF_NODELAY) == 0) && 17756 ((uint32_t)len + (uint32_t)sb_offset >= sbavail(sb)) && 17757 (len < segsiz)) { 17758 /* 17759 * No delay is not on and the 17760 * user is sending less than 1MSS. This 17761 * brings out SWS avoidance so we 17762 * don't send. Another app-limited case. 17763 */ 17764 app_limited = CTF_JR_APP_LIMITED; 17765 } else if (tp->t_flags & TF_NOPUSH) { 17766 /* 17767 * The user has requested no push of 17768 * the last segment and we are 17769 * at the last segment. Another app 17770 * limited case. 17771 */ 17772 app_limited = CTF_JR_APP_LIMITED; 17773 } else if ((ctf_outstanding(tp) + minseg) > cwnd_to_use) { 17774 /* Its the cwnd */ 17775 app_limited = CTF_JR_CWND_LIMITED; 17776 } else if (IN_FASTRECOVERY(tp->t_flags) && 17777 (rack->rack_no_prr == 0) && 17778 (rack->r_ctl.rc_prr_sndcnt < segsiz)) { 17779 app_limited = CTF_JR_PRR; 17780 } else { 17781 /* Now why here are we not sending? */ 17782 #ifdef NOW 17783 #ifdef INVARIANTS 17784 panic("rack:%p hit JR_ASSESSING case cwnd_to_use:%u?", rack, cwnd_to_use); 17785 #endif 17786 #endif 17787 app_limited = CTF_JR_ASSESSING; 17788 } 17789 /* 17790 * App limited in some fashion, for our pacing GP 17791 * measurements we don't want any gap (even cwnd). 17792 * Close down the measurement window. 17793 */ 17794 if (rack_cwnd_block_ends_measure && 17795 ((app_limited == CTF_JR_CWND_LIMITED) || 17796 (app_limited == CTF_JR_PRR))) { 17797 /* 17798 * The reason we are not sending is 17799 * the cwnd (or prr). We have been configured 17800 * to end the measurement window in 17801 * this case. 17802 */ 17803 end_window = 1; 17804 } else if (rack_rwnd_block_ends_measure && 17805 (app_limited == CTF_JR_RWND_LIMITED)) { 17806 /* 17807 * We are rwnd limited and have been 17808 * configured to end the measurement 17809 * window in this case. 17810 */ 17811 end_window = 1; 17812 } else if (app_limited == CTF_JR_APP_LIMITED) { 17813 /* 17814 * A true application limited period, we have 17815 * ran out of data. 17816 */ 17817 end_window = 1; 17818 } else if (app_limited == CTF_JR_ASSESSING) { 17819 /* 17820 * In the assessing case we hit the end of 17821 * the if/else and had no known reason 17822 * This will panic us under invariants.. 17823 * 17824 * If we get this out in logs we need to 17825 * investagate which reason we missed. 17826 */ 17827 end_window = 1; 17828 } 17829 if (end_window) { 17830 uint8_t log = 0; 17831 17832 /* Adjust the Gput measurement */ 17833 if ((tp->t_flags & TF_GPUTINPROG) && 17834 SEQ_GT(tp->gput_ack, tp->snd_max)) { 17835 tp->gput_ack = tp->snd_max; 17836 if ((tp->gput_ack - tp->gput_seq) < (MIN_GP_WIN * segsiz)) { 17837 /* 17838 * There is not enough to measure. 17839 */ 17840 tp->t_flags &= ~TF_GPUTINPROG; 17841 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/, 17842 rack->r_ctl.rc_gp_srtt /*flex1*/, 17843 tp->gput_seq, 17844 0, 0, 18, __LINE__, NULL, 0); 17845 } else 17846 log = 1; 17847 } 17848 /* Mark the last packet has app limited */ 17849 rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree); 17850 if (rsm && ((rsm->r_flags & RACK_APP_LIMITED) == 0)) { 17851 if (rack->r_ctl.rc_app_limited_cnt == 0) 17852 rack->r_ctl.rc_end_appl = rack->r_ctl.rc_first_appl = rsm; 17853 else { 17854 /* 17855 * Go out to the end app limited and mark 17856 * this new one as next and move the end_appl up 17857 * to this guy. 17858 */ 17859 if (rack->r_ctl.rc_end_appl) 17860 rack->r_ctl.rc_end_appl->r_nseq_appl = rsm->r_start; 17861 rack->r_ctl.rc_end_appl = rsm; 17862 } 17863 rsm->r_flags |= RACK_APP_LIMITED; 17864 rack->r_ctl.rc_app_limited_cnt++; 17865 } 17866 if (log) 17867 rack_log_pacing_delay_calc(rack, 17868 rack->r_ctl.rc_app_limited_cnt, seq, 17869 tp->gput_ack, 0, 0, 4, __LINE__, NULL, 0); 17870 } 17871 } 17872 /* Check if we need to go into persists or not */ 17873 if ((tp->snd_max == tp->snd_una) && 17874 TCPS_HAVEESTABLISHED(tp->t_state) && 17875 sbavail(sb) && 17876 (sbavail(sb) > tp->snd_wnd) && 17877 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), minseg))) { 17878 /* Yes lets make sure to move to persist before timer-start */ 17879 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime); 17880 } 17881 rack_start_hpts_timer(rack, tp, cts, slot, tot_len_this_send, sup_rack); 17882 rack_log_type_just_return(rack, cts, tot_len_this_send, slot, hpts_calling, app_limited, cwnd_to_use); 17883 } 17884 #ifdef NETFLIX_SHARED_CWND 17885 if ((sbavail(sb) == 0) && 17886 rack->r_ctl.rc_scw) { 17887 tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index); 17888 rack->rack_scwnd_is_idle = 1; 17889 } 17890 #endif 17891 #ifdef TCP_ACCOUNTING 17892 if (tot_len_this_send > 0) { 17893 crtsc = get_cyclecount(); 17894 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 17895 tp->tcp_cnt_counters[SND_OUT_DATA]++; 17896 } 17897 counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], 1); 17898 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 17899 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val); 17900 } 17901 counter_u64_add(tcp_proc_time[SND_OUT_DATA], (crtsc - ts_val)); 17902 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 17903 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len_this_send + segsiz - 1) / segsiz); 17904 } 17905 counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((tot_len_this_send + segsiz - 1) / segsiz)); 17906 } else { 17907 crtsc = get_cyclecount(); 17908 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 17909 tp->tcp_cnt_counters[SND_LIMITED]++; 17910 } 17911 counter_u64_add(tcp_cnt_counters[SND_LIMITED], 1); 17912 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 17913 tp->tcp_proc_time[SND_LIMITED] += (crtsc - ts_val); 17914 } 17915 counter_u64_add(tcp_proc_time[SND_LIMITED], (crtsc - ts_val)); 17916 } 17917 sched_unpin(); 17918 #endif 17919 return (0); 17920 17921 send: 17922 if (rsm || sack_rxmit) 17923 counter_u64_add(rack_nfto_resend, 1); 17924 else 17925 counter_u64_add(rack_non_fto_send, 1); 17926 if ((flags & TH_FIN) && 17927 sbavail(sb)) { 17928 /* 17929 * We do not transmit a FIN 17930 * with data outstanding. We 17931 * need to make it so all data 17932 * is acked first. 17933 */ 17934 flags &= ~TH_FIN; 17935 } 17936 /* Enforce stack imposed max seg size if we have one */ 17937 if (rack->r_ctl.rc_pace_max_segs && 17938 (len > rack->r_ctl.rc_pace_max_segs)) { 17939 mark = 1; 17940 len = rack->r_ctl.rc_pace_max_segs; 17941 } 17942 SOCKBUF_LOCK_ASSERT(sb); 17943 if (len > 0) { 17944 if (len >= segsiz) 17945 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT; 17946 else 17947 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT; 17948 } 17949 /* 17950 * Before ESTABLISHED, force sending of initial options unless TCP 17951 * set not to do any options. NOTE: we assume that the IP/TCP header 17952 * plus TCP options always fit in a single mbuf, leaving room for a 17953 * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr) 17954 * + optlen <= MCLBYTES 17955 */ 17956 optlen = 0; 17957 #ifdef INET6 17958 if (isipv6) 17959 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr); 17960 else 17961 #endif 17962 hdrlen = sizeof(struct tcpiphdr); 17963 17964 /* 17965 * Compute options for segment. We only have to care about SYN and 17966 * established connection segments. Options for SYN-ACK segments 17967 * are handled in TCP syncache. 17968 */ 17969 to.to_flags = 0; 17970 if ((tp->t_flags & TF_NOOPT) == 0) { 17971 /* Maximum segment size. */ 17972 if (flags & TH_SYN) { 17973 tp->snd_nxt = tp->iss; 17974 to.to_mss = tcp_mssopt(&inp->inp_inc); 17975 if (tp->t_port) 17976 to.to_mss -= V_tcp_udp_tunneling_overhead; 17977 to.to_flags |= TOF_MSS; 17978 17979 /* 17980 * On SYN or SYN|ACK transmits on TFO connections, 17981 * only include the TFO option if it is not a 17982 * retransmit, as the presence of the TFO option may 17983 * have caused the original SYN or SYN|ACK to have 17984 * been dropped by a middlebox. 17985 */ 17986 if (IS_FASTOPEN(tp->t_flags) && 17987 (tp->t_rxtshift == 0)) { 17988 if (tp->t_state == TCPS_SYN_RECEIVED) { 17989 to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN; 17990 to.to_tfo_cookie = 17991 (u_int8_t *)&tp->t_tfo_cookie.server; 17992 to.to_flags |= TOF_FASTOPEN; 17993 wanted_cookie = 1; 17994 } else if (tp->t_state == TCPS_SYN_SENT) { 17995 to.to_tfo_len = 17996 tp->t_tfo_client_cookie_len; 17997 to.to_tfo_cookie = 17998 tp->t_tfo_cookie.client; 17999 to.to_flags |= TOF_FASTOPEN; 18000 wanted_cookie = 1; 18001 /* 18002 * If we wind up having more data to 18003 * send with the SYN than can fit in 18004 * one segment, don't send any more 18005 * until the SYN|ACK comes back from 18006 * the other end. 18007 */ 18008 sendalot = 0; 18009 } 18010 } 18011 } 18012 /* Window scaling. */ 18013 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) { 18014 to.to_wscale = tp->request_r_scale; 18015 to.to_flags |= TOF_SCALE; 18016 } 18017 /* Timestamps. */ 18018 if ((tp->t_flags & TF_RCVD_TSTMP) || 18019 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) { 18020 to.to_tsval = ms_cts + tp->ts_offset; 18021 to.to_tsecr = tp->ts_recent; 18022 to.to_flags |= TOF_TS; 18023 } 18024 /* Set receive buffer autosizing timestamp. */ 18025 if (tp->rfbuf_ts == 0 && 18026 (so->so_rcv.sb_flags & SB_AUTOSIZE)) 18027 tp->rfbuf_ts = tcp_ts_getticks(); 18028 /* Selective ACK's. */ 18029 if (tp->t_flags & TF_SACK_PERMIT) { 18030 if (flags & TH_SYN) 18031 to.to_flags |= TOF_SACKPERM; 18032 else if (TCPS_HAVEESTABLISHED(tp->t_state) && 18033 tp->rcv_numsacks > 0) { 18034 to.to_flags |= TOF_SACK; 18035 to.to_nsacks = tp->rcv_numsacks; 18036 to.to_sacks = (u_char *)tp->sackblks; 18037 } 18038 } 18039 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) 18040 /* TCP-MD5 (RFC2385). */ 18041 if (tp->t_flags & TF_SIGNATURE) 18042 to.to_flags |= TOF_SIGNATURE; 18043 #endif /* TCP_SIGNATURE */ 18044 18045 /* Processing the options. */ 18046 hdrlen += optlen = tcp_addoptions(&to, opt); 18047 /* 18048 * If we wanted a TFO option to be added, but it was unable 18049 * to fit, ensure no data is sent. 18050 */ 18051 if (IS_FASTOPEN(tp->t_flags) && wanted_cookie && 18052 !(to.to_flags & TOF_FASTOPEN)) 18053 len = 0; 18054 } 18055 if (tp->t_port) { 18056 if (V_tcp_udp_tunneling_port == 0) { 18057 /* The port was removed?? */ 18058 SOCKBUF_UNLOCK(&so->so_snd); 18059 #ifdef TCP_ACCOUNTING 18060 crtsc = get_cyclecount(); 18061 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 18062 tp->tcp_cnt_counters[SND_OUT_FAIL]++; 18063 } 18064 counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1); 18065 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 18066 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val); 18067 } 18068 counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val)); 18069 sched_unpin(); 18070 #endif 18071 return (EHOSTUNREACH); 18072 } 18073 hdrlen += sizeof(struct udphdr); 18074 } 18075 #ifdef INET6 18076 if (isipv6) 18077 ipoptlen = ip6_optlen(inp); 18078 else 18079 #endif 18080 if (inp->inp_options) 18081 ipoptlen = inp->inp_options->m_len - 18082 offsetof(struct ipoption, ipopt_list); 18083 else 18084 ipoptlen = 0; 18085 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 18086 ipoptlen += ipsec_optlen; 18087 #endif 18088 18089 /* 18090 * Adjust data length if insertion of options will bump the packet 18091 * length beyond the t_maxseg length. Clear the FIN bit because we 18092 * cut off the tail of the segment. 18093 */ 18094 if (len + optlen + ipoptlen > tp->t_maxseg) { 18095 if (tso) { 18096 uint32_t if_hw_tsomax; 18097 uint32_t moff; 18098 int32_t max_len; 18099 18100 /* extract TSO information */ 18101 if_hw_tsomax = tp->t_tsomax; 18102 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount; 18103 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize; 18104 KASSERT(ipoptlen == 0, 18105 ("%s: TSO can't do IP options", __func__)); 18106 18107 /* 18108 * Check if we should limit by maximum payload 18109 * length: 18110 */ 18111 if (if_hw_tsomax != 0) { 18112 /* compute maximum TSO length */ 18113 max_len = (if_hw_tsomax - hdrlen - 18114 max_linkhdr); 18115 if (max_len <= 0) { 18116 len = 0; 18117 } else if (len > max_len) { 18118 sendalot = 1; 18119 len = max_len; 18120 mark = 2; 18121 } 18122 } 18123 /* 18124 * Prevent the last segment from being fractional 18125 * unless the send sockbuf can be emptied: 18126 */ 18127 max_len = (tp->t_maxseg - optlen); 18128 if ((sb_offset + len) < sbavail(sb)) { 18129 moff = len % (u_int)max_len; 18130 if (moff != 0) { 18131 mark = 3; 18132 len -= moff; 18133 } 18134 } 18135 /* 18136 * In case there are too many small fragments don't 18137 * use TSO: 18138 */ 18139 if (len <= segsiz) { 18140 mark = 4; 18141 tso = 0; 18142 } 18143 /* 18144 * Send the FIN in a separate segment after the bulk 18145 * sending is done. We don't trust the TSO 18146 * implementations to clear the FIN flag on all but 18147 * the last segment. 18148 */ 18149 if (tp->t_flags & TF_NEEDFIN) { 18150 sendalot = 4; 18151 } 18152 } else { 18153 mark = 5; 18154 if (optlen + ipoptlen >= tp->t_maxseg) { 18155 /* 18156 * Since we don't have enough space to put 18157 * the IP header chain and the TCP header in 18158 * one packet as required by RFC 7112, don't 18159 * send it. Also ensure that at least one 18160 * byte of the payload can be put into the 18161 * TCP segment. 18162 */ 18163 SOCKBUF_UNLOCK(&so->so_snd); 18164 error = EMSGSIZE; 18165 sack_rxmit = 0; 18166 goto out; 18167 } 18168 len = tp->t_maxseg - optlen - ipoptlen; 18169 sendalot = 5; 18170 } 18171 } else { 18172 tso = 0; 18173 mark = 6; 18174 } 18175 KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET, 18176 ("%s: len > IP_MAXPACKET", __func__)); 18177 #ifdef DIAGNOSTIC 18178 #ifdef INET6 18179 if (max_linkhdr + hdrlen > MCLBYTES) 18180 #else 18181 if (max_linkhdr + hdrlen > MHLEN) 18182 #endif 18183 panic("tcphdr too big"); 18184 #endif 18185 18186 /* 18187 * This KASSERT is here to catch edge cases at a well defined place. 18188 * Before, those had triggered (random) panic conditions further 18189 * down. 18190 */ 18191 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__)); 18192 if ((len == 0) && 18193 (flags & TH_FIN) && 18194 (sbused(sb))) { 18195 /* 18196 * We have outstanding data, don't send a fin by itself!. 18197 */ 18198 goto just_return; 18199 } 18200 /* 18201 * Grab a header mbuf, attaching a copy of data to be transmitted, 18202 * and initialize the header from the template for sends on this 18203 * connection. 18204 */ 18205 hw_tls = tp->t_nic_ktls_xmit != 0; 18206 if (len) { 18207 uint32_t max_val; 18208 uint32_t moff; 18209 18210 if (rack->r_ctl.rc_pace_max_segs) 18211 max_val = rack->r_ctl.rc_pace_max_segs; 18212 else if (rack->rc_user_set_max_segs) 18213 max_val = rack->rc_user_set_max_segs * segsiz; 18214 else 18215 max_val = len; 18216 /* 18217 * We allow a limit on sending with hptsi. 18218 */ 18219 if (len > max_val) { 18220 mark = 7; 18221 len = max_val; 18222 } 18223 #ifdef INET6 18224 if (MHLEN < hdrlen + max_linkhdr) 18225 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 18226 else 18227 #endif 18228 m = m_gethdr(M_NOWAIT, MT_DATA); 18229 18230 if (m == NULL) { 18231 SOCKBUF_UNLOCK(sb); 18232 error = ENOBUFS; 18233 sack_rxmit = 0; 18234 goto out; 18235 } 18236 m->m_data += max_linkhdr; 18237 m->m_len = hdrlen; 18238 18239 /* 18240 * Start the m_copy functions from the closest mbuf to the 18241 * sb_offset in the socket buffer chain. 18242 */ 18243 mb = sbsndptr_noadv(sb, sb_offset, &moff); 18244 s_mb = mb; 18245 s_moff = moff; 18246 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) { 18247 m_copydata(mb, moff, (int)len, 18248 mtod(m, caddr_t)+hdrlen); 18249 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) 18250 sbsndptr_adv(sb, mb, len); 18251 m->m_len += len; 18252 } else { 18253 struct sockbuf *msb; 18254 18255 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) 18256 msb = NULL; 18257 else 18258 msb = sb; 18259 m->m_next = tcp_m_copym( 18260 mb, moff, &len, 18261 if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, msb, 18262 ((rsm == NULL) ? hw_tls : 0) 18263 #ifdef NETFLIX_COPY_ARGS 18264 , &s_mb, &s_moff 18265 #endif 18266 ); 18267 if (len <= (tp->t_maxseg - optlen)) { 18268 /* 18269 * Must have ran out of mbufs for the copy 18270 * shorten it to no longer need tso. Lets 18271 * not put on sendalot since we are low on 18272 * mbufs. 18273 */ 18274 tso = 0; 18275 } 18276 if (m->m_next == NULL) { 18277 SOCKBUF_UNLOCK(sb); 18278 (void)m_free(m); 18279 error = ENOBUFS; 18280 sack_rxmit = 0; 18281 goto out; 18282 } 18283 } 18284 if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) { 18285 if (rsm && (rsm->r_flags & RACK_TLP)) { 18286 /* 18287 * TLP should not count in retran count, but 18288 * in its own bin 18289 */ 18290 counter_u64_add(rack_tlp_retran, 1); 18291 counter_u64_add(rack_tlp_retran_bytes, len); 18292 } else { 18293 tp->t_sndrexmitpack++; 18294 KMOD_TCPSTAT_INC(tcps_sndrexmitpack); 18295 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len); 18296 } 18297 #ifdef STATS 18298 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB, 18299 len); 18300 #endif 18301 } else { 18302 KMOD_TCPSTAT_INC(tcps_sndpack); 18303 KMOD_TCPSTAT_ADD(tcps_sndbyte, len); 18304 #ifdef STATS 18305 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB, 18306 len); 18307 #endif 18308 } 18309 /* 18310 * If we're sending everything we've got, set PUSH. (This 18311 * will keep happy those implementations which only give 18312 * data to the user when a buffer fills or a PUSH comes in.) 18313 */ 18314 if (sb_offset + len == sbused(sb) && 18315 sbused(sb) && 18316 !(flags & TH_SYN)) { 18317 flags |= TH_PUSH; 18318 add_flag |= RACK_HAD_PUSH; 18319 } 18320 18321 SOCKBUF_UNLOCK(sb); 18322 } else { 18323 SOCKBUF_UNLOCK(sb); 18324 if (tp->t_flags & TF_ACKNOW) 18325 KMOD_TCPSTAT_INC(tcps_sndacks); 18326 else if (flags & (TH_SYN | TH_FIN | TH_RST)) 18327 KMOD_TCPSTAT_INC(tcps_sndctrl); 18328 else 18329 KMOD_TCPSTAT_INC(tcps_sndwinup); 18330 18331 m = m_gethdr(M_NOWAIT, MT_DATA); 18332 if (m == NULL) { 18333 error = ENOBUFS; 18334 sack_rxmit = 0; 18335 goto out; 18336 } 18337 #ifdef INET6 18338 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) && 18339 MHLEN >= hdrlen) { 18340 M_ALIGN(m, hdrlen); 18341 } else 18342 #endif 18343 m->m_data += max_linkhdr; 18344 m->m_len = hdrlen; 18345 } 18346 SOCKBUF_UNLOCK_ASSERT(sb); 18347 m->m_pkthdr.rcvif = (struct ifnet *)0; 18348 #ifdef MAC 18349 mac_inpcb_create_mbuf(inp, m); 18350 #endif 18351 if ((ipoptlen == 0) && (rack->r_ctl.fsb.tcp_ip_hdr) && rack->r_fsb_inited) { 18352 #ifdef INET6 18353 if (isipv6) 18354 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr; 18355 else 18356 #endif /* INET6 */ 18357 #ifdef INET 18358 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr; 18359 #endif 18360 th = rack->r_ctl.fsb.th; 18361 udp = rack->r_ctl.fsb.udp; 18362 if (udp) { 18363 #ifdef INET6 18364 if (isipv6) 18365 ulen = hdrlen + len - sizeof(struct ip6_hdr); 18366 else 18367 #endif /* INET6 */ 18368 ulen = hdrlen + len - sizeof(struct ip); 18369 udp->uh_ulen = htons(ulen); 18370 } 18371 } else { 18372 #ifdef INET6 18373 if (isipv6) { 18374 ip6 = mtod(m, struct ip6_hdr *); 18375 if (tp->t_port) { 18376 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 18377 udp->uh_sport = htons(V_tcp_udp_tunneling_port); 18378 udp->uh_dport = tp->t_port; 18379 ulen = hdrlen + len - sizeof(struct ip6_hdr); 18380 udp->uh_ulen = htons(ulen); 18381 th = (struct tcphdr *)(udp + 1); 18382 } else 18383 th = (struct tcphdr *)(ip6 + 1); 18384 tcpip_fillheaders(inp, tp->t_port, ip6, th); 18385 } else 18386 #endif /* INET6 */ 18387 { 18388 #ifdef INET 18389 ip = mtod(m, struct ip *); 18390 if (tp->t_port) { 18391 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip)); 18392 udp->uh_sport = htons(V_tcp_udp_tunneling_port); 18393 udp->uh_dport = tp->t_port; 18394 ulen = hdrlen + len - sizeof(struct ip); 18395 udp->uh_ulen = htons(ulen); 18396 th = (struct tcphdr *)(udp + 1); 18397 } else 18398 th = (struct tcphdr *)(ip + 1); 18399 tcpip_fillheaders(inp, tp->t_port, ip, th); 18400 #endif 18401 } 18402 } 18403 /* 18404 * Fill in fields, remembering maximum advertised window for use in 18405 * delaying messages about window sizes. If resending a FIN, be sure 18406 * not to use a new sequence number. 18407 */ 18408 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN && 18409 tp->snd_nxt == tp->snd_max) 18410 tp->snd_nxt--; 18411 /* 18412 * If we are starting a connection, send ECN setup SYN packet. If we 18413 * are on a retransmit, we may resend those bits a number of times 18414 * as per RFC 3168. 18415 */ 18416 if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn) { 18417 flags |= tcp_ecn_output_syn_sent(tp); 18418 } 18419 /* Also handle parallel SYN for ECN */ 18420 if (TCPS_HAVERCVDSYN(tp->t_state) && 18421 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) { 18422 int ect = tcp_ecn_output_established(tp, &flags, len, sack_rxmit); 18423 if ((tp->t_state == TCPS_SYN_RECEIVED) && 18424 (tp->t_flags2 & TF2_ECN_SND_ECE)) 18425 tp->t_flags2 &= ~TF2_ECN_SND_ECE; 18426 #ifdef INET6 18427 if (isipv6) { 18428 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20); 18429 ip6->ip6_flow |= htonl(ect << 20); 18430 } 18431 else 18432 #endif 18433 { 18434 #ifdef INET 18435 ip->ip_tos &= ~IPTOS_ECN_MASK; 18436 ip->ip_tos |= ect; 18437 #endif 18438 } 18439 } 18440 /* 18441 * If we are doing retransmissions, then snd_nxt will not reflect 18442 * the first unsent octet. For ACK only packets, we do not want the 18443 * sequence number of the retransmitted packet, we want the sequence 18444 * number of the next unsent octet. So, if there is no data (and no 18445 * SYN or FIN), use snd_max instead of snd_nxt when filling in 18446 * ti_seq. But if we are in persist state, snd_max might reflect 18447 * one byte beyond the right edge of the window, so use snd_nxt in 18448 * that case, since we know we aren't doing a retransmission. 18449 * (retransmit and persist are mutually exclusive...) 18450 */ 18451 if (sack_rxmit == 0) { 18452 if (len || (flags & (TH_SYN | TH_FIN))) { 18453 th->th_seq = htonl(tp->snd_nxt); 18454 rack_seq = tp->snd_nxt; 18455 } else { 18456 th->th_seq = htonl(tp->snd_max); 18457 rack_seq = tp->snd_max; 18458 } 18459 } else { 18460 th->th_seq = htonl(rsm->r_start); 18461 rack_seq = rsm->r_start; 18462 } 18463 th->th_ack = htonl(tp->rcv_nxt); 18464 tcp_set_flags(th, flags); 18465 /* 18466 * Calculate receive window. Don't shrink window, but avoid silly 18467 * window syndrome. 18468 * If a RST segment is sent, advertise a window of zero. 18469 */ 18470 if (flags & TH_RST) { 18471 recwin = 0; 18472 } else { 18473 if (recwin < (long)(so->so_rcv.sb_hiwat / 4) && 18474 recwin < (long)segsiz) { 18475 recwin = 0; 18476 } 18477 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) && 18478 recwin < (long)(tp->rcv_adv - tp->rcv_nxt)) 18479 recwin = (long)(tp->rcv_adv - tp->rcv_nxt); 18480 } 18481 18482 /* 18483 * According to RFC1323 the window field in a SYN (i.e., a <SYN> or 18484 * <SYN,ACK>) segment itself is never scaled. The <SYN,ACK> case is 18485 * handled in syncache. 18486 */ 18487 if (flags & TH_SYN) 18488 th->th_win = htons((u_short) 18489 (min(sbspace(&so->so_rcv), TCP_MAXWIN))); 18490 else { 18491 /* Avoid shrinking window with window scaling. */ 18492 recwin = roundup2(recwin, 1 << tp->rcv_scale); 18493 th->th_win = htons((u_short)(recwin >> tp->rcv_scale)); 18494 } 18495 /* 18496 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0 18497 * window. This may cause the remote transmitter to stall. This 18498 * flag tells soreceive() to disable delayed acknowledgements when 18499 * draining the buffer. This can occur if the receiver is 18500 * attempting to read more data than can be buffered prior to 18501 * transmitting on the connection. 18502 */ 18503 if (th->th_win == 0) { 18504 tp->t_sndzerowin++; 18505 tp->t_flags |= TF_RXWIN0SENT; 18506 } else 18507 tp->t_flags &= ~TF_RXWIN0SENT; 18508 tp->snd_up = tp->snd_una; /* drag it along, its deprecated */ 18509 /* Now are we using fsb?, if so copy the template data to the mbuf */ 18510 if ((ipoptlen == 0) && (rack->r_ctl.fsb.tcp_ip_hdr) && rack->r_fsb_inited) { 18511 uint8_t *cpto; 18512 18513 cpto = mtod(m, uint8_t *); 18514 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len); 18515 /* 18516 * We have just copied in: 18517 * IP/IP6 18518 * <optional udphdr> 18519 * tcphdr (no options) 18520 * 18521 * We need to grab the correct pointers into the mbuf 18522 * for both the tcp header, and possibly the udp header (if tunneling). 18523 * We do this by using the offset in the copy buffer and adding it 18524 * to the mbuf base pointer (cpto). 18525 */ 18526 #ifdef INET6 18527 if (isipv6) 18528 ip6 = mtod(m, struct ip6_hdr *); 18529 else 18530 #endif /* INET6 */ 18531 #ifdef INET 18532 ip = mtod(m, struct ip *); 18533 #endif 18534 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr)); 18535 /* If we have a udp header lets set it into the mbuf as well */ 18536 if (udp) 18537 udp = (struct udphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.udp - rack->r_ctl.fsb.tcp_ip_hdr)); 18538 } 18539 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE) 18540 if (to.to_flags & TOF_SIGNATURE) { 18541 /* 18542 * Calculate MD5 signature and put it into the place 18543 * determined before. 18544 * NOTE: since TCP options buffer doesn't point into 18545 * mbuf's data, calculate offset and use it. 18546 */ 18547 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th, 18548 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) { 18549 /* 18550 * Do not send segment if the calculation of MD5 18551 * digest has failed. 18552 */ 18553 goto out; 18554 } 18555 } 18556 #endif 18557 if (optlen) { 18558 bcopy(opt, th + 1, optlen); 18559 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2; 18560 } 18561 /* 18562 * Put TCP length in extended header, and then checksum extended 18563 * header and data. 18564 */ 18565 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */ 18566 #ifdef INET6 18567 if (isipv6) { 18568 /* 18569 * ip6_plen is not need to be filled now, and will be filled 18570 * in ip6_output. 18571 */ 18572 if (tp->t_port) { 18573 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6; 18574 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 18575 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0); 18576 th->th_sum = htons(0); 18577 UDPSTAT_INC(udps_opackets); 18578 } else { 18579 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6; 18580 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 18581 th->th_sum = in6_cksum_pseudo(ip6, 18582 sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP, 18583 0); 18584 } 18585 } 18586 #endif 18587 #if defined(INET6) && defined(INET) 18588 else 18589 #endif 18590 #ifdef INET 18591 { 18592 if (tp->t_port) { 18593 m->m_pkthdr.csum_flags = CSUM_UDP; 18594 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum); 18595 udp->uh_sum = in_pseudo(ip->ip_src.s_addr, 18596 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP)); 18597 th->th_sum = htons(0); 18598 UDPSTAT_INC(udps_opackets); 18599 } else { 18600 m->m_pkthdr.csum_flags = CSUM_TCP; 18601 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum); 18602 th->th_sum = in_pseudo(ip->ip_src.s_addr, 18603 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) + 18604 IPPROTO_TCP + len + optlen)); 18605 } 18606 /* IP version must be set here for ipv4/ipv6 checking later */ 18607 KASSERT(ip->ip_v == IPVERSION, 18608 ("%s: IP version incorrect: %d", __func__, ip->ip_v)); 18609 } 18610 #endif 18611 /* 18612 * Enable TSO and specify the size of the segments. The TCP pseudo 18613 * header checksum is always provided. XXX: Fixme: This is currently 18614 * not the case for IPv6. 18615 */ 18616 if (tso) { 18617 KASSERT(len > tp->t_maxseg - optlen, 18618 ("%s: len <= tso_segsz", __func__)); 18619 m->m_pkthdr.csum_flags |= CSUM_TSO; 18620 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen; 18621 } 18622 KASSERT(len + hdrlen == m_length(m, NULL), 18623 ("%s: mbuf chain different than expected: %d + %u != %u", 18624 __func__, len, hdrlen, m_length(m, NULL))); 18625 18626 #ifdef TCP_HHOOK 18627 /* Run HHOOK_TCP_ESTABLISHED_OUT helper hooks. */ 18628 hhook_run_tcp_est_out(tp, th, &to, len, tso); 18629 #endif 18630 /* We're getting ready to send; log now. */ 18631 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 18632 union tcp_log_stackspecific log; 18633 18634 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 18635 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp); 18636 if (rack->rack_no_prr) 18637 log.u_bbr.flex1 = 0; 18638 else 18639 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt; 18640 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs; 18641 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs; 18642 log.u_bbr.flex4 = orig_len; 18643 /* Save off the early/late values */ 18644 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early; 18645 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed; 18646 log.u_bbr.bw_inuse = rack_get_bw(rack); 18647 log.u_bbr.flex8 = 0; 18648 if (rsm) { 18649 if (rsm->r_flags & RACK_RWND_COLLAPSED) { 18650 rack_log_collapse(rack, rsm->r_start, rsm->r_end, 0, __LINE__, 5, rsm->r_flags, rsm); 18651 counter_u64_add(rack_collapsed_win_rxt, 1); 18652 counter_u64_add(rack_collapsed_win_rxt_bytes, (rsm->r_end - rsm->r_start)); 18653 } 18654 if (doing_tlp) 18655 log.u_bbr.flex8 = 2; 18656 else 18657 log.u_bbr.flex8 = 1; 18658 } else { 18659 if (doing_tlp) 18660 log.u_bbr.flex8 = 3; 18661 else 18662 log.u_bbr.flex8 = 0; 18663 } 18664 log.u_bbr.pacing_gain = rack_get_output_gain(rack, rsm); 18665 log.u_bbr.flex7 = mark; 18666 log.u_bbr.flex7 <<= 8; 18667 log.u_bbr.flex7 |= pass; 18668 log.u_bbr.pkts_out = tp->t_maxseg; 18669 log.u_bbr.timeStamp = cts; 18670 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 18671 log.u_bbr.lt_epoch = cwnd_to_use; 18672 log.u_bbr.delivered = sendalot; 18673 lgb = tcp_log_event_(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK, 18674 len, &log, false, NULL, NULL, 0, &tv); 18675 } else 18676 lgb = NULL; 18677 18678 /* 18679 * Fill in IP length and desired time to live and send to IP level. 18680 * There should be a better way to handle ttl and tos; we could keep 18681 * them in the template, but need a way to checksum without them. 18682 */ 18683 /* 18684 * m->m_pkthdr.len should have been set before cksum calcuration, 18685 * because in6_cksum() need it. 18686 */ 18687 #ifdef INET6 18688 if (isipv6) { 18689 /* 18690 * we separately set hoplimit for every segment, since the 18691 * user might want to change the value via setsockopt. Also, 18692 * desired default hop limit might be changed via Neighbor 18693 * Discovery. 18694 */ 18695 rack->r_ctl.fsb.hoplimit = ip6->ip6_hlim = in6_selecthlim(inp, NULL); 18696 18697 /* 18698 * Set the packet size here for the benefit of DTrace 18699 * probes. ip6_output() will set it properly; it's supposed 18700 * to include the option header lengths as well. 18701 */ 18702 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6)); 18703 18704 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) 18705 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 18706 else 18707 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 18708 18709 if (tp->t_state == TCPS_SYN_SENT) 18710 TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th); 18711 18712 TCP_PROBE5(send, NULL, tp, ip6, tp, th); 18713 /* TODO: IPv6 IP6TOS_ECT bit on */ 18714 error = ip6_output(m, 18715 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 18716 inp->in6p_outputopts, 18717 #else 18718 NULL, 18719 #endif 18720 &inp->inp_route6, 18721 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 18722 NULL, NULL, inp); 18723 18724 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL) 18725 mtu = inp->inp_route6.ro_nh->nh_mtu; 18726 } 18727 #endif /* INET6 */ 18728 #if defined(INET) && defined(INET6) 18729 else 18730 #endif 18731 #ifdef INET 18732 { 18733 ip->ip_len = htons(m->m_pkthdr.len); 18734 #ifdef INET6 18735 if (inp->inp_vflag & INP_IPV6PROTO) 18736 ip->ip_ttl = in6_selecthlim(inp, NULL); 18737 #endif /* INET6 */ 18738 rack->r_ctl.fsb.hoplimit = ip->ip_ttl; 18739 /* 18740 * If we do path MTU discovery, then we set DF on every 18741 * packet. This might not be the best thing to do according 18742 * to RFC3390 Section 2. However the tcp hostcache migitates 18743 * the problem so it affects only the first tcp connection 18744 * with a host. 18745 * 18746 * NB: Don't set DF on small MTU/MSS to have a safe 18747 * fallback. 18748 */ 18749 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) { 18750 tp->t_flags2 |= TF2_PLPMTU_PMTUD; 18751 if (tp->t_port == 0 || len < V_tcp_minmss) { 18752 ip->ip_off |= htons(IP_DF); 18753 } 18754 } else { 18755 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD; 18756 } 18757 18758 if (tp->t_state == TCPS_SYN_SENT) 18759 TCP_PROBE5(connect__request, NULL, tp, ip, tp, th); 18760 18761 TCP_PROBE5(send, NULL, tp, ip, tp, th); 18762 18763 error = ip_output(m, 18764 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 18765 inp->inp_options, 18766 #else 18767 NULL, 18768 #endif 18769 &inp->inp_route, 18770 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0, 18771 inp); 18772 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL) 18773 mtu = inp->inp_route.ro_nh->nh_mtu; 18774 } 18775 #endif /* INET */ 18776 18777 out: 18778 if (lgb) { 18779 lgb->tlb_errno = error; 18780 lgb = NULL; 18781 } 18782 /* 18783 * In transmit state, time the transmission and arrange for the 18784 * retransmit. In persist state, just set snd_max. 18785 */ 18786 if (error == 0) { 18787 tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls); 18788 if (rsm && doing_tlp) { 18789 rack->rc_last_sent_tlp_past_cumack = 0; 18790 rack->rc_last_sent_tlp_seq_valid = 1; 18791 rack->r_ctl.last_sent_tlp_seq = rsm->r_start; 18792 rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start; 18793 } 18794 rack->forced_ack = 0; /* If we send something zap the FA flag */ 18795 if (rsm && (doing_tlp == 0)) { 18796 /* Set we retransmitted */ 18797 rack->rc_gp_saw_rec = 1; 18798 } else { 18799 if (cwnd_to_use > tp->snd_ssthresh) { 18800 /* Set we sent in CA */ 18801 rack->rc_gp_saw_ca = 1; 18802 } else { 18803 /* Set we sent in SS */ 18804 rack->rc_gp_saw_ss = 1; 18805 } 18806 } 18807 if (TCPS_HAVEESTABLISHED(tp->t_state) && 18808 (tp->t_flags & TF_SACK_PERMIT) && 18809 tp->rcv_numsacks > 0) 18810 tcp_clean_dsack_blocks(tp); 18811 tot_len_this_send += len; 18812 if (len == 0) 18813 counter_u64_add(rack_out_size[TCP_MSS_ACCT_SNDACK], 1); 18814 else if (len == 1) { 18815 counter_u64_add(rack_out_size[TCP_MSS_ACCT_PERSIST], 1); 18816 } else if (len > 1) { 18817 int idx; 18818 18819 idx = (len / segsiz) + 3; 18820 if (idx >= TCP_MSS_ACCT_ATIMER) 18821 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1); 18822 else 18823 counter_u64_add(rack_out_size[idx], 1); 18824 } 18825 } 18826 if ((rack->rack_no_prr == 0) && 18827 sub_from_prr && 18828 (error == 0)) { 18829 if (rack->r_ctl.rc_prr_sndcnt >= len) 18830 rack->r_ctl.rc_prr_sndcnt -= len; 18831 else 18832 rack->r_ctl.rc_prr_sndcnt = 0; 18833 } 18834 sub_from_prr = 0; 18835 if (doing_tlp) { 18836 /* Make sure the TLP is added */ 18837 add_flag |= RACK_TLP; 18838 } else if (rsm) { 18839 /* If its a resend without TLP then it must not have the flag */ 18840 rsm->r_flags &= ~RACK_TLP; 18841 } 18842 rack_log_output(tp, &to, len, rack_seq, (uint8_t) flags, error, 18843 rack_to_usec_ts(&tv), 18844 rsm, add_flag, s_mb, s_moff, hw_tls); 18845 18846 18847 if ((error == 0) && 18848 (len > 0) && 18849 (tp->snd_una == tp->snd_max)) 18850 rack->r_ctl.rc_tlp_rxt_last_time = cts; 18851 { 18852 tcp_seq startseq = tp->snd_nxt; 18853 18854 /* Track our lost count */ 18855 if (rsm && (doing_tlp == 0)) 18856 rack->r_ctl.rc_loss_count += rsm->r_end - rsm->r_start; 18857 /* 18858 * Advance snd_nxt over sequence space of this segment. 18859 */ 18860 if (error) 18861 /* We don't log or do anything with errors */ 18862 goto nomore; 18863 if (doing_tlp == 0) { 18864 if (rsm == NULL) { 18865 /* 18866 * Not a retransmission of some 18867 * sort, new data is going out so 18868 * clear our TLP count and flag. 18869 */ 18870 rack->rc_tlp_in_progress = 0; 18871 rack->r_ctl.rc_tlp_cnt_out = 0; 18872 } 18873 } else { 18874 /* 18875 * We have just sent a TLP, mark that it is true 18876 * and make sure our in progress is set so we 18877 * continue to check the count. 18878 */ 18879 rack->rc_tlp_in_progress = 1; 18880 rack->r_ctl.rc_tlp_cnt_out++; 18881 } 18882 if (flags & (TH_SYN | TH_FIN)) { 18883 if (flags & TH_SYN) 18884 tp->snd_nxt++; 18885 if (flags & TH_FIN) { 18886 tp->snd_nxt++; 18887 tp->t_flags |= TF_SENTFIN; 18888 } 18889 } 18890 /* In the ENOBUFS case we do *not* update snd_max */ 18891 if (sack_rxmit) 18892 goto nomore; 18893 18894 tp->snd_nxt += len; 18895 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) { 18896 if (tp->snd_una == tp->snd_max) { 18897 /* 18898 * Update the time we just added data since 18899 * none was outstanding. 18900 */ 18901 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__); 18902 tp->t_acktime = ticks; 18903 } 18904 tp->snd_max = tp->snd_nxt; 18905 /* 18906 * Time this transmission if not a retransmission and 18907 * not currently timing anything. 18908 * This is only relevant in case of switching back to 18909 * the base stack. 18910 */ 18911 if (tp->t_rtttime == 0) { 18912 tp->t_rtttime = ticks; 18913 tp->t_rtseq = startseq; 18914 KMOD_TCPSTAT_INC(tcps_segstimed); 18915 } 18916 if (len && 18917 ((tp->t_flags & TF_GPUTINPROG) == 0)) 18918 rack_start_gp_measurement(tp, rack, startseq, sb_offset); 18919 } 18920 /* 18921 * If we are doing FO we need to update the mbuf position and subtract 18922 * this happens when the peer sends us duplicate information and 18923 * we thus want to send a DSACK. 18924 * 18925 * XXXRRS: This brings to mind a ?, when we send a DSACK block is TSO 18926 * turned off? If not then we are going to echo multiple DSACK blocks 18927 * out (with the TSO), which we should not be doing. 18928 */ 18929 if (rack->r_fast_output && len) { 18930 if (rack->r_ctl.fsb.left_to_send > len) 18931 rack->r_ctl.fsb.left_to_send -= len; 18932 else 18933 rack->r_ctl.fsb.left_to_send = 0; 18934 if (rack->r_ctl.fsb.left_to_send < segsiz) 18935 rack->r_fast_output = 0; 18936 if (rack->r_fast_output) { 18937 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off); 18938 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; 18939 } 18940 } 18941 } 18942 nomore: 18943 if (error) { 18944 rack->r_ctl.rc_agg_delayed = 0; 18945 rack->r_early = 0; 18946 rack->r_late = 0; 18947 rack->r_ctl.rc_agg_early = 0; 18948 SOCKBUF_UNLOCK_ASSERT(sb); /* Check gotos. */ 18949 /* 18950 * Failures do not advance the seq counter above. For the 18951 * case of ENOBUFS we will fall out and retry in 1ms with 18952 * the hpts. Everything else will just have to retransmit 18953 * with the timer. 18954 * 18955 * In any case, we do not want to loop around for another 18956 * send without a good reason. 18957 */ 18958 sendalot = 0; 18959 switch (error) { 18960 case EPERM: 18961 tp->t_softerror = error; 18962 #ifdef TCP_ACCOUNTING 18963 crtsc = get_cyclecount(); 18964 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 18965 tp->tcp_cnt_counters[SND_OUT_FAIL]++; 18966 } 18967 counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1); 18968 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 18969 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val); 18970 } 18971 counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val)); 18972 sched_unpin(); 18973 #endif 18974 return (error); 18975 case ENOBUFS: 18976 /* 18977 * Pace us right away to retry in a some 18978 * time 18979 */ 18980 if (rack->r_ctl.crte != NULL) { 18981 rack_trace_point(rack, RACK_TP_HWENOBUF); 18982 } else 18983 rack_trace_point(rack, RACK_TP_ENOBUF); 18984 slot = ((1 + rack->rc_enobuf) * HPTS_USEC_IN_MSEC); 18985 if (rack->rc_enobuf < 0x7f) 18986 rack->rc_enobuf++; 18987 if (slot < (10 * HPTS_USEC_IN_MSEC)) 18988 slot = 10 * HPTS_USEC_IN_MSEC; 18989 if (rack->r_ctl.crte != NULL) { 18990 counter_u64_add(rack_saw_enobuf_hw, 1); 18991 tcp_rl_log_enobuf(rack->r_ctl.crte); 18992 } 18993 counter_u64_add(rack_saw_enobuf, 1); 18994 goto enobufs; 18995 case EMSGSIZE: 18996 /* 18997 * For some reason the interface we used initially 18998 * to send segments changed to another or lowered 18999 * its MTU. If TSO was active we either got an 19000 * interface without TSO capabilits or TSO was 19001 * turned off. If we obtained mtu from ip_output() 19002 * then update it and try again. 19003 */ 19004 if (tso) 19005 tp->t_flags &= ~TF_TSO; 19006 if (mtu != 0) { 19007 tcp_mss_update(tp, -1, mtu, NULL, NULL); 19008 goto again; 19009 } 19010 slot = 10 * HPTS_USEC_IN_MSEC; 19011 rack_start_hpts_timer(rack, tp, cts, slot, 0, 0); 19012 #ifdef TCP_ACCOUNTING 19013 crtsc = get_cyclecount(); 19014 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19015 tp->tcp_cnt_counters[SND_OUT_FAIL]++; 19016 } 19017 counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1); 19018 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19019 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val); 19020 } 19021 counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val)); 19022 sched_unpin(); 19023 #endif 19024 return (error); 19025 case ENETUNREACH: 19026 counter_u64_add(rack_saw_enetunreach, 1); 19027 case EHOSTDOWN: 19028 case EHOSTUNREACH: 19029 case ENETDOWN: 19030 if (TCPS_HAVERCVDSYN(tp->t_state)) { 19031 tp->t_softerror = error; 19032 } 19033 /* FALLTHROUGH */ 19034 default: 19035 slot = 10 * HPTS_USEC_IN_MSEC; 19036 rack_start_hpts_timer(rack, tp, cts, slot, 0, 0); 19037 #ifdef TCP_ACCOUNTING 19038 crtsc = get_cyclecount(); 19039 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19040 tp->tcp_cnt_counters[SND_OUT_FAIL]++; 19041 } 19042 counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1); 19043 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19044 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val); 19045 } 19046 counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val)); 19047 sched_unpin(); 19048 #endif 19049 return (error); 19050 } 19051 } else { 19052 rack->rc_enobuf = 0; 19053 if (IN_FASTRECOVERY(tp->t_flags) && rsm) 19054 rack->r_ctl.retran_during_recovery += len; 19055 } 19056 KMOD_TCPSTAT_INC(tcps_sndtotal); 19057 19058 /* 19059 * Data sent (as far as we can tell). If this advertises a larger 19060 * window than any other segment, then remember the size of the 19061 * advertised window. Any pending ACK has now been sent. 19062 */ 19063 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv)) 19064 tp->rcv_adv = tp->rcv_nxt + recwin; 19065 19066 tp->last_ack_sent = tp->rcv_nxt; 19067 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK); 19068 enobufs: 19069 if (sendalot) { 19070 /* Do we need to turn off sendalot? */ 19071 if (rack->r_ctl.rc_pace_max_segs && 19072 (tot_len_this_send >= rack->r_ctl.rc_pace_max_segs)) { 19073 /* We hit our max. */ 19074 sendalot = 0; 19075 } else if ((rack->rc_user_set_max_segs) && 19076 (tot_len_this_send >= (rack->rc_user_set_max_segs * segsiz))) { 19077 /* We hit the user defined max */ 19078 sendalot = 0; 19079 } 19080 } 19081 if ((error == 0) && (flags & TH_FIN)) 19082 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_FIN); 19083 if (flags & TH_RST) { 19084 /* 19085 * We don't send again after sending a RST. 19086 */ 19087 slot = 0; 19088 sendalot = 0; 19089 if (error == 0) 19090 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST); 19091 } else if ((slot == 0) && (sendalot == 0) && tot_len_this_send) { 19092 /* 19093 * Get our pacing rate, if an error 19094 * occurred in sending (ENOBUF) we would 19095 * hit the else if with slot preset. Other 19096 * errors return. 19097 */ 19098 slot = rack_get_pacing_delay(rack, tp, tot_len_this_send, rsm, segsiz); 19099 } 19100 if (rsm && 19101 (rsm->r_flags & RACK_HAS_SYN) == 0 && 19102 rack->use_rack_rr) { 19103 /* Its a retransmit and we use the rack cheat? */ 19104 if ((slot == 0) || 19105 (rack->rc_always_pace == 0) || 19106 (rack->r_rr_config == 1)) { 19107 /* 19108 * We have no pacing set or we 19109 * are using old-style rack or 19110 * we are overridden to use the old 1ms pacing. 19111 */ 19112 slot = rack->r_ctl.rc_min_to; 19113 } 19114 } 19115 /* We have sent clear the flag */ 19116 rack->r_ent_rec_ns = 0; 19117 if (rack->r_must_retran) { 19118 if (rsm) { 19119 rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start); 19120 if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) { 19121 /* 19122 * We have retransmitted all. 19123 */ 19124 rack->r_must_retran = 0; 19125 rack->r_ctl.rc_out_at_rto = 0; 19126 } 19127 } else if (SEQ_GEQ(tp->snd_max, rack->r_ctl.rc_snd_max_at_rto)) { 19128 /* 19129 * Sending new data will also kill 19130 * the loop. 19131 */ 19132 rack->r_must_retran = 0; 19133 rack->r_ctl.rc_out_at_rto = 0; 19134 } 19135 } 19136 rack->r_ctl.fsb.recwin = recwin; 19137 if ((tp->t_flags & (TF_WASCRECOVERY|TF_WASFRECOVERY)) && 19138 SEQ_GT(tp->snd_max, rack->r_ctl.rc_snd_max_at_rto)) { 19139 /* 19140 * We hit an RTO and now have past snd_max at the RTO 19141 * clear all the WAS flags. 19142 */ 19143 tp->t_flags &= ~(TF_WASCRECOVERY|TF_WASFRECOVERY); 19144 } 19145 if (slot) { 19146 /* set the rack tcb into the slot N */ 19147 if ((error == 0) && 19148 rack_use_rfo && 19149 ((flags & (TH_SYN|TH_FIN)) == 0) && 19150 (rsm == NULL) && 19151 (tp->snd_nxt == tp->snd_max) && 19152 (ipoptlen == 0) && 19153 (tp->rcv_numsacks == 0) && 19154 rack->r_fsb_inited && 19155 TCPS_HAVEESTABLISHED(tp->t_state) && 19156 (rack->r_must_retran == 0) && 19157 ((tp->t_flags & TF_NEEDFIN) == 0) && 19158 (len > 0) && (orig_len > 0) && 19159 (orig_len > len) && 19160 ((orig_len - len) >= segsiz) && 19161 ((optlen == 0) || 19162 ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) { 19163 /* We can send at least one more MSS using our fsb */ 19164 19165 rack->r_fast_output = 1; 19166 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off); 19167 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; 19168 rack->r_ctl.fsb.tcp_flags = flags; 19169 rack->r_ctl.fsb.left_to_send = orig_len - len; 19170 if (hw_tls) 19171 rack->r_ctl.fsb.hw_tls = 1; 19172 else 19173 rack->r_ctl.fsb.hw_tls = 0; 19174 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))), 19175 ("rack:%p left_to_send:%u sbavail:%u out:%u", 19176 rack, rack->r_ctl.fsb.left_to_send, sbavail(sb), 19177 (tp->snd_max - tp->snd_una))); 19178 if (rack->r_ctl.fsb.left_to_send < segsiz) 19179 rack->r_fast_output = 0; 19180 else { 19181 if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una))) 19182 rack->r_ctl.fsb.rfo_apply_push = 1; 19183 else 19184 rack->r_ctl.fsb.rfo_apply_push = 0; 19185 } 19186 } else 19187 rack->r_fast_output = 0; 19188 rack_log_fsb(rack, tp, so, flags, 19189 ipoptlen, orig_len, len, error, 19190 (rsm == NULL), optlen, __LINE__, 2); 19191 } else if (sendalot) { 19192 int ret; 19193 19194 sack_rxmit = 0; 19195 if ((error == 0) && 19196 rack_use_rfo && 19197 ((flags & (TH_SYN|TH_FIN)) == 0) && 19198 (rsm == NULL) && 19199 (ipoptlen == 0) && 19200 (tp->rcv_numsacks == 0) && 19201 (tp->snd_nxt == tp->snd_max) && 19202 (rack->r_must_retran == 0) && 19203 rack->r_fsb_inited && 19204 TCPS_HAVEESTABLISHED(tp->t_state) && 19205 ((tp->t_flags & TF_NEEDFIN) == 0) && 19206 (len > 0) && (orig_len > 0) && 19207 (orig_len > len) && 19208 ((orig_len - len) >= segsiz) && 19209 ((optlen == 0) || 19210 ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) { 19211 /* we can use fast_output for more */ 19212 19213 rack->r_fast_output = 1; 19214 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off); 19215 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len; 19216 rack->r_ctl.fsb.tcp_flags = flags; 19217 rack->r_ctl.fsb.left_to_send = orig_len - len; 19218 if (hw_tls) 19219 rack->r_ctl.fsb.hw_tls = 1; 19220 else 19221 rack->r_ctl.fsb.hw_tls = 0; 19222 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))), 19223 ("rack:%p left_to_send:%u sbavail:%u out:%u", 19224 rack, rack->r_ctl.fsb.left_to_send, sbavail(sb), 19225 (tp->snd_max - tp->snd_una))); 19226 if (rack->r_ctl.fsb.left_to_send < segsiz) { 19227 rack->r_fast_output = 0; 19228 } 19229 if (rack->r_fast_output) { 19230 if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una))) 19231 rack->r_ctl.fsb.rfo_apply_push = 1; 19232 else 19233 rack->r_ctl.fsb.rfo_apply_push = 0; 19234 rack_log_fsb(rack, tp, so, flags, 19235 ipoptlen, orig_len, len, error, 19236 (rsm == NULL), optlen, __LINE__, 3); 19237 error = 0; 19238 ret = rack_fast_output(tp, rack, ts_val, cts, ms_cts, &tv, tot_len_this_send, &error); 19239 if (ret >= 0) 19240 return (ret); 19241 else if (error) 19242 goto nomore; 19243 19244 } 19245 } 19246 goto again; 19247 } 19248 /* Assure when we leave that snd_nxt will point to top */ 19249 if (SEQ_GT(tp->snd_max, tp->snd_nxt)) 19250 tp->snd_nxt = tp->snd_max; 19251 rack_start_hpts_timer(rack, tp, cts, slot, tot_len_this_send, 0); 19252 #ifdef TCP_ACCOUNTING 19253 crtsc = get_cyclecount() - ts_val; 19254 if (tot_len_this_send) { 19255 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19256 tp->tcp_cnt_counters[SND_OUT_DATA]++; 19257 } 19258 counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], 1); 19259 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19260 tp->tcp_proc_time[SND_OUT_DATA] += crtsc; 19261 } 19262 counter_u64_add(tcp_proc_time[SND_OUT_DATA], crtsc); 19263 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19264 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len_this_send + segsiz - 1) /segsiz); 19265 } 19266 counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((tot_len_this_send + segsiz - 1) /segsiz)); 19267 } else { 19268 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19269 tp->tcp_cnt_counters[SND_OUT_ACK]++; 19270 } 19271 counter_u64_add(tcp_cnt_counters[SND_OUT_ACK], 1); 19272 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) { 19273 tp->tcp_proc_time[SND_OUT_ACK] += crtsc; 19274 } 19275 counter_u64_add(tcp_proc_time[SND_OUT_ACK], crtsc); 19276 } 19277 sched_unpin(); 19278 #endif 19279 if (error == ENOBUFS) 19280 error = 0; 19281 return (error); 19282 } 19283 19284 static void 19285 rack_update_seg(struct tcp_rack *rack) 19286 { 19287 uint32_t orig_val; 19288 19289 orig_val = rack->r_ctl.rc_pace_max_segs; 19290 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL); 19291 if (orig_val != rack->r_ctl.rc_pace_max_segs) 19292 rack_log_pacing_delay_calc(rack, 0, 0, orig_val, 0, 0, 15, __LINE__, NULL, 0); 19293 } 19294 19295 static void 19296 rack_mtu_change(struct tcpcb *tp) 19297 { 19298 /* 19299 * The MSS may have changed 19300 */ 19301 struct tcp_rack *rack; 19302 struct rack_sendmap *rsm; 19303 19304 rack = (struct tcp_rack *)tp->t_fb_ptr; 19305 if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) { 19306 /* 19307 * The MTU has changed we need to resend everything 19308 * since all we have sent is lost. We first fix 19309 * up the mtu though. 19310 */ 19311 rack_set_pace_segments(tp, rack, __LINE__, NULL); 19312 /* We treat this like a full retransmit timeout without the cwnd adjustment */ 19313 rack_remxt_tmr(tp); 19314 rack->r_fast_output = 0; 19315 rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp, 19316 rack->r_ctl.rc_sacked); 19317 rack->r_ctl.rc_snd_max_at_rto = tp->snd_max; 19318 rack->r_must_retran = 1; 19319 /* Mark all inflight to needing to be rxt'd */ 19320 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) { 19321 rsm->r_flags |= RACK_MUST_RXT; 19322 } 19323 } 19324 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una); 19325 /* We don't use snd_nxt to retransmit */ 19326 tp->snd_nxt = tp->snd_max; 19327 } 19328 19329 static int 19330 rack_set_profile(struct tcp_rack *rack, int prof) 19331 { 19332 int err = EINVAL; 19333 if (prof == 1) { 19334 /* pace_always=1 */ 19335 if (rack->rc_always_pace == 0) { 19336 if (tcp_can_enable_pacing() == 0) 19337 return (EBUSY); 19338 } 19339 rack->rc_always_pace = 1; 19340 if (rack->use_fixed_rate || rack->gp_ready) 19341 rack_set_cc_pacing(rack); 19342 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19343 rack->rack_attempt_hdwr_pace = 0; 19344 /* cmpack=1 */ 19345 if (rack_use_cmp_acks) 19346 rack->r_use_cmp_ack = 1; 19347 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state) && 19348 rack->r_use_cmp_ack) 19349 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP; 19350 /* scwnd=1 */ 19351 rack->rack_enable_scwnd = 1; 19352 /* dynamic=100 */ 19353 rack->rc_gp_dyn_mul = 1; 19354 /* gp_inc_ca */ 19355 rack->r_ctl.rack_per_of_gp_ca = 100; 19356 /* rrr_conf=3 */ 19357 rack->r_rr_config = 3; 19358 /* npush=2 */ 19359 rack->r_ctl.rc_no_push_at_mrtt = 2; 19360 /* fillcw=1 */ 19361 rack->rc_pace_to_cwnd = 1; 19362 rack->rc_pace_fill_if_rttin_range = 0; 19363 rack->rtt_limit_mul = 0; 19364 /* noprr=1 */ 19365 rack->rack_no_prr = 1; 19366 /* lscwnd=1 */ 19367 rack->r_limit_scw = 1; 19368 /* gp_inc_rec */ 19369 rack->r_ctl.rack_per_of_gp_rec = 90; 19370 err = 0; 19371 19372 } else if (prof == 3) { 19373 /* Same as profile one execept fill_cw becomes 2 (less aggressive set) */ 19374 /* pace_always=1 */ 19375 if (rack->rc_always_pace == 0) { 19376 if (tcp_can_enable_pacing() == 0) 19377 return (EBUSY); 19378 } 19379 rack->rc_always_pace = 1; 19380 if (rack->use_fixed_rate || rack->gp_ready) 19381 rack_set_cc_pacing(rack); 19382 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19383 rack->rack_attempt_hdwr_pace = 0; 19384 /* cmpack=1 */ 19385 if (rack_use_cmp_acks) 19386 rack->r_use_cmp_ack = 1; 19387 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state) && 19388 rack->r_use_cmp_ack) 19389 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP; 19390 /* scwnd=1 */ 19391 rack->rack_enable_scwnd = 1; 19392 /* dynamic=100 */ 19393 rack->rc_gp_dyn_mul = 1; 19394 /* gp_inc_ca */ 19395 rack->r_ctl.rack_per_of_gp_ca = 100; 19396 /* rrr_conf=3 */ 19397 rack->r_rr_config = 3; 19398 /* npush=2 */ 19399 rack->r_ctl.rc_no_push_at_mrtt = 2; 19400 /* fillcw=2 */ 19401 rack->rc_pace_to_cwnd = 1; 19402 rack->r_fill_less_agg = 1; 19403 rack->rc_pace_fill_if_rttin_range = 0; 19404 rack->rtt_limit_mul = 0; 19405 /* noprr=1 */ 19406 rack->rack_no_prr = 1; 19407 /* lscwnd=1 */ 19408 rack->r_limit_scw = 1; 19409 /* gp_inc_rec */ 19410 rack->r_ctl.rack_per_of_gp_rec = 90; 19411 err = 0; 19412 19413 19414 } else if (prof == 2) { 19415 /* cmpack=1 */ 19416 if (rack->rc_always_pace == 0) { 19417 if (tcp_can_enable_pacing() == 0) 19418 return (EBUSY); 19419 } 19420 rack->rc_always_pace = 1; 19421 if (rack->use_fixed_rate || rack->gp_ready) 19422 rack_set_cc_pacing(rack); 19423 rack->r_use_cmp_ack = 1; 19424 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state)) 19425 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP; 19426 /* pace_always=1 */ 19427 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19428 /* scwnd=1 */ 19429 rack->rack_enable_scwnd = 1; 19430 /* dynamic=100 */ 19431 rack->rc_gp_dyn_mul = 1; 19432 rack->r_ctl.rack_per_of_gp_ca = 100; 19433 /* rrr_conf=3 */ 19434 rack->r_rr_config = 3; 19435 /* npush=2 */ 19436 rack->r_ctl.rc_no_push_at_mrtt = 2; 19437 /* fillcw=1 */ 19438 rack->rc_pace_to_cwnd = 1; 19439 rack->rc_pace_fill_if_rttin_range = 0; 19440 rack->rtt_limit_mul = 0; 19441 /* noprr=1 */ 19442 rack->rack_no_prr = 1; 19443 /* lscwnd=0 */ 19444 rack->r_limit_scw = 0; 19445 err = 0; 19446 } else if (prof == 0) { 19447 /* This changes things back to the default settings */ 19448 err = 0; 19449 if (rack->rc_always_pace) { 19450 tcp_decrement_paced_conn(); 19451 rack_undo_cc_pacing(rack); 19452 rack->rc_always_pace = 0; 19453 } 19454 if (rack_pace_every_seg && tcp_can_enable_pacing()) { 19455 rack->rc_always_pace = 1; 19456 if (rack->use_fixed_rate || rack->gp_ready) 19457 rack_set_cc_pacing(rack); 19458 } else 19459 rack->rc_always_pace = 0; 19460 if (rack_dsack_std_based & 0x1) { 19461 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */ 19462 rack->rc_rack_tmr_std_based = 1; 19463 } 19464 if (rack_dsack_std_based & 0x2) { 19465 /* Basically this means rack timers are extended based on dsack by up to (2 * srtt) */ 19466 rack->rc_rack_use_dsack = 1; 19467 } 19468 if (rack_use_cmp_acks) 19469 rack->r_use_cmp_ack = 1; 19470 else 19471 rack->r_use_cmp_ack = 0; 19472 if (rack_disable_prr) 19473 rack->rack_no_prr = 1; 19474 else 19475 rack->rack_no_prr = 0; 19476 if (rack_gp_no_rec_chg) 19477 rack->rc_gp_no_rec_chg = 1; 19478 else 19479 rack->rc_gp_no_rec_chg = 0; 19480 if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack) { 19481 rack->r_mbuf_queue = 1; 19482 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state)) 19483 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP; 19484 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19485 } else { 19486 rack->r_mbuf_queue = 0; 19487 rack->rc_inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ; 19488 } 19489 if (rack_enable_shared_cwnd) 19490 rack->rack_enable_scwnd = 1; 19491 else 19492 rack->rack_enable_scwnd = 0; 19493 if (rack_do_dyn_mul) { 19494 /* When dynamic adjustment is on CA needs to start at 100% */ 19495 rack->rc_gp_dyn_mul = 1; 19496 if (rack_do_dyn_mul >= 100) 19497 rack->r_ctl.rack_per_of_gp_ca = rack_do_dyn_mul; 19498 } else { 19499 rack->r_ctl.rack_per_of_gp_ca = rack_per_of_gp_ca; 19500 rack->rc_gp_dyn_mul = 0; 19501 } 19502 rack->r_rr_config = 0; 19503 rack->r_ctl.rc_no_push_at_mrtt = 0; 19504 rack->rc_pace_to_cwnd = 0; 19505 rack->rc_pace_fill_if_rttin_range = 0; 19506 rack->rtt_limit_mul = 0; 19507 19508 if (rack_enable_hw_pacing) 19509 rack->rack_hdw_pace_ena = 1; 19510 else 19511 rack->rack_hdw_pace_ena = 0; 19512 if (rack_disable_prr) 19513 rack->rack_no_prr = 1; 19514 else 19515 rack->rack_no_prr = 0; 19516 if (rack_limits_scwnd) 19517 rack->r_limit_scw = 1; 19518 else 19519 rack->r_limit_scw = 0; 19520 err = 0; 19521 } 19522 return (err); 19523 } 19524 19525 static int 19526 rack_add_deferred_option(struct tcp_rack *rack, int sopt_name, uint64_t loptval) 19527 { 19528 struct deferred_opt_list *dol; 19529 19530 dol = malloc(sizeof(struct deferred_opt_list), 19531 M_TCPFSB, M_NOWAIT|M_ZERO); 19532 if (dol == NULL) { 19533 /* 19534 * No space yikes -- fail out.. 19535 */ 19536 return (0); 19537 } 19538 dol->optname = sopt_name; 19539 dol->optval = loptval; 19540 TAILQ_INSERT_TAIL(&rack->r_ctl.opt_list, dol, next); 19541 return (1); 19542 } 19543 19544 static int 19545 rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name, 19546 uint32_t optval, uint64_t loptval) 19547 { 19548 struct epoch_tracker et; 19549 struct sockopt sopt; 19550 struct cc_newreno_opts opt; 19551 struct inpcb *inp = tptoinpcb(tp); 19552 uint64_t val; 19553 int error = 0; 19554 uint16_t ca, ss; 19555 19556 switch (sopt_name) { 19557 19558 case TCP_RACK_DSACK_OPT: 19559 RACK_OPTS_INC(tcp_rack_dsack_opt); 19560 if (optval & 0x1) { 19561 rack->rc_rack_tmr_std_based = 1; 19562 } else { 19563 rack->rc_rack_tmr_std_based = 0; 19564 } 19565 if (optval & 0x2) { 19566 rack->rc_rack_use_dsack = 1; 19567 } else { 19568 rack->rc_rack_use_dsack = 0; 19569 } 19570 rack_log_dsack_event(rack, 5, __LINE__, 0, 0); 19571 break; 19572 case TCP_RACK_PACING_BETA: 19573 RACK_OPTS_INC(tcp_rack_beta); 19574 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) { 19575 /* This only works for newreno. */ 19576 error = EINVAL; 19577 break; 19578 } 19579 if (rack->rc_pacing_cc_set) { 19580 /* 19581 * Set them into the real CC module 19582 * whats in the rack pcb is the old values 19583 * to be used on restoral/ 19584 */ 19585 sopt.sopt_dir = SOPT_SET; 19586 opt.name = CC_NEWRENO_BETA; 19587 opt.val = optval; 19588 if (CC_ALGO(tp)->ctl_output != NULL) 19589 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt); 19590 else { 19591 error = ENOENT; 19592 break; 19593 } 19594 } else { 19595 /* 19596 * Not pacing yet so set it into our local 19597 * rack pcb storage. 19598 */ 19599 rack->r_ctl.rc_saved_beta.beta = optval; 19600 } 19601 break; 19602 case TCP_RACK_TIMER_SLOP: 19603 RACK_OPTS_INC(tcp_rack_timer_slop); 19604 rack->r_ctl.timer_slop = optval; 19605 if (rack->rc_tp->t_srtt) { 19606 /* 19607 * If we have an SRTT lets update t_rxtcur 19608 * to have the new slop. 19609 */ 19610 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp), 19611 rack_rto_min, rack_rto_max, 19612 rack->r_ctl.timer_slop); 19613 } 19614 break; 19615 case TCP_RACK_PACING_BETA_ECN: 19616 RACK_OPTS_INC(tcp_rack_beta_ecn); 19617 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) { 19618 /* This only works for newreno. */ 19619 error = EINVAL; 19620 break; 19621 } 19622 if (rack->rc_pacing_cc_set) { 19623 /* 19624 * Set them into the real CC module 19625 * whats in the rack pcb is the old values 19626 * to be used on restoral/ 19627 */ 19628 sopt.sopt_dir = SOPT_SET; 19629 opt.name = CC_NEWRENO_BETA_ECN; 19630 opt.val = optval; 19631 if (CC_ALGO(tp)->ctl_output != NULL) 19632 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt); 19633 else 19634 error = ENOENT; 19635 } else { 19636 /* 19637 * Not pacing yet so set it into our local 19638 * rack pcb storage. 19639 */ 19640 rack->r_ctl.rc_saved_beta.beta_ecn = optval; 19641 rack->r_ctl.rc_saved_beta.newreno_flags = CC_NEWRENO_BETA_ECN_ENABLED; 19642 } 19643 break; 19644 case TCP_DEFER_OPTIONS: 19645 RACK_OPTS_INC(tcp_defer_opt); 19646 if (optval) { 19647 if (rack->gp_ready) { 19648 /* Too late */ 19649 error = EINVAL; 19650 break; 19651 } 19652 rack->defer_options = 1; 19653 } else 19654 rack->defer_options = 0; 19655 break; 19656 case TCP_RACK_MEASURE_CNT: 19657 RACK_OPTS_INC(tcp_rack_measure_cnt); 19658 if (optval && (optval <= 0xff)) { 19659 rack->r_ctl.req_measurements = optval; 19660 } else 19661 error = EINVAL; 19662 break; 19663 case TCP_REC_ABC_VAL: 19664 RACK_OPTS_INC(tcp_rec_abc_val); 19665 if (optval > 0) 19666 rack->r_use_labc_for_rec = 1; 19667 else 19668 rack->r_use_labc_for_rec = 0; 19669 break; 19670 case TCP_RACK_ABC_VAL: 19671 RACK_OPTS_INC(tcp_rack_abc_val); 19672 if ((optval > 0) && (optval < 255)) 19673 rack->rc_labc = optval; 19674 else 19675 error = EINVAL; 19676 break; 19677 case TCP_HDWR_UP_ONLY: 19678 RACK_OPTS_INC(tcp_pacing_up_only); 19679 if (optval) 19680 rack->r_up_only = 1; 19681 else 19682 rack->r_up_only = 0; 19683 break; 19684 case TCP_PACING_RATE_CAP: 19685 RACK_OPTS_INC(tcp_pacing_rate_cap); 19686 rack->r_ctl.bw_rate_cap = loptval; 19687 break; 19688 case TCP_RACK_PROFILE: 19689 RACK_OPTS_INC(tcp_profile); 19690 error = rack_set_profile(rack, optval); 19691 break; 19692 case TCP_USE_CMP_ACKS: 19693 RACK_OPTS_INC(tcp_use_cmp_acks); 19694 if ((optval == 0) && (rack->rc_inp->inp_flags2 & INP_MBUF_ACKCMP)) { 19695 /* You can't turn it off once its on! */ 19696 error = EINVAL; 19697 } else if ((optval == 1) && (rack->r_use_cmp_ack == 0)) { 19698 rack->r_use_cmp_ack = 1; 19699 rack->r_mbuf_queue = 1; 19700 inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19701 } 19702 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state)) 19703 inp->inp_flags2 |= INP_MBUF_ACKCMP; 19704 break; 19705 case TCP_SHARED_CWND_TIME_LIMIT: 19706 RACK_OPTS_INC(tcp_lscwnd); 19707 if (optval) 19708 rack->r_limit_scw = 1; 19709 else 19710 rack->r_limit_scw = 0; 19711 break; 19712 case TCP_RACK_PACE_TO_FILL: 19713 RACK_OPTS_INC(tcp_fillcw); 19714 if (optval == 0) 19715 rack->rc_pace_to_cwnd = 0; 19716 else { 19717 rack->rc_pace_to_cwnd = 1; 19718 if (optval > 1) 19719 rack->r_fill_less_agg = 1; 19720 } 19721 if ((optval >= rack_gp_rtt_maxmul) && 19722 rack_gp_rtt_maxmul && 19723 (optval < 0xf)) { 19724 rack->rc_pace_fill_if_rttin_range = 1; 19725 rack->rtt_limit_mul = optval; 19726 } else { 19727 rack->rc_pace_fill_if_rttin_range = 0; 19728 rack->rtt_limit_mul = 0; 19729 } 19730 break; 19731 case TCP_RACK_NO_PUSH_AT_MAX: 19732 RACK_OPTS_INC(tcp_npush); 19733 if (optval == 0) 19734 rack->r_ctl.rc_no_push_at_mrtt = 0; 19735 else if (optval < 0xff) 19736 rack->r_ctl.rc_no_push_at_mrtt = optval; 19737 else 19738 error = EINVAL; 19739 break; 19740 case TCP_SHARED_CWND_ENABLE: 19741 RACK_OPTS_INC(tcp_rack_scwnd); 19742 if (optval == 0) 19743 rack->rack_enable_scwnd = 0; 19744 else 19745 rack->rack_enable_scwnd = 1; 19746 break; 19747 case TCP_RACK_MBUF_QUEUE: 19748 /* Now do we use the LRO mbuf-queue feature */ 19749 RACK_OPTS_INC(tcp_rack_mbufq); 19750 if (optval || rack->r_use_cmp_ack) 19751 rack->r_mbuf_queue = 1; 19752 else 19753 rack->r_mbuf_queue = 0; 19754 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack) 19755 inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19756 else 19757 inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ; 19758 break; 19759 case TCP_RACK_NONRXT_CFG_RATE: 19760 RACK_OPTS_INC(tcp_rack_cfg_rate); 19761 if (optval == 0) 19762 rack->rack_rec_nonrxt_use_cr = 0; 19763 else 19764 rack->rack_rec_nonrxt_use_cr = 1; 19765 break; 19766 case TCP_NO_PRR: 19767 RACK_OPTS_INC(tcp_rack_noprr); 19768 if (optval == 0) 19769 rack->rack_no_prr = 0; 19770 else if (optval == 1) 19771 rack->rack_no_prr = 1; 19772 else if (optval == 2) 19773 rack->no_prr_addback = 1; 19774 else 19775 error = EINVAL; 19776 break; 19777 case TCP_TIMELY_DYN_ADJ: 19778 RACK_OPTS_INC(tcp_timely_dyn); 19779 if (optval == 0) 19780 rack->rc_gp_dyn_mul = 0; 19781 else { 19782 rack->rc_gp_dyn_mul = 1; 19783 if (optval >= 100) { 19784 /* 19785 * If the user sets something 100 or more 19786 * its the gp_ca value. 19787 */ 19788 rack->r_ctl.rack_per_of_gp_ca = optval; 19789 } 19790 } 19791 break; 19792 case TCP_RACK_DO_DETECTION: 19793 RACK_OPTS_INC(tcp_rack_do_detection); 19794 if (optval == 0) 19795 rack->do_detection = 0; 19796 else 19797 rack->do_detection = 1; 19798 break; 19799 case TCP_RACK_TLP_USE: 19800 if ((optval < TLP_USE_ID) || (optval > TLP_USE_TWO_TWO)) { 19801 error = EINVAL; 19802 break; 19803 } 19804 RACK_OPTS_INC(tcp_tlp_use); 19805 rack->rack_tlp_threshold_use = optval; 19806 break; 19807 case TCP_RACK_TLP_REDUCE: 19808 /* RACK TLP cwnd reduction (bool) */ 19809 RACK_OPTS_INC(tcp_rack_tlp_reduce); 19810 rack->r_ctl.rc_tlp_cwnd_reduce = optval; 19811 break; 19812 /* Pacing related ones */ 19813 case TCP_RACK_PACE_ALWAYS: 19814 /* 19815 * zero is old rack method, 1 is new 19816 * method using a pacing rate. 19817 */ 19818 RACK_OPTS_INC(tcp_rack_pace_always); 19819 if (optval > 0) { 19820 if (rack->rc_always_pace) { 19821 error = EALREADY; 19822 break; 19823 } else if (tcp_can_enable_pacing()) { 19824 rack->rc_always_pace = 1; 19825 if (rack->use_fixed_rate || rack->gp_ready) 19826 rack_set_cc_pacing(rack); 19827 } 19828 else { 19829 error = ENOSPC; 19830 break; 19831 } 19832 } else { 19833 if (rack->rc_always_pace) { 19834 tcp_decrement_paced_conn(); 19835 rack->rc_always_pace = 0; 19836 rack_undo_cc_pacing(rack); 19837 } 19838 } 19839 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack) 19840 inp->inp_flags2 |= INP_SUPPORTS_MBUFQ; 19841 else 19842 inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ; 19843 /* A rate may be set irate or other, if so set seg size */ 19844 rack_update_seg(rack); 19845 break; 19846 case TCP_BBR_RACK_INIT_RATE: 19847 RACK_OPTS_INC(tcp_initial_rate); 19848 val = optval; 19849 /* Change from kbits per second to bytes per second */ 19850 val *= 1000; 19851 val /= 8; 19852 rack->r_ctl.init_rate = val; 19853 if (rack->rc_init_win != rack_default_init_window) { 19854 uint32_t win, snt; 19855 19856 /* 19857 * Options don't always get applied 19858 * in the order you think. So in order 19859 * to assure we update a cwnd we need 19860 * to check and see if we are still 19861 * where we should raise the cwnd. 19862 */ 19863 win = rc_init_window(rack); 19864 if (SEQ_GT(tp->snd_max, tp->iss)) 19865 snt = tp->snd_max - tp->iss; 19866 else 19867 snt = 0; 19868 if ((snt < win) && 19869 (tp->snd_cwnd < win)) 19870 tp->snd_cwnd = win; 19871 } 19872 if (rack->rc_always_pace) 19873 rack_update_seg(rack); 19874 break; 19875 case TCP_BBR_IWINTSO: 19876 RACK_OPTS_INC(tcp_initial_win); 19877 if (optval && (optval <= 0xff)) { 19878 uint32_t win, snt; 19879 19880 rack->rc_init_win = optval; 19881 win = rc_init_window(rack); 19882 if (SEQ_GT(tp->snd_max, tp->iss)) 19883 snt = tp->snd_max - tp->iss; 19884 else 19885 snt = 0; 19886 if ((snt < win) && 19887 (tp->t_srtt | 19888 #ifdef NETFLIX_PEAKRATE 19889 tp->t_maxpeakrate | 19890 #endif 19891 rack->r_ctl.init_rate)) { 19892 /* 19893 * We are not past the initial window 19894 * and we have some bases for pacing, 19895 * so we need to possibly adjust up 19896 * the cwnd. Note even if we don't set 19897 * the cwnd, its still ok to raise the rc_init_win 19898 * which can be used coming out of idle when we 19899 * would have a rate. 19900 */ 19901 if (tp->snd_cwnd < win) 19902 tp->snd_cwnd = win; 19903 } 19904 if (rack->rc_always_pace) 19905 rack_update_seg(rack); 19906 } else 19907 error = EINVAL; 19908 break; 19909 case TCP_RACK_FORCE_MSEG: 19910 RACK_OPTS_INC(tcp_rack_force_max_seg); 19911 if (optval) 19912 rack->rc_force_max_seg = 1; 19913 else 19914 rack->rc_force_max_seg = 0; 19915 break; 19916 case TCP_RACK_PACE_MAX_SEG: 19917 /* Max segments size in a pace in bytes */ 19918 RACK_OPTS_INC(tcp_rack_max_seg); 19919 rack->rc_user_set_max_segs = optval; 19920 rack_set_pace_segments(tp, rack, __LINE__, NULL); 19921 break; 19922 case TCP_RACK_PACE_RATE_REC: 19923 /* Set the fixed pacing rate in Bytes per second ca */ 19924 RACK_OPTS_INC(tcp_rack_pace_rate_rec); 19925 rack->r_ctl.rc_fixed_pacing_rate_rec = optval; 19926 if (rack->r_ctl.rc_fixed_pacing_rate_ca == 0) 19927 rack->r_ctl.rc_fixed_pacing_rate_ca = optval; 19928 if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0) 19929 rack->r_ctl.rc_fixed_pacing_rate_ss = optval; 19930 rack->use_fixed_rate = 1; 19931 if (rack->rc_always_pace) 19932 rack_set_cc_pacing(rack); 19933 rack_log_pacing_delay_calc(rack, 19934 rack->r_ctl.rc_fixed_pacing_rate_ss, 19935 rack->r_ctl.rc_fixed_pacing_rate_ca, 19936 rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8, 19937 __LINE__, NULL,0); 19938 break; 19939 19940 case TCP_RACK_PACE_RATE_SS: 19941 /* Set the fixed pacing rate in Bytes per second ca */ 19942 RACK_OPTS_INC(tcp_rack_pace_rate_ss); 19943 rack->r_ctl.rc_fixed_pacing_rate_ss = optval; 19944 if (rack->r_ctl.rc_fixed_pacing_rate_ca == 0) 19945 rack->r_ctl.rc_fixed_pacing_rate_ca = optval; 19946 if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0) 19947 rack->r_ctl.rc_fixed_pacing_rate_rec = optval; 19948 rack->use_fixed_rate = 1; 19949 if (rack->rc_always_pace) 19950 rack_set_cc_pacing(rack); 19951 rack_log_pacing_delay_calc(rack, 19952 rack->r_ctl.rc_fixed_pacing_rate_ss, 19953 rack->r_ctl.rc_fixed_pacing_rate_ca, 19954 rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8, 19955 __LINE__, NULL, 0); 19956 break; 19957 19958 case TCP_RACK_PACE_RATE_CA: 19959 /* Set the fixed pacing rate in Bytes per second ca */ 19960 RACK_OPTS_INC(tcp_rack_pace_rate_ca); 19961 rack->r_ctl.rc_fixed_pacing_rate_ca = optval; 19962 if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0) 19963 rack->r_ctl.rc_fixed_pacing_rate_ss = optval; 19964 if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0) 19965 rack->r_ctl.rc_fixed_pacing_rate_rec = optval; 19966 rack->use_fixed_rate = 1; 19967 if (rack->rc_always_pace) 19968 rack_set_cc_pacing(rack); 19969 rack_log_pacing_delay_calc(rack, 19970 rack->r_ctl.rc_fixed_pacing_rate_ss, 19971 rack->r_ctl.rc_fixed_pacing_rate_ca, 19972 rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8, 19973 __LINE__, NULL, 0); 19974 break; 19975 case TCP_RACK_GP_INCREASE_REC: 19976 RACK_OPTS_INC(tcp_gp_inc_rec); 19977 rack->r_ctl.rack_per_of_gp_rec = optval; 19978 rack_log_pacing_delay_calc(rack, 19979 rack->r_ctl.rack_per_of_gp_ss, 19980 rack->r_ctl.rack_per_of_gp_ca, 19981 rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1, 19982 __LINE__, NULL, 0); 19983 break; 19984 case TCP_RACK_GP_INCREASE_CA: 19985 RACK_OPTS_INC(tcp_gp_inc_ca); 19986 ca = optval; 19987 if (ca < 100) { 19988 /* 19989 * We don't allow any reduction 19990 * over the GP b/w. 19991 */ 19992 error = EINVAL; 19993 break; 19994 } 19995 rack->r_ctl.rack_per_of_gp_ca = ca; 19996 rack_log_pacing_delay_calc(rack, 19997 rack->r_ctl.rack_per_of_gp_ss, 19998 rack->r_ctl.rack_per_of_gp_ca, 19999 rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1, 20000 __LINE__, NULL, 0); 20001 break; 20002 case TCP_RACK_GP_INCREASE_SS: 20003 RACK_OPTS_INC(tcp_gp_inc_ss); 20004 ss = optval; 20005 if (ss < 100) { 20006 /* 20007 * We don't allow any reduction 20008 * over the GP b/w. 20009 */ 20010 error = EINVAL; 20011 break; 20012 } 20013 rack->r_ctl.rack_per_of_gp_ss = ss; 20014 rack_log_pacing_delay_calc(rack, 20015 rack->r_ctl.rack_per_of_gp_ss, 20016 rack->r_ctl.rack_per_of_gp_ca, 20017 rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1, 20018 __LINE__, NULL, 0); 20019 break; 20020 case TCP_RACK_RR_CONF: 20021 RACK_OPTS_INC(tcp_rack_rrr_no_conf_rate); 20022 if (optval && optval <= 3) 20023 rack->r_rr_config = optval; 20024 else 20025 rack->r_rr_config = 0; 20026 break; 20027 case TCP_HDWR_RATE_CAP: 20028 RACK_OPTS_INC(tcp_hdwr_rate_cap); 20029 if (optval) { 20030 if (rack->r_rack_hw_rate_caps == 0) 20031 rack->r_rack_hw_rate_caps = 1; 20032 else 20033 error = EALREADY; 20034 } else { 20035 rack->r_rack_hw_rate_caps = 0; 20036 } 20037 break; 20038 case TCP_BBR_HDWR_PACE: 20039 RACK_OPTS_INC(tcp_hdwr_pacing); 20040 if (optval){ 20041 if (rack->rack_hdrw_pacing == 0) { 20042 rack->rack_hdw_pace_ena = 1; 20043 rack->rack_attempt_hdwr_pace = 0; 20044 } else 20045 error = EALREADY; 20046 } else { 20047 rack->rack_hdw_pace_ena = 0; 20048 #ifdef RATELIMIT 20049 if (rack->r_ctl.crte != NULL) { 20050 rack->rack_hdrw_pacing = 0; 20051 rack->rack_attempt_hdwr_pace = 0; 20052 tcp_rel_pacing_rate(rack->r_ctl.crte, tp); 20053 rack->r_ctl.crte = NULL; 20054 } 20055 #endif 20056 } 20057 break; 20058 /* End Pacing related ones */ 20059 case TCP_RACK_PRR_SENDALOT: 20060 /* Allow PRR to send more than one seg */ 20061 RACK_OPTS_INC(tcp_rack_prr_sendalot); 20062 rack->r_ctl.rc_prr_sendalot = optval; 20063 break; 20064 case TCP_RACK_MIN_TO: 20065 /* Minimum time between rack t-o's in ms */ 20066 RACK_OPTS_INC(tcp_rack_min_to); 20067 rack->r_ctl.rc_min_to = optval; 20068 break; 20069 case TCP_RACK_EARLY_SEG: 20070 /* If early recovery max segments */ 20071 RACK_OPTS_INC(tcp_rack_early_seg); 20072 rack->r_ctl.rc_early_recovery_segs = optval; 20073 break; 20074 case TCP_RACK_ENABLE_HYSTART: 20075 { 20076 if (optval) { 20077 tp->t_ccv.flags |= CCF_HYSTART_ALLOWED; 20078 if (rack_do_hystart > RACK_HYSTART_ON) 20079 tp->t_ccv.flags |= CCF_HYSTART_CAN_SH_CWND; 20080 if (rack_do_hystart > RACK_HYSTART_ON_W_SC) 20081 tp->t_ccv.flags |= CCF_HYSTART_CONS_SSTH; 20082 } else { 20083 tp->t_ccv.flags &= ~(CCF_HYSTART_ALLOWED|CCF_HYSTART_CAN_SH_CWND|CCF_HYSTART_CONS_SSTH); 20084 } 20085 } 20086 break; 20087 case TCP_RACK_REORD_THRESH: 20088 /* RACK reorder threshold (shift amount) */ 20089 RACK_OPTS_INC(tcp_rack_reord_thresh); 20090 if ((optval > 0) && (optval < 31)) 20091 rack->r_ctl.rc_reorder_shift = optval; 20092 else 20093 error = EINVAL; 20094 break; 20095 case TCP_RACK_REORD_FADE: 20096 /* Does reordering fade after ms time */ 20097 RACK_OPTS_INC(tcp_rack_reord_fade); 20098 rack->r_ctl.rc_reorder_fade = optval; 20099 break; 20100 case TCP_RACK_TLP_THRESH: 20101 /* RACK TLP theshold i.e. srtt+(srtt/N) */ 20102 RACK_OPTS_INC(tcp_rack_tlp_thresh); 20103 if (optval) 20104 rack->r_ctl.rc_tlp_threshold = optval; 20105 else 20106 error = EINVAL; 20107 break; 20108 case TCP_BBR_USE_RACK_RR: 20109 RACK_OPTS_INC(tcp_rack_rr); 20110 if (optval) 20111 rack->use_rack_rr = 1; 20112 else 20113 rack->use_rack_rr = 0; 20114 break; 20115 case TCP_FAST_RSM_HACK: 20116 RACK_OPTS_INC(tcp_rack_fastrsm_hack); 20117 if (optval) 20118 rack->fast_rsm_hack = 1; 20119 else 20120 rack->fast_rsm_hack = 0; 20121 break; 20122 case TCP_RACK_PKT_DELAY: 20123 /* RACK added ms i.e. rack-rtt + reord + N */ 20124 RACK_OPTS_INC(tcp_rack_pkt_delay); 20125 rack->r_ctl.rc_pkt_delay = optval; 20126 break; 20127 case TCP_DELACK: 20128 RACK_OPTS_INC(tcp_rack_delayed_ack); 20129 if (optval == 0) 20130 tp->t_delayed_ack = 0; 20131 else 20132 tp->t_delayed_ack = 1; 20133 if (tp->t_flags & TF_DELACK) { 20134 tp->t_flags &= ~TF_DELACK; 20135 tp->t_flags |= TF_ACKNOW; 20136 NET_EPOCH_ENTER(et); 20137 rack_output(tp); 20138 NET_EPOCH_EXIT(et); 20139 } 20140 break; 20141 20142 case TCP_BBR_RACK_RTT_USE: 20143 RACK_OPTS_INC(tcp_rack_rtt_use); 20144 if ((optval != USE_RTT_HIGH) && 20145 (optval != USE_RTT_LOW) && 20146 (optval != USE_RTT_AVG)) 20147 error = EINVAL; 20148 else 20149 rack->r_ctl.rc_rate_sample_method = optval; 20150 break; 20151 case TCP_DATA_AFTER_CLOSE: 20152 RACK_OPTS_INC(tcp_data_after_close); 20153 if (optval) 20154 rack->rc_allow_data_af_clo = 1; 20155 else 20156 rack->rc_allow_data_af_clo = 0; 20157 break; 20158 default: 20159 break; 20160 } 20161 #ifdef NETFLIX_STATS 20162 tcp_log_socket_option(tp, sopt_name, optval, error); 20163 #endif 20164 return (error); 20165 } 20166 20167 20168 static void 20169 rack_apply_deferred_options(struct tcp_rack *rack) 20170 { 20171 struct deferred_opt_list *dol, *sdol; 20172 uint32_t s_optval; 20173 20174 TAILQ_FOREACH_SAFE(dol, &rack->r_ctl.opt_list, next, sdol) { 20175 TAILQ_REMOVE(&rack->r_ctl.opt_list, dol, next); 20176 /* Disadvantage of deferal is you loose the error return */ 20177 s_optval = (uint32_t)dol->optval; 20178 (void)rack_process_option(rack->rc_tp, rack, dol->optname, s_optval, dol->optval); 20179 free(dol, M_TCPDO); 20180 } 20181 } 20182 20183 static void 20184 rack_hw_tls_change(struct tcpcb *tp, int chg) 20185 { 20186 /* Update HW tls state */ 20187 struct tcp_rack *rack; 20188 20189 rack = (struct tcp_rack *)tp->t_fb_ptr; 20190 if (chg) 20191 rack->r_ctl.fsb.hw_tls = 1; 20192 else 20193 rack->r_ctl.fsb.hw_tls = 0; 20194 } 20195 20196 static int 20197 rack_pru_options(struct tcpcb *tp, int flags) 20198 { 20199 if (flags & PRUS_OOB) 20200 return (EOPNOTSUPP); 20201 return (0); 20202 } 20203 20204 static struct tcp_function_block __tcp_rack = { 20205 .tfb_tcp_block_name = __XSTRING(STACKNAME), 20206 .tfb_tcp_output = rack_output, 20207 .tfb_do_queued_segments = ctf_do_queued_segments, 20208 .tfb_do_segment_nounlock = rack_do_segment_nounlock, 20209 .tfb_tcp_do_segment = rack_do_segment, 20210 .tfb_tcp_ctloutput = rack_ctloutput, 20211 .tfb_tcp_fb_init = rack_init, 20212 .tfb_tcp_fb_fini = rack_fini, 20213 .tfb_tcp_timer_stop_all = rack_stopall, 20214 .tfb_tcp_rexmit_tmr = rack_remxt_tmr, 20215 .tfb_tcp_handoff_ok = rack_handoff_ok, 20216 .tfb_tcp_mtu_chg = rack_mtu_change, 20217 .tfb_pru_options = rack_pru_options, 20218 .tfb_hwtls_change = rack_hw_tls_change, 20219 .tfb_compute_pipe = rack_compute_pipe, 20220 .tfb_flags = TCP_FUNC_OUTPUT_CANDROP, 20221 }; 20222 20223 /* 20224 * rack_ctloutput() must drop the inpcb lock before performing copyin on 20225 * socket option arguments. When it re-acquires the lock after the copy, it 20226 * has to revalidate that the connection is still valid for the socket 20227 * option. 20228 */ 20229 static int 20230 rack_set_sockopt(struct inpcb *inp, struct sockopt *sopt) 20231 { 20232 #ifdef INET6 20233 struct ip6_hdr *ip6; 20234 #endif 20235 #ifdef INET 20236 struct ip *ip; 20237 #endif 20238 struct tcpcb *tp; 20239 struct tcp_rack *rack; 20240 uint64_t loptval; 20241 int32_t error = 0, optval; 20242 20243 tp = intotcpcb(inp); 20244 rack = (struct tcp_rack *)tp->t_fb_ptr; 20245 if (rack == NULL) { 20246 INP_WUNLOCK(inp); 20247 return (EINVAL); 20248 } 20249 #ifdef INET6 20250 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr; 20251 #endif 20252 #ifdef INET 20253 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr; 20254 #endif 20255 20256 switch (sopt->sopt_level) { 20257 #ifdef INET6 20258 case IPPROTO_IPV6: 20259 MPASS(inp->inp_vflag & INP_IPV6PROTO); 20260 switch (sopt->sopt_name) { 20261 case IPV6_USE_MIN_MTU: 20262 tcp6_use_min_mtu(tp); 20263 break; 20264 case IPV6_TCLASS: 20265 /* 20266 * The DSCP codepoint has changed, update the fsb. 20267 */ 20268 ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) | 20269 (rack->rc_inp->inp_flow & IPV6_FLOWINFO_MASK); 20270 break; 20271 } 20272 INP_WUNLOCK(inp); 20273 return (0); 20274 #endif 20275 #ifdef INET 20276 case IPPROTO_IP: 20277 switch (sopt->sopt_name) { 20278 case IP_TOS: 20279 /* 20280 * The DSCP codepoint has changed, update the fsb. 20281 */ 20282 ip->ip_tos = rack->rc_inp->inp_ip_tos; 20283 break; 20284 case IP_TTL: 20285 /* 20286 * The TTL has changed, update the fsb. 20287 */ 20288 ip->ip_ttl = rack->rc_inp->inp_ip_ttl; 20289 break; 20290 } 20291 INP_WUNLOCK(inp); 20292 return (0); 20293 #endif 20294 } 20295 20296 switch (sopt->sopt_name) { 20297 case TCP_RACK_TLP_REDUCE: /* URL:tlp_reduce */ 20298 /* Pacing related ones */ 20299 case TCP_RACK_PACE_ALWAYS: /* URL:pace_always */ 20300 case TCP_BBR_RACK_INIT_RATE: /* URL:irate */ 20301 case TCP_BBR_IWINTSO: /* URL:tso_iwin */ 20302 case TCP_RACK_PACE_MAX_SEG: /* URL:pace_max_seg */ 20303 case TCP_RACK_FORCE_MSEG: /* URL:force_max_seg */ 20304 case TCP_RACK_PACE_RATE_CA: /* URL:pr_ca */ 20305 case TCP_RACK_PACE_RATE_SS: /* URL:pr_ss*/ 20306 case TCP_RACK_PACE_RATE_REC: /* URL:pr_rec */ 20307 case TCP_RACK_GP_INCREASE_CA: /* URL:gp_inc_ca */ 20308 case TCP_RACK_GP_INCREASE_SS: /* URL:gp_inc_ss */ 20309 case TCP_RACK_GP_INCREASE_REC: /* URL:gp_inc_rec */ 20310 case TCP_RACK_RR_CONF: /* URL:rrr_conf */ 20311 case TCP_BBR_HDWR_PACE: /* URL:hdwrpace */ 20312 case TCP_HDWR_RATE_CAP: /* URL:hdwrcap boolean */ 20313 case TCP_PACING_RATE_CAP: /* URL:cap -- used by side-channel */ 20314 case TCP_HDWR_UP_ONLY: /* URL:uponly -- hardware pacing boolean */ 20315 /* End pacing related */ 20316 case TCP_FAST_RSM_HACK: /* URL:frsm_hack */ 20317 case TCP_DELACK: /* URL:delack (in base TCP i.e. tcp_hints along with cc etc ) */ 20318 case TCP_RACK_PRR_SENDALOT: /* URL:prr_sendalot */ 20319 case TCP_RACK_MIN_TO: /* URL:min_to */ 20320 case TCP_RACK_EARLY_SEG: /* URL:early_seg */ 20321 case TCP_RACK_REORD_THRESH: /* URL:reord_thresh */ 20322 case TCP_RACK_REORD_FADE: /* URL:reord_fade */ 20323 case TCP_RACK_TLP_THRESH: /* URL:tlp_thresh */ 20324 case TCP_RACK_PKT_DELAY: /* URL:pkt_delay */ 20325 case TCP_RACK_TLP_USE: /* URL:tlp_use */ 20326 case TCP_BBR_RACK_RTT_USE: /* URL:rttuse */ 20327 case TCP_BBR_USE_RACK_RR: /* URL:rackrr */ 20328 case TCP_RACK_DO_DETECTION: /* URL:detect */ 20329 case TCP_NO_PRR: /* URL:noprr */ 20330 case TCP_TIMELY_DYN_ADJ: /* URL:dynamic */ 20331 case TCP_DATA_AFTER_CLOSE: /* no URL */ 20332 case TCP_RACK_NONRXT_CFG_RATE: /* URL:nonrxtcr */ 20333 case TCP_SHARED_CWND_ENABLE: /* URL:scwnd */ 20334 case TCP_RACK_MBUF_QUEUE: /* URL:mqueue */ 20335 case TCP_RACK_NO_PUSH_AT_MAX: /* URL:npush */ 20336 case TCP_RACK_PACE_TO_FILL: /* URL:fillcw */ 20337 case TCP_SHARED_CWND_TIME_LIMIT: /* URL:lscwnd */ 20338 case TCP_RACK_PROFILE: /* URL:profile */ 20339 case TCP_USE_CMP_ACKS: /* URL:cmpack */ 20340 case TCP_RACK_ABC_VAL: /* URL:labc */ 20341 case TCP_REC_ABC_VAL: /* URL:reclabc */ 20342 case TCP_RACK_MEASURE_CNT: /* URL:measurecnt */ 20343 case TCP_DEFER_OPTIONS: /* URL:defer */ 20344 case TCP_RACK_DSACK_OPT: /* URL:dsack */ 20345 case TCP_RACK_PACING_BETA: /* URL:pacing_beta */ 20346 case TCP_RACK_PACING_BETA_ECN: /* URL:pacing_beta_ecn */ 20347 case TCP_RACK_TIMER_SLOP: /* URL:timer_slop */ 20348 case TCP_RACK_ENABLE_HYSTART: /* URL:hystart */ 20349 break; 20350 default: 20351 /* Filter off all unknown options to the base stack */ 20352 return (tcp_default_ctloutput(inp, sopt)); 20353 break; 20354 } 20355 INP_WUNLOCK(inp); 20356 if (sopt->sopt_name == TCP_PACING_RATE_CAP) { 20357 error = sooptcopyin(sopt, &loptval, sizeof(loptval), sizeof(loptval)); 20358 /* 20359 * We truncate it down to 32 bits for the socket-option trace this 20360 * means rates > 34Gbps won't show right, but thats probably ok. 20361 */ 20362 optval = (uint32_t)loptval; 20363 } else { 20364 error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval)); 20365 /* Save it in 64 bit form too */ 20366 loptval = optval; 20367 } 20368 if (error) 20369 return (error); 20370 INP_WLOCK(inp); 20371 if (inp->inp_flags & INP_DROPPED) { 20372 INP_WUNLOCK(inp); 20373 return (ECONNRESET); 20374 } 20375 if (tp->t_fb != &__tcp_rack) { 20376 INP_WUNLOCK(inp); 20377 return (ENOPROTOOPT); 20378 } 20379 if (rack->defer_options && (rack->gp_ready == 0) && 20380 (sopt->sopt_name != TCP_DEFER_OPTIONS) && 20381 (sopt->sopt_name != TCP_RACK_PACING_BETA) && 20382 (sopt->sopt_name != TCP_RACK_PACING_BETA_ECN) && 20383 (sopt->sopt_name != TCP_RACK_MEASURE_CNT)) { 20384 /* Options are beind deferred */ 20385 if (rack_add_deferred_option(rack, sopt->sopt_name, loptval)) { 20386 INP_WUNLOCK(inp); 20387 return (0); 20388 } else { 20389 /* No memory to defer, fail */ 20390 INP_WUNLOCK(inp); 20391 return (ENOMEM); 20392 } 20393 } 20394 error = rack_process_option(tp, rack, sopt->sopt_name, optval, loptval); 20395 INP_WUNLOCK(inp); 20396 return (error); 20397 } 20398 20399 static void 20400 rack_fill_info(struct tcpcb *tp, struct tcp_info *ti) 20401 { 20402 20403 INP_WLOCK_ASSERT(tptoinpcb(tp)); 20404 bzero(ti, sizeof(*ti)); 20405 20406 ti->tcpi_state = tp->t_state; 20407 if ((tp->t_flags & TF_REQ_TSTMP) && (tp->t_flags & TF_RCVD_TSTMP)) 20408 ti->tcpi_options |= TCPI_OPT_TIMESTAMPS; 20409 if (tp->t_flags & TF_SACK_PERMIT) 20410 ti->tcpi_options |= TCPI_OPT_SACK; 20411 if ((tp->t_flags & TF_REQ_SCALE) && (tp->t_flags & TF_RCVD_SCALE)) { 20412 ti->tcpi_options |= TCPI_OPT_WSCALE; 20413 ti->tcpi_snd_wscale = tp->snd_scale; 20414 ti->tcpi_rcv_wscale = tp->rcv_scale; 20415 } 20416 if (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT)) 20417 ti->tcpi_options |= TCPI_OPT_ECN; 20418 if (tp->t_flags & TF_FASTOPEN) 20419 ti->tcpi_options |= TCPI_OPT_TFO; 20420 /* still kept in ticks is t_rcvtime */ 20421 ti->tcpi_last_data_recv = ((uint32_t)ticks - tp->t_rcvtime) * tick; 20422 /* Since we hold everything in precise useconds this is easy */ 20423 ti->tcpi_rtt = tp->t_srtt; 20424 ti->tcpi_rttvar = tp->t_rttvar; 20425 ti->tcpi_rto = tp->t_rxtcur; 20426 ti->tcpi_snd_ssthresh = tp->snd_ssthresh; 20427 ti->tcpi_snd_cwnd = tp->snd_cwnd; 20428 /* 20429 * FreeBSD-specific extension fields for tcp_info. 20430 */ 20431 ti->tcpi_rcv_space = tp->rcv_wnd; 20432 ti->tcpi_rcv_nxt = tp->rcv_nxt; 20433 ti->tcpi_snd_wnd = tp->snd_wnd; 20434 ti->tcpi_snd_bwnd = 0; /* Unused, kept for compat. */ 20435 ti->tcpi_snd_nxt = tp->snd_nxt; 20436 ti->tcpi_snd_mss = tp->t_maxseg; 20437 ti->tcpi_rcv_mss = tp->t_maxseg; 20438 ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack; 20439 ti->tcpi_rcv_ooopack = tp->t_rcvoopack; 20440 ti->tcpi_snd_zerowin = tp->t_sndzerowin; 20441 #ifdef NETFLIX_STATS 20442 ti->tcpi_total_tlp = tp->t_sndtlppack; 20443 ti->tcpi_total_tlp_bytes = tp->t_sndtlpbyte; 20444 memcpy(&ti->tcpi_rxsyninfo, &tp->t_rxsyninfo, sizeof(struct tcpsyninfo)); 20445 #endif 20446 #ifdef TCP_OFFLOAD 20447 if (tp->t_flags & TF_TOE) { 20448 ti->tcpi_options |= TCPI_OPT_TOE; 20449 tcp_offload_tcp_info(tp, ti); 20450 } 20451 #endif 20452 } 20453 20454 static int 20455 rack_get_sockopt(struct inpcb *inp, struct sockopt *sopt) 20456 { 20457 struct tcpcb *tp; 20458 struct tcp_rack *rack; 20459 int32_t error, optval; 20460 uint64_t val, loptval; 20461 struct tcp_info ti; 20462 /* 20463 * Because all our options are either boolean or an int, we can just 20464 * pull everything into optval and then unlock and copy. If we ever 20465 * add a option that is not a int, then this will have quite an 20466 * impact to this routine. 20467 */ 20468 error = 0; 20469 tp = intotcpcb(inp); 20470 rack = (struct tcp_rack *)tp->t_fb_ptr; 20471 if (rack == NULL) { 20472 INP_WUNLOCK(inp); 20473 return (EINVAL); 20474 } 20475 switch (sopt->sopt_name) { 20476 case TCP_INFO: 20477 /* First get the info filled */ 20478 rack_fill_info(tp, &ti); 20479 /* Fix up the rtt related fields if needed */ 20480 INP_WUNLOCK(inp); 20481 error = sooptcopyout(sopt, &ti, sizeof ti); 20482 return (error); 20483 /* 20484 * Beta is the congestion control value for NewReno that influences how 20485 * much of a backoff happens when loss is detected. It is normally set 20486 * to 50 for 50% i.e. the cwnd is reduced to 50% of its previous value 20487 * when you exit recovery. 20488 */ 20489 case TCP_RACK_PACING_BETA: 20490 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) 20491 error = EINVAL; 20492 else if (rack->rc_pacing_cc_set == 0) 20493 optval = rack->r_ctl.rc_saved_beta.beta; 20494 else { 20495 /* 20496 * Reach out into the CC data and report back what 20497 * I have previously set. Yeah it looks hackish but 20498 * we don't want to report the saved values. 20499 */ 20500 if (tp->t_ccv.cc_data) 20501 optval = ((struct newreno *)tp->t_ccv.cc_data)->beta; 20502 else 20503 error = EINVAL; 20504 } 20505 break; 20506 /* 20507 * Beta_ecn is the congestion control value for NewReno that influences how 20508 * much of a backoff happens when a ECN mark is detected. It is normally set 20509 * to 80 for 80% i.e. the cwnd is reduced by 20% of its previous value when 20510 * you exit recovery. Note that classic ECN has a beta of 50, it is only 20511 * ABE Ecn that uses this "less" value, but we do too with pacing :) 20512 */ 20513 20514 case TCP_RACK_PACING_BETA_ECN: 20515 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) 20516 error = EINVAL; 20517 else if (rack->rc_pacing_cc_set == 0) 20518 optval = rack->r_ctl.rc_saved_beta.beta_ecn; 20519 else { 20520 /* 20521 * Reach out into the CC data and report back what 20522 * I have previously set. Yeah it looks hackish but 20523 * we don't want to report the saved values. 20524 */ 20525 if (tp->t_ccv.cc_data) 20526 optval = ((struct newreno *)tp->t_ccv.cc_data)->beta_ecn; 20527 else 20528 error = EINVAL; 20529 } 20530 break; 20531 case TCP_RACK_DSACK_OPT: 20532 optval = 0; 20533 if (rack->rc_rack_tmr_std_based) { 20534 optval |= 1; 20535 } 20536 if (rack->rc_rack_use_dsack) { 20537 optval |= 2; 20538 } 20539 break; 20540 case TCP_RACK_ENABLE_HYSTART: 20541 { 20542 if (tp->t_ccv.flags & CCF_HYSTART_ALLOWED) { 20543 optval = RACK_HYSTART_ON; 20544 if (tp->t_ccv.flags & CCF_HYSTART_CAN_SH_CWND) 20545 optval = RACK_HYSTART_ON_W_SC; 20546 if (tp->t_ccv.flags & CCF_HYSTART_CONS_SSTH) 20547 optval = RACK_HYSTART_ON_W_SC_C; 20548 } else { 20549 optval = RACK_HYSTART_OFF; 20550 } 20551 } 20552 break; 20553 case TCP_FAST_RSM_HACK: 20554 optval = rack->fast_rsm_hack; 20555 break; 20556 case TCP_DEFER_OPTIONS: 20557 optval = rack->defer_options; 20558 break; 20559 case TCP_RACK_MEASURE_CNT: 20560 optval = rack->r_ctl.req_measurements; 20561 break; 20562 case TCP_REC_ABC_VAL: 20563 optval = rack->r_use_labc_for_rec; 20564 break; 20565 case TCP_RACK_ABC_VAL: 20566 optval = rack->rc_labc; 20567 break; 20568 case TCP_HDWR_UP_ONLY: 20569 optval= rack->r_up_only; 20570 break; 20571 case TCP_PACING_RATE_CAP: 20572 loptval = rack->r_ctl.bw_rate_cap; 20573 break; 20574 case TCP_RACK_PROFILE: 20575 /* You cannot retrieve a profile, its write only */ 20576 error = EINVAL; 20577 break; 20578 case TCP_USE_CMP_ACKS: 20579 optval = rack->r_use_cmp_ack; 20580 break; 20581 case TCP_RACK_PACE_TO_FILL: 20582 optval = rack->rc_pace_to_cwnd; 20583 if (optval && rack->r_fill_less_agg) 20584 optval++; 20585 break; 20586 case TCP_RACK_NO_PUSH_AT_MAX: 20587 optval = rack->r_ctl.rc_no_push_at_mrtt; 20588 break; 20589 case TCP_SHARED_CWND_ENABLE: 20590 optval = rack->rack_enable_scwnd; 20591 break; 20592 case TCP_RACK_NONRXT_CFG_RATE: 20593 optval = rack->rack_rec_nonrxt_use_cr; 20594 break; 20595 case TCP_NO_PRR: 20596 if (rack->rack_no_prr == 1) 20597 optval = 1; 20598 else if (rack->no_prr_addback == 1) 20599 optval = 2; 20600 else 20601 optval = 0; 20602 break; 20603 case TCP_RACK_DO_DETECTION: 20604 optval = rack->do_detection; 20605 break; 20606 case TCP_RACK_MBUF_QUEUE: 20607 /* Now do we use the LRO mbuf-queue feature */ 20608 optval = rack->r_mbuf_queue; 20609 break; 20610 case TCP_TIMELY_DYN_ADJ: 20611 optval = rack->rc_gp_dyn_mul; 20612 break; 20613 case TCP_BBR_IWINTSO: 20614 optval = rack->rc_init_win; 20615 break; 20616 case TCP_RACK_TLP_REDUCE: 20617 /* RACK TLP cwnd reduction (bool) */ 20618 optval = rack->r_ctl.rc_tlp_cwnd_reduce; 20619 break; 20620 case TCP_BBR_RACK_INIT_RATE: 20621 val = rack->r_ctl.init_rate; 20622 /* convert to kbits per sec */ 20623 val *= 8; 20624 val /= 1000; 20625 optval = (uint32_t)val; 20626 break; 20627 case TCP_RACK_FORCE_MSEG: 20628 optval = rack->rc_force_max_seg; 20629 break; 20630 case TCP_RACK_PACE_MAX_SEG: 20631 /* Max segments in a pace */ 20632 optval = rack->rc_user_set_max_segs; 20633 break; 20634 case TCP_RACK_PACE_ALWAYS: 20635 /* Use the always pace method */ 20636 optval = rack->rc_always_pace; 20637 break; 20638 case TCP_RACK_PRR_SENDALOT: 20639 /* Allow PRR to send more than one seg */ 20640 optval = rack->r_ctl.rc_prr_sendalot; 20641 break; 20642 case TCP_RACK_MIN_TO: 20643 /* Minimum time between rack t-o's in ms */ 20644 optval = rack->r_ctl.rc_min_to; 20645 break; 20646 case TCP_RACK_EARLY_SEG: 20647 /* If early recovery max segments */ 20648 optval = rack->r_ctl.rc_early_recovery_segs; 20649 break; 20650 case TCP_RACK_REORD_THRESH: 20651 /* RACK reorder threshold (shift amount) */ 20652 optval = rack->r_ctl.rc_reorder_shift; 20653 break; 20654 case TCP_RACK_REORD_FADE: 20655 /* Does reordering fade after ms time */ 20656 optval = rack->r_ctl.rc_reorder_fade; 20657 break; 20658 case TCP_BBR_USE_RACK_RR: 20659 /* Do we use the rack cheat for rxt */ 20660 optval = rack->use_rack_rr; 20661 break; 20662 case TCP_RACK_RR_CONF: 20663 optval = rack->r_rr_config; 20664 break; 20665 case TCP_HDWR_RATE_CAP: 20666 optval = rack->r_rack_hw_rate_caps; 20667 break; 20668 case TCP_BBR_HDWR_PACE: 20669 optval = rack->rack_hdw_pace_ena; 20670 break; 20671 case TCP_RACK_TLP_THRESH: 20672 /* RACK TLP theshold i.e. srtt+(srtt/N) */ 20673 optval = rack->r_ctl.rc_tlp_threshold; 20674 break; 20675 case TCP_RACK_PKT_DELAY: 20676 /* RACK added ms i.e. rack-rtt + reord + N */ 20677 optval = rack->r_ctl.rc_pkt_delay; 20678 break; 20679 case TCP_RACK_TLP_USE: 20680 optval = rack->rack_tlp_threshold_use; 20681 break; 20682 case TCP_RACK_PACE_RATE_CA: 20683 optval = rack->r_ctl.rc_fixed_pacing_rate_ca; 20684 break; 20685 case TCP_RACK_PACE_RATE_SS: 20686 optval = rack->r_ctl.rc_fixed_pacing_rate_ss; 20687 break; 20688 case TCP_RACK_PACE_RATE_REC: 20689 optval = rack->r_ctl.rc_fixed_pacing_rate_rec; 20690 break; 20691 case TCP_RACK_GP_INCREASE_SS: 20692 optval = rack->r_ctl.rack_per_of_gp_ca; 20693 break; 20694 case TCP_RACK_GP_INCREASE_CA: 20695 optval = rack->r_ctl.rack_per_of_gp_ss; 20696 break; 20697 case TCP_BBR_RACK_RTT_USE: 20698 optval = rack->r_ctl.rc_rate_sample_method; 20699 break; 20700 case TCP_DELACK: 20701 optval = tp->t_delayed_ack; 20702 break; 20703 case TCP_DATA_AFTER_CLOSE: 20704 optval = rack->rc_allow_data_af_clo; 20705 break; 20706 case TCP_SHARED_CWND_TIME_LIMIT: 20707 optval = rack->r_limit_scw; 20708 break; 20709 case TCP_RACK_TIMER_SLOP: 20710 optval = rack->r_ctl.timer_slop; 20711 break; 20712 default: 20713 return (tcp_default_ctloutput(inp, sopt)); 20714 break; 20715 } 20716 INP_WUNLOCK(inp); 20717 if (error == 0) { 20718 if (TCP_PACING_RATE_CAP) 20719 error = sooptcopyout(sopt, &loptval, sizeof loptval); 20720 else 20721 error = sooptcopyout(sopt, &optval, sizeof optval); 20722 } 20723 return (error); 20724 } 20725 20726 static int 20727 rack_ctloutput(struct inpcb *inp, struct sockopt *sopt) 20728 { 20729 if (sopt->sopt_dir == SOPT_SET) { 20730 return (rack_set_sockopt(inp, sopt)); 20731 } else if (sopt->sopt_dir == SOPT_GET) { 20732 return (rack_get_sockopt(inp, sopt)); 20733 } else { 20734 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir); 20735 } 20736 } 20737 20738 static const char *rack_stack_names[] = { 20739 __XSTRING(STACKNAME), 20740 #ifdef STACKALIAS 20741 __XSTRING(STACKALIAS), 20742 #endif 20743 }; 20744 20745 static int 20746 rack_ctor(void *mem, int32_t size, void *arg, int32_t how) 20747 { 20748 memset(mem, 0, size); 20749 return (0); 20750 } 20751 20752 static void 20753 rack_dtor(void *mem, int32_t size, void *arg) 20754 { 20755 20756 } 20757 20758 static bool rack_mod_inited = false; 20759 20760 static int 20761 tcp_addrack(module_t mod, int32_t type, void *data) 20762 { 20763 int32_t err = 0; 20764 int num_stacks; 20765 20766 switch (type) { 20767 case MOD_LOAD: 20768 rack_zone = uma_zcreate(__XSTRING(MODNAME) "_map", 20769 sizeof(struct rack_sendmap), 20770 rack_ctor, rack_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); 20771 20772 rack_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb", 20773 sizeof(struct tcp_rack), 20774 rack_ctor, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0); 20775 20776 sysctl_ctx_init(&rack_sysctl_ctx); 20777 rack_sysctl_root = SYSCTL_ADD_NODE(&rack_sysctl_ctx, 20778 SYSCTL_STATIC_CHILDREN(_net_inet_tcp), 20779 OID_AUTO, 20780 #ifdef STACKALIAS 20781 __XSTRING(STACKALIAS), 20782 #else 20783 __XSTRING(STACKNAME), 20784 #endif 20785 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 20786 ""); 20787 if (rack_sysctl_root == NULL) { 20788 printf("Failed to add sysctl node\n"); 20789 err = EFAULT; 20790 goto free_uma; 20791 } 20792 rack_init_sysctls(); 20793 num_stacks = nitems(rack_stack_names); 20794 err = register_tcp_functions_as_names(&__tcp_rack, M_WAITOK, 20795 rack_stack_names, &num_stacks); 20796 if (err) { 20797 printf("Failed to register %s stack name for " 20798 "%s module\n", rack_stack_names[num_stacks], 20799 __XSTRING(MODNAME)); 20800 sysctl_ctx_free(&rack_sysctl_ctx); 20801 free_uma: 20802 uma_zdestroy(rack_zone); 20803 uma_zdestroy(rack_pcb_zone); 20804 rack_counter_destroy(); 20805 printf("Failed to register rack module -- err:%d\n", err); 20806 return (err); 20807 } 20808 tcp_lro_reg_mbufq(); 20809 rack_mod_inited = true; 20810 break; 20811 case MOD_QUIESCE: 20812 err = deregister_tcp_functions(&__tcp_rack, true, false); 20813 break; 20814 case MOD_UNLOAD: 20815 err = deregister_tcp_functions(&__tcp_rack, false, true); 20816 if (err == EBUSY) 20817 break; 20818 if (rack_mod_inited) { 20819 uma_zdestroy(rack_zone); 20820 uma_zdestroy(rack_pcb_zone); 20821 sysctl_ctx_free(&rack_sysctl_ctx); 20822 rack_counter_destroy(); 20823 rack_mod_inited = false; 20824 } 20825 tcp_lro_dereg_mbufq(); 20826 err = 0; 20827 break; 20828 default: 20829 return (EOPNOTSUPP); 20830 } 20831 return (err); 20832 } 20833 20834 static moduledata_t tcp_rack = { 20835 .name = __XSTRING(MODNAME), 20836 .evhand = tcp_addrack, 20837 .priv = 0 20838 }; 20839 20840 MODULE_VERSION(MODNAME, 1); 20841 DECLARE_MODULE(MODNAME, tcp_rack, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY); 20842 MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1); 20843 20844 #endif /* #if !defined(INET) && !defined(INET6) */ 20845