1 /*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
26
27 #include <sys/cdefs.h>
28 #include "opt_inet.h"
29 #include "opt_inet6.h"
30 #include "opt_ipsec.h"
31 #include "opt_ratelimit.h"
32 #include "opt_kern_tls.h"
33 #if defined(INET) || defined(INET6)
34 #include <sys/param.h>
35 #include <sys/arb.h>
36 #include <sys/module.h>
37 #include <sys/kernel.h>
38 #ifdef TCP_HHOOK
39 #include <sys/hhook.h>
40 #endif
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/mutex.h>
44 #include <sys/mbuf.h>
45 #include <sys/proc.h> /* for proc0 declaration */
46 #include <sys/socket.h>
47 #include <sys/socketvar.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50 #ifdef STATS
51 #include <sys/qmath.h>
52 #include <sys/tree.h>
53 #include <sys/stats.h> /* Must come after qmath.h and tree.h */
54 #else
55 #include <sys/tree.h>
56 #endif
57 #include <sys/refcount.h>
58 #include <sys/queue.h>
59 #include <sys/tim_filter.h>
60 #include <sys/smp.h>
61 #include <sys/kthread.h>
62 #include <sys/kern_prefetch.h>
63 #include <sys/protosw.h>
64 #ifdef TCP_ACCOUNTING
65 #include <sys/sched.h>
66 #include <machine/cpu.h>
67 #endif
68 #include <vm/uma.h>
69
70 #include <net/route.h>
71 #include <net/route/nhop.h>
72 #include <net/vnet.h>
73
74 #define TCPSTATES /* for logging */
75
76 #include <netinet/in.h>
77 #include <netinet/in_kdtrace.h>
78 #include <netinet/in_pcb.h>
79 #include <netinet/ip.h>
80 #include <netinet/ip_var.h>
81 #include <netinet/ip6.h>
82 #include <netinet6/in6_pcb.h>
83 #include <netinet6/ip6_var.h>
84 #include <netinet/tcp.h>
85 #define TCPOUTFLAGS
86 #include <netinet/tcp_fsm.h>
87 #include <netinet/tcp_seq.h>
88 #include <netinet/tcp_timer.h>
89 #include <netinet/tcp_var.h>
90 #include <netinet/tcp_log_buf.h>
91 #include <netinet/tcp_syncache.h>
92 #include <netinet/tcp_hpts.h>
93 #include <netinet/tcp_ratelimit.h>
94 #include <netinet/tcp_accounting.h>
95 #include <netinet/tcpip.h>
96 #include <netinet/cc/cc.h>
97 #include <netinet/cc/cc_newreno.h>
98 #include <netinet/tcp_fastopen.h>
99 #include <netinet/tcp_lro.h>
100 #ifdef NETFLIX_SHARED_CWND
101 #include <netinet/tcp_shared_cwnd.h>
102 #endif
103 #ifdef TCP_OFFLOAD
104 #include <netinet/tcp_offload.h>
105 #endif
106 #ifdef INET6
107 #include <netinet6/tcp6_var.h>
108 #endif
109 #include <netinet/tcp_ecn.h>
110
111 #include <netipsec/ipsec_support.h>
112
113 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
114 #include <netipsec/ipsec.h>
115 #include <netipsec/ipsec6.h>
116 #endif /* IPSEC */
117
118 #include <netinet/udp.h>
119 #include <netinet/udp_var.h>
120 #include <machine/in_cksum.h>
121
122 #ifdef MAC
123 #include <security/mac/mac_framework.h>
124 #endif
125 #include "sack_filter.h"
126 #include "tcp_rack.h"
127 #include "tailq_hash.h"
128 #include "rack_bbr_common.h"
129
130 uma_zone_t rack_zone;
131 uma_zone_t rack_pcb_zone;
132
133 #ifndef TICKS2SBT
134 #define TICKS2SBT(__t) (tick_sbt * ((sbintime_t)(__t)))
135 #endif
136
137 VNET_DECLARE(uint32_t, newreno_beta);
138 VNET_DECLARE(uint32_t, newreno_beta_ecn);
139 #define V_newreno_beta VNET(newreno_beta)
140 #define V_newreno_beta_ecn VNET(newreno_beta_ecn)
141
142 #define M_TCPFSB __CONCAT(M_TCPFSB, STACKNAME)
143 #define M_TCPDO __CONCAT(M_TCPDO, STACKNAME)
144
145 MALLOC_DEFINE(M_TCPFSB, "tcp_fsb_" __XSTRING(STACKNAME), "TCP fast send block");
146 MALLOC_DEFINE(M_TCPDO, "tcp_do_" __XSTRING(STACKNAME), "TCP deferred options");
147 MALLOC_DEFINE(M_TCPPCM, "tcp_pcm_" __XSTRING(STACKNAME), "TCP PCM measurement information");
148
149 struct sysctl_ctx_list rack_sysctl_ctx;
150 struct sysctl_oid *rack_sysctl_root;
151
152 #define CUM_ACKED 1
153 #define SACKED 2
154
155 /*
156 * The RACK module incorporates a number of
157 * TCP ideas that have been put out into the IETF
158 * over the last few years:
159 * - Matt Mathis's Rate Halving which slowly drops
160 * the congestion window so that the ack clock can
161 * be maintained during a recovery.
162 * - Yuchung Cheng's RACK TCP (for which its named) that
163 * will stop us using the number of dup acks and instead
164 * use time as the gage of when we retransmit.
165 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
166 * of Dukkipati et.al.
167 * RACK depends on SACK, so if an endpoint arrives that
168 * cannot do SACK the state machine below will shuttle the
169 * connection back to using the "default" TCP stack that is
170 * in FreeBSD.
171 *
172 * To implement RACK the original TCP stack was first decomposed
173 * into a functional state machine with individual states
174 * for each of the possible TCP connection states. The do_segment
175 * functions role in life is to mandate the connection supports SACK
176 * initially and then assure that the RACK state matches the conenction
177 * state before calling the states do_segment function. Each
178 * state is simplified due to the fact that the original do_segment
179 * has been decomposed and we *know* what state we are in (no
180 * switches on the state) and all tests for SACK are gone. This
181 * greatly simplifies what each state does.
182 *
183 * TCP output is also over-written with a new version since it
184 * must maintain the new rack scoreboard.
185 *
186 */
187 static int32_t rack_tlp_thresh = 1;
188 static int32_t rack_tlp_limit = 2; /* No more than 2 TLPs w-out new data */
189 static int32_t rack_tlp_use_greater = 1;
190 static int32_t rack_reorder_thresh = 2;
191 static int32_t rack_reorder_fade = 60000000; /* 0 - never fade, def 60,000,000
192 * - 60 seconds */
193 static uint32_t rack_pcm_every_n_rounds = 100;
194 static uint32_t rack_pcm_blast = 0;
195 static uint32_t rack_pcm_is_enabled = 1;
196 static uint8_t rack_ssthresh_rest_rto_rec = 0; /* Do we restore ssthresh when we have rec -> rto -> rec */
197
198 static uint32_t rack_gp_gain_req = 1200; /* Amount percent wise required to gain to record a round as "gaining" */
199 static uint32_t rack_rnd_cnt_req = 0x10005; /* Default number of rounds if we are below rack_gp_gain_req where we exit ss */
200
201
202 static int32_t rack_rxt_scoreboard_clear_thresh = 2;
203 static int32_t rack_dnd_default = 0; /* For rr_conf = 3, what is the default for dnd */
204 static int32_t rack_rxt_controls = 0;
205 static int32_t rack_fill_cw_state = 0;
206 static uint8_t rack_req_measurements = 1;
207 /* Attack threshold detections */
208 static uint32_t rack_highest_sack_thresh_seen = 0;
209 static uint32_t rack_highest_move_thresh_seen = 0;
210 static uint32_t rack_merge_out_sacks_on_attack = 0;
211 static int32_t rack_enable_hw_pacing = 0; /* Due to CCSP keep it off by default */
212 static int32_t rack_hw_rate_caps = 0; /* 1; */
213 static int32_t rack_hw_rate_cap_per = 0; /* 0 -- off */
214 static int32_t rack_hw_rate_min = 0; /* 1500000;*/
215 static int32_t rack_hw_rate_to_low = 0; /* 1200000; */
216 static int32_t rack_hw_up_only = 0;
217 static int32_t rack_stats_gets_ms_rtt = 1;
218 static int32_t rack_prr_addbackmax = 2;
219 static int32_t rack_do_hystart = 0;
220 static int32_t rack_apply_rtt_with_reduced_conf = 0;
221 static int32_t rack_hibeta_setting = 0;
222 static int32_t rack_default_pacing_divisor = 250;
223 static uint16_t rack_pacing_min_seg = 0;
224 static int32_t rack_timely_off = 0;
225
226 static uint32_t sad_seg_size_per = 800; /* 80.0 % */
227 static int32_t rack_pkt_delay = 1000;
228 static int32_t rack_send_a_lot_in_prr = 1;
229 static int32_t rack_min_to = 1000; /* Number of microsecond min timeout */
230 static int32_t rack_verbose_logging = 0;
231 static int32_t rack_ignore_data_after_close = 1;
232 static int32_t rack_enable_shared_cwnd = 1;
233 static int32_t rack_use_cmp_acks = 1;
234 static int32_t rack_use_fsb = 1;
235 static int32_t rack_use_rfo = 1;
236 static int32_t rack_use_rsm_rfo = 1;
237 static int32_t rack_max_abc_post_recovery = 2;
238 static int32_t rack_client_low_buf = 0;
239 static int32_t rack_dsack_std_based = 0x3; /* bit field bit 1 sets rc_rack_tmr_std_based and bit 2 sets rc_rack_use_dsack */
240 static int32_t rack_bw_multipler = 0; /* Limit on fill cw's jump up to be this x gp_est */
241 #ifdef TCP_ACCOUNTING
242 static int32_t rack_tcp_accounting = 0;
243 #endif
244 static int32_t rack_limits_scwnd = 1;
245 static int32_t rack_enable_mqueue_for_nonpaced = 0;
246 static int32_t rack_hybrid_allow_set_maxseg = 0;
247 static int32_t rack_disable_prr = 0;
248 static int32_t use_rack_rr = 1;
249 static int32_t rack_non_rxt_use_cr = 0; /* does a non-rxt in recovery use the configured rate (ss/ca)? */
250 static int32_t rack_persist_min = 250000; /* 250usec */
251 static int32_t rack_persist_max = 2000000; /* 2 Second in usec's */
252 static int32_t rack_honors_hpts_min_to = 1; /* Do we honor the hpts minimum time out for pacing timers */
253 static uint32_t rack_max_reduce = 10; /* Percent we can reduce pacing delay by */
254 static int32_t rack_sack_not_required = 1; /* set to one to allow non-sack to use rack */
255 static int32_t rack_limit_time_with_srtt = 0;
256 static int32_t rack_autosndbuf_inc = 20; /* In percentage form */
257 static int32_t rack_enobuf_hw_boost_mult = 0; /* How many times the hw rate we boost pacing delay using time_between */
258 static int32_t rack_enobuf_hw_max = 12000; /* 12 ms in usecs */
259 static int32_t rack_enobuf_hw_min = 10000; /* 10 ms in usecs */
260 static int32_t rack_hw_rwnd_factor = 2; /* How many max_segs the rwnd must be before we hold off sending */
261 static int32_t rack_hw_check_queue = 0; /* Do we always pre-check queue depth of a hw queue */
262
263 /*
264 * Currently regular tcp has a rto_min of 30ms
265 * the backoff goes 12 times so that ends up
266 * being a total of 122.850 seconds before a
267 * connection is killed.
268 */
269 static uint32_t rack_def_data_window = 20;
270 static uint32_t rack_goal_bdp = 2;
271 static uint32_t rack_min_srtts = 1;
272 static uint32_t rack_min_measure_usec = 0;
273 static int32_t rack_tlp_min = 10000; /* 10ms */
274 static int32_t rack_rto_min = 30000; /* 30,000 usec same as main freebsd */
275 static int32_t rack_rto_max = 4000000; /* 4 seconds in usec's */
276 static const int32_t rack_free_cache = 2;
277 static int32_t rack_hptsi_segments = 40;
278 static int32_t rack_rate_sample_method = USE_RTT_LOW;
279 static int32_t rack_pace_every_seg = 0;
280 static int32_t rack_delayed_ack_time = 40000; /* 40ms in usecs */
281 static int32_t rack_pacing_delay_reduction = 4;
282 static int32_t rack_wma_divisor = 8; /* For WMA calculation */
283 static int32_t rack_cwnd_block_ends_measure = 0;
284 static int32_t rack_rwnd_block_ends_measure = 0;
285 static int32_t rack_def_profile = 0;
286
287 static int32_t rack_lower_cwnd_at_tlp = 0;
288 static int32_t rack_always_send_oldest = 0;
289 static int32_t rack_tlp_threshold_use = TLP_USE_TWO_ONE;
290
291 static uint16_t rack_per_of_gp_ss = 250; /* 250 % slow-start */
292 static uint16_t rack_per_of_gp_ca = 200; /* 200 % congestion-avoidance */
293 static uint16_t rack_per_of_gp_rec = 200; /* 200 % of bw */
294
295 /* Probertt */
296 static uint16_t rack_per_of_gp_probertt = 60; /* 60% of bw */
297 static uint16_t rack_per_of_gp_lowthresh = 40; /* 40% is bottom */
298 static uint16_t rack_per_of_gp_probertt_reduce = 10; /* 10% reduction */
299 static uint16_t rack_atexit_prtt_hbp = 130; /* Clamp to 130% on exit prtt if highly buffered path */
300 static uint16_t rack_atexit_prtt = 130; /* Clamp to 100% on exit prtt if non highly buffered path */
301
302 static uint32_t rack_max_drain_wait = 2; /* How man gp srtt's before we give up draining */
303 static uint32_t rack_must_drain = 1; /* How many GP srtt's we *must* wait */
304 static uint32_t rack_probertt_use_min_rtt_entry = 1; /* Use the min to calculate the goal else gp_srtt */
305 static uint32_t rack_probertt_use_min_rtt_exit = 0;
306 static uint32_t rack_probe_rtt_sets_cwnd = 0;
307 static uint32_t rack_probe_rtt_safety_val = 2000000; /* No more than 2 sec in probe-rtt */
308 static uint32_t rack_time_between_probertt = 9600000; /* 9.6 sec in usecs */
309 static uint32_t rack_probertt_gpsrtt_cnt_mul = 0; /* How many srtt periods does probe-rtt last top fraction */
310 static uint32_t rack_probertt_gpsrtt_cnt_div = 0; /* How many srtt periods does probe-rtt last bottom fraction */
311 static uint32_t rack_min_probertt_hold = 40000; /* Equal to delayed ack time */
312 static uint32_t rack_probertt_filter_life = 10000000;
313 static uint32_t rack_probertt_lower_within = 10;
314 static uint32_t rack_min_rtt_movement = 250000; /* Must move at least 250ms (in microseconds) to count as a lowering */
315 static int32_t rack_pace_one_seg = 0; /* Shall we pace for less than 1.4Meg 1MSS at a time */
316 static int32_t rack_probertt_clear_is = 1;
317 static int32_t rack_max_drain_hbp = 1; /* Extra drain times gpsrtt for highly buffered paths */
318 static int32_t rack_hbp_thresh = 3; /* what is the divisor max_rtt/min_rtt to decided a hbp */
319
320 /* Part of pacing */
321 static int32_t rack_max_per_above = 30; /* When we go to increment stop if above 100+this% */
322
323 /* Timely information:
324 *
325 * Here we have various control parameters on how
326 * timely may change the multiplier. rack_gain_p5_ub
327 * is associated with timely but not directly influencing
328 * the rate decision like the other variables. It controls
329 * the way fill-cw interacts with timely and caps how much
330 * timely can boost the fill-cw b/w.
331 *
332 * The other values are various boost/shrink numbers as well
333 * as potential caps when adjustments are made to the timely
334 * gain (returned by rack_get_output_gain(). Remember too that
335 * the gain returned can be overriden by other factors such as
336 * probeRTT as well as fixed-rate-pacing.
337 */
338 static int32_t rack_gain_p5_ub = 250;
339 static int32_t rack_gp_per_bw_mul_up = 2; /* 2% */
340 static int32_t rack_gp_per_bw_mul_down = 4; /* 4% */
341 static int32_t rack_gp_rtt_maxmul = 3; /* 3 x maxmin */
342 static int32_t rack_gp_rtt_minmul = 1; /* minrtt + (minrtt/mindiv) is lower rtt */
343 static int32_t rack_gp_rtt_mindiv = 4; /* minrtt + (minrtt * minmul/mindiv) is lower rtt */
344 static int32_t rack_gp_decrease_per = 80; /* Beta value of timely decrease (.8) = 80 */
345 static int32_t rack_gp_increase_per = 2; /* 2% increase in multiplier */
346 static int32_t rack_per_lower_bound = 50; /* Don't allow to drop below this multiplier */
347 static int32_t rack_per_upper_bound_ss = 0; /* Don't allow SS to grow above this */
348 static int32_t rack_per_upper_bound_ca = 0; /* Don't allow CA to grow above this */
349 static int32_t rack_do_dyn_mul = 0; /* Are the rack gp multipliers dynamic */
350 static int32_t rack_gp_no_rec_chg = 1; /* Prohibit recovery from reducing it's multiplier */
351 static int32_t rack_timely_dec_clear = 6; /* Do we clear decrement count at a value (6)? */
352 static int32_t rack_timely_max_push_rise = 3; /* One round of pushing */
353 static int32_t rack_timely_max_push_drop = 3; /* Three round of pushing */
354 static int32_t rack_timely_min_segs = 4; /* 4 segment minimum */
355 static int32_t rack_timely_no_stopping = 0;
356 static int32_t rack_down_raise_thresh = 100;
357 static int32_t rack_req_segs = 1;
358 static uint64_t rack_bw_rate_cap = 0;
359 static uint64_t rack_fillcw_bw_cap = 3750000; /* Cap fillcw at 30Mbps */
360
361
362 /* Rack specific counters */
363 counter_u64_t rack_saw_enobuf;
364 counter_u64_t rack_saw_enobuf_hw;
365 counter_u64_t rack_saw_enetunreach;
366 counter_u64_t rack_persists_sends;
367 counter_u64_t rack_persists_acks;
368 counter_u64_t rack_persists_loss;
369 counter_u64_t rack_persists_lost_ends;
370 counter_u64_t rack_total_bytes;
371 #ifdef INVARIANTS
372 counter_u64_t rack_adjust_map_bw;
373 #endif
374 /* Tail loss probe counters */
375 counter_u64_t rack_tlp_tot;
376 counter_u64_t rack_tlp_newdata;
377 counter_u64_t rack_tlp_retran;
378 counter_u64_t rack_tlp_retran_bytes;
379 counter_u64_t rack_to_tot;
380 counter_u64_t rack_hot_alloc;
381 counter_u64_t rack_to_alloc;
382 counter_u64_t rack_to_alloc_hard;
383 counter_u64_t rack_to_alloc_emerg;
384 counter_u64_t rack_to_alloc_limited;
385 counter_u64_t rack_alloc_limited_conns;
386 counter_u64_t rack_split_limited;
387 counter_u64_t rack_rxt_clamps_cwnd;
388 counter_u64_t rack_rxt_clamps_cwnd_uniq;
389
390 counter_u64_t rack_multi_single_eq;
391 counter_u64_t rack_proc_non_comp_ack;
392
393 counter_u64_t rack_fto_send;
394 counter_u64_t rack_fto_rsm_send;
395 counter_u64_t rack_nfto_resend;
396 counter_u64_t rack_non_fto_send;
397 counter_u64_t rack_extended_rfo;
398
399 counter_u64_t rack_sack_proc_all;
400 counter_u64_t rack_sack_proc_short;
401 counter_u64_t rack_sack_proc_restart;
402 counter_u64_t rack_sack_attacks_detected;
403 counter_u64_t rack_sack_attacks_reversed;
404 counter_u64_t rack_sack_attacks_suspect;
405 counter_u64_t rack_sack_used_next_merge;
406 counter_u64_t rack_sack_splits;
407 counter_u64_t rack_sack_used_prev_merge;
408 counter_u64_t rack_sack_skipped_acked;
409 counter_u64_t rack_ack_total;
410 counter_u64_t rack_express_sack;
411 counter_u64_t rack_sack_total;
412 counter_u64_t rack_move_none;
413 counter_u64_t rack_move_some;
414
415 counter_u64_t rack_input_idle_reduces;
416 counter_u64_t rack_collapsed_win;
417 counter_u64_t rack_collapsed_win_seen;
418 counter_u64_t rack_collapsed_win_rxt;
419 counter_u64_t rack_collapsed_win_rxt_bytes;
420 counter_u64_t rack_try_scwnd;
421 counter_u64_t rack_hw_pace_init_fail;
422 counter_u64_t rack_hw_pace_lost;
423
424 counter_u64_t rack_out_size[TCP_MSS_ACCT_SIZE];
425 counter_u64_t rack_opts_arry[RACK_OPTS_SIZE];
426
427
428 #define RACK_REXMTVAL(tp) max(rack_rto_min, ((tp)->t_srtt + ((tp)->t_rttvar << 2)))
429
430 #define RACK_TCPT_RANGESET(tv, value, tvmin, tvmax, slop) do { \
431 (tv) = (value) + slop; \
432 if ((u_long)(tv) < (u_long)(tvmin)) \
433 (tv) = (tvmin); \
434 if ((u_long)(tv) > (u_long)(tvmax)) \
435 (tv) = (tvmax); \
436 } while (0)
437
438 static void
439 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line);
440
441 static int
442 rack_process_ack(struct mbuf *m, struct tcphdr *th,
443 struct socket *so, struct tcpcb *tp, struct tcpopt *to,
444 uint32_t tiwin, int32_t tlen, int32_t * ofia, int32_t thflags, int32_t * ret_val, int32_t orig_tlen);
445 static int
446 rack_process_data(struct mbuf *m, struct tcphdr *th,
447 struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
448 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt);
449 static void
450 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack,
451 uint32_t th_ack, uint16_t nsegs, uint16_t type, int32_t recovery);
452 static struct rack_sendmap *rack_alloc(struct tcp_rack *rack);
453 static struct rack_sendmap *rack_alloc_limit(struct tcp_rack *rack,
454 uint8_t limit_type);
455 static struct rack_sendmap *
456 rack_check_recovery_mode(struct tcpcb *tp,
457 uint32_t tsused);
458 static uint32_t
459 rack_grab_rtt(struct tcpcb *tp, struct tcp_rack *rack);
460 static void
461 rack_cong_signal(struct tcpcb *tp,
462 uint32_t type, uint32_t ack, int );
463 static void rack_counter_destroy(void);
464 static int
465 rack_ctloutput(struct tcpcb *tp, struct sockopt *sopt);
466 static int32_t rack_ctor(void *mem, int32_t size, void *arg, int32_t how);
467 static void
468 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override);
469 static void
470 rack_do_segment(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
471 int32_t drop_hdrlen, int32_t tlen, uint8_t iptos);
472 static void rack_dtor(void *mem, int32_t size, void *arg);
473 static void
474 rack_log_alt_to_to_cancel(struct tcp_rack *rack,
475 uint32_t flex1, uint32_t flex2,
476 uint32_t flex3, uint32_t flex4,
477 uint32_t flex5, uint32_t flex6,
478 uint16_t flex7, uint8_t mod);
479
480 static void
481 rack_log_pacing_delay_calc(struct tcp_rack *rack, uint32_t len, uint32_t pacing_delay,
482 uint64_t bw_est, uint64_t bw, uint64_t len_time, int method, int line,
483 struct rack_sendmap *rsm, uint8_t quality);
484 static struct rack_sendmap *
485 rack_find_high_nonack(struct tcp_rack *rack,
486 struct rack_sendmap *rsm);
487 static struct rack_sendmap *rack_find_lowest_rsm(struct tcp_rack *rack);
488 static void rack_free(struct tcp_rack *rack, struct rack_sendmap *rsm);
489 static void rack_fini(struct tcpcb *tp, int32_t tcb_is_purged);
490 static int rack_get_sockopt(struct tcpcb *tp, struct sockopt *sopt);
491 static void
492 rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
493 tcp_seq th_ack, int line, uint8_t quality);
494 static void
495 rack_log_type_pacing_sizes(struct tcpcb *tp, struct tcp_rack *rack, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint8_t frm);
496
497 static uint32_t
498 rack_get_pacing_len(struct tcp_rack *rack, uint64_t bw, uint32_t mss);
499 static int32_t rack_handoff_ok(struct tcpcb *tp);
500 static int32_t rack_init(struct tcpcb *tp, void **ptr);
501 static void rack_init_sysctls(void);
502
503 static void
504 rack_log_ack(struct tcpcb *tp, struct tcpopt *to,
505 struct tcphdr *th, int entered_rec, int dup_ack_struck,
506 int *dsack_seen, int *sacks_seen);
507 static void
508 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len,
509 uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t ts,
510 struct rack_sendmap *hintrsm, uint32_t add_flags, struct mbuf *s_mb, uint32_t s_moff, int hw_tls, int segsiz);
511
512 static uint64_t rack_get_gp_est(struct tcp_rack *rack);
513
514
515 static void
516 rack_log_sack_passed(struct tcpcb *tp, struct tcp_rack *rack,
517 struct rack_sendmap *rsm, uint32_t cts);
518 static void rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rsm);
519 static int32_t rack_output(struct tcpcb *tp);
520
521 static uint32_t
522 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack,
523 struct sackblk *sack, struct tcpopt *to, struct rack_sendmap **prsm,
524 uint32_t cts, uint32_t segsiz);
525 static void rack_post_recovery(struct tcpcb *tp, uint32_t th_seq);
526 static void rack_remxt_tmr(struct tcpcb *tp);
527 static int rack_set_sockopt(struct tcpcb *tp, struct sockopt *sopt);
528 static void rack_set_state(struct tcpcb *tp, struct tcp_rack *rack);
529 static int32_t rack_stopall(struct tcpcb *tp);
530 static void rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line);
531 static uint32_t
532 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack,
533 struct rack_sendmap *rsm, uint64_t ts, int32_t * lenp, uint32_t add_flag, int segsiz);
534 static void
535 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
536 struct rack_sendmap *rsm, uint64_t ts, uint32_t add_flag, int segsiz);
537 static int
538 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack,
539 struct rack_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, tcp_seq th_ack);
540 static int32_t tcp_addrack(module_t mod, int32_t type, void *data);
541 static int
542 rack_do_close_wait(struct mbuf *m, struct tcphdr *th,
543 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
544 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
545
546 static int
547 rack_do_closing(struct mbuf *m, struct tcphdr *th,
548 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
549 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
550 static int
551 rack_do_established(struct mbuf *m, struct tcphdr *th,
552 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
553 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
554 static int
555 rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th,
556 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
557 int32_t tlen, uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos);
558 static int
559 rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th,
560 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
561 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
562 static int
563 rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th,
564 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
565 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
566 static int
567 rack_do_lastack(struct mbuf *m, struct tcphdr *th,
568 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
569 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
570 static int
571 rack_do_syn_recv(struct mbuf *m, struct tcphdr *th,
572 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
573 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
574 static int
575 rack_do_syn_sent(struct mbuf *m, struct tcphdr *th,
576 struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
577 int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
578 static void rack_chk_req_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts);
579 struct rack_sendmap *
580 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack,
581 uint32_t tsused);
582 static void tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt,
583 uint32_t len, uint32_t us_tim, int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt);
584 static void
585 tcp_rack_partialack(struct tcpcb *tp);
586 static int
587 rack_set_profile(struct tcp_rack *rack, int prof);
588 static void
589 rack_apply_deferred_options(struct tcp_rack *rack);
590
591 int32_t rack_clear_counter=0;
592
593 static uint64_t
rack_get_lt_bw(struct tcp_rack * rack)594 rack_get_lt_bw(struct tcp_rack *rack)
595 {
596 struct timeval tv;
597 uint64_t tim, bytes;
598
599 tim = rack->r_ctl.lt_bw_time;
600 bytes = rack->r_ctl.lt_bw_bytes;
601 if (rack->lt_bw_up) {
602 /* Include all the current bytes too */
603 microuptime(&tv);
604 bytes += (rack->rc_tp->snd_una - rack->r_ctl.lt_seq);
605 tim += (tcp_tv_to_lusec(&tv) - rack->r_ctl.lt_timemark);
606 }
607 if ((bytes != 0) && (tim != 0))
608 return ((bytes * (uint64_t)1000000) / tim);
609 else
610 return (0);
611 }
612
613 static void
rack_swap_beta_values(struct tcp_rack * rack,uint8_t flex8)614 rack_swap_beta_values(struct tcp_rack *rack, uint8_t flex8)
615 {
616 struct sockopt sopt;
617 struct cc_newreno_opts opt;
618 struct tcpcb *tp;
619 uint32_t old_beta;
620 uint32_t old_beta_ecn;
621 int error = 0, failed = 0;
622
623 tp = rack->rc_tp;
624 if (tp->t_cc == NULL) {
625 /* Tcb is leaving */
626 return;
627 }
628 rack->rc_pacing_cc_set = 1;
629 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) {
630 /* Not new-reno we can't play games with beta! */
631 failed = 1;
632 goto out;
633
634 }
635 if (CC_ALGO(tp)->ctl_output == NULL) {
636 /* Huh, not using new-reno so no swaps.? */
637 failed = 2;
638 goto out;
639 }
640 /* Get the current values out */
641 sopt.sopt_valsize = sizeof(struct cc_newreno_opts);
642 sopt.sopt_dir = SOPT_GET;
643 opt.name = CC_NEWRENO_BETA;
644 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt);
645 if (error) {
646 failed = 3;
647 goto out;
648 }
649 old_beta = opt.val;
650 opt.name = CC_NEWRENO_BETA_ECN;
651 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt);
652 if (error) {
653 failed = 4;
654 goto out;
655 }
656 old_beta_ecn = opt.val;
657
658 /* Now lets set in the values we have stored */
659 sopt.sopt_dir = SOPT_SET;
660 opt.name = CC_NEWRENO_BETA;
661 opt.val = rack->r_ctl.rc_saved_beta;
662 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt);
663 if (error) {
664 failed = 5;
665 goto out;
666 }
667 opt.name = CC_NEWRENO_BETA_ECN;
668 opt.val = rack->r_ctl.rc_saved_beta_ecn;
669 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt);
670 if (error) {
671 failed = 6;
672 goto out;
673 }
674 /* Save off the values for restoral */
675 rack->r_ctl.rc_saved_beta = old_beta;
676 rack->r_ctl.rc_saved_beta_ecn = old_beta_ecn;
677 out:
678 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
679 union tcp_log_stackspecific log;
680 struct timeval tv;
681 struct newreno *ptr;
682
683 ptr = ((struct newreno *)tp->t_ccv.cc_data);
684 memset(&log, 0, sizeof(log));
685 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
686 log.u_bbr.flex1 = ptr->beta;
687 log.u_bbr.flex2 = ptr->beta_ecn;
688 log.u_bbr.flex3 = ptr->newreno_flags;
689 log.u_bbr.flex4 = rack->r_ctl.rc_saved_beta;
690 log.u_bbr.flex5 = rack->r_ctl.rc_saved_beta_ecn;
691 log.u_bbr.flex6 = failed;
692 log.u_bbr.flex7 = rack->gp_ready;
693 log.u_bbr.flex7 <<= 1;
694 log.u_bbr.flex7 |= rack->use_fixed_rate;
695 log.u_bbr.flex7 <<= 1;
696 log.u_bbr.flex7 |= rack->rc_pacing_cc_set;
697 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
698 log.u_bbr.flex8 = flex8;
699 tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_CWND, error,
700 0, &log, false, NULL, NULL, 0, &tv);
701 }
702 }
703
704 static void
rack_set_cc_pacing(struct tcp_rack * rack)705 rack_set_cc_pacing(struct tcp_rack *rack)
706 {
707 if (rack->rc_pacing_cc_set)
708 return;
709 /*
710 * Use the swap utility placing in 3 for flex8 to id a
711 * set of a new set of values.
712 */
713 rack->rc_pacing_cc_set = 1;
714 rack_swap_beta_values(rack, 3);
715 }
716
717 static void
rack_undo_cc_pacing(struct tcp_rack * rack)718 rack_undo_cc_pacing(struct tcp_rack *rack)
719 {
720 if (rack->rc_pacing_cc_set == 0)
721 return;
722 /*
723 * Use the swap utility placing in 4 for flex8 to id a
724 * restoral of the old values.
725 */
726 rack->rc_pacing_cc_set = 0;
727 rack_swap_beta_values(rack, 4);
728 }
729
730 static void
rack_remove_pacing(struct tcp_rack * rack)731 rack_remove_pacing(struct tcp_rack *rack)
732 {
733 if (rack->rc_pacing_cc_set)
734 rack_undo_cc_pacing(rack);
735 if (rack->r_ctl.pacing_method & RACK_REG_PACING)
736 tcp_decrement_paced_conn();
737 if (rack->r_ctl.pacing_method & RACK_DGP_PACING)
738 tcp_dec_dgp_pacing_cnt();
739 rack->rc_always_pace = 0;
740 rack->r_ctl.pacing_method = RACK_PACING_NONE;
741 rack->dgp_on = 0;
742 rack->rc_hybrid_mode = 0;
743 rack->use_fixed_rate = 0;
744 }
745
746 static void
rack_log_gpset(struct tcp_rack * rack,uint32_t seq_end,uint32_t ack_end_t,uint32_t send_end_t,int line,uint8_t mode,struct rack_sendmap * rsm)747 rack_log_gpset(struct tcp_rack *rack, uint32_t seq_end, uint32_t ack_end_t,
748 uint32_t send_end_t, int line, uint8_t mode, struct rack_sendmap *rsm)
749 {
750 if (tcp_bblogging_on(rack->rc_tp) && (rack_verbose_logging != 0)) {
751 union tcp_log_stackspecific log;
752 struct timeval tv;
753
754 memset(&log, 0, sizeof(log));
755 log.u_bbr.flex1 = seq_end;
756 log.u_bbr.flex2 = rack->rc_tp->gput_seq;
757 log.u_bbr.flex3 = ack_end_t;
758 log.u_bbr.flex4 = rack->rc_tp->gput_ts;
759 log.u_bbr.flex5 = send_end_t;
760 log.u_bbr.flex6 = rack->rc_tp->gput_ack;
761 log.u_bbr.flex7 = mode;
762 log.u_bbr.flex8 = 69;
763 log.u_bbr.rttProp = rack->r_ctl.rc_gp_cumack_ts;
764 log.u_bbr.delRate = rack->r_ctl.rc_gp_output_ts;
765 log.u_bbr.pkts_out = line;
766 log.u_bbr.cwnd_gain = rack->app_limited_needs_set;
767 log.u_bbr.pkt_epoch = rack->r_ctl.rc_app_limited_cnt;
768 log.u_bbr.epoch = rack->r_ctl.current_round;
769 log.u_bbr.lt_epoch = rack->r_ctl.rc_considered_lost;
770 if (rsm != NULL) {
771 log.u_bbr.applimited = rsm->r_start;
772 log.u_bbr.delivered = rsm->r_end;
773 log.u_bbr.epoch = rsm->r_flags;
774 }
775 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
776 TCP_LOG_EVENTP(rack->rc_tp, NULL,
777 &rack->rc_inp->inp_socket->so_rcv,
778 &rack->rc_inp->inp_socket->so_snd,
779 BBR_LOG_HPTSI_CALC, 0,
780 0, &log, false, &tv);
781 }
782 }
783
784 static int
sysctl_rack_clear(SYSCTL_HANDLER_ARGS)785 sysctl_rack_clear(SYSCTL_HANDLER_ARGS)
786 {
787 uint32_t stat;
788 int32_t error;
789
790 error = SYSCTL_OUT(req, &rack_clear_counter, sizeof(uint32_t));
791 if (error || req->newptr == NULL)
792 return error;
793
794 error = SYSCTL_IN(req, &stat, sizeof(uint32_t));
795 if (error)
796 return (error);
797 if (stat == 1) {
798 #ifdef INVARIANTS
799 printf("Clearing RACK counters\n");
800 #endif
801 counter_u64_zero(rack_tlp_tot);
802 counter_u64_zero(rack_tlp_newdata);
803 counter_u64_zero(rack_tlp_retran);
804 counter_u64_zero(rack_tlp_retran_bytes);
805 counter_u64_zero(rack_to_tot);
806 counter_u64_zero(rack_saw_enobuf);
807 counter_u64_zero(rack_saw_enobuf_hw);
808 counter_u64_zero(rack_saw_enetunreach);
809 counter_u64_zero(rack_persists_sends);
810 counter_u64_zero(rack_total_bytes);
811 counter_u64_zero(rack_persists_acks);
812 counter_u64_zero(rack_persists_loss);
813 counter_u64_zero(rack_persists_lost_ends);
814 #ifdef INVARIANTS
815 counter_u64_zero(rack_adjust_map_bw);
816 #endif
817 counter_u64_zero(rack_to_alloc_hard);
818 counter_u64_zero(rack_to_alloc_emerg);
819 counter_u64_zero(rack_sack_proc_all);
820 counter_u64_zero(rack_fto_send);
821 counter_u64_zero(rack_fto_rsm_send);
822 counter_u64_zero(rack_extended_rfo);
823 counter_u64_zero(rack_hw_pace_init_fail);
824 counter_u64_zero(rack_hw_pace_lost);
825 counter_u64_zero(rack_non_fto_send);
826 counter_u64_zero(rack_nfto_resend);
827 counter_u64_zero(rack_sack_proc_short);
828 counter_u64_zero(rack_sack_proc_restart);
829 counter_u64_zero(rack_to_alloc);
830 counter_u64_zero(rack_to_alloc_limited);
831 counter_u64_zero(rack_alloc_limited_conns);
832 counter_u64_zero(rack_split_limited);
833 counter_u64_zero(rack_rxt_clamps_cwnd);
834 counter_u64_zero(rack_rxt_clamps_cwnd_uniq);
835 counter_u64_zero(rack_multi_single_eq);
836 counter_u64_zero(rack_proc_non_comp_ack);
837 counter_u64_zero(rack_sack_attacks_detected);
838 counter_u64_zero(rack_sack_attacks_reversed);
839 counter_u64_zero(rack_sack_attacks_suspect);
840 counter_u64_zero(rack_sack_used_next_merge);
841 counter_u64_zero(rack_sack_used_prev_merge);
842 counter_u64_zero(rack_sack_splits);
843 counter_u64_zero(rack_sack_skipped_acked);
844 counter_u64_zero(rack_ack_total);
845 counter_u64_zero(rack_express_sack);
846 counter_u64_zero(rack_sack_total);
847 counter_u64_zero(rack_move_none);
848 counter_u64_zero(rack_move_some);
849 counter_u64_zero(rack_try_scwnd);
850 counter_u64_zero(rack_collapsed_win);
851 counter_u64_zero(rack_collapsed_win_rxt);
852 counter_u64_zero(rack_collapsed_win_seen);
853 counter_u64_zero(rack_collapsed_win_rxt_bytes);
854 } else if (stat == 2) {
855 #ifdef INVARIANTS
856 printf("Clearing RACK option array\n");
857 #endif
858 COUNTER_ARRAY_ZERO(rack_opts_arry, RACK_OPTS_SIZE);
859 } else if (stat == 3) {
860 printf("Rack has no stats counters to clear (use 1 to clear all stats in sysctl node)\n");
861 } else if (stat == 4) {
862 #ifdef INVARIANTS
863 printf("Clearing RACK out size array\n");
864 #endif
865 COUNTER_ARRAY_ZERO(rack_out_size, TCP_MSS_ACCT_SIZE);
866 }
867 rack_clear_counter = 0;
868 return (0);
869 }
870
871 static void
rack_init_sysctls(void)872 rack_init_sysctls(void)
873 {
874 struct sysctl_oid *rack_counters;
875 struct sysctl_oid *rack_attack;
876 struct sysctl_oid *rack_pacing;
877 struct sysctl_oid *rack_timely;
878 struct sysctl_oid *rack_timers;
879 struct sysctl_oid *rack_tlp;
880 struct sysctl_oid *rack_misc;
881 struct sysctl_oid *rack_features;
882 struct sysctl_oid *rack_measure;
883 struct sysctl_oid *rack_probertt;
884 struct sysctl_oid *rack_hw_pacing;
885
886 rack_attack = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
887 SYSCTL_CHILDREN(rack_sysctl_root),
888 OID_AUTO,
889 "sack_attack",
890 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
891 "Rack Sack Attack Counters and Controls");
892 rack_counters = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
893 SYSCTL_CHILDREN(rack_sysctl_root),
894 OID_AUTO,
895 "stats",
896 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
897 "Rack Counters");
898 SYSCTL_ADD_S32(&rack_sysctl_ctx,
899 SYSCTL_CHILDREN(rack_sysctl_root),
900 OID_AUTO, "rate_sample_method", CTLFLAG_RW,
901 &rack_rate_sample_method , USE_RTT_LOW,
902 "What method should we use for rate sampling 0=high, 1=low ");
903 /* Probe rtt related controls */
904 rack_probertt = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
905 SYSCTL_CHILDREN(rack_sysctl_root),
906 OID_AUTO,
907 "probertt",
908 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
909 "ProbeRTT related Controls");
910 SYSCTL_ADD_U16(&rack_sysctl_ctx,
911 SYSCTL_CHILDREN(rack_probertt),
912 OID_AUTO, "exit_per_hpb", CTLFLAG_RW,
913 &rack_atexit_prtt_hbp, 130,
914 "What percentage above goodput do we clamp CA/SS to at exit on high-BDP path 110%");
915 SYSCTL_ADD_U16(&rack_sysctl_ctx,
916 SYSCTL_CHILDREN(rack_probertt),
917 OID_AUTO, "exit_per_nonhpb", CTLFLAG_RW,
918 &rack_atexit_prtt, 130,
919 "What percentage above goodput do we clamp CA/SS to at exit on a non high-BDP path 100%");
920 SYSCTL_ADD_U16(&rack_sysctl_ctx,
921 SYSCTL_CHILDREN(rack_probertt),
922 OID_AUTO, "gp_per_mul", CTLFLAG_RW,
923 &rack_per_of_gp_probertt, 60,
924 "What percentage of goodput do we pace at in probertt");
925 SYSCTL_ADD_U16(&rack_sysctl_ctx,
926 SYSCTL_CHILDREN(rack_probertt),
927 OID_AUTO, "gp_per_reduce", CTLFLAG_RW,
928 &rack_per_of_gp_probertt_reduce, 10,
929 "What percentage of goodput do we reduce every gp_srtt");
930 SYSCTL_ADD_U16(&rack_sysctl_ctx,
931 SYSCTL_CHILDREN(rack_probertt),
932 OID_AUTO, "gp_per_low", CTLFLAG_RW,
933 &rack_per_of_gp_lowthresh, 40,
934 "What percentage of goodput do we allow the multiplier to fall to");
935 SYSCTL_ADD_U32(&rack_sysctl_ctx,
936 SYSCTL_CHILDREN(rack_probertt),
937 OID_AUTO, "time_between", CTLFLAG_RW,
938 &rack_time_between_probertt, 96000000,
939 "How many useconds between the lowest rtt falling must past before we enter probertt");
940 SYSCTL_ADD_U32(&rack_sysctl_ctx,
941 SYSCTL_CHILDREN(rack_probertt),
942 OID_AUTO, "safety", CTLFLAG_RW,
943 &rack_probe_rtt_safety_val, 2000000,
944 "If not zero, provides a maximum usecond that you can stay in probertt (2sec = 2000000)");
945 SYSCTL_ADD_U32(&rack_sysctl_ctx,
946 SYSCTL_CHILDREN(rack_probertt),
947 OID_AUTO, "sets_cwnd", CTLFLAG_RW,
948 &rack_probe_rtt_sets_cwnd, 0,
949 "Do we set the cwnd too (if always_lower is on)");
950 SYSCTL_ADD_U32(&rack_sysctl_ctx,
951 SYSCTL_CHILDREN(rack_probertt),
952 OID_AUTO, "maxdrainsrtts", CTLFLAG_RW,
953 &rack_max_drain_wait, 2,
954 "Maximum number of gp_srtt's to hold in drain waiting for flight to reach goal");
955 SYSCTL_ADD_U32(&rack_sysctl_ctx,
956 SYSCTL_CHILDREN(rack_probertt),
957 OID_AUTO, "mustdrainsrtts", CTLFLAG_RW,
958 &rack_must_drain, 1,
959 "We must drain this many gp_srtt's waiting for flight to reach goal");
960 SYSCTL_ADD_U32(&rack_sysctl_ctx,
961 SYSCTL_CHILDREN(rack_probertt),
962 OID_AUTO, "goal_use_min_entry", CTLFLAG_RW,
963 &rack_probertt_use_min_rtt_entry, 1,
964 "Should we use the min-rtt to calculate the goal rtt (else gp_srtt) at entry");
965 SYSCTL_ADD_U32(&rack_sysctl_ctx,
966 SYSCTL_CHILDREN(rack_probertt),
967 OID_AUTO, "goal_use_min_exit", CTLFLAG_RW,
968 &rack_probertt_use_min_rtt_exit, 0,
969 "How to set cwnd at exit, 0 - dynamic, 1 - use min-rtt, 2 - use curgprtt, 3 - entry gp-rtt");
970 SYSCTL_ADD_U32(&rack_sysctl_ctx,
971 SYSCTL_CHILDREN(rack_probertt),
972 OID_AUTO, "length_div", CTLFLAG_RW,
973 &rack_probertt_gpsrtt_cnt_div, 0,
974 "How many recent goodput srtt periods plus hold tim does probertt last (bottom of fraction)");
975 SYSCTL_ADD_U32(&rack_sysctl_ctx,
976 SYSCTL_CHILDREN(rack_probertt),
977 OID_AUTO, "length_mul", CTLFLAG_RW,
978 &rack_probertt_gpsrtt_cnt_mul, 0,
979 "How many recent goodput srtt periods plus hold tim does probertt last (top of fraction)");
980 SYSCTL_ADD_U32(&rack_sysctl_ctx,
981 SYSCTL_CHILDREN(rack_probertt),
982 OID_AUTO, "holdtim_at_target", CTLFLAG_RW,
983 &rack_min_probertt_hold, 200000,
984 "What is the minimum time we hold probertt at target");
985 SYSCTL_ADD_U32(&rack_sysctl_ctx,
986 SYSCTL_CHILDREN(rack_probertt),
987 OID_AUTO, "filter_life", CTLFLAG_RW,
988 &rack_probertt_filter_life, 10000000,
989 "What is the time for the filters life in useconds");
990 SYSCTL_ADD_U32(&rack_sysctl_ctx,
991 SYSCTL_CHILDREN(rack_probertt),
992 OID_AUTO, "lower_within", CTLFLAG_RW,
993 &rack_probertt_lower_within, 10,
994 "If the rtt goes lower within this percentage of the time, go into probe-rtt");
995 SYSCTL_ADD_U32(&rack_sysctl_ctx,
996 SYSCTL_CHILDREN(rack_probertt),
997 OID_AUTO, "must_move", CTLFLAG_RW,
998 &rack_min_rtt_movement, 250,
999 "How much is the minimum movement in rtt to count as a drop for probertt purposes");
1000 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1001 SYSCTL_CHILDREN(rack_probertt),
1002 OID_AUTO, "clear_is_cnts", CTLFLAG_RW,
1003 &rack_probertt_clear_is, 1,
1004 "Do we clear I/S counts on exiting probe-rtt");
1005 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1006 SYSCTL_CHILDREN(rack_probertt),
1007 OID_AUTO, "hbp_extra_drain", CTLFLAG_RW,
1008 &rack_max_drain_hbp, 1,
1009 "How many extra drain gpsrtt's do we get in highly buffered paths");
1010 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1011 SYSCTL_CHILDREN(rack_probertt),
1012 OID_AUTO, "hbp_threshold", CTLFLAG_RW,
1013 &rack_hbp_thresh, 3,
1014 "We are highly buffered if min_rtt_seen / max_rtt_seen > this-threshold");
1015 /* Pacing related sysctls */
1016 rack_pacing = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1017 SYSCTL_CHILDREN(rack_sysctl_root),
1018 OID_AUTO,
1019 "pacing",
1020 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1021 "Pacing related Controls");
1022 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1023 SYSCTL_CHILDREN(rack_pacing),
1024 OID_AUTO, "pcm_enabled", CTLFLAG_RW,
1025 &rack_pcm_is_enabled, 1,
1026 "Do we by default do PCM measurements?");
1027 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1028 SYSCTL_CHILDREN(rack_pacing),
1029 OID_AUTO, "pcm_rnds", CTLFLAG_RW,
1030 &rack_pcm_every_n_rounds, 100,
1031 "How many rounds before we need to do a PCM measurement");
1032 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1033 SYSCTL_CHILDREN(rack_pacing),
1034 OID_AUTO, "pcm_blast", CTLFLAG_RW,
1035 &rack_pcm_blast, 0,
1036 "Blast out the full cwnd/rwnd when doing a PCM measurement");
1037 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1038 SYSCTL_CHILDREN(rack_pacing),
1039 OID_AUTO, "rnd_gp_gain", CTLFLAG_RW,
1040 &rack_gp_gain_req, 1200,
1041 "How much do we have to increase the GP to record the round 1200 = 120.0");
1042 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1043 SYSCTL_CHILDREN(rack_pacing),
1044 OID_AUTO, "dgp_out_of_ss_at", CTLFLAG_RW,
1045 &rack_rnd_cnt_req, 0x10005,
1046 "How many rounds less than rnd_gp_gain will drop us out of SS");
1047 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1048 SYSCTL_CHILDREN(rack_pacing),
1049 OID_AUTO, "no_timely", CTLFLAG_RW,
1050 &rack_timely_off, 0,
1051 "Do we not use timely in DGP?");
1052 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1053 SYSCTL_CHILDREN(rack_pacing),
1054 OID_AUTO, "fillcw", CTLFLAG_RW,
1055 &rack_fill_cw_state, 0,
1056 "Enable fillcw on new connections (default=0 off)?");
1057 SYSCTL_ADD_U16(&rack_sysctl_ctx,
1058 SYSCTL_CHILDREN(rack_pacing),
1059 OID_AUTO, "min_burst", CTLFLAG_RW,
1060 &rack_pacing_min_seg, 0,
1061 "What is the min burst size for pacing (0 disables)?");
1062 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1063 SYSCTL_CHILDREN(rack_pacing),
1064 OID_AUTO, "divisor", CTLFLAG_RW,
1065 &rack_default_pacing_divisor, 250,
1066 "What is the default divisor given to the rl code?");
1067 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1068 SYSCTL_CHILDREN(rack_pacing),
1069 OID_AUTO, "fillcw_max_mult", CTLFLAG_RW,
1070 &rack_bw_multipler, 0,
1071 "What is the limit multiplier of the current gp_est that fillcw can increase the b/w too, 200 == 200% (0 = off)?");
1072 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1073 SYSCTL_CHILDREN(rack_pacing),
1074 OID_AUTO, "max_pace_over", CTLFLAG_RW,
1075 &rack_max_per_above, 30,
1076 "What is the maximum allowable percentage that we can pace above (so 30 = 130% of our goal)");
1077 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1078 SYSCTL_CHILDREN(rack_pacing),
1079 OID_AUTO, "allow1mss", CTLFLAG_RW,
1080 &rack_pace_one_seg, 0,
1081 "Do we allow low b/w pacing of 1MSS instead of two (1.2Meg and less)?");
1082 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1083 SYSCTL_CHILDREN(rack_pacing),
1084 OID_AUTO, "limit_wsrtt", CTLFLAG_RW,
1085 &rack_limit_time_with_srtt, 0,
1086 "Do we limit pacing time based on srtt");
1087 SYSCTL_ADD_U16(&rack_sysctl_ctx,
1088 SYSCTL_CHILDREN(rack_pacing),
1089 OID_AUTO, "gp_per_ss", CTLFLAG_RW,
1090 &rack_per_of_gp_ss, 250,
1091 "If non zero, what percentage of goodput to pace at in slow start");
1092 SYSCTL_ADD_U16(&rack_sysctl_ctx,
1093 SYSCTL_CHILDREN(rack_pacing),
1094 OID_AUTO, "gp_per_ca", CTLFLAG_RW,
1095 &rack_per_of_gp_ca, 150,
1096 "If non zero, what percentage of goodput to pace at in congestion avoidance");
1097 SYSCTL_ADD_U16(&rack_sysctl_ctx,
1098 SYSCTL_CHILDREN(rack_pacing),
1099 OID_AUTO, "gp_per_rec", CTLFLAG_RW,
1100 &rack_per_of_gp_rec, 200,
1101 "If non zero, what percentage of goodput to pace at in recovery");
1102 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1103 SYSCTL_CHILDREN(rack_pacing),
1104 OID_AUTO, "pace_max_seg", CTLFLAG_RW,
1105 &rack_hptsi_segments, 40,
1106 "What size is the max for TSO segments in pacing and burst mitigation");
1107 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1108 SYSCTL_CHILDREN(rack_pacing),
1109 OID_AUTO, "burst_reduces", CTLFLAG_RW,
1110 &rack_pacing_delay_reduction, 4,
1111 "When doing only burst mitigation what is the reduce divisor");
1112 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1113 SYSCTL_CHILDREN(rack_sysctl_root),
1114 OID_AUTO, "use_pacing", CTLFLAG_RW,
1115 &rack_pace_every_seg, 0,
1116 "If set we use pacing, if clear we use only the original burst mitigation");
1117 SYSCTL_ADD_U64(&rack_sysctl_ctx,
1118 SYSCTL_CHILDREN(rack_pacing),
1119 OID_AUTO, "rate_cap", CTLFLAG_RW,
1120 &rack_bw_rate_cap, 0,
1121 "If set we apply this value to the absolute rate cap used by pacing");
1122 SYSCTL_ADD_U64(&rack_sysctl_ctx,
1123 SYSCTL_CHILDREN(rack_pacing),
1124 OID_AUTO, "fillcw_cap", CTLFLAG_RW,
1125 &rack_fillcw_bw_cap, 3750000,
1126 "Do we have an absolute cap on the amount of b/w fillcw can specify (0 = no)?");
1127 SYSCTL_ADD_U8(&rack_sysctl_ctx,
1128 SYSCTL_CHILDREN(rack_sysctl_root),
1129 OID_AUTO, "req_measure_cnt", CTLFLAG_RW,
1130 &rack_req_measurements, 1,
1131 "If doing dynamic pacing, how many measurements must be in before we start pacing?");
1132 /* Hardware pacing */
1133 rack_hw_pacing = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1134 SYSCTL_CHILDREN(rack_sysctl_root),
1135 OID_AUTO,
1136 "hdwr_pacing",
1137 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1138 "Pacing related Controls");
1139 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1140 SYSCTL_CHILDREN(rack_hw_pacing),
1141 OID_AUTO, "rwnd_factor", CTLFLAG_RW,
1142 &rack_hw_rwnd_factor, 2,
1143 "How many times does snd_wnd need to be bigger than pace_max_seg so we will hold off and get more acks?");
1144 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1145 SYSCTL_CHILDREN(rack_hw_pacing),
1146 OID_AUTO, "precheck", CTLFLAG_RW,
1147 &rack_hw_check_queue, 0,
1148 "Do we always precheck the hdwr pacing queue to avoid ENOBUF's?");
1149 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1150 SYSCTL_CHILDREN(rack_hw_pacing),
1151 OID_AUTO, "pace_enobuf_mult", CTLFLAG_RW,
1152 &rack_enobuf_hw_boost_mult, 0,
1153 "By how many time_betweens should we boost the pacing time if we see a ENOBUFS?");
1154 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1155 SYSCTL_CHILDREN(rack_hw_pacing),
1156 OID_AUTO, "pace_enobuf_max", CTLFLAG_RW,
1157 &rack_enobuf_hw_max, 2,
1158 "What is the max boost the pacing time if we see a ENOBUFS?");
1159 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1160 SYSCTL_CHILDREN(rack_hw_pacing),
1161 OID_AUTO, "pace_enobuf_min", CTLFLAG_RW,
1162 &rack_enobuf_hw_min, 2,
1163 "What is the min boost the pacing time if we see a ENOBUFS?");
1164 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1165 SYSCTL_CHILDREN(rack_hw_pacing),
1166 OID_AUTO, "enable", CTLFLAG_RW,
1167 &rack_enable_hw_pacing, 0,
1168 "Should RACK attempt to use hw pacing?");
1169 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1170 SYSCTL_CHILDREN(rack_hw_pacing),
1171 OID_AUTO, "rate_cap", CTLFLAG_RW,
1172 &rack_hw_rate_caps, 0,
1173 "Does the highest hardware pacing rate cap the rate we will send at??");
1174 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1175 SYSCTL_CHILDREN(rack_hw_pacing),
1176 OID_AUTO, "uncap_per", CTLFLAG_RW,
1177 &rack_hw_rate_cap_per, 0,
1178 "If you go over b/w by this amount you will be uncapped (0 = never)");
1179 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1180 SYSCTL_CHILDREN(rack_hw_pacing),
1181 OID_AUTO, "rate_min", CTLFLAG_RW,
1182 &rack_hw_rate_min, 0,
1183 "Do we need a minimum estimate of this many bytes per second in order to engage hw pacing?");
1184 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1185 SYSCTL_CHILDREN(rack_hw_pacing),
1186 OID_AUTO, "rate_to_low", CTLFLAG_RW,
1187 &rack_hw_rate_to_low, 0,
1188 "If we fall below this rate, dis-engage hw pacing?");
1189 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1190 SYSCTL_CHILDREN(rack_hw_pacing),
1191 OID_AUTO, "up_only", CTLFLAG_RW,
1192 &rack_hw_up_only, 0,
1193 "Do we allow hw pacing to lower the rate selected?");
1194 rack_timely = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1195 SYSCTL_CHILDREN(rack_sysctl_root),
1196 OID_AUTO,
1197 "timely",
1198 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1199 "Rack Timely RTT Controls");
1200 /* Timely based GP dynmics */
1201 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1202 SYSCTL_CHILDREN(rack_timely),
1203 OID_AUTO, "upper", CTLFLAG_RW,
1204 &rack_gp_per_bw_mul_up, 2,
1205 "Rack timely upper range for equal b/w (in percentage)");
1206 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1207 SYSCTL_CHILDREN(rack_timely),
1208 OID_AUTO, "lower", CTLFLAG_RW,
1209 &rack_gp_per_bw_mul_down, 4,
1210 "Rack timely lower range for equal b/w (in percentage)");
1211 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1212 SYSCTL_CHILDREN(rack_timely),
1213 OID_AUTO, "rtt_max_mul", CTLFLAG_RW,
1214 &rack_gp_rtt_maxmul, 3,
1215 "Rack timely multiplier of lowest rtt for rtt_max");
1216 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1217 SYSCTL_CHILDREN(rack_timely),
1218 OID_AUTO, "rtt_min_div", CTLFLAG_RW,
1219 &rack_gp_rtt_mindiv, 4,
1220 "Rack timely divisor used for rtt + (rtt * mul/divisor) for check for lower rtt");
1221 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1222 SYSCTL_CHILDREN(rack_timely),
1223 OID_AUTO, "rtt_min_mul", CTLFLAG_RW,
1224 &rack_gp_rtt_minmul, 1,
1225 "Rack timely multiplier used for rtt + (rtt * mul/divisor) for check for lower rtt");
1226 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1227 SYSCTL_CHILDREN(rack_timely),
1228 OID_AUTO, "decrease", CTLFLAG_RW,
1229 &rack_gp_decrease_per, 80,
1230 "Rack timely Beta value 80 = .8 (scaled by 100)");
1231 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1232 SYSCTL_CHILDREN(rack_timely),
1233 OID_AUTO, "increase", CTLFLAG_RW,
1234 &rack_gp_increase_per, 2,
1235 "Rack timely increase perentage of our GP multiplication factor");
1236 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1237 SYSCTL_CHILDREN(rack_timely),
1238 OID_AUTO, "lowerbound", CTLFLAG_RW,
1239 &rack_per_lower_bound, 50,
1240 "Rack timely lowest percentage we allow GP multiplier to fall to");
1241 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1242 SYSCTL_CHILDREN(rack_timely),
1243 OID_AUTO, "p5_upper", CTLFLAG_RW,
1244 &rack_gain_p5_ub, 250,
1245 "Profile 5 upper bound to timely gain");
1246
1247 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1248 SYSCTL_CHILDREN(rack_timely),
1249 OID_AUTO, "upperboundss", CTLFLAG_RW,
1250 &rack_per_upper_bound_ss, 0,
1251 "Rack timely highest percentage we allow GP multiplier in SS to raise to (0 is no upperbound)");
1252 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1253 SYSCTL_CHILDREN(rack_timely),
1254 OID_AUTO, "upperboundca", CTLFLAG_RW,
1255 &rack_per_upper_bound_ca, 0,
1256 "Rack timely highest percentage we allow GP multiplier to CA raise to (0 is no upperbound)");
1257 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1258 SYSCTL_CHILDREN(rack_timely),
1259 OID_AUTO, "dynamicgp", CTLFLAG_RW,
1260 &rack_do_dyn_mul, 0,
1261 "Rack timely do we enable dynmaic timely goodput by default");
1262 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1263 SYSCTL_CHILDREN(rack_timely),
1264 OID_AUTO, "no_rec_red", CTLFLAG_RW,
1265 &rack_gp_no_rec_chg, 1,
1266 "Rack timely do we prohibit the recovery multiplier from being lowered");
1267 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1268 SYSCTL_CHILDREN(rack_timely),
1269 OID_AUTO, "red_clear_cnt", CTLFLAG_RW,
1270 &rack_timely_dec_clear, 6,
1271 "Rack timely what threshold do we count to before another boost during b/w decent");
1272 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1273 SYSCTL_CHILDREN(rack_timely),
1274 OID_AUTO, "max_push_rise", CTLFLAG_RW,
1275 &rack_timely_max_push_rise, 3,
1276 "Rack timely how many times do we push up with b/w increase");
1277 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1278 SYSCTL_CHILDREN(rack_timely),
1279 OID_AUTO, "max_push_drop", CTLFLAG_RW,
1280 &rack_timely_max_push_drop, 3,
1281 "Rack timely how many times do we push back on b/w decent");
1282 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1283 SYSCTL_CHILDREN(rack_timely),
1284 OID_AUTO, "min_segs", CTLFLAG_RW,
1285 &rack_timely_min_segs, 4,
1286 "Rack timely when setting the cwnd what is the min num segments");
1287 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1288 SYSCTL_CHILDREN(rack_timely),
1289 OID_AUTO, "nonstop", CTLFLAG_RW,
1290 &rack_timely_no_stopping, 0,
1291 "Rack timely don't stop increase");
1292 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1293 SYSCTL_CHILDREN(rack_timely),
1294 OID_AUTO, "dec_raise_thresh", CTLFLAG_RW,
1295 &rack_down_raise_thresh, 100,
1296 "If the CA or SS is below this threshold raise on the first 3 b/w lowers (0=always)");
1297 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1298 SYSCTL_CHILDREN(rack_timely),
1299 OID_AUTO, "bottom_drag_segs", CTLFLAG_RW,
1300 &rack_req_segs, 1,
1301 "Bottom dragging if not these many segments outstanding and room");
1302
1303 /* TLP and Rack related parameters */
1304 rack_tlp = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1305 SYSCTL_CHILDREN(rack_sysctl_root),
1306 OID_AUTO,
1307 "tlp",
1308 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1309 "TLP and Rack related Controls");
1310 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1311 SYSCTL_CHILDREN(rack_tlp),
1312 OID_AUTO, "use_rrr", CTLFLAG_RW,
1313 &use_rack_rr, 1,
1314 "Do we use Rack Rapid Recovery");
1315 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1316 SYSCTL_CHILDREN(rack_tlp),
1317 OID_AUTO, "post_rec_labc", CTLFLAG_RW,
1318 &rack_max_abc_post_recovery, 2,
1319 "Since we do early recovery, do we override the l_abc to a value, if so what?");
1320 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1321 SYSCTL_CHILDREN(rack_tlp),
1322 OID_AUTO, "nonrxt_use_cr", CTLFLAG_RW,
1323 &rack_non_rxt_use_cr, 0,
1324 "Do we use ss/ca rate if in recovery we are transmitting a new data chunk");
1325 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1326 SYSCTL_CHILDREN(rack_tlp),
1327 OID_AUTO, "tlpmethod", CTLFLAG_RW,
1328 &rack_tlp_threshold_use, TLP_USE_TWO_ONE,
1329 "What method do we do for TLP time calc 0=no-de-ack-comp, 1=ID, 2=2.1, 3=2.2");
1330 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1331 SYSCTL_CHILDREN(rack_tlp),
1332 OID_AUTO, "limit", CTLFLAG_RW,
1333 &rack_tlp_limit, 2,
1334 "How many TLP's can be sent without sending new data");
1335 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1336 SYSCTL_CHILDREN(rack_tlp),
1337 OID_AUTO, "use_greater", CTLFLAG_RW,
1338 &rack_tlp_use_greater, 1,
1339 "Should we use the rack_rtt time if its greater than srtt");
1340 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1341 SYSCTL_CHILDREN(rack_tlp),
1342 OID_AUTO, "tlpminto", CTLFLAG_RW,
1343 &rack_tlp_min, 10000,
1344 "TLP minimum timeout per the specification (in microseconds)");
1345 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1346 SYSCTL_CHILDREN(rack_tlp),
1347 OID_AUTO, "send_oldest", CTLFLAG_RW,
1348 &rack_always_send_oldest, 0,
1349 "Should we always send the oldest TLP and RACK-TLP");
1350 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1351 SYSCTL_CHILDREN(rack_tlp),
1352 OID_AUTO, "tlp_cwnd_flag", CTLFLAG_RW,
1353 &rack_lower_cwnd_at_tlp, 0,
1354 "When a TLP completes a retran should we enter recovery");
1355 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1356 SYSCTL_CHILDREN(rack_tlp),
1357 OID_AUTO, "reorder_thresh", CTLFLAG_RW,
1358 &rack_reorder_thresh, 2,
1359 "What factor for rack will be added when seeing reordering (shift right)");
1360 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1361 SYSCTL_CHILDREN(rack_tlp),
1362 OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW,
1363 &rack_tlp_thresh, 1,
1364 "What divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)");
1365 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1366 SYSCTL_CHILDREN(rack_tlp),
1367 OID_AUTO, "reorder_fade", CTLFLAG_RW,
1368 &rack_reorder_fade, 60000000,
1369 "Does reorder detection fade, if so how many microseconds (0 means never)");
1370 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1371 SYSCTL_CHILDREN(rack_tlp),
1372 OID_AUTO, "pktdelay", CTLFLAG_RW,
1373 &rack_pkt_delay, 1000,
1374 "Extra RACK time (in microseconds) besides reordering thresh");
1375
1376 /* Timer related controls */
1377 rack_timers = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1378 SYSCTL_CHILDREN(rack_sysctl_root),
1379 OID_AUTO,
1380 "timers",
1381 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1382 "Timer related controls");
1383 SYSCTL_ADD_U8(&rack_sysctl_ctx,
1384 SYSCTL_CHILDREN(rack_timers),
1385 OID_AUTO, "reset_ssth_rec_rto", CTLFLAG_RW,
1386 &rack_ssthresh_rest_rto_rec, 0,
1387 "When doing recovery -> rto -> recovery do we reset SSthresh?");
1388 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1389 SYSCTL_CHILDREN(rack_timers),
1390 OID_AUTO, "scoreboard_thresh", CTLFLAG_RW,
1391 &rack_rxt_scoreboard_clear_thresh, 2,
1392 "How many RTO's are allowed before we clear the scoreboard");
1393 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1394 SYSCTL_CHILDREN(rack_timers),
1395 OID_AUTO, "honor_hpts_min", CTLFLAG_RW,
1396 &rack_honors_hpts_min_to, 1,
1397 "Do rack pacing timers honor hpts min timeout");
1398 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1399 SYSCTL_CHILDREN(rack_timers),
1400 OID_AUTO, "hpts_max_reduce", CTLFLAG_RW,
1401 &rack_max_reduce, 10,
1402 "Max percentage we will reduce pacing delay by for pacing when we are behind");
1403 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1404 SYSCTL_CHILDREN(rack_timers),
1405 OID_AUTO, "persmin", CTLFLAG_RW,
1406 &rack_persist_min, 250000,
1407 "What is the minimum time in microseconds between persists");
1408 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1409 SYSCTL_CHILDREN(rack_timers),
1410 OID_AUTO, "persmax", CTLFLAG_RW,
1411 &rack_persist_max, 2000000,
1412 "What is the largest delay in microseconds between persists");
1413 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1414 SYSCTL_CHILDREN(rack_timers),
1415 OID_AUTO, "delayed_ack", CTLFLAG_RW,
1416 &rack_delayed_ack_time, 40000,
1417 "Delayed ack time (40ms in microseconds)");
1418 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1419 SYSCTL_CHILDREN(rack_timers),
1420 OID_AUTO, "minrto", CTLFLAG_RW,
1421 &rack_rto_min, 30000,
1422 "Minimum RTO in microseconds -- set with caution below 1000 due to TLP");
1423 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1424 SYSCTL_CHILDREN(rack_timers),
1425 OID_AUTO, "maxrto", CTLFLAG_RW,
1426 &rack_rto_max, 4000000,
1427 "Maximum RTO in microseconds -- should be at least as large as min_rto");
1428 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1429 SYSCTL_CHILDREN(rack_timers),
1430 OID_AUTO, "minto", CTLFLAG_RW,
1431 &rack_min_to, 1000,
1432 "Minimum rack timeout in microseconds");
1433 /* Measure controls */
1434 rack_measure = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1435 SYSCTL_CHILDREN(rack_sysctl_root),
1436 OID_AUTO,
1437 "measure",
1438 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1439 "Measure related controls");
1440 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1441 SYSCTL_CHILDREN(rack_measure),
1442 OID_AUTO, "wma_divisor", CTLFLAG_RW,
1443 &rack_wma_divisor, 8,
1444 "When doing b/w calculation what is the divisor for the WMA");
1445 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1446 SYSCTL_CHILDREN(rack_measure),
1447 OID_AUTO, "end_cwnd", CTLFLAG_RW,
1448 &rack_cwnd_block_ends_measure, 0,
1449 "Does a cwnd just-return end the measurement window (app limited)");
1450 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1451 SYSCTL_CHILDREN(rack_measure),
1452 OID_AUTO, "end_rwnd", CTLFLAG_RW,
1453 &rack_rwnd_block_ends_measure, 0,
1454 "Does an rwnd just-return end the measurement window (app limited -- not persists)");
1455 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1456 SYSCTL_CHILDREN(rack_measure),
1457 OID_AUTO, "min_target", CTLFLAG_RW,
1458 &rack_def_data_window, 20,
1459 "What is the minimum target window (in mss) for a GP measurements");
1460 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1461 SYSCTL_CHILDREN(rack_measure),
1462 OID_AUTO, "goal_bdp", CTLFLAG_RW,
1463 &rack_goal_bdp, 2,
1464 "What is the goal BDP to measure");
1465 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1466 SYSCTL_CHILDREN(rack_measure),
1467 OID_AUTO, "min_srtts", CTLFLAG_RW,
1468 &rack_min_srtts, 1,
1469 "What is the goal BDP to measure");
1470 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1471 SYSCTL_CHILDREN(rack_measure),
1472 OID_AUTO, "min_measure_tim", CTLFLAG_RW,
1473 &rack_min_measure_usec, 0,
1474 "What is the Minimum time time for a measurement if 0, this is off");
1475 /* Features */
1476 rack_features = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1477 SYSCTL_CHILDREN(rack_sysctl_root),
1478 OID_AUTO,
1479 "features",
1480 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1481 "Feature controls");
1482 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1483 SYSCTL_CHILDREN(rack_features),
1484 OID_AUTO, "hybrid_set_maxseg", CTLFLAG_RW,
1485 &rack_hybrid_allow_set_maxseg, 0,
1486 "Should hybrid pacing allow the setmss command");
1487 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1488 SYSCTL_CHILDREN(rack_features),
1489 OID_AUTO, "cmpack", CTLFLAG_RW,
1490 &rack_use_cmp_acks, 1,
1491 "Should RACK have LRO send compressed acks");
1492 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1493 SYSCTL_CHILDREN(rack_features),
1494 OID_AUTO, "fsb", CTLFLAG_RW,
1495 &rack_use_fsb, 1,
1496 "Should RACK use the fast send block?");
1497 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1498 SYSCTL_CHILDREN(rack_features),
1499 OID_AUTO, "rfo", CTLFLAG_RW,
1500 &rack_use_rfo, 1,
1501 "Should RACK use rack_fast_output()?");
1502 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1503 SYSCTL_CHILDREN(rack_features),
1504 OID_AUTO, "rsmrfo", CTLFLAG_RW,
1505 &rack_use_rsm_rfo, 1,
1506 "Should RACK use rack_fast_rsm_output()?");
1507 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1508 SYSCTL_CHILDREN(rack_features),
1509 OID_AUTO, "non_paced_lro_queue", CTLFLAG_RW,
1510 &rack_enable_mqueue_for_nonpaced, 0,
1511 "Should RACK use mbuf queuing for non-paced connections");
1512 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1513 SYSCTL_CHILDREN(rack_features),
1514 OID_AUTO, "hystartplusplus", CTLFLAG_RW,
1515 &rack_do_hystart, 0,
1516 "Should RACK enable HyStart++ on connections?");
1517 /* Misc rack controls */
1518 rack_misc = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1519 SYSCTL_CHILDREN(rack_sysctl_root),
1520 OID_AUTO,
1521 "misc",
1522 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1523 "Misc related controls");
1524 #ifdef TCP_ACCOUNTING
1525 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1526 SYSCTL_CHILDREN(rack_misc),
1527 OID_AUTO, "tcp_acct", CTLFLAG_RW,
1528 &rack_tcp_accounting, 0,
1529 "Should we turn on TCP accounting for all rack sessions?");
1530 #endif
1531 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1532 SYSCTL_CHILDREN(rack_misc),
1533 OID_AUTO, "dnd", CTLFLAG_RW,
1534 &rack_dnd_default, 0,
1535 "Do not disturb default for rack_rrr = 3");
1536 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1537 SYSCTL_CHILDREN(rack_misc),
1538 OID_AUTO, "sad_seg_per", CTLFLAG_RW,
1539 &sad_seg_size_per, 800,
1540 "Percentage of segment size needed in a sack 800 = 80.0?");
1541 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1542 SYSCTL_CHILDREN(rack_misc),
1543 OID_AUTO, "rxt_controls", CTLFLAG_RW,
1544 &rack_rxt_controls, 0,
1545 "Retransmit sending size controls (valid values 0, 1, 2 default=1)?");
1546 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1547 SYSCTL_CHILDREN(rack_misc),
1548 OID_AUTO, "rack_hibeta", CTLFLAG_RW,
1549 &rack_hibeta_setting, 0,
1550 "Do we ue a high beta (80 instead of 50)?");
1551 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1552 SYSCTL_CHILDREN(rack_misc),
1553 OID_AUTO, "apply_rtt_with_low_conf", CTLFLAG_RW,
1554 &rack_apply_rtt_with_reduced_conf, 0,
1555 "When a persist or keep-alive probe is not answered do we calculate rtt on subsequent answers?");
1556 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1557 SYSCTL_CHILDREN(rack_misc),
1558 OID_AUTO, "rack_dsack_ctl", CTLFLAG_RW,
1559 &rack_dsack_std_based, 3,
1560 "How do we process dsack with respect to rack timers, bit field, 3 is standards based?");
1561 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1562 SYSCTL_CHILDREN(rack_misc),
1563 OID_AUTO, "prr_addback_max", CTLFLAG_RW,
1564 &rack_prr_addbackmax, 2,
1565 "What is the maximum number of MSS we allow to be added back if prr can't send all its data?");
1566 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1567 SYSCTL_CHILDREN(rack_misc),
1568 OID_AUTO, "stats_gets_ms", CTLFLAG_RW,
1569 &rack_stats_gets_ms_rtt, 1,
1570 "What do we feed the stats framework (1 = ms_rtt, 0 = us_rtt, 2 = ms_rtt from hdwr, > 2 usec rtt from hdwr)?");
1571 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1572 SYSCTL_CHILDREN(rack_misc),
1573 OID_AUTO, "clientlowbuf", CTLFLAG_RW,
1574 &rack_client_low_buf, 0,
1575 "Client low buffer level (below this we are more aggressive in DGP exiting recovery (0 = off)?");
1576 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1577 SYSCTL_CHILDREN(rack_misc),
1578 OID_AUTO, "defprofile", CTLFLAG_RW,
1579 &rack_def_profile, 0,
1580 "Should RACK use a default profile (0=no, num == profile num)?");
1581 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1582 SYSCTL_CHILDREN(rack_misc),
1583 OID_AUTO, "shared_cwnd", CTLFLAG_RW,
1584 &rack_enable_shared_cwnd, 1,
1585 "Should RACK try to use the shared cwnd on connections where allowed");
1586 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1587 SYSCTL_CHILDREN(rack_misc),
1588 OID_AUTO, "limits_on_scwnd", CTLFLAG_RW,
1589 &rack_limits_scwnd, 1,
1590 "Should RACK place low end time limits on the shared cwnd feature");
1591 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1592 SYSCTL_CHILDREN(rack_misc),
1593 OID_AUTO, "no_prr", CTLFLAG_RW,
1594 &rack_disable_prr, 0,
1595 "Should RACK not use prr and only pace (must have pacing on)");
1596 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1597 SYSCTL_CHILDREN(rack_misc),
1598 OID_AUTO, "bb_verbose", CTLFLAG_RW,
1599 &rack_verbose_logging, 0,
1600 "Should RACK black box logging be verbose");
1601 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1602 SYSCTL_CHILDREN(rack_misc),
1603 OID_AUTO, "data_after_close", CTLFLAG_RW,
1604 &rack_ignore_data_after_close, 1,
1605 "Do we hold off sending a RST until all pending data is ack'd");
1606 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1607 SYSCTL_CHILDREN(rack_misc),
1608 OID_AUTO, "no_sack_needed", CTLFLAG_RW,
1609 &rack_sack_not_required, 1,
1610 "Do we allow rack to run on connections not supporting SACK");
1611 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1612 SYSCTL_CHILDREN(rack_misc),
1613 OID_AUTO, "prr_sendalot", CTLFLAG_RW,
1614 &rack_send_a_lot_in_prr, 1,
1615 "Send a lot in prr");
1616 SYSCTL_ADD_S32(&rack_sysctl_ctx,
1617 SYSCTL_CHILDREN(rack_misc),
1618 OID_AUTO, "autoscale", CTLFLAG_RW,
1619 &rack_autosndbuf_inc, 20,
1620 "What percentage should rack scale up its snd buffer by?");
1621
1622
1623 /* Sack Attacker detection stuff */
1624 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1625 SYSCTL_CHILDREN(rack_attack),
1626 OID_AUTO, "merge_out", CTLFLAG_RW,
1627 &rack_merge_out_sacks_on_attack, 0,
1628 "Do we merge the sendmap when we decide we are being attacked?");
1629
1630 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1631 SYSCTL_CHILDREN(rack_attack),
1632 OID_AUTO, "detect_highsackratio", CTLFLAG_RW,
1633 &rack_highest_sack_thresh_seen, 0,
1634 "Highest sack to ack ratio seen");
1635 SYSCTL_ADD_U32(&rack_sysctl_ctx,
1636 SYSCTL_CHILDREN(rack_attack),
1637 OID_AUTO, "detect_highmoveratio", CTLFLAG_RW,
1638 &rack_highest_move_thresh_seen, 0,
1639 "Highest move to non-move ratio seen");
1640 rack_ack_total = counter_u64_alloc(M_WAITOK);
1641 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1642 SYSCTL_CHILDREN(rack_attack),
1643 OID_AUTO, "acktotal", CTLFLAG_RD,
1644 &rack_ack_total,
1645 "Total number of Ack's");
1646 rack_express_sack = counter_u64_alloc(M_WAITOK);
1647 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1648 SYSCTL_CHILDREN(rack_attack),
1649 OID_AUTO, "exp_sacktotal", CTLFLAG_RD,
1650 &rack_express_sack,
1651 "Total expresss number of Sack's");
1652 rack_sack_total = counter_u64_alloc(M_WAITOK);
1653 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1654 SYSCTL_CHILDREN(rack_attack),
1655 OID_AUTO, "sacktotal", CTLFLAG_RD,
1656 &rack_sack_total,
1657 "Total number of SACKs");
1658 rack_move_none = counter_u64_alloc(M_WAITOK);
1659 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1660 SYSCTL_CHILDREN(rack_attack),
1661 OID_AUTO, "move_none", CTLFLAG_RD,
1662 &rack_move_none,
1663 "Total number of SACK index reuse of positions under threshold");
1664 rack_move_some = counter_u64_alloc(M_WAITOK);
1665 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1666 SYSCTL_CHILDREN(rack_attack),
1667 OID_AUTO, "move_some", CTLFLAG_RD,
1668 &rack_move_some,
1669 "Total number of SACK index reuse of positions over threshold");
1670 rack_sack_attacks_detected = counter_u64_alloc(M_WAITOK);
1671 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1672 SYSCTL_CHILDREN(rack_attack),
1673 OID_AUTO, "attacks", CTLFLAG_RD,
1674 &rack_sack_attacks_detected,
1675 "Total number of SACK attackers that had sack disabled");
1676 rack_sack_attacks_reversed = counter_u64_alloc(M_WAITOK);
1677 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1678 SYSCTL_CHILDREN(rack_attack),
1679 OID_AUTO, "reversed", CTLFLAG_RD,
1680 &rack_sack_attacks_reversed,
1681 "Total number of SACK attackers that were later determined false positive");
1682 rack_sack_attacks_suspect = counter_u64_alloc(M_WAITOK);
1683 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1684 SYSCTL_CHILDREN(rack_attack),
1685 OID_AUTO, "suspect", CTLFLAG_RD,
1686 &rack_sack_attacks_suspect,
1687 "Total number of SACKs that triggered early detection");
1688
1689 rack_sack_used_next_merge = counter_u64_alloc(M_WAITOK);
1690 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1691 SYSCTL_CHILDREN(rack_attack),
1692 OID_AUTO, "nextmerge", CTLFLAG_RD,
1693 &rack_sack_used_next_merge,
1694 "Total number of times we used the next merge");
1695 rack_sack_used_prev_merge = counter_u64_alloc(M_WAITOK);
1696 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1697 SYSCTL_CHILDREN(rack_attack),
1698 OID_AUTO, "prevmerge", CTLFLAG_RD,
1699 &rack_sack_used_prev_merge,
1700 "Total number of times we used the prev merge");
1701 /* Counters */
1702 rack_total_bytes = counter_u64_alloc(M_WAITOK);
1703 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1704 SYSCTL_CHILDREN(rack_counters),
1705 OID_AUTO, "totalbytes", CTLFLAG_RD,
1706 &rack_total_bytes,
1707 "Total number of bytes sent");
1708 rack_fto_send = counter_u64_alloc(M_WAITOK);
1709 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1710 SYSCTL_CHILDREN(rack_counters),
1711 OID_AUTO, "fto_send", CTLFLAG_RD,
1712 &rack_fto_send, "Total number of rack_fast_output sends");
1713 rack_fto_rsm_send = counter_u64_alloc(M_WAITOK);
1714 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1715 SYSCTL_CHILDREN(rack_counters),
1716 OID_AUTO, "fto_rsm_send", CTLFLAG_RD,
1717 &rack_fto_rsm_send, "Total number of rack_fast_rsm_output sends");
1718 rack_nfto_resend = counter_u64_alloc(M_WAITOK);
1719 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1720 SYSCTL_CHILDREN(rack_counters),
1721 OID_AUTO, "nfto_resend", CTLFLAG_RD,
1722 &rack_nfto_resend, "Total number of rack_output retransmissions");
1723 rack_non_fto_send = counter_u64_alloc(M_WAITOK);
1724 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1725 SYSCTL_CHILDREN(rack_counters),
1726 OID_AUTO, "nfto_send", CTLFLAG_RD,
1727 &rack_non_fto_send, "Total number of rack_output first sends");
1728 rack_extended_rfo = counter_u64_alloc(M_WAITOK);
1729 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1730 SYSCTL_CHILDREN(rack_counters),
1731 OID_AUTO, "rfo_extended", CTLFLAG_RD,
1732 &rack_extended_rfo, "Total number of times we extended rfo");
1733
1734 rack_hw_pace_init_fail = counter_u64_alloc(M_WAITOK);
1735 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1736 SYSCTL_CHILDREN(rack_counters),
1737 OID_AUTO, "hwpace_init_fail", CTLFLAG_RD,
1738 &rack_hw_pace_init_fail, "Total number of times we failed to initialize hw pacing");
1739 rack_hw_pace_lost = counter_u64_alloc(M_WAITOK);
1740
1741 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1742 SYSCTL_CHILDREN(rack_counters),
1743 OID_AUTO, "hwpace_lost", CTLFLAG_RD,
1744 &rack_hw_pace_lost, "Total number of times we failed to initialize hw pacing");
1745 rack_tlp_tot = counter_u64_alloc(M_WAITOK);
1746 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1747 SYSCTL_CHILDREN(rack_counters),
1748 OID_AUTO, "tlp_to_total", CTLFLAG_RD,
1749 &rack_tlp_tot,
1750 "Total number of tail loss probe expirations");
1751 rack_tlp_newdata = counter_u64_alloc(M_WAITOK);
1752 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1753 SYSCTL_CHILDREN(rack_counters),
1754 OID_AUTO, "tlp_new", CTLFLAG_RD,
1755 &rack_tlp_newdata,
1756 "Total number of tail loss probe sending new data");
1757 rack_tlp_retran = counter_u64_alloc(M_WAITOK);
1758 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1759 SYSCTL_CHILDREN(rack_counters),
1760 OID_AUTO, "tlp_retran", CTLFLAG_RD,
1761 &rack_tlp_retran,
1762 "Total number of tail loss probe sending retransmitted data");
1763 rack_tlp_retran_bytes = counter_u64_alloc(M_WAITOK);
1764 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1765 SYSCTL_CHILDREN(rack_counters),
1766 OID_AUTO, "tlp_retran_bytes", CTLFLAG_RD,
1767 &rack_tlp_retran_bytes,
1768 "Total bytes of tail loss probe sending retransmitted data");
1769 rack_to_tot = counter_u64_alloc(M_WAITOK);
1770 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1771 SYSCTL_CHILDREN(rack_counters),
1772 OID_AUTO, "rack_to_tot", CTLFLAG_RD,
1773 &rack_to_tot,
1774 "Total number of times the rack to expired");
1775 rack_saw_enobuf = counter_u64_alloc(M_WAITOK);
1776 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1777 SYSCTL_CHILDREN(rack_counters),
1778 OID_AUTO, "saw_enobufs", CTLFLAG_RD,
1779 &rack_saw_enobuf,
1780 "Total number of times a sends returned enobuf for non-hdwr paced connections");
1781 rack_saw_enobuf_hw = counter_u64_alloc(M_WAITOK);
1782 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1783 SYSCTL_CHILDREN(rack_counters),
1784 OID_AUTO, "saw_enobufs_hw", CTLFLAG_RD,
1785 &rack_saw_enobuf_hw,
1786 "Total number of times a send returned enobuf for hdwr paced connections");
1787 rack_saw_enetunreach = counter_u64_alloc(M_WAITOK);
1788 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1789 SYSCTL_CHILDREN(rack_counters),
1790 OID_AUTO, "saw_enetunreach", CTLFLAG_RD,
1791 &rack_saw_enetunreach,
1792 "Total number of times a send received a enetunreachable");
1793 rack_hot_alloc = counter_u64_alloc(M_WAITOK);
1794 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1795 SYSCTL_CHILDREN(rack_counters),
1796 OID_AUTO, "alloc_hot", CTLFLAG_RD,
1797 &rack_hot_alloc,
1798 "Total allocations from the top of our list");
1799 rack_to_alloc = counter_u64_alloc(M_WAITOK);
1800 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1801 SYSCTL_CHILDREN(rack_counters),
1802 OID_AUTO, "allocs", CTLFLAG_RD,
1803 &rack_to_alloc,
1804 "Total allocations of tracking structures");
1805 rack_to_alloc_hard = counter_u64_alloc(M_WAITOK);
1806 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1807 SYSCTL_CHILDREN(rack_counters),
1808 OID_AUTO, "allochard", CTLFLAG_RD,
1809 &rack_to_alloc_hard,
1810 "Total allocations done with sleeping the hard way");
1811 rack_to_alloc_emerg = counter_u64_alloc(M_WAITOK);
1812 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1813 SYSCTL_CHILDREN(rack_counters),
1814 OID_AUTO, "allocemerg", CTLFLAG_RD,
1815 &rack_to_alloc_emerg,
1816 "Total allocations done from emergency cache");
1817 rack_to_alloc_limited = counter_u64_alloc(M_WAITOK);
1818 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1819 SYSCTL_CHILDREN(rack_counters),
1820 OID_AUTO, "alloc_limited", CTLFLAG_RD,
1821 &rack_to_alloc_limited,
1822 "Total allocations dropped due to limit");
1823 rack_alloc_limited_conns = counter_u64_alloc(M_WAITOK);
1824 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1825 SYSCTL_CHILDREN(rack_counters),
1826 OID_AUTO, "alloc_limited_conns", CTLFLAG_RD,
1827 &rack_alloc_limited_conns,
1828 "Connections with allocations dropped due to limit");
1829 rack_split_limited = counter_u64_alloc(M_WAITOK);
1830 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1831 SYSCTL_CHILDREN(rack_counters),
1832 OID_AUTO, "split_limited", CTLFLAG_RD,
1833 &rack_split_limited,
1834 "Split allocations dropped due to limit");
1835 rack_rxt_clamps_cwnd = counter_u64_alloc(M_WAITOK);
1836 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1837 SYSCTL_CHILDREN(rack_counters),
1838 OID_AUTO, "rxt_clamps_cwnd", CTLFLAG_RD,
1839 &rack_rxt_clamps_cwnd,
1840 "Number of times that excessive rxt clamped the cwnd down");
1841 rack_rxt_clamps_cwnd_uniq = counter_u64_alloc(M_WAITOK);
1842 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1843 SYSCTL_CHILDREN(rack_counters),
1844 OID_AUTO, "rxt_clamps_cwnd_uniq", CTLFLAG_RD,
1845 &rack_rxt_clamps_cwnd_uniq,
1846 "Number of connections that have had excessive rxt clamped the cwnd down");
1847 rack_persists_sends = counter_u64_alloc(M_WAITOK);
1848 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1849 SYSCTL_CHILDREN(rack_counters),
1850 OID_AUTO, "persist_sends", CTLFLAG_RD,
1851 &rack_persists_sends,
1852 "Number of times we sent a persist probe");
1853 rack_persists_acks = counter_u64_alloc(M_WAITOK);
1854 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1855 SYSCTL_CHILDREN(rack_counters),
1856 OID_AUTO, "persist_acks", CTLFLAG_RD,
1857 &rack_persists_acks,
1858 "Number of times a persist probe was acked");
1859 rack_persists_loss = counter_u64_alloc(M_WAITOK);
1860 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1861 SYSCTL_CHILDREN(rack_counters),
1862 OID_AUTO, "persist_loss", CTLFLAG_RD,
1863 &rack_persists_loss,
1864 "Number of times we detected a lost persist probe (no ack)");
1865 rack_persists_lost_ends = counter_u64_alloc(M_WAITOK);
1866 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1867 SYSCTL_CHILDREN(rack_counters),
1868 OID_AUTO, "persist_loss_ends", CTLFLAG_RD,
1869 &rack_persists_lost_ends,
1870 "Number of lost persist probe (no ack) that the run ended with a PERSIST abort");
1871 #ifdef INVARIANTS
1872 rack_adjust_map_bw = counter_u64_alloc(M_WAITOK);
1873 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1874 SYSCTL_CHILDREN(rack_counters),
1875 OID_AUTO, "map_adjust_req", CTLFLAG_RD,
1876 &rack_adjust_map_bw,
1877 "Number of times we hit the case where the sb went up and down on a sendmap entry");
1878 #endif
1879 rack_multi_single_eq = counter_u64_alloc(M_WAITOK);
1880 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1881 SYSCTL_CHILDREN(rack_counters),
1882 OID_AUTO, "cmp_ack_equiv", CTLFLAG_RD,
1883 &rack_multi_single_eq,
1884 "Number of compressed acks total represented");
1885 rack_proc_non_comp_ack = counter_u64_alloc(M_WAITOK);
1886 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1887 SYSCTL_CHILDREN(rack_counters),
1888 OID_AUTO, "cmp_ack_not", CTLFLAG_RD,
1889 &rack_proc_non_comp_ack,
1890 "Number of non compresseds acks that we processed");
1891
1892
1893 rack_sack_proc_all = counter_u64_alloc(M_WAITOK);
1894 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1895 SYSCTL_CHILDREN(rack_counters),
1896 OID_AUTO, "sack_long", CTLFLAG_RD,
1897 &rack_sack_proc_all,
1898 "Total times we had to walk whole list for sack processing");
1899 rack_sack_proc_restart = counter_u64_alloc(M_WAITOK);
1900 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1901 SYSCTL_CHILDREN(rack_counters),
1902 OID_AUTO, "sack_restart", CTLFLAG_RD,
1903 &rack_sack_proc_restart,
1904 "Total times we had to walk whole list due to a restart");
1905 rack_sack_proc_short = counter_u64_alloc(M_WAITOK);
1906 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1907 SYSCTL_CHILDREN(rack_counters),
1908 OID_AUTO, "sack_short", CTLFLAG_RD,
1909 &rack_sack_proc_short,
1910 "Total times we took shortcut for sack processing");
1911 rack_sack_skipped_acked = counter_u64_alloc(M_WAITOK);
1912 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1913 SYSCTL_CHILDREN(rack_attack),
1914 OID_AUTO, "skipacked", CTLFLAG_RD,
1915 &rack_sack_skipped_acked,
1916 "Total number of times we skipped previously sacked");
1917 rack_sack_splits = counter_u64_alloc(M_WAITOK);
1918 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1919 SYSCTL_CHILDREN(rack_attack),
1920 OID_AUTO, "ofsplit", CTLFLAG_RD,
1921 &rack_sack_splits,
1922 "Total number of times we did the old fashion tree split");
1923 rack_input_idle_reduces = counter_u64_alloc(M_WAITOK);
1924 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1925 SYSCTL_CHILDREN(rack_counters),
1926 OID_AUTO, "idle_reduce_oninput", CTLFLAG_RD,
1927 &rack_input_idle_reduces,
1928 "Total number of idle reductions on input");
1929 rack_collapsed_win_seen = counter_u64_alloc(M_WAITOK);
1930 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1931 SYSCTL_CHILDREN(rack_counters),
1932 OID_AUTO, "collapsed_win_seen", CTLFLAG_RD,
1933 &rack_collapsed_win_seen,
1934 "Total number of collapsed window events seen (where our window shrinks)");
1935
1936 rack_collapsed_win = counter_u64_alloc(M_WAITOK);
1937 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1938 SYSCTL_CHILDREN(rack_counters),
1939 OID_AUTO, "collapsed_win", CTLFLAG_RD,
1940 &rack_collapsed_win,
1941 "Total number of collapsed window events where we mark packets");
1942 rack_collapsed_win_rxt = counter_u64_alloc(M_WAITOK);
1943 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1944 SYSCTL_CHILDREN(rack_counters),
1945 OID_AUTO, "collapsed_win_rxt", CTLFLAG_RD,
1946 &rack_collapsed_win_rxt,
1947 "Total number of packets that were retransmitted");
1948 rack_collapsed_win_rxt_bytes = counter_u64_alloc(M_WAITOK);
1949 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1950 SYSCTL_CHILDREN(rack_counters),
1951 OID_AUTO, "collapsed_win_bytes", CTLFLAG_RD,
1952 &rack_collapsed_win_rxt_bytes,
1953 "Total number of bytes that were retransmitted");
1954 rack_try_scwnd = counter_u64_alloc(M_WAITOK);
1955 SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1956 SYSCTL_CHILDREN(rack_counters),
1957 OID_AUTO, "tried_scwnd", CTLFLAG_RD,
1958 &rack_try_scwnd,
1959 "Total number of scwnd attempts");
1960 COUNTER_ARRAY_ALLOC(rack_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK);
1961 SYSCTL_ADD_COUNTER_U64_ARRAY(&rack_sysctl_ctx, SYSCTL_CHILDREN(rack_sysctl_root),
1962 OID_AUTO, "outsize", CTLFLAG_RD,
1963 rack_out_size, TCP_MSS_ACCT_SIZE, "MSS send sizes");
1964 COUNTER_ARRAY_ALLOC(rack_opts_arry, RACK_OPTS_SIZE, M_WAITOK);
1965 SYSCTL_ADD_COUNTER_U64_ARRAY(&rack_sysctl_ctx, SYSCTL_CHILDREN(rack_sysctl_root),
1966 OID_AUTO, "opts", CTLFLAG_RD,
1967 rack_opts_arry, RACK_OPTS_SIZE, "RACK Option Stats");
1968 SYSCTL_ADD_PROC(&rack_sysctl_ctx,
1969 SYSCTL_CHILDREN(rack_sysctl_root),
1970 OID_AUTO, "clear", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1971 &rack_clear_counter, 0, sysctl_rack_clear, "IU", "Clear counters");
1972 }
1973
1974 static uint32_t
rc_init_window(struct tcp_rack * rack)1975 rc_init_window(struct tcp_rack *rack)
1976 {
1977 return (tcp_compute_initwnd(tcp_maxseg(rack->rc_tp)));
1978
1979 }
1980
1981 static uint64_t
rack_get_fixed_pacing_bw(struct tcp_rack * rack)1982 rack_get_fixed_pacing_bw(struct tcp_rack *rack)
1983 {
1984 if (IN_FASTRECOVERY(rack->rc_tp->t_flags))
1985 return (rack->r_ctl.rc_fixed_pacing_rate_rec);
1986 else if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh)
1987 return (rack->r_ctl.rc_fixed_pacing_rate_ss);
1988 else
1989 return (rack->r_ctl.rc_fixed_pacing_rate_ca);
1990 }
1991
1992 static void
rack_log_hybrid_bw(struct tcp_rack * rack,uint32_t seq,uint64_t cbw,uint64_t tim,uint64_t data,uint8_t mod,uint16_t aux,struct tcp_sendfile_track * cur,int line)1993 rack_log_hybrid_bw(struct tcp_rack *rack, uint32_t seq, uint64_t cbw, uint64_t tim,
1994 uint64_t data, uint8_t mod, uint16_t aux,
1995 struct tcp_sendfile_track *cur, int line)
1996 {
1997 #ifdef TCP_REQUEST_TRK
1998 int do_log = 0;
1999
2000 /*
2001 * The rate cap one is noisy and only should come out when normal BB logging
2002 * is enabled, the other logs (not RATE_CAP and NOT CAP_CALC) only come out
2003 * once per chunk and make up the BBpoint that can be turned on by the client.
2004 */
2005 if ((mod == HYBRID_LOG_RATE_CAP) || (mod == HYBRID_LOG_CAP_CALC)) {
2006 /*
2007 * The very noisy two need to only come out when
2008 * we have verbose logging on.
2009 */
2010 if (rack_verbose_logging != 0)
2011 do_log = tcp_bblogging_on(rack->rc_tp);
2012 else
2013 do_log = 0;
2014 } else if (mod != HYBRID_LOG_BW_MEASURE) {
2015 /*
2016 * All other less noisy logs here except the measure which
2017 * also needs to come out on the point and the log.
2018 */
2019 do_log = tcp_bblogging_on(rack->rc_tp);
2020 } else {
2021 do_log = tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING);
2022 }
2023
2024 if (do_log) {
2025 union tcp_log_stackspecific log;
2026 struct timeval tv;
2027 uint64_t lt_bw;
2028
2029 /* Convert our ms to a microsecond */
2030 memset(&log, 0, sizeof(log));
2031
2032 log.u_bbr.cwnd_gain = line;
2033 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2034 log.u_bbr.rttProp = tim;
2035 log.u_bbr.bw_inuse = cbw;
2036 log.u_bbr.delRate = rack_get_gp_est(rack);
2037 lt_bw = rack_get_lt_bw(rack);
2038 log.u_bbr.flex1 = seq;
2039 log.u_bbr.pacing_gain = aux;
2040 /* lt_bw = < flex3 | flex2 > */
2041 log.u_bbr.flex2 = (uint32_t)(lt_bw & 0x00000000ffffffff);
2042 log.u_bbr.flex3 = (uint32_t)((lt_bw >> 32) & 0x00000000ffffffff);
2043 /* Record the last obtained us rtt in inflight */
2044 if (cur == NULL) {
2045 /* Make sure we are looking at the right log if an overide comes in */
2046 cur = rack->r_ctl.rc_last_sft;
2047 }
2048 if (rack->r_ctl.rack_rs.rs_flags != RACK_RTT_EMPTY)
2049 log.u_bbr.inflight = rack->r_ctl.rack_rs.rs_us_rtt;
2050 else {
2051 /* Use the last known rtt i.e. the rack-rtt */
2052 log.u_bbr.inflight = rack->rc_rack_rtt;
2053 }
2054 if (cur != NULL) {
2055 uint64_t off;
2056
2057 log.u_bbr.cur_del_rate = cur->deadline;
2058 if ((mod == HYBRID_LOG_RATE_CAP) || (mod == HYBRID_LOG_CAP_CALC)) {
2059 /* start = < lost | pkt_epoch > */
2060 log.u_bbr.pkt_epoch = (uint32_t)(cur->start & 0x00000000ffffffff);
2061 log.u_bbr.lost = (uint32_t)((cur->start >> 32) & 0x00000000ffffffff);
2062 log.u_bbr.flex6 = cur->start_seq;
2063 log.u_bbr.pkts_out = cur->end_seq;
2064 } else {
2065 /* start = < lost | pkt_epoch > */
2066 log.u_bbr.pkt_epoch = (uint32_t)(cur->start & 0x00000000ffffffff);
2067 log.u_bbr.lost = (uint32_t)((cur->start >> 32) & 0x00000000ffffffff);
2068 /* end = < pkts_out | flex6 > */
2069 log.u_bbr.flex6 = (uint32_t)(cur->end & 0x00000000ffffffff);
2070 log.u_bbr.pkts_out = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff);
2071 }
2072 /* first_send = <lt_epoch | epoch> */
2073 log.u_bbr.epoch = (uint32_t)(cur->first_send & 0x00000000ffffffff);
2074 log.u_bbr.lt_epoch = (uint32_t)((cur->first_send >> 32) & 0x00000000ffffffff);
2075 /* localtime = <delivered | applimited>*/
2076 log.u_bbr.applimited = (uint32_t)(cur->localtime & 0x00000000ffffffff);
2077 log.u_bbr.delivered = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);
2078 #ifdef TCP_REQUEST_TRK
2079 off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
2080 log.u_bbr.bbr_substate = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
2081 #endif
2082 log.u_bbr.inhpts = 1;
2083 log.u_bbr.flex4 = (uint32_t)(rack->rc_tp->t_sndbytes - cur->sent_at_fs);
2084 log.u_bbr.flex5 = (uint32_t)(rack->rc_tp->t_snd_rxt_bytes - cur->rxt_at_fs);
2085 log.u_bbr.flex7 = (uint16_t)cur->hybrid_flags;
2086 } else {
2087 log.u_bbr.flex7 = 0xffff;
2088 log.u_bbr.cur_del_rate = 0xffffffffffffffff;
2089 }
2090 /*
2091 * Compose bbr_state to be a bit wise 0000ADHF
2092 * where A is the always_pace flag
2093 * where D is the dgp_on flag
2094 * where H is the hybrid_mode on flag
2095 * where F is the use_fixed_rate flag.
2096 */
2097 log.u_bbr.bbr_state = rack->rc_always_pace;
2098 log.u_bbr.bbr_state <<= 1;
2099 log.u_bbr.bbr_state |= rack->dgp_on;
2100 log.u_bbr.bbr_state <<= 1;
2101 log.u_bbr.bbr_state |= rack->rc_hybrid_mode;
2102 log.u_bbr.bbr_state <<= 1;
2103 log.u_bbr.bbr_state |= rack->use_fixed_rate;
2104 log.u_bbr.flex8 = mod;
2105 tcp_log_event(rack->rc_tp, NULL,
2106 &rack->rc_inp->inp_socket->so_rcv,
2107 &rack->rc_inp->inp_socket->so_snd,
2108 TCP_HYBRID_PACING_LOG, 0,
2109 0, &log, false, NULL, __func__, __LINE__, &tv);
2110
2111 }
2112 #endif
2113 }
2114
2115 #ifdef TCP_REQUEST_TRK
2116 static void
rack_log_hybrid_sends(struct tcp_rack * rack,struct tcp_sendfile_track * cur,int line)2117 rack_log_hybrid_sends(struct tcp_rack *rack, struct tcp_sendfile_track *cur, int line)
2118 {
2119 if (tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING)) {
2120 union tcp_log_stackspecific log;
2121 struct timeval tv;
2122 uint64_t off;
2123
2124 /* Convert our ms to a microsecond */
2125 memset(&log, 0, sizeof(log));
2126
2127 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2128 log.u_bbr.delRate = cur->sent_at_fs;
2129
2130 if ((cur->flags & TCP_TRK_TRACK_FLG_LSND) == 0) {
2131 /*
2132 * We did not get a new Rules Applied to set so
2133 * no overlapping send occured, this means the
2134 * current byte counts are correct.
2135 */
2136 log.u_bbr.cur_del_rate = rack->rc_tp->t_sndbytes;
2137 log.u_bbr.rttProp = rack->rc_tp->t_snd_rxt_bytes;
2138 } else {
2139 /*
2140 * Overlapping send case, we switched to a new
2141 * send and did a rules applied.
2142 */
2143 log.u_bbr.cur_del_rate = cur->sent_at_ls;
2144 log.u_bbr.rttProp = cur->rxt_at_ls;
2145 }
2146 log.u_bbr.bw_inuse = cur->rxt_at_fs;
2147 log.u_bbr.cwnd_gain = line;
2148 off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
2149 log.u_bbr.bbr_substate = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
2150 /* start = < flex1 | flex2 > */
2151 log.u_bbr.flex2 = (uint32_t)(cur->start & 0x00000000ffffffff);
2152 log.u_bbr.flex1 = (uint32_t)((cur->start >> 32) & 0x00000000ffffffff);
2153 /* end = < flex3 | flex4 > */
2154 log.u_bbr.flex4 = (uint32_t)(cur->end & 0x00000000ffffffff);
2155 log.u_bbr.flex3 = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff);
2156
2157 /* localtime = <delivered | applimited>*/
2158 log.u_bbr.applimited = (uint32_t)(cur->localtime & 0x00000000ffffffff);
2159 log.u_bbr.delivered = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);
2160 /* client timestamp = <lt_epoch | epoch>*/
2161 log.u_bbr.epoch = (uint32_t)(cur->timestamp & 0x00000000ffffffff);
2162 log.u_bbr.lt_epoch = (uint32_t)((cur->timestamp >> 32) & 0x00000000ffffffff);
2163 /* now set all the flags in */
2164 log.u_bbr.pkts_out = cur->hybrid_flags;
2165 log.u_bbr.lost = cur->playout_ms;
2166 log.u_bbr.flex6 = cur->flags;
2167 /*
2168 * Last send time = <flex5 | pkt_epoch> note we do not distinguish cases
2169 * where a false retransmit occurred so first_send <-> lastsend may
2170 * include longer time then it actually took if we have a false rxt.
2171 */
2172 log.u_bbr.pkt_epoch = (uint32_t)(rack->r_ctl.last_tmit_time_acked & 0x00000000ffffffff);
2173 log.u_bbr.flex5 = (uint32_t)((rack->r_ctl.last_tmit_time_acked >> 32) & 0x00000000ffffffff);
2174 /*
2175 * Compose bbr_state to be a bit wise 0000ADHF
2176 * where A is the always_pace flag
2177 * where D is the dgp_on flag
2178 * where H is the hybrid_mode on flag
2179 * where F is the use_fixed_rate flag.
2180 */
2181 log.u_bbr.bbr_state = rack->rc_always_pace;
2182 log.u_bbr.bbr_state <<= 1;
2183 log.u_bbr.bbr_state |= rack->dgp_on;
2184 log.u_bbr.bbr_state <<= 1;
2185 log.u_bbr.bbr_state |= rack->rc_hybrid_mode;
2186 log.u_bbr.bbr_state <<= 1;
2187 log.u_bbr.bbr_state |= rack->use_fixed_rate;
2188
2189 log.u_bbr.flex8 = HYBRID_LOG_SENT_LOST;
2190 tcp_log_event(rack->rc_tp, NULL,
2191 &rack->rc_inp->inp_socket->so_rcv,
2192 &rack->rc_inp->inp_socket->so_snd,
2193 TCP_HYBRID_PACING_LOG, 0,
2194 0, &log, false, NULL, __func__, __LINE__, &tv);
2195 }
2196 }
2197 #endif
2198
2199 static inline uint64_t
rack_compensate_for_linerate(struct tcp_rack * rack,uint64_t bw)2200 rack_compensate_for_linerate(struct tcp_rack *rack, uint64_t bw)
2201 {
2202 uint64_t ret_bw, ether;
2203 uint64_t u_segsiz;
2204
2205 ether = rack->rc_tp->t_maxseg + sizeof(struct tcphdr);
2206 if (rack->r_is_v6){
2207 #ifdef INET6
2208 ether += sizeof(struct ip6_hdr);
2209 #endif
2210 ether += 14; /* eheader size 6+6+2 */
2211 } else {
2212 #ifdef INET
2213 ether += sizeof(struct ip);
2214 #endif
2215 ether += 14; /* eheader size 6+6+2 */
2216 }
2217 u_segsiz = (uint64_t)min(ctf_fixed_maxseg(rack->rc_tp), rack->r_ctl.rc_pace_min_segs);
2218 ret_bw = bw;
2219 ret_bw *= ether;
2220 ret_bw /= u_segsiz;
2221 return (ret_bw);
2222 }
2223
2224 static void
rack_rate_cap_bw(struct tcp_rack * rack,uint64_t * bw,int * capped)2225 rack_rate_cap_bw(struct tcp_rack *rack, uint64_t *bw, int *capped)
2226 {
2227 #ifdef TCP_REQUEST_TRK
2228 struct timeval tv;
2229 uint64_t timenow, timeleft, lenleft, lengone, calcbw;
2230 #endif
2231
2232 if (rack->r_ctl.bw_rate_cap == 0)
2233 return;
2234 #ifdef TCP_REQUEST_TRK
2235 if (rack->rc_catch_up && rack->rc_hybrid_mode &&
2236 (rack->r_ctl.rc_last_sft != NULL)) {
2237 /*
2238 * We have a dynamic cap. The original target
2239 * is in bw_rate_cap, but we need to look at
2240 * how long it is until we hit the deadline.
2241 */
2242 struct tcp_sendfile_track *ent;
2243
2244 ent = rack->r_ctl.rc_last_sft;
2245 microuptime(&tv);
2246 timenow = tcp_tv_to_lusec(&tv);
2247 if (timenow >= ent->deadline) {
2248 /* No time left we do DGP only */
2249 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2250 0, 0, 0, HYBRID_LOG_OUTOFTIME, 0, ent, __LINE__);
2251 rack->r_ctl.bw_rate_cap = 0;
2252 return;
2253 }
2254 /* We have the time */
2255 timeleft = rack->r_ctl.rc_last_sft->deadline - timenow;
2256 if (timeleft < HPTS_MSEC_IN_SEC) {
2257 /* If there is less than a ms left just use DGPs rate */
2258 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2259 0, timeleft, 0, HYBRID_LOG_OUTOFTIME, 0, ent, __LINE__);
2260 rack->r_ctl.bw_rate_cap = 0;
2261 return;
2262 }
2263 /*
2264 * Now lets find the amount of data left to send.
2265 *
2266 * Now ideally we want to use the end_seq to figure out how much more
2267 * but it might not be possible (only if we have the TRACK_FG_COMP on the entry..
2268 */
2269 if (ent->flags & TCP_TRK_TRACK_FLG_COMP) {
2270 if (SEQ_GT(ent->end_seq, rack->rc_tp->snd_una))
2271 lenleft = ent->end_seq - rack->rc_tp->snd_una;
2272 else {
2273 /* TSNH, we should catch it at the send */
2274 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2275 0, timeleft, 0, HYBRID_LOG_CAPERROR, 0, ent, __LINE__);
2276 rack->r_ctl.bw_rate_cap = 0;
2277 return;
2278 }
2279 } else {
2280 /*
2281 * The hard way, figure out how much is gone and then
2282 * take that away from the total the client asked for
2283 * (thats off by tls overhead if this is tls).
2284 */
2285 if (SEQ_GT(rack->rc_tp->snd_una, ent->start_seq))
2286 lengone = rack->rc_tp->snd_una - ent->start_seq;
2287 else
2288 lengone = 0;
2289 if (lengone < (ent->end - ent->start))
2290 lenleft = (ent->end - ent->start) - lengone;
2291 else {
2292 /* TSNH, we should catch it at the send */
2293 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2294 0, timeleft, lengone, HYBRID_LOG_CAPERROR, 0, ent, __LINE__);
2295 rack->r_ctl.bw_rate_cap = 0;
2296 return;
2297 }
2298 }
2299 if (lenleft == 0) {
2300 /* We have it all sent */
2301 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2302 0, timeleft, lenleft, HYBRID_LOG_ALLSENT, 0, ent, __LINE__);
2303 if (rack->r_ctl.bw_rate_cap)
2304 goto normal_ratecap;
2305 else
2306 return;
2307 }
2308 calcbw = lenleft * HPTS_USEC_IN_SEC;
2309 calcbw /= timeleft;
2310 /* Now we must compensate for IP/TCP overhead */
2311 calcbw = rack_compensate_for_linerate(rack, calcbw);
2312 /* Update the bit rate cap */
2313 rack->r_ctl.bw_rate_cap = calcbw;
2314 if ((rack->r_ctl.rc_last_sft->hybrid_flags & TCP_HYBRID_PACING_S_MSS) &&
2315 (rack_hybrid_allow_set_maxseg == 1) &&
2316 ((rack->r_ctl.rc_last_sft->hybrid_flags & TCP_HYBRID_PACING_SETMSS) == 0)) {
2317 /* Lets set in a smaller mss possibly here to match our rate-cap */
2318 uint32_t orig_max;
2319
2320 orig_max = rack->r_ctl.rc_pace_max_segs;
2321 rack->r_ctl.rc_last_sft->hybrid_flags |= TCP_HYBRID_PACING_SETMSS;
2322 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack, calcbw, ctf_fixed_maxseg(rack->rc_tp));
2323 rack_log_type_pacing_sizes(rack->rc_tp, rack, rack->r_ctl.client_suggested_maxseg, orig_max, __LINE__, 5);
2324 }
2325 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2326 calcbw, timeleft, lenleft, HYBRID_LOG_CAP_CALC, 0, ent, __LINE__);
2327 if ((calcbw > 0) && (*bw > calcbw)) {
2328 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2329 *bw, ent->deadline, lenleft, HYBRID_LOG_RATE_CAP, 0, ent, __LINE__);
2330 *capped = 1;
2331 *bw = calcbw;
2332 }
2333 return;
2334 }
2335 normal_ratecap:
2336 #endif
2337 if ((rack->r_ctl.bw_rate_cap > 0) && (*bw > rack->r_ctl.bw_rate_cap)) {
2338 #ifdef TCP_REQUEST_TRK
2339 if (rack->rc_hybrid_mode &&
2340 rack->rc_catch_up &&
2341 (rack->r_ctl.rc_last_sft != NULL) &&
2342 (rack->r_ctl.rc_last_sft->hybrid_flags & TCP_HYBRID_PACING_S_MSS) &&
2343 (rack_hybrid_allow_set_maxseg == 1) &&
2344 ((rack->r_ctl.rc_last_sft->hybrid_flags & TCP_HYBRID_PACING_SETMSS) == 0)) {
2345 /* Lets set in a smaller mss possibly here to match our rate-cap */
2346 uint32_t orig_max;
2347
2348 orig_max = rack->r_ctl.rc_pace_max_segs;
2349 rack->r_ctl.rc_last_sft->hybrid_flags |= TCP_HYBRID_PACING_SETMSS;
2350 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack, rack->r_ctl.bw_rate_cap, ctf_fixed_maxseg(rack->rc_tp));
2351 rack_log_type_pacing_sizes(rack->rc_tp, rack, rack->r_ctl.client_suggested_maxseg, orig_max, __LINE__, 5);
2352 }
2353 #endif
2354 *capped = 1;
2355 *bw = rack->r_ctl.bw_rate_cap;
2356 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
2357 *bw, 0, 0,
2358 HYBRID_LOG_RATE_CAP, 1, NULL, __LINE__);
2359 }
2360 }
2361
2362 static uint64_t
rack_get_gp_est(struct tcp_rack * rack)2363 rack_get_gp_est(struct tcp_rack *rack)
2364 {
2365 uint64_t bw, lt_bw, ret_bw;
2366
2367 if (rack->rc_gp_filled == 0) {
2368 /*
2369 * We have yet no b/w measurement,
2370 * if we have a user set initial bw
2371 * return it. If we don't have that and
2372 * we have an srtt, use the tcp IW (10) to
2373 * calculate a fictional b/w over the SRTT
2374 * which is more or less a guess. Note
2375 * we don't use our IW from rack on purpose
2376 * so if we have like IW=30, we are not
2377 * calculating a "huge" b/w.
2378 */
2379 uint64_t srtt;
2380
2381 if (rack->dis_lt_bw == 1)
2382 lt_bw = 0;
2383 else
2384 lt_bw = rack_get_lt_bw(rack);
2385 if (lt_bw) {
2386 /*
2387 * No goodput bw but a long-term b/w does exist
2388 * lets use that.
2389 */
2390 ret_bw = lt_bw;
2391 goto compensate;
2392 }
2393 if (rack->r_ctl.init_rate)
2394 return (rack->r_ctl.init_rate);
2395
2396 /* Ok lets come up with the IW guess, if we have a srtt */
2397 if (rack->rc_tp->t_srtt == 0) {
2398 /*
2399 * Go with old pacing method
2400 * i.e. burst mitigation only.
2401 */
2402 return (0);
2403 }
2404 /* Ok lets get the initial TCP win (not racks) */
2405 bw = tcp_compute_initwnd(tcp_maxseg(rack->rc_tp));
2406 srtt = (uint64_t)rack->rc_tp->t_srtt;
2407 bw *= (uint64_t)USECS_IN_SECOND;
2408 bw /= srtt;
2409 ret_bw = bw;
2410 goto compensate;
2411
2412 }
2413 if (rack->r_ctl.num_measurements >= RACK_REQ_AVG) {
2414 /* Averaging is done, we can return the value */
2415 bw = rack->r_ctl.gp_bw;
2416 } else {
2417 /* Still doing initial average must calculate */
2418 bw = rack->r_ctl.gp_bw / max(rack->r_ctl.num_measurements, 1);
2419 }
2420 if (rack->dis_lt_bw) {
2421 /* We are not using lt-bw */
2422 ret_bw = bw;
2423 goto compensate;
2424 }
2425 lt_bw = rack_get_lt_bw(rack);
2426 if (lt_bw == 0) {
2427 /* If we don't have one then equate it to the gp_bw */
2428 lt_bw = rack->r_ctl.gp_bw;
2429 }
2430 if (rack->use_lesser_lt_bw) {
2431 if (lt_bw < bw)
2432 ret_bw = lt_bw;
2433 else
2434 ret_bw = bw;
2435 } else {
2436 if (lt_bw > bw)
2437 ret_bw = lt_bw;
2438 else
2439 ret_bw = bw;
2440 }
2441 /*
2442 * Now lets compensate based on the TCP/IP overhead. Our
2443 * Goodput estimate does not include this so we must pace out
2444 * a bit faster since our pacing calculations do. The pacing
2445 * calculations use the base ETHERNET_SEGMENT_SIZE and the segsiz
2446 * we are using to do this, so we do that here in the opposite
2447 * direction as well. This means that if we are tunneled and the
2448 * segsiz is say 1200 bytes we will get quite a boost, but its
2449 * compensated for in the pacing time the opposite way.
2450 */
2451 compensate:
2452 ret_bw = rack_compensate_for_linerate(rack, ret_bw);
2453 return(ret_bw);
2454 }
2455
2456
2457 static uint64_t
rack_get_bw(struct tcp_rack * rack)2458 rack_get_bw(struct tcp_rack *rack)
2459 {
2460 uint64_t bw;
2461
2462 if (rack->use_fixed_rate) {
2463 /* Return the fixed pacing rate */
2464 return (rack_get_fixed_pacing_bw(rack));
2465 }
2466 bw = rack_get_gp_est(rack);
2467 return (bw);
2468 }
2469
2470 static uint16_t
rack_get_output_gain(struct tcp_rack * rack,struct rack_sendmap * rsm)2471 rack_get_output_gain(struct tcp_rack *rack, struct rack_sendmap *rsm)
2472 {
2473 if (rack->use_fixed_rate) {
2474 return (100);
2475 } else if (rack->in_probe_rtt && (rsm == NULL))
2476 return (rack->r_ctl.rack_per_of_gp_probertt);
2477 else if ((IN_FASTRECOVERY(rack->rc_tp->t_flags) &&
2478 rack->r_ctl.rack_per_of_gp_rec)) {
2479 if (rsm) {
2480 /* a retransmission always use the recovery rate */
2481 return (rack->r_ctl.rack_per_of_gp_rec);
2482 } else if (rack->rack_rec_nonrxt_use_cr) {
2483 /* Directed to use the configured rate */
2484 goto configured_rate;
2485 } else if (rack->rack_no_prr &&
2486 (rack->r_ctl.rack_per_of_gp_rec > 100)) {
2487 /* No PRR, lets just use the b/w estimate only */
2488 return (100);
2489 } else {
2490 /*
2491 * Here we may have a non-retransmit but we
2492 * have no overrides, so just use the recovery
2493 * rate (prr is in effect).
2494 */
2495 return (rack->r_ctl.rack_per_of_gp_rec);
2496 }
2497 }
2498 configured_rate:
2499 /* For the configured rate we look at our cwnd vs the ssthresh */
2500 if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh)
2501 return (rack->r_ctl.rack_per_of_gp_ss);
2502 else
2503 return (rack->r_ctl.rack_per_of_gp_ca);
2504 }
2505
2506 static void
rack_log_dsack_event(struct tcp_rack * rack,uint8_t mod,uint32_t flex4,uint32_t flex5,uint32_t flex6)2507 rack_log_dsack_event(struct tcp_rack *rack, uint8_t mod, uint32_t flex4, uint32_t flex5, uint32_t flex6)
2508 {
2509 /*
2510 * Types of logs (mod value)
2511 * 1 = dsack_persists reduced by 1 via T-O or fast recovery exit.
2512 * 2 = a dsack round begins, persist is reset to 16.
2513 * 3 = a dsack round ends
2514 * 4 = Dsack option increases rack rtt flex5 is the srtt input, flex6 is thresh
2515 * 5 = Socket option set changing the control flags rc_rack_tmr_std_based, rc_rack_use_dsack
2516 * 6 = Final rack rtt, flex4 is srtt and flex6 is final limited thresh.
2517 */
2518 if (tcp_bblogging_on(rack->rc_tp)) {
2519 union tcp_log_stackspecific log;
2520 struct timeval tv;
2521
2522 memset(&log, 0, sizeof(log));
2523 log.u_bbr.flex1 = rack->rc_rack_tmr_std_based;
2524 log.u_bbr.flex1 <<= 1;
2525 log.u_bbr.flex1 |= rack->rc_rack_use_dsack;
2526 log.u_bbr.flex1 <<= 1;
2527 log.u_bbr.flex1 |= rack->rc_dsack_round_seen;
2528 log.u_bbr.flex2 = rack->r_ctl.dsack_round_end;
2529 log.u_bbr.flex3 = rack->r_ctl.num_dsack;
2530 log.u_bbr.flex4 = flex4;
2531 log.u_bbr.flex5 = flex5;
2532 log.u_bbr.flex6 = flex6;
2533 log.u_bbr.flex7 = rack->r_ctl.dsack_persist;
2534 log.u_bbr.flex8 = mod;
2535 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2536 log.u_bbr.epoch = rack->r_ctl.current_round;
2537 log.u_bbr.lt_epoch = rack->r_ctl.rc_considered_lost;
2538 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2539 &rack->rc_inp->inp_socket->so_rcv,
2540 &rack->rc_inp->inp_socket->so_snd,
2541 RACK_DSACK_HANDLING, 0,
2542 0, &log, false, &tv);
2543 }
2544 }
2545
2546 static void
rack_log_hdwr_pacing(struct tcp_rack * rack,uint64_t rate,uint64_t hw_rate,int line,int error,uint16_t mod)2547 rack_log_hdwr_pacing(struct tcp_rack *rack,
2548 uint64_t rate, uint64_t hw_rate, int line,
2549 int error, uint16_t mod)
2550 {
2551 if (tcp_bblogging_on(rack->rc_tp)) {
2552 union tcp_log_stackspecific log;
2553 struct timeval tv;
2554 const struct ifnet *ifp;
2555 uint64_t ifp64;
2556
2557 memset(&log, 0, sizeof(log));
2558 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff);
2559 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff);
2560 if (rack->r_ctl.crte) {
2561 ifp = rack->r_ctl.crte->ptbl->rs_ifp;
2562 } else if (rack->rc_inp->inp_route.ro_nh &&
2563 rack->rc_inp->inp_route.ro_nh->nh_ifp) {
2564 ifp = rack->rc_inp->inp_route.ro_nh->nh_ifp;
2565 } else
2566 ifp = NULL;
2567 if (ifp) {
2568 ifp64 = (uintptr_t)ifp;
2569 log.u_bbr.flex3 = ((ifp64 >> 32) & 0x00000000ffffffff);
2570 log.u_bbr.flex4 = (ifp64 & 0x00000000ffffffff);
2571 }
2572 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2573 log.u_bbr.bw_inuse = rate;
2574 log.u_bbr.flex5 = line;
2575 log.u_bbr.flex6 = error;
2576 log.u_bbr.flex7 = mod;
2577 log.u_bbr.applimited = rack->r_ctl.rc_pace_max_segs;
2578 log.u_bbr.flex8 = rack->use_fixed_rate;
2579 log.u_bbr.flex8 <<= 1;
2580 log.u_bbr.flex8 |= rack->rack_hdrw_pacing;
2581 log.u_bbr.pkts_out = rack->rc_tp->t_maxseg;
2582 log.u_bbr.delRate = rack->r_ctl.crte_prev_rate;
2583 if (rack->r_ctl.crte)
2584 log.u_bbr.cur_del_rate = rack->r_ctl.crte->rate;
2585 else
2586 log.u_bbr.cur_del_rate = 0;
2587 log.u_bbr.rttProp = rack->r_ctl.last_hw_bw_req;
2588 log.u_bbr.epoch = rack->r_ctl.current_round;
2589 log.u_bbr.lt_epoch = rack->r_ctl.rc_considered_lost;
2590 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2591 &rack->rc_inp->inp_socket->so_rcv,
2592 &rack->rc_inp->inp_socket->so_snd,
2593 BBR_LOG_HDWR_PACE, 0,
2594 0, &log, false, &tv);
2595 }
2596 }
2597
2598 static uint64_t
rack_get_output_bw(struct tcp_rack * rack,uint64_t bw,struct rack_sendmap * rsm,int * capped)2599 rack_get_output_bw(struct tcp_rack *rack, uint64_t bw, struct rack_sendmap *rsm, int *capped)
2600 {
2601 /*
2602 * We allow rack_per_of_gp_xx to dictate our bw rate we want.
2603 */
2604 uint64_t bw_est, high_rate;
2605 uint64_t gain;
2606
2607 gain = (uint64_t)rack_get_output_gain(rack, rsm);
2608 bw_est = bw * gain;
2609 bw_est /= (uint64_t)100;
2610 /* Never fall below the minimum (def 64kbps) */
2611 if (bw_est < RACK_MIN_BW)
2612 bw_est = RACK_MIN_BW;
2613 if (rack->r_rack_hw_rate_caps) {
2614 /* Rate caps are in place */
2615 if (rack->r_ctl.crte != NULL) {
2616 /* We have a hdwr rate already */
2617 high_rate = tcp_hw_highest_rate(rack->r_ctl.crte);
2618 if (bw_est >= high_rate) {
2619 /* We are capping bw at the highest rate table entry */
2620 if (rack_hw_rate_cap_per &&
2621 (((high_rate * (100 + rack_hw_rate_cap_per)) / 100) < bw_est)) {
2622 rack->r_rack_hw_rate_caps = 0;
2623 goto done;
2624 }
2625 rack_log_hdwr_pacing(rack,
2626 bw_est, high_rate, __LINE__,
2627 0, 3);
2628 bw_est = high_rate;
2629 if (capped)
2630 *capped = 1;
2631 }
2632 } else if ((rack->rack_hdrw_pacing == 0) &&
2633 (rack->rack_hdw_pace_ena) &&
2634 (rack->rack_attempt_hdwr_pace == 0) &&
2635 (rack->rc_inp->inp_route.ro_nh != NULL) &&
2636 (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) {
2637 /*
2638 * Special case, we have not yet attempted hardware
2639 * pacing, and yet we may, when we do, find out if we are
2640 * above the highest rate. We need to know the maxbw for the interface
2641 * in question (if it supports ratelimiting). We get back
2642 * a 0, if the interface is not found in the RL lists.
2643 */
2644 high_rate = tcp_hw_highest_rate_ifp(rack->rc_inp->inp_route.ro_nh->nh_ifp, rack->rc_inp);
2645 if (high_rate) {
2646 /* Yep, we have a rate is it above this rate? */
2647 if (bw_est > high_rate) {
2648 bw_est = high_rate;
2649 if (capped)
2650 *capped = 1;
2651 }
2652 }
2653 }
2654 }
2655 done:
2656 return (bw_est);
2657 }
2658
2659 static void
rack_log_retran_reason(struct tcp_rack * rack,struct rack_sendmap * rsm,uint32_t tsused,uint32_t thresh,int mod)2660 rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t tsused, uint32_t thresh, int mod)
2661 {
2662 if (tcp_bblogging_on(rack->rc_tp)) {
2663 union tcp_log_stackspecific log;
2664 struct timeval tv;
2665
2666 if ((mod != 1) && (rack_verbose_logging == 0)) {
2667 /*
2668 * We get 3 values currently for mod
2669 * 1 - We are retransmitting and this tells the reason.
2670 * 2 - We are clearing a dup-ack count.
2671 * 3 - We are incrementing a dup-ack count.
2672 *
2673 * The clear/increment are only logged
2674 * if you have BBverbose on.
2675 */
2676 return;
2677 }
2678 memset(&log, 0, sizeof(log));
2679 log.u_bbr.flex1 = tsused;
2680 log.u_bbr.flex2 = thresh;
2681 log.u_bbr.flex3 = rsm->r_flags;
2682 log.u_bbr.flex4 = rsm->r_dupack;
2683 log.u_bbr.flex5 = rsm->r_start;
2684 log.u_bbr.flex6 = rsm->r_end;
2685 log.u_bbr.flex8 = mod;
2686 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
2687 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2688 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2689 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2690 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2691 log.u_bbr.pacing_gain = rack->r_must_retran;
2692 log.u_bbr.epoch = rack->r_ctl.current_round;
2693 log.u_bbr.lt_epoch = rack->r_ctl.rc_considered_lost;
2694 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2695 &rack->rc_inp->inp_socket->so_rcv,
2696 &rack->rc_inp->inp_socket->so_snd,
2697 BBR_LOG_SETTINGS_CHG, 0,
2698 0, &log, false, &tv);
2699 }
2700 }
2701
2702 static void
rack_log_to_start(struct tcp_rack * rack,uint32_t cts,uint32_t to,int32_t pacing_delay,uint8_t which)2703 rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t pacing_delay, uint8_t which)
2704 {
2705 if (tcp_bblogging_on(rack->rc_tp)) {
2706 union tcp_log_stackspecific log;
2707 struct timeval tv;
2708
2709 memset(&log, 0, sizeof(log));
2710 log.u_bbr.flex1 = rack->rc_tp->t_srtt;
2711 log.u_bbr.flex2 = to;
2712 log.u_bbr.flex3 = rack->r_ctl.rc_hpts_flags;
2713 log.u_bbr.flex4 = pacing_delay;
2714 log.u_bbr.flex5 = rack->rc_tp->t_hpts_slot;
2715 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur;
2716 log.u_bbr.flex7 = rack->rc_in_persist;
2717 log.u_bbr.flex8 = which;
2718 if (rack->rack_no_prr)
2719 log.u_bbr.pkts_out = 0;
2720 else
2721 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
2722 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
2723 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2724 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2725 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2726 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2727 log.u_bbr.pacing_gain = rack->r_must_retran;
2728 log.u_bbr.cwnd_gain = rack->rack_deferred_inited;
2729 log.u_bbr.pkt_epoch = rack->rc_has_collapsed;
2730 log.u_bbr.lt_epoch = rack->rc_tp->t_rxtshift;
2731 log.u_bbr.lost = rack_rto_min;
2732 log.u_bbr.epoch = rack->r_ctl.roundends;
2733 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
2734 log.u_bbr.bw_inuse <<= 32;
2735 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
2736 log.u_bbr.applimited = rack->rc_tp->t_flags2;
2737 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2738 &rack->rc_inp->inp_socket->so_rcv,
2739 &rack->rc_inp->inp_socket->so_snd,
2740 BBR_LOG_TIMERSTAR, 0,
2741 0, &log, false, &tv);
2742 }
2743 }
2744
2745 static void
rack_log_to_event(struct tcp_rack * rack,int32_t to_num,struct rack_sendmap * rsm)2746 rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rsm)
2747 {
2748 if (tcp_bblogging_on(rack->rc_tp)) {
2749 union tcp_log_stackspecific log;
2750 struct timeval tv;
2751
2752 memset(&log, 0, sizeof(log));
2753 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
2754 log.u_bbr.flex8 = to_num;
2755 log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt;
2756 log.u_bbr.flex2 = rack->rc_rack_rtt;
2757 if (rsm == NULL)
2758 log.u_bbr.flex3 = 0;
2759 else
2760 log.u_bbr.flex3 = rsm->r_end - rsm->r_start;
2761 if (rack->rack_no_prr)
2762 log.u_bbr.flex5 = 0;
2763 else
2764 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
2765 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2766 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2767 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2768 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2769 log.u_bbr.pacing_gain = rack->r_must_retran;
2770 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
2771 log.u_bbr.bw_inuse <<= 32;
2772 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
2773 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2774 &rack->rc_inp->inp_socket->so_rcv,
2775 &rack->rc_inp->inp_socket->so_snd,
2776 BBR_LOG_RTO, 0,
2777 0, &log, false, &tv);
2778 }
2779 }
2780
2781 static void
rack_log_map_chg(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * prev,struct rack_sendmap * rsm,struct rack_sendmap * next,int flag,uint32_t th_ack,int line)2782 rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack,
2783 struct rack_sendmap *prev,
2784 struct rack_sendmap *rsm,
2785 struct rack_sendmap *next,
2786 int flag, uint32_t th_ack, int line)
2787 {
2788 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
2789 union tcp_log_stackspecific log;
2790 struct timeval tv;
2791
2792 memset(&log, 0, sizeof(log));
2793 log.u_bbr.flex8 = flag;
2794 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
2795 log.u_bbr.cur_del_rate = (uintptr_t)prev;
2796 log.u_bbr.delRate = (uintptr_t)rsm;
2797 log.u_bbr.rttProp = (uintptr_t)next;
2798 log.u_bbr.flex7 = 0;
2799 if (prev) {
2800 log.u_bbr.flex1 = prev->r_start;
2801 log.u_bbr.flex2 = prev->r_end;
2802 log.u_bbr.flex7 |= 0x4;
2803 }
2804 if (rsm) {
2805 log.u_bbr.flex3 = rsm->r_start;
2806 log.u_bbr.flex4 = rsm->r_end;
2807 log.u_bbr.flex7 |= 0x2;
2808 }
2809 if (next) {
2810 log.u_bbr.flex5 = next->r_start;
2811 log.u_bbr.flex6 = next->r_end;
2812 log.u_bbr.flex7 |= 0x1;
2813 }
2814 log.u_bbr.applimited = line;
2815 log.u_bbr.pkts_out = th_ack;
2816 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2817 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2818 if (rack->rack_no_prr)
2819 log.u_bbr.lost = 0;
2820 else
2821 log.u_bbr.lost = rack->r_ctl.rc_prr_sndcnt;
2822 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
2823 log.u_bbr.bw_inuse <<= 32;
2824 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
2825 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2826 &rack->rc_inp->inp_socket->so_rcv,
2827 &rack->rc_inp->inp_socket->so_snd,
2828 TCP_LOG_MAPCHG, 0,
2829 0, &log, false, &tv);
2830 }
2831 }
2832
2833 static void
rack_log_rtt_upd(struct tcpcb * tp,struct tcp_rack * rack,uint32_t t,uint32_t len,struct rack_sendmap * rsm,int conf)2834 rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t len,
2835 struct rack_sendmap *rsm, int conf)
2836 {
2837 if (tcp_bblogging_on(tp)) {
2838 union tcp_log_stackspecific log;
2839 struct timeval tv;
2840 memset(&log, 0, sizeof(log));
2841 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
2842 log.u_bbr.flex1 = t;
2843 log.u_bbr.flex2 = len;
2844 log.u_bbr.flex3 = rack->r_ctl.rc_rack_min_rtt;
2845 log.u_bbr.flex4 = rack->r_ctl.rack_rs.rs_rtt_lowest;
2846 log.u_bbr.flex5 = rack->r_ctl.rack_rs.rs_rtt_highest;
2847 log.u_bbr.flex6 = rack->r_ctl.rack_rs.rs_us_rtrcnt;
2848 log.u_bbr.flex7 = conf;
2849 log.u_bbr.rttProp = (uint64_t)rack->r_ctl.rack_rs.rs_rtt_tot;
2850 log.u_bbr.flex8 = rack->r_ctl.rc_rate_sample_method;
2851 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2852 log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt;
2853 log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags;
2854 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2855 if (rsm) {
2856 log.u_bbr.pkt_epoch = rsm->r_start;
2857 log.u_bbr.lost = rsm->r_end;
2858 log.u_bbr.cwnd_gain = rsm->r_rtr_cnt;
2859 /* We loose any upper of the 24 bits */
2860 log.u_bbr.pacing_gain = (uint16_t)rsm->r_flags;
2861 } else {
2862 /* Its a SYN */
2863 log.u_bbr.pkt_epoch = rack->rc_tp->iss;
2864 log.u_bbr.lost = 0;
2865 log.u_bbr.cwnd_gain = 0;
2866 log.u_bbr.pacing_gain = 0;
2867 }
2868 /* Write out general bits of interest rrs here */
2869 log.u_bbr.use_lt_bw = rack->rc_highly_buffered;
2870 log.u_bbr.use_lt_bw <<= 1;
2871 log.u_bbr.use_lt_bw |= rack->forced_ack;
2872 log.u_bbr.use_lt_bw <<= 1;
2873 log.u_bbr.use_lt_bw |= rack->rc_gp_dyn_mul;
2874 log.u_bbr.use_lt_bw <<= 1;
2875 log.u_bbr.use_lt_bw |= rack->in_probe_rtt;
2876 log.u_bbr.use_lt_bw <<= 1;
2877 log.u_bbr.use_lt_bw |= rack->measure_saw_probe_rtt;
2878 log.u_bbr.use_lt_bw <<= 1;
2879 log.u_bbr.use_lt_bw |= rack->app_limited_needs_set;
2880 log.u_bbr.use_lt_bw <<= 1;
2881 log.u_bbr.use_lt_bw |= rack->rc_gp_filled;
2882 log.u_bbr.use_lt_bw <<= 1;
2883 log.u_bbr.use_lt_bw |= rack->rc_dragged_bottom;
2884 log.u_bbr.applimited = rack->r_ctl.rc_target_probertt_flight;
2885 log.u_bbr.epoch = rack->r_ctl.rc_time_probertt_starts;
2886 log.u_bbr.lt_epoch = rack->r_ctl.rc_time_probertt_entered;
2887 log.u_bbr.cur_del_rate = rack->r_ctl.rc_lower_rtt_us_cts;
2888 log.u_bbr.delRate = rack->r_ctl.rc_gp_srtt;
2889 log.u_bbr.bw_inuse = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
2890 log.u_bbr.bw_inuse <<= 32;
2891 if (rsm)
2892 log.u_bbr.bw_inuse |= ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
2893 TCP_LOG_EVENTP(tp, NULL,
2894 &rack->rc_inp->inp_socket->so_rcv,
2895 &rack->rc_inp->inp_socket->so_snd,
2896 BBR_LOG_BBRRTT, 0,
2897 0, &log, false, &tv);
2898
2899
2900 }
2901 }
2902
2903 static void
rack_log_rtt_sample(struct tcp_rack * rack,uint32_t rtt)2904 rack_log_rtt_sample(struct tcp_rack *rack, uint32_t rtt)
2905 {
2906 /*
2907 * Log the rtt sample we are
2908 * applying to the srtt algorithm in
2909 * useconds.
2910 */
2911 if (tcp_bblogging_on(rack->rc_tp)) {
2912 union tcp_log_stackspecific log;
2913 struct timeval tv;
2914
2915 /* Convert our ms to a microsecond */
2916 memset(&log, 0, sizeof(log));
2917 log.u_bbr.flex1 = rtt;
2918 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur;
2919 log.u_bbr.flex7 = 1;
2920 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2921 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2922 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2923 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2924 log.u_bbr.pacing_gain = rack->r_must_retran;
2925 /*
2926 * We capture in delRate the upper 32 bits as
2927 * the confidence level we had declared, and the
2928 * lower 32 bits as the actual RTT using the arrival
2929 * timestamp.
2930 */
2931 log.u_bbr.delRate = rack->r_ctl.rack_rs.confidence;
2932 log.u_bbr.delRate <<= 32;
2933 log.u_bbr.delRate |= rack->r_ctl.rack_rs.rs_us_rtt;
2934 /* Lets capture all the things that make up t_rtxcur */
2935 log.u_bbr.applimited = rack_rto_min;
2936 log.u_bbr.epoch = rack_rto_max;
2937 log.u_bbr.lt_epoch = rack->r_ctl.timer_slop;
2938 log.u_bbr.lost = rack_rto_min;
2939 log.u_bbr.pkt_epoch = TICKS_2_USEC(tcp_rexmit_slop);
2940 log.u_bbr.rttProp = RACK_REXMTVAL(rack->rc_tp);
2941 log.u_bbr.bw_inuse = rack->r_ctl.act_rcv_time.tv_sec;
2942 log.u_bbr.bw_inuse *= HPTS_USEC_IN_SEC;
2943 log.u_bbr.bw_inuse += rack->r_ctl.act_rcv_time.tv_usec;
2944 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2945 &rack->rc_inp->inp_socket->so_rcv,
2946 &rack->rc_inp->inp_socket->so_snd,
2947 TCP_LOG_RTT, 0,
2948 0, &log, false, &tv);
2949 }
2950 }
2951
2952 static void
rack_log_rtt_sample_calc(struct tcp_rack * rack,uint32_t rtt,uint32_t send_time,uint32_t ack_time,int where)2953 rack_log_rtt_sample_calc(struct tcp_rack *rack, uint32_t rtt, uint32_t send_time, uint32_t ack_time, int where)
2954 {
2955 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
2956 union tcp_log_stackspecific log;
2957 struct timeval tv;
2958
2959 /* Convert our ms to a microsecond */
2960 memset(&log, 0, sizeof(log));
2961 log.u_bbr.flex1 = rtt;
2962 log.u_bbr.flex2 = send_time;
2963 log.u_bbr.flex3 = ack_time;
2964 log.u_bbr.flex4 = where;
2965 log.u_bbr.flex7 = 2;
2966 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2967 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
2968 log.u_bbr.bw_inuse <<= 32;
2969 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
2970 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2971 &rack->rc_inp->inp_socket->so_rcv,
2972 &rack->rc_inp->inp_socket->so_snd,
2973 TCP_LOG_RTT, 0,
2974 0, &log, false, &tv);
2975 }
2976 }
2977
2978
2979 static void
rack_log_rtt_sendmap(struct tcp_rack * rack,uint32_t idx,uint64_t tsv,uint32_t tsecho)2980 rack_log_rtt_sendmap(struct tcp_rack *rack, uint32_t idx, uint64_t tsv, uint32_t tsecho)
2981 {
2982 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
2983 union tcp_log_stackspecific log;
2984 struct timeval tv;
2985
2986 /* Convert our ms to a microsecond */
2987 memset(&log, 0, sizeof(log));
2988 log.u_bbr.flex1 = idx;
2989 log.u_bbr.flex2 = rack_ts_to_msec(tsv);
2990 log.u_bbr.flex3 = tsecho;
2991 log.u_bbr.flex7 = 3;
2992 log.u_bbr.rttProp = tsv;
2993 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2994 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
2995 log.u_bbr.bw_inuse <<= 32;
2996 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
2997 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2998 &rack->rc_inp->inp_socket->so_rcv,
2999 &rack->rc_inp->inp_socket->so_snd,
3000 TCP_LOG_RTT, 0,
3001 0, &log, false, &tv);
3002 }
3003 }
3004
3005
3006 static inline void
rack_log_progress_event(struct tcp_rack * rack,struct tcpcb * tp,uint32_t tick,int event,int line)3007 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line)
3008 {
3009 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
3010 union tcp_log_stackspecific log;
3011 struct timeval tv;
3012
3013 memset(&log, 0, sizeof(log));
3014 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
3015 log.u_bbr.flex1 = line;
3016 log.u_bbr.flex2 = tick;
3017 log.u_bbr.flex3 = tp->t_maxunacktime;
3018 log.u_bbr.flex4 = tp->t_acktime;
3019 log.u_bbr.flex8 = event;
3020 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3021 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3022 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3023 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3024 log.u_bbr.pacing_gain = rack->r_must_retran;
3025 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
3026 log.u_bbr.bw_inuse <<= 32;
3027 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
3028 TCP_LOG_EVENTP(tp, NULL,
3029 &rack->rc_inp->inp_socket->so_rcv,
3030 &rack->rc_inp->inp_socket->so_snd,
3031 BBR_LOG_PROGRESS, 0,
3032 0, &log, false, &tv);
3033 }
3034 }
3035
3036 static void
rack_log_type_bbrsnd(struct tcp_rack * rack,uint32_t len,uint32_t pacing_delay,uint32_t cts,struct timeval * tv,int line)3037 rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t pacing_delay, uint32_t cts, struct timeval *tv, int line)
3038 {
3039 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
3040 union tcp_log_stackspecific log;
3041
3042 memset(&log, 0, sizeof(log));
3043 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
3044 log.u_bbr.flex1 = pacing_delay;
3045 if (rack->rack_no_prr)
3046 log.u_bbr.flex2 = 0;
3047 else
3048 log.u_bbr.flex2 = rack->r_ctl.rc_prr_sndcnt;
3049 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
3050 log.u_bbr.flex6 = line;
3051 log.u_bbr.flex7 = (0x0000ffff & rack->r_ctl.rc_hpts_flags);
3052 log.u_bbr.flex8 = rack->rc_in_persist;
3053 log.u_bbr.timeStamp = cts;
3054 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3055 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3056 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3057 log.u_bbr.pacing_gain = rack->r_must_retran;
3058 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3059 &rack->rc_inp->inp_socket->so_rcv,
3060 &rack->rc_inp->inp_socket->so_snd,
3061 BBR_LOG_BBRSND, 0,
3062 0, &log, false, tv);
3063 }
3064 }
3065
3066 static void
rack_log_doseg_done(struct tcp_rack * rack,uint32_t cts,int32_t nxt_pkt,int32_t did_out,int way_out,int nsegs)3067 rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_t did_out, int way_out, int nsegs)
3068 {
3069 if (tcp_bblogging_on(rack->rc_tp)) {
3070 union tcp_log_stackspecific log;
3071 struct timeval tv;
3072
3073 memset(&log, 0, sizeof(log));
3074 log.u_bbr.flex1 = did_out;
3075 log.u_bbr.flex2 = nxt_pkt;
3076 log.u_bbr.flex3 = way_out;
3077 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
3078 if (rack->rack_no_prr)
3079 log.u_bbr.flex5 = 0;
3080 else
3081 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
3082 log.u_bbr.flex6 = nsegs;
3083 log.u_bbr.applimited = rack->r_ctl.rc_pace_min_segs;
3084 log.u_bbr.flex7 = rack->rc_ack_can_sendout_data; /* Do we have ack-can-send set */
3085 log.u_bbr.flex7 <<= 1;
3086 log.u_bbr.flex7 |= rack->r_fast_output; /* is fast output primed */
3087 log.u_bbr.flex7 <<= 1;
3088 log.u_bbr.flex7 |= rack->r_wanted_output; /* Do we want output */
3089 log.u_bbr.flex8 = rack->rc_in_persist;
3090 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
3091 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3092 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3093 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
3094 log.u_bbr.use_lt_bw <<= 1;
3095 log.u_bbr.use_lt_bw |= rack->r_might_revert;
3096 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3097 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3098 log.u_bbr.pacing_gain = rack->r_must_retran;
3099 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
3100 log.u_bbr.bw_inuse <<= 32;
3101 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
3102 log.u_bbr.epoch = rack->rc_inp->inp_socket->so_snd.sb_hiwat;
3103 log.u_bbr.lt_epoch = rack->rc_inp->inp_socket->so_rcv.sb_hiwat;
3104 log.u_bbr.lost = rack->rc_tp->t_srtt;
3105 log.u_bbr.pkt_epoch = rack->rc_tp->rfbuf_cnt;
3106 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3107 &rack->rc_inp->inp_socket->so_rcv,
3108 &rack->rc_inp->inp_socket->so_snd,
3109 BBR_LOG_DOSEG_DONE, 0,
3110 0, &log, false, &tv);
3111 }
3112 }
3113
3114 static void
rack_log_type_pacing_sizes(struct tcpcb * tp,struct tcp_rack * rack,uint32_t arg1,uint32_t arg2,uint32_t arg3,uint8_t frm)3115 rack_log_type_pacing_sizes(struct tcpcb *tp, struct tcp_rack *rack, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint8_t frm)
3116 {
3117 if (tcp_bblogging_on(rack->rc_tp)) {
3118 union tcp_log_stackspecific log;
3119 struct timeval tv;
3120
3121 memset(&log, 0, sizeof(log));
3122 log.u_bbr.flex1 = rack->r_ctl.rc_pace_min_segs;
3123 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
3124 log.u_bbr.flex4 = arg1;
3125 log.u_bbr.flex5 = arg2;
3126 log.u_bbr.flex7 = rack->r_ctl.rc_user_set_min_segs;
3127 log.u_bbr.flex6 = arg3;
3128 log.u_bbr.flex8 = frm;
3129 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3130 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3131 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3132 log.u_bbr.applimited = rack->r_ctl.rc_sacked;
3133 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3134 log.u_bbr.pacing_gain = rack->r_must_retran;
3135 TCP_LOG_EVENTP(tp, NULL, &tptosocket(tp)->so_rcv,
3136 &tptosocket(tp)->so_snd,
3137 TCP_HDWR_PACE_SIZE, 0, 0, &log, false, &tv);
3138 }
3139 }
3140
3141 static void
rack_log_type_just_return(struct tcp_rack * rack,uint32_t cts,uint32_t tlen,uint32_t pacing_delay,uint8_t hpts_calling,int reason,uint32_t cwnd_to_use)3142 rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, uint32_t pacing_delay,
3143 uint8_t hpts_calling, int reason, uint32_t cwnd_to_use)
3144 {
3145 if (tcp_bblogging_on(rack->rc_tp)) {
3146 union tcp_log_stackspecific log;
3147 struct timeval tv;
3148
3149 memset(&log, 0, sizeof(log));
3150 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
3151 log.u_bbr.flex1 = pacing_delay;
3152 log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags;
3153 log.u_bbr.flex4 = reason;
3154 if (rack->rack_no_prr)
3155 log.u_bbr.flex5 = 0;
3156 else
3157 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
3158 log.u_bbr.flex7 = hpts_calling;
3159 log.u_bbr.flex8 = rack->rc_in_persist;
3160 log.u_bbr.lt_epoch = cwnd_to_use;
3161 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3162 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3163 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3164 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3165 log.u_bbr.pacing_gain = rack->r_must_retran;
3166 log.u_bbr.cwnd_gain = rack->rc_has_collapsed;
3167 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
3168 log.u_bbr.bw_inuse <<= 32;
3169 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
3170 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3171 &rack->rc_inp->inp_socket->so_rcv,
3172 &rack->rc_inp->inp_socket->so_snd,
3173 BBR_LOG_JUSTRET, 0,
3174 tlen, &log, false, &tv);
3175 }
3176 }
3177
3178 static void
rack_log_to_cancel(struct tcp_rack * rack,int32_t hpts_removed,int line,uint32_t us_cts,struct timeval * tv,uint32_t flags_on_entry)3179 rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32_t us_cts,
3180 struct timeval *tv, uint32_t flags_on_entry)
3181 {
3182 if (tcp_bblogging_on(rack->rc_tp)) {
3183 union tcp_log_stackspecific log;
3184
3185 memset(&log, 0, sizeof(log));
3186 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
3187 log.u_bbr.flex1 = line;
3188 log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to;
3189 log.u_bbr.flex3 = flags_on_entry;
3190 log.u_bbr.flex4 = us_cts;
3191 if (rack->rack_no_prr)
3192 log.u_bbr.flex5 = 0;
3193 else
3194 log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
3195 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur;
3196 log.u_bbr.flex7 = hpts_removed;
3197 log.u_bbr.flex8 = 1;
3198 log.u_bbr.applimited = rack->r_ctl.rc_hpts_flags;
3199 log.u_bbr.timeStamp = us_cts;
3200 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3201 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3202 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3203 log.u_bbr.pacing_gain = rack->r_must_retran;
3204 log.u_bbr.bw_inuse = rack->r_ctl.current_round;
3205 log.u_bbr.bw_inuse <<= 32;
3206 log.u_bbr.bw_inuse |= rack->r_ctl.rc_considered_lost;
3207 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3208 &rack->rc_inp->inp_socket->so_rcv,
3209 &rack->rc_inp->inp_socket->so_snd,
3210 BBR_LOG_TIMERCANC, 0,
3211 0, &log, false, tv);
3212 }
3213 }
3214
3215 static void
rack_log_alt_to_to_cancel(struct tcp_rack * rack,uint32_t flex1,uint32_t flex2,uint32_t flex3,uint32_t flex4,uint32_t flex5,uint32_t flex6,uint16_t flex7,uint8_t mod)3216 rack_log_alt_to_to_cancel(struct tcp_rack *rack,
3217 uint32_t flex1, uint32_t flex2,
3218 uint32_t flex3, uint32_t flex4,
3219 uint32_t flex5, uint32_t flex6,
3220 uint16_t flex7, uint8_t mod)
3221 {
3222 if (tcp_bblogging_on(rack->rc_tp)) {
3223 union tcp_log_stackspecific log;
3224 struct timeval tv;
3225
3226 if (mod == 1) {
3227 /* No you can't use 1, its for the real to cancel */
3228 return;
3229 }
3230 memset(&log, 0, sizeof(log));
3231 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3232 log.u_bbr.flex1 = flex1;
3233 log.u_bbr.flex2 = flex2;
3234 log.u_bbr.flex3 = flex3;
3235 log.u_bbr.flex4 = flex4;
3236 log.u_bbr.flex5 = flex5;
3237 log.u_bbr.flex6 = flex6;
3238 log.u_bbr.flex7 = flex7;
3239 log.u_bbr.flex8 = mod;
3240 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3241 &rack->rc_inp->inp_socket->so_rcv,
3242 &rack->rc_inp->inp_socket->so_snd,
3243 BBR_LOG_TIMERCANC, 0,
3244 0, &log, false, &tv);
3245 }
3246 }
3247
3248 static void
rack_log_to_processing(struct tcp_rack * rack,uint32_t cts,int32_t ret,int32_t timers)3249 rack_log_to_processing(struct tcp_rack *rack, uint32_t cts, int32_t ret, int32_t timers)
3250 {
3251 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
3252 union tcp_log_stackspecific log;
3253 struct timeval tv;
3254
3255 memset(&log, 0, sizeof(log));
3256 log.u_bbr.flex1 = timers;
3257 log.u_bbr.flex2 = ret;
3258 log.u_bbr.flex3 = rack->r_ctl.rc_timer_exp;
3259 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
3260 log.u_bbr.flex5 = cts;
3261 if (rack->rack_no_prr)
3262 log.u_bbr.flex6 = 0;
3263 else
3264 log.u_bbr.flex6 = rack->r_ctl.rc_prr_sndcnt;
3265 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
3266 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
3267 log.u_bbr.pacing_gain = rack->r_must_retran;
3268 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3269 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3270 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3271 &rack->rc_inp->inp_socket->so_rcv,
3272 &rack->rc_inp->inp_socket->so_snd,
3273 BBR_LOG_TO_PROCESS, 0,
3274 0, &log, false, &tv);
3275 }
3276 }
3277
3278 static void
rack_log_to_prr(struct tcp_rack * rack,int frm,int orig_cwnd,int line)3279 rack_log_to_prr(struct tcp_rack *rack, int frm, int orig_cwnd, int line)
3280 {
3281 if (tcp_bblogging_on(rack->rc_tp)) {
3282 union tcp_log_stackspecific log;
3283 struct timeval tv;
3284
3285 memset(&log, 0, sizeof(log));
3286 log.u_bbr.flex1 = rack->r_ctl.rc_prr_out;
3287 log.u_bbr.flex2 = rack->r_ctl.rc_prr_recovery_fs;
3288 if (rack->rack_no_prr)
3289 log.u_bbr.flex3 = 0;
3290 else
3291 log.u_bbr.flex3 = rack->r_ctl.rc_prr_sndcnt;
3292 log.u_bbr.flex4 = rack->r_ctl.rc_prr_delivered;
3293 log.u_bbr.flex5 = rack->r_ctl.rc_sacked;
3294 log.u_bbr.flex6 = rack->r_ctl.rc_holes_rxt;
3295 log.u_bbr.flex7 = line;
3296 log.u_bbr.flex8 = frm;
3297 log.u_bbr.pkts_out = orig_cwnd;
3298 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3299 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3300 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
3301 log.u_bbr.use_lt_bw <<= 1;
3302 log.u_bbr.use_lt_bw |= rack->r_might_revert;
3303 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3304 &rack->rc_inp->inp_socket->so_rcv,
3305 &rack->rc_inp->inp_socket->so_snd,
3306 BBR_LOG_BBRUPD, 0,
3307 0, &log, false, &tv);
3308 }
3309 }
3310
3311 static void
rack_counter_destroy(void)3312 rack_counter_destroy(void)
3313 {
3314 counter_u64_free(rack_total_bytes);
3315 counter_u64_free(rack_fto_send);
3316 counter_u64_free(rack_fto_rsm_send);
3317 counter_u64_free(rack_nfto_resend);
3318 counter_u64_free(rack_hw_pace_init_fail);
3319 counter_u64_free(rack_hw_pace_lost);
3320 counter_u64_free(rack_non_fto_send);
3321 counter_u64_free(rack_extended_rfo);
3322 counter_u64_free(rack_ack_total);
3323 counter_u64_free(rack_express_sack);
3324 counter_u64_free(rack_sack_total);
3325 counter_u64_free(rack_move_none);
3326 counter_u64_free(rack_move_some);
3327 counter_u64_free(rack_sack_attacks_detected);
3328 counter_u64_free(rack_sack_attacks_reversed);
3329 counter_u64_free(rack_sack_attacks_suspect);
3330 counter_u64_free(rack_sack_used_next_merge);
3331 counter_u64_free(rack_sack_used_prev_merge);
3332 counter_u64_free(rack_tlp_tot);
3333 counter_u64_free(rack_tlp_newdata);
3334 counter_u64_free(rack_tlp_retran);
3335 counter_u64_free(rack_tlp_retran_bytes);
3336 counter_u64_free(rack_to_tot);
3337 counter_u64_free(rack_saw_enobuf);
3338 counter_u64_free(rack_saw_enobuf_hw);
3339 counter_u64_free(rack_saw_enetunreach);
3340 counter_u64_free(rack_hot_alloc);
3341 counter_u64_free(rack_to_alloc);
3342 counter_u64_free(rack_to_alloc_hard);
3343 counter_u64_free(rack_to_alloc_emerg);
3344 counter_u64_free(rack_to_alloc_limited);
3345 counter_u64_free(rack_alloc_limited_conns);
3346 counter_u64_free(rack_split_limited);
3347 counter_u64_free(rack_multi_single_eq);
3348 counter_u64_free(rack_rxt_clamps_cwnd);
3349 counter_u64_free(rack_rxt_clamps_cwnd_uniq);
3350 counter_u64_free(rack_proc_non_comp_ack);
3351 counter_u64_free(rack_sack_proc_all);
3352 counter_u64_free(rack_sack_proc_restart);
3353 counter_u64_free(rack_sack_proc_short);
3354 counter_u64_free(rack_sack_skipped_acked);
3355 counter_u64_free(rack_sack_splits);
3356 counter_u64_free(rack_input_idle_reduces);
3357 counter_u64_free(rack_collapsed_win);
3358 counter_u64_free(rack_collapsed_win_rxt);
3359 counter_u64_free(rack_collapsed_win_rxt_bytes);
3360 counter_u64_free(rack_collapsed_win_seen);
3361 counter_u64_free(rack_try_scwnd);
3362 counter_u64_free(rack_persists_sends);
3363 counter_u64_free(rack_persists_acks);
3364 counter_u64_free(rack_persists_loss);
3365 counter_u64_free(rack_persists_lost_ends);
3366 #ifdef INVARIANTS
3367 counter_u64_free(rack_adjust_map_bw);
3368 #endif
3369 COUNTER_ARRAY_FREE(rack_out_size, TCP_MSS_ACCT_SIZE);
3370 COUNTER_ARRAY_FREE(rack_opts_arry, RACK_OPTS_SIZE);
3371 }
3372
3373 static struct rack_sendmap *
rack_alloc(struct tcp_rack * rack)3374 rack_alloc(struct tcp_rack *rack)
3375 {
3376 struct rack_sendmap *rsm;
3377
3378 /*
3379 * First get the top of the list it in
3380 * theory is the "hottest" rsm we have,
3381 * possibly just freed by ack processing.
3382 */
3383 if (rack->rc_free_cnt > rack_free_cache) {
3384 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
3385 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
3386 counter_u64_add(rack_hot_alloc, 1);
3387 rack->rc_free_cnt--;
3388 return (rsm);
3389 }
3390 /*
3391 * Once we get under our free cache we probably
3392 * no longer have a "hot" one available. Lets
3393 * get one from UMA.
3394 */
3395 rsm = uma_zalloc(rack_zone, M_NOWAIT);
3396 if (rsm) {
3397 rack->r_ctl.rc_num_maps_alloced++;
3398 counter_u64_add(rack_to_alloc, 1);
3399 return (rsm);
3400 }
3401 /*
3402 * Dig in to our aux rsm's (the last two) since
3403 * UMA failed to get us one.
3404 */
3405 if (rack->rc_free_cnt) {
3406 counter_u64_add(rack_to_alloc_emerg, 1);
3407 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
3408 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
3409 rack->rc_free_cnt--;
3410 return (rsm);
3411 }
3412 return (NULL);
3413 }
3414
3415 static struct rack_sendmap *
rack_alloc_full_limit(struct tcp_rack * rack)3416 rack_alloc_full_limit(struct tcp_rack *rack)
3417 {
3418 if ((V_tcp_map_entries_limit > 0) &&
3419 (rack->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
3420 counter_u64_add(rack_to_alloc_limited, 1);
3421 if (!rack->alloc_limit_reported) {
3422 rack->alloc_limit_reported = 1;
3423 counter_u64_add(rack_alloc_limited_conns, 1);
3424 }
3425 return (NULL);
3426 }
3427 return (rack_alloc(rack));
3428 }
3429
3430 /* wrapper to allocate a sendmap entry, subject to a specific limit */
3431 static struct rack_sendmap *
rack_alloc_limit(struct tcp_rack * rack,uint8_t limit_type)3432 rack_alloc_limit(struct tcp_rack *rack, uint8_t limit_type)
3433 {
3434 struct rack_sendmap *rsm;
3435
3436 if (limit_type) {
3437 /* currently there is only one limit type */
3438 if (rack->r_ctl.rc_split_limit > 0 &&
3439 rack->r_ctl.rc_num_split_allocs >= rack->r_ctl.rc_split_limit) {
3440 counter_u64_add(rack_split_limited, 1);
3441 if (!rack->alloc_limit_reported) {
3442 rack->alloc_limit_reported = 1;
3443 counter_u64_add(rack_alloc_limited_conns, 1);
3444 }
3445 return (NULL);
3446 }
3447 }
3448
3449 /* allocate and mark in the limit type, if set */
3450 rsm = rack_alloc(rack);
3451 if (rsm != NULL && limit_type) {
3452 rsm->r_limit_type = limit_type;
3453 rack->r_ctl.rc_num_split_allocs++;
3454 }
3455 return (rsm);
3456 }
3457
3458 static void
rack_free_trim(struct tcp_rack * rack)3459 rack_free_trim(struct tcp_rack *rack)
3460 {
3461 struct rack_sendmap *rsm;
3462
3463 /*
3464 * Free up all the tail entries until
3465 * we get our list down to the limit.
3466 */
3467 while (rack->rc_free_cnt > rack_free_cache) {
3468 rsm = TAILQ_LAST(&rack->r_ctl.rc_free, rack_head);
3469 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
3470 rack->rc_free_cnt--;
3471 rack->r_ctl.rc_num_maps_alloced--;
3472 uma_zfree(rack_zone, rsm);
3473 }
3474 }
3475
3476 static void
rack_free(struct tcp_rack * rack,struct rack_sendmap * rsm)3477 rack_free(struct tcp_rack *rack, struct rack_sendmap *rsm)
3478 {
3479 if (rsm->r_flags & RACK_APP_LIMITED) {
3480 KASSERT((rack->r_ctl.rc_app_limited_cnt > 0),
3481 ("app_cnt %u, rsm %p", rack->r_ctl.rc_app_limited_cnt, rsm));
3482 rack->r_ctl.rc_app_limited_cnt--;
3483 }
3484 if (rsm->r_limit_type) {
3485 /* currently there is only one limit type */
3486 rack->r_ctl.rc_num_split_allocs--;
3487 }
3488 if (rsm == rack->r_ctl.rc_first_appl) {
3489 rack->r_ctl.cleared_app_ack_seq = rsm->r_end;
3490 rack->r_ctl.cleared_app_ack = 1;
3491 if (rack->r_ctl.rc_app_limited_cnt == 0)
3492 rack->r_ctl.rc_first_appl = NULL;
3493 else
3494 rack->r_ctl.rc_first_appl = tqhash_find(rack->r_ctl.tqh, rsm->r_nseq_appl);
3495 }
3496 if (rsm == rack->r_ctl.rc_resend)
3497 rack->r_ctl.rc_resend = NULL;
3498 if (rsm == rack->r_ctl.rc_end_appl)
3499 rack->r_ctl.rc_end_appl = NULL;
3500 if (rack->r_ctl.rc_tlpsend == rsm)
3501 rack->r_ctl.rc_tlpsend = NULL;
3502 if (rack->r_ctl.rc_sacklast == rsm)
3503 rack->r_ctl.rc_sacklast = NULL;
3504 memset(rsm, 0, sizeof(struct rack_sendmap));
3505 /* Make sure we are not going to overrun our count limit of 0xff */
3506 if ((rack->rc_free_cnt + 1) > RACK_FREE_CNT_MAX) {
3507 rack_free_trim(rack);
3508 }
3509 TAILQ_INSERT_HEAD(&rack->r_ctl.rc_free, rsm, r_tnext);
3510 rack->rc_free_cnt++;
3511 }
3512
3513 static uint32_t
rack_get_measure_window(struct tcpcb * tp,struct tcp_rack * rack)3514 rack_get_measure_window(struct tcpcb *tp, struct tcp_rack *rack)
3515 {
3516 uint64_t srtt, bw, len, tim;
3517 uint32_t segsiz, def_len, minl;
3518
3519 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
3520 def_len = rack_def_data_window * segsiz;
3521 if (rack->rc_gp_filled == 0) {
3522 /*
3523 * We have no measurement (IW is in flight?) so
3524 * we can only guess using our data_window sysctl
3525 * value (usually 20MSS).
3526 */
3527 return (def_len);
3528 }
3529 /*
3530 * Now we have a number of factors to consider.
3531 *
3532 * 1) We have a desired BDP which is usually
3533 * at least 2.
3534 * 2) We have a minimum number of rtt's usually 1 SRTT
3535 * but we allow it too to be more.
3536 * 3) We want to make sure a measurement last N useconds (if
3537 * we have set rack_min_measure_usec.
3538 *
3539 * We handle the first concern here by trying to create a data
3540 * window of max(rack_def_data_window, DesiredBDP). The
3541 * second concern we handle in not letting the measurement
3542 * window end normally until at least the required SRTT's
3543 * have gone by which is done further below in
3544 * rack_enough_for_measurement(). Finally the third concern
3545 * we also handle here by calculating how long that time
3546 * would take at the current BW and then return the
3547 * max of our first calculation and that length. Note
3548 * that if rack_min_measure_usec is 0, we don't deal
3549 * with concern 3. Also for both Concern 1 and 3 an
3550 * application limited period could end the measurement
3551 * earlier.
3552 *
3553 * So lets calculate the BDP with the "known" b/w using
3554 * the SRTT as our rtt and then multiply it by the goal.
3555 */
3556 bw = rack_get_bw(rack);
3557 srtt = (uint64_t)tp->t_srtt;
3558 len = bw * srtt;
3559 len /= (uint64_t)HPTS_USEC_IN_SEC;
3560 len *= max(1, rack_goal_bdp);
3561 /* Now we need to round up to the nearest MSS */
3562 len = roundup(len, segsiz);
3563 if (rack_min_measure_usec) {
3564 /* Now calculate our min length for this b/w */
3565 tim = rack_min_measure_usec;
3566 minl = (tim * bw) / (uint64_t)HPTS_USEC_IN_SEC;
3567 if (minl == 0)
3568 minl = 1;
3569 minl = roundup(minl, segsiz);
3570 if (len < minl)
3571 len = minl;
3572 }
3573 /*
3574 * Now if we have a very small window we want
3575 * to attempt to get the window that is
3576 * as small as possible. This happens on
3577 * low b/w connections and we don't want to
3578 * span huge numbers of rtt's between measurements.
3579 *
3580 * We basically include 2 over our "MIN window" so
3581 * that the measurement can be shortened (possibly) by
3582 * an ack'ed packet.
3583 */
3584 if (len < def_len)
3585 return (max((uint32_t)len, ((MIN_GP_WIN+2) * segsiz)));
3586 else
3587 return (max((uint32_t)len, def_len));
3588
3589 }
3590
3591 static int
rack_enough_for_measurement(struct tcpcb * tp,struct tcp_rack * rack,tcp_seq th_ack,uint8_t * quality)3592 rack_enough_for_measurement(struct tcpcb *tp, struct tcp_rack *rack, tcp_seq th_ack, uint8_t *quality)
3593 {
3594 uint32_t tim, srtts, segsiz;
3595
3596 /*
3597 * Has enough time passed for the GP measurement to be valid?
3598 */
3599 if (SEQ_LT(th_ack, tp->gput_seq)) {
3600 /* Not enough bytes yet */
3601 return (0);
3602 }
3603 if ((tp->snd_max == tp->snd_una) ||
3604 (th_ack == tp->snd_max)){
3605 /*
3606 * All is acked quality of all acked is
3607 * usually low or medium, but we in theory could split
3608 * all acked into two cases, where you got
3609 * a signifigant amount of your window and
3610 * where you did not. For now we leave it
3611 * but it is something to contemplate in the
3612 * future. The danger here is that delayed ack
3613 * is effecting the last byte (which is a 50:50 chance).
3614 */
3615 *quality = RACK_QUALITY_ALLACKED;
3616 return (1);
3617 }
3618 if (SEQ_GEQ(th_ack, tp->gput_ack)) {
3619 /*
3620 * We obtained our entire window of data we wanted
3621 * no matter if we are in recovery or not then
3622 * its ok since expanding the window does not
3623 * make things fuzzy (or at least not as much).
3624 */
3625 *quality = RACK_QUALITY_HIGH;
3626 return (1);
3627 }
3628 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
3629 if (SEQ_LT(th_ack, tp->gput_ack) &&
3630 ((th_ack - tp->gput_seq) < max(rc_init_window(rack), (MIN_GP_WIN * segsiz)))) {
3631 /* Not enough bytes yet */
3632 return (0);
3633 }
3634 if (rack->r_ctl.rc_first_appl &&
3635 (SEQ_GEQ(th_ack, rack->r_ctl.rc_first_appl->r_end))) {
3636 /*
3637 * We are up to the app limited send point
3638 * we have to measure irrespective of the time..
3639 */
3640 *quality = RACK_QUALITY_APPLIMITED;
3641 return (1);
3642 }
3643 /* Now what about time? */
3644 srtts = (rack->r_ctl.rc_gp_srtt * rack_min_srtts);
3645 tim = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time) - tp->gput_ts;
3646 if ((tim >= srtts) && (IN_RECOVERY(rack->rc_tp->t_flags) == 0)) {
3647 /*
3648 * We do not allow a measurement if we are in recovery
3649 * that would shrink the goodput window we wanted.
3650 * This is to prevent cloudyness of when the last send
3651 * was actually made.
3652 */
3653 *quality = RACK_QUALITY_HIGH;
3654 return (1);
3655 }
3656 /* Nope not even a full SRTT has passed */
3657 return (0);
3658 }
3659
3660 static void
rack_log_timely(struct tcp_rack * rack,uint32_t logged,uint64_t cur_bw,uint64_t low_bnd,uint64_t up_bnd,int line,uint8_t method)3661 rack_log_timely(struct tcp_rack *rack,
3662 uint32_t logged, uint64_t cur_bw, uint64_t low_bnd,
3663 uint64_t up_bnd, int line, uint8_t method)
3664 {
3665 if (tcp_bblogging_on(rack->rc_tp)) {
3666 union tcp_log_stackspecific log;
3667 struct timeval tv;
3668
3669 memset(&log, 0, sizeof(log));
3670 log.u_bbr.flex1 = logged;
3671 log.u_bbr.flex2 = rack->rc_gp_timely_inc_cnt;
3672 log.u_bbr.flex2 <<= 4;
3673 log.u_bbr.flex2 |= rack->rc_gp_timely_dec_cnt;
3674 log.u_bbr.flex2 <<= 4;
3675 log.u_bbr.flex2 |= rack->rc_gp_incr;
3676 log.u_bbr.flex2 <<= 4;
3677 log.u_bbr.flex2 |= rack->rc_gp_bwred;
3678 log.u_bbr.flex3 = rack->rc_gp_incr;
3679 log.u_bbr.flex4 = rack->r_ctl.rack_per_of_gp_ss;
3680 log.u_bbr.flex5 = rack->r_ctl.rack_per_of_gp_ca;
3681 log.u_bbr.flex6 = rack->r_ctl.rack_per_of_gp_rec;
3682 log.u_bbr.flex7 = rack->rc_gp_bwred;
3683 log.u_bbr.flex8 = method;
3684 log.u_bbr.cur_del_rate = cur_bw;
3685 log.u_bbr.delRate = low_bnd;
3686 log.u_bbr.bw_inuse = up_bnd;
3687 log.u_bbr.rttProp = rack_get_bw(rack);
3688 log.u_bbr.pkt_epoch = line;
3689 log.u_bbr.pkts_out = rack->r_ctl.rc_rtt_diff;
3690 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3691 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3692 log.u_bbr.epoch = rack->r_ctl.rc_gp_srtt;
3693 log.u_bbr.lt_epoch = rack->r_ctl.rc_prev_gp_srtt;
3694 log.u_bbr.cwnd_gain = rack->rc_dragged_bottom;
3695 log.u_bbr.cwnd_gain <<= 1;
3696 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_rec;
3697 log.u_bbr.cwnd_gain <<= 1;
3698 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ss;
3699 log.u_bbr.cwnd_gain <<= 1;
3700 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ca;
3701 log.u_bbr.lost = rack->r_ctl.rc_loss_count;
3702 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3703 &rack->rc_inp->inp_socket->so_rcv,
3704 &rack->rc_inp->inp_socket->so_snd,
3705 TCP_TIMELY_WORK, 0,
3706 0, &log, false, &tv);
3707 }
3708 }
3709
3710 static int
rack_bw_can_be_raised(struct tcp_rack * rack,uint64_t cur_bw,uint64_t last_bw_est,uint16_t mult)3711 rack_bw_can_be_raised(struct tcp_rack *rack, uint64_t cur_bw, uint64_t last_bw_est, uint16_t mult)
3712 {
3713 /*
3714 * Before we increase we need to know if
3715 * the estimate just made was less than
3716 * our pacing goal (i.e. (cur_bw * mult) > last_bw_est)
3717 *
3718 * If we already are pacing at a fast enough
3719 * rate to push us faster there is no sense of
3720 * increasing.
3721 *
3722 * We first caculate our actual pacing rate (ss or ca multiplier
3723 * times our cur_bw).
3724 *
3725 * Then we take the last measured rate and multipy by our
3726 * maximum pacing overage to give us a max allowable rate.
3727 *
3728 * If our act_rate is smaller than our max_allowable rate
3729 * then we should increase. Else we should hold steady.
3730 *
3731 */
3732 uint64_t act_rate, max_allow_rate;
3733
3734 if (rack_timely_no_stopping)
3735 return (1);
3736
3737 if ((cur_bw == 0) || (last_bw_est == 0)) {
3738 /*
3739 * Initial startup case or
3740 * everything is acked case.
3741 */
3742 rack_log_timely(rack, mult, cur_bw, 0, 0,
3743 __LINE__, 9);
3744 return (1);
3745 }
3746 if (mult <= 100) {
3747 /*
3748 * We can always pace at or slightly above our rate.
3749 */
3750 rack_log_timely(rack, mult, cur_bw, 0, 0,
3751 __LINE__, 9);
3752 return (1);
3753 }
3754 act_rate = cur_bw * (uint64_t)mult;
3755 act_rate /= 100;
3756 max_allow_rate = last_bw_est * ((uint64_t)rack_max_per_above + (uint64_t)100);
3757 max_allow_rate /= 100;
3758 if (act_rate < max_allow_rate) {
3759 /*
3760 * Here the rate we are actually pacing at
3761 * is smaller than 10% above our last measurement.
3762 * This means we are pacing below what we would
3763 * like to try to achieve (plus some wiggle room).
3764 */
3765 rack_log_timely(rack, mult, cur_bw, act_rate, max_allow_rate,
3766 __LINE__, 9);
3767 return (1);
3768 } else {
3769 /*
3770 * Here we are already pacing at least rack_max_per_above(10%)
3771 * what we are getting back. This indicates most likely
3772 * that we are being limited (cwnd/rwnd/app) and can't
3773 * get any more b/w. There is no sense of trying to
3774 * raise up the pacing rate its not speeding us up
3775 * and we already are pacing faster than we are getting.
3776 */
3777 rack_log_timely(rack, mult, cur_bw, act_rate, max_allow_rate,
3778 __LINE__, 8);
3779 return (0);
3780 }
3781 }
3782
3783 static void
rack_validate_multipliers_at_or_above100(struct tcp_rack * rack)3784 rack_validate_multipliers_at_or_above100(struct tcp_rack *rack)
3785 {
3786 /*
3787 * When we drag bottom, we want to assure
3788 * that no multiplier is below 1.0, if so
3789 * we want to restore it to at least that.
3790 */
3791 if (rack->r_ctl.rack_per_of_gp_rec < 100) {
3792 /* This is unlikely we usually do not touch recovery */
3793 rack->r_ctl.rack_per_of_gp_rec = 100;
3794 }
3795 if (rack->r_ctl.rack_per_of_gp_ca < 100) {
3796 rack->r_ctl.rack_per_of_gp_ca = 100;
3797 }
3798 if (rack->r_ctl.rack_per_of_gp_ss < 100) {
3799 rack->r_ctl.rack_per_of_gp_ss = 100;
3800 }
3801 }
3802
3803 static void
rack_validate_multipliers_at_or_below_100(struct tcp_rack * rack)3804 rack_validate_multipliers_at_or_below_100(struct tcp_rack *rack)
3805 {
3806 if (rack->r_ctl.rack_per_of_gp_ca > 100) {
3807 rack->r_ctl.rack_per_of_gp_ca = 100;
3808 }
3809 if (rack->r_ctl.rack_per_of_gp_ss > 100) {
3810 rack->r_ctl.rack_per_of_gp_ss = 100;
3811 }
3812 }
3813
3814 static void
rack_increase_bw_mul(struct tcp_rack * rack,int timely_says,uint64_t cur_bw,uint64_t last_bw_est,int override)3815 rack_increase_bw_mul(struct tcp_rack *rack, int timely_says, uint64_t cur_bw, uint64_t last_bw_est, int override)
3816 {
3817 int32_t calc, logged, plus;
3818
3819 logged = 0;
3820
3821 if (rack->rc_skip_timely)
3822 return;
3823 if (override) {
3824 /*
3825 * override is passed when we are
3826 * loosing b/w and making one last
3827 * gasp at trying to not loose out
3828 * to a new-reno flow.
3829 */
3830 goto extra_boost;
3831 }
3832 /* In classic timely we boost by 5x if we have 5 increases in a row, lets not */
3833 if (rack->rc_gp_incr &&
3834 ((rack->rc_gp_timely_inc_cnt + 1) >= RACK_TIMELY_CNT_BOOST)) {
3835 /*
3836 * Reset and get 5 strokes more before the boost. Note
3837 * that the count is 0 based so we have to add one.
3838 */
3839 extra_boost:
3840 plus = (uint32_t)rack_gp_increase_per * RACK_TIMELY_CNT_BOOST;
3841 rack->rc_gp_timely_inc_cnt = 0;
3842 } else
3843 plus = (uint32_t)rack_gp_increase_per;
3844 /* Must be at least 1% increase for true timely increases */
3845 if ((plus < 1) &&
3846 ((rack->r_ctl.rc_rtt_diff <= 0) || (timely_says <= 0)))
3847 plus = 1;
3848 if (rack->rc_gp_saw_rec &&
3849 (rack->rc_gp_no_rec_chg == 0) &&
3850 rack_bw_can_be_raised(rack, cur_bw, last_bw_est,
3851 rack->r_ctl.rack_per_of_gp_rec)) {
3852 /* We have been in recovery ding it too */
3853 calc = rack->r_ctl.rack_per_of_gp_rec + plus;
3854 if (calc > 0xffff)
3855 calc = 0xffff;
3856 logged |= 1;
3857 rack->r_ctl.rack_per_of_gp_rec = (uint16_t)calc;
3858 if (rack->r_ctl.rack_per_upper_bound_ca &&
3859 (rack->rc_dragged_bottom == 0) &&
3860 (rack->r_ctl.rack_per_of_gp_rec > rack->r_ctl.rack_per_upper_bound_ca))
3861 rack->r_ctl.rack_per_of_gp_rec = rack->r_ctl.rack_per_upper_bound_ca;
3862 }
3863 if (rack->rc_gp_saw_ca &&
3864 (rack->rc_gp_saw_ss == 0) &&
3865 rack_bw_can_be_raised(rack, cur_bw, last_bw_est,
3866 rack->r_ctl.rack_per_of_gp_ca)) {
3867 /* In CA */
3868 calc = rack->r_ctl.rack_per_of_gp_ca + plus;
3869 if (calc > 0xffff)
3870 calc = 0xffff;
3871 logged |= 2;
3872 rack->r_ctl.rack_per_of_gp_ca = (uint16_t)calc;
3873 if (rack->r_ctl.rack_per_upper_bound_ca &&
3874 (rack->rc_dragged_bottom == 0) &&
3875 (rack->r_ctl.rack_per_of_gp_ca > rack->r_ctl.rack_per_upper_bound_ca))
3876 rack->r_ctl.rack_per_of_gp_ca = rack->r_ctl.rack_per_upper_bound_ca;
3877 }
3878 if (rack->rc_gp_saw_ss &&
3879 rack_bw_can_be_raised(rack, cur_bw, last_bw_est,
3880 rack->r_ctl.rack_per_of_gp_ss)) {
3881 /* In SS */
3882 calc = rack->r_ctl.rack_per_of_gp_ss + plus;
3883 if (calc > 0xffff)
3884 calc = 0xffff;
3885 rack->r_ctl.rack_per_of_gp_ss = (uint16_t)calc;
3886 if (rack->r_ctl.rack_per_upper_bound_ss &&
3887 (rack->rc_dragged_bottom == 0) &&
3888 (rack->r_ctl.rack_per_of_gp_ss > rack->r_ctl.rack_per_upper_bound_ss))
3889 rack->r_ctl.rack_per_of_gp_ss = rack->r_ctl.rack_per_upper_bound_ss;
3890 logged |= 4;
3891 }
3892 if (logged &&
3893 (rack->rc_gp_incr == 0)){
3894 /* Go into increment mode */
3895 rack->rc_gp_incr = 1;
3896 rack->rc_gp_timely_inc_cnt = 0;
3897 }
3898 if (rack->rc_gp_incr &&
3899 logged &&
3900 (rack->rc_gp_timely_inc_cnt < RACK_TIMELY_CNT_BOOST)) {
3901 rack->rc_gp_timely_inc_cnt++;
3902 }
3903 rack_log_timely(rack, logged, plus, 0, 0,
3904 __LINE__, 1);
3905 }
3906
3907 static uint32_t
rack_get_decrease(struct tcp_rack * rack,uint32_t curper,int32_t rtt_diff)3908 rack_get_decrease(struct tcp_rack *rack, uint32_t curper, int32_t rtt_diff)
3909 {
3910 /*-
3911 * norm_grad = rtt_diff / minrtt;
3912 * new_per = curper * (1 - B * norm_grad)
3913 *
3914 * B = rack_gp_decrease_per (default 80%)
3915 * rtt_dif = input var current rtt-diff
3916 * curper = input var current percentage
3917 * minrtt = from rack filter
3918 *
3919 * In order to do the floating point calculations above we
3920 * do an integer conversion. The code looks confusing so let me
3921 * translate it into something that use more variables and
3922 * is clearer for us humans :)
3923 *
3924 * uint64_t norm_grad, inverse, reduce_by, final_result;
3925 * uint32_t perf;
3926 *
3927 * norm_grad = (((uint64_t)rtt_diff * 1000000) /
3928 * (uint64_t)get_filter_small(&rack->r_ctl.rc_gp_min_rtt));
3929 * inverse = ((uint64_t)rack_gp_decrease * (uint64_t)1000000) * norm_grad;
3930 * inverse /= 1000000;
3931 * reduce_by = (1000000 - inverse);
3932 * final_result = (cur_per * reduce_by) / 1000000;
3933 * perf = (uint32_t)final_result;
3934 */
3935 uint64_t perf;
3936
3937 perf = (((uint64_t)curper * ((uint64_t)1000000 -
3938 ((uint64_t)rack_gp_decrease_per * (uint64_t)10000 *
3939 (((uint64_t)rtt_diff * (uint64_t)1000000)/
3940 (uint64_t)get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt)))/
3941 (uint64_t)1000000)) /
3942 (uint64_t)1000000);
3943 if (perf > curper) {
3944 /* TSNH */
3945 perf = curper - 1;
3946 }
3947 return ((uint32_t)perf);
3948 }
3949
3950 static uint32_t
rack_decrease_highrtt(struct tcp_rack * rack,uint32_t curper,uint32_t rtt)3951 rack_decrease_highrtt(struct tcp_rack *rack, uint32_t curper, uint32_t rtt)
3952 {
3953 /*
3954 * highrttthresh
3955 * result = curper * (1 - (B * ( 1 - ------ ))
3956 * gp_srtt
3957 *
3958 * B = rack_gp_decrease_per (default .8 i.e. 80)
3959 * highrttthresh = filter_min * rack_gp_rtt_maxmul
3960 */
3961 uint64_t perf;
3962 uint32_t highrttthresh;
3963
3964 highrttthresh = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_maxmul;
3965
3966 perf = (((uint64_t)curper * ((uint64_t)1000000 -
3967 ((uint64_t)rack_gp_decrease_per * ((uint64_t)1000000 -
3968 ((uint64_t)highrttthresh * (uint64_t)1000000) /
3969 (uint64_t)rtt)) / 100)) /(uint64_t)1000000);
3970 if (tcp_bblogging_on(rack->rc_tp)) {
3971 uint64_t log1;
3972
3973 log1 = rtt;
3974 log1 <<= 32;
3975 log1 |= highrttthresh;
3976 rack_log_timely(rack,
3977 rack_gp_decrease_per,
3978 (uint64_t)curper,
3979 log1,
3980 perf,
3981 __LINE__,
3982 15);
3983 }
3984 return (perf);
3985 }
3986
3987 static void
rack_decrease_bw_mul(struct tcp_rack * rack,int timely_says,uint32_t rtt,int32_t rtt_diff)3988 rack_decrease_bw_mul(struct tcp_rack *rack, int timely_says, uint32_t rtt, int32_t rtt_diff)
3989 {
3990 uint64_t logvar, logvar2, logvar3;
3991 uint32_t logged, new_per, ss_red, ca_red, rec_red, alt, val;
3992
3993 if (rack->rc_skip_timely)
3994 return;
3995 if (rack->rc_gp_incr) {
3996 /* Turn off increment counting */
3997 rack->rc_gp_incr = 0;
3998 rack->rc_gp_timely_inc_cnt = 0;
3999 }
4000 ss_red = ca_red = rec_red = 0;
4001 logged = 0;
4002 /* Calculate the reduction value */
4003 if (rtt_diff < 0) {
4004 rtt_diff *= -1;
4005 }
4006 /* Must be at least 1% reduction */
4007 if (rack->rc_gp_saw_rec && (rack->rc_gp_no_rec_chg == 0)) {
4008 /* We have been in recovery ding it too */
4009 if (timely_says == 2) {
4010 new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_rec, rtt);
4011 alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
4012 if (alt < new_per)
4013 val = alt;
4014 else
4015 val = new_per;
4016 } else
4017 val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
4018 if (rack->r_ctl.rack_per_of_gp_rec > val) {
4019 rec_red = (rack->r_ctl.rack_per_of_gp_rec - val);
4020 rack->r_ctl.rack_per_of_gp_rec = (uint16_t)val;
4021 } else {
4022 rack->r_ctl.rack_per_of_gp_rec = rack_per_lower_bound;
4023 rec_red = 0;
4024 }
4025 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_rec)
4026 rack->r_ctl.rack_per_of_gp_rec = rack_per_lower_bound;
4027 logged |= 1;
4028 }
4029 if (rack->rc_gp_saw_ss) {
4030 /* Sent in SS */
4031 if (timely_says == 2) {
4032 new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ss, rtt);
4033 alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ss, rtt_diff);
4034 if (alt < new_per)
4035 val = alt;
4036 else
4037 val = new_per;
4038 } else
4039 val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ss, rtt_diff);
4040 if (rack->r_ctl.rack_per_of_gp_ss > new_per) {
4041 ss_red = rack->r_ctl.rack_per_of_gp_ss - val;
4042 rack->r_ctl.rack_per_of_gp_ss = (uint16_t)val;
4043 } else {
4044 ss_red = new_per;
4045 rack->r_ctl.rack_per_of_gp_ss = rack_per_lower_bound;
4046 logvar = new_per;
4047 logvar <<= 32;
4048 logvar |= alt;
4049 logvar2 = (uint32_t)rtt;
4050 logvar2 <<= 32;
4051 logvar2 |= (uint32_t)rtt_diff;
4052 logvar3 = rack_gp_rtt_maxmul;
4053 logvar3 <<= 32;
4054 logvar3 |= get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
4055 rack_log_timely(rack, timely_says,
4056 logvar2, logvar3,
4057 logvar, __LINE__, 10);
4058 }
4059 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_ss)
4060 rack->r_ctl.rack_per_of_gp_ss = rack_per_lower_bound;
4061 logged |= 4;
4062 } else if (rack->rc_gp_saw_ca) {
4063 /* Sent in CA */
4064 if (timely_says == 2) {
4065 new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ca, rtt);
4066 alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ca, rtt_diff);
4067 if (alt < new_per)
4068 val = alt;
4069 else
4070 val = new_per;
4071 } else
4072 val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ca, rtt_diff);
4073 if (rack->r_ctl.rack_per_of_gp_ca > val) {
4074 ca_red = rack->r_ctl.rack_per_of_gp_ca - val;
4075 rack->r_ctl.rack_per_of_gp_ca = (uint16_t)val;
4076 } else {
4077 rack->r_ctl.rack_per_of_gp_ca = rack_per_lower_bound;
4078 ca_red = 0;
4079 logvar = new_per;
4080 logvar <<= 32;
4081 logvar |= alt;
4082 logvar2 = (uint32_t)rtt;
4083 logvar2 <<= 32;
4084 logvar2 |= (uint32_t)rtt_diff;
4085 logvar3 = rack_gp_rtt_maxmul;
4086 logvar3 <<= 32;
4087 logvar3 |= get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
4088 rack_log_timely(rack, timely_says,
4089 logvar2, logvar3,
4090 logvar, __LINE__, 10);
4091 }
4092 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_ca)
4093 rack->r_ctl.rack_per_of_gp_ca = rack_per_lower_bound;
4094 logged |= 2;
4095 }
4096 if (rack->rc_gp_timely_dec_cnt < 0x7) {
4097 rack->rc_gp_timely_dec_cnt++;
4098 if (rack_timely_dec_clear &&
4099 (rack->rc_gp_timely_dec_cnt == rack_timely_dec_clear))
4100 rack->rc_gp_timely_dec_cnt = 0;
4101 }
4102 logvar = ss_red;
4103 logvar <<= 32;
4104 logvar |= ca_red;
4105 rack_log_timely(rack, logged, rec_red, rack_per_lower_bound, logvar,
4106 __LINE__, 2);
4107 }
4108
4109 static void
rack_log_rtt_shrinks(struct tcp_rack * rack,uint32_t us_cts,uint32_t rtt,uint32_t line,uint8_t reas)4110 rack_log_rtt_shrinks(struct tcp_rack *rack, uint32_t us_cts,
4111 uint32_t rtt, uint32_t line, uint8_t reas)
4112 {
4113 if (tcp_bblogging_on(rack->rc_tp)) {
4114 union tcp_log_stackspecific log;
4115 struct timeval tv;
4116
4117 memset(&log, 0, sizeof(log));
4118 log.u_bbr.flex1 = line;
4119 log.u_bbr.flex2 = rack->r_ctl.rc_time_probertt_starts;
4120 log.u_bbr.flex3 = rack->r_ctl.rc_lower_rtt_us_cts;
4121 log.u_bbr.flex4 = rack->r_ctl.rack_per_of_gp_ss;
4122 log.u_bbr.flex5 = rtt;
4123 log.u_bbr.flex6 = rack->rc_highly_buffered;
4124 log.u_bbr.flex6 <<= 1;
4125 log.u_bbr.flex6 |= rack->forced_ack;
4126 log.u_bbr.flex6 <<= 1;
4127 log.u_bbr.flex6 |= rack->rc_gp_dyn_mul;
4128 log.u_bbr.flex6 <<= 1;
4129 log.u_bbr.flex6 |= rack->in_probe_rtt;
4130 log.u_bbr.flex6 <<= 1;
4131 log.u_bbr.flex6 |= rack->measure_saw_probe_rtt;
4132 log.u_bbr.flex7 = rack->r_ctl.rack_per_of_gp_probertt;
4133 log.u_bbr.pacing_gain = rack->r_ctl.rack_per_of_gp_ca;
4134 log.u_bbr.cwnd_gain = rack->r_ctl.rack_per_of_gp_rec;
4135 log.u_bbr.flex8 = reas;
4136 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4137 log.u_bbr.delRate = rack_get_bw(rack);
4138 log.u_bbr.cur_del_rate = rack->r_ctl.rc_highest_us_rtt;
4139 log.u_bbr.cur_del_rate <<= 32;
4140 log.u_bbr.cur_del_rate |= rack->r_ctl.rc_lowest_us_rtt;
4141 log.u_bbr.applimited = rack->r_ctl.rc_time_probertt_entered;
4142 log.u_bbr.pkts_out = rack->r_ctl.rc_rtt_diff;
4143 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
4144 log.u_bbr.epoch = rack->r_ctl.rc_gp_srtt;
4145 log.u_bbr.lt_epoch = rack->r_ctl.rc_prev_gp_srtt;
4146 log.u_bbr.pkt_epoch = rack->r_ctl.rc_lower_rtt_us_cts;
4147 log.u_bbr.delivered = rack->r_ctl.rc_target_probertt_flight;
4148 log.u_bbr.lost = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
4149 log.u_bbr.rttProp = us_cts;
4150 log.u_bbr.rttProp <<= 32;
4151 log.u_bbr.rttProp |= rack->r_ctl.rc_entry_gp_rtt;
4152 TCP_LOG_EVENTP(rack->rc_tp, NULL,
4153 &rack->rc_inp->inp_socket->so_rcv,
4154 &rack->rc_inp->inp_socket->so_snd,
4155 BBR_LOG_RTT_SHRINKS, 0,
4156 0, &log, false, &rack->r_ctl.act_rcv_time);
4157 }
4158 }
4159
4160 static void
rack_set_prtt_target(struct tcp_rack * rack,uint32_t segsiz,uint32_t rtt)4161 rack_set_prtt_target(struct tcp_rack *rack, uint32_t segsiz, uint32_t rtt)
4162 {
4163 uint64_t bwdp;
4164
4165 bwdp = rack_get_bw(rack);
4166 bwdp *= (uint64_t)rtt;
4167 bwdp /= (uint64_t)HPTS_USEC_IN_SEC;
4168 rack->r_ctl.rc_target_probertt_flight = roundup((uint32_t)bwdp, segsiz);
4169 if (rack->r_ctl.rc_target_probertt_flight < (segsiz * rack_timely_min_segs)) {
4170 /*
4171 * A window protocol must be able to have 4 packets
4172 * outstanding as the floor in order to function
4173 * (especially considering delayed ack :D).
4174 */
4175 rack->r_ctl.rc_target_probertt_flight = (segsiz * rack_timely_min_segs);
4176 }
4177 }
4178
4179 static void
rack_enter_probertt(struct tcp_rack * rack,uint32_t us_cts)4180 rack_enter_probertt(struct tcp_rack *rack, uint32_t us_cts)
4181 {
4182 /**
4183 * ProbeRTT is a bit different in rack_pacing than in
4184 * BBR. It is like BBR in that it uses the lowering of
4185 * the RTT as a signal that we saw something new and
4186 * counts from there for how long between. But it is
4187 * different in that its quite simple. It does not
4188 * play with the cwnd and wait until we get down
4189 * to N segments outstanding and hold that for
4190 * 200ms. Instead it just sets the pacing reduction
4191 * rate to a set percentage (70 by default) and hold
4192 * that for a number of recent GP Srtt's.
4193 */
4194 uint32_t segsiz;
4195
4196 rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
4197 if (rack->rc_gp_dyn_mul == 0)
4198 return;
4199
4200 if (rack->rc_tp->snd_max == rack->rc_tp->snd_una) {
4201 /* We are idle */
4202 return;
4203 }
4204 if ((rack->rc_tp->t_flags & TF_GPUTINPROG) &&
4205 SEQ_GT(rack->rc_tp->snd_una, rack->rc_tp->gput_seq)) {
4206 /*
4207 * Stop the goodput now, the idea here is
4208 * that future measurements with in_probe_rtt
4209 * won't register if they are not greater so
4210 * we want to get what info (if any) is available
4211 * now.
4212 */
4213 rack_do_goodput_measurement(rack->rc_tp, rack,
4214 rack->rc_tp->snd_una, __LINE__,
4215 RACK_QUALITY_PROBERTT);
4216 }
4217 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt;
4218 rack->r_ctl.rc_time_probertt_entered = us_cts;
4219 segsiz = min(ctf_fixed_maxseg(rack->rc_tp),
4220 rack->r_ctl.rc_pace_min_segs);
4221 rack->in_probe_rtt = 1;
4222 rack->measure_saw_probe_rtt = 1;
4223 rack->r_ctl.rc_time_probertt_starts = 0;
4224 rack->r_ctl.rc_entry_gp_rtt = rack->r_ctl.rc_gp_srtt;
4225 if (rack_probertt_use_min_rtt_entry)
4226 rack_set_prtt_target(rack, segsiz, get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt));
4227 else
4228 rack_set_prtt_target(rack, segsiz, rack->r_ctl.rc_gp_srtt);
4229 rack_log_rtt_shrinks(rack, us_cts, get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4230 __LINE__, RACK_RTTS_ENTERPROBE);
4231 }
4232
4233 static void
rack_exit_probertt(struct tcp_rack * rack,uint32_t us_cts)4234 rack_exit_probertt(struct tcp_rack *rack, uint32_t us_cts)
4235 {
4236 struct rack_sendmap *rsm;
4237 uint32_t segsiz;
4238
4239 segsiz = min(ctf_fixed_maxseg(rack->rc_tp),
4240 rack->r_ctl.rc_pace_min_segs);
4241 rack->in_probe_rtt = 0;
4242 if ((rack->rc_tp->t_flags & TF_GPUTINPROG) &&
4243 SEQ_GT(rack->rc_tp->snd_una, rack->rc_tp->gput_seq)) {
4244 /*
4245 * Stop the goodput now, the idea here is
4246 * that future measurements with in_probe_rtt
4247 * won't register if they are not greater so
4248 * we want to get what info (if any) is available
4249 * now.
4250 */
4251 rack_do_goodput_measurement(rack->rc_tp, rack,
4252 rack->rc_tp->snd_una, __LINE__,
4253 RACK_QUALITY_PROBERTT);
4254 } else if (rack->rc_tp->t_flags & TF_GPUTINPROG) {
4255 /*
4256 * We don't have enough data to make a measurement.
4257 * So lets just stop and start here after exiting
4258 * probe-rtt. We probably are not interested in
4259 * the results anyway.
4260 */
4261 rack->rc_tp->t_flags &= ~TF_GPUTINPROG;
4262 }
4263 /*
4264 * Measurements through the current snd_max are going
4265 * to be limited by the slower pacing rate.
4266 *
4267 * We need to mark these as app-limited so we
4268 * don't collapse the b/w.
4269 */
4270 rsm = tqhash_max(rack->r_ctl.tqh);
4271 if (rsm && ((rsm->r_flags & RACK_APP_LIMITED) == 0)) {
4272 if (rack->r_ctl.rc_app_limited_cnt == 0)
4273 rack->r_ctl.rc_end_appl = rack->r_ctl.rc_first_appl = rsm;
4274 else {
4275 /*
4276 * Go out to the end app limited and mark
4277 * this new one as next and move the end_appl up
4278 * to this guy.
4279 */
4280 if (rack->r_ctl.rc_end_appl)
4281 rack->r_ctl.rc_end_appl->r_nseq_appl = rsm->r_start;
4282 rack->r_ctl.rc_end_appl = rsm;
4283 }
4284 rsm->r_flags |= RACK_APP_LIMITED;
4285 rack->r_ctl.rc_app_limited_cnt++;
4286 }
4287 /*
4288 * Now, we need to examine our pacing rate multipliers.
4289 * If its under 100%, we need to kick it back up to
4290 * 100%. We also don't let it be over our "max" above
4291 * the actual rate i.e. 100% + rack_clamp_atexit_prtt.
4292 * Note setting clamp_atexit_prtt to 0 has the effect
4293 * of setting CA/SS to 100% always at exit (which is
4294 * the default behavior).
4295 */
4296 if (rack_probertt_clear_is) {
4297 rack->rc_gp_incr = 0;
4298 rack->rc_gp_bwred = 0;
4299 rack->rc_gp_timely_inc_cnt = 0;
4300 rack->rc_gp_timely_dec_cnt = 0;
4301 }
4302 /* Do we do any clamping at exit? */
4303 if (rack->rc_highly_buffered && rack_atexit_prtt_hbp) {
4304 rack->r_ctl.rack_per_of_gp_ca = rack_atexit_prtt_hbp;
4305 rack->r_ctl.rack_per_of_gp_ss = rack_atexit_prtt_hbp;
4306 }
4307 if ((rack->rc_highly_buffered == 0) && rack_atexit_prtt) {
4308 rack->r_ctl.rack_per_of_gp_ca = rack_atexit_prtt;
4309 rack->r_ctl.rack_per_of_gp_ss = rack_atexit_prtt;
4310 }
4311 /*
4312 * Lets set rtt_diff to 0, so that we will get a "boost"
4313 * after exiting.
4314 */
4315 rack->r_ctl.rc_rtt_diff = 0;
4316
4317 /* Clear all flags so we start fresh */
4318 rack->rc_tp->t_bytes_acked = 0;
4319 rack->rc_tp->t_ccv.flags &= ~CCF_ABC_SENTAWND;
4320 /*
4321 * If configured to, set the cwnd and ssthresh to
4322 * our targets.
4323 */
4324 if (rack_probe_rtt_sets_cwnd) {
4325 uint64_t ebdp;
4326 uint32_t setto;
4327
4328 /* Set ssthresh so we get into CA once we hit our target */
4329 if (rack_probertt_use_min_rtt_exit == 1) {
4330 /* Set to min rtt */
4331 rack_set_prtt_target(rack, segsiz,
4332 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt));
4333 } else if (rack_probertt_use_min_rtt_exit == 2) {
4334 /* Set to current gp rtt */
4335 rack_set_prtt_target(rack, segsiz,
4336 rack->r_ctl.rc_gp_srtt);
4337 } else if (rack_probertt_use_min_rtt_exit == 3) {
4338 /* Set to entry gp rtt */
4339 rack_set_prtt_target(rack, segsiz,
4340 rack->r_ctl.rc_entry_gp_rtt);
4341 } else {
4342 uint64_t sum;
4343 uint32_t setval;
4344
4345 sum = rack->r_ctl.rc_entry_gp_rtt;
4346 sum *= 10;
4347 sum /= (uint64_t)(max(1, rack->r_ctl.rc_gp_srtt));
4348 if (sum >= 20) {
4349 /*
4350 * A highly buffered path needs
4351 * cwnd space for timely to work.
4352 * Lets set things up as if
4353 * we are heading back here again.
4354 */
4355 setval = rack->r_ctl.rc_entry_gp_rtt;
4356 } else if (sum >= 15) {
4357 /*
4358 * Lets take the smaller of the
4359 * two since we are just somewhat
4360 * buffered.
4361 */
4362 setval = rack->r_ctl.rc_gp_srtt;
4363 if (setval > rack->r_ctl.rc_entry_gp_rtt)
4364 setval = rack->r_ctl.rc_entry_gp_rtt;
4365 } else {
4366 /*
4367 * Here we are not highly buffered
4368 * and should pick the min we can to
4369 * keep from causing loss.
4370 */
4371 setval = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
4372 }
4373 rack_set_prtt_target(rack, segsiz,
4374 setval);
4375 }
4376 if (rack_probe_rtt_sets_cwnd > 1) {
4377 /* There is a percentage here to boost */
4378 ebdp = rack->r_ctl.rc_target_probertt_flight;
4379 ebdp *= rack_probe_rtt_sets_cwnd;
4380 ebdp /= 100;
4381 setto = rack->r_ctl.rc_target_probertt_flight + ebdp;
4382 } else
4383 setto = rack->r_ctl.rc_target_probertt_flight;
4384 rack->rc_tp->snd_cwnd = roundup(setto, segsiz);
4385 if (rack->rc_tp->snd_cwnd < (segsiz * rack_timely_min_segs)) {
4386 /* Enforce a min */
4387 rack->rc_tp->snd_cwnd = segsiz * rack_timely_min_segs;
4388 }
4389 /* If we set in the cwnd also set the ssthresh point so we are in CA */
4390 rack->rc_tp->snd_ssthresh = (rack->rc_tp->snd_cwnd - 1);
4391 }
4392 rack_log_rtt_shrinks(rack, us_cts,
4393 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4394 __LINE__, RACK_RTTS_EXITPROBE);
4395 /* Clear times last so log has all the info */
4396 rack->r_ctl.rc_probertt_sndmax_atexit = rack->rc_tp->snd_max;
4397 rack->r_ctl.rc_time_probertt_entered = us_cts;
4398 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
4399 rack->r_ctl.rc_time_of_last_probertt = us_cts;
4400 }
4401
4402 static void
rack_check_probe_rtt(struct tcp_rack * rack,uint32_t us_cts)4403 rack_check_probe_rtt(struct tcp_rack *rack, uint32_t us_cts)
4404 {
4405 /* Check in on probe-rtt */
4406
4407 if (rack->rc_gp_filled == 0) {
4408 /* We do not do p-rtt unless we have gp measurements */
4409 return;
4410 }
4411 if (rack->in_probe_rtt) {
4412 uint64_t no_overflow;
4413 uint32_t endtime, must_stay;
4414
4415 if (rack->r_ctl.rc_went_idle_time &&
4416 ((us_cts - rack->r_ctl.rc_went_idle_time) > rack_min_probertt_hold)) {
4417 /*
4418 * We went idle during prtt, just exit now.
4419 */
4420 rack_exit_probertt(rack, us_cts);
4421 } else if (rack_probe_rtt_safety_val &&
4422 TSTMP_GT(us_cts, rack->r_ctl.rc_time_probertt_entered) &&
4423 ((us_cts - rack->r_ctl.rc_time_probertt_entered) > rack_probe_rtt_safety_val)) {
4424 /*
4425 * Probe RTT safety value triggered!
4426 */
4427 rack_log_rtt_shrinks(rack, us_cts,
4428 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4429 __LINE__, RACK_RTTS_SAFETY);
4430 rack_exit_probertt(rack, us_cts);
4431 }
4432 /* Calculate the max we will wait */
4433 endtime = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_max_drain_wait);
4434 if (rack->rc_highly_buffered)
4435 endtime += (rack->r_ctl.rc_gp_srtt * rack_max_drain_hbp);
4436 /* Calculate the min we must wait */
4437 must_stay = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_must_drain);
4438 if ((ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.rc_target_probertt_flight) &&
4439 TSTMP_LT(us_cts, endtime)) {
4440 uint32_t calc;
4441 /* Do we lower more? */
4442 no_exit:
4443 if (TSTMP_GT(us_cts, rack->r_ctl.rc_time_probertt_entered))
4444 calc = us_cts - rack->r_ctl.rc_time_probertt_entered;
4445 else
4446 calc = 0;
4447 calc /= max(rack->r_ctl.rc_gp_srtt, 1);
4448 if (calc) {
4449 /* Maybe */
4450 calc *= rack_per_of_gp_probertt_reduce;
4451 if (calc > rack_per_of_gp_probertt)
4452 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_lowthresh;
4453 else
4454 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt - calc;
4455 /* Limit it too */
4456 if (rack->r_ctl.rack_per_of_gp_probertt < rack_per_of_gp_lowthresh)
4457 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_lowthresh;
4458 }
4459 /* We must reach target or the time set */
4460 return;
4461 }
4462 if (rack->r_ctl.rc_time_probertt_starts == 0) {
4463 if ((TSTMP_LT(us_cts, must_stay) &&
4464 rack->rc_highly_buffered) ||
4465 (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) >
4466 rack->r_ctl.rc_target_probertt_flight)) {
4467 /* We are not past the must_stay time */
4468 goto no_exit;
4469 }
4470 rack_log_rtt_shrinks(rack, us_cts,
4471 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4472 __LINE__, RACK_RTTS_REACHTARGET);
4473 rack->r_ctl.rc_time_probertt_starts = us_cts;
4474 if (rack->r_ctl.rc_time_probertt_starts == 0)
4475 rack->r_ctl.rc_time_probertt_starts = 1;
4476 /* Restore back to our rate we want to pace at in prtt */
4477 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt;
4478 }
4479 /*
4480 * Setup our end time, some number of gp_srtts plus 200ms.
4481 */
4482 no_overflow = ((uint64_t)rack->r_ctl.rc_gp_srtt *
4483 (uint64_t)rack_probertt_gpsrtt_cnt_mul);
4484 if (rack_probertt_gpsrtt_cnt_div)
4485 endtime = (uint32_t)(no_overflow / (uint64_t)rack_probertt_gpsrtt_cnt_div);
4486 else
4487 endtime = 0;
4488 endtime += rack_min_probertt_hold;
4489 endtime += rack->r_ctl.rc_time_probertt_starts;
4490 if (TSTMP_GEQ(us_cts, endtime)) {
4491 /* yes, exit probertt */
4492 rack_exit_probertt(rack, us_cts);
4493 }
4494
4495 } else if ((rack->rc_skip_timely == 0) &&
4496 (TSTMP_GT(us_cts, rack->r_ctl.rc_lower_rtt_us_cts)) &&
4497 ((us_cts - rack->r_ctl.rc_lower_rtt_us_cts) >= rack_time_between_probertt)) {
4498 /* Go into probertt, its been too long since we went lower */
4499 rack_enter_probertt(rack, us_cts);
4500 }
4501 }
4502
4503 static void
rack_update_multiplier(struct tcp_rack * rack,int32_t timely_says,uint64_t last_bw_est,uint32_t rtt,int32_t rtt_diff)4504 rack_update_multiplier(struct tcp_rack *rack, int32_t timely_says, uint64_t last_bw_est,
4505 uint32_t rtt, int32_t rtt_diff)
4506 {
4507 uint64_t cur_bw, up_bnd, low_bnd, subfr;
4508 uint32_t losses;
4509
4510 if ((rack->rc_gp_dyn_mul == 0) ||
4511 (rack->use_fixed_rate) ||
4512 (rack->in_probe_rtt) ||
4513 (rack->rc_always_pace == 0)) {
4514 /* No dynamic GP multiplier in play */
4515 return;
4516 }
4517 losses = rack->r_ctl.rc_loss_count - rack->r_ctl.rc_loss_at_start;
4518 cur_bw = rack_get_bw(rack);
4519 /* Calculate our up and down range */
4520 up_bnd = rack->r_ctl.last_gp_comp_bw * (uint64_t)rack_gp_per_bw_mul_up;
4521 up_bnd /= 100;
4522 up_bnd += rack->r_ctl.last_gp_comp_bw;
4523
4524 subfr = (uint64_t)rack->r_ctl.last_gp_comp_bw * (uint64_t)rack_gp_per_bw_mul_down;
4525 subfr /= 100;
4526 low_bnd = rack->r_ctl.last_gp_comp_bw - subfr;
4527 if ((timely_says == 2) && (rack->r_ctl.rc_no_push_at_mrtt)) {
4528 /*
4529 * This is the case where our RTT is above
4530 * the max target and we have been configured
4531 * to just do timely no bonus up stuff in that case.
4532 *
4533 * There are two configurations, set to 1, and we
4534 * just do timely if we are over our max. If its
4535 * set above 1 then we slam the multipliers down
4536 * to 100 and then decrement per timely.
4537 */
4538 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd,
4539 __LINE__, 3);
4540 if (rack->r_ctl.rc_no_push_at_mrtt > 1)
4541 rack_validate_multipliers_at_or_below_100(rack);
4542 rack_decrease_bw_mul(rack, timely_says, rtt, rtt_diff);
4543 } else if ((timely_says != 0) && (last_bw_est < low_bnd) && !losses) {
4544 /*
4545 * We are decreasing this is a bit complicated this
4546 * means we are loosing ground. This could be
4547 * because another flow entered and we are competing
4548 * for b/w with it. This will push the RTT up which
4549 * makes timely unusable unless we want to get shoved
4550 * into a corner and just be backed off (the age
4551 * old problem with delay based CC).
4552 *
4553 * On the other hand if it was a route change we
4554 * would like to stay somewhat contained and not
4555 * blow out the buffers.
4556 */
4557 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd,
4558 __LINE__, 3);
4559 rack->r_ctl.last_gp_comp_bw = cur_bw;
4560 if (rack->rc_gp_bwred == 0) {
4561 /* Go into reduction counting */
4562 rack->rc_gp_bwred = 1;
4563 rack->rc_gp_timely_dec_cnt = 0;
4564 }
4565 if (rack->rc_gp_timely_dec_cnt < rack_timely_max_push_drop) {
4566 /*
4567 * Push another time with a faster pacing
4568 * to try to gain back (we include override to
4569 * get a full raise factor).
4570 */
4571 if ((rack->rc_gp_saw_ca && rack->r_ctl.rack_per_of_gp_ca <= rack_down_raise_thresh) ||
4572 (rack->rc_gp_saw_ss && rack->r_ctl.rack_per_of_gp_ss <= rack_down_raise_thresh) ||
4573 (timely_says == 0) ||
4574 (rack_down_raise_thresh == 0)) {
4575 /*
4576 * Do an override up in b/w if we were
4577 * below the threshold or if the threshold
4578 * is zero we always do the raise.
4579 */
4580 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 1);
4581 } else {
4582 /* Log it stays the same */
4583 rack_log_timely(rack, 0, last_bw_est, low_bnd, 0,
4584 __LINE__, 11);
4585 }
4586 rack->rc_gp_timely_dec_cnt++;
4587 /* We are not incrementing really no-count */
4588 rack->rc_gp_incr = 0;
4589 rack->rc_gp_timely_inc_cnt = 0;
4590 } else {
4591 /*
4592 * Lets just use the RTT
4593 * information and give up
4594 * pushing.
4595 */
4596 goto use_timely;
4597 }
4598 } else if ((timely_says != 2) &&
4599 !losses &&
4600 (last_bw_est > up_bnd)) {
4601 /*
4602 * We are increasing b/w lets keep going, updating
4603 * our b/w and ignoring any timely input, unless
4604 * of course we are at our max raise (if there is one).
4605 */
4606
4607 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd,
4608 __LINE__, 3);
4609 rack->r_ctl.last_gp_comp_bw = cur_bw;
4610 if (rack->rc_gp_saw_ss &&
4611 rack->r_ctl.rack_per_upper_bound_ss &&
4612 (rack->r_ctl.rack_per_of_gp_ss == rack->r_ctl.rack_per_upper_bound_ss)) {
4613 /*
4614 * In cases where we can't go higher
4615 * we should just use timely.
4616 */
4617 goto use_timely;
4618 }
4619 if (rack->rc_gp_saw_ca &&
4620 rack->r_ctl.rack_per_upper_bound_ca &&
4621 (rack->r_ctl.rack_per_of_gp_ca == rack->r_ctl.rack_per_upper_bound_ca)) {
4622 /*
4623 * In cases where we can't go higher
4624 * we should just use timely.
4625 */
4626 goto use_timely;
4627 }
4628 rack->rc_gp_bwred = 0;
4629 rack->rc_gp_timely_dec_cnt = 0;
4630 /* You get a set number of pushes if timely is trying to reduce */
4631 if ((rack->rc_gp_incr < rack_timely_max_push_rise) || (timely_says == 0)) {
4632 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0);
4633 } else {
4634 /* Log it stays the same */
4635 rack_log_timely(rack, 0, last_bw_est, up_bnd, 0,
4636 __LINE__, 12);
4637 }
4638 return;
4639 } else {
4640 /*
4641 * We are staying between the lower and upper range bounds
4642 * so use timely to decide.
4643 */
4644 rack_log_timely(rack, timely_says, cur_bw, low_bnd, up_bnd,
4645 __LINE__, 3);
4646 use_timely:
4647 if (timely_says) {
4648 rack->rc_gp_incr = 0;
4649 rack->rc_gp_timely_inc_cnt = 0;
4650 if ((rack->rc_gp_timely_dec_cnt < rack_timely_max_push_drop) &&
4651 !losses &&
4652 (last_bw_est < low_bnd)) {
4653 /* We are loosing ground */
4654 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0);
4655 rack->rc_gp_timely_dec_cnt++;
4656 /* We are not incrementing really no-count */
4657 rack->rc_gp_incr = 0;
4658 rack->rc_gp_timely_inc_cnt = 0;
4659 } else
4660 rack_decrease_bw_mul(rack, timely_says, rtt, rtt_diff);
4661 } else {
4662 rack->rc_gp_bwred = 0;
4663 rack->rc_gp_timely_dec_cnt = 0;
4664 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0);
4665 }
4666 }
4667 }
4668
4669 static int32_t
rack_make_timely_judgement(struct tcp_rack * rack,uint32_t rtt,int32_t rtt_diff,uint32_t prev_rtt)4670 rack_make_timely_judgement(struct tcp_rack *rack, uint32_t rtt, int32_t rtt_diff, uint32_t prev_rtt)
4671 {
4672 int32_t timely_says;
4673 uint64_t log_mult, log_rtt_a_diff;
4674
4675 log_rtt_a_diff = rtt;
4676 log_rtt_a_diff <<= 32;
4677 log_rtt_a_diff |= (uint32_t)rtt_diff;
4678 if (rtt >= (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) *
4679 rack_gp_rtt_maxmul)) {
4680 /* Reduce the b/w multiplier */
4681 timely_says = 2;
4682 log_mult = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_maxmul;
4683 log_mult <<= 32;
4684 log_mult |= prev_rtt;
4685 rack_log_timely(rack, timely_says, log_mult,
4686 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4687 log_rtt_a_diff, __LINE__, 4);
4688 } else if (rtt <= (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) +
4689 ((get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_minmul) /
4690 max(rack_gp_rtt_mindiv , 1)))) {
4691 /* Increase the b/w multiplier */
4692 log_mult = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) +
4693 ((get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_minmul) /
4694 max(rack_gp_rtt_mindiv , 1));
4695 log_mult <<= 32;
4696 log_mult |= prev_rtt;
4697 timely_says = 0;
4698 rack_log_timely(rack, timely_says, log_mult ,
4699 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4700 log_rtt_a_diff, __LINE__, 5);
4701 } else {
4702 /*
4703 * Use a gradient to find it the timely gradient
4704 * is:
4705 * grad = rc_rtt_diff / min_rtt;
4706 *
4707 * anything below or equal to 0 will be
4708 * a increase indication. Anything above
4709 * zero is a decrease. Note we take care
4710 * of the actual gradient calculation
4711 * in the reduction (its not needed for
4712 * increase).
4713 */
4714 log_mult = prev_rtt;
4715 if (rtt_diff <= 0) {
4716 /*
4717 * Rttdiff is less than zero, increase the
4718 * b/w multiplier (its 0 or negative)
4719 */
4720 timely_says = 0;
4721 rack_log_timely(rack, timely_says, log_mult,
4722 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), log_rtt_a_diff, __LINE__, 6);
4723 } else {
4724 /* Reduce the b/w multiplier */
4725 timely_says = 1;
4726 rack_log_timely(rack, timely_says, log_mult,
4727 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), log_rtt_a_diff, __LINE__, 7);
4728 }
4729 }
4730 return (timely_says);
4731 }
4732
4733 static __inline int
rack_in_gp_window(struct tcpcb * tp,struct rack_sendmap * rsm)4734 rack_in_gp_window(struct tcpcb *tp, struct rack_sendmap *rsm)
4735 {
4736 if (SEQ_GEQ(rsm->r_start, tp->gput_seq) &&
4737 SEQ_LEQ(rsm->r_end, tp->gput_ack)) {
4738 /**
4739 * This covers the case that the
4740 * resent is completely inside
4741 * the gp range or up to it.
4742 * |----------------|
4743 * |-----| <or>
4744 * |----|
4745 * <or> |---|
4746 */
4747 return (1);
4748 } else if (SEQ_LT(rsm->r_start, tp->gput_seq) &&
4749 SEQ_GT(rsm->r_end, tp->gput_seq)){
4750 /**
4751 * This covers the case of
4752 * |--------------|
4753 * |-------->|
4754 */
4755 return (1);
4756 } else if (SEQ_GEQ(rsm->r_start, tp->gput_seq) &&
4757 SEQ_LT(rsm->r_start, tp->gput_ack) &&
4758 SEQ_GEQ(rsm->r_end, tp->gput_ack)) {
4759
4760 /**
4761 * This covers the case of
4762 * |--------------|
4763 * |-------->|
4764 */
4765 return (1);
4766 }
4767 return (0);
4768 }
4769
4770 static __inline void
rack_mark_in_gp_win(struct tcpcb * tp,struct rack_sendmap * rsm)4771 rack_mark_in_gp_win(struct tcpcb *tp, struct rack_sendmap *rsm)
4772 {
4773
4774 if ((tp->t_flags & TF_GPUTINPROG) == 0)
4775 return;
4776 /*
4777 * We have a Goodput measurement in progress. Mark
4778 * the send if its within the window. If its not
4779 * in the window make sure it does not have the mark.
4780 */
4781 if (rack_in_gp_window(tp, rsm))
4782 rsm->r_flags |= RACK_IN_GP_WIN;
4783 else
4784 rsm->r_flags &= ~RACK_IN_GP_WIN;
4785 }
4786
4787 static __inline void
rack_clear_gp_marks(struct tcpcb * tp,struct tcp_rack * rack)4788 rack_clear_gp_marks(struct tcpcb *tp, struct tcp_rack *rack)
4789 {
4790 /* A GP measurement is ending, clear all marks on the send map*/
4791 struct rack_sendmap *rsm = NULL;
4792
4793 rsm = tqhash_find(rack->r_ctl.tqh, tp->gput_seq);
4794 if (rsm == NULL) {
4795 rsm = tqhash_min(rack->r_ctl.tqh);
4796 }
4797 /* Nothing left? */
4798 while ((rsm != NULL) && (SEQ_GEQ(tp->gput_ack, rsm->r_start))){
4799 rsm->r_flags &= ~RACK_IN_GP_WIN;
4800 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
4801 }
4802 }
4803
4804
4805 static __inline void
rack_tend_gp_marks(struct tcpcb * tp,struct tcp_rack * rack)4806 rack_tend_gp_marks(struct tcpcb *tp, struct tcp_rack *rack)
4807 {
4808 struct rack_sendmap *rsm = NULL;
4809
4810 if (tp->snd_una == tp->snd_max) {
4811 /* Nothing outstanding yet, nothing to do here */
4812 return;
4813 }
4814 if (SEQ_GT(tp->gput_seq, tp->snd_una)) {
4815 /*
4816 * We are measuring ahead of some outstanding
4817 * data. We need to walk through up until we get
4818 * to gp_seq marking so that no rsm is set incorrectly
4819 * with RACK_IN_GP_WIN.
4820 */
4821 rsm = tqhash_min(rack->r_ctl.tqh);
4822 while (rsm != NULL) {
4823 rack_mark_in_gp_win(tp, rsm);
4824 if (SEQ_GEQ(rsm->r_end, tp->gput_seq))
4825 break;
4826 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
4827 }
4828 }
4829 if (rsm == NULL) {
4830 /*
4831 * Need to find the GP seq, if rsm is
4832 * set we stopped as we hit it.
4833 */
4834 rsm = tqhash_find(rack->r_ctl.tqh, tp->gput_seq);
4835 if (rsm == NULL)
4836 return;
4837 rack_mark_in_gp_win(tp, rsm);
4838 }
4839 /*
4840 * Now we may need to mark already sent rsm, ahead of
4841 * gput_seq in the window since they may have been sent
4842 * *before* we started our measurment. The rsm, if non-null
4843 * has been marked (note if rsm would have been NULL we would have
4844 * returned in the previous block). So we go to the next, and continue
4845 * until we run out of entries or we exceed the gp_ack value.
4846 */
4847 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
4848 while (rsm) {
4849 rack_mark_in_gp_win(tp, rsm);
4850 if (SEQ_GT(rsm->r_end, tp->gput_ack))
4851 break;
4852 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
4853 }
4854 }
4855
4856 static void
rack_log_gp_calc(struct tcp_rack * rack,uint32_t add_part,uint32_t sub_part,uint32_t srtt,uint64_t meas_bw,uint64_t utim,uint8_t meth,uint32_t line)4857 rack_log_gp_calc(struct tcp_rack *rack, uint32_t add_part, uint32_t sub_part, uint32_t srtt, uint64_t meas_bw, uint64_t utim, uint8_t meth, uint32_t line)
4858 {
4859 if (tcp_bblogging_on(rack->rc_tp)) {
4860 union tcp_log_stackspecific log;
4861 struct timeval tv;
4862
4863 memset(&log, 0, sizeof(log));
4864 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4865 log.u_bbr.flex1 = add_part;
4866 log.u_bbr.flex2 = sub_part;
4867 log.u_bbr.flex3 = rack_wma_divisor;
4868 log.u_bbr.flex4 = srtt;
4869 log.u_bbr.flex7 = (uint16_t)line;
4870 log.u_bbr.flex8 = meth;
4871 log.u_bbr.delRate = rack->r_ctl.gp_bw;
4872 log.u_bbr.cur_del_rate = meas_bw;
4873 log.u_bbr.rttProp = utim;
4874 TCP_LOG_EVENTP(rack->rc_tp, NULL,
4875 &rack->rc_inp->inp_socket->so_rcv,
4876 &rack->rc_inp->inp_socket->so_snd,
4877 BBR_LOG_THRESH_CALC, 0,
4878 0, &log, false, &rack->r_ctl.act_rcv_time);
4879 }
4880 }
4881
4882 static void
rack_do_goodput_measurement(struct tcpcb * tp,struct tcp_rack * rack,tcp_seq th_ack,int line,uint8_t quality)4883 rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
4884 tcp_seq th_ack, int line, uint8_t quality)
4885 {
4886 uint64_t tim, bytes_ps, stim, utim;
4887 uint32_t segsiz, bytes, reqbytes, us_cts;
4888 int32_t gput, new_rtt_diff, timely_says;
4889 uint64_t resid_bw, subpart = 0, addpart = 0, srtt;
4890 int did_add = 0;
4891
4892 us_cts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
4893 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
4894 if (TSTMP_GEQ(us_cts, tp->gput_ts))
4895 tim = us_cts - tp->gput_ts;
4896 else
4897 tim = 0;
4898 if (rack->r_ctl.rc_gp_cumack_ts > rack->r_ctl.rc_gp_output_ts)
4899 stim = rack->r_ctl.rc_gp_cumack_ts - rack->r_ctl.rc_gp_output_ts;
4900 else
4901 stim = 0;
4902 /*
4903 * Use the larger of the send time or ack time. This prevents us
4904 * from being influenced by ack artifacts to come up with too
4905 * high of measurement. Note that since we are spanning over many more
4906 * bytes in most of our measurements hopefully that is less likely to
4907 * occur.
4908 */
4909 if (tim > stim)
4910 utim = max(tim, 1);
4911 else
4912 utim = max(stim, 1);
4913 reqbytes = min(rc_init_window(rack), (MIN_GP_WIN * segsiz));
4914 rack_log_gpset(rack, th_ack, us_cts, rack->r_ctl.rc_gp_cumack_ts, __LINE__, 3, NULL);
4915 if ((tim == 0) && (stim == 0)) {
4916 /*
4917 * Invalid measurement time, maybe
4918 * all on one ack/one send?
4919 */
4920 bytes = 0;
4921 bytes_ps = 0;
4922 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4923 0, 0, 0, 10, __LINE__, NULL, quality);
4924 goto skip_measurement;
4925 }
4926 if (rack->r_ctl.rc_gp_lowrtt == 0xffffffff) {
4927 /* We never made a us_rtt measurement? */
4928 bytes = 0;
4929 bytes_ps = 0;
4930 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4931 0, 0, 0, 10, __LINE__, NULL, quality);
4932 goto skip_measurement;
4933 }
4934 /*
4935 * Calculate the maximum possible b/w this connection
4936 * could have. We base our calculation on the lowest
4937 * rtt we have seen during the measurement and the
4938 * largest rwnd the client has given us in that time. This
4939 * forms a BDP that is the maximum that we could ever
4940 * get to the client. Anything larger is not valid.
4941 *
4942 * I originally had code here that rejected measurements
4943 * where the time was less than 1/2 the latest us_rtt.
4944 * But after thinking on that I realized its wrong since
4945 * say you had a 150Mbps or even 1Gbps link, and you
4946 * were a long way away.. example I am in Europe (100ms rtt)
4947 * talking to my 1Gbps link in S.C. Now measuring say 150,000
4948 * bytes my time would be 1.2ms, and yet my rtt would say
4949 * the measurement was invalid the time was < 50ms. The
4950 * same thing is true for 150Mb (8ms of time).
4951 *
4952 * A better way I realized is to look at what the maximum
4953 * the connection could possibly do. This is gated on
4954 * the lowest RTT we have seen and the highest rwnd.
4955 * We should in theory never exceed that, if we are
4956 * then something on the path is storing up packets
4957 * and then feeding them all at once to our endpoint
4958 * messing up our measurement.
4959 */
4960 rack->r_ctl.last_max_bw = rack->r_ctl.rc_gp_high_rwnd;
4961 rack->r_ctl.last_max_bw *= HPTS_USEC_IN_SEC;
4962 rack->r_ctl.last_max_bw /= rack->r_ctl.rc_gp_lowrtt;
4963 if (SEQ_LT(th_ack, tp->gput_seq)) {
4964 /* No measurement can be made */
4965 bytes = 0;
4966 bytes_ps = 0;
4967 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4968 0, 0, 0, 10, __LINE__, NULL, quality);
4969 goto skip_measurement;
4970 } else
4971 bytes = (th_ack - tp->gput_seq);
4972 bytes_ps = (uint64_t)bytes;
4973 /*
4974 * Don't measure a b/w for pacing unless we have gotten at least
4975 * an initial windows worth of data in this measurement interval.
4976 *
4977 * Small numbers of bytes get badly influenced by delayed ack and
4978 * other artifacts. Note we take the initial window or our
4979 * defined minimum GP (defaulting to 10 which hopefully is the
4980 * IW).
4981 */
4982 if (rack->rc_gp_filled == 0) {
4983 /*
4984 * The initial estimate is special. We
4985 * have blasted out an IW worth of packets
4986 * without a real valid ack ts results. We
4987 * then setup the app_limited_needs_set flag,
4988 * this should get the first ack in (probably 2
4989 * MSS worth) to be recorded as the timestamp.
4990 * We thus allow a smaller number of bytes i.e.
4991 * IW - 2MSS.
4992 */
4993 reqbytes -= (2 * segsiz);
4994 /* Also lets fill previous for our first measurement to be neutral */
4995 rack->r_ctl.rc_prev_gp_srtt = rack->r_ctl.rc_gp_srtt;
4996 }
4997 if ((bytes_ps < reqbytes) || rack->app_limited_needs_set) {
4998 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4999 rack->r_ctl.rc_app_limited_cnt,
5000 0, 0, 10, __LINE__, NULL, quality);
5001 goto skip_measurement;
5002 }
5003 /*
5004 * We now need to calculate the Timely like status so
5005 * we can update (possibly) the b/w multipliers.
5006 */
5007 new_rtt_diff = (int32_t)rack->r_ctl.rc_gp_srtt - (int32_t)rack->r_ctl.rc_prev_gp_srtt;
5008 if (rack->rc_gp_filled == 0) {
5009 /* No previous reading */
5010 rack->r_ctl.rc_rtt_diff = new_rtt_diff;
5011 } else {
5012 if (rack->measure_saw_probe_rtt == 0) {
5013 /*
5014 * We don't want a probertt to be counted
5015 * since it will be negative incorrectly. We
5016 * expect to be reducing the RTT when we
5017 * pace at a slower rate.
5018 */
5019 rack->r_ctl.rc_rtt_diff -= (rack->r_ctl.rc_rtt_diff / 8);
5020 rack->r_ctl.rc_rtt_diff += (new_rtt_diff / 8);
5021 }
5022 }
5023 timely_says = rack_make_timely_judgement(rack,
5024 rack->r_ctl.rc_gp_srtt,
5025 rack->r_ctl.rc_rtt_diff,
5026 rack->r_ctl.rc_prev_gp_srtt
5027 );
5028 bytes_ps *= HPTS_USEC_IN_SEC;
5029 bytes_ps /= utim;
5030 if (bytes_ps > rack->r_ctl.last_max_bw) {
5031 /*
5032 * Something is on path playing
5033 * since this b/w is not possible based
5034 * on our BDP (highest rwnd and lowest rtt
5035 * we saw in the measurement window).
5036 *
5037 * Another option here would be to
5038 * instead skip the measurement.
5039 */
5040 rack_log_pacing_delay_calc(rack, bytes, reqbytes,
5041 bytes_ps, rack->r_ctl.last_max_bw, 0,
5042 11, __LINE__, NULL, quality);
5043 bytes_ps = rack->r_ctl.last_max_bw;
5044 }
5045 /* We store gp for b/w in bytes per second */
5046 if (rack->rc_gp_filled == 0) {
5047 /* Initial measurement */
5048 if (bytes_ps) {
5049 rack->r_ctl.gp_bw = bytes_ps;
5050 rack->rc_gp_filled = 1;
5051 rack->r_ctl.num_measurements = 1;
5052 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
5053 } else {
5054 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
5055 rack->r_ctl.rc_app_limited_cnt,
5056 0, 0, 10, __LINE__, NULL, quality);
5057 }
5058 if (tcp_in_hpts(rack->rc_tp) &&
5059 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
5060 /*
5061 * Ok we can't trust the pacer in this case
5062 * where we transition from un-paced to paced.
5063 * Or for that matter when the burst mitigation
5064 * was making a wild guess and got it wrong.
5065 * Stop the pacer and clear up all the aggregate
5066 * delays etc.
5067 */
5068 tcp_hpts_remove(rack->rc_tp);
5069 rack->r_ctl.rc_hpts_flags = 0;
5070 rack->r_ctl.rc_last_output_to = 0;
5071 }
5072 did_add = 2;
5073 } else if (rack->r_ctl.num_measurements < RACK_REQ_AVG) {
5074 /* Still a small number run an average */
5075 rack->r_ctl.gp_bw += bytes_ps;
5076 addpart = rack->r_ctl.num_measurements;
5077 rack->r_ctl.num_measurements++;
5078 if (rack->r_ctl.num_measurements >= RACK_REQ_AVG) {
5079 /* We have collected enough to move forward */
5080 rack->r_ctl.gp_bw /= (uint64_t)rack->r_ctl.num_measurements;
5081 }
5082 rack_set_pace_segments(tp, rack, __LINE__, NULL);
5083 did_add = 3;
5084 } else {
5085 /*
5086 * We want to take 1/wma of the goodput and add in to 7/8th
5087 * of the old value weighted by the srtt. So if your measurement
5088 * period is say 2 SRTT's long you would get 1/4 as the
5089 * value, if it was like 1/2 SRTT then you would get 1/16th.
5090 *
5091 * But we must be careful not to take too much i.e. if the
5092 * srtt is say 20ms and the measurement is taken over
5093 * 400ms our weight would be 400/20 i.e. 20. On the
5094 * other hand if we get a measurement over 1ms with a
5095 * 10ms rtt we only want to take a much smaller portion.
5096 */
5097 uint8_t meth;
5098
5099 if (rack->r_ctl.num_measurements < 0xff) {
5100 rack->r_ctl.num_measurements++;
5101 }
5102 srtt = (uint64_t)tp->t_srtt;
5103 if (srtt == 0) {
5104 /*
5105 * Strange why did t_srtt go back to zero?
5106 */
5107 if (rack->r_ctl.rc_rack_min_rtt)
5108 srtt = rack->r_ctl.rc_rack_min_rtt;
5109 else
5110 srtt = HPTS_USEC_IN_MSEC;
5111 }
5112 /*
5113 * XXXrrs: Note for reviewers, in playing with
5114 * dynamic pacing I discovered this GP calculation
5115 * as done originally leads to some undesired results.
5116 * Basically you can get longer measurements contributing
5117 * too much to the WMA. Thus I changed it if you are doing
5118 * dynamic adjustments to only do the aportioned adjustment
5119 * if we have a very small (time wise) measurement. Longer
5120 * measurements just get there weight (defaulting to 1/8)
5121 * add to the WMA. We may want to think about changing
5122 * this to always do that for both sides i.e. dynamic
5123 * and non-dynamic... but considering lots of folks
5124 * were playing with this I did not want to change the
5125 * calculation per.se. without your thoughts.. Lawerence?
5126 * Peter??
5127 */
5128 if (rack->rc_gp_dyn_mul == 0) {
5129 subpart = rack->r_ctl.gp_bw * utim;
5130 subpart /= (srtt * 8);
5131 if (subpart < (rack->r_ctl.gp_bw / 2)) {
5132 /*
5133 * The b/w update takes no more
5134 * away then 1/2 our running total
5135 * so factor it in.
5136 */
5137 addpart = bytes_ps * utim;
5138 addpart /= (srtt * 8);
5139 meth = 1;
5140 } else {
5141 /*
5142 * Don't allow a single measurement
5143 * to account for more than 1/2 of the
5144 * WMA. This could happen on a retransmission
5145 * where utim becomes huge compared to
5146 * srtt (multiple retransmissions when using
5147 * the sending rate which factors in all the
5148 * transmissions from the first one).
5149 */
5150 subpart = rack->r_ctl.gp_bw / 2;
5151 addpart = bytes_ps / 2;
5152 meth = 2;
5153 }
5154 rack_log_gp_calc(rack, addpart, subpart, srtt, bytes_ps, utim, meth, __LINE__);
5155 resid_bw = rack->r_ctl.gp_bw - subpart;
5156 rack->r_ctl.gp_bw = resid_bw + addpart;
5157 did_add = 1;
5158 } else {
5159 if ((utim / srtt) <= 1) {
5160 /*
5161 * The b/w update was over a small period
5162 * of time. The idea here is to prevent a small
5163 * measurement time period from counting
5164 * too much. So we scale it based on the
5165 * time so it attributes less than 1/rack_wma_divisor
5166 * of its measurement.
5167 */
5168 subpart = rack->r_ctl.gp_bw * utim;
5169 subpart /= (srtt * rack_wma_divisor);
5170 addpart = bytes_ps * utim;
5171 addpart /= (srtt * rack_wma_divisor);
5172 meth = 3;
5173 } else {
5174 /*
5175 * The scaled measurement was long
5176 * enough so lets just add in the
5177 * portion of the measurement i.e. 1/rack_wma_divisor
5178 */
5179 subpart = rack->r_ctl.gp_bw / rack_wma_divisor;
5180 addpart = bytes_ps / rack_wma_divisor;
5181 meth = 4;
5182 }
5183 if ((rack->measure_saw_probe_rtt == 0) ||
5184 (bytes_ps > rack->r_ctl.gp_bw)) {
5185 /*
5186 * For probe-rtt we only add it in
5187 * if its larger, all others we just
5188 * add in.
5189 */
5190 did_add = 1;
5191 rack_log_gp_calc(rack, addpart, subpart, srtt, bytes_ps, utim, meth, __LINE__);
5192 resid_bw = rack->r_ctl.gp_bw - subpart;
5193 rack->r_ctl.gp_bw = resid_bw + addpart;
5194 }
5195 }
5196 rack_set_pace_segments(tp, rack, __LINE__, NULL);
5197 }
5198 /*
5199 * We only watch the growth of the GP during the initial startup
5200 * or first-slowstart that ensues. If we ever needed to watch
5201 * growth of gp outside of that period all we need to do is
5202 * remove the first clause of this if (rc_initial_ss_comp).
5203 */
5204 if ((rack->rc_initial_ss_comp == 0) &&
5205 (rack->r_ctl.num_measurements >= RACK_REQ_AVG)) {
5206 uint64_t gp_est;
5207
5208 gp_est = bytes_ps;
5209 if (tcp_bblogging_on(rack->rc_tp)) {
5210 union tcp_log_stackspecific log;
5211 struct timeval tv;
5212
5213 memset(&log, 0, sizeof(log));
5214 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
5215 log.u_bbr.flex1 = rack->r_ctl.current_round;
5216 log.u_bbr.flex2 = rack->r_ctl.last_rnd_of_gp_rise;
5217 log.u_bbr.delRate = gp_est;
5218 log.u_bbr.cur_del_rate = rack->r_ctl.last_gpest;
5219 log.u_bbr.flex8 = 41;
5220 (void)tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
5221 0, &log, false, NULL, __func__, __LINE__,&tv);
5222 }
5223 if ((rack->r_ctl.num_measurements == RACK_REQ_AVG) ||
5224 (rack->r_ctl.last_gpest == 0)) {
5225 /*
5226 * The round we get our measurement averaging going
5227 * is the base round so it always is the source point
5228 * for when we had our first increment. From there on
5229 * we only record the round that had a rise.
5230 */
5231 rack->r_ctl.last_rnd_of_gp_rise = rack->r_ctl.current_round;
5232 rack->r_ctl.last_gpest = rack->r_ctl.gp_bw;
5233 } else if (gp_est >= rack->r_ctl.last_gpest) {
5234 /*
5235 * Test to see if its gone up enough
5236 * to set the round count up to now. Note
5237 * that on the seeding of the 4th measurement we
5238 */
5239 gp_est *= 1000;
5240 gp_est /= rack->r_ctl.last_gpest;
5241 if ((uint32_t)gp_est > rack->r_ctl.gp_gain_req) {
5242 /*
5243 * We went up enough to record the round.
5244 */
5245 if (tcp_bblogging_on(rack->rc_tp)) {
5246 union tcp_log_stackspecific log;
5247 struct timeval tv;
5248
5249 memset(&log, 0, sizeof(log));
5250 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
5251 log.u_bbr.flex1 = rack->r_ctl.current_round;
5252 log.u_bbr.flex2 = (uint32_t)gp_est;
5253 log.u_bbr.flex3 = rack->r_ctl.gp_gain_req;
5254 log.u_bbr.delRate = gp_est;
5255 log.u_bbr.cur_del_rate = rack->r_ctl.last_gpest;
5256 log.u_bbr.flex8 = 42;
5257 (void)tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
5258 0, &log, false, NULL, __func__, __LINE__,&tv);
5259 }
5260 rack->r_ctl.last_rnd_of_gp_rise = rack->r_ctl.current_round;
5261 if (rack->r_ctl.use_gp_not_last == 1)
5262 rack->r_ctl.last_gpest = rack->r_ctl.gp_bw;
5263 else
5264 rack->r_ctl.last_gpest = bytes_ps;
5265 }
5266 }
5267 }
5268 if ((rack->gp_ready == 0) &&
5269 (rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) {
5270 /* We have enough measurements now */
5271 rack->gp_ready = 1;
5272 if (rack->dgp_on ||
5273 rack->rack_hibeta)
5274 rack_set_cc_pacing(rack);
5275 if (rack->defer_options)
5276 rack_apply_deferred_options(rack);
5277 }
5278 rack_log_pacing_delay_calc(rack, subpart, addpart, bytes_ps, stim,
5279 rack_get_bw(rack), 22, did_add, NULL, quality);
5280 /* We do not update any multipliers if we are in or have seen a probe-rtt */
5281
5282 if ((rack->measure_saw_probe_rtt == 0) &&
5283 rack->rc_gp_rtt_set) {
5284 if (rack->rc_skip_timely == 0) {
5285 rack_update_multiplier(rack, timely_says, bytes_ps,
5286 rack->r_ctl.rc_gp_srtt,
5287 rack->r_ctl.rc_rtt_diff);
5288 }
5289 }
5290 rack_log_pacing_delay_calc(rack, bytes, tim, bytes_ps, stim,
5291 rack_get_bw(rack), 3, line, NULL, quality);
5292 rack_log_pacing_delay_calc(rack,
5293 bytes, /* flex2 */
5294 tim, /* flex1 */
5295 bytes_ps, /* bw_inuse */
5296 rack->r_ctl.gp_bw, /* delRate */
5297 rack_get_lt_bw(rack), /* rttProp */
5298 20, line, NULL, 0);
5299 /* reset the gp srtt and setup the new prev */
5300 rack->r_ctl.rc_prev_gp_srtt = rack->r_ctl.rc_gp_srtt;
5301 /* Record the lost count for the next measurement */
5302 rack->r_ctl.rc_loss_at_start = rack->r_ctl.rc_loss_count;
5303 skip_measurement:
5304 /*
5305 * We restart our diffs based on the gpsrtt in the
5306 * measurement window.
5307 */
5308 rack->rc_gp_rtt_set = 0;
5309 rack->rc_gp_saw_rec = 0;
5310 rack->rc_gp_saw_ca = 0;
5311 rack->rc_gp_saw_ss = 0;
5312 rack->rc_dragged_bottom = 0;
5313 if (quality == RACK_QUALITY_HIGH) {
5314 /*
5315 * Gput in the stats world is in kbps where bytes_ps is
5316 * bytes per second so we do ((x * 8)/ 1000).
5317 */
5318 gput = (int32_t)((bytes_ps << 3) / (uint64_t)1000);
5319 #ifdef STATS
5320 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
5321 gput);
5322 /*
5323 * XXXLAS: This is a temporary hack, and should be
5324 * chained off VOI_TCP_GPUT when stats(9) grows an
5325 * API to deal with chained VOIs.
5326 */
5327 if (tp->t_stats_gput_prev > 0)
5328 stats_voi_update_abs_s32(tp->t_stats,
5329 VOI_TCP_GPUT_ND,
5330 ((gput - tp->t_stats_gput_prev) * 100) /
5331 tp->t_stats_gput_prev);
5332 #endif
5333 tp->t_stats_gput_prev = gput;
5334 }
5335 tp->t_flags &= ~TF_GPUTINPROG;
5336 /*
5337 * Now are we app limited now and there is space from where we
5338 * were to where we want to go?
5339 *
5340 * We don't do the other case i.e. non-applimited here since
5341 * the next send will trigger us picking up the missing data.
5342 */
5343 if (rack->r_ctl.rc_first_appl &&
5344 TCPS_HAVEESTABLISHED(tp->t_state) &&
5345 rack->r_ctl.rc_app_limited_cnt &&
5346 (SEQ_GT(rack->r_ctl.rc_first_appl->r_start, th_ack)) &&
5347 ((rack->r_ctl.rc_first_appl->r_end - th_ack) >
5348 max(rc_init_window(rack), (MIN_GP_WIN * segsiz)))) {
5349 /*
5350 * Yep there is enough outstanding to make a measurement here.
5351 */
5352 struct rack_sendmap *rsm;
5353
5354 rack->r_ctl.rc_gp_lowrtt = 0xffffffff;
5355 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd;
5356 tp->gput_ts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
5357 rack->app_limited_needs_set = 0;
5358 tp->gput_seq = th_ack;
5359 if (rack->in_probe_rtt)
5360 rack->measure_saw_probe_rtt = 1;
5361 else if ((rack->measure_saw_probe_rtt) &&
5362 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit)))
5363 rack->measure_saw_probe_rtt = 0;
5364 if ((rack->r_ctl.rc_first_appl->r_end - th_ack) >= rack_get_measure_window(tp, rack)) {
5365 /* There is a full window to gain info from */
5366 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack);
5367 } else {
5368 /* We can only measure up to the applimited point */
5369 tp->gput_ack = tp->gput_seq + (rack->r_ctl.rc_first_appl->r_end - th_ack);
5370 if ((tp->gput_ack - tp->gput_seq) < (MIN_GP_WIN * segsiz)) {
5371 /*
5372 * We don't have enough to make a measurement.
5373 */
5374 tp->t_flags &= ~TF_GPUTINPROG;
5375 rack_log_pacing_delay_calc(rack, tp->gput_ack, tp->gput_seq,
5376 0, 0, 0, 6, __LINE__, NULL, quality);
5377 return;
5378 }
5379 }
5380 if (tp->t_state >= TCPS_FIN_WAIT_1) {
5381 /*
5382 * We will get no more data into the SB
5383 * this means we need to have the data available
5384 * before we start a measurement.
5385 */
5386 if (sbavail(&tptosocket(tp)->so_snd) < (tp->gput_ack - tp->gput_seq)) {
5387 /* Nope not enough data. */
5388 return;
5389 }
5390 }
5391 tp->t_flags |= TF_GPUTINPROG;
5392 /*
5393 * Now we need to find the timestamp of the send at tp->gput_seq
5394 * for the send based measurement.
5395 */
5396 rack->r_ctl.rc_gp_cumack_ts = 0;
5397 rsm = tqhash_find(rack->r_ctl.tqh, tp->gput_seq);
5398 if (rsm) {
5399 /* Ok send-based limit is set */
5400 if (SEQ_LT(rsm->r_start, tp->gput_seq)) {
5401 /*
5402 * Move back to include the earlier part
5403 * so our ack time lines up right (this may
5404 * make an overlapping measurement but thats
5405 * ok).
5406 */
5407 tp->gput_seq = rsm->r_start;
5408 }
5409 if (rsm->r_flags & RACK_ACKED) {
5410 struct rack_sendmap *nrsm;
5411
5412 tp->gput_ts = (uint32_t)rsm->r_ack_arrival;
5413 tp->gput_seq = rsm->r_end;
5414 nrsm = tqhash_next(rack->r_ctl.tqh, rsm);
5415 if (nrsm)
5416 rsm = nrsm;
5417 else {
5418 rack->app_limited_needs_set = 1;
5419 }
5420 } else
5421 rack->app_limited_needs_set = 1;
5422 /* We always go from the first send */
5423 rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[0];
5424 } else {
5425 /*
5426 * If we don't find the rsm due to some
5427 * send-limit set the current time, which
5428 * basically disables the send-limit.
5429 */
5430 struct timeval tv;
5431
5432 microuptime(&tv);
5433 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv);
5434 }
5435 rack_tend_gp_marks(tp, rack);
5436 rack_log_pacing_delay_calc(rack,
5437 tp->gput_seq,
5438 tp->gput_ack,
5439 (uintptr_t)rsm,
5440 tp->gput_ts,
5441 (((uint64_t)rack->r_ctl.rc_app_limited_cnt << 32) | (uint64_t)rack->r_ctl.rc_gp_output_ts),
5442 9,
5443 __LINE__, rsm, quality);
5444 rack_log_gpset(rack, tp->gput_ack, 0, 0, __LINE__, 1, NULL);
5445 } else {
5446 /*
5447 * To make sure proper timestamp merging occurs, we need to clear
5448 * all GP marks if we don't start a measurement.
5449 */
5450 rack_clear_gp_marks(tp, rack);
5451 }
5452 }
5453
5454 /*
5455 * CC wrapper hook functions
5456 */
5457 static void
rack_ack_received(struct tcpcb * tp,struct tcp_rack * rack,uint32_t th_ack,uint16_t nsegs,uint16_t type,int32_t post_recovery)5458 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack, uint32_t th_ack, uint16_t nsegs,
5459 uint16_t type, int32_t post_recovery)
5460 {
5461 uint32_t prior_cwnd, acked;
5462 struct tcp_log_buffer *lgb = NULL;
5463 uint8_t labc_to_use, quality;
5464
5465 INP_WLOCK_ASSERT(tptoinpcb(tp));
5466 tp->t_ccv.nsegs = nsegs;
5467 acked = tp->t_ccv.bytes_this_ack = (th_ack - tp->snd_una);
5468 if ((post_recovery) && (rack->r_ctl.rc_early_recovery_segs)) {
5469 uint32_t max;
5470
5471 max = rack->r_ctl.rc_early_recovery_segs * ctf_fixed_maxseg(tp);
5472 if (tp->t_ccv.bytes_this_ack > max) {
5473 tp->t_ccv.bytes_this_ack = max;
5474 }
5475 }
5476 #ifdef STATS
5477 stats_voi_update_abs_s32(tp->t_stats, VOI_TCP_CALCFRWINDIFF,
5478 ((int32_t)rack->r_ctl.cwnd_to_use) - tp->snd_wnd);
5479 #endif
5480 if ((th_ack == tp->snd_max) && rack->lt_bw_up) {
5481 /*
5482 * We will ack all the data, time to end any
5483 * lt_bw_up we have running until something
5484 * new is sent. Note we need to use the actual
5485 * ack_rcv_time which with pacing may be different.
5486 */
5487 uint64_t tmark;
5488
5489 rack->r_ctl.lt_bw_bytes += (tp->snd_max - rack->r_ctl.lt_seq);
5490 rack->r_ctl.lt_seq = tp->snd_max;
5491 tmark = tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time);
5492 if (tmark >= rack->r_ctl.lt_timemark) {
5493 rack->r_ctl.lt_bw_time += (tmark - rack->r_ctl.lt_timemark);
5494 }
5495 rack->r_ctl.lt_timemark = tmark;
5496 rack->lt_bw_up = 0;
5497 }
5498 quality = RACK_QUALITY_NONE;
5499 if ((tp->t_flags & TF_GPUTINPROG) &&
5500 rack_enough_for_measurement(tp, rack, th_ack, &quality)) {
5501 /* Measure the Goodput */
5502 rack_do_goodput_measurement(tp, rack, th_ack, __LINE__, quality);
5503 }
5504 /* Which way our we limited, if not cwnd limited no advance in CA */
5505 if (tp->snd_cwnd <= tp->snd_wnd)
5506 tp->t_ccv.flags |= CCF_CWND_LIMITED;
5507 else
5508 tp->t_ccv.flags &= ~CCF_CWND_LIMITED;
5509 if (tp->snd_cwnd > tp->snd_ssthresh) {
5510 tp->t_bytes_acked += min(tp->t_ccv.bytes_this_ack,
5511 nsegs * V_tcp_abc_l_var * ctf_fixed_maxseg(tp));
5512 /* For the setting of a window past use the actual scwnd we are using */
5513 if (tp->t_bytes_acked >= rack->r_ctl.cwnd_to_use) {
5514 tp->t_bytes_acked -= rack->r_ctl.cwnd_to_use;
5515 tp->t_ccv.flags |= CCF_ABC_SENTAWND;
5516 }
5517 } else {
5518 tp->t_ccv.flags &= ~CCF_ABC_SENTAWND;
5519 tp->t_bytes_acked = 0;
5520 }
5521 prior_cwnd = tp->snd_cwnd;
5522 if ((post_recovery == 0) || (rack_max_abc_post_recovery == 0) || rack->r_use_labc_for_rec ||
5523 (rack_client_low_buf && rack->client_bufferlvl &&
5524 (rack->client_bufferlvl < rack_client_low_buf)))
5525 labc_to_use = rack->rc_labc;
5526 else
5527 labc_to_use = rack_max_abc_post_recovery;
5528 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
5529 union tcp_log_stackspecific log;
5530 struct timeval tv;
5531
5532 memset(&log, 0, sizeof(log));
5533 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
5534 log.u_bbr.flex1 = th_ack;
5535 log.u_bbr.flex2 = tp->t_ccv.flags;
5536 log.u_bbr.flex3 = tp->t_ccv.bytes_this_ack;
5537 log.u_bbr.flex4 = tp->t_ccv.nsegs;
5538 log.u_bbr.flex5 = labc_to_use;
5539 log.u_bbr.flex6 = prior_cwnd;
5540 log.u_bbr.flex7 = V_tcp_do_newsack;
5541 log.u_bbr.flex8 = 1;
5542 lgb = tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
5543 0, &log, false, NULL, __func__, __LINE__,&tv);
5544 }
5545 if (CC_ALGO(tp)->ack_received != NULL) {
5546 /* XXXLAS: Find a way to live without this */
5547 tp->t_ccv.curack = th_ack;
5548 tp->t_ccv.labc = labc_to_use;
5549 tp->t_ccv.flags |= CCF_USE_LOCAL_ABC;
5550 CC_ALGO(tp)->ack_received(&tp->t_ccv, type);
5551 }
5552 if (lgb) {
5553 lgb->tlb_stackinfo.u_bbr.flex6 = tp->snd_cwnd;
5554 }
5555 if (rack->r_must_retran) {
5556 if (SEQ_GEQ(th_ack, rack->r_ctl.rc_snd_max_at_rto)) {
5557 /*
5558 * We now are beyond the rxt point so lets disable
5559 * the flag.
5560 */
5561 rack->r_ctl.rc_out_at_rto = 0;
5562 rack->r_must_retran = 0;
5563 } else if ((prior_cwnd + ctf_fixed_maxseg(tp)) <= tp->snd_cwnd) {
5564 /*
5565 * Only decrement the rc_out_at_rto if the cwnd advances
5566 * at least a whole segment. Otherwise next time the peer
5567 * acks, we won't be able to send this generaly happens
5568 * when we are in Congestion Avoidance.
5569 */
5570 if (acked <= rack->r_ctl.rc_out_at_rto){
5571 rack->r_ctl.rc_out_at_rto -= acked;
5572 } else {
5573 rack->r_ctl.rc_out_at_rto = 0;
5574 }
5575 }
5576 }
5577 #ifdef STATS
5578 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_LCWIN, rack->r_ctl.cwnd_to_use);
5579 #endif
5580 if (rack->r_ctl.rc_rack_largest_cwnd < rack->r_ctl.cwnd_to_use) {
5581 rack->r_ctl.rc_rack_largest_cwnd = rack->r_ctl.cwnd_to_use;
5582 }
5583 if ((rack->rc_initial_ss_comp == 0) &&
5584 (tp->snd_cwnd >= tp->snd_ssthresh)) {
5585 /*
5586 * The cwnd has grown beyond ssthresh we have
5587 * entered ca and completed our first Slowstart.
5588 */
5589 rack->rc_initial_ss_comp = 1;
5590 }
5591 }
5592
5593 static void
tcp_rack_partialack(struct tcpcb * tp)5594 tcp_rack_partialack(struct tcpcb *tp)
5595 {
5596 struct tcp_rack *rack;
5597
5598 rack = (struct tcp_rack *)tp->t_fb_ptr;
5599 INP_WLOCK_ASSERT(tptoinpcb(tp));
5600 /*
5601 * If we are doing PRR and have enough
5602 * room to send <or> we are pacing and prr
5603 * is disabled we will want to see if we
5604 * can send data (by setting r_wanted_output to
5605 * true).
5606 */
5607 if ((rack->r_ctl.rc_prr_sndcnt > 0) ||
5608 rack->rack_no_prr)
5609 rack->r_wanted_output = 1;
5610 }
5611
5612 static void
rack_exit_recovery(struct tcpcb * tp,struct tcp_rack * rack,int how)5613 rack_exit_recovery(struct tcpcb *tp, struct tcp_rack *rack, int how)
5614 {
5615 /*
5616 * Now exit recovery.
5617 */
5618 EXIT_RECOVERY(tp->t_flags);
5619 }
5620
5621 static void
rack_post_recovery(struct tcpcb * tp,uint32_t th_ack)5622 rack_post_recovery(struct tcpcb *tp, uint32_t th_ack)
5623 {
5624 struct tcp_rack *rack;
5625 uint32_t orig_cwnd;
5626
5627 orig_cwnd = tp->snd_cwnd;
5628 INP_WLOCK_ASSERT(tptoinpcb(tp));
5629 rack = (struct tcp_rack *)tp->t_fb_ptr;
5630 /* only alert CC if we alerted when we entered */
5631 if (CC_ALGO(tp)->post_recovery != NULL) {
5632 tp->t_ccv.curack = th_ack;
5633 CC_ALGO(tp)->post_recovery(&tp->t_ccv);
5634 if (tp->snd_cwnd < tp->snd_ssthresh) {
5635 /*
5636 * Rack has burst control and pacing
5637 * so lets not set this any lower than
5638 * snd_ssthresh per RFC-6582 (option 2).
5639 */
5640 tp->snd_cwnd = tp->snd_ssthresh;
5641 }
5642 }
5643 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
5644 union tcp_log_stackspecific log;
5645 struct timeval tv;
5646
5647 memset(&log, 0, sizeof(log));
5648 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
5649 log.u_bbr.flex1 = th_ack;
5650 log.u_bbr.flex2 = tp->t_ccv.flags;
5651 log.u_bbr.flex3 = tp->t_ccv.bytes_this_ack;
5652 log.u_bbr.flex4 = tp->t_ccv.nsegs;
5653 log.u_bbr.flex5 = V_tcp_abc_l_var;
5654 log.u_bbr.flex6 = orig_cwnd;
5655 log.u_bbr.flex7 = V_tcp_do_newsack;
5656 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
5657 log.u_bbr.flex8 = 2;
5658 tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
5659 0, &log, false, NULL, __func__, __LINE__, &tv);
5660 }
5661 if ((rack->rack_no_prr == 0) &&
5662 (rack->no_prr_addback == 0) &&
5663 (rack->r_ctl.rc_prr_sndcnt > 0)) {
5664 /*
5665 * Suck the next prr cnt back into cwnd, but
5666 * only do that if we are not application limited.
5667 */
5668 if (ctf_outstanding(tp) <= sbavail(&tptosocket(tp)->so_snd)) {
5669 /*
5670 * We are allowed to add back to the cwnd the amount we did
5671 * not get out if:
5672 * a) no_prr_addback is off.
5673 * b) we are not app limited
5674 * c) we are doing prr
5675 * <and>
5676 * d) it is bounded by rack_prr_addbackmax (if addback is 0, then none).
5677 */
5678 tp->snd_cwnd += min((ctf_fixed_maxseg(tp) * rack_prr_addbackmax),
5679 rack->r_ctl.rc_prr_sndcnt);
5680 }
5681 rack->r_ctl.rc_prr_sndcnt = 0;
5682 rack_log_to_prr(rack, 1, 0, __LINE__);
5683 }
5684 rack_log_to_prr(rack, 14, orig_cwnd, __LINE__);
5685 tp->snd_recover = tp->snd_una;
5686 if (rack->r_ctl.dsack_persist) {
5687 rack->r_ctl.dsack_persist--;
5688 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) {
5689 rack->r_ctl.num_dsack = 0;
5690 }
5691 rack_log_dsack_event(rack, 1, __LINE__, 0, 0);
5692 }
5693 if (rack->rto_from_rec == 1) {
5694 rack->rto_from_rec = 0;
5695 if (rack->r_ctl.rto_ssthresh > tp->snd_ssthresh)
5696 tp->snd_ssthresh = rack->r_ctl.rto_ssthresh;
5697 }
5698 rack_exit_recovery(tp, rack, 1);
5699 }
5700
5701 static void
rack_cong_signal(struct tcpcb * tp,uint32_t type,uint32_t ack,int line)5702 rack_cong_signal(struct tcpcb *tp, uint32_t type, uint32_t ack, int line)
5703 {
5704 struct tcp_rack *rack;
5705 uint32_t ssthresh_enter, cwnd_enter, in_rec_at_entry, orig_cwnd;
5706
5707 INP_WLOCK_ASSERT(tptoinpcb(tp));
5708 #ifdef STATS
5709 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);
5710 #endif
5711 if (IN_RECOVERY(tp->t_flags) == 0) {
5712 in_rec_at_entry = 0;
5713 ssthresh_enter = tp->snd_ssthresh;
5714 cwnd_enter = tp->snd_cwnd;
5715 } else
5716 in_rec_at_entry = 1;
5717 rack = (struct tcp_rack *)tp->t_fb_ptr;
5718 switch (type) {
5719 case CC_NDUPACK:
5720 tp->t_flags &= ~TF_WASFRECOVERY;
5721 tp->t_flags &= ~TF_WASCRECOVERY;
5722 if (!IN_FASTRECOVERY(tp->t_flags)) {
5723 /* Check if this is the end of the initial Start-up i.e. initial slow-start */
5724 if (rack->rc_initial_ss_comp == 0) {
5725 /* Yep it is the end of the initial slowstart */
5726 rack->rc_initial_ss_comp = 1;
5727 }
5728 rack->r_ctl.rc_prr_delivered = 0;
5729 rack->r_ctl.rc_prr_out = 0;
5730 rack->r_fast_output = 0;
5731 if (rack->rack_no_prr == 0) {
5732 rack->r_ctl.rc_prr_sndcnt = ctf_fixed_maxseg(tp);
5733 rack_log_to_prr(rack, 2, in_rec_at_entry, line);
5734 }
5735 rack->r_ctl.rc_prr_recovery_fs = tp->snd_max - tp->snd_una;
5736 tp->snd_recover = tp->snd_max;
5737 if (tp->t_flags2 & TF2_ECN_PERMIT)
5738 tp->t_flags2 |= TF2_ECN_SND_CWR;
5739 }
5740 break;
5741 case CC_ECN:
5742 if (!IN_CONGRECOVERY(tp->t_flags) ||
5743 /*
5744 * Allow ECN reaction on ACK to CWR, if
5745 * that data segment was also CE marked.
5746 */
5747 SEQ_GEQ(ack, tp->snd_recover)) {
5748 EXIT_CONGRECOVERY(tp->t_flags);
5749 KMOD_TCPSTAT_INC(tcps_ecn_rcwnd);
5750 rack->r_fast_output = 0;
5751 tp->snd_recover = tp->snd_max + 1;
5752 if (tp->t_flags2 & TF2_ECN_PERMIT)
5753 tp->t_flags2 |= TF2_ECN_SND_CWR;
5754 }
5755 break;
5756 case CC_RTO:
5757 tp->t_dupacks = 0;
5758 tp->t_bytes_acked = 0;
5759 rack->r_fast_output = 0;
5760 if (IN_RECOVERY(tp->t_flags))
5761 rack_exit_recovery(tp, rack, 2);
5762 orig_cwnd = tp->snd_cwnd;
5763 rack_log_to_prr(rack, 16, orig_cwnd, line);
5764 if (CC_ALGO(tp)->cong_signal == NULL) {
5765 /* TSNH */
5766 tp->snd_ssthresh = max(2,
5767 min(tp->snd_wnd, rack->r_ctl.cwnd_to_use) / 2 /
5768 ctf_fixed_maxseg(tp)) * ctf_fixed_maxseg(tp);
5769 tp->snd_cwnd = ctf_fixed_maxseg(tp);
5770 }
5771 if (tp->t_flags2 & TF2_ECN_PERMIT)
5772 tp->t_flags2 |= TF2_ECN_SND_CWR;
5773 break;
5774 case CC_RTO_ERR:
5775 KMOD_TCPSTAT_INC(tcps_sndrexmitbad);
5776 /* RTO was unnecessary, so reset everything. */
5777 tp->snd_cwnd = tp->snd_cwnd_prev;
5778 tp->snd_ssthresh = tp->snd_ssthresh_prev;
5779 tp->snd_recover = tp->snd_recover_prev;
5780 if (tp->t_flags & TF_WASFRECOVERY) {
5781 ENTER_FASTRECOVERY(tp->t_flags);
5782 tp->t_flags &= ~TF_WASFRECOVERY;
5783 }
5784 if (tp->t_flags & TF_WASCRECOVERY) {
5785 ENTER_CONGRECOVERY(tp->t_flags);
5786 tp->t_flags &= ~TF_WASCRECOVERY;
5787 }
5788 tp->snd_nxt = tp->snd_max;
5789 tp->t_badrxtwin = 0;
5790 break;
5791 }
5792 if ((CC_ALGO(tp)->cong_signal != NULL) &&
5793 (type != CC_RTO)){
5794 tp->t_ccv.curack = ack;
5795 CC_ALGO(tp)->cong_signal(&tp->t_ccv, type);
5796 }
5797 if ((in_rec_at_entry == 0) && IN_RECOVERY(tp->t_flags)) {
5798 rack_log_to_prr(rack, 15, cwnd_enter, line);
5799 rack->r_ctl.dsack_byte_cnt = 0;
5800 rack->r_ctl.retran_during_recovery = 0;
5801 rack->r_ctl.rc_cwnd_at_erec = cwnd_enter;
5802 rack->r_ctl.rc_ssthresh_at_erec = ssthresh_enter;
5803 rack->r_ent_rec_ns = 1;
5804 }
5805 }
5806
5807 static inline void
rack_cc_after_idle(struct tcp_rack * rack,struct tcpcb * tp)5808 rack_cc_after_idle(struct tcp_rack *rack, struct tcpcb *tp)
5809 {
5810 uint32_t i_cwnd;
5811
5812 INP_WLOCK_ASSERT(tptoinpcb(tp));
5813
5814 if (CC_ALGO(tp)->after_idle != NULL)
5815 CC_ALGO(tp)->after_idle(&tp->t_ccv);
5816
5817 if (tp->snd_cwnd == 1)
5818 i_cwnd = tp->t_maxseg; /* SYN(-ACK) lost */
5819 else
5820 i_cwnd = rc_init_window(rack);
5821
5822 /*
5823 * Being idle is no different than the initial window. If the cc
5824 * clamps it down below the initial window raise it to the initial
5825 * window.
5826 */
5827 if (tp->snd_cwnd < i_cwnd) {
5828 tp->snd_cwnd = i_cwnd;
5829 }
5830 }
5831
5832 /*
5833 * Indicate whether this ack should be delayed. We can delay the ack if
5834 * following conditions are met:
5835 * - There is no delayed ack timer in progress.
5836 * - Our last ack wasn't a 0-sized window. We never want to delay
5837 * the ack that opens up a 0-sized window.
5838 * - LRO wasn't used for this segment. We make sure by checking that the
5839 * segment size is not larger than the MSS.
5840 * - Delayed acks are enabled or this is a half-synchronized T/TCP
5841 * connection.
5842 */
5843 #define DELAY_ACK(tp, tlen) \
5844 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
5845 ((tp->t_flags & TF_DELACK) == 0) && \
5846 (tlen <= tp->t_maxseg) && \
5847 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
5848
5849 static struct rack_sendmap *
rack_find_lowest_rsm(struct tcp_rack * rack)5850 rack_find_lowest_rsm(struct tcp_rack *rack)
5851 {
5852 struct rack_sendmap *rsm;
5853
5854 /*
5855 * Walk the time-order transmitted list looking for an rsm that is
5856 * not acked. This will be the one that was sent the longest time
5857 * ago that is still outstanding.
5858 */
5859 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
5860 if (rsm->r_flags & RACK_ACKED) {
5861 continue;
5862 }
5863 goto finish;
5864 }
5865 finish:
5866 return (rsm);
5867 }
5868
5869 static struct rack_sendmap *
rack_find_high_nonack(struct tcp_rack * rack,struct rack_sendmap * rsm)5870 rack_find_high_nonack(struct tcp_rack *rack, struct rack_sendmap *rsm)
5871 {
5872 struct rack_sendmap *prsm;
5873
5874 /*
5875 * Walk the sequence order list backward until we hit and arrive at
5876 * the highest seq not acked. In theory when this is called it
5877 * should be the last segment (which it was not).
5878 */
5879 prsm = rsm;
5880
5881 TQHASH_FOREACH_REVERSE_FROM(prsm, rack->r_ctl.tqh) {
5882 if (prsm->r_flags & (RACK_ACKED | RACK_HAS_FIN)) {
5883 continue;
5884 }
5885 return (prsm);
5886 }
5887 return (NULL);
5888 }
5889
5890 static uint32_t
rack_calc_thresh_rack(struct tcp_rack * rack,uint32_t srtt,uint32_t cts,int line,int log_allowed)5891 rack_calc_thresh_rack(struct tcp_rack *rack, uint32_t srtt, uint32_t cts, int line, int log_allowed)
5892 {
5893 int32_t lro;
5894 uint32_t thresh;
5895
5896 /*
5897 * lro is the flag we use to determine if we have seen reordering.
5898 * If it gets set we have seen reordering. The reorder logic either
5899 * works in one of two ways:
5900 *
5901 * If reorder-fade is configured, then we track the last time we saw
5902 * re-ordering occur. If we reach the point where enough time as
5903 * passed we no longer consider reordering as occurring.
5904 *
5905 * Or if reorder-face is 0, then once we see reordering we consider
5906 * the connection to alway be subject to reordering and just set lro
5907 * to 1.
5908 *
5909 * In the end if lro is non-zero we add the extra time for
5910 * reordering in.
5911 */
5912 if (srtt == 0)
5913 srtt = 1;
5914 if (rack->r_ctl.rc_reorder_ts) {
5915 if (rack->r_ctl.rc_reorder_fade) {
5916 if (SEQ_GEQ(cts, rack->r_ctl.rc_reorder_ts)) {
5917 lro = cts - rack->r_ctl.rc_reorder_ts;
5918 if (lro == 0) {
5919 /*
5920 * No time as passed since the last
5921 * reorder, mark it as reordering.
5922 */
5923 lro = 1;
5924 }
5925 } else {
5926 /* Negative time? */
5927 lro = 0;
5928 }
5929 if (lro > rack->r_ctl.rc_reorder_fade) {
5930 /* Turn off reordering seen too */
5931 rack->r_ctl.rc_reorder_ts = 0;
5932 lro = 0;
5933 }
5934 } else {
5935 /* Reodering does not fade */
5936 lro = 1;
5937 }
5938 } else {
5939 lro = 0;
5940 }
5941 if (rack->rc_rack_tmr_std_based == 0) {
5942 thresh = srtt + rack->r_ctl.rc_pkt_delay;
5943 } else {
5944 /* Standards based pkt-delay is 1/4 srtt */
5945 thresh = srtt + (srtt >> 2);
5946 }
5947 if (lro && (rack->rc_rack_tmr_std_based == 0)) {
5948 /* It must be set, if not you get 1/4 rtt */
5949 if (rack->r_ctl.rc_reorder_shift)
5950 thresh += (srtt >> rack->r_ctl.rc_reorder_shift);
5951 else
5952 thresh += (srtt >> 2);
5953 }
5954 if (rack->rc_rack_use_dsack &&
5955 lro &&
5956 (rack->r_ctl.num_dsack > 0)) {
5957 /*
5958 * We only increase the reordering window if we
5959 * have seen reordering <and> we have a DSACK count.
5960 */
5961 thresh += rack->r_ctl.num_dsack * (srtt >> 2);
5962 if (log_allowed)
5963 rack_log_dsack_event(rack, 4, line, srtt, thresh);
5964 }
5965 /* SRTT * 2 is the ceiling */
5966 if (thresh > (srtt * 2)) {
5967 thresh = srtt * 2;
5968 }
5969 /* And we don't want it above the RTO max either */
5970 if (thresh > rack_rto_max) {
5971 thresh = rack_rto_max;
5972 }
5973 if (log_allowed)
5974 rack_log_dsack_event(rack, 6, line, srtt, thresh);
5975 return (thresh);
5976 }
5977
5978 static uint32_t
rack_calc_thresh_tlp(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint32_t srtt)5979 rack_calc_thresh_tlp(struct tcpcb *tp, struct tcp_rack *rack,
5980 struct rack_sendmap *rsm, uint32_t srtt)
5981 {
5982 struct rack_sendmap *prsm;
5983 uint32_t thresh, len;
5984 int segsiz;
5985
5986 if (srtt == 0)
5987 srtt = 1;
5988 if (rack->r_ctl.rc_tlp_threshold)
5989 thresh = srtt + (srtt / rack->r_ctl.rc_tlp_threshold);
5990 else
5991 thresh = (srtt * 2);
5992
5993 /* Get the previous sent packet, if any */
5994 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
5995 len = rsm->r_end - rsm->r_start;
5996 if (rack->rack_tlp_threshold_use == TLP_USE_ID) {
5997 /* Exactly like the ID */
5998 if (((tp->snd_max - tp->snd_una) - rack->r_ctl.rc_sacked + rack->r_ctl.rc_holes_rxt) <= segsiz) {
5999 uint32_t alt_thresh;
6000 /*
6001 * Compensate for delayed-ack with the d-ack time.
6002 */
6003 alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time;
6004 if (alt_thresh > thresh)
6005 thresh = alt_thresh;
6006 }
6007 } else if (rack->rack_tlp_threshold_use == TLP_USE_TWO_ONE) {
6008 /* 2.1 behavior */
6009 prsm = TAILQ_PREV(rsm, rack_head, r_tnext);
6010 if (prsm && (len <= segsiz)) {
6011 /*
6012 * Two packets outstanding, thresh should be (2*srtt) +
6013 * possible inter-packet delay (if any).
6014 */
6015 uint32_t inter_gap = 0;
6016 int idx, nidx;
6017
6018 idx = rsm->r_rtr_cnt - 1;
6019 nidx = prsm->r_rtr_cnt - 1;
6020 if (rsm->r_tim_lastsent[nidx] >= prsm->r_tim_lastsent[idx]) {
6021 /* Yes it was sent later (or at the same time) */
6022 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
6023 }
6024 thresh += inter_gap;
6025 } else if (len <= segsiz) {
6026 /*
6027 * Possibly compensate for delayed-ack.
6028 */
6029 uint32_t alt_thresh;
6030
6031 alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time;
6032 if (alt_thresh > thresh)
6033 thresh = alt_thresh;
6034 }
6035 } else if (rack->rack_tlp_threshold_use == TLP_USE_TWO_TWO) {
6036 /* 2.2 behavior */
6037 if (len <= segsiz) {
6038 uint32_t alt_thresh;
6039 /*
6040 * Compensate for delayed-ack with the d-ack time.
6041 */
6042 alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time;
6043 if (alt_thresh > thresh)
6044 thresh = alt_thresh;
6045 }
6046 }
6047 /* Not above an RTO */
6048 if (thresh > tp->t_rxtcur) {
6049 thresh = tp->t_rxtcur;
6050 }
6051 /* Not above a RTO max */
6052 if (thresh > rack_rto_max) {
6053 thresh = rack_rto_max;
6054 }
6055 /* Apply user supplied min TLP */
6056 if (thresh < rack_tlp_min) {
6057 thresh = rack_tlp_min;
6058 }
6059 return (thresh);
6060 }
6061
6062 static uint32_t
rack_grab_rtt(struct tcpcb * tp,struct tcp_rack * rack)6063 rack_grab_rtt(struct tcpcb *tp, struct tcp_rack *rack)
6064 {
6065 /*
6066 * We want the rack_rtt which is the
6067 * last rtt we measured. However if that
6068 * does not exist we fallback to the srtt (which
6069 * we probably will never do) and then as a last
6070 * resort we use RACK_INITIAL_RTO if no srtt is
6071 * yet set.
6072 */
6073 if (rack->rc_rack_rtt)
6074 return (rack->rc_rack_rtt);
6075 else if (tp->t_srtt == 0)
6076 return (RACK_INITIAL_RTO);
6077 return (tp->t_srtt);
6078 }
6079
6080 static struct rack_sendmap *
rack_check_recovery_mode(struct tcpcb * tp,uint32_t tsused)6081 rack_check_recovery_mode(struct tcpcb *tp, uint32_t tsused)
6082 {
6083 /*
6084 * Check to see that we don't need to fall into recovery. We will
6085 * need to do so if our oldest transmit is past the time we should
6086 * have had an ack.
6087 */
6088 struct tcp_rack *rack;
6089 struct rack_sendmap *rsm;
6090 int32_t idx;
6091 uint32_t srtt, thresh;
6092
6093 rack = (struct tcp_rack *)tp->t_fb_ptr;
6094 if (tqhash_empty(rack->r_ctl.tqh)) {
6095 return (NULL);
6096 }
6097 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
6098 if (rsm == NULL)
6099 return (NULL);
6100
6101
6102 if (rsm->r_flags & RACK_ACKED) {
6103 rsm = rack_find_lowest_rsm(rack);
6104 if (rsm == NULL)
6105 return (NULL);
6106 }
6107 idx = rsm->r_rtr_cnt - 1;
6108 srtt = rack_grab_rtt(tp, rack);
6109 thresh = rack_calc_thresh_rack(rack, srtt, tsused, __LINE__, 1);
6110 if (TSTMP_LT(tsused, ((uint32_t)rsm->r_tim_lastsent[idx]))) {
6111 return (NULL);
6112 }
6113 if ((tsused - ((uint32_t)rsm->r_tim_lastsent[idx])) < thresh) {
6114 return (NULL);
6115 }
6116 /* Ok if we reach here we are over-due and this guy can be sent */
6117 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__);
6118 return (rsm);
6119 }
6120
6121 static uint32_t
rack_get_persists_timer_val(struct tcpcb * tp,struct tcp_rack * rack)6122 rack_get_persists_timer_val(struct tcpcb *tp, struct tcp_rack *rack)
6123 {
6124 int32_t t;
6125 int32_t tt;
6126 uint32_t ret_val;
6127
6128 t = (tp->t_srtt + (tp->t_rttvar << 2));
6129 RACK_TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift],
6130 rack_persist_min, rack_persist_max, rack->r_ctl.timer_slop);
6131 rack->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
6132 ret_val = (uint32_t)tt;
6133 return (ret_val);
6134 }
6135
6136 static uint32_t
rack_timer_start(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,int sup_rack)6137 rack_timer_start(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int sup_rack)
6138 {
6139 /*
6140 * Start the FR timer, we do this based on getting the first one in
6141 * the rc_tmap. Note that if its NULL we must stop the timer. in all
6142 * events we need to stop the running timer (if its running) before
6143 * starting the new one.
6144 */
6145 uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse;
6146 uint32_t srtt_cur;
6147 int32_t idx;
6148 int32_t is_tlp_timer = 0;
6149 struct rack_sendmap *rsm;
6150
6151 if (rack->t_timers_stopped) {
6152 /* All timers have been stopped none are to run */
6153 return (0);
6154 }
6155 if (rack->rc_in_persist) {
6156 /* We can't start any timer in persists */
6157 return (rack_get_persists_timer_val(tp, rack));
6158 }
6159 rack->rc_on_min_to = 0;
6160 if ((tp->t_state < TCPS_ESTABLISHED) ||
6161 ((tp->t_flags & TF_SACK_PERMIT) == 0)) {
6162 goto activate_rxt;
6163 }
6164 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
6165 if ((rsm == NULL) || sup_rack) {
6166 /* Nothing on the send map or no rack */
6167 activate_rxt:
6168 time_since_sent = 0;
6169 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
6170 if (rsm) {
6171 /*
6172 * Should we discount the RTX timer any?
6173 *
6174 * We want to discount it the smallest amount.
6175 * If a timer (Rack/TLP or RXT) has gone off more
6176 * recently thats the discount we want to use (now - timer time).
6177 * If the retransmit of the oldest packet was more recent then
6178 * we want to use that (now - oldest-packet-last_transmit_time).
6179 *
6180 */
6181 idx = rsm->r_rtr_cnt - 1;
6182 if (TSTMP_GEQ(rack->r_ctl.rc_tlp_rxt_last_time, ((uint32_t)rsm->r_tim_lastsent[idx])))
6183 tstmp_touse = (uint32_t)rack->r_ctl.rc_tlp_rxt_last_time;
6184 else
6185 tstmp_touse = (uint32_t)rsm->r_tim_lastsent[idx];
6186 if (TSTMP_GT(cts, tstmp_touse))
6187 time_since_sent = cts - tstmp_touse;
6188 }
6189 if (SEQ_LT(tp->snd_una, tp->snd_max) ||
6190 sbavail(&tptosocket(tp)->so_snd)) {
6191 rack->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
6192 to = tp->t_rxtcur;
6193 if (to > time_since_sent)
6194 to -= time_since_sent;
6195 else
6196 to = rack->r_ctl.rc_min_to;
6197 if (to == 0)
6198 to = 1;
6199 /* Special case for KEEPINIT */
6200 if ((TCPS_HAVEESTABLISHED(tp->t_state) == 0) &&
6201 (TP_KEEPINIT(tp) != 0) &&
6202 rsm) {
6203 /*
6204 * We have to put a ceiling on the rxt timer
6205 * of the keep-init timeout.
6206 */
6207 uint32_t max_time, red;
6208
6209 max_time = TICKS_2_USEC(TP_KEEPINIT(tp));
6210 if (TSTMP_GT(cts, (uint32_t)rsm->r_tim_lastsent[0])) {
6211 red = (cts - (uint32_t)rsm->r_tim_lastsent[0]);
6212 if (red < max_time)
6213 max_time -= red;
6214 else
6215 max_time = 1;
6216 }
6217 /* Reduce timeout to the keep value if needed */
6218 if (max_time < to)
6219 to = max_time;
6220 }
6221 return (to);
6222 }
6223 return (0);
6224 }
6225 if (rsm->r_flags & RACK_ACKED) {
6226 rsm = rack_find_lowest_rsm(rack);
6227 if (rsm == NULL) {
6228 /* No lowest? */
6229 goto activate_rxt;
6230 }
6231 }
6232 /* Convert from ms to usecs */
6233 if ((rsm->r_flags & RACK_SACK_PASSED) ||
6234 (rsm->r_flags & RACK_RWND_COLLAPSED) ||
6235 (rsm->r_dupack >= DUP_ACK_THRESHOLD)) {
6236 if ((tp->t_flags & TF_SENTFIN) &&
6237 ((tp->snd_max - tp->snd_una) == 1) &&
6238 (rsm->r_flags & RACK_HAS_FIN)) {
6239 /*
6240 * We don't start a rack timer if all we have is a
6241 * FIN outstanding.
6242 */
6243 goto activate_rxt;
6244 }
6245 if ((rack->use_rack_rr == 0) &&
6246 (IN_FASTRECOVERY(tp->t_flags)) &&
6247 (rack->rack_no_prr == 0) &&
6248 (rack->r_ctl.rc_prr_sndcnt < ctf_fixed_maxseg(tp))) {
6249 /*
6250 * We are not cheating, in recovery and
6251 * not enough ack's to yet get our next
6252 * retransmission out.
6253 *
6254 * Note that classified attackers do not
6255 * get to use the rack-cheat.
6256 */
6257 goto activate_tlp;
6258 }
6259 srtt = rack_grab_rtt(tp, rack);
6260 thresh = rack_calc_thresh_rack(rack, srtt, cts, __LINE__, 1);
6261 idx = rsm->r_rtr_cnt - 1;
6262 exp = ((uint32_t)rsm->r_tim_lastsent[idx]) + thresh;
6263 if (SEQ_GEQ(exp, cts)) {
6264 to = exp - cts;
6265 if (to < rack->r_ctl.rc_min_to) {
6266 to = rack->r_ctl.rc_min_to;
6267 if (rack->r_rr_config == 3)
6268 rack->rc_on_min_to = 1;
6269 }
6270 } else {
6271 to = rack->r_ctl.rc_min_to;
6272 if (rack->r_rr_config == 3)
6273 rack->rc_on_min_to = 1;
6274 }
6275 } else {
6276 /* Ok we need to do a TLP not RACK */
6277 activate_tlp:
6278 if ((rack->rc_tlp_in_progress != 0) &&
6279 (rack->r_ctl.rc_tlp_cnt_out >= rack_tlp_limit)) {
6280 /*
6281 * The previous send was a TLP and we have sent
6282 * N TLP's without sending new data.
6283 */
6284 goto activate_rxt;
6285 }
6286 rsm = TAILQ_LAST_FAST(&rack->r_ctl.rc_tmap, rack_sendmap, r_tnext);
6287 if (rsm == NULL) {
6288 /* We found no rsm to TLP with. */
6289 goto activate_rxt;
6290 }
6291 if (rsm->r_flags & RACK_HAS_FIN) {
6292 /* If its a FIN we dont do TLP */
6293 rsm = NULL;
6294 goto activate_rxt;
6295 }
6296 idx = rsm->r_rtr_cnt - 1;
6297 time_since_sent = 0;
6298 if (TSTMP_GEQ(((uint32_t)rsm->r_tim_lastsent[idx]), rack->r_ctl.rc_tlp_rxt_last_time))
6299 tstmp_touse = (uint32_t)rsm->r_tim_lastsent[idx];
6300 else
6301 tstmp_touse = (uint32_t)rack->r_ctl.rc_tlp_rxt_last_time;
6302 if (TSTMP_GT(cts, tstmp_touse))
6303 time_since_sent = cts - tstmp_touse;
6304 is_tlp_timer = 1;
6305 if (tp->t_srtt) {
6306 if ((rack->rc_srtt_measure_made == 0) &&
6307 (tp->t_srtt == 1)) {
6308 /*
6309 * If another stack as run and set srtt to 1,
6310 * then the srtt was 0, so lets use the initial.
6311 */
6312 srtt = RACK_INITIAL_RTO;
6313 } else {
6314 srtt_cur = tp->t_srtt;
6315 srtt = srtt_cur;
6316 }
6317 } else
6318 srtt = RACK_INITIAL_RTO;
6319 /*
6320 * If the SRTT is not keeping up and the
6321 * rack RTT has spiked we want to use
6322 * the last RTT not the smoothed one.
6323 */
6324 if (rack_tlp_use_greater &&
6325 tp->t_srtt &&
6326 (srtt < rack_grab_rtt(tp, rack))) {
6327 srtt = rack_grab_rtt(tp, rack);
6328 }
6329 thresh = rack_calc_thresh_tlp(tp, rack, rsm, srtt);
6330 if (thresh > time_since_sent) {
6331 to = thresh - time_since_sent;
6332 } else {
6333 to = rack->r_ctl.rc_min_to;
6334 rack_log_alt_to_to_cancel(rack,
6335 thresh, /* flex1 */
6336 time_since_sent, /* flex2 */
6337 tstmp_touse, /* flex3 */
6338 rack->r_ctl.rc_tlp_rxt_last_time, /* flex4 */
6339 (uint32_t)rsm->r_tim_lastsent[idx],
6340 srtt,
6341 idx, 99);
6342 }
6343 if (to < rack_tlp_min) {
6344 to = rack_tlp_min;
6345 }
6346 if (to > TICKS_2_USEC(tcp_rexmit_max)) {
6347 /*
6348 * If the TLP time works out to larger than the max
6349 * RTO lets not do TLP.. just RTO.
6350 */
6351 goto activate_rxt;
6352 }
6353 }
6354 if (is_tlp_timer == 0) {
6355 rack->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
6356 } else {
6357 rack->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
6358 }
6359 if (to == 0)
6360 to = 1;
6361 return (to);
6362 }
6363
6364 static void
rack_enter_persist(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,tcp_seq snd_una)6365 rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, tcp_seq snd_una)
6366 {
6367 if (rack->rc_in_persist == 0) {
6368 if (tp->t_flags & TF_GPUTINPROG) {
6369 /*
6370 * Stop the goodput now, the calling of the
6371 * measurement function clears the flag.
6372 */
6373 rack_do_goodput_measurement(tp, rack, tp->snd_una, __LINE__,
6374 RACK_QUALITY_PERSIST);
6375 }
6376 #ifdef NETFLIX_SHARED_CWND
6377 if (rack->r_ctl.rc_scw) {
6378 tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
6379 rack->rack_scwnd_is_idle = 1;
6380 }
6381 #endif
6382 rack->r_ctl.rc_went_idle_time = cts;
6383 if (rack->r_ctl.rc_went_idle_time == 0)
6384 rack->r_ctl.rc_went_idle_time = 1;
6385 if (rack->lt_bw_up) {
6386 /* Suspend our LT BW measurement */
6387 uint64_t tmark;
6388
6389 rack->r_ctl.lt_bw_bytes += (snd_una - rack->r_ctl.lt_seq);
6390 rack->r_ctl.lt_seq = snd_una;
6391 tmark = tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time);
6392 if (tmark >= rack->r_ctl.lt_timemark) {
6393 rack->r_ctl.lt_bw_time += (tmark - rack->r_ctl.lt_timemark);
6394 }
6395 rack->r_ctl.lt_timemark = tmark;
6396 rack->lt_bw_up = 0;
6397 rack->r_persist_lt_bw_off = 1;
6398 }
6399 rack_timer_cancel(tp, rack, cts, __LINE__);
6400 rack->r_ctl.persist_lost_ends = 0;
6401 rack->probe_not_answered = 0;
6402 rack->forced_ack = 0;
6403 tp->t_rxtshift = 0;
6404 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
6405 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
6406 rack->rc_in_persist = 1;
6407 }
6408 }
6409
6410 static void
rack_exit_persist(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts)6411 rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
6412 {
6413 if (tcp_in_hpts(rack->rc_tp)) {
6414 tcp_hpts_remove(rack->rc_tp);
6415 rack->r_ctl.rc_hpts_flags = 0;
6416 }
6417 #ifdef NETFLIX_SHARED_CWND
6418 if (rack->r_ctl.rc_scw) {
6419 tcp_shared_cwnd_active(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
6420 rack->rack_scwnd_is_idle = 0;
6421 }
6422 #endif
6423 if (rack->rc_gp_dyn_mul &&
6424 (rack->use_fixed_rate == 0) &&
6425 (rack->rc_always_pace)) {
6426 /*
6427 * Do we count this as if a probe-rtt just
6428 * finished?
6429 */
6430 uint32_t time_idle, idle_min;
6431
6432 time_idle = cts - rack->r_ctl.rc_went_idle_time;
6433 idle_min = rack_min_probertt_hold;
6434 if (rack_probertt_gpsrtt_cnt_div) {
6435 uint64_t extra;
6436 extra = (uint64_t)rack->r_ctl.rc_gp_srtt *
6437 (uint64_t)rack_probertt_gpsrtt_cnt_mul;
6438 extra /= (uint64_t)rack_probertt_gpsrtt_cnt_div;
6439 idle_min += (uint32_t)extra;
6440 }
6441 if (time_idle >= idle_min) {
6442 /* Yes, we count it as a probe-rtt. */
6443 uint32_t us_cts;
6444
6445 us_cts = tcp_get_usecs(NULL);
6446 if (rack->in_probe_rtt == 0) {
6447 rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
6448 rack->r_ctl.rc_time_probertt_entered = rack->r_ctl.rc_lower_rtt_us_cts;
6449 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts;
6450 rack->r_ctl.rc_time_of_last_probertt = rack->r_ctl.rc_lower_rtt_us_cts;
6451 } else {
6452 rack_exit_probertt(rack, us_cts);
6453 }
6454 }
6455 }
6456 if (rack->r_persist_lt_bw_off) {
6457 /* Continue where we left off */
6458 rack->r_ctl.lt_timemark = tcp_get_u64_usecs(NULL);
6459 rack->lt_bw_up = 1;
6460 rack->r_persist_lt_bw_off = 0;
6461 }
6462 rack->rc_in_persist = 0;
6463 rack->r_ctl.rc_went_idle_time = 0;
6464 tp->t_rxtshift = 0;
6465 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
6466 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
6467 rack->r_ctl.rc_agg_delayed = 0;
6468 rack->r_early = 0;
6469 rack->r_late = 0;
6470 rack->r_ctl.rc_agg_early = 0;
6471 }
6472
6473 static void
rack_log_hpts_diag(struct tcp_rack * rack,uint32_t cts,struct hpts_diag * diag,struct timeval * tv)6474 rack_log_hpts_diag(struct tcp_rack *rack, uint32_t cts,
6475 struct hpts_diag *diag, struct timeval *tv)
6476 {
6477 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
6478 union tcp_log_stackspecific log;
6479
6480 memset(&log, 0, sizeof(log));
6481 log.u_bbr.flex1 = diag->p_nxt_slot;
6482 log.u_bbr.flex2 = diag->p_cur_slot;
6483 log.u_bbr.flex3 = diag->slot_req;
6484 log.u_bbr.flex4 = diag->inp_hptsslot;
6485 log.u_bbr.flex5 = diag->time_remaining;
6486 log.u_bbr.flex6 = diag->need_new_to;
6487 log.u_bbr.flex7 = diag->p_hpts_active;
6488 log.u_bbr.flex8 = diag->p_on_min_sleep;
6489 /* Hijack other fields as needed */
6490 log.u_bbr.epoch = diag->have_slept;
6491 log.u_bbr.lt_epoch = diag->yet_to_sleep;
6492 log.u_bbr.pkts_out = diag->co_ret;
6493 log.u_bbr.applimited = diag->hpts_sleep_time;
6494 log.u_bbr.delivered = diag->p_prev_slot;
6495 log.u_bbr.inflight = diag->p_runningslot;
6496 log.u_bbr.bw_inuse = diag->wheel_slot;
6497 log.u_bbr.rttProp = diag->wheel_cts;
6498 log.u_bbr.timeStamp = cts;
6499 log.u_bbr.delRate = diag->maxslots;
6500 TCP_LOG_EVENTP(rack->rc_tp, NULL,
6501 &rack->rc_inp->inp_socket->so_rcv,
6502 &rack->rc_inp->inp_socket->so_snd,
6503 BBR_LOG_HPTSDIAG, 0,
6504 0, &log, false, tv);
6505 }
6506
6507 }
6508
6509 static void
rack_log_wakeup(struct tcpcb * tp,struct tcp_rack * rack,struct sockbuf * sb,uint32_t len,int type)6510 rack_log_wakeup(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb, uint32_t len, int type)
6511 {
6512 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
6513 union tcp_log_stackspecific log;
6514 struct timeval tv;
6515
6516 memset(&log, 0, sizeof(log));
6517 log.u_bbr.flex1 = sb->sb_flags;
6518 log.u_bbr.flex2 = len;
6519 log.u_bbr.flex3 = sb->sb_state;
6520 log.u_bbr.flex8 = type;
6521 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
6522 TCP_LOG_EVENTP(rack->rc_tp, NULL,
6523 &rack->rc_inp->inp_socket->so_rcv,
6524 &rack->rc_inp->inp_socket->so_snd,
6525 TCP_LOG_SB_WAKE, 0,
6526 len, &log, false, &tv);
6527 }
6528 }
6529
6530 static void
rack_start_hpts_timer(struct tcp_rack * rack,struct tcpcb * tp,uint32_t cts,int32_t usecs,uint32_t tot_len_this_send,int sup_rack)6531 rack_start_hpts_timer (struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
6532 int32_t usecs, uint32_t tot_len_this_send, int sup_rack)
6533 {
6534 struct hpts_diag diag;
6535 struct inpcb *inp = tptoinpcb(tp);
6536 struct timeval tv;
6537 uint32_t delayed_ack = 0;
6538 uint32_t hpts_timeout;
6539 uint32_t entry_usecs = usecs;
6540 uint8_t stopped;
6541 uint32_t left = 0;
6542 uint32_t us_cts;
6543
6544 if ((tp->t_state == TCPS_CLOSED) ||
6545 (tp->t_state == TCPS_LISTEN)) {
6546 return;
6547 }
6548 if (tcp_in_hpts(tp)) {
6549 /* Already on the pacer */
6550 return;
6551 }
6552 stopped = rack->rc_tmr_stopped;
6553 if (stopped && TSTMP_GT(rack->r_ctl.rc_timer_exp, cts)) {
6554 left = rack->r_ctl.rc_timer_exp - cts;
6555 }
6556 rack->r_ctl.rc_timer_exp = 0;
6557 rack->r_ctl.rc_hpts_flags = 0;
6558 us_cts = tcp_get_usecs(&tv);
6559 /* Now early/late accounting */
6560 rack_log_pacing_delay_calc(rack, entry_usecs, usecs, 0, 0, 0, 26, __LINE__, NULL, 0);
6561 if (rack->r_early && (rack->rc_ack_can_sendout_data == 0)) {
6562 /*
6563 * We have a early carry over set,
6564 * we can always add more time so we
6565 * can always make this compensation.
6566 *
6567 * Note if ack's are allowed to wake us do not
6568 * penalize the next timer for being awoke
6569 * by an ack aka the rc_agg_early (non-paced mode).
6570 */
6571 usecs += rack->r_ctl.rc_agg_early;
6572 rack->r_early = 0;
6573 rack->r_ctl.rc_agg_early = 0;
6574 }
6575 if ((rack->r_late) &&
6576 ((rack->r_use_hpts_min == 0) || (rack->dgp_on == 0))) {
6577 /*
6578 * This is harder, we can
6579 * compensate some but it
6580 * really depends on what
6581 * the current pacing time is.
6582 */
6583 if (rack->r_ctl.rc_agg_delayed >= usecs) {
6584 /*
6585 * We can't compensate for it all.
6586 * And we have to have some time
6587 * on the clock. We always have a min
6588 * 10 HPTS timer units (10 x 10 i.e. 100 usecs).
6589 */
6590 if (usecs <= HPTS_USECS_PER_SLOT) {
6591 /* We gain delay */
6592 rack->r_ctl.rc_agg_delayed += (HPTS_USECS_PER_SLOT - usecs);
6593 usecs = HPTS_USECS_PER_SLOT;
6594 } else {
6595 /* We take off some */
6596 rack->r_ctl.rc_agg_delayed -= (usecs - HPTS_USECS_PER_SLOT);
6597 usecs = HPTS_USECS_PER_SLOT;
6598 }
6599 } else {
6600 usecs -= rack->r_ctl.rc_agg_delayed;
6601 rack->r_ctl.rc_agg_delayed = 0;
6602 /* Make sure we have 100 useconds at minimum */
6603 if (usecs < HPTS_USECS_PER_SLOT) {
6604 rack->r_ctl.rc_agg_delayed = HPTS_USECS_PER_SLOT - usecs;
6605 usecs = HPTS_USECS_PER_SLOT;
6606 }
6607 if (rack->r_ctl.rc_agg_delayed == 0)
6608 rack->r_late = 0;
6609 }
6610 } else if (rack->r_late) {
6611 /* r_use_hpts_min is on and so is DGP */
6612 uint32_t max_red;
6613
6614 max_red = (usecs * rack->r_ctl.max_reduction) / 100;
6615 if (max_red >= rack->r_ctl.rc_agg_delayed) {
6616 usecs -= rack->r_ctl.rc_agg_delayed;
6617 rack->r_ctl.rc_agg_delayed = 0;
6618 } else {
6619 usecs -= max_red;
6620 rack->r_ctl.rc_agg_delayed -= max_red;
6621 }
6622 }
6623 if ((rack->r_use_hpts_min == 1) &&
6624 (usecs > 0) &&
6625 (rack->dgp_on == 1)) {
6626 /*
6627 * We are enforcing a min pacing timer
6628 * based on our hpts min timeout.
6629 */
6630 uint32_t min;
6631
6632 min = get_hpts_min_sleep_time();
6633 if (min > usecs) {
6634 usecs = min;
6635 }
6636 }
6637 hpts_timeout = rack_timer_start(tp, rack, cts, sup_rack);
6638 if (tp->t_flags & TF_DELACK) {
6639 delayed_ack = TICKS_2_USEC(tcp_delacktime);
6640 rack->r_ctl.rc_hpts_flags |= PACE_TMR_DELACK;
6641 }
6642 if (delayed_ack && ((hpts_timeout == 0) ||
6643 (delayed_ack < hpts_timeout)))
6644 hpts_timeout = delayed_ack;
6645 else
6646 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
6647 /*
6648 * If no timers are going to run and we will fall off the hptsi
6649 * wheel, we resort to a keep-alive timer if its configured.
6650 */
6651 if ((hpts_timeout == 0) &&
6652 (usecs == 0)) {
6653 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
6654 (tp->t_state <= TCPS_CLOSING)) {
6655 /*
6656 * Ok we have no timer (persists, rack, tlp, rxt or
6657 * del-ack), we don't have segments being paced. So
6658 * all that is left is the keepalive timer.
6659 */
6660 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
6661 /* Get the established keep-alive time */
6662 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp));
6663 } else {
6664 /*
6665 * Get the initial setup keep-alive time,
6666 * note that this is probably not going to
6667 * happen, since rack will be running a rxt timer
6668 * if a SYN of some sort is outstanding. It is
6669 * actually handled in rack_timeout_rxt().
6670 */
6671 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp));
6672 }
6673 rack->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
6674 if (rack->in_probe_rtt) {
6675 /*
6676 * We want to instead not wake up a long time from
6677 * now but to wake up about the time we would
6678 * exit probe-rtt and initiate a keep-alive ack.
6679 * This will get us out of probe-rtt and update
6680 * our min-rtt.
6681 */
6682 hpts_timeout = rack_min_probertt_hold;
6683 }
6684 }
6685 }
6686 if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) ==
6687 (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
6688 /*
6689 * RACK, TLP, persists and RXT timers all are restartable
6690 * based on actions input .. i.e we received a packet (ack
6691 * or sack) and that changes things (rw, or snd_una etc).
6692 * Thus we can restart them with a new value. For
6693 * keep-alive, delayed_ack we keep track of what was left
6694 * and restart the timer with a smaller value.
6695 */
6696 if (left < hpts_timeout)
6697 hpts_timeout = left;
6698 }
6699 if (hpts_timeout) {
6700 /*
6701 * Hack alert for now we can't time-out over 2,147,483
6702 * seconds (a bit more than 596 hours), which is probably ok
6703 * :).
6704 */
6705 if (hpts_timeout > 0x7ffffffe)
6706 hpts_timeout = 0x7ffffffe;
6707 rack->r_ctl.rc_timer_exp = cts + hpts_timeout;
6708 }
6709 rack_log_pacing_delay_calc(rack, entry_usecs, usecs, hpts_timeout, 0, 0, 27, __LINE__, NULL, 0);
6710 if ((rack->gp_ready == 0) &&
6711 (rack->use_fixed_rate == 0) &&
6712 (hpts_timeout < usecs) &&
6713 (rack->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
6714 /*
6715 * We have no good estimate yet for the
6716 * old clunky burst mitigation or the
6717 * real pacing. And the tlp or rxt is smaller
6718 * than the pacing calculation. Lets not
6719 * pace that long since we know the calculation
6720 * so far is not accurate.
6721 */
6722 usecs = hpts_timeout;
6723 }
6724 /**
6725 * Turn off all the flags for queuing by default. The
6726 * flags have important meanings to what happens when
6727 * LRO interacts with the transport. Most likely (by default now)
6728 * mbuf_queueing and ack compression are on. So the transport
6729 * has a couple of flags that control what happens (if those
6730 * are not on then these flags won't have any effect since it
6731 * won't go through the queuing LRO path).
6732 *
6733 * TF2_MBUF_QUEUE_READY - This flags says that I am busy
6734 * pacing output, so don't disturb. But
6735 * it also means LRO can wake me if there
6736 * is a SACK arrival.
6737 *
6738 * TF2_DONT_SACK_QUEUE - This flag is used in conjunction
6739 * with the above flag (QUEUE_READY) and
6740 * when present it says don't even wake me
6741 * if a SACK arrives.
6742 *
6743 * The idea behind these flags is that if we are pacing we
6744 * set the MBUF_QUEUE_READY and only get woken up if
6745 * a SACK arrives (which could change things) or if
6746 * our pacing timer expires. If, however, we have a rack
6747 * timer running, then we don't even want a sack to wake
6748 * us since the rack timer has to expire before we can send.
6749 *
6750 * Other cases should usually have none of the flags set
6751 * so LRO can call into us.
6752 */
6753 tp->t_flags2 &= ~(TF2_DONT_SACK_QUEUE|TF2_MBUF_QUEUE_READY);
6754 if (usecs) {
6755 rack->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
6756 rack->r_ctl.rc_last_output_to = us_cts + usecs;
6757 /*
6758 * A pacing timer (usecs microseconds) is being set, in
6759 * such a case we cannot send (we are blocked by
6760 * the timer). So lets tell LRO that it should not
6761 * wake us unless there is a SACK. Note this only
6762 * will be effective if mbuf queueing is on or
6763 * compressed acks are being processed.
6764 */
6765 tp->t_flags2 |= TF2_MBUF_QUEUE_READY;
6766 /*
6767 * But wait if we have a Rack timer running
6768 * even a SACK should not disturb us (with
6769 * the exception of r_rr_config 3).
6770 */
6771 if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK) ||
6772 (IN_RECOVERY(tp->t_flags))) {
6773 if (rack->r_rr_config != 3)
6774 tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
6775 else if (rack->rc_pace_dnd) {
6776 /*
6777 * When DND is on, we only let a sack
6778 * interrupt us if we are not in recovery.
6779 *
6780 * If DND is off, then we never hit here
6781 * and let all sacks wake us up.
6782 *
6783 */
6784 tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
6785 }
6786 }
6787 if (rack->rc_ack_can_sendout_data) {
6788 /*
6789 * Ahh but wait, this is that special case
6790 * where the pacing timer can be disturbed
6791 * backout the changes (used for non-paced
6792 * burst limiting).
6793 */
6794 tp->t_flags2 &= ~(TF2_DONT_SACK_QUEUE |
6795 TF2_MBUF_QUEUE_READY);
6796 }
6797 if ((rack->use_rack_rr) &&
6798 (rack->r_rr_config < 2) &&
6799 ((hpts_timeout) && (hpts_timeout < usecs))) {
6800 /*
6801 * Arrange for the hpts to kick back in after the
6802 * t-o if the t-o does not cause a send.
6803 */
6804 tcp_hpts_insert(tp, hpts_timeout, &diag);
6805 rack_log_hpts_diag(rack, us_cts, &diag, &tv);
6806 rack_log_to_start(rack, cts, hpts_timeout, usecs, 0);
6807 } else {
6808 tcp_hpts_insert(tp, usecs, &diag);
6809 rack_log_hpts_diag(rack, us_cts, &diag, &tv);
6810 rack_log_to_start(rack, cts, hpts_timeout, usecs, 1);
6811 }
6812 } else if (hpts_timeout) {
6813 /*
6814 * With respect to t_flags2(?) here, lets let any new acks wake
6815 * us up here. Since we are not pacing (no pacing timer), output
6816 * can happen so we should let it. If its a Rack timer, then any inbound
6817 * packet probably won't change the sending (we will be blocked)
6818 * but it may change the prr stats so letting it in (the set defaults
6819 * at the start of this block) are good enough.
6820 */
6821 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
6822 tcp_hpts_insert(tp, hpts_timeout, &diag);
6823 rack_log_hpts_diag(rack, us_cts, &diag, &tv);
6824 rack_log_to_start(rack, cts, hpts_timeout, usecs, 0);
6825 } else {
6826 /* No timer starting */
6827 #ifdef INVARIANTS
6828 if (SEQ_GT(tp->snd_max, tp->snd_una)) {
6829 panic("tp:%p rack:%p tlts:%d cts:%u usecs:%u pto:%u -- no timer started?",
6830 tp, rack, tot_len_this_send, cts, usecs, hpts_timeout);
6831 }
6832 #endif
6833 }
6834 rack->rc_tmr_stopped = 0;
6835 if (usecs)
6836 rack_log_type_bbrsnd(rack, tot_len_this_send, usecs, us_cts, &tv, __LINE__);
6837 }
6838
6839 static void
rack_mark_lost(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint32_t cts)6840 rack_mark_lost(struct tcpcb *tp,
6841 struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t cts)
6842 {
6843 struct rack_sendmap *nrsm;
6844 uint32_t thresh, exp;
6845
6846 thresh = rack_calc_thresh_rack(rack, rack_grab_rtt(tp, rack), cts, __LINE__, 0);
6847 nrsm = rsm;
6848 TAILQ_FOREACH_FROM(nrsm, &rack->r_ctl.rc_tmap, r_tnext) {
6849 if ((nrsm->r_flags & RACK_SACK_PASSED) == 0) {
6850 /* Got up to all that were marked sack-passed */
6851 break;
6852 }
6853 if ((nrsm->r_flags & RACK_WAS_LOST) == 0) {
6854 exp = ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) + thresh;
6855 if (TSTMP_LT(exp, cts) || (exp == cts)) {
6856 /* We now consider it lost */
6857 nrsm->r_flags |= RACK_WAS_LOST;
6858 rack->r_ctl.rc_considered_lost += nrsm->r_end - nrsm->r_start;
6859 } else {
6860 /* Past here it won't be lost so stop */
6861 break;
6862 }
6863 }
6864 }
6865 }
6866
6867 /*
6868 * RACK Timer, here we simply do logging and house keeping.
6869 * the normal rack_output() function will call the
6870 * appropriate thing to check if we need to do a RACK retransmit.
6871 * We return 1, saying don't proceed with rack_output only
6872 * when all timers have been stopped (destroyed PCB?).
6873 */
6874 static int
rack_timeout_rack(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts)6875 rack_timeout_rack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
6876 {
6877 /*
6878 * This timer simply provides an internal trigger to send out data.
6879 * The check_recovery_mode call will see if there are needed
6880 * retransmissions, if so we will enter fast-recovery. The output
6881 * call may or may not do the same thing depending on sysctl
6882 * settings.
6883 */
6884 struct rack_sendmap *rsm;
6885
6886 counter_u64_add(rack_to_tot, 1);
6887 if (rack->r_state && (rack->r_state != tp->t_state))
6888 rack_set_state(tp, rack);
6889 rack->rc_on_min_to = 0;
6890 rsm = rack_check_recovery_mode(tp, cts);
6891 rack_log_to_event(rack, RACK_TO_FRM_RACK, rsm);
6892 if (rsm) {
6893 /* We need to stroke any lost that are now declared as lost */
6894 rack_mark_lost(tp, rack, rsm, cts);
6895 rack->r_ctl.rc_resend = rsm;
6896 rack->r_timer_override = 1;
6897 if (rack->use_rack_rr) {
6898 /*
6899 * Don't accumulate extra pacing delay
6900 * we are allowing the rack timer to
6901 * over-ride pacing i.e. rrr takes precedence
6902 * if the pacing interval is longer than the rrr
6903 * time (in other words we get the min pacing
6904 * time versus rrr pacing time).
6905 */
6906 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
6907 }
6908 }
6909 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
6910 if (rsm == NULL) {
6911 /* restart a timer and return 1 */
6912 rack_start_hpts_timer(rack, tp, cts,
6913 0, 0, 0);
6914 return (1);
6915 }
6916 return (0);
6917 }
6918
6919
6920
6921 static void
rack_adjust_orig_mlen(struct rack_sendmap * rsm)6922 rack_adjust_orig_mlen(struct rack_sendmap *rsm)
6923 {
6924
6925 if ((M_TRAILINGROOM(rsm->m) != rsm->orig_t_space)) {
6926 /*
6927 * The trailing space changed, mbufs can grow
6928 * at the tail but they can't shrink from
6929 * it, KASSERT that. Adjust the orig_m_len to
6930 * compensate for this change.
6931 */
6932 KASSERT((rsm->orig_t_space > M_TRAILINGROOM(rsm->m)),
6933 ("mbuf:%p rsm:%p trailing_space:%jd ots:%u oml:%u mlen:%u\n",
6934 rsm->m,
6935 rsm,
6936 (intmax_t)M_TRAILINGROOM(rsm->m),
6937 rsm->orig_t_space,
6938 rsm->orig_m_len,
6939 rsm->m->m_len));
6940 rsm->orig_m_len += (rsm->orig_t_space - M_TRAILINGROOM(rsm->m));
6941 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
6942 }
6943 if (rsm->m->m_len < rsm->orig_m_len) {
6944 /*
6945 * Mbuf shrank, trimmed off the top by an ack, our
6946 * offset changes.
6947 */
6948 KASSERT((rsm->soff >= (rsm->orig_m_len - rsm->m->m_len)),
6949 ("mbuf:%p len:%u rsm:%p oml:%u soff:%u\n",
6950 rsm->m, rsm->m->m_len,
6951 rsm, rsm->orig_m_len,
6952 rsm->soff));
6953 if (rsm->soff >= (rsm->orig_m_len - rsm->m->m_len))
6954 rsm->soff -= (rsm->orig_m_len - rsm->m->m_len);
6955 else
6956 rsm->soff = 0;
6957 rsm->orig_m_len = rsm->m->m_len;
6958 #ifdef INVARIANTS
6959 } else if (rsm->m->m_len > rsm->orig_m_len) {
6960 panic("rsm:%p m:%p m_len grew outside of t_space compensation",
6961 rsm, rsm->m);
6962 #endif
6963 }
6964 }
6965
6966 static void
rack_setup_offset_for_rsm(struct tcp_rack * rack,struct rack_sendmap * src_rsm,struct rack_sendmap * rsm)6967 rack_setup_offset_for_rsm(struct tcp_rack *rack, struct rack_sendmap *src_rsm, struct rack_sendmap *rsm)
6968 {
6969 struct mbuf *m;
6970 uint32_t soff;
6971
6972 if (src_rsm->m &&
6973 ((src_rsm->orig_m_len != src_rsm->m->m_len) ||
6974 (M_TRAILINGROOM(src_rsm->m) != src_rsm->orig_t_space))) {
6975 /* Fix up the orig_m_len and possibly the mbuf offset */
6976 rack_adjust_orig_mlen(src_rsm);
6977 }
6978 m = src_rsm->m;
6979 soff = src_rsm->soff + (src_rsm->r_end - src_rsm->r_start);
6980 while (soff >= m->m_len) {
6981 /* Move out past this mbuf */
6982 soff -= m->m_len;
6983 m = m->m_next;
6984 KASSERT((m != NULL),
6985 ("rsm:%p nrsm:%p hit at soff:%u null m",
6986 src_rsm, rsm, soff));
6987 if (m == NULL) {
6988 /* This should *not* happen which is why there is a kassert */
6989 src_rsm->m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd,
6990 (src_rsm->r_start - rack->rc_tp->snd_una),
6991 &src_rsm->soff);
6992 src_rsm->orig_m_len = src_rsm->m->m_len;
6993 src_rsm->orig_t_space = M_TRAILINGROOM(src_rsm->m);
6994 rsm->m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd,
6995 (rsm->r_start - rack->rc_tp->snd_una),
6996 &rsm->soff);
6997 rsm->orig_m_len = rsm->m->m_len;
6998 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
6999 return;
7000 }
7001 }
7002 rsm->m = m;
7003 rsm->soff = soff;
7004 rsm->orig_m_len = m->m_len;
7005 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
7006 }
7007
7008 static __inline void
rack_clone_rsm(struct tcp_rack * rack,struct rack_sendmap * nrsm,struct rack_sendmap * rsm,uint32_t start)7009 rack_clone_rsm(struct tcp_rack *rack, struct rack_sendmap *nrsm,
7010 struct rack_sendmap *rsm, uint32_t start)
7011 {
7012 int idx;
7013
7014 nrsm->r_start = start;
7015 nrsm->r_end = rsm->r_end;
7016 nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
7017 nrsm->r_act_rxt_cnt = rsm->r_act_rxt_cnt;
7018 nrsm->r_flags = rsm->r_flags;
7019 nrsm->r_dupack = rsm->r_dupack;
7020 nrsm->r_no_rtt_allowed = rsm->r_no_rtt_allowed;
7021 nrsm->r_rtr_bytes = 0;
7022 nrsm->r_fas = rsm->r_fas;
7023 nrsm->r_bas = rsm->r_bas;
7024 tqhash_update_end(rack->r_ctl.tqh, rsm, nrsm->r_start);
7025 nrsm->r_just_ret = rsm->r_just_ret;
7026 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {
7027 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
7028 }
7029 /* Now if we have SYN flag we keep it on the left edge */
7030 if (nrsm->r_flags & RACK_HAS_SYN)
7031 nrsm->r_flags &= ~RACK_HAS_SYN;
7032 /* Now if we have a FIN flag we keep it on the right edge */
7033 if (rsm->r_flags & RACK_HAS_FIN)
7034 rsm->r_flags &= ~RACK_HAS_FIN;
7035 /* Push bit must go to the right edge as well */
7036 if (rsm->r_flags & RACK_HAD_PUSH)
7037 rsm->r_flags &= ~RACK_HAD_PUSH;
7038 /* Update the count if app limited */
7039 if (nrsm->r_flags & RACK_APP_LIMITED)
7040 rack->r_ctl.rc_app_limited_cnt++;
7041 /* Clone over the state of the hw_tls flag */
7042 nrsm->r_hw_tls = rsm->r_hw_tls;
7043 /*
7044 * Now we need to find nrsm's new location in the mbuf chain
7045 * we basically calculate a new offset, which is soff +
7046 * how much is left in original rsm. Then we walk out the mbuf
7047 * chain to find the righ position, it may be the same mbuf
7048 * or maybe not.
7049 */
7050 KASSERT(((rsm->m != NULL) ||
7051 (rsm->r_flags & (RACK_HAS_SYN|RACK_HAS_FIN))),
7052 ("rsm:%p nrsm:%p rack:%p -- rsm->m is NULL?", rsm, nrsm, rack));
7053 if (rsm->m)
7054 rack_setup_offset_for_rsm(rack, rsm, nrsm);
7055 }
7056
7057 static struct rack_sendmap *
rack_merge_rsm(struct tcp_rack * rack,struct rack_sendmap * l_rsm,struct rack_sendmap * r_rsm)7058 rack_merge_rsm(struct tcp_rack *rack,
7059 struct rack_sendmap *l_rsm,
7060 struct rack_sendmap *r_rsm)
7061 {
7062 /*
7063 * We are merging two ack'd RSM's,
7064 * the l_rsm is on the left (lower seq
7065 * values) and the r_rsm is on the right
7066 * (higher seq value). The simplest way
7067 * to merge these is to move the right
7068 * one into the left. I don't think there
7069 * is any reason we need to try to find
7070 * the oldest (or last oldest retransmitted).
7071 */
7072 rack_log_map_chg(rack->rc_tp, rack, NULL,
7073 l_rsm, r_rsm, MAP_MERGE, r_rsm->r_end, __LINE__);
7074 tqhash_update_end(rack->r_ctl.tqh, l_rsm, r_rsm->r_end);
7075 if (l_rsm->r_dupack < r_rsm->r_dupack)
7076 l_rsm->r_dupack = r_rsm->r_dupack;
7077 if (r_rsm->r_rtr_bytes)
7078 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;
7079 if (r_rsm->r_in_tmap) {
7080 /* This really should not happen */
7081 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, r_rsm, r_tnext);
7082 r_rsm->r_in_tmap = 0;
7083 }
7084
7085 /* Now the flags */
7086 if (r_rsm->r_flags & RACK_HAS_FIN)
7087 l_rsm->r_flags |= RACK_HAS_FIN;
7088 if (r_rsm->r_flags & RACK_TLP)
7089 l_rsm->r_flags |= RACK_TLP;
7090 if (r_rsm->r_flags & RACK_RWND_COLLAPSED)
7091 l_rsm->r_flags |= RACK_RWND_COLLAPSED;
7092 if ((r_rsm->r_flags & RACK_APP_LIMITED) &&
7093 ((l_rsm->r_flags & RACK_APP_LIMITED) == 0)) {
7094 /*
7095 * If both are app-limited then let the
7096 * free lower the count. If right is app
7097 * limited and left is not, transfer.
7098 */
7099 l_rsm->r_flags |= RACK_APP_LIMITED;
7100 r_rsm->r_flags &= ~RACK_APP_LIMITED;
7101 if (r_rsm == rack->r_ctl.rc_first_appl)
7102 rack->r_ctl.rc_first_appl = l_rsm;
7103 }
7104 tqhash_remove(rack->r_ctl.tqh, r_rsm, REMOVE_TYPE_MERGE);
7105 /*
7106 * We keep the largest value, which is the newest
7107 * send. We do this in case a segment that is
7108 * joined together and not part of a GP estimate
7109 * later gets expanded into the GP estimate.
7110 *
7111 * We prohibit the merging of unlike kinds i.e.
7112 * all pieces that are in the GP estimate can be
7113 * merged and all pieces that are not in a GP estimate
7114 * can be merged, but not disimilar pieces. Combine
7115 * this with taking the highest here and we should
7116 * be ok unless of course the client reneges. Then
7117 * all bets are off.
7118 */
7119 if(l_rsm->r_tim_lastsent[(l_rsm->r_rtr_cnt-1)] <
7120 r_rsm->r_tim_lastsent[(r_rsm->r_rtr_cnt-1)]) {
7121 l_rsm->r_tim_lastsent[(l_rsm->r_rtr_cnt-1)] = r_rsm->r_tim_lastsent[(r_rsm->r_rtr_cnt-1)];
7122 }
7123 /*
7124 * When merging two RSM's we also need to consider the ack time and keep
7125 * newest. If the ack gets merged into a measurement then that is the
7126 * one we will want to be using.
7127 */
7128 if(l_rsm->r_ack_arrival < r_rsm->r_ack_arrival)
7129 l_rsm->r_ack_arrival = r_rsm->r_ack_arrival;
7130
7131 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {
7132 /* Transfer the split limit to the map we free */
7133 r_rsm->r_limit_type = l_rsm->r_limit_type;
7134 l_rsm->r_limit_type = 0;
7135 }
7136 rack_free(rack, r_rsm);
7137 l_rsm->r_flags |= RACK_MERGED;
7138 return (l_rsm);
7139 }
7140
7141 /*
7142 * TLP Timer, here we simply setup what segment we want to
7143 * have the TLP expire on, the normal rack_output() will then
7144 * send it out.
7145 *
7146 * We return 1, saying don't proceed with rack_output only
7147 * when all timers have been stopped (destroyed PCB?).
7148 */
7149 static int
rack_timeout_tlp(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,uint8_t * doing_tlp)7150 rack_timeout_tlp(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t *doing_tlp)
7151 {
7152 /*
7153 * Tail Loss Probe.
7154 */
7155 struct rack_sendmap *rsm = NULL;
7156 int insret __diagused;
7157 struct socket *so = tptosocket(tp);
7158 uint32_t amm;
7159 uint32_t out, avail;
7160 int collapsed_win = 0;
7161
7162 if (TSTMP_LT(cts, rack->r_ctl.rc_timer_exp)) {
7163 /* Its not time yet */
7164 return (0);
7165 }
7166 if (ctf_progress_timeout_check(tp, true)) {
7167 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
7168 return (-ETIMEDOUT); /* tcp_drop() */
7169 }
7170 /*
7171 * A TLP timer has expired. We have been idle for 2 rtts. So we now
7172 * need to figure out how to force a full MSS segment out.
7173 */
7174 rack_log_to_event(rack, RACK_TO_FRM_TLP, NULL);
7175 rack->r_ctl.retran_during_recovery = 0;
7176 rack->r_might_revert = 0;
7177 rack->r_ctl.dsack_byte_cnt = 0;
7178 counter_u64_add(rack_tlp_tot, 1);
7179 if (rack->r_state && (rack->r_state != tp->t_state))
7180 rack_set_state(tp, rack);
7181 avail = sbavail(&so->so_snd);
7182 out = tp->snd_max - tp->snd_una;
7183 if ((out > tp->snd_wnd) || rack->rc_has_collapsed) {
7184 /* special case, we need a retransmission */
7185 collapsed_win = 1;
7186 goto need_retran;
7187 }
7188 if (rack->r_ctl.dsack_persist && (rack->r_ctl.rc_tlp_cnt_out >= 1)) {
7189 rack->r_ctl.dsack_persist--;
7190 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) {
7191 rack->r_ctl.num_dsack = 0;
7192 }
7193 rack_log_dsack_event(rack, 1, __LINE__, 0, 0);
7194 }
7195 if ((tp->t_flags & TF_GPUTINPROG) &&
7196 (rack->r_ctl.rc_tlp_cnt_out == 1)) {
7197 /*
7198 * If this is the second in a row
7199 * TLP and we are doing a measurement
7200 * its time to abandon the measurement.
7201 * Something is likely broken on
7202 * the clients network and measuring a
7203 * broken network does us no good.
7204 */
7205 tp->t_flags &= ~TF_GPUTINPROG;
7206 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
7207 rack->r_ctl.rc_gp_srtt /*flex1*/,
7208 tp->gput_seq,
7209 0, 0, 18, __LINE__, NULL, 0);
7210 }
7211 /*
7212 * Check our send oldest always settings, and if
7213 * there is an oldest to send jump to the need_retran.
7214 */
7215 if (rack_always_send_oldest && (TAILQ_EMPTY(&rack->r_ctl.rc_tmap) == 0))
7216 goto need_retran;
7217
7218 if (avail > out) {
7219 /* New data is available */
7220 amm = avail - out;
7221 if (amm > ctf_fixed_maxseg(tp)) {
7222 amm = ctf_fixed_maxseg(tp);
7223 if ((amm + out) > tp->snd_wnd) {
7224 /* We are rwnd limited */
7225 goto need_retran;
7226 }
7227 } else if (amm < ctf_fixed_maxseg(tp)) {
7228 /* not enough to fill a MTU */
7229 goto need_retran;
7230 }
7231 if (IN_FASTRECOVERY(tp->t_flags)) {
7232 /* Unlikely */
7233 if (rack->rack_no_prr == 0) {
7234 if (out + amm <= tp->snd_wnd) {
7235 rack->r_ctl.rc_prr_sndcnt = amm;
7236 rack->r_ctl.rc_tlp_new_data = amm;
7237 rack_log_to_prr(rack, 4, 0, __LINE__);
7238 }
7239 } else
7240 goto need_retran;
7241 } else {
7242 /* Set the send-new override */
7243 if (out + amm <= tp->snd_wnd)
7244 rack->r_ctl.rc_tlp_new_data = amm;
7245 else
7246 goto need_retran;
7247 }
7248 rack->r_ctl.rc_tlpsend = NULL;
7249 counter_u64_add(rack_tlp_newdata, 1);
7250 goto send;
7251 }
7252 need_retran:
7253 /*
7254 * Ok we need to arrange the last un-acked segment to be re-sent, or
7255 * optionally the first un-acked segment.
7256 */
7257 if (collapsed_win == 0) {
7258 if (rack_always_send_oldest)
7259 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
7260 else {
7261 rsm = tqhash_max(rack->r_ctl.tqh);
7262 if (rsm && (rsm->r_flags & (RACK_ACKED | RACK_HAS_FIN))) {
7263 rsm = rack_find_high_nonack(rack, rsm);
7264 }
7265 }
7266 if (rsm == NULL) {
7267 #ifdef TCP_BLACKBOX
7268 tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true);
7269 #endif
7270 goto out;
7271 }
7272 } else {
7273 /*
7274 * We had a collapsed window, lets find
7275 * the point before the collapse.
7276 */
7277 if (SEQ_GT((rack->r_ctl.last_collapse_point - 1), rack->rc_tp->snd_una))
7278 rsm = tqhash_find(rack->r_ctl.tqh, (rack->r_ctl.last_collapse_point - 1));
7279 else {
7280 rsm = tqhash_min(rack->r_ctl.tqh);
7281 }
7282 if (rsm == NULL) {
7283 /* Huh */
7284 goto out;
7285 }
7286 }
7287 if ((rsm->r_end - rsm->r_start) > ctf_fixed_maxseg(tp)) {
7288 /*
7289 * We need to split this the last segment in two.
7290 */
7291 struct rack_sendmap *nrsm;
7292
7293 nrsm = rack_alloc_full_limit(rack);
7294 if (nrsm == NULL) {
7295 /*
7296 * No memory to split, we will just exit and punt
7297 * off to the RXT timer.
7298 */
7299 goto out;
7300 }
7301 rack_clone_rsm(rack, nrsm, rsm,
7302 (rsm->r_end - ctf_fixed_maxseg(tp)));
7303 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__);
7304 #ifndef INVARIANTS
7305 (void)tqhash_insert(rack->r_ctl.tqh, nrsm);
7306 #else
7307 if ((insret = tqhash_insert(rack->r_ctl.tqh, nrsm)) != 0) {
7308 panic("Insert in tailq_hash of %p fails ret:%d rack:%p rsm:%p",
7309 nrsm, insret, rack, rsm);
7310 }
7311 #endif
7312 if (rsm->r_in_tmap) {
7313 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7314 nrsm->r_in_tmap = 1;
7315 }
7316 rsm = nrsm;
7317 }
7318 rack->r_ctl.rc_tlpsend = rsm;
7319 send:
7320 /* Make sure output path knows we are doing a TLP */
7321 *doing_tlp = 1;
7322 rack->r_timer_override = 1;
7323 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
7324 return (0);
7325 out:
7326 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
7327 return (0);
7328 }
7329
7330 /*
7331 * Delayed ack Timer, here we simply need to setup the
7332 * ACK_NOW flag and remove the DELACK flag. From there
7333 * the output routine will send the ack out.
7334 *
7335 * We only return 1, saying don't proceed, if all timers
7336 * are stopped (destroyed PCB?).
7337 */
7338 static int
rack_timeout_delack(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts)7339 rack_timeout_delack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
7340 {
7341
7342 rack_log_to_event(rack, RACK_TO_FRM_DELACK, NULL);
7343 tp->t_flags &= ~TF_DELACK;
7344 tp->t_flags |= TF_ACKNOW;
7345 KMOD_TCPSTAT_INC(tcps_delack);
7346 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
7347 return (0);
7348 }
7349
7350 static inline int
rack_send_ack_challange(struct tcp_rack * rack)7351 rack_send_ack_challange(struct tcp_rack *rack)
7352 {
7353 struct tcptemp *t_template;
7354
7355 t_template = tcpip_maketemplate(rack->rc_inp);
7356 if (t_template) {
7357 if (rack->forced_ack == 0) {
7358 rack->forced_ack = 1;
7359 rack->r_ctl.forced_ack_ts = tcp_get_usecs(NULL);
7360 } else {
7361 rack->probe_not_answered = 1;
7362 }
7363 tcp_respond(rack->rc_tp, t_template->tt_ipgen,
7364 &t_template->tt_t, (struct mbuf *)NULL,
7365 rack->rc_tp->rcv_nxt, rack->rc_tp->snd_una - 1, 0);
7366 free(t_template, M_TEMP);
7367 /* This does send an ack so kill any D-ack timer */
7368 if (rack->rc_tp->t_flags & TF_DELACK)
7369 rack->rc_tp->t_flags &= ~TF_DELACK;
7370 return(1);
7371 } else
7372 return (0);
7373
7374 }
7375
7376 /*
7377 * Persists timer, here we simply send the
7378 * same thing as a keepalive will.
7379 * the one byte send.
7380 *
7381 * We only return 1, saying don't proceed, if all timers
7382 * are stopped (destroyed PCB?).
7383 */
7384 static int
rack_timeout_persist(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts)7385 rack_timeout_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
7386 {
7387 int32_t retval = 1;
7388
7389 if (rack->rc_in_persist == 0)
7390 return (0);
7391 if (ctf_progress_timeout_check(tp, false)) {
7392 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
7393 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
7394 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends);
7395 return (-ETIMEDOUT); /* tcp_drop() */
7396 }
7397 /*
7398 * Persistence timer into zero window. Force a byte to be output, if
7399 * possible.
7400 */
7401 KMOD_TCPSTAT_INC(tcps_persisttimeo);
7402 /*
7403 * Hack: if the peer is dead/unreachable, we do not time out if the
7404 * window is closed. After a full backoff, drop the connection if
7405 * the idle time (no responses to probes) reaches the maximum
7406 * backoff that we would use if retransmitting.
7407 */
7408 if (tp->t_rxtshift >= V_tcp_retries &&
7409 (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
7410 TICKS_2_USEC(ticks - tp->t_rcvtime) >= RACK_REXMTVAL(tp) * tcp_totbackoff)) {
7411 KMOD_TCPSTAT_INC(tcps_persistdrop);
7412 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
7413 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends);
7414 retval = -ETIMEDOUT; /* tcp_drop() */
7415 goto out;
7416 }
7417 if ((sbavail(&rack->rc_inp->inp_socket->so_snd) == 0) &&
7418 tp->snd_una == tp->snd_max)
7419 rack_exit_persist(tp, rack, cts);
7420 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
7421 /*
7422 * If the user has closed the socket then drop a persisting
7423 * connection after a much reduced timeout.
7424 */
7425 if (tp->t_state > TCPS_CLOSE_WAIT &&
7426 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
7427 KMOD_TCPSTAT_INC(tcps_persistdrop);
7428 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
7429 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends);
7430 retval = -ETIMEDOUT; /* tcp_drop() */
7431 goto out;
7432 }
7433 if (rack_send_ack_challange(rack)) {
7434 /* only set it if we were answered */
7435 if (rack->probe_not_answered) {
7436 counter_u64_add(rack_persists_loss, 1);
7437 rack->r_ctl.persist_lost_ends++;
7438 }
7439 counter_u64_add(rack_persists_sends, 1);
7440 counter_u64_add(rack_out_size[TCP_MSS_ACCT_PERSIST], 1);
7441 }
7442 if (tp->t_rxtshift < V_tcp_retries)
7443 tp->t_rxtshift++;
7444 out:
7445 rack_log_to_event(rack, RACK_TO_FRM_PERSIST, NULL);
7446 rack_start_hpts_timer(rack, tp, cts,
7447 0, 0, 0);
7448 return (retval);
7449 }
7450
7451 /*
7452 * If a keepalive goes off, we had no other timers
7453 * happening. We always return 1 here since this
7454 * routine either drops the connection or sends
7455 * out a segment with respond.
7456 */
7457 static int
rack_timeout_keepalive(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts)7458 rack_timeout_keepalive(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
7459 {
7460 struct inpcb *inp = tptoinpcb(tp);
7461
7462 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
7463 rack_log_to_event(rack, RACK_TO_FRM_KEEP, NULL);
7464 /*
7465 * Keep-alive timer went off; send something or drop connection if
7466 * idle for too long.
7467 */
7468 KMOD_TCPSTAT_INC(tcps_keeptimeo);
7469 if (tp->t_state < TCPS_ESTABLISHED)
7470 goto dropit;
7471 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
7472 tp->t_state <= TCPS_CLOSING) {
7473 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
7474 goto dropit;
7475 /*
7476 * Send a packet designed to force a response if the peer is
7477 * up and reachable: either an ACK if the connection is
7478 * still alive, or an RST if the peer has closed the
7479 * connection due to timeout or reboot. Using sequence
7480 * number tp->snd_una-1 causes the transmitted zero-length
7481 * segment to lie outside the receive window; by the
7482 * protocol spec, this requires the correspondent TCP to
7483 * respond.
7484 */
7485 KMOD_TCPSTAT_INC(tcps_keepprobe);
7486 rack_send_ack_challange(rack);
7487 }
7488 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
7489 return (1);
7490 dropit:
7491 KMOD_TCPSTAT_INC(tcps_keepdrops);
7492 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX);
7493 return (-ETIMEDOUT); /* tcp_drop() */
7494 }
7495
7496 /*
7497 * Retransmit helper function, clear up all the ack
7498 * flags and take care of important book keeping.
7499 */
7500 static void
rack_remxt_tmr(struct tcpcb * tp)7501 rack_remxt_tmr(struct tcpcb *tp)
7502 {
7503 /*
7504 * The retransmit timer went off, all sack'd blocks must be
7505 * un-acked.
7506 */
7507 struct rack_sendmap *rsm, *trsm = NULL;
7508 struct tcp_rack *rack;
7509
7510 rack = (struct tcp_rack *)tp->t_fb_ptr;
7511 rack_timer_cancel(tp, rack, tcp_get_usecs(NULL), __LINE__);
7512 rack_log_to_event(rack, RACK_TO_FRM_TMR, NULL);
7513 rack->r_timer_override = 1;
7514 rack->r_ctl.rc_snd_max_at_rto = tp->snd_max;
7515 rack->r_ctl.rc_last_timeout_snduna = tp->snd_una;
7516 rack->r_late = 0;
7517 rack->r_early = 0;
7518 rack->r_ctl.rc_agg_delayed = 0;
7519 rack->r_ctl.rc_agg_early = 0;
7520 if (rack->r_state && (rack->r_state != tp->t_state))
7521 rack_set_state(tp, rack);
7522 if (tp->t_rxtshift <= rack_rxt_scoreboard_clear_thresh) {
7523 /*
7524 * We do not clear the scoreboard until we have had
7525 * more than rack_rxt_scoreboard_clear_thresh time-outs.
7526 */
7527 rack->r_ctl.rc_resend = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
7528 if (rack->r_ctl.rc_resend != NULL)
7529 rack->r_ctl.rc_resend->r_flags |= RACK_TO_REXT;
7530
7531 return;
7532 }
7533 /*
7534 * Ideally we would like to be able to
7535 * mark SACK-PASS on anything not acked here.
7536 *
7537 * However, if we do that we would burst out
7538 * all that data 1ms apart. This would be unwise,
7539 * so for now we will just let the normal rxt timer
7540 * and tlp timer take care of it.
7541 *
7542 * Also we really need to stick them back in sequence
7543 * order. This way we send in the proper order and any
7544 * sacks that come floating in will "re-ack" the data.
7545 * To do this we zap the tmap with an INIT and then
7546 * walk through and place every rsm in the tail queue
7547 * hash table back in its seq ordered place.
7548 */
7549 TAILQ_INIT(&rack->r_ctl.rc_tmap);
7550
7551 TQHASH_FOREACH(rsm, rack->r_ctl.tqh) {
7552 rsm->r_dupack = 0;
7553 if (rack_verbose_logging)
7554 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
7555 /* We must re-add it back to the tlist */
7556 if (trsm == NULL) {
7557 TAILQ_INSERT_HEAD(&rack->r_ctl.rc_tmap, rsm, r_tnext);
7558 } else {
7559 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext);
7560 }
7561 rsm->r_in_tmap = 1;
7562 trsm = rsm;
7563 if (rsm->r_flags & RACK_ACKED)
7564 rsm->r_flags |= RACK_WAS_ACKED;
7565 rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS | RACK_RWND_COLLAPSED | RACK_WAS_LOST);
7566 rsm->r_flags |= RACK_MUST_RXT;
7567 }
7568 /* zero the lost since it's all gone */
7569 rack->r_ctl.rc_considered_lost = 0;
7570 /* Clear the count (we just un-acked them) */
7571 rack->r_ctl.rc_sacked = 0;
7572 rack->r_ctl.rc_sacklast = NULL;
7573 /* Clear the tlp rtx mark */
7574 rack->r_ctl.rc_resend = tqhash_min(rack->r_ctl.tqh);
7575 if (rack->r_ctl.rc_resend != NULL)
7576 rack->r_ctl.rc_resend->r_flags |= RACK_TO_REXT;
7577 rack->r_ctl.rc_prr_sndcnt = 0;
7578 rack_log_to_prr(rack, 6, 0, __LINE__);
7579 rack->r_ctl.rc_resend = tqhash_min(rack->r_ctl.tqh);
7580 if (rack->r_ctl.rc_resend != NULL)
7581 rack->r_ctl.rc_resend->r_flags |= RACK_TO_REXT;
7582 if (((tp->t_flags & TF_SACK_PERMIT) == 0) &&
7583 ((tp->t_flags & TF_SENTFIN) == 0)) {
7584 /*
7585 * For non-sack customers new data
7586 * needs to go out as retransmits until
7587 * we retransmit up to snd_max.
7588 */
7589 rack->r_must_retran = 1;
7590 rack->r_ctl.rc_out_at_rto = ctf_flight_size(rack->rc_tp,
7591 rack->r_ctl.rc_sacked);
7592 }
7593 }
7594
7595 static void
rack_convert_rtts(struct tcpcb * tp)7596 rack_convert_rtts(struct tcpcb *tp)
7597 {
7598 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_USEC);
7599 tp->t_rxtcur = RACK_REXMTVAL(tp);
7600 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
7601 tp->t_rxtcur += TICKS_2_USEC(tcp_rexmit_slop);
7602 }
7603 if (tp->t_rxtcur > rack_rto_max) {
7604 tp->t_rxtcur = rack_rto_max;
7605 }
7606 }
7607
7608 static void
rack_cc_conn_init(struct tcpcb * tp)7609 rack_cc_conn_init(struct tcpcb *tp)
7610 {
7611 struct tcp_rack *rack;
7612 uint32_t srtt;
7613
7614 rack = (struct tcp_rack *)tp->t_fb_ptr;
7615 srtt = tp->t_srtt;
7616 cc_conn_init(tp);
7617 /*
7618 * Now convert to rack's internal format,
7619 * if required.
7620 */
7621 if ((srtt == 0) && (tp->t_srtt != 0))
7622 rack_convert_rtts(tp);
7623 /*
7624 * We want a chance to stay in slowstart as
7625 * we create a connection. TCP spec says that
7626 * initially ssthresh is infinite. For our
7627 * purposes that is the snd_wnd.
7628 */
7629 if (tp->snd_ssthresh < tp->snd_wnd) {
7630 tp->snd_ssthresh = tp->snd_wnd;
7631 }
7632 /*
7633 * We also want to assure a IW worth of
7634 * data can get inflight.
7635 */
7636 if (rc_init_window(rack) < tp->snd_cwnd)
7637 tp->snd_cwnd = rc_init_window(rack);
7638 }
7639
7640 /*
7641 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
7642 * we will setup to retransmit the lowest seq number outstanding.
7643 */
7644 static int
rack_timeout_rxt(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts)7645 rack_timeout_rxt(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
7646 {
7647 struct inpcb *inp = tptoinpcb(tp);
7648 int32_t rexmt;
7649 int32_t retval = 0;
7650 bool isipv6;
7651
7652 if ((tp->t_flags & TF_GPUTINPROG) &&
7653 (tp->t_rxtshift)) {
7654 /*
7655 * We have had a second timeout
7656 * measurements on successive rxt's are not profitable.
7657 * It is unlikely to be of any use (the network is
7658 * broken or the client went away).
7659 */
7660 tp->t_flags &= ~TF_GPUTINPROG;
7661 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
7662 rack->r_ctl.rc_gp_srtt /*flex1*/,
7663 tp->gput_seq,
7664 0, 0, 18, __LINE__, NULL, 0);
7665 }
7666 if (ctf_progress_timeout_check(tp, false)) {
7667 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN);
7668 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
7669 return (-ETIMEDOUT); /* tcp_drop() */
7670 }
7671 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
7672 rack->r_ctl.retran_during_recovery = 0;
7673 rack->rc_ack_required = 1;
7674 rack->r_ctl.dsack_byte_cnt = 0;
7675 if (IN_RECOVERY(tp->t_flags) &&
7676 (rack->rto_from_rec == 0)) {
7677 /*
7678 * Mark that we had a rto while in recovery
7679 * and save the ssthresh so if we go back
7680 * into recovery we will have a chance
7681 * to slowstart back to the level.
7682 */
7683 rack->rto_from_rec = 1;
7684 rack->r_ctl.rto_ssthresh = tp->snd_ssthresh;
7685 }
7686 if (IN_FASTRECOVERY(tp->t_flags))
7687 tp->t_flags |= TF_WASFRECOVERY;
7688 else
7689 tp->t_flags &= ~TF_WASFRECOVERY;
7690 if (IN_CONGRECOVERY(tp->t_flags))
7691 tp->t_flags |= TF_WASCRECOVERY;
7692 else
7693 tp->t_flags &= ~TF_WASCRECOVERY;
7694 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
7695 (tp->snd_una == tp->snd_max)) {
7696 /* Nothing outstanding .. nothing to do */
7697 return (0);
7698 }
7699 if (rack->r_ctl.dsack_persist) {
7700 rack->r_ctl.dsack_persist--;
7701 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) {
7702 rack->r_ctl.num_dsack = 0;
7703 }
7704 rack_log_dsack_event(rack, 1, __LINE__, 0, 0);
7705 }
7706 /*
7707 * Rack can only run one timer at a time, so we cannot
7708 * run a KEEPINIT (gating SYN sending) and a retransmit
7709 * timer for the SYN. So if we are in a front state and
7710 * have a KEEPINIT timer we need to check the first transmit
7711 * against now to see if we have exceeded the KEEPINIT time
7712 * (if one is set).
7713 */
7714 if ((TCPS_HAVEESTABLISHED(tp->t_state) == 0) &&
7715 (TP_KEEPINIT(tp) != 0)) {
7716 struct rack_sendmap *rsm;
7717
7718 rsm = tqhash_min(rack->r_ctl.tqh);
7719 if (rsm) {
7720 /* Ok we have something outstanding to test keepinit with */
7721 if ((TSTMP_GT(cts, (uint32_t)rsm->r_tim_lastsent[0])) &&
7722 ((cts - (uint32_t)rsm->r_tim_lastsent[0]) >= TICKS_2_USEC(TP_KEEPINIT(tp)))) {
7723 /* We have exceeded the KEEPINIT time */
7724 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX);
7725 goto drop_it;
7726 }
7727 }
7728 }
7729 /*
7730 * Retransmission timer went off. Message has not been acked within
7731 * retransmit interval. Back off to a longer retransmit interval
7732 * and retransmit one segment.
7733 */
7734 if ((rack->r_ctl.rc_resend == NULL) ||
7735 ((rack->r_ctl.rc_resend->r_flags & RACK_RWND_COLLAPSED) == 0)) {
7736 /*
7737 * If the rwnd collapsed on
7738 * the one we are retransmitting
7739 * it does not count against the
7740 * rxt count.
7741 */
7742 tp->t_rxtshift++;
7743 }
7744 rack_remxt_tmr(tp);
7745 if (tp->t_rxtshift > V_tcp_retries) {
7746 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN);
7747 drop_it:
7748 tp->t_rxtshift = V_tcp_retries;
7749 KMOD_TCPSTAT_INC(tcps_timeoutdrop);
7750 /* XXXGL: previously t_softerror was casted to uint16_t */
7751 MPASS(tp->t_softerror >= 0);
7752 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;
7753 goto out; /* tcp_drop() */
7754 }
7755 if (tp->t_state == TCPS_SYN_SENT) {
7756 /*
7757 * If the SYN was retransmitted, indicate CWND to be limited
7758 * to 1 segment in cc_conn_init().
7759 */
7760 tp->snd_cwnd = 1;
7761 } else if (tp->t_rxtshift == 1) {
7762 /*
7763 * first retransmit; record ssthresh and cwnd so they can be
7764 * recovered if this turns out to be a "bad" retransmit. A
7765 * retransmit is considered "bad" if an ACK for this segment
7766 * is received within RTT/2 interval; the assumption here is
7767 * that the ACK was already in flight. See "On Estimating
7768 * End-to-End Network Path Properties" by Allman and Paxson
7769 * for more details.
7770 */
7771 tp->snd_cwnd_prev = tp->snd_cwnd;
7772 tp->snd_ssthresh_prev = tp->snd_ssthresh;
7773 tp->snd_recover_prev = tp->snd_recover;
7774 tp->t_badrxtwin = ticks + (USEC_2_TICKS(tp->t_srtt)/2);
7775 tp->t_flags |= TF_PREVVALID;
7776 } else if ((tp->t_flags & TF_RCVD_TSTMP) == 0)
7777 tp->t_flags &= ~TF_PREVVALID;
7778 KMOD_TCPSTAT_INC(tcps_rexmttimeo);
7779 if ((tp->t_state == TCPS_SYN_SENT) ||
7780 (tp->t_state == TCPS_SYN_RECEIVED))
7781 rexmt = RACK_INITIAL_RTO * tcp_backoff[tp->t_rxtshift];
7782 else
7783 rexmt = max(rack_rto_min, (tp->t_srtt + (tp->t_rttvar << 2))) * tcp_backoff[tp->t_rxtshift];
7784
7785 RACK_TCPT_RANGESET(tp->t_rxtcur, rexmt,
7786 max(rack_rto_min, rexmt), rack_rto_max, rack->r_ctl.timer_slop);
7787 /*
7788 * We enter the path for PLMTUD if connection is established or, if
7789 * connection is FIN_WAIT_1 status, reason for the last is that if
7790 * amount of data we send is very small, we could send it in couple
7791 * of packets and process straight to FIN. In that case we won't
7792 * catch ESTABLISHED state.
7793 */
7794 #ifdef INET6
7795 isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false;
7796 #else
7797 isipv6 = false;
7798 #endif
7799 if (((V_tcp_pmtud_blackhole_detect == 1) ||
7800 (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) ||
7801 (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) &&
7802 ((tp->t_state == TCPS_ESTABLISHED) ||
7803 (tp->t_state == TCPS_FIN_WAIT_1))) {
7804 /*
7805 * Idea here is that at each stage of mtu probe (usually,
7806 * 1448 -> 1188 -> 524) should be given 2 chances to recover
7807 * before further clamping down. 'tp->t_rxtshift % 2 == 0'
7808 * should take care of that.
7809 */
7810 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
7811 (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) &&
7812 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
7813 tp->t_rxtshift % 2 == 0)) {
7814 /*
7815 * Enter Path MTU Black-hole Detection mechanism: -
7816 * Disable Path MTU Discovery (IP "DF" bit). -
7817 * Reduce MTU to lower value than what we negotiated
7818 * with peer.
7819 */
7820 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
7821 /* Record that we may have found a black hole. */
7822 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
7823 /* Keep track of previous MSS. */
7824 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
7825 }
7826
7827 /*
7828 * Reduce the MSS to blackhole value or to the
7829 * default in an attempt to retransmit.
7830 */
7831 #ifdef INET6
7832 if (isipv6 &&
7833 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
7834 /* Use the sysctl tuneable blackhole MSS. */
7835 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
7836 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
7837 } else if (isipv6) {
7838 /* Use the default MSS. */
7839 tp->t_maxseg = V_tcp_v6mssdflt;
7840 /*
7841 * Disable Path MTU Discovery when we switch
7842 * to minmss.
7843 */
7844 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
7845 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
7846 }
7847 #endif
7848 #if defined(INET6) && defined(INET)
7849 else
7850 #endif
7851 #ifdef INET
7852 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
7853 /* Use the sysctl tuneable blackhole MSS. */
7854 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
7855 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
7856 } else {
7857 /* Use the default MSS. */
7858 tp->t_maxseg = V_tcp_mssdflt;
7859 /*
7860 * Disable Path MTU Discovery when we switch
7861 * to minmss.
7862 */
7863 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
7864 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
7865 }
7866 #endif
7867 } else {
7868 /*
7869 * If further retransmissions are still unsuccessful
7870 * with a lowered MTU, maybe this isn't a blackhole
7871 * and we restore the previous MSS and blackhole
7872 * detection flags. The limit '6' is determined by
7873 * giving each probe stage (1448, 1188, 524) 2
7874 * chances to recover.
7875 */
7876 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
7877 (tp->t_rxtshift >= 6)) {
7878 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
7879 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
7880 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
7881 if (tp->t_maxseg < V_tcp_mssdflt) {
7882 /*
7883 * The MSS is so small we should not
7884 * process incoming SACK's since we are
7885 * subject to attack in such a case.
7886 */
7887 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
7888 } else {
7889 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
7890 }
7891 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed);
7892 }
7893 }
7894 }
7895 /*
7896 * Disable RFC1323 and SACK if we haven't got any response to
7897 * our third SYN to work-around some broken terminal servers
7898 * (most of which have hopefully been retired) that have bad VJ
7899 * header compression code which trashes TCP segments containing
7900 * unknown-to-them TCP options.
7901 */
7902 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
7903 (tp->t_rxtshift == 3))
7904 tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_SACK_PERMIT);
7905 /*
7906 * If we backed off this far, our srtt estimate is probably bogus.
7907 * Clobber it so we'll take the next rtt measurement as our srtt;
7908 * move the current srtt into rttvar to keep the current retransmit
7909 * times until then.
7910 */
7911 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
7912 #ifdef INET6
7913 if ((inp->inp_vflag & INP_IPV6) != 0)
7914 in6_losing(inp);
7915 else
7916 #endif
7917 in_losing(inp);
7918 tp->t_rttvar += tp->t_srtt;
7919 tp->t_srtt = 0;
7920 }
7921 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
7922 tp->snd_recover = tp->snd_max;
7923 tp->t_flags |= TF_ACKNOW;
7924 tp->t_rtttime = 0;
7925 rack_cong_signal(tp, CC_RTO, tp->snd_una, __LINE__);
7926 out:
7927 return (retval);
7928 }
7929
7930 static int
rack_process_timers(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,uint8_t hpts_calling,uint8_t * doing_tlp)7931 rack_process_timers(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t hpts_calling, uint8_t *doing_tlp)
7932 {
7933 int32_t ret = 0;
7934 int32_t timers = (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
7935
7936 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
7937 (tp->t_flags & TF_GPUTINPROG)) {
7938 /*
7939 * We have a goodput in progress
7940 * and we have entered a late state.
7941 * Do we have enough data in the sb
7942 * to handle the GPUT request?
7943 */
7944 uint32_t bytes;
7945
7946 bytes = tp->gput_ack - tp->gput_seq;
7947 if (SEQ_GT(tp->gput_seq, tp->snd_una))
7948 bytes += tp->gput_seq - tp->snd_una;
7949 if (bytes > sbavail(&tptosocket(tp)->so_snd)) {
7950 /*
7951 * There are not enough bytes in the socket
7952 * buffer that have been sent to cover this
7953 * measurement. Cancel it.
7954 */
7955 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
7956 rack->r_ctl.rc_gp_srtt /*flex1*/,
7957 tp->gput_seq,
7958 0, 0, 18, __LINE__, NULL, 0);
7959 tp->t_flags &= ~TF_GPUTINPROG;
7960 }
7961 }
7962 if (timers == 0) {
7963 return (0);
7964 }
7965 if (tp->t_state == TCPS_LISTEN) {
7966 /* no timers on listen sockets */
7967 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
7968 return (0);
7969 return (1);
7970 }
7971 if ((timers & PACE_TMR_RACK) &&
7972 rack->rc_on_min_to) {
7973 /*
7974 * For the rack timer when we
7975 * are on a min-timeout (which means rrr_conf = 3)
7976 * we don't want to check the timer. It may
7977 * be going off for a pace and thats ok we
7978 * want to send the retransmit (if its ready).
7979 *
7980 * If its on a normal rack timer (non-min) then
7981 * we will check if its expired.
7982 */
7983 goto skip_time_check;
7984 }
7985 if (TSTMP_LT(cts, rack->r_ctl.rc_timer_exp)) {
7986 uint32_t left;
7987
7988 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
7989 ret = -1;
7990 rack_log_to_processing(rack, cts, ret, 0);
7991 return (0);
7992 }
7993 if (hpts_calling == 0) {
7994 /*
7995 * A user send or queued mbuf (sack) has called us? We
7996 * return 0 and let the pacing guards
7997 * deal with it if they should or
7998 * should not cause a send.
7999 */
8000 ret = -2;
8001 rack_log_to_processing(rack, cts, ret, 0);
8002 return (0);
8003 }
8004 /*
8005 * Ok our timer went off early and we are not paced false
8006 * alarm, go back to sleep. We make sure we don't have
8007 * no-sack wakeup on since we no longer have a PKT_OUTPUT
8008 * flag in place.
8009 */
8010 rack->rc_tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
8011 ret = -3;
8012 left = rack->r_ctl.rc_timer_exp - cts;
8013 tcp_hpts_insert(tp, left, NULL);
8014 rack_log_to_processing(rack, cts, ret, left);
8015 return (1);
8016 }
8017 skip_time_check:
8018 rack->rc_tmr_stopped = 0;
8019 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
8020 if (timers & PACE_TMR_DELACK) {
8021 ret = rack_timeout_delack(tp, rack, cts);
8022 } else if (timers & PACE_TMR_RACK) {
8023 rack->r_ctl.rc_tlp_rxt_last_time = cts;
8024 rack->r_fast_output = 0;
8025 ret = rack_timeout_rack(tp, rack, cts);
8026 } else if (timers & PACE_TMR_TLP) {
8027 rack->r_ctl.rc_tlp_rxt_last_time = cts;
8028 rack->r_fast_output = 0;
8029 ret = rack_timeout_tlp(tp, rack, cts, doing_tlp);
8030 } else if (timers & PACE_TMR_RXT) {
8031 rack->r_ctl.rc_tlp_rxt_last_time = cts;
8032 rack->r_fast_output = 0;
8033 ret = rack_timeout_rxt(tp, rack, cts);
8034 } else if (timers & PACE_TMR_PERSIT) {
8035 ret = rack_timeout_persist(tp, rack, cts);
8036 } else if (timers & PACE_TMR_KEEP) {
8037 ret = rack_timeout_keepalive(tp, rack, cts);
8038 }
8039 rack_log_to_processing(rack, cts, ret, timers);
8040 return (ret);
8041 }
8042
8043 static void
rack_timer_cancel(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cts,int line)8044 rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line)
8045 {
8046 struct timeval tv;
8047 uint32_t us_cts, flags_on_entry;
8048 uint8_t hpts_removed = 0;
8049
8050 flags_on_entry = rack->r_ctl.rc_hpts_flags;
8051 us_cts = tcp_get_usecs(&tv);
8052 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
8053 ((TSTMP_GEQ(us_cts, rack->r_ctl.rc_last_output_to)) ||
8054 ((tp->snd_max - tp->snd_una) == 0))) {
8055 tcp_hpts_remove(rack->rc_tp);
8056 hpts_removed = 1;
8057 /* If we were not delayed cancel out the flag. */
8058 if ((tp->snd_max - tp->snd_una) == 0)
8059 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
8060 rack_log_to_cancel(rack, hpts_removed, line, us_cts, &tv, flags_on_entry);
8061 }
8062 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
8063 rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
8064 if (tcp_in_hpts(rack->rc_tp) &&
8065 ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) {
8066 /*
8067 * Canceling timer's when we have no output being
8068 * paced. We also must remove ourselves from the
8069 * hpts.
8070 */
8071 tcp_hpts_remove(rack->rc_tp);
8072 hpts_removed = 1;
8073 }
8074 rack->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
8075 }
8076 if (hpts_removed == 0)
8077 rack_log_to_cancel(rack, hpts_removed, line, us_cts, &tv, flags_on_entry);
8078 }
8079
8080 static int
rack_stopall(struct tcpcb * tp)8081 rack_stopall(struct tcpcb *tp)
8082 {
8083 struct tcp_rack *rack;
8084
8085 rack = (struct tcp_rack *)tp->t_fb_ptr;
8086 rack->t_timers_stopped = 1;
8087
8088 tcp_hpts_remove(tp);
8089
8090 return (0);
8091 }
8092
8093 static void
rack_stop_all_timers(struct tcpcb * tp,struct tcp_rack * rack)8094 rack_stop_all_timers(struct tcpcb *tp, struct tcp_rack *rack)
8095 {
8096 /*
8097 * Assure no timers are running.
8098 */
8099 if (tcp_timer_active(tp, TT_PERSIST)) {
8100 /* We enter in persists, set the flag appropriately */
8101 rack->rc_in_persist = 1;
8102 }
8103 if (tcp_in_hpts(rack->rc_tp)) {
8104 tcp_hpts_remove(rack->rc_tp);
8105 }
8106 }
8107
8108 static void
rack_update_rsm(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint64_t ts,uint32_t add_flag,int segsiz)8109 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
8110 struct rack_sendmap *rsm, uint64_t ts, uint32_t add_flag, int segsiz)
8111 {
8112 int32_t idx;
8113
8114 rsm->r_rtr_cnt++;
8115 if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) {
8116 rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS;
8117 rsm->r_flags |= RACK_OVERMAX;
8118 }
8119 rsm->r_act_rxt_cnt++;
8120 /* Peg the count/index */
8121 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
8122 rsm->r_dupack = 0;
8123 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) {
8124 rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
8125 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
8126 }
8127 if (rsm->r_flags & RACK_WAS_LOST) {
8128 /*
8129 * We retransmitted it putting it back in flight
8130 * remove the lost desgination and reduce the
8131 * bytes considered lost.
8132 */
8133 rsm->r_flags &= ~RACK_WAS_LOST;
8134 KASSERT((rack->r_ctl.rc_considered_lost >= (rsm->r_end - rsm->r_start)),
8135 ("rsm:%p rack:%p rc_considered_lost goes negative", rsm, rack));
8136 if (rack->r_ctl.rc_considered_lost >= (rsm->r_end - rsm->r_start))
8137 rack->r_ctl.rc_considered_lost -= rsm->r_end - rsm->r_start;
8138 else
8139 rack->r_ctl.rc_considered_lost = 0;
8140 }
8141 idx = rsm->r_rtr_cnt - 1;
8142 rsm->r_tim_lastsent[idx] = ts;
8143 /*
8144 * Here we don't add in the len of send, since its already
8145 * in snduna <->snd_max.
8146 */
8147 rsm->r_fas = ctf_flight_size(rack->rc_tp,
8148 rack->r_ctl.rc_sacked);
8149 if (rsm->r_flags & RACK_ACKED) {
8150 /* Problably MTU discovery messing with us */
8151 rsm->r_flags &= ~RACK_ACKED;
8152 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
8153 }
8154 if (rsm->r_in_tmap) {
8155 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
8156 rsm->r_in_tmap = 0;
8157 }
8158 /* Lets make sure it really is in or not the GP window */
8159 rack_mark_in_gp_win(tp, rsm);
8160 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
8161 rsm->r_in_tmap = 1;
8162 rsm->r_bas = (uint8_t)(((rsm->r_end - rsm->r_start) + segsiz - 1) / segsiz);
8163 /* Take off the must retransmit flag, if its on */
8164 if (rsm->r_flags & RACK_MUST_RXT) {
8165 if (rack->r_must_retran)
8166 rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start);
8167 if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) {
8168 /*
8169 * We have retransmitted all we need. Clear
8170 * any must retransmit flags.
8171 */
8172 rack->r_must_retran = 0;
8173 rack->r_ctl.rc_out_at_rto = 0;
8174 }
8175 rsm->r_flags &= ~RACK_MUST_RXT;
8176 }
8177 /* Remove any collapsed flag */
8178 rsm->r_flags &= ~RACK_RWND_COLLAPSED;
8179 if (rsm->r_flags & RACK_SACK_PASSED) {
8180 /* We have retransmitted due to the SACK pass */
8181 rsm->r_flags &= ~RACK_SACK_PASSED;
8182 rsm->r_flags |= RACK_WAS_SACKPASS;
8183 }
8184 }
8185
8186 static uint32_t
rack_update_entry(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint64_t ts,int32_t * lenp,uint32_t add_flag,int segsiz)8187 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack,
8188 struct rack_sendmap *rsm, uint64_t ts, int32_t *lenp, uint32_t add_flag, int segsiz)
8189 {
8190 /*
8191 * We (re-)transmitted starting at rsm->r_start for some length
8192 * (possibly less than r_end.
8193 */
8194 struct rack_sendmap *nrsm;
8195 int insret __diagused;
8196 uint32_t c_end;
8197 int32_t len;
8198
8199 len = *lenp;
8200 c_end = rsm->r_start + len;
8201 if (SEQ_GEQ(c_end, rsm->r_end)) {
8202 /*
8203 * We retransmitted the whole piece or more than the whole
8204 * slopping into the next rsm.
8205 */
8206 rack_update_rsm(tp, rack, rsm, ts, add_flag, segsiz);
8207 if (c_end == rsm->r_end) {
8208 *lenp = 0;
8209 return (0);
8210 } else {
8211 int32_t act_len;
8212
8213 /* Hangs over the end return whats left */
8214 act_len = rsm->r_end - rsm->r_start;
8215 *lenp = (len - act_len);
8216 return (rsm->r_end);
8217 }
8218 /* We don't get out of this block. */
8219 }
8220 /*
8221 * Here we retransmitted less than the whole thing which means we
8222 * have to split this into what was transmitted and what was not.
8223 */
8224 nrsm = rack_alloc_full_limit(rack);
8225 if (nrsm == NULL) {
8226 /*
8227 * We can't get memory, so lets not proceed.
8228 */
8229 *lenp = 0;
8230 return (0);
8231 }
8232 /*
8233 * So here we are going to take the original rsm and make it what we
8234 * retransmitted. nrsm will be the tail portion we did not
8235 * retransmit. For example say the chunk was 1, 11 (10 bytes). And
8236 * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to
8237 * 1, 6 and the new piece will be 6, 11.
8238 */
8239 rack_clone_rsm(rack, nrsm, rsm, c_end);
8240 nrsm->r_dupack = 0;
8241 rack_log_retran_reason(rack, nrsm, __LINE__, 0, 2);
8242 #ifndef INVARIANTS
8243 (void)tqhash_insert(rack->r_ctl.tqh, nrsm);
8244 #else
8245 if ((insret = tqhash_insert(rack->r_ctl.tqh, nrsm)) != 0) {
8246 panic("Insert in tailq_hash of %p fails ret:%d rack:%p rsm:%p",
8247 nrsm, insret, rack, rsm);
8248 }
8249 #endif
8250 if (rsm->r_in_tmap) {
8251 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8252 nrsm->r_in_tmap = 1;
8253 }
8254 rsm->r_flags &= (~RACK_HAS_FIN);
8255 rack_update_rsm(tp, rack, rsm, ts, add_flag, segsiz);
8256 /* Log a split of rsm into rsm and nrsm */
8257 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__);
8258 *lenp = 0;
8259 return (0);
8260 }
8261
8262 static void
rack_log_output(struct tcpcb * tp,struct tcpopt * to,int32_t len,uint32_t seq_out,uint16_t th_flags,int32_t err,uint64_t cts,struct rack_sendmap * hintrsm,uint32_t add_flag,struct mbuf * s_mb,uint32_t s_moff,int hw_tls,int segsiz)8263 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len,
8264 uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t cts,
8265 struct rack_sendmap *hintrsm, uint32_t add_flag, struct mbuf *s_mb,
8266 uint32_t s_moff, int hw_tls, int segsiz)
8267 {
8268 struct tcp_rack *rack;
8269 struct rack_sendmap *rsm, *nrsm;
8270 int insret __diagused;
8271
8272 register uint32_t snd_max, snd_una;
8273
8274 /*
8275 * Add to the RACK log of packets in flight or retransmitted. If
8276 * there is a TS option we will use the TS echoed, if not we will
8277 * grab a TS.
8278 *
8279 * Retransmissions will increment the count and move the ts to its
8280 * proper place. Note that if options do not include TS's then we
8281 * won't be able to effectively use the ACK for an RTT on a retran.
8282 *
8283 * Notes about r_start and r_end. Lets consider a send starting at
8284 * sequence 1 for 10 bytes. In such an example the r_start would be
8285 * 1 (starting sequence) but the r_end would be r_start+len i.e. 11.
8286 * This means that r_end is actually the first sequence for the next
8287 * slot (11).
8288 *
8289 */
8290 /*
8291 * If err is set what do we do XXXrrs? should we not add the thing?
8292 * -- i.e. return if err != 0 or should we pretend we sent it? --
8293 * i.e. proceed with add ** do this for now.
8294 */
8295 INP_WLOCK_ASSERT(tptoinpcb(tp));
8296 if (err)
8297 /*
8298 * We don't log errors -- we could but snd_max does not
8299 * advance in this case either.
8300 */
8301 return;
8302
8303 if (th_flags & TH_RST) {
8304 /*
8305 * We don't log resets and we return immediately from
8306 * sending
8307 */
8308 return;
8309 }
8310 rack = (struct tcp_rack *)tp->t_fb_ptr;
8311 snd_una = tp->snd_una;
8312 snd_max = tp->snd_max;
8313 if (th_flags & (TH_SYN | TH_FIN)) {
8314 /*
8315 * The call to rack_log_output is made before bumping
8316 * snd_max. This means we can record one extra byte on a SYN
8317 * or FIN if seq_out is adding more on and a FIN is present
8318 * (and we are not resending).
8319 */
8320 if ((th_flags & TH_SYN) && (seq_out == tp->iss))
8321 len++;
8322 if (th_flags & TH_FIN)
8323 len++;
8324 }
8325 if (SEQ_LEQ((seq_out + len), snd_una)) {
8326 /* Are sending an old segment to induce an ack (keep-alive)? */
8327 return;
8328 }
8329 if (SEQ_LT(seq_out, snd_una)) {
8330 /* huh? should we panic? */
8331 uint32_t end;
8332
8333 end = seq_out + len;
8334 seq_out = snd_una;
8335 if (SEQ_GEQ(end, seq_out))
8336 len = end - seq_out;
8337 else
8338 len = 0;
8339 }
8340 if (len == 0) {
8341 /* We don't log zero window probes */
8342 return;
8343 }
8344 if (IN_FASTRECOVERY(tp->t_flags)) {
8345 rack->r_ctl.rc_prr_out += len;
8346 }
8347 /* First question is it a retransmission or new? */
8348 if (seq_out == snd_max) {
8349 /* Its new */
8350 rack_chk_req_and_hybrid_on_out(rack, seq_out, len, cts);
8351 again:
8352 rsm = rack_alloc(rack);
8353 if (rsm == NULL) {
8354 /*
8355 * Hmm out of memory and the tcb got destroyed while
8356 * we tried to wait.
8357 */
8358 return;
8359 }
8360 if (th_flags & TH_FIN) {
8361 rsm->r_flags = RACK_HAS_FIN|add_flag;
8362 } else {
8363 rsm->r_flags = add_flag;
8364 }
8365 if (hw_tls)
8366 rsm->r_hw_tls = 1;
8367 rsm->r_tim_lastsent[0] = cts;
8368 rsm->r_rtr_cnt = 1;
8369 rsm->r_act_rxt_cnt = 0;
8370 rsm->r_rtr_bytes = 0;
8371 if (th_flags & TH_SYN) {
8372 /* The data space is one beyond snd_una */
8373 rsm->r_flags |= RACK_HAS_SYN;
8374 }
8375 rsm->r_start = seq_out;
8376 rsm->r_end = rsm->r_start + len;
8377 rack_mark_in_gp_win(tp, rsm);
8378 rsm->r_dupack = 0;
8379 /*
8380 * save off the mbuf location that
8381 * sndmbuf_noadv returned (which is
8382 * where we started copying from)..
8383 */
8384 rsm->m = s_mb;
8385 rsm->soff = s_moff;
8386 /*
8387 * Here we do add in the len of send, since its not yet
8388 * reflected in in snduna <->snd_max
8389 */
8390 rsm->r_fas = (ctf_flight_size(rack->rc_tp,
8391 rack->r_ctl.rc_sacked) +
8392 (rsm->r_end - rsm->r_start));
8393 if ((rack->rc_initial_ss_comp == 0) &&
8394 (rack->r_ctl.ss_hi_fs < rsm->r_fas)) {
8395 rack->r_ctl.ss_hi_fs = rsm->r_fas;
8396 }
8397 /* rsm->m will be NULL if RACK_HAS_SYN or RACK_HAS_FIN is set */
8398 if (rsm->m) {
8399 if (rsm->m->m_len <= rsm->soff) {
8400 /*
8401 * XXXrrs Question, will this happen?
8402 *
8403 * If sbsndptr is set at the correct place
8404 * then s_moff should always be somewhere
8405 * within rsm->m. But if the sbsndptr was
8406 * off then that won't be true. If it occurs
8407 * we need to walkout to the correct location.
8408 */
8409 struct mbuf *lm;
8410
8411 lm = rsm->m;
8412 while (lm->m_len <= rsm->soff) {
8413 rsm->soff -= lm->m_len;
8414 lm = lm->m_next;
8415 KASSERT(lm != NULL, ("%s rack:%p lm goes null orig_off:%u origmb:%p rsm->soff:%u",
8416 __func__, rack, s_moff, s_mb, rsm->soff));
8417 }
8418 rsm->m = lm;
8419 }
8420 rsm->orig_m_len = rsm->m->m_len;
8421 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
8422 } else {
8423 rsm->orig_m_len = 0;
8424 rsm->orig_t_space = 0;
8425 }
8426 rsm->r_bas = (uint8_t)((len + segsiz - 1) / segsiz);
8427 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
8428 /* Log a new rsm */
8429 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_NEW, 0, __LINE__);
8430 #ifndef INVARIANTS
8431 (void)tqhash_insert(rack->r_ctl.tqh, rsm);
8432 #else
8433 if ((insret = tqhash_insert(rack->r_ctl.tqh, rsm)) != 0) {
8434 panic("Insert in tailq_hash of %p fails ret:%d rack:%p rsm:%p",
8435 nrsm, insret, rack, rsm);
8436 }
8437 #endif
8438 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
8439 rsm->r_in_tmap = 1;
8440 if (rsm->r_flags & RACK_IS_PCM) {
8441 rack->r_ctl.pcm_i.send_time = cts;
8442 rack->r_ctl.pcm_i.eseq = rsm->r_end;
8443 /* First time through we set the start too */
8444 if (rack->pcm_in_progress == 0)
8445 rack->r_ctl.pcm_i.sseq = rsm->r_start;
8446 }
8447 /*
8448 * Special case detection, is there just a single
8449 * packet outstanding when we are not in recovery?
8450 *
8451 * If this is true mark it so.
8452 */
8453 if ((IN_FASTRECOVERY(tp->t_flags) == 0) &&
8454 (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) == ctf_fixed_maxseg(tp))) {
8455 struct rack_sendmap *prsm;
8456
8457 prsm = tqhash_prev(rack->r_ctl.tqh, rsm);
8458 if (prsm)
8459 prsm->r_one_out_nr = 1;
8460 }
8461 return;
8462 }
8463 /*
8464 * If we reach here its a retransmission and we need to find it.
8465 */
8466 more:
8467 if (hintrsm && (hintrsm->r_start == seq_out)) {
8468 rsm = hintrsm;
8469 hintrsm = NULL;
8470 } else {
8471 /* No hints sorry */
8472 rsm = NULL;
8473 }
8474 if ((rsm) && (rsm->r_start == seq_out)) {
8475 seq_out = rack_update_entry(tp, rack, rsm, cts, &len, add_flag, segsiz);
8476 if (len == 0) {
8477 return;
8478 } else {
8479 goto more;
8480 }
8481 }
8482 /* Ok it was not the last pointer go through it the hard way. */
8483 refind:
8484 rsm = tqhash_find(rack->r_ctl.tqh, seq_out);
8485 if (rsm) {
8486 if (rsm->r_start == seq_out) {
8487 seq_out = rack_update_entry(tp, rack, rsm, cts, &len, add_flag, segsiz);
8488 if (len == 0) {
8489 return;
8490 } else {
8491 goto refind;
8492 }
8493 }
8494 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
8495 /* Transmitted within this piece */
8496 /*
8497 * Ok we must split off the front and then let the
8498 * update do the rest
8499 */
8500 nrsm = rack_alloc_full_limit(rack);
8501 if (nrsm == NULL) {
8502 rack_update_rsm(tp, rack, rsm, cts, add_flag, segsiz);
8503 return;
8504 }
8505 /*
8506 * copy rsm to nrsm and then trim the front of rsm
8507 * to not include this part.
8508 */
8509 rack_clone_rsm(rack, nrsm, rsm, seq_out);
8510 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__);
8511 #ifndef INVARIANTS
8512 (void)tqhash_insert(rack->r_ctl.tqh, nrsm);
8513 #else
8514 if ((insret = tqhash_insert(rack->r_ctl.tqh, nrsm)) != 0) {
8515 panic("Insert in tailq_hash of %p fails ret:%d rack:%p rsm:%p",
8516 nrsm, insret, rack, rsm);
8517 }
8518 #endif
8519 if (rsm->r_in_tmap) {
8520 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8521 nrsm->r_in_tmap = 1;
8522 }
8523 rsm->r_flags &= (~RACK_HAS_FIN);
8524 seq_out = rack_update_entry(tp, rack, nrsm, cts, &len, add_flag, segsiz);
8525 if (len == 0) {
8526 return;
8527 } else if (len > 0)
8528 goto refind;
8529 }
8530 }
8531 /*
8532 * Hmm not found in map did they retransmit both old and on into the
8533 * new?
8534 */
8535 if (seq_out == tp->snd_max) {
8536 goto again;
8537 } else if (SEQ_LT(seq_out, tp->snd_max)) {
8538 #ifdef INVARIANTS
8539 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
8540 seq_out, len, tp->snd_una, tp->snd_max);
8541 printf("Starting Dump of all rack entries\n");
8542 TQHASH_FOREACH(rsm, rack->r_ctl.tqh) {
8543 printf("rsm:%p start:%u end:%u\n",
8544 rsm, rsm->r_start, rsm->r_end);
8545 }
8546 printf("Dump complete\n");
8547 panic("seq_out not found rack:%p tp:%p",
8548 rack, tp);
8549 #endif
8550 } else {
8551 #ifdef INVARIANTS
8552 /*
8553 * Hmm beyond sndmax? (only if we are using the new rtt-pack
8554 * flag)
8555 */
8556 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p",
8557 seq_out, len, tp->snd_max, tp);
8558 #endif
8559 }
8560 }
8561
8562 /*
8563 * Record one of the RTT updates from an ack into
8564 * our sample structure.
8565 */
8566
8567 static void
tcp_rack_xmit_timer(struct tcp_rack * rack,int32_t rtt,uint32_t len,uint32_t us_rtt,int confidence,struct rack_sendmap * rsm,uint16_t rtrcnt)8568 tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt, uint32_t len, uint32_t us_rtt,
8569 int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt)
8570 {
8571 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) ||
8572 (rack->r_ctl.rack_rs.rs_rtt_lowest > rtt)) {
8573 rack->r_ctl.rack_rs.rs_rtt_lowest = rtt;
8574 }
8575 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) ||
8576 (rack->r_ctl.rack_rs.rs_rtt_highest < rtt)) {
8577 rack->r_ctl.rack_rs.rs_rtt_highest = rtt;
8578 }
8579 if (rack->rc_tp->t_flags & TF_GPUTINPROG) {
8580 if (us_rtt < rack->r_ctl.rc_gp_lowrtt)
8581 rack->r_ctl.rc_gp_lowrtt = us_rtt;
8582 if (rack->rc_tp->snd_wnd > rack->r_ctl.rc_gp_high_rwnd)
8583 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd;
8584 }
8585 if ((confidence == 1) &&
8586 ((rsm == NULL) ||
8587 (rsm->r_just_ret) ||
8588 (rsm->r_one_out_nr &&
8589 len < (ctf_fixed_maxseg(rack->rc_tp) * 2)))) {
8590 /*
8591 * If the rsm had a just return
8592 * hit it then we can't trust the
8593 * rtt measurement for buffer deterimination
8594 * Note that a confidence of 2, indicates
8595 * SACK'd which overrides the r_just_ret or
8596 * the r_one_out_nr. If it was a CUM-ACK and
8597 * we had only two outstanding, but get an
8598 * ack for only 1. Then that also lowers our
8599 * confidence.
8600 */
8601 confidence = 0;
8602 }
8603 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) ||
8604 (rack->r_ctl.rack_rs.rs_us_rtt > us_rtt)) {
8605 if (rack->r_ctl.rack_rs.confidence == 0) {
8606 /*
8607 * We take anything with no current confidence
8608 * saved.
8609 */
8610 rack->r_ctl.rack_rs.rs_us_rtt = us_rtt;
8611 rack->r_ctl.rack_rs.confidence = confidence;
8612 rack->r_ctl.rack_rs.rs_us_rtrcnt = rtrcnt;
8613 } else if (confidence != 0) {
8614 /*
8615 * Once we have a confident number,
8616 * we can update it with a smaller
8617 * value since this confident number
8618 * may include the DSACK time until
8619 * the next segment (the second one) arrived.
8620 */
8621 rack->r_ctl.rack_rs.rs_us_rtt = us_rtt;
8622 rack->r_ctl.rack_rs.confidence = confidence;
8623 rack->r_ctl.rack_rs.rs_us_rtrcnt = rtrcnt;
8624 }
8625 }
8626 rack_log_rtt_upd(rack->rc_tp, rack, us_rtt, len, rsm, confidence);
8627 rack->r_ctl.rack_rs.rs_flags = RACK_RTT_VALID;
8628 rack->r_ctl.rack_rs.rs_rtt_tot += rtt;
8629 rack->r_ctl.rack_rs.rs_rtt_cnt++;
8630 }
8631
8632 /*
8633 * Collect new round-trip time estimate
8634 * and update averages and current timeout.
8635 */
8636 static void
tcp_rack_xmit_timer_commit(struct tcp_rack * rack,struct tcpcb * tp)8637 tcp_rack_xmit_timer_commit(struct tcp_rack *rack, struct tcpcb *tp)
8638 {
8639 int32_t delta;
8640 int32_t rtt;
8641
8642 if (rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY)
8643 /* No valid sample */
8644 return;
8645 if (rack->r_ctl.rc_rate_sample_method == USE_RTT_LOW) {
8646 /* We are to use the lowest RTT seen in a single ack */
8647 rtt = rack->r_ctl.rack_rs.rs_rtt_lowest;
8648 } else if (rack->r_ctl.rc_rate_sample_method == USE_RTT_HIGH) {
8649 /* We are to use the highest RTT seen in a single ack */
8650 rtt = rack->r_ctl.rack_rs.rs_rtt_highest;
8651 } else if (rack->r_ctl.rc_rate_sample_method == USE_RTT_AVG) {
8652 /* We are to use the average RTT seen in a single ack */
8653 rtt = (int32_t)(rack->r_ctl.rack_rs.rs_rtt_tot /
8654 (uint64_t)rack->r_ctl.rack_rs.rs_rtt_cnt);
8655 } else {
8656 #ifdef INVARIANTS
8657 panic("Unknown rtt variant %d", rack->r_ctl.rc_rate_sample_method);
8658 #endif
8659 return;
8660 }
8661 if (rtt == 0)
8662 rtt = 1;
8663 if (rack->rc_gp_rtt_set == 0) {
8664 /*
8665 * With no RTT we have to accept
8666 * even one we are not confident of.
8667 */
8668 rack->r_ctl.rc_gp_srtt = rack->r_ctl.rack_rs.rs_us_rtt;
8669 rack->rc_gp_rtt_set = 1;
8670 } else if (rack->r_ctl.rack_rs.confidence) {
8671 /* update the running gp srtt */
8672 rack->r_ctl.rc_gp_srtt -= (rack->r_ctl.rc_gp_srtt/8);
8673 rack->r_ctl.rc_gp_srtt += rack->r_ctl.rack_rs.rs_us_rtt / 8;
8674 }
8675 if (rack->r_ctl.rack_rs.confidence) {
8676 /*
8677 * record the low and high for highly buffered path computation,
8678 * we only do this if we are confident (not a retransmission).
8679 */
8680 if (rack->r_ctl.rc_highest_us_rtt < rack->r_ctl.rack_rs.rs_us_rtt) {
8681 rack->r_ctl.rc_highest_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt;
8682 }
8683 if (rack->rc_highly_buffered == 0) {
8684 /*
8685 * Currently once we declare a path has
8686 * highly buffered there is no going
8687 * back, which may be a problem...
8688 */
8689 if ((rack->r_ctl.rc_highest_us_rtt / rack->r_ctl.rc_lowest_us_rtt) > rack_hbp_thresh) {
8690 rack_log_rtt_shrinks(rack, rack->r_ctl.rack_rs.rs_us_rtt,
8691 rack->r_ctl.rc_highest_us_rtt,
8692 rack->r_ctl.rc_lowest_us_rtt,
8693 RACK_RTTS_SEEHBP);
8694 rack->rc_highly_buffered = 1;
8695 }
8696 }
8697 }
8698 if ((rack->r_ctl.rack_rs.confidence) ||
8699 (rack->r_ctl.rack_rs.rs_us_rtrcnt == 1)) {
8700 /*
8701 * If we are highly confident of it <or> it was
8702 * never retransmitted we accept it as the last us_rtt.
8703 */
8704 rack->r_ctl.rc_last_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt;
8705 /* The lowest rtt can be set if its was not retransmited */
8706 if (rack->r_ctl.rc_lowest_us_rtt > rack->r_ctl.rack_rs.rs_us_rtt) {
8707 rack->r_ctl.rc_lowest_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt;
8708 if (rack->r_ctl.rc_lowest_us_rtt == 0)
8709 rack->r_ctl.rc_lowest_us_rtt = 1;
8710 }
8711 }
8712 rack = (struct tcp_rack *)tp->t_fb_ptr;
8713 if (tp->t_srtt != 0) {
8714 /*
8715 * We keep a simple srtt in microseconds, like our rtt
8716 * measurement. We don't need to do any tricks with shifting
8717 * etc. Instead we just add in 1/8th of the new measurement
8718 * and subtract out 1/8 of the old srtt. We do the same with
8719 * the variance after finding the absolute value of the
8720 * difference between this sample and the current srtt.
8721 */
8722 delta = tp->t_srtt - rtt;
8723 /* Take off 1/8th of the current sRTT */
8724 tp->t_srtt -= (tp->t_srtt >> 3);
8725 /* Add in 1/8th of the new RTT just measured */
8726 tp->t_srtt += (rtt >> 3);
8727 if (tp->t_srtt <= 0)
8728 tp->t_srtt = 1;
8729 /* Now lets make the absolute value of the variance */
8730 if (delta < 0)
8731 delta = -delta;
8732 /* Subtract out 1/8th */
8733 tp->t_rttvar -= (tp->t_rttvar >> 3);
8734 /* Add in 1/8th of the new variance we just saw */
8735 tp->t_rttvar += (delta >> 3);
8736 if (tp->t_rttvar <= 0)
8737 tp->t_rttvar = 1;
8738 } else {
8739 /*
8740 * No rtt measurement yet - use the unsmoothed rtt. Set the
8741 * variance to half the rtt (so our first retransmit happens
8742 * at 3*rtt).
8743 */
8744 tp->t_srtt = rtt;
8745 tp->t_rttvar = rtt >> 1;
8746 }
8747 rack->rc_srtt_measure_made = 1;
8748 KMOD_TCPSTAT_INC(tcps_rttupdated);
8749 if (tp->t_rttupdated < UCHAR_MAX)
8750 tp->t_rttupdated++;
8751 #ifdef STATS
8752 if (rack_stats_gets_ms_rtt == 0) {
8753 /* Send in the microsecond rtt used for rxt timeout purposes */
8754 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt));
8755 } else if (rack_stats_gets_ms_rtt == 1) {
8756 /* Send in the millisecond rtt used for rxt timeout purposes */
8757 int32_t ms_rtt;
8758
8759 /* Round up */
8760 ms_rtt = (rtt + HPTS_USEC_IN_MSEC - 1) / HPTS_USEC_IN_MSEC;
8761 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, ms_rtt));
8762 } else if (rack_stats_gets_ms_rtt == 2) {
8763 /* Send in the millisecond rtt has close to the path RTT as we can get */
8764 int32_t ms_rtt;
8765
8766 /* Round up */
8767 ms_rtt = (rack->r_ctl.rack_rs.rs_us_rtt + HPTS_USEC_IN_MSEC - 1) / HPTS_USEC_IN_MSEC;
8768 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, ms_rtt));
8769 } else {
8770 /* Send in the microsecond rtt has close to the path RTT as we can get */
8771 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rack->r_ctl.rack_rs.rs_us_rtt));
8772 }
8773 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rack->r_ctl.rack_rs.rs_us_rtt));
8774 #endif
8775 rack->r_ctl.last_rcv_tstmp_for_rtt = tcp_tv_to_msec(&rack->r_ctl.act_rcv_time);
8776 /*
8777 * the retransmit should happen at rtt + 4 * rttvar. Because of the
8778 * way we do the smoothing, srtt and rttvar will each average +1/2
8779 * tick of bias. When we compute the retransmit timer, we want 1/2
8780 * tick of rounding and 1 extra tick because of +-1/2 tick
8781 * uncertainty in the firing of the timer. The bias will give us
8782 * exactly the 1.5 tick we need. But, because the bias is
8783 * statistical, we have to test that we don't drop below the minimum
8784 * feasible timer (which is 2 ticks).
8785 */
8786 tp->t_rxtshift = 0;
8787 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
8788 max(rack_rto_min, rtt + 2), rack_rto_max, rack->r_ctl.timer_slop);
8789 rack_log_rtt_sample(rack, rtt);
8790 tp->t_softerror = 0;
8791 }
8792
8793
8794 static void
rack_apply_updated_usrtt(struct tcp_rack * rack,uint32_t us_rtt,uint32_t us_cts)8795 rack_apply_updated_usrtt(struct tcp_rack *rack, uint32_t us_rtt, uint32_t us_cts)
8796 {
8797 /*
8798 * Apply to filter the inbound us-rtt at us_cts.
8799 */
8800 uint32_t old_rtt;
8801
8802 old_rtt = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
8803 apply_filter_min_small(&rack->r_ctl.rc_gp_min_rtt,
8804 us_rtt, us_cts);
8805 if (old_rtt > us_rtt) {
8806 /* We just hit a new lower rtt time */
8807 rack_log_rtt_shrinks(rack, us_cts, old_rtt,
8808 __LINE__, RACK_RTTS_NEWRTT);
8809 /*
8810 * Only count it if its lower than what we saw within our
8811 * calculated range.
8812 */
8813 if ((old_rtt - us_rtt) > rack_min_rtt_movement) {
8814 if (rack_probertt_lower_within &&
8815 rack->rc_gp_dyn_mul &&
8816 (rack->use_fixed_rate == 0) &&
8817 (rack->rc_always_pace)) {
8818 /*
8819 * We are seeing a new lower rtt very close
8820 * to the time that we would have entered probe-rtt.
8821 * This is probably due to the fact that a peer flow
8822 * has entered probe-rtt. Lets go in now too.
8823 */
8824 uint32_t val;
8825
8826 val = rack_probertt_lower_within * rack_time_between_probertt;
8827 val /= 100;
8828 if ((rack->in_probe_rtt == 0) &&
8829 (rack->rc_skip_timely == 0) &&
8830 ((us_cts - rack->r_ctl.rc_lower_rtt_us_cts) >= (rack_time_between_probertt - val))) {
8831 rack_enter_probertt(rack, us_cts);
8832 }
8833 }
8834 rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
8835 }
8836 }
8837 }
8838
8839 static int
rack_update_rtt(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,struct tcpopt * to,uint32_t cts,int32_t ack_type,tcp_seq th_ack)8840 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack,
8841 struct rack_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, tcp_seq th_ack)
8842 {
8843 uint32_t us_rtt;
8844 int32_t i, all;
8845 uint32_t t, len_acked;
8846
8847 if ((rsm->r_flags & RACK_ACKED) ||
8848 (rsm->r_flags & RACK_WAS_ACKED))
8849 /* Already done */
8850 return (0);
8851 if (rsm->r_no_rtt_allowed) {
8852 /* Not allowed */
8853 return (0);
8854 }
8855 if (ack_type == CUM_ACKED) {
8856 if (SEQ_GT(th_ack, rsm->r_end)) {
8857 len_acked = rsm->r_end - rsm->r_start;
8858 all = 1;
8859 } else {
8860 len_acked = th_ack - rsm->r_start;
8861 all = 0;
8862 }
8863 } else {
8864 len_acked = rsm->r_end - rsm->r_start;
8865 all = 0;
8866 }
8867 if (rsm->r_rtr_cnt == 1) {
8868
8869 t = cts - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
8870 if ((int)t <= 0)
8871 t = 1;
8872 if (!tp->t_rttlow || tp->t_rttlow > t)
8873 tp->t_rttlow = t;
8874 if (!rack->r_ctl.rc_rack_min_rtt ||
8875 SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
8876 rack->r_ctl.rc_rack_min_rtt = t;
8877 if (rack->r_ctl.rc_rack_min_rtt == 0) {
8878 rack->r_ctl.rc_rack_min_rtt = 1;
8879 }
8880 }
8881 if (TSTMP_GT(tcp_tv_to_usec(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]))
8882 us_rtt = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
8883 else
8884 us_rtt = tcp_get_usecs(NULL) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
8885 if (us_rtt == 0)
8886 us_rtt = 1;
8887 if (CC_ALGO(tp)->rttsample != NULL) {
8888 /* Kick the RTT to the CC */
8889 CC_ALGO(tp)->rttsample(&tp->t_ccv, us_rtt, 1, rsm->r_fas);
8890 }
8891 rack_apply_updated_usrtt(rack, us_rtt, tcp_tv_to_usec(&rack->r_ctl.act_rcv_time));
8892 if (ack_type == SACKED) {
8893 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 1);
8894 tcp_rack_xmit_timer(rack, t + 1, len_acked, us_rtt, 2 , rsm, rsm->r_rtr_cnt);
8895 } else {
8896 /*
8897 * We need to setup what our confidence
8898 * is in this ack.
8899 *
8900 * If the rsm was app limited and it is
8901 * less than a mss in length (the end
8902 * of the send) then we have a gap. If we
8903 * were app limited but say we were sending
8904 * multiple MSS's then we are more confident
8905 * int it.
8906 *
8907 * When we are not app-limited then we see if
8908 * the rsm is being included in the current
8909 * measurement, we tell this by the app_limited_needs_set
8910 * flag.
8911 *
8912 * Note that being cwnd blocked is not applimited
8913 * as well as the pacing delay between packets which
8914 * are sending only 1 or 2 MSS's also will show up
8915 * in the RTT. We probably need to examine this algorithm
8916 * a bit more and enhance it to account for the delay
8917 * between rsm's. We could do that by saving off the
8918 * pacing delay of each rsm (in an rsm) and then
8919 * factoring that in somehow though for now I am
8920 * not sure how :)
8921 */
8922 int calc_conf = 0;
8923
8924 if (rsm->r_flags & RACK_APP_LIMITED) {
8925 if (all && (len_acked <= ctf_fixed_maxseg(tp)))
8926 calc_conf = 0;
8927 else
8928 calc_conf = 1;
8929 } else if (rack->app_limited_needs_set == 0) {
8930 calc_conf = 1;
8931 } else {
8932 calc_conf = 0;
8933 }
8934 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 2);
8935 tcp_rack_xmit_timer(rack, t + 1, len_acked, us_rtt,
8936 calc_conf, rsm, rsm->r_rtr_cnt);
8937 }
8938 if ((rsm->r_flags & RACK_TLP) &&
8939 (!IN_FASTRECOVERY(tp->t_flags))) {
8940 /* Segment was a TLP and our retrans matched */
8941 if (rack->r_ctl.rc_tlp_cwnd_reduce) {
8942 rack_cong_signal(tp, CC_NDUPACK, th_ack, __LINE__);
8943 }
8944 }
8945 if ((rack->r_ctl.rc_rack_tmit_time == 0) ||
8946 (SEQ_LT(rack->r_ctl.rc_rack_tmit_time,
8947 (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]))) {
8948 /* New more recent rack_tmit_time */
8949 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
8950 if (rack->r_ctl.rc_rack_tmit_time == 0)
8951 rack->r_ctl.rc_rack_tmit_time = 1;
8952 rack->rc_rack_rtt = t;
8953 }
8954 return (1);
8955 }
8956 /*
8957 * We clear the soft/rxtshift since we got an ack.
8958 * There is no assurance we will call the commit() function
8959 * so we need to clear these to avoid incorrect handling.
8960 */
8961 tp->t_rxtshift = 0;
8962 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
8963 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
8964 tp->t_softerror = 0;
8965 if (to && (to->to_flags & TOF_TS) &&
8966 (ack_type == CUM_ACKED) &&
8967 (to->to_tsecr) &&
8968 ((rsm->r_flags & RACK_OVERMAX) == 0)) {
8969 /*
8970 * Now which timestamp does it match? In this block the ACK
8971 * must be coming from a previous transmission.
8972 */
8973 for (i = 0; i < rsm->r_rtr_cnt; i++) {
8974 if (rack_ts_to_msec(rsm->r_tim_lastsent[i]) == to->to_tsecr) {
8975 t = cts - (uint32_t)rsm->r_tim_lastsent[i];
8976 if ((int)t <= 0)
8977 t = 1;
8978 if (CC_ALGO(tp)->rttsample != NULL) {
8979 /*
8980 * Kick the RTT to the CC, here
8981 * we lie a bit in that we know the
8982 * retransmission is correct even though
8983 * we retransmitted. This is because
8984 * we match the timestamps.
8985 */
8986 if (TSTMP_GT(tcp_tv_to_usec(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[i]))
8987 us_rtt = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[i];
8988 else
8989 us_rtt = tcp_get_usecs(NULL) - (uint32_t)rsm->r_tim_lastsent[i];
8990 CC_ALGO(tp)->rttsample(&tp->t_ccv, us_rtt, 1, rsm->r_fas);
8991 }
8992 if ((i + 1) < rsm->r_rtr_cnt) {
8993 /*
8994 * The peer ack'd from our previous
8995 * transmission. We have a spurious
8996 * retransmission and thus we dont
8997 * want to update our rack_rtt.
8998 *
8999 * Hmm should there be a CC revert here?
9000 *
9001 */
9002 return (0);
9003 }
9004 if (!tp->t_rttlow || tp->t_rttlow > t)
9005 tp->t_rttlow = t;
9006 if (!rack->r_ctl.rc_rack_min_rtt || SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
9007 rack->r_ctl.rc_rack_min_rtt = t;
9008 if (rack->r_ctl.rc_rack_min_rtt == 0) {
9009 rack->r_ctl.rc_rack_min_rtt = 1;
9010 }
9011 }
9012 if ((rack->r_ctl.rc_rack_tmit_time == 0) ||
9013 (SEQ_LT(rack->r_ctl.rc_rack_tmit_time,
9014 (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]))) {
9015 /* New more recent rack_tmit_time */
9016 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
9017 if (rack->r_ctl.rc_rack_tmit_time == 0)
9018 rack->r_ctl.rc_rack_tmit_time = 1;
9019 rack->rc_rack_rtt = t;
9020 }
9021 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[i], cts, 3);
9022 tcp_rack_xmit_timer(rack, t + 1, len_acked, t, 0, rsm,
9023 rsm->r_rtr_cnt);
9024 return (1);
9025 }
9026 }
9027 /* If we are logging log out the sendmap */
9028 if (tcp_bblogging_on(rack->rc_tp)) {
9029 for (i = 0; i < rsm->r_rtr_cnt; i++) {
9030 rack_log_rtt_sendmap(rack, i, rsm->r_tim_lastsent[i], to->to_tsecr);
9031 }
9032 }
9033 goto ts_not_found;
9034 } else {
9035 /*
9036 * Ok its a SACK block that we retransmitted. or a windows
9037 * machine without timestamps. We can tell nothing from the
9038 * time-stamp since its not there or the time the peer last
9039 * received a segment that moved forward its cum-ack point.
9040 */
9041 ts_not_found:
9042 i = rsm->r_rtr_cnt - 1;
9043 t = cts - (uint32_t)rsm->r_tim_lastsent[i];
9044 if ((int)t <= 0)
9045 t = 1;
9046 if (rack->r_ctl.rc_rack_min_rtt && SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
9047 /*
9048 * We retransmitted and the ack came back in less
9049 * than the smallest rtt we have observed. We most
9050 * likely did an improper retransmit as outlined in
9051 * 6.2 Step 2 point 2 in the rack-draft so we
9052 * don't want to update our rack_rtt. We in
9053 * theory (in future) might want to think about reverting our
9054 * cwnd state but we won't for now.
9055 */
9056 return (0);
9057 } else if (rack->r_ctl.rc_rack_min_rtt) {
9058 /*
9059 * We retransmitted it and the retransmit did the
9060 * job.
9061 */
9062 if (!rack->r_ctl.rc_rack_min_rtt ||
9063 SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
9064 rack->r_ctl.rc_rack_min_rtt = t;
9065 if (rack->r_ctl.rc_rack_min_rtt == 0) {
9066 rack->r_ctl.rc_rack_min_rtt = 1;
9067 }
9068 }
9069 if ((rack->r_ctl.rc_rack_tmit_time == 0) ||
9070 (SEQ_LT(rack->r_ctl.rc_rack_tmit_time,
9071 (uint32_t)rsm->r_tim_lastsent[i]))) {
9072 /* New more recent rack_tmit_time */
9073 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[i];
9074 if (rack->r_ctl.rc_rack_tmit_time == 0)
9075 rack->r_ctl.rc_rack_tmit_time = 1;
9076 rack->rc_rack_rtt = t;
9077 }
9078 return (1);
9079 }
9080 }
9081 return (0);
9082 }
9083
9084 /*
9085 * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
9086 */
9087 static void
rack_log_sack_passed(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint32_t cts)9088 rack_log_sack_passed(struct tcpcb *tp,
9089 struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t cts)
9090 {
9091 struct rack_sendmap *nrsm;
9092 uint32_t thresh;
9093
9094 /* Get our rxt threshold for lost consideration */
9095 thresh = rack_calc_thresh_rack(rack, rack_grab_rtt(tp, rack), cts, __LINE__, 0);
9096 /* Now start looking at rsm's */
9097 nrsm = rsm;
9098 TAILQ_FOREACH_REVERSE_FROM(nrsm, &rack->r_ctl.rc_tmap,
9099 rack_head, r_tnext) {
9100 if (nrsm == rsm) {
9101 /* Skip original segment he is acked */
9102 continue;
9103 }
9104 if (nrsm->r_flags & RACK_ACKED) {
9105 /*
9106 * Skip ack'd segments, though we
9107 * should not see these, since tmap
9108 * should not have ack'd segments.
9109 */
9110 continue;
9111 }
9112 if (nrsm->r_flags & RACK_RWND_COLLAPSED) {
9113 /*
9114 * If the peer dropped the rwnd on
9115 * these then we don't worry about them.
9116 */
9117 continue;
9118 }
9119 /* Check lost state */
9120 if ((nrsm->r_flags & RACK_WAS_LOST) == 0) {
9121 uint32_t exp;
9122
9123 exp = ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) + thresh;
9124 if (TSTMP_LT(exp, cts) || (exp == cts)) {
9125 /* We consider it lost */
9126 nrsm->r_flags |= RACK_WAS_LOST;
9127 rack->r_ctl.rc_considered_lost += nrsm->r_end - nrsm->r_start;
9128 }
9129 }
9130 if (nrsm->r_flags & RACK_SACK_PASSED) {
9131 /*
9132 * We found one that is already marked
9133 * passed, we have been here before and
9134 * so all others below this are marked.
9135 */
9136 break;
9137 }
9138 nrsm->r_flags |= RACK_SACK_PASSED;
9139 nrsm->r_flags &= ~RACK_WAS_SACKPASS;
9140 }
9141 }
9142
9143 static void
rack_need_set_test(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,tcp_seq th_ack,int line,int use_which)9144 rack_need_set_test(struct tcpcb *tp,
9145 struct tcp_rack *rack,
9146 struct rack_sendmap *rsm,
9147 tcp_seq th_ack,
9148 int line,
9149 int use_which)
9150 {
9151 struct rack_sendmap *s_rsm;
9152
9153 if ((tp->t_flags & TF_GPUTINPROG) &&
9154 SEQ_GEQ(rsm->r_end, tp->gput_seq)) {
9155 /*
9156 * We were app limited, and this ack
9157 * butts up or goes beyond the point where we want
9158 * to start our next measurement. We need
9159 * to record the new gput_ts as here and
9160 * possibly update the start sequence.
9161 */
9162 uint32_t seq, ts;
9163
9164 if (rsm->r_rtr_cnt > 1) {
9165 /*
9166 * This is a retransmit, can we
9167 * really make any assessment at this
9168 * point? We are not really sure of
9169 * the timestamp, is it this or the
9170 * previous transmission?
9171 *
9172 * Lets wait for something better that
9173 * is not retransmitted.
9174 */
9175 return;
9176 }
9177 seq = tp->gput_seq;
9178 ts = tp->gput_ts;
9179 rack->app_limited_needs_set = 0;
9180 tp->gput_ts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
9181 /* Do we start at a new end? */
9182 if ((use_which == RACK_USE_BEG) &&
9183 SEQ_GEQ(rsm->r_start, tp->gput_seq)) {
9184 /*
9185 * When we get an ACK that just eats
9186 * up some of the rsm, we set RACK_USE_BEG
9187 * since whats at r_start (i.e. th_ack)
9188 * is left unacked and thats where the
9189 * measurement now starts.
9190 */
9191 tp->gput_seq = rsm->r_start;
9192 }
9193 if ((use_which == RACK_USE_END) &&
9194 SEQ_GEQ(rsm->r_end, tp->gput_seq)) {
9195 /*
9196 * We use the end when the cumack
9197 * is moving forward and completely
9198 * deleting the rsm passed so basically
9199 * r_end holds th_ack.
9200 *
9201 * For SACK's we also want to use the end
9202 * since this piece just got sacked and
9203 * we want to target anything after that
9204 * in our measurement.
9205 */
9206 tp->gput_seq = rsm->r_end;
9207 }
9208 if (use_which == RACK_USE_END_OR_THACK) {
9209 /*
9210 * special case for ack moving forward,
9211 * not a sack, we need to move all the
9212 * way up to where this ack cum-ack moves
9213 * to.
9214 */
9215 if (SEQ_GT(th_ack, rsm->r_end))
9216 tp->gput_seq = th_ack;
9217 else
9218 tp->gput_seq = rsm->r_end;
9219 }
9220 if (SEQ_LT(tp->gput_seq, tp->snd_max))
9221 s_rsm = tqhash_find(rack->r_ctl.tqh, tp->gput_seq);
9222 else
9223 s_rsm = NULL;
9224 /*
9225 * Pick up the correct send time if we can the rsm passed in
9226 * may be equal to s_rsm if the RACK_USE_BEG was set. For the other
9227 * two cases (RACK_USE_THACK or RACK_USE_END) most likely we will
9228 * find a different seq i.e. the next send up.
9229 *
9230 * If that has not been sent, s_rsm will be NULL and we must
9231 * arrange it so this function will get called again by setting
9232 * app_limited_needs_set.
9233 */
9234 if (s_rsm)
9235 rack->r_ctl.rc_gp_output_ts = s_rsm->r_tim_lastsent[0];
9236 else {
9237 /* If we hit here we have to have *not* sent tp->gput_seq */
9238 rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[0];
9239 /* Set it up so we will go through here again */
9240 rack->app_limited_needs_set = 1;
9241 }
9242 if (SEQ_GT(tp->gput_seq, tp->gput_ack)) {
9243 /*
9244 * We moved beyond this guy's range, re-calculate
9245 * the new end point.
9246 */
9247 if (rack->rc_gp_filled == 0) {
9248 tp->gput_ack = tp->gput_seq + max(rc_init_window(rack), (MIN_GP_WIN * ctf_fixed_maxseg(tp)));
9249 } else {
9250 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack);
9251 }
9252 }
9253 /*
9254 * We are moving the goal post, we may be able to clear the
9255 * measure_saw_probe_rtt flag.
9256 */
9257 if ((rack->in_probe_rtt == 0) &&
9258 (rack->measure_saw_probe_rtt) &&
9259 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit)))
9260 rack->measure_saw_probe_rtt = 0;
9261 rack_log_pacing_delay_calc(rack, ts, tp->gput_ts,
9262 seq, tp->gput_seq,
9263 (((uint64_t)rack->r_ctl.rc_app_limited_cnt << 32) |
9264 (uint64_t)rack->r_ctl.rc_gp_output_ts),
9265 5, line, NULL, 0);
9266 if (rack->rc_gp_filled &&
9267 ((tp->gput_ack - tp->gput_seq) <
9268 max(rc_init_window(rack), (MIN_GP_WIN *
9269 ctf_fixed_maxseg(tp))))) {
9270 uint32_t ideal_amount;
9271
9272 ideal_amount = rack_get_measure_window(tp, rack);
9273 if (ideal_amount > sbavail(&tptosocket(tp)->so_snd)) {
9274 /*
9275 * There is no sense of continuing this measurement
9276 * because its too small to gain us anything we
9277 * trust. Skip it and that way we can start a new
9278 * measurement quicker.
9279 */
9280 tp->t_flags &= ~TF_GPUTINPROG;
9281 rack_log_pacing_delay_calc(rack, tp->gput_ack, tp->gput_seq,
9282 0, 0,
9283 (((uint64_t)rack->r_ctl.rc_app_limited_cnt << 32) |
9284 (uint64_t)rack->r_ctl.rc_gp_output_ts),
9285 6, __LINE__, NULL, 0);
9286 } else {
9287 /*
9288 * Reset the window further out.
9289 */
9290 tp->gput_ack = tp->gput_seq + ideal_amount;
9291 }
9292 }
9293 rack_tend_gp_marks(tp, rack);
9294 rack_log_gpset(rack, tp->gput_ack, 0, 0, line, 2, rsm);
9295 }
9296 }
9297
9298 static inline int
is_rsm_inside_declared_tlp_block(struct tcp_rack * rack,struct rack_sendmap * rsm)9299 is_rsm_inside_declared_tlp_block(struct tcp_rack *rack, struct rack_sendmap *rsm)
9300 {
9301 if (SEQ_LT(rsm->r_end, rack->r_ctl.last_tlp_acked_start)) {
9302 /* Behind our TLP definition or right at */
9303 return (0);
9304 }
9305 if (SEQ_GT(rsm->r_start, rack->r_ctl.last_tlp_acked_end)) {
9306 /* The start is beyond or right at our end of TLP definition */
9307 return (0);
9308 }
9309 /* It has to be a sub-part of the original TLP recorded */
9310 return (1);
9311 }
9312
9313 static uint32_t
rack_proc_sack_blk(struct tcpcb * tp,struct tcp_rack * rack,struct sackblk * sack,struct tcpopt * to,struct rack_sendmap ** prsm,uint32_t cts,uint32_t segsiz)9314 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack, struct sackblk *sack,
9315 struct tcpopt *to, struct rack_sendmap **prsm, uint32_t cts,
9316 uint32_t segsiz)
9317 {
9318 uint32_t start, end, changed = 0;
9319 struct rack_sendmap stack_map;
9320 struct rack_sendmap *rsm, *nrsm, *prev, *next;
9321 int insret __diagused;
9322 int32_t used_ref = 1;
9323 int can_use_hookery = 0;
9324
9325 start = sack->start;
9326 end = sack->end;
9327 rsm = *prsm;
9328
9329 do_rest_ofb:
9330 if ((rsm == NULL) ||
9331 (SEQ_LT(end, rsm->r_start)) ||
9332 (SEQ_GEQ(start, rsm->r_end)) ||
9333 (SEQ_LT(start, rsm->r_start))) {
9334 /*
9335 * We are not in the right spot,
9336 * find the correct spot in the tree.
9337 */
9338 used_ref = 0;
9339 rsm = tqhash_find(rack->r_ctl.tqh, start);
9340 }
9341 if (rsm == NULL) {
9342 /* TSNH */
9343 goto out;
9344 }
9345 /* Ok we have an ACK for some piece of this rsm */
9346 if (rsm->r_start != start) {
9347 if ((rsm->r_flags & RACK_ACKED) == 0) {
9348 /*
9349 * Before any splitting or hookery is
9350 * done is it a TLP of interest i.e. rxt?
9351 */
9352 if ((rsm->r_flags & RACK_TLP) &&
9353 (rsm->r_rtr_cnt > 1)) {
9354 /*
9355 * We are splitting a rxt TLP, check
9356 * if we need to save off the start/end
9357 */
9358 if (rack->rc_last_tlp_acked_set &&
9359 (is_rsm_inside_declared_tlp_block(rack, rsm))) {
9360 /*
9361 * We already turned this on since we are inside
9362 * the previous one was a partially sack now we
9363 * are getting another one (maybe all of it).
9364 *
9365 */
9366 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
9367 /*
9368 * Lets make sure we have all of it though.
9369 */
9370 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
9371 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9372 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9373 rack->r_ctl.last_tlp_acked_end);
9374 }
9375 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
9376 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9377 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9378 rack->r_ctl.last_tlp_acked_end);
9379 }
9380 } else {
9381 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9382 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9383 rack->rc_last_tlp_past_cumack = 0;
9384 rack->rc_last_tlp_acked_set = 1;
9385 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
9386 }
9387 }
9388 /**
9389 * Need to split this in two pieces the before and after,
9390 * the before remains in the map, the after must be
9391 * added. In other words we have:
9392 * rsm |--------------|
9393 * sackblk |------->
9394 * rsm will become
9395 * rsm |---|
9396 * and nrsm will be the sacked piece
9397 * nrsm |----------|
9398 *
9399 * But before we start down that path lets
9400 * see if the sack spans over on top of
9401 * the next guy and it is already sacked.
9402 *
9403 */
9404 /*
9405 * Hookery can only be used if the two entries
9406 * are in the same bucket and neither one of
9407 * them staddle the bucket line.
9408 */
9409 next = tqhash_next(rack->r_ctl.tqh, rsm);
9410 if (next &&
9411 (rsm->bindex == next->bindex) &&
9412 ((rsm->r_flags & RACK_STRADDLE) == 0) &&
9413 ((next->r_flags & RACK_STRADDLE) == 0) &&
9414 ((rsm->r_flags & RACK_IS_PCM) == 0) &&
9415 ((next->r_flags & RACK_IS_PCM) == 0) &&
9416 (rsm->r_flags & RACK_IN_GP_WIN) &&
9417 (next->r_flags & RACK_IN_GP_WIN))
9418 can_use_hookery = 1;
9419 else
9420 can_use_hookery = 0;
9421 if (next && can_use_hookery &&
9422 (next->r_flags & RACK_ACKED) &&
9423 SEQ_GEQ(end, next->r_start)) {
9424 /**
9425 * So the next one is already acked, and
9426 * we can thus by hookery use our stack_map
9427 * to reflect the piece being sacked and
9428 * then adjust the two tree entries moving
9429 * the start and ends around. So we start like:
9430 * rsm |------------| (not-acked)
9431 * next |-----------| (acked)
9432 * sackblk |-------->
9433 * We want to end like so:
9434 * rsm |------| (not-acked)
9435 * next |-----------------| (acked)
9436 * nrsm |-----|
9437 * Where nrsm is a temporary stack piece we
9438 * use to update all the gizmos.
9439 */
9440 /* Copy up our fudge block */
9441 nrsm = &stack_map;
9442 memcpy(nrsm, rsm, sizeof(struct rack_sendmap));
9443 /* Now adjust our tree blocks */
9444 tqhash_update_end(rack->r_ctl.tqh, rsm, start);
9445 next->r_start = start;
9446 rsm->r_flags |= RACK_SHUFFLED;
9447 next->r_flags |= RACK_SHUFFLED;
9448 /* Now we must adjust back where next->m is */
9449 rack_setup_offset_for_rsm(rack, rsm, next);
9450 /*
9451 * Which timestamp do we keep? It is rather
9452 * important in GP measurements to have the
9453 * accurate end of the send window.
9454 *
9455 * We keep the largest value, which is the newest
9456 * send. We do this in case a segment that is
9457 * joined together and not part of a GP estimate
9458 * later gets expanded into the GP estimate.
9459 *
9460 * We prohibit the merging of unlike kinds i.e.
9461 * all pieces that are in the GP estimate can be
9462 * merged and all pieces that are not in a GP estimate
9463 * can be merged, but not disimilar pieces. Combine
9464 * this with taking the highest here and we should
9465 * be ok unless of course the client reneges. Then
9466 * all bets are off.
9467 */
9468 if (next->r_tim_lastsent[(next->r_rtr_cnt-1)] <
9469 nrsm->r_tim_lastsent[(nrsm->r_rtr_cnt-1)])
9470 next->r_tim_lastsent[(next->r_rtr_cnt-1)] = nrsm->r_tim_lastsent[(nrsm->r_rtr_cnt-1)];
9471 /*
9472 * And we must keep the newest ack arrival time.
9473 */
9474 if (next->r_ack_arrival <
9475 rack_to_usec_ts(&rack->r_ctl.act_rcv_time))
9476 next->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
9477
9478
9479 /* We don't need to adjust rsm, it did not change */
9480 /* Clear out the dup ack count of the remainder */
9481 rsm->r_dupack = 0;
9482 rsm->r_just_ret = 0;
9483 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
9484 /* Now lets make sure our fudge block is right */
9485 nrsm->r_start = start;
9486 /* Now lets update all the stats and such */
9487 rack_update_rtt(tp, rack, nrsm, to, cts, SACKED, 0);
9488 if (rack->app_limited_needs_set)
9489 rack_need_set_test(tp, rack, nrsm, tp->snd_una, __LINE__, RACK_USE_END);
9490 changed += (nrsm->r_end - nrsm->r_start);
9491 rack->r_ctl.rc_sacked += (nrsm->r_end - nrsm->r_start);
9492 if (rsm->r_flags & RACK_WAS_LOST) {
9493 int my_chg;
9494
9495 my_chg = (nrsm->r_end - nrsm->r_start);
9496 KASSERT((rack->r_ctl.rc_considered_lost >= my_chg),
9497 ("rsm:%p rack:%p rc_considered_lost goes negative", rsm, rack));
9498 if (my_chg <= rack->r_ctl.rc_considered_lost)
9499 rack->r_ctl.rc_considered_lost -= my_chg;
9500 else
9501 rack->r_ctl.rc_considered_lost = 0;
9502 }
9503 if (nrsm->r_flags & RACK_SACK_PASSED) {
9504 rack->r_ctl.rc_reorder_ts = cts;
9505 if (rack->r_ctl.rc_reorder_ts == 0)
9506 rack->r_ctl.rc_reorder_ts = 1;
9507 }
9508 /*
9509 * Now we want to go up from rsm (the
9510 * one left un-acked) to the next one
9511 * in the tmap. We do this so when
9512 * we walk backwards we include marking
9513 * sack-passed on rsm (The one passed in
9514 * is skipped since it is generally called
9515 * on something sacked before removing it
9516 * from the tmap).
9517 */
9518 if (rsm->r_in_tmap) {
9519 nrsm = TAILQ_NEXT(rsm, r_tnext);
9520 /*
9521 * Now that we have the next
9522 * one walk backwards from there.
9523 */
9524 if (nrsm && nrsm->r_in_tmap)
9525 rack_log_sack_passed(tp, rack, nrsm, cts);
9526 }
9527 /* Now are we done? */
9528 if (SEQ_LT(end, next->r_end) ||
9529 (end == next->r_end)) {
9530 /* Done with block */
9531 goto out;
9532 }
9533 rack_log_map_chg(tp, rack, &stack_map, rsm, next, MAP_SACK_M1, end, __LINE__);
9534 counter_u64_add(rack_sack_used_next_merge, 1);
9535 /* Postion for the next block */
9536 start = next->r_end;
9537 rsm = tqhash_next(rack->r_ctl.tqh, next);
9538 if (rsm == NULL)
9539 goto out;
9540 } else {
9541 /**
9542 * We can't use any hookery here, so we
9543 * need to split the map. We enter like
9544 * so:
9545 * rsm |--------|
9546 * sackblk |----->
9547 * We will add the new block nrsm and
9548 * that will be the new portion, and then
9549 * fall through after reseting rsm. So we
9550 * split and look like this:
9551 * rsm |----|
9552 * sackblk |----->
9553 * nrsm |---|
9554 * We then fall through reseting
9555 * rsm to nrsm, so the next block
9556 * picks it up.
9557 */
9558 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT);
9559 if (nrsm == NULL) {
9560 /*
9561 * failed XXXrrs what can we do but loose the sack
9562 * info?
9563 */
9564 goto out;
9565 }
9566 counter_u64_add(rack_sack_splits, 1);
9567 rack_clone_rsm(rack, nrsm, rsm, start);
9568 rsm->r_just_ret = 0;
9569 #ifndef INVARIANTS
9570 (void)tqhash_insert(rack->r_ctl.tqh, nrsm);
9571 #else
9572 if ((insret = tqhash_insert(rack->r_ctl.tqh, nrsm)) != 0) {
9573 panic("Insert in tailq_hash of %p fails ret:%d rack:%p rsm:%p",
9574 nrsm, insret, rack, rsm);
9575 }
9576 #endif
9577 if (rsm->r_in_tmap) {
9578 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
9579 nrsm->r_in_tmap = 1;
9580 }
9581 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SACK_M2, end, __LINE__);
9582 rsm->r_flags &= (~RACK_HAS_FIN);
9583 /* Position us to point to the new nrsm that starts the sack blk */
9584 rsm = nrsm;
9585 }
9586 } else {
9587 /* Already sacked this piece */
9588 counter_u64_add(rack_sack_skipped_acked, 1);
9589 if (end == rsm->r_end) {
9590 /* Done with block */
9591 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
9592 goto out;
9593 } else if (SEQ_LT(end, rsm->r_end)) {
9594 /* A partial sack to a already sacked block */
9595 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
9596 goto out;
9597 } else {
9598 /*
9599 * The end goes beyond this guy
9600 * reposition the start to the
9601 * next block.
9602 */
9603 start = rsm->r_end;
9604 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
9605 if (rsm == NULL)
9606 goto out;
9607 }
9608 }
9609 }
9610 if (SEQ_GEQ(end, rsm->r_end)) {
9611 /**
9612 * The end of this block is either beyond this guy or right
9613 * at this guy. I.e.:
9614 * rsm --- |-----|
9615 * end |-----|
9616 * <or>
9617 * end |---------|
9618 */
9619 if ((rsm->r_flags & RACK_ACKED) == 0) {
9620 /*
9621 * Is it a TLP of interest?
9622 */
9623 if ((rsm->r_flags & RACK_TLP) &&
9624 (rsm->r_rtr_cnt > 1)) {
9625 /*
9626 * We are splitting a rxt TLP, check
9627 * if we need to save off the start/end
9628 */
9629 if (rack->rc_last_tlp_acked_set &&
9630 (is_rsm_inside_declared_tlp_block(rack, rsm))) {
9631 /*
9632 * We already turned this on since we are inside
9633 * the previous one was a partially sack now we
9634 * are getting another one (maybe all of it).
9635 */
9636 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
9637 /*
9638 * Lets make sure we have all of it though.
9639 */
9640 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
9641 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9642 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9643 rack->r_ctl.last_tlp_acked_end);
9644 }
9645 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
9646 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9647 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9648 rack->r_ctl.last_tlp_acked_end);
9649 }
9650 } else {
9651 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9652 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9653 rack->rc_last_tlp_past_cumack = 0;
9654 rack->rc_last_tlp_acked_set = 1;
9655 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
9656 }
9657 }
9658 rack_update_rtt(tp, rack, rsm, to, cts, SACKED, 0);
9659 changed += (rsm->r_end - rsm->r_start);
9660 /* You get a count for acking a whole segment or more */
9661 if (rsm->r_flags & RACK_WAS_LOST) {
9662 int my_chg;
9663
9664 my_chg = (rsm->r_end - rsm->r_start);
9665 rsm->r_flags &= ~RACK_WAS_LOST;
9666 KASSERT((rack->r_ctl.rc_considered_lost >= my_chg),
9667 ("rsm:%p rack:%p rc_considered_lost goes negative", rsm, rack));
9668 if (my_chg <= rack->r_ctl.rc_considered_lost)
9669 rack->r_ctl.rc_considered_lost -= my_chg;
9670 else
9671 rack->r_ctl.rc_considered_lost = 0;
9672 }
9673 rack->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
9674 if (rsm->r_in_tmap) /* should be true */
9675 rack_log_sack_passed(tp, rack, rsm, cts);
9676 /* Is Reordering occuring? */
9677 if (rsm->r_flags & RACK_SACK_PASSED) {
9678 rsm->r_flags &= ~RACK_SACK_PASSED;
9679 rack->r_ctl.rc_reorder_ts = cts;
9680 if (rack->r_ctl.rc_reorder_ts == 0)
9681 rack->r_ctl.rc_reorder_ts = 1;
9682 }
9683 if (rack->app_limited_needs_set)
9684 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_END);
9685 rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
9686 rsm->r_flags |= RACK_ACKED;
9687 rack_update_pcm_ack(rack, 0, rsm->r_start, rsm->r_end);
9688 if (rsm->r_in_tmap) {
9689 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
9690 rsm->r_in_tmap = 0;
9691 }
9692 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_SACK_M3, end, __LINE__);
9693 } else {
9694 counter_u64_add(rack_sack_skipped_acked, 1);
9695 }
9696 if (end == rsm->r_end) {
9697 /* This block only - done, setup for next */
9698 goto out;
9699 }
9700 /*
9701 * There is more not coverend by this rsm move on
9702 * to the next block in the tail queue hash table.
9703 */
9704 nrsm = tqhash_next(rack->r_ctl.tqh, rsm);
9705 start = rsm->r_end;
9706 rsm = nrsm;
9707 if (rsm == NULL)
9708 goto out;
9709 goto do_rest_ofb;
9710 }
9711 /**
9712 * The end of this sack block is smaller than
9713 * our rsm i.e.:
9714 * rsm --- |-----|
9715 * end |--|
9716 */
9717 if ((rsm->r_flags & RACK_ACKED) == 0) {
9718 /*
9719 * Is it a TLP of interest?
9720 */
9721 if ((rsm->r_flags & RACK_TLP) &&
9722 (rsm->r_rtr_cnt > 1)) {
9723 /*
9724 * We are splitting a rxt TLP, check
9725 * if we need to save off the start/end
9726 */
9727 if (rack->rc_last_tlp_acked_set &&
9728 (is_rsm_inside_declared_tlp_block(rack, rsm))) {
9729 /*
9730 * We already turned this on since we are inside
9731 * the previous one was a partially sack now we
9732 * are getting another one (maybe all of it).
9733 */
9734 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
9735 /*
9736 * Lets make sure we have all of it though.
9737 */
9738 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
9739 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9740 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9741 rack->r_ctl.last_tlp_acked_end);
9742 }
9743 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
9744 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9745 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9746 rack->r_ctl.last_tlp_acked_end);
9747 }
9748 } else {
9749 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9750 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9751 rack->rc_last_tlp_past_cumack = 0;
9752 rack->rc_last_tlp_acked_set = 1;
9753 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
9754 }
9755 }
9756 /*
9757 * Hookery can only be used if the two entries
9758 * are in the same bucket and neither one of
9759 * them staddle the bucket line.
9760 */
9761 prev = tqhash_prev(rack->r_ctl.tqh, rsm);
9762 if (prev &&
9763 (rsm->bindex == prev->bindex) &&
9764 ((rsm->r_flags & RACK_STRADDLE) == 0) &&
9765 ((prev->r_flags & RACK_STRADDLE) == 0) &&
9766 ((rsm->r_flags & RACK_IS_PCM) == 0) &&
9767 ((prev->r_flags & RACK_IS_PCM) == 0) &&
9768 (rsm->r_flags & RACK_IN_GP_WIN) &&
9769 (prev->r_flags & RACK_IN_GP_WIN))
9770 can_use_hookery = 1;
9771 else
9772 can_use_hookery = 0;
9773 if (prev && can_use_hookery &&
9774 (prev->r_flags & RACK_ACKED)) {
9775 /**
9776 * Goal, we want the right remainder of rsm to shrink
9777 * in place and span from (rsm->r_start = end) to rsm->r_end.
9778 * We want to expand prev to go all the way
9779 * to prev->r_end <- end.
9780 * so in the tree we have before:
9781 * prev |--------| (acked)
9782 * rsm |-------| (non-acked)
9783 * sackblk |-|
9784 * We churn it so we end up with
9785 * prev |----------| (acked)
9786 * rsm |-----| (non-acked)
9787 * nrsm |-| (temporary)
9788 *
9789 * Note if either prev/rsm is a TLP we don't
9790 * do this.
9791 */
9792 nrsm = &stack_map;
9793 memcpy(nrsm, rsm, sizeof(struct rack_sendmap));
9794 tqhash_update_end(rack->r_ctl.tqh, prev, end);
9795 rsm->r_start = end;
9796 rsm->r_flags |= RACK_SHUFFLED;
9797 prev->r_flags |= RACK_SHUFFLED;
9798 /* Now adjust nrsm (stack copy) to be
9799 * the one that is the small
9800 * piece that was "sacked".
9801 */
9802 nrsm->r_end = end;
9803 rsm->r_dupack = 0;
9804 /*
9805 * Which timestamp do we keep? It is rather
9806 * important in GP measurements to have the
9807 * accurate end of the send window.
9808 *
9809 * We keep the largest value, which is the newest
9810 * send. We do this in case a segment that is
9811 * joined together and not part of a GP estimate
9812 * later gets expanded into the GP estimate.
9813 *
9814 * We prohibit the merging of unlike kinds i.e.
9815 * all pieces that are in the GP estimate can be
9816 * merged and all pieces that are not in a GP estimate
9817 * can be merged, but not disimilar pieces. Combine
9818 * this with taking the highest here and we should
9819 * be ok unless of course the client reneges. Then
9820 * all bets are off.
9821 */
9822 if(prev->r_tim_lastsent[(prev->r_rtr_cnt-1)] <
9823 nrsm->r_tim_lastsent[(nrsm->r_rtr_cnt-1)]) {
9824 prev->r_tim_lastsent[(prev->r_rtr_cnt-1)] = nrsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
9825 }
9826 /*
9827 * And we must keep the newest ack arrival time.
9828 */
9829
9830 if(prev->r_ack_arrival <
9831 rack_to_usec_ts(&rack->r_ctl.act_rcv_time))
9832 prev->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
9833
9834 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
9835 /*
9836 * Now that the rsm has had its start moved forward
9837 * lets go ahead and get its new place in the world.
9838 */
9839 rack_setup_offset_for_rsm(rack, prev, rsm);
9840 /*
9841 * Now nrsm is our new little piece
9842 * that is acked (which was merged
9843 * to prev). Update the rtt and changed
9844 * based on that. Also check for reordering.
9845 */
9846 rack_update_rtt(tp, rack, nrsm, to, cts, SACKED, 0);
9847 if (rack->app_limited_needs_set)
9848 rack_need_set_test(tp, rack, nrsm, tp->snd_una, __LINE__, RACK_USE_END);
9849 changed += (nrsm->r_end - nrsm->r_start);
9850 rack->r_ctl.rc_sacked += (nrsm->r_end - nrsm->r_start);
9851 if (rsm->r_flags & RACK_WAS_LOST) {
9852 int my_chg;
9853
9854 my_chg = (nrsm->r_end - nrsm->r_start);
9855 KASSERT((rack->r_ctl.rc_considered_lost >= my_chg),
9856 ("rsm:%p rack:%p rc_considered_lost goes negative", rsm, rack));
9857 if (my_chg <= rack->r_ctl.rc_considered_lost)
9858 rack->r_ctl.rc_considered_lost -= my_chg;
9859 else
9860 rack->r_ctl.rc_considered_lost = 0;
9861 }
9862 if (nrsm->r_flags & RACK_SACK_PASSED) {
9863 rack->r_ctl.rc_reorder_ts = cts;
9864 if (rack->r_ctl.rc_reorder_ts == 0)
9865 rack->r_ctl.rc_reorder_ts = 1;
9866 }
9867 rack_log_map_chg(tp, rack, prev, &stack_map, rsm, MAP_SACK_M4, end, __LINE__);
9868 rsm = prev;
9869 counter_u64_add(rack_sack_used_prev_merge, 1);
9870 } else {
9871 /**
9872 * This is the case where our previous
9873 * block is not acked either, so we must
9874 * split the block in two.
9875 */
9876 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT);
9877 if (nrsm == NULL) {
9878 /* failed rrs what can we do but loose the sack info? */
9879 goto out;
9880 }
9881 if ((rsm->r_flags & RACK_TLP) &&
9882 (rsm->r_rtr_cnt > 1)) {
9883 /*
9884 * We are splitting a rxt TLP, check
9885 * if we need to save off the start/end
9886 */
9887 if (rack->rc_last_tlp_acked_set &&
9888 (is_rsm_inside_declared_tlp_block(rack, rsm))) {
9889 /*
9890 * We already turned this on since this block is inside
9891 * the previous one was a partially sack now we
9892 * are getting another one (maybe all of it).
9893 */
9894 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
9895 /*
9896 * Lets make sure we have all of it though.
9897 */
9898 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
9899 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9900 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9901 rack->r_ctl.last_tlp_acked_end);
9902 }
9903 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
9904 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9905 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9906 rack->r_ctl.last_tlp_acked_end);
9907 }
9908 } else {
9909 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9910 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9911 rack->rc_last_tlp_acked_set = 1;
9912 rack->rc_last_tlp_past_cumack = 0;
9913 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
9914 }
9915 }
9916 /**
9917 * In this case nrsm becomes
9918 * nrsm->r_start = end;
9919 * nrsm->r_end = rsm->r_end;
9920 * which is un-acked.
9921 * <and>
9922 * rsm->r_end = nrsm->r_start;
9923 * i.e. the remaining un-acked
9924 * piece is left on the left
9925 * hand side.
9926 *
9927 * So we start like this
9928 * rsm |----------| (not acked)
9929 * sackblk |---|
9930 * build it so we have
9931 * rsm |---| (acked)
9932 * nrsm |------| (not acked)
9933 */
9934 counter_u64_add(rack_sack_splits, 1);
9935 rack_clone_rsm(rack, nrsm, rsm, end);
9936 rsm->r_flags &= (~RACK_HAS_FIN);
9937 rsm->r_just_ret = 0;
9938 #ifndef INVARIANTS
9939 (void)tqhash_insert(rack->r_ctl.tqh, nrsm);
9940 #else
9941 if ((insret = tqhash_insert(rack->r_ctl.tqh, nrsm)) != 0) {
9942 panic("Insert in tailq_hash of %p fails ret:% rack:%p rsm:%p",
9943 nrsm, insret, rack, rsm);
9944 }
9945 #endif
9946 if (rsm->r_in_tmap) {
9947 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
9948 nrsm->r_in_tmap = 1;
9949 }
9950 nrsm->r_dupack = 0;
9951 rack_log_retran_reason(rack, nrsm, __LINE__, 0, 2);
9952 rack_update_rtt(tp, rack, rsm, to, cts, SACKED, 0);
9953 changed += (rsm->r_end - rsm->r_start);
9954 if (rsm->r_flags & RACK_WAS_LOST) {
9955 int my_chg;
9956
9957 my_chg = (rsm->r_end - rsm->r_start);
9958 rsm->r_flags &= ~RACK_WAS_LOST;
9959 KASSERT((rack->r_ctl.rc_considered_lost >= my_chg),
9960 ("rsm:%p rack:%p rc_considered_lost goes negative", rsm, rack));
9961 if (my_chg <= rack->r_ctl.rc_considered_lost)
9962 rack->r_ctl.rc_considered_lost -= my_chg;
9963 else
9964 rack->r_ctl.rc_considered_lost = 0;
9965 }
9966 rack->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
9967
9968 if (rsm->r_in_tmap) /* should be true */
9969 rack_log_sack_passed(tp, rack, rsm, cts);
9970 /* Is Reordering occuring? */
9971 if (rsm->r_flags & RACK_SACK_PASSED) {
9972 rsm->r_flags &= ~RACK_SACK_PASSED;
9973 rack->r_ctl.rc_reorder_ts = cts;
9974 if (rack->r_ctl.rc_reorder_ts == 0)
9975 rack->r_ctl.rc_reorder_ts = 1;
9976 }
9977 if (rack->app_limited_needs_set)
9978 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_END);
9979 rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
9980 rsm->r_flags |= RACK_ACKED;
9981 rack_update_pcm_ack(rack, 0, rsm->r_start, rsm->r_end);
9982 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SACK_M5, end, __LINE__);
9983 if (rsm->r_in_tmap) {
9984 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
9985 rsm->r_in_tmap = 0;
9986 }
9987 }
9988 } else if (start != end){
9989 /*
9990 * The block was already acked.
9991 */
9992 counter_u64_add(rack_sack_skipped_acked, 1);
9993 }
9994 out:
9995 if (rsm &&
9996 ((rsm->r_flags & RACK_TLP) == 0) &&
9997 (rsm->r_flags & RACK_ACKED)) {
9998 /*
9999 * Now can we merge where we worked
10000 * with either the previous or
10001 * next block?
10002 */
10003 next = tqhash_next(rack->r_ctl.tqh, rsm);
10004 while (next) {
10005 if (next->r_flags & RACK_TLP)
10006 break;
10007 /* Only allow merges between ones in or out of GP window */
10008 if ((next->r_flags & RACK_IN_GP_WIN) &&
10009 ((rsm->r_flags & RACK_IN_GP_WIN) == 0)) {
10010 break;
10011 }
10012 if ((rsm->r_flags & RACK_IN_GP_WIN) &&
10013 ((next->r_flags & RACK_IN_GP_WIN) == 0)) {
10014 break;
10015 }
10016 if (rsm->bindex != next->bindex)
10017 break;
10018 if (rsm->r_flags & RACK_STRADDLE)
10019 break;
10020 if (rsm->r_flags & RACK_IS_PCM)
10021 break;
10022 if (next->r_flags & RACK_STRADDLE)
10023 break;
10024 if (next->r_flags & RACK_IS_PCM)
10025 break;
10026 if (next->r_flags & RACK_ACKED) {
10027 /* yep this and next can be merged */
10028 rsm = rack_merge_rsm(rack, rsm, next);
10029 next = tqhash_next(rack->r_ctl.tqh, rsm);
10030 } else
10031 break;
10032 }
10033 /* Now what about the previous? */
10034 prev = tqhash_prev(rack->r_ctl.tqh, rsm);
10035 while (prev) {
10036 if (prev->r_flags & RACK_TLP)
10037 break;
10038 /* Only allow merges between ones in or out of GP window */
10039 if ((prev->r_flags & RACK_IN_GP_WIN) &&
10040 ((rsm->r_flags & RACK_IN_GP_WIN) == 0)) {
10041 break;
10042 }
10043 if ((rsm->r_flags & RACK_IN_GP_WIN) &&
10044 ((prev->r_flags & RACK_IN_GP_WIN) == 0)) {
10045 break;
10046 }
10047 if (rsm->bindex != prev->bindex)
10048 break;
10049 if (rsm->r_flags & RACK_STRADDLE)
10050 break;
10051 if (rsm->r_flags & RACK_IS_PCM)
10052 break;
10053 if (prev->r_flags & RACK_STRADDLE)
10054 break;
10055 if (prev->r_flags & RACK_IS_PCM)
10056 break;
10057 if (prev->r_flags & RACK_ACKED) {
10058 /* yep the previous and this can be merged */
10059 rsm = rack_merge_rsm(rack, prev, rsm);
10060 prev = tqhash_prev(rack->r_ctl.tqh, rsm);
10061 } else
10062 break;
10063 }
10064 }
10065 if (used_ref == 0) {
10066 counter_u64_add(rack_sack_proc_all, 1);
10067 } else {
10068 counter_u64_add(rack_sack_proc_short, 1);
10069 }
10070 /* Save off the next one for quick reference. */
10071 nrsm = tqhash_find(rack->r_ctl.tqh, end);
10072 *prsm = rack->r_ctl.rc_sacklast = nrsm;
10073 return (changed);
10074 }
10075
10076 static void inline
rack_peer_reneges(struct tcp_rack * rack,struct rack_sendmap * rsm,tcp_seq th_ack)10077 rack_peer_reneges(struct tcp_rack *rack, struct rack_sendmap *rsm, tcp_seq th_ack)
10078 {
10079 struct rack_sendmap *tmap;
10080
10081 tmap = NULL;
10082 while (rsm && (rsm->r_flags & RACK_ACKED)) {
10083 /* Its no longer sacked, mark it so */
10084 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
10085 #ifdef INVARIANTS
10086 if (rsm->r_in_tmap) {
10087 panic("rack:%p rsm:%p flags:0x%x in tmap?",
10088 rack, rsm, rsm->r_flags);
10089 }
10090 #endif
10091 rsm->r_flags &= ~(RACK_ACKED|RACK_SACK_PASSED|RACK_WAS_SACKPASS);
10092 /* Rebuild it into our tmap */
10093 if (tmap == NULL) {
10094 TAILQ_INSERT_HEAD(&rack->r_ctl.rc_tmap, rsm, r_tnext);
10095 tmap = rsm;
10096 } else {
10097 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, tmap, rsm, r_tnext);
10098 tmap = rsm;
10099 }
10100 tmap->r_in_tmap = 1;
10101 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
10102 }
10103 /*
10104 * Now lets possibly clear the sack filter so we start
10105 * recognizing sacks that cover this area.
10106 */
10107 sack_filter_clear(&rack->r_ctl.rack_sf, th_ack);
10108
10109 }
10110
10111
10112 static void inline
rack_rsm_sender_update(struct tcp_rack * rack,struct tcpcb * tp,struct rack_sendmap * rsm,uint8_t from)10113 rack_rsm_sender_update(struct tcp_rack *rack, struct tcpcb *tp, struct rack_sendmap *rsm, uint8_t from)
10114 {
10115 /*
10116 * We look at advancing the end send time for our GP
10117 * measurement tracking only as the cumulative acknowledgment
10118 * moves forward. You might wonder about this, why not
10119 * at every transmission or retransmission within the
10120 * GP window update the rc_gp_cumack_ts? Well its rather
10121 * nuanced but basically the GP window *may* expand (as
10122 * it does below) or worse and harder to track it may shrink.
10123 *
10124 * This last makes it impossible to track at the time of
10125 * the send, since you may set forward your rc_gp_cumack_ts
10126 * when you send, because that send *is* in your currently
10127 * "guessed" window, but then it shrinks. Now which was
10128 * the send time of the last bytes in the window, by the
10129 * time you ask that question that part of the sendmap
10130 * is freed. So you don't know and you will have too
10131 * long of send window. Instead by updating the time
10132 * marker only when the cumack advances this assures us
10133 * that we will have only the sends in the window of our
10134 * GP measurement.
10135 *
10136 * Another complication from this is the
10137 * merging of sendmap entries. During SACK processing this
10138 * can happen to conserve the sendmap size. That breaks
10139 * everything down in tracking the send window of the GP
10140 * estimate. So to prevent that and keep it working with
10141 * a tiny bit more limited merging, we only allow like
10142 * types to be merged. I.e. if two sends are in the GP window
10143 * then its ok to merge them together. If two sends are not
10144 * in the GP window its ok to merge them together too. Though
10145 * one send in and one send out cannot be merged. We combine
10146 * this with never allowing the shrinking of the GP window when
10147 * we are in recovery so that we can properly calculate the
10148 * sending times.
10149 *
10150 * This all of course seems complicated, because it is.. :)
10151 *
10152 * The cum-ack is being advanced upon the sendmap.
10153 * If we are not doing a GP estimate don't
10154 * proceed.
10155 */
10156 uint64_t ts;
10157
10158 if ((tp->t_flags & TF_GPUTINPROG) == 0)
10159 return;
10160 /*
10161 * If this sendmap entry is going
10162 * beyond the measurement window we had picked,
10163 * expand the measurement window by that much.
10164 */
10165 if (SEQ_GT(rsm->r_end, tp->gput_ack)) {
10166 tp->gput_ack = rsm->r_end;
10167 }
10168 /*
10169 * If we have not setup a ack, then we
10170 * have no idea if the newly acked pieces
10171 * will be "in our seq measurement range". If
10172 * it is when we clear the app_limited_needs_set
10173 * flag the timestamp will be updated.
10174 */
10175 if (rack->app_limited_needs_set)
10176 return;
10177 /*
10178 * Finally, we grab out the latest timestamp
10179 * that this packet was sent and then see
10180 * if:
10181 * a) The packet touches are newly defined GP range.
10182 * b) The time is greater than (newer) than the
10183 * one we currently have. If so we update
10184 * our sending end time window.
10185 *
10186 * Note we *do not* do this at send time. The reason
10187 * is that if you do you *may* pick up a newer timestamp
10188 * for a range you are not going to measure. We project
10189 * out how far and then sometimes modify that to be
10190 * smaller. If that occurs then you will have a send
10191 * that does not belong to the range included.
10192 */
10193 if ((ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]) <=
10194 rack->r_ctl.rc_gp_cumack_ts)
10195 return;
10196 if (rack_in_gp_window(tp, rsm)) {
10197 rack->r_ctl.rc_gp_cumack_ts = ts;
10198 rack_log_gpset(rack, tp->gput_ack, (uint32_t)ts, rsm->r_end,
10199 __LINE__, from, rsm);
10200 }
10201 }
10202
10203 static void
rack_process_to_cumack(struct tcpcb * tp,struct tcp_rack * rack,register uint32_t th_ack,uint32_t cts,struct tcpopt * to,uint64_t acktime)10204 rack_process_to_cumack(struct tcpcb *tp, struct tcp_rack *rack, register uint32_t th_ack, uint32_t cts, struct tcpopt *to, uint64_t acktime)
10205 {
10206 struct rack_sendmap *rsm;
10207 /*
10208 * The ACK point is advancing to th_ack, we must drop off
10209 * the packets in the rack log and calculate any eligble
10210 * RTT's.
10211 */
10212
10213 if (sack_filter_blks_used(&rack->r_ctl.rack_sf)) {
10214 /*
10215 * If we have some sack blocks in the filter
10216 * lets prune them out by calling sfb with no blocks.
10217 */
10218 sack_filter_blks(tp, &rack->r_ctl.rack_sf, NULL, 0, th_ack);
10219 }
10220 if (SEQ_GT(th_ack, tp->snd_una)) {
10221 /* Clear any app ack remembered settings */
10222 rack->r_ctl.cleared_app_ack = 0;
10223 }
10224 rack->r_wanted_output = 1;
10225 if (SEQ_GT(th_ack, tp->snd_una))
10226 rack->r_ctl.last_cumack_advance = acktime;
10227
10228 /* Tend any TLP that has been marked for 1/2 the seq space (its old) */
10229 if ((rack->rc_last_tlp_acked_set == 1)&&
10230 (rack->rc_last_tlp_past_cumack == 1) &&
10231 (SEQ_GT(rack->r_ctl.last_tlp_acked_start, th_ack))) {
10232 /*
10233 * We have reached the point where our last rack
10234 * tlp retransmit sequence is ahead of the cum-ack.
10235 * This can only happen when the cum-ack moves all
10236 * the way around (its been a full 2^^31+1 bytes
10237 * or more since we sent a retransmitted TLP). Lets
10238 * turn off the valid flag since its not really valid.
10239 *
10240 * Note since sack's also turn on this event we have
10241 * a complication, we have to wait to age it out until
10242 * the cum-ack is by the TLP before checking which is
10243 * what the next else clause does.
10244 */
10245 rack_log_dsack_event(rack, 9, __LINE__,
10246 rack->r_ctl.last_tlp_acked_start,
10247 rack->r_ctl.last_tlp_acked_end);
10248 rack->rc_last_tlp_acked_set = 0;
10249 rack->rc_last_tlp_past_cumack = 0;
10250 } else if ((rack->rc_last_tlp_acked_set == 1) &&
10251 (rack->rc_last_tlp_past_cumack == 0) &&
10252 (SEQ_GEQ(th_ack, rack->r_ctl.last_tlp_acked_end))) {
10253 /*
10254 * It is safe to start aging TLP's out.
10255 */
10256 rack->rc_last_tlp_past_cumack = 1;
10257 }
10258 /* We do the same for the tlp send seq as well */
10259 if ((rack->rc_last_sent_tlp_seq_valid == 1) &&
10260 (rack->rc_last_sent_tlp_past_cumack == 1) &&
10261 (SEQ_GT(rack->r_ctl.last_sent_tlp_seq, th_ack))) {
10262 rack_log_dsack_event(rack, 9, __LINE__,
10263 rack->r_ctl.last_sent_tlp_seq,
10264 (rack->r_ctl.last_sent_tlp_seq +
10265 rack->r_ctl.last_sent_tlp_len));
10266 rack->rc_last_sent_tlp_seq_valid = 0;
10267 rack->rc_last_sent_tlp_past_cumack = 0;
10268 } else if ((rack->rc_last_sent_tlp_seq_valid == 1) &&
10269 (rack->rc_last_sent_tlp_past_cumack == 0) &&
10270 (SEQ_GEQ(th_ack, rack->r_ctl.last_sent_tlp_seq))) {
10271 /*
10272 * It is safe to start aging TLP's send.
10273 */
10274 rack->rc_last_sent_tlp_past_cumack = 1;
10275 }
10276 more:
10277 rsm = tqhash_min(rack->r_ctl.tqh);
10278 if (rsm == NULL) {
10279 if ((th_ack - 1) == tp->iss) {
10280 /*
10281 * For the SYN incoming case we will not
10282 * have called tcp_output for the sending of
10283 * the SYN, so there will be no map. All
10284 * other cases should probably be a panic.
10285 */
10286 return;
10287 }
10288 if (tp->t_flags & TF_SENTFIN) {
10289 /* if we sent a FIN we often will not have map */
10290 return;
10291 }
10292 #ifdef INVARIANTS
10293 panic("No rack map tp:%p for state:%d ack:%u rack:%p snd_una:%u snd_max:%u\n",
10294 tp,
10295 tp->t_state, th_ack, rack,
10296 tp->snd_una, tp->snd_max);
10297 #endif
10298 return;
10299 }
10300 if (SEQ_LT(th_ack, rsm->r_start)) {
10301 /* Huh map is missing this */
10302 #ifdef INVARIANTS
10303 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d\n",
10304 rsm->r_start,
10305 th_ack, tp->t_state, rack->r_state);
10306 #endif
10307 return;
10308 }
10309 rack_update_rtt(tp, rack, rsm, to, cts, CUM_ACKED, th_ack);
10310
10311 /* Now was it a retransmitted TLP? */
10312 if ((rsm->r_flags & RACK_TLP) &&
10313 (rsm->r_rtr_cnt > 1)) {
10314 /*
10315 * Yes, this rsm was a TLP and retransmitted, remember that
10316 * since if a DSACK comes back on this we don't want
10317 * to think of it as a reordered segment. This may
10318 * get updated again with possibly even other TLPs
10319 * in flight, but thats ok. Only when we don't send
10320 * a retransmitted TLP for 1/2 the sequences space
10321 * will it get turned off (above).
10322 */
10323 if (rack->rc_last_tlp_acked_set &&
10324 (is_rsm_inside_declared_tlp_block(rack, rsm))) {
10325 /*
10326 * We already turned this on since the end matches,
10327 * the previous one was a partially ack now we
10328 * are getting another one (maybe all of it).
10329 */
10330 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
10331 /*
10332 * Lets make sure we have all of it though.
10333 */
10334 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
10335 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
10336 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
10337 rack->r_ctl.last_tlp_acked_end);
10338 }
10339 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
10340 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
10341 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
10342 rack->r_ctl.last_tlp_acked_end);
10343 }
10344 } else {
10345 rack->rc_last_tlp_past_cumack = 1;
10346 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
10347 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
10348 rack->rc_last_tlp_acked_set = 1;
10349 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
10350 }
10351 }
10352 /* Now do we consume the whole thing? */
10353 rack->r_ctl.last_tmit_time_acked = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
10354 if (SEQ_GEQ(th_ack, rsm->r_end)) {
10355 /* Its all consumed. */
10356 uint32_t left;
10357 uint8_t newly_acked;
10358
10359 if (rsm->r_flags & RACK_WAS_LOST) {
10360 /*
10361 * This can happen when we marked it as lost
10362 * and yet before retransmitting we get an ack
10363 * which can happen due to reordering.
10364 */
10365 rsm->r_flags &= ~RACK_WAS_LOST;
10366 KASSERT((rack->r_ctl.rc_considered_lost >= (rsm->r_end - rsm->r_start)),
10367 ("rsm:%p rack:%p rc_considered_lost goes negative", rsm, rack));
10368 if (rack->r_ctl.rc_considered_lost >= (rsm->r_end - rsm->r_start))
10369 rack->r_ctl.rc_considered_lost -= rsm->r_end - rsm->r_start;
10370 else
10371 rack->r_ctl.rc_considered_lost = 0;
10372 }
10373 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_FREE, rsm->r_end, __LINE__);
10374 rack->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
10375 rsm->r_rtr_bytes = 0;
10376 /*
10377 * Record the time of highest cumack sent if its in our measurement
10378 * window and possibly bump out the end.
10379 */
10380 rack_rsm_sender_update(rack, tp, rsm, 4);
10381 tqhash_remove(rack->r_ctl.tqh, rsm, REMOVE_TYPE_CUMACK);
10382 if (rsm->r_in_tmap) {
10383 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
10384 rsm->r_in_tmap = 0;
10385 }
10386 newly_acked = 1;
10387 if (rsm->r_flags & RACK_ACKED) {
10388 /*
10389 * It was acked on the scoreboard -- remove
10390 * it from total
10391 */
10392 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
10393 newly_acked = 0;
10394 } else if (rsm->r_flags & RACK_SACK_PASSED) {
10395 /*
10396 * There are segments ACKED on the
10397 * scoreboard further up. We are seeing
10398 * reordering.
10399 */
10400 rsm->r_flags &= ~RACK_SACK_PASSED;
10401 rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
10402 rsm->r_flags |= RACK_ACKED;
10403 rack->r_ctl.rc_reorder_ts = cts;
10404 if (rack->r_ctl.rc_reorder_ts == 0)
10405 rack->r_ctl.rc_reorder_ts = 1;
10406 if (rack->r_ent_rec_ns) {
10407 /*
10408 * We have sent no more, and we saw an sack
10409 * then ack arrive.
10410 */
10411 rack->r_might_revert = 1;
10412 }
10413 rack_update_pcm_ack(rack, 1, rsm->r_start, rsm->r_end);
10414 } else {
10415 rack_update_pcm_ack(rack, 1, rsm->r_start, rsm->r_end);
10416 }
10417 if ((rsm->r_flags & RACK_TO_REXT) &&
10418 (tp->t_flags & TF_RCVD_TSTMP) &&
10419 (to->to_flags & TOF_TS) &&
10420 (to->to_tsecr != 0) &&
10421 (tp->t_flags & TF_PREVVALID)) {
10422 /*
10423 * We can use the timestamp to see
10424 * if this retransmission was from the
10425 * first transmit. If so we made a mistake.
10426 */
10427 tp->t_flags &= ~TF_PREVVALID;
10428 if (to->to_tsecr == rack_ts_to_msec(rsm->r_tim_lastsent[0])) {
10429 /* The first transmit is what this ack is for */
10430 rack_cong_signal(tp, CC_RTO_ERR, th_ack, __LINE__);
10431 }
10432 }
10433 left = th_ack - rsm->r_end;
10434 if (rack->app_limited_needs_set && newly_acked)
10435 rack_need_set_test(tp, rack, rsm, th_ack, __LINE__, RACK_USE_END_OR_THACK);
10436 /* Free back to zone */
10437 rack_free(rack, rsm);
10438 if (left) {
10439 goto more;
10440 }
10441 /* Check for reneging */
10442 rsm = tqhash_min(rack->r_ctl.tqh);
10443 if (rsm && (rsm->r_flags & RACK_ACKED) && (th_ack == rsm->r_start)) {
10444 /*
10445 * The peer has moved snd_una up to
10446 * the edge of this send, i.e. one
10447 * that it had previously acked. The only
10448 * way that can be true if the peer threw
10449 * away data (space issues) that it had
10450 * previously sacked (else it would have
10451 * given us snd_una up to (rsm->r_end).
10452 * We need to undo the acked markings here.
10453 *
10454 * Note we have to look to make sure th_ack is
10455 * our rsm->r_start in case we get an old ack
10456 * where th_ack is behind snd_una.
10457 */
10458 rack_peer_reneges(rack, rsm, th_ack);
10459 }
10460 return;
10461 }
10462 if (rsm->r_flags & RACK_ACKED) {
10463 /*
10464 * It was acked on the scoreboard -- remove it from
10465 * total for the part being cum-acked.
10466 */
10467 rack->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
10468 } else {
10469 rack_update_pcm_ack(rack, 1, rsm->r_start, th_ack);
10470 }
10471 /* And what about the lost flag? */
10472 if (rsm->r_flags & RACK_WAS_LOST) {
10473 /*
10474 * This can happen when we marked it as lost
10475 * and yet before retransmitting we get an ack
10476 * which can happen due to reordering. In this
10477 * case its only a partial ack of the send.
10478 */
10479 KASSERT((rack->r_ctl.rc_considered_lost >= (th_ack - rsm->r_start)),
10480 ("rsm:%p rack:%p rc_considered_lost goes negative th_ack:%u", rsm, rack, th_ack));
10481 if (rack->r_ctl.rc_considered_lost >= (th_ack - rsm->r_start))
10482 rack->r_ctl.rc_considered_lost -= th_ack - rsm->r_start;
10483 else
10484 rack->r_ctl.rc_considered_lost = 0;
10485 }
10486 /*
10487 * Clear the dup ack count for
10488 * the piece that remains.
10489 */
10490 rsm->r_dupack = 0;
10491 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
10492 if (rsm->r_rtr_bytes) {
10493 /*
10494 * It was retransmitted adjust the
10495 * sack holes for what was acked.
10496 */
10497 int ack_am;
10498
10499 ack_am = (th_ack - rsm->r_start);
10500 if (ack_am >= rsm->r_rtr_bytes) {
10501 rack->r_ctl.rc_holes_rxt -= ack_am;
10502 rsm->r_rtr_bytes -= ack_am;
10503 }
10504 }
10505 /*
10506 * Update where the piece starts and record
10507 * the time of send of highest cumack sent if
10508 * its in our GP range.
10509 */
10510 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_TRIM_HEAD, th_ack, __LINE__);
10511 /* Now we need to move our offset forward too */
10512 if (rsm->m &&
10513 ((rsm->orig_m_len != rsm->m->m_len) ||
10514 (M_TRAILINGROOM(rsm->m) != rsm->orig_t_space))) {
10515 /* Fix up the orig_m_len and possibly the mbuf offset */
10516 rack_adjust_orig_mlen(rsm);
10517 }
10518 rsm->soff += (th_ack - rsm->r_start);
10519 rack_rsm_sender_update(rack, tp, rsm, 5);
10520 /* The trim will move th_ack into r_start for us */
10521 tqhash_trim(rack->r_ctl.tqh, th_ack);
10522 /* Now do we need to move the mbuf fwd too? */
10523 {
10524 struct mbuf *m;
10525 uint32_t soff;
10526
10527 m = rsm->m;
10528 soff = rsm->soff;
10529 if (m) {
10530 while (soff >= m->m_len) {
10531 soff -= m->m_len;
10532 KASSERT((m->m_next != NULL),
10533 (" rsm:%p off:%u soff:%u m:%p",
10534 rsm, rsm->soff, soff, m));
10535 m = m->m_next;
10536 if (m == NULL) {
10537 /*
10538 * This is a fall-back that prevents a panic. In reality
10539 * we should be able to walk the mbuf's and find our place.
10540 * At this point snd_una has not been updated with the sbcut() yet
10541 * but tqhash_trim did update rsm->r_start so the offset calcuation
10542 * should work fine. This is undesirable since we will take cache
10543 * hits to access the socket buffer. And even more puzzling is that
10544 * it happens occasionally. It should not :(
10545 */
10546 m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd,
10547 (rsm->r_start - tp->snd_una),
10548 &soff);
10549 break;
10550 }
10551 }
10552 /*
10553 * Now save in our updated values.
10554 */
10555 rsm->m = m;
10556 rsm->soff = soff;
10557 rsm->orig_m_len = rsm->m->m_len;
10558 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
10559 }
10560 }
10561 if (rack->app_limited_needs_set &&
10562 SEQ_GEQ(th_ack, tp->gput_seq))
10563 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_BEG);
10564 }
10565
10566 static void
rack_handle_might_revert(struct tcpcb * tp,struct tcp_rack * rack)10567 rack_handle_might_revert(struct tcpcb *tp, struct tcp_rack *rack)
10568 {
10569 struct rack_sendmap *rsm;
10570 int sack_pass_fnd = 0;
10571
10572 if (rack->r_might_revert) {
10573 /*
10574 * Ok we have reordering, have not sent anything, we
10575 * might want to revert the congestion state if nothing
10576 * further has SACK_PASSED on it. Lets check.
10577 *
10578 * We also get here when we have DSACKs come in for
10579 * all the data that we FR'd. Note that a rxt or tlp
10580 * timer clears this from happening.
10581 */
10582
10583 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
10584 if (rsm->r_flags & RACK_SACK_PASSED) {
10585 sack_pass_fnd = 1;
10586 break;
10587 }
10588 }
10589 if (sack_pass_fnd == 0) {
10590 /*
10591 * We went into recovery
10592 * incorrectly due to reordering!
10593 */
10594 int orig_cwnd;
10595
10596 rack->r_ent_rec_ns = 0;
10597 orig_cwnd = tp->snd_cwnd;
10598 tp->snd_ssthresh = rack->r_ctl.rc_ssthresh_at_erec;
10599 tp->snd_recover = tp->snd_una;
10600 rack_log_to_prr(rack, 14, orig_cwnd, __LINE__);
10601 if (IN_RECOVERY(tp->t_flags)) {
10602 rack_exit_recovery(tp, rack, 3);
10603 if ((rack->rto_from_rec == 1) && (rack_ssthresh_rest_rto_rec != 0) ){
10604 /*
10605 * We were in recovery, had an RTO
10606 * and then re-entered recovery (more sack's arrived)
10607 * and we have properly recorded the old ssthresh from
10608 * the first recovery. We want to be able to slow-start
10609 * back to this level. The ssthresh from the timeout
10610 * and then back into recovery will end up most likely
10611 * to be min(cwnd=1mss, 2mss). Which makes it basically
10612 * so we get no slow-start after our RTO.
10613 */
10614 rack->rto_from_rec = 0;
10615 if (rack->r_ctl.rto_ssthresh > tp->snd_ssthresh)
10616 tp->snd_ssthresh = rack->r_ctl.rto_ssthresh;
10617 }
10618 }
10619 }
10620 rack->r_might_revert = 0;
10621 }
10622 }
10623
10624
10625 static int
rack_note_dsack(struct tcp_rack * rack,tcp_seq start,tcp_seq end)10626 rack_note_dsack(struct tcp_rack *rack, tcp_seq start, tcp_seq end)
10627 {
10628
10629 uint32_t am, l_end;
10630 int was_tlp = 0;
10631
10632 if (SEQ_GT(end, start))
10633 am = end - start;
10634 else
10635 am = 0;
10636 if ((rack->rc_last_tlp_acked_set ) &&
10637 (SEQ_GEQ(start, rack->r_ctl.last_tlp_acked_start)) &&
10638 (SEQ_LEQ(end, rack->r_ctl.last_tlp_acked_end))) {
10639 /*
10640 * The DSACK is because of a TLP which we don't
10641 * do anything with the reordering window over since
10642 * it was not reordering that caused the DSACK but
10643 * our previous retransmit TLP.
10644 */
10645 rack_log_dsack_event(rack, 7, __LINE__, start, end);
10646 was_tlp = 1;
10647 goto skip_dsack_round;
10648 }
10649 if (rack->rc_last_sent_tlp_seq_valid) {
10650 l_end = rack->r_ctl.last_sent_tlp_seq + rack->r_ctl.last_sent_tlp_len;
10651 if (SEQ_GEQ(start, rack->r_ctl.last_sent_tlp_seq) &&
10652 (SEQ_LEQ(end, l_end))) {
10653 /*
10654 * This dsack is from the last sent TLP, ignore it
10655 * for reordering purposes.
10656 */
10657 rack_log_dsack_event(rack, 7, __LINE__, start, end);
10658 was_tlp = 1;
10659 goto skip_dsack_round;
10660 }
10661 }
10662 if (rack->rc_dsack_round_seen == 0) {
10663 rack->rc_dsack_round_seen = 1;
10664 rack->r_ctl.dsack_round_end = rack->rc_tp->snd_max;
10665 rack->r_ctl.num_dsack++;
10666 rack->r_ctl.dsack_persist = 16; /* 16 is from the standard */
10667 rack_log_dsack_event(rack, 2, __LINE__, 0, 0);
10668 }
10669 skip_dsack_round:
10670 /*
10671 * We keep track of how many DSACK blocks we get
10672 * after a recovery incident.
10673 */
10674 rack->r_ctl.dsack_byte_cnt += am;
10675 if (!IN_FASTRECOVERY(rack->rc_tp->t_flags) &&
10676 rack->r_ctl.retran_during_recovery &&
10677 (rack->r_ctl.dsack_byte_cnt >= rack->r_ctl.retran_during_recovery)) {
10678 /*
10679 * False recovery most likely culprit is reordering. If
10680 * nothing else is missing we need to revert.
10681 */
10682 rack->r_might_revert = 1;
10683 rack_handle_might_revert(rack->rc_tp, rack);
10684 rack->r_might_revert = 0;
10685 rack->r_ctl.retran_during_recovery = 0;
10686 rack->r_ctl.dsack_byte_cnt = 0;
10687 }
10688 return (was_tlp);
10689 }
10690
10691 static uint32_t
do_rack_compute_pipe(struct tcpcb * tp,struct tcp_rack * rack,uint32_t snd_una)10692 do_rack_compute_pipe(struct tcpcb *tp, struct tcp_rack *rack, uint32_t snd_una)
10693 {
10694 return (((tp->snd_max - snd_una) -
10695 (rack->r_ctl.rc_sacked + rack->r_ctl.rc_considered_lost)) + rack->r_ctl.rc_holes_rxt);
10696 }
10697
10698 static int32_t
rack_compute_pipe(struct tcpcb * tp)10699 rack_compute_pipe(struct tcpcb *tp)
10700 {
10701 return ((int32_t)do_rack_compute_pipe(tp,
10702 (struct tcp_rack *)tp->t_fb_ptr,
10703 tp->snd_una));
10704 }
10705
10706 static void
rack_update_prr(struct tcpcb * tp,struct tcp_rack * rack,uint32_t changed,tcp_seq th_ack)10707 rack_update_prr(struct tcpcb *tp, struct tcp_rack *rack, uint32_t changed, tcp_seq th_ack)
10708 {
10709 /* Deal with changed and PRR here (in recovery only) */
10710 uint32_t pipe, snd_una;
10711
10712 rack->r_ctl.rc_prr_delivered += changed;
10713
10714 if (sbavail(&rack->rc_inp->inp_socket->so_snd) <= (tp->snd_max - tp->snd_una)) {
10715 /*
10716 * It is all outstanding, we are application limited
10717 * and thus we don't need more room to send anything.
10718 * Note we use tp->snd_una here and not th_ack because
10719 * the data as yet not been cut from the sb.
10720 */
10721 rack->r_ctl.rc_prr_sndcnt = 0;
10722 return;
10723 }
10724 /* Compute prr_sndcnt */
10725 if (SEQ_GT(tp->snd_una, th_ack)) {
10726 snd_una = tp->snd_una;
10727 } else {
10728 snd_una = th_ack;
10729 }
10730 pipe = do_rack_compute_pipe(tp, rack, snd_una);
10731 if (pipe > tp->snd_ssthresh) {
10732 long sndcnt;
10733
10734 sndcnt = rack->r_ctl.rc_prr_delivered * tp->snd_ssthresh;
10735 if (rack->r_ctl.rc_prr_recovery_fs > 0)
10736 sndcnt /= (long)rack->r_ctl.rc_prr_recovery_fs;
10737 else {
10738 rack->r_ctl.rc_prr_sndcnt = 0;
10739 rack_log_to_prr(rack, 9, 0, __LINE__);
10740 sndcnt = 0;
10741 }
10742 sndcnt++;
10743 if (sndcnt > (long)rack->r_ctl.rc_prr_out)
10744 sndcnt -= rack->r_ctl.rc_prr_out;
10745 else
10746 sndcnt = 0;
10747 rack->r_ctl.rc_prr_sndcnt = sndcnt;
10748 rack_log_to_prr(rack, 10, 0, __LINE__);
10749 } else {
10750 uint32_t limit;
10751
10752 if (rack->r_ctl.rc_prr_delivered > rack->r_ctl.rc_prr_out)
10753 limit = (rack->r_ctl.rc_prr_delivered - rack->r_ctl.rc_prr_out);
10754 else
10755 limit = 0;
10756 if (changed > limit)
10757 limit = changed;
10758 limit += ctf_fixed_maxseg(tp);
10759 if (tp->snd_ssthresh > pipe) {
10760 rack->r_ctl.rc_prr_sndcnt = min((tp->snd_ssthresh - pipe), limit);
10761 rack_log_to_prr(rack, 11, 0, __LINE__);
10762 } else {
10763 rack->r_ctl.rc_prr_sndcnt = min(0, limit);
10764 rack_log_to_prr(rack, 12, 0, __LINE__);
10765 }
10766 }
10767 }
10768
10769 static void
rack_log_ack(struct tcpcb * tp,struct tcpopt * to,struct tcphdr * th,int entered_recovery,int dup_ack_struck,int * dsack_seen,int * sacks_seen)10770 rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered_recovery, int dup_ack_struck,
10771 int *dsack_seen, int *sacks_seen)
10772 {
10773 uint32_t changed;
10774 struct tcp_rack *rack;
10775 struct rack_sendmap *rsm;
10776 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1];
10777 register uint32_t th_ack;
10778 int32_t i, j, k, num_sack_blks = 0;
10779 uint32_t cts, acked, ack_point;
10780 int loop_start = 0;
10781 uint32_t tsused;
10782 uint32_t segsiz;
10783
10784
10785 INP_WLOCK_ASSERT(tptoinpcb(tp));
10786 if (tcp_get_flags(th) & TH_RST) {
10787 /* We don't log resets */
10788 return;
10789 }
10790 rack = (struct tcp_rack *)tp->t_fb_ptr;
10791 cts = tcp_get_usecs(NULL);
10792 rsm = tqhash_min(rack->r_ctl.tqh);
10793 changed = 0;
10794 th_ack = th->th_ack;
10795 segsiz = ctf_fixed_maxseg(rack->rc_tp);
10796 if (BYTES_THIS_ACK(tp, th) >= segsiz) {
10797 /*
10798 * You only get credit for
10799 * MSS and greater (and you get extra
10800 * credit for larger cum-ack moves).
10801 */
10802 int ac;
10803
10804 ac = BYTES_THIS_ACK(tp, th) / ctf_fixed_maxseg(rack->rc_tp);
10805 counter_u64_add(rack_ack_total, ac);
10806 }
10807 if (SEQ_GT(th_ack, tp->snd_una)) {
10808 rack_log_progress_event(rack, tp, ticks, PROGRESS_UPDATE, __LINE__);
10809 tp->t_acktime = ticks;
10810 }
10811 if (rsm && SEQ_GT(th_ack, rsm->r_start))
10812 changed = th_ack - rsm->r_start;
10813 if (changed) {
10814 rack_process_to_cumack(tp, rack, th_ack, cts, to,
10815 tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time));
10816 }
10817 if ((to->to_flags & TOF_SACK) == 0) {
10818 /* We are done nothing left and no sack. */
10819 rack_handle_might_revert(tp, rack);
10820 /*
10821 * For cases where we struck a dup-ack
10822 * with no SACK, add to the changes so
10823 * PRR will work right.
10824 */
10825 if (dup_ack_struck && (changed == 0)) {
10826 changed += ctf_fixed_maxseg(rack->rc_tp);
10827 }
10828 goto out;
10829 }
10830 /* Sack block processing */
10831 if (SEQ_GT(th_ack, tp->snd_una))
10832 ack_point = th_ack;
10833 else
10834 ack_point = tp->snd_una;
10835 for (i = 0; i < to->to_nsacks; i++) {
10836 bcopy((to->to_sacks + i * TCPOLEN_SACK),
10837 &sack, sizeof(sack));
10838 sack.start = ntohl(sack.start);
10839 sack.end = ntohl(sack.end);
10840 if (SEQ_GT(sack.end, sack.start) &&
10841 SEQ_GT(sack.start, ack_point) &&
10842 SEQ_LT(sack.start, tp->snd_max) &&
10843 SEQ_GT(sack.end, ack_point) &&
10844 SEQ_LEQ(sack.end, tp->snd_max)) {
10845 sack_blocks[num_sack_blks] = sack;
10846 num_sack_blks++;
10847 } else if (SEQ_LEQ(sack.start, th_ack) &&
10848 SEQ_LEQ(sack.end, th_ack)) {
10849 int was_tlp;
10850
10851 if (dsack_seen != NULL)
10852 *dsack_seen = 1;
10853 was_tlp = rack_note_dsack(rack, sack.start, sack.end);
10854 /*
10855 * Its a D-SACK block.
10856 */
10857 tcp_record_dsack(tp, sack.start, sack.end, was_tlp);
10858 }
10859 }
10860 if (rack->rc_dsack_round_seen) {
10861 /* Is the dsack roound over? */
10862 if (SEQ_GEQ(th_ack, rack->r_ctl.dsack_round_end)) {
10863 /* Yes it is */
10864 rack->rc_dsack_round_seen = 0;
10865 rack_log_dsack_event(rack, 3, __LINE__, 0, 0);
10866 }
10867 }
10868 /*
10869 * Sort the SACK blocks so we can update the rack scoreboard with
10870 * just one pass.
10871 */
10872 num_sack_blks = sack_filter_blks(tp, &rack->r_ctl.rack_sf, sack_blocks,
10873 num_sack_blks, th->th_ack);
10874 ctf_log_sack_filter(rack->rc_tp, num_sack_blks, sack_blocks);
10875 if (sacks_seen != NULL)
10876 *sacks_seen = num_sack_blks;
10877 if (num_sack_blks == 0) {
10878 /* Nothing to sack, but we need to update counts */
10879 goto out_with_totals;
10880 }
10881 /* Its a sack of some sort */
10882 if (num_sack_blks < 2) {
10883 /* Only one, we don't need to sort */
10884 goto do_sack_work;
10885 }
10886 /* Sort the sacks */
10887 for (i = 0; i < num_sack_blks; i++) {
10888 for (j = i + 1; j < num_sack_blks; j++) {
10889 if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) {
10890 sack = sack_blocks[i];
10891 sack_blocks[i] = sack_blocks[j];
10892 sack_blocks[j] = sack;
10893 }
10894 }
10895 }
10896 /*
10897 * Now are any of the sack block ends the same (yes some
10898 * implementations send these)?
10899 */
10900 again:
10901 if (num_sack_blks == 0)
10902 goto out_with_totals;
10903 if (num_sack_blks > 1) {
10904 for (i = 0; i < num_sack_blks; i++) {
10905 for (j = i + 1; j < num_sack_blks; j++) {
10906 if (sack_blocks[i].end == sack_blocks[j].end) {
10907 /*
10908 * Ok these two have the same end we
10909 * want the smallest end and then
10910 * throw away the larger and start
10911 * again.
10912 */
10913 if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) {
10914 /*
10915 * The second block covers
10916 * more area use that
10917 */
10918 sack_blocks[i].start = sack_blocks[j].start;
10919 }
10920 /*
10921 * Now collapse out the dup-sack and
10922 * lower the count
10923 */
10924 for (k = (j + 1); k < num_sack_blks; k++) {
10925 sack_blocks[j].start = sack_blocks[k].start;
10926 sack_blocks[j].end = sack_blocks[k].end;
10927 j++;
10928 }
10929 num_sack_blks--;
10930 goto again;
10931 }
10932 }
10933 }
10934 }
10935 do_sack_work:
10936 /*
10937 * First lets look to see if
10938 * we have retransmitted and
10939 * can use the transmit next?
10940 */
10941 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
10942 if (rsm &&
10943 SEQ_GT(sack_blocks[0].end, rsm->r_start) &&
10944 SEQ_LT(sack_blocks[0].start, rsm->r_end)) {
10945 /*
10946 * We probably did the FR and the next
10947 * SACK in continues as we would expect.
10948 */
10949 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[0], to, &rsm, cts, segsiz);
10950 if (acked) {
10951 rack->r_wanted_output = 1;
10952 changed += acked;
10953 }
10954 if (num_sack_blks == 1) {
10955 /*
10956 * This is what we would expect from
10957 * a normal implementation to happen
10958 * after we have retransmitted the FR,
10959 * i.e the sack-filter pushes down
10960 * to 1 block and the next to be retransmitted
10961 * is the sequence in the sack block (has more
10962 * are acked). Count this as ACK'd data to boost
10963 * up the chances of recovering any false positives.
10964 */
10965 counter_u64_add(rack_ack_total, (acked / ctf_fixed_maxseg(rack->rc_tp)));
10966 counter_u64_add(rack_express_sack, 1);
10967 goto out_with_totals;
10968 } else {
10969 /*
10970 * Start the loop through the
10971 * rest of blocks, past the first block.
10972 */
10973 loop_start = 1;
10974 }
10975 }
10976 counter_u64_add(rack_sack_total, 1);
10977 rsm = rack->r_ctl.rc_sacklast;
10978 for (i = loop_start; i < num_sack_blks; i++) {
10979 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[i], to, &rsm, cts, segsiz);
10980 if (acked) {
10981 rack->r_wanted_output = 1;
10982 changed += acked;
10983 }
10984 }
10985 out_with_totals:
10986 if (num_sack_blks > 1) {
10987 /*
10988 * You get an extra stroke if
10989 * you have more than one sack-blk, this
10990 * could be where we are skipping forward
10991 * and the sack-filter is still working, or
10992 * it could be an attacker constantly
10993 * moving us.
10994 */
10995 counter_u64_add(rack_move_some, 1);
10996 }
10997 out:
10998 if (changed) {
10999 /* Something changed cancel the rack timer */
11000 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
11001 }
11002 tsused = tcp_get_usecs(NULL);
11003 rsm = tcp_rack_output(tp, rack, tsused);
11004 if ((!IN_FASTRECOVERY(tp->t_flags)) &&
11005 rsm &&
11006 ((rsm->r_flags & RACK_MUST_RXT) == 0)) {
11007 /* Enter recovery */
11008 entered_recovery = 1;
11009 rack_cong_signal(tp, CC_NDUPACK, th_ack, __LINE__);
11010 /*
11011 * When we enter recovery we need to assure we send
11012 * one packet.
11013 */
11014 if (rack->rack_no_prr == 0) {
11015 rack->r_ctl.rc_prr_sndcnt = ctf_fixed_maxseg(tp);
11016 rack_log_to_prr(rack, 8, 0, __LINE__);
11017 }
11018 rack->r_timer_override = 1;
11019 rack->r_early = 0;
11020 rack->r_ctl.rc_agg_early = 0;
11021 } else if (IN_FASTRECOVERY(tp->t_flags) &&
11022 rsm &&
11023 (rack->r_rr_config == 3)) {
11024 /*
11025 * Assure we can output and we get no
11026 * remembered pace time except the retransmit.
11027 */
11028 rack->r_timer_override = 1;
11029 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
11030 rack->r_ctl.rc_resend = rsm;
11031 }
11032 if (IN_FASTRECOVERY(tp->t_flags) &&
11033 (rack->rack_no_prr == 0) &&
11034 (entered_recovery == 0)) {
11035 rack_update_prr(tp, rack, changed, th_ack);
11036 if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) &&
11037 ((tcp_in_hpts(rack->rc_tp) == 0) &&
11038 ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) {
11039 /*
11040 * If you are pacing output you don't want
11041 * to override.
11042 */
11043 rack->r_early = 0;
11044 rack->r_ctl.rc_agg_early = 0;
11045 rack->r_timer_override = 1;
11046 }
11047 }
11048 }
11049
11050 static void
rack_strike_dupack(struct tcp_rack * rack,tcp_seq th_ack)11051 rack_strike_dupack(struct tcp_rack *rack, tcp_seq th_ack)
11052 {
11053 struct rack_sendmap *rsm;
11054
11055 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
11056 while (rsm) {
11057 /*
11058 * We need to skip anything already set
11059 * to be retransmitted.
11060 */
11061 if ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
11062 (rsm->r_flags & RACK_MUST_RXT)) {
11063 rsm = TAILQ_NEXT(rsm, r_tnext);
11064 continue;
11065 }
11066 break;
11067 }
11068 if (rsm && (rsm->r_dupack < 0xff)) {
11069 rsm->r_dupack++;
11070 if (rsm->r_dupack >= DUP_ACK_THRESHOLD) {
11071 struct timeval tv;
11072 uint32_t cts;
11073 /*
11074 * Here we see if we need to retransmit. For
11075 * a SACK type connection if enough time has passed
11076 * we will get a return of the rsm. For a non-sack
11077 * connection we will get the rsm returned if the
11078 * dupack value is 3 or more.
11079 */
11080 cts = tcp_get_usecs(&tv);
11081 rack->r_ctl.rc_resend = tcp_rack_output(rack->rc_tp, rack, cts);
11082 if (rack->r_ctl.rc_resend != NULL) {
11083 if (!IN_FASTRECOVERY(rack->rc_tp->t_flags)) {
11084 rack_cong_signal(rack->rc_tp, CC_NDUPACK,
11085 th_ack, __LINE__);
11086 }
11087 rack->r_wanted_output = 1;
11088 rack->r_timer_override = 1;
11089 rack_log_retran_reason(rack, rsm, __LINE__, 1, 3);
11090 }
11091 } else {
11092 rack_log_retran_reason(rack, rsm, __LINE__, 0, 3);
11093 }
11094 }
11095 }
11096
11097 static void
rack_check_bottom_drag(struct tcpcb * tp,struct tcp_rack * rack,struct socket * so)11098 rack_check_bottom_drag(struct tcpcb *tp,
11099 struct tcp_rack *rack,
11100 struct socket *so)
11101 {
11102 /*
11103 * So what is dragging bottom?
11104 *
11105 * Dragging bottom means you were under pacing and had a
11106 * delay in processing inbound acks waiting on our pacing
11107 * timer to expire. While you were waiting all of the acknowledgments
11108 * for the packets you sent have arrived. This means we are pacing
11109 * way underneath the bottleneck to the point where our Goodput
11110 * measurements stop working, since they require more than one
11111 * ack (usually at least 8 packets worth with multiple acks so we can
11112 * gauge the inter-ack times). If that occurs we have a real problem
11113 * since we are stuck in a hole that we can't get out of without
11114 * something speeding us up.
11115 *
11116 * We also check to see if we are widdling down to just one segment
11117 * outstanding. If this occurs and we have room to send in our cwnd/rwnd
11118 * then we are adding the delayed ack interval into our measurments and
11119 * we need to speed up slightly.
11120 */
11121 uint32_t segsiz, minseg;
11122
11123 segsiz = ctf_fixed_maxseg(tp);
11124 minseg = segsiz;
11125 if (tp->snd_max == tp->snd_una) {
11126 /*
11127 * We are doing dynamic pacing and we are way
11128 * under. Basically everything got acked while
11129 * we were still waiting on the pacer to expire.
11130 *
11131 * This means we need to boost the b/w in
11132 * addition to any earlier boosting of
11133 * the multiplier.
11134 */
11135 uint64_t lt_bw;
11136
11137 tcp_trace_point(rack->rc_tp, TCP_TP_PACED_BOTTOM);
11138 lt_bw = rack_get_lt_bw(rack);
11139 rack->rc_dragged_bottom = 1;
11140 rack_validate_multipliers_at_or_above100(rack);
11141 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_VALID) &&
11142 (rack->dis_lt_bw == 0) &&
11143 (rack->use_lesser_lt_bw == 0) &&
11144 (lt_bw > 0)) {
11145 /*
11146 * Lets use the long-term b/w we have
11147 * been getting as a base.
11148 */
11149 if (rack->rc_gp_filled == 0) {
11150 if (lt_bw > ONE_POINT_TWO_MEG) {
11151 /*
11152 * If we have no measurement
11153 * don't let us set in more than
11154 * 1.2Mbps. If we are still too
11155 * low after pacing with this we
11156 * will hopefully have a max b/w
11157 * available to sanity check things.
11158 */
11159 lt_bw = ONE_POINT_TWO_MEG;
11160 }
11161 rack->r_ctl.rc_rtt_diff = 0;
11162 rack->r_ctl.gp_bw = lt_bw;
11163 rack->rc_gp_filled = 1;
11164 if (rack->r_ctl.num_measurements < RACK_REQ_AVG)
11165 rack->r_ctl.num_measurements = RACK_REQ_AVG;
11166 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
11167 } else if (lt_bw > rack->r_ctl.gp_bw) {
11168 rack->r_ctl.rc_rtt_diff = 0;
11169 if (rack->r_ctl.num_measurements < RACK_REQ_AVG)
11170 rack->r_ctl.num_measurements = RACK_REQ_AVG;
11171 rack->r_ctl.gp_bw = lt_bw;
11172 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
11173 } else
11174 rack_increase_bw_mul(rack, -1, 0, 0, 1);
11175 if ((rack->gp_ready == 0) &&
11176 (rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) {
11177 /* We have enough measurements now */
11178 rack->gp_ready = 1;
11179 if (rack->dgp_on ||
11180 rack->rack_hibeta)
11181 rack_set_cc_pacing(rack);
11182 if (rack->defer_options)
11183 rack_apply_deferred_options(rack);
11184 }
11185 } else {
11186 /*
11187 * zero rtt possibly?, settle for just an old increase.
11188 */
11189 rack_increase_bw_mul(rack, -1, 0, 0, 1);
11190 }
11191 } else if ((IN_FASTRECOVERY(tp->t_flags) == 0) &&
11192 (sbavail(&so->so_snd) > max((segsiz * (4 + rack_req_segs)),
11193 minseg)) &&
11194 (rack->r_ctl.cwnd_to_use > max((segsiz * (rack_req_segs + 2)), minseg)) &&
11195 (tp->snd_wnd > max((segsiz * (rack_req_segs + 2)), minseg)) &&
11196 (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) <=
11197 (segsiz * rack_req_segs))) {
11198 /*
11199 * We are doing dynamic GP pacing and
11200 * we have everything except 1MSS or less
11201 * bytes left out. We are still pacing away.
11202 * And there is data that could be sent, This
11203 * means we are inserting delayed ack time in
11204 * our measurements because we are pacing too slow.
11205 */
11206 rack_validate_multipliers_at_or_above100(rack);
11207 rack->rc_dragged_bottom = 1;
11208 rack_increase_bw_mul(rack, -1, 0, 0, 1);
11209 }
11210 }
11211
11212 #ifdef TCP_REQUEST_TRK
11213 static void
rack_log_hybrid(struct tcp_rack * rack,uint32_t seq,struct tcp_sendfile_track * cur,uint8_t mod,int line,int err)11214 rack_log_hybrid(struct tcp_rack *rack, uint32_t seq,
11215 struct tcp_sendfile_track *cur, uint8_t mod, int line, int err)
11216 {
11217 int do_log;
11218
11219 do_log = tcp_bblogging_on(rack->rc_tp);
11220 if (do_log == 0) {
11221 if ((do_log = tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) )== 0)
11222 return;
11223 /* We only allow the three below with point logging on */
11224 if ((mod != HYBRID_LOG_RULES_APP) &&
11225 (mod != HYBRID_LOG_RULES_SET) &&
11226 (mod != HYBRID_LOG_REQ_COMP))
11227 return;
11228
11229 }
11230 if (do_log) {
11231 union tcp_log_stackspecific log;
11232 struct timeval tv;
11233
11234 /* Convert our ms to a microsecond */
11235 memset(&log, 0, sizeof(log));
11236 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
11237 log.u_bbr.flex1 = seq;
11238 log.u_bbr.cwnd_gain = line;
11239 if (cur != NULL) {
11240 uint64_t off;
11241
11242 log.u_bbr.flex2 = cur->start_seq;
11243 log.u_bbr.flex3 = cur->end_seq;
11244 log.u_bbr.flex4 = (uint32_t)((cur->localtime >> 32) & 0x00000000ffffffff);
11245 log.u_bbr.flex5 = (uint32_t)(cur->localtime & 0x00000000ffffffff);
11246 log.u_bbr.flex6 = cur->flags;
11247 log.u_bbr.pkts_out = cur->hybrid_flags;
11248 log.u_bbr.rttProp = cur->timestamp;
11249 log.u_bbr.cur_del_rate = cur->cspr;
11250 log.u_bbr.bw_inuse = cur->start;
11251 log.u_bbr.applimited = (uint32_t)(cur->end & 0x00000000ffffffff);
11252 log.u_bbr.delivered = (uint32_t)((cur->end >> 32) & 0x00000000ffffffff) ;
11253 log.u_bbr.epoch = (uint32_t)(cur->deadline & 0x00000000ffffffff);
11254 log.u_bbr.lt_epoch = (uint32_t)((cur->deadline >> 32) & 0x00000000ffffffff) ;
11255 log.u_bbr.inhpts = 1;
11256 #ifdef TCP_REQUEST_TRK
11257 off = (uint64_t)(cur) - (uint64_t)(&rack->rc_tp->t_tcpreq_info[0]);
11258 log.u_bbr.use_lt_bw = (uint8_t)(off / sizeof(struct tcp_sendfile_track));
11259 #endif
11260 } else {
11261 log.u_bbr.flex2 = err;
11262 }
11263 /*
11264 * Fill in flex7 to be CHD (catchup|hybrid|DGP)
11265 */
11266 log.u_bbr.flex7 = rack->rc_catch_up;
11267 log.u_bbr.flex7 <<= 1;
11268 log.u_bbr.flex7 |= rack->rc_hybrid_mode;
11269 log.u_bbr.flex7 <<= 1;
11270 log.u_bbr.flex7 |= rack->dgp_on;
11271 /*
11272 * Compose bbr_state to be a bit wise 0000ADHF
11273 * where A is the always_pace flag
11274 * where D is the dgp_on flag
11275 * where H is the hybrid_mode on flag
11276 * where F is the use_fixed_rate flag.
11277 */
11278 log.u_bbr.bbr_state = rack->rc_always_pace;
11279 log.u_bbr.bbr_state <<= 1;
11280 log.u_bbr.bbr_state |= rack->dgp_on;
11281 log.u_bbr.bbr_state <<= 1;
11282 log.u_bbr.bbr_state |= rack->rc_hybrid_mode;
11283 log.u_bbr.bbr_state <<= 1;
11284 log.u_bbr.bbr_state |= rack->use_fixed_rate;
11285 log.u_bbr.flex8 = mod;
11286 log.u_bbr.delRate = rack->r_ctl.bw_rate_cap;
11287 log.u_bbr.bbr_substate = rack->r_ctl.client_suggested_maxseg;
11288 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
11289 log.u_bbr.pkt_epoch = rack->rc_tp->tcp_hybrid_start;
11290 log.u_bbr.lost = rack->rc_tp->tcp_hybrid_error;
11291 log.u_bbr.pacing_gain = (uint16_t)rack->rc_tp->tcp_hybrid_stop;
11292 tcp_log_event(rack->rc_tp, NULL,
11293 &rack->rc_inp->inp_socket->so_rcv,
11294 &rack->rc_inp->inp_socket->so_snd,
11295 TCP_HYBRID_PACING_LOG, 0,
11296 0, &log, false, NULL, __func__, __LINE__, &tv);
11297 }
11298 }
11299 #endif
11300
11301 #ifdef TCP_REQUEST_TRK
11302 static void
rack_set_dgp_hybrid_mode(struct tcp_rack * rack,tcp_seq seq,uint32_t len,uint64_t cts)11303 rack_set_dgp_hybrid_mode(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts)
11304 {
11305 struct tcp_sendfile_track *rc_cur, *orig_ent;
11306 struct tcpcb *tp;
11307 int err = 0;
11308
11309 orig_ent = rack->r_ctl.rc_last_sft;
11310 rc_cur = tcp_req_find_req_for_seq(rack->rc_tp, seq);
11311 if (rc_cur == NULL) {
11312 /* If not in the beginning what about the end piece */
11313 if (rack->rc_hybrid_mode)
11314 rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
11315 rc_cur = tcp_req_find_req_for_seq(rack->rc_tp, (seq + len - 1));
11316 } else {
11317 err = 12345;
11318 }
11319 /* If we find no parameters we are in straight DGP mode */
11320 if(rc_cur == NULL) {
11321 /* None found for this seq, just DGP for now */
11322 if (rack->rc_hybrid_mode) {
11323 rack->r_ctl.client_suggested_maxseg = 0;
11324 rack->rc_catch_up = 0;
11325 if (rack->cspr_is_fcc == 0)
11326 rack->r_ctl.bw_rate_cap = 0;
11327 else
11328 rack->r_ctl.fillcw_cap = rack_fillcw_bw_cap;
11329 }
11330 if (rack->rc_hybrid_mode) {
11331 rack_log_hybrid(rack, (seq + len - 1), NULL, HYBRID_LOG_NO_RANGE, __LINE__, err);
11332 }
11333 if (rack->r_ctl.rc_last_sft) {
11334 rack->r_ctl.rc_last_sft = NULL;
11335 }
11336 return;
11337 }
11338 if ((rc_cur->hybrid_flags & TCP_HYBRID_PACING_WASSET) == 0) {
11339 /* This entry was never setup for hybrid pacing on/off etc */
11340 if (rack->rc_hybrid_mode) {
11341 rack->r_ctl.client_suggested_maxseg = 0;
11342 rack->rc_catch_up = 0;
11343 rack->r_ctl.bw_rate_cap = 0;
11344 }
11345 if (rack->r_ctl.rc_last_sft) {
11346 rack->r_ctl.rc_last_sft = NULL;
11347 }
11348 if ((rc_cur->flags & TCP_TRK_TRACK_FLG_FSND) == 0) {
11349 rc_cur->flags |= TCP_TRK_TRACK_FLG_FSND;
11350 rc_cur->first_send = cts;
11351 rc_cur->sent_at_fs = rack->rc_tp->t_sndbytes;
11352 rc_cur->rxt_at_fs = rack->rc_tp->t_snd_rxt_bytes;
11353 }
11354 return;
11355 }
11356 /*
11357 * Ok if we have a new entry *or* have never
11358 * set up an entry we need to proceed. If
11359 * we have already set it up this entry we
11360 * just continue along with what we already
11361 * setup.
11362 */
11363 tp = rack->rc_tp;
11364 if ((rack->r_ctl.rc_last_sft != NULL) &&
11365 (rack->r_ctl.rc_last_sft == rc_cur)) {
11366 /* Its already in place */
11367 if (rack->rc_hybrid_mode)
11368 rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_ISSAME, __LINE__, 0);
11369 return;
11370 }
11371 if (rack->rc_hybrid_mode == 0) {
11372 rack->r_ctl.rc_last_sft = rc_cur;
11373 if (orig_ent) {
11374 orig_ent->sent_at_ls = rack->rc_tp->t_sndbytes;
11375 orig_ent->rxt_at_ls = rack->rc_tp->t_snd_rxt_bytes;
11376 orig_ent->flags |= TCP_TRK_TRACK_FLG_LSND;
11377 }
11378 rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_RULES_APP, __LINE__, 0);
11379 return;
11380 }
11381 if ((rc_cur->hybrid_flags & TCP_HYBRID_PACING_CSPR) && rc_cur->cspr){
11382 /* Compensate for all the header overhead's */
11383 if (rack->cspr_is_fcc == 0)
11384 rack->r_ctl.bw_rate_cap = rack_compensate_for_linerate(rack, rc_cur->cspr);
11385 else
11386 rack->r_ctl.fillcw_cap = rack_compensate_for_linerate(rack, rc_cur->cspr);
11387 } else {
11388 if (rack->rc_hybrid_mode) {
11389 if (rack->cspr_is_fcc == 0)
11390 rack->r_ctl.bw_rate_cap = 0;
11391 else
11392 rack->r_ctl.fillcw_cap = rack_fillcw_bw_cap;
11393 }
11394 }
11395 if (rc_cur->hybrid_flags & TCP_HYBRID_PACING_H_MS)
11396 rack->r_ctl.client_suggested_maxseg = rc_cur->hint_maxseg;
11397 else
11398 rack->r_ctl.client_suggested_maxseg = 0;
11399 if (rc_cur->timestamp == rack->r_ctl.last_tm_mark) {
11400 /*
11401 * It is the same timestamp as the previous one
11402 * add the hybrid flag that will indicate we use
11403 * sendtime not arrival time for catch-up mode.
11404 */
11405 rc_cur->hybrid_flags |= TCP_HYBRID_PACING_SENDTIME;
11406 }
11407 if ((rc_cur->hybrid_flags & TCP_HYBRID_PACING_CU) &&
11408 (rc_cur->cspr > 0)) {
11409 uint64_t len;
11410
11411 rack->rc_catch_up = 1;
11412 /*
11413 * Calculate the deadline time, first set the
11414 * time to when the request arrived.
11415 */
11416 if (rc_cur->hybrid_flags & TCP_HYBRID_PACING_SENDTIME) {
11417 /*
11418 * For cases where its a duplicate tm (we received more
11419 * than one request for a tm) we want to use now, the point
11420 * where we are just sending the first bit of the request.
11421 */
11422 rc_cur->deadline = cts;
11423 } else {
11424 /*
11425 * Here we have a different tm from the last request
11426 * so we want to use arrival time as our base.
11427 */
11428 rc_cur->deadline = rc_cur->localtime;
11429 }
11430 /*
11431 * Next calculate the length and compensate for
11432 * TLS if need be.
11433 */
11434 len = rc_cur->end - rc_cur->start;
11435 if (tp->t_inpcb.inp_socket->so_snd.sb_tls_info) {
11436 /*
11437 * This session is doing TLS. Take a swag guess
11438 * at the overhead.
11439 */
11440 len += tcp_estimate_tls_overhead(tp->t_inpcb.inp_socket, len);
11441 }
11442 /*
11443 * Now considering the size, and the cspr, what is the time that
11444 * would be required at the cspr rate. Here we use the raw
11445 * cspr value since the client only looks at the raw data. We
11446 * do use len which includes TLS overhead, but not the TCP/IP etc.
11447 * That will get made up for in the CU pacing rate set.
11448 */
11449 len *= HPTS_USEC_IN_SEC;
11450 len /= rc_cur->cspr;
11451 rc_cur->deadline += len;
11452 } else {
11453 rack->rc_catch_up = 0;
11454 rc_cur->deadline = 0;
11455 }
11456 if (rack->r_ctl.client_suggested_maxseg != 0) {
11457 /*
11458 * We need to reset the max pace segs if we have a
11459 * client_suggested_maxseg.
11460 */
11461 rack_set_pace_segments(tp, rack, __LINE__, NULL);
11462 }
11463 if (orig_ent) {
11464 orig_ent->sent_at_ls = rack->rc_tp->t_sndbytes;
11465 orig_ent->rxt_at_ls = rack->rc_tp->t_snd_rxt_bytes;
11466 orig_ent->flags |= TCP_TRK_TRACK_FLG_LSND;
11467 }
11468 rack_log_hybrid(rack, seq, rc_cur, HYBRID_LOG_RULES_APP, __LINE__, 0);
11469 /* Remember it for next time and for CU mode */
11470 rack->r_ctl.rc_last_sft = rc_cur;
11471 rack->r_ctl.last_tm_mark = rc_cur->timestamp;
11472 }
11473 #endif
11474
11475 static void
rack_chk_req_and_hybrid_on_out(struct tcp_rack * rack,tcp_seq seq,uint32_t len,uint64_t cts)11476 rack_chk_req_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts)
11477 {
11478 #ifdef TCP_REQUEST_TRK
11479 struct tcp_sendfile_track *ent;
11480
11481 ent = rack->r_ctl.rc_last_sft;
11482 if ((ent == NULL) ||
11483 (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) ||
11484 (SEQ_GEQ(seq, ent->end_seq))) {
11485 /* Time to update the track. */
11486 rack_set_dgp_hybrid_mode(rack, seq, len, cts);
11487 ent = rack->r_ctl.rc_last_sft;
11488 }
11489 /* Out of all */
11490 if (ent == NULL) {
11491 return;
11492 }
11493 if (SEQ_LT(ent->end_seq, (seq + len))) {
11494 /*
11495 * This is the case where our end_seq guess
11496 * was wrong. This is usually due to TLS having
11497 * more bytes then our guess. It could also be the
11498 * case that the client sent in two requests closely
11499 * and the SB is full of both so we are sending part
11500 * of each (end|beg). In such a case lets move this
11501 * guys end to match the end of this send. That
11502 * way it will complete when all of it is acked.
11503 */
11504 ent->end_seq = (seq + len);
11505 if (rack->rc_hybrid_mode)
11506 rack_log_hybrid_bw(rack, seq, len, 0, 0, HYBRID_LOG_EXTEND, 0, ent, __LINE__);
11507 }
11508 /* Now validate we have set the send time of this one */
11509 if ((ent->flags & TCP_TRK_TRACK_FLG_FSND) == 0) {
11510 ent->flags |= TCP_TRK_TRACK_FLG_FSND;
11511 ent->first_send = cts;
11512 ent->sent_at_fs = rack->rc_tp->t_sndbytes;
11513 ent->rxt_at_fs = rack->rc_tp->t_snd_rxt_bytes;
11514 }
11515 #endif
11516 }
11517
11518 static void
rack_gain_for_fastoutput(struct tcp_rack * rack,struct tcpcb * tp,struct socket * so,uint32_t acked_amount)11519 rack_gain_for_fastoutput(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t acked_amount)
11520 {
11521 /*
11522 * The fast output path is enabled and we
11523 * have moved the cumack forward. Lets see if
11524 * we can expand forward the fast path length by
11525 * that amount. What we would ideally like to
11526 * do is increase the number of bytes in the
11527 * fast path block (left_to_send) by the
11528 * acked amount. However we have to gate that
11529 * by two factors:
11530 * 1) The amount outstanding and the rwnd of the peer
11531 * (i.e. we don't want to exceed the rwnd of the peer).
11532 * <and>
11533 * 2) The amount of data left in the socket buffer (i.e.
11534 * we can't send beyond what is in the buffer).
11535 *
11536 * Note that this does not take into account any increase
11537 * in the cwnd. We will only extend the fast path by
11538 * what was acked.
11539 */
11540 uint32_t new_total, gating_val;
11541
11542 new_total = acked_amount + rack->r_ctl.fsb.left_to_send;
11543 gating_val = min((sbavail(&so->so_snd) - (tp->snd_max - tp->snd_una)),
11544 (tp->snd_wnd - (tp->snd_max - tp->snd_una)));
11545 if (new_total <= gating_val) {
11546 /* We can increase left_to_send by the acked amount */
11547 counter_u64_add(rack_extended_rfo, 1);
11548 rack->r_ctl.fsb.left_to_send = new_total;
11549 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(&rack->rc_inp->inp_socket->so_snd) - (tp->snd_max - tp->snd_una))),
11550 ("rack:%p left_to_send:%u sbavail:%u out:%u",
11551 rack, rack->r_ctl.fsb.left_to_send,
11552 sbavail(&rack->rc_inp->inp_socket->so_snd),
11553 (tp->snd_max - tp->snd_una)));
11554
11555 }
11556 }
11557
11558 static void
rack_adjust_sendmap_head(struct tcp_rack * rack,struct sockbuf * sb)11559 rack_adjust_sendmap_head(struct tcp_rack *rack, struct sockbuf *sb)
11560 {
11561 /*
11562 * Here any sendmap entry that points to the
11563 * beginning mbuf must be adjusted to the correct
11564 * offset. This must be called with:
11565 * 1) The socket buffer locked
11566 * 2) snd_una adjusted to its new position.
11567 *
11568 * Note that (2) implies rack_ack_received has also
11569 * been called and all the sbcut's have been done.
11570 *
11571 * We grab the first mbuf in the socket buffer and
11572 * then go through the front of the sendmap, recalculating
11573 * the stored offset for any sendmap entry that has
11574 * that mbuf. We must use the sb functions to do this
11575 * since its possible an add was done has well as
11576 * the subtraction we may have just completed. This should
11577 * not be a penalty though, since we just referenced the sb
11578 * to go in and trim off the mbufs that we freed (of course
11579 * there will be a penalty for the sendmap references though).
11580 *
11581 * Note also with INVARIANT on, we validate with a KASSERT
11582 * that the first sendmap entry has a soff of 0.
11583 *
11584 */
11585 struct mbuf *m;
11586 struct rack_sendmap *rsm;
11587 tcp_seq snd_una;
11588 #ifdef INVARIANTS
11589 int first_processed = 0;
11590 #endif
11591
11592 snd_una = rack->rc_tp->snd_una;
11593 SOCKBUF_LOCK_ASSERT(sb);
11594 m = sb->sb_mb;
11595 rsm = tqhash_min(rack->r_ctl.tqh);
11596 if ((rsm == NULL) || (m == NULL)) {
11597 /* Nothing outstanding */
11598 return;
11599 }
11600 /* The very first RSM's mbuf must point to the head mbuf in the sb */
11601 KASSERT((rsm->m == m),
11602 ("Rack:%p sb:%p rsm:%p -- first rsm mbuf not aligned to sb",
11603 rack, sb, rsm));
11604 while (rsm->m && (rsm->m == m)) {
11605 /* one to adjust */
11606 #ifdef INVARIANTS
11607 struct mbuf *tm;
11608 uint32_t soff;
11609
11610 tm = sbsndmbuf(sb, (rsm->r_start - snd_una), &soff);
11611 if ((rsm->orig_m_len != m->m_len) ||
11612 (rsm->orig_t_space != M_TRAILINGROOM(m))){
11613 rack_adjust_orig_mlen(rsm);
11614 }
11615 if (first_processed == 0) {
11616 KASSERT((rsm->soff == 0),
11617 ("Rack:%p rsm:%p -- rsm at head but soff not zero",
11618 rack, rsm));
11619 first_processed = 1;
11620 }
11621 if ((rsm->soff != soff) || (rsm->m != tm)) {
11622 /*
11623 * This is not a fatal error, we anticipate it
11624 * might happen (the else code), so we count it here
11625 * so that under invariant we can see that it really
11626 * does happen.
11627 */
11628 counter_u64_add(rack_adjust_map_bw, 1);
11629 }
11630 rsm->m = tm;
11631 rsm->soff = soff;
11632 if (tm) {
11633 rsm->orig_m_len = rsm->m->m_len;
11634 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
11635 } else {
11636 rsm->orig_m_len = 0;
11637 rsm->orig_t_space = 0;
11638 }
11639 #else
11640 rsm->m = sbsndmbuf(sb, (rsm->r_start - snd_una), &rsm->soff);
11641 if (rsm->m) {
11642 rsm->orig_m_len = rsm->m->m_len;
11643 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
11644 } else {
11645 rsm->orig_m_len = 0;
11646 rsm->orig_t_space = 0;
11647 }
11648 #endif
11649 rsm = tqhash_next(rack->r_ctl.tqh, rsm);
11650 if (rsm == NULL)
11651 break;
11652 }
11653 }
11654
11655 #ifdef TCP_REQUEST_TRK
11656 static inline void
rack_req_check_for_comp(struct tcp_rack * rack,tcp_seq th_ack)11657 rack_req_check_for_comp(struct tcp_rack *rack, tcp_seq th_ack)
11658 {
11659 struct tcp_sendfile_track *ent;
11660 int i;
11661
11662 if ((rack->rc_hybrid_mode == 0) &&
11663 (tcp_bblogging_point_on(rack->rc_tp, TCP_BBPOINT_REQ_LEVEL_LOGGING) == 0)) {
11664 /*
11665 * Just do normal completions hybrid pacing is not on
11666 * and CLDL is off as well.
11667 */
11668 tcp_req_check_for_comp(rack->rc_tp, th_ack);
11669 return;
11670 }
11671 /*
11672 * Originally I was just going to find the th_ack associated
11673 * with an entry. But then I realized a large strech ack could
11674 * in theory ack two or more requests at once. So instead we
11675 * need to find all entries that are completed by th_ack not
11676 * just a single entry and do our logging.
11677 */
11678 ent = tcp_req_find_a_req_that_is_completed_by(rack->rc_tp, th_ack, &i);
11679 while (ent != NULL) {
11680 /*
11681 * We may be doing hybrid pacing or CLDL and need more details possibly
11682 * so we do it manually instead of calling
11683 * tcp_req_check_for_comp()
11684 */
11685 uint64_t laa, tim, data, cbw, ftim;
11686
11687 /* Ok this ack frees it */
11688 rack_log_hybrid(rack, th_ack,
11689 ent, HYBRID_LOG_REQ_COMP, __LINE__, 0);
11690 rack_log_hybrid_sends(rack, ent, __LINE__);
11691 /* calculate the time based on the ack arrival */
11692 data = ent->end - ent->start;
11693 laa = tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time);
11694 if (ent->flags & TCP_TRK_TRACK_FLG_FSND) {
11695 if (ent->first_send > ent->localtime)
11696 ftim = ent->first_send;
11697 else
11698 ftim = ent->localtime;
11699 } else {
11700 /* TSNH */
11701 ftim = ent->localtime;
11702 }
11703 if (laa > ent->localtime)
11704 tim = laa - ftim;
11705 else
11706 tim = 0;
11707 cbw = data * HPTS_USEC_IN_SEC;
11708 if (tim > 0)
11709 cbw /= tim;
11710 else
11711 cbw = 0;
11712 rack_log_hybrid_bw(rack, th_ack, cbw, tim, data, HYBRID_LOG_BW_MEASURE, 0, ent, __LINE__);
11713 /*
11714 * Check to see if we are freeing what we are pointing to send wise
11715 * if so be sure to NULL the pointer so we know we are no longer
11716 * set to anything.
11717 */
11718 if (ent == rack->r_ctl.rc_last_sft) {
11719 rack->r_ctl.rc_last_sft = NULL;
11720 if (rack->rc_hybrid_mode) {
11721 rack->rc_catch_up = 0;
11722 if (rack->cspr_is_fcc == 0)
11723 rack->r_ctl.bw_rate_cap = 0;
11724 else
11725 rack->r_ctl.fillcw_cap = rack_fillcw_bw_cap;
11726 rack->r_ctl.client_suggested_maxseg = 0;
11727 }
11728 }
11729 /* Generate the log that the tcp_netflix call would have */
11730 tcp_req_log_req_info(rack->rc_tp, ent,
11731 i, TCP_TRK_REQ_LOG_FREED, 0, 0);
11732 /* Free it and see if there is another one */
11733 tcp_req_free_a_slot(rack->rc_tp, ent);
11734 ent = tcp_req_find_a_req_that_is_completed_by(rack->rc_tp, th_ack, &i);
11735 }
11736 }
11737 #endif
11738
11739
11740 /*
11741 * Return value of 1, we do not need to call rack_process_data().
11742 * return value of 0, rack_process_data can be called.
11743 * For ret_val if its 0 the TCP is locked, if its non-zero
11744 * its unlocked and probably unsafe to touch the TCB.
11745 */
11746 static int
rack_process_ack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,uint32_t tiwin,int32_t tlen,int32_t * ofia,int32_t thflags,int32_t * ret_val,int32_t orig_tlen)11747 rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so,
11748 struct tcpcb *tp, struct tcpopt *to,
11749 uint32_t tiwin, int32_t tlen,
11750 int32_t * ofia, int32_t thflags, int32_t *ret_val, int32_t orig_tlen)
11751 {
11752 int32_t ourfinisacked = 0;
11753 int32_t nsegs, acked_amount;
11754 int32_t acked;
11755 struct mbuf *mfree;
11756 struct tcp_rack *rack;
11757 int32_t under_pacing = 0;
11758 int32_t post_recovery = 0;
11759 uint32_t p_cwnd;
11760
11761 INP_WLOCK_ASSERT(tptoinpcb(tp));
11762
11763 rack = (struct tcp_rack *)tp->t_fb_ptr;
11764 if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) {
11765 /* Checking SEG.ACK against ISS is definitely redundant. */
11766 tp->t_flags2 |= TF2_NO_ISS_CHECK;
11767 }
11768 if (!V_tcp_insecure_ack) {
11769 tcp_seq seq_min;
11770 bool ghost_ack_check;
11771
11772 if (tp->t_flags2 & TF2_NO_ISS_CHECK) {
11773 /* Check for too old ACKs (RFC 5961, Section 5.2). */
11774 seq_min = tp->snd_una - tp->max_sndwnd;
11775 ghost_ack_check = false;
11776 } else {
11777 if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) {
11778 /* Checking for ghost ACKs is stricter. */
11779 seq_min = tp->iss + 1;
11780 ghost_ack_check = true;
11781 } else {
11782 /*
11783 * Checking for too old ACKs (RFC 5961,
11784 * Section 5.2) is stricter.
11785 */
11786 seq_min = tp->snd_una - tp->max_sndwnd;
11787 ghost_ack_check = false;
11788 }
11789 }
11790 if (SEQ_LT(th->th_ack, seq_min)) {
11791 if (ghost_ack_check)
11792 TCPSTAT_INC(tcps_rcvghostack);
11793 else
11794 TCPSTAT_INC(tcps_rcvacktooold);
11795 /* Send challenge ACK. */
11796 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
11797 rack->r_wanted_output = 1;
11798 return (1);
11799 }
11800 }
11801 if (SEQ_GT(th->th_ack, tp->snd_max)) {
11802 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
11803 rack->r_wanted_output = 1;
11804 return (1);
11805 }
11806 if (rack->gp_ready &&
11807 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
11808 under_pacing = 1;
11809 }
11810 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
11811 int in_rec, dup_ack_struck = 0;
11812 int dsack_seen = 0, sacks_seen = 0;
11813
11814 in_rec = IN_FASTRECOVERY(tp->t_flags);
11815 if (rack->rc_in_persist) {
11816 tp->t_rxtshift = 0;
11817 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
11818 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
11819 }
11820
11821 if ((th->th_ack == tp->snd_una) &&
11822 (tiwin == tp->snd_wnd) &&
11823 (orig_tlen == 0) &&
11824 ((to->to_flags & TOF_SACK) == 0)) {
11825 rack_strike_dupack(rack, th->th_ack);
11826 dup_ack_struck = 1;
11827 }
11828 rack_log_ack(tp, to, th, ((in_rec == 0) && IN_FASTRECOVERY(tp->t_flags)),
11829 dup_ack_struck, &dsack_seen, &sacks_seen);
11830
11831 }
11832 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
11833 /*
11834 * Old ack, behind (or duplicate to) the last one rcv'd
11835 * Note: We mark reordering is occuring if its
11836 * less than and we have not closed our window.
11837 */
11838 if (SEQ_LT(th->th_ack, tp->snd_una) && (sbspace(&so->so_rcv) > ctf_fixed_maxseg(tp))) {
11839 rack->r_ctl.rc_reorder_ts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
11840 if (rack->r_ctl.rc_reorder_ts == 0)
11841 rack->r_ctl.rc_reorder_ts = 1;
11842 }
11843 return (0);
11844 }
11845 /*
11846 * If we reach this point, ACK is not a duplicate, i.e., it ACKs
11847 * something we sent.
11848 */
11849 if (tp->t_flags & TF_NEEDSYN) {
11850 /*
11851 * T/TCP: Connection was half-synchronized, and our SYN has
11852 * been ACK'd (so connection is now fully synchronized). Go
11853 * to non-starred state, increment snd_una for ACK of SYN,
11854 * and check if we can do window scaling.
11855 */
11856 tp->t_flags &= ~TF_NEEDSYN;
11857 tp->snd_una++;
11858 /* Do window scaling? */
11859 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
11860 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
11861 tp->rcv_scale = tp->request_r_scale;
11862 /* Send window already scaled. */
11863 }
11864 }
11865 nsegs = max(1, m->m_pkthdr.lro_nsegs);
11866
11867 acked = BYTES_THIS_ACK(tp, th);
11868 if (acked) {
11869 /*
11870 * Any time we move the cum-ack forward clear
11871 * keep-alive tied probe-not-answered. The
11872 * persists clears its own on entry.
11873 */
11874 rack->probe_not_answered = 0;
11875 }
11876 KMOD_TCPSTAT_ADD(tcps_rcvackpack, nsegs);
11877 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
11878 /*
11879 * If we just performed our first retransmit, and the ACK arrives
11880 * within our recovery window, then it was a mistake to do the
11881 * retransmit in the first place. Recover our original cwnd and
11882 * ssthresh, and proceed to transmit where we left off.
11883 */
11884 if ((tp->t_flags & TF_PREVVALID) &&
11885 ((tp->t_flags & TF_RCVD_TSTMP) == 0)) {
11886 tp->t_flags &= ~TF_PREVVALID;
11887 if (tp->t_rxtshift == 1 &&
11888 (int)(ticks - tp->t_badrxtwin) < 0)
11889 rack_cong_signal(tp, CC_RTO_ERR, th->th_ack, __LINE__);
11890 }
11891 if (acked) {
11892 /* assure we are not backed off */
11893 tp->t_rxtshift = 0;
11894 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
11895 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
11896 rack->rc_tlp_in_progress = 0;
11897 rack->r_ctl.rc_tlp_cnt_out = 0;
11898 /*
11899 * If it is the RXT timer we want to
11900 * stop it, so we can restart a TLP.
11901 */
11902 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
11903 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
11904 #ifdef TCP_REQUEST_TRK
11905 rack_req_check_for_comp(rack, th->th_ack);
11906 #endif
11907 }
11908 /*
11909 * If we have a timestamp reply, update smoothed round trip time. If
11910 * no timestamp is present but transmit timer is running and timed
11911 * sequence number was acked, update smoothed round trip time. Since
11912 * we now have an rtt measurement, cancel the timer backoff (cf.,
11913 * Phil Karn's retransmit alg.). Recompute the initial retransmit
11914 * timer.
11915 *
11916 * Some boxes send broken timestamp replies during the SYN+ACK
11917 * phase, ignore timestamps of 0 or we could calculate a huge RTT
11918 * and blow up the retransmit timer.
11919 */
11920 /*
11921 * If all outstanding data is acked, stop retransmit timer and
11922 * remember to restart (more output or persist). If there is more
11923 * data to be acked, restart retransmit timer, using current
11924 * (possibly backed-off) value.
11925 */
11926 if (acked == 0) {
11927 if (ofia)
11928 *ofia = ourfinisacked;
11929 return (0);
11930 }
11931 if (IN_RECOVERY(tp->t_flags)) {
11932 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
11933 (SEQ_LT(th->th_ack, tp->snd_max))) {
11934 tcp_rack_partialack(tp);
11935 } else {
11936 rack_post_recovery(tp, th->th_ack);
11937 post_recovery = 1;
11938 /*
11939 * Grab the segsiz, multiply by 2 and add the snd_cwnd
11940 * that is the max the CC should add if we are exiting
11941 * recovery and doing a late add.
11942 */
11943 p_cwnd = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
11944 p_cwnd <<= 1;
11945 p_cwnd += tp->snd_cwnd;
11946 }
11947 } else if ((rack->rto_from_rec == 1) &&
11948 SEQ_GEQ(th->th_ack, tp->snd_recover)) {
11949 /*
11950 * We were in recovery, hit a rxt timeout
11951 * and never re-entered recovery. The timeout(s)
11952 * made up all the lost data. In such a case
11953 * we need to clear the rto_from_rec flag.
11954 */
11955 rack->rto_from_rec = 0;
11956 }
11957 /*
11958 * Let the congestion control algorithm update congestion control
11959 * related information. This typically means increasing the
11960 * congestion window.
11961 */
11962 rack_ack_received(tp, rack, th->th_ack, nsegs, CC_ACK, post_recovery);
11963 if (post_recovery &&
11964 (tp->snd_cwnd > p_cwnd)) {
11965 /* Must be non-newreno (cubic) getting too ahead of itself */
11966 tp->snd_cwnd = p_cwnd;
11967 }
11968 SOCK_SENDBUF_LOCK(so);
11969 acked_amount = min(acked, (int)sbavail(&so->so_snd));
11970 tp->snd_wnd -= acked_amount;
11971 mfree = sbcut_locked(&so->so_snd, acked_amount);
11972 if ((sbused(&so->so_snd) == 0) &&
11973 (acked > acked_amount) &&
11974 (tp->t_state >= TCPS_FIN_WAIT_1) &&
11975 (tp->t_flags & TF_SENTFIN)) {
11976 /*
11977 * We must be sure our fin
11978 * was sent and acked (we can be
11979 * in FIN_WAIT_1 without having
11980 * sent the fin).
11981 */
11982 ourfinisacked = 1;
11983 }
11984 tp->snd_una = th->th_ack;
11985 /* wakeups? */
11986 if (acked_amount && sbavail(&so->so_snd))
11987 rack_adjust_sendmap_head(rack, &so->so_snd);
11988 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2);
11989 /* NB: sowwakeup_locked() does an implicit unlock. */
11990 sowwakeup_locked(so);
11991 m_freem(mfree);
11992 if (SEQ_GT(tp->snd_una, tp->snd_recover))
11993 tp->snd_recover = tp->snd_una;
11994
11995 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
11996 tp->snd_nxt = tp->snd_max;
11997 }
11998 if (under_pacing &&
11999 (rack->use_fixed_rate == 0) &&
12000 (rack->in_probe_rtt == 0) &&
12001 rack->rc_gp_dyn_mul &&
12002 rack->rc_always_pace) {
12003 /* Check if we are dragging bottom */
12004 rack_check_bottom_drag(tp, rack, so);
12005 }
12006 if (tp->snd_una == tp->snd_max) {
12007 /* Nothing left outstanding */
12008 tp->t_flags &= ~TF_PREVVALID;
12009 if (rack->r_ctl.rc_went_idle_time == 0)
12010 rack->r_ctl.rc_went_idle_time = 1;
12011 rack->r_ctl.retran_during_recovery = 0;
12012 rack->r_ctl.dsack_byte_cnt = 0;
12013 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
12014 if (sbavail(&tptosocket(tp)->so_snd) == 0)
12015 tp->t_acktime = 0;
12016 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
12017 rack->rc_suspicious = 0;
12018 /* Set need output so persist might get set */
12019 rack->r_wanted_output = 1;
12020 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
12021 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
12022 (sbavail(&so->so_snd) == 0) &&
12023 (tp->t_flags2 & TF2_DROP_AF_DATA)) {
12024 /*
12025 * The socket was gone and the
12026 * peer sent data (now or in the past), time to
12027 * reset him.
12028 */
12029 *ret_val = 1;
12030 /* tcp_close will kill the inp pre-log the Reset */
12031 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
12032 tp = tcp_close(tp);
12033 ctf_do_dropwithreset(m, tp, th, tlen);
12034 return (1);
12035 }
12036 }
12037 if (ofia)
12038 *ofia = ourfinisacked;
12039 return (0);
12040 }
12041
12042
12043 static void
rack_log_collapse(struct tcp_rack * rack,uint32_t cnt,uint32_t split,uint32_t out,int line,int dir,uint32_t flags,struct rack_sendmap * rsm)12044 rack_log_collapse(struct tcp_rack *rack, uint32_t cnt, uint32_t split, uint32_t out, int line,
12045 int dir, uint32_t flags, struct rack_sendmap *rsm)
12046 {
12047 if (tcp_bblogging_on(rack->rc_tp)) {
12048 union tcp_log_stackspecific log;
12049 struct timeval tv;
12050
12051 memset(&log, 0, sizeof(log));
12052 log.u_bbr.flex1 = cnt;
12053 log.u_bbr.flex2 = split;
12054 log.u_bbr.flex3 = out;
12055 log.u_bbr.flex4 = line;
12056 log.u_bbr.flex5 = rack->r_must_retran;
12057 log.u_bbr.flex6 = flags;
12058 log.u_bbr.flex7 = rack->rc_has_collapsed;
12059 log.u_bbr.flex8 = dir; /*
12060 * 1 is collapsed, 0 is uncollapsed,
12061 * 2 is log of a rsm being marked, 3 is a split.
12062 */
12063 if (rsm == NULL)
12064 log.u_bbr.rttProp = 0;
12065 else
12066 log.u_bbr.rttProp = (uintptr_t)rsm;
12067 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
12068 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
12069 TCP_LOG_EVENTP(rack->rc_tp, NULL,
12070 &rack->rc_inp->inp_socket->so_rcv,
12071 &rack->rc_inp->inp_socket->so_snd,
12072 TCP_RACK_LOG_COLLAPSE, 0,
12073 0, &log, false, &tv);
12074 }
12075 }
12076
12077 static void
rack_collapsed_window(struct tcp_rack * rack,uint32_t out,tcp_seq th_ack,int line)12078 rack_collapsed_window(struct tcp_rack *rack, uint32_t out, tcp_seq th_ack, int line)
12079 {
12080 /*
12081 * Here all we do is mark the collapsed point and set the flag.
12082 * This may happen again and again, but there is no
12083 * sense splitting our map until we know where the
12084 * peer finally lands in the collapse.
12085 */
12086 tcp_trace_point(rack->rc_tp, TCP_TP_COLLAPSED_WND);
12087 if ((rack->rc_has_collapsed == 0) ||
12088 (rack->r_ctl.last_collapse_point != (th_ack + rack->rc_tp->snd_wnd)))
12089 counter_u64_add(rack_collapsed_win_seen, 1);
12090 rack->r_ctl.last_collapse_point = th_ack + rack->rc_tp->snd_wnd;
12091 rack->r_ctl.high_collapse_point = rack->rc_tp->snd_max;
12092 rack->rc_has_collapsed = 1;
12093 rack->r_collapse_point_valid = 1;
12094 rack_log_collapse(rack, 0, th_ack, rack->r_ctl.last_collapse_point, line, 1, 0, NULL);
12095 }
12096
12097 static void
rack_un_collapse_window(struct tcp_rack * rack,int line)12098 rack_un_collapse_window(struct tcp_rack *rack, int line)
12099 {
12100 struct rack_sendmap *nrsm, *rsm;
12101 int cnt = 0, split = 0;
12102 int insret __diagused;
12103
12104
12105 tcp_trace_point(rack->rc_tp, TCP_TP_COLLAPSED_WND);
12106 rack->rc_has_collapsed = 0;
12107 rsm = tqhash_find(rack->r_ctl.tqh, rack->r_ctl.last_collapse_point);
12108 if (rsm == NULL) {
12109 /* Nothing to do maybe the peer ack'ed it all */
12110 rack_log_collapse(rack, 0, 0, ctf_outstanding(rack->rc_tp), line, 0, 0, NULL);
12111 return;
12112 }
12113 /* Now do we need to split this one? */
12114 if (SEQ_GT(rack->r_ctl.last_collapse_point, rsm->r_start)) {
12115 rack_log_collapse(rack, rsm->r_start, rsm->r_end,
12116 rack->r_ctl.last_collapse_point, line, 3, rsm->r_flags, rsm);
12117 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT);
12118 if (nrsm == NULL) {
12119 /* We can't get a rsm, mark all? */
12120 nrsm = rsm;
12121 goto no_split;
12122 }
12123 /* Clone it */
12124 split = 1;
12125 rack_clone_rsm(rack, nrsm, rsm, rack->r_ctl.last_collapse_point);
12126 #ifndef INVARIANTS
12127 (void)tqhash_insert(rack->r_ctl.tqh, nrsm);
12128 #else
12129 if ((insret = tqhash_insert(rack->r_ctl.tqh, nrsm)) != 0) {
12130 panic("Insert in tailq_hash of %p fails ret:%d rack:%p rsm:%p",
12131 nrsm, insret, rack, rsm);
12132 }
12133 #endif
12134 rack_log_map_chg(rack->rc_tp, rack, NULL, rsm, nrsm, MAP_SPLIT,
12135 rack->r_ctl.last_collapse_point, __LINE__);
12136 if (rsm->r_in_tmap) {
12137 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
12138 nrsm->r_in_tmap = 1;
12139 }
12140 /*
12141 * Set in the new RSM as the
12142 * collapsed starting point
12143 */
12144 rsm = nrsm;
12145 }
12146
12147 no_split:
12148 TQHASH_FOREACH_FROM(nrsm, rack->r_ctl.tqh, rsm) {
12149 cnt++;
12150 nrsm->r_flags |= RACK_RWND_COLLAPSED;
12151 rack_log_collapse(rack, nrsm->r_start, nrsm->r_end, 0, line, 4, nrsm->r_flags, nrsm);
12152 cnt++;
12153 }
12154 if (cnt) {
12155 counter_u64_add(rack_collapsed_win, 1);
12156 }
12157 rack_log_collapse(rack, cnt, split, ctf_outstanding(rack->rc_tp), line, 0, 0, NULL);
12158 }
12159
12160 static void
rack_handle_delayed_ack(struct tcpcb * tp,struct tcp_rack * rack,int32_t tlen,int32_t tfo_syn)12161 rack_handle_delayed_ack(struct tcpcb *tp, struct tcp_rack *rack,
12162 int32_t tlen, int32_t tfo_syn)
12163 {
12164 if (DELAY_ACK(tp, tlen) || tfo_syn) {
12165 rack_timer_cancel(tp, rack,
12166 rack->r_ctl.rc_rcvtime, __LINE__);
12167 tp->t_flags |= TF_DELACK;
12168 } else {
12169 rack->r_wanted_output = 1;
12170 tp->t_flags |= TF_ACKNOW;
12171 }
12172 }
12173
12174 static void
rack_validate_fo_sendwin_up(struct tcpcb * tp,struct tcp_rack * rack)12175 rack_validate_fo_sendwin_up(struct tcpcb *tp, struct tcp_rack *rack)
12176 {
12177 /*
12178 * If fast output is in progress, lets validate that
12179 * the new window did not shrink on us and make it
12180 * so fast output should end.
12181 */
12182 if (rack->r_fast_output) {
12183 uint32_t out;
12184
12185 /*
12186 * Calculate what we will send if left as is
12187 * and compare that to our send window.
12188 */
12189 out = ctf_outstanding(tp);
12190 if ((out + rack->r_ctl.fsb.left_to_send) > tp->snd_wnd) {
12191 /* ok we have an issue */
12192 if (out >= tp->snd_wnd) {
12193 /* Turn off fast output the window is met or collapsed */
12194 rack->r_fast_output = 0;
12195 } else {
12196 /* we have some room left */
12197 rack->r_ctl.fsb.left_to_send = tp->snd_wnd - out;
12198 if (rack->r_ctl.fsb.left_to_send < ctf_fixed_maxseg(tp)) {
12199 /* If not at least 1 full segment never mind */
12200 rack->r_fast_output = 0;
12201 }
12202 }
12203 }
12204 }
12205 }
12206
12207 /*
12208 * Return value of 1, the TCB is unlocked and most
12209 * likely gone, return value of 0, the TCP is still
12210 * locked.
12211 */
12212 static int
rack_process_data(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt)12213 rack_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so,
12214 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
12215 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
12216 {
12217 /*
12218 * Update window information. Don't look at window if no ACK: TAC's
12219 * send garbage on first SYN.
12220 */
12221 int32_t nsegs;
12222 int32_t tfo_syn;
12223 struct tcp_rack *rack;
12224
12225 INP_WLOCK_ASSERT(tptoinpcb(tp));
12226
12227 rack = (struct tcp_rack *)tp->t_fb_ptr;
12228 nsegs = max(1, m->m_pkthdr.lro_nsegs);
12229 if ((thflags & TH_ACK) &&
12230 (SEQ_LT(tp->snd_wl1, th->th_seq) ||
12231 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
12232 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
12233 /* keep track of pure window updates */
12234 if (tlen == 0 &&
12235 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
12236 KMOD_TCPSTAT_INC(tcps_rcvwinupd);
12237 tp->snd_wnd = tiwin;
12238 rack_validate_fo_sendwin_up(tp, rack);
12239 tp->snd_wl1 = th->th_seq;
12240 tp->snd_wl2 = th->th_ack;
12241 if (tp->snd_wnd > tp->max_sndwnd)
12242 tp->max_sndwnd = tp->snd_wnd;
12243 rack->r_wanted_output = 1;
12244 } else if (thflags & TH_ACK) {
12245 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
12246 tp->snd_wnd = tiwin;
12247 rack_validate_fo_sendwin_up(tp, rack);
12248 tp->snd_wl1 = th->th_seq;
12249 tp->snd_wl2 = th->th_ack;
12250 }
12251 }
12252 if (tp->snd_wnd < ctf_outstanding(tp))
12253 /* The peer collapsed the window */
12254 rack_collapsed_window(rack, ctf_outstanding(tp), th->th_ack, __LINE__);
12255 else if (rack->rc_has_collapsed)
12256 rack_un_collapse_window(rack, __LINE__);
12257 if ((rack->r_collapse_point_valid) &&
12258 (SEQ_GT(th->th_ack, rack->r_ctl.high_collapse_point)))
12259 rack->r_collapse_point_valid = 0;
12260 /* Was persist timer active and now we have window space? */
12261 if ((rack->rc_in_persist != 0) &&
12262 (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2),
12263 rack->r_ctl.rc_pace_min_segs))) {
12264 rack_exit_persist(tp, rack, rack->r_ctl.rc_rcvtime);
12265 tp->snd_nxt = tp->snd_max;
12266 /* Make sure we output to start the timer */
12267 rack->r_wanted_output = 1;
12268 }
12269 /* Do we enter persists? */
12270 if ((rack->rc_in_persist == 0) &&
12271 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) &&
12272 TCPS_HAVEESTABLISHED(tp->t_state) &&
12273 ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) &&
12274 sbavail(&tptosocket(tp)->so_snd) &&
12275 (sbavail(&tptosocket(tp)->so_snd) > tp->snd_wnd)) {
12276 /*
12277 * Here the rwnd is less than
12278 * the pacing size, we are established,
12279 * nothing is outstanding, and there is
12280 * data to send. Enter persists.
12281 */
12282 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime, tp->snd_una);
12283 }
12284 if (tp->t_flags2 & TF2_DROP_AF_DATA) {
12285 m_freem(m);
12286 return (0);
12287 }
12288 /*
12289 * don't process the URG bit, ignore them drag
12290 * along the up.
12291 */
12292 tp->rcv_up = tp->rcv_nxt;
12293
12294 /*
12295 * Process the segment text, merging it into the TCP sequencing
12296 * queue, and arranging for acknowledgment of receipt if necessary.
12297 * This process logically involves adjusting tp->rcv_wnd as data is
12298 * presented to the user (this happens in tcp_usrreq.c, case
12299 * PRU_RCVD). If a FIN has already been received on this connection
12300 * then we just ignore the text.
12301 */
12302 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
12303 (tp->t_flags & TF_FASTOPEN));
12304 if ((tlen || (thflags & TH_FIN) || (tfo_syn && tlen > 0)) &&
12305 TCPS_HAVERCVDFIN(tp->t_state) == 0) {
12306 tcp_seq save_start = th->th_seq;
12307 tcp_seq save_rnxt = tp->rcv_nxt;
12308 int save_tlen = tlen;
12309
12310 m_adj(m, drop_hdrlen); /* delayed header drop */
12311 /*
12312 * Insert segment which includes th into TCP reassembly
12313 * queue with control block tp. Set thflags to whether
12314 * reassembly now includes a segment with FIN. This handles
12315 * the common case inline (segment is the next to be
12316 * received on an established connection, and the queue is
12317 * empty), avoiding linkage into and removal from the queue
12318 * and repetition of various conversions. Set DELACK for
12319 * segments received in order, but ack immediately when
12320 * segments are out of order (so fast retransmit can work).
12321 */
12322 if (th->th_seq == tp->rcv_nxt &&
12323 SEGQ_EMPTY(tp) &&
12324 (TCPS_HAVEESTABLISHED(tp->t_state) ||
12325 tfo_syn)) {
12326 #ifdef NETFLIX_SB_LIMITS
12327 u_int mcnt, appended;
12328
12329 if (so->so_rcv.sb_shlim) {
12330 mcnt = m_memcnt(m);
12331 appended = 0;
12332 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
12333 CFO_NOSLEEP, NULL) == false) {
12334 counter_u64_add(tcp_sb_shlim_fails, 1);
12335 m_freem(m);
12336 return (0);
12337 }
12338 }
12339 #endif
12340 rack_handle_delayed_ack(tp, rack, tlen, tfo_syn);
12341 tp->rcv_nxt += tlen;
12342 if (tlen &&
12343 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
12344 (tp->t_fbyte_in == 0)) {
12345 tp->t_fbyte_in = ticks;
12346 if (tp->t_fbyte_in == 0)
12347 tp->t_fbyte_in = 1;
12348 if (tp->t_fbyte_out && tp->t_fbyte_in)
12349 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
12350 }
12351 thflags = tcp_get_flags(th) & TH_FIN;
12352 KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs);
12353 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
12354 SOCK_RECVBUF_LOCK(so);
12355 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
12356 m_freem(m);
12357 } else {
12358 int32_t newsize;
12359
12360 if (tlen > 0) {
12361 newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
12362 if (newsize)
12363 if (!sbreserve_locked(so, SO_RCV, newsize, NULL))
12364 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
12365 }
12366 #ifdef NETFLIX_SB_LIMITS
12367 appended =
12368 #endif
12369 sbappendstream_locked(&so->so_rcv, m, 0);
12370 }
12371 rack_log_wakeup(tp,rack, &so->so_rcv, tlen, 1);
12372 /* NB: sorwakeup_locked() does an implicit unlock. */
12373 sorwakeup_locked(so);
12374 #ifdef NETFLIX_SB_LIMITS
12375 if (so->so_rcv.sb_shlim && appended != mcnt)
12376 counter_fo_release(so->so_rcv.sb_shlim,
12377 mcnt - appended);
12378 #endif
12379 } else {
12380 /*
12381 * XXX: Due to the header drop above "th" is
12382 * theoretically invalid by now. Fortunately
12383 * m_adj() doesn't actually frees any mbufs when
12384 * trimming from the head.
12385 */
12386 tcp_seq temp = save_start;
12387
12388 thflags = tcp_reass(tp, th, &temp, &tlen, m);
12389 tp->t_flags |= TF_ACKNOW;
12390 if (tp->t_flags & TF_WAKESOR) {
12391 tp->t_flags &= ~TF_WAKESOR;
12392 /* NB: sorwakeup_locked() does an implicit unlock. */
12393 sorwakeup_locked(so);
12394 }
12395 }
12396 if ((tp->t_flags & TF_SACK_PERMIT) &&
12397 (save_tlen > 0) &&
12398 TCPS_HAVEESTABLISHED(tp->t_state)) {
12399 if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) {
12400 /*
12401 * DSACK actually handled in the fastpath
12402 * above.
12403 */
12404 tcp_update_sack_list(tp, save_start,
12405 save_start + save_tlen);
12406 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
12407 if ((tp->rcv_numsacks >= 1) &&
12408 (tp->sackblks[0].end == save_start)) {
12409 /*
12410 * Partial overlap, recorded at todrop
12411 * above.
12412 */
12413 tcp_update_sack_list(tp,
12414 tp->sackblks[0].start,
12415 tp->sackblks[0].end);
12416 } else {
12417 tcp_update_dsack_list(tp, save_start,
12418 save_start + save_tlen);
12419 }
12420 } else if (tlen >= save_tlen) {
12421 /* Update of sackblks. */
12422 tcp_update_dsack_list(tp, save_start,
12423 save_start + save_tlen);
12424 } else if (tlen > 0) {
12425 tcp_update_dsack_list(tp, save_start,
12426 save_start + tlen);
12427 }
12428 }
12429 } else {
12430 m_freem(m);
12431 thflags &= ~TH_FIN;
12432 }
12433
12434 /*
12435 * If FIN is received ACK the FIN and let the user know that the
12436 * connection is closing.
12437 */
12438 if (thflags & TH_FIN) {
12439 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
12440 /* The socket upcall is handled by socantrcvmore. */
12441 socantrcvmore(so);
12442 /*
12443 * If connection is half-synchronized (ie NEEDSYN
12444 * flag on) then delay ACK, so it may be piggybacked
12445 * when SYN is sent. Otherwise, since we received a
12446 * FIN then no more input can be expected, send ACK
12447 * now.
12448 */
12449 if (tp->t_flags & TF_NEEDSYN) {
12450 rack_timer_cancel(tp, rack,
12451 rack->r_ctl.rc_rcvtime, __LINE__);
12452 tp->t_flags |= TF_DELACK;
12453 } else {
12454 tp->t_flags |= TF_ACKNOW;
12455 }
12456 tp->rcv_nxt++;
12457 }
12458 switch (tp->t_state) {
12459 /*
12460 * In SYN_RECEIVED and ESTABLISHED STATES enter the
12461 * CLOSE_WAIT state.
12462 */
12463 case TCPS_SYN_RECEIVED:
12464 tp->t_starttime = ticks;
12465 /* FALLTHROUGH */
12466 case TCPS_ESTABLISHED:
12467 rack_timer_cancel(tp, rack,
12468 rack->r_ctl.rc_rcvtime, __LINE__);
12469 tcp_state_change(tp, TCPS_CLOSE_WAIT);
12470 break;
12471
12472 /*
12473 * If still in FIN_WAIT_1 STATE FIN has not been
12474 * acked so enter the CLOSING state.
12475 */
12476 case TCPS_FIN_WAIT_1:
12477 rack_timer_cancel(tp, rack,
12478 rack->r_ctl.rc_rcvtime, __LINE__);
12479 tcp_state_change(tp, TCPS_CLOSING);
12480 break;
12481
12482 /*
12483 * In FIN_WAIT_2 state enter the TIME_WAIT state,
12484 * starting the time-wait timer, turning off the
12485 * other standard timers.
12486 */
12487 case TCPS_FIN_WAIT_2:
12488 rack_timer_cancel(tp, rack,
12489 rack->r_ctl.rc_rcvtime, __LINE__);
12490 tcp_twstart(tp);
12491 return (1);
12492 }
12493 }
12494 /*
12495 * Return any desired output.
12496 */
12497 if ((tp->t_flags & TF_ACKNOW) ||
12498 (sbavail(&so->so_snd) > (tp->snd_max - tp->snd_una))) {
12499 rack->r_wanted_output = 1;
12500 }
12501 return (0);
12502 }
12503
12504 /*
12505 * Here nothing is really faster, its just that we
12506 * have broken out the fast-data path also just like
12507 * the fast-ack.
12508 */
12509 static int
rack_do_fastnewdata(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t nxt_pkt,uint8_t iptos)12510 rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so,
12511 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
12512 uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos)
12513 {
12514 int32_t nsegs;
12515 int32_t newsize = 0; /* automatic sockbuf scaling */
12516 struct tcp_rack *rack;
12517 #ifdef NETFLIX_SB_LIMITS
12518 u_int mcnt, appended;
12519 #endif
12520
12521 /*
12522 * If last ACK falls within this segment's sequence numbers, record
12523 * the timestamp. NOTE that the test is modified according to the
12524 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
12525 */
12526 if (__predict_false(th->th_seq != tp->rcv_nxt)) {
12527 return (0);
12528 }
12529 if (tiwin && tiwin != tp->snd_wnd) {
12530 return (0);
12531 }
12532 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
12533 return (0);
12534 }
12535 if (__predict_false((to->to_flags & TOF_TS) &&
12536 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
12537 return (0);
12538 }
12539 if (__predict_false((th->th_ack != tp->snd_una))) {
12540 return (0);
12541 }
12542 if (__predict_false(tlen > sbspace(&so->so_rcv))) {
12543 return (0);
12544 }
12545 if ((to->to_flags & TOF_TS) != 0 &&
12546 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
12547 tp->ts_recent_age = tcp_ts_getticks();
12548 tp->ts_recent = to->to_tsval;
12549 }
12550 rack = (struct tcp_rack *)tp->t_fb_ptr;
12551 /*
12552 * This is a pure, in-sequence data packet with nothing on the
12553 * reassembly queue and we have enough buffer space to take it.
12554 */
12555 nsegs = max(1, m->m_pkthdr.lro_nsegs);
12556
12557 #ifdef NETFLIX_SB_LIMITS
12558 if (so->so_rcv.sb_shlim) {
12559 mcnt = m_memcnt(m);
12560 appended = 0;
12561 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
12562 CFO_NOSLEEP, NULL) == false) {
12563 counter_u64_add(tcp_sb_shlim_fails, 1);
12564 m_freem(m);
12565 return (1);
12566 }
12567 }
12568 #endif
12569 /* Clean receiver SACK report if present */
12570 if (tp->rcv_numsacks)
12571 tcp_clean_sackreport(tp);
12572 KMOD_TCPSTAT_INC(tcps_preddat);
12573 tp->rcv_nxt += tlen;
12574 if (tlen &&
12575 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
12576 (tp->t_fbyte_in == 0)) {
12577 tp->t_fbyte_in = ticks;
12578 if (tp->t_fbyte_in == 0)
12579 tp->t_fbyte_in = 1;
12580 if (tp->t_fbyte_out && tp->t_fbyte_in)
12581 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
12582 }
12583 /*
12584 * Pull snd_wl1 up to prevent seq wrap relative to th_seq.
12585 */
12586 tp->snd_wl1 = th->th_seq;
12587 /*
12588 * Pull rcv_up up to prevent seq wrap relative to rcv_nxt.
12589 */
12590 tp->rcv_up = tp->rcv_nxt;
12591 KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs);
12592 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
12593 newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
12594
12595 /* Add data to socket buffer. */
12596 SOCK_RECVBUF_LOCK(so);
12597 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
12598 m_freem(m);
12599 } else {
12600 /*
12601 * Set new socket buffer size. Give up when limit is
12602 * reached.
12603 */
12604 if (newsize)
12605 if (!sbreserve_locked(so, SO_RCV, newsize, NULL))
12606 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
12607 m_adj(m, drop_hdrlen); /* delayed header drop */
12608 #ifdef NETFLIX_SB_LIMITS
12609 appended =
12610 #endif
12611 sbappendstream_locked(&so->so_rcv, m, 0);
12612 ctf_calc_rwin(so, tp);
12613 }
12614 rack_log_wakeup(tp,rack, &so->so_rcv, tlen, 1);
12615 /* NB: sorwakeup_locked() does an implicit unlock. */
12616 sorwakeup_locked(so);
12617 #ifdef NETFLIX_SB_LIMITS
12618 if (so->so_rcv.sb_shlim && mcnt != appended)
12619 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);
12620 #endif
12621 rack_handle_delayed_ack(tp, rack, tlen, 0);
12622 if (tp->snd_una == tp->snd_max)
12623 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
12624 return (1);
12625 }
12626
12627 /*
12628 * This subfunction is used to try to highly optimize the
12629 * fast path. We again allow window updates that are
12630 * in sequence to remain in the fast-path. We also add
12631 * in the __predict's to attempt to help the compiler.
12632 * Note that if we return a 0, then we can *not* process
12633 * it and the caller should push the packet into the
12634 * slow-path.
12635 */
12636 static int
rack_fastack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t nxt_pkt,uint32_t cts)12637 rack_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
12638 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
12639 uint32_t tiwin, int32_t nxt_pkt, uint32_t cts)
12640 {
12641 int32_t acked;
12642 int32_t nsegs;
12643 int32_t under_pacing = 0;
12644 struct tcp_rack *rack;
12645
12646 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
12647 /* Old ack, behind (or duplicate to) the last one rcv'd */
12648 return (0);
12649 }
12650 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
12651 /* Above what we have sent? */
12652 return (0);
12653 }
12654 if (__predict_false(tiwin == 0)) {
12655 /* zero window */
12656 return (0);
12657 }
12658 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
12659 /* We need a SYN or a FIN, unlikely.. */
12660 return (0);
12661 }
12662 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
12663 /* Timestamp is behind .. old ack with seq wrap? */
12664 return (0);
12665 }
12666 if (__predict_false(IN_RECOVERY(tp->t_flags))) {
12667 /* Still recovering */
12668 return (0);
12669 }
12670 rack = (struct tcp_rack *)tp->t_fb_ptr;
12671 if (rack->r_ctl.rc_sacked) {
12672 /* We have sack holes on our scoreboard */
12673 return (0);
12674 }
12675 /* Ok if we reach here, we can process a fast-ack */
12676 if (rack->gp_ready &&
12677 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
12678 under_pacing = 1;
12679 }
12680 nsegs = max(1, m->m_pkthdr.lro_nsegs);
12681 rack_log_ack(tp, to, th, 0, 0, NULL, NULL);
12682 /* Did the window get updated? */
12683 if (tiwin != tp->snd_wnd) {
12684 tp->snd_wnd = tiwin;
12685 rack_validate_fo_sendwin_up(tp, rack);
12686 tp->snd_wl1 = th->th_seq;
12687 if (tp->snd_wnd > tp->max_sndwnd)
12688 tp->max_sndwnd = tp->snd_wnd;
12689 }
12690 /* Do we exit persists? */
12691 if ((rack->rc_in_persist != 0) &&
12692 (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2),
12693 rack->r_ctl.rc_pace_min_segs))) {
12694 rack_exit_persist(tp, rack, cts);
12695 }
12696 /* Do we enter persists? */
12697 if ((rack->rc_in_persist == 0) &&
12698 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) &&
12699 TCPS_HAVEESTABLISHED(tp->t_state) &&
12700 ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) &&
12701 sbavail(&tptosocket(tp)->so_snd) &&
12702 (sbavail(&tptosocket(tp)->so_snd) > tp->snd_wnd)) {
12703 /*
12704 * Here the rwnd is less than
12705 * the pacing size, we are established,
12706 * nothing is outstanding, and there is
12707 * data to send. Enter persists.
12708 */
12709 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime, th->th_ack);
12710 }
12711 /*
12712 * If last ACK falls within this segment's sequence numbers, record
12713 * the timestamp. NOTE that the test is modified according to the
12714 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
12715 */
12716 if ((to->to_flags & TOF_TS) != 0 &&
12717 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
12718 tp->ts_recent_age = tcp_ts_getticks();
12719 tp->ts_recent = to->to_tsval;
12720 }
12721 /*
12722 * This is a pure ack for outstanding data.
12723 */
12724 KMOD_TCPSTAT_INC(tcps_predack);
12725
12726 /*
12727 * "bad retransmit" recovery.
12728 */
12729 if ((tp->t_flags & TF_PREVVALID) &&
12730 ((tp->t_flags & TF_RCVD_TSTMP) == 0)) {
12731 tp->t_flags &= ~TF_PREVVALID;
12732 if (tp->t_rxtshift == 1 &&
12733 (int)(ticks - tp->t_badrxtwin) < 0)
12734 rack_cong_signal(tp, CC_RTO_ERR, th->th_ack, __LINE__);
12735 }
12736 /*
12737 * Recalculate the transmit timer / rtt.
12738 *
12739 * Some boxes send broken timestamp replies during the SYN+ACK
12740 * phase, ignore timestamps of 0 or we could calculate a huge RTT
12741 * and blow up the retransmit timer.
12742 */
12743 acked = BYTES_THIS_ACK(tp, th);
12744
12745 #ifdef TCP_HHOOK
12746 /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */
12747 hhook_run_tcp_est_in(tp, th, to);
12748 #endif
12749 KMOD_TCPSTAT_ADD(tcps_rcvackpack, nsegs);
12750 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
12751 if (acked) {
12752 struct mbuf *mfree;
12753
12754 rack_ack_received(tp, rack, th->th_ack, nsegs, CC_ACK, 0);
12755 SOCK_SENDBUF_LOCK(so);
12756 mfree = sbcut_locked(&so->so_snd, acked);
12757 tp->snd_una = th->th_ack;
12758 /* Note we want to hold the sb lock through the sendmap adjust */
12759 rack_adjust_sendmap_head(rack, &so->so_snd);
12760 /* Wake up the socket if we have room to write more */
12761 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2);
12762 sowwakeup_locked(so);
12763 m_freem(mfree);
12764 tp->t_rxtshift = 0;
12765 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
12766 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
12767 rack->rc_tlp_in_progress = 0;
12768 rack->r_ctl.rc_tlp_cnt_out = 0;
12769 /*
12770 * If it is the RXT timer we want to
12771 * stop it, so we can restart a TLP.
12772 */
12773 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
12774 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
12775
12776 #ifdef TCP_REQUEST_TRK
12777 rack_req_check_for_comp(rack, th->th_ack);
12778 #endif
12779 }
12780 /*
12781 * Let the congestion control algorithm update congestion control
12782 * related information. This typically means increasing the
12783 * congestion window.
12784 */
12785 if (tp->snd_wnd < ctf_outstanding(tp)) {
12786 /* The peer collapsed the window */
12787 rack_collapsed_window(rack, ctf_outstanding(tp), th->th_ack, __LINE__);
12788 } else if (rack->rc_has_collapsed)
12789 rack_un_collapse_window(rack, __LINE__);
12790 if ((rack->r_collapse_point_valid) &&
12791 (SEQ_GT(tp->snd_una, rack->r_ctl.high_collapse_point)))
12792 rack->r_collapse_point_valid = 0;
12793 /*
12794 * Pull snd_wl2 up to prevent seq wrap relative to th_ack.
12795 */
12796 tp->snd_wl2 = th->th_ack;
12797 tp->t_dupacks = 0;
12798 m_freem(m);
12799 /* ND6_HINT(tp); *//* Some progress has been made. */
12800
12801 /*
12802 * If all outstanding data are acked, stop retransmit timer,
12803 * otherwise restart timer using current (possibly backed-off)
12804 * value. If process is waiting for space, wakeup/selwakeup/signal.
12805 * If data are ready to send, let tcp_output decide between more
12806 * output or persist.
12807 */
12808 if (under_pacing &&
12809 (rack->use_fixed_rate == 0) &&
12810 (rack->in_probe_rtt == 0) &&
12811 rack->rc_gp_dyn_mul &&
12812 rack->rc_always_pace) {
12813 /* Check if we are dragging bottom */
12814 rack_check_bottom_drag(tp, rack, so);
12815 }
12816 if (tp->snd_una == tp->snd_max) {
12817 tp->t_flags &= ~TF_PREVVALID;
12818 rack->r_ctl.retran_during_recovery = 0;
12819 rack->rc_suspicious = 0;
12820 rack->r_ctl.dsack_byte_cnt = 0;
12821 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL);
12822 if (rack->r_ctl.rc_went_idle_time == 0)
12823 rack->r_ctl.rc_went_idle_time = 1;
12824 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
12825 if (sbavail(&tptosocket(tp)->so_snd) == 0)
12826 tp->t_acktime = 0;
12827 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
12828 }
12829 if (acked && rack->r_fast_output)
12830 rack_gain_for_fastoutput(rack, tp, so, (uint32_t)acked);
12831 if (sbavail(&so->so_snd)) {
12832 rack->r_wanted_output = 1;
12833 }
12834 return (1);
12835 }
12836
12837 /*
12838 * Return value of 1, the TCB is unlocked and most
12839 * likely gone, return value of 0, the TCP is still
12840 * locked.
12841 */
12842 static int
rack_do_syn_sent(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)12843 rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so,
12844 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
12845 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
12846 {
12847 int32_t ret_val = 0;
12848 int32_t orig_tlen = tlen;
12849 int32_t todrop;
12850 int32_t ourfinisacked = 0;
12851 struct tcp_rack *rack;
12852
12853 INP_WLOCK_ASSERT(tptoinpcb(tp));
12854
12855 ctf_calc_rwin(so, tp);
12856 /*
12857 * If the state is SYN_SENT: if seg contains an ACK, but not for our
12858 * SYN, drop the input. if seg contains a RST, then drop the
12859 * connection. if seg does not contain SYN, then drop it. Otherwise
12860 * this is an acceptable SYN segment initialize tp->rcv_nxt and
12861 * tp->irs if seg contains ack then advance tp->snd_una if seg
12862 * contains an ECE and ECN support is enabled, the stream is ECN
12863 * capable. if SYN has been acked change to ESTABLISHED else
12864 * SYN_RCVD state arrange for segment to be acked (eventually)
12865 * continue processing rest of data/controls.
12866 */
12867 if ((thflags & TH_ACK) &&
12868 (SEQ_LEQ(th->th_ack, tp->iss) ||
12869 SEQ_GT(th->th_ack, tp->snd_max))) {
12870 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
12871 ctf_do_dropwithreset(m, tp, th, tlen);
12872 return (1);
12873 }
12874 if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) {
12875 TCP_PROBE5(connect__refused, NULL, tp,
12876 mtod(m, const char *), tp, th);
12877 tp = tcp_drop(tp, ECONNREFUSED);
12878 ctf_do_drop(m, tp);
12879 return (1);
12880 }
12881 if (thflags & TH_RST) {
12882 ctf_do_drop(m, tp);
12883 return (1);
12884 }
12885 if (!(thflags & TH_SYN)) {
12886 ctf_do_drop(m, tp);
12887 return (1);
12888 }
12889 tp->irs = th->th_seq;
12890 tcp_rcvseqinit(tp);
12891 rack = (struct tcp_rack *)tp->t_fb_ptr;
12892 if (thflags & TH_ACK) {
12893 int tfo_partial = 0;
12894
12895 KMOD_TCPSTAT_INC(tcps_connects);
12896 soisconnected(so);
12897 #ifdef MAC
12898 mac_socketpeer_set_from_mbuf(m, so);
12899 #endif
12900 /* Do window scaling on this connection? */
12901 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
12902 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
12903 tp->rcv_scale = tp->request_r_scale;
12904 }
12905 tp->rcv_adv += min(tp->rcv_wnd,
12906 TCP_MAXWIN << tp->rcv_scale);
12907 /*
12908 * If not all the data that was sent in the TFO SYN
12909 * has been acked, resend the remainder right away.
12910 */
12911 if ((tp->t_flags & TF_FASTOPEN) &&
12912 (tp->snd_una != tp->snd_max)) {
12913 /* Was it a partial ack? */
12914 if (SEQ_LT(th->th_ack, tp->snd_max))
12915 tfo_partial = 1;
12916 }
12917 /*
12918 * If there's data, delay ACK; if there's also a FIN ACKNOW
12919 * will be turned on later.
12920 */
12921 if (DELAY_ACK(tp, tlen) && tlen != 0 && !tfo_partial) {
12922 rack_timer_cancel(tp, rack,
12923 rack->r_ctl.rc_rcvtime, __LINE__);
12924 tp->t_flags |= TF_DELACK;
12925 } else {
12926 rack->r_wanted_output = 1;
12927 tp->t_flags |= TF_ACKNOW;
12928 }
12929
12930 tcp_ecn_input_syn_sent(tp, thflags, iptos);
12931
12932 if (SEQ_GT(th->th_ack, tp->snd_una)) {
12933 /*
12934 * We advance snd_una for the
12935 * fast open case. If th_ack is
12936 * acknowledging data beyond
12937 * snd_una we can't just call
12938 * ack-processing since the
12939 * data stream in our send-map
12940 * will start at snd_una + 1 (one
12941 * beyond the SYN). If its just
12942 * equal we don't need to do that
12943 * and there is no send_map.
12944 */
12945 tp->snd_una++;
12946 if (tfo_partial && (SEQ_GT(tp->snd_max, tp->snd_una))) {
12947 /*
12948 * We sent a SYN with data, and thus have a
12949 * sendmap entry with a SYN set. Lets find it
12950 * and take off the send bit and the byte and
12951 * set it up to be what we send (send it next).
12952 */
12953 struct rack_sendmap *rsm;
12954
12955 rsm = tqhash_min(rack->r_ctl.tqh);
12956 if (rsm) {
12957 if (rsm->r_flags & RACK_HAS_SYN) {
12958 rsm->r_flags &= ~RACK_HAS_SYN;
12959 rsm->r_start++;
12960 }
12961 rack->r_ctl.rc_resend = rsm;
12962 }
12963 }
12964 }
12965 /*
12966 * Received <SYN,ACK> in SYN_SENT[*] state. Transitions:
12967 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
12968 */
12969 tp->t_starttime = ticks;
12970 if (tp->t_flags & TF_NEEDFIN) {
12971 tcp_state_change(tp, TCPS_FIN_WAIT_1);
12972 tp->t_flags &= ~TF_NEEDFIN;
12973 thflags &= ~TH_SYN;
12974 } else {
12975 tcp_state_change(tp, TCPS_ESTABLISHED);
12976 TCP_PROBE5(connect__established, NULL, tp,
12977 mtod(m, const char *), tp, th);
12978 rack_cc_conn_init(tp);
12979 }
12980 } else {
12981 /*
12982 * Received initial SYN in SYN-SENT[*] state => simultaneous
12983 * open. If segment contains CC option and there is a
12984 * cached CC, apply TAO test. If it succeeds, connection is *
12985 * half-synchronized. Otherwise, do 3-way handshake:
12986 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
12987 * there was no CC option, clear cached CC value.
12988 */
12989 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN);
12990 tcp_state_change(tp, TCPS_SYN_RECEIVED);
12991 }
12992 /*
12993 * Advance th->th_seq to correspond to first data byte. If data,
12994 * trim to stay within window, dropping FIN if necessary.
12995 */
12996 th->th_seq++;
12997 if (tlen > tp->rcv_wnd) {
12998 todrop = tlen - tp->rcv_wnd;
12999 m_adj(m, -todrop);
13000 tlen = tp->rcv_wnd;
13001 thflags &= ~TH_FIN;
13002 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin);
13003 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
13004 }
13005 tp->snd_wl1 = th->th_seq - 1;
13006 tp->rcv_up = th->th_seq;
13007 /*
13008 * Client side of transaction: already sent SYN and data. If the
13009 * remote host used T/TCP to validate the SYN, our data will be
13010 * ACK'd; if so, enter normal data segment processing in the middle
13011 * of step 5, ack processing. Otherwise, goto step 6.
13012 */
13013 if (thflags & TH_ACK) {
13014 /* For syn-sent we need to possibly update the rtt */
13015 if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) {
13016 uint32_t t, mcts;
13017
13018 mcts = tcp_ts_getticks();
13019 t = (mcts - to->to_tsecr) * HPTS_USEC_IN_MSEC;
13020 if (!tp->t_rttlow || tp->t_rttlow > t)
13021 tp->t_rttlow = t;
13022 rack_log_rtt_sample_calc(rack, t, (to->to_tsecr * 1000), (mcts * 1000), 4);
13023 tcp_rack_xmit_timer(rack, t + 1, 1, t, 0, NULL, 2);
13024 tcp_rack_xmit_timer_commit(rack, tp);
13025 }
13026 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen))
13027 return (ret_val);
13028 /* We may have changed to FIN_WAIT_1 above */
13029 if (tp->t_state == TCPS_FIN_WAIT_1) {
13030 /*
13031 * In FIN_WAIT_1 STATE in addition to the processing
13032 * for the ESTABLISHED state if our FIN is now
13033 * acknowledged then enter FIN_WAIT_2.
13034 */
13035 if (ourfinisacked) {
13036 /*
13037 * If we can't receive any more data, then
13038 * closing user can proceed. Starting the
13039 * timer is contrary to the specification,
13040 * but if we don't get a FIN we'll hang
13041 * forever.
13042 *
13043 * XXXjl: we should release the tp also, and
13044 * use a compressed state.
13045 */
13046 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
13047 soisdisconnected(so);
13048 tcp_timer_activate(tp, TT_2MSL,
13049 (tcp_fast_finwait2_recycle ?
13050 tcp_finwait2_timeout :
13051 TP_MAXIDLE(tp)));
13052 }
13053 tcp_state_change(tp, TCPS_FIN_WAIT_2);
13054 }
13055 }
13056 }
13057 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13058 tiwin, thflags, nxt_pkt));
13059 }
13060
13061 /*
13062 * Return value of 1, the TCB is unlocked and most
13063 * likely gone, return value of 0, the TCP is still
13064 * locked.
13065 */
13066 static int
rack_do_syn_recv(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13067 rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so,
13068 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13069 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13070 {
13071 struct tcp_rack *rack;
13072 int32_t orig_tlen = tlen;
13073 int32_t ret_val = 0;
13074 int32_t ourfinisacked = 0;
13075
13076 rack = (struct tcp_rack *)tp->t_fb_ptr;
13077 ctf_calc_rwin(so, tp);
13078 if ((thflags & TH_RST) ||
13079 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13080 return (ctf_process_rst(m, th, so, tp));
13081 if ((thflags & TH_ACK) &&
13082 (SEQ_LEQ(th->th_ack, tp->snd_una) ||
13083 SEQ_GT(th->th_ack, tp->snd_max))) {
13084 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
13085 ctf_do_dropwithreset(m, tp, th, tlen);
13086 return (1);
13087 }
13088 if (tp->t_flags & TF_FASTOPEN) {
13089 /*
13090 * When a TFO connection is in SYN_RECEIVED, the
13091 * only valid packets are the initial SYN, a
13092 * retransmit/copy of the initial SYN (possibly with
13093 * a subset of the original data), a valid ACK, a
13094 * FIN, or a RST.
13095 */
13096 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
13097 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
13098 ctf_do_dropwithreset(m, tp, th, tlen);
13099 return (1);
13100 } else if (thflags & TH_SYN) {
13101 /* non-initial SYN is ignored */
13102 if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
13103 (rack->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
13104 (rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
13105 ctf_do_drop(m, NULL);
13106 return (0);
13107 }
13108 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) {
13109 ctf_do_drop(m, NULL);
13110 return (0);
13111 }
13112 }
13113
13114 /*
13115 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13116 * it's less than ts_recent, drop it.
13117 */
13118 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13119 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13120 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13121 return (ret_val);
13122 }
13123 /*
13124 * In the SYN-RECEIVED state, validate that the packet belongs to
13125 * this connection before trimming the data to fit the receive
13126 * window. Check the sequence number versus IRS since we know the
13127 * sequence numbers haven't wrapped. This is a partial fix for the
13128 * "LAND" DoS attack.
13129 */
13130 if (SEQ_LT(th->th_seq, tp->irs)) {
13131 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
13132 ctf_do_dropwithreset(m, tp, th, tlen);
13133 return (1);
13134 }
13135 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13136 return (ret_val);
13137 }
13138 /*
13139 * If last ACK falls within this segment's sequence numbers, record
13140 * its timestamp. NOTE: 1) That the test incorporates suggestions
13141 * from the latest proposal of the tcplw@cray.com list (Braden
13142 * 1993/04/26). 2) That updating only on newer timestamps interferes
13143 * with our earlier PAWS tests, so this check should be solely
13144 * predicated on the sequence space of this segment. 3) That we
13145 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13146 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13147 * SEG.Len, This modified check allows us to overcome RFC1323's
13148 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13149 * p.869. In such cases, we can still calculate the RTT correctly
13150 * when RCV.NXT == Last.ACK.Sent.
13151 */
13152 if ((to->to_flags & TOF_TS) != 0 &&
13153 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13154 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13155 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13156 tp->ts_recent_age = tcp_ts_getticks();
13157 tp->ts_recent = to->to_tsval;
13158 }
13159 tp->snd_wnd = tiwin;
13160 rack_validate_fo_sendwin_up(tp, rack);
13161 /*
13162 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13163 * is on (half-synchronized state), then queue data for later
13164 * processing; else drop segment and return.
13165 */
13166 if ((thflags & TH_ACK) == 0) {
13167 if (tp->t_flags & TF_FASTOPEN) {
13168 rack_cc_conn_init(tp);
13169 }
13170 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13171 tiwin, thflags, nxt_pkt));
13172 }
13173 KMOD_TCPSTAT_INC(tcps_connects);
13174 if (tp->t_flags & TF_SONOTCONN) {
13175 tp->t_flags &= ~TF_SONOTCONN;
13176 soisconnected(so);
13177 }
13178 /* Do window scaling? */
13179 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
13180 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
13181 tp->rcv_scale = tp->request_r_scale;
13182 }
13183 /*
13184 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* ->
13185 * FIN-WAIT-1
13186 */
13187 tp->t_starttime = ticks;
13188 if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) {
13189 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
13190 tp->t_tfo_pending = NULL;
13191 }
13192 if (tp->t_flags & TF_NEEDFIN) {
13193 tcp_state_change(tp, TCPS_FIN_WAIT_1);
13194 tp->t_flags &= ~TF_NEEDFIN;
13195 } else {
13196 tcp_state_change(tp, TCPS_ESTABLISHED);
13197 TCP_PROBE5(accept__established, NULL, tp,
13198 mtod(m, const char *), tp, th);
13199 /*
13200 * TFO connections call cc_conn_init() during SYN
13201 * processing. Calling it again here for such connections
13202 * is not harmless as it would undo the snd_cwnd reduction
13203 * that occurs when a TFO SYN|ACK is retransmitted.
13204 */
13205 if (!(tp->t_flags & TF_FASTOPEN))
13206 rack_cc_conn_init(tp);
13207 }
13208 /*
13209 * Account for the ACK of our SYN prior to
13210 * regular ACK processing below, except for
13211 * simultaneous SYN, which is handled later.
13212 */
13213 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))
13214 tp->snd_una++;
13215 /*
13216 * If segment contains data or ACK, will call tcp_reass() later; if
13217 * not, do so now to pass queued data to user.
13218 */
13219 if (tlen == 0 && (thflags & TH_FIN) == 0) {
13220 (void) tcp_reass(tp, (struct tcphdr *)0, NULL, 0,
13221 (struct mbuf *)0);
13222 if (tp->t_flags & TF_WAKESOR) {
13223 tp->t_flags &= ~TF_WAKESOR;
13224 /* NB: sorwakeup_locked() does an implicit unlock. */
13225 sorwakeup_locked(so);
13226 }
13227 }
13228 tp->snd_wl1 = th->th_seq - 1;
13229 /* For syn-recv we need to possibly update the rtt */
13230 if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) {
13231 uint32_t t, mcts;
13232
13233 mcts = tcp_ts_getticks();
13234 t = (mcts - to->to_tsecr) * HPTS_USEC_IN_MSEC;
13235 if (!tp->t_rttlow || tp->t_rttlow > t)
13236 tp->t_rttlow = t;
13237 rack_log_rtt_sample_calc(rack, t, (to->to_tsecr * 1000), (mcts * 1000), 5);
13238 tcp_rack_xmit_timer(rack, t + 1, 1, t, 0, NULL, 2);
13239 tcp_rack_xmit_timer_commit(rack, tp);
13240 }
13241 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13242 return (ret_val);
13243 }
13244 if (tp->t_state == TCPS_FIN_WAIT_1) {
13245 /* We could have went to FIN_WAIT_1 (or EST) above */
13246 /*
13247 * In FIN_WAIT_1 STATE in addition to the processing for the
13248 * ESTABLISHED state if our FIN is now acknowledged then
13249 * enter FIN_WAIT_2.
13250 */
13251 if (ourfinisacked) {
13252 /*
13253 * If we can't receive any more data, then closing
13254 * user can proceed. Starting the timer is contrary
13255 * to the specification, but if we don't get a FIN
13256 * we'll hang forever.
13257 *
13258 * XXXjl: we should release the tp also, and use a
13259 * compressed state.
13260 */
13261 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
13262 soisdisconnected(so);
13263 tcp_timer_activate(tp, TT_2MSL,
13264 (tcp_fast_finwait2_recycle ?
13265 tcp_finwait2_timeout :
13266 TP_MAXIDLE(tp)));
13267 }
13268 tcp_state_change(tp, TCPS_FIN_WAIT_2);
13269 }
13270 }
13271 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13272 tiwin, thflags, nxt_pkt));
13273 }
13274
13275 /*
13276 * Return value of 1, the TCB is unlocked and most
13277 * likely gone, return value of 0, the TCP is still
13278 * locked.
13279 */
13280 static int
rack_do_established(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13281 rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so,
13282 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13283 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13284 {
13285 int32_t ret_val = 0;
13286 int32_t orig_tlen = tlen;
13287 struct tcp_rack *rack;
13288
13289 /*
13290 * Header prediction: check for the two common cases of a
13291 * uni-directional data xfer. If the packet has no control flags,
13292 * is in-sequence, the window didn't change and we're not
13293 * retransmitting, it's a candidate. If the length is zero and the
13294 * ack moved forward, we're the sender side of the xfer. Just free
13295 * the data acked & wake any higher level process that was blocked
13296 * waiting for space. If the length is non-zero and the ack didn't
13297 * move, we're the receiver side. If we're getting packets in-order
13298 * (the reassembly queue is empty), add the data toc The socket
13299 * buffer and note that we need a delayed ack. Make sure that the
13300 * hidden state-flags are also off. Since we check for
13301 * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN.
13302 */
13303 rack = (struct tcp_rack *)tp->t_fb_ptr;
13304 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&
13305 __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_ACK)) == TH_ACK) &&
13306 __predict_true(SEGQ_EMPTY(tp)) &&
13307 __predict_true(th->th_seq == tp->rcv_nxt)) {
13308 if (tlen == 0) {
13309 if (rack_fastack(m, th, so, tp, to, drop_hdrlen, tlen,
13310 tiwin, nxt_pkt, rack->r_ctl.rc_rcvtime)) {
13311 return (0);
13312 }
13313 } else {
13314 if (rack_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen,
13315 tiwin, nxt_pkt, iptos)) {
13316 return (0);
13317 }
13318 }
13319 }
13320 ctf_calc_rwin(so, tp);
13321
13322 if ((thflags & TH_RST) ||
13323 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13324 return (ctf_process_rst(m, th, so, tp));
13325
13326 /*
13327 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13328 * synchronized state.
13329 */
13330 if (thflags & TH_SYN) {
13331 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
13332 return (ret_val);
13333 }
13334 /*
13335 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13336 * it's less than ts_recent, drop it.
13337 */
13338 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13339 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13340 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13341 return (ret_val);
13342 }
13343 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13344 return (ret_val);
13345 }
13346 /*
13347 * If last ACK falls within this segment's sequence numbers, record
13348 * its timestamp. NOTE: 1) That the test incorporates suggestions
13349 * from the latest proposal of the tcplw@cray.com list (Braden
13350 * 1993/04/26). 2) That updating only on newer timestamps interferes
13351 * with our earlier PAWS tests, so this check should be solely
13352 * predicated on the sequence space of this segment. 3) That we
13353 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13354 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13355 * SEG.Len, This modified check allows us to overcome RFC1323's
13356 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13357 * p.869. In such cases, we can still calculate the RTT correctly
13358 * when RCV.NXT == Last.ACK.Sent.
13359 */
13360 if ((to->to_flags & TOF_TS) != 0 &&
13361 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13362 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13363 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13364 tp->ts_recent_age = tcp_ts_getticks();
13365 tp->ts_recent = to->to_tsval;
13366 }
13367 /*
13368 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13369 * is on (half-synchronized state), then queue data for later
13370 * processing; else drop segment and return.
13371 */
13372 if ((thflags & TH_ACK) == 0) {
13373 if (tp->t_flags & TF_NEEDSYN) {
13374 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13375 tiwin, thflags, nxt_pkt));
13376
13377 } else if (tp->t_flags & TF_ACKNOW) {
13378 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
13379 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
13380 return (ret_val);
13381 } else {
13382 ctf_do_drop(m, NULL);
13383 return (0);
13384 }
13385 }
13386 /*
13387 * Ack processing.
13388 */
13389 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val, orig_tlen)) {
13390 return (ret_val);
13391 }
13392 if (sbavail(&so->so_snd)) {
13393 if (ctf_progress_timeout_check(tp, true)) {
13394 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
13395 ctf_do_dropwithreset_conn(m, tp, th, tlen);
13396 return (1);
13397 }
13398 }
13399 /* State changes only happen in rack_process_data() */
13400 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13401 tiwin, thflags, nxt_pkt));
13402 }
13403
13404 /*
13405 * Return value of 1, the TCB is unlocked and most
13406 * likely gone, return value of 0, the TCP is still
13407 * locked.
13408 */
13409 static int
rack_do_close_wait(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13410 rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so,
13411 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13412 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13413 {
13414 int32_t ret_val = 0;
13415 int32_t orig_tlen = tlen;
13416
13417 ctf_calc_rwin(so, tp);
13418 if ((thflags & TH_RST) ||
13419 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13420 return (ctf_process_rst(m, th, so, tp));
13421 /*
13422 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13423 * synchronized state.
13424 */
13425 if (thflags & TH_SYN) {
13426 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
13427 return (ret_val);
13428 }
13429 /*
13430 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13431 * it's less than ts_recent, drop it.
13432 */
13433 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13434 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13435 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13436 return (ret_val);
13437 }
13438 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13439 return (ret_val);
13440 }
13441 /*
13442 * If last ACK falls within this segment's sequence numbers, record
13443 * its timestamp. NOTE: 1) That the test incorporates suggestions
13444 * from the latest proposal of the tcplw@cray.com list (Braden
13445 * 1993/04/26). 2) That updating only on newer timestamps interferes
13446 * with our earlier PAWS tests, so this check should be solely
13447 * predicated on the sequence space of this segment. 3) That we
13448 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13449 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13450 * SEG.Len, This modified check allows us to overcome RFC1323's
13451 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13452 * p.869. In such cases, we can still calculate the RTT correctly
13453 * when RCV.NXT == Last.ACK.Sent.
13454 */
13455 if ((to->to_flags & TOF_TS) != 0 &&
13456 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13457 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13458 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13459 tp->ts_recent_age = tcp_ts_getticks();
13460 tp->ts_recent = to->to_tsval;
13461 }
13462 /*
13463 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13464 * is on (half-synchronized state), then queue data for later
13465 * processing; else drop segment and return.
13466 */
13467 if ((thflags & TH_ACK) == 0) {
13468 if (tp->t_flags & TF_NEEDSYN) {
13469 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13470 tiwin, thflags, nxt_pkt));
13471
13472 } else if (tp->t_flags & TF_ACKNOW) {
13473 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
13474 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
13475 return (ret_val);
13476 } else {
13477 ctf_do_drop(m, NULL);
13478 return (0);
13479 }
13480 }
13481 /*
13482 * Ack processing.
13483 */
13484 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val, orig_tlen)) {
13485 return (ret_val);
13486 }
13487 if (sbavail(&so->so_snd)) {
13488 if (ctf_progress_timeout_check(tp, true)) {
13489 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
13490 tp, tick, PROGRESS_DROP, __LINE__);
13491 ctf_do_dropwithreset_conn(m, tp, th, tlen);
13492 return (1);
13493 }
13494 }
13495 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13496 tiwin, thflags, nxt_pkt));
13497 }
13498
13499 static int
rack_check_data_after_close(struct mbuf * m,struct tcpcb * tp,int32_t * tlen,struct tcphdr * th,struct socket * so)13500 rack_check_data_after_close(struct mbuf *m,
13501 struct tcpcb *tp, int32_t *tlen, struct tcphdr *th, struct socket *so)
13502 {
13503 struct tcp_rack *rack;
13504
13505 rack = (struct tcp_rack *)tp->t_fb_ptr;
13506 if (rack->rc_allow_data_af_clo == 0) {
13507 close_now:
13508 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
13509 /* tcp_close will kill the inp pre-log the Reset */
13510 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
13511 tp = tcp_close(tp);
13512 KMOD_TCPSTAT_INC(tcps_rcvafterclose);
13513 ctf_do_dropwithreset(m, tp, th, *tlen);
13514 return (1);
13515 }
13516 if (sbavail(&so->so_snd) == 0)
13517 goto close_now;
13518 /* Ok we allow data that is ignored and a followup reset */
13519 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
13520 tp->rcv_nxt = th->th_seq + *tlen;
13521 tp->t_flags2 |= TF2_DROP_AF_DATA;
13522 rack->r_wanted_output = 1;
13523 *tlen = 0;
13524 return (0);
13525 }
13526
13527 /*
13528 * Return value of 1, the TCB is unlocked and most
13529 * likely gone, return value of 0, the TCP is still
13530 * locked.
13531 */
13532 static int
rack_do_fin_wait_1(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13533 rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so,
13534 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13535 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13536 {
13537 int32_t ret_val = 0;
13538 int32_t orig_tlen = tlen;
13539 int32_t ourfinisacked = 0;
13540
13541 ctf_calc_rwin(so, tp);
13542
13543 if ((thflags & TH_RST) ||
13544 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13545 return (ctf_process_rst(m, th, so, tp));
13546 /*
13547 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13548 * synchronized state.
13549 */
13550 if (thflags & TH_SYN) {
13551 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
13552 return (ret_val);
13553 }
13554 /*
13555 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13556 * it's less than ts_recent, drop it.
13557 */
13558 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13559 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13560 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13561 return (ret_val);
13562 }
13563 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13564 return (ret_val);
13565 }
13566 /*
13567 * If new data are received on a connection after the user processes
13568 * are gone, then RST the other end.
13569 */
13570 if ((tp->t_flags & TF_CLOSED) && tlen &&
13571 rack_check_data_after_close(m, tp, &tlen, th, so))
13572 return (1);
13573 /*
13574 * If last ACK falls within this segment's sequence numbers, record
13575 * its timestamp. NOTE: 1) That the test incorporates suggestions
13576 * from the latest proposal of the tcplw@cray.com list (Braden
13577 * 1993/04/26). 2) That updating only on newer timestamps interferes
13578 * with our earlier PAWS tests, so this check should be solely
13579 * predicated on the sequence space of this segment. 3) That we
13580 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13581 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13582 * SEG.Len, This modified check allows us to overcome RFC1323's
13583 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13584 * p.869. In such cases, we can still calculate the RTT correctly
13585 * when RCV.NXT == Last.ACK.Sent.
13586 */
13587 if ((to->to_flags & TOF_TS) != 0 &&
13588 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13589 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13590 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13591 tp->ts_recent_age = tcp_ts_getticks();
13592 tp->ts_recent = to->to_tsval;
13593 }
13594 /*
13595 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13596 * is on (half-synchronized state), then queue data for later
13597 * processing; else drop segment and return.
13598 */
13599 if ((thflags & TH_ACK) == 0) {
13600 if (tp->t_flags & TF_NEEDSYN) {
13601 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13602 tiwin, thflags, nxt_pkt));
13603 } else if (tp->t_flags & TF_ACKNOW) {
13604 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
13605 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
13606 return (ret_val);
13607 } else {
13608 ctf_do_drop(m, NULL);
13609 return (0);
13610 }
13611 }
13612 /*
13613 * Ack processing.
13614 */
13615 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13616 return (ret_val);
13617 }
13618 if (ourfinisacked) {
13619 /*
13620 * If we can't receive any more data, then closing user can
13621 * proceed. Starting the timer is contrary to the
13622 * specification, but if we don't get a FIN we'll hang
13623 * forever.
13624 *
13625 * XXXjl: we should release the tp also, and use a
13626 * compressed state.
13627 */
13628 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
13629 soisdisconnected(so);
13630 tcp_timer_activate(tp, TT_2MSL,
13631 (tcp_fast_finwait2_recycle ?
13632 tcp_finwait2_timeout :
13633 TP_MAXIDLE(tp)));
13634 }
13635 tcp_state_change(tp, TCPS_FIN_WAIT_2);
13636 }
13637 if (sbavail(&so->so_snd)) {
13638 if (ctf_progress_timeout_check(tp, true)) {
13639 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
13640 tp, tick, PROGRESS_DROP, __LINE__);
13641 ctf_do_dropwithreset_conn(m, tp, th, tlen);
13642 return (1);
13643 }
13644 }
13645 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13646 tiwin, thflags, nxt_pkt));
13647 }
13648
13649 /*
13650 * Return value of 1, the TCB is unlocked and most
13651 * likely gone, return value of 0, the TCP is still
13652 * locked.
13653 */
13654 static int
rack_do_closing(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13655 rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so,
13656 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13657 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13658 {
13659 int32_t ret_val = 0;
13660 int32_t orig_tlen = tlen;
13661 int32_t ourfinisacked = 0;
13662
13663 ctf_calc_rwin(so, tp);
13664
13665 if ((thflags & TH_RST) ||
13666 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13667 return (ctf_process_rst(m, th, so, tp));
13668 /*
13669 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13670 * synchronized state.
13671 */
13672 if (thflags & TH_SYN) {
13673 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
13674 return (ret_val);
13675 }
13676 /*
13677 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13678 * it's less than ts_recent, drop it.
13679 */
13680 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13681 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13682 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13683 return (ret_val);
13684 }
13685 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13686 return (ret_val);
13687 }
13688 /*
13689 * If last ACK falls within this segment's sequence numbers, record
13690 * its timestamp. NOTE: 1) That the test incorporates suggestions
13691 * from the latest proposal of the tcplw@cray.com list (Braden
13692 * 1993/04/26). 2) That updating only on newer timestamps interferes
13693 * with our earlier PAWS tests, so this check should be solely
13694 * predicated on the sequence space of this segment. 3) That we
13695 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13696 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13697 * SEG.Len, This modified check allows us to overcome RFC1323's
13698 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13699 * p.869. In such cases, we can still calculate the RTT correctly
13700 * when RCV.NXT == Last.ACK.Sent.
13701 */
13702 if ((to->to_flags & TOF_TS) != 0 &&
13703 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13704 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13705 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13706 tp->ts_recent_age = tcp_ts_getticks();
13707 tp->ts_recent = to->to_tsval;
13708 }
13709 /*
13710 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13711 * is on (half-synchronized state), then queue data for later
13712 * processing; else drop segment and return.
13713 */
13714 if ((thflags & TH_ACK) == 0) {
13715 if (tp->t_flags & TF_NEEDSYN) {
13716 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13717 tiwin, thflags, nxt_pkt));
13718 } else if (tp->t_flags & TF_ACKNOW) {
13719 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
13720 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
13721 return (ret_val);
13722 } else {
13723 ctf_do_drop(m, NULL);
13724 return (0);
13725 }
13726 }
13727 /*
13728 * Ack processing.
13729 */
13730 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13731 return (ret_val);
13732 }
13733 if (ourfinisacked) {
13734 tcp_twstart(tp);
13735 m_freem(m);
13736 return (1);
13737 }
13738 if (sbavail(&so->so_snd)) {
13739 if (ctf_progress_timeout_check(tp, true)) {
13740 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
13741 tp, tick, PROGRESS_DROP, __LINE__);
13742 ctf_do_dropwithreset_conn(m, tp, th, tlen);
13743 return (1);
13744 }
13745 }
13746 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13747 tiwin, thflags, nxt_pkt));
13748 }
13749
13750 /*
13751 * Return value of 1, the TCB is unlocked and most
13752 * likely gone, return value of 0, the TCP is still
13753 * locked.
13754 */
13755 static int
rack_do_lastack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13756 rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
13757 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13758 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13759 {
13760 int32_t ret_val = 0;
13761 int32_t orig_tlen;
13762 int32_t ourfinisacked = 0;
13763
13764 ctf_calc_rwin(so, tp);
13765
13766 if ((thflags & TH_RST) ||
13767 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13768 return (ctf_process_rst(m, th, so, tp));
13769 /*
13770 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13771 * synchronized state.
13772 */
13773 if (thflags & TH_SYN) {
13774 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
13775 return (ret_val);
13776 }
13777 /*
13778 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13779 * it's less than ts_recent, drop it.
13780 */
13781 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13782 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13783 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13784 return (ret_val);
13785 }
13786 orig_tlen = tlen;
13787 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13788 return (ret_val);
13789 }
13790 /*
13791 * If last ACK falls within this segment's sequence numbers, record
13792 * its timestamp. NOTE: 1) That the test incorporates suggestions
13793 * from the latest proposal of the tcplw@cray.com list (Braden
13794 * 1993/04/26). 2) That updating only on newer timestamps interferes
13795 * with our earlier PAWS tests, so this check should be solely
13796 * predicated on the sequence space of this segment. 3) That we
13797 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13798 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13799 * SEG.Len, This modified check allows us to overcome RFC1323's
13800 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13801 * p.869. In such cases, we can still calculate the RTT correctly
13802 * when RCV.NXT == Last.ACK.Sent.
13803 */
13804 if ((to->to_flags & TOF_TS) != 0 &&
13805 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13806 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13807 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13808 tp->ts_recent_age = tcp_ts_getticks();
13809 tp->ts_recent = to->to_tsval;
13810 }
13811 /*
13812 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13813 * is on (half-synchronized state), then queue data for later
13814 * processing; else drop segment and return.
13815 */
13816 if ((thflags & TH_ACK) == 0) {
13817 if (tp->t_flags & TF_NEEDSYN) {
13818 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13819 tiwin, thflags, nxt_pkt));
13820 } else if (tp->t_flags & TF_ACKNOW) {
13821 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
13822 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
13823 return (ret_val);
13824 } else {
13825 ctf_do_drop(m, NULL);
13826 return (0);
13827 }
13828 }
13829 /*
13830 * case TCPS_LAST_ACK: Ack processing.
13831 */
13832 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13833 return (ret_val);
13834 }
13835 if (ourfinisacked) {
13836 tp = tcp_close(tp);
13837 ctf_do_drop(m, tp);
13838 return (1);
13839 }
13840 if (sbavail(&so->so_snd)) {
13841 if (ctf_progress_timeout_check(tp, true)) {
13842 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
13843 tp, tick, PROGRESS_DROP, __LINE__);
13844 ctf_do_dropwithreset_conn(m, tp, th, tlen);
13845 return (1);
13846 }
13847 }
13848 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13849 tiwin, thflags, nxt_pkt));
13850 }
13851
13852 /*
13853 * Return value of 1, the TCB is unlocked and most
13854 * likely gone, return value of 0, the TCP is still
13855 * locked.
13856 */
13857 static int
rack_do_fin_wait_2(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)13858 rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so,
13859 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
13860 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
13861 {
13862 int32_t ret_val = 0;
13863 int32_t orig_tlen = tlen;
13864 int32_t ourfinisacked = 0;
13865
13866 ctf_calc_rwin(so, tp);
13867
13868 /* Reset receive buffer auto scaling when not in bulk receive mode. */
13869 if ((thflags & TH_RST) ||
13870 (tp->t_fin_is_rst && (thflags & TH_FIN)))
13871 return (ctf_process_rst(m, th, so, tp));
13872 /*
13873 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13874 * synchronized state.
13875 */
13876 if (thflags & TH_SYN) {
13877 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
13878 return (ret_val);
13879 }
13880 /*
13881 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13882 * it's less than ts_recent, drop it.
13883 */
13884 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
13885 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
13886 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
13887 return (ret_val);
13888 }
13889 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
13890 return (ret_val);
13891 }
13892 /*
13893 * If new data are received on a connection after the user processes
13894 * are gone, then RST the other end.
13895 */
13896 if ((tp->t_flags & TF_CLOSED) && tlen &&
13897 rack_check_data_after_close(m, tp, &tlen, th, so))
13898 return (1);
13899 /*
13900 * If last ACK falls within this segment's sequence numbers, record
13901 * its timestamp. NOTE: 1) That the test incorporates suggestions
13902 * from the latest proposal of the tcplw@cray.com list (Braden
13903 * 1993/04/26). 2) That updating only on newer timestamps interferes
13904 * with our earlier PAWS tests, so this check should be solely
13905 * predicated on the sequence space of this segment. 3) That we
13906 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13907 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13908 * SEG.Len, This modified check allows us to overcome RFC1323's
13909 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13910 * p.869. In such cases, we can still calculate the RTT correctly
13911 * when RCV.NXT == Last.ACK.Sent.
13912 */
13913 if ((to->to_flags & TOF_TS) != 0 &&
13914 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
13915 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
13916 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
13917 tp->ts_recent_age = tcp_ts_getticks();
13918 tp->ts_recent = to->to_tsval;
13919 }
13920 /*
13921 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13922 * is on (half-synchronized state), then queue data for later
13923 * processing; else drop segment and return.
13924 */
13925 if ((thflags & TH_ACK) == 0) {
13926 if (tp->t_flags & TF_NEEDSYN) {
13927 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13928 tiwin, thflags, nxt_pkt));
13929 } else if (tp->t_flags & TF_ACKNOW) {
13930 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
13931 ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
13932 return (ret_val);
13933 } else {
13934 ctf_do_drop(m, NULL);
13935 return (0);
13936 }
13937 }
13938 /*
13939 * Ack processing.
13940 */
13941 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13942 return (ret_val);
13943 }
13944 if (sbavail(&so->so_snd)) {
13945 if (ctf_progress_timeout_check(tp, true)) {
13946 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
13947 tp, tick, PROGRESS_DROP, __LINE__);
13948 ctf_do_dropwithreset_conn(m, tp, th, tlen);
13949 return (1);
13950 }
13951 }
13952 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
13953 tiwin, thflags, nxt_pkt));
13954 }
13955
13956 static void inline
rack_clear_rate_sample(struct tcp_rack * rack)13957 rack_clear_rate_sample(struct tcp_rack *rack)
13958 {
13959 rack->r_ctl.rack_rs.rs_flags = RACK_RTT_EMPTY;
13960 rack->r_ctl.rack_rs.rs_rtt_cnt = 0;
13961 rack->r_ctl.rack_rs.rs_rtt_tot = 0;
13962 }
13963
13964 static void
rack_set_pace_segments(struct tcpcb * tp,struct tcp_rack * rack,uint32_t line,uint64_t * fill_override)13965 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override)
13966 {
13967 uint64_t bw_est, rate_wanted;
13968 int chged = 0;
13969 uint32_t user_max, orig_min, orig_max;
13970
13971 #ifdef TCP_REQUEST_TRK
13972 if (rack->rc_hybrid_mode &&
13973 (rack->r_ctl.rc_pace_max_segs != 0) &&
13974 (rack_hybrid_allow_set_maxseg == 1) &&
13975 (rack->r_ctl.rc_last_sft != NULL)) {
13976 rack->r_ctl.rc_last_sft->hybrid_flags &= ~TCP_HYBRID_PACING_SETMSS;
13977 return;
13978 }
13979 #endif
13980 orig_min = rack->r_ctl.rc_pace_min_segs;
13981 orig_max = rack->r_ctl.rc_pace_max_segs;
13982 user_max = ctf_fixed_maxseg(tp) * rack->rc_user_set_max_segs;
13983 if (ctf_fixed_maxseg(tp) != rack->r_ctl.rc_pace_min_segs)
13984 chged = 1;
13985 rack->r_ctl.rc_pace_min_segs = ctf_fixed_maxseg(tp);
13986 if (rack->use_fixed_rate || rack->rc_force_max_seg) {
13987 if (user_max != rack->r_ctl.rc_pace_max_segs)
13988 chged = 1;
13989 }
13990 if (rack->rc_force_max_seg) {
13991 rack->r_ctl.rc_pace_max_segs = user_max;
13992 } else if (rack->use_fixed_rate) {
13993 bw_est = rack_get_bw(rack);
13994 if ((rack->r_ctl.crte == NULL) ||
13995 (bw_est != rack->r_ctl.crte->rate)) {
13996 rack->r_ctl.rc_pace_max_segs = user_max;
13997 } else {
13998 /* We are pacing right at the hardware rate */
13999 uint32_t segsiz, pace_one;
14000
14001 if (rack_pace_one_seg ||
14002 (rack->r_ctl.rc_user_set_min_segs == 1))
14003 pace_one = 1;
14004 else
14005 pace_one = 0;
14006 segsiz = min(ctf_fixed_maxseg(tp),
14007 rack->r_ctl.rc_pace_min_segs);
14008 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size_w_divisor(
14009 tp, bw_est, segsiz, pace_one,
14010 rack->r_ctl.crte, NULL, rack->r_ctl.pace_len_divisor);
14011 }
14012 } else if (rack->rc_always_pace) {
14013 if (rack->r_ctl.gp_bw ||
14014 rack->r_ctl.init_rate) {
14015 /* We have a rate of some sort set */
14016 uint32_t orig;
14017
14018 bw_est = rack_get_bw(rack);
14019 orig = rack->r_ctl.rc_pace_max_segs;
14020 if (fill_override)
14021 rate_wanted = *fill_override;
14022 else
14023 rate_wanted = rack_get_gp_est(rack);
14024 if (rate_wanted) {
14025 /* We have something */
14026 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack,
14027 rate_wanted,
14028 ctf_fixed_maxseg(rack->rc_tp));
14029 } else
14030 rack->r_ctl.rc_pace_max_segs = rack->r_ctl.rc_pace_min_segs;
14031 if (orig != rack->r_ctl.rc_pace_max_segs)
14032 chged = 1;
14033 } else if ((rack->r_ctl.gp_bw == 0) &&
14034 (rack->r_ctl.rc_pace_max_segs == 0)) {
14035 /*
14036 * If we have nothing limit us to bursting
14037 * out IW sized pieces.
14038 */
14039 chged = 1;
14040 rack->r_ctl.rc_pace_max_segs = rc_init_window(rack);
14041 }
14042 }
14043 if (rack->r_ctl.rc_pace_max_segs > PACE_MAX_IP_BYTES) {
14044 chged = 1;
14045 rack->r_ctl.rc_pace_max_segs = PACE_MAX_IP_BYTES;
14046 }
14047 if (chged)
14048 rack_log_type_pacing_sizes(tp, rack, orig_min, orig_max, line, 2);
14049 }
14050
14051
14052 static void
rack_init_fsb_block(struct tcpcb * tp,struct tcp_rack * rack,int32_t flags)14053 rack_init_fsb_block(struct tcpcb *tp, struct tcp_rack *rack, int32_t flags)
14054 {
14055 #ifdef INET6
14056 struct ip6_hdr *ip6 = NULL;
14057 #endif
14058 #ifdef INET
14059 struct ip *ip = NULL;
14060 #endif
14061 struct udphdr *udp = NULL;
14062
14063 /* Ok lets fill in the fast block, it can only be used with no IP options! */
14064 #ifdef INET6
14065 if (rack->r_is_v6) {
14066 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
14067 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
14068 if (tp->t_port) {
14069 rack->r_ctl.fsb.tcp_ip_hdr_len += sizeof(struct udphdr);
14070 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
14071 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
14072 udp->uh_dport = tp->t_port;
14073 rack->r_ctl.fsb.udp = udp;
14074 rack->r_ctl.fsb.th = (struct tcphdr *)(udp + 1);
14075 } else
14076 {
14077 rack->r_ctl.fsb.th = (struct tcphdr *)(ip6 + 1);
14078 rack->r_ctl.fsb.udp = NULL;
14079 }
14080 tcpip_fillheaders(rack->rc_inp,
14081 tp->t_port,
14082 ip6, rack->r_ctl.fsb.th);
14083 rack->r_ctl.fsb.hoplimit = in6_selecthlim(rack->rc_inp, NULL);
14084 } else
14085 #endif /* INET6 */
14086 #ifdef INET
14087 {
14088 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct tcpiphdr);
14089 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
14090 if (tp->t_port) {
14091 rack->r_ctl.fsb.tcp_ip_hdr_len += sizeof(struct udphdr);
14092 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
14093 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
14094 udp->uh_dport = tp->t_port;
14095 rack->r_ctl.fsb.udp = udp;
14096 rack->r_ctl.fsb.th = (struct tcphdr *)(udp + 1);
14097 } else
14098 {
14099 rack->r_ctl.fsb.udp = NULL;
14100 rack->r_ctl.fsb.th = (struct tcphdr *)(ip + 1);
14101 }
14102 tcpip_fillheaders(rack->rc_inp,
14103 tp->t_port,
14104 ip, rack->r_ctl.fsb.th);
14105 rack->r_ctl.fsb.hoplimit = tptoinpcb(tp)->inp_ip_ttl;
14106 }
14107 #endif
14108 rack->r_ctl.fsb.recwin = lmin(lmax(sbspace(&tptosocket(tp)->so_rcv), 0),
14109 (long)TCP_MAXWIN << tp->rcv_scale);
14110 rack->r_fsb_inited = 1;
14111 }
14112
14113 static int
rack_init_fsb(struct tcpcb * tp,struct tcp_rack * rack)14114 rack_init_fsb(struct tcpcb *tp, struct tcp_rack *rack)
14115 {
14116 /*
14117 * Allocate the larger of spaces V6 if available else just
14118 * V4 and include udphdr (overbook)
14119 */
14120 #ifdef INET6
14121 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + sizeof(struct udphdr);
14122 #else
14123 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct tcpiphdr) + sizeof(struct udphdr);
14124 #endif
14125 rack->r_ctl.fsb.tcp_ip_hdr = malloc(rack->r_ctl.fsb.tcp_ip_hdr_len,
14126 M_TCPFSB, M_NOWAIT|M_ZERO);
14127 if (rack->r_ctl.fsb.tcp_ip_hdr == NULL) {
14128 return (ENOMEM);
14129 }
14130 rack->r_fsb_inited = 0;
14131 return (0);
14132 }
14133
14134 static void
rack_log_hystart_event(struct tcp_rack * rack,uint32_t high_seq,uint8_t mod)14135 rack_log_hystart_event(struct tcp_rack *rack, uint32_t high_seq, uint8_t mod)
14136 {
14137 /*
14138 * Types of logs (mod value)
14139 * 20 - Initial round setup
14140 * 21 - Rack declares a new round.
14141 */
14142 struct tcpcb *tp;
14143
14144 tp = rack->rc_tp;
14145 if (tcp_bblogging_on(tp)) {
14146 union tcp_log_stackspecific log;
14147 struct timeval tv;
14148
14149 memset(&log, 0, sizeof(log));
14150 log.u_bbr.flex1 = rack->r_ctl.current_round;
14151 log.u_bbr.flex2 = rack->r_ctl.roundends;
14152 log.u_bbr.flex3 = high_seq;
14153 log.u_bbr.flex4 = tp->snd_max;
14154 log.u_bbr.flex8 = mod;
14155 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
14156 log.u_bbr.cur_del_rate = rack->rc_tp->t_sndbytes;
14157 log.u_bbr.delRate = rack->rc_tp->t_snd_rxt_bytes;
14158 TCP_LOG_EVENTP(tp, NULL,
14159 &tptosocket(tp)->so_rcv,
14160 &tptosocket(tp)->so_snd,
14161 TCP_HYSTART, 0,
14162 0, &log, false, &tv);
14163 }
14164 }
14165
14166 static void
rack_deferred_init(struct tcpcb * tp,struct tcp_rack * rack)14167 rack_deferred_init(struct tcpcb *tp, struct tcp_rack *rack)
14168 {
14169 rack->rack_deferred_inited = 1;
14170 rack->r_ctl.roundends = tp->snd_max;
14171 rack->r_ctl.rc_high_rwnd = tp->snd_wnd;
14172 rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
14173 }
14174
14175 static void
rack_init_retransmit_value(struct tcp_rack * rack,int ctl)14176 rack_init_retransmit_value(struct tcp_rack *rack, int ctl)
14177 {
14178 /* Retransmit bit controls.
14179 *
14180 * The setting of these values control one of
14181 * three settings you can have and dictate
14182 * how rack does retransmissions. Note this
14183 * is in *any* mode i.e. pacing on or off DGP
14184 * fixed rate pacing, or just bursting rack.
14185 *
14186 * 1 - Use full sized retransmits i.e. limit
14187 * the size to whatever the pace_max_segments
14188 * size is.
14189 *
14190 * 2 - Use pacer min granularity as a guide to
14191 * the size combined with the current calculated
14192 * goodput b/w measurement. So for example if
14193 * the goodput is measured at 20Mbps we would
14194 * calculate 8125 (pacer minimum 250usec in
14195 * that b/w) and then round it up to the next
14196 * MSS i.e. for 1448 mss 6 MSS or 8688 bytes.
14197 *
14198 * 0 - The rack default 1 MSS (anything not 0/1/2
14199 * fall here too if we are setting via rack_init()).
14200 *
14201 */
14202 if (ctl == 1) {
14203 rack->full_size_rxt = 1;
14204 rack->shape_rxt_to_pacing_min = 0;
14205 } else if (ctl == 2) {
14206 rack->full_size_rxt = 0;
14207 rack->shape_rxt_to_pacing_min = 1;
14208 } else {
14209 rack->full_size_rxt = 0;
14210 rack->shape_rxt_to_pacing_min = 0;
14211 }
14212 }
14213
14214 static void
rack_log_chg_info(struct tcpcb * tp,struct tcp_rack * rack,uint8_t mod,uint32_t flex1,uint32_t flex2,uint32_t flex3)14215 rack_log_chg_info(struct tcpcb *tp, struct tcp_rack *rack, uint8_t mod,
14216 uint32_t flex1,
14217 uint32_t flex2,
14218 uint32_t flex3)
14219 {
14220 if (tcp_bblogging_on(rack->rc_tp)) {
14221 union tcp_log_stackspecific log;
14222 struct timeval tv;
14223
14224 memset(&log, 0, sizeof(log));
14225 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
14226 log.u_bbr.flex8 = mod;
14227 log.u_bbr.flex1 = flex1;
14228 log.u_bbr.flex2 = flex2;
14229 log.u_bbr.flex3 = flex3;
14230 tcp_log_event(tp, NULL, NULL, NULL, TCP_CHG_QUERY, 0,
14231 0, &log, false, NULL, __func__, __LINE__, &tv);
14232 }
14233 }
14234
14235 static int
rack_chg_query(struct tcpcb * tp,struct tcp_query_resp * reqr)14236 rack_chg_query(struct tcpcb *tp, struct tcp_query_resp *reqr)
14237 {
14238 struct tcp_rack *rack;
14239 struct rack_sendmap *rsm;
14240 int i;
14241
14242
14243 rack = (struct tcp_rack *)tp->t_fb_ptr;
14244 switch (reqr->req) {
14245 case TCP_QUERY_SENDMAP:
14246 if ((reqr->req_param == tp->snd_max) ||
14247 (tp->snd_max == tp->snd_una)){
14248 /* Unlikely */
14249 return (0);
14250 }
14251 rsm = tqhash_find(rack->r_ctl.tqh, reqr->req_param);
14252 if (rsm == NULL) {
14253 /* Can't find that seq -- unlikely */
14254 return (0);
14255 }
14256 reqr->sendmap_start = rsm->r_start;
14257 reqr->sendmap_end = rsm->r_end;
14258 reqr->sendmap_send_cnt = rsm->r_rtr_cnt;
14259 reqr->sendmap_fas = rsm->r_fas;
14260 if (reqr->sendmap_send_cnt > SNDMAP_NRTX)
14261 reqr->sendmap_send_cnt = SNDMAP_NRTX;
14262 for(i=0; i<reqr->sendmap_send_cnt; i++)
14263 reqr->sendmap_time[i] = rsm->r_tim_lastsent[i];
14264 reqr->sendmap_ack_arrival = rsm->r_ack_arrival;
14265 reqr->sendmap_flags = rsm->r_flags & SNDMAP_MASK;
14266 reqr->sendmap_r_rtr_bytes = rsm->r_rtr_bytes;
14267 reqr->sendmap_dupacks = rsm->r_dupack;
14268 rack_log_chg_info(tp, rack, 1,
14269 rsm->r_start,
14270 rsm->r_end,
14271 rsm->r_flags);
14272 return(1);
14273 break;
14274 case TCP_QUERY_TIMERS_UP:
14275 if (rack->r_ctl.rc_hpts_flags == 0) {
14276 /* no timers up */
14277 return (0);
14278 }
14279 reqr->timer_hpts_flags = rack->r_ctl.rc_hpts_flags;
14280 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
14281 reqr->timer_pacing_to = rack->r_ctl.rc_last_output_to;
14282 }
14283 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
14284 reqr->timer_timer_exp = rack->r_ctl.rc_timer_exp;
14285 }
14286 rack_log_chg_info(tp, rack, 2,
14287 rack->r_ctl.rc_hpts_flags,
14288 rack->r_ctl.rc_last_output_to,
14289 rack->r_ctl.rc_timer_exp);
14290 return (1);
14291 break;
14292 case TCP_QUERY_RACK_TIMES:
14293 /* Reordering items */
14294 reqr->rack_num_dsacks = rack->r_ctl.num_dsack;
14295 reqr->rack_reorder_ts = rack->r_ctl.rc_reorder_ts;
14296 /* Timerstamps and timers */
14297 reqr->rack_rxt_last_time = rack->r_ctl.rc_tlp_rxt_last_time;
14298 reqr->rack_min_rtt = rack->r_ctl.rc_rack_min_rtt;
14299 reqr->rack_rtt = rack->rc_rack_rtt;
14300 reqr->rack_tmit_time = rack->r_ctl.rc_rack_tmit_time;
14301 reqr->rack_srtt_measured = rack->rc_srtt_measure_made;
14302 /* PRR data */
14303 reqr->rack_sacked = rack->r_ctl.rc_sacked;
14304 reqr->rack_holes_rxt = rack->r_ctl.rc_holes_rxt;
14305 reqr->rack_prr_delivered = rack->r_ctl.rc_prr_delivered;
14306 reqr->rack_prr_recovery_fs = rack->r_ctl.rc_prr_recovery_fs;
14307 reqr->rack_prr_sndcnt = rack->r_ctl.rc_prr_sndcnt;
14308 reqr->rack_prr_out = rack->r_ctl.rc_prr_out;
14309 /* TLP and persists info */
14310 reqr->rack_tlp_out = rack->rc_tlp_in_progress;
14311 reqr->rack_tlp_cnt_out = rack->r_ctl.rc_tlp_cnt_out;
14312 if (rack->rc_in_persist) {
14313 reqr->rack_time_went_idle = rack->r_ctl.rc_went_idle_time;
14314 reqr->rack_in_persist = 1;
14315 } else {
14316 reqr->rack_time_went_idle = 0;
14317 reqr->rack_in_persist = 0;
14318 }
14319 if (rack->r_wanted_output)
14320 reqr->rack_wanted_output = 1;
14321 else
14322 reqr->rack_wanted_output = 0;
14323 return (1);
14324 break;
14325 default:
14326 return (-EINVAL);
14327 }
14328 }
14329
14330 static void
rack_switch_failed(struct tcpcb * tp)14331 rack_switch_failed(struct tcpcb *tp)
14332 {
14333 /*
14334 * This method gets called if a stack switch was
14335 * attempted and it failed. We are left
14336 * but our hpts timers were stopped and we
14337 * need to validate time units and t_flags2.
14338 */
14339 struct tcp_rack *rack;
14340 struct timeval tv;
14341 uint32_t cts;
14342 uint32_t toval;
14343 struct hpts_diag diag;
14344
14345 rack = (struct tcp_rack *)tp->t_fb_ptr;
14346 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_USEC);
14347 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
14348 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
14349 else
14350 tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
14351 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
14352 tp->t_flags2 |= TF2_MBUF_ACKCMP;
14353 if (tp->t_in_hpts > IHPTS_NONE) {
14354 /* Strange */
14355 return;
14356 }
14357 cts = tcp_get_usecs(&tv);
14358 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
14359 if (TSTMP_GT(rack->r_ctl.rc_last_output_to, cts)) {
14360 toval = rack->r_ctl.rc_last_output_to - cts;
14361 } else {
14362 /* one slot please */
14363 toval = HPTS_USECS_PER_SLOT;
14364 }
14365 } else if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
14366 if (TSTMP_GT(rack->r_ctl.rc_timer_exp, cts)) {
14367 toval = rack->r_ctl.rc_timer_exp - cts;
14368 } else {
14369 /* one slot please */
14370 toval = HPTS_USECS_PER_SLOT;
14371 }
14372 } else
14373 toval = HPTS_USECS_PER_SLOT;
14374 tcp_hpts_insert(tp, toval, &diag);
14375 rack_log_hpts_diag(rack, cts, &diag, &tv);
14376 }
14377
14378 static int
rack_init_outstanding(struct tcpcb * tp,struct tcp_rack * rack,uint32_t us_cts,void * ptr)14379 rack_init_outstanding(struct tcpcb *tp, struct tcp_rack *rack, uint32_t us_cts, void *ptr)
14380 {
14381 struct rack_sendmap *rsm, *ersm;
14382 int insret __diagused;
14383 /*
14384 * When initing outstanding, we must be quite careful
14385 * to not refer to tp->t_fb_ptr. This has the old rack
14386 * pointer in it, not the "new" one (when we are doing
14387 * a stack switch).
14388 */
14389
14390
14391 if (tp->t_fb->tfb_chg_query == NULL) {
14392 /* Create a send map for the current outstanding data */
14393
14394 rsm = rack_alloc(rack);
14395 if (rsm == NULL) {
14396 uma_zfree(rack_pcb_zone, ptr);
14397 return (ENOMEM);
14398 }
14399 rsm->r_no_rtt_allowed = 1;
14400 rsm->r_tim_lastsent[0] = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
14401 rsm->r_rtr_cnt = 1;
14402 rsm->r_rtr_bytes = 0;
14403 if (tp->t_flags & TF_SENTFIN)
14404 rsm->r_flags |= RACK_HAS_FIN;
14405 rsm->r_end = tp->snd_max;
14406 if (tp->snd_una == tp->iss) {
14407 /* The data space is one beyond snd_una */
14408 rsm->r_flags |= RACK_HAS_SYN;
14409 rsm->r_start = tp->iss;
14410 rsm->r_end = rsm->r_start + (tp->snd_max - tp->snd_una);
14411 } else
14412 rsm->r_start = tp->snd_una;
14413 rsm->r_dupack = 0;
14414 if (rack->rc_inp->inp_socket->so_snd.sb_mb != NULL) {
14415 rsm->m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd, 0, &rsm->soff);
14416 if (rsm->m) {
14417 rsm->orig_m_len = rsm->m->m_len;
14418 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
14419 } else {
14420 rsm->orig_m_len = 0;
14421 rsm->orig_t_space = 0;
14422 }
14423 } else {
14424 /*
14425 * This can happen if we have a stand-alone FIN or
14426 * SYN.
14427 */
14428 rsm->m = NULL;
14429 rsm->orig_m_len = 0;
14430 rsm->orig_t_space = 0;
14431 rsm->soff = 0;
14432 }
14433 #ifdef INVARIANTS
14434 if ((insret = tqhash_insert(rack->r_ctl.tqh, rsm)) != 0) {
14435 panic("Insert in tailq_hash fails ret:%d rack:%p rsm:%p",
14436 insret, rack, rsm);
14437 }
14438 #else
14439 (void)tqhash_insert(rack->r_ctl.tqh, rsm);
14440 #endif
14441 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
14442 rsm->r_in_tmap = 1;
14443 } else {
14444 /* We have a query mechanism, lets use it */
14445 struct tcp_query_resp qr;
14446 int i;
14447 tcp_seq at;
14448
14449 at = tp->snd_una;
14450 while (at != tp->snd_max) {
14451 memset(&qr, 0, sizeof(qr));
14452 qr.req = TCP_QUERY_SENDMAP;
14453 qr.req_param = at;
14454 if ((*tp->t_fb->tfb_chg_query)(tp, &qr) == 0)
14455 break;
14456 /* Move forward */
14457 at = qr.sendmap_end;
14458 /* Now lets build the entry for this one */
14459 rsm = rack_alloc(rack);
14460 if (rsm == NULL) {
14461 uma_zfree(rack_pcb_zone, ptr);
14462 return (ENOMEM);
14463 }
14464 memset(rsm, 0, sizeof(struct rack_sendmap));
14465 /* Now configure the rsm and insert it */
14466 rsm->r_dupack = qr.sendmap_dupacks;
14467 rsm->r_start = qr.sendmap_start;
14468 rsm->r_end = qr.sendmap_end;
14469 if (qr.sendmap_fas)
14470 rsm->r_fas = qr.sendmap_end;
14471 else
14472 rsm->r_fas = rsm->r_start - tp->snd_una;
14473 /*
14474 * We have carefully aligned the bits
14475 * so that all we have to do is copy over
14476 * the bits with the mask.
14477 */
14478 rsm->r_flags = qr.sendmap_flags & SNDMAP_MASK;
14479 rsm->r_rtr_bytes = qr.sendmap_r_rtr_bytes;
14480 rsm->r_rtr_cnt = qr.sendmap_send_cnt;
14481 rsm->r_ack_arrival = qr.sendmap_ack_arrival;
14482 for (i=0 ; i<rsm->r_rtr_cnt; i++)
14483 rsm->r_tim_lastsent[i] = qr.sendmap_time[i];
14484 rsm->m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd,
14485 (rsm->r_start - tp->snd_una), &rsm->soff);
14486 if (rsm->m) {
14487 rsm->orig_m_len = rsm->m->m_len;
14488 rsm->orig_t_space = M_TRAILINGROOM(rsm->m);
14489 } else {
14490 rsm->orig_m_len = 0;
14491 rsm->orig_t_space = 0;
14492 }
14493 #ifdef INVARIANTS
14494 if ((insret = tqhash_insert(rack->r_ctl.tqh, rsm)) != 0) {
14495 panic("Insert in tailq_hash fails ret:%d rack:%p rsm:%p",
14496 insret, rack, rsm);
14497 }
14498 #else
14499 (void)tqhash_insert(rack->r_ctl.tqh, rsm);
14500 #endif
14501 if ((rsm->r_flags & RACK_ACKED) == 0) {
14502 TAILQ_FOREACH(ersm, &rack->r_ctl.rc_tmap, r_tnext) {
14503 if (ersm->r_tim_lastsent[(ersm->r_rtr_cnt-1)] >
14504 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]) {
14505 /*
14506 * If the existing ersm was sent at
14507 * a later time than the new one, then
14508 * the new one should appear ahead of this
14509 * ersm.
14510 */
14511 rsm->r_in_tmap = 1;
14512 TAILQ_INSERT_BEFORE(ersm, rsm, r_tnext);
14513 break;
14514 }
14515 }
14516 if (rsm->r_in_tmap == 0) {
14517 /*
14518 * Not found so shove it on the tail.
14519 */
14520 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
14521 rsm->r_in_tmap = 1;
14522 }
14523 } else {
14524 if ((rack->r_ctl.rc_sacklast == NULL) ||
14525 (SEQ_GT(rsm->r_end, rack->r_ctl.rc_sacklast->r_end))) {
14526 rack->r_ctl.rc_sacklast = rsm;
14527 }
14528 }
14529 rack_log_chg_info(tp, rack, 3,
14530 rsm->r_start,
14531 rsm->r_end,
14532 rsm->r_flags);
14533 }
14534 }
14535 return (0);
14536 }
14537
14538
14539 static int32_t
rack_init(struct tcpcb * tp,void ** ptr)14540 rack_init(struct tcpcb *tp, void **ptr)
14541 {
14542 struct inpcb *inp = tptoinpcb(tp);
14543 struct tcp_rack *rack = NULL;
14544 uint32_t iwin, snt, us_cts;
14545 size_t sz;
14546 int err, no_query;
14547
14548 tcp_hpts_init(tp);
14549
14550 /*
14551 * First are we the initial or are we a switched stack?
14552 * If we are initing via tcp_newtcppcb the ptr passed
14553 * will be tp->t_fb_ptr. If its a stack switch that
14554 * has a previous stack we can query it will be a local
14555 * var that will in the end be set into t_fb_ptr.
14556 */
14557 if (ptr == &tp->t_fb_ptr)
14558 no_query = 1;
14559 else
14560 no_query = 0;
14561 *ptr = uma_zalloc(rack_pcb_zone, M_NOWAIT);
14562 if (*ptr == NULL) {
14563 /*
14564 * We need to allocate memory but cant. The INP and INP_INFO
14565 * locks and they are recursive (happens during setup. So a
14566 * scheme to drop the locks fails :(
14567 *
14568 */
14569 return(ENOMEM);
14570 }
14571 memset(*ptr, 0, sizeof(struct tcp_rack));
14572 rack = (struct tcp_rack *)*ptr;
14573 rack->r_ctl.tqh = malloc(sizeof(struct tailq_hash), M_TCPFSB, M_NOWAIT);
14574 if (rack->r_ctl.tqh == NULL) {
14575 uma_zfree(rack_pcb_zone, rack);
14576 return(ENOMEM);
14577 }
14578 tqhash_init(rack->r_ctl.tqh);
14579 TAILQ_INIT(&rack->r_ctl.rc_free);
14580 TAILQ_INIT(&rack->r_ctl.rc_tmap);
14581 rack->rc_tp = tp;
14582 rack->rc_inp = inp;
14583 /* Set the flag */
14584 rack->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0;
14585 /* Probably not needed but lets be sure */
14586 rack_clear_rate_sample(rack);
14587 /*
14588 * Save off the default values, socket options will poke
14589 * at these if pacing is not on or we have not yet
14590 * reached where pacing is on (gp_ready/fixed enabled).
14591 * When they get set into the CC module (when gp_ready
14592 * is enabled or we enable fixed) then we will set these
14593 * values into the CC and place in here the old values
14594 * so we have a restoral. Then we will set the flag
14595 * rc_pacing_cc_set. That way whenever we turn off pacing
14596 * or switch off this stack, we will know to go restore
14597 * the saved values.
14598 *
14599 * We specifically put into the beta the ecn value for pacing.
14600 */
14601 rack->rc_new_rnd_needed = 1;
14602 rack->r_ctl.rc_split_limit = V_tcp_map_split_limit;
14603 /* We want abe like behavior as well */
14604
14605 rack->r_ctl.rc_reorder_fade = rack_reorder_fade;
14606 rack->rc_allow_data_af_clo = rack_ignore_data_after_close;
14607 rack->r_ctl.rc_tlp_threshold = rack_tlp_thresh;
14608 if (rack_fill_cw_state)
14609 rack->rc_pace_to_cwnd = 1;
14610 if (rack_pacing_min_seg)
14611 rack->r_ctl.rc_user_set_min_segs = rack_pacing_min_seg;
14612 if (use_rack_rr)
14613 rack->use_rack_rr = 1;
14614 if (rack_dnd_default) {
14615 rack->rc_pace_dnd = 1;
14616 }
14617 if (V_tcp_delack_enabled)
14618 tp->t_delayed_ack = 1;
14619 else
14620 tp->t_delayed_ack = 0;
14621 #ifdef TCP_ACCOUNTING
14622 if (rack_tcp_accounting) {
14623 tp->t_flags2 |= TF2_TCP_ACCOUNTING;
14624 }
14625 #endif
14626 rack->r_ctl.pcm_i.cnt_alloc = RACK_DEFAULT_PCM_ARRAY;
14627 sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc);
14628 rack->r_ctl.pcm_s = malloc(sz,M_TCPPCM, M_NOWAIT);
14629 if (rack->r_ctl.pcm_s == NULL) {
14630 rack->r_ctl.pcm_i.cnt_alloc = 0;
14631 }
14632 rack->r_ctl.rack_per_upper_bound_ss = (uint8_t)rack_per_upper_bound_ss;
14633 rack->r_ctl.rack_per_upper_bound_ca = (uint8_t)rack_per_upper_bound_ca;
14634 if (rack_enable_shared_cwnd)
14635 rack->rack_enable_scwnd = 1;
14636 rack->r_ctl.pace_len_divisor = rack_default_pacing_divisor;
14637 rack->rc_user_set_max_segs = rack_hptsi_segments;
14638 rack->r_ctl.max_reduction = rack_max_reduce;
14639 rack->rc_force_max_seg = 0;
14640 TAILQ_INIT(&rack->r_ctl.opt_list);
14641 rack->r_ctl.rc_saved_beta = V_newreno_beta_ecn;
14642 rack->r_ctl.rc_saved_beta_ecn = V_newreno_beta_ecn;
14643 if (rack_hibeta_setting) {
14644 rack->rack_hibeta = 1;
14645 if ((rack_hibeta_setting >= 50) &&
14646 (rack_hibeta_setting <= 100)) {
14647 rack->r_ctl.rc_saved_beta = rack_hibeta_setting;
14648 rack->r_ctl.saved_hibeta = rack_hibeta_setting;
14649 }
14650 } else {
14651 rack->r_ctl.saved_hibeta = 50;
14652 }
14653 /*
14654 * We initialize to all ones so we never match 0
14655 * just in case the client sends in 0, it hopefully
14656 * will never have all 1's in ms :-)
14657 */
14658 rack->r_ctl.last_tm_mark = 0xffffffffffffffff;
14659 rack->r_ctl.rc_reorder_shift = rack_reorder_thresh;
14660 rack->r_ctl.rc_pkt_delay = rack_pkt_delay;
14661 rack->r_ctl.rc_tlp_cwnd_reduce = rack_lower_cwnd_at_tlp;
14662 rack->r_ctl.rc_lowest_us_rtt = 0xffffffff;
14663 rack->r_ctl.rc_highest_us_rtt = 0;
14664 rack->r_ctl.bw_rate_cap = rack_bw_rate_cap;
14665 rack->pcm_enabled = rack_pcm_is_enabled;
14666 if (rack_fillcw_bw_cap)
14667 rack->r_ctl.fillcw_cap = rack_fillcw_bw_cap;
14668 rack->r_ctl.timer_slop = TICKS_2_USEC(tcp_rexmit_slop);
14669 if (rack_use_cmp_acks)
14670 rack->r_use_cmp_ack = 1;
14671 if (rack_disable_prr)
14672 rack->rack_no_prr = 1;
14673 if (rack_gp_no_rec_chg)
14674 rack->rc_gp_no_rec_chg = 1;
14675 if (rack_pace_every_seg && tcp_can_enable_pacing()) {
14676 rack->r_ctl.pacing_method |= RACK_REG_PACING;
14677 rack->rc_always_pace = 1;
14678 if (rack->rack_hibeta)
14679 rack_set_cc_pacing(rack);
14680 } else
14681 rack->rc_always_pace = 0;
14682 if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack)
14683 rack->r_mbuf_queue = 1;
14684 else
14685 rack->r_mbuf_queue = 0;
14686 rack_set_pace_segments(tp, rack, __LINE__, NULL);
14687 if (rack_limits_scwnd)
14688 rack->r_limit_scw = 1;
14689 else
14690 rack->r_limit_scw = 0;
14691 rack_init_retransmit_value(rack, rack_rxt_controls);
14692 rack->rc_labc = V_tcp_abc_l_var;
14693 if (rack_honors_hpts_min_to)
14694 rack->r_use_hpts_min = 1;
14695 if (tp->snd_una != 0) {
14696 rack->rc_sendvars_notset = 0;
14697 /*
14698 * Make sure any TCP timers are not running.
14699 */
14700 tcp_timer_stop(tp);
14701 } else {
14702 /*
14703 * Server side, we are called from the
14704 * syn-cache. This means none of the
14705 * snd_una/max are set yet so we have
14706 * to defer this until the first send.
14707 */
14708 rack->rc_sendvars_notset = 1;
14709 }
14710
14711 rack->r_ctl.rc_rate_sample_method = rack_rate_sample_method;
14712 rack->rack_tlp_threshold_use = rack_tlp_threshold_use;
14713 rack->r_ctl.rc_prr_sendalot = rack_send_a_lot_in_prr;
14714 rack->r_ctl.rc_min_to = rack_min_to;
14715 microuptime(&rack->r_ctl.act_rcv_time);
14716 rack->r_ctl.rc_last_time_decay = rack->r_ctl.act_rcv_time;
14717 rack->r_ctl.rack_per_of_gp_ss = rack_per_of_gp_ss;
14718 if (rack_hw_up_only)
14719 rack->r_up_only = 1;
14720 if (rack_do_dyn_mul) {
14721 /* When dynamic adjustment is on CA needs to start at 100% */
14722 rack->rc_gp_dyn_mul = 1;
14723 if (rack_do_dyn_mul >= 100)
14724 rack->r_ctl.rack_per_of_gp_ca = rack_do_dyn_mul;
14725 } else
14726 rack->r_ctl.rack_per_of_gp_ca = rack_per_of_gp_ca;
14727 rack->r_ctl.rack_per_of_gp_rec = rack_per_of_gp_rec;
14728 if (rack_timely_off) {
14729 rack->rc_skip_timely = 1;
14730 }
14731 if (rack->rc_skip_timely) {
14732 rack->r_ctl.rack_per_of_gp_rec = 90;
14733 rack->r_ctl.rack_per_of_gp_ca = 100;
14734 rack->r_ctl.rack_per_of_gp_ss = 250;
14735 }
14736 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt;
14737 rack->r_ctl.rc_tlp_rxt_last_time = tcp_tv_to_msec(&rack->r_ctl.act_rcv_time);
14738 rack->r_ctl.last_rcv_tstmp_for_rtt = tcp_tv_to_msec(&rack->r_ctl.act_rcv_time);
14739
14740 setup_time_filter_small(&rack->r_ctl.rc_gp_min_rtt, FILTER_TYPE_MIN,
14741 rack_probertt_filter_life);
14742 us_cts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
14743 rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
14744 rack->r_ctl.rc_time_of_last_probertt = us_cts;
14745 rack->r_ctl.rc_went_idle_time = us_cts;
14746 rack->r_ctl.rc_time_probertt_starts = 0;
14747
14748 rack->r_ctl.gp_rnd_thresh = rack_rnd_cnt_req & 0xff;
14749 if (rack_rnd_cnt_req & 0x10000)
14750 rack->r_ctl.gate_to_fs = 1;
14751 rack->r_ctl.gp_gain_req = rack_gp_gain_req;
14752 if ((rack_rnd_cnt_req & 0x100) > 0) {
14753
14754 }
14755 if (rack_dsack_std_based & 0x1) {
14756 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */
14757 rack->rc_rack_tmr_std_based = 1;
14758 }
14759 if (rack_dsack_std_based & 0x2) {
14760 /* Basically this means rack timers are extended based on dsack by up to (2 * srtt) */
14761 rack->rc_rack_use_dsack = 1;
14762 }
14763 /* We require at least one measurement, even if the sysctl is 0 */
14764 if (rack_req_measurements)
14765 rack->r_ctl.req_measurements = rack_req_measurements;
14766 else
14767 rack->r_ctl.req_measurements = 1;
14768 if (rack_enable_hw_pacing)
14769 rack->rack_hdw_pace_ena = 1;
14770 if (rack_hw_rate_caps)
14771 rack->r_rack_hw_rate_caps = 1;
14772 if (rack_non_rxt_use_cr)
14773 rack->rack_rec_nonrxt_use_cr = 1;
14774 /* Lets setup the fsb block */
14775 err = rack_init_fsb(tp, rack);
14776 if (err) {
14777 uma_zfree(rack_pcb_zone, *ptr);
14778 *ptr = NULL;
14779 return (err);
14780 }
14781 if (rack_do_hystart) {
14782 tp->t_ccv.flags |= CCF_HYSTART_ALLOWED;
14783 if (rack_do_hystart > 1)
14784 tp->t_ccv.flags |= CCF_HYSTART_CAN_SH_CWND;
14785 if (rack_do_hystart > 2)
14786 tp->t_ccv.flags |= CCF_HYSTART_CONS_SSTH;
14787 }
14788 /* Log what we will do with queries */
14789 rack_log_chg_info(tp, rack, 7,
14790 no_query, 0, 0);
14791 if (rack_def_profile)
14792 rack_set_profile(rack, rack_def_profile);
14793 /* Cancel the GP measurement in progress */
14794 tp->t_flags &= ~TF_GPUTINPROG;
14795 if ((tp->t_state != TCPS_CLOSED) &&
14796 (tp->t_state != TCPS_TIME_WAIT)) {
14797 /*
14798 * We are already open, we may
14799 * need to adjust a few things.
14800 */
14801 if (SEQ_GT(tp->snd_max, tp->iss))
14802 snt = tp->snd_max - tp->iss;
14803 else
14804 snt = 0;
14805 iwin = rc_init_window(rack);
14806 if ((snt < iwin) &&
14807 (no_query == 1)) {
14808 /* We are not past the initial window
14809 * on the first init (i.e. a stack switch
14810 * has not yet occured) so we need to make
14811 * sure cwnd and ssthresh is correct.
14812 */
14813 if (tp->snd_cwnd < iwin)
14814 tp->snd_cwnd = iwin;
14815 /*
14816 * If we are within the initial window
14817 * we want ssthresh to be unlimited. Setting
14818 * it to the rwnd (which the default stack does
14819 * and older racks) is not really a good idea
14820 * since we want to be in SS and grow both the
14821 * cwnd and the rwnd (via dynamic rwnd growth). If
14822 * we set it to the rwnd then as the peer grows its
14823 * rwnd we will be stuck in CA and never hit SS.
14824 *
14825 * Its far better to raise it up high (this takes the
14826 * risk that there as been a loss already, probably
14827 * we should have an indicator in all stacks of loss
14828 * but we don't), but considering the normal use this
14829 * is a risk worth taking. The consequences of not
14830 * hitting SS are far worse than going one more time
14831 * into it early on (before we have sent even a IW).
14832 * It is highly unlikely that we will have had a loss
14833 * before getting the IW out.
14834 */
14835 tp->snd_ssthresh = 0xffffffff;
14836 }
14837 /*
14838 * Any init based on sequence numbers
14839 * should be done in the deferred init path
14840 * since we can be CLOSED and not have them
14841 * inited when rack_init() is called. We
14842 * are not closed so lets call it.
14843 */
14844 rack_deferred_init(tp, rack);
14845 }
14846 if ((tp->t_state != TCPS_CLOSED) &&
14847 (tp->t_state != TCPS_TIME_WAIT) &&
14848 (no_query == 0) &&
14849 (tp->snd_una != tp->snd_max)) {
14850 err = rack_init_outstanding(tp, rack, us_cts, *ptr);
14851 if (err) {
14852 *ptr = NULL;
14853 return(err);
14854 }
14855 }
14856 rack_stop_all_timers(tp, rack);
14857 /* Setup all the t_flags2 */
14858 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
14859 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
14860 else
14861 tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
14862 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
14863 tp->t_flags2 |= TF2_MBUF_ACKCMP;
14864 /*
14865 * Timers in Rack are kept in microseconds so lets
14866 * convert any initial incoming variables
14867 * from ticks into usecs. Note that we
14868 * also change the values of t_srtt and t_rttvar, if
14869 * they are non-zero. They are kept with a 5
14870 * bit decimal so we have to carefully convert
14871 * these to get the full precision.
14872 */
14873 rack_convert_rtts(tp);
14874 rack_log_hystart_event(rack, rack->r_ctl.roundends, 20);
14875 if ((tptoinpcb(tp)->inp_flags & INP_DROPPED) == 0) {
14876 /* We do not start any timers on DROPPED connections */
14877 if (tp->t_fb->tfb_chg_query == NULL) {
14878 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
14879 } else {
14880 struct tcp_query_resp qr;
14881 int ret;
14882
14883 memset(&qr, 0, sizeof(qr));
14884
14885 /* Get the misc time stamps and such for rack */
14886 qr.req = TCP_QUERY_RACK_TIMES;
14887 ret = (*tp->t_fb->tfb_chg_query)(tp, &qr);
14888 if (ret == 1) {
14889 rack->r_ctl.rc_reorder_ts = qr.rack_reorder_ts;
14890 rack->r_ctl.num_dsack = qr.rack_num_dsacks;
14891 rack->r_ctl.rc_tlp_rxt_last_time = qr.rack_rxt_last_time;
14892 rack->r_ctl.rc_rack_min_rtt = qr.rack_min_rtt;
14893 rack->rc_rack_rtt = qr.rack_rtt;
14894 rack->r_ctl.rc_rack_tmit_time = qr.rack_tmit_time;
14895 rack->r_ctl.rc_sacked = qr.rack_sacked;
14896 rack->r_ctl.rc_holes_rxt = qr.rack_holes_rxt;
14897 rack->r_ctl.rc_prr_delivered = qr.rack_prr_delivered;
14898 rack->r_ctl.rc_prr_recovery_fs = qr.rack_prr_recovery_fs;
14899 rack->r_ctl.rc_prr_sndcnt = qr.rack_prr_sndcnt;
14900 rack->r_ctl.rc_prr_out = qr.rack_prr_out;
14901 if (qr.rack_tlp_out) {
14902 rack->rc_tlp_in_progress = 1;
14903 rack->r_ctl.rc_tlp_cnt_out = qr.rack_tlp_cnt_out;
14904 } else {
14905 rack->rc_tlp_in_progress = 0;
14906 rack->r_ctl.rc_tlp_cnt_out = 0;
14907 }
14908 if (qr.rack_srtt_measured)
14909 rack->rc_srtt_measure_made = 1;
14910 if (qr.rack_in_persist == 1) {
14911 rack->r_ctl.rc_went_idle_time = qr.rack_time_went_idle;
14912 #ifdef NETFLIX_SHARED_CWND
14913 if (rack->r_ctl.rc_scw) {
14914 tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
14915 rack->rack_scwnd_is_idle = 1;
14916 }
14917 #endif
14918 rack->r_ctl.persist_lost_ends = 0;
14919 rack->probe_not_answered = 0;
14920 rack->forced_ack = 0;
14921 tp->t_rxtshift = 0;
14922 rack->rc_in_persist = 1;
14923 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
14924 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
14925 }
14926 if (qr.rack_wanted_output)
14927 rack->r_wanted_output = 1;
14928 rack_log_chg_info(tp, rack, 6,
14929 qr.rack_min_rtt,
14930 qr.rack_rtt,
14931 qr.rack_reorder_ts);
14932 }
14933 /* Get the old stack timers */
14934 qr.req_param = 0;
14935 qr.req = TCP_QUERY_TIMERS_UP;
14936 ret = (*tp->t_fb->tfb_chg_query)(tp, &qr);
14937 if (ret) {
14938 /*
14939 * non-zero return means we have a timer('s)
14940 * to start. Zero means no timer (no keepalive
14941 * I suppose).
14942 */
14943 uint32_t tov = 0;
14944
14945 rack->r_ctl.rc_hpts_flags = qr.timer_hpts_flags;
14946 if (qr.timer_hpts_flags & PACE_PKT_OUTPUT) {
14947 rack->r_ctl.rc_last_output_to = qr.timer_pacing_to;
14948 if (TSTMP_GT(qr.timer_pacing_to, us_cts))
14949 tov = qr.timer_pacing_to - us_cts;
14950 else
14951 tov = HPTS_USECS_PER_SLOT;
14952 }
14953 if (qr.timer_hpts_flags & PACE_TMR_MASK) {
14954 rack->r_ctl.rc_timer_exp = qr.timer_timer_exp;
14955 if (tov == 0) {
14956 if (TSTMP_GT(qr.timer_timer_exp, us_cts))
14957 tov = qr.timer_timer_exp - us_cts;
14958 else
14959 tov = HPTS_USECS_PER_SLOT;
14960 }
14961 }
14962 rack_log_chg_info(tp, rack, 4,
14963 rack->r_ctl.rc_hpts_flags,
14964 rack->r_ctl.rc_last_output_to,
14965 rack->r_ctl.rc_timer_exp);
14966 if (tov) {
14967 struct hpts_diag diag;
14968
14969 tcp_hpts_insert(tp, tov, &diag);
14970 rack_log_hpts_diag(rack, us_cts, &diag, &rack->r_ctl.act_rcv_time);
14971 }
14972 }
14973 }
14974 rack_log_rtt_shrinks(rack, us_cts, tp->t_rxtcur,
14975 __LINE__, RACK_RTTS_INIT);
14976 }
14977 return (0);
14978 }
14979
14980 static int
rack_handoff_ok(struct tcpcb * tp)14981 rack_handoff_ok(struct tcpcb *tp)
14982 {
14983 if ((tp->t_state == TCPS_CLOSED) ||
14984 (tp->t_state == TCPS_LISTEN)) {
14985 /* Sure no problem though it may not stick */
14986 return (0);
14987 }
14988 if ((tp->t_state == TCPS_SYN_SENT) ||
14989 (tp->t_state == TCPS_SYN_RECEIVED)) {
14990 /*
14991 * We really don't know if you support sack,
14992 * you have to get to ESTAB or beyond to tell.
14993 */
14994 return (EAGAIN);
14995 }
14996 if ((tp->t_flags & TF_SENTFIN) && ((tp->snd_max - tp->snd_una) > 1)) {
14997 /*
14998 * Rack will only send a FIN after all data is acknowledged.
14999 * So in this case we have more data outstanding. We can't
15000 * switch stacks until either all data and only the FIN
15001 * is left (in which case rack_init() now knows how
15002 * to deal with that) <or> all is acknowledged and we
15003 * are only left with incoming data, though why you
15004 * would want to switch to rack after all data is acknowledged
15005 * I have no idea (rrs)!
15006 */
15007 return (EAGAIN);
15008 }
15009 if ((tp->t_flags & TF_SACK_PERMIT) || rack_sack_not_required){
15010 return (0);
15011 }
15012 /*
15013 * If we reach here we don't do SACK on this connection so we can
15014 * never do rack.
15015 */
15016 return (EINVAL);
15017 }
15018
15019 static void
rack_fini(struct tcpcb * tp,int32_t tcb_is_purged)15020 rack_fini(struct tcpcb *tp, int32_t tcb_is_purged)
15021 {
15022
15023 if (tp->t_fb_ptr) {
15024 uint32_t cnt_free = 0;
15025 struct tcp_rack *rack;
15026 struct rack_sendmap *rsm;
15027
15028 tcp_handle_orphaned_packets(tp);
15029 tp->t_flags &= ~TF_FORCEDATA;
15030 rack = (struct tcp_rack *)tp->t_fb_ptr;
15031 rack_log_pacing_delay_calc(rack,
15032 0,
15033 0,
15034 0,
15035 rack_get_gp_est(rack), /* delRate */
15036 rack_get_lt_bw(rack), /* rttProp */
15037 20, __LINE__, NULL, 0);
15038 #ifdef NETFLIX_SHARED_CWND
15039 if (rack->r_ctl.rc_scw) {
15040 uint32_t limit;
15041
15042 if (rack->r_limit_scw)
15043 limit = max(1, rack->r_ctl.rc_lowest_us_rtt);
15044 else
15045 limit = 0;
15046 tcp_shared_cwnd_free_full(tp, rack->r_ctl.rc_scw,
15047 rack->r_ctl.rc_scw_index,
15048 limit);
15049 rack->r_ctl.rc_scw = NULL;
15050 }
15051 #endif
15052 if (rack->r_ctl.fsb.tcp_ip_hdr) {
15053 free(rack->r_ctl.fsb.tcp_ip_hdr, M_TCPFSB);
15054 rack->r_ctl.fsb.tcp_ip_hdr = NULL;
15055 rack->r_ctl.fsb.th = NULL;
15056 }
15057 if (rack->rc_always_pace == 1) {
15058 rack_remove_pacing(rack);
15059 }
15060 /* Clean up any options if they were not applied */
15061 while (!TAILQ_EMPTY(&rack->r_ctl.opt_list)) {
15062 struct deferred_opt_list *dol;
15063
15064 dol = TAILQ_FIRST(&rack->r_ctl.opt_list);
15065 TAILQ_REMOVE(&rack->r_ctl.opt_list, dol, next);
15066 free(dol, M_TCPDO);
15067 }
15068 /* rack does not use force data but other stacks may clear it */
15069 if (rack->r_ctl.crte != NULL) {
15070 tcp_rel_pacing_rate(rack->r_ctl.crte, tp);
15071 rack->rack_hdrw_pacing = 0;
15072 rack->r_ctl.crte = NULL;
15073 }
15074 #ifdef TCP_BLACKBOX
15075 tcp_log_flowend(tp);
15076 #endif
15077 /*
15078 * Lets take a different approach to purging just
15079 * get each one and free it like a cum-ack would and
15080 * not use a foreach loop.
15081 */
15082 rsm = tqhash_min(rack->r_ctl.tqh);
15083 while (rsm) {
15084 tqhash_remove(rack->r_ctl.tqh, rsm, REMOVE_TYPE_CUMACK);
15085 rack->r_ctl.rc_num_maps_alloced--;
15086 uma_zfree(rack_zone, rsm);
15087 rsm = tqhash_min(rack->r_ctl.tqh);
15088 }
15089 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
15090 while (rsm) {
15091 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
15092 rack->r_ctl.rc_num_maps_alloced--;
15093 rack->rc_free_cnt--;
15094 cnt_free++;
15095 uma_zfree(rack_zone, rsm);
15096 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
15097 }
15098 if (rack->r_ctl.pcm_s != NULL) {
15099 free(rack->r_ctl.pcm_s, M_TCPPCM);
15100 rack->r_ctl.pcm_s = NULL;
15101 rack->r_ctl.pcm_i.cnt_alloc = 0;
15102 rack->r_ctl.pcm_i.cnt = 0;
15103 }
15104 if ((rack->r_ctl.rc_num_maps_alloced > 0) &&
15105 (tcp_bblogging_on(tp))) {
15106 union tcp_log_stackspecific log;
15107 struct timeval tv;
15108
15109 memset(&log, 0, sizeof(log));
15110 log.u_bbr.flex8 = 10;
15111 log.u_bbr.flex1 = rack->r_ctl.rc_num_maps_alloced;
15112 log.u_bbr.flex2 = rack->rc_free_cnt;
15113 log.u_bbr.flex3 = cnt_free;
15114 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
15115 rsm = tqhash_min(rack->r_ctl.tqh);
15116 log.u_bbr.delRate = (uintptr_t)rsm;
15117 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
15118 log.u_bbr.cur_del_rate = (uintptr_t)rsm;
15119 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
15120 log.u_bbr.pkt_epoch = __LINE__;
15121 (void)tcp_log_event(tp, NULL, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK,
15122 0, &log, false, NULL, NULL, 0, &tv);
15123 }
15124 KASSERT((rack->r_ctl.rc_num_maps_alloced == 0),
15125 ("rack:%p num_aloc:%u after freeing all?",
15126 rack,
15127 rack->r_ctl.rc_num_maps_alloced));
15128 rack->rc_free_cnt = 0;
15129 free(rack->r_ctl.tqh, M_TCPFSB);
15130 rack->r_ctl.tqh = NULL;
15131 uma_zfree(rack_pcb_zone, tp->t_fb_ptr);
15132 tp->t_fb_ptr = NULL;
15133 }
15134 /* Make sure snd_nxt is correctly set */
15135 tp->snd_nxt = tp->snd_max;
15136 }
15137
15138 static void
rack_set_state(struct tcpcb * tp,struct tcp_rack * rack)15139 rack_set_state(struct tcpcb *tp, struct tcp_rack *rack)
15140 {
15141 if ((rack->r_state == TCPS_CLOSED) && (tp->t_state != TCPS_CLOSED)) {
15142 rack->r_is_v6 = (tptoinpcb(tp)->inp_vflag & INP_IPV6) != 0;
15143 }
15144 switch (tp->t_state) {
15145 case TCPS_SYN_SENT:
15146 rack->r_state = TCPS_SYN_SENT;
15147 rack->r_substate = rack_do_syn_sent;
15148 break;
15149 case TCPS_SYN_RECEIVED:
15150 rack->r_state = TCPS_SYN_RECEIVED;
15151 rack->r_substate = rack_do_syn_recv;
15152 break;
15153 case TCPS_ESTABLISHED:
15154 rack_set_pace_segments(tp, rack, __LINE__, NULL);
15155 rack->r_state = TCPS_ESTABLISHED;
15156 rack->r_substate = rack_do_established;
15157 break;
15158 case TCPS_CLOSE_WAIT:
15159 rack->r_state = TCPS_CLOSE_WAIT;
15160 rack->r_substate = rack_do_close_wait;
15161 break;
15162 case TCPS_FIN_WAIT_1:
15163 rack_set_pace_segments(tp, rack, __LINE__, NULL);
15164 rack->r_state = TCPS_FIN_WAIT_1;
15165 rack->r_substate = rack_do_fin_wait_1;
15166 break;
15167 case TCPS_CLOSING:
15168 rack_set_pace_segments(tp, rack, __LINE__, NULL);
15169 rack->r_state = TCPS_CLOSING;
15170 rack->r_substate = rack_do_closing;
15171 break;
15172 case TCPS_LAST_ACK:
15173 rack_set_pace_segments(tp, rack, __LINE__, NULL);
15174 rack->r_state = TCPS_LAST_ACK;
15175 rack->r_substate = rack_do_lastack;
15176 break;
15177 case TCPS_FIN_WAIT_2:
15178 rack->r_state = TCPS_FIN_WAIT_2;
15179 rack->r_substate = rack_do_fin_wait_2;
15180 break;
15181 case TCPS_LISTEN:
15182 case TCPS_CLOSED:
15183 case TCPS_TIME_WAIT:
15184 default:
15185 break;
15186 };
15187 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
15188 rack->rc_tp->t_flags2 |= TF2_MBUF_ACKCMP;
15189
15190 }
15191
15192 static void
rack_timer_audit(struct tcpcb * tp,struct tcp_rack * rack,struct sockbuf * sb)15193 rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb)
15194 {
15195 /*
15196 * We received an ack, and then did not
15197 * call send or were bounced out due to the
15198 * hpts was running. Now a timer is up as well, is
15199 * it the right timer?
15200 */
15201 struct rack_sendmap *rsm;
15202 int tmr_up;
15203
15204 tmr_up = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
15205 if (tcp_in_hpts(rack->rc_tp) == 0) {
15206 /*
15207 * Ok we probably need some timer up, but no
15208 * matter what the mask we are not in hpts. We
15209 * may have received an old ack and thus did nothing.
15210 */
15211 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
15212 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
15213 return;
15214 }
15215 if (rack->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
15216 return;
15217 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
15218 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
15219 (tmr_up == PACE_TMR_RXT)) {
15220 /* Should be an RXT */
15221 return;
15222 }
15223 if (rsm == NULL) {
15224 /* Nothing outstanding? */
15225 if (tp->t_flags & TF_DELACK) {
15226 if (tmr_up == PACE_TMR_DELACK)
15227 /* We are supposed to have delayed ack up and we do */
15228 return;
15229 } else if (((V_tcp_always_keepalive ||
15230 rack->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
15231 (tp->t_state <= TCPS_CLOSING)) &&
15232 (tmr_up == PACE_TMR_KEEP) &&
15233 (tp->snd_max == tp->snd_una)) {
15234 /* We should have keep alive up and we do */
15235 return;
15236 }
15237 }
15238 if (SEQ_GT(tp->snd_max, tp->snd_una) &&
15239 ((tmr_up == PACE_TMR_TLP) ||
15240 (tmr_up == PACE_TMR_RACK) ||
15241 (tmr_up == PACE_TMR_RXT))) {
15242 /*
15243 * Either a Rack, TLP or RXT is fine if we
15244 * have outstanding data.
15245 */
15246 return;
15247 } else if (tmr_up == PACE_TMR_DELACK) {
15248 /*
15249 * If the delayed ack was going to go off
15250 * before the rtx/tlp/rack timer were going to
15251 * expire, then that would be the timer in control.
15252 * Note we don't check the time here trusting the
15253 * code is correct.
15254 */
15255 return;
15256 }
15257 /*
15258 * Ok the timer originally started is not what we want now.
15259 * We will force the hpts to be stopped if any, and restart
15260 * with the slot set to what was in the saved slot.
15261 */
15262 if (tcp_in_hpts(rack->rc_tp)) {
15263 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
15264 uint32_t us_cts;
15265
15266 us_cts = tcp_get_usecs(NULL);
15267 if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) {
15268 rack->r_early = 1;
15269 rack->r_ctl.rc_agg_early += (rack->r_ctl.rc_last_output_to - us_cts);
15270 }
15271 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
15272 }
15273 tcp_hpts_remove(rack->rc_tp);
15274 }
15275 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
15276 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
15277 }
15278
15279
15280 static void
rack_do_win_updates(struct tcpcb * tp,struct tcp_rack * rack,uint32_t tiwin,uint32_t seq,uint32_t ack,uint32_t cts)15281 rack_do_win_updates(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tiwin, uint32_t seq, uint32_t ack, uint32_t cts)
15282 {
15283 if ((SEQ_LT(tp->snd_wl1, seq) ||
15284 (tp->snd_wl1 == seq && (SEQ_LT(tp->snd_wl2, ack) ||
15285 (tp->snd_wl2 == ack && tiwin > tp->snd_wnd))))) {
15286 /* keep track of pure window updates */
15287 if ((tp->snd_wl2 == ack) && (tiwin > tp->snd_wnd))
15288 KMOD_TCPSTAT_INC(tcps_rcvwinupd);
15289 tp->snd_wnd = tiwin;
15290 rack_validate_fo_sendwin_up(tp, rack);
15291 tp->snd_wl1 = seq;
15292 tp->snd_wl2 = ack;
15293 if (tp->snd_wnd > tp->max_sndwnd)
15294 tp->max_sndwnd = tp->snd_wnd;
15295 rack->r_wanted_output = 1;
15296 } else if ((tp->snd_wl2 == ack) && (tiwin < tp->snd_wnd)) {
15297 tp->snd_wnd = tiwin;
15298 rack_validate_fo_sendwin_up(tp, rack);
15299 tp->snd_wl1 = seq;
15300 tp->snd_wl2 = ack;
15301 } else {
15302 /* Not a valid win update */
15303 return;
15304 }
15305 if (tp->snd_wnd > tp->max_sndwnd)
15306 tp->max_sndwnd = tp->snd_wnd;
15307 /* Do we exit persists? */
15308 if ((rack->rc_in_persist != 0) &&
15309 (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2),
15310 rack->r_ctl.rc_pace_min_segs))) {
15311 rack_exit_persist(tp, rack, cts);
15312 }
15313 /* Do we enter persists? */
15314 if ((rack->rc_in_persist == 0) &&
15315 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) &&
15316 TCPS_HAVEESTABLISHED(tp->t_state) &&
15317 ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) &&
15318 sbavail(&tptosocket(tp)->so_snd) &&
15319 (sbavail(&tptosocket(tp)->so_snd) > tp->snd_wnd)) {
15320 /*
15321 * Here the rwnd is less than
15322 * the pacing size, we are established,
15323 * nothing is outstanding, and there is
15324 * data to send. Enter persists.
15325 */
15326 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime, ack);
15327 }
15328 }
15329
15330 static void
rack_log_input_packet(struct tcpcb * tp,struct tcp_rack * rack,struct tcp_ackent * ae,int ackval,uint32_t high_seq)15331 rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent *ae, int ackval, uint32_t high_seq)
15332 {
15333
15334 if (tcp_bblogging_on(rack->rc_tp)) {
15335 struct inpcb *inp = tptoinpcb(tp);
15336 union tcp_log_stackspecific log;
15337 struct timeval ltv;
15338 char tcp_hdr_buf[60];
15339 struct tcphdr *th;
15340 struct timespec ts;
15341 uint32_t orig_snd_una;
15342 uint8_t xx = 0;
15343
15344 #ifdef TCP_REQUEST_TRK
15345 struct tcp_sendfile_track *tcp_req;
15346
15347 if (SEQ_GT(ae->ack, tp->snd_una)) {
15348 tcp_req = tcp_req_find_req_for_seq(tp, (ae->ack-1));
15349 } else {
15350 tcp_req = tcp_req_find_req_for_seq(tp, ae->ack);
15351 }
15352 #endif
15353 memset(&log, 0, sizeof(log));
15354 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
15355 if (rack->rack_no_prr == 0)
15356 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
15357 else
15358 log.u_bbr.flex1 = 0;
15359 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
15360 log.u_bbr.use_lt_bw <<= 1;
15361 log.u_bbr.use_lt_bw |= rack->r_might_revert;
15362 log.u_bbr.flex2 = rack->r_ctl.rc_num_maps_alloced;
15363 log.u_bbr.bbr_state = rack->rc_free_cnt;
15364 log.u_bbr.inflight = ctf_flight_size(tp, rack->r_ctl.rc_sacked);
15365 log.u_bbr.pkts_out = tp->t_maxseg;
15366 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
15367 log.u_bbr.flex7 = 1;
15368 log.u_bbr.lost = ae->flags;
15369 log.u_bbr.cwnd_gain = ackval;
15370 log.u_bbr.pacing_gain = 0x2;
15371 if (ae->flags & TSTMP_HDWR) {
15372 /* Record the hardware timestamp if present */
15373 log.u_bbr.flex3 = M_TSTMP;
15374 ts.tv_sec = ae->timestamp / 1000000000;
15375 ts.tv_nsec = ae->timestamp % 1000000000;
15376 ltv.tv_sec = ts.tv_sec;
15377 ltv.tv_usec = ts.tv_nsec / 1000;
15378 log.u_bbr.lt_epoch = tcp_tv_to_usec(<v);
15379 } else if (ae->flags & TSTMP_LRO) {
15380 /* Record the LRO the arrival timestamp */
15381 log.u_bbr.flex3 = M_TSTMP_LRO;
15382 ts.tv_sec = ae->timestamp / 1000000000;
15383 ts.tv_nsec = ae->timestamp % 1000000000;
15384 ltv.tv_sec = ts.tv_sec;
15385 ltv.tv_usec = ts.tv_nsec / 1000;
15386 log.u_bbr.flex5 = tcp_tv_to_usec(<v);
15387 }
15388 log.u_bbr.timeStamp = tcp_get_usecs(<v);
15389 /* Log the rcv time */
15390 log.u_bbr.delRate = ae->timestamp;
15391 #ifdef TCP_REQUEST_TRK
15392 log.u_bbr.applimited = tp->t_tcpreq_closed;
15393 log.u_bbr.applimited <<= 8;
15394 log.u_bbr.applimited |= tp->t_tcpreq_open;
15395 log.u_bbr.applimited <<= 8;
15396 log.u_bbr.applimited |= tp->t_tcpreq_req;
15397 if (tcp_req) {
15398 /* Copy out any client req info */
15399 /* seconds */
15400 log.u_bbr.pkt_epoch = (tcp_req->localtime / HPTS_USEC_IN_SEC);
15401 /* useconds */
15402 log.u_bbr.delivered = (tcp_req->localtime % HPTS_USEC_IN_SEC);
15403 log.u_bbr.rttProp = tcp_req->timestamp;
15404 log.u_bbr.cur_del_rate = tcp_req->start;
15405 if (tcp_req->flags & TCP_TRK_TRACK_FLG_OPEN) {
15406 log.u_bbr.flex8 |= 1;
15407 } else {
15408 log.u_bbr.flex8 |= 2;
15409 log.u_bbr.bw_inuse = tcp_req->end;
15410 }
15411 log.u_bbr.flex6 = tcp_req->start_seq;
15412 if (tcp_req->flags & TCP_TRK_TRACK_FLG_COMP) {
15413 log.u_bbr.flex8 |= 4;
15414 log.u_bbr.epoch = tcp_req->end_seq;
15415 }
15416 }
15417 #endif
15418 memset(tcp_hdr_buf, 0, sizeof(tcp_hdr_buf));
15419 th = (struct tcphdr *)tcp_hdr_buf;
15420 th->th_seq = ae->seq;
15421 th->th_ack = ae->ack;
15422 th->th_win = ae->win;
15423 /* Now fill in the ports */
15424 th->th_sport = inp->inp_fport;
15425 th->th_dport = inp->inp_lport;
15426 tcp_set_flags(th, ae->flags);
15427 /* Now do we have a timestamp option? */
15428 if (ae->flags & HAS_TSTMP) {
15429 u_char *cp;
15430 uint32_t val;
15431
15432 th->th_off = ((sizeof(struct tcphdr) + TCPOLEN_TSTAMP_APPA) >> 2);
15433 cp = (u_char *)(th + 1);
15434 *cp = TCPOPT_NOP;
15435 cp++;
15436 *cp = TCPOPT_NOP;
15437 cp++;
15438 *cp = TCPOPT_TIMESTAMP;
15439 cp++;
15440 *cp = TCPOLEN_TIMESTAMP;
15441 cp++;
15442 val = htonl(ae->ts_value);
15443 bcopy((char *)&val,
15444 (char *)cp, sizeof(uint32_t));
15445 val = htonl(ae->ts_echo);
15446 bcopy((char *)&val,
15447 (char *)(cp + 4), sizeof(uint32_t));
15448 } else
15449 th->th_off = (sizeof(struct tcphdr) >> 2);
15450
15451 /*
15452 * For sane logging we need to play a little trick.
15453 * If the ack were fully processed we would have moved
15454 * snd_una to high_seq, but since compressed acks are
15455 * processed in two phases, at this point (logging) snd_una
15456 * won't be advanced. So we would see multiple acks showing
15457 * the advancement. We can prevent that by "pretending" that
15458 * snd_una was advanced and then un-advancing it so that the
15459 * logging code has the right value for tlb_snd_una.
15460 */
15461 if (tp->snd_una != high_seq) {
15462 orig_snd_una = tp->snd_una;
15463 tp->snd_una = high_seq;
15464 xx = 1;
15465 } else
15466 xx = 0;
15467 TCP_LOG_EVENTP(tp, th,
15468 &tptosocket(tp)->so_rcv,
15469 &tptosocket(tp)->so_snd, TCP_LOG_IN, 0,
15470 0, &log, true, <v);
15471 if (xx) {
15472 tp->snd_una = orig_snd_una;
15473 }
15474 }
15475
15476 }
15477
15478 static void
rack_handle_probe_response(struct tcp_rack * rack,uint32_t tiwin,uint32_t us_cts)15479 rack_handle_probe_response(struct tcp_rack *rack, uint32_t tiwin, uint32_t us_cts)
15480 {
15481 uint32_t us_rtt;
15482 /*
15483 * A persist or keep-alive was forced out, update our
15484 * min rtt time. Note now worry about lost responses.
15485 * When a subsequent keep-alive or persist times out
15486 * and forced_ack is still on, then the last probe
15487 * was not responded to. In such cases we have a
15488 * sysctl that controls the behavior. Either we apply
15489 * the rtt but with reduced confidence (0). Or we just
15490 * plain don't apply the rtt estimate. Having data flow
15491 * will clear the probe_not_answered flag i.e. cum-ack
15492 * move forward <or> exiting and reentering persists.
15493 */
15494
15495 rack->forced_ack = 0;
15496 rack->rc_tp->t_rxtshift = 0;
15497 if ((rack->rc_in_persist &&
15498 (tiwin == rack->rc_tp->snd_wnd)) ||
15499 (rack->rc_in_persist == 0)) {
15500 /*
15501 * In persists only apply the RTT update if this is
15502 * a response to our window probe. And that
15503 * means the rwnd sent must match the current
15504 * snd_wnd. If it does not, then we got a
15505 * window update ack instead. For keepalive
15506 * we allow the answer no matter what the window.
15507 *
15508 * Note that if the probe_not_answered is set then
15509 * the forced_ack_ts is the oldest one i.e. the first
15510 * probe sent that might have been lost. This assures
15511 * us that if we do calculate an RTT it is longer not
15512 * some short thing.
15513 */
15514 if (rack->rc_in_persist)
15515 counter_u64_add(rack_persists_acks, 1);
15516 us_rtt = us_cts - rack->r_ctl.forced_ack_ts;
15517 if (us_rtt == 0)
15518 us_rtt = 1;
15519 if (rack->probe_not_answered == 0) {
15520 rack_apply_updated_usrtt(rack, us_rtt, us_cts);
15521 tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 3, NULL, 1);
15522 } else {
15523 /* We have a retransmitted probe here too */
15524 if (rack_apply_rtt_with_reduced_conf) {
15525 rack_apply_updated_usrtt(rack, us_rtt, us_cts);
15526 tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1);
15527 }
15528 }
15529 }
15530 }
15531
15532 static void
rack_new_round_starts(struct tcpcb * tp,struct tcp_rack * rack,uint32_t high_seq)15533 rack_new_round_starts(struct tcpcb *tp, struct tcp_rack *rack, uint32_t high_seq)
15534 {
15535 /*
15536 * The next send has occurred mark the end of the round
15537 * as when that data gets acknowledged. We can
15538 * also do common things we might need to do when
15539 * a round begins.
15540 */
15541 rack->r_ctl.roundends = tp->snd_max;
15542 rack->rc_new_rnd_needed = 0;
15543 rack_log_hystart_event(rack, tp->snd_max, 4);
15544 }
15545
15546
15547 static void
rack_log_pcm(struct tcp_rack * rack,uint8_t mod,uint32_t flex1,uint32_t flex2,uint32_t flex3)15548 rack_log_pcm(struct tcp_rack *rack, uint8_t mod, uint32_t flex1, uint32_t flex2,
15549 uint32_t flex3)
15550 {
15551 if (tcp_bblogging_on(rack->rc_tp)) {
15552 union tcp_log_stackspecific log;
15553 struct timeval tv;
15554
15555 (void)tcp_get_usecs(&tv);
15556 memset(&log, 0, sizeof(log));
15557 log.u_bbr.timeStamp = tcp_tv_to_usec(&tv);
15558 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
15559 log.u_bbr.flex8 = mod;
15560 log.u_bbr.flex1 = flex1;
15561 log.u_bbr.flex2 = flex2;
15562 log.u_bbr.flex3 = flex3;
15563 log.u_bbr.flex4 = rack_pcm_every_n_rounds;
15564 log.u_bbr.flex5 = rack->r_ctl.pcm_idle_rounds;
15565 log.u_bbr.bbr_substate = rack->pcm_needed;
15566 log.u_bbr.bbr_substate <<= 1;
15567 log.u_bbr.bbr_substate |= rack->pcm_in_progress;
15568 log.u_bbr.bbr_substate <<= 1;
15569 log.u_bbr.bbr_substate |= rack->pcm_enabled; /* bits are NIE for Needed, Inprogress, Enabled */
15570 (void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK,
15571 0, &log, false, NULL, NULL, 0, &tv);
15572 }
15573 }
15574
15575 static void
rack_new_round_setup(struct tcpcb * tp,struct tcp_rack * rack,uint32_t high_seq)15576 rack_new_round_setup(struct tcpcb *tp, struct tcp_rack *rack, uint32_t high_seq)
15577 {
15578 /*
15579 * The round (current_round) has ended. We now
15580 * setup for the next round by incrementing the
15581 * round numnber and doing any round specific
15582 * things.
15583 */
15584 rack_log_hystart_event(rack, high_seq, 21);
15585 rack->r_ctl.current_round++;
15586 /* New round (current_round) begins at next send */
15587 rack->rc_new_rnd_needed = 1;
15588 if ((rack->pcm_enabled == 1) &&
15589 (rack->pcm_needed == 0) &&
15590 (rack->pcm_in_progress == 0)) {
15591 /*
15592 * If we have enabled PCM, then we need to
15593 * check if the round has adanced to the state
15594 * where one is required.
15595 */
15596 int rnds;
15597
15598 rnds = rack->r_ctl.current_round - rack->r_ctl.last_pcm_round;
15599 if ((rnds + rack->r_ctl.pcm_idle_rounds) >= rack_pcm_every_n_rounds) {
15600 rack->pcm_needed = 1;
15601 rack_log_pcm(rack, 3, rack->r_ctl.last_pcm_round, rack_pcm_every_n_rounds, rack->r_ctl.current_round );
15602 } else if (rack_verbose_logging) {
15603 rack_log_pcm(rack, 3, rack->r_ctl.last_pcm_round, rack_pcm_every_n_rounds, rack->r_ctl.current_round );
15604 }
15605 }
15606 if (tp->t_ccv.flags & CCF_HYSTART_ALLOWED) {
15607 /* We have hystart enabled send the round info in */
15608 if (CC_ALGO(tp)->newround != NULL) {
15609 CC_ALGO(tp)->newround(&tp->t_ccv, rack->r_ctl.current_round);
15610 }
15611 }
15612 /*
15613 * For DGP an initial startup check. We want to validate
15614 * that we are not just pushing on slow-start and just
15615 * not gaining.. i.e. filling buffers without getting any
15616 * boost in b/w during the inital slow-start.
15617 */
15618 if (rack->dgp_on &&
15619 (rack->rc_initial_ss_comp == 0) &&
15620 (tp->snd_cwnd < tp->snd_ssthresh) &&
15621 (rack->r_ctl.num_measurements >= RACK_REQ_AVG) &&
15622 (rack->r_ctl.gp_rnd_thresh > 0) &&
15623 ((rack->r_ctl.current_round - rack->r_ctl.last_rnd_of_gp_rise) >= rack->r_ctl.gp_rnd_thresh)) {
15624
15625 /*
15626 * We are in the initial SS and we have hd rack_rnd_cnt_req rounds(def:5) where
15627 * we have not gained the required amount in the gp_est (120.0% aka 1200). Lets
15628 * exit SS.
15629 *
15630 * Pick up the flight size now as we enter slowstart (not the
15631 * cwnd which may be inflated).
15632 */
15633 rack->rc_initial_ss_comp = 1;
15634
15635 if (tcp_bblogging_on(rack->rc_tp)) {
15636 union tcp_log_stackspecific log;
15637 struct timeval tv;
15638
15639 memset(&log, 0, sizeof(log));
15640 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
15641 log.u_bbr.flex1 = rack->r_ctl.current_round;
15642 log.u_bbr.flex2 = rack->r_ctl.last_rnd_of_gp_rise;
15643 log.u_bbr.flex3 = rack->r_ctl.gp_rnd_thresh;
15644 log.u_bbr.flex4 = rack->r_ctl.gate_to_fs;
15645 log.u_bbr.flex5 = rack->r_ctl.ss_hi_fs;
15646 log.u_bbr.flex8 = 40;
15647 (void)tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
15648 0, &log, false, NULL, __func__, __LINE__,&tv);
15649 }
15650 if ((rack->r_ctl.gate_to_fs == 1) &&
15651 (tp->snd_cwnd > rack->r_ctl.ss_hi_fs)) {
15652 tp->snd_cwnd = rack->r_ctl.ss_hi_fs;
15653 }
15654 tp->snd_ssthresh = tp->snd_cwnd - 1;
15655 /* Turn off any fast output running */
15656 rack->r_fast_output = 0;
15657 }
15658 }
15659
15660 static int
rack_do_compressed_ack_processing(struct tcpcb * tp,struct socket * so,struct mbuf * m,int nxt_pkt,struct timeval * tv)15661 rack_do_compressed_ack_processing(struct tcpcb *tp, struct socket *so, struct mbuf *m, int nxt_pkt, struct timeval *tv)
15662 {
15663 /*
15664 * Handle a "special" compressed ack mbuf. Each incoming
15665 * ack has only four possible dispositions:
15666 *
15667 * A) It moves the cum-ack forward
15668 * B) It is behind the cum-ack.
15669 * C) It is a window-update ack.
15670 * D) It is a dup-ack.
15671 *
15672 * Note that we can have between 1 -> TCP_COMP_ACK_ENTRIES
15673 * in the incoming mbuf. We also need to still pay attention
15674 * to nxt_pkt since there may be another packet after this
15675 * one.
15676 */
15677 #ifdef TCP_ACCOUNTING
15678 uint64_t ts_val;
15679 uint64_t rdstc;
15680 #endif
15681 int segsiz;
15682 struct timespec ts;
15683 struct tcp_rack *rack;
15684 struct tcp_ackent *ae;
15685 uint32_t tiwin, ms_cts, cts, acked, acked_amount, high_seq, win_seq, the_win, win_upd_ack;
15686 int cnt, i, did_out, ourfinisacked = 0;
15687 struct tcpopt to_holder, *to = NULL;
15688 #ifdef TCP_ACCOUNTING
15689 int win_up_req = 0;
15690 #endif
15691 int nsegs = 0;
15692 int under_pacing = 0;
15693 int post_recovery = 0;
15694 #ifdef TCP_ACCOUNTING
15695 sched_pin();
15696 #endif
15697 rack = (struct tcp_rack *)tp->t_fb_ptr;
15698 if (rack->gp_ready &&
15699 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT))
15700 under_pacing = 1;
15701
15702 if (rack->r_state != tp->t_state)
15703 rack_set_state(tp, rack);
15704 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
15705 (tp->t_flags & TF_GPUTINPROG)) {
15706 /*
15707 * We have a goodput in progress
15708 * and we have entered a late state.
15709 * Do we have enough data in the sb
15710 * to handle the GPUT request?
15711 */
15712 uint32_t bytes;
15713
15714 bytes = tp->gput_ack - tp->gput_seq;
15715 if (SEQ_GT(tp->gput_seq, tp->snd_una))
15716 bytes += tp->gput_seq - tp->snd_una;
15717 if (bytes > sbavail(&tptosocket(tp)->so_snd)) {
15718 /*
15719 * There are not enough bytes in the socket
15720 * buffer that have been sent to cover this
15721 * measurement. Cancel it.
15722 */
15723 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
15724 rack->r_ctl.rc_gp_srtt /*flex1*/,
15725 tp->gput_seq,
15726 0, 0, 18, __LINE__, NULL, 0);
15727 tp->t_flags &= ~TF_GPUTINPROG;
15728 }
15729 }
15730 to = &to_holder;
15731 to->to_flags = 0;
15732 KASSERT((m->m_len >= sizeof(struct tcp_ackent)),
15733 ("tp:%p m_cmpack:%p with invalid len:%u", tp, m, m->m_len));
15734 cnt = m->m_len / sizeof(struct tcp_ackent);
15735 counter_u64_add(rack_multi_single_eq, cnt);
15736 high_seq = tp->snd_una;
15737 the_win = tp->snd_wnd;
15738 win_seq = tp->snd_wl1;
15739 win_upd_ack = tp->snd_wl2;
15740 cts = tcp_tv_to_usec(tv);
15741 ms_cts = tcp_tv_to_msec(tv);
15742 rack->r_ctl.rc_rcvtime = cts;
15743 segsiz = ctf_fixed_maxseg(tp);
15744 if ((rack->rc_gp_dyn_mul) &&
15745 (rack->use_fixed_rate == 0) &&
15746 (rack->rc_always_pace)) {
15747 /* Check in on probertt */
15748 rack_check_probe_rtt(rack, cts);
15749 }
15750 for (i = 0; i < cnt; i++) {
15751 #ifdef TCP_ACCOUNTING
15752 ts_val = get_cyclecount();
15753 #endif
15754 rack_clear_rate_sample(rack);
15755 ae = ((mtod(m, struct tcp_ackent *)) + i);
15756 if (ae->flags & TH_FIN)
15757 rack_log_pacing_delay_calc(rack,
15758 0,
15759 0,
15760 0,
15761 rack_get_gp_est(rack), /* delRate */
15762 rack_get_lt_bw(rack), /* rttProp */
15763 20, __LINE__, NULL, 0);
15764 /* Setup the window */
15765 tiwin = ae->win << tp->snd_scale;
15766 if (tiwin > rack->r_ctl.rc_high_rwnd)
15767 rack->r_ctl.rc_high_rwnd = tiwin;
15768 /* figure out the type of ack */
15769 if (SEQ_LT(ae->ack, high_seq)) {
15770 /* Case B*/
15771 ae->ack_val_set = ACK_BEHIND;
15772 } else if (SEQ_GT(ae->ack, high_seq)) {
15773 /* Case A */
15774 ae->ack_val_set = ACK_CUMACK;
15775 } else if ((tiwin == the_win) && (rack->rc_in_persist == 0)){
15776 /* Case D */
15777 ae->ack_val_set = ACK_DUPACK;
15778 } else {
15779 /* Case C */
15780 ae->ack_val_set = ACK_RWND;
15781 }
15782 rack_log_type_bbrsnd(rack, 0, 0, cts, tv, __LINE__);
15783 rack_log_input_packet(tp, rack, ae, ae->ack_val_set, high_seq);
15784 /* Validate timestamp */
15785 if (ae->flags & HAS_TSTMP) {
15786 /* Setup for a timestamp */
15787 to->to_flags = TOF_TS;
15788 ae->ts_echo -= tp->ts_offset;
15789 to->to_tsecr = ae->ts_echo;
15790 to->to_tsval = ae->ts_value;
15791 /*
15792 * If echoed timestamp is later than the current time, fall back to
15793 * non RFC1323 RTT calculation. Normalize timestamp if syncookies
15794 * were used when this connection was established.
15795 */
15796 if (TSTMP_GT(ae->ts_echo, ms_cts))
15797 to->to_tsecr = 0;
15798 if (tp->ts_recent &&
15799 TSTMP_LT(ae->ts_value, tp->ts_recent)) {
15800 if (ctf_ts_check_ac(tp, (ae->flags & 0xff))) {
15801 #ifdef TCP_ACCOUNTING
15802 rdstc = get_cyclecount();
15803 if (rdstc > ts_val) {
15804 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
15805 tp->tcp_proc_time[ae->ack_val_set] += (rdstc - ts_val);
15806 }
15807 }
15808 #endif
15809 continue;
15810 }
15811 }
15812 if (SEQ_LEQ(ae->seq, tp->last_ack_sent) &&
15813 SEQ_LEQ(tp->last_ack_sent, ae->seq)) {
15814 tp->ts_recent_age = tcp_ts_getticks();
15815 tp->ts_recent = ae->ts_value;
15816 }
15817 } else {
15818 /* Setup for a no options */
15819 to->to_flags = 0;
15820 }
15821 /* Update the rcv time and perform idle reduction possibly */
15822 if (tp->t_idle_reduce &&
15823 (tp->snd_max == tp->snd_una) &&
15824 (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) {
15825 counter_u64_add(rack_input_idle_reduces, 1);
15826 rack_cc_after_idle(rack, tp);
15827 }
15828 tp->t_rcvtime = ticks;
15829 /* Now what about ECN of a chain of pure ACKs? */
15830 if (tcp_ecn_input_segment(tp, ae->flags, 0,
15831 tcp_packets_this_ack(tp, ae->ack),
15832 ae->codepoint))
15833 rack_cong_signal(tp, CC_ECN, ae->ack, __LINE__);
15834 #ifdef TCP_ACCOUNTING
15835 /* Count for the specific type of ack in */
15836 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
15837 tp->tcp_cnt_counters[ae->ack_val_set]++;
15838 }
15839 #endif
15840 /*
15841 * Note how we could move up these in the determination
15842 * above, but we don't so that way the timestamp checks (and ECN)
15843 * is done first before we do any processing on the ACK.
15844 * The non-compressed path through the code has this
15845 * weakness (noted by @jtl) that it actually does some
15846 * processing before verifying the timestamp information.
15847 * We don't take that path here which is why we set
15848 * the ack_val_set first, do the timestamp and ecn
15849 * processing, and then look at what we have setup.
15850 */
15851 if (ae->ack_val_set == ACK_BEHIND) {
15852 /*
15853 * Case B flag reordering, if window is not closed
15854 * or it could be a keep-alive or persists
15855 */
15856 if (SEQ_LT(ae->ack, tp->snd_una) && (sbspace(&so->so_rcv) > segsiz)) {
15857 rack->r_ctl.rc_reorder_ts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
15858 if (rack->r_ctl.rc_reorder_ts == 0)
15859 rack->r_ctl.rc_reorder_ts = 1;
15860 }
15861 } else if (ae->ack_val_set == ACK_DUPACK) {
15862 /* Case D */
15863 rack_strike_dupack(rack, ae->ack);
15864 } else if (ae->ack_val_set == ACK_RWND) {
15865 /* Case C */
15866 if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) {
15867 ts.tv_sec = ae->timestamp / 1000000000;
15868 ts.tv_nsec = ae->timestamp % 1000000000;
15869 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
15870 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
15871 } else {
15872 rack->r_ctl.act_rcv_time = *tv;
15873 }
15874 if (rack->forced_ack) {
15875 rack_handle_probe_response(rack, tiwin,
15876 tcp_tv_to_usec(&rack->r_ctl.act_rcv_time));
15877 }
15878 #ifdef TCP_ACCOUNTING
15879 win_up_req = 1;
15880 #endif
15881 win_upd_ack = ae->ack;
15882 win_seq = ae->seq;
15883 the_win = tiwin;
15884 rack_do_win_updates(tp, rack, the_win, win_seq, win_upd_ack, cts);
15885 } else {
15886 /* Case A */
15887 if (SEQ_GT(ae->ack, tp->snd_max)) {
15888 /*
15889 * We just send an ack since the incoming
15890 * ack is beyond the largest seq we sent.
15891 */
15892 if ((tp->t_flags & TF_ACKNOW) == 0) {
15893 ctf_ack_war_checks(tp);
15894 if (tp->t_flags && TF_ACKNOW)
15895 rack->r_wanted_output = 1;
15896 }
15897 } else {
15898 nsegs++;
15899 /* If the window changed setup to update */
15900 if (tiwin != tp->snd_wnd) {
15901 win_upd_ack = ae->ack;
15902 win_seq = ae->seq;
15903 the_win = tiwin;
15904 rack_do_win_updates(tp, rack, the_win, win_seq, win_upd_ack, cts);
15905 }
15906 #ifdef TCP_ACCOUNTING
15907 /* Account for the acks */
15908 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
15909 tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz);
15910 }
15911 #endif
15912 high_seq = ae->ack;
15913 /* Setup our act_rcv_time */
15914 if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) {
15915 ts.tv_sec = ae->timestamp / 1000000000;
15916 ts.tv_nsec = ae->timestamp % 1000000000;
15917 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
15918 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
15919 } else {
15920 rack->r_ctl.act_rcv_time = *tv;
15921 }
15922 rack_process_to_cumack(tp, rack, ae->ack, cts, to,
15923 tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time));
15924 #ifdef TCP_REQUEST_TRK
15925 rack_req_check_for_comp(rack, high_seq);
15926 #endif
15927 if (rack->rc_dsack_round_seen) {
15928 /* Is the dsack round over? */
15929 if (SEQ_GEQ(ae->ack, rack->r_ctl.dsack_round_end)) {
15930 /* Yes it is */
15931 rack->rc_dsack_round_seen = 0;
15932 rack_log_dsack_event(rack, 3, __LINE__, 0, 0);
15933 }
15934 }
15935 }
15936 }
15937 /* And lets be sure to commit the rtt measurements for this ack */
15938 tcp_rack_xmit_timer_commit(rack, tp);
15939 #ifdef TCP_ACCOUNTING
15940 rdstc = get_cyclecount();
15941 if (rdstc > ts_val) {
15942 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
15943 tp->tcp_proc_time[ae->ack_val_set] += (rdstc - ts_val);
15944 if (ae->ack_val_set == ACK_CUMACK)
15945 tp->tcp_proc_time[CYC_HANDLE_MAP] += (rdstc - ts_val);
15946 }
15947 }
15948 #endif
15949 }
15950 #ifdef TCP_ACCOUNTING
15951 ts_val = get_cyclecount();
15952 #endif
15953 /* Tend to any collapsed window */
15954 if (SEQ_GT(tp->snd_max, high_seq) && (tp->snd_wnd < (tp->snd_max - high_seq))) {
15955 /* The peer collapsed the window */
15956 rack_collapsed_window(rack, (tp->snd_max - high_seq), high_seq, __LINE__);
15957 } else if (rack->rc_has_collapsed)
15958 rack_un_collapse_window(rack, __LINE__);
15959 if ((rack->r_collapse_point_valid) &&
15960 (SEQ_GT(high_seq, rack->r_ctl.high_collapse_point)))
15961 rack->r_collapse_point_valid = 0;
15962 acked_amount = acked = (high_seq - tp->snd_una);
15963 if (acked) {
15964 /*
15965 * The draft (v3) calls for us to use SEQ_GEQ, but that
15966 * causes issues when we are just going app limited. Lets
15967 * instead use SEQ_GT <or> where its equal but more data
15968 * is outstanding.
15969 *
15970 * Also make sure we are on the last ack of a series. We
15971 * have to have all the ack's processed in queue to know
15972 * if there is something left outstanding.
15973 *
15974 */
15975 if (SEQ_GEQ(high_seq, rack->r_ctl.roundends) &&
15976 (rack->rc_new_rnd_needed == 0) &&
15977 (nxt_pkt == 0)) {
15978 /*
15979 * We have crossed into a new round with
15980 * this th_ack value.
15981 */
15982 rack_new_round_setup(tp, rack, high_seq);
15983 }
15984 /*
15985 * Clear the probe not answered flag
15986 * since cum-ack moved forward.
15987 */
15988 rack->probe_not_answered = 0;
15989 if (tp->t_flags & TF_NEEDSYN) {
15990 /*
15991 * T/TCP: Connection was half-synchronized, and our SYN has
15992 * been ACK'd (so connection is now fully synchronized). Go
15993 * to non-starred state, increment snd_una for ACK of SYN,
15994 * and check if we can do window scaling.
15995 */
15996 tp->t_flags &= ~TF_NEEDSYN;
15997 tp->snd_una++;
15998 acked_amount = acked = (high_seq - tp->snd_una);
15999 }
16000 if (acked > sbavail(&so->so_snd))
16001 acked_amount = sbavail(&so->so_snd);
16002 if (IN_FASTRECOVERY(tp->t_flags) &&
16003 (rack->rack_no_prr == 0))
16004 rack_update_prr(tp, rack, acked_amount, high_seq);
16005 if (IN_RECOVERY(tp->t_flags)) {
16006 if (SEQ_LT(high_seq, tp->snd_recover) &&
16007 (SEQ_LT(high_seq, tp->snd_max))) {
16008 tcp_rack_partialack(tp);
16009 } else {
16010 rack_post_recovery(tp, high_seq);
16011 post_recovery = 1;
16012 }
16013 } else if ((rack->rto_from_rec == 1) &&
16014 SEQ_GEQ(high_seq, tp->snd_recover)) {
16015 /*
16016 * We were in recovery, hit a rxt timeout
16017 * and never re-entered recovery. The timeout(s)
16018 * made up all the lost data. In such a case
16019 * we need to clear the rto_from_rec flag.
16020 */
16021 rack->rto_from_rec = 0;
16022 }
16023 /* Handle the rack-log-ack part (sendmap) */
16024 if ((sbused(&so->so_snd) == 0) &&
16025 (acked > acked_amount) &&
16026 (tp->t_state >= TCPS_FIN_WAIT_1) &&
16027 (tp->t_flags & TF_SENTFIN)) {
16028 /*
16029 * We must be sure our fin
16030 * was sent and acked (we can be
16031 * in FIN_WAIT_1 without having
16032 * sent the fin).
16033 */
16034 ourfinisacked = 1;
16035 /*
16036 * Lets make sure snd_una is updated
16037 * since most likely acked_amount = 0 (it
16038 * should be).
16039 */
16040 tp->snd_una = high_seq;
16041 }
16042 /* Did we make a RTO error? */
16043 if ((tp->t_flags & TF_PREVVALID) &&
16044 ((tp->t_flags & TF_RCVD_TSTMP) == 0)) {
16045 tp->t_flags &= ~TF_PREVVALID;
16046 if (tp->t_rxtshift == 1 &&
16047 (int)(ticks - tp->t_badrxtwin) < 0)
16048 rack_cong_signal(tp, CC_RTO_ERR, high_seq, __LINE__);
16049 }
16050 /* Handle the data in the socket buffer */
16051 KMOD_TCPSTAT_ADD(tcps_rcvackpack, 1);
16052 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
16053 if (acked_amount > 0) {
16054 uint32_t p_cwnd;
16055 struct mbuf *mfree;
16056
16057 if (post_recovery) {
16058 /*
16059 * Grab the segsiz, multiply by 2 and add the snd_cwnd
16060 * that is the max the CC should add if we are exiting
16061 * recovery and doing a late add.
16062 */
16063 p_cwnd = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
16064 p_cwnd <<= 1;
16065 p_cwnd += tp->snd_cwnd;
16066 }
16067 rack_ack_received(tp, rack, high_seq, nsegs, CC_ACK, post_recovery);
16068 if (post_recovery && (tp->snd_cwnd > p_cwnd)) {
16069 /* Must be non-newreno (cubic) getting too ahead of itself */
16070 tp->snd_cwnd = p_cwnd;
16071 }
16072 SOCK_SENDBUF_LOCK(so);
16073 mfree = sbcut_locked(&so->so_snd, acked_amount);
16074 tp->snd_una = high_seq;
16075 /* Note we want to hold the sb lock through the sendmap adjust */
16076 rack_adjust_sendmap_head(rack, &so->so_snd);
16077 /* Wake up the socket if we have room to write more */
16078 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2);
16079 sowwakeup_locked(so);
16080 m_freem(mfree);
16081 }
16082 /* update progress */
16083 tp->t_acktime = ticks;
16084 rack_log_progress_event(rack, tp, tp->t_acktime,
16085 PROGRESS_UPDATE, __LINE__);
16086 /* Clear out shifts and such */
16087 tp->t_rxtshift = 0;
16088 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
16089 rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
16090 rack->rc_tlp_in_progress = 0;
16091 rack->r_ctl.rc_tlp_cnt_out = 0;
16092 /* Send recover and snd_nxt must be dragged along */
16093 if (SEQ_GT(tp->snd_una, tp->snd_recover))
16094 tp->snd_recover = tp->snd_una;
16095 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
16096 tp->snd_nxt = tp->snd_max;
16097 /*
16098 * If the RXT timer is running we want to
16099 * stop it, so we can restart a TLP (or new RXT).
16100 */
16101 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
16102 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
16103 tp->snd_wl2 = high_seq;
16104 tp->t_dupacks = 0;
16105 if (under_pacing &&
16106 (rack->use_fixed_rate == 0) &&
16107 (rack->in_probe_rtt == 0) &&
16108 rack->rc_gp_dyn_mul &&
16109 rack->rc_always_pace) {
16110 /* Check if we are dragging bottom */
16111 rack_check_bottom_drag(tp, rack, so);
16112 }
16113 if (tp->snd_una == tp->snd_max) {
16114 tp->t_flags &= ~TF_PREVVALID;
16115 rack->r_ctl.retran_during_recovery = 0;
16116 rack->rc_suspicious = 0;
16117 rack->r_ctl.dsack_byte_cnt = 0;
16118 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL);
16119 if (rack->r_ctl.rc_went_idle_time == 0)
16120 rack->r_ctl.rc_went_idle_time = 1;
16121 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
16122 if (sbavail(&tptosocket(tp)->so_snd) == 0)
16123 tp->t_acktime = 0;
16124 /* Set so we might enter persists... */
16125 rack->r_wanted_output = 1;
16126 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
16127 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
16128 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
16129 (sbavail(&so->so_snd) == 0) &&
16130 (tp->t_flags2 & TF2_DROP_AF_DATA)) {
16131 /*
16132 * The socket was gone and the
16133 * peer sent data (not now in the past), time to
16134 * reset him.
16135 */
16136 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
16137 /* tcp_close will kill the inp pre-log the Reset */
16138 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
16139 #ifdef TCP_ACCOUNTING
16140 rdstc = get_cyclecount();
16141 if (rdstc > ts_val) {
16142 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16143 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
16144 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
16145 }
16146 }
16147 #endif
16148 m_freem(m);
16149 tp = tcp_close(tp);
16150 if (tp == NULL) {
16151 #ifdef TCP_ACCOUNTING
16152 sched_unpin();
16153 #endif
16154 return (1);
16155 }
16156 /*
16157 * We would normally do drop-with-reset which would
16158 * send back a reset. We can't since we don't have
16159 * all the needed bits. Instead lets arrange for
16160 * a call to tcp_output(). That way since we
16161 * are in the closed state we will generate a reset.
16162 *
16163 * Note if tcp_accounting is on we don't unpin since
16164 * we do that after the goto label.
16165 */
16166 goto send_out_a_rst;
16167 }
16168 if ((sbused(&so->so_snd) == 0) &&
16169 (tp->t_state >= TCPS_FIN_WAIT_1) &&
16170 (tp->t_flags & TF_SENTFIN)) {
16171 /*
16172 * If we can't receive any more data, then closing user can
16173 * proceed. Starting the timer is contrary to the
16174 * specification, but if we don't get a FIN we'll hang
16175 * forever.
16176 *
16177 */
16178 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
16179 soisdisconnected(so);
16180 tcp_timer_activate(tp, TT_2MSL,
16181 (tcp_fast_finwait2_recycle ?
16182 tcp_finwait2_timeout :
16183 TP_MAXIDLE(tp)));
16184 }
16185 if (ourfinisacked == 0) {
16186 /*
16187 * We don't change to fin-wait-2 if we have our fin acked
16188 * which means we are probably in TCPS_CLOSING.
16189 */
16190 tcp_state_change(tp, TCPS_FIN_WAIT_2);
16191 }
16192 }
16193 }
16194 /* Wake up the socket if we have room to write more */
16195 if (sbavail(&so->so_snd)) {
16196 rack->r_wanted_output = 1;
16197 if (ctf_progress_timeout_check(tp, true)) {
16198 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
16199 tp, tick, PROGRESS_DROP, __LINE__);
16200 /*
16201 * We cheat here and don't send a RST, we should send one
16202 * when the pacer drops the connection.
16203 */
16204 #ifdef TCP_ACCOUNTING
16205 rdstc = get_cyclecount();
16206 if (rdstc > ts_val) {
16207 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16208 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
16209 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
16210 }
16211 }
16212 sched_unpin();
16213 #endif
16214 (void)tcp_drop(tp, ETIMEDOUT);
16215 m_freem(m);
16216 return (1);
16217 }
16218 }
16219 if (ourfinisacked) {
16220 switch(tp->t_state) {
16221 case TCPS_CLOSING:
16222 #ifdef TCP_ACCOUNTING
16223 rdstc = get_cyclecount();
16224 if (rdstc > ts_val) {
16225 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16226 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
16227 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
16228 }
16229 }
16230 sched_unpin();
16231 #endif
16232 tcp_twstart(tp);
16233 m_freem(m);
16234 return (1);
16235 break;
16236 case TCPS_LAST_ACK:
16237 #ifdef TCP_ACCOUNTING
16238 rdstc = get_cyclecount();
16239 if (rdstc > ts_val) {
16240 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16241 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
16242 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
16243 }
16244 }
16245 sched_unpin();
16246 #endif
16247 tp = tcp_close(tp);
16248 ctf_do_drop(m, tp);
16249 return (1);
16250 break;
16251 case TCPS_FIN_WAIT_1:
16252 #ifdef TCP_ACCOUNTING
16253 rdstc = get_cyclecount();
16254 if (rdstc > ts_val) {
16255 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16256 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
16257 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
16258 }
16259 }
16260 #endif
16261 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
16262 soisdisconnected(so);
16263 tcp_timer_activate(tp, TT_2MSL,
16264 (tcp_fast_finwait2_recycle ?
16265 tcp_finwait2_timeout :
16266 TP_MAXIDLE(tp)));
16267 }
16268 tcp_state_change(tp, TCPS_FIN_WAIT_2);
16269 break;
16270 default:
16271 break;
16272 }
16273 }
16274 if (rack->r_fast_output) {
16275 /*
16276 * We re doing fast output.. can we expand that?
16277 */
16278 rack_gain_for_fastoutput(rack, tp, so, acked_amount);
16279 }
16280 #ifdef TCP_ACCOUNTING
16281 rdstc = get_cyclecount();
16282 if (rdstc > ts_val) {
16283 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16284 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
16285 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
16286 }
16287 }
16288
16289 } else if (win_up_req) {
16290 rdstc = get_cyclecount();
16291 if (rdstc > ts_val) {
16292 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16293 tp->tcp_proc_time[ACK_RWND] += (rdstc - ts_val);
16294 }
16295 }
16296 #endif
16297 }
16298 /* Now is there a next packet, if so we are done */
16299 m_freem(m);
16300 did_out = 0;
16301 if (nxt_pkt) {
16302 #ifdef TCP_ACCOUNTING
16303 sched_unpin();
16304 #endif
16305 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, 5, nsegs);
16306 return (0);
16307 }
16308 rack_handle_might_revert(tp, rack);
16309 ctf_calc_rwin(so, tp);
16310 if ((rack->r_wanted_output != 0) ||
16311 (rack->r_fast_output != 0) ||
16312 (tp->t_flags & TF_ACKNOW )) {
16313 send_out_a_rst:
16314 if (tcp_output(tp) < 0) {
16315 #ifdef TCP_ACCOUNTING
16316 sched_unpin();
16317 #endif
16318 return (1);
16319 }
16320 did_out = 1;
16321 }
16322 if (tp->t_flags2 & TF2_HPTS_CALLS)
16323 tp->t_flags2 &= ~TF2_HPTS_CALLS;
16324 rack_free_trim(rack);
16325 #ifdef TCP_ACCOUNTING
16326 sched_unpin();
16327 #endif
16328 rack_timer_audit(tp, rack, &so->so_snd);
16329 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, 6, nsegs);
16330 return (0);
16331 }
16332
16333 #define TCP_LRO_TS_OPTION \
16334 ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | \
16335 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)
16336
16337 static int
rack_do_segment_nounlock(struct tcpcb * tp,struct mbuf * m,struct tcphdr * th,int32_t drop_hdrlen,int32_t tlen,uint8_t iptos,int32_t nxt_pkt,struct timeval * tv)16338 rack_do_segment_nounlock(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
16339 int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, int32_t nxt_pkt,
16340 struct timeval *tv)
16341 {
16342 struct inpcb *inp = tptoinpcb(tp);
16343 struct socket *so = tptosocket(tp);
16344 #ifdef TCP_ACCOUNTING
16345 uint64_t ts_val;
16346 #endif
16347 int32_t thflags, retval, did_out = 0;
16348 int32_t way_out = 0;
16349 /*
16350 * cts - is the current time from tv (caller gets ts) in microseconds.
16351 * ms_cts - is the current time from tv in milliseconds.
16352 * us_cts - is the time that LRO or hardware actually got the packet in microseconds.
16353 */
16354 uint32_t cts, us_cts, ms_cts;
16355 uint32_t tiwin;
16356 struct timespec ts;
16357 struct tcpopt to;
16358 struct tcp_rack *rack;
16359 struct rack_sendmap *rsm;
16360 int32_t prev_state = 0;
16361 int no_output = 0;
16362 int time_remaining = 0;
16363 #ifdef TCP_ACCOUNTING
16364 int ack_val_set = 0xf;
16365 #endif
16366 int nsegs;
16367
16368 NET_EPOCH_ASSERT();
16369 INP_WLOCK_ASSERT(inp);
16370
16371 /*
16372 * tv passed from common code is from either M_TSTMP_LRO or
16373 * tcp_get_usecs() if no LRO m_pkthdr timestamp is present.
16374 */
16375 rack = (struct tcp_rack *)tp->t_fb_ptr;
16376 if (rack->rack_deferred_inited == 0) {
16377 /*
16378 * If we are the connecting socket we will
16379 * hit rack_init() when no sequence numbers
16380 * are setup. This makes it so we must defer
16381 * some initialization. Call that now.
16382 */
16383 rack_deferred_init(tp, rack);
16384 }
16385 /*
16386 * Check to see if we need to skip any output plans. This
16387 * can happen in the non-LRO path where we are pacing and
16388 * must process the ack coming in but need to defer sending
16389 * anything becase a pacing timer is running.
16390 */
16391 us_cts = tcp_tv_to_usec(tv);
16392 if (m->m_flags & M_ACKCMP) {
16393 /*
16394 * All compressed ack's are ack's by definition so
16395 * remove any ack required flag and then do the processing.
16396 */
16397 rack->rc_ack_required = 0;
16398 return (rack_do_compressed_ack_processing(tp, so, m, nxt_pkt, tv));
16399 }
16400 thflags = tcp_get_flags(th);
16401 if ((rack->rc_always_pace == 1) &&
16402 (rack->rc_ack_can_sendout_data == 0) &&
16403 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
16404 (TSTMP_LT(us_cts, rack->r_ctl.rc_last_output_to))) {
16405 /*
16406 * Ok conditions are right for queuing the packets
16407 * but we do have to check the flags in the inp, it
16408 * could be, if a sack is present, we want to be awoken and
16409 * so should process the packets.
16410 */
16411 time_remaining = rack->r_ctl.rc_last_output_to - us_cts;
16412 if (rack->rc_tp->t_flags2 & TF2_DONT_SACK_QUEUE) {
16413 no_output = 1;
16414 } else {
16415 /*
16416 * If there is no options, or just a
16417 * timestamp option, we will want to queue
16418 * the packets. This is the same that LRO does
16419 * and will need to change with accurate ECN.
16420 */
16421 uint32_t *ts_ptr;
16422 int optlen;
16423
16424 optlen = (th->th_off << 2) - sizeof(struct tcphdr);
16425 ts_ptr = (uint32_t *)(th + 1);
16426 if ((optlen == 0) ||
16427 ((optlen == TCPOLEN_TSTAMP_APPA) &&
16428 (*ts_ptr == TCP_LRO_TS_OPTION)))
16429 no_output = 1;
16430 }
16431 if ((no_output == 1) && (time_remaining < tcp_min_hptsi_time)) {
16432 /*
16433 * It is unrealistic to think we can pace in less than
16434 * the minimum granularity of the pacer (def:250usec). So
16435 * if we have less than that time remaining we should go
16436 * ahead and allow output to be "early". We will attempt to
16437 * make up for it in any pacing time we try to apply on
16438 * the outbound packet.
16439 */
16440 no_output = 0;
16441 }
16442 }
16443 /*
16444 * If there is a RST or FIN lets dump out the bw
16445 * with a FIN the connection may go on but we
16446 * may not.
16447 */
16448 if ((thflags & TH_FIN) || (thflags & TH_RST))
16449 rack_log_pacing_delay_calc(rack,
16450 rack->r_ctl.gp_bw,
16451 0,
16452 0,
16453 rack_get_gp_est(rack), /* delRate */
16454 rack_get_lt_bw(rack), /* rttProp */
16455 20, __LINE__, NULL, 0);
16456 if (m->m_flags & M_ACKCMP) {
16457 panic("Impossible reach m has ackcmp? m:%p tp:%p", m, tp);
16458 }
16459 cts = tcp_tv_to_usec(tv);
16460 ms_cts = tcp_tv_to_msec(tv);
16461 nsegs = m->m_pkthdr.lro_nsegs;
16462 counter_u64_add(rack_proc_non_comp_ack, 1);
16463 #ifdef TCP_ACCOUNTING
16464 sched_pin();
16465 if (thflags & TH_ACK)
16466 ts_val = get_cyclecount();
16467 #endif
16468 if ((m->m_flags & M_TSTMP) ||
16469 (m->m_flags & M_TSTMP_LRO)) {
16470 mbuf_tstmp2timespec(m, &ts);
16471 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
16472 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
16473 } else
16474 rack->r_ctl.act_rcv_time = *tv;
16475 kern_prefetch(rack, &prev_state);
16476 prev_state = 0;
16477 /*
16478 * Unscale the window into a 32-bit value. For the SYN_SENT state
16479 * the scale is zero.
16480 */
16481 tiwin = th->th_win << tp->snd_scale;
16482 #ifdef TCP_ACCOUNTING
16483 if (thflags & TH_ACK) {
16484 /*
16485 * We have a tradeoff here. We can either do what we are
16486 * doing i.e. pinning to this CPU and then doing the accounting
16487 * <or> we could do a critical enter, setup the rdtsc and cpu
16488 * as in below, and then validate we are on the same CPU on
16489 * exit. I have choosen to not do the critical enter since
16490 * that often will gain you a context switch, and instead lock
16491 * us (line above this if) to the same CPU with sched_pin(). This
16492 * means we may be context switched out for a higher priority
16493 * interupt but we won't be moved to another CPU.
16494 *
16495 * If this occurs (which it won't very often since we most likely
16496 * are running this code in interupt context and only a higher
16497 * priority will bump us ... clock?) we will falsely add in
16498 * to the time the interupt processing time plus the ack processing
16499 * time. This is ok since its a rare event.
16500 */
16501 ack_val_set = tcp_do_ack_accounting(tp, th, &to, tiwin,
16502 ctf_fixed_maxseg(tp));
16503 }
16504 #endif
16505 /*
16506 * Parse options on any incoming segment.
16507 */
16508 memset(&to, 0, sizeof(to));
16509 tcp_dooptions(&to, (u_char *)(th + 1),
16510 (th->th_off << 2) - sizeof(struct tcphdr),
16511 (thflags & TH_SYN) ? TO_SYN : 0);
16512 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
16513 __func__));
16514 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
16515 __func__));
16516 if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) {
16517 /*
16518 * We don't look at sack's from the
16519 * peer because the MSS is too small which
16520 * can subject us to an attack.
16521 */
16522 to.to_flags &= ~TOF_SACK;
16523 }
16524 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
16525 (tp->t_flags & TF_GPUTINPROG)) {
16526 /*
16527 * We have a goodput in progress
16528 * and we have entered a late state.
16529 * Do we have enough data in the sb
16530 * to handle the GPUT request?
16531 */
16532 uint32_t bytes;
16533
16534 bytes = tp->gput_ack - tp->gput_seq;
16535 if (SEQ_GT(tp->gput_seq, tp->snd_una))
16536 bytes += tp->gput_seq - tp->snd_una;
16537 if (bytes > sbavail(&tptosocket(tp)->so_snd)) {
16538 /*
16539 * There are not enough bytes in the socket
16540 * buffer that have been sent to cover this
16541 * measurement. Cancel it.
16542 */
16543 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
16544 rack->r_ctl.rc_gp_srtt /*flex1*/,
16545 tp->gput_seq,
16546 0, 0, 18, __LINE__, NULL, 0);
16547 tp->t_flags &= ~TF_GPUTINPROG;
16548 }
16549 }
16550 if (tcp_bblogging_on(rack->rc_tp)) {
16551 union tcp_log_stackspecific log;
16552 struct timeval ltv;
16553 #ifdef TCP_REQUEST_TRK
16554 struct tcp_sendfile_track *tcp_req;
16555
16556 if (SEQ_GT(th->th_ack, tp->snd_una)) {
16557 tcp_req = tcp_req_find_req_for_seq(tp, (th->th_ack-1));
16558 } else {
16559 tcp_req = tcp_req_find_req_for_seq(tp, th->th_ack);
16560 }
16561 #endif
16562 memset(&log, 0, sizeof(log));
16563 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
16564 if (rack->rack_no_prr == 0)
16565 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
16566 else
16567 log.u_bbr.flex1 = 0;
16568 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
16569 log.u_bbr.use_lt_bw <<= 1;
16570 log.u_bbr.use_lt_bw |= rack->r_might_revert;
16571 log.u_bbr.flex2 = rack->r_ctl.rc_num_maps_alloced;
16572 log.u_bbr.bbr_state = rack->rc_free_cnt;
16573 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
16574 log.u_bbr.pkts_out = rack->rc_tp->t_maxseg;
16575 log.u_bbr.flex3 = m->m_flags;
16576 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
16577 log.u_bbr.lost = thflags;
16578 log.u_bbr.pacing_gain = 0x1;
16579 #ifdef TCP_ACCOUNTING
16580 log.u_bbr.cwnd_gain = ack_val_set;
16581 #endif
16582 log.u_bbr.flex7 = 2;
16583 if (m->m_flags & M_TSTMP) {
16584 /* Record the hardware timestamp if present */
16585 mbuf_tstmp2timespec(m, &ts);
16586 ltv.tv_sec = ts.tv_sec;
16587 ltv.tv_usec = ts.tv_nsec / 1000;
16588 log.u_bbr.lt_epoch = tcp_tv_to_usec(<v);
16589 } else if (m->m_flags & M_TSTMP_LRO) {
16590 /* Record the LRO the arrival timestamp */
16591 mbuf_tstmp2timespec(m, &ts);
16592 ltv.tv_sec = ts.tv_sec;
16593 ltv.tv_usec = ts.tv_nsec / 1000;
16594 log.u_bbr.flex5 = tcp_tv_to_usec(<v);
16595 }
16596 log.u_bbr.timeStamp = tcp_get_usecs(<v);
16597 /* Log the rcv time */
16598 log.u_bbr.delRate = m->m_pkthdr.rcv_tstmp;
16599 #ifdef TCP_REQUEST_TRK
16600 log.u_bbr.applimited = tp->t_tcpreq_closed;
16601 log.u_bbr.applimited <<= 8;
16602 log.u_bbr.applimited |= tp->t_tcpreq_open;
16603 log.u_bbr.applimited <<= 8;
16604 log.u_bbr.applimited |= tp->t_tcpreq_req;
16605 if (tcp_req) {
16606 /* Copy out any client req info */
16607 /* seconds */
16608 log.u_bbr.pkt_epoch = (tcp_req->localtime / HPTS_USEC_IN_SEC);
16609 /* useconds */
16610 log.u_bbr.delivered = (tcp_req->localtime % HPTS_USEC_IN_SEC);
16611 log.u_bbr.rttProp = tcp_req->timestamp;
16612 log.u_bbr.cur_del_rate = tcp_req->start;
16613 if (tcp_req->flags & TCP_TRK_TRACK_FLG_OPEN) {
16614 log.u_bbr.flex8 |= 1;
16615 } else {
16616 log.u_bbr.flex8 |= 2;
16617 log.u_bbr.bw_inuse = tcp_req->end;
16618 }
16619 log.u_bbr.flex6 = tcp_req->start_seq;
16620 if (tcp_req->flags & TCP_TRK_TRACK_FLG_COMP) {
16621 log.u_bbr.flex8 |= 4;
16622 log.u_bbr.epoch = tcp_req->end_seq;
16623 }
16624 }
16625 #endif
16626 TCP_LOG_EVENTP(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0,
16627 tlen, &log, true, <v);
16628 }
16629 /* Remove ack required flag if set, we have one */
16630 if (thflags & TH_ACK)
16631 rack->rc_ack_required = 0;
16632 rack_log_type_bbrsnd(rack, 0, 0, cts, tv, __LINE__);
16633 if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) {
16634 way_out = 4;
16635 retval = 0;
16636 m_freem(m);
16637 goto done_with_input;
16638 }
16639 /*
16640 * If a segment with the ACK-bit set arrives in the SYN-SENT state
16641 * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9.
16642 */
16643 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&
16644 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {
16645 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
16646 ctf_do_dropwithreset(m, tp, th, tlen);
16647 #ifdef TCP_ACCOUNTING
16648 sched_unpin();
16649 #endif
16650 return (1);
16651 }
16652 /*
16653 * If timestamps were negotiated during SYN/ACK and a
16654 * segment without a timestamp is received, silently drop
16655 * the segment, unless it is a RST segment or missing timestamps are
16656 * tolerated.
16657 * See section 3.2 of RFC 7323.
16658 */
16659 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&
16660 ((thflags & TH_RST) == 0) && (V_tcp_tolerate_missing_ts == 0)) {
16661 way_out = 5;
16662 retval = 0;
16663 m_freem(m);
16664 goto done_with_input;
16665 }
16666 /*
16667 * Segment received on connection. Reset idle time and keep-alive
16668 * timer. XXX: This should be done after segment validation to
16669 * ignore broken/spoofed segs.
16670 */
16671 if (tp->t_idle_reduce &&
16672 (tp->snd_max == tp->snd_una) &&
16673 (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) {
16674 counter_u64_add(rack_input_idle_reduces, 1);
16675 rack_cc_after_idle(rack, tp);
16676 }
16677 tp->t_rcvtime = ticks;
16678 #ifdef STATS
16679 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
16680 #endif
16681 if (tiwin > rack->r_ctl.rc_high_rwnd)
16682 rack->r_ctl.rc_high_rwnd = tiwin;
16683 /*
16684 * TCP ECN processing. XXXJTL: If we ever use ECN, we need to move
16685 * this to occur after we've validated the segment.
16686 */
16687 if (tcp_ecn_input_segment(tp, thflags, tlen,
16688 tcp_packets_this_ack(tp, th->th_ack),
16689 iptos))
16690 rack_cong_signal(tp, CC_ECN, th->th_ack, __LINE__);
16691
16692 /*
16693 * If echoed timestamp is later than the current time, fall back to
16694 * non RFC1323 RTT calculation. Normalize timestamp if syncookies
16695 * were used when this connection was established.
16696 */
16697 if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
16698 to.to_tsecr -= tp->ts_offset;
16699 if (TSTMP_GT(to.to_tsecr, ms_cts))
16700 to.to_tsecr = 0;
16701 }
16702 if ((rack->r_rcvpath_rtt_up == 1) &&
16703 (to.to_flags & TOF_TS) &&
16704 (TSTMP_GEQ(to.to_tsecr, rack->r_ctl.last_rcv_tstmp_for_rtt))) {
16705 uint32_t rtt = 0;
16706
16707 /*
16708 * We are receiving only and thus not sending
16709 * data to do an RTT. We set a flag when we first
16710 * sent this TS to the peer. We now have it back
16711 * and have an RTT to share. We log it as a conf
16712 * 4, we are not so sure about it.. since we
16713 * may have lost an ack.
16714 */
16715 if (TSTMP_GT(cts, rack->r_ctl.last_time_of_arm_rcv))
16716 rtt = (cts - rack->r_ctl.last_time_of_arm_rcv);
16717 rack->r_rcvpath_rtt_up = 0;
16718 /* Submit and commit the timer */
16719 if (rtt > 0) {
16720 tcp_rack_xmit_timer(rack, rtt, 0, rtt, 4, NULL, 1);
16721 tcp_rack_xmit_timer_commit(rack, tp);
16722 }
16723 }
16724 /*
16725 * If its the first time in we need to take care of options and
16726 * verify we can do SACK for rack!
16727 */
16728 if (rack->r_state == 0) {
16729 /* Should be init'd by rack_init() */
16730 KASSERT(rack->rc_inp != NULL,
16731 ("%s: rack->rc_inp unexpectedly NULL", __func__));
16732 if (rack->rc_inp == NULL) {
16733 rack->rc_inp = inp;
16734 }
16735
16736 /*
16737 * Process options only when we get SYN/ACK back. The SYN
16738 * case for incoming connections is handled in tcp_syncache.
16739 * According to RFC1323 the window field in a SYN (i.e., a
16740 * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX
16741 * this is traditional behavior, may need to be cleaned up.
16742 */
16743 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
16744 /* Handle parallel SYN for ECN */
16745 tcp_ecn_input_parallel_syn(tp, thflags, iptos);
16746 if ((to.to_flags & TOF_SCALE) &&
16747 (tp->t_flags & TF_REQ_SCALE)) {
16748 tp->t_flags |= TF_RCVD_SCALE;
16749 tp->snd_scale = to.to_wscale;
16750 } else
16751 tp->t_flags &= ~TF_REQ_SCALE;
16752 /*
16753 * Initial send window. It will be updated with the
16754 * next incoming segment to the scaled value.
16755 */
16756 tp->snd_wnd = th->th_win;
16757 rack_validate_fo_sendwin_up(tp, rack);
16758 if ((to.to_flags & TOF_TS) &&
16759 (tp->t_flags & TF_REQ_TSTMP)) {
16760 tp->t_flags |= TF_RCVD_TSTMP;
16761 tp->ts_recent = to.to_tsval;
16762 tp->ts_recent_age = cts;
16763 } else
16764 tp->t_flags &= ~TF_REQ_TSTMP;
16765 if (to.to_flags & TOF_MSS) {
16766 tcp_mss(tp, to.to_mss);
16767 }
16768 if ((tp->t_flags & TF_SACK_PERMIT) &&
16769 (to.to_flags & TOF_SACKPERM) == 0)
16770 tp->t_flags &= ~TF_SACK_PERMIT;
16771 if (tp->t_flags & TF_FASTOPEN) {
16772 if (to.to_flags & TOF_FASTOPEN) {
16773 uint16_t mss;
16774
16775 if (to.to_flags & TOF_MSS)
16776 mss = to.to_mss;
16777 else
16778 if ((inp->inp_vflag & INP_IPV6) != 0)
16779 mss = TCP6_MSS;
16780 else
16781 mss = TCP_MSS;
16782 tcp_fastopen_update_cache(tp, mss,
16783 to.to_tfo_len, to.to_tfo_cookie);
16784 } else
16785 tcp_fastopen_disable_path(tp);
16786 }
16787 }
16788 /*
16789 * At this point we are at the initial call. Here we decide
16790 * if we are doing RACK or not. We do this by seeing if
16791 * TF_SACK_PERMIT is set and the sack-not-required is clear.
16792 * The code now does do dup-ack counting so if you don't
16793 * switch back you won't get rack & TLP, but you will still
16794 * get this stack.
16795 */
16796
16797 if ((rack_sack_not_required == 0) &&
16798 ((tp->t_flags & TF_SACK_PERMIT) == 0)) {
16799 tcp_switch_back_to_default(tp);
16800 (*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen,
16801 tlen, iptos);
16802 #ifdef TCP_ACCOUNTING
16803 sched_unpin();
16804 #endif
16805 return (1);
16806 }
16807 tcp_set_hpts(tp);
16808 sack_filter_clear(&rack->r_ctl.rack_sf, th->th_ack);
16809 }
16810 if (thflags & TH_FIN)
16811 tcp_log_end_status(tp, TCP_EI_STATUS_CLIENT_FIN);
16812 us_cts = tcp_tv_to_usec(&rack->r_ctl.act_rcv_time);
16813 if ((rack->rc_gp_dyn_mul) &&
16814 (rack->use_fixed_rate == 0) &&
16815 (rack->rc_always_pace)) {
16816 /* Check in on probertt */
16817 rack_check_probe_rtt(rack, cts);
16818 }
16819 rack_clear_rate_sample(rack);
16820 if ((rack->forced_ack) &&
16821 ((tcp_get_flags(th) & TH_RST) == 0)) {
16822 rack_handle_probe_response(rack, tiwin, us_cts);
16823 }
16824 /*
16825 * This is the one exception case where we set the rack state
16826 * always. All other times (timers etc) we must have a rack-state
16827 * set (so we assure we have done the checks above for SACK).
16828 */
16829 rack->r_ctl.rc_rcvtime = cts;
16830 if (rack->r_state != tp->t_state)
16831 rack_set_state(tp, rack);
16832 if (SEQ_GT(th->th_ack, tp->snd_una) &&
16833 (rsm = tqhash_min(rack->r_ctl.tqh)) != NULL)
16834 kern_prefetch(rsm, &prev_state);
16835 prev_state = rack->r_state;
16836 if ((thflags & TH_RST) &&
16837 ((SEQ_GEQ(th->th_seq, tp->last_ack_sent) &&
16838 SEQ_LT(th->th_seq, tp->last_ack_sent + tp->rcv_wnd)) ||
16839 (tp->rcv_wnd == 0 && tp->last_ack_sent == th->th_seq))) {
16840 /* The connection will be killed by a reset check the tracepoint */
16841 tcp_trace_point(rack->rc_tp, TCP_TP_RESET_RCV);
16842 }
16843 retval = (*rack->r_substate) (m, th, so,
16844 tp, &to, drop_hdrlen,
16845 tlen, tiwin, thflags, nxt_pkt, iptos);
16846 if (retval == 0) {
16847 /*
16848 * If retval is 1 the tcb is unlocked and most likely the tp
16849 * is gone.
16850 */
16851 INP_WLOCK_ASSERT(inp);
16852 if ((rack->rc_gp_dyn_mul) &&
16853 (rack->rc_always_pace) &&
16854 (rack->use_fixed_rate == 0) &&
16855 rack->in_probe_rtt &&
16856 (rack->r_ctl.rc_time_probertt_starts == 0)) {
16857 /*
16858 * If we are going for target, lets recheck before
16859 * we output.
16860 */
16861 rack_check_probe_rtt(rack, cts);
16862 }
16863 if (rack->set_pacing_done_a_iw == 0) {
16864 /* How much has been acked? */
16865 if ((tp->snd_una - tp->iss) > (ctf_fixed_maxseg(tp) * 10)) {
16866 /* We have enough to set in the pacing segment size */
16867 rack->set_pacing_done_a_iw = 1;
16868 rack_set_pace_segments(tp, rack, __LINE__, NULL);
16869 }
16870 }
16871 tcp_rack_xmit_timer_commit(rack, tp);
16872 #ifdef TCP_ACCOUNTING
16873 /*
16874 * If we set the ack_val_se to what ack processing we are doing
16875 * we also want to track how many cycles we burned. Note
16876 * the bits after tcp_output we let be "free". This is because
16877 * we are also tracking the tcp_output times as well. Note the
16878 * use of 0xf here since we only have 11 counter (0 - 0xa) and
16879 * 0xf cannot be returned and is what we initialize it too to
16880 * indicate we are not doing the tabulations.
16881 */
16882 if (ack_val_set != 0xf) {
16883 uint64_t crtsc;
16884
16885 crtsc = get_cyclecount();
16886 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16887 tp->tcp_proc_time[ack_val_set] += (crtsc - ts_val);
16888 }
16889 }
16890 #endif
16891 if ((nxt_pkt == 0) && (no_output == 0)) {
16892 if ((rack->r_wanted_output != 0) ||
16893 (tp->t_flags & TF_ACKNOW) ||
16894 (rack->r_fast_output != 0)) {
16895
16896 do_output_now:
16897 if (tcp_output(tp) < 0) {
16898 #ifdef TCP_ACCOUNTING
16899 sched_unpin();
16900 #endif
16901 return (1);
16902 }
16903 did_out = 1;
16904 }
16905 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
16906 rack_free_trim(rack);
16907 } else if ((nxt_pkt == 0) && (tp->t_flags & TF_ACKNOW)) {
16908 goto do_output_now;
16909 } else if ((no_output == 1) &&
16910 (nxt_pkt == 0) &&
16911 (tcp_in_hpts(rack->rc_tp) == 0)) {
16912 /*
16913 * We are not in hpts and we had a pacing timer up. Use
16914 * the remaining time (time_remaining) to restart the timer.
16915 */
16916 KASSERT ((time_remaining != 0), ("slot remaining is zero for rack:%p tp:%p", rack, tp));
16917 rack_start_hpts_timer(rack, tp, cts, time_remaining, 0, 0);
16918 rack_free_trim(rack);
16919 }
16920 /* Clear the flag, it may have been cleared by output but we may not have */
16921 if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS))
16922 tp->t_flags2 &= ~TF2_HPTS_CALLS;
16923 /*
16924 * The draft (v3) calls for us to use SEQ_GEQ, but that
16925 * causes issues when we are just going app limited. Lets
16926 * instead use SEQ_GT <or> where its equal but more data
16927 * is outstanding.
16928 *
16929 * Also make sure we are on the last ack of a series. We
16930 * have to have all the ack's processed in queue to know
16931 * if there is something left outstanding.
16932 */
16933 if (SEQ_GEQ(tp->snd_una, rack->r_ctl.roundends) &&
16934 (rack->rc_new_rnd_needed == 0) &&
16935 (nxt_pkt == 0)) {
16936 /*
16937 * We have crossed into a new round with
16938 * the new snd_unae.
16939 */
16940 rack_new_round_setup(tp, rack, tp->snd_una);
16941 }
16942 if ((nxt_pkt == 0) &&
16943 ((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
16944 (SEQ_GT(tp->snd_max, tp->snd_una) ||
16945 (tp->t_flags & TF_DELACK) ||
16946 ((V_tcp_always_keepalive || rack->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
16947 (tp->t_state <= TCPS_CLOSING)))) {
16948 /* We could not send (probably in the hpts but stopped the timer earlier)? */
16949 if ((tp->snd_max == tp->snd_una) &&
16950 ((tp->t_flags & TF_DELACK) == 0) &&
16951 (tcp_in_hpts(rack->rc_tp)) &&
16952 (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
16953 /* keep alive not needed if we are hptsi output yet */
16954 ;
16955 } else {
16956 int late = 0;
16957 if (tcp_in_hpts(tp)) {
16958 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
16959 us_cts = tcp_get_usecs(NULL);
16960 if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) {
16961 rack->r_early = 1;
16962 rack->r_ctl.rc_agg_early += (rack->r_ctl.rc_last_output_to - us_cts);
16963 } else
16964 late = 1;
16965 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
16966 }
16967 tcp_hpts_remove(tp);
16968 }
16969 if (late && (did_out == 0)) {
16970 /*
16971 * We are late in the sending
16972 * and we did not call the output
16973 * (this probably should not happen).
16974 */
16975 goto do_output_now;
16976 }
16977 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
16978 }
16979 way_out = 1;
16980 } else if (nxt_pkt == 0) {
16981 /* Do we have the correct timer running? */
16982 rack_timer_audit(tp, rack, &so->so_snd);
16983 way_out = 2;
16984 }
16985 done_with_input:
16986 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, way_out, max(1, nsegs));
16987 if (did_out)
16988 rack->r_wanted_output = 0;
16989 }
16990
16991 #ifdef TCP_ACCOUNTING
16992 sched_unpin();
16993 #endif
16994 return (retval);
16995 }
16996
16997 static void
rack_do_segment(struct tcpcb * tp,struct mbuf * m,struct tcphdr * th,int32_t drop_hdrlen,int32_t tlen,uint8_t iptos)16998 rack_do_segment(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
16999 int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
17000 {
17001 struct timeval tv;
17002
17003 /* First lets see if we have old packets */
17004 if (!STAILQ_EMPTY(&tp->t_inqueue)) {
17005 if (ctf_do_queued_segments(tp, 1)) {
17006 m_freem(m);
17007 return;
17008 }
17009 }
17010 if (m->m_flags & M_TSTMP_LRO) {
17011 mbuf_tstmp2timeval(m, &tv);
17012 } else {
17013 /* Should not be should we kassert instead? */
17014 tcp_get_usecs(&tv);
17015 }
17016 if (rack_do_segment_nounlock(tp, m, th, drop_hdrlen, tlen, iptos, 0,
17017 &tv) == 0) {
17018 INP_WUNLOCK(tptoinpcb(tp));
17019 }
17020 }
17021
17022 struct rack_sendmap *
tcp_rack_output(struct tcpcb * tp,struct tcp_rack * rack,uint32_t tsused)17023 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tsused)
17024 {
17025 struct rack_sendmap *rsm = NULL;
17026 int32_t idx;
17027 uint32_t srtt = 0, thresh = 0, ts_low = 0;
17028
17029 /* Return the next guy to be re-transmitted */
17030 if (tqhash_empty(rack->r_ctl.tqh)) {
17031 return (NULL);
17032 }
17033 if (tp->t_flags & TF_SENTFIN) {
17034 /* retran the end FIN? */
17035 return (NULL);
17036 }
17037 /* ok lets look at this one */
17038 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
17039 if (rack->r_must_retran && rsm && (rsm->r_flags & RACK_MUST_RXT)) {
17040 return (rsm);
17041 }
17042 if (rsm && ((rsm->r_flags & RACK_ACKED) == 0)) {
17043 goto check_it;
17044 }
17045 rsm = rack_find_lowest_rsm(rack);
17046 if (rsm == NULL) {
17047 return (NULL);
17048 }
17049 check_it:
17050 if (((rack->rc_tp->t_flags & TF_SACK_PERMIT) == 0) &&
17051 (rsm->r_dupack >= DUP_ACK_THRESHOLD)) {
17052 /*
17053 * No sack so we automatically do the 3 strikes and
17054 * retransmit (no rack timer would be started).
17055 */
17056 return (rsm);
17057 }
17058 if (rsm->r_flags & RACK_ACKED) {
17059 return (NULL);
17060 }
17061 if (((rsm->r_flags & RACK_SACK_PASSED) == 0) &&
17062 (rsm->r_dupack < DUP_ACK_THRESHOLD)) {
17063 /* Its not yet ready */
17064 return (NULL);
17065 }
17066 srtt = rack_grab_rtt(tp, rack);
17067 idx = rsm->r_rtr_cnt - 1;
17068 ts_low = (uint32_t)rsm->r_tim_lastsent[idx];
17069 thresh = rack_calc_thresh_rack(rack, srtt, tsused, __LINE__, 1);
17070 if ((tsused == ts_low) ||
17071 (TSTMP_LT(tsused, ts_low))) {
17072 /* No time since sending */
17073 return (NULL);
17074 }
17075 if ((tsused - ts_low) < thresh) {
17076 /* It has not been long enough yet */
17077 return (NULL);
17078 }
17079 if ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
17080 ((rsm->r_flags & RACK_SACK_PASSED))) {
17081 /*
17082 * We have passed the dup-ack threshold <or>
17083 * a SACK has indicated this is missing.
17084 * Note that if you are a declared attacker
17085 * it is only the dup-ack threshold that
17086 * will cause retransmits.
17087 */
17088 /* log retransmit reason */
17089 rack_log_retran_reason(rack, rsm, (tsused - ts_low), thresh, 1);
17090 rack->r_fast_output = 0;
17091 return (rsm);
17092 }
17093 return (NULL);
17094 }
17095
17096 static void
rack_log_pacing_delay_calc(struct tcp_rack * rack,uint32_t len,uint32_t pacing_delay,uint64_t bw_est,uint64_t bw,uint64_t len_time,int method,int line,struct rack_sendmap * rsm,uint8_t quality)17097 rack_log_pacing_delay_calc (struct tcp_rack *rack, uint32_t len, uint32_t pacing_delay,
17098 uint64_t bw_est, uint64_t bw, uint64_t len_time, int method,
17099 int line, struct rack_sendmap *rsm, uint8_t quality)
17100 {
17101 if (tcp_bblogging_on(rack->rc_tp)) {
17102 union tcp_log_stackspecific log;
17103 struct timeval tv;
17104
17105 if (rack_verbose_logging == 0) {
17106 /*
17107 * We are not verbose screen out all but
17108 * ones we always want.
17109 */
17110 if ((method != 2) &&
17111 (method != 3) &&
17112 (method != 7) &&
17113 (method != 89) &&
17114 (method != 14) &&
17115 (method != 20)) {
17116 return;
17117 }
17118 }
17119 memset(&log, 0, sizeof(log));
17120 log.u_bbr.flex1 = pacing_delay;
17121 log.u_bbr.flex2 = len;
17122 log.u_bbr.flex3 = rack->r_ctl.rc_pace_min_segs;
17123 log.u_bbr.flex4 = rack->r_ctl.rc_pace_max_segs;
17124 log.u_bbr.flex5 = rack->r_ctl.rack_per_of_gp_ss;
17125 log.u_bbr.flex6 = rack->r_ctl.rack_per_of_gp_ca;
17126 log.u_bbr.use_lt_bw = rack->rc_ack_can_sendout_data;
17127 log.u_bbr.use_lt_bw <<= 1;
17128 log.u_bbr.use_lt_bw |= rack->r_late;
17129 log.u_bbr.use_lt_bw <<= 1;
17130 log.u_bbr.use_lt_bw |= rack->r_early;
17131 log.u_bbr.use_lt_bw <<= 1;
17132 log.u_bbr.use_lt_bw |= rack->app_limited_needs_set;
17133 log.u_bbr.use_lt_bw <<= 1;
17134 log.u_bbr.use_lt_bw |= rack->rc_gp_filled;
17135 log.u_bbr.use_lt_bw <<= 1;
17136 log.u_bbr.use_lt_bw |= rack->measure_saw_probe_rtt;
17137 log.u_bbr.use_lt_bw <<= 1;
17138 log.u_bbr.use_lt_bw |= rack->in_probe_rtt;
17139 log.u_bbr.use_lt_bw <<= 1;
17140 log.u_bbr.use_lt_bw |= rack->gp_ready;
17141 log.u_bbr.pkt_epoch = line;
17142 log.u_bbr.epoch = rack->r_ctl.rc_agg_delayed;
17143 log.u_bbr.lt_epoch = rack->r_ctl.rc_agg_early;
17144 log.u_bbr.applimited = rack->r_ctl.rack_per_of_gp_rec;
17145 log.u_bbr.bw_inuse = bw_est;
17146 log.u_bbr.delRate = bw;
17147 if (rack->r_ctl.gp_bw == 0)
17148 log.u_bbr.cur_del_rate = 0;
17149 else
17150 log.u_bbr.cur_del_rate = rack_get_bw(rack);
17151 log.u_bbr.rttProp = len_time;
17152 log.u_bbr.pkts_out = rack->r_ctl.rc_rack_min_rtt;
17153 log.u_bbr.lost = rack->r_ctl.rc_probertt_sndmax_atexit;
17154 log.u_bbr.pacing_gain = rack_get_output_gain(rack, rsm);
17155 if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh) {
17156 /* We are in slow start */
17157 log.u_bbr.flex7 = 1;
17158 } else {
17159 /* we are on congestion avoidance */
17160 log.u_bbr.flex7 = 0;
17161 }
17162 log.u_bbr.flex8 = method;
17163 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
17164 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
17165 log.u_bbr.cwnd_gain = rack->rc_gp_saw_rec;
17166 log.u_bbr.cwnd_gain <<= 1;
17167 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ss;
17168 log.u_bbr.cwnd_gain <<= 1;
17169 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ca;
17170 log.u_bbr.cwnd_gain <<= 1;
17171 log.u_bbr.cwnd_gain |= rack->use_fixed_rate;
17172 log.u_bbr.cwnd_gain <<= 1;
17173 log.u_bbr.cwnd_gain |= rack->rc_always_pace;
17174 log.u_bbr.cwnd_gain <<= 1;
17175 log.u_bbr.cwnd_gain |= rack->gp_ready;
17176 log.u_bbr.bbr_substate = quality;
17177 log.u_bbr.bbr_state = rack->dgp_on;
17178 log.u_bbr.bbr_state <<= 1;
17179 log.u_bbr.bbr_state |= rack->rc_pace_to_cwnd;
17180 log.u_bbr.bbr_state <<= 2;
17181 TCP_LOG_EVENTP(rack->rc_tp, NULL,
17182 &rack->rc_inp->inp_socket->so_rcv,
17183 &rack->rc_inp->inp_socket->so_snd,
17184 BBR_LOG_HPTSI_CALC, 0,
17185 0, &log, false, &tv);
17186 }
17187 }
17188
17189 static uint32_t
rack_get_pacing_len(struct tcp_rack * rack,uint64_t bw,uint32_t mss)17190 rack_get_pacing_len(struct tcp_rack *rack, uint64_t bw, uint32_t mss)
17191 {
17192 uint32_t new_tso, user_max, pace_one;
17193
17194 user_max = rack->rc_user_set_max_segs * mss;
17195 if (rack->rc_force_max_seg) {
17196 return (user_max);
17197 }
17198 if (rack->use_fixed_rate &&
17199 ((rack->r_ctl.crte == NULL) ||
17200 (bw != rack->r_ctl.crte->rate))) {
17201 /* Use the user mss since we are not exactly matched */
17202 return (user_max);
17203 }
17204 if (rack_pace_one_seg ||
17205 (rack->r_ctl.rc_user_set_min_segs == 1))
17206 pace_one = 1;
17207 else
17208 pace_one = 0;
17209
17210 new_tso = tcp_get_pacing_burst_size_w_divisor(rack->rc_tp, bw, mss,
17211 pace_one, rack->r_ctl.crte, NULL, rack->r_ctl.pace_len_divisor);
17212 if (new_tso > user_max)
17213 new_tso = user_max;
17214 if (rack->rc_hybrid_mode && rack->r_ctl.client_suggested_maxseg) {
17215 if (((uint32_t)rack->r_ctl.client_suggested_maxseg * mss) > new_tso)
17216 new_tso = (uint32_t)rack->r_ctl.client_suggested_maxseg * mss;
17217 }
17218 if (rack->r_ctl.rc_user_set_min_segs &&
17219 ((rack->r_ctl.rc_user_set_min_segs * mss) > new_tso))
17220 new_tso = rack->r_ctl.rc_user_set_min_segs * mss;
17221 return (new_tso);
17222 }
17223
17224 static uint64_t
rack_arrive_at_discounted_rate(struct tcp_rack * rack,uint64_t window_input,uint32_t * rate_set,uint32_t * gain_b)17225 rack_arrive_at_discounted_rate(struct tcp_rack *rack, uint64_t window_input, uint32_t *rate_set, uint32_t *gain_b)
17226 {
17227 uint64_t reduced_win;
17228 uint32_t gain;
17229
17230 if (window_input < rc_init_window(rack)) {
17231 /*
17232 * The cwnd is collapsed to
17233 * nearly zero, maybe because of a time-out?
17234 * Lets drop back to the lt-bw.
17235 */
17236 reduced_win = rack_get_lt_bw(rack);
17237 /* Set the flag so the caller knows its a rate and not a reduced window */
17238 *rate_set = 1;
17239 gain = 100;
17240 } else if (IN_RECOVERY(rack->rc_tp->t_flags)) {
17241 /*
17242 * If we are in recover our cwnd needs to be less for
17243 * our pacing consideration.
17244 */
17245 if (rack->rack_hibeta == 0) {
17246 reduced_win = window_input / 2;
17247 gain = 50;
17248 } else {
17249 reduced_win = window_input * rack->r_ctl.saved_hibeta;
17250 reduced_win /= 100;
17251 gain = rack->r_ctl.saved_hibeta;
17252 }
17253 } else {
17254 /*
17255 * Apply Timely factor to increase/decrease the
17256 * amount we are pacing at.
17257 */
17258 gain = rack_get_output_gain(rack, NULL);
17259 if (gain > rack_gain_p5_ub) {
17260 gain = rack_gain_p5_ub;
17261 }
17262 reduced_win = window_input * gain;
17263 reduced_win /= 100;
17264 }
17265 if (gain_b != NULL)
17266 *gain_b = gain;
17267 /*
17268 * What is being returned here is a trimmed down
17269 * window values in all cases where rate_set is left
17270 * at 0. In one case we actually return the rate (lt_bw).
17271 * the "reduced_win" is returned as a slimmed down cwnd that
17272 * is then calculated by the caller into a rate when rate_set
17273 * is 0.
17274 */
17275 return (reduced_win);
17276 }
17277
17278 static int32_t
pace_to_fill_cwnd(struct tcp_rack * rack,int32_t pacing_delay,uint32_t len,uint32_t segsiz,int * capped,uint64_t * rate_wanted,uint8_t non_paced)17279 pace_to_fill_cwnd(struct tcp_rack *rack, int32_t pacing_delay, uint32_t len, uint32_t segsiz, int *capped, uint64_t *rate_wanted, uint8_t non_paced)
17280 {
17281 uint64_t lentim, fill_bw;
17282
17283 rack->r_via_fill_cw = 0;
17284 if (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.cwnd_to_use)
17285 return (pacing_delay);
17286 if ((ctf_outstanding(rack->rc_tp) + (segsiz-1)) > rack->rc_tp->snd_wnd)
17287 return (pacing_delay);
17288 if (rack->r_ctl.rc_last_us_rtt == 0)
17289 return (pacing_delay);
17290 if (rack->rc_pace_fill_if_rttin_range &&
17291 (rack->r_ctl.rc_last_us_rtt >=
17292 (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack->rtt_limit_mul))) {
17293 /* The rtt is huge, N * smallest, lets not fill */
17294 return (pacing_delay);
17295 }
17296 if (rack->r_ctl.fillcw_cap && *rate_wanted >= rack->r_ctl.fillcw_cap)
17297 return (pacing_delay);
17298 /*
17299 * first lets calculate the b/w based on the last us-rtt
17300 * and the the smallest send window.
17301 */
17302 fill_bw = min(rack->rc_tp->snd_cwnd, rack->r_ctl.cwnd_to_use);
17303 if (rack->rc_fillcw_apply_discount) {
17304 uint32_t rate_set = 0;
17305
17306 fill_bw = rack_arrive_at_discounted_rate(rack, fill_bw, &rate_set, NULL);
17307 if (rate_set) {
17308 goto at_lt_bw;
17309 }
17310 }
17311 /* Take the rwnd if its smaller */
17312 if (fill_bw > rack->rc_tp->snd_wnd)
17313 fill_bw = rack->rc_tp->snd_wnd;
17314 /* Now lets make it into a b/w */
17315 fill_bw *= (uint64_t)HPTS_USEC_IN_SEC;
17316 fill_bw /= (uint64_t)rack->r_ctl.rc_last_us_rtt;
17317 /* Adjust to any cap */
17318 if (rack->r_ctl.fillcw_cap && fill_bw >= rack->r_ctl.fillcw_cap)
17319 fill_bw = rack->r_ctl.fillcw_cap;
17320
17321 at_lt_bw:
17322 if (rack_bw_multipler > 0) {
17323 /*
17324 * We want to limit fill-cw to the some multiplier
17325 * of the max(lt_bw, gp_est). The normal default
17326 * is 0 for off, so a sysctl has enabled it.
17327 */
17328 uint64_t lt_bw, gp, rate;
17329
17330 gp = rack_get_gp_est(rack);
17331 lt_bw = rack_get_lt_bw(rack);
17332 if (lt_bw > gp)
17333 rate = lt_bw;
17334 else
17335 rate = gp;
17336 rate *= rack_bw_multipler;
17337 rate /= 100;
17338 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
17339 union tcp_log_stackspecific log;
17340 struct timeval tv;
17341
17342 memset(&log, 0, sizeof(log));
17343 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
17344 log.u_bbr.flex1 = rack_bw_multipler;
17345 log.u_bbr.flex2 = len;
17346 log.u_bbr.cur_del_rate = gp;
17347 log.u_bbr.delRate = lt_bw;
17348 log.u_bbr.bw_inuse = rate;
17349 log.u_bbr.rttProp = fill_bw;
17350 log.u_bbr.flex8 = 44;
17351 tcp_log_event(rack->rc_tp, NULL, NULL, NULL,
17352 BBR_LOG_CWND, 0,
17353 0, &log, false, NULL,
17354 __func__, __LINE__, &tv);
17355 }
17356 if (fill_bw > rate)
17357 fill_bw = rate;
17358 }
17359 /* We are below the min b/w */
17360 if (non_paced)
17361 *rate_wanted = fill_bw;
17362 if ((fill_bw < RACK_MIN_BW) || (fill_bw < *rate_wanted))
17363 return (pacing_delay);
17364 rack->r_via_fill_cw = 1;
17365 if (rack->r_rack_hw_rate_caps &&
17366 (rack->r_ctl.crte != NULL)) {
17367 uint64_t high_rate;
17368
17369 high_rate = tcp_hw_highest_rate(rack->r_ctl.crte);
17370 if (fill_bw > high_rate) {
17371 /* We are capping bw at the highest rate table entry */
17372 if (*rate_wanted > high_rate) {
17373 /* The original rate was also capped */
17374 rack->r_via_fill_cw = 0;
17375 }
17376 rack_log_hdwr_pacing(rack,
17377 fill_bw, high_rate, __LINE__,
17378 0, 3);
17379 fill_bw = high_rate;
17380 if (capped)
17381 *capped = 1;
17382 }
17383 } else if ((rack->r_ctl.crte == NULL) &&
17384 (rack->rack_hdrw_pacing == 0) &&
17385 (rack->rack_hdw_pace_ena) &&
17386 rack->r_rack_hw_rate_caps &&
17387 (rack->rack_attempt_hdwr_pace == 0) &&
17388 (rack->rc_inp->inp_route.ro_nh != NULL) &&
17389 (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) {
17390 /*
17391 * Ok we may have a first attempt that is greater than our top rate
17392 * lets check.
17393 */
17394 uint64_t high_rate;
17395
17396 high_rate = tcp_hw_highest_rate_ifp(rack->rc_inp->inp_route.ro_nh->nh_ifp, rack->rc_inp);
17397 if (high_rate) {
17398 if (fill_bw > high_rate) {
17399 fill_bw = high_rate;
17400 if (capped)
17401 *capped = 1;
17402 }
17403 }
17404 }
17405 if (rack->r_ctl.bw_rate_cap && (fill_bw > rack->r_ctl.bw_rate_cap)) {
17406 rack_log_hybrid_bw(rack, rack->rc_tp->snd_max,
17407 fill_bw, 0, 0, HYBRID_LOG_RATE_CAP, 2, NULL, __LINE__);
17408 fill_bw = rack->r_ctl.bw_rate_cap;
17409 }
17410 /*
17411 * Ok fill_bw holds our mythical b/w to fill the cwnd
17412 * in an rtt (unless it was capped), what does that
17413 * time wise equate too?
17414 */
17415 lentim = (uint64_t)(len) * (uint64_t)HPTS_USEC_IN_SEC;
17416 lentim /= fill_bw;
17417 *rate_wanted = fill_bw;
17418 if (non_paced || (lentim < pacing_delay)) {
17419 rack_log_pacing_delay_calc(rack, len, pacing_delay, fill_bw,
17420 0, lentim, 12, __LINE__, NULL, 0);
17421 return ((int32_t)lentim);
17422 } else
17423 return (pacing_delay);
17424 }
17425
17426 static int32_t
rack_get_pacing_delay(struct tcp_rack * rack,struct tcpcb * tp,uint32_t len,struct rack_sendmap * rsm,uint32_t segsiz,int line)17427 rack_get_pacing_delay(struct tcp_rack *rack, struct tcpcb *tp, uint32_t len, struct rack_sendmap *rsm, uint32_t segsiz, int line)
17428 {
17429 uint64_t srtt;
17430 int32_t pacing_delay = 0;
17431 int can_start_hw_pacing = 1;
17432 int err;
17433 int pace_one;
17434
17435 if (rack_pace_one_seg ||
17436 (rack->r_ctl.rc_user_set_min_segs == 1))
17437 pace_one = 1;
17438 else
17439 pace_one = 0;
17440 if (rack->rc_always_pace == 0) {
17441 /*
17442 * We use the most optimistic possible cwnd/srtt for
17443 * sending calculations. This will make our
17444 * calculation anticipate getting more through
17445 * quicker then possible. But thats ok we don't want
17446 * the peer to have a gap in data sending.
17447 */
17448 uint64_t cwnd, tr_perms = 0;
17449 int32_t reduce;
17450
17451 old_method:
17452 /*
17453 * We keep no precise pacing with the old method
17454 * instead we use the pacer to mitigate bursts.
17455 */
17456 if (rack->r_ctl.rc_rack_min_rtt)
17457 srtt = rack->r_ctl.rc_rack_min_rtt;
17458 else
17459 srtt = max(tp->t_srtt, 1);
17460 if (rack->r_ctl.rc_rack_largest_cwnd)
17461 cwnd = rack->r_ctl.rc_rack_largest_cwnd;
17462 else
17463 cwnd = rack->r_ctl.cwnd_to_use;
17464 /* Inflate cwnd by 1000 so srtt of usecs is in ms */
17465 tr_perms = (cwnd * 1000) / srtt;
17466 if (tr_perms == 0) {
17467 tr_perms = ctf_fixed_maxseg(tp);
17468 }
17469 /*
17470 * Calculate how long this will take to drain, if
17471 * the calculation comes out to zero, thats ok we
17472 * will use send_a_lot to possibly spin around for
17473 * more increasing tot_len_this_send to the point
17474 * that its going to require a pace, or we hit the
17475 * cwnd. Which in that case we are just waiting for
17476 * a ACK.
17477 */
17478 pacing_delay = len / tr_perms;
17479 /* Now do we reduce the time so we don't run dry? */
17480 if (pacing_delay && rack_pacing_delay_reduction) {
17481 reduce = (pacing_delay / rack_pacing_delay_reduction);
17482 if (reduce < pacing_delay) {
17483 pacing_delay -= reduce;
17484 } else
17485 pacing_delay = 0;
17486 } else
17487 reduce = 0;
17488 pacing_delay *= HPTS_USEC_IN_MSEC;
17489 if (rack->rc_pace_to_cwnd) {
17490 uint64_t rate_wanted = 0;
17491
17492 pacing_delay = pace_to_fill_cwnd(rack, pacing_delay, len, segsiz, NULL, &rate_wanted, 1);
17493 rack->rc_ack_can_sendout_data = 1;
17494 rack_log_pacing_delay_calc(rack, len, pacing_delay, rate_wanted, 0, 0, 14, __LINE__, NULL, 0);
17495 } else
17496 rack_log_pacing_delay_calc(rack, len, pacing_delay, tr_perms, reduce, 0, 7, __LINE__, NULL, 0);
17497 /*******************************************************/
17498 /* RRS: We insert non-paced call to stats here for len */
17499 /*******************************************************/
17500 } else {
17501 uint64_t bw_est, res, lentim, rate_wanted;
17502 uint32_t segs, oh;
17503 int capped = 0;
17504 int prev_fill;
17505
17506 if ((rack->r_rr_config == 1) && rsm) {
17507 return (rack->r_ctl.rc_min_to);
17508 }
17509 if (rack->use_fixed_rate) {
17510 rate_wanted = bw_est = rack_get_fixed_pacing_bw(rack);
17511 } else if ((rack->r_ctl.init_rate == 0) &&
17512 (rack->r_ctl.gp_bw == 0)) {
17513 /* no way to yet do an estimate */
17514 bw_est = rate_wanted = 0;
17515 } else if (rack->dgp_on) {
17516 bw_est = rack_get_bw(rack);
17517 rate_wanted = rack_get_output_bw(rack, bw_est, rsm, &capped);
17518 } else {
17519 uint32_t gain, rate_set = 0;
17520
17521 rate_wanted = min(rack->rc_tp->snd_cwnd, rack->r_ctl.cwnd_to_use);
17522 rate_wanted = rack_arrive_at_discounted_rate(rack, rate_wanted, &rate_set, &gain);
17523 if (rate_set == 0) {
17524 if (rate_wanted > rack->rc_tp->snd_wnd)
17525 rate_wanted = rack->rc_tp->snd_wnd;
17526 /* Now lets make it into a b/w */
17527 rate_wanted *= (uint64_t)HPTS_USEC_IN_SEC;
17528 rate_wanted /= (uint64_t)rack->r_ctl.rc_last_us_rtt;
17529 }
17530 bw_est = rate_wanted;
17531 rack_log_pacing_delay_calc(rack, rack->rc_tp->snd_cwnd,
17532 rack->r_ctl.cwnd_to_use,
17533 rate_wanted, bw_est,
17534 rack->r_ctl.rc_last_us_rtt,
17535 88, __LINE__, NULL, gain);
17536 }
17537 if (((bw_est == 0) || (rate_wanted == 0) || (rack->gp_ready == 0)) &&
17538 (rack->use_fixed_rate == 0)) {
17539 /*
17540 * No way yet to make a b/w estimate or
17541 * our raise is set incorrectly.
17542 */
17543 goto old_method;
17544 }
17545 rack_rate_cap_bw(rack, &rate_wanted, &capped);
17546 /* We need to account for all the overheads */
17547 segs = (len + segsiz - 1) / segsiz;
17548 /*
17549 * We need the diff between 1514 bytes (e-mtu with e-hdr)
17550 * and how much data we put in each packet. Yes this
17551 * means we may be off if we are larger than 1500 bytes
17552 * or smaller. But this just makes us more conservative.
17553 */
17554
17555 oh = (tp->t_maxseg - segsiz) + sizeof(struct tcphdr);
17556 if (rack->r_is_v6) {
17557 #ifdef INET6
17558 oh += sizeof(struct ip6_hdr);
17559 #endif
17560 } else {
17561 #ifdef INET
17562 oh += sizeof(struct ip);
17563 #endif
17564 }
17565 /* We add a fixed 14 for the ethernet header */
17566 oh += 14;
17567 segs *= oh;
17568 lentim = (uint64_t)(len + segs) * (uint64_t)HPTS_USEC_IN_SEC;
17569 res = lentim / rate_wanted;
17570 pacing_delay = (uint32_t)res;
17571 if (rack_hw_rate_min &&
17572 (rate_wanted < rack_hw_rate_min)) {
17573 can_start_hw_pacing = 0;
17574 if (rack->r_ctl.crte) {
17575 /*
17576 * Ok we need to release it, we
17577 * have fallen too low.
17578 */
17579 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
17580 rack->r_ctl.crte = NULL;
17581 rack->rack_attempt_hdwr_pace = 0;
17582 rack->rack_hdrw_pacing = 0;
17583 }
17584 }
17585 if (rack->r_ctl.crte &&
17586 (tcp_hw_highest_rate(rack->r_ctl.crte) < rate_wanted)) {
17587 /*
17588 * We want more than the hardware can give us,
17589 * don't start any hw pacing.
17590 */
17591 can_start_hw_pacing = 0;
17592 if (rack->r_rack_hw_rate_caps == 0) {
17593 /*
17594 * Ok we need to release it, we
17595 * want more than the card can give us and
17596 * no rate cap is in place. Set it up so
17597 * when we want less we can retry.
17598 */
17599 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
17600 rack->r_ctl.crte = NULL;
17601 rack->rack_attempt_hdwr_pace = 0;
17602 rack->rack_hdrw_pacing = 0;
17603 }
17604 }
17605 if ((rack->r_ctl.crte != NULL) && (rack->rc_inp->inp_snd_tag == NULL)) {
17606 /*
17607 * We lost our rate somehow, this can happen
17608 * if the interface changed underneath us.
17609 */
17610 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
17611 rack->r_ctl.crte = NULL;
17612 /* Lets re-allow attempting to setup pacing */
17613 rack->rack_hdrw_pacing = 0;
17614 rack->rack_attempt_hdwr_pace = 0;
17615 rack_log_hdwr_pacing(rack,
17616 rate_wanted, bw_est, __LINE__,
17617 0, 6);
17618 }
17619 prev_fill = rack->r_via_fill_cw;
17620 if ((rack->rc_pace_to_cwnd) &&
17621 (capped == 0) &&
17622 (rack->dgp_on == 1) &&
17623 (rack->use_fixed_rate == 0) &&
17624 (rack->in_probe_rtt == 0) &&
17625 (IN_FASTRECOVERY(rack->rc_tp->t_flags) == 0)) {
17626 /*
17627 * We want to pace at our rate *or* faster to
17628 * fill the cwnd to the max if its not full.
17629 */
17630 pacing_delay = pace_to_fill_cwnd(rack, pacing_delay, (len+segs), segsiz, &capped, &rate_wanted, 0);
17631 /* Re-check to make sure we are not exceeding our max b/w */
17632 if ((rack->r_ctl.crte != NULL) &&
17633 (tcp_hw_highest_rate(rack->r_ctl.crte) < rate_wanted)) {
17634 /*
17635 * We want more than the hardware can give us,
17636 * don't start any hw pacing.
17637 */
17638 can_start_hw_pacing = 0;
17639 if (rack->r_rack_hw_rate_caps == 0) {
17640 /*
17641 * Ok we need to release it, we
17642 * want more than the card can give us and
17643 * no rate cap is in place. Set it up so
17644 * when we want less we can retry.
17645 */
17646 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
17647 rack->r_ctl.crte = NULL;
17648 rack->rack_attempt_hdwr_pace = 0;
17649 rack->rack_hdrw_pacing = 0;
17650 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
17651 }
17652 }
17653 }
17654 if ((rack->rc_inp->inp_route.ro_nh != NULL) &&
17655 (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) {
17656 if ((rack->rack_hdw_pace_ena) &&
17657 (can_start_hw_pacing > 0) &&
17658 (rack->rack_hdrw_pacing == 0) &&
17659 (rack->rack_attempt_hdwr_pace == 0)) {
17660 /*
17661 * Lets attempt to turn on hardware pacing
17662 * if we can.
17663 */
17664 rack->rack_attempt_hdwr_pace = 1;
17665 rack->r_ctl.crte = tcp_set_pacing_rate(rack->rc_tp,
17666 rack->rc_inp->inp_route.ro_nh->nh_ifp,
17667 rate_wanted,
17668 RS_PACING_GEQ,
17669 &err, &rack->r_ctl.crte_prev_rate);
17670 if (rack->r_ctl.crte) {
17671 rack->rack_hdrw_pacing = 1;
17672 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size_w_divisor(tp, rate_wanted, segsiz,
17673 pace_one, rack->r_ctl.crte,
17674 NULL, rack->r_ctl.pace_len_divisor);
17675 rack_log_hdwr_pacing(rack,
17676 rate_wanted, rack->r_ctl.crte->rate, __LINE__,
17677 err, 0);
17678 rack->r_ctl.last_hw_bw_req = rate_wanted;
17679 } else {
17680 counter_u64_add(rack_hw_pace_init_fail, 1);
17681 }
17682 } else if (rack->rack_hdrw_pacing &&
17683 (rack->r_ctl.last_hw_bw_req != rate_wanted)) {
17684 /* Do we need to adjust our rate? */
17685 const struct tcp_hwrate_limit_table *nrte;
17686
17687 if (rack->r_up_only &&
17688 (rate_wanted < rack->r_ctl.crte->rate)) {
17689 /**
17690 * We have four possible states here
17691 * having to do with the previous time
17692 * and this time.
17693 * previous | this-time
17694 * A) 0 | 0 -- fill_cw not in the picture
17695 * B) 1 | 0 -- we were doing a fill-cw but now are not
17696 * C) 1 | 1 -- all rates from fill_cw
17697 * D) 0 | 1 -- we were doing non-fill and now we are filling
17698 *
17699 * For case A, C and D we don't allow a drop. But for
17700 * case B where we now our on our steady rate we do
17701 * allow a drop.
17702 *
17703 */
17704 if (!((prev_fill == 1) && (rack->r_via_fill_cw == 0)))
17705 goto done_w_hdwr;
17706 }
17707 if ((rate_wanted > rack->r_ctl.crte->rate) ||
17708 (rate_wanted <= rack->r_ctl.crte_prev_rate)) {
17709 if (rack_hw_rate_to_low &&
17710 (bw_est < rack_hw_rate_to_low)) {
17711 /*
17712 * The pacing rate is too low for hardware, but
17713 * do allow hardware pacing to be restarted.
17714 */
17715 rack_log_hdwr_pacing(rack,
17716 bw_est, rack->r_ctl.crte->rate, __LINE__,
17717 0, 5);
17718 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
17719 rack->r_ctl.crte = NULL;
17720 rack->rack_attempt_hdwr_pace = 0;
17721 rack->rack_hdrw_pacing = 0;
17722 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted);
17723 goto done_w_hdwr;
17724 }
17725 nrte = tcp_chg_pacing_rate(rack->r_ctl.crte,
17726 rack->rc_tp,
17727 rack->rc_inp->inp_route.ro_nh->nh_ifp,
17728 rate_wanted,
17729 RS_PACING_GEQ,
17730 &err, &rack->r_ctl.crte_prev_rate);
17731 if (nrte == NULL) {
17732 /*
17733 * Lost the rate, lets drop hardware pacing
17734 * period.
17735 */
17736 rack->rack_hdrw_pacing = 0;
17737 rack->r_ctl.crte = NULL;
17738 rack_log_hdwr_pacing(rack,
17739 rate_wanted, 0, __LINE__,
17740 err, 1);
17741 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted);
17742 counter_u64_add(rack_hw_pace_lost, 1);
17743 } else if (nrte != rack->r_ctl.crte) {
17744 rack->r_ctl.crte = nrte;
17745 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size_w_divisor(tp, rate_wanted,
17746 segsiz, pace_one, rack->r_ctl.crte,
17747 NULL, rack->r_ctl.pace_len_divisor);
17748 rack_log_hdwr_pacing(rack,
17749 rate_wanted, rack->r_ctl.crte->rate, __LINE__,
17750 err, 2);
17751 rack->r_ctl.last_hw_bw_req = rate_wanted;
17752 }
17753 } else {
17754 /* We just need to adjust the segment size */
17755 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted);
17756 rack_log_hdwr_pacing(rack,
17757 rate_wanted, rack->r_ctl.crte->rate, __LINE__,
17758 0, 4);
17759 rack->r_ctl.last_hw_bw_req = rate_wanted;
17760 }
17761 }
17762 }
17763 done_w_hdwr:
17764 if (rack_limit_time_with_srtt &&
17765 (rack->use_fixed_rate == 0) &&
17766 (rack->rack_hdrw_pacing == 0)) {
17767 /*
17768 * Sanity check, we do not allow the pacing delay
17769 * to be longer than the SRTT of the path. If it is
17770 * a slow path, then adding a packet should increase
17771 * the RTT and compensate for this i.e. the srtt will
17772 * be greater so the allowed pacing time will be greater.
17773 *
17774 * Note this restriction is not for where a peak rate
17775 * is set, we are doing fixed pacing or hardware pacing.
17776 */
17777 if (rack->rc_tp->t_srtt)
17778 srtt = rack->rc_tp->t_srtt;
17779 else
17780 srtt = RACK_INITIAL_RTO * HPTS_USEC_IN_MSEC; /* its in ms convert */
17781 if (srtt < (uint64_t)pacing_delay) {
17782 rack_log_pacing_delay_calc(rack, srtt, pacing_delay, rate_wanted, bw_est, lentim, 99, __LINE__, NULL, 0);
17783 pacing_delay = srtt;
17784 }
17785 }
17786 /*******************************************************************/
17787 /* RRS: We insert paced call to stats here for len and rate_wanted */
17788 /*******************************************************************/
17789 rack_log_pacing_delay_calc(rack, len, pacing_delay, rate_wanted, bw_est, lentim, 2, __LINE__, rsm, 0);
17790 }
17791 if (rack->r_ctl.crte && (rack->r_ctl.crte->rs_num_enobufs > 0)) {
17792 /*
17793 * If this rate is seeing enobufs when it
17794 * goes to send then either the nic is out
17795 * of gas or we are mis-estimating the time
17796 * somehow and not letting the queue empty
17797 * completely. Lets add to the pacing time.
17798 */
17799 int hw_boost_delay;
17800
17801 hw_boost_delay = rack->r_ctl.crte->time_between * rack_enobuf_hw_boost_mult;
17802 if (hw_boost_delay > rack_enobuf_hw_max)
17803 hw_boost_delay = rack_enobuf_hw_max;
17804 else if (hw_boost_delay < rack_enobuf_hw_min)
17805 hw_boost_delay = rack_enobuf_hw_min;
17806 pacing_delay += hw_boost_delay;
17807 }
17808 return (pacing_delay);
17809 }
17810
17811 static void
rack_start_gp_measurement(struct tcpcb * tp,struct tcp_rack * rack,tcp_seq startseq,uint32_t sb_offset)17812 rack_start_gp_measurement(struct tcpcb *tp, struct tcp_rack *rack,
17813 tcp_seq startseq, uint32_t sb_offset)
17814 {
17815 struct rack_sendmap *my_rsm = NULL;
17816
17817 if (tp->t_state < TCPS_ESTABLISHED) {
17818 /*
17819 * We don't start any measurements if we are
17820 * not at least established.
17821 */
17822 return;
17823 }
17824 if (tp->t_state >= TCPS_FIN_WAIT_1) {
17825 /*
17826 * We will get no more data into the SB
17827 * this means we need to have the data available
17828 * before we start a measurement.
17829 */
17830
17831 if (sbavail(&tptosocket(tp)->so_snd) <
17832 max(rc_init_window(rack),
17833 (MIN_GP_WIN * ctf_fixed_maxseg(tp)))) {
17834 /* Nope not enough data */
17835 return;
17836 }
17837 }
17838 tp->t_flags |= TF_GPUTINPROG;
17839 rack->r_ctl.rc_gp_cumack_ts = 0;
17840 rack->r_ctl.rc_gp_lowrtt = 0xffffffff;
17841 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd;
17842 tp->gput_seq = startseq;
17843 rack->app_limited_needs_set = 0;
17844 if (rack->in_probe_rtt)
17845 rack->measure_saw_probe_rtt = 1;
17846 else if ((rack->measure_saw_probe_rtt) &&
17847 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit)))
17848 rack->measure_saw_probe_rtt = 0;
17849 if (rack->rc_gp_filled)
17850 tp->gput_ts = rack->r_ctl.last_cumack_advance;
17851 else {
17852 /* Special case initial measurement */
17853 struct timeval tv;
17854
17855 tp->gput_ts = tcp_get_usecs(&tv);
17856 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv);
17857 }
17858 /*
17859 * We take a guess out into the future,
17860 * if we have no measurement and no
17861 * initial rate, we measure the first
17862 * initial-windows worth of data to
17863 * speed up getting some GP measurement and
17864 * thus start pacing.
17865 */
17866 if ((rack->rc_gp_filled == 0) && (rack->r_ctl.init_rate == 0)) {
17867 rack->app_limited_needs_set = 1;
17868 tp->gput_ack = startseq + max(rc_init_window(rack),
17869 (MIN_GP_WIN * ctf_fixed_maxseg(tp)));
17870 rack_log_pacing_delay_calc(rack,
17871 tp->gput_seq,
17872 tp->gput_ack,
17873 0,
17874 tp->gput_ts,
17875 (((uint64_t)rack->r_ctl.rc_app_limited_cnt << 32) | (uint64_t)rack->r_ctl.rc_gp_output_ts),
17876 9,
17877 __LINE__, NULL, 0);
17878 rack_tend_gp_marks(tp, rack);
17879 rack_log_gpset(rack, tp->gput_ack, 0, 0, __LINE__, 1, NULL);
17880 return;
17881 }
17882 if (sb_offset) {
17883 /*
17884 * We are out somewhere in the sb
17885 * can we use the already outstanding data?
17886 */
17887
17888 if (rack->r_ctl.rc_app_limited_cnt == 0) {
17889 /*
17890 * Yes first one is good and in this case
17891 * the tp->gput_ts is correctly set based on
17892 * the last ack that arrived (no need to
17893 * set things up when an ack comes in).
17894 */
17895 my_rsm = tqhash_min(rack->r_ctl.tqh);
17896 if ((my_rsm == NULL) ||
17897 (my_rsm->r_rtr_cnt != 1)) {
17898 /* retransmission? */
17899 goto use_latest;
17900 }
17901 } else {
17902 if (rack->r_ctl.rc_first_appl == NULL) {
17903 /*
17904 * If rc_first_appl is NULL
17905 * then the cnt should be 0.
17906 * This is probably an error, maybe
17907 * a KASSERT would be approprate.
17908 */
17909 goto use_latest;
17910 }
17911 /*
17912 * If we have a marker pointer to the last one that is
17913 * app limited we can use that, but we need to set
17914 * things up so that when it gets ack'ed we record
17915 * the ack time (if its not already acked).
17916 */
17917 rack->app_limited_needs_set = 1;
17918 /*
17919 * We want to get to the rsm that is either
17920 * next with space i.e. over 1 MSS or the one
17921 * after that (after the app-limited).
17922 */
17923 my_rsm = tqhash_next(rack->r_ctl.tqh, rack->r_ctl.rc_first_appl);
17924 if (my_rsm) {
17925 if ((my_rsm->r_end - my_rsm->r_start) <= ctf_fixed_maxseg(tp))
17926 /* Have to use the next one */
17927 my_rsm = tqhash_next(rack->r_ctl.tqh, my_rsm);
17928 else {
17929 /* Use after the first MSS of it is acked */
17930 tp->gput_seq = my_rsm->r_start + ctf_fixed_maxseg(tp);
17931 goto start_set;
17932 }
17933 }
17934 if ((my_rsm == NULL) ||
17935 (my_rsm->r_rtr_cnt != 1)) {
17936 /*
17937 * Either its a retransmit or
17938 * the last is the app-limited one.
17939 */
17940 goto use_latest;
17941 }
17942 }
17943 tp->gput_seq = my_rsm->r_start;
17944 start_set:
17945 if (my_rsm->r_flags & RACK_ACKED) {
17946 /*
17947 * This one has been acked use the arrival ack time
17948 */
17949 struct rack_sendmap *nrsm;
17950
17951 tp->gput_ts = (uint32_t)my_rsm->r_ack_arrival;
17952 rack->app_limited_needs_set = 0;
17953 /*
17954 * Ok in this path we need to use the r_end now
17955 * since this guy is the starting ack.
17956 */
17957 tp->gput_seq = my_rsm->r_end;
17958 /*
17959 * We also need to adjust up the sendtime
17960 * to the send of the next data after my_rsm.
17961 */
17962 nrsm = tqhash_next(rack->r_ctl.tqh, my_rsm);
17963 if (nrsm != NULL)
17964 my_rsm = nrsm;
17965 else {
17966 /*
17967 * The next as not been sent, thats the
17968 * case for using the latest.
17969 */
17970 goto use_latest;
17971 }
17972 }
17973 rack->r_ctl.rc_gp_output_ts = my_rsm->r_tim_lastsent[0];
17974 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack);
17975 rack->r_ctl.rc_gp_cumack_ts = 0;
17976 if ((rack->r_ctl.cleared_app_ack == 1) &&
17977 (SEQ_GEQ(tp->gput_seq, rack->r_ctl.cleared_app_ack_seq))) {
17978 /*
17979 * We just cleared an application limited period
17980 * so the next seq out needs to skip the first
17981 * ack.
17982 */
17983 rack->app_limited_needs_set = 1;
17984 rack->r_ctl.cleared_app_ack = 0;
17985 }
17986 rack_log_pacing_delay_calc(rack,
17987 tp->gput_seq,
17988 tp->gput_ack,
17989 (uintptr_t)my_rsm,
17990 tp->gput_ts,
17991 (((uint64_t)rack->r_ctl.rc_app_limited_cnt << 32) | (uint64_t)rack->r_ctl.rc_gp_output_ts),
17992 9,
17993 __LINE__, my_rsm, 0);
17994 /* Now lets make sure all are marked as they should be */
17995 rack_tend_gp_marks(tp, rack);
17996 rack_log_gpset(rack, tp->gput_ack, 0, 0, __LINE__, 1, NULL);
17997 return;
17998 }
17999
18000 use_latest:
18001 /*
18002 * We don't know how long we may have been
18003 * idle or if this is the first-send. Lets
18004 * setup the flag so we will trim off
18005 * the first ack'd data so we get a true
18006 * measurement.
18007 */
18008 rack->app_limited_needs_set = 1;
18009 tp->gput_ack = startseq + rack_get_measure_window(tp, rack);
18010 rack->r_ctl.rc_gp_cumack_ts = 0;
18011 /* Find this guy so we can pull the send time */
18012 my_rsm = tqhash_find(rack->r_ctl.tqh, startseq);
18013 if (my_rsm) {
18014 rack->r_ctl.rc_gp_output_ts = my_rsm->r_tim_lastsent[0];
18015 if (my_rsm->r_flags & RACK_ACKED) {
18016 /*
18017 * Unlikely since its probably what was
18018 * just transmitted (but I am paranoid).
18019 */
18020 tp->gput_ts = (uint32_t)my_rsm->r_ack_arrival;
18021 rack->app_limited_needs_set = 0;
18022 }
18023 if (SEQ_LT(my_rsm->r_start, tp->gput_seq)) {
18024 /* This also is unlikely */
18025 tp->gput_seq = my_rsm->r_start;
18026 }
18027 } else {
18028 /*
18029 * TSNH unless we have some send-map limit,
18030 * and even at that it should not be hitting
18031 * that limit (we should have stopped sending).
18032 */
18033 struct timeval tv;
18034
18035 microuptime(&tv);
18036 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv);
18037 }
18038 rack_tend_gp_marks(tp, rack);
18039 rack_log_pacing_delay_calc(rack,
18040 tp->gput_seq,
18041 tp->gput_ack,
18042 (uintptr_t)my_rsm,
18043 tp->gput_ts,
18044 (((uint64_t)rack->r_ctl.rc_app_limited_cnt << 32) | (uint64_t)rack->r_ctl.rc_gp_output_ts),
18045 9, __LINE__, NULL, 0);
18046 rack_log_gpset(rack, tp->gput_ack, 0, 0, __LINE__, 1, NULL);
18047 }
18048
18049 static inline uint32_t
rack_what_can_we_send(struct tcpcb * tp,struct tcp_rack * rack,uint32_t cwnd_to_use,uint32_t avail,int32_t sb_offset)18050 rack_what_can_we_send(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cwnd_to_use,
18051 uint32_t avail, int32_t sb_offset)
18052 {
18053 uint32_t len;
18054 uint32_t sendwin;
18055
18056 if (tp->snd_wnd > cwnd_to_use)
18057 sendwin = cwnd_to_use;
18058 else
18059 sendwin = tp->snd_wnd;
18060 if (ctf_outstanding(tp) >= tp->snd_wnd) {
18061 /* We never want to go over our peers rcv-window */
18062 len = 0;
18063 } else {
18064 uint32_t flight;
18065
18066 flight = ctf_flight_size(tp, rack->r_ctl.rc_sacked);
18067 if (flight >= sendwin) {
18068 /*
18069 * We have in flight what we are allowed by cwnd (if
18070 * it was rwnd blocking it would have hit above out
18071 * >= tp->snd_wnd).
18072 */
18073 return (0);
18074 }
18075 len = sendwin - flight;
18076 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {
18077 /* We would send too much (beyond the rwnd) */
18078 len = tp->snd_wnd - ctf_outstanding(tp);
18079 }
18080 if ((len + sb_offset) > avail) {
18081 /*
18082 * We don't have that much in the SB, how much is
18083 * there?
18084 */
18085 len = avail - sb_offset;
18086 }
18087 }
18088 return (len);
18089 }
18090
18091 static void
rack_log_fsb(struct tcp_rack * rack,struct tcpcb * tp,struct socket * so,uint32_t flags,unsigned ipoptlen,int32_t orig_len,int32_t len,int error,int rsm_is_null,int optlen,int line,uint16_t mode)18092 rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t flags,
18093 unsigned ipoptlen, int32_t orig_len, int32_t len, int error,
18094 int rsm_is_null, int optlen, int line, uint16_t mode)
18095 {
18096 if (rack_verbose_logging && tcp_bblogging_on(rack->rc_tp)) {
18097 union tcp_log_stackspecific log;
18098 struct timeval tv;
18099
18100 memset(&log, 0, sizeof(log));
18101 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
18102 log.u_bbr.flex1 = error;
18103 log.u_bbr.flex2 = flags;
18104 log.u_bbr.flex3 = rsm_is_null;
18105 log.u_bbr.flex4 = ipoptlen;
18106 log.u_bbr.flex5 = tp->rcv_numsacks;
18107 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
18108 log.u_bbr.flex7 = optlen;
18109 log.u_bbr.flex8 = rack->r_fsb_inited;
18110 log.u_bbr.applimited = rack->r_fast_output;
18111 log.u_bbr.bw_inuse = rack_get_bw(rack);
18112 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL);
18113 log.u_bbr.cwnd_gain = mode;
18114 log.u_bbr.pkts_out = orig_len;
18115 log.u_bbr.lt_epoch = len;
18116 log.u_bbr.delivered = line;
18117 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
18118 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
18119 tcp_log_event(tp, NULL, &so->so_rcv, &so->so_snd, TCP_LOG_FSB, 0,
18120 len, &log, false, NULL, __func__, __LINE__, &tv);
18121 }
18122 }
18123
18124
18125 static struct mbuf *
rack_fo_base_copym(struct mbuf * the_m,uint32_t the_off,int32_t * plen,struct rack_fast_send_blk * fsb,int32_t seglimit,int32_t segsize,int hw_tls)18126 rack_fo_base_copym(struct mbuf *the_m, uint32_t the_off, int32_t *plen,
18127 struct rack_fast_send_blk *fsb,
18128 int32_t seglimit, int32_t segsize, int hw_tls)
18129 {
18130 #ifdef KERN_TLS
18131 struct ktls_session *tls, *ntls;
18132 #ifdef INVARIANTS
18133 struct mbuf *start;
18134 #endif
18135 #endif
18136 struct mbuf *m, *n, **np, *smb;
18137 struct mbuf *top;
18138 int32_t off, soff;
18139 int32_t len = *plen;
18140 int32_t fragsize;
18141 int32_t len_cp = 0;
18142 uint32_t mlen, frags;
18143
18144 soff = off = the_off;
18145 smb = m = the_m;
18146 np = ⊤
18147 top = NULL;
18148 #ifdef KERN_TLS
18149 if (hw_tls && (m->m_flags & M_EXTPG))
18150 tls = m->m_epg_tls;
18151 else
18152 tls = NULL;
18153 #ifdef INVARIANTS
18154 start = m;
18155 #endif
18156 #endif
18157 while (len > 0) {
18158 if (m == NULL) {
18159 *plen = len_cp;
18160 break;
18161 }
18162 #ifdef KERN_TLS
18163 if (hw_tls) {
18164 if (m->m_flags & M_EXTPG)
18165 ntls = m->m_epg_tls;
18166 else
18167 ntls = NULL;
18168
18169 /*
18170 * Avoid mixing TLS records with handshake
18171 * data or TLS records from different
18172 * sessions.
18173 */
18174 if (tls != ntls) {
18175 MPASS(m != start);
18176 *plen = len_cp;
18177 break;
18178 }
18179 }
18180 #endif
18181 mlen = min(len, m->m_len - off);
18182 if (seglimit) {
18183 /*
18184 * For M_EXTPG mbufs, add 3 segments
18185 * + 1 in case we are crossing page boundaries
18186 * + 2 in case the TLS hdr/trailer are used
18187 * It is cheaper to just add the segments
18188 * than it is to take the cache miss to look
18189 * at the mbuf ext_pgs state in detail.
18190 */
18191 if (m->m_flags & M_EXTPG) {
18192 fragsize = min(segsize, PAGE_SIZE);
18193 frags = 3;
18194 } else {
18195 fragsize = segsize;
18196 frags = 0;
18197 }
18198
18199 /* Break if we really can't fit anymore. */
18200 if ((frags + 1) >= seglimit) {
18201 *plen = len_cp;
18202 break;
18203 }
18204
18205 /*
18206 * Reduce size if you can't copy the whole
18207 * mbuf. If we can't copy the whole mbuf, also
18208 * adjust len so the loop will end after this
18209 * mbuf.
18210 */
18211 if ((frags + howmany(mlen, fragsize)) >= seglimit) {
18212 mlen = (seglimit - frags - 1) * fragsize;
18213 len = mlen;
18214 *plen = len_cp + len;
18215 }
18216 frags += howmany(mlen, fragsize);
18217 if (frags == 0)
18218 frags++;
18219 seglimit -= frags;
18220 KASSERT(seglimit > 0,
18221 ("%s: seglimit went too low", __func__));
18222 }
18223 n = m_get(M_NOWAIT, m->m_type);
18224 *np = n;
18225 if (n == NULL)
18226 goto nospace;
18227 n->m_len = mlen;
18228 soff += mlen;
18229 len_cp += n->m_len;
18230 if (m->m_flags & (M_EXT | M_EXTPG)) {
18231 n->m_data = m->m_data + off;
18232 mb_dupcl(n, m);
18233 } else {
18234 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
18235 (u_int)n->m_len);
18236 }
18237 len -= n->m_len;
18238 off = 0;
18239 m = m->m_next;
18240 np = &n->m_next;
18241 if (len || (soff == smb->m_len)) {
18242 /*
18243 * We have more so we move forward or
18244 * we have consumed the entire mbuf and
18245 * len has fell to 0.
18246 */
18247 soff = 0;
18248 smb = m;
18249 }
18250
18251 }
18252 if (fsb != NULL) {
18253 fsb->m = smb;
18254 fsb->off = soff;
18255 if (smb) {
18256 /*
18257 * Save off the size of the mbuf. We do
18258 * this so that we can recognize when it
18259 * has been trimmed by sbcut() as acks
18260 * come in.
18261 */
18262 fsb->o_m_len = smb->m_len;
18263 fsb->o_t_len = M_TRAILINGROOM(smb);
18264 } else {
18265 /*
18266 * This is the case where the next mbuf went to NULL. This
18267 * means with this copy we have sent everything in the sb.
18268 * In theory we could clear the fast_output flag, but lets
18269 * not since its possible that we could get more added
18270 * and acks that call the extend function which would let
18271 * us send more.
18272 */
18273 fsb->o_m_len = 0;
18274 fsb->o_t_len = 0;
18275 }
18276 }
18277 return (top);
18278 nospace:
18279 if (top)
18280 m_freem(top);
18281 return (NULL);
18282
18283 }
18284
18285 /*
18286 * This is a copy of m_copym(), taking the TSO segment size/limit
18287 * constraints into account, and advancing the sndptr as it goes.
18288 */
18289 static struct mbuf *
rack_fo_m_copym(struct tcp_rack * rack,int32_t * plen,int32_t seglimit,int32_t segsize,struct mbuf ** s_mb,int * s_soff)18290 rack_fo_m_copym(struct tcp_rack *rack, int32_t *plen,
18291 int32_t seglimit, int32_t segsize, struct mbuf **s_mb, int *s_soff)
18292 {
18293 struct mbuf *m, *n;
18294 int32_t soff;
18295
18296 m = rack->r_ctl.fsb.m;
18297 if (M_TRAILINGROOM(m) != rack->r_ctl.fsb.o_t_len) {
18298 /*
18299 * The trailing space changed, mbufs can grow
18300 * at the tail but they can't shrink from
18301 * it, KASSERT that. Adjust the orig_m_len to
18302 * compensate for this change.
18303 */
18304 KASSERT((rack->r_ctl.fsb.o_t_len > M_TRAILINGROOM(m)),
18305 ("mbuf:%p rack:%p trailing_space:%jd ots:%u oml:%u mlen:%u\n",
18306 m,
18307 rack,
18308 (intmax_t)M_TRAILINGROOM(m),
18309 rack->r_ctl.fsb.o_t_len,
18310 rack->r_ctl.fsb.o_m_len,
18311 m->m_len));
18312 rack->r_ctl.fsb.o_m_len += (rack->r_ctl.fsb.o_t_len - M_TRAILINGROOM(m));
18313 rack->r_ctl.fsb.o_t_len = M_TRAILINGROOM(m);
18314 }
18315 if (m->m_len < rack->r_ctl.fsb.o_m_len) {
18316 /*
18317 * Mbuf shrank, trimmed off the top by an ack, our
18318 * offset changes.
18319 */
18320 KASSERT((rack->r_ctl.fsb.off >= (rack->r_ctl.fsb.o_m_len - m->m_len)),
18321 ("mbuf:%p len:%u rack:%p oml:%u soff:%u\n",
18322 m, m->m_len,
18323 rack, rack->r_ctl.fsb.o_m_len,
18324 rack->r_ctl.fsb.off));
18325
18326 if (rack->r_ctl.fsb.off >= (rack->r_ctl.fsb.o_m_len- m->m_len))
18327 rack->r_ctl.fsb.off -= (rack->r_ctl.fsb.o_m_len - m->m_len);
18328 else
18329 rack->r_ctl.fsb.off = 0;
18330 rack->r_ctl.fsb.o_m_len = m->m_len;
18331 #ifdef INVARIANTS
18332 } else if (m->m_len > rack->r_ctl.fsb.o_m_len) {
18333 panic("rack:%p m:%p m_len grew outside of t_space compensation",
18334 rack, m);
18335 #endif
18336 }
18337 soff = rack->r_ctl.fsb.off;
18338 KASSERT(soff >= 0, ("%s, negative off %d", __FUNCTION__, soff));
18339 KASSERT(*plen >= 0, ("%s, negative len %d", __FUNCTION__, *plen));
18340 KASSERT(soff < m->m_len, ("%s rack:%p len:%u m:%p m->m_len:%u < off?",
18341 __FUNCTION__,
18342 rack, *plen, m, m->m_len));
18343 /* Save off the right location before we copy and advance */
18344 *s_soff = soff;
18345 *s_mb = rack->r_ctl.fsb.m;
18346 n = rack_fo_base_copym(m, soff, plen,
18347 &rack->r_ctl.fsb,
18348 seglimit, segsize, rack->r_ctl.fsb.hw_tls);
18349 return (n);
18350 }
18351
18352 /* Log the buffer level */
18353 static void
rack_log_queue_level(struct tcpcb * tp,struct tcp_rack * rack,int len,struct timeval * tv,uint32_t cts)18354 rack_log_queue_level(struct tcpcb *tp, struct tcp_rack *rack,
18355 int len, struct timeval *tv,
18356 uint32_t cts)
18357 {
18358 uint32_t p_rate = 0, p_queue = 0, err = 0;
18359 union tcp_log_stackspecific log;
18360
18361 #ifdef RATELIMIT
18362 err = in_pcbquery_txrlevel(rack->rc_inp, &p_queue);
18363 err = in_pcbquery_txrtlmt(rack->rc_inp, &p_rate);
18364 #endif
18365 memset(&log, 0, sizeof(log));
18366 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
18367 log.u_bbr.flex1 = p_rate;
18368 log.u_bbr.flex2 = p_queue;
18369 log.u_bbr.flex4 = (uint32_t)rack->r_ctl.crte->using;
18370 log.u_bbr.flex5 = (uint32_t)rack->r_ctl.crte->rs_num_enobufs;
18371 log.u_bbr.flex6 = rack->r_ctl.crte->time_between;
18372 log.u_bbr.flex7 = 99;
18373 log.u_bbr.flex8 = 0;
18374 log.u_bbr.pkts_out = err;
18375 log.u_bbr.delRate = rack->r_ctl.crte->rate;
18376 log.u_bbr.timeStamp = cts;
18377 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
18378 tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_HDWR_PACE, 0,
18379 len, &log, false, NULL, __func__, __LINE__, tv);
18380
18381 }
18382
18383 static uint32_t
rack_check_queue_level(struct tcp_rack * rack,struct tcpcb * tp,struct timeval * tv,uint32_t cts,int len,uint32_t segsiz)18384 rack_check_queue_level(struct tcp_rack *rack, struct tcpcb *tp,
18385 struct timeval *tv, uint32_t cts, int len, uint32_t segsiz)
18386 {
18387 uint64_t lentime = 0;
18388 #ifdef RATELIMIT
18389 uint32_t p_rate = 0, p_queue = 0, err;
18390 union tcp_log_stackspecific log;
18391 uint64_t bw;
18392
18393 err = in_pcbquery_txrlevel(rack->rc_inp, &p_queue);
18394 /* Failed or queue is zero */
18395 if (err || (p_queue == 0)) {
18396 lentime = 0;
18397 goto out;
18398 }
18399 err = in_pcbquery_txrtlmt(rack->rc_inp, &p_rate);
18400 if (err) {
18401 lentime = 0;
18402 goto out;
18403 }
18404 /*
18405 * If we reach here we have some bytes in
18406 * the queue. The number returned is a value
18407 * between 0 and 0xffff where ffff is full
18408 * and 0 is empty. So how best to make this into
18409 * something usable?
18410 *
18411 * The "safer" way is lets take the b/w gotten
18412 * from the query (which should be our b/w rate)
18413 * and pretend that a full send (our rc_pace_max_segs)
18414 * is outstanding. We factor it so its as if a full
18415 * number of our MSS segment is terms of full
18416 * ethernet segments are outstanding.
18417 */
18418 bw = p_rate / 8;
18419 if (bw) {
18420 lentime = (rack->r_ctl.rc_pace_max_segs / segsiz);
18421 lentime *= ETHERNET_SEGMENT_SIZE;
18422 lentime *= (uint64_t)HPTS_USEC_IN_SEC;
18423 lentime /= bw;
18424 } else {
18425 /* TSNH -- KASSERT? */
18426 lentime = 0;
18427 }
18428 out:
18429 if (tcp_bblogging_on(tp)) {
18430 memset(&log, 0, sizeof(log));
18431 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
18432 log.u_bbr.flex1 = p_rate;
18433 log.u_bbr.flex2 = p_queue;
18434 log.u_bbr.flex4 = (uint32_t)rack->r_ctl.crte->using;
18435 log.u_bbr.flex5 = (uint32_t)rack->r_ctl.crte->rs_num_enobufs;
18436 log.u_bbr.flex6 = rack->r_ctl.crte->time_between;
18437 log.u_bbr.flex7 = 99;
18438 log.u_bbr.flex8 = 0;
18439 log.u_bbr.pkts_out = err;
18440 log.u_bbr.delRate = rack->r_ctl.crte->rate;
18441 log.u_bbr.cur_del_rate = lentime;
18442 log.u_bbr.timeStamp = cts;
18443 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
18444 tcp_log_event(tp, NULL, NULL, NULL, BBR_LOG_HDWR_PACE, 0,
18445 len, &log, false, NULL, __func__, __LINE__,tv);
18446 }
18447 #endif
18448 return ((uint32_t)lentime);
18449 }
18450
18451 static int
rack_fast_rsm_output(struct tcpcb * tp,struct tcp_rack * rack,struct rack_sendmap * rsm,uint64_t ts_val,uint32_t cts,uint32_t ms_cts,struct timeval * tv,int len,uint8_t doing_tlp)18452 rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendmap *rsm,
18453 uint64_t ts_val, uint32_t cts, uint32_t ms_cts, struct timeval *tv, int len, uint8_t doing_tlp)
18454 {
18455 /*
18456 * Enter the fast retransmit path. We are given that a sched_pin is
18457 * in place (if accounting is compliled in) and the cycle count taken
18458 * at the entry is in the ts_val. The concept her is that the rsm
18459 * now holds the mbuf offsets and such so we can directly transmit
18460 * without a lot of overhead, the len field is already set for
18461 * us to prohibit us from sending too much (usually its 1MSS).
18462 */
18463 struct ip *ip = NULL;
18464 struct udphdr *udp = NULL;
18465 struct tcphdr *th = NULL;
18466 struct mbuf *m = NULL;
18467 struct inpcb *inp;
18468 uint8_t *cpto;
18469 struct tcp_log_buffer *lgb;
18470 #ifdef TCP_ACCOUNTING
18471 uint64_t crtsc;
18472 int cnt_thru = 1;
18473 #endif
18474 struct tcpopt to;
18475 u_char opt[TCP_MAXOLEN];
18476 uint32_t hdrlen, optlen;
18477 int32_t pacing_delay, segsiz, max_val, tso = 0, error = 0, ulen = 0;
18478 uint16_t flags;
18479 uint32_t if_hw_tsomaxsegcount = 0, startseq;
18480 uint32_t if_hw_tsomaxsegsize;
18481 int32_t ip_sendflag = IP_NO_SND_TAG_RL;
18482
18483 #ifdef INET6
18484 struct ip6_hdr *ip6 = NULL;
18485
18486 if (rack->r_is_v6) {
18487 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
18488 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
18489 } else
18490 #endif /* INET6 */
18491 {
18492 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
18493 hdrlen = sizeof(struct tcpiphdr);
18494 }
18495 if (tp->t_port && (V_tcp_udp_tunneling_port == 0)) {
18496 goto failed;
18497 }
18498 if (doing_tlp) {
18499 /* Its a TLP add the flag, it may already be there but be sure */
18500 rsm->r_flags |= RACK_TLP;
18501 } else {
18502 /* If it was a TLP it is not not on this retransmit */
18503 rsm->r_flags &= ~RACK_TLP;
18504 }
18505 startseq = rsm->r_start;
18506 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
18507 inp = rack->rc_inp;
18508 to.to_flags = 0;
18509 flags = tcp_outflags[tp->t_state];
18510 if (flags & (TH_SYN|TH_RST)) {
18511 goto failed;
18512 }
18513 if (rsm->r_flags & RACK_HAS_FIN) {
18514 /* We can't send a FIN here */
18515 goto failed;
18516 }
18517 if (flags & TH_FIN) {
18518 /* We never send a FIN */
18519 flags &= ~TH_FIN;
18520 }
18521 if (tp->t_flags & TF_RCVD_TSTMP) {
18522 to.to_tsval = ms_cts + tp->ts_offset;
18523 to.to_tsecr = tp->ts_recent;
18524 to.to_flags = TOF_TS;
18525 }
18526 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
18527 /* TCP-MD5 (RFC2385). */
18528 if (tp->t_flags & TF_SIGNATURE)
18529 to.to_flags |= TOF_SIGNATURE;
18530 #endif
18531 optlen = tcp_addoptions(&to, opt);
18532 hdrlen += optlen;
18533 udp = rack->r_ctl.fsb.udp;
18534 if (udp)
18535 hdrlen += sizeof(struct udphdr);
18536 if (rack->r_ctl.rc_pace_max_segs)
18537 max_val = rack->r_ctl.rc_pace_max_segs;
18538 else if (rack->rc_user_set_max_segs)
18539 max_val = rack->rc_user_set_max_segs * segsiz;
18540 else
18541 max_val = len;
18542 if ((tp->t_flags & TF_TSO) &&
18543 V_tcp_do_tso &&
18544 (len > segsiz) &&
18545 (tp->t_port == 0))
18546 tso = 1;
18547 #ifdef INET6
18548 if (MHLEN < hdrlen + max_linkhdr)
18549 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
18550 else
18551 #endif
18552 m = m_gethdr(M_NOWAIT, MT_DATA);
18553 if (m == NULL)
18554 goto failed;
18555 m->m_data += max_linkhdr;
18556 m->m_len = hdrlen;
18557 th = rack->r_ctl.fsb.th;
18558 /* Establish the len to send */
18559 if (len > max_val)
18560 len = max_val;
18561 if ((tso) && (len + optlen > segsiz)) {
18562 uint32_t if_hw_tsomax;
18563 int32_t max_len;
18564
18565 /* extract TSO information */
18566 if_hw_tsomax = tp->t_tsomax;
18567 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
18568 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
18569 /*
18570 * Check if we should limit by maximum payload
18571 * length:
18572 */
18573 if (if_hw_tsomax != 0) {
18574 /* compute maximum TSO length */
18575 max_len = (if_hw_tsomax - hdrlen -
18576 max_linkhdr);
18577 if (max_len <= 0) {
18578 goto failed;
18579 } else if (len > max_len) {
18580 len = max_len;
18581 }
18582 }
18583 if (len <= segsiz) {
18584 /*
18585 * In case there are too many small fragments don't
18586 * use TSO:
18587 */
18588 tso = 0;
18589 }
18590 } else {
18591 tso = 0;
18592 }
18593 if ((tso == 0) && (len > segsiz))
18594 len = segsiz;
18595 (void)tcp_get_usecs(tv);
18596 if ((len == 0) ||
18597 (len <= MHLEN - hdrlen - max_linkhdr)) {
18598 goto failed;
18599 }
18600 th->th_seq = htonl(rsm->r_start);
18601 th->th_ack = htonl(tp->rcv_nxt);
18602 /*
18603 * The PUSH bit should only be applied
18604 * if the full retransmission is made. If
18605 * we are sending less than this is the
18606 * left hand edge and should not have
18607 * the PUSH bit.
18608 */
18609 if ((rsm->r_flags & RACK_HAD_PUSH) &&
18610 (len == (rsm->r_end - rsm->r_start)))
18611 flags |= TH_PUSH;
18612 th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale));
18613 if (th->th_win == 0) {
18614 tp->t_sndzerowin++;
18615 tp->t_flags |= TF_RXWIN0SENT;
18616 } else
18617 tp->t_flags &= ~TF_RXWIN0SENT;
18618 if (rsm->r_flags & RACK_TLP) {
18619 /*
18620 * TLP should not count in retran count, but
18621 * in its own bin
18622 */
18623 counter_u64_add(rack_tlp_retran, 1);
18624 counter_u64_add(rack_tlp_retran_bytes, len);
18625 } else {
18626 tp->t_sndrexmitpack++;
18627 KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
18628 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
18629 }
18630 #ifdef STATS
18631 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
18632 len);
18633 #endif
18634 if (rsm->m == NULL)
18635 goto failed;
18636 if (rsm->m &&
18637 ((rsm->orig_m_len != rsm->m->m_len) ||
18638 (M_TRAILINGROOM(rsm->m) != rsm->orig_t_space))) {
18639 /* Fix up the orig_m_len and possibly the mbuf offset */
18640 rack_adjust_orig_mlen(rsm);
18641 }
18642 m->m_next = rack_fo_base_copym(rsm->m, rsm->soff, &len, NULL, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, rsm->r_hw_tls);
18643 if (len <= segsiz) {
18644 /*
18645 * Must have ran out of mbufs for the copy
18646 * shorten it to no longer need tso. Lets
18647 * not put on sendalot since we are low on
18648 * mbufs.
18649 */
18650 tso = 0;
18651 }
18652 if ((m->m_next == NULL) || (len <= 0)){
18653 goto failed;
18654 }
18655 if (udp) {
18656 if (rack->r_is_v6)
18657 ulen = hdrlen + len - sizeof(struct ip6_hdr);
18658 else
18659 ulen = hdrlen + len - sizeof(struct ip);
18660 udp->uh_ulen = htons(ulen);
18661 }
18662 m->m_pkthdr.rcvif = (struct ifnet *)0;
18663 if (TCPS_HAVERCVDSYN(tp->t_state) &&
18664 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) {
18665 int ect = tcp_ecn_output_established(tp, &flags, len, true);
18666 if ((tp->t_state == TCPS_SYN_RECEIVED) &&
18667 (tp->t_flags2 & TF2_ECN_SND_ECE))
18668 tp->t_flags2 &= ~TF2_ECN_SND_ECE;
18669 #ifdef INET6
18670 if (rack->r_is_v6) {
18671 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20);
18672 ip6->ip6_flow |= htonl(ect << 20);
18673 }
18674 else
18675 #endif
18676 {
18677 ip->ip_tos &= ~IPTOS_ECN_MASK;
18678 ip->ip_tos |= ect;
18679 }
18680 }
18681 if (rack->r_ctl.crte != NULL) {
18682 /* See if we can send via the hw queue */
18683 pacing_delay = rack_check_queue_level(rack, tp, tv, cts, len, segsiz);
18684 /* If there is nothing in queue (no pacing time) we can send via the hw queue */
18685 if (pacing_delay == 0)
18686 ip_sendflag = 0;
18687 }
18688 tcp_set_flags(th, flags);
18689 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
18690 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
18691 if (to.to_flags & TOF_SIGNATURE) {
18692 /*
18693 * Calculate MD5 signature and put it into the place
18694 * determined before.
18695 * NOTE: since TCP options buffer doesn't point into
18696 * mbuf's data, calculate offset and use it.
18697 */
18698 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
18699 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
18700 /*
18701 * Do not send segment if the calculation of MD5
18702 * digest has failed.
18703 */
18704 goto failed;
18705 }
18706 }
18707 #endif
18708 #ifdef INET6
18709 if (rack->r_is_v6) {
18710 if (tp->t_port) {
18711 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
18712 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
18713 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
18714 th->th_sum = htons(0);
18715 UDPSTAT_INC(udps_opackets);
18716 } else {
18717 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
18718 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
18719 th->th_sum = in6_cksum_pseudo(ip6,
18720 sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
18721 0);
18722 }
18723 }
18724 #endif
18725 #if defined(INET6) && defined(INET)
18726 else
18727 #endif
18728 #ifdef INET
18729 {
18730 if (tp->t_port) {
18731 m->m_pkthdr.csum_flags = CSUM_UDP;
18732 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
18733 udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
18734 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
18735 th->th_sum = htons(0);
18736 UDPSTAT_INC(udps_opackets);
18737 } else {
18738 m->m_pkthdr.csum_flags = CSUM_TCP;
18739 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
18740 th->th_sum = in_pseudo(ip->ip_src.s_addr,
18741 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
18742 IPPROTO_TCP + len + optlen));
18743 }
18744 /* IP version must be set here for ipv4/ipv6 checking later */
18745 KASSERT(ip->ip_v == IPVERSION,
18746 ("%s: IP version incorrect: %d", __func__, ip->ip_v));
18747 }
18748 #endif
18749 if (tso) {
18750 /*
18751 * Here we use segsiz since we have no added options besides
18752 * any standard timestamp options (no DSACKs or SACKS are sent
18753 * via either fast-path).
18754 */
18755 KASSERT(len > segsiz,
18756 ("%s: len <= tso_segsz tp:%p", __func__, tp));
18757 m->m_pkthdr.csum_flags |= CSUM_TSO;
18758 m->m_pkthdr.tso_segsz = segsiz;
18759 }
18760 #ifdef INET6
18761 if (rack->r_is_v6) {
18762 ip6->ip6_hlim = rack->r_ctl.fsb.hoplimit;
18763 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
18764 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
18765 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
18766 else
18767 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
18768 }
18769 #endif
18770 #if defined(INET) && defined(INET6)
18771 else
18772 #endif
18773 #ifdef INET
18774 {
18775 ip->ip_len = htons(m->m_pkthdr.len);
18776 ip->ip_ttl = rack->r_ctl.fsb.hoplimit;
18777 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
18778 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
18779 if (tp->t_port == 0 || len < V_tcp_minmss) {
18780 ip->ip_off |= htons(IP_DF);
18781 }
18782 } else {
18783 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
18784 }
18785 }
18786 #endif
18787 if (doing_tlp == 0) {
18788 /* Set we retransmitted */
18789 rack->rc_gp_saw_rec = 1;
18790 } else {
18791 /* Its a TLP set ca or ss */
18792 if (tp->snd_cwnd > tp->snd_ssthresh) {
18793 /* Set we sent in CA */
18794 rack->rc_gp_saw_ca = 1;
18795 } else {
18796 /* Set we sent in SS */
18797 rack->rc_gp_saw_ss = 1;
18798 }
18799 }
18800 /* Time to copy in our header */
18801 cpto = mtod(m, uint8_t *);
18802 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len);
18803 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr));
18804 if (optlen) {
18805 bcopy(opt, th + 1, optlen);
18806 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
18807 } else {
18808 th->th_off = sizeof(struct tcphdr) >> 2;
18809 }
18810 if (tcp_bblogging_on(rack->rc_tp)) {
18811 union tcp_log_stackspecific log;
18812
18813 if (rsm->r_flags & RACK_RWND_COLLAPSED) {
18814 rack_log_collapse(rack, rsm->r_start, rsm->r_end, 0, __LINE__, 5, rsm->r_flags, rsm);
18815 counter_u64_add(rack_collapsed_win_rxt, 1);
18816 counter_u64_add(rack_collapsed_win_rxt_bytes, (rsm->r_end - rsm->r_start));
18817 }
18818 memset(&log, 0, sizeof(log));
18819 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
18820 if (rack->rack_no_prr)
18821 log.u_bbr.flex1 = 0;
18822 else
18823 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
18824 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs;
18825 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
18826 log.u_bbr.flex4 = max_val;
18827 /* Save off the early/late values */
18828 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
18829 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed;
18830 log.u_bbr.bw_inuse = rack_get_bw(rack);
18831 log.u_bbr.cur_del_rate = rack->r_ctl.gp_bw;
18832 if (doing_tlp == 0)
18833 log.u_bbr.flex8 = 1;
18834 else
18835 log.u_bbr.flex8 = 2;
18836 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL);
18837 log.u_bbr.flex7 = 55;
18838 log.u_bbr.pkts_out = tp->t_maxseg;
18839 log.u_bbr.timeStamp = cts;
18840 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
18841 if (rsm->r_rtr_cnt > 0) {
18842 /*
18843 * When we have a retransmit we want to log the
18844 * burst at send and flight at send from before.
18845 */
18846 log.u_bbr.flex5 = rsm->r_fas;
18847 log.u_bbr.bbr_substate = rsm->r_bas;
18848 } else {
18849 /*
18850 * This is currently unlikely until we do the
18851 * packet pair probes but I will add it for completeness.
18852 */
18853 log.u_bbr.flex5 = log.u_bbr.inflight;
18854 log.u_bbr.bbr_substate = (uint8_t)((len + segsiz - 1)/segsiz);
18855 }
18856 log.u_bbr.lt_epoch = rack->r_ctl.cwnd_to_use;
18857 log.u_bbr.delivered = 0;
18858 log.u_bbr.rttProp = (uintptr_t)rsm;
18859 log.u_bbr.delRate = rsm->r_flags;
18860 log.u_bbr.delRate <<= 31;
18861 log.u_bbr.delRate |= rack->r_must_retran;
18862 log.u_bbr.delRate <<= 1;
18863 log.u_bbr.delRate |= 1;
18864 log.u_bbr.pkt_epoch = __LINE__;
18865 lgb = tcp_log_event(tp, th, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK,
18866 len, &log, false, NULL, __func__, __LINE__, tv);
18867 } else
18868 lgb = NULL;
18869 if ((rack->r_ctl.crte != NULL) &&
18870 tcp_bblogging_on(tp)) {
18871 rack_log_queue_level(tp, rack, len, tv, cts);
18872 }
18873 #ifdef INET6
18874 if (rack->r_is_v6) {
18875 error = ip6_output(m, inp->in6p_outputopts,
18876 &inp->inp_route6,
18877 ip_sendflag, NULL, NULL, inp);
18878 }
18879 else
18880 #endif
18881 #ifdef INET
18882 {
18883 error = ip_output(m, NULL,
18884 &inp->inp_route,
18885 ip_sendflag, 0, inp);
18886 }
18887 #endif
18888 m = NULL;
18889 if (lgb) {
18890 lgb->tlb_errno = error;
18891 lgb = NULL;
18892 }
18893 /* Move snd_nxt to snd_max so we don't have false retransmissions */
18894 tp->snd_nxt = tp->snd_max;
18895 if (error) {
18896 goto failed;
18897 } else if (rack->rc_hw_nobuf && (ip_sendflag != IP_NO_SND_TAG_RL)) {
18898 rack->rc_hw_nobuf = 0;
18899 rack->r_ctl.rc_agg_delayed = 0;
18900 rack->r_early = 0;
18901 rack->r_late = 0;
18902 rack->r_ctl.rc_agg_early = 0;
18903 }
18904 rack_log_output(tp, &to, len, rsm->r_start, flags, error, rack_to_usec_ts(tv),
18905 rsm, RACK_SENT_FP, rsm->m, rsm->soff, rsm->r_hw_tls, segsiz);
18906 if (doing_tlp) {
18907 rack->rc_tlp_in_progress = 1;
18908 rack->r_ctl.rc_tlp_cnt_out++;
18909 }
18910 if (error == 0) {
18911 counter_u64_add(rack_total_bytes, len);
18912 tcp_account_for_send(tp, len, 1, doing_tlp, rsm->r_hw_tls);
18913 if (doing_tlp) {
18914 rack->rc_last_sent_tlp_past_cumack = 0;
18915 rack->rc_last_sent_tlp_seq_valid = 1;
18916 rack->r_ctl.last_sent_tlp_seq = rsm->r_start;
18917 rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start;
18918 }
18919 if (rack->r_ctl.rc_prr_sndcnt >= len)
18920 rack->r_ctl.rc_prr_sndcnt -= len;
18921 else
18922 rack->r_ctl.rc_prr_sndcnt = 0;
18923 }
18924 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
18925 rack->forced_ack = 0; /* If we send something zap the FA flag */
18926 if (IN_FASTRECOVERY(tp->t_flags) && rsm)
18927 rack->r_ctl.retran_during_recovery += len;
18928 {
18929 int idx;
18930
18931 idx = (len / segsiz) + 3;
18932 if (idx >= TCP_MSS_ACCT_ATIMER)
18933 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1);
18934 else
18935 counter_u64_add(rack_out_size[idx], 1);
18936 }
18937 if (tp->t_rtttime == 0) {
18938 tp->t_rtttime = ticks;
18939 tp->t_rtseq = startseq;
18940 KMOD_TCPSTAT_INC(tcps_segstimed);
18941 }
18942 counter_u64_add(rack_fto_rsm_send, 1);
18943 if (error && (error == ENOBUFS)) {
18944 if (rack->r_ctl.crte != NULL) {
18945 tcp_trace_point(rack->rc_tp, TCP_TP_HWENOBUF);
18946 if (tcp_bblogging_on(rack->rc_tp))
18947 rack_log_queue_level(tp, rack, len, tv, cts);
18948 } else
18949 tcp_trace_point(rack->rc_tp, TCP_TP_ENOBUF);
18950 pacing_delay = ((1 + rack->rc_enobuf) * HPTS_USEC_IN_MSEC);
18951 if (rack->rc_enobuf < 0x7f)
18952 rack->rc_enobuf++;
18953 if (pacing_delay < (10 * HPTS_USEC_IN_MSEC))
18954 pacing_delay = 10 * HPTS_USEC_IN_MSEC;
18955 if (rack->r_ctl.crte != NULL) {
18956 counter_u64_add(rack_saw_enobuf_hw, 1);
18957 tcp_rl_log_enobuf(rack->r_ctl.crte);
18958 }
18959 counter_u64_add(rack_saw_enobuf, 1);
18960 } else {
18961 pacing_delay = rack_get_pacing_delay(rack, tp, len, NULL, segsiz, __LINE__);
18962 }
18963 rack_start_hpts_timer(rack, tp, cts, pacing_delay, len, 0);
18964 #ifdef TCP_ACCOUNTING
18965 crtsc = get_cyclecount();
18966 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18967 tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru;
18968 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
18969 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((len + segsiz - 1) / segsiz);
18970 }
18971 sched_unpin();
18972 #endif
18973 return (0);
18974 failed:
18975 if (m)
18976 m_free(m);
18977 return (-1);
18978 }
18979
18980 static void
rack_sndbuf_autoscale(struct tcp_rack * rack)18981 rack_sndbuf_autoscale(struct tcp_rack *rack)
18982 {
18983 /*
18984 * Automatic sizing of send socket buffer. Often the send buffer
18985 * size is not optimally adjusted to the actual network conditions
18986 * at hand (delay bandwidth product). Setting the buffer size too
18987 * small limits throughput on links with high bandwidth and high
18988 * delay (eg. trans-continental/oceanic links). Setting the
18989 * buffer size too big consumes too much real kernel memory,
18990 * especially with many connections on busy servers.
18991 *
18992 * The criteria to step up the send buffer one notch are:
18993 * 1. receive window of remote host is larger than send buffer
18994 * (with a fudge factor of 5/4th);
18995 * 2. send buffer is filled to 7/8th with data (so we actually
18996 * have data to make use of it);
18997 * 3. send buffer fill has not hit maximal automatic size;
18998 * 4. our send window (slow start and cogestion controlled) is
18999 * larger than sent but unacknowledged data in send buffer.
19000 *
19001 * Note that the rack version moves things much faster since
19002 * we want to avoid hitting cache lines in the rack_fast_output()
19003 * path so this is called much less often and thus moves
19004 * the SB forward by a percentage.
19005 */
19006 struct socket *so;
19007 struct tcpcb *tp;
19008 uint32_t sendwin, scaleup;
19009
19010 tp = rack->rc_tp;
19011 so = rack->rc_inp->inp_socket;
19012 sendwin = min(rack->r_ctl.cwnd_to_use, tp->snd_wnd);
19013 if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) {
19014 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
19015 sbused(&so->so_snd) >=
19016 (so->so_snd.sb_hiwat / 8 * 7) &&
19017 sbused(&so->so_snd) < V_tcp_autosndbuf_max &&
19018 sendwin >= (sbused(&so->so_snd) -
19019 (tp->snd_max - tp->snd_una))) {
19020 if (rack_autosndbuf_inc)
19021 scaleup = (rack_autosndbuf_inc * so->so_snd.sb_hiwat) / 100;
19022 else
19023 scaleup = V_tcp_autosndbuf_inc;
19024 if (scaleup < V_tcp_autosndbuf_inc)
19025 scaleup = V_tcp_autosndbuf_inc;
19026 scaleup += so->so_snd.sb_hiwat;
19027 if (scaleup > V_tcp_autosndbuf_max)
19028 scaleup = V_tcp_autosndbuf_max;
19029 if (!sbreserve_locked(so, SO_SND, scaleup, curthread))
19030 so->so_snd.sb_flags &= ~SB_AUTOSIZE;
19031 }
19032 }
19033 }
19034
19035 static int
rack_fast_output(struct tcpcb * tp,struct tcp_rack * rack,uint64_t ts_val,uint32_t cts,uint32_t ms_cts,struct timeval * tv,long * tot_len,int * send_err,int line)19036 rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val,
19037 uint32_t cts, uint32_t ms_cts, struct timeval *tv, long *tot_len, int *send_err, int line)
19038 {
19039 /*
19040 * Enter to do fast output. We are given that the sched_pin is
19041 * in place (if accounting is compiled in) and the cycle count taken
19042 * at entry is in place in ts_val. The idea here is that
19043 * we know how many more bytes needs to be sent (presumably either
19044 * during pacing or to fill the cwnd and that was greater than
19045 * the max-burst). We have how much to send and all the info we
19046 * need to just send.
19047 */
19048 #ifdef INET
19049 struct ip *ip = NULL;
19050 #endif
19051 struct udphdr *udp = NULL;
19052 struct tcphdr *th = NULL;
19053 struct mbuf *m, *s_mb;
19054 struct inpcb *inp;
19055 uint8_t *cpto;
19056 struct tcp_log_buffer *lgb;
19057 #ifdef TCP_ACCOUNTING
19058 uint64_t crtsc;
19059 #endif
19060 struct tcpopt to;
19061 u_char opt[TCP_MAXOLEN];
19062 uint32_t hdrlen, optlen;
19063 #ifdef TCP_ACCOUNTING
19064 int cnt_thru = 1;
19065 #endif
19066 int32_t pacing_delay, segsiz, len, max_val, tso = 0, sb_offset, error, ulen = 0;
19067 uint16_t flags;
19068 uint32_t s_soff;
19069 uint32_t if_hw_tsomaxsegcount = 0, startseq;
19070 uint32_t if_hw_tsomaxsegsize;
19071 uint32_t add_flag = RACK_SENT_FP;
19072 #ifdef INET6
19073 struct ip6_hdr *ip6 = NULL;
19074
19075 if (rack->r_is_v6) {
19076 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
19077 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
19078 } else
19079 #endif /* INET6 */
19080 {
19081 #ifdef INET
19082 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
19083 hdrlen = sizeof(struct tcpiphdr);
19084 #endif
19085 }
19086 if (tp->t_port && (V_tcp_udp_tunneling_port == 0)) {
19087 m = NULL;
19088 goto failed;
19089 }
19090 rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
19091 startseq = tp->snd_max;
19092 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
19093 inp = rack->rc_inp;
19094 len = rack->r_ctl.fsb.left_to_send;
19095 to.to_flags = 0;
19096 flags = rack->r_ctl.fsb.tcp_flags;
19097 if (tp->t_flags & TF_RCVD_TSTMP) {
19098 to.to_tsval = ms_cts + tp->ts_offset;
19099 to.to_tsecr = tp->ts_recent;
19100 to.to_flags = TOF_TS;
19101 }
19102 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
19103 /* TCP-MD5 (RFC2385). */
19104 if (tp->t_flags & TF_SIGNATURE)
19105 to.to_flags |= TOF_SIGNATURE;
19106 #endif
19107 optlen = tcp_addoptions(&to, opt);
19108 hdrlen += optlen;
19109 udp = rack->r_ctl.fsb.udp;
19110 if (udp)
19111 hdrlen += sizeof(struct udphdr);
19112 if (rack->r_ctl.rc_pace_max_segs)
19113 max_val = rack->r_ctl.rc_pace_max_segs;
19114 else if (rack->rc_user_set_max_segs)
19115 max_val = rack->rc_user_set_max_segs * segsiz;
19116 else
19117 max_val = len;
19118 if ((tp->t_flags & TF_TSO) &&
19119 V_tcp_do_tso &&
19120 (len > segsiz) &&
19121 (tp->t_port == 0))
19122 tso = 1;
19123 again:
19124 #ifdef INET6
19125 if (MHLEN < hdrlen + max_linkhdr)
19126 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
19127 else
19128 #endif
19129 m = m_gethdr(M_NOWAIT, MT_DATA);
19130 if (m == NULL)
19131 goto failed;
19132 m->m_data += max_linkhdr;
19133 m->m_len = hdrlen;
19134 th = rack->r_ctl.fsb.th;
19135 /* Establish the len to send */
19136 if (len > max_val)
19137 len = max_val;
19138 if ((tso) && (len + optlen > segsiz)) {
19139 uint32_t if_hw_tsomax;
19140 int32_t max_len;
19141
19142 /* extract TSO information */
19143 if_hw_tsomax = tp->t_tsomax;
19144 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
19145 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
19146 /*
19147 * Check if we should limit by maximum payload
19148 * length:
19149 */
19150 if (if_hw_tsomax != 0) {
19151 /* compute maximum TSO length */
19152 max_len = (if_hw_tsomax - hdrlen -
19153 max_linkhdr);
19154 if (max_len <= 0) {
19155 goto failed;
19156 } else if (len > max_len) {
19157 len = max_len;
19158 }
19159 }
19160 if (len <= segsiz) {
19161 /*
19162 * In case there are too many small fragments don't
19163 * use TSO:
19164 */
19165 tso = 0;
19166 }
19167 } else {
19168 tso = 0;
19169 }
19170 if ((tso == 0) && (len > segsiz))
19171 len = segsiz;
19172 (void)tcp_get_usecs(tv);
19173 if ((len == 0) ||
19174 (len <= MHLEN - hdrlen - max_linkhdr)) {
19175 goto failed;
19176 }
19177 sb_offset = tp->snd_max - tp->snd_una;
19178 th->th_seq = htonl(tp->snd_max);
19179 th->th_ack = htonl(tp->rcv_nxt);
19180 th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale));
19181 if (th->th_win == 0) {
19182 tp->t_sndzerowin++;
19183 tp->t_flags |= TF_RXWIN0SENT;
19184 } else
19185 tp->t_flags &= ~TF_RXWIN0SENT;
19186 tp->snd_up = tp->snd_una; /* drag it along, its deprecated */
19187 KMOD_TCPSTAT_INC(tcps_sndpack);
19188 KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
19189 #ifdef STATS
19190 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
19191 len);
19192 #endif
19193 if (rack->r_ctl.fsb.m == NULL)
19194 goto failed;
19195
19196 /* s_mb and s_soff are saved for rack_log_output */
19197 m->m_next = rack_fo_m_copym(rack, &len, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize,
19198 &s_mb, &s_soff);
19199 if (len <= segsiz) {
19200 /*
19201 * Must have ran out of mbufs for the copy
19202 * shorten it to no longer need tso. Lets
19203 * not put on sendalot since we are low on
19204 * mbufs.
19205 */
19206 tso = 0;
19207 }
19208 if (rack->r_ctl.fsb.rfo_apply_push &&
19209 (len == rack->r_ctl.fsb.left_to_send)) {
19210 flags |= TH_PUSH;
19211 add_flag |= RACK_HAD_PUSH;
19212 }
19213 if ((m->m_next == NULL) || (len <= 0)){
19214 goto failed;
19215 }
19216 if (udp) {
19217 if (rack->r_is_v6)
19218 ulen = hdrlen + len - sizeof(struct ip6_hdr);
19219 else
19220 ulen = hdrlen + len - sizeof(struct ip);
19221 udp->uh_ulen = htons(ulen);
19222 }
19223 m->m_pkthdr.rcvif = (struct ifnet *)0;
19224 if (TCPS_HAVERCVDSYN(tp->t_state) &&
19225 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) {
19226 int ect = tcp_ecn_output_established(tp, &flags, len, false);
19227 if ((tp->t_state == TCPS_SYN_RECEIVED) &&
19228 (tp->t_flags2 & TF2_ECN_SND_ECE))
19229 tp->t_flags2 &= ~TF2_ECN_SND_ECE;
19230 #ifdef INET6
19231 if (rack->r_is_v6) {
19232 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20);
19233 ip6->ip6_flow |= htonl(ect << 20);
19234 }
19235 else
19236 #endif
19237 {
19238 #ifdef INET
19239 ip->ip_tos &= ~IPTOS_ECN_MASK;
19240 ip->ip_tos |= ect;
19241 #endif
19242 }
19243 }
19244 tcp_set_flags(th, flags);
19245 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
19246 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
19247 if (to.to_flags & TOF_SIGNATURE) {
19248 /*
19249 * Calculate MD5 signature and put it into the place
19250 * determined before.
19251 * NOTE: since TCP options buffer doesn't point into
19252 * mbuf's data, calculate offset and use it.
19253 */
19254 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
19255 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
19256 /*
19257 * Do not send segment if the calculation of MD5
19258 * digest has failed.
19259 */
19260 goto failed;
19261 }
19262 }
19263 #endif
19264 #ifdef INET6
19265 if (rack->r_is_v6) {
19266 if (tp->t_port) {
19267 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
19268 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
19269 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
19270 th->th_sum = htons(0);
19271 UDPSTAT_INC(udps_opackets);
19272 } else {
19273 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
19274 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
19275 th->th_sum = in6_cksum_pseudo(ip6,
19276 sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
19277 0);
19278 }
19279 }
19280 #endif
19281 #if defined(INET6) && defined(INET)
19282 else
19283 #endif
19284 #ifdef INET
19285 {
19286 if (tp->t_port) {
19287 m->m_pkthdr.csum_flags = CSUM_UDP;
19288 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
19289 udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
19290 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
19291 th->th_sum = htons(0);
19292 UDPSTAT_INC(udps_opackets);
19293 } else {
19294 m->m_pkthdr.csum_flags = CSUM_TCP;
19295 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
19296 th->th_sum = in_pseudo(ip->ip_src.s_addr,
19297 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
19298 IPPROTO_TCP + len + optlen));
19299 }
19300 /* IP version must be set here for ipv4/ipv6 checking later */
19301 KASSERT(ip->ip_v == IPVERSION,
19302 ("%s: IP version incorrect: %d", __func__, ip->ip_v));
19303 }
19304 #endif
19305 if (tso) {
19306 /*
19307 * Here we use segsiz since we have no added options besides
19308 * any standard timestamp options (no DSACKs or SACKS are sent
19309 * via either fast-path).
19310 */
19311 KASSERT(len > segsiz,
19312 ("%s: len <= tso_segsz tp:%p", __func__, tp));
19313 m->m_pkthdr.csum_flags |= CSUM_TSO;
19314 m->m_pkthdr.tso_segsz = segsiz;
19315 }
19316 #ifdef INET6
19317 if (rack->r_is_v6) {
19318 ip6->ip6_hlim = rack->r_ctl.fsb.hoplimit;
19319 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
19320 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
19321 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
19322 else
19323 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
19324 }
19325 #endif
19326 #if defined(INET) && defined(INET6)
19327 else
19328 #endif
19329 #ifdef INET
19330 {
19331 ip->ip_len = htons(m->m_pkthdr.len);
19332 ip->ip_ttl = rack->r_ctl.fsb.hoplimit;
19333 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
19334 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
19335 if (tp->t_port == 0 || len < V_tcp_minmss) {
19336 ip->ip_off |= htons(IP_DF);
19337 }
19338 } else {
19339 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
19340 }
19341 }
19342 #endif
19343 if (tp->snd_cwnd > tp->snd_ssthresh) {
19344 /* Set we sent in CA */
19345 rack->rc_gp_saw_ca = 1;
19346 } else {
19347 /* Set we sent in SS */
19348 rack->rc_gp_saw_ss = 1;
19349 }
19350 /* Time to copy in our header */
19351 cpto = mtod(m, uint8_t *);
19352 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len);
19353 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr));
19354 if (optlen) {
19355 bcopy(opt, th + 1, optlen);
19356 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
19357 } else {
19358 th->th_off = sizeof(struct tcphdr) >> 2;
19359 }
19360 if ((rack->r_ctl.crte != NULL) &&
19361 tcp_bblogging_on(tp)) {
19362 rack_log_queue_level(tp, rack, len, tv, cts);
19363 }
19364 if (tcp_bblogging_on(rack->rc_tp)) {
19365 union tcp_log_stackspecific log;
19366
19367 memset(&log, 0, sizeof(log));
19368 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
19369 if (rack->rack_no_prr)
19370 log.u_bbr.flex1 = 0;
19371 else
19372 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
19373 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs;
19374 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
19375 log.u_bbr.flex4 = max_val;
19376 /* Save off the early/late values */
19377 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
19378 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed;
19379 log.u_bbr.bw_inuse = rack_get_bw(rack);
19380 log.u_bbr.cur_del_rate = rack->r_ctl.gp_bw;
19381 log.u_bbr.flex8 = 0;
19382 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL);
19383 log.u_bbr.flex7 = 44;
19384 log.u_bbr.pkts_out = tp->t_maxseg;
19385 log.u_bbr.timeStamp = cts;
19386 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
19387 log.u_bbr.flex5 = log.u_bbr.inflight;
19388 log.u_bbr.lt_epoch = rack->r_ctl.cwnd_to_use;
19389 log.u_bbr.delivered = rack->r_ctl.fsb.left_to_send;
19390 log.u_bbr.rttProp = 0;
19391 log.u_bbr.delRate = rack->r_must_retran;
19392 log.u_bbr.delRate <<= 1;
19393 log.u_bbr.pkt_epoch = line;
19394 /* For fast output no retrans so just inflight and how many mss we send */
19395 log.u_bbr.flex5 = log.u_bbr.inflight;
19396 log.u_bbr.bbr_substate = (uint8_t)((len + segsiz - 1)/segsiz);
19397 lgb = tcp_log_event(tp, th, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK,
19398 len, &log, false, NULL, __func__, __LINE__, tv);
19399 } else
19400 lgb = NULL;
19401 #ifdef INET6
19402 if (rack->r_is_v6) {
19403 error = ip6_output(m, inp->in6p_outputopts,
19404 &inp->inp_route6,
19405 0, NULL, NULL, inp);
19406 }
19407 #endif
19408 #if defined(INET) && defined(INET6)
19409 else
19410 #endif
19411 #ifdef INET
19412 {
19413 error = ip_output(m, NULL,
19414 &inp->inp_route,
19415 0, 0, inp);
19416 }
19417 #endif
19418 if (lgb) {
19419 lgb->tlb_errno = error;
19420 lgb = NULL;
19421 }
19422 if (error) {
19423 *send_err = error;
19424 m = NULL;
19425 goto failed;
19426 } else if (rack->rc_hw_nobuf) {
19427 rack->rc_hw_nobuf = 0;
19428 rack->r_ctl.rc_agg_delayed = 0;
19429 rack->r_early = 0;
19430 rack->r_late = 0;
19431 rack->r_ctl.rc_agg_early = 0;
19432 }
19433 if ((error == 0) && (rack->lt_bw_up == 0)) {
19434 /* Unlikely */
19435 rack->r_ctl.lt_timemark = tcp_tv_to_lusec(tv);
19436 rack->r_ctl.lt_seq = tp->snd_una;
19437 rack->lt_bw_up = 1;
19438 } else if ((error == 0) &&
19439 (((tp->snd_max + len) - rack->r_ctl.lt_seq) > 0x7fffffff)) {
19440 /*
19441 * Need to record what we have since we are
19442 * approaching seq wrap.
19443 */
19444 struct timeval tv;
19445 uint64_t tmark;
19446
19447 rack->r_ctl.lt_bw_bytes += (tp->snd_una - rack->r_ctl.lt_seq);
19448 rack->r_ctl.lt_seq = tp->snd_una;
19449 tmark = tcp_get_u64_usecs(&tv);
19450 if (tmark > rack->r_ctl.lt_timemark) {
19451 rack->r_ctl.lt_bw_time += (tmark - rack->r_ctl.lt_timemark);
19452 rack->r_ctl.lt_timemark = tmark;
19453 }
19454 }
19455 rack_log_output(tp, &to, len, tp->snd_max, flags, error, rack_to_usec_ts(tv),
19456 NULL, add_flag, s_mb, s_soff, rack->r_ctl.fsb.hw_tls, segsiz);
19457 if (tp->snd_una == tp->snd_max) {
19458 rack->r_ctl.rc_tlp_rxt_last_time = cts;
19459 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__);
19460 tp->t_acktime = ticks;
19461 }
19462 counter_u64_add(rack_total_bytes, len);
19463 tcp_account_for_send(tp, len, 0, 0, rack->r_ctl.fsb.hw_tls);
19464
19465 rack->forced_ack = 0; /* If we send something zap the FA flag */
19466 *tot_len += len;
19467 if ((tp->t_flags & TF_GPUTINPROG) == 0)
19468 rack_start_gp_measurement(tp, rack, tp->snd_max, sb_offset);
19469 tp->snd_max += len;
19470 tp->snd_nxt = tp->snd_max;
19471 if (rack->rc_new_rnd_needed) {
19472 rack_new_round_starts(tp, rack, tp->snd_max);
19473 }
19474 {
19475 int idx;
19476
19477 idx = (len / segsiz) + 3;
19478 if (idx >= TCP_MSS_ACCT_ATIMER)
19479 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1);
19480 else
19481 counter_u64_add(rack_out_size[idx], 1);
19482 }
19483 if (len <= rack->r_ctl.fsb.left_to_send)
19484 rack->r_ctl.fsb.left_to_send -= len;
19485 else
19486 rack->r_ctl.fsb.left_to_send = 0;
19487 if (rack->r_ctl.fsb.left_to_send < segsiz) {
19488 rack->r_fast_output = 0;
19489 rack->r_ctl.fsb.left_to_send = 0;
19490 /* At the end of fast_output scale up the sb */
19491 SOCK_SENDBUF_LOCK(rack->rc_inp->inp_socket);
19492 rack_sndbuf_autoscale(rack);
19493 SOCK_SENDBUF_UNLOCK(rack->rc_inp->inp_socket);
19494 }
19495 if (tp->t_rtttime == 0) {
19496 tp->t_rtttime = ticks;
19497 tp->t_rtseq = startseq;
19498 KMOD_TCPSTAT_INC(tcps_segstimed);
19499 }
19500 if ((rack->r_ctl.fsb.left_to_send >= segsiz) &&
19501 (max_val > len) &&
19502 (*tot_len < rack->r_ctl.rc_pace_max_segs) &&
19503 (tso == 0)) {
19504 max_val -= len;
19505 len = segsiz;
19506 th = rack->r_ctl.fsb.th;
19507 #ifdef TCP_ACCOUNTING
19508 cnt_thru++;
19509 #endif
19510 goto again;
19511 }
19512 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
19513 counter_u64_add(rack_fto_send, 1);
19514 pacing_delay = rack_get_pacing_delay(rack, tp, *tot_len, NULL, segsiz, __LINE__);
19515 rack_start_hpts_timer(rack, tp, cts, pacing_delay, *tot_len, 0);
19516 #ifdef TCP_ACCOUNTING
19517 crtsc = get_cyclecount();
19518 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19519 tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru;
19520 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
19521 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((*tot_len + segsiz - 1) / segsiz);
19522 }
19523 sched_unpin();
19524 #endif
19525 return (0);
19526 failed:
19527 if (m)
19528 m_free(m);
19529 rack->r_fast_output = 0;
19530 return (-1);
19531 }
19532
19533 static inline void
rack_setup_fast_output(struct tcpcb * tp,struct tcp_rack * rack,struct sockbuf * sb,int len,int orig_len,int segsiz,uint32_t pace_max_seg,bool hw_tls,uint16_t flags)19534 rack_setup_fast_output(struct tcpcb *tp, struct tcp_rack *rack,
19535 struct sockbuf *sb,
19536 int len, int orig_len, int segsiz, uint32_t pace_max_seg,
19537 bool hw_tls,
19538 uint16_t flags)
19539 {
19540 rack->r_fast_output = 1;
19541 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off);
19542 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len;
19543 rack->r_ctl.fsb.o_t_len = M_TRAILINGROOM(rack->r_ctl.fsb.m);
19544 rack->r_ctl.fsb.tcp_flags = flags;
19545 rack->r_ctl.fsb.left_to_send = orig_len - len;
19546 if (rack->r_ctl.fsb.left_to_send < pace_max_seg) {
19547 /* Less than a full sized pace, lets not */
19548 rack->r_fast_output = 0;
19549 return;
19550 } else {
19551 /* Round down to the nearest pace_max_seg */
19552 rack->r_ctl.fsb.left_to_send = rounddown(rack->r_ctl.fsb.left_to_send, pace_max_seg);
19553 }
19554 if (hw_tls)
19555 rack->r_ctl.fsb.hw_tls = 1;
19556 else
19557 rack->r_ctl.fsb.hw_tls = 0;
19558 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))),
19559 ("rack:%p left_to_send:%u sbavail:%u out:%u",
19560 rack, rack->r_ctl.fsb.left_to_send, sbavail(sb),
19561 (tp->snd_max - tp->snd_una)));
19562 if (rack->r_ctl.fsb.left_to_send < segsiz)
19563 rack->r_fast_output = 0;
19564 else {
19565 if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una)))
19566 rack->r_ctl.fsb.rfo_apply_push = 1;
19567 else
19568 rack->r_ctl.fsb.rfo_apply_push = 0;
19569 }
19570 }
19571
19572 static uint32_t
rack_get_hpts_pacing_min_for_bw(struct tcp_rack * rack,int32_t segsiz)19573 rack_get_hpts_pacing_min_for_bw(struct tcp_rack *rack, int32_t segsiz)
19574 {
19575 uint64_t min_time;
19576 uint32_t maxlen;
19577
19578 min_time = (uint64_t)get_hpts_min_sleep_time();
19579 maxlen = (uint32_t)((rack->r_ctl.gp_bw * min_time) / (uint64_t)HPTS_USEC_IN_SEC);
19580 maxlen = roundup(maxlen, segsiz);
19581 return (maxlen);
19582 }
19583
19584 static struct rack_sendmap *
rack_check_collapsed(struct tcp_rack * rack,uint32_t cts)19585 rack_check_collapsed(struct tcp_rack *rack, uint32_t cts)
19586 {
19587 struct rack_sendmap *rsm = NULL;
19588 int thresh;
19589
19590 restart:
19591 rsm = tqhash_find(rack->r_ctl.tqh, rack->r_ctl.last_collapse_point);
19592 if ((rsm == NULL) || ((rsm->r_flags & RACK_RWND_COLLAPSED) == 0)) {
19593 /* Nothing, strange turn off validity */
19594 rack->r_collapse_point_valid = 0;
19595 return (NULL);
19596 }
19597 /* Can we send it yet? */
19598 if (rsm->r_end > (rack->rc_tp->snd_una + rack->rc_tp->snd_wnd)) {
19599 /*
19600 * Receiver window has not grown enough for
19601 * the segment to be put on the wire.
19602 */
19603 return (NULL);
19604 }
19605 if (rsm->r_flags & RACK_ACKED) {
19606 /*
19607 * It has been sacked, lets move to the
19608 * next one if possible.
19609 */
19610 rack->r_ctl.last_collapse_point = rsm->r_end;
19611 /* Are we done? */
19612 if (SEQ_GEQ(rack->r_ctl.last_collapse_point,
19613 rack->r_ctl.high_collapse_point)) {
19614 rack->r_collapse_point_valid = 0;
19615 return (NULL);
19616 }
19617 goto restart;
19618 }
19619 /* Now has it been long enough ? */
19620 thresh = rack_calc_thresh_rack(rack, rack_grab_rtt(rack->rc_tp, rack), cts, __LINE__, 1);
19621 if ((cts - ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])) > thresh) {
19622 rack_log_collapse(rack, rsm->r_start,
19623 (cts - ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])),
19624 thresh, __LINE__, 6, rsm->r_flags, rsm);
19625 return (rsm);
19626 }
19627 /* Not enough time */
19628 rack_log_collapse(rack, rsm->r_start,
19629 (cts - ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)])),
19630 thresh, __LINE__, 7, rsm->r_flags, rsm);
19631 return (NULL);
19632 }
19633
19634 static inline void
rack_validate_sizes(struct tcp_rack * rack,int32_t * len,int32_t segsiz,uint32_t pace_max_seg)19635 rack_validate_sizes(struct tcp_rack *rack, int32_t *len, int32_t segsiz, uint32_t pace_max_seg)
19636 {
19637 if ((rack->full_size_rxt == 0) &&
19638 (rack->shape_rxt_to_pacing_min == 0) &&
19639 (*len >= segsiz)) {
19640 *len = segsiz;
19641 } else if (rack->shape_rxt_to_pacing_min &&
19642 rack->gp_ready) {
19643 /* We use pacing min as shaping len req */
19644 uint32_t maxlen;
19645
19646 maxlen = rack_get_hpts_pacing_min_for_bw(rack, segsiz);
19647 if (*len > maxlen)
19648 *len = maxlen;
19649 } else {
19650 /*
19651 * The else is full_size_rxt is on so send it all
19652 * note we do need to check this for exceeding
19653 * our max segment size due to the fact that
19654 * we do sometimes merge chunks together i.e.
19655 * we cannot just assume that we will never have
19656 * a chunk greater than pace_max_seg
19657 */
19658 if (*len > pace_max_seg)
19659 *len = pace_max_seg;
19660 }
19661 }
19662
19663 static int
rack_output(struct tcpcb * tp)19664 rack_output(struct tcpcb *tp)
19665 {
19666 struct socket *so;
19667 uint32_t recwin;
19668 uint32_t sb_offset, s_moff = 0;
19669 int32_t len, error = 0;
19670 uint16_t flags;
19671 struct mbuf *m, *s_mb = NULL;
19672 struct mbuf *mb;
19673 uint32_t if_hw_tsomaxsegcount = 0;
19674 uint32_t if_hw_tsomaxsegsize;
19675 int32_t segsiz, minseg;
19676 long tot_len_this_send = 0;
19677 #ifdef INET
19678 struct ip *ip = NULL;
19679 #endif
19680 struct udphdr *udp = NULL;
19681 struct tcp_rack *rack;
19682 struct tcphdr *th;
19683 uint8_t pass = 0;
19684 uint8_t mark = 0;
19685 uint8_t check_done = 0;
19686 uint8_t wanted_cookie = 0;
19687 u_char opt[TCP_MAXOLEN];
19688 unsigned ipoptlen, optlen, hdrlen, ulen=0;
19689 uint32_t rack_seq;
19690
19691 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
19692 unsigned ipsec_optlen = 0;
19693
19694 #endif
19695 int32_t idle, sendalot;
19696 uint32_t tot_idle;
19697 int32_t sub_from_prr = 0;
19698 volatile int32_t sack_rxmit;
19699 struct rack_sendmap *rsm = NULL;
19700 int32_t tso, mtu;
19701 struct tcpopt to;
19702 int32_t pacing_delay = 0;
19703 int32_t sup_rack = 0;
19704 uint32_t cts, ms_cts, delayed, early;
19705 uint32_t add_flag = RACK_SENT_SP;
19706 /* The doing_tlp flag will be set by the actual rack_timeout_tlp() */
19707 uint8_t doing_tlp = 0;
19708 uint32_t cwnd_to_use, pace_max_seg;
19709 int32_t do_a_prefetch = 0;
19710 int32_t prefetch_rsm = 0;
19711 int32_t orig_len = 0;
19712 struct timeval tv;
19713 int32_t prefetch_so_done = 0;
19714 struct tcp_log_buffer *lgb;
19715 struct inpcb *inp = tptoinpcb(tp);
19716 struct sockbuf *sb;
19717 uint64_t ts_val = 0;
19718 #ifdef TCP_ACCOUNTING
19719 uint64_t crtsc;
19720 #endif
19721 #ifdef INET6
19722 struct ip6_hdr *ip6 = NULL;
19723 int32_t isipv6;
19724 #endif
19725 bool hpts_calling, hw_tls = false;
19726
19727 NET_EPOCH_ASSERT();
19728 INP_WLOCK_ASSERT(inp);
19729
19730 /* setup and take the cache hits here */
19731 rack = (struct tcp_rack *)tp->t_fb_ptr;
19732 #ifdef TCP_ACCOUNTING
19733 sched_pin();
19734 ts_val = get_cyclecount();
19735 #endif
19736 hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS);
19737 tp->t_flags2 &= ~TF2_HPTS_CALLS;
19738 #ifdef TCP_OFFLOAD
19739 if (tp->t_flags & TF_TOE) {
19740 #ifdef TCP_ACCOUNTING
19741 sched_unpin();
19742 #endif
19743 return (tcp_offload_output(tp));
19744 }
19745 #endif
19746 if (rack->rack_deferred_inited == 0) {
19747 /*
19748 * If we are the connecting socket we will
19749 * hit rack_init() when no sequence numbers
19750 * are setup. This makes it so we must defer
19751 * some initialization. Call that now.
19752 */
19753 rack_deferred_init(tp, rack);
19754 }
19755 /*
19756 * For TFO connections in SYN_RECEIVED, only allow the initial
19757 * SYN|ACK and those sent by the retransmit timer.
19758 */
19759 if ((tp->t_flags & TF_FASTOPEN) &&
19760 (tp->t_state == TCPS_SYN_RECEIVED) &&
19761 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN|ACK sent */
19762 (rack->r_ctl.rc_resend == NULL)) { /* not a retransmit */
19763 #ifdef TCP_ACCOUNTING
19764 sched_unpin();
19765 #endif
19766 return (0);
19767 }
19768 #ifdef INET6
19769 if (rack->r_state) {
19770 /* Use the cache line loaded if possible */
19771 isipv6 = rack->r_is_v6;
19772 } else {
19773 isipv6 = (rack->rc_inp->inp_vflag & INP_IPV6) != 0;
19774 }
19775 #endif
19776 early = 0;
19777 cts = tcp_get_usecs(&tv);
19778 ms_cts = tcp_tv_to_msec(&tv);
19779 if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
19780 tcp_in_hpts(rack->rc_tp)) {
19781 /*
19782 * We are on the hpts for some timer but not hptsi output.
19783 * Remove from the hpts unconditionally.
19784 */
19785 rack_timer_cancel(tp, rack, cts, __LINE__);
19786 }
19787 /* Are we pacing and late? */
19788 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
19789 TSTMP_GEQ(cts, rack->r_ctl.rc_last_output_to)) {
19790 /* We are delayed */
19791 delayed = cts - rack->r_ctl.rc_last_output_to;
19792 } else {
19793 delayed = 0;
19794 }
19795 /* Do the timers, which may override the pacer */
19796 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
19797 int retval;
19798
19799 retval = rack_process_timers(tp, rack, cts, hpts_calling,
19800 &doing_tlp);
19801 if (retval != 0) {
19802 counter_u64_add(rack_out_size[TCP_MSS_ACCT_ATIMER], 1);
19803 #ifdef TCP_ACCOUNTING
19804 sched_unpin();
19805 #endif
19806 /*
19807 * If timers want tcp_drop(), then pass error out,
19808 * otherwise suppress it.
19809 */
19810 return (retval < 0 ? retval : 0);
19811 }
19812 }
19813 if (rack->rc_in_persist) {
19814 if (tcp_in_hpts(rack->rc_tp) == 0) {
19815 /* Timer is not running */
19816 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
19817 }
19818 #ifdef TCP_ACCOUNTING
19819 sched_unpin();
19820 #endif
19821 return (0);
19822 }
19823 if ((rack->rc_ack_required == 1) &&
19824 (rack->r_timer_override == 0)){
19825 /* A timeout occurred and no ack has arrived */
19826 if (tcp_in_hpts(rack->rc_tp) == 0) {
19827 /* Timer is not running */
19828 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
19829 }
19830 #ifdef TCP_ACCOUNTING
19831 sched_unpin();
19832 #endif
19833 return (0);
19834 }
19835 if ((rack->r_timer_override) ||
19836 (rack->rc_ack_can_sendout_data) ||
19837 (delayed) ||
19838 (tp->t_state < TCPS_ESTABLISHED)) {
19839 rack->rc_ack_can_sendout_data = 0;
19840 if (tcp_in_hpts(rack->rc_tp))
19841 tcp_hpts_remove(rack->rc_tp);
19842 } else if (tcp_in_hpts(rack->rc_tp)) {
19843 /*
19844 * On the hpts you can't pass even if ACKNOW is on, we will
19845 * when the hpts fires.
19846 */
19847 #ifdef TCP_ACCOUNTING
19848 crtsc = get_cyclecount();
19849 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19850 tp->tcp_proc_time[SND_BLOCKED] += (crtsc - ts_val);
19851 tp->tcp_cnt_counters[SND_BLOCKED]++;
19852 }
19853 sched_unpin();
19854 #endif
19855 counter_u64_add(rack_out_size[TCP_MSS_ACCT_INPACE], 1);
19856 return (0);
19857 }
19858 /* Finish out both pacing early and late accounting */
19859 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
19860 TSTMP_GT(rack->r_ctl.rc_last_output_to, cts)) {
19861 early = rack->r_ctl.rc_last_output_to - cts;
19862 } else
19863 early = 0;
19864 if (delayed && (rack->rc_always_pace == 1)) {
19865 rack->r_ctl.rc_agg_delayed += delayed;
19866 rack->r_late = 1;
19867 } else if (early && (rack->rc_always_pace == 1)) {
19868 rack->r_ctl.rc_agg_early += early;
19869 rack->r_early = 1;
19870 } else if (rack->rc_always_pace == 0) {
19871 /* Non-paced we are not late */
19872 rack->r_ctl.rc_agg_delayed = rack->r_ctl.rc_agg_early = 0;
19873 rack->r_early = rack->r_late = 0;
19874 }
19875 /* Now that early/late accounting is done turn off the flag */
19876 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
19877 rack->r_wanted_output = 0;
19878 rack->r_timer_override = 0;
19879 if ((tp->t_state != rack->r_state) &&
19880 TCPS_HAVEESTABLISHED(tp->t_state)) {
19881 rack_set_state(tp, rack);
19882 }
19883 segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
19884 minseg = segsiz;
19885 if (rack->r_ctl.rc_pace_max_segs == 0)
19886 pace_max_seg = rack->rc_user_set_max_segs * segsiz;
19887 else
19888 pace_max_seg = rack->r_ctl.rc_pace_max_segs;
19889 if ((rack->r_fast_output) &&
19890 (doing_tlp == 0) &&
19891 (tp->rcv_numsacks == 0)) {
19892 int ret;
19893
19894 error = 0;
19895 ret = rack_fast_output(tp, rack, ts_val, cts, ms_cts, &tv, &tot_len_this_send, &error, __LINE__);
19896 if (ret > 0)
19897 return(ret);
19898 else if (error) {
19899 inp = rack->rc_inp;
19900 so = inp->inp_socket;
19901 sb = &so->so_snd;
19902 goto nomore;
19903 } else {
19904 /* Return == 0, if there is more we can send tot_len wise fall through and send */
19905 if (tot_len_this_send >= pace_max_seg)
19906 return (ret);
19907 #ifdef TCP_ACCOUNTING
19908 /* We need to re-pin since fast_output un-pined */
19909 sched_pin();
19910 ts_val = get_cyclecount();
19911 #endif
19912 /* Fall back out so we can send any more that may bring us to pace_max_seg */
19913 }
19914 }
19915 inp = rack->rc_inp;
19916 /*
19917 * For TFO connections in SYN_SENT or SYN_RECEIVED,
19918 * only allow the initial SYN or SYN|ACK and those sent
19919 * by the retransmit timer.
19920 */
19921 if ((tp->t_flags & TF_FASTOPEN) &&
19922 ((tp->t_state == TCPS_SYN_RECEIVED) ||
19923 (tp->t_state == TCPS_SYN_SENT)) &&
19924 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
19925 (tp->t_rxtshift == 0)) { /* not a retransmit */
19926 rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
19927 #ifdef TCP_ACCOUNTING
19928 sched_unpin();
19929 #endif
19930 return (0);
19931 }
19932 /*
19933 * Determine length of data that should be transmitted, and flags
19934 * that will be used. If there is some data or critical controls
19935 * (SYN, RST) to send, then transmit; otherwise, investigate
19936 * further.
19937 */
19938 idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
19939 if (tp->t_idle_reduce) {
19940 if (idle && (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur))
19941 rack_cc_after_idle(rack, tp);
19942 }
19943 tp->t_flags &= ~TF_LASTIDLE;
19944 if (idle) {
19945 if (tp->t_flags & TF_MORETOCOME) {
19946 tp->t_flags |= TF_LASTIDLE;
19947 idle = 0;
19948 }
19949 }
19950 if ((tp->snd_una == tp->snd_max) &&
19951 rack->r_ctl.rc_went_idle_time &&
19952 (cts > rack->r_ctl.rc_went_idle_time)) {
19953 tot_idle = (cts - rack->r_ctl.rc_went_idle_time);
19954 if (tot_idle > rack_min_probertt_hold) {
19955 /* Count as a probe rtt */
19956 if (rack->in_probe_rtt == 0) {
19957 rack->r_ctl.rc_lower_rtt_us_cts = cts;
19958 rack->r_ctl.rc_time_probertt_entered = rack->r_ctl.rc_lower_rtt_us_cts;
19959 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts;
19960 rack->r_ctl.rc_time_of_last_probertt = rack->r_ctl.rc_lower_rtt_us_cts;
19961 } else {
19962 rack_exit_probertt(rack, cts);
19963 }
19964 }
19965 } else
19966 tot_idle = 0;
19967 if (rack_use_fsb &&
19968 (rack->r_ctl.fsb.tcp_ip_hdr) &&
19969 (rack->r_fsb_inited == 0) &&
19970 (rack->r_state != TCPS_CLOSED))
19971 rack_init_fsb_block(tp, rack, tcp_outflags[tp->t_state]);
19972 if (rack->rc_sendvars_notset == 1) {
19973 rack->rc_sendvars_notset = 0;
19974 /*
19975 * Make sure any TCP timers (keep-alive) is not running.
19976 */
19977 tcp_timer_stop(tp);
19978 }
19979 if ((rack->rack_no_prr == 1) &&
19980 (rack->rc_always_pace == 0)) {
19981 /*
19982 * Sanity check before sending, if we have
19983 * no-pacing enabled and prr is turned off that
19984 * is a logistics error. Correct this by turnning
19985 * prr back on. A user *must* set some form of
19986 * pacing in order to turn PRR off. We do this
19987 * in the output path so that we can avoid socket
19988 * option ordering issues that would occur if we
19989 * tried to do it while setting rack_no_prr on.
19990 */
19991 rack->rack_no_prr = 0;
19992 }
19993 if ((rack->pcm_enabled == 1) &&
19994 (rack->pcm_needed == 0) &&
19995 (tot_idle > 0)) {
19996 /*
19997 * We have been idle some micro seconds. We need
19998 * to factor this in to see if a PCM is needed.
19999 */
20000 uint32_t rtts_idle, rnds;
20001
20002 if (tp->t_srtt)
20003 rtts_idle = tot_idle / tp->t_srtt;
20004 else
20005 rtts_idle = 0;
20006 rnds = rack->r_ctl.current_round - rack->r_ctl.last_pcm_round;
20007 rack->r_ctl.pcm_idle_rounds += rtts_idle;
20008 if ((rnds + rack->r_ctl.pcm_idle_rounds) >= rack_pcm_every_n_rounds) {
20009 rack->pcm_needed = 1;
20010 rack_log_pcm(rack, 8, rack->r_ctl.last_pcm_round, rtts_idle, rack->r_ctl.current_round );
20011 }
20012 }
20013 again:
20014 sendalot = 0;
20015 cts = tcp_get_usecs(&tv);
20016 ms_cts = tcp_tv_to_msec(&tv);
20017 tso = 0;
20018 mtu = 0;
20019 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
20020 (rack->r_ctl.pcm_max_seg == 0)) {
20021 /*
20022 * We set in our first send so we know that the ctf_fixed_maxseg
20023 * has been fully set. If we do it in rack_init() we most likely
20024 * see 512 bytes so we end up at 5120, not desirable.
20025 */
20026 rack->r_ctl.pcm_max_seg = rc_init_window(rack);
20027 if (rack->r_ctl.pcm_max_seg < (ctf_fixed_maxseg(tp) * 10)) {
20028 /*
20029 * Assure our initial PCM probe is at least 10 MSS.
20030 */
20031 rack->r_ctl.pcm_max_seg = ctf_fixed_maxseg(tp) * 10;
20032 }
20033 }
20034 if ((rack->r_ctl.pcm_max_seg != 0) && (rack->pcm_needed == 1)) {
20035 uint32_t rw_avail, cwa;
20036
20037 if (tp->snd_wnd > ctf_outstanding(tp))
20038 rw_avail = tp->snd_wnd - ctf_outstanding(tp);
20039 else
20040 rw_avail = 0;
20041 if (tp->snd_cwnd > ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked))
20042 cwa = tp->snd_cwnd -ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
20043 else
20044 cwa = 0;
20045 if ((cwa >= rack->r_ctl.pcm_max_seg) &&
20046 (rw_avail > rack->r_ctl.pcm_max_seg)) {
20047 /* Raise up the max seg for this trip through */
20048 pace_max_seg = rack->r_ctl.pcm_max_seg;
20049 /* Disable any fast output */
20050 rack->r_fast_output = 0;
20051 }
20052 if (rack_verbose_logging) {
20053 rack_log_pcm(rack, 4,
20054 cwa, rack->r_ctl.pcm_max_seg, rw_avail);
20055 }
20056 }
20057 sb_offset = tp->snd_max - tp->snd_una;
20058 cwnd_to_use = rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
20059 flags = tcp_outflags[tp->t_state];
20060 while (rack->rc_free_cnt < rack_free_cache) {
20061 rsm = rack_alloc(rack);
20062 if (rsm == NULL) {
20063 if (hpts_calling)
20064 /* Retry in a ms */
20065 pacing_delay = (1 * HPTS_USEC_IN_MSEC);
20066 so = inp->inp_socket;
20067 sb = &so->so_snd;
20068 goto just_return_nolock;
20069 }
20070 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_free, rsm, r_tnext);
20071 rack->rc_free_cnt++;
20072 rsm = NULL;
20073 }
20074 sack_rxmit = 0;
20075 len = 0;
20076 rsm = NULL;
20077 if (flags & TH_RST) {
20078 SOCK_SENDBUF_LOCK(inp->inp_socket);
20079 so = inp->inp_socket;
20080 sb = &so->so_snd;
20081 goto send;
20082 }
20083 if (rack->r_ctl.rc_resend) {
20084 /* Retransmit timer */
20085 rsm = rack->r_ctl.rc_resend;
20086 rack->r_ctl.rc_resend = NULL;
20087 len = rsm->r_end - rsm->r_start;
20088 sack_rxmit = 1;
20089 sendalot = 0;
20090 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
20091 ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p",
20092 __func__, __LINE__,
20093 rsm->r_start, tp->snd_una, tp, rack, rsm));
20094 sb_offset = rsm->r_start - tp->snd_una;
20095 rack_validate_sizes(rack, &len, segsiz, pace_max_seg);
20096 } else if (rack->r_collapse_point_valid &&
20097 ((rsm = rack_check_collapsed(rack, cts)) != NULL)) {
20098 /*
20099 * If an RSM is returned then enough time has passed
20100 * for us to retransmit it. Move up the collapse point,
20101 * since this rsm has its chance to retransmit now.
20102 */
20103 tcp_trace_point(rack->rc_tp, TCP_TP_COLLAPSED_RXT);
20104 rack->r_ctl.last_collapse_point = rsm->r_end;
20105 /* Are we done? */
20106 if (SEQ_GEQ(rack->r_ctl.last_collapse_point,
20107 rack->r_ctl.high_collapse_point))
20108 rack->r_collapse_point_valid = 0;
20109 sack_rxmit = 1;
20110 /* We are not doing a TLP */
20111 doing_tlp = 0;
20112 len = rsm->r_end - rsm->r_start;
20113 sb_offset = rsm->r_start - tp->snd_una;
20114 sendalot = 0;
20115 rack_validate_sizes(rack, &len, segsiz, pace_max_seg);
20116 } else if ((rsm = tcp_rack_output(tp, rack, cts)) != NULL) {
20117 /* We have a retransmit that takes precedence */
20118 if ((!IN_FASTRECOVERY(tp->t_flags)) &&
20119 ((rsm->r_flags & RACK_MUST_RXT) == 0) &&
20120 ((tp->t_flags & TF_WASFRECOVERY) == 0)) {
20121 /* Enter recovery if not induced by a time-out */
20122 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__);
20123 }
20124 #ifdef INVARIANTS
20125 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
20126 panic("Huh, tp:%p rack:%p rsm:%p start:%u < snd_una:%u\n",
20127 tp, rack, rsm, rsm->r_start, tp->snd_una);
20128 }
20129 #endif
20130 len = rsm->r_end - rsm->r_start;
20131 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
20132 ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p",
20133 __func__, __LINE__,
20134 rsm->r_start, tp->snd_una, tp, rack, rsm));
20135 sb_offset = rsm->r_start - tp->snd_una;
20136 sendalot = 0;
20137 rack_validate_sizes(rack, &len, segsiz, pace_max_seg);
20138 if (len > 0) {
20139 sack_rxmit = 1;
20140 KMOD_TCPSTAT_INC(tcps_sack_rexmits);
20141 KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes,
20142 min(len, segsiz));
20143 }
20144 } else if (rack->r_ctl.rc_tlpsend) {
20145 /* Tail loss probe */
20146 long cwin;
20147 long tlen;
20148
20149 /*
20150 * Check if we can do a TLP with a RACK'd packet
20151 * this can happen if we are not doing the rack
20152 * cheat and we skipped to a TLP and it
20153 * went off.
20154 */
20155 rsm = rack->r_ctl.rc_tlpsend;
20156 /* We are doing a TLP make sure the flag is preent */
20157 rsm->r_flags |= RACK_TLP;
20158 rack->r_ctl.rc_tlpsend = NULL;
20159 sack_rxmit = 1;
20160 tlen = rsm->r_end - rsm->r_start;
20161 if (tlen > segsiz)
20162 tlen = segsiz;
20163 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
20164 ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p",
20165 __func__, __LINE__,
20166 rsm->r_start, tp->snd_una, tp, rack, rsm));
20167 sb_offset = rsm->r_start - tp->snd_una;
20168 cwin = min(tp->snd_wnd, tlen);
20169 len = cwin;
20170 }
20171 if (rack->r_must_retran &&
20172 (doing_tlp == 0) &&
20173 (SEQ_GT(tp->snd_max, tp->snd_una)) &&
20174 (rsm == NULL)) {
20175 /*
20176 * There are two different ways that we
20177 * can get into this block:
20178 * a) This is a non-sack connection, we had a time-out
20179 * and thus r_must_retran was set and everything
20180 * left outstanding as been marked for retransmit.
20181 * b) The MTU of the path shrank, so that everything
20182 * was marked to be retransmitted with the smaller
20183 * mtu and r_must_retran was set.
20184 *
20185 * This means that we expect the sendmap (outstanding)
20186 * to all be marked must. We can use the tmap to
20187 * look at them.
20188 *
20189 */
20190 int sendwin, flight;
20191
20192 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
20193 flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto);
20194 if (flight >= sendwin) {
20195 /*
20196 * We can't send yet.
20197 */
20198 so = inp->inp_socket;
20199 sb = &so->so_snd;
20200 goto just_return_nolock;
20201 }
20202 /*
20203 * This is the case a/b mentioned above. All
20204 * outstanding/not-acked should be marked.
20205 * We can use the tmap to find them.
20206 */
20207 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
20208 if (rsm == NULL) {
20209 /* TSNH */
20210 rack->r_must_retran = 0;
20211 rack->r_ctl.rc_out_at_rto = 0;
20212 so = inp->inp_socket;
20213 sb = &so->so_snd;
20214 goto just_return_nolock;
20215 }
20216 if ((rsm->r_flags & RACK_MUST_RXT) == 0) {
20217 /*
20218 * The first one does not have the flag, did we collapse
20219 * further up in our list?
20220 */
20221 rack->r_must_retran = 0;
20222 rack->r_ctl.rc_out_at_rto = 0;
20223 rsm = NULL;
20224 sack_rxmit = 0;
20225 } else {
20226 sack_rxmit = 1;
20227 len = rsm->r_end - rsm->r_start;
20228 sb_offset = rsm->r_start - tp->snd_una;
20229 sendalot = 0;
20230 if ((rack->full_size_rxt == 0) &&
20231 (rack->shape_rxt_to_pacing_min == 0) &&
20232 (len >= segsiz))
20233 len = segsiz;
20234 else if (rack->shape_rxt_to_pacing_min &&
20235 rack->gp_ready) {
20236 /* We use pacing min as shaping len req */
20237 uint32_t maxlen;
20238
20239 maxlen = rack_get_hpts_pacing_min_for_bw(rack, segsiz);
20240 if (len > maxlen)
20241 len = maxlen;
20242 }
20243 /*
20244 * Delay removing the flag RACK_MUST_RXT so
20245 * that the fastpath for retransmit will
20246 * work with this rsm.
20247 */
20248 }
20249 }
20250 /*
20251 * Enforce a connection sendmap count limit if set
20252 * as long as we are not retransmiting.
20253 */
20254 if ((rsm == NULL) &&
20255 (V_tcp_map_entries_limit > 0) &&
20256 (rack->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
20257 counter_u64_add(rack_to_alloc_limited, 1);
20258 if (!rack->alloc_limit_reported) {
20259 rack->alloc_limit_reported = 1;
20260 counter_u64_add(rack_alloc_limited_conns, 1);
20261 }
20262 so = inp->inp_socket;
20263 sb = &so->so_snd;
20264 goto just_return_nolock;
20265 }
20266 if (rsm && (rsm->r_flags & RACK_HAS_FIN)) {
20267 /* we are retransmitting the fin */
20268 len--;
20269 if (len) {
20270 /*
20271 * When retransmitting data do *not* include the
20272 * FIN. This could happen from a TLP probe.
20273 */
20274 flags &= ~TH_FIN;
20275 }
20276 }
20277 if (rsm && rack->r_fsb_inited &&
20278 rack_use_rsm_rfo &&
20279 ((rsm->r_flags & RACK_HAS_FIN) == 0)) {
20280 int ret;
20281
20282 ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp);
20283 if (ret == 0)
20284 return (0);
20285 }
20286 so = inp->inp_socket;
20287 sb = &so->so_snd;
20288 if (do_a_prefetch == 0) {
20289 kern_prefetch(sb, &do_a_prefetch);
20290 do_a_prefetch = 1;
20291 }
20292 #ifdef NETFLIX_SHARED_CWND
20293 if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) &&
20294 rack->rack_enable_scwnd) {
20295 /* We are doing cwnd sharing */
20296 if (rack->gp_ready &&
20297 (rack->rack_attempted_scwnd == 0) &&
20298 (rack->r_ctl.rc_scw == NULL) &&
20299 tp->t_lib) {
20300 /* The pcbid is in, lets make an attempt */
20301 counter_u64_add(rack_try_scwnd, 1);
20302 rack->rack_attempted_scwnd = 1;
20303 rack->r_ctl.rc_scw = tcp_shared_cwnd_alloc(tp,
20304 &rack->r_ctl.rc_scw_index,
20305 segsiz);
20306 }
20307 if (rack->r_ctl.rc_scw &&
20308 (rack->rack_scwnd_is_idle == 1) &&
20309 sbavail(&so->so_snd)) {
20310 /* we are no longer out of data */
20311 tcp_shared_cwnd_active(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
20312 rack->rack_scwnd_is_idle = 0;
20313 }
20314 if (rack->r_ctl.rc_scw) {
20315 /* First lets update and get the cwnd */
20316 rack->r_ctl.cwnd_to_use = cwnd_to_use = tcp_shared_cwnd_update(rack->r_ctl.rc_scw,
20317 rack->r_ctl.rc_scw_index,
20318 tp->snd_cwnd, tp->snd_wnd, segsiz);
20319 }
20320 }
20321 #endif
20322 /*
20323 * Get standard flags, and add SYN or FIN if requested by 'hidden'
20324 * state flags.
20325 */
20326 if (tp->t_flags & TF_NEEDFIN)
20327 flags |= TH_FIN;
20328 if (tp->t_flags & TF_NEEDSYN)
20329 flags |= TH_SYN;
20330 if ((sack_rxmit == 0) && (prefetch_rsm == 0)) {
20331 void *end_rsm;
20332 end_rsm = TAILQ_LAST_FAST(&rack->r_ctl.rc_tmap, rack_sendmap, r_tnext);
20333 if (end_rsm)
20334 kern_prefetch(end_rsm, &prefetch_rsm);
20335 prefetch_rsm = 1;
20336 }
20337 SOCK_SENDBUF_LOCK(so);
20338 if ((sack_rxmit == 0) &&
20339 (TCPS_HAVEESTABLISHED(tp->t_state) ||
20340 (tp->t_flags & TF_FASTOPEN))) {
20341 /*
20342 * We are not retransmitting (sack_rxmit is 0) so we
20343 * are sending new data. This is always based on snd_max.
20344 * Now in theory snd_max may be equal to snd_una, if so
20345 * then nothing is outstanding and the offset would be 0.
20346 */
20347 uint32_t avail;
20348
20349 avail = sbavail(sb);
20350 if (SEQ_GT(tp->snd_max, tp->snd_una) && avail)
20351 sb_offset = tp->snd_max - tp->snd_una;
20352 else
20353 sb_offset = 0;
20354 if ((IN_FASTRECOVERY(tp->t_flags) == 0) || rack->rack_no_prr) {
20355 if (rack->r_ctl.rc_tlp_new_data) {
20356 /* TLP is forcing out new data */
20357 if (rack->r_ctl.rc_tlp_new_data > (uint32_t) (avail - sb_offset)) {
20358 rack->r_ctl.rc_tlp_new_data = (uint32_t) (avail - sb_offset);
20359 }
20360 if ((rack->r_ctl.rc_tlp_new_data + sb_offset) > tp->snd_wnd) {
20361 if (tp->snd_wnd > sb_offset)
20362 len = tp->snd_wnd - sb_offset;
20363 else
20364 len = 0;
20365 } else {
20366 len = rack->r_ctl.rc_tlp_new_data;
20367 }
20368 rack->r_ctl.rc_tlp_new_data = 0;
20369 } else {
20370 len = rack_what_can_we_send(tp, rack, cwnd_to_use, avail, sb_offset);
20371 }
20372 if ((rack->r_ctl.crte == NULL) &&
20373 IN_FASTRECOVERY(tp->t_flags) &&
20374 (rack->full_size_rxt == 0) &&
20375 (rack->shape_rxt_to_pacing_min == 0) &&
20376 (len > segsiz)) {
20377 /*
20378 * For prr=off, we need to send only 1 MSS
20379 * at a time. We do this because another sack could
20380 * be arriving that causes us to send retransmits and
20381 * we don't want to be on a long pace due to a larger send
20382 * that keeps us from sending out the retransmit.
20383 */
20384 len = segsiz;
20385 } else if (rack->shape_rxt_to_pacing_min &&
20386 rack->gp_ready) {
20387 /* We use pacing min as shaping len req */
20388 uint32_t maxlen;
20389
20390 maxlen = rack_get_hpts_pacing_min_for_bw(rack, segsiz);
20391 if (len > maxlen)
20392 len = maxlen;
20393 }/* The else is full_size_rxt is on so send it all */
20394 } else {
20395 uint32_t outstanding;
20396 /*
20397 * We are inside of a Fast recovery episode, this
20398 * is caused by a SACK or 3 dup acks. At this point
20399 * we have sent all the retransmissions and we rely
20400 * on PRR to dictate what we will send in the form of
20401 * new data.
20402 */
20403
20404 outstanding = tp->snd_max - tp->snd_una;
20405 if ((rack->r_ctl.rc_prr_sndcnt + outstanding) > tp->snd_wnd) {
20406 if (tp->snd_wnd > outstanding) {
20407 len = tp->snd_wnd - outstanding;
20408 /* Check to see if we have the data */
20409 if ((sb_offset + len) > avail) {
20410 /* It does not all fit */
20411 if (avail > sb_offset)
20412 len = avail - sb_offset;
20413 else
20414 len = 0;
20415 }
20416 } else {
20417 len = 0;
20418 }
20419 } else if (avail > sb_offset) {
20420 len = avail - sb_offset;
20421 } else {
20422 len = 0;
20423 }
20424 if (len > 0) {
20425 if (len > rack->r_ctl.rc_prr_sndcnt) {
20426 len = rack->r_ctl.rc_prr_sndcnt;
20427 }
20428 if (len > 0) {
20429 sub_from_prr = 1;
20430 }
20431 }
20432 if (len > segsiz) {
20433 /*
20434 * We should never send more than a MSS when
20435 * retransmitting or sending new data in prr
20436 * mode unless the override flag is on. Most
20437 * likely the PRR algorithm is not going to
20438 * let us send a lot as well :-)
20439 */
20440 if (rack->r_ctl.rc_prr_sendalot == 0) {
20441 len = segsiz;
20442 }
20443 } else if (len < segsiz) {
20444 /*
20445 * Do we send any? The idea here is if the
20446 * send empty's the socket buffer we want to
20447 * do it. However if not then lets just wait
20448 * for our prr_sndcnt to get bigger.
20449 */
20450 long leftinsb;
20451
20452 leftinsb = sbavail(sb) - sb_offset;
20453 if (leftinsb > len) {
20454 /* This send does not empty the sb */
20455 len = 0;
20456 }
20457 }
20458 }
20459 } else if (!TCPS_HAVEESTABLISHED(tp->t_state)) {
20460 /*
20461 * If you have not established
20462 * and are not doing FAST OPEN
20463 * no data please.
20464 */
20465 if ((sack_rxmit == 0) &&
20466 !(tp->t_flags & TF_FASTOPEN)) {
20467 len = 0;
20468 sb_offset = 0;
20469 }
20470 }
20471 if (prefetch_so_done == 0) {
20472 kern_prefetch(so, &prefetch_so_done);
20473 prefetch_so_done = 1;
20474 }
20475 orig_len = len;
20476 /*
20477 * Lop off SYN bit if it has already been sent. However, if this is
20478 * SYN-SENT state and if segment contains data and if we don't know
20479 * that foreign host supports TAO, suppress sending segment.
20480 */
20481 if ((flags & TH_SYN) &&
20482 SEQ_GT(tp->snd_max, tp->snd_una) &&
20483 ((sack_rxmit == 0) &&
20484 (tp->t_rxtshift == 0))) {
20485 /*
20486 * When sending additional segments following a TFO SYN|ACK,
20487 * do not include the SYN bit.
20488 */
20489 if ((tp->t_flags & TF_FASTOPEN) &&
20490 (tp->t_state == TCPS_SYN_RECEIVED))
20491 flags &= ~TH_SYN;
20492 }
20493 /*
20494 * Be careful not to send data and/or FIN on SYN segments. This
20495 * measure is needed to prevent interoperability problems with not
20496 * fully conformant TCP implementations.
20497 */
20498 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
20499 len = 0;
20500 flags &= ~TH_FIN;
20501 }
20502 /*
20503 * On TFO sockets, ensure no data is sent in the following cases:
20504 *
20505 * - When retransmitting SYN|ACK on a passively-created socket
20506 *
20507 * - When retransmitting SYN on an actively created socket
20508 *
20509 * - When sending a zero-length cookie (cookie request) on an
20510 * actively created socket
20511 *
20512 * - When the socket is in the CLOSED state (RST is being sent)
20513 */
20514 if ((tp->t_flags & TF_FASTOPEN) &&
20515 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
20516 ((tp->t_state == TCPS_SYN_SENT) &&
20517 (tp->t_tfo_client_cookie_len == 0)) ||
20518 (flags & TH_RST))) {
20519 sack_rxmit = 0;
20520 len = 0;
20521 }
20522 /* Without fast-open there should never be data sent on a SYN */
20523 if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN)) {
20524 len = 0;
20525 }
20526 if ((len > segsiz) && (tcp_dsack_block_exists(tp))) {
20527 /* We only send 1 MSS if we have a DSACK block */
20528 add_flag |= RACK_SENT_W_DSACK;
20529 len = segsiz;
20530 }
20531 if (len <= 0) {
20532 /*
20533 * We have nothing to send, or the window shrank, or
20534 * is closed, do we need to go into persists?
20535 */
20536 len = 0;
20537 if ((tp->snd_wnd == 0) &&
20538 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
20539 (tp->snd_una == tp->snd_max) &&
20540 (sb_offset < (int)sbavail(sb))) {
20541 rack_enter_persist(tp, rack, cts, tp->snd_una);
20542 }
20543 } else if ((rsm == NULL) &&
20544 (doing_tlp == 0) &&
20545 (len < pace_max_seg)) {
20546 /*
20547 * We are not sending a maximum sized segment for
20548 * some reason. Should we not send anything (think
20549 * sws or persists)?
20550 */
20551 if ((tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), minseg)) &&
20552 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
20553 (len < minseg) &&
20554 (len < (int)(sbavail(sb) - sb_offset))) {
20555 /*
20556 * Here the rwnd is less than
20557 * the minimum pacing size, this is not a retransmit,
20558 * we are established and
20559 * the send is not the last in the socket buffer
20560 * we send nothing, and we may enter persists
20561 * if nothing is outstanding.
20562 */
20563 len = 0;
20564 if (tp->snd_max == tp->snd_una) {
20565 /*
20566 * Nothing out we can
20567 * go into persists.
20568 */
20569 rack_enter_persist(tp, rack, cts, tp->snd_una);
20570 }
20571 } else if ((cwnd_to_use >= max(minseg, (segsiz * 4))) &&
20572 (ctf_flight_size(tp, rack->r_ctl.rc_sacked) > (2 * segsiz)) &&
20573 (len < (int)(sbavail(sb) - sb_offset)) &&
20574 (len < minseg)) {
20575 /*
20576 * Here we are not retransmitting, and
20577 * the cwnd is not so small that we could
20578 * not send at least a min size (rxt timer
20579 * not having gone off), We have 2 segments or
20580 * more already in flight, its not the tail end
20581 * of the socket buffer and the cwnd is blocking
20582 * us from sending out a minimum pacing segment size.
20583 * Lets not send anything.
20584 */
20585 len = 0;
20586 } else if (((tp->snd_wnd - ctf_outstanding(tp)) <
20587 min((rack->r_ctl.rc_high_rwnd/2), minseg)) &&
20588 (ctf_flight_size(tp, rack->r_ctl.rc_sacked) > (2 * segsiz)) &&
20589 (len < (int)(sbavail(sb) - sb_offset)) &&
20590 (TCPS_HAVEESTABLISHED(tp->t_state))) {
20591 /*
20592 * Here we have a send window but we have
20593 * filled it up and we can't send another pacing segment.
20594 * We also have in flight more than 2 segments
20595 * and we are not completing the sb i.e. we allow
20596 * the last bytes of the sb to go out even if
20597 * its not a full pacing segment.
20598 */
20599 len = 0;
20600 } else if ((rack->r_ctl.crte != NULL) &&
20601 (tp->snd_wnd >= (pace_max_seg * max(1, rack_hw_rwnd_factor))) &&
20602 (cwnd_to_use >= (pace_max_seg + (4 * segsiz))) &&
20603 (ctf_flight_size(tp, rack->r_ctl.rc_sacked) >= (2 * segsiz)) &&
20604 (len < (int)(sbavail(sb) - sb_offset))) {
20605 /*
20606 * Here we are doing hardware pacing, this is not a TLP,
20607 * we are not sending a pace max segment size, there is rwnd
20608 * room to send at least N pace_max_seg, the cwnd is greater
20609 * than or equal to a full pacing segments plus 4 mss and we have 2 or
20610 * more segments in flight and its not the tail of the socket buffer.
20611 *
20612 * We don't want to send instead we need to get more ack's in to
20613 * allow us to send a full pacing segment. Normally, if we are pacing
20614 * about the right speed, we should have finished our pacing
20615 * send as most of the acks have come back if we are at the
20616 * right rate. This is a bit fuzzy since return path delay
20617 * can delay the acks, which is why we want to make sure we
20618 * have cwnd space to have a bit more than a max pace segments in flight.
20619 *
20620 * If we have not gotten our acks back we are pacing at too high a
20621 * rate delaying will not hurt and will bring our GP estimate down by
20622 * injecting the delay. If we don't do this we will send
20623 * 2 MSS out in response to the acks being clocked in which
20624 * defeats the point of hw-pacing (i.e. to help us get
20625 * larger TSO's out).
20626 */
20627 len = 0;
20628 }
20629
20630 }
20631 /* len will be >= 0 after this point. */
20632 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
20633 rack_sndbuf_autoscale(rack);
20634 /*
20635 * Decide if we can use TCP Segmentation Offloading (if supported by
20636 * hardware).
20637 *
20638 * TSO may only be used if we are in a pure bulk sending state. The
20639 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
20640 * options prevent using TSO. With TSO the TCP header is the same
20641 * (except for the sequence number) for all generated packets. This
20642 * makes it impossible to transmit any options which vary per
20643 * generated segment or packet.
20644 *
20645 * IPv4 handling has a clear separation of ip options and ip header
20646 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen() does
20647 * the right thing below to provide length of just ip options and thus
20648 * checking for ipoptlen is enough to decide if ip options are present.
20649 */
20650 ipoptlen = 0;
20651 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
20652 /*
20653 * Pre-calculate here as we save another lookup into the darknesses
20654 * of IPsec that way and can actually decide if TSO is ok.
20655 */
20656 #ifdef INET6
20657 if (isipv6 && IPSEC_ENABLED(ipv6))
20658 ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp);
20659 #ifdef INET
20660 else
20661 #endif
20662 #endif /* INET6 */
20663 #ifdef INET
20664 if (IPSEC_ENABLED(ipv4))
20665 ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp);
20666 #endif /* INET */
20667 #endif
20668
20669 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
20670 ipoptlen += ipsec_optlen;
20671 #endif
20672 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > segsiz &&
20673 (tp->t_port == 0) &&
20674 ((tp->t_flags & TF_SIGNATURE) == 0) &&
20675 sack_rxmit == 0 &&
20676 ipoptlen == 0)
20677 tso = 1;
20678 {
20679 uint32_t outstanding __unused;
20680
20681 outstanding = tp->snd_max - tp->snd_una;
20682 if (tp->t_flags & TF_SENTFIN) {
20683 /*
20684 * If we sent a fin, snd_max is 1 higher than
20685 * snd_una
20686 */
20687 outstanding--;
20688 }
20689 if (sack_rxmit) {
20690 if ((rsm->r_flags & RACK_HAS_FIN) == 0)
20691 flags &= ~TH_FIN;
20692 }
20693 }
20694 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
20695 (long)TCP_MAXWIN << tp->rcv_scale);
20696
20697 /*
20698 * Sender silly window avoidance. We transmit under the following
20699 * conditions when len is non-zero:
20700 *
20701 * - We have a full segment (or more with TSO) - This is the last
20702 * buffer in a write()/send() and we are either idle or running
20703 * NODELAY - we've timed out (e.g. persist timer) - we have more
20704 * then 1/2 the maximum send window's worth of data (receiver may be
20705 * limited the window size) - we need to retransmit
20706 */
20707 if (len) {
20708 if (len >= segsiz) {
20709 goto send;
20710 }
20711 /*
20712 * NOTE! on localhost connections an 'ack' from the remote
20713 * end may occur synchronously with the output and cause us
20714 * to flush a buffer queued with moretocome. XXX
20715 *
20716 */
20717 if (!(tp->t_flags & TF_MORETOCOME) && /* normal case */
20718 (idle || (tp->t_flags & TF_NODELAY)) &&
20719 ((uint32_t)len + (uint32_t)sb_offset >= sbavail(sb)) &&
20720 (tp->t_flags & TF_NOPUSH) == 0) {
20721 pass = 2;
20722 goto send;
20723 }
20724 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */
20725 pass = 22;
20726 goto send;
20727 }
20728 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
20729 pass = 4;
20730 goto send;
20731 }
20732 if (sack_rxmit) {
20733 pass = 6;
20734 goto send;
20735 }
20736 if (((tp->snd_wnd - ctf_outstanding(tp)) < segsiz) &&
20737 (ctf_outstanding(tp) < (segsiz * 2))) {
20738 /*
20739 * We have less than two MSS outstanding (delayed ack)
20740 * and our rwnd will not let us send a full sized
20741 * MSS. Lets go ahead and let this small segment
20742 * out because we want to try to have at least two
20743 * packets inflight to not be caught by delayed ack.
20744 */
20745 pass = 12;
20746 goto send;
20747 }
20748 }
20749 /*
20750 * Sending of standalone window updates.
20751 *
20752 * Window updates are important when we close our window due to a
20753 * full socket buffer and are opening it again after the application
20754 * reads data from it. Once the window has opened again and the
20755 * remote end starts to send again the ACK clock takes over and
20756 * provides the most current window information.
20757 *
20758 * We must avoid the silly window syndrome whereas every read from
20759 * the receive buffer, no matter how small, causes a window update
20760 * to be sent. We also should avoid sending a flurry of window
20761 * updates when the socket buffer had queued a lot of data and the
20762 * application is doing small reads.
20763 *
20764 * Prevent a flurry of pointless window updates by only sending an
20765 * update when we can increase the advertized window by more than
20766 * 1/4th of the socket buffer capacity. When the buffer is getting
20767 * full or is very small be more aggressive and send an update
20768 * whenever we can increase by two mss sized segments. In all other
20769 * situations the ACK's to new incoming data will carry further
20770 * window increases.
20771 *
20772 * Don't send an independent window update if a delayed ACK is
20773 * pending (it will get piggy-backed on it) or the remote side
20774 * already has done a half-close and won't send more data. Skip
20775 * this if the connection is in T/TCP half-open state.
20776 */
20777 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
20778 !(tp->t_flags & TF_DELACK) &&
20779 !TCPS_HAVERCVDFIN(tp->t_state)) {
20780 /*
20781 * "adv" is the amount we could increase the window, taking
20782 * into account that we are limited by TCP_MAXWIN <<
20783 * tp->rcv_scale.
20784 */
20785 int32_t adv;
20786 int oldwin;
20787
20788 adv = recwin;
20789 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
20790 oldwin = (tp->rcv_adv - tp->rcv_nxt);
20791 if (adv > oldwin)
20792 adv -= oldwin;
20793 else {
20794 /* We can't increase the window */
20795 adv = 0;
20796 }
20797 } else
20798 oldwin = 0;
20799
20800 /*
20801 * If the new window size ends up being the same as or less
20802 * than the old size when it is scaled, then don't force
20803 * a window update.
20804 */
20805 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)
20806 goto dontupdate;
20807
20808 if (adv >= (int32_t)(2 * segsiz) &&
20809 (adv >= (int32_t)(so->so_rcv.sb_hiwat / 4) ||
20810 recwin <= (int32_t)(so->so_rcv.sb_hiwat / 8) ||
20811 so->so_rcv.sb_hiwat <= 8 * segsiz)) {
20812 pass = 7;
20813 goto send;
20814 }
20815 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat) {
20816 pass = 23;
20817 goto send;
20818 }
20819 }
20820 dontupdate:
20821
20822 /*
20823 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
20824 * is also a catch-all for the retransmit timer timeout case.
20825 */
20826 if (tp->t_flags & TF_ACKNOW) {
20827 pass = 8;
20828 goto send;
20829 }
20830 if (((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0)) {
20831 pass = 9;
20832 goto send;
20833 }
20834 /*
20835 * If our state indicates that FIN should be sent and we have not
20836 * yet done so, then we need to send.
20837 */
20838 if ((flags & TH_FIN) &&
20839 (tp->snd_max == tp->snd_una)) {
20840 pass = 11;
20841 goto send;
20842 }
20843 /*
20844 * No reason to send a segment, just return.
20845 */
20846 just_return:
20847 SOCK_SENDBUF_UNLOCK(so);
20848 just_return_nolock:
20849 {
20850 int app_limited = CTF_JR_SENT_DATA;
20851
20852 if ((tp->t_flags & TF_FASTOPEN) == 0 &&
20853 (flags & TH_FIN) &&
20854 (len == 0) &&
20855 (sbused(sb) == (tp->snd_max - tp->snd_una)) &&
20856 ((tp->snd_max - tp->snd_una) <= segsiz)) {
20857 /*
20858 * Ok less than or right at a MSS is
20859 * outstanding. The original FreeBSD stack would
20860 * have sent a FIN, which can speed things up for
20861 * a transactional application doing a MSG_WAITALL.
20862 * To speed things up since we do *not* send a FIN
20863 * if data is outstanding, we send a "challenge ack".
20864 * The idea behind that is instead of having to have
20865 * the peer wait for the delayed-ack timer to run off
20866 * we send an ack that makes the peer send us an ack.
20867 */
20868 rack_send_ack_challange(rack);
20869 }
20870 if (tot_len_this_send > 0) {
20871 rack->r_ctl.fsb.recwin = recwin;
20872 pacing_delay = rack_get_pacing_delay(rack, tp, tot_len_this_send, NULL, segsiz, __LINE__);
20873 if ((error == 0) &&
20874 rack_use_rfo &&
20875 ((flags & (TH_SYN|TH_FIN)) == 0) &&
20876 (ipoptlen == 0) &&
20877 rack->r_fsb_inited &&
20878 TCPS_HAVEESTABLISHED(tp->t_state) &&
20879 ((IN_RECOVERY(tp->t_flags)) == 0) &&
20880 (doing_tlp == 0) &&
20881 (rack->r_must_retran == 0) &&
20882 ((tp->t_flags & TF_NEEDFIN) == 0) &&
20883 (len > 0) && (orig_len > 0) &&
20884 (orig_len > len) &&
20885 ((orig_len - len) >= segsiz) &&
20886 ((optlen == 0) ||
20887 ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) {
20888 /* We can send at least one more MSS using our fsb */
20889 rack_setup_fast_output(tp, rack, sb, len, orig_len,
20890 segsiz, pace_max_seg, hw_tls, flags);
20891 } else
20892 rack->r_fast_output = 0;
20893 rack_log_fsb(rack, tp, so, flags,
20894 ipoptlen, orig_len, len, 0,
20895 1, optlen, __LINE__, 1);
20896 /* Assure when we leave that snd_nxt will point to top */
20897 if (SEQ_GT(tp->snd_max, tp->snd_nxt))
20898 tp->snd_nxt = tp->snd_max;
20899 } else {
20900 int end_window = 0;
20901 uint32_t seq = tp->gput_ack;
20902
20903 rsm = tqhash_max(rack->r_ctl.tqh);
20904 if (rsm) {
20905 /*
20906 * Mark the last sent that we just-returned (hinting
20907 * that delayed ack may play a role in any rtt measurement).
20908 */
20909 rsm->r_just_ret = 1;
20910 }
20911 counter_u64_add(rack_out_size[TCP_MSS_ACCT_JUSTRET], 1);
20912 rack->r_ctl.rc_agg_delayed = 0;
20913 rack->r_early = 0;
20914 rack->r_late = 0;
20915 rack->r_ctl.rc_agg_early = 0;
20916 if ((ctf_outstanding(tp) +
20917 min(max(segsiz, (rack->r_ctl.rc_high_rwnd/2)),
20918 minseg)) >= tp->snd_wnd) {
20919 /* We are limited by the rwnd */
20920 app_limited = CTF_JR_RWND_LIMITED;
20921 if (IN_FASTRECOVERY(tp->t_flags))
20922 rack->r_ctl.rc_prr_sndcnt = 0;
20923 } else if (ctf_outstanding(tp) >= sbavail(sb)) {
20924 /* We are limited by whats available -- app limited */
20925 app_limited = CTF_JR_APP_LIMITED;
20926 if (IN_FASTRECOVERY(tp->t_flags))
20927 rack->r_ctl.rc_prr_sndcnt = 0;
20928 } else if ((idle == 0) &&
20929 ((tp->t_flags & TF_NODELAY) == 0) &&
20930 ((uint32_t)len + (uint32_t)sb_offset >= sbavail(sb)) &&
20931 (len < segsiz)) {
20932 /*
20933 * No delay is not on and the
20934 * user is sending less than 1MSS. This
20935 * brings out SWS avoidance so we
20936 * don't send. Another app-limited case.
20937 */
20938 app_limited = CTF_JR_APP_LIMITED;
20939 } else if (tp->t_flags & TF_NOPUSH) {
20940 /*
20941 * The user has requested no push of
20942 * the last segment and we are
20943 * at the last segment. Another app
20944 * limited case.
20945 */
20946 app_limited = CTF_JR_APP_LIMITED;
20947 } else if ((ctf_outstanding(tp) + minseg) > cwnd_to_use) {
20948 /* Its the cwnd */
20949 app_limited = CTF_JR_CWND_LIMITED;
20950 } else if (IN_FASTRECOVERY(tp->t_flags) &&
20951 (rack->rack_no_prr == 0) &&
20952 (rack->r_ctl.rc_prr_sndcnt < segsiz)) {
20953 app_limited = CTF_JR_PRR;
20954 } else {
20955 /* Now why here are we not sending? */
20956 #ifdef NOW
20957 #ifdef INVARIANTS
20958 panic("rack:%p hit JR_ASSESSING case cwnd_to_use:%u?", rack, cwnd_to_use);
20959 #endif
20960 #endif
20961 app_limited = CTF_JR_ASSESSING;
20962 }
20963 /*
20964 * App limited in some fashion, for our pacing GP
20965 * measurements we don't want any gap (even cwnd).
20966 * Close down the measurement window.
20967 */
20968 if (rack_cwnd_block_ends_measure &&
20969 ((app_limited == CTF_JR_CWND_LIMITED) ||
20970 (app_limited == CTF_JR_PRR))) {
20971 /*
20972 * The reason we are not sending is
20973 * the cwnd (or prr). We have been configured
20974 * to end the measurement window in
20975 * this case.
20976 */
20977 end_window = 1;
20978 } else if (rack_rwnd_block_ends_measure &&
20979 (app_limited == CTF_JR_RWND_LIMITED)) {
20980 /*
20981 * We are rwnd limited and have been
20982 * configured to end the measurement
20983 * window in this case.
20984 */
20985 end_window = 1;
20986 } else if (app_limited == CTF_JR_APP_LIMITED) {
20987 /*
20988 * A true application limited period, we have
20989 * ran out of data.
20990 */
20991 end_window = 1;
20992 } else if (app_limited == CTF_JR_ASSESSING) {
20993 /*
20994 * In the assessing case we hit the end of
20995 * the if/else and had no known reason
20996 * This will panic us under invariants..
20997 *
20998 * If we get this out in logs we need to
20999 * investagate which reason we missed.
21000 */
21001 end_window = 1;
21002 }
21003 if (end_window) {
21004 uint8_t log = 0;
21005
21006 /* Adjust the Gput measurement */
21007 if ((tp->t_flags & TF_GPUTINPROG) &&
21008 SEQ_GT(tp->gput_ack, tp->snd_max)) {
21009 tp->gput_ack = tp->snd_max;
21010 if ((tp->gput_ack - tp->gput_seq) < (MIN_GP_WIN * segsiz)) {
21011 /*
21012 * There is not enough to measure.
21013 */
21014 tp->t_flags &= ~TF_GPUTINPROG;
21015 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
21016 rack->r_ctl.rc_gp_srtt /*flex1*/,
21017 tp->gput_seq,
21018 0, 0, 18, __LINE__, NULL, 0);
21019 } else
21020 log = 1;
21021 }
21022 /* Mark the last packet as app limited */
21023 rsm = tqhash_max(rack->r_ctl.tqh);
21024 if (rsm && ((rsm->r_flags & RACK_APP_LIMITED) == 0)) {
21025 if (rack->r_ctl.rc_app_limited_cnt == 0)
21026 rack->r_ctl.rc_end_appl = rack->r_ctl.rc_first_appl = rsm;
21027 else {
21028 /*
21029 * Go out to the end app limited and mark
21030 * this new one as next and move the end_appl up
21031 * to this guy.
21032 */
21033 if (rack->r_ctl.rc_end_appl)
21034 rack->r_ctl.rc_end_appl->r_nseq_appl = rsm->r_start;
21035 rack->r_ctl.rc_end_appl = rsm;
21036 }
21037 rsm->r_flags |= RACK_APP_LIMITED;
21038 rack->r_ctl.rc_app_limited_cnt++;
21039 }
21040 if (log)
21041 rack_log_pacing_delay_calc(rack,
21042 rack->r_ctl.rc_app_limited_cnt, seq,
21043 tp->gput_ack, 0, 0, 4, __LINE__, NULL, 0);
21044 }
21045 }
21046 /* Check if we need to go into persists or not */
21047 if ((tp->snd_max == tp->snd_una) &&
21048 TCPS_HAVEESTABLISHED(tp->t_state) &&
21049 sbavail(sb) &&
21050 (sbavail(sb) > tp->snd_wnd) &&
21051 (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), minseg))) {
21052 /* Yes lets make sure to move to persist before timer-start */
21053 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime, tp->snd_una);
21054 }
21055 rack_start_hpts_timer(rack, tp, cts, pacing_delay, tot_len_this_send, sup_rack);
21056 rack_log_type_just_return(rack, cts, tot_len_this_send, pacing_delay, hpts_calling, app_limited, cwnd_to_use);
21057 }
21058 #ifdef NETFLIX_SHARED_CWND
21059 if ((sbavail(sb) == 0) &&
21060 rack->r_ctl.rc_scw) {
21061 tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
21062 rack->rack_scwnd_is_idle = 1;
21063 }
21064 #endif
21065 #ifdef TCP_ACCOUNTING
21066 if (tot_len_this_send > 0) {
21067 crtsc = get_cyclecount();
21068 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
21069 tp->tcp_cnt_counters[SND_OUT_DATA]++;
21070 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
21071 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len_this_send + segsiz - 1) / segsiz);
21072 }
21073 } else {
21074 crtsc = get_cyclecount();
21075 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
21076 tp->tcp_cnt_counters[SND_LIMITED]++;
21077 tp->tcp_proc_time[SND_LIMITED] += (crtsc - ts_val);
21078 }
21079 }
21080 sched_unpin();
21081 #endif
21082 return (0);
21083
21084 send:
21085 if ((rack->r_ctl.crte != NULL) &&
21086 (rsm == NULL) &&
21087 ((rack->rc_hw_nobuf == 1) ||
21088 (rack_hw_check_queue && (check_done == 0)))) {
21089 /*
21090 * We only want to do this once with the hw_check_queue,
21091 * for the enobuf case we would only do it once if
21092 * we come around to again, the flag will be clear.
21093 */
21094 check_done = 1;
21095 pacing_delay = rack_check_queue_level(rack, tp, &tv, cts, len, segsiz);
21096 if (pacing_delay) {
21097 rack->r_ctl.rc_agg_delayed = 0;
21098 rack->r_ctl.rc_agg_early = 0;
21099 rack->r_early = 0;
21100 rack->r_late = 0;
21101 SOCK_SENDBUF_UNLOCK(so);
21102 goto skip_all_send;
21103 }
21104 }
21105 if (rsm || sack_rxmit)
21106 counter_u64_add(rack_nfto_resend, 1);
21107 else
21108 counter_u64_add(rack_non_fto_send, 1);
21109 if ((flags & TH_FIN) &&
21110 sbavail(sb)) {
21111 /*
21112 * We do not transmit a FIN
21113 * with data outstanding. We
21114 * need to make it so all data
21115 * is acked first.
21116 */
21117 flags &= ~TH_FIN;
21118 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
21119 (sbused(sb) == (tp->snd_max - tp->snd_una)) &&
21120 ((tp->snd_max - tp->snd_una) <= segsiz)) {
21121 /*
21122 * Ok less than or right at a MSS is
21123 * outstanding. The original FreeBSD stack would
21124 * have sent a FIN, which can speed things up for
21125 * a transactional application doing a MSG_WAITALL.
21126 * To speed things up since we do *not* send a FIN
21127 * if data is outstanding, we send a "challenge ack".
21128 * The idea behind that is instead of having to have
21129 * the peer wait for the delayed-ack timer to run off
21130 * we send an ack that makes the peer send us an ack.
21131 */
21132 rack_send_ack_challange(rack);
21133 }
21134 }
21135 /* Enforce stack imposed max seg size if we have one */
21136 if (pace_max_seg &&
21137 (len > pace_max_seg)) {
21138 mark = 1;
21139 len = pace_max_seg;
21140 }
21141 if ((rsm == NULL) &&
21142 (rack->pcm_in_progress == 0) &&
21143 (rack->r_ctl.pcm_max_seg > 0) &&
21144 (len >= rack->r_ctl.pcm_max_seg)) {
21145 /* It is large enough for a measurement */
21146 add_flag |= RACK_IS_PCM;
21147 rack_log_pcm(rack, 5, len, rack->r_ctl.pcm_max_seg, add_flag);
21148 } else if (rack_verbose_logging) {
21149 rack_log_pcm(rack, 6, len, rack->r_ctl.pcm_max_seg, add_flag);
21150 }
21151
21152 SOCKBUF_LOCK_ASSERT(sb);
21153 if (len > 0) {
21154 if (len >= segsiz)
21155 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
21156 else
21157 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
21158 }
21159 /*
21160 * Before ESTABLISHED, force sending of initial options unless TCP
21161 * set not to do any options. NOTE: we assume that the IP/TCP header
21162 * plus TCP options always fit in a single mbuf, leaving room for a
21163 * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr)
21164 * + optlen <= MCLBYTES
21165 */
21166 optlen = 0;
21167 #ifdef INET6
21168 if (isipv6)
21169 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
21170 else
21171 #endif
21172 hdrlen = sizeof(struct tcpiphdr);
21173
21174 /*
21175 * Ok what seq are we sending from. If we have
21176 * no rsm to use, then we look at various bits,
21177 * if we are putting out a SYN it will be ISS.
21178 * If we are retransmitting a FIN it will
21179 * be snd_max-1 else its snd_max.
21180 */
21181 if (rsm == NULL) {
21182 if (flags & TH_SYN)
21183 rack_seq = tp->iss;
21184 else if ((flags & TH_FIN) &&
21185 (tp->t_flags & TF_SENTFIN))
21186 rack_seq = tp->snd_max - 1;
21187 else
21188 rack_seq = tp->snd_max;
21189 } else {
21190 rack_seq = rsm->r_start;
21191 }
21192 /*
21193 * Compute options for segment. We only have to care about SYN and
21194 * established connection segments. Options for SYN-ACK segments
21195 * are handled in TCP syncache.
21196 */
21197 to.to_flags = 0;
21198 if ((tp->t_flags & TF_NOOPT) == 0) {
21199 /* Maximum segment size. */
21200 if (flags & TH_SYN) {
21201 to.to_mss = tcp_mssopt(&inp->inp_inc);
21202 if (tp->t_port)
21203 to.to_mss -= V_tcp_udp_tunneling_overhead;
21204 to.to_flags |= TOF_MSS;
21205
21206 /*
21207 * On SYN or SYN|ACK transmits on TFO connections,
21208 * only include the TFO option if it is not a
21209 * retransmit, as the presence of the TFO option may
21210 * have caused the original SYN or SYN|ACK to have
21211 * been dropped by a middlebox.
21212 */
21213 if ((tp->t_flags & TF_FASTOPEN) &&
21214 (tp->t_rxtshift == 0)) {
21215 if (tp->t_state == TCPS_SYN_RECEIVED) {
21216 to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
21217 to.to_tfo_cookie =
21218 (u_int8_t *)&tp->t_tfo_cookie.server;
21219 to.to_flags |= TOF_FASTOPEN;
21220 wanted_cookie = 1;
21221 } else if (tp->t_state == TCPS_SYN_SENT) {
21222 to.to_tfo_len =
21223 tp->t_tfo_client_cookie_len;
21224 to.to_tfo_cookie =
21225 tp->t_tfo_cookie.client;
21226 to.to_flags |= TOF_FASTOPEN;
21227 wanted_cookie = 1;
21228 /*
21229 * If we wind up having more data to
21230 * send with the SYN than can fit in
21231 * one segment, don't send any more
21232 * until the SYN|ACK comes back from
21233 * the other end.
21234 */
21235 sendalot = 0;
21236 }
21237 }
21238 }
21239 /* Window scaling. */
21240 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
21241 to.to_wscale = tp->request_r_scale;
21242 to.to_flags |= TOF_SCALE;
21243 }
21244 /* Timestamps. */
21245 if ((tp->t_flags & TF_RCVD_TSTMP) ||
21246 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
21247 uint32_t ts_to_use;
21248
21249 if ((rack->r_rcvpath_rtt_up == 1) &&
21250 (ms_cts == rack->r_ctl.last_rcv_tstmp_for_rtt)) {
21251 /*
21252 * When we are doing a rcv_rtt probe all
21253 * other timestamps use the next msec. This
21254 * is safe since our previous ack is in the
21255 * air and we will just have a few more
21256 * on the next ms. This assures that only
21257 * the one ack has the ms_cts that was on
21258 * our ack-probe.
21259 */
21260 ts_to_use = ms_cts + 1;
21261 } else {
21262 ts_to_use = ms_cts;
21263 }
21264 to.to_tsval = ts_to_use + tp->ts_offset;
21265 to.to_tsecr = tp->ts_recent;
21266 to.to_flags |= TOF_TS;
21267 if ((len == 0) &&
21268 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
21269 ((ms_cts - rack->r_ctl.last_rcv_tstmp_for_rtt) > RCV_PATH_RTT_MS) &&
21270 (tp->snd_una == tp->snd_max) &&
21271 (flags & TH_ACK) &&
21272 (sbavail(sb) == 0) &&
21273 (rack->r_ctl.current_round != 0) &&
21274 ((flags & (TH_SYN|TH_FIN)) == 0) &&
21275 (rack->r_rcvpath_rtt_up == 0)) {
21276 rack->r_ctl.last_rcv_tstmp_for_rtt = ms_cts;
21277 rack->r_ctl.last_time_of_arm_rcv = cts;
21278 rack->r_rcvpath_rtt_up = 1;
21279 /* Subtract 1 from seq to force a response */
21280 rack_seq--;
21281 }
21282 }
21283 /* Set receive buffer autosizing timestamp. */
21284 if (tp->rfbuf_ts == 0 &&
21285 (so->so_rcv.sb_flags & SB_AUTOSIZE)) {
21286 tp->rfbuf_ts = ms_cts;
21287 }
21288 /* Selective ACK's. */
21289 if (tp->t_flags & TF_SACK_PERMIT) {
21290 if (flags & TH_SYN)
21291 to.to_flags |= TOF_SACKPERM;
21292 else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
21293 tp->rcv_numsacks > 0) {
21294 to.to_flags |= TOF_SACK;
21295 to.to_nsacks = tp->rcv_numsacks;
21296 to.to_sacks = (u_char *)tp->sackblks;
21297 }
21298 }
21299 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
21300 /* TCP-MD5 (RFC2385). */
21301 if (tp->t_flags & TF_SIGNATURE)
21302 to.to_flags |= TOF_SIGNATURE;
21303 #endif
21304
21305 /* Processing the options. */
21306 hdrlen += optlen = tcp_addoptions(&to, opt);
21307 /*
21308 * If we wanted a TFO option to be added, but it was unable
21309 * to fit, ensure no data is sent.
21310 */
21311 if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie &&
21312 !(to.to_flags & TOF_FASTOPEN))
21313 len = 0;
21314 }
21315 if (tp->t_port) {
21316 if (V_tcp_udp_tunneling_port == 0) {
21317 /* The port was removed?? */
21318 SOCK_SENDBUF_UNLOCK(so);
21319 #ifdef TCP_ACCOUNTING
21320 crtsc = get_cyclecount();
21321 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
21322 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
21323 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
21324 }
21325 sched_unpin();
21326 #endif
21327 return (EHOSTUNREACH);
21328 }
21329 hdrlen += sizeof(struct udphdr);
21330 }
21331 #ifdef INET6
21332 if (isipv6)
21333 ipoptlen = ip6_optlen(inp);
21334 else
21335 #endif
21336 if (inp->inp_options)
21337 ipoptlen = inp->inp_options->m_len -
21338 offsetof(struct ipoption, ipopt_list);
21339 else
21340 ipoptlen = 0;
21341 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
21342 ipoptlen += ipsec_optlen;
21343 #endif
21344
21345 /*
21346 * Adjust data length if insertion of options will bump the packet
21347 * length beyond the t_maxseg length. Clear the FIN bit because we
21348 * cut off the tail of the segment.
21349 */
21350 if (len + optlen + ipoptlen > tp->t_maxseg) {
21351 if (tso) {
21352 uint32_t if_hw_tsomax;
21353 uint32_t moff;
21354 int32_t max_len;
21355
21356 /* extract TSO information */
21357 if_hw_tsomax = tp->t_tsomax;
21358 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
21359 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
21360 KASSERT(ipoptlen == 0,
21361 ("%s: TSO can't do IP options", __func__));
21362
21363 /*
21364 * Check if we should limit by maximum payload
21365 * length:
21366 */
21367 if (if_hw_tsomax != 0) {
21368 /* compute maximum TSO length */
21369 max_len = (if_hw_tsomax - hdrlen -
21370 max_linkhdr);
21371 if (max_len <= 0) {
21372 len = 0;
21373 } else if (len > max_len) {
21374 if (doing_tlp == 0)
21375 sendalot = 1;
21376 len = max_len;
21377 mark = 2;
21378 }
21379 }
21380 /*
21381 * Prevent the last segment from being fractional
21382 * unless the send sockbuf can be emptied:
21383 */
21384 max_len = (tp->t_maxseg - optlen);
21385 if ((sb_offset + len) < sbavail(sb)) {
21386 moff = len % (u_int)max_len;
21387 if (moff != 0) {
21388 mark = 3;
21389 len -= moff;
21390 }
21391 }
21392 /*
21393 * In case there are too many small fragments don't
21394 * use TSO:
21395 */
21396 if (len <= max_len) {
21397 mark = 4;
21398 tso = 0;
21399 }
21400 /*
21401 * Send the FIN in a separate segment after the bulk
21402 * sending is done. We don't trust the TSO
21403 * implementations to clear the FIN flag on all but
21404 * the last segment.
21405 */
21406 if (tp->t_flags & TF_NEEDFIN) {
21407 sendalot = 4;
21408 }
21409 } else {
21410 mark = 5;
21411 if (optlen + ipoptlen >= tp->t_maxseg) {
21412 /*
21413 * Since we don't have enough space to put
21414 * the IP header chain and the TCP header in
21415 * one packet as required by RFC 7112, don't
21416 * send it. Also ensure that at least one
21417 * byte of the payload can be put into the
21418 * TCP segment.
21419 */
21420 SOCK_SENDBUF_UNLOCK(so);
21421 error = EMSGSIZE;
21422 sack_rxmit = 0;
21423 goto out;
21424 }
21425 len = tp->t_maxseg - optlen - ipoptlen;
21426 sendalot = 5;
21427 }
21428 } else {
21429 tso = 0;
21430 mark = 6;
21431 }
21432 KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
21433 ("%s: len > IP_MAXPACKET", __func__));
21434 #ifdef DIAGNOSTIC
21435 #ifdef INET6
21436 if (max_linkhdr + hdrlen > MCLBYTES)
21437 #else
21438 if (max_linkhdr + hdrlen > MHLEN)
21439 #endif
21440 panic("tcphdr too big");
21441 #endif
21442
21443 /*
21444 * This KASSERT is here to catch edge cases at a well defined place.
21445 * Before, those had triggered (random) panic conditions further
21446 * down.
21447 */
21448 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
21449 if ((len == 0) &&
21450 (flags & TH_FIN) &&
21451 (sbused(sb))) {
21452 /*
21453 * We have outstanding data, don't send a fin by itself!.
21454 *
21455 * Check to see if we need to send a challenge ack.
21456 */
21457 if ((sbused(sb) == (tp->snd_max - tp->snd_una)) &&
21458 ((tp->snd_max - tp->snd_una) <= segsiz)) {
21459 /*
21460 * Ok less than or right at a MSS is
21461 * outstanding. The original FreeBSD stack would
21462 * have sent a FIN, which can speed things up for
21463 * a transactional application doing a MSG_WAITALL.
21464 * To speed things up since we do *not* send a FIN
21465 * if data is outstanding, we send a "challenge ack".
21466 * The idea behind that is instead of having to have
21467 * the peer wait for the delayed-ack timer to run off
21468 * we send an ack that makes the peer send us an ack.
21469 */
21470 rack_send_ack_challange(rack);
21471 }
21472 goto just_return;
21473 }
21474 /*
21475 * Grab a header mbuf, attaching a copy of data to be transmitted,
21476 * and initialize the header from the template for sends on this
21477 * connection.
21478 */
21479 hw_tls = tp->t_nic_ktls_xmit != 0;
21480 if (len) {
21481 uint32_t max_val;
21482 uint32_t moff;
21483
21484 if (pace_max_seg)
21485 max_val = pace_max_seg;
21486 else
21487 max_val = len;
21488 /*
21489 * We allow a limit on sending with hptsi.
21490 */
21491 if (len > max_val) {
21492 mark = 7;
21493 len = max_val;
21494 }
21495 #ifdef INET6
21496 if (MHLEN < hdrlen + max_linkhdr)
21497 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
21498 else
21499 #endif
21500 m = m_gethdr(M_NOWAIT, MT_DATA);
21501
21502 if (m == NULL) {
21503 SOCK_SENDBUF_UNLOCK(so);
21504 error = ENOBUFS;
21505 sack_rxmit = 0;
21506 goto out;
21507 }
21508 m->m_data += max_linkhdr;
21509 m->m_len = hdrlen;
21510
21511 /*
21512 * Start the m_copy functions from the closest mbuf to the
21513 * sb_offset in the socket buffer chain.
21514 */
21515 mb = sbsndptr_noadv(sb, sb_offset, &moff);
21516 s_mb = mb;
21517 s_moff = moff;
21518 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
21519 m_copydata(mb, moff, (int)len,
21520 mtod(m, caddr_t)+hdrlen);
21521 /*
21522 * If we are not retransmitting advance the
21523 * sndptr to help remember the next place in
21524 * the sb.
21525 */
21526 if (rsm == NULL)
21527 sbsndptr_adv(sb, mb, len);
21528 m->m_len += len;
21529 } else {
21530 struct sockbuf *msb;
21531
21532 /*
21533 * If we are not retransmitting pass in msb so
21534 * the socket buffer can be advanced. Otherwise
21535 * set it to NULL if its a retransmission since
21536 * we don't want to change the sb remembered
21537 * location.
21538 */
21539 if (rsm == NULL)
21540 msb = sb;
21541 else
21542 msb = NULL;
21543 m->m_next = tcp_m_copym(
21544 mb, moff, &len,
21545 if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, msb,
21546 ((rsm == NULL) ? hw_tls : 0));
21547 if (len <= (tp->t_maxseg - optlen)) {
21548 /*
21549 * Must have ran out of mbufs for the copy
21550 * shorten it to no longer need tso. Lets
21551 * not put on sendalot since we are low on
21552 * mbufs.
21553 */
21554 tso = 0;
21555 }
21556 if (m->m_next == NULL) {
21557 SOCK_SENDBUF_UNLOCK(so);
21558 (void)m_free(m);
21559 error = ENOBUFS;
21560 sack_rxmit = 0;
21561 goto out;
21562 }
21563 }
21564 if (sack_rxmit) {
21565 if (rsm && (rsm->r_flags & RACK_TLP)) {
21566 /*
21567 * TLP should not count in retran count, but
21568 * in its own bin
21569 */
21570 counter_u64_add(rack_tlp_retran, 1);
21571 counter_u64_add(rack_tlp_retran_bytes, len);
21572 } else {
21573 tp->t_sndrexmitpack++;
21574 KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
21575 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
21576 }
21577 #ifdef STATS
21578 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
21579 len);
21580 #endif
21581 } else {
21582 KMOD_TCPSTAT_INC(tcps_sndpack);
21583 KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
21584 #ifdef STATS
21585 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
21586 len);
21587 #endif
21588 }
21589 /*
21590 * If we're sending everything we've got, set PUSH. (This
21591 * will keep happy those implementations which only give
21592 * data to the user when a buffer fills or a PUSH comes in.)
21593 */
21594 if (sb_offset + len == sbused(sb) &&
21595 sbused(sb) &&
21596 !(flags & TH_SYN)) {
21597 flags |= TH_PUSH;
21598 add_flag |= RACK_HAD_PUSH;
21599 }
21600 SOCK_SENDBUF_UNLOCK(so);
21601 } else {
21602 SOCK_SENDBUF_UNLOCK(so);
21603 if (tp->t_flags & TF_ACKNOW)
21604 KMOD_TCPSTAT_INC(tcps_sndacks);
21605 else if (flags & (TH_SYN | TH_FIN | TH_RST))
21606 KMOD_TCPSTAT_INC(tcps_sndctrl);
21607 else
21608 KMOD_TCPSTAT_INC(tcps_sndwinup);
21609
21610 m = m_gethdr(M_NOWAIT, MT_DATA);
21611 if (m == NULL) {
21612 error = ENOBUFS;
21613 sack_rxmit = 0;
21614 goto out;
21615 }
21616 #ifdef INET6
21617 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
21618 MHLEN >= hdrlen) {
21619 M_ALIGN(m, hdrlen);
21620 } else
21621 #endif
21622 m->m_data += max_linkhdr;
21623 m->m_len = hdrlen;
21624 }
21625 SOCK_SENDBUF_UNLOCK_ASSERT(so);
21626 m->m_pkthdr.rcvif = (struct ifnet *)0;
21627 #ifdef MAC
21628 mac_inpcb_create_mbuf(inp, m);
21629 #endif
21630 if ((ipoptlen == 0) && (rack->r_ctl.fsb.tcp_ip_hdr) && rack->r_fsb_inited) {
21631 #ifdef INET6
21632 if (isipv6)
21633 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
21634 else
21635 #endif /* INET6 */
21636 #ifdef INET
21637 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
21638 #endif
21639 th = rack->r_ctl.fsb.th;
21640 udp = rack->r_ctl.fsb.udp;
21641 if (udp) {
21642 #ifdef INET6
21643 if (isipv6)
21644 ulen = hdrlen + len - sizeof(struct ip6_hdr);
21645 else
21646 #endif /* INET6 */
21647 ulen = hdrlen + len - sizeof(struct ip);
21648 udp->uh_ulen = htons(ulen);
21649 }
21650 } else {
21651 #ifdef INET6
21652 if (isipv6) {
21653 ip6 = mtod(m, struct ip6_hdr *);
21654 if (tp->t_port) {
21655 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
21656 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
21657 udp->uh_dport = tp->t_port;
21658 ulen = hdrlen + len - sizeof(struct ip6_hdr);
21659 udp->uh_ulen = htons(ulen);
21660 th = (struct tcphdr *)(udp + 1);
21661 } else
21662 th = (struct tcphdr *)(ip6 + 1);
21663 tcpip_fillheaders(inp, tp->t_port, ip6, th);
21664 } else
21665 #endif /* INET6 */
21666 {
21667 #ifdef INET
21668 ip = mtod(m, struct ip *);
21669 if (tp->t_port) {
21670 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
21671 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
21672 udp->uh_dport = tp->t_port;
21673 ulen = hdrlen + len - sizeof(struct ip);
21674 udp->uh_ulen = htons(ulen);
21675 th = (struct tcphdr *)(udp + 1);
21676 } else
21677 th = (struct tcphdr *)(ip + 1);
21678 tcpip_fillheaders(inp, tp->t_port, ip, th);
21679 #endif
21680 }
21681 }
21682 /*
21683 * If we are starting a connection, send ECN setup SYN packet. If we
21684 * are on a retransmit, we may resend those bits a number of times
21685 * as per RFC 3168.
21686 */
21687 if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn) {
21688 flags |= tcp_ecn_output_syn_sent(tp);
21689 }
21690 /* Also handle parallel SYN for ECN */
21691 if (TCPS_HAVERCVDSYN(tp->t_state) &&
21692 (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) {
21693 int ect = tcp_ecn_output_established(tp, &flags, len, sack_rxmit);
21694 if ((tp->t_state == TCPS_SYN_RECEIVED) &&
21695 (tp->t_flags2 & TF2_ECN_SND_ECE))
21696 tp->t_flags2 &= ~TF2_ECN_SND_ECE;
21697 #ifdef INET6
21698 if (isipv6) {
21699 ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20);
21700 ip6->ip6_flow |= htonl(ect << 20);
21701 }
21702 else
21703 #endif
21704 {
21705 #ifdef INET
21706 ip->ip_tos &= ~IPTOS_ECN_MASK;
21707 ip->ip_tos |= ect;
21708 #endif
21709 }
21710 }
21711 th->th_seq = htonl(rack_seq);
21712 th->th_ack = htonl(tp->rcv_nxt);
21713 tcp_set_flags(th, flags);
21714 /*
21715 * Calculate receive window. Don't shrink window, but avoid silly
21716 * window syndrome.
21717 * If a RST segment is sent, advertise a window of zero.
21718 */
21719 if (flags & TH_RST) {
21720 recwin = 0;
21721 } else {
21722 if (recwin < (long)(so->so_rcv.sb_hiwat / 4) &&
21723 recwin < (long)segsiz) {
21724 recwin = 0;
21725 }
21726 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
21727 recwin < (long)(tp->rcv_adv - tp->rcv_nxt))
21728 recwin = (long)(tp->rcv_adv - tp->rcv_nxt);
21729 }
21730
21731 /*
21732 * According to RFC1323 the window field in a SYN (i.e., a <SYN> or
21733 * <SYN,ACK>) segment itself is never scaled. The <SYN,ACK> case is
21734 * handled in syncache.
21735 */
21736 if (flags & TH_SYN)
21737 th->th_win = htons((u_short)
21738 (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
21739 else {
21740 /* Avoid shrinking window with window scaling. */
21741 recwin = roundup2(recwin, 1 << tp->rcv_scale);
21742 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
21743 }
21744 /*
21745 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
21746 * window. This may cause the remote transmitter to stall. This
21747 * flag tells soreceive() to disable delayed acknowledgements when
21748 * draining the buffer. This can occur if the receiver is
21749 * attempting to read more data than can be buffered prior to
21750 * transmitting on the connection.
21751 */
21752 if (th->th_win == 0) {
21753 tp->t_sndzerowin++;
21754 tp->t_flags |= TF_RXWIN0SENT;
21755 } else
21756 tp->t_flags &= ~TF_RXWIN0SENT;
21757 tp->snd_up = tp->snd_una; /* drag it along, its deprecated */
21758 /* Now are we using fsb?, if so copy the template data to the mbuf */
21759 if ((ipoptlen == 0) && (rack->r_ctl.fsb.tcp_ip_hdr) && rack->r_fsb_inited) {
21760 uint8_t *cpto;
21761
21762 cpto = mtod(m, uint8_t *);
21763 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len);
21764 /*
21765 * We have just copied in:
21766 * IP/IP6
21767 * <optional udphdr>
21768 * tcphdr (no options)
21769 *
21770 * We need to grab the correct pointers into the mbuf
21771 * for both the tcp header, and possibly the udp header (if tunneling).
21772 * We do this by using the offset in the copy buffer and adding it
21773 * to the mbuf base pointer (cpto).
21774 */
21775 #ifdef INET6
21776 if (isipv6)
21777 ip6 = mtod(m, struct ip6_hdr *);
21778 else
21779 #endif /* INET6 */
21780 #ifdef INET
21781 ip = mtod(m, struct ip *);
21782 #endif
21783 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr));
21784 /* If we have a udp header lets set it into the mbuf as well */
21785 if (udp)
21786 udp = (struct udphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.udp - rack->r_ctl.fsb.tcp_ip_hdr));
21787 }
21788 if (optlen) {
21789 bcopy(opt, th + 1, optlen);
21790 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
21791 }
21792 /*
21793 * Put TCP length in extended header, and then checksum extended
21794 * header and data.
21795 */
21796 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
21797 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
21798 if (to.to_flags & TOF_SIGNATURE) {
21799 /*
21800 * Calculate MD5 signature and put it into the place
21801 * determined before.
21802 * NOTE: since TCP options buffer doesn't point into
21803 * mbuf's data, calculate offset and use it.
21804 */
21805 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
21806 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
21807 /*
21808 * Do not send segment if the calculation of MD5
21809 * digest has failed.
21810 */
21811 goto out;
21812 }
21813 }
21814 #endif
21815 #ifdef INET6
21816 if (isipv6) {
21817 /*
21818 * ip6_plen is not need to be filled now, and will be filled
21819 * in ip6_output.
21820 */
21821 if (tp->t_port) {
21822 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
21823 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
21824 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
21825 th->th_sum = htons(0);
21826 UDPSTAT_INC(udps_opackets);
21827 } else {
21828 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
21829 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
21830 th->th_sum = in6_cksum_pseudo(ip6,
21831 sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
21832 0);
21833 }
21834 }
21835 #endif
21836 #if defined(INET6) && defined(INET)
21837 else
21838 #endif
21839 #ifdef INET
21840 {
21841 if (tp->t_port) {
21842 m->m_pkthdr.csum_flags = CSUM_UDP;
21843 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
21844 udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
21845 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
21846 th->th_sum = htons(0);
21847 UDPSTAT_INC(udps_opackets);
21848 } else {
21849 m->m_pkthdr.csum_flags = CSUM_TCP;
21850 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
21851 th->th_sum = in_pseudo(ip->ip_src.s_addr,
21852 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
21853 IPPROTO_TCP + len + optlen));
21854 }
21855 /* IP version must be set here for ipv4/ipv6 checking later */
21856 KASSERT(ip->ip_v == IPVERSION,
21857 ("%s: IP version incorrect: %d", __func__, ip->ip_v));
21858 }
21859 #endif
21860 /*
21861 * Enable TSO and specify the size of the segments. The TCP pseudo
21862 * header checksum is always provided. XXX: Fixme: This is currently
21863 * not the case for IPv6.
21864 */
21865 if (tso) {
21866 /*
21867 * Here we must use t_maxseg and the optlen since
21868 * the optlen may include SACK's (or DSACK).
21869 */
21870 KASSERT(len > tp->t_maxseg - optlen,
21871 ("%s: len <= tso_segsz", __func__));
21872 m->m_pkthdr.csum_flags |= CSUM_TSO;
21873 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
21874 }
21875 KASSERT(len + hdrlen == m_length(m, NULL),
21876 ("%s: mbuf chain different than expected: %d + %u != %u",
21877 __func__, len, hdrlen, m_length(m, NULL)));
21878
21879 #ifdef TCP_HHOOK
21880 /* Run HHOOK_TCP_ESTABLISHED_OUT helper hooks. */
21881 hhook_run_tcp_est_out(tp, th, &to, len, tso);
21882 #endif
21883 if ((rack->r_ctl.crte != NULL) &&
21884 (rack->rc_hw_nobuf == 0) &&
21885 tcp_bblogging_on(tp)) {
21886 rack_log_queue_level(tp, rack, len, &tv, cts);
21887 }
21888 /* We're getting ready to send; log now. */
21889 if (tcp_bblogging_on(rack->rc_tp)) {
21890 union tcp_log_stackspecific log;
21891
21892 memset(&log, 0, sizeof(log));
21893 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_tp);
21894 if (rack->rack_no_prr)
21895 log.u_bbr.flex1 = 0;
21896 else
21897 log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
21898 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs;
21899 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
21900 log.u_bbr.flex4 = orig_len;
21901 /* Save off the early/late values */
21902 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
21903 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed;
21904 log.u_bbr.bw_inuse = rack_get_bw(rack);
21905 log.u_bbr.cur_del_rate = rack->r_ctl.gp_bw;
21906 log.u_bbr.flex8 = 0;
21907 if (rsm) {
21908 if (rsm->r_flags & RACK_RWND_COLLAPSED) {
21909 rack_log_collapse(rack, rsm->r_start, rsm->r_end, 0, __LINE__, 5, rsm->r_flags, rsm);
21910 counter_u64_add(rack_collapsed_win_rxt, 1);
21911 counter_u64_add(rack_collapsed_win_rxt_bytes, (rsm->r_end - rsm->r_start));
21912 }
21913 if (doing_tlp)
21914 log.u_bbr.flex8 = 2;
21915 else
21916 log.u_bbr.flex8 = 1;
21917 } else {
21918 if (doing_tlp)
21919 log.u_bbr.flex8 = 3;
21920 }
21921 log.u_bbr.pacing_gain = rack_get_output_gain(rack, rsm);
21922 log.u_bbr.flex7 = mark;
21923 log.u_bbr.flex7 <<= 8;
21924 log.u_bbr.flex7 |= pass;
21925 log.u_bbr.pkts_out = tp->t_maxseg;
21926 log.u_bbr.timeStamp = cts;
21927 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
21928 if (rsm && (rsm->r_rtr_cnt > 0)) {
21929 /*
21930 * When we have a retransmit we want to log the
21931 * burst at send and flight at send from before.
21932 */
21933 log.u_bbr.flex5 = rsm->r_fas;
21934 log.u_bbr.bbr_substate = rsm->r_bas;
21935 } else {
21936 /*
21937 * New transmits we log in flex5 the inflight again as
21938 * well as the number of segments in our send in the
21939 * substate field.
21940 */
21941 log.u_bbr.flex5 = log.u_bbr.inflight;
21942 log.u_bbr.bbr_substate = (uint8_t)((len + segsiz - 1)/segsiz);
21943 }
21944 log.u_bbr.lt_epoch = cwnd_to_use;
21945 log.u_bbr.delivered = sendalot;
21946 log.u_bbr.rttProp = (uintptr_t)rsm;
21947 log.u_bbr.pkt_epoch = __LINE__;
21948 if (rsm) {
21949 log.u_bbr.delRate = rsm->r_flags;
21950 log.u_bbr.delRate <<= 31;
21951 log.u_bbr.delRate |= rack->r_must_retran;
21952 log.u_bbr.delRate <<= 1;
21953 log.u_bbr.delRate |= (sack_rxmit & 0x00000001);
21954 } else {
21955 log.u_bbr.delRate = rack->r_must_retran;
21956 log.u_bbr.delRate <<= 1;
21957 log.u_bbr.delRate |= (sack_rxmit & 0x00000001);
21958 }
21959 lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
21960 len, &log, false, NULL, __func__, __LINE__, &tv);
21961 } else
21962 lgb = NULL;
21963
21964 /*
21965 * Fill in IP length and desired time to live and send to IP level.
21966 * There should be a better way to handle ttl and tos; we could keep
21967 * them in the template, but need a way to checksum without them.
21968 */
21969 /*
21970 * m->m_pkthdr.len should have been set before cksum calcuration,
21971 * because in6_cksum() need it.
21972 */
21973 #ifdef INET6
21974 if (isipv6) {
21975 /*
21976 * we separately set hoplimit for every segment, since the
21977 * user might want to change the value via setsockopt. Also,
21978 * desired default hop limit might be changed via Neighbor
21979 * Discovery.
21980 */
21981 rack->r_ctl.fsb.hoplimit = ip6->ip6_hlim = in6_selecthlim(inp, NULL);
21982
21983 /*
21984 * Set the packet size here for the benefit of DTrace
21985 * probes. ip6_output() will set it properly; it's supposed
21986 * to include the option header lengths as well.
21987 */
21988 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
21989
21990 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
21991 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
21992 else
21993 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
21994
21995 if (tp->t_state == TCPS_SYN_SENT)
21996 TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
21997
21998 TCP_PROBE5(send, NULL, tp, ip6, tp, th);
21999 /* TODO: IPv6 IP6TOS_ECT bit on */
22000 error = ip6_output(m,
22001 inp->in6p_outputopts,
22002 &inp->inp_route6,
22003 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
22004 NULL, NULL, inp);
22005
22006 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
22007 mtu = inp->inp_route6.ro_nh->nh_mtu;
22008 }
22009 #endif /* INET6 */
22010 #if defined(INET) && defined(INET6)
22011 else
22012 #endif
22013 #ifdef INET
22014 {
22015 ip->ip_len = htons(m->m_pkthdr.len);
22016 #ifdef INET6
22017 if (inp->inp_vflag & INP_IPV6PROTO)
22018 ip->ip_ttl = in6_selecthlim(inp, NULL);
22019 #endif /* INET6 */
22020 rack->r_ctl.fsb.hoplimit = ip->ip_ttl;
22021 /*
22022 * If we do path MTU discovery, then we set DF on every
22023 * packet. This might not be the best thing to do according
22024 * to RFC3390 Section 2. However the tcp hostcache migitates
22025 * the problem so it affects only the first tcp connection
22026 * with a host.
22027 *
22028 * NB: Don't set DF on small MTU/MSS to have a safe
22029 * fallback.
22030 */
22031 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
22032 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
22033 if (tp->t_port == 0 || len < V_tcp_minmss) {
22034 ip->ip_off |= htons(IP_DF);
22035 }
22036 } else {
22037 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
22038 }
22039
22040 if (tp->t_state == TCPS_SYN_SENT)
22041 TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
22042
22043 TCP_PROBE5(send, NULL, tp, ip, tp, th);
22044
22045 error = ip_output(m,
22046 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
22047 inp->inp_options,
22048 #else
22049 NULL,
22050 #endif
22051 &inp->inp_route,
22052 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
22053 inp);
22054 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
22055 mtu = inp->inp_route.ro_nh->nh_mtu;
22056 }
22057 #endif /* INET */
22058 if (lgb) {
22059 lgb->tlb_errno = error;
22060 lgb = NULL;
22061 }
22062
22063 out:
22064 /*
22065 * In transmit state, time the transmission and arrange for the
22066 * retransmit. In persist state, just set snd_max.
22067 */
22068 if ((rsm == NULL) && doing_tlp)
22069 add_flag |= RACK_TLP;
22070 rack_log_output(tp, &to, len, rack_seq, (uint8_t) flags, error,
22071 rack_to_usec_ts(&tv),
22072 rsm, add_flag, s_mb, s_moff, hw_tls, segsiz);
22073 if (error == 0) {
22074 if (add_flag & RACK_IS_PCM) {
22075 /* We just launched a PCM */
22076 /* rrs here log */
22077 rack->pcm_in_progress = 1;
22078 rack->pcm_needed = 0;
22079 rack_log_pcm(rack, 7, len, rack->r_ctl.pcm_max_seg, add_flag);
22080 }
22081 if (rsm == NULL) {
22082 if (rack->lt_bw_up == 0) {
22083 rack->r_ctl.lt_timemark = tcp_tv_to_lusec(&tv);
22084 rack->r_ctl.lt_seq = tp->snd_una;
22085 rack->lt_bw_up = 1;
22086 } else if (((rack_seq + len) - rack->r_ctl.lt_seq) > 0x7fffffff) {
22087 /*
22088 * Need to record what we have since we are
22089 * approaching seq wrap.
22090 */
22091 uint64_t tmark;
22092
22093 rack->r_ctl.lt_bw_bytes += (tp->snd_una - rack->r_ctl.lt_seq);
22094 rack->r_ctl.lt_seq = tp->snd_una;
22095 tmark = tcp_get_u64_usecs(&tv);
22096 if (tmark > rack->r_ctl.lt_timemark) {
22097 rack->r_ctl.lt_bw_time += (tmark - rack->r_ctl.lt_timemark);
22098 rack->r_ctl.lt_timemark = tmark;
22099 }
22100 }
22101 }
22102 rack->forced_ack = 0; /* If we send something zap the FA flag */
22103 counter_u64_add(rack_total_bytes, len);
22104 tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls);
22105 if (rsm && doing_tlp) {
22106 rack->rc_last_sent_tlp_past_cumack = 0;
22107 rack->rc_last_sent_tlp_seq_valid = 1;
22108 rack->r_ctl.last_sent_tlp_seq = rsm->r_start;
22109 rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start;
22110 }
22111 if (rack->rc_hw_nobuf) {
22112 rack->rc_hw_nobuf = 0;
22113 rack->r_ctl.rc_agg_delayed = 0;
22114 rack->r_early = 0;
22115 rack->r_late = 0;
22116 rack->r_ctl.rc_agg_early = 0;
22117 }
22118 if (rsm && (doing_tlp == 0)) {
22119 /* Set we retransmitted */
22120 rack->rc_gp_saw_rec = 1;
22121 } else {
22122 if (cwnd_to_use > tp->snd_ssthresh) {
22123 /* Set we sent in CA */
22124 rack->rc_gp_saw_ca = 1;
22125 } else {
22126 /* Set we sent in SS */
22127 rack->rc_gp_saw_ss = 1;
22128 }
22129 }
22130 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
22131 (tp->t_flags & TF_SACK_PERMIT) &&
22132 tp->rcv_numsacks > 0)
22133 tcp_clean_dsack_blocks(tp);
22134 tot_len_this_send += len;
22135 if (len == 0) {
22136 counter_u64_add(rack_out_size[TCP_MSS_ACCT_SNDACK], 1);
22137 } else {
22138 int idx;
22139
22140 idx = (len / segsiz) + 3;
22141 if (idx >= TCP_MSS_ACCT_ATIMER)
22142 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1);
22143 else
22144 counter_u64_add(rack_out_size[idx], 1);
22145 }
22146 }
22147 if ((rack->rack_no_prr == 0) &&
22148 sub_from_prr &&
22149 (error == 0)) {
22150 if (rack->r_ctl.rc_prr_sndcnt >= len)
22151 rack->r_ctl.rc_prr_sndcnt -= len;
22152 else
22153 rack->r_ctl.rc_prr_sndcnt = 0;
22154 }
22155 sub_from_prr = 0;
22156 if (rsm != NULL) {
22157 if (doing_tlp)
22158 /* Make sure the TLP is added */
22159 rsm->r_flags |= RACK_TLP;
22160 else
22161 /* If its a resend without TLP then it must not have the flag */
22162 rsm->r_flags &= ~RACK_TLP;
22163 }
22164 if ((error == 0) &&
22165 (len > 0) &&
22166 (tp->snd_una == tp->snd_max))
22167 rack->r_ctl.rc_tlp_rxt_last_time = cts;
22168
22169 {
22170 /*
22171 * This block is not associated with the above error == 0 test.
22172 * It is used to advance snd_max if we have a new transmit.
22173 */
22174 tcp_seq startseq = tp->snd_max;
22175
22176
22177 if (rsm && (doing_tlp == 0))
22178 rack->r_ctl.rc_loss_count += rsm->r_end - rsm->r_start;
22179 if (error)
22180 /* We don't log or do anything with errors */
22181 goto nomore;
22182 if (doing_tlp == 0) {
22183 if (rsm == NULL) {
22184 /*
22185 * Not a retransmission of some
22186 * sort, new data is going out so
22187 * clear our TLP count and flag.
22188 */
22189 rack->rc_tlp_in_progress = 0;
22190 rack->r_ctl.rc_tlp_cnt_out = 0;
22191 }
22192 } else {
22193 /*
22194 * We have just sent a TLP, mark that it is true
22195 * and make sure our in progress is set so we
22196 * continue to check the count.
22197 */
22198 rack->rc_tlp_in_progress = 1;
22199 rack->r_ctl.rc_tlp_cnt_out++;
22200 }
22201 /*
22202 * If we are retransmitting we are done, snd_max
22203 * does not get updated.
22204 */
22205 if (sack_rxmit)
22206 goto nomore;
22207 if ((tp->snd_una == tp->snd_max) && (len > 0)) {
22208 /*
22209 * Update the time we just added data since
22210 * nothing was outstanding.
22211 */
22212 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__);
22213 tp->t_acktime = ticks;
22214 }
22215 /*
22216 * Now for special SYN/FIN handling.
22217 */
22218 if (flags & (TH_SYN | TH_FIN)) {
22219 if ((flags & TH_SYN) &&
22220 ((tp->t_flags & TF_SENTSYN) == 0)) {
22221 tp->snd_max++;
22222 tp->t_flags |= TF_SENTSYN;
22223 }
22224 if ((flags & TH_FIN) &&
22225 ((tp->t_flags & TF_SENTFIN) == 0)) {
22226 tp->snd_max++;
22227 tp->t_flags |= TF_SENTFIN;
22228 }
22229 }
22230 tp->snd_max += len;
22231 if (rack->rc_new_rnd_needed) {
22232 rack_new_round_starts(tp, rack, tp->snd_max);
22233 }
22234 /*
22235 * Time this transmission if not a retransmission and
22236 * not currently timing anything.
22237 * This is only relevant in case of switching back to
22238 * the base stack.
22239 */
22240 if (tp->t_rtttime == 0) {
22241 tp->t_rtttime = ticks;
22242 tp->t_rtseq = startseq;
22243 KMOD_TCPSTAT_INC(tcps_segstimed);
22244 }
22245 if (len &&
22246 ((tp->t_flags & TF_GPUTINPROG) == 0))
22247 rack_start_gp_measurement(tp, rack, startseq, sb_offset);
22248 /*
22249 * If we are doing FO we need to update the mbuf position and subtract
22250 * this happens when the peer sends us duplicate information and
22251 * we thus want to send a DSACK.
22252 *
22253 * XXXRRS: This brings to mind a ?, when we send a DSACK block is TSO
22254 * turned off? If not then we are going to echo multiple DSACK blocks
22255 * out (with the TSO), which we should not be doing.
22256 */
22257 if (rack->r_fast_output && len) {
22258 if (rack->r_ctl.fsb.left_to_send > len)
22259 rack->r_ctl.fsb.left_to_send -= len;
22260 else
22261 rack->r_ctl.fsb.left_to_send = 0;
22262 if (rack->r_ctl.fsb.left_to_send < segsiz)
22263 rack->r_fast_output = 0;
22264 if (rack->r_fast_output) {
22265 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off);
22266 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len;
22267 rack->r_ctl.fsb.o_t_len = M_TRAILINGROOM(rack->r_ctl.fsb.m);
22268 }
22269 }
22270 if (rack_pcm_blast == 0) {
22271 if ((orig_len > len) &&
22272 (add_flag & RACK_IS_PCM) &&
22273 (len < pace_max_seg) &&
22274 ((pace_max_seg - len) > segsiz)) {
22275 /*
22276 * We are doing a PCM measurement and we did
22277 * not get enough data in the TSO to meet the
22278 * burst requirement.
22279 */
22280 uint32_t n_len;
22281
22282 n_len = (orig_len - len);
22283 orig_len -= len;
22284 pace_max_seg -= len;
22285 len = n_len;
22286 sb_offset = tp->snd_max - tp->snd_una;
22287 /* Re-lock for the next spin */
22288 SOCK_SENDBUF_LOCK(so);
22289 goto send;
22290 }
22291 } else {
22292 if ((orig_len > len) &&
22293 (add_flag & RACK_IS_PCM) &&
22294 ((orig_len - len) > segsiz)) {
22295 /*
22296 * We are doing a PCM measurement and we did
22297 * not get enough data in the TSO to meet the
22298 * burst requirement.
22299 */
22300 uint32_t n_len;
22301
22302 n_len = (orig_len - len);
22303 orig_len -= len;
22304 len = n_len;
22305 sb_offset = tp->snd_max - tp->snd_una;
22306 /* Re-lock for the next spin */
22307 SOCK_SENDBUF_LOCK(so);
22308 goto send;
22309 }
22310 }
22311 }
22312 nomore:
22313 if (error) {
22314 rack->r_ctl.rc_agg_delayed = 0;
22315 rack->r_early = 0;
22316 rack->r_late = 0;
22317 rack->r_ctl.rc_agg_early = 0;
22318 SOCKBUF_UNLOCK_ASSERT(sb); /* Check gotos. */
22319 /*
22320 * Failures do not advance the seq counter above. For the
22321 * case of ENOBUFS we will fall out and retry in 1ms with
22322 * the hpts. Everything else will just have to retransmit
22323 * with the timer.
22324 *
22325 * In any case, we do not want to loop around for another
22326 * send without a good reason.
22327 */
22328 sendalot = 0;
22329 switch (error) {
22330 case EPERM:
22331 case EACCES:
22332 tp->t_softerror = error;
22333 #ifdef TCP_ACCOUNTING
22334 crtsc = get_cyclecount();
22335 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
22336 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
22337 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
22338 }
22339 sched_unpin();
22340 #endif
22341 return (error);
22342 case ENOBUFS:
22343 /*
22344 * Pace us right away to retry in a some
22345 * time
22346 */
22347 if (rack->r_ctl.crte != NULL) {
22348 tcp_trace_point(rack->rc_tp, TCP_TP_HWENOBUF);
22349 if (tcp_bblogging_on(rack->rc_tp))
22350 rack_log_queue_level(tp, rack, len, &tv, cts);
22351 } else
22352 tcp_trace_point(rack->rc_tp, TCP_TP_ENOBUF);
22353 pacing_delay = ((1 + rack->rc_enobuf) * HPTS_USEC_IN_MSEC);
22354 if (rack->rc_enobuf < 0x7f)
22355 rack->rc_enobuf++;
22356 if (pacing_delay < (10 * HPTS_USEC_IN_MSEC))
22357 pacing_delay = 10 * HPTS_USEC_IN_MSEC;
22358 if (rack->r_ctl.crte != NULL) {
22359 counter_u64_add(rack_saw_enobuf_hw, 1);
22360 tcp_rl_log_enobuf(rack->r_ctl.crte);
22361 }
22362 counter_u64_add(rack_saw_enobuf, 1);
22363 goto enobufs;
22364 case EMSGSIZE:
22365 /*
22366 * For some reason the interface we used initially
22367 * to send segments changed to another or lowered
22368 * its MTU. If TSO was active we either got an
22369 * interface without TSO capabilits or TSO was
22370 * turned off. If we obtained mtu from ip_output()
22371 * then update it and try again.
22372 */
22373 if (tso)
22374 tp->t_flags &= ~TF_TSO;
22375 if (mtu != 0) {
22376 int saved_mtu;
22377
22378 saved_mtu = tp->t_maxseg;
22379 tcp_mss_update(tp, -1, mtu, NULL, NULL);
22380 if (saved_mtu > tp->t_maxseg) {
22381 goto again;
22382 }
22383 }
22384 pacing_delay = 10 * HPTS_USEC_IN_MSEC;
22385 rack_start_hpts_timer(rack, tp, cts, pacing_delay, 0, 0);
22386 #ifdef TCP_ACCOUNTING
22387 crtsc = get_cyclecount();
22388 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
22389 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
22390 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
22391 }
22392 sched_unpin();
22393 #endif
22394 return (error);
22395 case ENETUNREACH:
22396 counter_u64_add(rack_saw_enetunreach, 1);
22397 /* FALLTHROUGH */
22398 case EHOSTDOWN:
22399 case EHOSTUNREACH:
22400 case ENETDOWN:
22401 if (TCPS_HAVERCVDSYN(tp->t_state)) {
22402 tp->t_softerror = error;
22403 error = 0;
22404 }
22405 /* FALLTHROUGH */
22406 default:
22407 pacing_delay = 10 * HPTS_USEC_IN_MSEC;
22408 rack_start_hpts_timer(rack, tp, cts, pacing_delay, 0, 0);
22409 #ifdef TCP_ACCOUNTING
22410 crtsc = get_cyclecount();
22411 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
22412 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
22413 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
22414 }
22415 sched_unpin();
22416 #endif
22417 return (error);
22418 }
22419 } else {
22420 rack->rc_enobuf = 0;
22421 if (IN_FASTRECOVERY(tp->t_flags) && rsm)
22422 rack->r_ctl.retran_during_recovery += len;
22423 }
22424 KMOD_TCPSTAT_INC(tcps_sndtotal);
22425
22426 /*
22427 * Data sent (as far as we can tell). If this advertises a larger
22428 * window than any other segment, then remember the size of the
22429 * advertised window. Any pending ACK has now been sent.
22430 */
22431 if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
22432 tp->rcv_adv = tp->rcv_nxt + recwin;
22433
22434 tp->last_ack_sent = tp->rcv_nxt;
22435 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
22436 enobufs:
22437 if (sendalot) {
22438 /* Do we need to turn off sendalot? */
22439 if (pace_max_seg &&
22440 (tot_len_this_send >= pace_max_seg)) {
22441 /* We hit our max. */
22442 sendalot = 0;
22443 }
22444 }
22445 if ((error == 0) && (flags & TH_FIN))
22446 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_FIN);
22447 if (flags & TH_RST) {
22448 /*
22449 * We don't send again after sending a RST.
22450 */
22451 pacing_delay = 0;
22452 sendalot = 0;
22453 if (error == 0)
22454 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
22455 } else if ((pacing_delay == 0) && (sendalot == 0) && tot_len_this_send) {
22456 /*
22457 * Get our pacing rate, if an error
22458 * occurred in sending (ENOBUF) we would
22459 * hit the else if with slot preset. Other
22460 * errors return.
22461 */
22462 pacing_delay = rack_get_pacing_delay(rack, tp, tot_len_this_send, rsm, segsiz, __LINE__);
22463 }
22464 /* We have sent clear the flag */
22465 rack->r_ent_rec_ns = 0;
22466 if (rack->r_must_retran) {
22467 if (rsm) {
22468 rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start);
22469 if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) {
22470 /*
22471 * We have retransmitted all.
22472 */
22473 rack->r_must_retran = 0;
22474 rack->r_ctl.rc_out_at_rto = 0;
22475 }
22476 } else if (SEQ_GEQ(tp->snd_max, rack->r_ctl.rc_snd_max_at_rto)) {
22477 /*
22478 * Sending new data will also kill
22479 * the loop.
22480 */
22481 rack->r_must_retran = 0;
22482 rack->r_ctl.rc_out_at_rto = 0;
22483 }
22484 }
22485 rack->r_ctl.fsb.recwin = recwin;
22486 if ((tp->t_flags & (TF_WASCRECOVERY|TF_WASFRECOVERY)) &&
22487 SEQ_GT(tp->snd_max, rack->r_ctl.rc_snd_max_at_rto)) {
22488 /*
22489 * We hit an RTO and now have past snd_max at the RTO
22490 * clear all the WAS flags.
22491 */
22492 tp->t_flags &= ~(TF_WASCRECOVERY|TF_WASFRECOVERY);
22493 }
22494 if (pacing_delay) {
22495 /* set the rack tcb into the slot N */
22496 if ((error == 0) &&
22497 rack_use_rfo &&
22498 ((flags & (TH_SYN|TH_FIN)) == 0) &&
22499 (rsm == NULL) &&
22500 (ipoptlen == 0) &&
22501 (doing_tlp == 0) &&
22502 rack->r_fsb_inited &&
22503 TCPS_HAVEESTABLISHED(tp->t_state) &&
22504 ((IN_RECOVERY(tp->t_flags)) == 0) &&
22505 (rack->r_must_retran == 0) &&
22506 ((tp->t_flags & TF_NEEDFIN) == 0) &&
22507 (len > 0) && (orig_len > 0) &&
22508 (orig_len > len) &&
22509 ((orig_len - len) >= segsiz) &&
22510 ((optlen == 0) ||
22511 ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) {
22512 /* We can send at least one more MSS using our fsb */
22513 rack_setup_fast_output(tp, rack, sb, len, orig_len,
22514 segsiz, pace_max_seg, hw_tls, flags);
22515 } else
22516 rack->r_fast_output = 0;
22517 rack_log_fsb(rack, tp, so, flags,
22518 ipoptlen, orig_len, len, error,
22519 (rsm == NULL), optlen, __LINE__, 2);
22520 } else if (sendalot) {
22521 int ret;
22522
22523 sack_rxmit = 0;
22524 if ((error == 0) &&
22525 rack_use_rfo &&
22526 ((flags & (TH_SYN|TH_FIN)) == 0) &&
22527 (rsm == NULL) &&
22528 (doing_tlp == 0) &&
22529 (ipoptlen == 0) &&
22530 (rack->r_must_retran == 0) &&
22531 rack->r_fsb_inited &&
22532 TCPS_HAVEESTABLISHED(tp->t_state) &&
22533 ((IN_RECOVERY(tp->t_flags)) == 0) &&
22534 ((tp->t_flags & TF_NEEDFIN) == 0) &&
22535 (len > 0) && (orig_len > 0) &&
22536 (orig_len > len) &&
22537 ((orig_len - len) >= segsiz) &&
22538 ((optlen == 0) ||
22539 ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) {
22540 /* we can use fast_output for more */
22541 rack_setup_fast_output(tp, rack, sb, len, orig_len,
22542 segsiz, pace_max_seg, hw_tls, flags);
22543 if (rack->r_fast_output) {
22544 error = 0;
22545 ret = rack_fast_output(tp, rack, ts_val, cts, ms_cts, &tv, &tot_len_this_send, &error, __LINE__);
22546 if (ret >= 0)
22547 return (ret);
22548 else if (error)
22549 goto nomore;
22550
22551 }
22552 }
22553 goto again;
22554 }
22555 skip_all_send:
22556 /* Assure when we leave that snd_nxt will point to top */
22557 if (SEQ_GT(tp->snd_max, tp->snd_nxt))
22558 tp->snd_nxt = tp->snd_max;
22559 rack_start_hpts_timer(rack, tp, cts, pacing_delay, tot_len_this_send, 0);
22560 #ifdef TCP_ACCOUNTING
22561 crtsc = get_cyclecount() - ts_val;
22562 if (tot_len_this_send) {
22563 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
22564 tp->tcp_cnt_counters[SND_OUT_DATA]++;
22565 tp->tcp_proc_time[SND_OUT_DATA] += crtsc;
22566 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len_this_send + segsiz - 1) /segsiz);
22567 }
22568 } else {
22569 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
22570 tp->tcp_cnt_counters[SND_OUT_ACK]++;
22571 tp->tcp_proc_time[SND_OUT_ACK] += crtsc;
22572 }
22573 }
22574 sched_unpin();
22575 #endif
22576 if (error == ENOBUFS)
22577 error = 0;
22578 return (error);
22579 }
22580
22581 static void
rack_update_seg(struct tcp_rack * rack)22582 rack_update_seg(struct tcp_rack *rack)
22583 {
22584 uint32_t orig_val;
22585
22586 orig_val = rack->r_ctl.rc_pace_max_segs;
22587 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
22588 if (orig_val != rack->r_ctl.rc_pace_max_segs)
22589 rack_log_pacing_delay_calc(rack, 0, 0, orig_val, 0, 0, 15, __LINE__, NULL, 0);
22590 }
22591
22592 static void
rack_mtu_change(struct tcpcb * tp)22593 rack_mtu_change(struct tcpcb *tp)
22594 {
22595 /*
22596 * The MSS may have changed
22597 */
22598 struct tcp_rack *rack;
22599 struct rack_sendmap *rsm;
22600
22601 rack = (struct tcp_rack *)tp->t_fb_ptr;
22602 if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) {
22603 /*
22604 * The MTU has changed we need to resend everything
22605 * since all we have sent is lost. We first fix
22606 * up the mtu though.
22607 */
22608 rack_set_pace_segments(tp, rack, __LINE__, NULL);
22609 /* We treat this like a full retransmit timeout without the cwnd adjustment */
22610 rack_remxt_tmr(tp);
22611 rack->r_fast_output = 0;
22612 rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp,
22613 rack->r_ctl.rc_sacked);
22614 rack->r_ctl.rc_snd_max_at_rto = tp->snd_max;
22615 rack->r_must_retran = 1;
22616 /* Mark all inflight to needing to be rxt'd */
22617 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
22618 rsm->r_flags |= (RACK_MUST_RXT|RACK_PMTU_CHG);
22619 }
22620 }
22621 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
22622 /* We don't use snd_nxt to retransmit */
22623 tp->snd_nxt = tp->snd_max;
22624 }
22625
22626 static int
rack_set_dgp(struct tcp_rack * rack)22627 rack_set_dgp(struct tcp_rack *rack)
22628 {
22629 if (rack->dgp_on == 1)
22630 return(0);
22631 if ((rack->use_fixed_rate == 1) &&
22632 (rack->rc_always_pace == 1)) {
22633 /*
22634 * We are already pacing another
22635 * way.
22636 */
22637 return (EBUSY);
22638 }
22639 if (rack->rc_always_pace == 1) {
22640 rack_remove_pacing(rack);
22641 }
22642 if (tcp_incr_dgp_pacing_cnt() == 0)
22643 return (ENOSPC);
22644 rack->r_ctl.pacing_method |= RACK_DGP_PACING;
22645 rack->rc_fillcw_apply_discount = 0;
22646 rack->dgp_on = 1;
22647 rack->rc_always_pace = 1;
22648 rack->rc_pace_dnd = 1;
22649 rack->use_fixed_rate = 0;
22650 if (rack->gp_ready)
22651 rack_set_cc_pacing(rack);
22652 rack->rc_tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
22653 rack->rack_attempt_hdwr_pace = 0;
22654 /* rxt settings */
22655 rack->full_size_rxt = 1;
22656 rack->shape_rxt_to_pacing_min = 0;
22657 /* cmpack=1 */
22658 rack->r_use_cmp_ack = 1;
22659 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state) &&
22660 rack->r_use_cmp_ack)
22661 rack->rc_tp->t_flags2 |= TF2_MBUF_ACKCMP;
22662 /* scwnd=1 */
22663 rack->rack_enable_scwnd = 1;
22664 /* dynamic=100 */
22665 rack->rc_gp_dyn_mul = 1;
22666 /* gp_inc_ca */
22667 rack->r_ctl.rack_per_of_gp_ca = 100;
22668 /* rrr_conf=3 */
22669 rack->r_rr_config = 3;
22670 /* npush=2 */
22671 rack->r_ctl.rc_no_push_at_mrtt = 2;
22672 /* fillcw=1 */
22673 rack->rc_pace_to_cwnd = 1;
22674 rack->rc_pace_fill_if_rttin_range = 0;
22675 rack->rtt_limit_mul = 0;
22676 /* noprr=1 */
22677 rack->rack_no_prr = 1;
22678 /* lscwnd=1 */
22679 rack->r_limit_scw = 1;
22680 /* gp_inc_rec */
22681 rack->r_ctl.rack_per_of_gp_rec = 90;
22682 return (0);
22683 }
22684
22685 static int
rack_set_profile(struct tcp_rack * rack,int prof)22686 rack_set_profile(struct tcp_rack *rack, int prof)
22687 {
22688 int err = EINVAL;
22689 if (prof == 1) {
22690 /*
22691 * Profile 1 is "standard" DGP. It ignores
22692 * client buffer level.
22693 */
22694 err = rack_set_dgp(rack);
22695 if (err)
22696 return (err);
22697 } else if (prof == 6) {
22698 err = rack_set_dgp(rack);
22699 if (err)
22700 return (err);
22701 /*
22702 * Profile 6 tweaks DGP so that it will apply to
22703 * fill-cw the same settings that profile5 does
22704 * to replace DGP. It gets then the max(dgp-rate, fillcw(discounted).
22705 */
22706 rack->rc_fillcw_apply_discount = 1;
22707 } else if (prof == 0) {
22708 /* This changes things back to the default settings */
22709 if (rack->rc_always_pace == 1) {
22710 rack_remove_pacing(rack);
22711 } else {
22712 /* Make sure any stray flags are off */
22713 rack->dgp_on = 0;
22714 rack->rc_hybrid_mode = 0;
22715 rack->use_fixed_rate = 0;
22716 }
22717 err = 0;
22718 if (rack_fill_cw_state)
22719 rack->rc_pace_to_cwnd = 1;
22720 else
22721 rack->rc_pace_to_cwnd = 0;
22722
22723 if (rack_pace_every_seg && tcp_can_enable_pacing()) {
22724 rack->r_ctl.pacing_method |= RACK_REG_PACING;
22725 rack->rc_always_pace = 1;
22726 if (rack->rack_hibeta)
22727 rack_set_cc_pacing(rack);
22728 } else
22729 rack->rc_always_pace = 0;
22730 if (rack_dsack_std_based & 0x1) {
22731 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */
22732 rack->rc_rack_tmr_std_based = 1;
22733 }
22734 if (rack_dsack_std_based & 0x2) {
22735 /* Basically this means rack timers are extended based on dsack by up to (2 * srtt) */
22736 rack->rc_rack_use_dsack = 1;
22737 }
22738 if (rack_use_cmp_acks)
22739 rack->r_use_cmp_ack = 1;
22740 else
22741 rack->r_use_cmp_ack = 0;
22742 if (rack_disable_prr)
22743 rack->rack_no_prr = 1;
22744 else
22745 rack->rack_no_prr = 0;
22746 if (rack_gp_no_rec_chg)
22747 rack->rc_gp_no_rec_chg = 1;
22748 else
22749 rack->rc_gp_no_rec_chg = 0;
22750 if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack) {
22751 rack->r_mbuf_queue = 1;
22752 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state))
22753 rack->rc_tp->t_flags2 |= TF2_MBUF_ACKCMP;
22754 rack->rc_tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
22755 } else {
22756 rack->r_mbuf_queue = 0;
22757 rack->rc_tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
22758 }
22759 if (rack_enable_shared_cwnd)
22760 rack->rack_enable_scwnd = 1;
22761 else
22762 rack->rack_enable_scwnd = 0;
22763 if (rack_do_dyn_mul) {
22764 /* When dynamic adjustment is on CA needs to start at 100% */
22765 rack->rc_gp_dyn_mul = 1;
22766 if (rack_do_dyn_mul >= 100)
22767 rack->r_ctl.rack_per_of_gp_ca = rack_do_dyn_mul;
22768 } else {
22769 rack->r_ctl.rack_per_of_gp_ca = rack_per_of_gp_ca;
22770 rack->rc_gp_dyn_mul = 0;
22771 }
22772 rack->r_rr_config = 0;
22773 rack->r_ctl.rc_no_push_at_mrtt = 0;
22774 rack->rc_pace_fill_if_rttin_range = 0;
22775 rack->rtt_limit_mul = 0;
22776
22777 if (rack_enable_hw_pacing)
22778 rack->rack_hdw_pace_ena = 1;
22779 else
22780 rack->rack_hdw_pace_ena = 0;
22781 if (rack_disable_prr)
22782 rack->rack_no_prr = 1;
22783 else
22784 rack->rack_no_prr = 0;
22785 if (rack_limits_scwnd)
22786 rack->r_limit_scw = 1;
22787 else
22788 rack->r_limit_scw = 0;
22789 rack_init_retransmit_value(rack, rack_rxt_controls);
22790 err = 0;
22791 }
22792 return (err);
22793 }
22794
22795 static int
rack_add_deferred_option(struct tcp_rack * rack,int sopt_name,uint64_t loptval)22796 rack_add_deferred_option(struct tcp_rack *rack, int sopt_name, uint64_t loptval)
22797 {
22798 struct deferred_opt_list *dol;
22799
22800 dol = malloc(sizeof(struct deferred_opt_list),
22801 M_TCPDO, M_NOWAIT|M_ZERO);
22802 if (dol == NULL) {
22803 /*
22804 * No space yikes -- fail out..
22805 */
22806 return (0);
22807 }
22808 dol->optname = sopt_name;
22809 dol->optval = loptval;
22810 TAILQ_INSERT_TAIL(&rack->r_ctl.opt_list, dol, next);
22811 return (1);
22812 }
22813
22814 static int
process_hybrid_pacing(struct tcp_rack * rack,struct tcp_hybrid_req * hybrid)22815 process_hybrid_pacing(struct tcp_rack *rack, struct tcp_hybrid_req *hybrid)
22816 {
22817 #ifdef TCP_REQUEST_TRK
22818 struct tcp_sendfile_track *sft;
22819 struct timeval tv;
22820 tcp_seq seq;
22821 int err;
22822
22823 microuptime(&tv);
22824
22825 /* Make sure no fixed rate is on */
22826 rack->use_fixed_rate = 0;
22827 rack->r_ctl.rc_fixed_pacing_rate_rec = 0;
22828 rack->r_ctl.rc_fixed_pacing_rate_ca = 0;
22829 rack->r_ctl.rc_fixed_pacing_rate_ss = 0;
22830 /* Now allocate or find our entry that will have these settings */
22831 sft = tcp_req_alloc_req_full(rack->rc_tp, &hybrid->req, tcp_tv_to_lusec(&tv), 0);
22832 if (sft == NULL) {
22833 rack->rc_tp->tcp_hybrid_error++;
22834 /* no space, where would it have gone? */
22835 seq = rack->rc_tp->snd_una + rack->rc_tp->t_inpcb.inp_socket->so_snd.sb_ccc;
22836 rack_log_hybrid(rack, seq, NULL, HYBRID_LOG_NO_ROOM, __LINE__, 0);
22837 return (ENOSPC);
22838 }
22839 /* mask our internal flags */
22840 hybrid->hybrid_flags &= TCP_HYBRID_PACING_USER_MASK;
22841 /* The seq will be snd_una + everything in the buffer */
22842 seq = sft->start_seq;
22843 if ((hybrid->hybrid_flags & TCP_HYBRID_PACING_ENABLE) == 0) {
22844 /* Disabling hybrid pacing */
22845 if (rack->rc_hybrid_mode) {
22846 rack_set_profile(rack, 0);
22847 rack->rc_tp->tcp_hybrid_stop++;
22848 }
22849 rack_log_hybrid(rack, seq, sft, HYBRID_LOG_TURNED_OFF, __LINE__, 0);
22850 return (0);
22851 }
22852 if (rack->dgp_on == 0) {
22853 /*
22854 * If we have not yet turned DGP on, do so
22855 * now setting pure DGP mode, no buffer level
22856 * response.
22857 */
22858 if ((err = rack_set_profile(rack, 1)) != 0){
22859 /* Failed to turn pacing on */
22860 rack->rc_tp->tcp_hybrid_error++;
22861 rack_log_hybrid(rack, seq, sft, HYBRID_LOG_NO_PACING, __LINE__, 0);
22862 return (err);
22863 }
22864 }
22865 /*
22866 * Now we must switch to hybrid mode as well which also
22867 * means moving to regular pacing.
22868 */
22869 if (rack->rc_hybrid_mode == 0) {
22870 /* First time */
22871 if (tcp_can_enable_pacing()) {
22872 rack->r_ctl.pacing_method |= RACK_REG_PACING;
22873 rack->rc_hybrid_mode = 1;
22874 } else {
22875 return (ENOSPC);
22876 }
22877 if (rack->r_ctl.pacing_method & RACK_DGP_PACING) {
22878 /*
22879 * This should be true.
22880 */
22881 tcp_dec_dgp_pacing_cnt();
22882 rack->r_ctl.pacing_method &= ~RACK_DGP_PACING;
22883 }
22884 }
22885 /* Now set in our flags */
22886 sft->hybrid_flags = hybrid->hybrid_flags | TCP_HYBRID_PACING_WASSET;
22887 if (hybrid->hybrid_flags & TCP_HYBRID_PACING_CSPR)
22888 sft->cspr = hybrid->cspr;
22889 else
22890 sft->cspr = 0;
22891 if (hybrid->hybrid_flags & TCP_HYBRID_PACING_H_MS)
22892 sft->hint_maxseg = hybrid->hint_maxseg;
22893 else
22894 sft->hint_maxseg = 0;
22895 rack->rc_tp->tcp_hybrid_start++;
22896 rack_log_hybrid(rack, seq, sft, HYBRID_LOG_RULES_SET, __LINE__,0);
22897 return (0);
22898 #else
22899 return (ENOTSUP);
22900 #endif
22901 }
22902
22903 static int
rack_stack_information(struct tcpcb * tp,struct stack_specific_info * si)22904 rack_stack_information(struct tcpcb *tp, struct stack_specific_info *si)
22905 {
22906 /* We pulled a SSI info log out what was there */
22907 si->bytes_transmitted = tp->t_sndbytes;
22908 si->bytes_retransmitted = tp->t_snd_rxt_bytes;
22909 return (0);
22910 }
22911
22912 static int
rack_process_option(struct tcpcb * tp,struct tcp_rack * rack,int sopt_name,uint32_t optval,uint64_t loptval,struct tcp_hybrid_req * hybrid)22913 rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name,
22914 uint32_t optval, uint64_t loptval, struct tcp_hybrid_req *hybrid)
22915
22916 {
22917 struct epoch_tracker et;
22918 struct sockopt sopt;
22919 struct cc_newreno_opts opt;
22920 uint64_t val;
22921 int error = 0;
22922 uint16_t ca, ss;
22923
22924 switch (sopt_name) {
22925 case TCP_RACK_SET_RXT_OPTIONS:
22926 if (optval <= 2) {
22927 rack_init_retransmit_value(rack, optval);
22928 } else {
22929 /*
22930 * You must send in 0, 1 or 2 all else is
22931 * invalid.
22932 */
22933 error = EINVAL;
22934 }
22935 break;
22936 case TCP_RACK_DSACK_OPT:
22937 RACK_OPTS_INC(tcp_rack_dsack_opt);
22938 if (optval & 0x1) {
22939 rack->rc_rack_tmr_std_based = 1;
22940 } else {
22941 rack->rc_rack_tmr_std_based = 0;
22942 }
22943 if (optval & 0x2) {
22944 rack->rc_rack_use_dsack = 1;
22945 } else {
22946 rack->rc_rack_use_dsack = 0;
22947 }
22948 rack_log_dsack_event(rack, 5, __LINE__, 0, 0);
22949 break;
22950 case TCP_RACK_PACING_DIVISOR:
22951 RACK_OPTS_INC(tcp_rack_pacing_divisor);
22952 if (optval == 0) {
22953 rack->r_ctl.pace_len_divisor = rack_default_pacing_divisor;
22954 } else {
22955 if (optval < RL_MIN_DIVISOR)
22956 rack->r_ctl.pace_len_divisor = RL_MIN_DIVISOR;
22957 else
22958 rack->r_ctl.pace_len_divisor = optval;
22959 }
22960 break;
22961 case TCP_RACK_HI_BETA:
22962 RACK_OPTS_INC(tcp_rack_hi_beta);
22963 if (optval > 0) {
22964 rack->rack_hibeta = 1;
22965 if ((optval >= 50) &&
22966 (optval <= 100)) {
22967 /*
22968 * User wants to set a custom beta.
22969 */
22970 rack->r_ctl.saved_hibeta = optval;
22971 if (rack->rc_pacing_cc_set)
22972 rack_undo_cc_pacing(rack);
22973 rack->r_ctl.rc_saved_beta = optval;
22974 }
22975 if (rack->rc_pacing_cc_set == 0)
22976 rack_set_cc_pacing(rack);
22977 } else {
22978 rack->rack_hibeta = 0;
22979 if (rack->rc_pacing_cc_set)
22980 rack_undo_cc_pacing(rack);
22981 }
22982 break;
22983 case TCP_RACK_PACING_BETA:
22984 error = EINVAL;
22985 break;
22986 case TCP_RACK_TIMER_SLOP:
22987 RACK_OPTS_INC(tcp_rack_timer_slop);
22988 rack->r_ctl.timer_slop = optval;
22989 if (rack->rc_tp->t_srtt) {
22990 /*
22991 * If we have an SRTT lets update t_rxtcur
22992 * to have the new slop.
22993 */
22994 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
22995 rack_rto_min, rack_rto_max,
22996 rack->r_ctl.timer_slop);
22997 }
22998 break;
22999 case TCP_RACK_PACING_BETA_ECN:
23000 RACK_OPTS_INC(tcp_rack_beta_ecn);
23001 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0) {
23002 /* This only works for newreno. */
23003 error = EINVAL;
23004 break;
23005 }
23006 if (rack->rc_pacing_cc_set) {
23007 /*
23008 * Set them into the real CC module
23009 * whats in the rack pcb is the old values
23010 * to be used on restoral/
23011 */
23012 sopt.sopt_dir = SOPT_SET;
23013 opt.name = CC_NEWRENO_BETA_ECN;
23014 opt.val = optval;
23015 if (CC_ALGO(tp)->ctl_output != NULL)
23016 error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, &sopt, &opt);
23017 else
23018 error = ENOENT;
23019 } else {
23020 /*
23021 * Not pacing yet so set it into our local
23022 * rack pcb storage.
23023 */
23024 rack->r_ctl.rc_saved_beta_ecn = optval;
23025 }
23026 break;
23027 case TCP_DEFER_OPTIONS:
23028 RACK_OPTS_INC(tcp_defer_opt);
23029 if (optval) {
23030 if (rack->gp_ready) {
23031 /* Too late */
23032 error = EINVAL;
23033 break;
23034 }
23035 rack->defer_options = 1;
23036 } else
23037 rack->defer_options = 0;
23038 break;
23039 case TCP_RACK_MEASURE_CNT:
23040 RACK_OPTS_INC(tcp_rack_measure_cnt);
23041 if (optval && (optval <= 0xff)) {
23042 rack->r_ctl.req_measurements = optval;
23043 } else
23044 error = EINVAL;
23045 break;
23046 case TCP_REC_ABC_VAL:
23047 RACK_OPTS_INC(tcp_rec_abc_val);
23048 if (optval > 0)
23049 rack->r_use_labc_for_rec = 1;
23050 else
23051 rack->r_use_labc_for_rec = 0;
23052 break;
23053 case TCP_RACK_ABC_VAL:
23054 RACK_OPTS_INC(tcp_rack_abc_val);
23055 if ((optval > 0) && (optval < 255))
23056 rack->rc_labc = optval;
23057 else
23058 error = EINVAL;
23059 break;
23060 case TCP_HDWR_UP_ONLY:
23061 RACK_OPTS_INC(tcp_pacing_up_only);
23062 if (optval)
23063 rack->r_up_only = 1;
23064 else
23065 rack->r_up_only = 0;
23066 break;
23067 case TCP_FILLCW_RATE_CAP: /* URL:fillcw_cap */
23068 RACK_OPTS_INC(tcp_fillcw_rate_cap);
23069 rack->r_ctl.fillcw_cap = loptval;
23070 break;
23071 case TCP_PACING_RATE_CAP:
23072 RACK_OPTS_INC(tcp_pacing_rate_cap);
23073 if ((rack->dgp_on == 1) &&
23074 (rack->r_ctl.pacing_method & RACK_DGP_PACING)) {
23075 /*
23076 * If we are doing DGP we need to switch
23077 * to using the pacing limit.
23078 */
23079 if (tcp_can_enable_pacing() == 0) {
23080 error = ENOSPC;
23081 break;
23082 }
23083 /*
23084 * Now change up the flags and counts to be correct.
23085 */
23086 rack->r_ctl.pacing_method |= RACK_REG_PACING;
23087 tcp_dec_dgp_pacing_cnt();
23088 rack->r_ctl.pacing_method &= ~RACK_DGP_PACING;
23089 }
23090 rack->r_ctl.bw_rate_cap = loptval;
23091 break;
23092 case TCP_HYBRID_PACING:
23093 if (hybrid == NULL) {
23094 error = EINVAL;
23095 break;
23096 }
23097 if (rack->r_ctl.side_chan_dis_mask & HYBRID_DIS_MASK) {
23098 error = EPERM;
23099 break;
23100 }
23101 error = process_hybrid_pacing(rack, hybrid);
23102 break;
23103 case TCP_SIDECHAN_DIS: /* URL:scodm */
23104 if (optval)
23105 rack->r_ctl.side_chan_dis_mask = optval;
23106 else
23107 rack->r_ctl.side_chan_dis_mask = 0;
23108 break;
23109 case TCP_RACK_PROFILE:
23110 RACK_OPTS_INC(tcp_profile);
23111 error = rack_set_profile(rack, optval);
23112 break;
23113 case TCP_USE_CMP_ACKS:
23114 RACK_OPTS_INC(tcp_use_cmp_acks);
23115 if ((optval == 0) && (tp->t_flags2 & TF2_MBUF_ACKCMP)) {
23116 /* You can't turn it off once its on! */
23117 error = EINVAL;
23118 } else if ((optval == 1) && (rack->r_use_cmp_ack == 0)) {
23119 rack->r_use_cmp_ack = 1;
23120 rack->r_mbuf_queue = 1;
23121 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
23122 }
23123 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
23124 tp->t_flags2 |= TF2_MBUF_ACKCMP;
23125 break;
23126 case TCP_SHARED_CWND_TIME_LIMIT:
23127 RACK_OPTS_INC(tcp_lscwnd);
23128 if (optval)
23129 rack->r_limit_scw = 1;
23130 else
23131 rack->r_limit_scw = 0;
23132 break;
23133 case TCP_RACK_DGP_IN_REC:
23134 error = EINVAL;
23135 break;
23136 case TCP_RACK_PACE_TO_FILL:
23137 RACK_OPTS_INC(tcp_fillcw);
23138 if (optval == 0)
23139 rack->rc_pace_to_cwnd = 0;
23140 else {
23141 rack->rc_pace_to_cwnd = 1;
23142 }
23143 if ((optval >= rack_gp_rtt_maxmul) &&
23144 rack_gp_rtt_maxmul &&
23145 (optval < 0xf)) {
23146 rack->rc_pace_fill_if_rttin_range = 1;
23147 rack->rtt_limit_mul = optval;
23148 } else {
23149 rack->rc_pace_fill_if_rttin_range = 0;
23150 rack->rtt_limit_mul = 0;
23151 }
23152 break;
23153 case TCP_RACK_NO_PUSH_AT_MAX:
23154 RACK_OPTS_INC(tcp_npush);
23155 if (optval == 0)
23156 rack->r_ctl.rc_no_push_at_mrtt = 0;
23157 else if (optval < 0xff)
23158 rack->r_ctl.rc_no_push_at_mrtt = optval;
23159 else
23160 error = EINVAL;
23161 break;
23162 case TCP_SHARED_CWND_ENABLE:
23163 RACK_OPTS_INC(tcp_rack_scwnd);
23164 if (optval == 0)
23165 rack->rack_enable_scwnd = 0;
23166 else
23167 rack->rack_enable_scwnd = 1;
23168 break;
23169 case TCP_RACK_MBUF_QUEUE:
23170 /* Now do we use the LRO mbuf-queue feature */
23171 RACK_OPTS_INC(tcp_rack_mbufq);
23172 if (optval || rack->r_use_cmp_ack)
23173 rack->r_mbuf_queue = 1;
23174 else
23175 rack->r_mbuf_queue = 0;
23176 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
23177 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
23178 else
23179 tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
23180 break;
23181 case TCP_RACK_NONRXT_CFG_RATE:
23182 RACK_OPTS_INC(tcp_rack_cfg_rate);
23183 if (optval == 0)
23184 rack->rack_rec_nonrxt_use_cr = 0;
23185 else
23186 rack->rack_rec_nonrxt_use_cr = 1;
23187 break;
23188 case TCP_NO_PRR:
23189 RACK_OPTS_INC(tcp_rack_noprr);
23190 if (optval == 0)
23191 rack->rack_no_prr = 0;
23192 else if (optval == 1)
23193 rack->rack_no_prr = 1;
23194 else if (optval == 2)
23195 rack->no_prr_addback = 1;
23196 else
23197 error = EINVAL;
23198 break;
23199 case RACK_CSPR_IS_FCC: /* URL:csprisfcc */
23200 if (optval > 0)
23201 rack->cspr_is_fcc = 1;
23202 else
23203 rack->cspr_is_fcc = 0;
23204 break;
23205 case TCP_TIMELY_DYN_ADJ:
23206 RACK_OPTS_INC(tcp_timely_dyn);
23207 if (optval == 0)
23208 rack->rc_gp_dyn_mul = 0;
23209 else {
23210 rack->rc_gp_dyn_mul = 1;
23211 if (optval >= 100) {
23212 /*
23213 * If the user sets something 100 or more
23214 * its the gp_ca value.
23215 */
23216 rack->r_ctl.rack_per_of_gp_ca = optval;
23217 }
23218 }
23219 break;
23220 case TCP_RACK_DO_DETECTION:
23221 error = EINVAL;
23222 break;
23223 case TCP_RACK_TLP_USE:
23224 if ((optval < TLP_USE_ID) || (optval > TLP_USE_TWO_TWO)) {
23225 error = EINVAL;
23226 break;
23227 }
23228 RACK_OPTS_INC(tcp_tlp_use);
23229 rack->rack_tlp_threshold_use = optval;
23230 break;
23231 case TCP_RACK_TLP_REDUCE:
23232 /* RACK TLP cwnd reduction (bool) */
23233 RACK_OPTS_INC(tcp_rack_tlp_reduce);
23234 rack->r_ctl.rc_tlp_cwnd_reduce = optval;
23235 break;
23236 /* Pacing related ones */
23237 case TCP_RACK_PACE_ALWAYS:
23238 /*
23239 * zero is old rack method, 1 is new
23240 * method using a pacing rate.
23241 */
23242 RACK_OPTS_INC(tcp_rack_pace_always);
23243 if (rack->r_ctl.side_chan_dis_mask & CCSP_DIS_MASK) {
23244 error = EPERM;
23245 break;
23246 }
23247 if (optval > 0) {
23248 if (rack->rc_always_pace) {
23249 error = EALREADY;
23250 break;
23251 } else if (tcp_can_enable_pacing()) {
23252 rack->r_ctl.pacing_method |= RACK_REG_PACING;
23253 rack->rc_always_pace = 1;
23254 if (rack->rack_hibeta)
23255 rack_set_cc_pacing(rack);
23256 }
23257 else {
23258 error = ENOSPC;
23259 break;
23260 }
23261 } else {
23262 if (rack->rc_always_pace == 1) {
23263 rack_remove_pacing(rack);
23264 }
23265 }
23266 if (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
23267 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
23268 else
23269 tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
23270 /* A rate may be set irate or other, if so set seg size */
23271 rack_update_seg(rack);
23272 break;
23273 case TCP_BBR_RACK_INIT_RATE:
23274 RACK_OPTS_INC(tcp_initial_rate);
23275 val = optval;
23276 /* Change from kbits per second to bytes per second */
23277 val *= 1000;
23278 val /= 8;
23279 rack->r_ctl.init_rate = val;
23280 if (rack->rc_always_pace)
23281 rack_update_seg(rack);
23282 break;
23283 case TCP_BBR_IWINTSO:
23284 error = EINVAL;
23285 break;
23286 case TCP_RACK_FORCE_MSEG:
23287 RACK_OPTS_INC(tcp_rack_force_max_seg);
23288 if (optval)
23289 rack->rc_force_max_seg = 1;
23290 else
23291 rack->rc_force_max_seg = 0;
23292 break;
23293 case TCP_RACK_PACE_MIN_SEG:
23294 RACK_OPTS_INC(tcp_rack_min_seg);
23295 rack->r_ctl.rc_user_set_min_segs = (0x0000ffff & optval);
23296 rack_set_pace_segments(tp, rack, __LINE__, NULL);
23297 break;
23298 case TCP_RACK_PACE_MAX_SEG:
23299 /* Max segments size in a pace in bytes */
23300 RACK_OPTS_INC(tcp_rack_max_seg);
23301 if ((rack->dgp_on == 1) &&
23302 (rack->r_ctl.pacing_method & RACK_DGP_PACING)) {
23303 /*
23304 * If we set a max-seg and are doing DGP then
23305 * we now fall under the pacing limits not the
23306 * DGP ones.
23307 */
23308 if (tcp_can_enable_pacing() == 0) {
23309 error = ENOSPC;
23310 break;
23311 }
23312 /*
23313 * Now change up the flags and counts to be correct.
23314 */
23315 rack->r_ctl.pacing_method |= RACK_REG_PACING;
23316 tcp_dec_dgp_pacing_cnt();
23317 rack->r_ctl.pacing_method &= ~RACK_DGP_PACING;
23318 }
23319 if (optval <= MAX_USER_SET_SEG)
23320 rack->rc_user_set_max_segs = optval;
23321 else
23322 rack->rc_user_set_max_segs = MAX_USER_SET_SEG;
23323 rack_set_pace_segments(tp, rack, __LINE__, NULL);
23324 break;
23325 case TCP_RACK_PACE_RATE_REC:
23326 /* Set the fixed pacing rate in Bytes per second ca */
23327 RACK_OPTS_INC(tcp_rack_pace_rate_rec);
23328 if (rack->r_ctl.side_chan_dis_mask & CCSP_DIS_MASK) {
23329 error = EPERM;
23330 break;
23331 }
23332 if (rack->dgp_on) {
23333 /*
23334 * We are already pacing another
23335 * way.
23336 */
23337 error = EBUSY;
23338 break;
23339 }
23340 rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
23341 if (rack->r_ctl.rc_fixed_pacing_rate_ca == 0)
23342 rack->r_ctl.rc_fixed_pacing_rate_ca = optval;
23343 if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0)
23344 rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
23345 rack->use_fixed_rate = 1;
23346 if (rack->rack_hibeta)
23347 rack_set_cc_pacing(rack);
23348 rack_log_pacing_delay_calc(rack,
23349 rack->r_ctl.rc_fixed_pacing_rate_ss,
23350 rack->r_ctl.rc_fixed_pacing_rate_ca,
23351 rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8,
23352 __LINE__, NULL,0);
23353 break;
23354
23355 case TCP_RACK_PACE_RATE_SS:
23356 /* Set the fixed pacing rate in Bytes per second ca */
23357 RACK_OPTS_INC(tcp_rack_pace_rate_ss);
23358 if (rack->r_ctl.side_chan_dis_mask & CCSP_DIS_MASK) {
23359 error = EPERM;
23360 break;
23361 }
23362 if (rack->dgp_on) {
23363 /*
23364 * We are already pacing another
23365 * way.
23366 */
23367 error = EBUSY;
23368 break;
23369 }
23370 rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
23371 if (rack->r_ctl.rc_fixed_pacing_rate_ca == 0)
23372 rack->r_ctl.rc_fixed_pacing_rate_ca = optval;
23373 if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0)
23374 rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
23375 rack->use_fixed_rate = 1;
23376 if (rack->rack_hibeta)
23377 rack_set_cc_pacing(rack);
23378 rack_log_pacing_delay_calc(rack,
23379 rack->r_ctl.rc_fixed_pacing_rate_ss,
23380 rack->r_ctl.rc_fixed_pacing_rate_ca,
23381 rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8,
23382 __LINE__, NULL, 0);
23383 break;
23384
23385 case TCP_RACK_PACE_RATE_CA:
23386 /* Set the fixed pacing rate in Bytes per second ca */
23387 RACK_OPTS_INC(tcp_rack_pace_rate_ca);
23388 if (rack->r_ctl.side_chan_dis_mask & CCSP_DIS_MASK) {
23389 error = EPERM;
23390 break;
23391 }
23392 if (rack->dgp_on) {
23393 /*
23394 * We are already pacing another
23395 * way.
23396 */
23397 error = EBUSY;
23398 break;
23399 }
23400 rack->r_ctl.rc_fixed_pacing_rate_ca = optval;
23401 if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0)
23402 rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
23403 if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0)
23404 rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
23405 rack->use_fixed_rate = 1;
23406 if (rack->rack_hibeta)
23407 rack_set_cc_pacing(rack);
23408 rack_log_pacing_delay_calc(rack,
23409 rack->r_ctl.rc_fixed_pacing_rate_ss,
23410 rack->r_ctl.rc_fixed_pacing_rate_ca,
23411 rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8,
23412 __LINE__, NULL, 0);
23413 break;
23414 case TCP_RACK_GP_INCREASE_REC:
23415 RACK_OPTS_INC(tcp_gp_inc_rec);
23416 rack->r_ctl.rack_per_of_gp_rec = optval;
23417 rack_log_pacing_delay_calc(rack,
23418 rack->r_ctl.rack_per_of_gp_ss,
23419 rack->r_ctl.rack_per_of_gp_ca,
23420 rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1,
23421 __LINE__, NULL, 0);
23422 break;
23423 case TCP_RACK_GP_INCREASE_CA:
23424 RACK_OPTS_INC(tcp_gp_inc_ca);
23425 ca = optval;
23426 if (ca < 100) {
23427 /*
23428 * We don't allow any reduction
23429 * over the GP b/w.
23430 */
23431 error = EINVAL;
23432 break;
23433 }
23434 rack->r_ctl.rack_per_of_gp_ca = ca;
23435 rack_log_pacing_delay_calc(rack,
23436 rack->r_ctl.rack_per_of_gp_ss,
23437 rack->r_ctl.rack_per_of_gp_ca,
23438 rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1,
23439 __LINE__, NULL, 0);
23440 break;
23441 case TCP_RACK_GP_INCREASE_SS:
23442 RACK_OPTS_INC(tcp_gp_inc_ss);
23443 ss = optval;
23444 if (ss < 100) {
23445 /*
23446 * We don't allow any reduction
23447 * over the GP b/w.
23448 */
23449 error = EINVAL;
23450 break;
23451 }
23452 rack->r_ctl.rack_per_of_gp_ss = ss;
23453 rack_log_pacing_delay_calc(rack,
23454 rack->r_ctl.rack_per_of_gp_ss,
23455 rack->r_ctl.rack_per_of_gp_ca,
23456 rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1,
23457 __LINE__, NULL, 0);
23458 break;
23459 case TCP_RACK_RR_CONF:
23460 RACK_OPTS_INC(tcp_rack_rrr_no_conf_rate);
23461 if (optval && optval <= 3)
23462 rack->r_rr_config = optval;
23463 else
23464 rack->r_rr_config = 0;
23465 break;
23466 case TCP_PACING_DND: /* URL:dnd */
23467 if (optval > 0)
23468 rack->rc_pace_dnd = 1;
23469 else
23470 rack->rc_pace_dnd = 0;
23471 break;
23472 case TCP_HDWR_RATE_CAP:
23473 RACK_OPTS_INC(tcp_hdwr_rate_cap);
23474 if (optval) {
23475 if (rack->r_rack_hw_rate_caps == 0)
23476 rack->r_rack_hw_rate_caps = 1;
23477 else
23478 error = EALREADY;
23479 } else {
23480 rack->r_rack_hw_rate_caps = 0;
23481 }
23482 break;
23483 case TCP_DGP_UPPER_BOUNDS:
23484 {
23485 uint8_t val;
23486 val = optval & 0x0000ff;
23487 rack->r_ctl.rack_per_upper_bound_ca = val;
23488 val = (optval >> 16) & 0x0000ff;
23489 rack->r_ctl.rack_per_upper_bound_ss = val;
23490 break;
23491 }
23492 case TCP_SS_EEXIT: /* URL:eexit */
23493 if (optval > 0) {
23494 rack->r_ctl.gp_rnd_thresh = optval & 0x0ff;
23495 if (optval & 0x10000) {
23496 rack->r_ctl.gate_to_fs = 1;
23497 } else {
23498 rack->r_ctl.gate_to_fs = 0;
23499 }
23500 if (optval & 0x20000) {
23501 rack->r_ctl.use_gp_not_last = 1;
23502 } else {
23503 rack->r_ctl.use_gp_not_last = 0;
23504 }
23505 if (optval & 0xfffc0000) {
23506 uint32_t v;
23507
23508 v = (optval >> 18) & 0x00003fff;
23509 if (v >= 1000)
23510 rack->r_ctl.gp_gain_req = v;
23511 }
23512 } else {
23513 /* We do not do ss early exit at all */
23514 rack->rc_initial_ss_comp = 1;
23515 rack->r_ctl.gp_rnd_thresh = 0;
23516 }
23517 break;
23518 case TCP_RACK_SPLIT_LIMIT:
23519 RACK_OPTS_INC(tcp_split_limit);
23520 rack->r_ctl.rc_split_limit = optval;
23521 break;
23522 case TCP_BBR_HDWR_PACE:
23523 RACK_OPTS_INC(tcp_hdwr_pacing);
23524 if (optval){
23525 if (rack->rack_hdrw_pacing == 0) {
23526 rack->rack_hdw_pace_ena = 1;
23527 rack->rack_attempt_hdwr_pace = 0;
23528 } else
23529 error = EALREADY;
23530 } else {
23531 rack->rack_hdw_pace_ena = 0;
23532 #ifdef RATELIMIT
23533 if (rack->r_ctl.crte != NULL) {
23534 rack->rack_hdrw_pacing = 0;
23535 rack->rack_attempt_hdwr_pace = 0;
23536 tcp_rel_pacing_rate(rack->r_ctl.crte, tp);
23537 rack->r_ctl.crte = NULL;
23538 }
23539 #endif
23540 }
23541 break;
23542 /* End Pacing related ones */
23543 case TCP_RACK_PRR_SENDALOT:
23544 /* Allow PRR to send more than one seg */
23545 RACK_OPTS_INC(tcp_rack_prr_sendalot);
23546 rack->r_ctl.rc_prr_sendalot = optval;
23547 break;
23548 case TCP_RACK_MIN_TO:
23549 /* Minimum time between rack t-o's in ms */
23550 RACK_OPTS_INC(tcp_rack_min_to);
23551 rack->r_ctl.rc_min_to = optval;
23552 break;
23553 case TCP_RACK_EARLY_SEG:
23554 /* If early recovery max segments */
23555 RACK_OPTS_INC(tcp_rack_early_seg);
23556 rack->r_ctl.rc_early_recovery_segs = optval;
23557 break;
23558 case TCP_RACK_ENABLE_HYSTART:
23559 {
23560 if (optval) {
23561 tp->t_ccv.flags |= CCF_HYSTART_ALLOWED;
23562 if (rack_do_hystart > RACK_HYSTART_ON)
23563 tp->t_ccv.flags |= CCF_HYSTART_CAN_SH_CWND;
23564 if (rack_do_hystart > RACK_HYSTART_ON_W_SC)
23565 tp->t_ccv.flags |= CCF_HYSTART_CONS_SSTH;
23566 } else {
23567 tp->t_ccv.flags &= ~(CCF_HYSTART_ALLOWED|CCF_HYSTART_CAN_SH_CWND|CCF_HYSTART_CONS_SSTH);
23568 }
23569 }
23570 break;
23571 case TCP_RACK_REORD_THRESH:
23572 /* RACK reorder threshold (shift amount) */
23573 RACK_OPTS_INC(tcp_rack_reord_thresh);
23574 if ((optval > 0) && (optval < 31))
23575 rack->r_ctl.rc_reorder_shift = optval;
23576 else
23577 error = EINVAL;
23578 break;
23579 case TCP_RACK_REORD_FADE:
23580 /* Does reordering fade after ms time */
23581 RACK_OPTS_INC(tcp_rack_reord_fade);
23582 rack->r_ctl.rc_reorder_fade = optval;
23583 break;
23584 case TCP_RACK_TLP_THRESH:
23585 /* RACK TLP theshold i.e. srtt+(srtt/N) */
23586 RACK_OPTS_INC(tcp_rack_tlp_thresh);
23587 if (optval)
23588 rack->r_ctl.rc_tlp_threshold = optval;
23589 else
23590 error = EINVAL;
23591 break;
23592 case TCP_BBR_USE_RACK_RR:
23593 RACK_OPTS_INC(tcp_rack_rr);
23594 if (optval)
23595 rack->use_rack_rr = 1;
23596 else
23597 rack->use_rack_rr = 0;
23598 break;
23599 case TCP_RACK_PKT_DELAY:
23600 /* RACK added ms i.e. rack-rtt + reord + N */
23601 RACK_OPTS_INC(tcp_rack_pkt_delay);
23602 rack->r_ctl.rc_pkt_delay = optval;
23603 break;
23604 case TCP_DELACK:
23605 RACK_OPTS_INC(tcp_rack_delayed_ack);
23606 if (optval == 0)
23607 tp->t_delayed_ack = 0;
23608 else
23609 tp->t_delayed_ack = 1;
23610 if (tp->t_flags & TF_DELACK) {
23611 tp->t_flags &= ~TF_DELACK;
23612 tp->t_flags |= TF_ACKNOW;
23613 NET_EPOCH_ENTER(et);
23614 rack_output(tp);
23615 NET_EPOCH_EXIT(et);
23616 }
23617 break;
23618
23619 case TCP_BBR_RACK_RTT_USE:
23620 RACK_OPTS_INC(tcp_rack_rtt_use);
23621 if ((optval != USE_RTT_HIGH) &&
23622 (optval != USE_RTT_LOW) &&
23623 (optval != USE_RTT_AVG))
23624 error = EINVAL;
23625 else
23626 rack->r_ctl.rc_rate_sample_method = optval;
23627 break;
23628 case TCP_HONOR_HPTS_MIN:
23629 RACK_OPTS_INC(tcp_honor_hpts);
23630 if (optval) {
23631 rack->r_use_hpts_min = 1;
23632 /*
23633 * Must be between 2 - 80% to be a reduction else
23634 * we keep the default (10%).
23635 */
23636 if ((optval > 1) && (optval <= 80)) {
23637 rack->r_ctl.max_reduction = optval;
23638 }
23639 } else
23640 rack->r_use_hpts_min = 0;
23641 break;
23642 case TCP_REC_IS_DYN: /* URL:dynrec */
23643 RACK_OPTS_INC(tcp_dyn_rec);
23644 if (optval)
23645 rack->rc_gp_no_rec_chg = 1;
23646 else
23647 rack->rc_gp_no_rec_chg = 0;
23648 break;
23649 case TCP_NO_TIMELY:
23650 RACK_OPTS_INC(tcp_notimely);
23651 if (optval) {
23652 rack->rc_skip_timely = 1;
23653 rack->r_ctl.rack_per_of_gp_rec = 90;
23654 rack->r_ctl.rack_per_of_gp_ca = 100;
23655 rack->r_ctl.rack_per_of_gp_ss = 250;
23656 } else {
23657 rack->rc_skip_timely = 0;
23658 }
23659 break;
23660 case TCP_GP_USE_LTBW:
23661 if (optval == 0) {
23662 rack->use_lesser_lt_bw = 0;
23663 rack->dis_lt_bw = 1;
23664 } else if (optval == 1) {
23665 rack->use_lesser_lt_bw = 1;
23666 rack->dis_lt_bw = 0;
23667 } else if (optval == 2) {
23668 rack->use_lesser_lt_bw = 0;
23669 rack->dis_lt_bw = 0;
23670 }
23671 break;
23672 case TCP_DATA_AFTER_CLOSE:
23673 RACK_OPTS_INC(tcp_data_after_close);
23674 if (optval)
23675 rack->rc_allow_data_af_clo = 1;
23676 else
23677 rack->rc_allow_data_af_clo = 0;
23678 break;
23679 default:
23680 break;
23681 }
23682 tcp_log_socket_option(tp, sopt_name, optval, error);
23683 return (error);
23684 }
23685
23686 static void
rack_inherit(struct tcpcb * tp,struct inpcb * parent)23687 rack_inherit(struct tcpcb *tp, struct inpcb *parent)
23688 {
23689 /*
23690 * A new connection has been created (tp) and
23691 * the parent is the inpcb given. We want to
23692 * apply a read-lock to the parent (we are already
23693 * holding a write lock on the tp) and copy anything
23694 * out of the rack specific data as long as its tfb is
23695 * the same as ours i.e. we are the same stack. Otherwise
23696 * we just return.
23697 */
23698 struct tcpcb *par;
23699 struct tcp_rack *dest, *src;
23700 int cnt = 0;
23701
23702 par = intotcpcb(parent);
23703 if (par->t_fb != tp->t_fb) {
23704 /* Not the same stack */
23705 tcp_log_socket_option(tp, 0, 0, 1);
23706 return;
23707 }
23708 /* Ok if we reach here lets setup the two rack pointers */
23709 dest = (struct tcp_rack *)tp->t_fb_ptr;
23710 src = (struct tcp_rack *)par->t_fb_ptr;
23711 if ((src == NULL) || (dest == NULL)) {
23712 /* Huh? */
23713 tcp_log_socket_option(tp, 0, 0, 2);
23714 return;
23715 }
23716 /* Now copy out anything we wish to inherit i.e. things in socket-options */
23717 /* TCP_RACK_PROFILE we can't know but we can set DGP if its on */
23718 if ((src->dgp_on) && (dest->dgp_on == 0)) {
23719 /* Profile 1 had to be set via sock opt */
23720 rack_set_dgp(dest);
23721 cnt++;
23722 }
23723 /* TCP_RACK_SET_RXT_OPTIONS */
23724 if (dest->full_size_rxt != src->full_size_rxt) {
23725 dest->full_size_rxt = src->full_size_rxt;
23726 cnt++;
23727 }
23728 if (dest->shape_rxt_to_pacing_min != src->shape_rxt_to_pacing_min) {
23729 dest->shape_rxt_to_pacing_min = src->shape_rxt_to_pacing_min;
23730 cnt++;
23731 }
23732 /* TCP_RACK_DSACK_OPT */
23733 if (dest->rc_rack_tmr_std_based != src->rc_rack_tmr_std_based) {
23734 dest->rc_rack_tmr_std_based = src->rc_rack_tmr_std_based;
23735 cnt++;
23736 }
23737 if (dest->rc_rack_use_dsack != src->rc_rack_use_dsack) {
23738 dest->rc_rack_use_dsack = src->rc_rack_use_dsack;
23739 cnt++;
23740 }
23741 /* TCP_RACK_PACING_DIVISOR */
23742 if (dest->r_ctl.pace_len_divisor != src->r_ctl.pace_len_divisor) {
23743 dest->r_ctl.pace_len_divisor = src->r_ctl.pace_len_divisor;
23744 cnt++;
23745 }
23746 /* TCP_RACK_HI_BETA */
23747 if (src->rack_hibeta != dest->rack_hibeta) {
23748 cnt++;
23749 if (src->rack_hibeta) {
23750 dest->r_ctl.rc_saved_beta = src->r_ctl.rc_saved_beta;
23751 dest->rack_hibeta = 1;
23752 } else {
23753 dest->rack_hibeta = 0;
23754 }
23755 }
23756 /* TCP_RACK_TIMER_SLOP */
23757 if (dest->r_ctl.timer_slop != src->r_ctl.timer_slop) {
23758 dest->r_ctl.timer_slop = src->r_ctl.timer_slop;
23759 cnt++;
23760 }
23761 /* TCP_RACK_PACING_BETA_ECN */
23762 if (dest->r_ctl.rc_saved_beta_ecn != src->r_ctl.rc_saved_beta_ecn) {
23763 dest->r_ctl.rc_saved_beta_ecn = src->r_ctl.rc_saved_beta_ecn;
23764 cnt++;
23765 }
23766 /* We do not do TCP_DEFER_OPTIONS */
23767 /* TCP_RACK_MEASURE_CNT */
23768 if (dest->r_ctl.req_measurements != src->r_ctl.req_measurements) {
23769 dest->r_ctl.req_measurements = src->r_ctl.req_measurements;
23770 cnt++;
23771 }
23772 /* TCP_HDWR_UP_ONLY */
23773 if (dest->r_up_only != src->r_up_only) {
23774 dest->r_up_only = src->r_up_only;
23775 cnt++;
23776 }
23777 /* TCP_FILLCW_RATE_CAP */
23778 if (dest->r_ctl.fillcw_cap != src->r_ctl.fillcw_cap) {
23779 dest->r_ctl.fillcw_cap = src->r_ctl.fillcw_cap;
23780 cnt++;
23781 }
23782 /* TCP_PACING_RATE_CAP */
23783 if (dest->r_ctl.bw_rate_cap != src->r_ctl.bw_rate_cap) {
23784 dest->r_ctl.bw_rate_cap = src->r_ctl.bw_rate_cap;
23785 cnt++;
23786 }
23787 /* A listener can't set TCP_HYBRID_PACING */
23788 /* TCP_SIDECHAN_DIS */
23789 if (dest->r_ctl.side_chan_dis_mask != src->r_ctl.side_chan_dis_mask) {
23790 dest->r_ctl.side_chan_dis_mask = src->r_ctl.side_chan_dis_mask;
23791 cnt++;
23792 }
23793 /* TCP_SHARED_CWND_TIME_LIMIT */
23794 if (dest->r_limit_scw != src->r_limit_scw) {
23795 dest->r_limit_scw = src->r_limit_scw;
23796 cnt++;
23797 }
23798 /* TCP_RACK_PACE_TO_FILL */
23799 if (dest->rc_pace_to_cwnd != src->rc_pace_to_cwnd) {
23800 dest->rc_pace_to_cwnd = src->rc_pace_to_cwnd;
23801 cnt++;
23802 }
23803 if (dest->rc_pace_fill_if_rttin_range != src->rc_pace_fill_if_rttin_range) {
23804 dest->rc_pace_fill_if_rttin_range = src->rc_pace_fill_if_rttin_range;
23805 cnt++;
23806 }
23807 if (dest->rtt_limit_mul != src->rtt_limit_mul) {
23808 dest->rtt_limit_mul = src->rtt_limit_mul;
23809 cnt++;
23810 }
23811 /* TCP_RACK_NO_PUSH_AT_MAX */
23812 if (dest->r_ctl.rc_no_push_at_mrtt != src->r_ctl.rc_no_push_at_mrtt) {
23813 dest->r_ctl.rc_no_push_at_mrtt = src->r_ctl.rc_no_push_at_mrtt;
23814 cnt++;
23815 }
23816 /* TCP_SHARED_CWND_ENABLE */
23817 if (dest->rack_enable_scwnd != src->rack_enable_scwnd) {
23818 dest->rack_enable_scwnd = src->rack_enable_scwnd;
23819 cnt++;
23820 }
23821 /* TCP_USE_CMP_ACKS */
23822 if (dest->r_use_cmp_ack != src->r_use_cmp_ack) {
23823 dest->r_use_cmp_ack = src->r_use_cmp_ack;
23824 cnt++;
23825 }
23826
23827 if (dest->r_mbuf_queue != src->r_mbuf_queue) {
23828 dest->r_mbuf_queue = src->r_mbuf_queue;
23829 cnt++;
23830 }
23831 /* TCP_RACK_MBUF_QUEUE */
23832 if (dest->r_mbuf_queue != src->r_mbuf_queue) {
23833 dest->r_mbuf_queue = src->r_mbuf_queue;
23834 cnt++;
23835 }
23836 if (dest->r_mbuf_queue || dest->rc_always_pace || dest->r_use_cmp_ack) {
23837 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
23838 } else {
23839 tp->t_flags2 &= ~TF2_SUPPORTS_MBUFQ;
23840 }
23841 if (dest->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state)) {
23842 tp->t_flags2 |= TF2_MBUF_ACKCMP;
23843 }
23844 /* TCP_RACK_NONRXT_CFG_RATE */
23845 if (dest->rack_rec_nonrxt_use_cr != src->rack_rec_nonrxt_use_cr) {
23846 dest->rack_rec_nonrxt_use_cr = src->rack_rec_nonrxt_use_cr;
23847 cnt++;
23848 }
23849 /* TCP_NO_PRR */
23850 if (dest->rack_no_prr != src->rack_no_prr) {
23851 dest->rack_no_prr = src->rack_no_prr;
23852 cnt++;
23853 }
23854 if (dest->no_prr_addback != src->no_prr_addback) {
23855 dest->no_prr_addback = src->no_prr_addback;
23856 cnt++;
23857 }
23858 /* RACK_CSPR_IS_FCC */
23859 if (dest->cspr_is_fcc != src->cspr_is_fcc) {
23860 dest->cspr_is_fcc = src->cspr_is_fcc;
23861 cnt++;
23862 }
23863 /* TCP_TIMELY_DYN_ADJ */
23864 if (dest->rc_gp_dyn_mul != src->rc_gp_dyn_mul) {
23865 dest->rc_gp_dyn_mul = src->rc_gp_dyn_mul;
23866 cnt++;
23867 }
23868 if (dest->r_ctl.rack_per_of_gp_ca != src->r_ctl.rack_per_of_gp_ca) {
23869 dest->r_ctl.rack_per_of_gp_ca = src->r_ctl.rack_per_of_gp_ca;
23870 cnt++;
23871 }
23872 /* TCP_RACK_TLP_USE */
23873 if (dest->rack_tlp_threshold_use != src->rack_tlp_threshold_use) {
23874 dest->rack_tlp_threshold_use = src->rack_tlp_threshold_use;
23875 cnt++;
23876 }
23877 /* we don't allow inheritence of TCP_RACK_PACE_ALWAYS */
23878 /* TCP_BBR_RACK_INIT_RATE */
23879 if (dest->r_ctl.init_rate != src->r_ctl.init_rate) {
23880 dest->r_ctl.init_rate = src->r_ctl.init_rate;
23881 cnt++;
23882 }
23883 /* TCP_RACK_FORCE_MSEG */
23884 if (dest->rc_force_max_seg != src->rc_force_max_seg) {
23885 dest->rc_force_max_seg = src->rc_force_max_seg;
23886 cnt++;
23887 }
23888 /* TCP_RACK_PACE_MIN_SEG */
23889 if (dest->r_ctl.rc_user_set_min_segs != src->r_ctl.rc_user_set_min_segs) {
23890 dest->r_ctl.rc_user_set_min_segs = src->r_ctl.rc_user_set_min_segs;
23891 cnt++;
23892 }
23893 /* we don't allow TCP_RACK_PACE_MAX_SEG */
23894 /* TCP_RACK_PACE_RATE_REC, TCP_RACK_PACE_RATE_SS, TCP_RACK_PACE_RATE_CA */
23895 if (dest->r_ctl.rc_fixed_pacing_rate_ca != src->r_ctl.rc_fixed_pacing_rate_ca) {
23896 dest->r_ctl.rc_fixed_pacing_rate_ca = src->r_ctl.rc_fixed_pacing_rate_ca;
23897 cnt++;
23898 }
23899 if (dest->r_ctl.rc_fixed_pacing_rate_ss != src->r_ctl.rc_fixed_pacing_rate_ss) {
23900 dest->r_ctl.rc_fixed_pacing_rate_ss = src->r_ctl.rc_fixed_pacing_rate_ss;
23901 cnt++;
23902 }
23903 if (dest->r_ctl.rc_fixed_pacing_rate_rec != src->r_ctl.rc_fixed_pacing_rate_rec) {
23904 dest->r_ctl.rc_fixed_pacing_rate_rec = src->r_ctl.rc_fixed_pacing_rate_rec;
23905 cnt++;
23906 }
23907 /* TCP_RACK_GP_INCREASE_REC, TCP_RACK_GP_INCREASE_CA, TCP_RACK_GP_INCREASE_SS */
23908 if (dest->r_ctl.rack_per_of_gp_rec != src->r_ctl.rack_per_of_gp_rec) {
23909 dest->r_ctl.rack_per_of_gp_rec = src->r_ctl.rack_per_of_gp_rec;
23910 cnt++;
23911 }
23912 if (dest->r_ctl.rack_per_of_gp_ca != src->r_ctl.rack_per_of_gp_ca) {
23913 dest->r_ctl.rack_per_of_gp_ca = src->r_ctl.rack_per_of_gp_ca;
23914 cnt++;
23915 }
23916
23917 if (dest->r_ctl.rack_per_of_gp_ss != src->r_ctl.rack_per_of_gp_ss) {
23918 dest->r_ctl.rack_per_of_gp_ss = src->r_ctl.rack_per_of_gp_ss;
23919 cnt++;
23920 }
23921 /* TCP_RACK_RR_CONF */
23922 if (dest->r_rr_config != src->r_rr_config) {
23923 dest->r_rr_config = src->r_rr_config;
23924 cnt++;
23925 }
23926 /* TCP_PACING_DND */
23927 if (dest->rc_pace_dnd != src->rc_pace_dnd) {
23928 dest->rc_pace_dnd = src->rc_pace_dnd;
23929 cnt++;
23930 }
23931 /* TCP_HDWR_RATE_CAP */
23932 if (dest->r_rack_hw_rate_caps != src->r_rack_hw_rate_caps) {
23933 dest->r_rack_hw_rate_caps = src->r_rack_hw_rate_caps;
23934 cnt++;
23935 }
23936 /* TCP_DGP_UPPER_BOUNDS */
23937 if (dest->r_ctl.rack_per_upper_bound_ca != src->r_ctl.rack_per_upper_bound_ca) {
23938 dest->r_ctl.rack_per_upper_bound_ca = src->r_ctl.rack_per_upper_bound_ca;
23939 cnt++;
23940 }
23941 if (dest->r_ctl.rack_per_upper_bound_ss != src->r_ctl.rack_per_upper_bound_ss) {
23942 dest->r_ctl.rack_per_upper_bound_ss = src->r_ctl.rack_per_upper_bound_ss;
23943 cnt++;
23944 }
23945 /* TCP_SS_EEXIT */
23946 if (dest->r_ctl.gp_rnd_thresh != src->r_ctl.gp_rnd_thresh) {
23947 dest->r_ctl.gp_rnd_thresh = src->r_ctl.gp_rnd_thresh;
23948 cnt++;
23949 }
23950 if (dest->r_ctl.gate_to_fs != src->r_ctl.gate_to_fs) {
23951 dest->r_ctl.gate_to_fs = src->r_ctl.gate_to_fs;
23952 cnt++;
23953 }
23954 if (dest->r_ctl.use_gp_not_last != src->r_ctl.use_gp_not_last) {
23955 dest->r_ctl.use_gp_not_last = src->r_ctl.use_gp_not_last;
23956 cnt++;
23957 }
23958 if (dest->r_ctl.gp_gain_req != src->r_ctl.gp_gain_req) {
23959 dest->r_ctl.gp_gain_req = src->r_ctl.gp_gain_req;
23960 cnt++;
23961 }
23962 /* TCP_BBR_HDWR_PACE */
23963 if (dest->rack_hdw_pace_ena != src->rack_hdw_pace_ena) {
23964 dest->rack_hdw_pace_ena = src->rack_hdw_pace_ena;
23965 cnt++;
23966 }
23967 if (dest->rack_attempt_hdwr_pace != src->rack_attempt_hdwr_pace) {
23968 dest->rack_attempt_hdwr_pace = src->rack_attempt_hdwr_pace;
23969 cnt++;
23970 }
23971 /* TCP_RACK_PRR_SENDALOT */
23972 if (dest->r_ctl.rc_prr_sendalot != src->r_ctl.rc_prr_sendalot) {
23973 dest->r_ctl.rc_prr_sendalot = src->r_ctl.rc_prr_sendalot;
23974 cnt++;
23975 }
23976 /* TCP_RACK_MIN_TO */
23977 if (dest->r_ctl.rc_min_to != src->r_ctl.rc_min_to) {
23978 dest->r_ctl.rc_min_to = src->r_ctl.rc_min_to;
23979 cnt++;
23980 }
23981 /* TCP_RACK_EARLY_SEG */
23982 if (dest->r_ctl.rc_early_recovery_segs != src->r_ctl.rc_early_recovery_segs) {
23983 dest->r_ctl.rc_early_recovery_segs = src->r_ctl.rc_early_recovery_segs;
23984 cnt++;
23985 }
23986 /* TCP_RACK_ENABLE_HYSTART */
23987 if (par->t_ccv.flags != tp->t_ccv.flags) {
23988 cnt++;
23989 if (par->t_ccv.flags & CCF_HYSTART_ALLOWED) {
23990 tp->t_ccv.flags |= CCF_HYSTART_ALLOWED;
23991 if (rack_do_hystart > RACK_HYSTART_ON)
23992 tp->t_ccv.flags |= CCF_HYSTART_CAN_SH_CWND;
23993 if (rack_do_hystart > RACK_HYSTART_ON_W_SC)
23994 tp->t_ccv.flags |= CCF_HYSTART_CONS_SSTH;
23995 } else {
23996 tp->t_ccv.flags &= ~(CCF_HYSTART_ALLOWED|CCF_HYSTART_CAN_SH_CWND|CCF_HYSTART_CONS_SSTH);
23997 }
23998 }
23999 /* TCP_RACK_REORD_THRESH */
24000 if (dest->r_ctl.rc_reorder_shift != src->r_ctl.rc_reorder_shift) {
24001 dest->r_ctl.rc_reorder_shift = src->r_ctl.rc_reorder_shift;
24002 cnt++;
24003 }
24004 /* TCP_RACK_REORD_FADE */
24005 if (dest->r_ctl.rc_reorder_fade != src->r_ctl.rc_reorder_fade) {
24006 dest->r_ctl.rc_reorder_fade = src->r_ctl.rc_reorder_fade;
24007 cnt++;
24008 }
24009 /* TCP_RACK_TLP_THRESH */
24010 if (dest->r_ctl.rc_tlp_threshold != src->r_ctl.rc_tlp_threshold) {
24011 dest->r_ctl.rc_tlp_threshold = src->r_ctl.rc_tlp_threshold;
24012 cnt++;
24013 }
24014 /* TCP_BBR_USE_RACK_RR */
24015 if (dest->use_rack_rr != src->use_rack_rr) {
24016 dest->use_rack_rr = src->use_rack_rr;
24017 cnt++;
24018 }
24019 /* TCP_RACK_PKT_DELAY */
24020 if (dest->r_ctl.rc_pkt_delay != src->r_ctl.rc_pkt_delay) {
24021 dest->r_ctl.rc_pkt_delay = src->r_ctl.rc_pkt_delay;
24022 cnt++;
24023 }
24024 /* TCP_DELACK will get copied via the main code if applicable */
24025 /* TCP_BBR_RACK_RTT_USE */
24026 if (dest->r_ctl.rc_rate_sample_method != src->r_ctl.rc_rate_sample_method) {
24027 dest->r_ctl.rc_rate_sample_method = src->r_ctl.rc_rate_sample_method;
24028 cnt++;
24029 }
24030 /* TCP_HONOR_HPTS_MIN */
24031 if (dest->r_use_hpts_min != src->r_use_hpts_min) {
24032 dest->r_use_hpts_min = src->r_use_hpts_min;
24033 cnt++;
24034 }
24035 if (dest->r_ctl.max_reduction != src->r_ctl.max_reduction) {
24036 dest->r_ctl.max_reduction = src->r_ctl.max_reduction;
24037 cnt++;
24038 }
24039 /* TCP_REC_IS_DYN */
24040 if (dest->rc_gp_no_rec_chg != src->rc_gp_no_rec_chg) {
24041 dest->rc_gp_no_rec_chg = src->rc_gp_no_rec_chg;
24042 cnt++;
24043 }
24044 if (dest->rc_skip_timely != src->rc_skip_timely) {
24045 dest->rc_skip_timely = src->rc_skip_timely;
24046 cnt++;
24047 }
24048 /* TCP_DATA_AFTER_CLOSE */
24049 if (dest->rc_allow_data_af_clo != src->rc_allow_data_af_clo) {
24050 dest->rc_allow_data_af_clo = src->rc_allow_data_af_clo;
24051 cnt++;
24052 }
24053 /* TCP_GP_USE_LTBW */
24054 if (src->use_lesser_lt_bw != dest->use_lesser_lt_bw) {
24055 dest->use_lesser_lt_bw = src->use_lesser_lt_bw;
24056 cnt++;
24057 }
24058 if (dest->dis_lt_bw != src->dis_lt_bw) {
24059 dest->dis_lt_bw = src->dis_lt_bw;
24060 cnt++;
24061 }
24062 tcp_log_socket_option(tp, 0, cnt, 0);
24063 }
24064
24065
24066 static void
rack_apply_deferred_options(struct tcp_rack * rack)24067 rack_apply_deferred_options(struct tcp_rack *rack)
24068 {
24069 struct deferred_opt_list *dol, *sdol;
24070 uint32_t s_optval;
24071
24072 TAILQ_FOREACH_SAFE(dol, &rack->r_ctl.opt_list, next, sdol) {
24073 TAILQ_REMOVE(&rack->r_ctl.opt_list, dol, next);
24074 /* Disadvantage of deferal is you loose the error return */
24075 s_optval = (uint32_t)dol->optval;
24076 (void)rack_process_option(rack->rc_tp, rack, dol->optname, s_optval, dol->optval, NULL);
24077 free(dol, M_TCPDO);
24078 }
24079 }
24080
24081 static void
rack_hw_tls_change(struct tcpcb * tp,int chg)24082 rack_hw_tls_change(struct tcpcb *tp, int chg)
24083 {
24084 /* Update HW tls state */
24085 struct tcp_rack *rack;
24086
24087 rack = (struct tcp_rack *)tp->t_fb_ptr;
24088 if (chg)
24089 rack->r_ctl.fsb.hw_tls = 1;
24090 else
24091 rack->r_ctl.fsb.hw_tls = 0;
24092 }
24093
24094 static int
rack_pru_options(struct tcpcb * tp,int flags)24095 rack_pru_options(struct tcpcb *tp, int flags)
24096 {
24097 if (flags & PRUS_OOB)
24098 return (EOPNOTSUPP);
24099 return (0);
24100 }
24101
24102 static bool
rack_wake_check(struct tcpcb * tp)24103 rack_wake_check(struct tcpcb *tp)
24104 {
24105 struct tcp_rack *rack;
24106 struct timeval tv;
24107 uint32_t cts;
24108
24109 rack = (struct tcp_rack *)tp->t_fb_ptr;
24110 if (rack->r_ctl.rc_hpts_flags) {
24111 cts = tcp_get_usecs(&tv);
24112 if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == PACE_PKT_OUTPUT){
24113 /*
24114 * Pacing timer is up, check if we are ready.
24115 */
24116 if (TSTMP_GEQ(cts, rack->r_ctl.rc_last_output_to))
24117 return (true);
24118 } else if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) != 0) {
24119 /*
24120 * A timer is up, check if we are ready.
24121 */
24122 if (TSTMP_GEQ(cts, rack->r_ctl.rc_timer_exp))
24123 return (true);
24124 }
24125 }
24126 return (false);
24127 }
24128
24129 static struct tcp_function_block __tcp_rack = {
24130 .tfb_tcp_block_name = __XSTRING(STACKNAME),
24131 .tfb_tcp_output = rack_output,
24132 .tfb_do_queued_segments = ctf_do_queued_segments,
24133 .tfb_do_segment_nounlock = rack_do_segment_nounlock,
24134 .tfb_tcp_do_segment = rack_do_segment,
24135 .tfb_tcp_ctloutput = rack_ctloutput,
24136 .tfb_tcp_fb_init = rack_init,
24137 .tfb_tcp_fb_fini = rack_fini,
24138 .tfb_tcp_timer_stop_all = rack_stopall,
24139 .tfb_tcp_rexmit_tmr = rack_remxt_tmr,
24140 .tfb_tcp_handoff_ok = rack_handoff_ok,
24141 .tfb_tcp_mtu_chg = rack_mtu_change,
24142 .tfb_pru_options = rack_pru_options,
24143 .tfb_hwtls_change = rack_hw_tls_change,
24144 .tfb_chg_query = rack_chg_query,
24145 .tfb_switch_failed = rack_switch_failed,
24146 .tfb_early_wake_check = rack_wake_check,
24147 .tfb_compute_pipe = rack_compute_pipe,
24148 .tfb_stack_info = rack_stack_information,
24149 .tfb_inherit = rack_inherit,
24150 .tfb_flags = TCP_FUNC_OUTPUT_CANDROP | TCP_FUNC_DEFAULT_OK,
24151
24152 };
24153
24154 /*
24155 * rack_ctloutput() must drop the inpcb lock before performing copyin on
24156 * socket option arguments. When it re-acquires the lock after the copy, it
24157 * has to revalidate that the connection is still valid for the socket
24158 * option.
24159 */
24160 static int
rack_set_sockopt(struct tcpcb * tp,struct sockopt * sopt)24161 rack_set_sockopt(struct tcpcb *tp, struct sockopt *sopt)
24162 {
24163 struct inpcb *inp = tptoinpcb(tp);
24164 #ifdef INET
24165 struct ip *ip;
24166 #endif
24167 struct tcp_rack *rack;
24168 struct tcp_hybrid_req hybrid;
24169 uint64_t loptval;
24170 int32_t error = 0, optval;
24171
24172 rack = (struct tcp_rack *)tp->t_fb_ptr;
24173 if (rack == NULL) {
24174 INP_WUNLOCK(inp);
24175 return (EINVAL);
24176 }
24177 #ifdef INET
24178 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
24179 #endif
24180
24181 switch (sopt->sopt_level) {
24182 #ifdef INET6
24183 case IPPROTO_IPV6:
24184 MPASS(inp->inp_vflag & INP_IPV6PROTO);
24185 switch (sopt->sopt_name) {
24186 case IPV6_USE_MIN_MTU:
24187 tcp6_use_min_mtu(tp);
24188 break;
24189 }
24190 INP_WUNLOCK(inp);
24191 return (0);
24192 #endif
24193 #ifdef INET
24194 case IPPROTO_IP:
24195 switch (sopt->sopt_name) {
24196 case IP_TOS:
24197 /*
24198 * The DSCP codepoint has changed, update the fsb.
24199 */
24200 ip->ip_tos = rack->rc_inp->inp_ip_tos;
24201 break;
24202 case IP_TTL:
24203 /*
24204 * The TTL has changed, update the fsb.
24205 */
24206 ip->ip_ttl = rack->rc_inp->inp_ip_ttl;
24207 break;
24208 }
24209 INP_WUNLOCK(inp);
24210 return (0);
24211 #endif
24212 #ifdef SO_PEERPRIO
24213 case SOL_SOCKET:
24214 switch (sopt->sopt_name) {
24215 case SO_PEERPRIO: /* SC-URL:bs */
24216 /* Already read in and sanity checked in sosetopt(). */
24217 if (inp->inp_socket) {
24218 rack->client_bufferlvl = inp->inp_socket->so_peerprio;
24219 }
24220 break;
24221 }
24222 INP_WUNLOCK(inp);
24223 return (0);
24224 #endif
24225 case IPPROTO_TCP:
24226 switch (sopt->sopt_name) {
24227 case TCP_RACK_TLP_REDUCE: /* URL:tlp_reduce */
24228 /* Pacing related ones */
24229 case TCP_RACK_PACE_ALWAYS: /* URL:pace_always */
24230 case TCP_BBR_RACK_INIT_RATE: /* URL:irate */
24231 case TCP_RACK_PACE_MIN_SEG: /* URL:pace_min_seg */
24232 case TCP_RACK_PACE_MAX_SEG: /* URL:pace_max_seg */
24233 case TCP_RACK_FORCE_MSEG: /* URL:force_max_seg */
24234 case TCP_RACK_PACE_RATE_CA: /* URL:pr_ca */
24235 case TCP_RACK_PACE_RATE_SS: /* URL:pr_ss*/
24236 case TCP_RACK_PACE_RATE_REC: /* URL:pr_rec */
24237 case TCP_RACK_GP_INCREASE_CA: /* URL:gp_inc_ca */
24238 case TCP_RACK_GP_INCREASE_SS: /* URL:gp_inc_ss */
24239 case TCP_RACK_GP_INCREASE_REC: /* URL:gp_inc_rec */
24240 case TCP_RACK_RR_CONF: /* URL:rrr_conf */
24241 case TCP_BBR_HDWR_PACE: /* URL:hdwrpace */
24242 case TCP_HDWR_RATE_CAP: /* URL:hdwrcap boolean */
24243 case TCP_PACING_RATE_CAP: /* URL:cap -- used by side-channel */
24244 case TCP_HDWR_UP_ONLY: /* URL:uponly -- hardware pacing boolean */
24245 case TCP_FILLCW_RATE_CAP: /* URL:fillcw_cap */
24246 case TCP_RACK_PACING_BETA_ECN: /* URL:pacing_beta_ecn */
24247 case TCP_RACK_PACE_TO_FILL: /* URL:fillcw */
24248 /* End pacing related */
24249 case TCP_DELACK: /* URL:delack (in base TCP i.e. tcp_hints along with cc etc ) */
24250 case TCP_RACK_PRR_SENDALOT: /* URL:prr_sendalot */
24251 case TCP_RACK_MIN_TO: /* URL:min_to */
24252 case TCP_RACK_EARLY_SEG: /* URL:early_seg */
24253 case TCP_RACK_REORD_THRESH: /* URL:reord_thresh */
24254 case TCP_RACK_REORD_FADE: /* URL:reord_fade */
24255 case TCP_RACK_TLP_THRESH: /* URL:tlp_thresh */
24256 case TCP_RACK_PKT_DELAY: /* URL:pkt_delay */
24257 case TCP_RACK_TLP_USE: /* URL:tlp_use */
24258 case TCP_BBR_RACK_RTT_USE: /* URL:rttuse */
24259 case TCP_BBR_USE_RACK_RR: /* URL:rackrr */
24260 case TCP_NO_PRR: /* URL:noprr */
24261 case TCP_TIMELY_DYN_ADJ: /* URL:dynamic */
24262 case TCP_DATA_AFTER_CLOSE: /* no URL */
24263 case TCP_RACK_NONRXT_CFG_RATE: /* URL:nonrxtcr */
24264 case TCP_SHARED_CWND_ENABLE: /* URL:scwnd */
24265 case TCP_RACK_MBUF_QUEUE: /* URL:mqueue */
24266 case TCP_RACK_NO_PUSH_AT_MAX: /* URL:npush */
24267 case TCP_SHARED_CWND_TIME_LIMIT: /* URL:lscwnd */
24268 case TCP_RACK_PROFILE: /* URL:profile */
24269 case TCP_SIDECHAN_DIS: /* URL:scodm */
24270 case TCP_HYBRID_PACING: /* URL:pacing=hybrid */
24271 case TCP_USE_CMP_ACKS: /* URL:cmpack */
24272 case TCP_RACK_ABC_VAL: /* URL:labc */
24273 case TCP_REC_ABC_VAL: /* URL:reclabc */
24274 case TCP_RACK_MEASURE_CNT: /* URL:measurecnt */
24275 case TCP_DEFER_OPTIONS: /* URL:defer */
24276 case TCP_RACK_DSACK_OPT: /* URL:dsack */
24277 case TCP_RACK_TIMER_SLOP: /* URL:timer_slop */
24278 case TCP_RACK_ENABLE_HYSTART: /* URL:hystart */
24279 case TCP_RACK_SET_RXT_OPTIONS: /* URL:rxtsz */
24280 case TCP_RACK_HI_BETA: /* URL:hibeta */
24281 case TCP_RACK_SPLIT_LIMIT: /* URL:split */
24282 case TCP_SS_EEXIT: /* URL:eexit */
24283 case TCP_DGP_UPPER_BOUNDS: /* URL:upper */
24284 case TCP_RACK_PACING_DIVISOR: /* URL:divisor */
24285 case TCP_PACING_DND: /* URL:dnd */
24286 case TCP_NO_TIMELY: /* URL:notimely */
24287 case RACK_CSPR_IS_FCC: /* URL:csprisfcc */
24288 case TCP_HONOR_HPTS_MIN: /* URL:hptsmin */
24289 case TCP_REC_IS_DYN: /* URL:dynrec */
24290 case TCP_GP_USE_LTBW: /* URL:useltbw */
24291 goto process_opt;
24292 break;
24293 default:
24294 /* Filter off all unknown options to the base stack */
24295 return (tcp_default_ctloutput(tp, sopt));
24296 break;
24297 }
24298 default:
24299 INP_WUNLOCK(inp);
24300 return (0);
24301 }
24302 process_opt:
24303 INP_WUNLOCK(inp);
24304 if ((sopt->sopt_name == TCP_PACING_RATE_CAP) ||
24305 (sopt->sopt_name == TCP_FILLCW_RATE_CAP)) {
24306 error = sooptcopyin(sopt, &loptval, sizeof(loptval), sizeof(loptval));
24307 /*
24308 * We truncate it down to 32 bits for the socket-option trace this
24309 * means rates > 34Gbps won't show right, but thats probably ok.
24310 */
24311 optval = (uint32_t)loptval;
24312 } else if (sopt->sopt_name == TCP_HYBRID_PACING) {
24313 error = sooptcopyin(sopt, &hybrid, sizeof(hybrid), sizeof(hybrid));
24314 } else {
24315 error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval));
24316 /* Save it in 64 bit form too */
24317 loptval = optval;
24318 }
24319 if (error)
24320 return (error);
24321 INP_WLOCK(inp);
24322 if (tp->t_fb != &__tcp_rack) {
24323 INP_WUNLOCK(inp);
24324 return (ENOPROTOOPT);
24325 }
24326 if (rack->defer_options && (rack->gp_ready == 0) &&
24327 (sopt->sopt_name != TCP_DEFER_OPTIONS) &&
24328 (sopt->sopt_name != TCP_HYBRID_PACING) &&
24329 (sopt->sopt_name != TCP_RACK_SET_RXT_OPTIONS) &&
24330 (sopt->sopt_name != TCP_RACK_PACING_BETA_ECN) &&
24331 (sopt->sopt_name != TCP_RACK_MEASURE_CNT)) {
24332 /* Options are being deferred */
24333 if (rack_add_deferred_option(rack, sopt->sopt_name, loptval)) {
24334 INP_WUNLOCK(inp);
24335 return (0);
24336 } else {
24337 /* No memory to defer, fail */
24338 INP_WUNLOCK(inp);
24339 return (ENOMEM);
24340 }
24341 }
24342 error = rack_process_option(tp, rack, sopt->sopt_name, optval, loptval, &hybrid);
24343 INP_WUNLOCK(inp);
24344 return (error);
24345 }
24346
24347 static void
rack_fill_info(struct tcpcb * tp,struct tcp_info * ti)24348 rack_fill_info(struct tcpcb *tp, struct tcp_info *ti)
24349 {
24350
24351 INP_WLOCK_ASSERT(tptoinpcb(tp));
24352 bzero(ti, sizeof(*ti));
24353
24354 ti->tcpi_state = tp->t_state;
24355 if ((tp->t_flags & TF_REQ_TSTMP) && (tp->t_flags & TF_RCVD_TSTMP))
24356 ti->tcpi_options |= TCPI_OPT_TIMESTAMPS;
24357 if (tp->t_flags & TF_SACK_PERMIT)
24358 ti->tcpi_options |= TCPI_OPT_SACK;
24359 if ((tp->t_flags & TF_REQ_SCALE) && (tp->t_flags & TF_RCVD_SCALE)) {
24360 ti->tcpi_options |= TCPI_OPT_WSCALE;
24361 ti->tcpi_snd_wscale = tp->snd_scale;
24362 ti->tcpi_rcv_wscale = tp->rcv_scale;
24363 }
24364 if (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))
24365 ti->tcpi_options |= TCPI_OPT_ECN;
24366 if (tp->t_flags & TF_FASTOPEN)
24367 ti->tcpi_options |= TCPI_OPT_TFO;
24368 /* still kept in ticks is t_rcvtime */
24369 ti->tcpi_last_data_recv = ((uint32_t)ticks - tp->t_rcvtime) * tick;
24370 /* Since we hold everything in precise useconds this is easy */
24371 ti->tcpi_rtt = tp->t_srtt;
24372 ti->tcpi_rttvar = tp->t_rttvar;
24373 ti->tcpi_rto = tp->t_rxtcur;
24374 ti->tcpi_snd_ssthresh = tp->snd_ssthresh;
24375 ti->tcpi_snd_cwnd = tp->snd_cwnd;
24376 /*
24377 * FreeBSD-specific extension fields for tcp_info.
24378 */
24379 ti->tcpi_rcv_space = tp->rcv_wnd;
24380 ti->tcpi_rcv_nxt = tp->rcv_nxt;
24381 ti->tcpi_snd_wnd = tp->snd_wnd;
24382 ti->tcpi_snd_bwnd = 0; /* Unused, kept for compat. */
24383 ti->tcpi_snd_nxt = tp->snd_nxt;
24384 ti->tcpi_snd_mss = tp->t_maxseg;
24385 ti->tcpi_rcv_mss = tp->t_maxseg;
24386 ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack;
24387 ti->tcpi_rcv_ooopack = tp->t_rcvoopack;
24388 ti->tcpi_snd_zerowin = tp->t_sndzerowin;
24389 ti->tcpi_total_tlp = tp->t_sndtlppack;
24390 ti->tcpi_total_tlp_bytes = tp->t_sndtlpbyte;
24391 ti->tcpi_rttmin = tp->t_rttlow;
24392 #ifdef NETFLIX_STATS
24393 memcpy(&ti->tcpi_rxsyninfo, &tp->t_rxsyninfo, sizeof(struct tcpsyninfo));
24394 #endif
24395 #ifdef TCP_OFFLOAD
24396 if (tp->t_flags & TF_TOE) {
24397 ti->tcpi_options |= TCPI_OPT_TOE;
24398 tcp_offload_tcp_info(tp, ti);
24399 }
24400 #endif
24401 }
24402
24403 static int
rack_get_sockopt(struct tcpcb * tp,struct sockopt * sopt)24404 rack_get_sockopt(struct tcpcb *tp, struct sockopt *sopt)
24405 {
24406 struct inpcb *inp = tptoinpcb(tp);
24407 struct tcp_rack *rack;
24408 int32_t error, optval;
24409 uint64_t val, loptval;
24410 struct tcp_info ti;
24411 /*
24412 * Because all our options are either boolean or an int, we can just
24413 * pull everything into optval and then unlock and copy. If we ever
24414 * add a option that is not a int, then this will have quite an
24415 * impact to this routine.
24416 */
24417 error = 0;
24418 rack = (struct tcp_rack *)tp->t_fb_ptr;
24419 if (rack == NULL) {
24420 INP_WUNLOCK(inp);
24421 return (EINVAL);
24422 }
24423 switch (sopt->sopt_name) {
24424 case TCP_INFO:
24425 /* First get the info filled */
24426 rack_fill_info(tp, &ti);
24427 /* Fix up the rtt related fields if needed */
24428 INP_WUNLOCK(inp);
24429 error = sooptcopyout(sopt, &ti, sizeof ti);
24430 return (error);
24431 /*
24432 * Beta is the congestion control value for NewReno that influences how
24433 * much of a backoff happens when loss is detected. It is normally set
24434 * to 50 for 50% i.e. the cwnd is reduced to 50% of its previous value
24435 * when you exit recovery.
24436 */
24437 case TCP_RACK_PACING_BETA:
24438 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0)
24439 error = EINVAL;
24440 else if (rack->rc_pacing_cc_set == 0)
24441 optval = rack->r_ctl.rc_saved_beta;
24442 else {
24443 /*
24444 * Reach out into the CC data and report back what
24445 * I have previously set. Yeah it looks hackish but
24446 * we don't want to report the saved values.
24447 */
24448 if (tp->t_ccv.cc_data)
24449 optval = ((struct newreno *)tp->t_ccv.cc_data)->beta;
24450 else
24451 error = EINVAL;
24452 }
24453 break;
24454 /*
24455 * Beta_ecn is the congestion control value for NewReno that influences how
24456 * much of a backoff happens when a ECN mark is detected. It is normally set
24457 * to 80 for 80% i.e. the cwnd is reduced by 20% of its previous value when
24458 * you exit recovery. Note that classic ECN has a beta of 50, it is only
24459 * ABE Ecn that uses this "less" value, but we do too with pacing :)
24460 */
24461 case TCP_RACK_PACING_BETA_ECN:
24462 if (strcmp(tp->t_cc->name, CCALGONAME_NEWRENO) != 0)
24463 error = EINVAL;
24464 else if (rack->rc_pacing_cc_set == 0)
24465 optval = rack->r_ctl.rc_saved_beta_ecn;
24466 else {
24467 /*
24468 * Reach out into the CC data and report back what
24469 * I have previously set. Yeah it looks hackish but
24470 * we don't want to report the saved values.
24471 */
24472 if (tp->t_ccv.cc_data)
24473 optval = ((struct newreno *)tp->t_ccv.cc_data)->beta_ecn;
24474 else
24475 error = EINVAL;
24476 }
24477 break;
24478 case TCP_RACK_DSACK_OPT:
24479 optval = 0;
24480 if (rack->rc_rack_tmr_std_based) {
24481 optval |= 1;
24482 }
24483 if (rack->rc_rack_use_dsack) {
24484 optval |= 2;
24485 }
24486 break;
24487 case TCP_RACK_ENABLE_HYSTART:
24488 {
24489 if (tp->t_ccv.flags & CCF_HYSTART_ALLOWED) {
24490 optval = RACK_HYSTART_ON;
24491 if (tp->t_ccv.flags & CCF_HYSTART_CAN_SH_CWND)
24492 optval = RACK_HYSTART_ON_W_SC;
24493 if (tp->t_ccv.flags & CCF_HYSTART_CONS_SSTH)
24494 optval = RACK_HYSTART_ON_W_SC_C;
24495 } else {
24496 optval = RACK_HYSTART_OFF;
24497 }
24498 }
24499 break;
24500 case TCP_RACK_DGP_IN_REC:
24501 error = EINVAL;
24502 break;
24503 case TCP_RACK_HI_BETA:
24504 optval = rack->rack_hibeta;
24505 break;
24506 case TCP_DEFER_OPTIONS:
24507 optval = rack->defer_options;
24508 break;
24509 case TCP_RACK_MEASURE_CNT:
24510 optval = rack->r_ctl.req_measurements;
24511 break;
24512 case TCP_REC_ABC_VAL:
24513 optval = rack->r_use_labc_for_rec;
24514 break;
24515 case TCP_RACK_ABC_VAL:
24516 optval = rack->rc_labc;
24517 break;
24518 case TCP_HDWR_UP_ONLY:
24519 optval= rack->r_up_only;
24520 break;
24521 case TCP_FILLCW_RATE_CAP:
24522 loptval = rack->r_ctl.fillcw_cap;
24523 break;
24524 case TCP_PACING_RATE_CAP:
24525 loptval = rack->r_ctl.bw_rate_cap;
24526 break;
24527 case TCP_RACK_PROFILE:
24528 /* You cannot retrieve a profile, its write only */
24529 error = EINVAL;
24530 break;
24531 case TCP_SIDECHAN_DIS:
24532 optval = rack->r_ctl.side_chan_dis_mask;
24533 break;
24534 case TCP_HYBRID_PACING:
24535 /* You cannot retrieve hybrid pacing information, its write only */
24536 error = EINVAL;
24537 break;
24538 case TCP_USE_CMP_ACKS:
24539 optval = rack->r_use_cmp_ack;
24540 break;
24541 case TCP_RACK_PACE_TO_FILL:
24542 optval = rack->rc_pace_to_cwnd;
24543 break;
24544 case TCP_RACK_NO_PUSH_AT_MAX:
24545 optval = rack->r_ctl.rc_no_push_at_mrtt;
24546 break;
24547 case TCP_SHARED_CWND_ENABLE:
24548 optval = rack->rack_enable_scwnd;
24549 break;
24550 case TCP_RACK_NONRXT_CFG_RATE:
24551 optval = rack->rack_rec_nonrxt_use_cr;
24552 break;
24553 case TCP_NO_PRR:
24554 if (rack->rack_no_prr == 1)
24555 optval = 1;
24556 else if (rack->no_prr_addback == 1)
24557 optval = 2;
24558 else
24559 optval = 0;
24560 break;
24561 case TCP_GP_USE_LTBW:
24562 if (rack->dis_lt_bw) {
24563 /* It is not used */
24564 optval = 0;
24565 } else if (rack->use_lesser_lt_bw) {
24566 /* we use min() */
24567 optval = 1;
24568 } else {
24569 /* we use max() */
24570 optval = 2;
24571 }
24572 break;
24573 case TCP_RACK_DO_DETECTION:
24574 error = EINVAL;
24575 break;
24576 case TCP_RACK_MBUF_QUEUE:
24577 /* Now do we use the LRO mbuf-queue feature */
24578 optval = rack->r_mbuf_queue;
24579 break;
24580 case RACK_CSPR_IS_FCC:
24581 optval = rack->cspr_is_fcc;
24582 break;
24583 case TCP_TIMELY_DYN_ADJ:
24584 optval = rack->rc_gp_dyn_mul;
24585 break;
24586 case TCP_BBR_IWINTSO:
24587 error = EINVAL;
24588 break;
24589 case TCP_RACK_TLP_REDUCE:
24590 /* RACK TLP cwnd reduction (bool) */
24591 optval = rack->r_ctl.rc_tlp_cwnd_reduce;
24592 break;
24593 case TCP_BBR_RACK_INIT_RATE:
24594 val = rack->r_ctl.init_rate;
24595 /* convert to kbits per sec */
24596 val *= 8;
24597 val /= 1000;
24598 optval = (uint32_t)val;
24599 break;
24600 case TCP_RACK_FORCE_MSEG:
24601 optval = rack->rc_force_max_seg;
24602 break;
24603 case TCP_RACK_PACE_MIN_SEG:
24604 optval = rack->r_ctl.rc_user_set_min_segs;
24605 break;
24606 case TCP_RACK_PACE_MAX_SEG:
24607 /* Max segments in a pace */
24608 optval = rack->rc_user_set_max_segs;
24609 break;
24610 case TCP_RACK_PACE_ALWAYS:
24611 /* Use the always pace method */
24612 optval = rack->rc_always_pace;
24613 break;
24614 case TCP_RACK_PRR_SENDALOT:
24615 /* Allow PRR to send more than one seg */
24616 optval = rack->r_ctl.rc_prr_sendalot;
24617 break;
24618 case TCP_RACK_MIN_TO:
24619 /* Minimum time between rack t-o's in ms */
24620 optval = rack->r_ctl.rc_min_to;
24621 break;
24622 case TCP_RACK_SPLIT_LIMIT:
24623 optval = rack->r_ctl.rc_split_limit;
24624 break;
24625 case TCP_RACK_EARLY_SEG:
24626 /* If early recovery max segments */
24627 optval = rack->r_ctl.rc_early_recovery_segs;
24628 break;
24629 case TCP_RACK_REORD_THRESH:
24630 /* RACK reorder threshold (shift amount) */
24631 optval = rack->r_ctl.rc_reorder_shift;
24632 break;
24633 case TCP_SS_EEXIT:
24634 if (rack->r_ctl.gp_rnd_thresh) {
24635 uint32_t v;
24636
24637 v = rack->r_ctl.gp_gain_req;
24638 v <<= 17;
24639 optval = v | (rack->r_ctl.gp_rnd_thresh & 0xff);
24640 if (rack->r_ctl.gate_to_fs == 1)
24641 optval |= 0x10000;
24642 } else
24643 optval = 0;
24644 break;
24645 case TCP_RACK_REORD_FADE:
24646 /* Does reordering fade after ms time */
24647 optval = rack->r_ctl.rc_reorder_fade;
24648 break;
24649 case TCP_BBR_USE_RACK_RR:
24650 /* Do we use the rack cheat for rxt */
24651 optval = rack->use_rack_rr;
24652 break;
24653 case TCP_RACK_RR_CONF:
24654 optval = rack->r_rr_config;
24655 break;
24656 case TCP_HDWR_RATE_CAP:
24657 optval = rack->r_rack_hw_rate_caps;
24658 break;
24659 case TCP_BBR_HDWR_PACE:
24660 optval = rack->rack_hdw_pace_ena;
24661 break;
24662 case TCP_RACK_TLP_THRESH:
24663 /* RACK TLP theshold i.e. srtt+(srtt/N) */
24664 optval = rack->r_ctl.rc_tlp_threshold;
24665 break;
24666 case TCP_RACK_PKT_DELAY:
24667 /* RACK added ms i.e. rack-rtt + reord + N */
24668 optval = rack->r_ctl.rc_pkt_delay;
24669 break;
24670 case TCP_RACK_TLP_USE:
24671 optval = rack->rack_tlp_threshold_use;
24672 break;
24673 case TCP_PACING_DND:
24674 optval = rack->rc_pace_dnd;
24675 break;
24676 case TCP_RACK_PACE_RATE_CA:
24677 optval = rack->r_ctl.rc_fixed_pacing_rate_ca;
24678 break;
24679 case TCP_RACK_PACE_RATE_SS:
24680 optval = rack->r_ctl.rc_fixed_pacing_rate_ss;
24681 break;
24682 case TCP_RACK_PACE_RATE_REC:
24683 optval = rack->r_ctl.rc_fixed_pacing_rate_rec;
24684 break;
24685 case TCP_DGP_UPPER_BOUNDS:
24686 optval = rack->r_ctl.rack_per_upper_bound_ss;
24687 optval <<= 16;
24688 optval |= rack->r_ctl.rack_per_upper_bound_ca;
24689 break;
24690 case TCP_RACK_GP_INCREASE_SS:
24691 optval = rack->r_ctl.rack_per_of_gp_ca;
24692 break;
24693 case TCP_RACK_GP_INCREASE_CA:
24694 optval = rack->r_ctl.rack_per_of_gp_ss;
24695 break;
24696 case TCP_RACK_PACING_DIVISOR:
24697 optval = rack->r_ctl.pace_len_divisor;
24698 break;
24699 case TCP_BBR_RACK_RTT_USE:
24700 optval = rack->r_ctl.rc_rate_sample_method;
24701 break;
24702 case TCP_DELACK:
24703 optval = tp->t_delayed_ack;
24704 break;
24705 case TCP_DATA_AFTER_CLOSE:
24706 optval = rack->rc_allow_data_af_clo;
24707 break;
24708 case TCP_SHARED_CWND_TIME_LIMIT:
24709 optval = rack->r_limit_scw;
24710 break;
24711 case TCP_HONOR_HPTS_MIN:
24712 if (rack->r_use_hpts_min)
24713 optval = rack->r_ctl.max_reduction;
24714 else
24715 optval = 0;
24716 break;
24717 case TCP_REC_IS_DYN:
24718 optval = rack->rc_gp_no_rec_chg;
24719 break;
24720 case TCP_NO_TIMELY:
24721 optval = rack->rc_skip_timely;
24722 break;
24723 case TCP_RACK_TIMER_SLOP:
24724 optval = rack->r_ctl.timer_slop;
24725 break;
24726 default:
24727 return (tcp_default_ctloutput(tp, sopt));
24728 break;
24729 }
24730 INP_WUNLOCK(inp);
24731 if (error == 0) {
24732 if ((sopt->sopt_name == TCP_PACING_RATE_CAP) ||
24733 (sopt->sopt_name == TCP_FILLCW_RATE_CAP))
24734 error = sooptcopyout(sopt, &loptval, sizeof loptval);
24735 else
24736 error = sooptcopyout(sopt, &optval, sizeof optval);
24737 }
24738 return (error);
24739 }
24740
24741 static int
rack_ctloutput(struct tcpcb * tp,struct sockopt * sopt)24742 rack_ctloutput(struct tcpcb *tp, struct sockopt *sopt)
24743 {
24744 if (sopt->sopt_dir == SOPT_SET) {
24745 return (rack_set_sockopt(tp, sopt));
24746 } else if (sopt->sopt_dir == SOPT_GET) {
24747 return (rack_get_sockopt(tp, sopt));
24748 } else {
24749 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
24750 }
24751 }
24752
24753 static const char *rack_stack_names[] = {
24754 __XSTRING(STACKNAME),
24755 #ifdef STACKALIAS
24756 __XSTRING(STACKALIAS),
24757 #endif
24758 };
24759
24760 static int
rack_ctor(void * mem,int32_t size,void * arg,int32_t how)24761 rack_ctor(void *mem, int32_t size, void *arg, int32_t how)
24762 {
24763 memset(mem, 0, size);
24764 return (0);
24765 }
24766
24767 static void
rack_dtor(void * mem,int32_t size,void * arg)24768 rack_dtor(void *mem, int32_t size, void *arg)
24769 {
24770
24771 }
24772
24773 static bool rack_mod_inited = false;
24774
24775 static int
tcp_addrack(module_t mod,int32_t type,void * data)24776 tcp_addrack(module_t mod, int32_t type, void *data)
24777 {
24778 int32_t err = 0;
24779 int num_stacks;
24780
24781 switch (type) {
24782 case MOD_LOAD:
24783 rack_zone = uma_zcreate(__XSTRING(MODNAME) "_map",
24784 sizeof(struct rack_sendmap),
24785 rack_ctor, rack_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
24786
24787 rack_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb",
24788 sizeof(struct tcp_rack),
24789 rack_ctor, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
24790
24791 sysctl_ctx_init(&rack_sysctl_ctx);
24792 rack_sysctl_root = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
24793 SYSCTL_STATIC_CHILDREN(_net_inet_tcp),
24794 OID_AUTO,
24795 #ifdef STACKALIAS
24796 __XSTRING(STACKALIAS),
24797 #else
24798 __XSTRING(STACKNAME),
24799 #endif
24800 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
24801 "");
24802 if (rack_sysctl_root == NULL) {
24803 printf("Failed to add sysctl node\n");
24804 err = EFAULT;
24805 goto free_uma;
24806 }
24807 rack_init_sysctls();
24808 num_stacks = nitems(rack_stack_names);
24809 err = register_tcp_functions_as_names(&__tcp_rack, M_WAITOK,
24810 rack_stack_names, &num_stacks);
24811 if (err) {
24812 printf("Failed to register %s stack name for "
24813 "%s module\n", rack_stack_names[num_stacks],
24814 __XSTRING(MODNAME));
24815 sysctl_ctx_free(&rack_sysctl_ctx);
24816 free_uma:
24817 uma_zdestroy(rack_zone);
24818 uma_zdestroy(rack_pcb_zone);
24819 rack_counter_destroy();
24820 printf("Failed to register rack module -- err:%d\n", err);
24821 return (err);
24822 }
24823 tcp_lro_reg_mbufq();
24824 rack_mod_inited = true;
24825 break;
24826 case MOD_QUIESCE:
24827 err = deregister_tcp_functions(&__tcp_rack, true, false);
24828 break;
24829 case MOD_UNLOAD:
24830 err = deregister_tcp_functions(&__tcp_rack, false, true);
24831 if (err == EBUSY)
24832 break;
24833 if (rack_mod_inited) {
24834 uma_zdestroy(rack_zone);
24835 uma_zdestroy(rack_pcb_zone);
24836 sysctl_ctx_free(&rack_sysctl_ctx);
24837 rack_counter_destroy();
24838 rack_mod_inited = false;
24839 }
24840 tcp_lro_dereg_mbufq();
24841 err = 0;
24842 break;
24843 default:
24844 return (EOPNOTSUPP);
24845 }
24846 return (err);
24847 }
24848
24849 static moduledata_t tcp_rack = {
24850 .name = __XSTRING(MODNAME),
24851 .evhand = tcp_addrack,
24852 .priv = 0
24853 };
24854
24855 MODULE_VERSION(MODNAME, 1);
24856 DECLARE_MODULE(MODNAME, tcp_rack, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
24857 MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1);
24858
24859 #endif /* #if !defined(INET) && !defined(INET6) */
24860