1 /*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
26 /**
27 * Author: Randall Stewart <rrs@netflix.com>
28 * This work is based on the ACM Queue paper
29 * BBR - Congestion Based Congestion Control
30 * and also numerous discussions with Neal, Yuchung and Van.
31 */
32
33 #include <sys/cdefs.h>
34 #include "opt_inet.h"
35 #include "opt_inet6.h"
36 #include "opt_ipsec.h"
37 #include "opt_ratelimit.h"
38 #include <sys/param.h>
39 #include <sys/arb.h>
40 #include <sys/module.h>
41 #include <sys/kernel.h>
42 #include <sys/libkern.h>
43 #ifdef TCP_HHOOK
44 #include <sys/hhook.h>
45 #endif
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/proc.h>
49 #include <sys/socket.h>
50 #include <sys/socketvar.h>
51 #include <sys/sysctl.h>
52 #include <sys/systm.h>
53 #ifdef STATS
54 #include <sys/qmath.h>
55 #include <sys/tree.h>
56 #include <sys/stats.h> /* Must come after qmath.h and tree.h */
57 #endif
58 #include <sys/refcount.h>
59 #include <sys/queue.h>
60 #include <sys/eventhandler.h>
61 #include <sys/smp.h>
62 #include <sys/kthread.h>
63 #include <sys/lock.h>
64 #include <sys/mutex.h>
65 #include <sys/tim_filter.h>
66 #include <sys/time.h>
67 #include <sys/protosw.h>
68 #include <vm/uma.h>
69 #include <sys/kern_prefetch.h>
70
71 #include <net/route.h>
72 #include <net/route/nhop.h>
73 #include <net/vnet.h>
74
75 #define TCPSTATES /* for logging */
76
77 #include <netinet/in.h>
78 #include <netinet/in_kdtrace.h>
79 #include <netinet/in_pcb.h>
80 #include <netinet/ip.h>
81 #include <netinet/ip_var.h>
82 #include <netinet/ip6.h>
83 #include <netinet6/in6_pcb.h>
84 #include <netinet6/ip6_var.h>
85 #define TCPOUTFLAGS
86 #include <netinet/tcp.h>
87 #include <netinet/tcp_fsm.h>
88 #include <netinet/tcp_seq.h>
89 #include <netinet/tcp_timer.h>
90 #include <netinet/tcp_var.h>
91 #include <netinet/tcpip.h>
92 #include <netinet/tcp_hpts.h>
93 #include <netinet/cc/cc.h>
94 #include <netinet/tcp_log_buf.h>
95 #include <netinet/tcp_ratelimit.h>
96 #include <netinet/tcp_lro.h>
97 #ifdef TCP_OFFLOAD
98 #include <netinet/tcp_offload.h>
99 #endif
100 #ifdef INET6
101 #include <netinet6/tcp6_var.h>
102 #endif
103 #include <netinet/tcp_fastopen.h>
104
105 #include <netipsec/ipsec_support.h>
106 #include <net/if.h>
107 #include <net/if_var.h>
108 #include <net/ethernet.h>
109
110 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
111 #include <netipsec/ipsec.h>
112 #include <netipsec/ipsec6.h>
113 #endif /* IPSEC */
114
115 #include <netinet/udp.h>
116 #include <netinet/udp_var.h>
117 #include <machine/in_cksum.h>
118
119 #ifdef MAC
120 #include <security/mac/mac_framework.h>
121 #endif
122
123 #include "sack_filter.h"
124 #include "tcp_bbr.h"
125 #include "rack_bbr_common.h"
126 uma_zone_t bbr_zone;
127 uma_zone_t bbr_pcb_zone;
128
129 struct sysctl_ctx_list bbr_sysctl_ctx;
130 struct sysctl_oid *bbr_sysctl_root;
131
132 #define TCPT_RANGESET_NOSLOP(tv, value, tvmin, tvmax) do { \
133 (tv) = (value); \
134 if ((u_long)(tv) < (u_long)(tvmin)) \
135 (tv) = (tvmin); \
136 if ((u_long)(tv) > (u_long)(tvmax)) \
137 (tv) = (tvmax); \
138 } while(0)
139
140 /*#define BBR_INVARIANT 1*/
141
142 /*
143 * initial window
144 */
145 static uint32_t bbr_def_init_win = 10;
146 static int32_t bbr_persist_min = 250000; /* 250ms */
147 static int32_t bbr_persist_max = 1000000; /* 1 Second */
148 static int32_t bbr_cwnd_may_shrink = 0;
149 static int32_t bbr_cwndtarget_rtt_touse = BBR_RTT_PROP;
150 static int32_t bbr_num_pktepo_for_del_limit = BBR_NUM_RTTS_FOR_DEL_LIMIT;
151 static int32_t bbr_hardware_pacing_limit = 8000;
152 static int32_t bbr_quanta = 3; /* How much extra quanta do we get? */
153 static int32_t bbr_no_retran = 0;
154
155 static int32_t bbr_error_base_paceout = 10000; /* usec to pace */
156 static int32_t bbr_max_net_error_cnt = 10;
157 /* Should the following be dynamic too -- loss wise */
158 static int32_t bbr_rtt_gain_thresh = 0;
159 /* Measurement controls */
160 static int32_t bbr_use_google_algo = 1;
161 static int32_t bbr_ts_limiting = 1;
162 static int32_t bbr_ts_can_raise = 0;
163 static int32_t bbr_do_red = 600;
164 static int32_t bbr_red_scale = 20000;
165 static int32_t bbr_red_mul = 1;
166 static int32_t bbr_red_div = 2;
167 static int32_t bbr_red_growth_restrict = 1;
168 static int32_t bbr_target_is_bbunit = 0;
169 static int32_t bbr_drop_limit = 0;
170 /*
171 * How much gain do we need to see to
172 * stay in startup?
173 */
174 static int32_t bbr_marks_rxt_sack_passed = 0;
175 static int32_t bbr_start_exit = 25;
176 static int32_t bbr_low_start_exit = 25; /* When we are in reduced gain */
177 static int32_t bbr_startup_loss_thresh = 2000; /* 20.00% loss */
178 static int32_t bbr_hptsi_max_mul = 1; /* These two mul/div assure a min pacing */
179 static int32_t bbr_hptsi_max_div = 2; /* time, 0 means turned off. We need this
180 * if we go back ever to where the pacer
181 * has priority over timers.
182 */
183 static int32_t bbr_policer_call_from_rack_to = 0;
184 static int32_t bbr_policer_detection_enabled = 1;
185 static int32_t bbr_min_measurements_req = 1; /* We need at least 2
186 * measurements before we are
187 * "good" note that 2 == 1.
188 * This is because we use a >
189 * comparison. This means if
190 * min_measure was 0, it takes
191 * num-measures > min(0) and
192 * you get 1 measurement and
193 * you are good. Set to 1, you
194 * have to have two
195 * measurements (this is done
196 * to prevent it from being ok
197 * to have no measurements). */
198 static int32_t bbr_no_pacing_until = 4;
199
200 static int32_t bbr_min_usec_delta = 20000; /* 20,000 usecs */
201 static int32_t bbr_min_peer_delta = 20; /* 20 units */
202 static int32_t bbr_delta_percent = 150; /* 15.0 % */
203
204 static int32_t bbr_target_cwnd_mult_limit = 8;
205 /*
206 * bbr_cwnd_min_val is the number of
207 * segments we hold to in the RTT probe
208 * state typically 4.
209 */
210 static int32_t bbr_cwnd_min_val = BBR_PROBERTT_NUM_MSS;
211
212 static int32_t bbr_cwnd_min_val_hs = BBR_HIGHSPEED_NUM_MSS;
213
214 static int32_t bbr_gain_to_target = 1;
215 static int32_t bbr_gain_gets_extra_too = 1;
216 /*
217 * bbr_high_gain is the 2/ln(2) value we need
218 * to double the sending rate in startup. This
219 * is used for both cwnd and hptsi gain's.
220 */
221 static int32_t bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1;
222 static int32_t bbr_startup_lower = BBR_UNIT * 1500 / 1000 + 1;
223 static int32_t bbr_use_lower_gain_in_startup = 1;
224
225 /* thresholds for reduction on drain in sub-states/drain */
226 static int32_t bbr_drain_rtt = BBR_SRTT;
227 static int32_t bbr_drain_floor = 88;
228 static int32_t google_allow_early_out = 1;
229 static int32_t google_consider_lost = 1;
230 static int32_t bbr_drain_drop_mul = 4;
231 static int32_t bbr_drain_drop_div = 5;
232 static int32_t bbr_rand_ot = 50;
233 static int32_t bbr_can_force_probertt = 0;
234 static int32_t bbr_can_adjust_probertt = 1;
235 static int32_t bbr_probertt_sets_rtt = 0;
236 static int32_t bbr_can_use_ts_for_rtt = 1;
237 static int32_t bbr_is_ratio = 0;
238 static int32_t bbr_sub_drain_app_limit = 1;
239 static int32_t bbr_prtt_slam_cwnd = 1;
240 static int32_t bbr_sub_drain_slam_cwnd = 1;
241 static int32_t bbr_slam_cwnd_in_main_drain = 1;
242 static int32_t bbr_filter_len_sec = 6; /* How long does the rttProp filter
243 * hold */
244 static uint32_t bbr_rtt_probe_limit = (USECS_IN_SECOND * 4);
245 /*
246 * bbr_drain_gain is the reverse of the high_gain
247 * designed to drain back out the standing queue
248 * that is formed in startup by causing a larger
249 * hptsi gain and thus drainging the packets
250 * in flight.
251 */
252 static int32_t bbr_drain_gain = BBR_UNIT * 1000 / 2885;
253 static int32_t bbr_rttprobe_gain = 192;
254
255 /*
256 * The cwnd_gain is the default cwnd gain applied when
257 * calculating a target cwnd. Note that the cwnd is
258 * a secondary factor in the way BBR works (see the
259 * paper and think about it, it will take some time).
260 * Basically the hptsi_gain spreads the packets out
261 * so you never get more than BDP to the peer even
262 * if the cwnd is high. In our implemenation that
263 * means in non-recovery/retransmission scenarios
264 * cwnd will never be reached by the flight-size.
265 */
266 static int32_t bbr_cwnd_gain = BBR_UNIT * 2;
267 static int32_t bbr_tlp_type_to_use = BBR_SRTT;
268 static int32_t bbr_delack_time = 100000; /* 100ms in useconds */
269 static int32_t bbr_sack_not_required = 0; /* set to one to allow non-sack to use bbr */
270 static int32_t bbr_initial_bw_bps = 62500; /* 500kbps in bytes ps */
271 static int32_t bbr_ignore_data_after_close = 1;
272 static int16_t bbr_hptsi_gain[] = {
273 (BBR_UNIT *5 / 4),
274 (BBR_UNIT * 3 / 4),
275 BBR_UNIT,
276 BBR_UNIT,
277 BBR_UNIT,
278 BBR_UNIT,
279 BBR_UNIT,
280 BBR_UNIT
281 };
282 int32_t bbr_use_rack_resend_cheat = 1;
283 int32_t bbr_sends_full_iwnd = 1;
284
285 #define BBR_HPTSI_GAIN_MAX 8
286 /*
287 * The BBR module incorporates a number of
288 * TCP ideas that have been put out into the IETF
289 * over the last few years:
290 * - Yuchung Cheng's RACK TCP (for which its named) that
291 * will stop us using the number of dup acks and instead
292 * use time as the gage of when we retransmit.
293 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
294 * of Dukkipati et.al.
295 * - Van Jacobson's et.al BBR.
296 *
297 * RACK depends on SACK, so if an endpoint arrives that
298 * cannot do SACK the state machine below will shuttle the
299 * connection back to using the "default" TCP stack that is
300 * in FreeBSD.
301 *
302 * To implement BBR and RACK the original TCP stack was first decomposed
303 * into a functional state machine with individual states
304 * for each of the possible TCP connection states. The do_segment
305 * functions role in life is to mandate the connection supports SACK
306 * initially and then assure that the RACK state matches the conenction
307 * state before calling the states do_segment function. Data processing
308 * of inbound segments also now happens in the hpts_do_segment in general
309 * with only one exception. This is so we can keep the connection on
310 * a single CPU.
311 *
312 * Each state is simplified due to the fact that the original do_segment
313 * has been decomposed and we *know* what state we are in (no
314 * switches on the state) and all tests for SACK are gone. This
315 * greatly simplifies what each state does.
316 *
317 * TCP output is also over-written with a new version since it
318 * must maintain the new rack scoreboard and has had hptsi
319 * integrated as a requirment. Still todo is to eliminate the
320 * use of the callout_() system and use the hpts for all
321 * timers as well.
322 */
323 static uint32_t bbr_rtt_probe_time = 200000; /* 200ms in micro seconds */
324 static uint32_t bbr_rtt_probe_cwndtarg = 4; /* How many mss's outstanding */
325 static const int32_t bbr_min_req_free = 2; /* The min we must have on the
326 * free list */
327 static int32_t bbr_tlp_thresh = 1;
328 static int32_t bbr_reorder_thresh = 2;
329 static int32_t bbr_reorder_fade = 60000000; /* 0 - never fade, def
330 * 60,000,000 - 60 seconds */
331 static int32_t bbr_pkt_delay = 1000;
332 static int32_t bbr_min_to = 1000; /* Number of usec's minimum timeout */
333 static int32_t bbr_incr_timers = 1;
334
335 static int32_t bbr_tlp_min = 10000; /* 10ms in usecs */
336 static int32_t bbr_delayed_ack_time = 200000; /* 200ms in usecs */
337 static int32_t bbr_exit_startup_at_loss = 1;
338
339 /*
340 * bbr_lt_bw_ratio is 1/8th
341 * bbr_lt_bw_diff is < 4 Kbit/sec
342 */
343 static uint64_t bbr_lt_bw_diff = 4000 / 8; /* In bytes per second */
344 static uint64_t bbr_lt_bw_ratio = 8; /* For 1/8th */
345 static uint32_t bbr_lt_bw_max_rtts = 48; /* How many rtt's do we use
346 * the lt_bw for */
347 static uint32_t bbr_lt_intvl_min_rtts = 4; /* Min num of RTT's to measure
348 * lt_bw */
349 static int32_t bbr_lt_intvl_fp = 0; /* False positive epoch diff */
350 static int32_t bbr_lt_loss_thresh = 196; /* Lost vs delivered % */
351 static int32_t bbr_lt_fd_thresh = 100; /* false detection % */
352
353 static int32_t bbr_verbose_logging = 0;
354 /*
355 * Currently regular tcp has a rto_min of 30ms
356 * the backoff goes 12 times so that ends up
357 * being a total of 122.850 seconds before a
358 * connection is killed.
359 */
360 static int32_t bbr_rto_min_ms = 30; /* 30ms same as main freebsd */
361 static int32_t bbr_rto_max_sec = 4; /* 4 seconds */
362
363 /****************************************************/
364 /* DEFAULT TSO SIZING (cpu performance impacting) */
365 /****************************************************/
366 /* What amount is our formula using to get TSO size */
367 static int32_t bbr_hptsi_per_second = 1000;
368
369 /*
370 * For hptsi under bbr_cross_over connections what is delay
371 * target 7ms (in usec) combined with a seg_max of 2
372 * gets us close to identical google behavior in
373 * TSO size selection (possibly more 1MSS sends).
374 */
375 static int32_t bbr_hptsi_segments_delay_tar = 7000;
376
377 /* Does pacing delay include overhead's in its time calculations? */
378 static int32_t bbr_include_enet_oh = 0;
379 static int32_t bbr_include_ip_oh = 1;
380 static int32_t bbr_include_tcp_oh = 1;
381 static int32_t bbr_google_discount = 10;
382
383 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
384 static int32_t bbr_state_is_pkt_epoch = 0;
385 static int32_t bbr_state_drain_2_tar = 1;
386 /* What is the max the 0 - bbr_cross_over MBPS TSO target
387 * can reach using our delay target. Note that this
388 * value becomes the floor for the cross over
389 * algorithm.
390 */
391 static int32_t bbr_hptsi_segments_max = 2;
392 static int32_t bbr_hptsi_segments_floor = 1;
393 static int32_t bbr_hptsi_utter_max = 0;
394
395 /* What is the min the 0 - bbr_cross-over MBPS TSO target can be */
396 static int32_t bbr_hptsi_bytes_min = 1460;
397 static int32_t bbr_all_get_min = 0;
398
399 /* Cross over point from algo-a to algo-b */
400 static uint32_t bbr_cross_over = TWENTY_THREE_MBPS;
401
402 /* Do we deal with our restart state? */
403 static int32_t bbr_uses_idle_restart = 0;
404 static int32_t bbr_idle_restart_threshold = 100000; /* 100ms in useconds */
405
406 /* Do we allow hardware pacing? */
407 static int32_t bbr_allow_hdwr_pacing = 0;
408 static int32_t bbr_hdwr_pace_adjust = 2; /* multipler when we calc the tso size */
409 static int32_t bbr_hdwr_pace_floor = 1;
410 static int32_t bbr_hdwr_pacing_delay_cnt = 10;
411
412 /****************************************************/
413 static int32_t bbr_resends_use_tso = 0;
414 static int32_t bbr_tlp_max_resend = 2;
415 static int32_t bbr_sack_block_limit = 128;
416
417 #define BBR_MAX_STAT 19
418 counter_u64_t bbr_state_time[BBR_MAX_STAT];
419 counter_u64_t bbr_state_lost[BBR_MAX_STAT];
420 counter_u64_t bbr_state_resend[BBR_MAX_STAT];
421 counter_u64_t bbr_stat_arry[BBR_STAT_SIZE];
422 counter_u64_t bbr_opts_arry[BBR_OPTS_SIZE];
423 counter_u64_t bbr_out_size[TCP_MSS_ACCT_SIZE];
424 counter_u64_t bbr_flows_whdwr_pacing;
425 counter_u64_t bbr_flows_nohdwr_pacing;
426
427 counter_u64_t bbr_nohdwr_pacing_enobuf;
428 counter_u64_t bbr_hdwr_pacing_enobuf;
429
430 static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr);
431
432 /*
433 * Static defintions we need for forward declarations.
434 */
435 static uint32_t
436 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain,
437 uint32_t useconds_time, uint64_t bw);
438 static uint32_t
439 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain);
440 static void
441 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win);
442 static void
443 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses);
444 static void
445 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line,
446 int dolog);
447 static uint32_t
448 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain);
449 static void
450 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch,
451 int32_t pkt_epoch, uint32_t losses);
452 static uint32_t
453 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts,
454 struct bbr_sendmap *rsm);
455 static uint32_t
456 bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp);
457 static uint32_t
458 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
459 struct bbr_sendmap *rsm, uint32_t srtt, uint32_t cts);
460 static void
461 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
462 int32_t line);
463 static void
464 bbr_set_state_target(struct tcp_bbr *bbr, int line);
465 static void
466 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line);
467 static void
468 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick,
469 int event, int line);
470 static void
471 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts);
472 static void
473 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts);
474 static void
475 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
476 uint32_t rtt, uint32_t line, uint8_t is_start,
477 uint16_t set);
478 static struct bbr_sendmap *
479 bbr_find_lowest_rsm(struct tcp_bbr *bbr);
480 static __inline uint32_t
481 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type);
482 static void
483 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t pacing_delay,
484 uint8_t which);
485 static void
486 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts,
487 uint32_t time_since_sent, uint32_t srtt,
488 uint32_t thresh, uint32_t to);
489 static void
490 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag);
491 static void
492 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t pacing_delay,
493 uint32_t del_by, uint32_t cts, uint32_t sloton,
494 uint32_t prev_delay);
495 static void
496 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
497 int32_t line);
498 static void
499 bbr_stop_all_timers(struct tcpcb *tp, struct tcp_bbr *bbr);
500 static void
501 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts);
502 static void
503 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);
504 static void
505 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);
506 static void
507 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
508 uint32_t cts, uint32_t usecs, uint64_t bw,
509 uint32_t override, int mod);
510 static int bbr_ctloutput(struct tcpcb *tp, struct sockopt *sopt);
511
512 static inline uint8_t
bbr_state_val(struct tcp_bbr * bbr)513 bbr_state_val(struct tcp_bbr *bbr)
514 {
515 return(bbr->rc_bbr_substate);
516 }
517
518 static inline uint32_t
get_min_cwnd(struct tcp_bbr * bbr)519 get_min_cwnd(struct tcp_bbr *bbr)
520 {
521 int mss;
522
523 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
524 bbr->r_ctl.rc_pace_max_segs);
525 if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED)
526 return (bbr_cwnd_min_val_hs * mss);
527 else
528 return (bbr_cwnd_min_val * mss);
529 }
530
531 static uint32_t
bbr_get_persists_timer_val(struct tcpcb * tp,struct tcp_bbr * bbr)532 bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr)
533 {
534 uint64_t srtt, var;
535 uint64_t ret_val;
536
537 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
538 if (tp->t_srtt == 0) {
539 srtt = (uint64_t)BBR_INITIAL_RTO;
540 var = 0;
541 } else {
542 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
543 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT);
544 }
545 TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]),
546 bbr_persist_min, bbr_persist_max);
547 return ((uint32_t)ret_val);
548 }
549
550 static uint32_t
bbr_timer_start(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)551 bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
552 {
553 /*
554 * Start the FR timer, we do this based on getting the first one in
555 * the rc_tmap. Note that if its NULL we must stop the timer. in all
556 * events we need to stop the running timer (if its running) before
557 * starting the new one.
558 */
559 uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse;
560 int32_t idx;
561 int32_t is_tlp_timer = 0;
562 struct bbr_sendmap *rsm;
563
564 if (bbr->rc_all_timers_stopped) {
565 /* All timers have been stopped none are to run */
566 return (0);
567 }
568 if (bbr->rc_in_persist) {
569 /* We can't start any timer in persists */
570 return (bbr_get_persists_timer_val(tp, bbr));
571 }
572 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
573 if ((rsm == NULL) ||
574 ((tp->t_flags & TF_SACK_PERMIT) == 0) ||
575 (tp->t_state < TCPS_ESTABLISHED)) {
576 /* Nothing on the send map */
577 activate_rxt:
578 if (SEQ_LT(tp->snd_una, tp->snd_max) ||
579 sbavail(&tptosocket(tp)->so_snd)) {
580 uint64_t tov;
581
582 time_since_sent = 0;
583 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
584 if (rsm) {
585 idx = rsm->r_rtr_cnt - 1;
586 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
587 tstmp_touse = rsm->r_tim_lastsent[idx];
588 else
589 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
590 if (TSTMP_GT(tstmp_touse, cts))
591 time_since_sent = cts - tstmp_touse;
592 }
593 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
594 if (tp->t_srtt == 0)
595 tov = BBR_INITIAL_RTO;
596 else
597 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) +
598 ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT);
599 if (tp->t_rxtshift)
600 tov *= tcp_backoff[tp->t_rxtshift];
601 if (tov > time_since_sent)
602 tov -= time_since_sent;
603 else
604 tov = bbr->r_ctl.rc_min_to;
605 TCPT_RANGESET_NOSLOP(to, tov,
606 (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),
607 (bbr->rc_max_rto_sec * USECS_IN_SECOND));
608 bbr_log_timer_var(bbr, 2, cts, 0, bbr_get_rtt(bbr, BBR_SRTT), 0, to);
609 return (to);
610 }
611 return (0);
612 }
613 if (rsm->r_flags & BBR_ACKED) {
614 rsm = bbr_find_lowest_rsm(bbr);
615 if (rsm == NULL) {
616 /* No lowest? */
617 goto activate_rxt;
618 }
619 }
620 /* Convert from ms to usecs */
621 if (rsm->r_flags & BBR_SACK_PASSED) {
622 if ((tp->t_flags & TF_SENTFIN) &&
623 ((tp->snd_max - tp->snd_una) == 1) &&
624 (rsm->r_flags & BBR_HAS_FIN)) {
625 /*
626 * We don't start a bbr rack timer if all we have is
627 * a FIN outstanding.
628 */
629 goto activate_rxt;
630 }
631 srtt = bbr_get_rtt(bbr, BBR_RTT_RACK);
632 thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm);
633 idx = rsm->r_rtr_cnt - 1;
634 exp = rsm->r_tim_lastsent[idx] + thresh;
635 if (SEQ_GEQ(exp, cts)) {
636 to = exp - cts;
637 if (to < bbr->r_ctl.rc_min_to) {
638 to = bbr->r_ctl.rc_min_to;
639 }
640 } else {
641 to = bbr->r_ctl.rc_min_to;
642 }
643 } else {
644 /* Ok we need to do a TLP not RACK */
645 if (bbr->rc_tlp_in_progress != 0) {
646 /*
647 * The previous send was a TLP.
648 */
649 goto activate_rxt;
650 }
651 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
652 if (rsm == NULL) {
653 /* We found no rsm to TLP with. */
654 goto activate_rxt;
655 }
656 if (rsm->r_flags & BBR_HAS_FIN) {
657 /* If its a FIN we don't do TLP */
658 rsm = NULL;
659 goto activate_rxt;
660 }
661 time_since_sent = 0;
662 idx = rsm->r_rtr_cnt - 1;
663 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
664 tstmp_touse = rsm->r_tim_lastsent[idx];
665 else
666 tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
667 if (TSTMP_GT(tstmp_touse, cts))
668 time_since_sent = cts - tstmp_touse;
669 is_tlp_timer = 1;
670 srtt = bbr_get_rtt(bbr, bbr_tlp_type_to_use);
671 thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts);
672 if (thresh > time_since_sent)
673 to = thresh - time_since_sent;
674 else
675 to = bbr->r_ctl.rc_min_to;
676 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
677 /*
678 * If the TLP time works out to larger than the max
679 * RTO lets not do TLP.. just RTO.
680 */
681 goto activate_rxt;
682 }
683 if ((bbr->rc_tlp_rtx_out == 1) &&
684 (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {
685 /*
686 * Second retransmit of the same TLP
687 * lets not.
688 */
689 bbr->rc_tlp_rtx_out = 0;
690 goto activate_rxt;
691 }
692 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {
693 /*
694 * The tail is no longer the last one I did a probe
695 * on
696 */
697 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
698 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
699 }
700 }
701 if (is_tlp_timer == 0) {
702 BBR_STAT_INC(bbr_to_arm_rack);
703 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
704 } else {
705 bbr_log_timer_var(bbr, 1, cts, time_since_sent, srtt, thresh, to);
706 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
707 /*
708 * We have exceeded how many times we can retran the
709 * current TLP timer, switch to the RTO timer.
710 */
711 goto activate_rxt;
712 } else {
713 BBR_STAT_INC(bbr_to_arm_tlp);
714 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
715 }
716 }
717 return (to);
718 }
719
720 static inline int32_t
bbr_minseg(struct tcp_bbr * bbr)721 bbr_minseg(struct tcp_bbr *bbr)
722 {
723 return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);
724 }
725
726 static void
bbr_start_hpts_timer(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t cts,int32_t frm,int32_t pacing_delay,uint32_t tot_len)727 bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t pacing_delay, uint32_t tot_len)
728 {
729 struct inpcb *inp = tptoinpcb(tp);
730 struct hpts_diag diag;
731 uint32_t delayed_ack = 0;
732 uint32_t left = 0;
733 uint32_t hpts_timeout;
734 uint8_t stopped;
735 int32_t delay_calc = 0;
736 uint32_t prev_delay = 0;
737
738 if (tcp_in_hpts(tp)) {
739 /* A previous call is already set up */
740 return;
741 }
742 if ((tp->t_state == TCPS_CLOSED) ||
743 (tp->t_state == TCPS_LISTEN)) {
744 return;
745 }
746 stopped = bbr->rc_tmr_stopped;
747 if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
748 left = bbr->r_ctl.rc_timer_exp - cts;
749 }
750 bbr->r_ctl.rc_hpts_flags = 0;
751 bbr->r_ctl.rc_timer_exp = 0;
752 prev_delay = bbr->r_ctl.rc_last_delay_val;
753 if (bbr->r_ctl.rc_last_delay_val &&
754 (pacing_delay == 0)) {
755 /*
756 * If a previous pacer delay was in place we
757 * are not coming from the output side (where
758 * we calculate a delay, more likely a timer).
759 */
760 pacing_delay = bbr->r_ctl.rc_last_delay_val;
761 if (TSTMP_GT(cts, bbr->rc_pacer_started)) {
762 /* Compensate for time passed */
763 delay_calc = cts - bbr->rc_pacer_started;
764 if (delay_calc <= pacing_delay)
765 pacing_delay -= delay_calc;
766 }
767 }
768 /* Do we have early to make up for by pushing out the pacing time? */
769 if (bbr->r_agg_early_set) {
770 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, pacing_delay, 0, bbr->r_agg_early_set, 2);
771 pacing_delay += bbr->r_ctl.rc_agg_early;
772 bbr->r_ctl.rc_agg_early = 0;
773 bbr->r_agg_early_set = 0;
774 }
775 /* Are we running a total debt that needs to be compensated for? */
776 if (bbr->r_ctl.rc_hptsi_agg_delay) {
777 if (pacing_delay > bbr->r_ctl.rc_hptsi_agg_delay) {
778 /* We nuke the delay */
779 pacing_delay -= bbr->r_ctl.rc_hptsi_agg_delay;
780 bbr->r_ctl.rc_hptsi_agg_delay = 0;
781 } else {
782 /* We nuke some of the delay, put in a minimal 100usecs */
783 bbr->r_ctl.rc_hptsi_agg_delay -= pacing_delay;
784 bbr->r_ctl.rc_last_delay_val = pacing_delay = 100;
785 }
786 }
787 bbr->r_ctl.rc_last_delay_val = pacing_delay;
788 hpts_timeout = bbr_timer_start(tp, bbr, cts);
789 if (tp->t_flags & TF_DELACK) {
790 if (bbr->rc_in_persist == 0) {
791 delayed_ack = bbr_delack_time;
792 } else {
793 /*
794 * We are in persists and have
795 * gotten a new data element.
796 */
797 if (hpts_timeout > bbr_delack_time) {
798 /*
799 * Lets make the persists timer (which acks)
800 * be the smaller of hpts_timeout and bbr_delack_time.
801 */
802 hpts_timeout = bbr_delack_time;
803 }
804 }
805 }
806 if (delayed_ack &&
807 ((hpts_timeout == 0) ||
808 (delayed_ack < hpts_timeout))) {
809 /* We need a Delayed ack timer */
810 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
811 hpts_timeout = delayed_ack;
812 }
813 if (pacing_delay) {
814 /* Mark that we have a pacing timer up */
815 BBR_STAT_INC(bbr_paced_segments);
816 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
817 }
818 /*
819 * If no timers are going to run and we will fall off thfe hptsi
820 * wheel, we resort to a keep-alive timer if its configured.
821 */
822 if ((hpts_timeout == 0) &&
823 (pacing_delay == 0)) {
824 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
825 (tp->t_state <= TCPS_CLOSING)) {
826 /*
827 * Ok we have no timer (persists, rack, tlp, rxt or
828 * del-ack), we don't have segments being paced. So
829 * all that is left is the keepalive timer.
830 */
831 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
832 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp));
833 } else {
834 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp));
835 }
836 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
837 }
838 }
839 if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) ==
840 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
841 /*
842 * RACK, TLP, persists and RXT timers all are restartable
843 * based on actions input .. i.e we received a packet (ack
844 * or sack) and that changes things (rw, or snd_una etc).
845 * Thus we can restart them with a new value. For
846 * keep-alive, delayed_ack we keep track of what was left
847 * and restart the timer with a smaller value.
848 */
849 if (left < hpts_timeout)
850 hpts_timeout = left;
851 }
852 if (bbr->r_ctl.rc_incr_tmrs && pacing_delay &&
853 (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
854 /*
855 * If configured to do so, and the timer is either
856 * the TLP or RXT timer, we need to increase the timeout
857 * by the pacing time. Consider the bottleneck at my
858 * machine as an example, we are sending something
859 * to start a TLP on. The last packet won't be emitted
860 * fully until the pacing time (the bottleneck will hold
861 * the data in place). Once the packet is emitted that
862 * is when we want to start waiting for the TLP. This
863 * is most evident with hardware pacing (where the nic
864 * is holding the packet(s) before emitting). But it
865 * can also show up in the network so we do it for all
866 * cases. Technically we would take off one packet from
867 * this extra delay but this is easier and being more
868 * conservative is probably better.
869 */
870 hpts_timeout += pacing_delay;
871 }
872 if (hpts_timeout) {
873 /*
874 * Hack alert for now we can't time-out over 2147 seconds (a
875 * bit more than 35min)
876 */
877 if (hpts_timeout > 0x7ffffffe)
878 hpts_timeout = 0x7ffffffe;
879 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
880 } else
881 bbr->r_ctl.rc_timer_exp = 0;
882 if ((pacing_delay) &&
883 (bbr->rc_use_google ||
884 bbr->output_error_seen ||
885 (pacing_delay <= hpts_timeout)) ) {
886 /*
887 * Tell LRO that it can queue packets while
888 * we pace.
889 */
890 bbr->rc_tp->t_flags2 |= TF2_MBUF_QUEUE_READY;
891 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
892 (bbr->rc_cwnd_limited == 0)) {
893 /*
894 * If we are not cwnd limited and we
895 * are running a rack timer we put on
896 * the do not disturbe even for sack.
897 */
898 tp->t_flags2 |= TF2_DONT_SACK_QUEUE;
899 } else
900 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
901 bbr->rc_pacer_started = cts;
902
903 tcp_hpts_insert(tp, pacing_delay, &diag);
904 bbr->rc_timer_first = 0;
905 bbr->bbr_timer_src = frm;
906 bbr_log_to_start(bbr, cts, hpts_timeout, pacing_delay, 1);
907 bbr_log_hpts_diag(bbr, cts, &diag);
908 } else if (hpts_timeout) {
909 tcp_hpts_insert(tp, hpts_timeout, &diag);
910 /*
911 * We add the flag here as well if the pacing delay is set,
912 * since hpts will call in to clear the queue first before
913 * calling the output routine (which does our timers).
914 * We don't want to set the flag if its just a timer
915 * else the arrival of data might (that causes us
916 * to send more) might get delayed. Imagine being
917 * on a keep-alive timer and a request comes in for
918 * more data.
919 */
920 if (pacing_delay)
921 bbr->rc_pacer_started = cts;
922 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
923 (bbr->rc_cwnd_limited == 0)) {
924 /*
925 * For a rack timer, don't wake us even
926 * if a sack arrives as long as we are
927 * not cwnd limited.
928 */
929 tp->t_flags2 |= (TF2_MBUF_QUEUE_READY |
930 TF2_DONT_SACK_QUEUE);
931 } else {
932 /* All other timers wake us up */
933 tp->t_flags2 &= ~(TF2_MBUF_QUEUE_READY |
934 TF2_DONT_SACK_QUEUE);
935 }
936 bbr->bbr_timer_src = frm;
937 bbr_log_to_start(bbr, cts, hpts_timeout, pacing_delay, 0);
938 bbr_log_hpts_diag(bbr, cts, &diag);
939 bbr->rc_timer_first = 1;
940 }
941 bbr->rc_tmr_stopped = 0;
942 bbr_log_type_bbrsnd(bbr, tot_len, pacing_delay, delay_calc, cts, frm, prev_delay);
943 }
944
945 static void
bbr_timer_audit(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,struct sockbuf * sb)946 bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb)
947 {
948 /*
949 * We received an ack, and then did not call send or were bounced
950 * out due to the hpts was running. Now a timer is up as well, is it
951 * the right timer?
952 */
953 struct inpcb *inp;
954 struct bbr_sendmap *rsm;
955 uint32_t hpts_timeout;
956 int tmr_up;
957
958 tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
959 if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
960 return;
961 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
962 if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
963 (tmr_up == PACE_TMR_RXT)) {
964 /* Should be an RXT */
965 return;
966 }
967 inp = bbr->rc_inp;
968 if (rsm == NULL) {
969 /* Nothing outstanding? */
970 if (tp->t_flags & TF_DELACK) {
971 if (tmr_up == PACE_TMR_DELACK)
972 /*
973 * We are supposed to have delayed ack up
974 * and we do
975 */
976 return;
977 } else if (((V_tcp_always_keepalive ||
978 inp->inp_socket->so_options & SO_KEEPALIVE) &&
979 (tp->t_state <= TCPS_CLOSING)) &&
980 (tmr_up == PACE_TMR_KEEP) &&
981 (tp->snd_max == tp->snd_una)) {
982 /* We should have keep alive up and we do */
983 return;
984 }
985 }
986 if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {
987 if ((tp->t_flags & TF_SENTFIN) &&
988 ((tp->snd_max - tp->snd_una) == 1) &&
989 (rsm->r_flags & BBR_HAS_FIN)) {
990 /* needs to be a RXT */
991 if (tmr_up == PACE_TMR_RXT)
992 return;
993 else
994 goto wrong_timer;
995 } else if (tmr_up == PACE_TMR_RACK)
996 return;
997 else
998 goto wrong_timer;
999 } else if (rsm && (tmr_up == PACE_TMR_RACK)) {
1000 /* Rack timer has priority if we have data out */
1001 return;
1002 } else if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1003 ((tmr_up == PACE_TMR_TLP) ||
1004 (tmr_up == PACE_TMR_RXT))) {
1005 /*
1006 * Either a TLP or RXT is fine if no sack-passed is in place
1007 * and data is outstanding.
1008 */
1009 return;
1010 } else if (tmr_up == PACE_TMR_DELACK) {
1011 /*
1012 * If the delayed ack was going to go off before the
1013 * rtx/tlp/rack timer were going to expire, then that would
1014 * be the timer in control. Note we don't check the time
1015 * here trusting the code is correct.
1016 */
1017 return;
1018 }
1019 if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1020 ((tmr_up == PACE_TMR_RXT) ||
1021 (tmr_up == PACE_TMR_TLP) ||
1022 (tmr_up == PACE_TMR_RACK))) {
1023 /*
1024 * We have outstanding data and
1025 * we *do* have a RACK, TLP or RXT
1026 * timer running. We won't restart
1027 * anything here since thats probably ok we
1028 * will get called with some timer here shortly.
1029 */
1030 return;
1031 }
1032 /*
1033 * Ok the timer originally started is not what we want now. We will
1034 * force the hpts to be stopped if any, and restart with the pacing
1035 * delay set to what was in the saved delay.
1036 */
1037 wrong_timer:
1038 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
1039 if (tcp_in_hpts(tp))
1040 tcp_hpts_remove(tp);
1041 bbr_timer_cancel(bbr, __LINE__, cts);
1042 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
1043 0);
1044 } else {
1045 /*
1046 * Output is hptsi so we just need to switch the type of
1047 * timer. We don't bother with keep-alive, since when we
1048 * jump through the output, it will start the keep-alive if
1049 * nothing is sent.
1050 *
1051 * We only need a delayed-ack added and or the hpts_timeout.
1052 */
1053 hpts_timeout = bbr_timer_start(tp, bbr, cts);
1054 if (tp->t_flags & TF_DELACK) {
1055 if (hpts_timeout == 0) {
1056 hpts_timeout = bbr_delack_time;
1057 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1058 }
1059 else if (hpts_timeout > bbr_delack_time) {
1060 hpts_timeout = bbr_delack_time;
1061 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1062 }
1063 }
1064 if (hpts_timeout) {
1065 if (hpts_timeout > 0x7ffffffe)
1066 hpts_timeout = 0x7ffffffe;
1067 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
1068 }
1069 }
1070 }
1071
1072 int32_t bbr_clear_lost = 0;
1073
1074 /*
1075 * Considers the two time values now (cts) and earlier.
1076 * If cts is smaller than earlier, we could have
1077 * had a sequence wrap (our counter wraps every
1078 * 70 min or so) or it could be just clock skew
1079 * getting us two different time values. Clock skew
1080 * will show up within 10ms or so. So in such
1081 * a case (where cts is behind earlier time by
1082 * less than 10ms) we return 0. Otherwise we
1083 * return the true difference between them.
1084 */
1085 static inline uint32_t
bbr_calc_time(uint32_t cts,uint32_t earlier_time)1086 bbr_calc_time(uint32_t cts, uint32_t earlier_time) {
1087 /*
1088 * Given two timestamps, the current time stamp cts, and some other
1089 * time-stamp taken in theory earlier return the difference. The
1090 * trick is here sometimes locking will get the other timestamp
1091 * after the cts. If this occurs we need to return 0.
1092 */
1093 if (TSTMP_GEQ(cts, earlier_time))
1094 return (cts - earlier_time);
1095 /*
1096 * cts is behind earlier_time if its less than 10ms consider it 0.
1097 * If its more than 10ms difference then we had a time wrap. Else
1098 * its just the normal locking foo. I wonder if we should not go to
1099 * 64bit TS and get rid of this issue.
1100 */
1101 if (TSTMP_GEQ((cts + 10000), earlier_time))
1102 return (0);
1103 /*
1104 * Ok the time must have wrapped. So we need to answer a large
1105 * amount of time, which the normal subtraction should do.
1106 */
1107 return (cts - earlier_time);
1108 }
1109
1110 static int
sysctl_bbr_clear_lost(SYSCTL_HANDLER_ARGS)1111 sysctl_bbr_clear_lost(SYSCTL_HANDLER_ARGS)
1112 {
1113 uint32_t stat;
1114 int32_t error;
1115
1116 error = SYSCTL_OUT(req, &bbr_clear_lost, sizeof(uint32_t));
1117 if (error || req->newptr == NULL)
1118 return error;
1119
1120 error = SYSCTL_IN(req, &stat, sizeof(uint32_t));
1121 if (error)
1122 return (error);
1123 if (stat == 1) {
1124 #ifdef BBR_INVARIANTS
1125 printf("Clearing BBR lost counters\n");
1126 #endif
1127 COUNTER_ARRAY_ZERO(bbr_state_lost, BBR_MAX_STAT);
1128 COUNTER_ARRAY_ZERO(bbr_state_time, BBR_MAX_STAT);
1129 COUNTER_ARRAY_ZERO(bbr_state_resend, BBR_MAX_STAT);
1130 } else if (stat == 2) {
1131 #ifdef BBR_INVARIANTS
1132 printf("Clearing BBR option counters\n");
1133 #endif
1134 COUNTER_ARRAY_ZERO(bbr_opts_arry, BBR_OPTS_SIZE);
1135 } else if (stat == 3) {
1136 #ifdef BBR_INVARIANTS
1137 printf("Clearing BBR stats counters\n");
1138 #endif
1139 COUNTER_ARRAY_ZERO(bbr_stat_arry, BBR_STAT_SIZE);
1140 } else if (stat == 4) {
1141 #ifdef BBR_INVARIANTS
1142 printf("Clearing BBR out-size counters\n");
1143 #endif
1144 COUNTER_ARRAY_ZERO(bbr_out_size, TCP_MSS_ACCT_SIZE);
1145 }
1146 bbr_clear_lost = 0;
1147 return (0);
1148 }
1149
1150 static void
bbr_init_sysctls(void)1151 bbr_init_sysctls(void)
1152 {
1153 struct sysctl_oid *bbr_probertt;
1154 struct sysctl_oid *bbr_hptsi;
1155 struct sysctl_oid *bbr_measure;
1156 struct sysctl_oid *bbr_cwnd;
1157 struct sysctl_oid *bbr_timeout;
1158 struct sysctl_oid *bbr_states;
1159 struct sysctl_oid *bbr_startup;
1160 struct sysctl_oid *bbr_policer;
1161
1162 /* Probe rtt controls */
1163 bbr_probertt = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1164 SYSCTL_CHILDREN(bbr_sysctl_root),
1165 OID_AUTO,
1166 "probertt",
1167 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1168 "");
1169 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1170 SYSCTL_CHILDREN(bbr_probertt),
1171 OID_AUTO, "gain", CTLFLAG_RW,
1172 &bbr_rttprobe_gain, 192,
1173 "What is the filter gain drop in probe_rtt (0=disable)?");
1174 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1175 SYSCTL_CHILDREN(bbr_probertt),
1176 OID_AUTO, "cwnd", CTLFLAG_RW,
1177 &bbr_rtt_probe_cwndtarg, 4,
1178 "How many mss's are outstanding during probe-rtt");
1179 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1180 SYSCTL_CHILDREN(bbr_probertt),
1181 OID_AUTO, "int", CTLFLAG_RW,
1182 &bbr_rtt_probe_limit, 4000000,
1183 "If RTT has not shrank in this many micro-seconds enter probe-rtt");
1184 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1185 SYSCTL_CHILDREN(bbr_probertt),
1186 OID_AUTO, "mintime", CTLFLAG_RW,
1187 &bbr_rtt_probe_time, 200000,
1188 "How many microseconds in probe-rtt");
1189 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1190 SYSCTL_CHILDREN(bbr_probertt),
1191 OID_AUTO, "filter_len_sec", CTLFLAG_RW,
1192 &bbr_filter_len_sec, 6,
1193 "How long in seconds does the rttProp filter run?");
1194 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1195 SYSCTL_CHILDREN(bbr_probertt),
1196 OID_AUTO, "drain_rtt", CTLFLAG_RW,
1197 &bbr_drain_rtt, BBR_SRTT,
1198 "What is the drain rtt to use in probeRTT (rtt_prop=0, rtt_rack=1, rtt_pkt=2, rtt_srtt=3?");
1199 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1200 SYSCTL_CHILDREN(bbr_probertt),
1201 OID_AUTO, "can_force", CTLFLAG_RW,
1202 &bbr_can_force_probertt, 0,
1203 "If we keep setting new low rtt's but delay going in probe-rtt can we force in??");
1204 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1205 SYSCTL_CHILDREN(bbr_probertt),
1206 OID_AUTO, "enter_sets_force", CTLFLAG_RW,
1207 &bbr_probertt_sets_rtt, 0,
1208 "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?");
1209 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1210 SYSCTL_CHILDREN(bbr_probertt),
1211 OID_AUTO, "can_adjust", CTLFLAG_RW,
1212 &bbr_can_adjust_probertt, 1,
1213 "Can we dynamically adjust the probe-rtt limits and times?");
1214 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1215 SYSCTL_CHILDREN(bbr_probertt),
1216 OID_AUTO, "is_ratio", CTLFLAG_RW,
1217 &bbr_is_ratio, 0,
1218 "is the limit to filter a ratio?");
1219 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1220 SYSCTL_CHILDREN(bbr_probertt),
1221 OID_AUTO, "use_cwnd", CTLFLAG_RW,
1222 &bbr_prtt_slam_cwnd, 0,
1223 "Should we set/recover cwnd?");
1224 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1225 SYSCTL_CHILDREN(bbr_probertt),
1226 OID_AUTO, "can_use_ts", CTLFLAG_RW,
1227 &bbr_can_use_ts_for_rtt, 1,
1228 "Can we use the ms timestamp if available for retransmistted rtt calculations?");
1229
1230 /* Pacing controls */
1231 bbr_hptsi = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1232 SYSCTL_CHILDREN(bbr_sysctl_root),
1233 OID_AUTO,
1234 "pacing",
1235 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1236 "");
1237 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1238 SYSCTL_CHILDREN(bbr_hptsi),
1239 OID_AUTO, "hw_pacing", CTLFLAG_RW,
1240 &bbr_allow_hdwr_pacing, 1,
1241 "Do we allow hardware pacing?");
1242 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1243 SYSCTL_CHILDREN(bbr_hptsi),
1244 OID_AUTO, "hw_pacing_limit", CTLFLAG_RW,
1245 &bbr_hardware_pacing_limit, 4000,
1246 "Do we have a limited number of connections for pacing chelsio (0=no limit)?");
1247 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1248 SYSCTL_CHILDREN(bbr_hptsi),
1249 OID_AUTO, "hw_pacing_adj", CTLFLAG_RW,
1250 &bbr_hdwr_pace_adjust, 2,
1251 "Multiplier to calculated tso size?");
1252 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1253 SYSCTL_CHILDREN(bbr_hptsi),
1254 OID_AUTO, "hw_pacing_floor", CTLFLAG_RW,
1255 &bbr_hdwr_pace_floor, 1,
1256 "Do we invoke the hardware pacing floor?");
1257 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1258 SYSCTL_CHILDREN(bbr_hptsi),
1259 OID_AUTO, "hw_pacing_delay_cnt", CTLFLAG_RW,
1260 &bbr_hdwr_pacing_delay_cnt, 10,
1261 "How many packets must be sent after hdwr pacing is enabled");
1262 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1263 SYSCTL_CHILDREN(bbr_hptsi),
1264 OID_AUTO, "bw_cross", CTLFLAG_RW,
1265 &bbr_cross_over, 3000000,
1266 "What is the point where we cross over to linux like TSO size set");
1267 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1268 SYSCTL_CHILDREN(bbr_hptsi),
1269 OID_AUTO, "seg_deltarg", CTLFLAG_RW,
1270 &bbr_hptsi_segments_delay_tar, 7000,
1271 "What is the worse case delay target for hptsi < 48Mbp connections");
1272 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1273 SYSCTL_CHILDREN(bbr_hptsi),
1274 OID_AUTO, "enet_oh", CTLFLAG_RW,
1275 &bbr_include_enet_oh, 0,
1276 "Do we include the ethernet overhead in calculating pacing delay?");
1277 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1278 SYSCTL_CHILDREN(bbr_hptsi),
1279 OID_AUTO, "ip_oh", CTLFLAG_RW,
1280 &bbr_include_ip_oh, 1,
1281 "Do we include the IP overhead in calculating pacing delay?");
1282 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1283 SYSCTL_CHILDREN(bbr_hptsi),
1284 OID_AUTO, "tcp_oh", CTLFLAG_RW,
1285 &bbr_include_tcp_oh, 0,
1286 "Do we include the TCP overhead in calculating pacing delay?");
1287 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1288 SYSCTL_CHILDREN(bbr_hptsi),
1289 OID_AUTO, "google_discount", CTLFLAG_RW,
1290 &bbr_google_discount, 10,
1291 "What is the default google discount percentage wise for pacing (11 = 1.1%%)?");
1292 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1293 SYSCTL_CHILDREN(bbr_hptsi),
1294 OID_AUTO, "all_get_min", CTLFLAG_RW,
1295 &bbr_all_get_min, 0,
1296 "If you are less than a MSS do you just get the min?");
1297 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1298 SYSCTL_CHILDREN(bbr_hptsi),
1299 OID_AUTO, "tso_min", CTLFLAG_RW,
1300 &bbr_hptsi_bytes_min, 1460,
1301 "For 0 -> 24Mbps what is floor number of segments for TSO");
1302 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1303 SYSCTL_CHILDREN(bbr_hptsi),
1304 OID_AUTO, "seg_tso_max", CTLFLAG_RW,
1305 &bbr_hptsi_segments_max, 6,
1306 "For 0 -> 24Mbps what is top number of segments for TSO");
1307 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1308 SYSCTL_CHILDREN(bbr_hptsi),
1309 OID_AUTO, "seg_floor", CTLFLAG_RW,
1310 &bbr_hptsi_segments_floor, 1,
1311 "Minimum TSO size we will fall too in segments");
1312 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1313 SYSCTL_CHILDREN(bbr_hptsi),
1314 OID_AUTO, "utter_max", CTLFLAG_RW,
1315 &bbr_hptsi_utter_max, 0,
1316 "The absolute maximum that any pacing (outside of hardware) can be");
1317 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1318 SYSCTL_CHILDREN(bbr_hptsi),
1319 OID_AUTO, "seg_divisor", CTLFLAG_RW,
1320 &bbr_hptsi_per_second, 100,
1321 "What is the divisor in our hptsi TSO calculation 512Mbps < X > 24Mbps ");
1322 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1323 SYSCTL_CHILDREN(bbr_hptsi),
1324 OID_AUTO, "srtt_mul", CTLFLAG_RW,
1325 &bbr_hptsi_max_mul, 1,
1326 "The multiplier for pace len max");
1327 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1328 SYSCTL_CHILDREN(bbr_hptsi),
1329 OID_AUTO, "srtt_div", CTLFLAG_RW,
1330 &bbr_hptsi_max_div, 2,
1331 "The divisor for pace len max");
1332 /* Measurement controls */
1333 bbr_measure = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1334 SYSCTL_CHILDREN(bbr_sysctl_root),
1335 OID_AUTO,
1336 "measure",
1337 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1338 "Measurement controls");
1339 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1340 SYSCTL_CHILDREN(bbr_measure),
1341 OID_AUTO, "min_i_bw", CTLFLAG_RW,
1342 &bbr_initial_bw_bps, 62500,
1343 "Minimum initial b/w in bytes per second");
1344 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1345 SYSCTL_CHILDREN(bbr_measure),
1346 OID_AUTO, "no_sack_needed", CTLFLAG_RW,
1347 &bbr_sack_not_required, 0,
1348 "Do we allow bbr to run on connections not supporting SACK?");
1349 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1350 SYSCTL_CHILDREN(bbr_measure),
1351 OID_AUTO, "use_google", CTLFLAG_RW,
1352 &bbr_use_google_algo, 0,
1353 "Use has close to google V1.0 has possible?");
1354 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1355 SYSCTL_CHILDREN(bbr_measure),
1356 OID_AUTO, "ts_limiting", CTLFLAG_RW,
1357 &bbr_ts_limiting, 1,
1358 "Do we attempt to use the peers timestamp to limit b/w caculations?");
1359 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1360 SYSCTL_CHILDREN(bbr_measure),
1361 OID_AUTO, "ts_can_raise", CTLFLAG_RW,
1362 &bbr_ts_can_raise, 0,
1363 "Can we raise the b/w via timestamp b/w calculation?");
1364 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1365 SYSCTL_CHILDREN(bbr_measure),
1366 OID_AUTO, "ts_delta", CTLFLAG_RW,
1367 &bbr_min_usec_delta, 20000,
1368 "How long in usec between ts of our sends in ts validation code?");
1369 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1370 SYSCTL_CHILDREN(bbr_measure),
1371 OID_AUTO, "ts_peer_delta", CTLFLAG_RW,
1372 &bbr_min_peer_delta, 20,
1373 "What min numerical value should be between the peer deltas?");
1374 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1375 SYSCTL_CHILDREN(bbr_measure),
1376 OID_AUTO, "ts_delta_percent", CTLFLAG_RW,
1377 &bbr_delta_percent, 150,
1378 "What percentage (150 = 15.0) do we allow variance for?");
1379 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1380 SYSCTL_CHILDREN(bbr_measure),
1381 OID_AUTO, "min_measure_good_bw", CTLFLAG_RW,
1382 &bbr_min_measurements_req, 1,
1383 "What is the minimum measurement count we need before we switch to our b/w estimate");
1384 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1385 SYSCTL_CHILDREN(bbr_measure),
1386 OID_AUTO, "min_measure_before_pace", CTLFLAG_RW,
1387 &bbr_no_pacing_until, 4,
1388 "How many pkt-epoch's (0 is off) do we need before pacing is on?");
1389 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1390 SYSCTL_CHILDREN(bbr_measure),
1391 OID_AUTO, "quanta", CTLFLAG_RW,
1392 &bbr_quanta, 2,
1393 "Extra quanta to add when calculating the target (ID section 4.2.3.2).");
1394 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1395 SYSCTL_CHILDREN(bbr_measure),
1396 OID_AUTO, "noretran", CTLFLAG_RW,
1397 &bbr_no_retran, 0,
1398 "Should google mode not use retransmission measurements for the b/w estimation?");
1399 /* State controls */
1400 bbr_states = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1401 SYSCTL_CHILDREN(bbr_sysctl_root),
1402 OID_AUTO,
1403 "states",
1404 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1405 "State controls");
1406 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1407 SYSCTL_CHILDREN(bbr_states),
1408 OID_AUTO, "idle_restart", CTLFLAG_RW,
1409 &bbr_uses_idle_restart, 0,
1410 "Do we use a new special idle_restart state to ramp back up quickly?");
1411 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1412 SYSCTL_CHILDREN(bbr_states),
1413 OID_AUTO, "idle_restart_threshold", CTLFLAG_RW,
1414 &bbr_idle_restart_threshold, 100000,
1415 "How long must we be idle before we restart??");
1416 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1417 SYSCTL_CHILDREN(bbr_states),
1418 OID_AUTO, "use_pkt_epoch", CTLFLAG_RW,
1419 &bbr_state_is_pkt_epoch, 0,
1420 "Do we use a pkt-epoch for substate if 0 rttProp?");
1421 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1422 SYSCTL_CHILDREN(bbr_states),
1423 OID_AUTO, "startup_rtt_gain", CTLFLAG_RW,
1424 &bbr_rtt_gain_thresh, 0,
1425 "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?");
1426 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1427 SYSCTL_CHILDREN(bbr_states),
1428 OID_AUTO, "drain_floor", CTLFLAG_RW,
1429 &bbr_drain_floor, 88,
1430 "What is the lowest we can drain (pg) too?");
1431 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1432 SYSCTL_CHILDREN(bbr_states),
1433 OID_AUTO, "drain_2_target", CTLFLAG_RW,
1434 &bbr_state_drain_2_tar, 1,
1435 "Do we drain to target in drain substate?");
1436 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1437 SYSCTL_CHILDREN(bbr_states),
1438 OID_AUTO, "gain_2_target", CTLFLAG_RW,
1439 &bbr_gain_to_target, 1,
1440 "Does probe bw gain to target??");
1441 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1442 SYSCTL_CHILDREN(bbr_states),
1443 OID_AUTO, "gain_extra_time", CTLFLAG_RW,
1444 &bbr_gain_gets_extra_too, 1,
1445 "Does probe bw gain get the extra time too?");
1446 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1447 SYSCTL_CHILDREN(bbr_states),
1448 OID_AUTO, "ld_div", CTLFLAG_RW,
1449 &bbr_drain_drop_div, 5,
1450 "Long drain drop divider?");
1451 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1452 SYSCTL_CHILDREN(bbr_states),
1453 OID_AUTO, "ld_mul", CTLFLAG_RW,
1454 &bbr_drain_drop_mul, 4,
1455 "Long drain drop multiplier?");
1456 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1457 SYSCTL_CHILDREN(bbr_states),
1458 OID_AUTO, "rand_ot_disc", CTLFLAG_RW,
1459 &bbr_rand_ot, 50,
1460 "Random discount of the ot?");
1461 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1462 SYSCTL_CHILDREN(bbr_states),
1463 OID_AUTO, "dr_filter_life", CTLFLAG_RW,
1464 &bbr_num_pktepo_for_del_limit, BBR_NUM_RTTS_FOR_DEL_LIMIT,
1465 "How many packet-epochs does the b/w delivery rate last?");
1466 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1467 SYSCTL_CHILDREN(bbr_states),
1468 OID_AUTO, "subdrain_applimited", CTLFLAG_RW,
1469 &bbr_sub_drain_app_limit, 0,
1470 "Does our sub-state drain invoke app limited if its long?");
1471 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1472 SYSCTL_CHILDREN(bbr_states),
1473 OID_AUTO, "use_cwnd_subdrain", CTLFLAG_RW,
1474 &bbr_sub_drain_slam_cwnd, 0,
1475 "Should we set/recover cwnd for sub-state drain?");
1476 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1477 SYSCTL_CHILDREN(bbr_states),
1478 OID_AUTO, "use_cwnd_maindrain", CTLFLAG_RW,
1479 &bbr_slam_cwnd_in_main_drain, 0,
1480 "Should we set/recover cwnd for main-state drain?");
1481 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1482 SYSCTL_CHILDREN(bbr_states),
1483 OID_AUTO, "google_gets_earlyout", CTLFLAG_RW,
1484 &google_allow_early_out, 1,
1485 "Should we allow google probe-bw/drain to exit early at flight target?");
1486 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1487 SYSCTL_CHILDREN(bbr_states),
1488 OID_AUTO, "google_exit_loss", CTLFLAG_RW,
1489 &google_consider_lost, 1,
1490 "Should we have losses exit gain of probebw in google mode??");
1491 /* Startup controls */
1492 bbr_startup = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1493 SYSCTL_CHILDREN(bbr_sysctl_root),
1494 OID_AUTO,
1495 "startup",
1496 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1497 "Startup controls");
1498 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1499 SYSCTL_CHILDREN(bbr_startup),
1500 OID_AUTO, "cheat_iwnd", CTLFLAG_RW,
1501 &bbr_sends_full_iwnd, 1,
1502 "Do we not pace but burst out initial windows has our TSO size?");
1503 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1504 SYSCTL_CHILDREN(bbr_startup),
1505 OID_AUTO, "loss_threshold", CTLFLAG_RW,
1506 &bbr_startup_loss_thresh, 2000,
1507 "In startup what is the loss threshold in a pe that will exit us from startup?");
1508 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1509 SYSCTL_CHILDREN(bbr_startup),
1510 OID_AUTO, "use_lowerpg", CTLFLAG_RW,
1511 &bbr_use_lower_gain_in_startup, 1,
1512 "Should we use a lower hptsi gain if we see loss in startup?");
1513 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1514 SYSCTL_CHILDREN(bbr_startup),
1515 OID_AUTO, "gain", CTLFLAG_RW,
1516 &bbr_start_exit, 25,
1517 "What gain percent do we need to see to stay in startup??");
1518 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1519 SYSCTL_CHILDREN(bbr_startup),
1520 OID_AUTO, "low_gain", CTLFLAG_RW,
1521 &bbr_low_start_exit, 15,
1522 "What gain percent do we need to see to stay in the lower gain startup??");
1523 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1524 SYSCTL_CHILDREN(bbr_startup),
1525 OID_AUTO, "loss_exit", CTLFLAG_RW,
1526 &bbr_exit_startup_at_loss, 1,
1527 "Should we exit startup at loss in an epoch if we are not gaining?");
1528 /* CWND controls */
1529 bbr_cwnd = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1530 SYSCTL_CHILDREN(bbr_sysctl_root),
1531 OID_AUTO,
1532 "cwnd",
1533 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1534 "Cwnd controls");
1535 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1536 SYSCTL_CHILDREN(bbr_cwnd),
1537 OID_AUTO, "tar_rtt", CTLFLAG_RW,
1538 &bbr_cwndtarget_rtt_touse, 0,
1539 "Target cwnd rtt measurement to use (0=rtt_prop, 1=rtt_rack, 2=pkt_rtt, 3=srtt)?");
1540 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1541 SYSCTL_CHILDREN(bbr_cwnd),
1542 OID_AUTO, "may_shrink", CTLFLAG_RW,
1543 &bbr_cwnd_may_shrink, 0,
1544 "Can the cwnd shrink if it would grow to more than the target?");
1545 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1546 SYSCTL_CHILDREN(bbr_cwnd),
1547 OID_AUTO, "max_target_limit", CTLFLAG_RW,
1548 &bbr_target_cwnd_mult_limit, 8,
1549 "Do we limit the cwnd to some multiple of the cwnd target if cwnd can't shrink 0=no?");
1550 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1551 SYSCTL_CHILDREN(bbr_cwnd),
1552 OID_AUTO, "highspeed_min", CTLFLAG_RW,
1553 &bbr_cwnd_min_val_hs, BBR_HIGHSPEED_NUM_MSS,
1554 "What is the high-speed min cwnd (rttProp under 1ms)");
1555 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1556 SYSCTL_CHILDREN(bbr_cwnd),
1557 OID_AUTO, "lowspeed_min", CTLFLAG_RW,
1558 &bbr_cwnd_min_val, BBR_PROBERTT_NUM_MSS,
1559 "What is the min cwnd (rttProp > 1ms)");
1560 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1561 SYSCTL_CHILDREN(bbr_cwnd),
1562 OID_AUTO, "initwin", CTLFLAG_RW,
1563 &bbr_def_init_win, 10,
1564 "What is the BBR initial window, if 0 use tcp version");
1565 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1566 SYSCTL_CHILDREN(bbr_cwnd),
1567 OID_AUTO, "do_loss_red", CTLFLAG_RW,
1568 &bbr_do_red, 600,
1569 "Do we reduce the b/w at exit from recovery based on ratio of prop/srtt (800=80.0, 0=off)?");
1570 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1571 SYSCTL_CHILDREN(bbr_cwnd),
1572 OID_AUTO, "red_scale", CTLFLAG_RW,
1573 &bbr_red_scale, 20000,
1574 "What RTT do we scale with?");
1575 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1576 SYSCTL_CHILDREN(bbr_cwnd),
1577 OID_AUTO, "red_growslow", CTLFLAG_RW,
1578 &bbr_red_growth_restrict, 1,
1579 "Do we restrict cwnd growth for whats in flight?");
1580 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1581 SYSCTL_CHILDREN(bbr_cwnd),
1582 OID_AUTO, "red_div", CTLFLAG_RW,
1583 &bbr_red_div, 2,
1584 "If we reduce whats the divisor?");
1585 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1586 SYSCTL_CHILDREN(bbr_cwnd),
1587 OID_AUTO, "red_mul", CTLFLAG_RW,
1588 &bbr_red_mul, 1,
1589 "If we reduce whats the mulitiplier?");
1590 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1591 SYSCTL_CHILDREN(bbr_cwnd),
1592 OID_AUTO, "target_is_unit", CTLFLAG_RW,
1593 &bbr_target_is_bbunit, 0,
1594 "Is the state target the pacing_gain or BBR_UNIT?");
1595 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1596 SYSCTL_CHILDREN(bbr_cwnd),
1597 OID_AUTO, "drop_limit", CTLFLAG_RW,
1598 &bbr_drop_limit, 0,
1599 "Number of segments limit for drop (0=use min_cwnd w/flight)?");
1600
1601 /* Timeout controls */
1602 bbr_timeout = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1603 SYSCTL_CHILDREN(bbr_sysctl_root),
1604 OID_AUTO,
1605 "timeout",
1606 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1607 "Time out controls");
1608 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1609 SYSCTL_CHILDREN(bbr_timeout),
1610 OID_AUTO, "delack", CTLFLAG_RW,
1611 &bbr_delack_time, 100000,
1612 "BBR's delayed ack time");
1613 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1614 SYSCTL_CHILDREN(bbr_timeout),
1615 OID_AUTO, "tlp_uses", CTLFLAG_RW,
1616 &bbr_tlp_type_to_use, 3,
1617 "RTT that TLP uses in its calculations, 0=rttProp, 1=Rack_rtt, 2=pkt_rtt and 3=srtt");
1618 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1619 SYSCTL_CHILDREN(bbr_timeout),
1620 OID_AUTO, "persmin", CTLFLAG_RW,
1621 &bbr_persist_min, 250000,
1622 "What is the minimum time in microseconds between persists");
1623 SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1624 SYSCTL_CHILDREN(bbr_timeout),
1625 OID_AUTO, "persmax", CTLFLAG_RW,
1626 &bbr_persist_max, 1000000,
1627 "What is the largest delay in microseconds between persists");
1628 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1629 SYSCTL_CHILDREN(bbr_timeout),
1630 OID_AUTO, "tlp_minto", CTLFLAG_RW,
1631 &bbr_tlp_min, 10000,
1632 "TLP Min timeout in usecs");
1633 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1634 SYSCTL_CHILDREN(bbr_timeout),
1635 OID_AUTO, "tlp_dack_time", CTLFLAG_RW,
1636 &bbr_delayed_ack_time, 200000,
1637 "TLP delayed ack compensation value");
1638 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1639 SYSCTL_CHILDREN(bbr_sysctl_root),
1640 OID_AUTO, "minrto", CTLFLAG_RW,
1641 &bbr_rto_min_ms, 30,
1642 "Minimum RTO in ms");
1643 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1644 SYSCTL_CHILDREN(bbr_timeout),
1645 OID_AUTO, "maxrto", CTLFLAG_RW,
1646 &bbr_rto_max_sec, 4,
1647 "Maximum RTO in seconds -- should be at least as large as min_rto");
1648 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1649 SYSCTL_CHILDREN(bbr_timeout),
1650 OID_AUTO, "tlp_retry", CTLFLAG_RW,
1651 &bbr_tlp_max_resend, 2,
1652 "How many times does TLP retry a single segment or multiple with no ACK");
1653 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1654 SYSCTL_CHILDREN(bbr_timeout),
1655 OID_AUTO, "minto", CTLFLAG_RW,
1656 &bbr_min_to, 1000,
1657 "Minimum rack timeout in useconds");
1658 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1659 SYSCTL_CHILDREN(bbr_timeout),
1660 OID_AUTO, "pktdelay", CTLFLAG_RW,
1661 &bbr_pkt_delay, 1000,
1662 "Extra RACK time (in useconds) besides reordering thresh");
1663 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1664 SYSCTL_CHILDREN(bbr_timeout),
1665 OID_AUTO, "incr_tmrs", CTLFLAG_RW,
1666 &bbr_incr_timers, 1,
1667 "Increase the RXT/TLP timer by the pacing time used?");
1668 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1669 SYSCTL_CHILDREN(bbr_timeout),
1670 OID_AUTO, "rxtmark_sackpassed", CTLFLAG_RW,
1671 &bbr_marks_rxt_sack_passed, 0,
1672 "Mark sack passed on all those not ack'd when a RXT hits?");
1673 /* Policer controls */
1674 bbr_policer = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1675 SYSCTL_CHILDREN(bbr_sysctl_root),
1676 OID_AUTO,
1677 "policer",
1678 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1679 "Policer controls");
1680 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1681 SYSCTL_CHILDREN(bbr_policer),
1682 OID_AUTO, "detect_enable", CTLFLAG_RW,
1683 &bbr_policer_detection_enabled, 1,
1684 "Is policer detection enabled??");
1685 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1686 SYSCTL_CHILDREN(bbr_policer),
1687 OID_AUTO, "min_pes", CTLFLAG_RW,
1688 &bbr_lt_intvl_min_rtts, 4,
1689 "Minimum number of PE's?");
1690 SYSCTL_ADD_U64(&bbr_sysctl_ctx,
1691 SYSCTL_CHILDREN(bbr_policer),
1692 OID_AUTO, "bwdiff", CTLFLAG_RW,
1693 &bbr_lt_bw_diff, (4000/8),
1694 "Minimal bw diff?");
1695 SYSCTL_ADD_U64(&bbr_sysctl_ctx,
1696 SYSCTL_CHILDREN(bbr_policer),
1697 OID_AUTO, "bwratio", CTLFLAG_RW,
1698 &bbr_lt_bw_ratio, 8,
1699 "Minimal bw diff?");
1700 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1701 SYSCTL_CHILDREN(bbr_policer),
1702 OID_AUTO, "from_rack_rxt", CTLFLAG_RW,
1703 &bbr_policer_call_from_rack_to, 0,
1704 "Do we call the policer detection code from a rack-timeout?");
1705 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1706 SYSCTL_CHILDREN(bbr_policer),
1707 OID_AUTO, "false_postive", CTLFLAG_RW,
1708 &bbr_lt_intvl_fp, 0,
1709 "What packet epoch do we do false-positive detection at (0=no)?");
1710 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1711 SYSCTL_CHILDREN(bbr_policer),
1712 OID_AUTO, "loss_thresh", CTLFLAG_RW,
1713 &bbr_lt_loss_thresh, 196,
1714 "Loss threshold 196 = 19.6%?");
1715 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1716 SYSCTL_CHILDREN(bbr_policer),
1717 OID_AUTO, "false_postive_thresh", CTLFLAG_RW,
1718 &bbr_lt_fd_thresh, 100,
1719 "What percentage is the false detection threshold (150=15.0)?");
1720 /* All the rest */
1721 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1722 SYSCTL_CHILDREN(bbr_sysctl_root),
1723 OID_AUTO, "cheat_rxt", CTLFLAG_RW,
1724 &bbr_use_rack_resend_cheat, 0,
1725 "Do we burst 1ms between sends on retransmissions (like rack)?");
1726 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1727 SYSCTL_CHILDREN(bbr_sysctl_root),
1728 OID_AUTO, "error_paceout", CTLFLAG_RW,
1729 &bbr_error_base_paceout, 10000,
1730 "When we hit an error what is the min to pace out in usec's?");
1731 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1732 SYSCTL_CHILDREN(bbr_sysctl_root),
1733 OID_AUTO, "kill_paceout", CTLFLAG_RW,
1734 &bbr_max_net_error_cnt, 10,
1735 "When we hit this many errors in a row, kill the session?");
1736 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1737 SYSCTL_CHILDREN(bbr_sysctl_root),
1738 OID_AUTO, "data_after_close", CTLFLAG_RW,
1739 &bbr_ignore_data_after_close, 1,
1740 "Do we hold off sending a RST until all pending data is ack'd");
1741 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1742 SYSCTL_CHILDREN(bbr_sysctl_root),
1743 OID_AUTO, "resend_use_tso", CTLFLAG_RW,
1744 &bbr_resends_use_tso, 0,
1745 "Can resends use TSO?");
1746 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1747 SYSCTL_CHILDREN(bbr_sysctl_root),
1748 OID_AUTO, "sblklimit", CTLFLAG_RW,
1749 &bbr_sack_block_limit, 128,
1750 "When do we start ignoring small sack blocks");
1751 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1752 SYSCTL_CHILDREN(bbr_sysctl_root),
1753 OID_AUTO, "bb_verbose", CTLFLAG_RW,
1754 &bbr_verbose_logging, 0,
1755 "Should BBR black box logging be verbose");
1756 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1757 SYSCTL_CHILDREN(bbr_sysctl_root),
1758 OID_AUTO, "reorder_thresh", CTLFLAG_RW,
1759 &bbr_reorder_thresh, 2,
1760 "What factor for rack will be added when seeing reordering (shift right)");
1761 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1762 SYSCTL_CHILDREN(bbr_sysctl_root),
1763 OID_AUTO, "reorder_fade", CTLFLAG_RW,
1764 &bbr_reorder_fade, 0,
1765 "Does reorder detection fade, if so how many ms (0 means never)");
1766 SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1767 SYSCTL_CHILDREN(bbr_sysctl_root),
1768 OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW,
1769 &bbr_tlp_thresh, 1,
1770 "what divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)");
1771 /* Stats and counters */
1772 /* The pacing counters for hdwr/software can't be in the array */
1773 bbr_nohdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK);
1774 bbr_hdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK);
1775 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1776 SYSCTL_CHILDREN(bbr_sysctl_root),
1777 OID_AUTO, "enob_hdwr_pacing", CTLFLAG_RD,
1778 &bbr_hdwr_pacing_enobuf,
1779 "Total number of enobufs for hardware paced flows");
1780 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1781 SYSCTL_CHILDREN(bbr_sysctl_root),
1782 OID_AUTO, "enob_no_hdwr_pacing", CTLFLAG_RD,
1783 &bbr_nohdwr_pacing_enobuf,
1784 "Total number of enobufs for non-hardware paced flows");
1785
1786 bbr_flows_whdwr_pacing = counter_u64_alloc(M_WAITOK);
1787 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1788 SYSCTL_CHILDREN(bbr_sysctl_root),
1789 OID_AUTO, "hdwr_pacing", CTLFLAG_RD,
1790 &bbr_flows_whdwr_pacing,
1791 "Total number of hardware paced flows");
1792 bbr_flows_nohdwr_pacing = counter_u64_alloc(M_WAITOK);
1793 SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1794 SYSCTL_CHILDREN(bbr_sysctl_root),
1795 OID_AUTO, "software_pacing", CTLFLAG_RD,
1796 &bbr_flows_nohdwr_pacing,
1797 "Total number of software paced flows");
1798 COUNTER_ARRAY_ALLOC(bbr_stat_arry, BBR_STAT_SIZE, M_WAITOK);
1799 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1800 OID_AUTO, "stats", CTLFLAG_RD,
1801 bbr_stat_arry, BBR_STAT_SIZE, "BBR Stats");
1802 COUNTER_ARRAY_ALLOC(bbr_opts_arry, BBR_OPTS_SIZE, M_WAITOK);
1803 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1804 OID_AUTO, "opts", CTLFLAG_RD,
1805 bbr_opts_arry, BBR_OPTS_SIZE, "BBR Option Stats");
1806 COUNTER_ARRAY_ALLOC(bbr_state_lost, BBR_MAX_STAT, M_WAITOK);
1807 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1808 OID_AUTO, "lost", CTLFLAG_RD,
1809 bbr_state_lost, BBR_MAX_STAT, "Stats of when losses occur");
1810 COUNTER_ARRAY_ALLOC(bbr_state_resend, BBR_MAX_STAT, M_WAITOK);
1811 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1812 OID_AUTO, "stateresend", CTLFLAG_RD,
1813 bbr_state_resend, BBR_MAX_STAT, "Stats of what states resend");
1814 COUNTER_ARRAY_ALLOC(bbr_state_time, BBR_MAX_STAT, M_WAITOK);
1815 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1816 OID_AUTO, "statetime", CTLFLAG_RD,
1817 bbr_state_time, BBR_MAX_STAT, "Stats of time spent in the states");
1818 COUNTER_ARRAY_ALLOC(bbr_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK);
1819 SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1820 OID_AUTO, "outsize", CTLFLAG_RD,
1821 bbr_out_size, TCP_MSS_ACCT_SIZE, "Size of output calls");
1822 SYSCTL_ADD_PROC(&bbr_sysctl_ctx,
1823 SYSCTL_CHILDREN(bbr_sysctl_root),
1824 OID_AUTO, "clrlost", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1825 &bbr_clear_lost, 0, sysctl_bbr_clear_lost, "IU", "Clear lost counters");
1826 }
1827
1828 static void
bbr_counter_destroy(void)1829 bbr_counter_destroy(void)
1830 {
1831 COUNTER_ARRAY_FREE(bbr_stat_arry, BBR_STAT_SIZE);
1832 COUNTER_ARRAY_FREE(bbr_opts_arry, BBR_OPTS_SIZE);
1833 COUNTER_ARRAY_FREE(bbr_out_size, TCP_MSS_ACCT_SIZE);
1834 COUNTER_ARRAY_FREE(bbr_state_lost, BBR_MAX_STAT);
1835 COUNTER_ARRAY_FREE(bbr_state_time, BBR_MAX_STAT);
1836 COUNTER_ARRAY_FREE(bbr_state_resend, BBR_MAX_STAT);
1837 counter_u64_free(bbr_nohdwr_pacing_enobuf);
1838 counter_u64_free(bbr_hdwr_pacing_enobuf);
1839 counter_u64_free(bbr_flows_whdwr_pacing);
1840 counter_u64_free(bbr_flows_nohdwr_pacing);
1841
1842 }
1843
1844 static __inline void
bbr_fill_in_logging_data(struct tcp_bbr * bbr,struct tcp_log_bbr * l,uint32_t cts)1845 bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts)
1846 {
1847 memset(l, 0, sizeof(union tcp_log_stackspecific));
1848 l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;
1849 l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);
1850 l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
1851 l->bw_inuse = bbr_get_bw(bbr);
1852 l->inflight = ctf_flight_size(bbr->rc_tp,
1853 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
1854 l->applimited = bbr->r_ctl.r_app_limited_until;
1855 l->delivered = bbr->r_ctl.rc_delivered;
1856 l->timeStamp = cts;
1857 l->lost = bbr->r_ctl.rc_lost;
1858 l->bbr_state = bbr->rc_bbr_state;
1859 l->bbr_substate = bbr_state_val(bbr);
1860 l->epoch = bbr->r_ctl.rc_rtt_epoch;
1861 l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
1862 l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
1863 l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
1864 l->inhpts = tcp_in_hpts(bbr->rc_tp);
1865 l->use_lt_bw = bbr->rc_lt_use_bw;
1866 l->pkts_out = bbr->r_ctl.rc_flight_at_input;
1867 l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;
1868 }
1869
1870 static void
bbr_log_type_bw_reduce(struct tcp_bbr * bbr,int reason)1871 bbr_log_type_bw_reduce(struct tcp_bbr *bbr, int reason)
1872 {
1873 if (tcp_bblogging_on(bbr->rc_tp)) {
1874 union tcp_log_stackspecific log;
1875
1876 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1877 log.u_bbr.flex1 = 0;
1878 log.u_bbr.flex2 = 0;
1879 log.u_bbr.flex5 = 0;
1880 log.u_bbr.flex3 = 0;
1881 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;
1882 log.u_bbr.flex7 = reason;
1883 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;
1884 log.u_bbr.flex8 = 0;
1885 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1886 &bbr->rc_inp->inp_socket->so_rcv,
1887 &bbr->rc_inp->inp_socket->so_snd,
1888 BBR_LOG_BW_RED_EV, 0,
1889 0, &log, false, &bbr->rc_tv);
1890 }
1891 }
1892
1893 static void
bbr_log_type_rwnd_collapse(struct tcp_bbr * bbr,int seq,int mode,uint32_t count)1894 bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count)
1895 {
1896 if (tcp_bblogging_on(bbr->rc_tp)) {
1897 union tcp_log_stackspecific log;
1898
1899 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1900 log.u_bbr.flex1 = seq;
1901 log.u_bbr.flex2 = count;
1902 log.u_bbr.flex8 = mode;
1903 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1904 &bbr->rc_inp->inp_socket->so_rcv,
1905 &bbr->rc_inp->inp_socket->so_snd,
1906 BBR_LOG_LOWGAIN, 0,
1907 0, &log, false, &bbr->rc_tv);
1908 }
1909 }
1910
1911 static void
bbr_log_type_just_return(struct tcp_bbr * bbr,uint32_t cts,uint32_t tlen,uint8_t hpts_calling,uint8_t reason,uint32_t p_maxseg,int len)1912 bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling,
1913 uint8_t reason, uint32_t p_maxseg, int len)
1914 {
1915 if (tcp_bblogging_on(bbr->rc_tp)) {
1916 union tcp_log_stackspecific log;
1917
1918 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1919 log.u_bbr.flex1 = p_maxseg;
1920 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;
1921 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
1922 log.u_bbr.flex4 = reason;
1923 log.u_bbr.flex5 = bbr->rc_in_persist;
1924 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;
1925 log.u_bbr.flex7 = p_maxseg;
1926 log.u_bbr.flex8 = bbr->rc_in_persist;
1927 log.u_bbr.pkts_out = 0;
1928 log.u_bbr.applimited = len;
1929 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1930 &bbr->rc_inp->inp_socket->so_rcv,
1931 &bbr->rc_inp->inp_socket->so_snd,
1932 BBR_LOG_JUSTRET, 0,
1933 tlen, &log, false, &bbr->rc_tv);
1934 }
1935 }
1936
1937 static void
bbr_log_type_enter_rec(struct tcp_bbr * bbr,uint32_t seq)1938 bbr_log_type_enter_rec(struct tcp_bbr *bbr, uint32_t seq)
1939 {
1940 if (tcp_bblogging_on(bbr->rc_tp)) {
1941 union tcp_log_stackspecific log;
1942
1943 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1944 log.u_bbr.flex1 = seq;
1945 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
1946 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;
1947 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1948 &bbr->rc_inp->inp_socket->so_rcv,
1949 &bbr->rc_inp->inp_socket->so_snd,
1950 BBR_LOG_ENTREC, 0,
1951 0, &log, false, &bbr->rc_tv);
1952 }
1953 }
1954
1955 static void
bbr_log_msgsize_fail(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t len,uint32_t maxseg,uint32_t mtu,int32_t csum_flags,int32_t tso,uint32_t cts)1956 bbr_log_msgsize_fail(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)
1957 {
1958 if (tcp_bblogging_on(tp)) {
1959 union tcp_log_stackspecific log;
1960
1961 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1962 log.u_bbr.flex1 = tso;
1963 log.u_bbr.flex2 = maxseg;
1964 log.u_bbr.flex3 = mtu;
1965 log.u_bbr.flex4 = csum_flags;
1966 TCP_LOG_EVENTP(tp, NULL,
1967 &bbr->rc_inp->inp_socket->so_rcv,
1968 &bbr->rc_inp->inp_socket->so_snd,
1969 BBR_LOG_MSGSIZE, 0,
1970 0, &log, false, &bbr->rc_tv);
1971 }
1972 }
1973
1974 static void
bbr_log_flowend(struct tcp_bbr * bbr)1975 bbr_log_flowend(struct tcp_bbr *bbr)
1976 {
1977 if (tcp_bblogging_on(bbr->rc_tp)) {
1978 union tcp_log_stackspecific log;
1979 struct sockbuf *r, *s;
1980 struct timeval tv;
1981
1982 if (bbr->rc_inp->inp_socket) {
1983 r = &bbr->rc_inp->inp_socket->so_rcv;
1984 s = &bbr->rc_inp->inp_socket->so_snd;
1985 } else {
1986 r = s = NULL;
1987 }
1988 bbr_fill_in_logging_data(bbr, &log.u_bbr, tcp_get_usecs(&tv));
1989 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1990 r, s,
1991 TCP_LOG_FLOWEND, 0,
1992 0, &log, false, &tv);
1993 }
1994 }
1995
1996 static void
bbr_log_pkt_epoch(struct tcp_bbr * bbr,uint32_t cts,uint32_t line,uint32_t lost,uint32_t del)1997 bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line,
1998 uint32_t lost, uint32_t del)
1999 {
2000 if (tcp_bblogging_on(bbr->rc_tp)) {
2001 union tcp_log_stackspecific log;
2002
2003 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2004 log.u_bbr.flex1 = lost;
2005 log.u_bbr.flex2 = del;
2006 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;
2007 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;
2008 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2009 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2010 log.u_bbr.flex7 = line;
2011 log.u_bbr.flex8 = 0;
2012 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;
2013 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2014 &bbr->rc_inp->inp_socket->so_rcv,
2015 &bbr->rc_inp->inp_socket->so_snd,
2016 BBR_LOG_PKT_EPOCH, 0,
2017 0, &log, false, &bbr->rc_tv);
2018 }
2019 }
2020
2021 static void
bbr_log_time_epoch(struct tcp_bbr * bbr,uint32_t cts,uint32_t line,uint32_t epoch_time)2022 bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time)
2023 {
2024 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2025 union tcp_log_stackspecific log;
2026
2027 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2028 log.u_bbr.flex1 = bbr->r_ctl.rc_lost;
2029 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;
2030 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;
2031 log.u_bbr.flex7 = line;
2032 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2033 &bbr->rc_inp->inp_socket->so_rcv,
2034 &bbr->rc_inp->inp_socket->so_snd,
2035 BBR_LOG_TIME_EPOCH, 0,
2036 0, &log, false, &bbr->rc_tv);
2037 }
2038 }
2039
2040 static void
bbr_log_set_of_state_target(struct tcp_bbr * bbr,uint32_t new_tar,int line,int meth)2041 bbr_log_set_of_state_target(struct tcp_bbr *bbr, uint32_t new_tar, int line, int meth)
2042 {
2043 if (tcp_bblogging_on(bbr->rc_tp)) {
2044 union tcp_log_stackspecific log;
2045
2046 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2047 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2048 log.u_bbr.flex2 = new_tar;
2049 log.u_bbr.flex3 = line;
2050 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2051 log.u_bbr.flex5 = bbr_quanta;
2052 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;
2053 log.u_bbr.flex7 = bbr->rc_last_options;
2054 log.u_bbr.flex8 = meth;
2055 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2056 &bbr->rc_inp->inp_socket->so_rcv,
2057 &bbr->rc_inp->inp_socket->so_snd,
2058 BBR_LOG_STATE_TARGET, 0,
2059 0, &log, false, &bbr->rc_tv);
2060 }
2061
2062 }
2063
2064 static void
bbr_log_type_statechange(struct tcp_bbr * bbr,uint32_t cts,int32_t line)2065 bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2066 {
2067 if (tcp_bblogging_on(bbr->rc_tp)) {
2068 union tcp_log_stackspecific log;
2069
2070 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2071 log.u_bbr.flex1 = line;
2072 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2073 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2074 if (bbr_state_is_pkt_epoch)
2075 log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
2076 else
2077 log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PROP);
2078 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2079 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2080 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);
2081 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;
2082 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;
2083 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2084 &bbr->rc_inp->inp_socket->so_rcv,
2085 &bbr->rc_inp->inp_socket->so_snd,
2086 BBR_LOG_STATE, 0,
2087 0, &log, false, &bbr->rc_tv);
2088 }
2089 }
2090
2091 static void
bbr_log_rtt_shrinks(struct tcp_bbr * bbr,uint32_t cts,uint32_t applied,uint32_t rtt,uint32_t line,uint8_t reas,uint16_t cond)2092 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
2093 uint32_t rtt, uint32_t line, uint8_t reas, uint16_t cond)
2094 {
2095 if (tcp_bblogging_on(bbr->rc_tp)) {
2096 union tcp_log_stackspecific log;
2097
2098 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2099 log.u_bbr.flex1 = line;
2100 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2101 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;
2102 log.u_bbr.flex4 = applied;
2103 log.u_bbr.flex5 = rtt;
2104 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2105 log.u_bbr.flex7 = cond;
2106 log.u_bbr.flex8 = reas;
2107 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2108 &bbr->rc_inp->inp_socket->so_rcv,
2109 &bbr->rc_inp->inp_socket->so_snd,
2110 BBR_LOG_RTT_SHRINKS, 0,
2111 0, &log, false, &bbr->rc_tv);
2112 }
2113 }
2114
2115 static void
bbr_log_type_exit_rec(struct tcp_bbr * bbr)2116 bbr_log_type_exit_rec(struct tcp_bbr *bbr)
2117 {
2118 if (tcp_bblogging_on(bbr->rc_tp)) {
2119 union tcp_log_stackspecific log;
2120
2121 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2122 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;
2123 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
2124 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2125 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2126 &bbr->rc_inp->inp_socket->so_rcv,
2127 &bbr->rc_inp->inp_socket->so_snd,
2128 BBR_LOG_EXITREC, 0,
2129 0, &log, false, &bbr->rc_tv);
2130 }
2131 }
2132
2133 static void
bbr_log_type_cwndupd(struct tcp_bbr * bbr,uint32_t bytes_this_ack,uint32_t chg,uint32_t prev_acked,int32_t meth,uint32_t target,uint32_t th_ack,int32_t line)2134 bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg,
2135 uint32_t prev_acked, int32_t meth, uint32_t target, uint32_t th_ack, int32_t line)
2136 {
2137 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2138 union tcp_log_stackspecific log;
2139
2140 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2141 log.u_bbr.flex1 = line;
2142 log.u_bbr.flex2 = prev_acked;
2143 log.u_bbr.flex3 = bytes_this_ack;
2144 log.u_bbr.flex4 = chg;
2145 log.u_bbr.flex5 = th_ack;
2146 log.u_bbr.flex6 = target;
2147 log.u_bbr.flex8 = meth;
2148 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2149 &bbr->rc_inp->inp_socket->so_rcv,
2150 &bbr->rc_inp->inp_socket->so_snd,
2151 BBR_LOG_CWND, 0,
2152 0, &log, false, &bbr->rc_tv);
2153 }
2154 }
2155
2156 static void
bbr_log_rtt_sample(struct tcp_bbr * bbr,uint32_t rtt,uint32_t tsin)2157 bbr_log_rtt_sample(struct tcp_bbr *bbr, uint32_t rtt, uint32_t tsin)
2158 {
2159 /*
2160 * Log the rtt sample we are applying to the srtt algorithm in
2161 * useconds.
2162 */
2163 if (tcp_bblogging_on(bbr->rc_tp)) {
2164 union tcp_log_stackspecific log;
2165
2166 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2167 log.u_bbr.flex1 = rtt;
2168 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;
2169 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;
2170 log.u_bbr.flex4 = bbr->rc_tp->ts_offset;
2171 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2172 log.u_bbr.pkts_out = tcp_tv_to_msec(&bbr->rc_tv);
2173 log.u_bbr.flex6 = tsin;
2174 log.u_bbr.flex7 = 0;
2175 log.u_bbr.flex8 = bbr->rc_ack_was_delayed;
2176 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2177 &bbr->rc_inp->inp_socket->so_rcv,
2178 &bbr->rc_inp->inp_socket->so_snd,
2179 TCP_LOG_RTT, 0,
2180 0, &log, false, &bbr->rc_tv);
2181 }
2182 }
2183
2184 static void
bbr_log_type_pesist(struct tcp_bbr * bbr,uint32_t cts,uint32_t time_in,int32_t line,uint8_t enter_exit)2185 bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t enter_exit)
2186 {
2187 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2188 union tcp_log_stackspecific log;
2189
2190 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2191 log.u_bbr.flex1 = time_in;
2192 log.u_bbr.flex2 = line;
2193 log.u_bbr.flex8 = enter_exit;
2194 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2195 &bbr->rc_inp->inp_socket->so_rcv,
2196 &bbr->rc_inp->inp_socket->so_snd,
2197 BBR_LOG_PERSIST, 0,
2198 0, &log, false, &bbr->rc_tv);
2199 }
2200 }
2201 static void
bbr_log_ack_clear(struct tcp_bbr * bbr,uint32_t cts)2202 bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts)
2203 {
2204 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2205 union tcp_log_stackspecific log;
2206
2207 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2208 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;
2209 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2210 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2211 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;
2212 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2213 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2214 &bbr->rc_inp->inp_socket->so_rcv,
2215 &bbr->rc_inp->inp_socket->so_snd,
2216 BBR_LOG_ACKCLEAR, 0,
2217 0, &log, false, &bbr->rc_tv);
2218 }
2219 }
2220
2221 static void
bbr_log_ack_event(struct tcp_bbr * bbr,struct tcphdr * th,struct tcpopt * to,uint32_t tlen,uint16_t nsegs,uint32_t cts,int32_t nxt_pkt,struct mbuf * m)2222 bbr_log_ack_event(struct tcp_bbr *bbr, struct tcphdr *th, struct tcpopt *to, uint32_t tlen,
2223 uint16_t nsegs, uint32_t cts, int32_t nxt_pkt, struct mbuf *m)
2224 {
2225 if (tcp_bblogging_on(bbr->rc_tp)) {
2226 union tcp_log_stackspecific log;
2227 struct timeval tv;
2228
2229 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2230 log.u_bbr.flex1 = nsegs;
2231 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;
2232 if (m) {
2233 struct timespec ts;
2234
2235 log.u_bbr.flex3 = m->m_flags;
2236 if (m->m_flags & M_TSTMP) {
2237 mbuf_tstmp2timespec(m, &ts);
2238 tv.tv_sec = ts.tv_sec;
2239 tv.tv_usec = ts.tv_nsec / 1000;
2240 log.u_bbr.lt_epoch = tcp_tv_to_usec(&tv);
2241 } else {
2242 log.u_bbr.lt_epoch = 0;
2243 }
2244 if (m->m_flags & M_TSTMP_LRO) {
2245 mbuf_tstmp2timeval(m, &tv);
2246 log.u_bbr.flex5 = tcp_tv_to_usec(&tv);
2247 } else {
2248 /* No arrival timestamp */
2249 log.u_bbr.flex5 = 0;
2250 }
2251
2252 log.u_bbr.pkts_out = tcp_get_usecs(&tv);
2253 } else {
2254 log.u_bbr.flex3 = 0;
2255 log.u_bbr.flex5 = 0;
2256 log.u_bbr.flex6 = 0;
2257 log.u_bbr.pkts_out = 0;
2258 }
2259 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;
2260 log.u_bbr.flex7 = bbr->r_wanted_output;
2261 log.u_bbr.flex8 = bbr->rc_in_persist;
2262 TCP_LOG_EVENTP(bbr->rc_tp, th,
2263 &bbr->rc_inp->inp_socket->so_rcv,
2264 &bbr->rc_inp->inp_socket->so_snd,
2265 TCP_LOG_IN, 0,
2266 tlen, &log, true, &bbr->rc_tv);
2267 }
2268 }
2269
2270 static void
bbr_log_doseg_done(struct tcp_bbr * bbr,uint32_t cts,int32_t nxt_pkt,int32_t did_out)2271 bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out)
2272 {
2273 if (tcp_bblogging_on(bbr->rc_tp)) {
2274 union tcp_log_stackspecific log;
2275
2276 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2277 log.u_bbr.flex1 = did_out;
2278 log.u_bbr.flex2 = nxt_pkt;
2279 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;
2280 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2281 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;
2282 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;
2283 log.u_bbr.flex7 = bbr->r_wanted_output;
2284 log.u_bbr.flex8 = bbr->rc_in_persist;
2285 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;
2286 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2287 &bbr->rc_inp->inp_socket->so_rcv,
2288 &bbr->rc_inp->inp_socket->so_snd,
2289 BBR_LOG_DOSEG_DONE, 0,
2290 0, &log, true, &bbr->rc_tv);
2291 }
2292 }
2293
2294 static void
bbr_log_enobuf_jmp(struct tcp_bbr * bbr,uint32_t len,uint32_t cts,int32_t line,uint32_t o_len,uint32_t segcnt,uint32_t segsiz)2295 bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts,
2296 int32_t line, uint32_t o_len, uint32_t segcnt, uint32_t segsiz)
2297 {
2298 if (tcp_bblogging_on(bbr->rc_tp)) {
2299 union tcp_log_stackspecific log;
2300
2301 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2302 log.u_bbr.flex1 = line;
2303 log.u_bbr.flex2 = o_len;
2304 log.u_bbr.flex3 = segcnt;
2305 log.u_bbr.flex4 = segsiz;
2306 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2307 &bbr->rc_inp->inp_socket->so_rcv,
2308 &bbr->rc_inp->inp_socket->so_snd,
2309 BBR_LOG_ENOBUF_JMP, ENOBUFS,
2310 len, &log, true, &bbr->rc_tv);
2311 }
2312 }
2313
2314 static void
bbr_log_to_processing(struct tcp_bbr * bbr,uint32_t cts,int32_t ret,int32_t timers,uint8_t hpts_calling)2315 bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_calling)
2316 {
2317 if (tcp_bblogging_on(bbr->rc_tp)) {
2318 union tcp_log_stackspecific log;
2319
2320 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2321 log.u_bbr.flex1 = timers;
2322 log.u_bbr.flex2 = ret;
2323 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
2324 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2325 log.u_bbr.flex5 = cts;
2326 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2327 log.u_bbr.flex8 = hpts_calling;
2328 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2329 &bbr->rc_inp->inp_socket->so_rcv,
2330 &bbr->rc_inp->inp_socket->so_snd,
2331 BBR_LOG_TO_PROCESS, 0,
2332 0, &log, false, &bbr->rc_tv);
2333 }
2334 }
2335
2336 static void
bbr_log_to_event(struct tcp_bbr * bbr,uint32_t cts,int32_t to_num)2337 bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num)
2338 {
2339 if (tcp_bblogging_on(bbr->rc_tp)) {
2340 union tcp_log_stackspecific log;
2341 uint64_t ar;
2342
2343 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2344 log.u_bbr.flex1 = bbr->bbr_timer_src;
2345 log.u_bbr.flex2 = 0;
2346 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2347 ar = (uintptr_t)(bbr->r_ctl.rc_resend);
2348 ar >>= 32;
2349 ar &= 0x00000000ffffffff;
2350 log.u_bbr.flex4 = (uint32_t)ar;
2351 ar = (uintptr_t)bbr->r_ctl.rc_resend;
2352 ar &= 0x00000000ffffffff;
2353 log.u_bbr.flex5 = (uint32_t)ar;
2354 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2355 log.u_bbr.flex8 = to_num;
2356 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2357 &bbr->rc_inp->inp_socket->so_rcv,
2358 &bbr->rc_inp->inp_socket->so_snd,
2359 BBR_LOG_RTO, 0,
2360 0, &log, false, &bbr->rc_tv);
2361 }
2362 }
2363
2364 static void
bbr_log_startup_event(struct tcp_bbr * bbr,uint32_t cts,uint32_t flex1,uint32_t flex2,uint32_t flex3,uint8_t reason)2365 bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t flex3, uint8_t reason)
2366 {
2367 if (tcp_bblogging_on(bbr->rc_tp)) {
2368 union tcp_log_stackspecific log;
2369
2370 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2371 log.u_bbr.flex1 = flex1;
2372 log.u_bbr.flex2 = flex2;
2373 log.u_bbr.flex3 = flex3;
2374 log.u_bbr.flex4 = 0;
2375 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2376 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2377 log.u_bbr.flex8 = reason;
2378 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;
2379 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2380 &bbr->rc_inp->inp_socket->so_rcv,
2381 &bbr->rc_inp->inp_socket->so_snd,
2382 BBR_LOG_REDUCE, 0,
2383 0, &log, false, &bbr->rc_tv);
2384 }
2385 }
2386
2387 static void
bbr_log_hpts_diag(struct tcp_bbr * bbr,uint32_t cts,struct hpts_diag * diag)2388 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag)
2389 {
2390 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2391 union tcp_log_stackspecific log;
2392
2393 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2394 log.u_bbr.flex1 = diag->p_nxt_slot;
2395 log.u_bbr.flex2 = diag->p_cur_slot;
2396 log.u_bbr.flex3 = diag->slot_req;
2397 log.u_bbr.flex4 = diag->inp_hptsslot;
2398 log.u_bbr.flex5 = diag->time_remaining;
2399 log.u_bbr.flex6 = diag->need_new_to;
2400 log.u_bbr.flex7 = diag->p_hpts_active;
2401 log.u_bbr.flex8 = diag->p_on_min_sleep;
2402 /* Hijack other fields as needed */
2403 log.u_bbr.epoch = diag->have_slept;
2404 log.u_bbr.lt_epoch = diag->yet_to_sleep;
2405 log.u_bbr.pkts_out = diag->co_ret;
2406 log.u_bbr.applimited = diag->hpts_sleep_time;
2407 log.u_bbr.delivered = diag->p_prev_slot;
2408 log.u_bbr.inflight = diag->p_runningslot;
2409 log.u_bbr.bw_inuse = diag->wheel_slot;
2410 log.u_bbr.rttProp = diag->wheel_cts;
2411 log.u_bbr.delRate = diag->maxslots;
2412 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2413 &bbr->rc_inp->inp_socket->so_rcv,
2414 &bbr->rc_inp->inp_socket->so_snd,
2415 BBR_LOG_HPTSDIAG, 0,
2416 0, &log, false, &bbr->rc_tv);
2417 }
2418 }
2419
2420 static void
bbr_log_timer_var(struct tcp_bbr * bbr,int mode,uint32_t cts,uint32_t time_since_sent,uint32_t srtt,uint32_t thresh,uint32_t to)2421 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
2422 uint32_t thresh, uint32_t to)
2423 {
2424 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2425 union tcp_log_stackspecific log;
2426
2427 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2428 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;
2429 log.u_bbr.flex2 = time_since_sent;
2430 log.u_bbr.flex3 = srtt;
2431 log.u_bbr.flex4 = thresh;
2432 log.u_bbr.flex5 = to;
2433 log.u_bbr.flex6 = bbr->rc_tp->t_srtt;
2434 log.u_bbr.flex8 = mode;
2435 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2436 &bbr->rc_inp->inp_socket->so_rcv,
2437 &bbr->rc_inp->inp_socket->so_snd,
2438 BBR_LOG_TIMERPREP, 0,
2439 0, &log, false, &bbr->rc_tv);
2440 }
2441 }
2442
2443 static void
bbr_log_pacing_delay_calc(struct tcp_bbr * bbr,uint16_t gain,uint32_t len,uint32_t cts,uint32_t usecs,uint64_t bw,uint32_t override,int mod)2444 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
2445 uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod)
2446 {
2447 if (tcp_bblogging_on(bbr->rc_tp)) {
2448 union tcp_log_stackspecific log;
2449
2450 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2451 log.u_bbr.flex1 = usecs;
2452 log.u_bbr.flex2 = len;
2453 log.u_bbr.flex3 = (uint32_t)((bw >> 32) & 0x00000000ffffffff);
2454 log.u_bbr.flex4 = (uint32_t)(bw & 0x00000000ffffffff);
2455 if (override)
2456 log.u_bbr.flex5 = (1 << 2);
2457 else
2458 log.u_bbr.flex5 = 0;
2459 log.u_bbr.flex6 = override;
2460 log.u_bbr.flex7 = gain;
2461 log.u_bbr.flex8 = mod;
2462 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2463 &bbr->rc_inp->inp_socket->so_rcv,
2464 &bbr->rc_inp->inp_socket->so_snd,
2465 BBR_LOG_HPTSI_CALC, 0,
2466 len, &log, false, &bbr->rc_tv);
2467 }
2468 }
2469
2470 static void
bbr_log_to_start(struct tcp_bbr * bbr,uint32_t cts,uint32_t to,int32_t pacing_delay,uint8_t which)2471 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t pacing_delay, uint8_t which)
2472 {
2473 if (tcp_bblogging_on(bbr->rc_tp)) {
2474 union tcp_log_stackspecific log;
2475
2476 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2477
2478 log.u_bbr.flex1 = bbr->bbr_timer_src;
2479 log.u_bbr.flex2 = to;
2480 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2481 log.u_bbr.flex4 = pacing_delay;
2482 log.u_bbr.flex5 = bbr->rc_tp->t_hpts_slot;
2483 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2484 log.u_bbr.pkts_out = bbr->rc_tp->t_flags2;
2485 log.u_bbr.flex8 = which;
2486 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2487 &bbr->rc_inp->inp_socket->so_rcv,
2488 &bbr->rc_inp->inp_socket->so_snd,
2489 BBR_LOG_TIMERSTAR, 0,
2490 0, &log, false, &bbr->rc_tv);
2491 }
2492 }
2493
2494 static void
bbr_log_thresh_choice(struct tcp_bbr * bbr,uint32_t cts,uint32_t thresh,uint32_t lro,uint32_t srtt,struct bbr_sendmap * rsm,uint8_t frm)2495 bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm)
2496 {
2497 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2498 union tcp_log_stackspecific log;
2499
2500 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2501 log.u_bbr.flex1 = thresh;
2502 log.u_bbr.flex2 = lro;
2503 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;
2504 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
2505 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2506 log.u_bbr.flex6 = srtt;
2507 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;
2508 log.u_bbr.flex8 = frm;
2509 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2510 &bbr->rc_inp->inp_socket->so_rcv,
2511 &bbr->rc_inp->inp_socket->so_snd,
2512 BBR_LOG_THRESH_CALC, 0,
2513 0, &log, false, &bbr->rc_tv);
2514 }
2515 }
2516
2517 static void
bbr_log_to_cancel(struct tcp_bbr * bbr,int32_t line,uint32_t cts,uint8_t hpts_removed)2518 bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed)
2519 {
2520 if (tcp_bblogging_on(bbr->rc_tp)) {
2521 union tcp_log_stackspecific log;
2522
2523 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2524 log.u_bbr.flex1 = line;
2525 log.u_bbr.flex2 = bbr->bbr_timer_src;
2526 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2527 log.u_bbr.flex4 = bbr->rc_in_persist;
2528 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2529 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2530 log.u_bbr.flex8 = hpts_removed;
2531 log.u_bbr.pkts_out = bbr->rc_pacer_started;
2532 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2533 &bbr->rc_inp->inp_socket->so_rcv,
2534 &bbr->rc_inp->inp_socket->so_snd,
2535 BBR_LOG_TIMERCANC, 0,
2536 0, &log, false, &bbr->rc_tv);
2537 }
2538 }
2539
2540 static void
bbr_log_tstmp_validation(struct tcp_bbr * bbr,uint64_t peer_delta,uint64_t delta)2541 bbr_log_tstmp_validation(struct tcp_bbr *bbr, uint64_t peer_delta, uint64_t delta)
2542 {
2543 if (tcp_bblogging_on(bbr->rc_tp)) {
2544 union tcp_log_stackspecific log;
2545
2546 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2547 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;
2548 log.u_bbr.flex2 = (peer_delta >> 32);
2549 log.u_bbr.flex3 = (peer_delta & 0x00000000ffffffff);
2550 log.u_bbr.flex4 = (delta >> 32);
2551 log.u_bbr.flex5 = (delta & 0x00000000ffffffff);
2552 log.u_bbr.flex7 = bbr->rc_ts_clock_set;
2553 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;
2554 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2555 &bbr->rc_inp->inp_socket->so_rcv,
2556 &bbr->rc_inp->inp_socket->so_snd,
2557 BBR_LOG_TSTMP_VAL, 0,
2558 0, &log, false, &bbr->rc_tv);
2559 }
2560 }
2561
2562 static void
bbr_log_type_tsosize(struct tcp_bbr * bbr,uint32_t cts,uint32_t tsosz,uint32_t tls,uint32_t old_val,uint32_t maxseg,int hdwr)2563 bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_val, uint32_t maxseg, int hdwr)
2564 {
2565 if (tcp_bblogging_on(bbr->rc_tp)) {
2566 union tcp_log_stackspecific log;
2567
2568 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2569 log.u_bbr.flex1 = tsosz;
2570 log.u_bbr.flex2 = tls;
2571 log.u_bbr.flex3 = tcp_min_hptsi_time;
2572 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;
2573 log.u_bbr.flex5 = old_val;
2574 log.u_bbr.flex6 = maxseg;
2575 log.u_bbr.flex7 = bbr->rc_no_pacing;
2576 log.u_bbr.flex7 <<= 1;
2577 log.u_bbr.flex7 |= bbr->rc_past_init_win;
2578 if (hdwr)
2579 log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;
2580 else
2581 log.u_bbr.flex8 = bbr->rc_use_google;
2582 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2583 &bbr->rc_inp->inp_socket->so_rcv,
2584 &bbr->rc_inp->inp_socket->so_snd,
2585 BBR_LOG_BBRTSO, 0,
2586 0, &log, false, &bbr->rc_tv);
2587 }
2588 }
2589
2590 static void
bbr_log_type_rsmclear(struct tcp_bbr * bbr,uint32_t cts,struct bbr_sendmap * rsm,uint32_t flags,uint32_t line)2591 bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm,
2592 uint32_t flags, uint32_t line)
2593 {
2594 if (tcp_bblogging_on(bbr->rc_tp)) {
2595 union tcp_log_stackspecific log;
2596
2597 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2598 log.u_bbr.flex1 = line;
2599 log.u_bbr.flex2 = rsm->r_start;
2600 log.u_bbr.flex3 = rsm->r_end;
2601 log.u_bbr.flex4 = rsm->r_delivered;
2602 log.u_bbr.flex5 = rsm->r_rtr_cnt;
2603 log.u_bbr.flex6 = rsm->r_dupack;
2604 log.u_bbr.flex7 = rsm->r_tim_lastsent[0];
2605 log.u_bbr.flex8 = rsm->r_flags;
2606 /* Hijack the pkts_out fids */
2607 log.u_bbr.applimited = flags;
2608 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2609 &bbr->rc_inp->inp_socket->so_rcv,
2610 &bbr->rc_inp->inp_socket->so_snd,
2611 BBR_RSM_CLEARED, 0,
2612 0, &log, false, &bbr->rc_tv);
2613 }
2614 }
2615
2616 static void
bbr_log_type_bbrupd(struct tcp_bbr * bbr,uint8_t flex8,uint32_t cts,uint32_t flex3,uint32_t flex2,uint32_t flex5,uint32_t flex6,uint32_t pkts_out,int flex7,uint32_t flex4,uint32_t flex1)2617 bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts,
2618 uint32_t flex3, uint32_t flex2, uint32_t flex5,
2619 uint32_t flex6, uint32_t pkts_out, int flex7,
2620 uint32_t flex4, uint32_t flex1)
2621 {
2622
2623 if (tcp_bblogging_on(bbr->rc_tp)) {
2624 union tcp_log_stackspecific log;
2625
2626 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2627 log.u_bbr.flex1 = flex1;
2628 log.u_bbr.flex2 = flex2;
2629 log.u_bbr.flex3 = flex3;
2630 log.u_bbr.flex4 = flex4;
2631 log.u_bbr.flex5 = flex5;
2632 log.u_bbr.flex6 = flex6;
2633 log.u_bbr.flex7 = flex7;
2634 /* Hijack the pkts_out fids */
2635 log.u_bbr.pkts_out = pkts_out;
2636 log.u_bbr.flex8 = flex8;
2637 if (bbr->rc_ack_was_delayed)
2638 log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;
2639 else
2640 log.u_bbr.epoch = 0;
2641 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2642 &bbr->rc_inp->inp_socket->so_rcv,
2643 &bbr->rc_inp->inp_socket->so_snd,
2644 BBR_LOG_BBRUPD, 0,
2645 flex2, &log, false, &bbr->rc_tv);
2646 }
2647 }
2648
2649 static void
bbr_log_type_ltbw(struct tcp_bbr * bbr,uint32_t cts,int32_t reason,uint32_t newbw,uint32_t obw,uint32_t diff,uint32_t tim)2650 bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason,
2651 uint32_t newbw, uint32_t obw, uint32_t diff,
2652 uint32_t tim)
2653 {
2654 if (/*bbr_verbose_logging && */tcp_bblogging_on(bbr->rc_tp)) {
2655 union tcp_log_stackspecific log;
2656
2657 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2658 log.u_bbr.flex1 = reason;
2659 log.u_bbr.flex2 = newbw;
2660 log.u_bbr.flex3 = obw;
2661 log.u_bbr.flex4 = diff;
2662 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;
2663 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;
2664 log.u_bbr.flex7 = bbr->rc_lt_is_sampling;
2665 log.u_bbr.pkts_out = tim;
2666 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;
2667 if (bbr->rc_lt_use_bw == 0)
2668 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
2669 else
2670 log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
2671 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2672 &bbr->rc_inp->inp_socket->so_rcv,
2673 &bbr->rc_inp->inp_socket->so_snd,
2674 BBR_LOG_BWSAMP, 0,
2675 0, &log, false, &bbr->rc_tv);
2676 }
2677 }
2678
2679 static inline void
bbr_log_progress_event(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t tick,int event,int line)2680 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line)
2681 {
2682 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2683 union tcp_log_stackspecific log;
2684
2685 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2686 log.u_bbr.flex1 = line;
2687 log.u_bbr.flex2 = tick;
2688 log.u_bbr.flex3 = tp->t_maxunacktime;
2689 log.u_bbr.flex4 = tp->t_acktime;
2690 log.u_bbr.flex8 = event;
2691 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2692 &bbr->rc_inp->inp_socket->so_rcv,
2693 &bbr->rc_inp->inp_socket->so_snd,
2694 BBR_LOG_PROGRESS, 0,
2695 0, &log, false, &bbr->rc_tv);
2696 }
2697 }
2698
2699 static void
bbr_type_log_hdwr_pacing(struct tcp_bbr * bbr,const struct ifnet * ifp,uint64_t rate,uint64_t hw_rate,int line,uint32_t cts,int error)2700 bbr_type_log_hdwr_pacing(struct tcp_bbr *bbr, const struct ifnet *ifp,
2701 uint64_t rate, uint64_t hw_rate, int line, uint32_t cts,
2702 int error)
2703 {
2704 if (tcp_bblogging_on(bbr->rc_tp)) {
2705 union tcp_log_stackspecific log;
2706 uint64_t ifp64 = (uintptr_t)ifp;
2707
2708 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2709 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff);
2710 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff);
2711 log.u_bbr.flex3 = ((ifp64 >> 32) & 0x00000000ffffffff);
2712 log.u_bbr.flex4 = (ifp64 & 0x00000000ffffffff);
2713 log.u_bbr.bw_inuse = rate;
2714 log.u_bbr.flex5 = line;
2715 log.u_bbr.flex6 = error;
2716 log.u_bbr.flex8 = bbr->skip_gain;
2717 log.u_bbr.flex8 <<= 1;
2718 log.u_bbr.flex8 |= bbr->gain_is_limited;
2719 log.u_bbr.flex8 <<= 1;
2720 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;
2721 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
2722 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2723 &bbr->rc_inp->inp_socket->so_rcv,
2724 &bbr->rc_inp->inp_socket->so_snd,
2725 BBR_LOG_HDWR_PACE, 0,
2726 0, &log, false, &bbr->rc_tv);
2727 }
2728 }
2729
2730 static void
bbr_log_type_bbrsnd(struct tcp_bbr * bbr,uint32_t len,uint32_t pacing_delay,uint32_t del_by,uint32_t cts,uint32_t line,uint32_t prev_delay)2731 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t pacing_delay, uint32_t del_by, uint32_t cts, uint32_t line, uint32_t prev_delay)
2732 {
2733 if (tcp_bblogging_on(bbr->rc_tp)) {
2734 union tcp_log_stackspecific log;
2735
2736 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2737 log.u_bbr.flex1 = pacing_delay;
2738 log.u_bbr.flex2 = del_by;
2739 log.u_bbr.flex3 = prev_delay;
2740 log.u_bbr.flex4 = line;
2741 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;
2742 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;
2743 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);
2744 log.u_bbr.flex8 = bbr->rc_in_persist;
2745 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2746 &bbr->rc_inp->inp_socket->so_rcv,
2747 &bbr->rc_inp->inp_socket->so_snd,
2748 BBR_LOG_BBRSND, 0,
2749 len, &log, false, &bbr->rc_tv);
2750 }
2751 }
2752
2753 static void
bbr_log_type_bbrrttprop(struct tcp_bbr * bbr,uint32_t t,uint32_t end,uint32_t tsconv,uint32_t cts,int32_t match,uint32_t seq,uint8_t flags)2754 bbr_log_type_bbrrttprop(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uint32_t seq, uint8_t flags)
2755 {
2756 if (tcp_bblogging_on(bbr->rc_tp)) {
2757 union tcp_log_stackspecific log;
2758
2759 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2760 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;
2761 log.u_bbr.flex2 = 0;
2762 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;
2763 log.u_bbr.flex4 = end;
2764 log.u_bbr.flex5 = seq;
2765 log.u_bbr.flex6 = t;
2766 log.u_bbr.flex7 = match;
2767 log.u_bbr.flex8 = flags;
2768 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2769 &bbr->rc_inp->inp_socket->so_rcv,
2770 &bbr->rc_inp->inp_socket->so_snd,
2771 BBR_LOG_BBRRTT, 0,
2772 0, &log, false, &bbr->rc_tv);
2773 }
2774 }
2775
2776 static void
bbr_log_exit_gain(struct tcp_bbr * bbr,uint32_t cts,int32_t entry_method)2777 bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method)
2778 {
2779 if (tcp_bblogging_on(bbr->rc_tp)) {
2780 union tcp_log_stackspecific log;
2781
2782 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2783 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2784 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
2785 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;
2786 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2787 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;
2788 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;
2789 log.u_bbr.flex7 = 0;
2790 log.u_bbr.flex8 = entry_method;
2791 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2792 &bbr->rc_inp->inp_socket->so_rcv,
2793 &bbr->rc_inp->inp_socket->so_snd,
2794 BBR_LOG_EXIT_GAIN, 0,
2795 0, &log, false, &bbr->rc_tv);
2796 }
2797 }
2798
2799 static void
bbr_log_settings_change(struct tcp_bbr * bbr,int settings_desired)2800 bbr_log_settings_change(struct tcp_bbr *bbr, int settings_desired)
2801 {
2802 if (bbr_verbose_logging && tcp_bblogging_on(bbr->rc_tp)) {
2803 union tcp_log_stackspecific log;
2804
2805 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2806 /* R-HU */
2807 log.u_bbr.flex1 = 0;
2808 log.u_bbr.flex2 = 0;
2809 log.u_bbr.flex3 = 0;
2810 log.u_bbr.flex4 = 0;
2811 log.u_bbr.flex7 = 0;
2812 log.u_bbr.flex8 = settings_desired;
2813
2814 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2815 &bbr->rc_inp->inp_socket->so_rcv,
2816 &bbr->rc_inp->inp_socket->so_snd,
2817 BBR_LOG_SETTINGS_CHG, 0,
2818 0, &log, false, &bbr->rc_tv);
2819 }
2820 }
2821
2822 /*
2823 * Returns the bw from the our filter.
2824 */
2825 static inline uint64_t
bbr_get_full_bw(struct tcp_bbr * bbr)2826 bbr_get_full_bw(struct tcp_bbr *bbr)
2827 {
2828 uint64_t bw;
2829
2830 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2831
2832 return (bw);
2833 }
2834
2835 static inline void
bbr_set_pktepoch(struct tcp_bbr * bbr,uint32_t cts,int32_t line)2836 bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2837 {
2838 uint64_t calclr;
2839 uint32_t lost, del;
2840
2841 if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)
2842 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;
2843 else
2844 lost = 0;
2845 del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;
2846 if (lost == 0) {
2847 calclr = 0;
2848 } else if (del) {
2849 calclr = lost;
2850 calclr *= (uint64_t)1000;
2851 calclr /= (uint64_t)del;
2852 } else {
2853 /* Nothing delivered? 100.0% loss */
2854 calclr = 1000;
2855 }
2856 bbr->r_ctl.rc_pkt_epoch_loss_rate = (uint32_t)calclr;
2857 if (IN_RECOVERY(bbr->rc_tp->t_flags))
2858 bbr->r_ctl.recovery_lr += (uint32_t)calclr;
2859 bbr->r_ctl.rc_pkt_epoch++;
2860 if (bbr->rc_no_pacing &&
2861 (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {
2862 bbr->rc_no_pacing = 0;
2863 tcp_bbr_tso_size_check(bbr, cts);
2864 }
2865 bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);
2866 bbr->r_ctl.rc_pkt_epoch_time = cts;
2867 /* What was our loss rate */
2868 bbr_log_pkt_epoch(bbr, cts, line, lost, del);
2869 bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;
2870 bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;
2871 }
2872
2873 static inline void
bbr_set_epoch(struct tcp_bbr * bbr,uint32_t cts,int32_t line)2874 bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2875 {
2876 uint32_t epoch_time;
2877
2878 /* Tick the RTT clock */
2879 bbr->r_ctl.rc_rtt_epoch++;
2880 epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;
2881 bbr_log_time_epoch(bbr, cts, line, epoch_time);
2882 bbr->r_ctl.rc_rcv_epoch_start = cts;
2883 }
2884
2885 static inline void
bbr_isit_a_pkt_epoch(struct tcp_bbr * bbr,uint32_t cts,struct bbr_sendmap * rsm,int32_t line,int32_t cum_acked)2886 bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked)
2887 {
2888 if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {
2889 bbr->rc_is_pkt_epoch_now = 1;
2890 }
2891 }
2892
2893 /*
2894 * Returns the bw from either the b/w filter
2895 * or from the lt_bw (if the connection is being
2896 * policed).
2897 */
2898 static inline uint64_t
__bbr_get_bw(struct tcp_bbr * bbr)2899 __bbr_get_bw(struct tcp_bbr *bbr)
2900 {
2901 uint64_t bw, min_bw;
2902 uint64_t rtt;
2903 int gm_measure_cnt = 1;
2904
2905 /*
2906 * For startup we make, like google, a
2907 * minimum b/w. This is generated from the
2908 * IW and the rttProp. We do fall back to srtt
2909 * if for some reason (initial handshake) we don't
2910 * have a rttProp. We, in the worst case, fall back
2911 * to the configured min_bw (rc_initial_hptsi_bw).
2912 */
2913 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
2914 /* Attempt first to use rttProp */
2915 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2916 if (rtt && (rtt < 0xffffffff)) {
2917 measure:
2918 min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2919 ((uint64_t)1000000);
2920 min_bw /= rtt;
2921 if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2922 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2923 }
2924
2925 } else if (bbr->rc_tp->t_srtt != 0) {
2926 /* No rttProp, use srtt? */
2927 rtt = bbr_get_rtt(bbr, BBR_SRTT);
2928 goto measure;
2929 } else {
2930 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2931 }
2932 } else
2933 min_bw = 0;
2934
2935 if ((bbr->rc_past_init_win == 0) &&
2936 (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))
2937 bbr->rc_past_init_win = 1;
2938 if ((bbr->rc_use_google) && (bbr->r_ctl.r_measurement_count >= 1))
2939 gm_measure_cnt = 0;
2940 if (gm_measure_cnt &&
2941 ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||
2942 (bbr->rc_past_init_win == 0))) {
2943 /* For google we use our guess rate until we get 1 measurement */
2944
2945 use_initial_window:
2946 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2947 if (rtt && (rtt < 0xffffffff)) {
2948 /*
2949 * We have an RTT measurement. Use that in
2950 * combination with our initial window to calculate
2951 * a b/w.
2952 */
2953 bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2954 ((uint64_t)1000000);
2955 bw /= rtt;
2956 if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2957 bw = bbr->r_ctl.rc_initial_hptsi_bw;
2958 }
2959 } else {
2960 /* Drop back to the 40 and punt to a default */
2961 bw = bbr->r_ctl.rc_initial_hptsi_bw;
2962 }
2963 if (bw < 1)
2964 /* Probably should panic */
2965 bw = 1;
2966 if (bw > min_bw)
2967 return (bw);
2968 else
2969 return (min_bw);
2970 }
2971 if (bbr->rc_lt_use_bw)
2972 bw = bbr->r_ctl.rc_lt_bw;
2973 else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))
2974 bw = bbr->r_ctl.red_bw;
2975 else
2976 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2977 if (bw == 0) {
2978 /* We should not be at 0, go to the initial window then */
2979 goto use_initial_window;
2980 }
2981 if (bw < min_bw)
2982 bw = min_bw;
2983 return (bw);
2984 }
2985
2986 static inline uint64_t
bbr_get_bw(struct tcp_bbr * bbr)2987 bbr_get_bw(struct tcp_bbr *bbr)
2988 {
2989 uint64_t bw;
2990
2991 bw = __bbr_get_bw(bbr);
2992 return (bw);
2993 }
2994
2995 static inline void
bbr_reset_lt_bw_interval(struct tcp_bbr * bbr,uint32_t cts)2996 bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts)
2997 {
2998 bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;
2999 bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;
3000 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3001 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3002 }
3003
3004 static inline void
bbr_reset_lt_bw_sampling(struct tcp_bbr * bbr,uint32_t cts)3005 bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts)
3006 {
3007 bbr->rc_lt_is_sampling = 0;
3008 bbr->rc_lt_use_bw = 0;
3009 bbr->r_ctl.rc_lt_bw = 0;
3010 bbr_reset_lt_bw_interval(bbr, cts);
3011 }
3012
3013 static inline void
bbr_lt_bw_samp_done(struct tcp_bbr * bbr,uint64_t bw,uint32_t cts,uint32_t timin)3014 bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin)
3015 {
3016 uint64_t diff;
3017
3018 /* Do we have a previous sample? */
3019 if (bbr->r_ctl.rc_lt_bw) {
3020 /* Get the diff in bytes per second */
3021 if (bbr->r_ctl.rc_lt_bw > bw)
3022 diff = bbr->r_ctl.rc_lt_bw - bw;
3023 else
3024 diff = bw - bbr->r_ctl.rc_lt_bw;
3025 if ((diff <= bbr_lt_bw_diff) ||
3026 (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {
3027 /* Consider us policed */
3028 uint32_t saved_bw;
3029
3030 saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;
3031 bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2; /* average of two */
3032 bbr->rc_lt_use_bw = 1;
3033 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
3034 /*
3035 * Use pkt based epoch for measuring length of
3036 * policer up
3037 */
3038 bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;
3039 /*
3040 * reason 4 is we need to start consider being
3041 * policed
3042 */
3043 bbr_log_type_ltbw(bbr, cts, 4, (uint32_t)bw, saved_bw, (uint32_t)diff, timin);
3044 return;
3045 }
3046 }
3047 bbr->r_ctl.rc_lt_bw = bw;
3048 bbr_reset_lt_bw_interval(bbr, cts);
3049 bbr_log_type_ltbw(bbr, cts, 5, 0, (uint32_t)bw, 0, timin);
3050 }
3051
3052 static void
bbr_randomize_extra_state_time(struct tcp_bbr * bbr)3053 bbr_randomize_extra_state_time(struct tcp_bbr *bbr)
3054 {
3055 uint32_t ran, deduct;
3056
3057 ran = arc4random_uniform(bbr_rand_ot);
3058 if (ran) {
3059 deduct = bbr->r_ctl.rc_level_state_extra / ran;
3060 bbr->r_ctl.rc_level_state_extra -= deduct;
3061 }
3062 }
3063 /*
3064 * Return randomly the starting state
3065 * to use in probebw.
3066 */
3067 static uint8_t
bbr_pick_probebw_substate(struct tcp_bbr * bbr,uint32_t cts)3068 bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts)
3069 {
3070 uint32_t ran;
3071 uint8_t ret_val;
3072
3073 /* Initialize the offset to 0 */
3074 bbr->r_ctl.rc_exta_time_gd = 0;
3075 bbr->rc_hit_state_1 = 0;
3076 bbr->r_ctl.rc_level_state_extra = 0;
3077 ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1));
3078 /*
3079 * The math works funny here :) the return value is used to set the
3080 * substate and then the state change is called which increments by
3081 * one. So if we return 1 (DRAIN) we will increment to 2 (LEVEL1) when
3082 * we fully enter the state. Note that the (8 - 1 - ran) assures that
3083 * we return 1 - 7, so we dont return 0 and end up starting in
3084 * state 1 (DRAIN).
3085 */
3086 ret_val = BBR_SUBSTATE_COUNT - 1 - ran;
3087 /* Set an epoch */
3088 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))
3089 bbr_set_epoch(bbr, cts, __LINE__);
3090
3091 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
3092 return (ret_val);
3093 }
3094
3095 static void
bbr_lt_bw_sampling(struct tcp_bbr * bbr,uint32_t cts,int32_t loss_detected)3096 bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected)
3097 {
3098 uint32_t diff, d_time;
3099 uint64_t del_time, bw, lost, delivered;
3100
3101 if (bbr->r_use_policer == 0)
3102 return;
3103 if (bbr->rc_lt_use_bw) {
3104 /* We are using lt bw do we stop yet? */
3105 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
3106 if (diff > bbr_lt_bw_max_rtts) {
3107 /* Reset it all */
3108 reset_all:
3109 bbr_reset_lt_bw_sampling(bbr, cts);
3110 if (bbr->rc_filled_pipe) {
3111 bbr_set_epoch(bbr, cts, __LINE__);
3112 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
3113 bbr_substate_change(bbr, cts, __LINE__, 0);
3114 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
3115 bbr_log_type_statechange(bbr, cts, __LINE__);
3116 } else {
3117 /*
3118 * This should not happen really
3119 * unless we remove the startup/drain
3120 * restrictions above.
3121 */
3122 bbr->rc_bbr_state = BBR_STATE_STARTUP;
3123 bbr_set_epoch(bbr, cts, __LINE__);
3124 bbr->r_ctl.rc_bbr_state_time = cts;
3125 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
3126 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
3127 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
3128 bbr_set_state_target(bbr, __LINE__);
3129 bbr_log_type_statechange(bbr, cts, __LINE__);
3130 }
3131 /* reason 0 is to stop using lt-bw */
3132 bbr_log_type_ltbw(bbr, cts, 0, 0, 0, 0, 0);
3133 return;
3134 }
3135 if (bbr_lt_intvl_fp == 0) {
3136 /* Not doing false-positive detection */
3137 return;
3138 }
3139 /* False positive detection */
3140 if (diff == bbr_lt_intvl_fp) {
3141 /* At bbr_lt_intvl_fp we record the lost */
3142 bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3143 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3144 } else if (diff > (bbr_lt_intvl_min_rtts + bbr_lt_intvl_fp)) {
3145 /* Now is our loss rate still high? */
3146 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3147 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3148 if ((delivered == 0) ||
3149 (((lost * 1000)/delivered) < bbr_lt_fd_thresh)) {
3150 /* No still below our threshold */
3151 bbr_log_type_ltbw(bbr, cts, 7, lost, delivered, 0, 0);
3152 } else {
3153 /* Yikes its still high, it must be a false positive */
3154 bbr_log_type_ltbw(bbr, cts, 8, lost, delivered, 0, 0);
3155 goto reset_all;
3156 }
3157 }
3158 return;
3159 }
3160 /*
3161 * Wait for the first loss before sampling, to let the policer
3162 * exhaust its tokens and estimate the steady-state rate allowed by
3163 * the policer. Starting samples earlier includes bursts that
3164 * over-estimate the bw.
3165 */
3166 if (bbr->rc_lt_is_sampling == 0) {
3167 /* reason 1 is to begin doing the sampling */
3168 if (loss_detected == 0)
3169 return;
3170 bbr_reset_lt_bw_interval(bbr, cts);
3171 bbr->rc_lt_is_sampling = 1;
3172 bbr_log_type_ltbw(bbr, cts, 1, 0, 0, 0, 0);
3173 return;
3174 }
3175 /* Now how long were we delivering long term last> */
3176 if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))
3177 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;
3178 else
3179 d_time = 0;
3180
3181 /* To avoid underestimates, reset sampling if we run out of data. */
3182 if (bbr->r_ctl.r_app_limited_until) {
3183 /* Can not measure in app-limited state */
3184 bbr_reset_lt_bw_sampling(bbr, cts);
3185 /* reason 2 is to reset sampling due to app limits */
3186 bbr_log_type_ltbw(bbr, cts, 2, 0, 0, 0, d_time);
3187 return;
3188 }
3189 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
3190 if (diff < bbr_lt_intvl_min_rtts) {
3191 /*
3192 * need more samples (we don't
3193 * start on a round like linux so
3194 * we need 1 more).
3195 */
3196 /* 6 is not_enough time or no-loss */
3197 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3198 return;
3199 }
3200 if (diff > (4 * bbr_lt_intvl_min_rtts)) {
3201 /*
3202 * For now if we wait too long, reset all sampling. We need
3203 * to do some research here, its possible that we should
3204 * base this on how much loss as occurred.. something like
3205 * if its under 10% (or some thresh) reset all otherwise
3206 * don't. Thats for phase II I guess.
3207 */
3208 bbr_reset_lt_bw_sampling(bbr, cts);
3209 /* reason 3 is to reset sampling due too long of sampling */
3210 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3211 return;
3212 }
3213 /*
3214 * End sampling interval when a packet is lost, so we estimate the
3215 * policer tokens were exhausted. Stopping the sampling before the
3216 * tokens are exhausted under-estimates the policed rate.
3217 */
3218 if (loss_detected == 0) {
3219 /* 6 is not_enough time or no-loss */
3220 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3221 return;
3222 }
3223 /* Calculate packets lost and delivered in sampling interval. */
3224 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3225 delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3226 if ((delivered == 0) ||
3227 (((lost * 1000)/delivered) < bbr_lt_loss_thresh)) {
3228 bbr_log_type_ltbw(bbr, cts, 6, lost, delivered, 0, d_time);
3229 return;
3230 }
3231 if (d_time < 1000) {
3232 /* Not enough time. wait */
3233 /* 6 is not_enough time or no-loss */
3234 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3235 return;
3236 }
3237 if (d_time >= (0xffffffff / USECS_IN_MSEC)) {
3238 /* Too long */
3239 bbr_reset_lt_bw_sampling(bbr, cts);
3240 /* reason 3 is to reset sampling due too long of sampling */
3241 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3242 return;
3243 }
3244 del_time = d_time;
3245 bw = delivered;
3246 bw *= (uint64_t)USECS_IN_SECOND;
3247 bw /= del_time;
3248 bbr_lt_bw_samp_done(bbr, bw, cts, d_time);
3249 }
3250
3251 /*
3252 * Allocate a sendmap from our zone.
3253 */
3254 static struct bbr_sendmap *
bbr_alloc(struct tcp_bbr * bbr)3255 bbr_alloc(struct tcp_bbr *bbr)
3256 {
3257 struct bbr_sendmap *rsm;
3258
3259 BBR_STAT_INC(bbr_to_alloc);
3260 rsm = uma_zalloc(bbr_zone, (M_NOWAIT | M_ZERO));
3261 if (rsm) {
3262 bbr->r_ctl.rc_num_maps_alloced++;
3263 return (rsm);
3264 }
3265 if (bbr->r_ctl.rc_free_cnt) {
3266 BBR_STAT_INC(bbr_to_alloc_emerg);
3267 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
3268 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
3269 bbr->r_ctl.rc_free_cnt--;
3270 return (rsm);
3271 }
3272 BBR_STAT_INC(bbr_to_alloc_failed);
3273 return (NULL);
3274 }
3275
3276 static struct bbr_sendmap *
bbr_alloc_full_limit(struct tcp_bbr * bbr)3277 bbr_alloc_full_limit(struct tcp_bbr *bbr)
3278 {
3279 if ((V_tcp_map_entries_limit > 0) &&
3280 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
3281 BBR_STAT_INC(bbr_alloc_limited);
3282 if (!bbr->alloc_limit_reported) {
3283 bbr->alloc_limit_reported = 1;
3284 BBR_STAT_INC(bbr_alloc_limited_conns);
3285 }
3286 return (NULL);
3287 }
3288 return (bbr_alloc(bbr));
3289 }
3290
3291 /* wrapper to allocate a sendmap entry, subject to a specific limit */
3292 static struct bbr_sendmap *
bbr_alloc_limit(struct tcp_bbr * bbr,uint8_t limit_type)3293 bbr_alloc_limit(struct tcp_bbr *bbr, uint8_t limit_type)
3294 {
3295 struct bbr_sendmap *rsm;
3296
3297 if (limit_type) {
3298 /* currently there is only one limit type */
3299 if (V_tcp_map_split_limit > 0 &&
3300 bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
3301 BBR_STAT_INC(bbr_split_limited);
3302 if (!bbr->alloc_limit_reported) {
3303 bbr->alloc_limit_reported = 1;
3304 BBR_STAT_INC(bbr_alloc_limited_conns);
3305 }
3306 return (NULL);
3307 }
3308 }
3309
3310 /* allocate and mark in the limit type, if set */
3311 rsm = bbr_alloc(bbr);
3312 if (rsm != NULL && limit_type) {
3313 rsm->r_limit_type = limit_type;
3314 bbr->r_ctl.rc_num_split_allocs++;
3315 }
3316 return (rsm);
3317 }
3318
3319 static void
bbr_free(struct tcp_bbr * bbr,struct bbr_sendmap * rsm)3320 bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
3321 {
3322 if (rsm->r_limit_type) {
3323 /* currently there is only one limit type */
3324 bbr->r_ctl.rc_num_split_allocs--;
3325 }
3326 if (rsm->r_is_smallmap)
3327 bbr->r_ctl.rc_num_small_maps_alloced--;
3328 if (bbr->r_ctl.rc_tlp_send == rsm)
3329 bbr->r_ctl.rc_tlp_send = NULL;
3330 if (bbr->r_ctl.rc_resend == rsm) {
3331 bbr->r_ctl.rc_resend = NULL;
3332 }
3333 if (bbr->r_ctl.rc_next == rsm)
3334 bbr->r_ctl.rc_next = NULL;
3335 if (bbr->r_ctl.rc_sacklast == rsm)
3336 bbr->r_ctl.rc_sacklast = NULL;
3337 if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
3338 memset(rsm, 0, sizeof(struct bbr_sendmap));
3339 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
3340 rsm->r_limit_type = 0;
3341 bbr->r_ctl.rc_free_cnt++;
3342 return;
3343 }
3344 bbr->r_ctl.rc_num_maps_alloced--;
3345 uma_zfree(bbr_zone, rsm);
3346 }
3347
3348 /*
3349 * Returns the BDP.
3350 */
3351 static uint64_t
bbr_get_bw_delay_prod(uint64_t rtt,uint64_t bw)3352 bbr_get_bw_delay_prod(uint64_t rtt, uint64_t bw) {
3353 /*
3354 * Calculate the bytes in flight needed given the bw (in bytes per
3355 * second) and the specifyed rtt in useconds. We need to put out the
3356 * returned value per RTT to match that rate. Gain will normally
3357 * raise it up from there.
3358 *
3359 * This should not overflow as long as the bandwidth is below 1
3360 * TByte per second (bw < 10**12 = 2**40) and the rtt is smaller
3361 * than 1000 seconds (rtt < 10**3 * 10**6 = 10**9 = 2**30).
3362 */
3363 uint64_t usec_per_sec;
3364
3365 usec_per_sec = USECS_IN_SECOND;
3366 return ((rtt * bw) / usec_per_sec);
3367 }
3368
3369 /*
3370 * Return the initial cwnd.
3371 */
3372 static uint32_t
bbr_initial_cwnd(struct tcp_bbr * bbr,struct tcpcb * tp)3373 bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp)
3374 {
3375 uint32_t i_cwnd;
3376
3377 if (bbr->rc_init_win) {
3378 i_cwnd = bbr->rc_init_win * tp->t_maxseg;
3379 } else if (V_tcp_initcwnd_segments)
3380 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),
3381 max(2 * tp->t_maxseg, 14600));
3382 else if (V_tcp_do_rfc3390)
3383 i_cwnd = min(4 * tp->t_maxseg,
3384 max(2 * tp->t_maxseg, 4380));
3385 else {
3386 /* Per RFC5681 Section 3.1 */
3387 if (tp->t_maxseg > 2190)
3388 i_cwnd = 2 * tp->t_maxseg;
3389 else if (tp->t_maxseg > 1095)
3390 i_cwnd = 3 * tp->t_maxseg;
3391 else
3392 i_cwnd = 4 * tp->t_maxseg;
3393 }
3394 return (i_cwnd);
3395 }
3396
3397 /*
3398 * Given a specified gain, return the target
3399 * cwnd based on that gain.
3400 */
3401 static uint32_t
bbr_get_raw_target_cwnd(struct tcp_bbr * bbr,uint32_t gain,uint64_t bw)3402 bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw)
3403 {
3404 uint64_t bdp, rtt;
3405 uint32_t cwnd;
3406
3407 if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||
3408 (bbr_get_full_bw(bbr) == 0)) {
3409 /* No measurements yet */
3410 return (bbr_initial_cwnd(bbr, bbr->rc_tp));
3411 }
3412 /*
3413 * Get bytes per RTT needed (rttProp is normally in
3414 * bbr_cwndtarget_rtt_touse)
3415 */
3416 rtt = bbr_get_rtt(bbr, bbr_cwndtarget_rtt_touse);
3417 /* Get the bdp from the two values */
3418 bdp = bbr_get_bw_delay_prod(rtt, bw);
3419 /* Now apply the gain */
3420 cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT));
3421
3422 return (cwnd);
3423 }
3424
3425 static uint32_t
bbr_get_target_cwnd(struct tcp_bbr * bbr,uint64_t bw,uint32_t gain)3426 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain)
3427 {
3428 uint32_t cwnd, mss;
3429
3430 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
3431 /* Get the base cwnd with gain rounded to a mss */
3432 cwnd = roundup(bbr_get_raw_target_cwnd(bbr, bw, gain), mss);
3433 /*
3434 * Add in N (2 default since we do not have a
3435 * fq layer to trap packets in) quanta's per the I-D
3436 * section 4.2.3.2 quanta adjust.
3437 */
3438 cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);
3439 if (bbr->rc_use_google) {
3440 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3441 (bbr_state_val(bbr) == BBR_SUB_GAIN)) {
3442 /*
3443 * The linux implementation adds
3444 * an extra 2 x mss in gain cycle which
3445 * is documented no-where except in the code.
3446 * so we add more for Neal undocumented feature
3447 */
3448 cwnd += 2 * mss;
3449 }
3450 if ((cwnd / mss) & 0x1) {
3451 /* Round up for odd num mss */
3452 cwnd += mss;
3453 }
3454 }
3455 /* Are we below the min cwnd? */
3456 if (cwnd < get_min_cwnd(bbr))
3457 return (get_min_cwnd(bbr));
3458 return (cwnd);
3459 }
3460
3461 static uint16_t
bbr_gain_adjust(struct tcp_bbr * bbr,uint16_t gain)3462 bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain)
3463 {
3464 if (gain < 1)
3465 gain = 1;
3466 return (gain);
3467 }
3468
3469 static uint32_t
bbr_get_header_oh(struct tcp_bbr * bbr)3470 bbr_get_header_oh(struct tcp_bbr *bbr)
3471 {
3472 int seg_oh;
3473
3474 seg_oh = 0;
3475 if (bbr->r_ctl.rc_inc_tcp_oh) {
3476 /* Do we include TCP overhead? */
3477 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));
3478 }
3479 if (bbr->r_ctl.rc_inc_ip_oh) {
3480 /* Do we include IP overhead? */
3481 #ifdef INET6
3482 if (bbr->r_is_v6) {
3483 seg_oh += sizeof(struct ip6_hdr);
3484 } else
3485 #endif
3486 {
3487
3488 #ifdef INET
3489 seg_oh += sizeof(struct ip);
3490 #endif
3491 }
3492 }
3493 if (bbr->r_ctl.rc_inc_enet_oh) {
3494 /* Do we include the ethernet overhead? */
3495 seg_oh += sizeof(struct ether_header);
3496 }
3497 return(seg_oh);
3498 }
3499
3500 static uint32_t
bbr_get_pacing_length(struct tcp_bbr * bbr,uint16_t gain,uint32_t useconds_time,uint64_t bw)3501 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw)
3502 {
3503 uint64_t divor, res, tim;
3504
3505 if (useconds_time == 0)
3506 return (0);
3507 gain = bbr_gain_adjust(bbr, gain);
3508 divor = (uint64_t)USECS_IN_SECOND * (uint64_t)BBR_UNIT;
3509 tim = useconds_time;
3510 res = (tim * bw * gain) / divor;
3511 if (res == 0)
3512 res = 1;
3513 return ((uint32_t)res);
3514 }
3515
3516 /*
3517 * Given a gain and a length return the delay in useconds that
3518 * should be used to evenly space out packets
3519 * on the connection (based on the gain factor).
3520 */
3521 static uint32_t
bbr_get_pacing_delay(struct tcp_bbr * bbr,uint16_t gain,int32_t len,uint32_t cts,int nolog)3522 bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog)
3523 {
3524 uint64_t bw, lentim, res;
3525 uint32_t usecs, srtt, over = 0;
3526 uint32_t seg_oh, num_segs, maxseg;
3527
3528 if (len == 0)
3529 return (0);
3530
3531 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
3532 num_segs = (len + maxseg - 1) / maxseg;
3533 if (bbr->rc_use_google == 0) {
3534 seg_oh = bbr_get_header_oh(bbr);
3535 len += (num_segs * seg_oh);
3536 }
3537 gain = bbr_gain_adjust(bbr, gain);
3538 bw = bbr_get_bw(bbr);
3539 if (bbr->rc_use_google) {
3540 uint64_t cbw;
3541
3542 /*
3543 * Reduce the b/w by the google discount
3544 * factor 10 = 1%.
3545 */
3546 cbw = bw * (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);
3547 cbw /= (uint64_t)1000;
3548 /* We don't apply a discount if it results in 0 */
3549 if (cbw > 0)
3550 bw = cbw;
3551 }
3552 lentim = ((uint64_t)len *
3553 (uint64_t)USECS_IN_SECOND *
3554 (uint64_t)BBR_UNIT);
3555 res = lentim / ((uint64_t)gain * bw);
3556 if (res == 0)
3557 res = 1;
3558 usecs = (uint32_t)res;
3559 srtt = bbr_get_rtt(bbr, BBR_SRTT);
3560 if (bbr_hptsi_max_mul && bbr_hptsi_max_div &&
3561 (bbr->rc_use_google == 0) &&
3562 (usecs > ((srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div))) {
3563 /*
3564 * We cannot let the delay be more than 1/2 the srtt time.
3565 * Otherwise we cannot pace out or send properly.
3566 */
3567 over = usecs = (srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div;
3568 BBR_STAT_INC(bbr_hpts_min_time);
3569 }
3570 if (!nolog)
3571 bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1);
3572 return (usecs);
3573 }
3574
3575 static void
bbr_ack_received(struct tcpcb * tp,struct tcp_bbr * bbr,struct tcphdr * th,uint32_t bytes_this_ack,uint32_t sack_changed,uint32_t prev_acked,int32_t line,uint32_t losses)3576 bbr_ack_received(struct tcpcb *tp, struct tcp_bbr *bbr, struct tcphdr *th, uint32_t bytes_this_ack,
3577 uint32_t sack_changed, uint32_t prev_acked, int32_t line, uint32_t losses)
3578 {
3579 uint64_t bw;
3580 uint32_t cwnd, target_cwnd, saved_bytes, maxseg;
3581 int32_t meth;
3582
3583 INP_WLOCK_ASSERT(tptoinpcb(tp));
3584
3585 #ifdef STATS
3586 if ((tp->t_flags & TF_GPUTINPROG) &&
3587 SEQ_GEQ(th->th_ack, tp->gput_ack)) {
3588 /*
3589 * Strech acks and compressed acks will cause this to
3590 * oscillate but we are doing it the same way as the main
3591 * stack so it will be compariable (though possibly not
3592 * ideal).
3593 */
3594 int32_t cgput;
3595 int64_t gput, time_stamp;
3596
3597 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8;
3598 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));
3599 cgput = gput / time_stamp;
3600 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
3601 cgput);
3602 if (tp->t_stats_gput_prev > 0)
3603 stats_voi_update_abs_s32(tp->t_stats,
3604 VOI_TCP_GPUT_ND,
3605 ((gput - tp->t_stats_gput_prev) * 100) /
3606 tp->t_stats_gput_prev);
3607 tp->t_flags &= ~TF_GPUTINPROG;
3608 tp->t_stats_gput_prev = cgput;
3609 }
3610 #endif
3611 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3612 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
3613 /* We don't change anything in probe-rtt */
3614 return;
3615 }
3616 maxseg = tp->t_maxseg - bbr->rc_last_options;
3617 saved_bytes = bytes_this_ack;
3618 bytes_this_ack += sack_changed;
3619 if (bytes_this_ack > prev_acked) {
3620 bytes_this_ack -= prev_acked;
3621 /*
3622 * A byte ack'd gives us a full mss
3623 * to be like linux i.e. they count packets.
3624 */
3625 if ((bytes_this_ack < maxseg) && bbr->rc_use_google)
3626 bytes_this_ack = maxseg;
3627 } else {
3628 /* Unlikely */
3629 bytes_this_ack = 0;
3630 }
3631 cwnd = tp->snd_cwnd;
3632 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3633 if (bw)
3634 target_cwnd = bbr_get_target_cwnd(bbr,
3635 bw,
3636 (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);
3637 else
3638 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);
3639 if (IN_RECOVERY(tp->t_flags) &&
3640 (bbr->bbr_prev_in_rec == 0)) {
3641 /*
3642 * We are entering recovery and
3643 * thus packet conservation.
3644 */
3645 bbr->pkt_conservation = 1;
3646 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;
3647 cwnd = ctf_flight_size(tp,
3648 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3649 bytes_this_ack;
3650 }
3651 if (IN_RECOVERY(tp->t_flags)) {
3652 uint32_t flight;
3653
3654 bbr->bbr_prev_in_rec = 1;
3655 if (cwnd > losses) {
3656 cwnd -= losses;
3657 if (cwnd < maxseg)
3658 cwnd = maxseg;
3659 } else
3660 cwnd = maxseg;
3661 flight = ctf_flight_size(tp,
3662 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3663 bbr_log_type_cwndupd(bbr, flight, 0,
3664 losses, 10, 0, 0, line);
3665 if (bbr->pkt_conservation) {
3666 uint32_t time_in;
3667
3668 if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))
3669 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;
3670 else
3671 time_in = 0;
3672
3673 if (time_in >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
3674 /* Clear packet conservation after an rttProp */
3675 bbr->pkt_conservation = 0;
3676 } else {
3677 if ((flight + bytes_this_ack) > cwnd)
3678 cwnd = flight + bytes_this_ack;
3679 if (cwnd < get_min_cwnd(bbr))
3680 cwnd = get_min_cwnd(bbr);
3681 tp->snd_cwnd = cwnd;
3682 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed,
3683 prev_acked, 1, target_cwnd, th->th_ack, line);
3684 return;
3685 }
3686 }
3687 } else
3688 bbr->bbr_prev_in_rec = 0;
3689 if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {
3690 bbr->r_ctl.restrict_growth--;
3691 if (bytes_this_ack > maxseg)
3692 bytes_this_ack = maxseg;
3693 }
3694 if (bbr->rc_filled_pipe) {
3695 /*
3696 * Here we have exited startup and filled the pipe. We will
3697 * thus allow the cwnd to shrink to the target. We hit here
3698 * mostly.
3699 */
3700 uint32_t s_cwnd;
3701
3702 meth = 2;
3703 s_cwnd = min((cwnd + bytes_this_ack), target_cwnd);
3704 if (s_cwnd > cwnd)
3705 cwnd = s_cwnd;
3706 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)
3707 cwnd = s_cwnd;
3708 } else {
3709 /*
3710 * Here we are still in startup, we increase cwnd by what
3711 * has been acked.
3712 */
3713 if ((cwnd < target_cwnd) ||
3714 (bbr->rc_past_init_win == 0)) {
3715 meth = 3;
3716 cwnd += bytes_this_ack;
3717 } else {
3718 /*
3719 * Method 4 means we are at target so no gain in
3720 * startup and past the initial window.
3721 */
3722 meth = 4;
3723 }
3724 }
3725 tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));
3726 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, line);
3727 }
3728
3729 static void
tcp_bbr_partialack(struct tcpcb * tp)3730 tcp_bbr_partialack(struct tcpcb *tp)
3731 {
3732 struct tcp_bbr *bbr;
3733
3734 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3735 INP_WLOCK_ASSERT(tptoinpcb(tp));
3736 if (ctf_flight_size(tp,
3737 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
3738 tp->snd_cwnd) {
3739 bbr->r_wanted_output = 1;
3740 }
3741 }
3742
3743 static void
bbr_post_recovery(struct tcpcb * tp)3744 bbr_post_recovery(struct tcpcb *tp)
3745 {
3746 struct tcp_bbr *bbr;
3747 uint32_t flight;
3748
3749 INP_WLOCK_ASSERT(tptoinpcb(tp));
3750 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3751 /*
3752 * Here we just exit recovery.
3753 */
3754 EXIT_RECOVERY(tp->t_flags);
3755 /* Lock in our b/w reduction for the specified number of pkt-epochs */
3756 bbr->r_recovery_bw = 0;
3757 tp->snd_recover = tp->snd_una;
3758 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3759 bbr->pkt_conservation = 0;
3760 if (bbr->rc_use_google == 0) {
3761 /*
3762 * For non-google mode lets
3763 * go ahead and make sure we clear
3764 * the recovery state so if we
3765 * bounce back in to recovery we
3766 * will do PC.
3767 */
3768 bbr->bbr_prev_in_rec = 0;
3769 }
3770 bbr_log_type_exit_rec(bbr);
3771 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3772 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3773 bbr_log_type_cwndupd(bbr, 0, 0, 0, 15, 0, 0, __LINE__);
3774 } else {
3775 /* For probe-rtt case lets fix up its saved_cwnd */
3776 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {
3777 bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;
3778 bbr_log_type_cwndupd(bbr, 0, 0, 0, 16, 0, 0, __LINE__);
3779 }
3780 }
3781 flight = ctf_flight_size(tp,
3782 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3783 if ((bbr->rc_use_google == 0) &&
3784 bbr_do_red) {
3785 uint64_t val, lr2use;
3786 uint32_t maxseg, newcwnd, acks_inflight, ratio, cwnd;
3787 uint32_t *cwnd_p;
3788
3789 if (bbr_get_rtt(bbr, BBR_SRTT)) {
3790 val = ((uint64_t)bbr_get_rtt(bbr, BBR_RTT_PROP) * (uint64_t)1000);
3791 val /= bbr_get_rtt(bbr, BBR_SRTT);
3792 ratio = (uint32_t)val;
3793 } else
3794 ratio = 1000;
3795
3796 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div,
3797 bbr->r_ctl.recovery_lr, 21,
3798 ratio,
3799 bbr->r_ctl.rc_red_cwnd_pe,
3800 __LINE__);
3801 if ((ratio < bbr_do_red) || (bbr_do_red == 0))
3802 goto done;
3803 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3804 bbr_prtt_slam_cwnd) ||
3805 (bbr_sub_drain_slam_cwnd &&
3806 (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3807 bbr->rc_hit_state_1 &&
3808 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) ||
3809 ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&
3810 bbr_slam_cwnd_in_main_drain)) {
3811 /*
3812 * Here we must poke at the saved cwnd
3813 * as well as the cwnd.
3814 */
3815 cwnd = bbr->r_ctl.rc_saved_cwnd;
3816 cwnd_p = &bbr->r_ctl.rc_saved_cwnd;
3817 } else {
3818 cwnd = tp->snd_cwnd;
3819 cwnd_p = &tp->snd_cwnd;
3820 }
3821 maxseg = tp->t_maxseg - bbr->rc_last_options;
3822 /* Add the overall lr with the recovery lr */
3823 if (bbr->r_ctl.rc_lost == 0)
3824 lr2use = 0;
3825 else if (bbr->r_ctl.rc_delivered == 0)
3826 lr2use = 1000;
3827 else {
3828 lr2use = (uint64_t)bbr->r_ctl.rc_lost * (uint64_t)1000;
3829 lr2use /= bbr->r_ctl.rc_delivered;
3830 }
3831 lr2use += bbr->r_ctl.recovery_lr;
3832 acks_inflight = (flight / (maxseg * 2));
3833 if (bbr_red_scale) {
3834 lr2use *= bbr_get_rtt(bbr, BBR_SRTT);
3835 lr2use /= bbr_red_scale;
3836 if ((bbr_red_growth_restrict) &&
3837 ((bbr_get_rtt(bbr, BBR_SRTT)/bbr_red_scale) > 1))
3838 bbr->r_ctl.restrict_growth += acks_inflight;
3839 }
3840 if (lr2use) {
3841 val = (uint64_t)cwnd * lr2use;
3842 val /= 1000;
3843 if (cwnd > val)
3844 newcwnd = roundup((cwnd - val), maxseg);
3845 else
3846 newcwnd = maxseg;
3847 } else {
3848 val = (uint64_t)cwnd * (uint64_t)bbr_red_mul;
3849 val /= (uint64_t)bbr_red_div;
3850 newcwnd = roundup((uint32_t)val, maxseg);
3851 }
3852 /* with standard delayed acks how many acks can I expect? */
3853 if (bbr_drop_limit == 0) {
3854 /*
3855 * Anticpate how much we will
3856 * raise the cwnd based on the acks.
3857 */
3858 if ((newcwnd + (acks_inflight * maxseg)) < get_min_cwnd(bbr)) {
3859 /* We do enforce the min (with the acks) */
3860 newcwnd = (get_min_cwnd(bbr) - acks_inflight);
3861 }
3862 } else {
3863 /*
3864 * A strict drop limit of N is inplace
3865 */
3866 if (newcwnd < (bbr_drop_limit * maxseg)) {
3867 newcwnd = bbr_drop_limit * maxseg;
3868 }
3869 }
3870 /* For the next N acks do we restrict the growth */
3871 *cwnd_p = newcwnd;
3872 if (tp->snd_cwnd > newcwnd)
3873 tp->snd_cwnd = newcwnd;
3874 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div, val, 22,
3875 (uint32_t)lr2use,
3876 bbr_get_rtt(bbr, BBR_SRTT), __LINE__);
3877 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;
3878 }
3879 done:
3880 bbr->r_ctl.recovery_lr = 0;
3881 if (flight <= tp->snd_cwnd) {
3882 bbr->r_wanted_output = 1;
3883 }
3884 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3885 }
3886
3887 static void
bbr_setup_red_bw(struct tcp_bbr * bbr,uint32_t cts)3888 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts)
3889 {
3890 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3891 /* Limit the drop in b/w to 1/2 our current filter. */
3892 if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)
3893 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;
3894 if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))
3895 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;
3896 tcp_bbr_tso_size_check(bbr, cts);
3897 }
3898
3899 static void
bbr_cong_signal(struct tcpcb * tp,struct tcphdr * th,uint32_t type,struct bbr_sendmap * rsm)3900 bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_sendmap *rsm)
3901 {
3902 struct tcp_bbr *bbr;
3903
3904 INP_WLOCK_ASSERT(tptoinpcb(tp));
3905 #ifdef STATS
3906 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);
3907 #endif
3908 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3909 switch (type) {
3910 case CC_NDUPACK:
3911 if (!IN_RECOVERY(tp->t_flags)) {
3912 tp->snd_recover = tp->snd_max;
3913 /* Start a new epoch */
3914 bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
3915 if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {
3916 /*
3917 * Move forward the lt epoch
3918 * so it won't count the truncated
3919 * epoch.
3920 */
3921 bbr->r_ctl.rc_lt_epoch++;
3922 }
3923 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
3924 /*
3925 * Just like the policer detection code
3926 * if we are in startup we must push
3927 * forward the last startup epoch
3928 * to hide the truncated PE.
3929 */
3930 bbr->r_ctl.rc_bbr_last_startup_epoch++;
3931 }
3932 bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;
3933 ENTER_RECOVERY(tp->t_flags);
3934 bbr->rc_tlp_rtx_out = 0;
3935 bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
3936 tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3937 if (tcp_in_hpts(bbr->rc_tp) &&
3938 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
3939 /*
3940 * When we enter recovery, we need to restart
3941 * any timers. This may mean we gain an agg
3942 * early, which will be made up for at the last
3943 * rxt out.
3944 */
3945 bbr->rc_timer_first = 1;
3946 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
3947 }
3948 /*
3949 * Calculate a new cwnd based on to the current
3950 * delivery rate with no gain. We get the bdp
3951 * without gaining it up like we normally would and
3952 * we use the last cur_del_rate.
3953 */
3954 if ((bbr->rc_use_google == 0) &&
3955 (bbr->r_ctl.bbr_rttprobe_gain_val ||
3956 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {
3957 tp->snd_cwnd = ctf_flight_size(tp,
3958 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3959 (tp->t_maxseg - bbr->rc_last_options);
3960 if (tp->snd_cwnd < get_min_cwnd(bbr)) {
3961 /* We always gate to min cwnd */
3962 tp->snd_cwnd = get_min_cwnd(bbr);
3963 }
3964 bbr_log_type_cwndupd(bbr, 0, 0, 0, 14, 0, 0, __LINE__);
3965 }
3966 bbr_log_type_enter_rec(bbr, rsm->r_start);
3967 }
3968 break;
3969 case CC_RTO_ERR:
3970 KMOD_TCPSTAT_INC(tcps_sndrexmitbad);
3971 /* RTO was unnecessary, so reset everything. */
3972 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);
3973 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3974 tp->snd_cwnd = tp->snd_cwnd_prev;
3975 tp->snd_ssthresh = tp->snd_ssthresh_prev;
3976 tp->snd_recover = tp->snd_recover_prev;
3977 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3978 bbr_log_type_cwndupd(bbr, 0, 0, 0, 13, 0, 0, __LINE__);
3979 }
3980 tp->t_badrxtwin = 0;
3981 break;
3982 }
3983 }
3984
3985 /*
3986 * Indicate whether this ack should be delayed. We can delay the ack if
3987 * following conditions are met:
3988 * - There is no delayed ack timer in progress.
3989 * - Our last ack wasn't a 0-sized window. We never want to delay
3990 * the ack that opens up a 0-sized window.
3991 * - LRO wasn't used for this segment. We make sure by checking that the
3992 * segment size is not larger than the MSS.
3993 * - Delayed acks are enabled or this is a half-synchronized T/TCP
3994 * connection.
3995 * - The data being acked is less than a full segment (a stretch ack
3996 * of more than a segment we should ack.
3997 * - nsegs is 1 (if its more than that we received more than 1 ack).
3998 */
3999 #define DELAY_ACK(tp, bbr, nsegs) \
4000 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
4001 ((tp->t_flags & TF_DELACK) == 0) && \
4002 ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) && \
4003 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4004
4005 /*
4006 * Return the lowest RSM in the map of
4007 * packets still in flight that is not acked.
4008 * This should normally find on the first one
4009 * since we remove packets from the send
4010 * map after they are marked ACKED.
4011 */
4012 static struct bbr_sendmap *
bbr_find_lowest_rsm(struct tcp_bbr * bbr)4013 bbr_find_lowest_rsm(struct tcp_bbr *bbr)
4014 {
4015 struct bbr_sendmap *rsm;
4016
4017 /*
4018 * Walk the time-order transmitted list looking for an rsm that is
4019 * not acked. This will be the one that was sent the longest time
4020 * ago that is still outstanding.
4021 */
4022 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {
4023 if (rsm->r_flags & BBR_ACKED) {
4024 continue;
4025 }
4026 goto finish;
4027 }
4028 finish:
4029 return (rsm);
4030 }
4031
4032 static struct bbr_sendmap *
bbr_find_high_nonack(struct tcp_bbr * bbr,struct bbr_sendmap * rsm)4033 bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
4034 {
4035 struct bbr_sendmap *prsm;
4036
4037 /*
4038 * Walk the sequence order list backward until we hit and arrive at
4039 * the highest seq not acked. In theory when this is called it
4040 * should be the last segment (which it was not).
4041 */
4042 prsm = rsm;
4043 TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4044 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) {
4045 continue;
4046 }
4047 return (prsm);
4048 }
4049 return (NULL);
4050 }
4051
4052 /*
4053 * Returns to the caller the number of microseconds that
4054 * the packet can be outstanding before we think we
4055 * should have had an ack returned.
4056 */
4057 static uint32_t
bbr_calc_thresh_rack(struct tcp_bbr * bbr,uint32_t srtt,uint32_t cts,struct bbr_sendmap * rsm)4058 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm)
4059 {
4060 /*
4061 * lro is the flag we use to determine if we have seen reordering.
4062 * If it gets set we have seen reordering. The reorder logic either
4063 * works in one of two ways:
4064 *
4065 * If reorder-fade is configured, then we track the last time we saw
4066 * re-ordering occur. If we reach the point where enough time as
4067 * passed we no longer consider reordering has occuring.
4068 *
4069 * Or if reorder-face is 0, then once we see reordering we consider
4070 * the connection to alway be subject to reordering and just set lro
4071 * to 1.
4072 *
4073 * In the end if lro is non-zero we add the extra time for
4074 * reordering in.
4075 */
4076 int32_t lro;
4077 uint32_t thresh, t_rxtcur;
4078
4079 if (srtt == 0)
4080 srtt = 1;
4081 if (bbr->r_ctl.rc_reorder_ts) {
4082 if (bbr->r_ctl.rc_reorder_fade) {
4083 if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {
4084 lro = cts - bbr->r_ctl.rc_reorder_ts;
4085 if (lro == 0) {
4086 /*
4087 * No time as passed since the last
4088 * reorder, mark it as reordering.
4089 */
4090 lro = 1;
4091 }
4092 } else {
4093 /* Negative time? */
4094 lro = 0;
4095 }
4096 if (lro > bbr->r_ctl.rc_reorder_fade) {
4097 /* Turn off reordering seen too */
4098 bbr->r_ctl.rc_reorder_ts = 0;
4099 lro = 0;
4100 }
4101 } else {
4102 /* Reodering does not fade */
4103 lro = 1;
4104 }
4105 } else {
4106 lro = 0;
4107 }
4108 thresh = srtt + bbr->r_ctl.rc_pkt_delay;
4109 if (lro) {
4110 /* It must be set, if not you get 1/4 rtt */
4111 if (bbr->r_ctl.rc_reorder_shift)
4112 thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);
4113 else
4114 thresh += (srtt >> 2);
4115 } else {
4116 thresh += 1000;
4117 }
4118 /* We don't let the rack timeout be above a RTO */
4119 if ((bbr->rc_tp)->t_srtt == 0)
4120 t_rxtcur = BBR_INITIAL_RTO;
4121 else
4122 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
4123 if (thresh > t_rxtcur) {
4124 thresh = t_rxtcur;
4125 }
4126 /* And we don't want it above the RTO max either */
4127 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4128 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4129 }
4130 bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK);
4131 return (thresh);
4132 }
4133
4134 /*
4135 * Return to the caller the amount of time in mico-seconds
4136 * that should be used for the TLP timer from the last
4137 * send time of this packet.
4138 */
4139 static uint32_t
bbr_calc_thresh_tlp(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t srtt,uint32_t cts)4140 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
4141 struct bbr_sendmap *rsm, uint32_t srtt,
4142 uint32_t cts)
4143 {
4144 uint32_t thresh, len, maxseg, t_rxtcur;
4145 struct bbr_sendmap *prsm;
4146
4147 if (srtt == 0)
4148 srtt = 1;
4149 if (bbr->rc_tlp_threshold)
4150 thresh = srtt + (srtt / bbr->rc_tlp_threshold);
4151 else
4152 thresh = (srtt * 2);
4153 maxseg = tp->t_maxseg - bbr->rc_last_options;
4154 /* Get the previous sent packet, if any */
4155 len = rsm->r_end - rsm->r_start;
4156
4157 /* 2.1 behavior */
4158 prsm = TAILQ_PREV(rsm, bbr_head, r_tnext);
4159 if (prsm && (len <= maxseg)) {
4160 /*
4161 * Two packets outstanding, thresh should be (2*srtt) +
4162 * possible inter-packet delay (if any).
4163 */
4164 uint32_t inter_gap = 0;
4165 int idx, nidx;
4166
4167 idx = rsm->r_rtr_cnt - 1;
4168 nidx = prsm->r_rtr_cnt - 1;
4169 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {
4170 /* Yes it was sent later (or at the same time) */
4171 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
4172 }
4173 thresh += inter_gap;
4174 } else if (len <= maxseg) {
4175 /*
4176 * Possibly compensate for delayed-ack.
4177 */
4178 uint32_t alt_thresh;
4179
4180 alt_thresh = srtt + (srtt / 2) + bbr_delayed_ack_time;
4181 if (alt_thresh > thresh)
4182 thresh = alt_thresh;
4183 }
4184 /* Not above the current RTO */
4185 if (tp->t_srtt == 0)
4186 t_rxtcur = BBR_INITIAL_RTO;
4187 else
4188 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur);
4189
4190 bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP);
4191 /* Not above an RTO */
4192 if (thresh > t_rxtcur) {
4193 thresh = t_rxtcur;
4194 }
4195 /* Not above a RTO max */
4196 if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4197 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4198 }
4199 /* And now apply the user TLP min */
4200 if (thresh < bbr_tlp_min) {
4201 thresh = bbr_tlp_min;
4202 }
4203 return (thresh);
4204 }
4205
4206 /*
4207 * Return one of three RTTs to use (in microseconds).
4208 */
4209 static __inline uint32_t
bbr_get_rtt(struct tcp_bbr * bbr,int32_t rtt_type)4210 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type)
4211 {
4212 uint32_t f_rtt;
4213 uint32_t srtt;
4214
4215 f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
4216 if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {
4217 /* We have no rtt at all */
4218 if (bbr->rc_tp->t_srtt == 0)
4219 f_rtt = BBR_INITIAL_RTO;
4220 else
4221 f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4222 /*
4223 * Since we don't know how good the rtt is apply a
4224 * delayed-ack min
4225 */
4226 if (f_rtt < bbr_delayed_ack_time) {
4227 f_rtt = bbr_delayed_ack_time;
4228 }
4229 }
4230 /* Take the filter version or last measured pkt-rtt */
4231 if (rtt_type == BBR_RTT_PROP) {
4232 srtt = f_rtt;
4233 } else if (rtt_type == BBR_RTT_PKTRTT) {
4234 if (bbr->r_ctl.rc_pkt_epoch_rtt) {
4235 srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
4236 } else {
4237 /* No pkt rtt yet */
4238 srtt = f_rtt;
4239 }
4240 } else if (rtt_type == BBR_RTT_RACK) {
4241 srtt = bbr->r_ctl.rc_last_rtt;
4242 /* We need to add in any internal delay for our timer */
4243 if (bbr->rc_ack_was_delayed)
4244 srtt += bbr->r_ctl.rc_ack_hdwr_delay;
4245 } else if (rtt_type == BBR_SRTT) {
4246 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4247 } else {
4248 /* TSNH */
4249 srtt = f_rtt;
4250 #ifdef BBR_INVARIANTS
4251 panic("Unknown rtt request type %d", rtt_type);
4252 #endif
4253 }
4254 return (srtt);
4255 }
4256
4257 static int
bbr_is_lost(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t cts)4258 bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts)
4259 {
4260 uint32_t thresh;
4261
4262 thresh = bbr_calc_thresh_rack(bbr, bbr_get_rtt(bbr, BBR_RTT_RACK),
4263 cts, rsm);
4264 if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {
4265 /* It is lost (past time) */
4266 return (1);
4267 }
4268 return (0);
4269 }
4270
4271 /*
4272 * Return a sendmap if we need to retransmit something.
4273 */
4274 static struct bbr_sendmap *
bbr_check_recovery_mode(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4275 bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4276 {
4277 /*
4278 * Check to see that we don't need to fall into recovery. We will
4279 * need to do so if our oldest transmit is past the time we should
4280 * have had an ack.
4281 */
4282
4283 struct bbr_sendmap *rsm;
4284 int32_t idx;
4285
4286 if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {
4287 /* Nothing outstanding that we know of */
4288 return (NULL);
4289 }
4290 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
4291 if (rsm == NULL) {
4292 /* Nothing in the transmit map */
4293 return (NULL);
4294 }
4295 if (tp->t_flags & TF_SENTFIN) {
4296 /* Fin restricted, don't find anything once a fin is sent */
4297 return (NULL);
4298 }
4299 if (rsm->r_flags & BBR_ACKED) {
4300 /*
4301 * Ok the first one is acked (this really should not happen
4302 * since we remove the from the tmap once they are acked)
4303 */
4304 rsm = bbr_find_lowest_rsm(bbr);
4305 if (rsm == NULL)
4306 return (NULL);
4307 }
4308 idx = rsm->r_rtr_cnt - 1;
4309 if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {
4310 /* Send timestamp is the same or less? can't be ready */
4311 return (NULL);
4312 }
4313 /* Get our RTT time */
4314 if (bbr_is_lost(bbr, rsm, cts) &&
4315 ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
4316 (rsm->r_flags & BBR_SACK_PASSED))) {
4317 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4318 rsm->r_flags |= BBR_MARKED_LOST;
4319 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4320 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4321 }
4322 bbr_cong_signal(tp, NULL, CC_NDUPACK, rsm);
4323 #ifdef BBR_INVARIANTS
4324 if ((rsm->r_end - rsm->r_start) == 0)
4325 panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm);
4326 #endif
4327 return (rsm);
4328 }
4329 return (NULL);
4330 }
4331
4332 /*
4333 * RACK Timer, here we simply do logging and house keeping.
4334 * the normal bbr_output_wtime() function will call the
4335 * appropriate thing to check if we need to do a RACK retransmit.
4336 * We return 1, saying don't proceed with bbr_output_wtime only
4337 * when all timers have been stopped (destroyed PCB?).
4338 */
4339 static int
bbr_timeout_rack(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4340 bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4341 {
4342 /*
4343 * This timer simply provides an internal trigger to send out data.
4344 * The check_recovery_mode call will see if there are needed
4345 * retransmissions, if so we will enter fast-recovery. The output
4346 * call may or may not do the same thing depending on sysctl
4347 * settings.
4348 */
4349 uint32_t lost;
4350
4351 if (bbr->rc_all_timers_stopped) {
4352 return (1);
4353 }
4354 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4355 /* Its not time yet */
4356 return (0);
4357 }
4358 BBR_STAT_INC(bbr_to_tot);
4359 lost = bbr->r_ctl.rc_lost;
4360 if (bbr->r_state && (bbr->r_state != tp->t_state))
4361 bbr_set_state(tp, bbr, 0);
4362 bbr_log_to_event(bbr, cts, BBR_TO_FRM_RACK);
4363 if (bbr->r_ctl.rc_resend == NULL) {
4364 /* Lets do the check here */
4365 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
4366 }
4367 if (bbr_policer_call_from_rack_to)
4368 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4369 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
4370 return (0);
4371 }
4372
4373 static __inline void
bbr_clone_rsm(struct tcp_bbr * bbr,struct bbr_sendmap * nrsm,struct bbr_sendmap * rsm,uint32_t start)4374 bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start)
4375 {
4376 int idx;
4377
4378 nrsm->r_start = start;
4379 nrsm->r_end = rsm->r_end;
4380 nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
4381 nrsm-> r_rtt_not_allowed = rsm->r_rtt_not_allowed;
4382 nrsm->r_flags = rsm->r_flags;
4383 /* We don't transfer forward the SYN flag */
4384 nrsm->r_flags &= ~BBR_HAS_SYN;
4385 /* We move forward the FIN flag, not that this should happen */
4386 rsm->r_flags &= ~BBR_HAS_FIN;
4387 nrsm->r_dupack = rsm->r_dupack;
4388 nrsm->r_rtr_bytes = 0;
4389 nrsm->r_is_gain = rsm->r_is_gain;
4390 nrsm->r_is_drain = rsm->r_is_drain;
4391 nrsm->r_delivered = rsm->r_delivered;
4392 nrsm->r_ts_valid = rsm->r_ts_valid;
4393 nrsm->r_del_ack_ts = rsm->r_del_ack_ts;
4394 nrsm->r_del_time = rsm->r_del_time;
4395 nrsm->r_app_limited = rsm->r_app_limited;
4396 nrsm->r_first_sent_time = rsm->r_first_sent_time;
4397 nrsm->r_flight_at_send = rsm->r_flight_at_send;
4398 /* We split a piece the lower section looses any just_ret flag. */
4399 nrsm->r_bbr_state = rsm->r_bbr_state;
4400 for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {
4401 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
4402 }
4403 rsm->r_end = nrsm->r_start;
4404 idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
4405 idx /= 8;
4406 /* Check if we got too small */
4407 if ((rsm->r_is_smallmap == 0) &&
4408 ((rsm->r_end - rsm->r_start) <= idx)) {
4409 bbr->r_ctl.rc_num_small_maps_alloced++;
4410 rsm->r_is_smallmap = 1;
4411 }
4412 /* Check the new one as well */
4413 if ((nrsm->r_end - nrsm->r_start) <= idx) {
4414 bbr->r_ctl.rc_num_small_maps_alloced++;
4415 nrsm->r_is_smallmap = 1;
4416 }
4417 }
4418
4419 static int
bbr_sack_mergable(struct bbr_sendmap * at,uint32_t start,uint32_t end)4420 bbr_sack_mergable(struct bbr_sendmap *at,
4421 uint32_t start, uint32_t end)
4422 {
4423 /*
4424 * Given a sack block defined by
4425 * start and end, and a current position
4426 * at. Return 1 if either side of at
4427 * would show that the block is mergable
4428 * to that side. A block to be mergable
4429 * must have overlap with the start/end
4430 * and be in the SACK'd state.
4431 */
4432 struct bbr_sendmap *l_rsm;
4433 struct bbr_sendmap *r_rsm;
4434
4435 /* first get the either side blocks */
4436 l_rsm = TAILQ_PREV(at, bbr_head, r_next);
4437 r_rsm = TAILQ_NEXT(at, r_next);
4438 if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) {
4439 /* Potentially mergeable */
4440 if ((l_rsm->r_end == start) ||
4441 (SEQ_LT(start, l_rsm->r_end) &&
4442 SEQ_GT(end, l_rsm->r_end))) {
4443 /*
4444 * map blk |------|
4445 * sack blk |------|
4446 * <or>
4447 * map blk |------|
4448 * sack blk |------|
4449 */
4450 return (1);
4451 }
4452 }
4453 if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) {
4454 /* Potentially mergeable */
4455 if ((r_rsm->r_start == end) ||
4456 (SEQ_LT(start, r_rsm->r_start) &&
4457 SEQ_GT(end, r_rsm->r_start))) {
4458 /*
4459 * map blk |---------|
4460 * sack blk |----|
4461 * <or>
4462 * map blk |---------|
4463 * sack blk |-------|
4464 */
4465 return (1);
4466 }
4467 }
4468 return (0);
4469 }
4470
4471 static struct bbr_sendmap *
bbr_merge_rsm(struct tcp_bbr * bbr,struct bbr_sendmap * l_rsm,struct bbr_sendmap * r_rsm)4472 bbr_merge_rsm(struct tcp_bbr *bbr,
4473 struct bbr_sendmap *l_rsm,
4474 struct bbr_sendmap *r_rsm)
4475 {
4476 /*
4477 * We are merging two ack'd RSM's,
4478 * the l_rsm is on the left (lower seq
4479 * values) and the r_rsm is on the right
4480 * (higher seq value). The simplest way
4481 * to merge these is to move the right
4482 * one into the left. I don't think there
4483 * is any reason we need to try to find
4484 * the oldest (or last oldest retransmitted).
4485 */
4486 l_rsm->r_end = r_rsm->r_end;
4487 if (l_rsm->r_dupack < r_rsm->r_dupack)
4488 l_rsm->r_dupack = r_rsm->r_dupack;
4489 if (r_rsm->r_rtr_bytes)
4490 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;
4491 if (r_rsm->r_in_tmap) {
4492 /* This really should not happen */
4493 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);
4494 }
4495 if (r_rsm->r_app_limited)
4496 l_rsm->r_app_limited = r_rsm->r_app_limited;
4497 /* Now the flags */
4498 if (r_rsm->r_flags & BBR_HAS_FIN)
4499 l_rsm->r_flags |= BBR_HAS_FIN;
4500 if (r_rsm->r_flags & BBR_TLP)
4501 l_rsm->r_flags |= BBR_TLP;
4502 if (r_rsm->r_flags & BBR_RWND_COLLAPSED)
4503 l_rsm->r_flags |= BBR_RWND_COLLAPSED;
4504 if (r_rsm->r_flags & BBR_MARKED_LOST) {
4505 /* This really should not happen */
4506 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;
4507 }
4508 TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);
4509 if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {
4510 /* Transfer the split limit to the map we free */
4511 r_rsm->r_limit_type = l_rsm->r_limit_type;
4512 l_rsm->r_limit_type = 0;
4513 }
4514 bbr_free(bbr, r_rsm);
4515 return(l_rsm);
4516 }
4517
4518 /*
4519 * TLP Timer, here we simply setup what segment we want to
4520 * have the TLP expire on, the normal bbr_output_wtime() will then
4521 * send it out.
4522 *
4523 * We return 1, saying don't proceed with bbr_output_wtime only
4524 * when all timers have been stopped (destroyed PCB?).
4525 */
4526 static int
bbr_timeout_tlp(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4527 bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4528 {
4529 /*
4530 * Tail Loss Probe.
4531 */
4532 struct bbr_sendmap *rsm = NULL;
4533 struct socket *so;
4534 uint32_t amm;
4535 uint32_t out, avail;
4536 uint32_t maxseg;
4537 int collapsed_win = 0;
4538
4539 if (bbr->rc_all_timers_stopped) {
4540 return (1);
4541 }
4542 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4543 /* Its not time yet */
4544 return (0);
4545 }
4546 if (ctf_progress_timeout_check(tp, true)) {
4547 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4548 return (-ETIMEDOUT); /* tcp_drop() */
4549 }
4550 /* Did we somehow get into persists? */
4551 if (bbr->rc_in_persist) {
4552 return (0);
4553 }
4554 if (bbr->r_state && (bbr->r_state != tp->t_state))
4555 bbr_set_state(tp, bbr, 0);
4556 BBR_STAT_INC(bbr_tlp_tot);
4557 maxseg = tp->t_maxseg - bbr->rc_last_options;
4558 /*
4559 * A TLP timer has expired. We have been idle for 2 rtts. So we now
4560 * need to figure out how to force a full MSS segment out.
4561 */
4562 so = tptosocket(tp);
4563 avail = sbavail(&so->so_snd);
4564 out = ctf_outstanding(tp);
4565 if (out > tp->snd_wnd) {
4566 /* special case, we need a retransmission */
4567 collapsed_win = 1;
4568 goto need_retran;
4569 }
4570 if (avail > out) {
4571 /* New data is available */
4572 amm = avail - out;
4573 if (amm > maxseg) {
4574 amm = maxseg;
4575 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {
4576 /* not enough to fill a MTU and no-delay is off */
4577 goto need_retran;
4578 }
4579 /* Set the send-new override */
4580 if ((out + amm) <= tp->snd_wnd) {
4581 bbr->rc_tlp_new_data = 1;
4582 } else {
4583 goto need_retran;
4584 }
4585 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4586 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;
4587 bbr->r_ctl.rc_tlp_send = NULL;
4588 /* cap any slots */
4589 BBR_STAT_INC(bbr_tlp_newdata);
4590 goto send;
4591 }
4592 need_retran:
4593 /*
4594 * Ok we need to arrange the last un-acked segment to be re-sent, or
4595 * optionally the first un-acked segment.
4596 */
4597 if (collapsed_win == 0) {
4598 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
4599 if (rsm && (rsm->r_flags & (BBR_ACKED | BBR_HAS_FIN))) {
4600 rsm = bbr_find_high_nonack(bbr, rsm);
4601 }
4602 if (rsm == NULL) {
4603 goto restore;
4604 }
4605 } else {
4606 /*
4607 * We must find the last segment
4608 * that was acceptable by the client.
4609 */
4610 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4611 if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {
4612 /* Found one */
4613 break;
4614 }
4615 }
4616 if (rsm == NULL) {
4617 /* None? if so send the first */
4618 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4619 if (rsm == NULL)
4620 goto restore;
4621 }
4622 }
4623 if ((rsm->r_end - rsm->r_start) > maxseg) {
4624 /*
4625 * We need to split this the last segment in two.
4626 */
4627 struct bbr_sendmap *nrsm;
4628
4629 nrsm = bbr_alloc_full_limit(bbr);
4630 if (nrsm == NULL) {
4631 /*
4632 * We can't get memory to split, we can either just
4633 * not split it. Or retransmit the whole piece, lets
4634 * do the large send (BTLP :-) ).
4635 */
4636 goto go_for_it;
4637 }
4638 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));
4639 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
4640 if (rsm->r_in_tmap) {
4641 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
4642 nrsm->r_in_tmap = 1;
4643 }
4644 rsm->r_flags &= (~BBR_HAS_FIN);
4645 rsm = nrsm;
4646 }
4647 go_for_it:
4648 bbr->r_ctl.rc_tlp_send = rsm;
4649 bbr->rc_tlp_rtx_out = 1;
4650 if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {
4651 bbr->r_ctl.rc_tlp_seg_send_cnt++;
4652 tp->t_rxtshift++;
4653 } else {
4654 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
4655 bbr->r_ctl.rc_tlp_seg_send_cnt = 1;
4656 }
4657 send:
4658 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
4659 /*
4660 * Can't [re]/transmit a segment we have retransmitted the
4661 * max times. We need the retransmit timer to take over.
4662 */
4663 restore:
4664 bbr->rc_tlp_new_data = 0;
4665 bbr->r_ctl.rc_tlp_send = NULL;
4666 if (rsm)
4667 rsm->r_flags &= ~BBR_TLP;
4668 BBR_STAT_INC(bbr_tlp_retran_fail);
4669 return (0);
4670 } else if (rsm) {
4671 rsm->r_flags |= BBR_TLP;
4672 }
4673 if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&
4674 (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {
4675 /*
4676 * We have retransmitted to many times for TLP. Switch to
4677 * the regular RTO timer
4678 */
4679 goto restore;
4680 }
4681 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TLP);
4682 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
4683 return (0);
4684 }
4685
4686 /*
4687 * Delayed ack Timer, here we simply need to setup the
4688 * ACK_NOW flag and remove the DELACK flag. From there
4689 * the output routine will send the ack out.
4690 *
4691 * We only return 1, saying don't proceed, if all timers
4692 * are stopped (destroyed PCB?).
4693 */
4694 static int
bbr_timeout_delack(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4695 bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4696 {
4697 if (bbr->rc_all_timers_stopped) {
4698 return (1);
4699 }
4700 bbr_log_to_event(bbr, cts, BBR_TO_FRM_DELACK);
4701 tp->t_flags &= ~TF_DELACK;
4702 tp->t_flags |= TF_ACKNOW;
4703 KMOD_TCPSTAT_INC(tcps_delack);
4704 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
4705 return (0);
4706 }
4707
4708 /*
4709 * Here we send a KEEP-ALIVE like probe to the
4710 * peer, we do not send data.
4711 *
4712 * We only return 1, saying don't proceed, if all timers
4713 * are stopped (destroyed PCB?).
4714 */
4715 static int
bbr_timeout_persist(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4716 bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4717 {
4718 struct tcptemp *t_template;
4719 int32_t retval = 1;
4720
4721 if (bbr->rc_all_timers_stopped) {
4722 return (1);
4723 }
4724 if (bbr->rc_in_persist == 0)
4725 return (0);
4726
4727 /*
4728 * Persistence timer into zero window. Force a byte to be output, if
4729 * possible.
4730 */
4731 bbr_log_to_event(bbr, cts, BBR_TO_FRM_PERSIST);
4732 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
4733 KMOD_TCPSTAT_INC(tcps_persisttimeo);
4734 /*
4735 * Have we exceeded the user specified progress time?
4736 */
4737 if (ctf_progress_timeout_check(tp, true)) {
4738 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4739 return (-ETIMEDOUT); /* tcp_drop() */
4740 }
4741 /*
4742 * Hack: if the peer is dead/unreachable, we do not time out if the
4743 * window is closed. After a full backoff, drop the connection if
4744 * the idle time (no responses to probes) reaches the maximum
4745 * backoff that we would use if retransmitting.
4746 */
4747 if (tp->t_rxtshift >= V_tcp_retries &&
4748 (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
4749 ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {
4750 KMOD_TCPSTAT_INC(tcps_persistdrop);
4751 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
4752 return (-ETIMEDOUT); /* tcp_drop() */
4753 }
4754 if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&
4755 tp->snd_una == tp->snd_max) {
4756 bbr_exit_persist(tp, bbr, cts, __LINE__);
4757 retval = 0;
4758 goto out;
4759 }
4760 /*
4761 * If the user has closed the socket then drop a persisting
4762 * connection after a much reduced timeout.
4763 */
4764 if (tp->t_state > TCPS_CLOSE_WAIT &&
4765 (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
4766 KMOD_TCPSTAT_INC(tcps_persistdrop);
4767 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
4768 return (-ETIMEDOUT); /* tcp_drop() */
4769 }
4770 t_template = tcpip_maketemplate(bbr->rc_inp);
4771 if (t_template) {
4772 tcp_respond(tp, t_template->tt_ipgen,
4773 &t_template->tt_t, (struct mbuf *)NULL,
4774 tp->rcv_nxt, tp->snd_una - 1, 0);
4775 /* This sends an ack */
4776 if (tp->t_flags & TF_DELACK)
4777 tp->t_flags &= ~TF_DELACK;
4778 free(t_template, M_TEMP);
4779 }
4780 if (tp->t_rxtshift < V_tcp_retries)
4781 tp->t_rxtshift++;
4782 bbr_start_hpts_timer(bbr, tp, cts, 3, 0, 0);
4783 out:
4784 return (retval);
4785 }
4786
4787 /*
4788 * If a keepalive goes off, we had no other timers
4789 * happening. We always return 1 here since this
4790 * routine either drops the connection or sends
4791 * out a segment with respond.
4792 */
4793 static int
bbr_timeout_keepalive(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4794 bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4795 {
4796 struct tcptemp *t_template;
4797 struct inpcb *inp = tptoinpcb(tp);
4798
4799 if (bbr->rc_all_timers_stopped) {
4800 return (1);
4801 }
4802 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
4803 bbr_log_to_event(bbr, cts, BBR_TO_FRM_KEEP);
4804 /*
4805 * Keep-alive timer went off; send something or drop connection if
4806 * idle for too long.
4807 */
4808 KMOD_TCPSTAT_INC(tcps_keeptimeo);
4809 if (tp->t_state < TCPS_ESTABLISHED)
4810 goto dropit;
4811 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
4812 tp->t_state <= TCPS_CLOSING) {
4813 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
4814 goto dropit;
4815 /*
4816 * Send a packet designed to force a response if the peer is
4817 * up and reachable: either an ACK if the connection is
4818 * still alive, or an RST if the peer has closed the
4819 * connection due to timeout or reboot. Using sequence
4820 * number tp->snd_una-1 causes the transmitted zero-length
4821 * segment to lie outside the receive window; by the
4822 * protocol spec, this requires the correspondent TCP to
4823 * respond.
4824 */
4825 KMOD_TCPSTAT_INC(tcps_keepprobe);
4826 t_template = tcpip_maketemplate(inp);
4827 if (t_template) {
4828 tcp_respond(tp, t_template->tt_ipgen,
4829 &t_template->tt_t, (struct mbuf *)NULL,
4830 tp->rcv_nxt, tp->snd_una - 1, 0);
4831 free(t_template, M_TEMP);
4832 }
4833 }
4834 bbr_start_hpts_timer(bbr, tp, cts, 4, 0, 0);
4835 return (1);
4836 dropit:
4837 KMOD_TCPSTAT_INC(tcps_keepdrops);
4838 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX);
4839 return (-ETIMEDOUT); /* tcp_drop() */
4840 }
4841
4842 /*
4843 * Retransmit helper function, clear up all the ack
4844 * flags and take care of important book keeping.
4845 */
4846 static void
bbr_remxt_tmr(struct tcpcb * tp)4847 bbr_remxt_tmr(struct tcpcb *tp)
4848 {
4849 /*
4850 * The retransmit timer went off, all sack'd blocks must be
4851 * un-acked.
4852 */
4853 struct bbr_sendmap *rsm, *trsm = NULL;
4854 struct tcp_bbr *bbr;
4855 uint32_t cts, lost;
4856
4857 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
4858 cts = tcp_get_usecs(&bbr->rc_tv);
4859 lost = bbr->r_ctl.rc_lost;
4860 if (bbr->r_state && (bbr->r_state != tp->t_state))
4861 bbr_set_state(tp, bbr, 0);
4862
4863 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
4864 if (rsm->r_flags & BBR_ACKED) {
4865 uint32_t old_flags;
4866
4867 rsm->r_dupack = 0;
4868 if (rsm->r_in_tmap == 0) {
4869 /* We must re-add it back to the tlist */
4870 if (trsm == NULL) {
4871 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
4872 } else {
4873 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);
4874 }
4875 rsm->r_in_tmap = 1;
4876 }
4877 old_flags = rsm->r_flags;
4878 rsm->r_flags |= BBR_RXT_CLEARED;
4879 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);
4880 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
4881 } else {
4882 if ((tp->t_state < TCPS_ESTABLISHED) &&
4883 (rsm->r_start == tp->snd_una)) {
4884 /*
4885 * Special case for TCP FO. Where
4886 * we sent more data beyond the snd_max.
4887 * We don't mark that as lost and stop here.
4888 */
4889 break;
4890 }
4891 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4892 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4893 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4894 }
4895 if (bbr_marks_rxt_sack_passed) {
4896 /*
4897 * With this option, we will rack out
4898 * in 1ms increments the rest of the packets.
4899 */
4900 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;
4901 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4902 } else {
4903 /*
4904 * With this option we only mark them lost
4905 * and remove all sack'd markings. We will run
4906 * another RXT or a TLP. This will cause
4907 * us to eventually send more based on what
4908 * ack's come in.
4909 */
4910 rsm->r_flags |= BBR_MARKED_LOST;
4911 rsm->r_flags &= ~BBR_WAS_SACKPASS;
4912 rsm->r_flags &= ~BBR_SACK_PASSED;
4913 }
4914 }
4915 trsm = rsm;
4916 }
4917 bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4918 /* Clear the count (we just un-acked them) */
4919 bbr_log_to_event(bbr, cts, BBR_TO_FRM_TMR);
4920 bbr->rc_tlp_new_data = 0;
4921 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4922 /* zap the behindness on a rxt */
4923 bbr->r_ctl.rc_hptsi_agg_delay = 0;
4924 bbr->r_agg_early_set = 0;
4925 bbr->r_ctl.rc_agg_early = 0;
4926 bbr->rc_tlp_rtx_out = 0;
4927 bbr->r_ctl.rc_sacked = 0;
4928 bbr->r_ctl.rc_sacklast = NULL;
4929 bbr->r_timer_override = 1;
4930 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4931 }
4932
4933 /*
4934 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
4935 * we will setup to retransmit the lowest seq number outstanding.
4936 */
4937 static int
bbr_timeout_rxt(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)4938 bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4939 {
4940 struct inpcb *inp = tptoinpcb(tp);
4941 int32_t rexmt;
4942 int32_t retval = 0;
4943 bool isipv6;
4944
4945 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
4946 if (bbr->rc_all_timers_stopped) {
4947 return (1);
4948 }
4949 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
4950 (tp->snd_una == tp->snd_max)) {
4951 /* Nothing outstanding .. nothing to do */
4952 return (0);
4953 }
4954 /*
4955 * Retransmission timer went off. Message has not been acked within
4956 * retransmit interval. Back off to a longer retransmit interval
4957 * and retransmit one segment.
4958 */
4959 if (ctf_progress_timeout_check(tp, true)) {
4960 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
4961 return (-ETIMEDOUT); /* tcp_drop() */
4962 }
4963 bbr_remxt_tmr(tp);
4964 if ((bbr->r_ctl.rc_resend == NULL) ||
4965 ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {
4966 /*
4967 * If the rwnd collapsed on
4968 * the one we are retransmitting
4969 * it does not count against the
4970 * rxt count.
4971 */
4972 tp->t_rxtshift++;
4973 }
4974 if (tp->t_rxtshift > V_tcp_retries) {
4975 tp->t_rxtshift = V_tcp_retries;
4976 KMOD_TCPSTAT_INC(tcps_timeoutdrop);
4977 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN);
4978 /* XXXGL: previously t_softerror was casted to uint16_t */
4979 MPASS(tp->t_softerror >= 0);
4980 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;
4981 return (retval); /* tcp_drop() */
4982 }
4983 if (tp->t_state == TCPS_SYN_SENT) {
4984 /*
4985 * If the SYN was retransmitted, indicate CWND to be limited
4986 * to 1 segment in cc_conn_init().
4987 */
4988 tp->snd_cwnd = 1;
4989 } else if (tp->t_rxtshift == 1) {
4990 /*
4991 * first retransmit; record ssthresh and cwnd so they can be
4992 * recovered if this turns out to be a "bad" retransmit. A
4993 * retransmit is considered "bad" if an ACK for this segment
4994 * is received within RTT/2 interval; the assumption here is
4995 * that the ACK was already in flight. See "On Estimating
4996 * End-to-End Network Path Properties" by Allman and Paxson
4997 * for more details.
4998 */
4999 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5000 if (!IN_RECOVERY(tp->t_flags)) {
5001 tp->snd_cwnd_prev = tp->snd_cwnd;
5002 tp->snd_ssthresh_prev = tp->snd_ssthresh;
5003 tp->snd_recover_prev = tp->snd_recover;
5004 tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
5005 tp->t_flags |= TF_PREVVALID;
5006 } else {
5007 tp->t_flags &= ~TF_PREVVALID;
5008 }
5009 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5010 } else {
5011 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5012 tp->t_flags &= ~TF_PREVVALID;
5013 }
5014 KMOD_TCPSTAT_INC(tcps_rexmttimeo);
5015 if ((tp->t_state == TCPS_SYN_SENT) ||
5016 (tp->t_state == TCPS_SYN_RECEIVED))
5017 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift];
5018 else
5019 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
5020 TCPT_RANGESET(tp->t_rxtcur, rexmt,
5021 MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),
5022 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
5023 /*
5024 * We enter the path for PLMTUD if connection is established or, if
5025 * connection is FIN_WAIT_1 status, reason for the last is that if
5026 * amount of data we send is very small, we could send it in couple
5027 * of packets and process straight to FIN. In that case we won't
5028 * catch ESTABLISHED state.
5029 */
5030 #ifdef INET6
5031 isipv6 = (inp->inp_vflag & INP_IPV6) ? true : false;
5032 #else
5033 isipv6 = false;
5034 #endif
5035 if (((V_tcp_pmtud_blackhole_detect == 1) ||
5036 (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) ||
5037 (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) &&
5038 ((tp->t_state == TCPS_ESTABLISHED) ||
5039 (tp->t_state == TCPS_FIN_WAIT_1))) {
5040 /*
5041 * Idea here is that at each stage of mtu probe (usually,
5042 * 1448 -> 1188 -> 524) should be given 2 chances to recover
5043 * before further clamping down. 'tp->t_rxtshift % 2 == 0'
5044 * should take care of that.
5045 */
5046 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
5047 (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) &&
5048 (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
5049 tp->t_rxtshift % 2 == 0)) {
5050 /*
5051 * Enter Path MTU Black-hole Detection mechanism: -
5052 * Disable Path MTU Discovery (IP "DF" bit). -
5053 * Reduce MTU to lower value than what we negotiated
5054 * with peer.
5055 */
5056 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
5057 /*
5058 * Record that we may have found a black
5059 * hole.
5060 */
5061 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
5062 /* Keep track of previous MSS. */
5063 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
5064 }
5065 /*
5066 * Reduce the MSS to blackhole value or to the
5067 * default in an attempt to retransmit.
5068 */
5069 #ifdef INET6
5070 isipv6 = bbr->r_is_v6;
5071 if (isipv6 &&
5072 tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
5073 /* Use the sysctl tuneable blackhole MSS. */
5074 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
5075 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
5076 } else if (isipv6) {
5077 /* Use the default MSS. */
5078 tp->t_maxseg = V_tcp_v6mssdflt;
5079 /*
5080 * Disable Path MTU Discovery when we switch
5081 * to minmss.
5082 */
5083 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5084 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
5085 }
5086 #endif
5087 #if defined(INET6) && defined(INET)
5088 else
5089 #endif
5090 #ifdef INET
5091 if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
5092 /* Use the sysctl tuneable blackhole MSS. */
5093 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
5094 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
5095 } else {
5096 /* Use the default MSS. */
5097 tp->t_maxseg = V_tcp_mssdflt;
5098 /*
5099 * Disable Path MTU Discovery when we switch
5100 * to minmss.
5101 */
5102 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5103 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
5104 }
5105 #endif
5106 } else {
5107 /*
5108 * If further retransmissions are still unsuccessful
5109 * with a lowered MTU, maybe this isn't a blackhole
5110 * and we restore the previous MSS and blackhole
5111 * detection flags. The limit '6' is determined by
5112 * giving each probe stage (1448, 1188, 524) 2
5113 * chances to recover.
5114 */
5115 if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
5116 (tp->t_rxtshift >= 6)) {
5117 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
5118 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
5119 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
5120 if (tp->t_maxseg < V_tcp_mssdflt) {
5121 /*
5122 * The MSS is so small we should not
5123 * process incoming SACK's since we are
5124 * subject to attack in such a case.
5125 */
5126 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
5127 } else {
5128 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
5129 }
5130 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed);
5131 }
5132 }
5133 }
5134 /*
5135 * Disable RFC1323 and SACK if we haven't got any response to our
5136 * third SYN to work-around some broken terminal servers (most of
5137 * which have hopefully been retired) that have bad VJ header
5138 * compression code which trashes TCP segments containing
5139 * unknown-to-them TCP options.
5140 */
5141 if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
5142 (tp->t_rxtshift == 3))
5143 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);
5144 /*
5145 * If we backed off this far, our srtt estimate is probably bogus.
5146 * Clobber it so we'll take the next rtt measurement as our srtt;
5147 * move the current srtt into rttvar to keep the current retransmit
5148 * times until then.
5149 */
5150 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
5151 #ifdef INET6
5152 if (bbr->r_is_v6)
5153 in6_losing(inp);
5154 else
5155 #endif
5156 in_losing(inp);
5157 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
5158 tp->t_srtt = 0;
5159 }
5160 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
5161 tp->snd_recover = tp->snd_max;
5162 tp->t_flags |= TF_ACKNOW;
5163 tp->t_rtttime = 0;
5164
5165 return (retval);
5166 }
5167
5168 static int
bbr_process_timers(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,uint8_t hpts_calling)5169 bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling)
5170 {
5171 int32_t ret = 0;
5172 int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
5173
5174 if (timers == 0) {
5175 return (0);
5176 }
5177 if (tp->t_state == TCPS_LISTEN) {
5178 /* no timers on listen sockets */
5179 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
5180 return (0);
5181 return (1);
5182 }
5183 if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
5184 uint32_t left;
5185
5186 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
5187 ret = -1;
5188 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5189 return (0);
5190 }
5191 if (hpts_calling == 0) {
5192 ret = -2;
5193 bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5194 return (0);
5195 }
5196 /*
5197 * Ok our timer went off early and we are not paced false
5198 * alarm, go back to sleep.
5199 */
5200 left = bbr->r_ctl.rc_timer_exp - cts;
5201 ret = -3;
5202 bbr_log_to_processing(bbr, cts, ret, left, hpts_calling);
5203 tcp_hpts_insert(tp, left, NULL);
5204 return (1);
5205 }
5206 bbr->rc_tmr_stopped = 0;
5207 bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
5208 if (timers & PACE_TMR_DELACK) {
5209 ret = bbr_timeout_delack(tp, bbr, cts);
5210 } else if (timers & PACE_TMR_PERSIT) {
5211 ret = bbr_timeout_persist(tp, bbr, cts);
5212 } else if (timers & PACE_TMR_RACK) {
5213 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5214 ret = bbr_timeout_rack(tp, bbr, cts);
5215 } else if (timers & PACE_TMR_TLP) {
5216 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5217 ret = bbr_timeout_tlp(tp, bbr, cts);
5218 } else if (timers & PACE_TMR_RXT) {
5219 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5220 ret = bbr_timeout_rxt(tp, bbr, cts);
5221 } else if (timers & PACE_TMR_KEEP) {
5222 ret = bbr_timeout_keepalive(tp, bbr, cts);
5223 }
5224 bbr_log_to_processing(bbr, cts, ret, timers, hpts_calling);
5225 return (ret);
5226 }
5227
5228 static void
bbr_timer_cancel(struct tcp_bbr * bbr,int32_t line,uint32_t cts)5229 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
5230 {
5231 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
5232 uint8_t hpts_removed = 0;
5233
5234 if (tcp_in_hpts(bbr->rc_tp) &&
5235 (bbr->rc_timer_first == 1)) {
5236 /*
5237 * If we are canceling timer's when we have the
5238 * timer ahead of the output being paced. We also
5239 * must remove ourselves from the hpts.
5240 */
5241 hpts_removed = 1;
5242 tcp_hpts_remove(bbr->rc_tp);
5243 if (bbr->r_ctl.rc_last_delay_val) {
5244 /* Update the last hptsi delay too */
5245 uint32_t time_since_send;
5246
5247 if (TSTMP_GT(cts, bbr->rc_pacer_started))
5248 time_since_send = cts - bbr->rc_pacer_started;
5249 else
5250 time_since_send = 0;
5251 if (bbr->r_ctl.rc_last_delay_val > time_since_send) {
5252 /* Cut down our pacing_delay time */
5253 bbr->r_ctl.rc_last_delay_val -= time_since_send;
5254 } else {
5255 bbr->r_ctl.rc_last_delay_val = 0;
5256 }
5257 bbr->rc_pacer_started = cts;
5258 }
5259 }
5260 bbr->rc_timer_first = 0;
5261 bbr_log_to_cancel(bbr, line, cts, hpts_removed);
5262 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
5263 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
5264 }
5265 }
5266
5267 static int
bbr_stopall(struct tcpcb * tp)5268 bbr_stopall(struct tcpcb *tp)
5269 {
5270 struct tcp_bbr *bbr;
5271
5272 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
5273 bbr->rc_all_timers_stopped = 1;
5274
5275 tcp_hpts_remove(tp);
5276
5277 return (0);
5278 }
5279
5280 static uint32_t
bbr_get_earliest_send_outstanding(struct tcp_bbr * bbr,struct bbr_sendmap * u_rsm,uint32_t cts)5281 bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts)
5282 {
5283 struct bbr_sendmap *rsm;
5284
5285 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
5286 if ((rsm == NULL) || (u_rsm == rsm))
5287 return (cts);
5288 return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
5289 }
5290
5291 static void
bbr_update_rsm(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t cts,uint32_t pacing_time)5292 bbr_update_rsm(struct tcpcb *tp, struct tcp_bbr *bbr,
5293 struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time)
5294 {
5295 int32_t idx;
5296
5297 rsm->r_rtr_cnt++;
5298 rsm->r_dupack = 0;
5299 if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {
5300 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;
5301 rsm->r_flags |= BBR_OVERMAX;
5302 }
5303 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
5304 /* Take off the collapsed flag at rxt */
5305 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
5306 }
5307 if (rsm->r_flags & BBR_MARKED_LOST) {
5308 /* We have retransmitted, its no longer lost */
5309 rsm->r_flags &= ~BBR_MARKED_LOST;
5310 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
5311 }
5312 if (rsm->r_flags & BBR_RXT_CLEARED) {
5313 /*
5314 * We hit a RXT timer on it and
5315 * we cleared the "acked" flag.
5316 * We now have it going back into
5317 * flight, we can remove the cleared
5318 * flag and possibly do accounting on
5319 * this piece.
5320 */
5321 rsm->r_flags &= ~BBR_RXT_CLEARED;
5322 }
5323 if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {
5324 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
5325 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
5326 }
5327 idx = rsm->r_rtr_cnt - 1;
5328 rsm->r_tim_lastsent[idx] = cts;
5329 rsm->r_pacing_delay = pacing_time;
5330 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5331 rsm->r_ts_valid = bbr->rc_ts_valid;
5332 if (bbr->rc_ts_valid)
5333 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5334 if (bbr->r_ctl.r_app_limited_until)
5335 rsm->r_app_limited = 1;
5336 else
5337 rsm->r_app_limited = 0;
5338 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5339 rsm->r_bbr_state = bbr_state_val(bbr);
5340 else
5341 rsm->r_bbr_state = 8;
5342 if (rsm->r_flags & BBR_ACKED) {
5343 /* Problably MTU discovery messing with us */
5344 uint32_t old_flags;
5345
5346 old_flags = rsm->r_flags;
5347 rsm->r_flags &= ~BBR_ACKED;
5348 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
5349 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
5350 if (bbr->r_ctl.rc_sacked == 0)
5351 bbr->r_ctl.rc_sacklast = NULL;
5352 }
5353 if (rsm->r_in_tmap) {
5354 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5355 }
5356 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5357 rsm->r_in_tmap = 1;
5358 if (rsm->r_flags & BBR_SACK_PASSED) {
5359 /* We have retransmitted due to the SACK pass */
5360 rsm->r_flags &= ~BBR_SACK_PASSED;
5361 rsm->r_flags |= BBR_WAS_SACKPASS;
5362 }
5363 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5364 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5365 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5366 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
5367 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5368 rsm->r_is_gain = 1;
5369 rsm->r_is_drain = 0;
5370 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5371 rsm->r_is_drain = 1;
5372 rsm->r_is_gain = 0;
5373 } else {
5374 rsm->r_is_drain = 0;
5375 rsm->r_is_gain = 0;
5376 }
5377 rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */
5378 }
5379
5380 /*
5381 * Returns 0, or the sequence where we stopped
5382 * updating. We also update the lenp to be the amount
5383 * of data left.
5384 */
5385
5386 static uint32_t
bbr_update_entry(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t cts,int32_t * lenp,uint32_t pacing_time)5387 bbr_update_entry(struct tcpcb *tp, struct tcp_bbr *bbr,
5388 struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time)
5389 {
5390 /*
5391 * We (re-)transmitted starting at rsm->r_start for some length
5392 * (possibly less than r_end.
5393 */
5394 struct bbr_sendmap *nrsm;
5395 uint32_t c_end;
5396 int32_t len;
5397
5398 len = *lenp;
5399 c_end = rsm->r_start + len;
5400 if (SEQ_GEQ(c_end, rsm->r_end)) {
5401 /*
5402 * We retransmitted the whole piece or more than the whole
5403 * slopping into the next rsm.
5404 */
5405 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5406 if (c_end == rsm->r_end) {
5407 *lenp = 0;
5408 return (0);
5409 } else {
5410 int32_t act_len;
5411
5412 /* Hangs over the end return whats left */
5413 act_len = rsm->r_end - rsm->r_start;
5414 *lenp = (len - act_len);
5415 return (rsm->r_end);
5416 }
5417 /* We don't get out of this block. */
5418 }
5419 /*
5420 * Here we retransmitted less than the whole thing which means we
5421 * have to split this into what was transmitted and what was not.
5422 */
5423 nrsm = bbr_alloc_full_limit(bbr);
5424 if (nrsm == NULL) {
5425 *lenp = 0;
5426 return (0);
5427 }
5428 /*
5429 * So here we are going to take the original rsm and make it what we
5430 * retransmitted. nrsm will be the tail portion we did not
5431 * retransmit. For example say the chunk was 1, 11 (10 bytes). And
5432 * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to
5433 * 1, 6 and the new piece will be 6, 11.
5434 */
5435 bbr_clone_rsm(bbr, nrsm, rsm, c_end);
5436 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
5437 nrsm->r_dupack = 0;
5438 if (rsm->r_in_tmap) {
5439 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
5440 nrsm->r_in_tmap = 1;
5441 }
5442 rsm->r_flags &= (~BBR_HAS_FIN);
5443 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5444 *lenp = 0;
5445 return (0);
5446 }
5447
5448 static uint64_t
bbr_get_hardware_rate(struct tcp_bbr * bbr)5449 bbr_get_hardware_rate(struct tcp_bbr *bbr)
5450 {
5451 uint64_t bw;
5452
5453 bw = bbr_get_bw(bbr);
5454 bw *= (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN];
5455 bw /= (uint64_t)BBR_UNIT;
5456 return(bw);
5457 }
5458
5459 static void
bbr_setup_less_of_rate(struct tcp_bbr * bbr,uint32_t cts,uint64_t act_rate,uint64_t rate_wanted)5460 bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts,
5461 uint64_t act_rate, uint64_t rate_wanted)
5462 {
5463 /*
5464 * We could not get a full gains worth
5465 * of rate.
5466 */
5467 if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {
5468 /* we can't even get the real rate */
5469 uint64_t red;
5470
5471 bbr->skip_gain = 1;
5472 bbr->gain_is_limited = 0;
5473 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;
5474 if (red)
5475 filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);
5476 } else {
5477 /* We can use a lower gain */
5478 bbr->skip_gain = 0;
5479 bbr->gain_is_limited = 1;
5480 }
5481 }
5482
5483 static void
bbr_update_hardware_pacing_rate(struct tcp_bbr * bbr,uint32_t cts)5484 bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts)
5485 {
5486 const struct tcp_hwrate_limit_table *nrte;
5487 int error, rate = -1;
5488
5489 if (bbr->r_ctl.crte == NULL)
5490 return;
5491 if ((bbr->rc_inp->inp_route.ro_nh == NULL) ||
5492 (bbr->rc_inp->inp_route.ro_nh->nh_ifp == NULL)) {
5493 /* Lost our routes? */
5494 /* Clear the way for a re-attempt */
5495 bbr->bbr_attempt_hdwr_pace = 0;
5496 lost_rate:
5497 bbr->gain_is_limited = 0;
5498 bbr->skip_gain = 0;
5499 bbr->bbr_hdrw_pacing = 0;
5500 counter_u64_add(bbr_flows_whdwr_pacing, -1);
5501 counter_u64_add(bbr_flows_nohdwr_pacing, 1);
5502 tcp_bbr_tso_size_check(bbr, cts);
5503 return;
5504 }
5505 rate = bbr_get_hardware_rate(bbr);
5506 nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,
5507 bbr->rc_tp,
5508 bbr->rc_inp->inp_route.ro_nh->nh_ifp,
5509 rate,
5510 (RS_PACING_GEQ|RS_PACING_SUB_OK),
5511 &error, NULL);
5512 if (nrte == NULL) {
5513 goto lost_rate;
5514 }
5515 if (nrte != bbr->r_ctl.crte) {
5516 bbr->r_ctl.crte = nrte;
5517 if (error == 0) {
5518 BBR_STAT_INC(bbr_hdwr_rl_mod_ok);
5519 if (bbr->r_ctl.crte->rate < rate) {
5520 /* We have a problem */
5521 bbr_setup_less_of_rate(bbr, cts,
5522 bbr->r_ctl.crte->rate, rate);
5523 } else {
5524 /* We are good */
5525 bbr->gain_is_limited = 0;
5526 bbr->skip_gain = 0;
5527 }
5528 } else {
5529 /* A failure should release the tag */
5530 BBR_STAT_INC(bbr_hdwr_rl_mod_fail);
5531 bbr->gain_is_limited = 0;
5532 bbr->skip_gain = 0;
5533 bbr->bbr_hdrw_pacing = 0;
5534 }
5535 bbr_type_log_hdwr_pacing(bbr,
5536 bbr->r_ctl.crte->ptbl->rs_ifp,
5537 rate,
5538 bbr->r_ctl.crte->rate,
5539 __LINE__,
5540 cts,
5541 error);
5542 }
5543 }
5544
5545 static void
bbr_adjust_for_hw_pacing(struct tcp_bbr * bbr,uint32_t cts)5546 bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts)
5547 {
5548 /*
5549 * If we have hardware pacing support
5550 * we need to factor that in for our
5551 * TSO size.
5552 */
5553 const struct tcp_hwrate_limit_table *rlp;
5554 uint32_t cur_delay, seg_sz, maxseg, new_tso, delta, hdwr_delay;
5555
5556 if ((bbr->bbr_hdrw_pacing == 0) ||
5557 (IN_RECOVERY(bbr->rc_tp->t_flags)) ||
5558 (bbr->r_ctl.crte == NULL))
5559 return;
5560 if (bbr->hw_pacing_set == 0) {
5561 /* Not yet by the hdwr pacing count delay */
5562 return;
5563 }
5564 if (bbr_hdwr_pace_adjust == 0) {
5565 /* No adjustment */
5566 return;
5567 }
5568 rlp = bbr->r_ctl.crte;
5569 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)
5570 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5571 else
5572 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5573 /*
5574 * So lets first get the
5575 * time we will take between
5576 * TSO sized sends currently without
5577 * hardware help.
5578 */
5579 cur_delay = bbr_get_pacing_delay(bbr, BBR_UNIT,
5580 bbr->r_ctl.rc_pace_max_segs, cts, 1);
5581 hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;
5582 hdwr_delay *= rlp->time_between;
5583 if (cur_delay > hdwr_delay)
5584 delta = cur_delay - hdwr_delay;
5585 else
5586 delta = 0;
5587 bbr_log_type_tsosize(bbr, cts, delta, cur_delay, hdwr_delay,
5588 (bbr->r_ctl.rc_pace_max_segs / maxseg),
5589 1);
5590 if (delta &&
5591 (delta < (max(rlp->time_between,
5592 bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {
5593 /*
5594 * Now lets divide by the pacing
5595 * time between each segment the
5596 * hardware sends rounding up and
5597 * derive a bytes from that. We multiply
5598 * that by bbr_hdwr_pace_adjust to get
5599 * more bang for our buck.
5600 *
5601 * The goal is to have the software pacer
5602 * waiting no more than an additional
5603 * pacing delay if we can (without the
5604 * compensation i.e. x bbr_hdwr_pace_adjust).
5605 */
5606 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between),
5607 (bbr->r_ctl.rc_pace_max_segs/maxseg));
5608 seg_sz *= bbr_hdwr_pace_adjust;
5609 if (bbr_hdwr_pace_floor &&
5610 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5611 /* Currently hardware paces
5612 * out rs_min_seg segments at a time.
5613 * We need to make sure we always send at least
5614 * a full burst of bbr_hdwr_pace_floor down.
5615 */
5616 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5617 }
5618 seg_sz *= maxseg;
5619 } else if (delta == 0) {
5620 /*
5621 * The highest pacing rate is
5622 * above our b/w gained. This means
5623 * we probably are going quite fast at
5624 * the hardware highest rate. Lets just multiply
5625 * the calculated TSO size by the
5626 * multiplier factor (its probably
5627 * 4 segments in the default config for
5628 * mlx).
5629 */
5630 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;
5631 if (bbr_hdwr_pace_floor &&
5632 (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5633 /* Currently hardware paces
5634 * out rs_min_seg segments at a time.
5635 * We need to make sure we always send at least
5636 * a full burst of bbr_hdwr_pace_floor down.
5637 */
5638 seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5639 }
5640 } else {
5641 /*
5642 * The pacing time difference is so
5643 * big that the hardware will
5644 * pace out more rapidly then we
5645 * really want and then we
5646 * will have a long delay. Lets just keep
5647 * the same TSO size so its as if
5648 * we were not using hdwr pacing (we
5649 * just gain a bit of spacing from the
5650 * hardware if seg_sz > 1).
5651 */
5652 seg_sz = bbr->r_ctl.rc_pace_max_segs;
5653 }
5654 if (seg_sz > bbr->r_ctl.rc_pace_max_segs)
5655 new_tso = seg_sz;
5656 else
5657 new_tso = bbr->r_ctl.rc_pace_max_segs;
5658 if (new_tso >= (PACE_MAX_IP_BYTES-maxseg))
5659 new_tso = PACE_MAX_IP_BYTES - maxseg;
5660
5661 if (new_tso != bbr->r_ctl.rc_pace_max_segs) {
5662 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);
5663 bbr->r_ctl.rc_pace_max_segs = new_tso;
5664 }
5665 }
5666
5667 static void
tcp_bbr_tso_size_check(struct tcp_bbr * bbr,uint32_t cts)5668 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts)
5669 {
5670 uint64_t bw;
5671 uint32_t old_tso = 0, new_tso;
5672 uint32_t maxseg, bytes;
5673 uint32_t tls_seg=0;
5674 /*
5675 * Google/linux uses the following algorithm to determine
5676 * the TSO size based on the b/w of the link (from Neal Cardwell email 9/27/18):
5677 *
5678 * bytes = bw_in_bytes_per_second / 1000
5679 * bytes = min(bytes, 64k)
5680 * tso_segs = bytes / MSS
5681 * if (bw < 1.2Mbs)
5682 * min_tso_segs = 1
5683 * else
5684 * min_tso_segs = 2
5685 * tso_segs = max(tso_segs, min_tso_segs)
5686 *
5687 * * Note apply a device specific limit (we apply this in the
5688 * tcp_m_copym).
5689 * Note that before the initial measurement is made google bursts out
5690 * a full iwnd just like new-reno/cubic.
5691 *
5692 * We do not use this algorithm. Instead we
5693 * use a two phased approach:
5694 *
5695 * if ( bw <= per-tcb-cross-over)
5696 * goal_tso = calculate how much with this bw we
5697 * can send in goal-time seconds.
5698 * if (goal_tso > mss)
5699 * seg = goal_tso / mss
5700 * tso = seg * mss
5701 * else
5702 * tso = mss
5703 * if (tso > per-tcb-max)
5704 * tso = per-tcb-max
5705 * else if ( bw > 512Mbps)
5706 * tso = max-tso (64k/mss)
5707 * else
5708 * goal_tso = bw / per-tcb-divsor
5709 * seg = (goal_tso + mss-1)/mss
5710 * tso = seg * mss
5711 *
5712 * if (tso < per-tcb-floor)
5713 * tso = per-tcb-floor
5714 * if (tso > per-tcb-utter_max)
5715 * tso = per-tcb-utter_max
5716 *
5717 * Note the default per-tcb-divisor is 1000 (same as google).
5718 * the goal cross over is 30Mbps however. To recreate googles
5719 * algorithm you need to set:
5720 *
5721 * cross-over = 23,168,000 bps
5722 * goal-time = 18000
5723 * per-tcb-max = 2
5724 * per-tcb-divisor = 1000
5725 * per-tcb-floor = 1
5726 *
5727 * This will get you "google bbr" behavior with respect to tso size.
5728 *
5729 * Note we do set anything TSO size until we are past the initial
5730 * window. Before that we gnerally use either a single MSS
5731 * or we use the full IW size (so we burst a IW at a time)
5732 */
5733
5734 if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {
5735 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5736 } else {
5737 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5738 }
5739 old_tso = bbr->r_ctl.rc_pace_max_segs;
5740 if (bbr->rc_past_init_win == 0) {
5741 /*
5742 * Not enough data has been acknowledged to make a
5743 * judgement. Set up the initial TSO based on if we
5744 * are sending a full IW at once or not.
5745 */
5746 if (bbr->rc_use_google)
5747 bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);
5748 else if (bbr->bbr_init_win_cheat)
5749 bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);
5750 else
5751 bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5752 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)
5753 bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;
5754 if (bbr->r_ctl.rc_pace_max_segs == 0) {
5755 bbr->r_ctl.rc_pace_max_segs = maxseg;
5756 }
5757 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);
5758 bbr_adjust_for_hw_pacing(bbr, cts);
5759 return;
5760 }
5761 /**
5762 * Now lets set the TSO goal based on our delivery rate in
5763 * bytes per second. Note we only do this if
5764 * we have acked at least the initial cwnd worth of data.
5765 */
5766 bw = bbr_get_bw(bbr);
5767 if (IN_RECOVERY(bbr->rc_tp->t_flags) &&
5768 (bbr->rc_use_google == 0)) {
5769 /* We clamp to one MSS in recovery */
5770 new_tso = maxseg;
5771 } else if (bbr->rc_use_google) {
5772 int min_tso_segs;
5773
5774 /* Google considers the gain too */
5775 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {
5776 bw *= bbr->r_ctl.rc_bbr_hptsi_gain;
5777 bw /= BBR_UNIT;
5778 }
5779 bytes = bw / 1024;
5780 if (bytes > (64 * 1024))
5781 bytes = 64 * 1024;
5782 new_tso = bytes / maxseg;
5783 if (bw < ONE_POINT_TWO_MEG)
5784 min_tso_segs = 1;
5785 else
5786 min_tso_segs = 2;
5787 if (new_tso < min_tso_segs)
5788 new_tso = min_tso_segs;
5789 new_tso *= maxseg;
5790 } else if (bbr->rc_no_pacing) {
5791 new_tso = (PACE_MAX_IP_BYTES / maxseg) * maxseg;
5792 } else if (bw <= bbr->r_ctl.bbr_cross_over) {
5793 /*
5794 * Calculate the worse case b/w TSO if we are inserting no
5795 * more than a delay_target number of TSO's.
5796 */
5797 uint32_t tso_len, min_tso;
5798
5799 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);
5800 if (tso_len > maxseg) {
5801 new_tso = tso_len / maxseg;
5802 if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)
5803 new_tso = bbr->r_ctl.bbr_hptsi_segments_max;
5804 new_tso *= maxseg;
5805 } else {
5806 /*
5807 * less than a full sized frame yikes.. long rtt or
5808 * low bw?
5809 */
5810 min_tso = bbr_minseg(bbr);
5811 if ((tso_len > min_tso) && (bbr_all_get_min == 0))
5812 new_tso = rounddown(tso_len, min_tso);
5813 else
5814 new_tso = min_tso;
5815 }
5816 } else if (bw > FIVETWELVE_MBPS) {
5817 /*
5818 * This guy is so fast b/w wise that we can TSO as large as
5819 * possible of segments that the NIC will allow.
5820 */
5821 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg);
5822 } else {
5823 /*
5824 * This formula is based on attempting to send a segment or
5825 * more every bbr_hptsi_per_second. The default is 1000
5826 * which means you are targeting what you can send every 1ms
5827 * based on the peers bw.
5828 *
5829 * If the number drops to say 500, then you are looking more
5830 * at 2ms and you will raise how much we send in a single
5831 * TSO thus saving CPU (less bbr_output_wtime() calls). The
5832 * trade off of course is you will send more at once and
5833 * thus tend to clump up the sends into larger "bursts"
5834 * building a queue.
5835 */
5836 bw /= bbr->r_ctl.bbr_hptsi_per_second;
5837 new_tso = roundup(bw, (uint64_t)maxseg);
5838 /*
5839 * Gate the floor to match what our lower than 48Mbps
5840 * algorithm does. The ceiling (bbr_hptsi_segments_max) thus
5841 * becomes the floor for this calculation.
5842 */
5843 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))
5844 new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);
5845 }
5846 if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor)))
5847 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;
5848 if (new_tso > PACE_MAX_IP_BYTES)
5849 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg);
5850 /* Enforce an utter maximum. */
5851 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {
5852 new_tso = bbr->r_ctl.bbr_utter_max * maxseg;
5853 }
5854 if (old_tso != new_tso) {
5855 /* Only log changes */
5856 bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0);
5857 bbr->r_ctl.rc_pace_max_segs = new_tso;
5858 }
5859 /* We have hardware pacing! */
5860 bbr_adjust_for_hw_pacing(bbr, cts);
5861 }
5862
5863 static void
bbr_log_output(struct tcp_bbr * bbr,struct tcpcb * tp,struct tcpopt * to,int32_t len,uint32_t seq_out,uint16_t th_flags,int32_t err,uint32_t cts,struct mbuf * mb,int32_t * abandon,struct bbr_sendmap * hintrsm,uint32_t delay_calc,struct sockbuf * sb)5864 bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len,
5865 uint32_t seq_out, uint16_t th_flags, int32_t err, uint32_t cts,
5866 struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc,
5867 struct sockbuf *sb)
5868 {
5869
5870 struct bbr_sendmap *rsm, *nrsm;
5871 register uint32_t snd_max, snd_una;
5872 uint32_t pacing_time;
5873 /*
5874 * Add to the RACK log of packets in flight or retransmitted. If
5875 * there is a TS option we will use the TS echoed, if not we will
5876 * grab a TS.
5877 *
5878 * Retransmissions will increment the count and move the ts to its
5879 * proper place. Note that if options do not include TS's then we
5880 * won't be able to effectively use the ACK for an RTT on a retran.
5881 *
5882 * Notes about r_start and r_end. Lets consider a send starting at
5883 * sequence 1 for 10 bytes. In such an example the r_start would be
5884 * 1 (starting sequence) but the r_end would be r_start+len i.e. 11.
5885 * This means that r_end is actually the first sequence for the next
5886 * pacing delay (11).
5887 *
5888 */
5889 INP_WLOCK_ASSERT(tptoinpcb(tp));
5890 if (err) {
5891 /*
5892 * We don't log errors -- we could but snd_max does not
5893 * advance in this case either.
5894 */
5895 return;
5896 }
5897 if (th_flags & TH_RST) {
5898 /*
5899 * We don't log resets and we return immediately from
5900 * sending
5901 */
5902 *abandon = 1;
5903 return;
5904 }
5905 snd_una = tp->snd_una;
5906 if (th_flags & (TH_SYN | TH_FIN) && (hintrsm == NULL)) {
5907 /*
5908 * The call to bbr_log_output is made before bumping
5909 * snd_max. This means we can record one extra byte on a SYN
5910 * or FIN if seq_out is adding more on and a FIN is present
5911 * (and we are not resending).
5912 */
5913 if ((th_flags & TH_SYN) && (tp->iss == seq_out))
5914 len++;
5915 if (th_flags & TH_FIN)
5916 len++;
5917 }
5918 if (SEQ_LEQ((seq_out + len), snd_una)) {
5919 /* Are sending an old segment to induce an ack (keep-alive)? */
5920 return;
5921 }
5922 if (SEQ_LT(seq_out, snd_una)) {
5923 /* huh? should we panic? */
5924 uint32_t end;
5925
5926 end = seq_out + len;
5927 seq_out = snd_una;
5928 len = end - seq_out;
5929 }
5930 snd_max = tp->snd_max;
5931 if (len == 0) {
5932 /* We don't log zero window probes */
5933 return;
5934 }
5935 pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);
5936 /* First question is it a retransmission? */
5937 if (seq_out == snd_max) {
5938 again:
5939 rsm = bbr_alloc(bbr);
5940 if (rsm == NULL) {
5941 return;
5942 }
5943 rsm->r_flags = 0;
5944 if (th_flags & TH_SYN)
5945 rsm->r_flags |= BBR_HAS_SYN;
5946 if (th_flags & TH_FIN)
5947 rsm->r_flags |= BBR_HAS_FIN;
5948 rsm->r_tim_lastsent[0] = cts;
5949 rsm->r_rtr_cnt = 1;
5950 rsm->r_rtr_bytes = 0;
5951 rsm->r_start = seq_out;
5952 rsm->r_end = rsm->r_start + len;
5953 rsm->r_dupack = 0;
5954 rsm->r_delivered = bbr->r_ctl.rc_delivered;
5955 rsm->r_pacing_delay = pacing_time;
5956 rsm->r_ts_valid = bbr->rc_ts_valid;
5957 if (bbr->rc_ts_valid)
5958 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5959 rsm->r_del_time = bbr->r_ctl.rc_del_time;
5960 if (bbr->r_ctl.r_app_limited_until)
5961 rsm->r_app_limited = 1;
5962 else
5963 rsm->r_app_limited = 0;
5964 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5965 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5966 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5967 /*
5968 * Here we must also add in this rsm since snd_max
5969 * is updated after we return from a new send.
5970 */
5971 rsm->r_flight_at_send += len;
5972 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
5973 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5974 rsm->r_in_tmap = 1;
5975 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5976 rsm->r_bbr_state = bbr_state_val(bbr);
5977 else
5978 rsm->r_bbr_state = 8;
5979 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5980 rsm->r_is_gain = 1;
5981 rsm->r_is_drain = 0;
5982 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5983 rsm->r_is_drain = 1;
5984 rsm->r_is_gain = 0;
5985 } else {
5986 rsm->r_is_drain = 0;
5987 rsm->r_is_gain = 0;
5988 }
5989 return;
5990 }
5991 /*
5992 * If we reach here its a retransmission and we need to find it.
5993 */
5994 more:
5995 if (hintrsm && (hintrsm->r_start == seq_out)) {
5996 rsm = hintrsm;
5997 hintrsm = NULL;
5998 } else if (bbr->r_ctl.rc_next) {
5999 /* We have a hint from a previous run */
6000 rsm = bbr->r_ctl.rc_next;
6001 } else {
6002 /* No hints sorry */
6003 rsm = NULL;
6004 }
6005 if ((rsm) && (rsm->r_start == seq_out)) {
6006 /*
6007 * We used rc_next or hintrsm to retransmit, hopefully the
6008 * likely case.
6009 */
6010 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6011 if (len == 0) {
6012 return;
6013 } else {
6014 goto more;
6015 }
6016 }
6017 /* Ok it was not the last pointer go through it the hard way. */
6018 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6019 if (rsm->r_start == seq_out) {
6020 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6021 bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
6022 if (len == 0) {
6023 return;
6024 } else {
6025 continue;
6026 }
6027 }
6028 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
6029 /* Transmitted within this piece */
6030 /*
6031 * Ok we must split off the front and then let the
6032 * update do the rest
6033 */
6034 nrsm = bbr_alloc_full_limit(bbr);
6035 if (nrsm == NULL) {
6036 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
6037 return;
6038 }
6039 /*
6040 * copy rsm to nrsm and then trim the front of rsm
6041 * to not include this part.
6042 */
6043 bbr_clone_rsm(bbr, nrsm, rsm, seq_out);
6044 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
6045 if (rsm->r_in_tmap) {
6046 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6047 nrsm->r_in_tmap = 1;
6048 }
6049 rsm->r_flags &= (~BBR_HAS_FIN);
6050 seq_out = bbr_update_entry(tp, bbr, nrsm, cts, &len, pacing_time);
6051 if (len == 0) {
6052 return;
6053 }
6054 }
6055 }
6056 /*
6057 * Hmm not found in map did they retransmit both old and on into the
6058 * new?
6059 */
6060 if (seq_out == tp->snd_max) {
6061 goto again;
6062 } else if (SEQ_LT(seq_out, tp->snd_max)) {
6063 #ifdef BBR_INVARIANTS
6064 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
6065 seq_out, len, tp->snd_una, tp->snd_max);
6066 printf("Starting Dump of all rack entries\n");
6067 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6068 printf("rsm:%p start:%u end:%u\n",
6069 rsm, rsm->r_start, rsm->r_end);
6070 }
6071 printf("Dump complete\n");
6072 panic("seq_out not found rack:%p tp:%p",
6073 bbr, tp);
6074 #endif
6075 } else {
6076 #ifdef BBR_INVARIANTS
6077 /*
6078 * Hmm beyond sndmax? (only if we are using the new rtt-pack
6079 * flag)
6080 */
6081 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p",
6082 seq_out, len, tp->snd_max, tp);
6083 #endif
6084 }
6085 }
6086
6087 static void
bbr_collapse_rtt(struct tcpcb * tp,struct tcp_bbr * bbr,int32_t rtt)6088 bbr_collapse_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, int32_t rtt)
6089 {
6090 /*
6091 * Collapse timeout back the cum-ack moved.
6092 */
6093 tp->t_rxtshift = 0;
6094 tp->t_softerror = 0;
6095 }
6096
6097 static void
tcp_bbr_xmit_timer(struct tcp_bbr * bbr,uint32_t rtt_usecs,uint32_t rsm_send_time,uint32_t r_start,uint32_t tsin)6098 tcp_bbr_xmit_timer(struct tcp_bbr *bbr, uint32_t rtt_usecs, uint32_t rsm_send_time, uint32_t r_start, uint32_t tsin)
6099 {
6100 bbr->rtt_valid = 1;
6101 bbr->r_ctl.cur_rtt = rtt_usecs;
6102 bbr->r_ctl.ts_in = tsin;
6103 if (rsm_send_time)
6104 bbr->r_ctl.cur_rtt_send_time = rsm_send_time;
6105 }
6106
6107 static void
bbr_make_timestamp_determination(struct tcp_bbr * bbr)6108 bbr_make_timestamp_determination(struct tcp_bbr *bbr)
6109 {
6110 /**
6111 * We have in our bbr control:
6112 * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp).
6113 * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).
6114 * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts)
6115 * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time)
6116 *
6117 * Now we can calculate the time between the sends by doing:
6118 *
6119 * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts
6120 *
6121 * And the peer's time between receiving them by doing:
6122 *
6123 * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp
6124 *
6125 * We want to figure out if the timestamp values are in msec, 10msec or usec.
6126 * We also may find that we can't use the timestamps if say we see
6127 * that the peer_delta indicates that though we may have taken 10ms to
6128 * pace out the data, it only saw 1ms between the two packets. This would
6129 * indicate that somewhere on the path is a batching entity that is giving
6130 * out time-slices of the actual b/w. This would mean we could not use
6131 * reliably the peers timestamps.
6132 *
6133 * We expect delta > peer_delta initially. Until we figure out the
6134 * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio.
6135 * If we place 1000 there then its a ms vs our usec. If we place 10000 there
6136 * then its 10ms vs our usec. If the peer is running a usec clock we would
6137 * put a 1 there. If the value is faster then ours, we will disable the
6138 * use of timestamps (though we could revist this later if we find it to be not
6139 * just an isolated one or two flows)).
6140 *
6141 * To detect the batching middle boxes we will come up with our compensation and
6142 * if with it in place, we find the peer is drastically off (by some margin) in
6143 * the smaller direction, then we will assume the worst case and disable use of timestamps.
6144 *
6145 */
6146 uint64_t delta, peer_delta, delta_up;
6147
6148 delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;
6149 if (delta < bbr_min_usec_delta) {
6150 /*
6151 * Have not seen a min amount of time
6152 * between our send times so we can
6153 * make a determination of the timestamp
6154 * yet.
6155 */
6156 return;
6157 }
6158 peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;
6159 if (peer_delta < bbr_min_peer_delta) {
6160 /*
6161 * We may have enough in the form of
6162 * our delta but the peers number
6163 * has not changed that much. It could
6164 * be its clock ratio is such that
6165 * we need more data (10ms tick) or
6166 * there may be other compression scenarios
6167 * going on. In any event we need the
6168 * spread to be larger.
6169 */
6170 return;
6171 }
6172 /* Ok lets first see which way our delta is going */
6173 if (peer_delta > delta) {
6174 /* Very unlikely, the peer without
6175 * compensation shows that it saw
6176 * the two sends arrive further apart
6177 * then we saw then in micro-seconds.
6178 */
6179 if (peer_delta < (delta + ((delta * (uint64_t)1000)/ (uint64_t)bbr_delta_percent))) {
6180 /* well it looks like the peer is a micro-second clock. */
6181 bbr->rc_ts_clock_set = 1;
6182 bbr->r_ctl.bbr_peer_tsratio = 1;
6183 } else {
6184 bbr->rc_ts_cant_be_used = 1;
6185 bbr->rc_ts_clock_set = 1;
6186 }
6187 return;
6188 }
6189 /* Ok we know that the peer_delta is smaller than our send distance */
6190 bbr->rc_ts_clock_set = 1;
6191 /* First question is it within the percentage that they are using usec time? */
6192 delta_up = (peer_delta * 1000) / (uint64_t)bbr_delta_percent;
6193 if ((peer_delta + delta_up) >= delta) {
6194 /* Its a usec clock */
6195 bbr->r_ctl.bbr_peer_tsratio = 1;
6196 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6197 return;
6198 }
6199 /* Ok if not usec, what about 10usec (though unlikely)? */
6200 delta_up = (peer_delta * 1000 * 10) / (uint64_t)bbr_delta_percent;
6201 if (((peer_delta * 10) + delta_up) >= delta) {
6202 bbr->r_ctl.bbr_peer_tsratio = 10;
6203 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6204 return;
6205 }
6206 /* And what about 100usec (though again unlikely)? */
6207 delta_up = (peer_delta * 1000 * 100) / (uint64_t)bbr_delta_percent;
6208 if (((peer_delta * 100) + delta_up) >= delta) {
6209 bbr->r_ctl.bbr_peer_tsratio = 100;
6210 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6211 return;
6212 }
6213 /* And how about 1 msec (the most likely one)? */
6214 delta_up = (peer_delta * 1000 * 1000) / (uint64_t)bbr_delta_percent;
6215 if (((peer_delta * 1000) + delta_up) >= delta) {
6216 bbr->r_ctl.bbr_peer_tsratio = 1000;
6217 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6218 return;
6219 }
6220 /* Ok if not msec could it be 10 msec? */
6221 delta_up = (peer_delta * 1000 * 10000) / (uint64_t)bbr_delta_percent;
6222 if (((peer_delta * 10000) + delta_up) >= delta) {
6223 bbr->r_ctl.bbr_peer_tsratio = 10000;
6224 return;
6225 }
6226 /* If we fall down here the clock tick so slowly we can't use it */
6227 bbr->rc_ts_cant_be_used = 1;
6228 bbr->r_ctl.bbr_peer_tsratio = 0;
6229 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6230 }
6231
6232 /*
6233 * Collect new round-trip time estimate
6234 * and update averages and current timeout.
6235 */
6236 static void
tcp_bbr_xmit_timer_commit(struct tcp_bbr * bbr,struct tcpcb * tp,uint32_t cts)6237 tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts)
6238 {
6239 int32_t delta;
6240 uint32_t rtt, tsin;
6241 int32_t rtt_ticks;
6242
6243 if (bbr->rtt_valid == 0)
6244 /* No valid sample */
6245 return;
6246
6247 rtt = bbr->r_ctl.cur_rtt;
6248 tsin = bbr->r_ctl.ts_in;
6249 if (bbr->rc_prtt_set_ts) {
6250 /*
6251 * We are to force feed the rttProp filter due
6252 * to an entry into PROBE_RTT. This assures
6253 * that the times are sync'd between when we
6254 * go into PROBE_RTT and the filter expiration.
6255 *
6256 * Google does not use a true filter, so they do
6257 * this implicitly since they only keep one value
6258 * and when they enter probe-rtt they update the
6259 * value to the newest rtt.
6260 */
6261 uint32_t rtt_prop;
6262
6263 bbr->rc_prtt_set_ts = 0;
6264 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
6265 if (rtt > rtt_prop)
6266 filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);
6267 else
6268 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6269 }
6270 #ifdef STATS
6271 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rtt));
6272 #endif
6273 if (bbr->rc_ack_was_delayed)
6274 rtt += bbr->r_ctl.rc_ack_hdwr_delay;
6275
6276 if (rtt < bbr->r_ctl.rc_lowest_rtt)
6277 bbr->r_ctl.rc_lowest_rtt = rtt;
6278 bbr_log_rtt_sample(bbr, rtt, tsin);
6279 if (bbr->r_init_rtt) {
6280 /*
6281 * The initial rtt is not-trusted, nuke it and lets get
6282 * our first valid measurement in.
6283 */
6284 bbr->r_init_rtt = 0;
6285 tp->t_srtt = 0;
6286 }
6287 if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {
6288 /*
6289 * So we have not yet figured out
6290 * what the peers TSTMP value is
6291 * in (most likely ms). We need a
6292 * series of cum-ack's to determine
6293 * this reliably.
6294 */
6295 if (bbr->rc_ack_is_cumack) {
6296 if (bbr->rc_ts_data_set) {
6297 /* Lets attempt to determine the timestamp granularity. */
6298 bbr_make_timestamp_determination(bbr);
6299 } else {
6300 bbr->rc_ts_data_set = 1;
6301 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;
6302 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;
6303 }
6304 } else {
6305 /*
6306 * We have to have consecutive acks
6307 * reset any "filled" state to none.
6308 */
6309 bbr->rc_ts_data_set = 0;
6310 }
6311 }
6312 /* Round it up */
6313 rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1)));
6314 if (tp->t_srtt != 0) {
6315 /*
6316 * srtt is stored as fixed point with 5 bits after the
6317 * binary point (i.e., scaled by 8). The following magic is
6318 * equivalent to the smoothing algorithm in rfc793 with an
6319 * alpha of .875 (srtt = rtt/8 + srtt*7/8 in fixed point).
6320 * Adjust rtt to origin 0.
6321 */
6322
6323 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT)
6324 - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));
6325
6326 tp->t_srtt += delta;
6327 if (tp->t_srtt <= 0)
6328 tp->t_srtt = 1;
6329
6330 /*
6331 * We accumulate a smoothed rtt variance (actually, a
6332 * smoothed mean difference), then set the retransmit timer
6333 * to smoothed rtt + 4 times the smoothed variance. rttvar
6334 * is stored as fixed point with 4 bits after the binary
6335 * point (scaled by 16). The following is equivalent to
6336 * rfc793 smoothing with an alpha of .75 (rttvar =
6337 * rttvar*3/4 + |delta| / 4). This replaces rfc793's
6338 * wired-in beta.
6339 */
6340 if (delta < 0)
6341 delta = -delta;
6342 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
6343 tp->t_rttvar += delta;
6344 if (tp->t_rttvar <= 0)
6345 tp->t_rttvar = 1;
6346 } else {
6347 /*
6348 * No rtt measurement yet - use the unsmoothed rtt. Set the
6349 * variance to half the rtt (so our first retransmit happens
6350 * at 3*rtt).
6351 */
6352 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT;
6353 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1);
6354 }
6355 KMOD_TCPSTAT_INC(tcps_rttupdated);
6356 if (tp->t_rttupdated < UCHAR_MAX)
6357 tp->t_rttupdated++;
6358 #ifdef STATS
6359 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));
6360 #endif
6361 /*
6362 * the retransmit should happen at rtt + 4 * rttvar. Because of the
6363 * way we do the smoothing, srtt and rttvar will each average +1/2
6364 * tick of bias. When we compute the retransmit timer, we want 1/2
6365 * tick of rounding and 1 extra tick because of +-1/2 tick
6366 * uncertainty in the firing of the timer. The bias will give us
6367 * exactly the 1.5 tick we need. But, because the bias is
6368 * statistical, we have to test that we don't drop below the minimum
6369 * feasible timer (which is 2 ticks).
6370 */
6371 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
6372 max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),
6373 MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
6374
6375 /*
6376 * We received an ack for a packet that wasn't retransmitted; it is
6377 * probably safe to discard any error indications we've received
6378 * recently. This isn't quite right, but close enough for now (a
6379 * route might have failed after we sent a segment, and the return
6380 * path might not be symmetrical).
6381 */
6382 tp->t_softerror = 0;
6383 rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
6384 if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)
6385 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;
6386 }
6387
6388 static void
bbr_set_reduced_rtt(struct tcp_bbr * bbr,uint32_t cts,uint32_t line)6389 bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line)
6390 {
6391 bbr->r_ctl.rc_rtt_shrinks = cts;
6392 if (bbr_can_force_probertt &&
6393 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
6394 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
6395 /*
6396 * We should enter probe-rtt its been too long
6397 * since we have been there.
6398 */
6399 bbr_enter_probe_rtt(bbr, cts, __LINE__);
6400 } else
6401 bbr_check_probe_rtt_limits(bbr, cts);
6402 }
6403
6404 static void
tcp_bbr_commit_bw(struct tcp_bbr * bbr,uint32_t cts)6405 tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts)
6406 {
6407 uint64_t orig_bw;
6408
6409 if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {
6410 /* We never apply a zero measurement */
6411 bbr_log_type_bbrupd(bbr, 20, cts, 0, 0,
6412 0, 0, 0, 0, 0, 0);
6413 return;
6414 }
6415 if (bbr->r_ctl.r_measurement_count < 0xffffffff)
6416 bbr->r_ctl.r_measurement_count++;
6417 orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
6418 apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);
6419 bbr_log_type_bbrupd(bbr, 21, cts, (uint32_t)orig_bw,
6420 (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),
6421 0, 0, 0, 0, 0, 0);
6422 if (orig_bw &&
6423 (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {
6424 if (bbr->bbr_hdrw_pacing) {
6425 /*
6426 * Apply a new rate to the hardware
6427 * possibly.
6428 */
6429 bbr_update_hardware_pacing_rate(bbr, cts);
6430 }
6431 bbr_set_state_target(bbr, __LINE__);
6432 tcp_bbr_tso_size_check(bbr, cts);
6433 if (bbr->r_recovery_bw) {
6434 bbr_setup_red_bw(bbr, cts);
6435 bbr_log_type_bw_reduce(bbr, BBR_RED_BW_USELRBW);
6436 }
6437 } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))
6438 tcp_bbr_tso_size_check(bbr, cts);
6439 }
6440
6441 static void
bbr_nf_measurement(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t rtt,uint32_t cts)6442 bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6443 {
6444 if (bbr->rc_in_persist == 0) {
6445 /* We log only when not in persist */
6446 /* Translate to a Bytes Per Second */
6447 uint64_t tim, bw, ts_diff, ts_bw;
6448 uint32_t delivered;
6449
6450 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6451 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6452 else
6453 tim = 1;
6454 /*
6455 * Now that we have processed the tim (skipping the sample
6456 * or possibly updating the time, go ahead and
6457 * calculate the cdr.
6458 */
6459 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6460 bw = (uint64_t)delivered;
6461 bw *= (uint64_t)USECS_IN_SECOND;
6462 bw /= tim;
6463 if (bw == 0) {
6464 /* We must have a calculatable amount */
6465 return;
6466 }
6467 /*
6468 * If we are using this b/w shove it in now so we
6469 * can see in the trace viewer if it gets over-ridden.
6470 */
6471 if (rsm->r_ts_valid &&
6472 bbr->rc_ts_valid &&
6473 bbr->rc_ts_clock_set &&
6474 (bbr->rc_ts_cant_be_used == 0) &&
6475 bbr->rc_use_ts_limit) {
6476 ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);
6477 ts_diff *= bbr->r_ctl.bbr_peer_tsratio;
6478 if ((delivered == 0) ||
6479 (rtt < 1000)) {
6480 /* Can't use the ts */
6481 bbr_log_type_bbrupd(bbr, 61, cts,
6482 ts_diff,
6483 bbr->r_ctl.last_inbound_ts,
6484 rsm->r_del_ack_ts, 0,
6485 0, 0, 0, delivered);
6486 } else {
6487 ts_bw = (uint64_t)delivered;
6488 ts_bw *= (uint64_t)USECS_IN_SECOND;
6489 ts_bw /= ts_diff;
6490 bbr_log_type_bbrupd(bbr, 62, cts,
6491 (ts_bw >> 32),
6492 (ts_bw & 0xffffffff), 0, 0,
6493 0, 0, ts_diff, delivered);
6494 if ((bbr->ts_can_raise) &&
6495 (ts_bw > bw)) {
6496 bbr_log_type_bbrupd(bbr, 8, cts,
6497 delivered,
6498 ts_diff,
6499 (bw >> 32),
6500 (bw & 0x00000000ffffffff),
6501 0, 0, 0, 0);
6502 bw = ts_bw;
6503 } else if (ts_bw && (ts_bw < bw)) {
6504 bbr_log_type_bbrupd(bbr, 7, cts,
6505 delivered,
6506 ts_diff,
6507 (bw >> 32),
6508 (bw & 0x00000000ffffffff),
6509 0, 0, 0, 0);
6510 bw = ts_bw;
6511 }
6512 }
6513 }
6514 if (rsm->r_first_sent_time &&
6515 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6516 uint64_t sbw, sti;
6517 /*
6518 * We use what was in flight at the time of our
6519 * send and the size of this send to figure
6520 * out what we have been sending at (amount).
6521 * For the time we take from the time of
6522 * the send of the first send outstanding
6523 * until this send plus this sends pacing
6524 * time. This gives us a good calculation
6525 * as to the rate we have been sending at.
6526 */
6527
6528 sbw = (uint64_t)(rsm->r_flight_at_send);
6529 sbw *= (uint64_t)USECS_IN_SECOND;
6530 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6531 sti += rsm->r_pacing_delay;
6532 sbw /= sti;
6533 if (sbw < bw) {
6534 bbr_log_type_bbrupd(bbr, 6, cts,
6535 delivered,
6536 (uint32_t)sti,
6537 (bw >> 32),
6538 (uint32_t)bw,
6539 rsm->r_first_sent_time, 0, (sbw >> 32),
6540 (uint32_t)sbw);
6541 bw = sbw;
6542 }
6543 }
6544 /* Use the google algorithm for b/w measurements */
6545 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6546 if ((rsm->r_app_limited == 0) ||
6547 (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {
6548 tcp_bbr_commit_bw(bbr, cts);
6549 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6550 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6551 }
6552 }
6553 }
6554
6555 static void
bbr_google_measurement(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t rtt,uint32_t cts)6556 bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6557 {
6558 if (bbr->rc_in_persist == 0) {
6559 /* We log only when not in persist */
6560 /* Translate to a Bytes Per Second */
6561 uint64_t tim, bw;
6562 uint32_t delivered;
6563 int no_apply = 0;
6564
6565 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6566 tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6567 else
6568 tim = 1;
6569 /*
6570 * Now that we have processed the tim (skipping the sample
6571 * or possibly updating the time, go ahead and
6572 * calculate the cdr.
6573 */
6574 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6575 bw = (uint64_t)delivered;
6576 bw *= (uint64_t)USECS_IN_SECOND;
6577 bw /= tim;
6578 if (tim < bbr->r_ctl.rc_lowest_rtt) {
6579 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6580 tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6581
6582 no_apply = 1;
6583 }
6584 /*
6585 * If we are using this b/w shove it in now so we
6586 * can see in the trace viewer if it gets over-ridden.
6587 */
6588 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6589 /* Gate by the sending rate */
6590 if (rsm->r_first_sent_time &&
6591 TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6592 uint64_t sbw, sti;
6593 /*
6594 * We use what was in flight at the time of our
6595 * send and the size of this send to figure
6596 * out what we have been sending at (amount).
6597 * For the time we take from the time of
6598 * the send of the first send outstanding
6599 * until this send plus this sends pacing
6600 * time. This gives us a good calculation
6601 * as to the rate we have been sending at.
6602 */
6603
6604 sbw = (uint64_t)(rsm->r_flight_at_send);
6605 sbw *= (uint64_t)USECS_IN_SECOND;
6606 sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6607 sti += rsm->r_pacing_delay;
6608 sbw /= sti;
6609 if (sbw < bw) {
6610 bbr_log_type_bbrupd(bbr, 6, cts,
6611 delivered,
6612 (uint32_t)sti,
6613 (bw >> 32),
6614 (uint32_t)bw,
6615 rsm->r_first_sent_time, 0, (sbw >> 32),
6616 (uint32_t)sbw);
6617 bw = sbw;
6618 }
6619 if ((sti > tim) &&
6620 (sti < bbr->r_ctl.rc_lowest_rtt)) {
6621 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6622 (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6623 no_apply = 1;
6624 } else
6625 no_apply = 0;
6626 }
6627 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6628 if ((no_apply == 0) &&
6629 ((rsm->r_app_limited == 0) ||
6630 (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {
6631 tcp_bbr_commit_bw(bbr, cts);
6632 bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6633 0, 0, 0, 0, bbr->r_ctl.rc_del_time, rsm->r_del_time);
6634 }
6635 }
6636 }
6637
6638 static void
bbr_update_bbr_info(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,uint32_t rtt,uint32_t cts,uint32_t tsin,uint32_t uts,int32_t match,uint32_t rsm_send_time,int32_t ack_type,struct tcpopt * to)6639 bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin,
6640 uint32_t uts, int32_t match, uint32_t rsm_send_time, int32_t ack_type, struct tcpopt *to)
6641 {
6642 uint64_t old_rttprop;
6643
6644 /* Update our delivery time and amount */
6645 bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);
6646 bbr->r_ctl.rc_del_time = cts;
6647 if (rtt == 0) {
6648 /*
6649 * 0 means its a retransmit, for now we don't use these for
6650 * the rest of BBR.
6651 */
6652 return;
6653 }
6654 if ((bbr->rc_use_google == 0) &&
6655 (match != BBR_RTT_BY_EXACTMATCH) &&
6656 (match != BBR_RTT_BY_TIMESTAMP)){
6657 /*
6658 * We get a lot of rtt updates, lets not pay attention to
6659 * any that are not an exact match. That way we don't have
6660 * to worry about timestamps and the whole nonsense of
6661 * unsure if its a retransmission etc (if we ever had the
6662 * timestamp fixed to always have the last thing sent this
6663 * would not be a issue).
6664 */
6665 return;
6666 }
6667 if ((bbr_no_retran && bbr->rc_use_google) &&
6668 (match != BBR_RTT_BY_EXACTMATCH) &&
6669 (match != BBR_RTT_BY_TIMESTAMP)){
6670 /*
6671 * We only do measurements in google mode
6672 * with bbr_no_retran on for sure things.
6673 */
6674 return;
6675 }
6676 /* Only update srtt if we know by exact match */
6677 tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);
6678 if (ack_type == BBR_CUM_ACKED)
6679 bbr->rc_ack_is_cumack = 1;
6680 else
6681 bbr->rc_ack_is_cumack = 0;
6682 old_rttprop = bbr_get_rtt(bbr, BBR_RTT_PROP);
6683 /*
6684 * Note the following code differs to the original
6685 * BBR spec. It calls for <= not <. However after a
6686 * long discussion in email with Neal, he acknowledged
6687 * that it should be < than so that we will have flows
6688 * going into probe-rtt (we were seeing cases where that
6689 * did not happen and caused ugly things to occur). We
6690 * have added this agreed upon fix to our code base.
6691 */
6692 if (rtt < old_rttprop) {
6693 /* Update when we last saw a rtt drop */
6694 bbr_log_rtt_shrinks(bbr, cts, 0, rtt, __LINE__, BBR_RTTS_NEWRTT, 0);
6695 bbr_set_reduced_rtt(bbr, cts, __LINE__);
6696 }
6697 bbr_log_type_bbrrttprop(bbr, rtt, rsm->r_end, uts, cts,
6698 match, rsm->r_start, rsm->r_flags);
6699 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6700 if (old_rttprop != bbr_get_rtt(bbr, BBR_RTT_PROP)) {
6701 /*
6702 * The RTT-prop moved, reset the target (may be a
6703 * nop for some states).
6704 */
6705 bbr_set_state_target(bbr, __LINE__);
6706 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)
6707 bbr_log_rtt_shrinks(bbr, cts, 0, 0,
6708 __LINE__, BBR_RTTS_NEW_TARGET, 0);
6709 else if (old_rttprop < bbr_get_rtt(bbr, BBR_RTT_PROP))
6710 /* It went up */
6711 bbr_check_probe_rtt_limits(bbr, cts);
6712 }
6713 if ((bbr->rc_use_google == 0) &&
6714 (match == BBR_RTT_BY_TIMESTAMP)) {
6715 /*
6716 * We don't do b/w update with
6717 * these since they are not really
6718 * reliable.
6719 */
6720 return;
6721 }
6722 if (bbr->r_ctl.r_app_limited_until &&
6723 (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {
6724 /* We are no longer app-limited */
6725 bbr->r_ctl.r_app_limited_until = 0;
6726 }
6727 if (bbr->rc_use_google) {
6728 bbr_google_measurement(bbr, rsm, rtt, cts);
6729 } else {
6730 bbr_nf_measurement(bbr, rsm, rtt, cts);
6731 }
6732 }
6733
6734 /*
6735 * Convert a timestamp that the main stack
6736 * uses (milliseconds) into one that bbr uses
6737 * (microseconds). Return that converted timestamp.
6738 */
6739 static uint32_t
bbr_ts_convert(uint32_t cts)6740 bbr_ts_convert(uint32_t cts) {
6741 uint32_t sec, msec;
6742
6743 sec = cts / MS_IN_USEC;
6744 msec = cts - (MS_IN_USEC * sec);
6745 return ((sec * USECS_IN_SECOND) + (msec * MS_IN_USEC));
6746 }
6747
6748 /*
6749 * Return 0 if we did not update the RTT time, return
6750 * 1 if we did.
6751 */
6752 static int
bbr_update_rtt(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,struct tcpopt * to,uint32_t cts,int32_t ack_type,uint32_t th_ack)6753 bbr_update_rtt(struct tcpcb *tp, struct tcp_bbr *bbr,
6754 struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack)
6755 {
6756 int32_t i;
6757 uint32_t t, uts = 0;
6758
6759 if ((rsm->r_flags & BBR_ACKED) ||
6760 (rsm->r_flags & BBR_WAS_RENEGED) ||
6761 (rsm->r_flags & BBR_RXT_CLEARED)) {
6762 /* Already done */
6763 return (0);
6764 }
6765 if (rsm->r_rtt_not_allowed) {
6766 /* Not allowed */
6767 return (0);
6768 }
6769 if (rsm->r_rtr_cnt == 1) {
6770 /*
6771 * Only one transmit. Hopefully the normal case.
6772 */
6773 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))
6774 t = cts - rsm->r_tim_lastsent[0];
6775 else
6776 t = 1;
6777 bbr->r_ctl.rc_last_rtt = t;
6778 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6779 BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);
6780 return (1);
6781 }
6782 /* Convert to usecs */
6783 if ((bbr_can_use_ts_for_rtt == 1) &&
6784 (bbr->rc_use_google == 1) &&
6785 (ack_type == BBR_CUM_ACKED) &&
6786 (to->to_flags & TOF_TS) &&
6787 (to->to_tsecr != 0)) {
6788 t = tcp_tv_to_msec(&bbr->rc_tv) - to->to_tsecr;
6789 if (t < 1)
6790 t = 1;
6791 t *= MS_IN_USEC;
6792 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
6793 BBR_RTT_BY_TIMESTAMP,
6794 rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],
6795 ack_type, to);
6796 return (1);
6797 }
6798 uts = bbr_ts_convert(to->to_tsecr);
6799 if ((to->to_flags & TOF_TS) &&
6800 (to->to_tsecr != 0) &&
6801 (ack_type == BBR_CUM_ACKED) &&
6802 ((rsm->r_flags & BBR_OVERMAX) == 0)) {
6803 /*
6804 * Now which timestamp does it match? In this block the ACK
6805 * may be coming from a previous transmission.
6806 */
6807 uint32_t fudge;
6808
6809 fudge = BBR_TIMER_FUDGE;
6810 for (i = 0; i < rsm->r_rtr_cnt; i++) {
6811 if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&
6812 (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {
6813 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6814 t = cts - rsm->r_tim_lastsent[i];
6815 else
6816 t = 1;
6817 bbr->r_ctl.rc_last_rtt = t;
6818 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,
6819 rsm->r_tim_lastsent[i], ack_type, to);
6820 if ((i + 1) < rsm->r_rtr_cnt) {
6821 /* Likely */
6822 return (0);
6823 } else if (rsm->r_flags & BBR_TLP) {
6824 bbr->rc_tlp_rtx_out = 0;
6825 }
6826 return (1);
6827 }
6828 }
6829 /* Fall through if we can't find a matching timestamp */
6830 }
6831 /*
6832 * Ok its a SACK block that we retransmitted. or a windows
6833 * machine without timestamps. We can tell nothing from the
6834 * time-stamp since its not there or the time the peer last
6835 * received a segment that moved forward its cum-ack point.
6836 *
6837 * Lets look at the last retransmit and see what we can tell
6838 * (with BBR for space we only keep 2 note we have to keep
6839 * at least 2 so the map can not be condensed more).
6840 */
6841 i = rsm->r_rtr_cnt - 1;
6842 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6843 t = cts - rsm->r_tim_lastsent[i];
6844 else
6845 goto not_sure;
6846 if (t < bbr->r_ctl.rc_lowest_rtt) {
6847 /*
6848 * We retransmitted and the ack came back in less
6849 * than the smallest rtt we have observed in the
6850 * windowed rtt. We most likey did an improper
6851 * retransmit as outlined in 4.2 Step 3 point 2 in
6852 * the rack-draft.
6853 *
6854 * Use the prior transmission to update all the
6855 * information as long as there is only one prior
6856 * transmission.
6857 */
6858 if ((rsm->r_flags & BBR_OVERMAX) == 0) {
6859 #ifdef BBR_INVARIANTS
6860 if (rsm->r_rtr_cnt == 1)
6861 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);
6862 #endif
6863 i = rsm->r_rtr_cnt - 2;
6864 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
6865 t = cts - rsm->r_tim_lastsent[i];
6866 else
6867 t = 1;
6868 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,
6869 rsm->r_tim_lastsent[i], ack_type, to);
6870 return (0);
6871 } else {
6872 /*
6873 * Too many prior transmissions, just
6874 * updated BBR delivered
6875 */
6876 not_sure:
6877 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6878 BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to);
6879 }
6880 } else {
6881 /*
6882 * We retransmitted it and the retransmit did the
6883 * job.
6884 */
6885 if (rsm->r_flags & BBR_TLP)
6886 bbr->rc_tlp_rtx_out = 0;
6887 if ((rsm->r_flags & BBR_OVERMAX) == 0)
6888 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,
6889 BBR_RTT_BY_THIS_RETRAN, 0, ack_type, to);
6890 else
6891 bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
6892 BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to);
6893 return (1);
6894 }
6895 return (0);
6896 }
6897
6898 /*
6899 * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
6900 */
6901 static void
bbr_log_sack_passed(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm)6902 bbr_log_sack_passed(struct tcpcb *tp,
6903 struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
6904 {
6905 struct bbr_sendmap *nrsm;
6906
6907 nrsm = rsm;
6908 TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,
6909 bbr_head, r_tnext) {
6910 if (nrsm == rsm) {
6911 /* Skip original segment he is acked */
6912 continue;
6913 }
6914 if (nrsm->r_flags & BBR_ACKED) {
6915 /* Skip ack'd segments */
6916 continue;
6917 }
6918 if (nrsm->r_flags & BBR_SACK_PASSED) {
6919 /*
6920 * We found one that is already marked
6921 * passed, we have been here before and
6922 * so all others below this are marked.
6923 */
6924 break;
6925 }
6926 BBR_STAT_INC(bbr_sack_passed);
6927 nrsm->r_flags |= BBR_SACK_PASSED;
6928 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) &&
6929 bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {
6930 bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;
6931 bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;
6932 nrsm->r_flags |= BBR_MARKED_LOST;
6933 }
6934 nrsm->r_flags &= ~BBR_WAS_SACKPASS;
6935 }
6936 }
6937
6938 /*
6939 * Returns the number of bytes that were
6940 * newly ack'd by sack blocks.
6941 */
6942 static uint32_t
bbr_proc_sack_blk(struct tcpcb * tp,struct tcp_bbr * bbr,struct sackblk * sack,struct tcpopt * to,struct bbr_sendmap ** prsm,uint32_t cts)6943 bbr_proc_sack_blk(struct tcpcb *tp, struct tcp_bbr *bbr, struct sackblk *sack,
6944 struct tcpopt *to, struct bbr_sendmap **prsm, uint32_t cts)
6945 {
6946 int32_t times = 0;
6947 uint32_t start, end, changed = 0;
6948 struct bbr_sendmap *rsm, *nrsm;
6949 int32_t used_ref = 1;
6950 uint8_t went_back = 0, went_fwd = 0;
6951
6952 start = sack->start;
6953 end = sack->end;
6954 rsm = *prsm;
6955 if (rsm == NULL)
6956 used_ref = 0;
6957
6958 /* Do we locate the block behind where we last were? */
6959 if (rsm && SEQ_LT(start, rsm->r_start)) {
6960 went_back = 1;
6961 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
6962 if (SEQ_GEQ(start, rsm->r_start) &&
6963 SEQ_LT(start, rsm->r_end)) {
6964 goto do_rest_ofb;
6965 }
6966 }
6967 }
6968 start_at_beginning:
6969 went_fwd = 1;
6970 /*
6971 * Ok lets locate the block where this guy is fwd from rsm (if its
6972 * set)
6973 */
6974 TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {
6975 if (SEQ_GEQ(start, rsm->r_start) &&
6976 SEQ_LT(start, rsm->r_end)) {
6977 break;
6978 }
6979 }
6980 do_rest_ofb:
6981 if (rsm == NULL) {
6982 /*
6983 * This happens when we get duplicate sack blocks with the
6984 * same end. For example SACK 4: 100 SACK 3: 100 The sort
6985 * will not change there location so we would just start at
6986 * the end of the first one and get lost.
6987 */
6988 if (tp->t_flags & TF_SENTFIN) {
6989 /*
6990 * Check to see if we have not logged the FIN that
6991 * went out.
6992 */
6993 nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
6994 if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {
6995 /*
6996 * Ok we did not get the FIN logged.
6997 */
6998 nrsm->r_end++;
6999 rsm = nrsm;
7000 goto do_rest_ofb;
7001 }
7002 }
7003 if (times == 1) {
7004 #ifdef BBR_INVARIANTS
7005 panic("tp:%p bbr:%p sack:%p to:%p prsm:%p",
7006 tp, bbr, sack, to, prsm);
7007 #else
7008 goto out;
7009 #endif
7010 }
7011 times++;
7012 BBR_STAT_INC(bbr_sack_proc_restart);
7013 rsm = NULL;
7014 goto start_at_beginning;
7015 }
7016 /* Ok we have an ACK for some piece of rsm */
7017 if (rsm->r_start != start) {
7018 /*
7019 * Need to split this in two pieces the before and after.
7020 */
7021 if (bbr_sack_mergable(rsm, start, end))
7022 nrsm = bbr_alloc_full_limit(bbr);
7023 else
7024 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7025 if (nrsm == NULL) {
7026 /* We could not allocate ignore the sack */
7027 struct sackblk blk;
7028
7029 blk.start = start;
7030 blk.end = end;
7031 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7032 goto out;
7033 }
7034 bbr_clone_rsm(bbr, nrsm, rsm, start);
7035 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7036 if (rsm->r_in_tmap) {
7037 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7038 nrsm->r_in_tmap = 1;
7039 }
7040 rsm->r_flags &= (~BBR_HAS_FIN);
7041 rsm = nrsm;
7042 }
7043 if (SEQ_GEQ(end, rsm->r_end)) {
7044 /*
7045 * The end of this block is either beyond this guy or right
7046 * at this guy.
7047 */
7048 if ((rsm->r_flags & BBR_ACKED) == 0) {
7049 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7050 changed += (rsm->r_end - rsm->r_start);
7051 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7052 bbr_log_sack_passed(tp, bbr, rsm);
7053 if (rsm->r_flags & BBR_MARKED_LOST) {
7054 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7055 }
7056 /* Is Reordering occuring? */
7057 if (rsm->r_flags & BBR_SACK_PASSED) {
7058 BBR_STAT_INC(bbr_reorder_seen);
7059 bbr->r_ctl.rc_reorder_ts = cts;
7060 if (rsm->r_flags & BBR_MARKED_LOST) {
7061 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7062 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7063 /* LT sampling also needs adjustment */
7064 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7065 }
7066 }
7067 rsm->r_flags |= BBR_ACKED;
7068 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7069 if (rsm->r_in_tmap) {
7070 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7071 rsm->r_in_tmap = 0;
7072 }
7073 }
7074 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7075 if (end == rsm->r_end) {
7076 /* This block only - done */
7077 goto out;
7078 }
7079 /* There is more not coverend by this rsm move on */
7080 start = rsm->r_end;
7081 nrsm = TAILQ_NEXT(rsm, r_next);
7082 rsm = nrsm;
7083 times = 0;
7084 goto do_rest_ofb;
7085 }
7086 if (rsm->r_flags & BBR_ACKED) {
7087 /* Been here done that */
7088 goto out;
7089 }
7090 /* Ok we need to split off this one at the tail */
7091 if (bbr_sack_mergable(rsm, start, end))
7092 nrsm = bbr_alloc_full_limit(bbr);
7093 else
7094 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7095 if (nrsm == NULL) {
7096 /* failed XXXrrs what can we do but loose the sack info? */
7097 struct sackblk blk;
7098
7099 blk.start = start;
7100 blk.end = end;
7101 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7102 goto out;
7103 }
7104 /* Clone it */
7105 bbr_clone_rsm(bbr, nrsm, rsm, end);
7106 /* The sack block does not cover this guy fully */
7107 rsm->r_flags &= (~BBR_HAS_FIN);
7108 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7109 if (rsm->r_in_tmap) {
7110 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7111 nrsm->r_in_tmap = 1;
7112 }
7113 nrsm->r_dupack = 0;
7114 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7115 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7116 changed += (rsm->r_end - rsm->r_start);
7117 bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7118 bbr_log_sack_passed(tp, bbr, rsm);
7119 /* Is Reordering occuring? */
7120 if (rsm->r_flags & BBR_MARKED_LOST) {
7121 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7122 }
7123 if (rsm->r_flags & BBR_SACK_PASSED) {
7124 BBR_STAT_INC(bbr_reorder_seen);
7125 bbr->r_ctl.rc_reorder_ts = cts;
7126 if (rsm->r_flags & BBR_MARKED_LOST) {
7127 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7128 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7129 /* LT sampling also needs adjustment */
7130 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7131 }
7132 }
7133 rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7134 rsm->r_flags |= BBR_ACKED;
7135 if (rsm->r_in_tmap) {
7136 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7137 rsm->r_in_tmap = 0;
7138 }
7139 out:
7140 if (rsm && (rsm->r_flags & BBR_ACKED)) {
7141 /*
7142 * Now can we merge this newly acked
7143 * block with either the previous or
7144 * next block?
7145 */
7146 nrsm = TAILQ_NEXT(rsm, r_next);
7147 if (nrsm &&
7148 (nrsm->r_flags & BBR_ACKED)) {
7149 /* yep this and next can be merged */
7150 rsm = bbr_merge_rsm(bbr, rsm, nrsm);
7151 }
7152 /* Now what about the previous? */
7153 nrsm = TAILQ_PREV(rsm, bbr_head, r_next);
7154 if (nrsm &&
7155 (nrsm->r_flags & BBR_ACKED)) {
7156 /* yep the previous and this can be merged */
7157 rsm = bbr_merge_rsm(bbr, nrsm, rsm);
7158 }
7159 }
7160 if (used_ref == 0) {
7161 BBR_STAT_INC(bbr_sack_proc_all);
7162 } else {
7163 BBR_STAT_INC(bbr_sack_proc_short);
7164 }
7165 if (went_fwd && went_back) {
7166 BBR_STAT_INC(bbr_sack_search_both);
7167 } else if (went_fwd) {
7168 BBR_STAT_INC(bbr_sack_search_fwd);
7169 } else if (went_back) {
7170 BBR_STAT_INC(bbr_sack_search_back);
7171 }
7172 /* Save off where the next seq is */
7173 if (rsm)
7174 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);
7175 else
7176 bbr->r_ctl.rc_sacklast = NULL;
7177 *prsm = rsm;
7178 return (changed);
7179 }
7180
7181 static void inline
bbr_peer_reneges(struct tcp_bbr * bbr,struct bbr_sendmap * rsm,tcp_seq th_ack)7182 bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack)
7183 {
7184 struct bbr_sendmap *tmap;
7185
7186 BBR_STAT_INC(bbr_reneges_seen);
7187 tmap = NULL;
7188 while (rsm && (rsm->r_flags & BBR_ACKED)) {
7189 /* Its no longer sacked, mark it so */
7190 uint32_t oflags;
7191 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7192 #ifdef BBR_INVARIANTS
7193 if (rsm->r_in_tmap) {
7194 panic("bbr:%p rsm:%p flags:0x%x in tmap?",
7195 bbr, rsm, rsm->r_flags);
7196 }
7197 #endif
7198 oflags = rsm->r_flags;
7199 if (rsm->r_flags & BBR_MARKED_LOST) {
7200 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7201 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7202 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7203 /* LT sampling also needs adjustment */
7204 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7205 }
7206 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);
7207 rsm->r_flags |= BBR_WAS_RENEGED;
7208 rsm->r_flags |= BBR_RXT_CLEARED;
7209 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);
7210 /* Rebuild it into our tmap */
7211 if (tmap == NULL) {
7212 TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7213 tmap = rsm;
7214 } else {
7215 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);
7216 tmap = rsm;
7217 }
7218 tmap->r_in_tmap = 1;
7219 /*
7220 * XXXrrs Delivered? Should we do anything here?
7221 *
7222 * Of course we don't on a rxt timeout so maybe its ok that
7223 * we don't?
7224 *
7225 * For now lets not.
7226 */
7227 rsm = TAILQ_NEXT(rsm, r_next);
7228 }
7229 /*
7230 * Now lets possibly clear the sack filter so we start recognizing
7231 * sacks that cover this area.
7232 */
7233 sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);
7234 }
7235
7236 static void
bbr_log_syn(struct tcpcb * tp,struct tcpopt * to)7237 bbr_log_syn(struct tcpcb *tp, struct tcpopt *to)
7238 {
7239 struct tcp_bbr *bbr;
7240 struct bbr_sendmap *rsm;
7241 uint32_t cts;
7242
7243 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7244 cts = bbr->r_ctl.rc_rcvtime;
7245 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7246 if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {
7247 if ((rsm->r_end - rsm->r_start) <= 1) {
7248 /* Log out the SYN completely */
7249 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7250 rsm->r_rtr_bytes = 0;
7251 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7252 if (rsm->r_in_tmap) {
7253 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7254 rsm->r_in_tmap = 0;
7255 }
7256 if (bbr->r_ctl.rc_next == rsm) {
7257 /* scoot along the marker */
7258 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7259 }
7260 if (to != NULL)
7261 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0);
7262 bbr_free(bbr, rsm);
7263 } else {
7264 /* There is more (Fast open)? strip out SYN. */
7265 rsm->r_flags &= ~BBR_HAS_SYN;
7266 rsm->r_start++;
7267 }
7268 }
7269 }
7270
7271 /*
7272 * Returns the number of bytes that were
7273 * acknowledged by SACK blocks.
7274 */
7275
7276 static uint32_t
bbr_log_ack(struct tcpcb * tp,struct tcpopt * to,struct tcphdr * th,uint32_t * prev_acked)7277 bbr_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th,
7278 uint32_t *prev_acked)
7279 {
7280 uint32_t changed, last_seq, entered_recovery = 0;
7281 struct tcp_bbr *bbr;
7282 struct bbr_sendmap *rsm;
7283 struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1];
7284 register uint32_t th_ack;
7285 int32_t i, j, k, new_sb, num_sack_blks = 0;
7286 uint32_t cts, acked, ack_point, sack_changed = 0;
7287 uint32_t p_maxseg, maxseg, p_acked = 0;
7288
7289 INP_WLOCK_ASSERT(tptoinpcb(tp));
7290 if (tcp_get_flags(th) & TH_RST) {
7291 /* We don't log resets */
7292 return (0);
7293 }
7294 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7295 cts = bbr->r_ctl.rc_rcvtime;
7296
7297 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7298 changed = 0;
7299 maxseg = tp->t_maxseg - bbr->rc_last_options;
7300 p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);
7301 th_ack = th->th_ack;
7302 if (SEQ_GT(th_ack, tp->snd_una)) {
7303 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_UPDATE, __LINE__);
7304 bbr->rc_tp->t_acktime = ticks;
7305 }
7306 if (SEQ_LEQ(th_ack, tp->snd_una)) {
7307 /* Only sent here for sack processing */
7308 goto proc_sack;
7309 }
7310 if (rsm && SEQ_GT(th_ack, rsm->r_start)) {
7311 changed = th_ack - rsm->r_start;
7312 } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {
7313 /*
7314 * For the SYN incoming case we will not have called
7315 * tcp_output for the sending of the SYN, so there will be
7316 * no map. All other cases should probably be a panic.
7317 */
7318 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) {
7319 /*
7320 * We have a timestamp that can be used to generate
7321 * an initial RTT.
7322 */
7323 uint32_t ts, now, rtt;
7324
7325 ts = bbr_ts_convert(to->to_tsecr);
7326 now = bbr_ts_convert(tcp_tv_to_msec(&bbr->rc_tv));
7327 rtt = now - ts;
7328 if (rtt < 1)
7329 rtt = 1;
7330 bbr_log_type_bbrrttprop(bbr, rtt,
7331 tp->iss, 0, cts,
7332 BBR_RTT_BY_TIMESTAMP, tp->iss, 0);
7333 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
7334 changed = 1;
7335 bbr->r_wanted_output = 1;
7336 goto out;
7337 }
7338 goto proc_sack;
7339 } else if (rsm == NULL) {
7340 goto out;
7341 }
7342 if (changed) {
7343 /*
7344 * The ACK point is advancing to th_ack, we must drop off
7345 * the packets in the rack log and calculate any eligble
7346 * RTT's.
7347 */
7348 bbr->r_wanted_output = 1;
7349 more:
7350 if (rsm == NULL) {
7351 if (tp->t_flags & TF_SENTFIN) {
7352 /* if we send a FIN we will not hav a map */
7353 goto proc_sack;
7354 }
7355 #ifdef BBR_INVARIANTS
7356 panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n",
7357 tp,
7358 th, tp->t_state, bbr,
7359 tp->snd_una, tp->snd_max, changed);
7360 #endif
7361 goto proc_sack;
7362 }
7363 }
7364 if (SEQ_LT(th_ack, rsm->r_start)) {
7365 /* Huh map is missing this */
7366 #ifdef BBR_INVARIANTS
7367 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d bbr:%p\n",
7368 rsm->r_start,
7369 th_ack, tp->t_state,
7370 bbr->r_state, bbr);
7371 panic("th-ack is bad bbr:%p tp:%p", bbr, tp);
7372 #endif
7373 goto proc_sack;
7374 } else if (th_ack == rsm->r_start) {
7375 /* None here to ack */
7376 goto proc_sack;
7377 }
7378 /*
7379 * Clear the dup ack counter, it will
7380 * either be freed or if there is some
7381 * remaining we need to start it at zero.
7382 */
7383 rsm->r_dupack = 0;
7384 /* Now do we consume the whole thing? */
7385 if (SEQ_GEQ(th_ack, rsm->r_end)) {
7386 /* Its all consumed. */
7387 uint32_t left;
7388
7389 if (rsm->r_flags & BBR_ACKED) {
7390 /*
7391 * It was acked on the scoreboard -- remove it from
7392 * total
7393 */
7394 p_acked += (rsm->r_end - rsm->r_start);
7395 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7396 if (bbr->r_ctl.rc_sacked == 0)
7397 bbr->r_ctl.rc_sacklast = NULL;
7398 } else {
7399 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack);
7400 if (rsm->r_flags & BBR_MARKED_LOST) {
7401 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7402 }
7403 if (rsm->r_flags & BBR_SACK_PASSED) {
7404 /*
7405 * There are acked segments ACKED on the
7406 * scoreboard further up. We are seeing
7407 * reordering.
7408 */
7409 BBR_STAT_INC(bbr_reorder_seen);
7410 bbr->r_ctl.rc_reorder_ts = cts;
7411 if (rsm->r_flags & BBR_MARKED_LOST) {
7412 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7413 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7414 /* LT sampling also needs adjustment */
7415 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7416 }
7417 }
7418 rsm->r_flags &= ~BBR_MARKED_LOST;
7419 }
7420 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7421 rsm->r_rtr_bytes = 0;
7422 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7423 if (rsm->r_in_tmap) {
7424 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7425 rsm->r_in_tmap = 0;
7426 }
7427 if (bbr->r_ctl.rc_next == rsm) {
7428 /* scoot along the marker */
7429 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7430 }
7431 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7432 /* Adjust the packet counts */
7433 left = th_ack - rsm->r_end;
7434 /* Free back to zone */
7435 bbr_free(bbr, rsm);
7436 if (left) {
7437 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7438 goto more;
7439 }
7440 goto proc_sack;
7441 }
7442 if (rsm->r_flags & BBR_ACKED) {
7443 /*
7444 * It was acked on the scoreboard -- remove it from total
7445 * for the part being cum-acked.
7446 */
7447 p_acked += (rsm->r_end - rsm->r_start);
7448 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
7449 if (bbr->r_ctl.rc_sacked == 0)
7450 bbr->r_ctl.rc_sacklast = NULL;
7451 } else {
7452 /*
7453 * It was acked up to th_ack point for the first time
7454 */
7455 struct bbr_sendmap lrsm;
7456
7457 memcpy(&lrsm, rsm, sizeof(struct bbr_sendmap));
7458 lrsm.r_end = th_ack;
7459 bbr_update_rtt(tp, bbr, &lrsm, to, cts, BBR_CUM_ACKED, th_ack);
7460 }
7461 if ((rsm->r_flags & BBR_MARKED_LOST) &&
7462 ((rsm->r_flags & BBR_ACKED) == 0)) {
7463 /*
7464 * It was marked lost and partly ack'd now
7465 * for the first time. We lower the rc_lost_bytes
7466 * and still leave it MARKED.
7467 */
7468 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;
7469 }
7470 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7471 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7472 rsm->r_rtr_bytes = 0;
7473 /* adjust packet count */
7474 rsm->r_start = th_ack;
7475 proc_sack:
7476 /* Check for reneging */
7477 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7478 if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {
7479 /*
7480 * The peer has moved snd_una up to the edge of this send,
7481 * i.e. one that it had previously acked. The only way that
7482 * can be true if the peer threw away data (space issues)
7483 * that it had previously sacked (else it would have given
7484 * us snd_una up to (rsm->r_end). We need to undo the acked
7485 * markings here.
7486 *
7487 * Note we have to look to make sure th_ack is our
7488 * rsm->r_start in case we get an old ack where th_ack is
7489 * behind snd_una.
7490 */
7491 bbr_peer_reneges(bbr, rsm, th->th_ack);
7492 }
7493 if ((to->to_flags & TOF_SACK) == 0) {
7494 /* We are done nothing left to log */
7495 goto out;
7496 }
7497 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7498 if (rsm) {
7499 last_seq = rsm->r_end;
7500 } else {
7501 last_seq = tp->snd_max;
7502 }
7503 /* Sack block processing */
7504 if (SEQ_GT(th_ack, tp->snd_una))
7505 ack_point = th_ack;
7506 else
7507 ack_point = tp->snd_una;
7508 for (i = 0; i < to->to_nsacks; i++) {
7509 bcopy((to->to_sacks + i * TCPOLEN_SACK),
7510 &sack, sizeof(sack));
7511 sack.start = ntohl(sack.start);
7512 sack.end = ntohl(sack.end);
7513 if (SEQ_GT(sack.end, sack.start) &&
7514 SEQ_GT(sack.start, ack_point) &&
7515 SEQ_LT(sack.start, tp->snd_max) &&
7516 SEQ_GT(sack.end, ack_point) &&
7517 SEQ_LEQ(sack.end, tp->snd_max)) {
7518 if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&
7519 (SEQ_LT(sack.end, last_seq)) &&
7520 ((sack.end - sack.start) < (p_maxseg / 8))) {
7521 /*
7522 * Not the last piece and its smaller than
7523 * 1/8th of a p_maxseg. We ignore this.
7524 */
7525 BBR_STAT_INC(bbr_runt_sacks);
7526 continue;
7527 }
7528 sack_blocks[num_sack_blks] = sack;
7529 num_sack_blks++;
7530 } else if (SEQ_LEQ(sack.start, th_ack) &&
7531 SEQ_LEQ(sack.end, th_ack)) {
7532 /*
7533 * Its a D-SACK block.
7534 */
7535 tcp_record_dsack(tp, sack.start, sack.end, 0);
7536 }
7537 }
7538 if (num_sack_blks == 0)
7539 goto out;
7540 /*
7541 * Sort the SACK blocks so we can update the rack scoreboard with
7542 * just one pass.
7543 */
7544 new_sb = sack_filter_blks(tp, &bbr->r_ctl.bbr_sf, sack_blocks,
7545 num_sack_blks, th->th_ack);
7546 ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);
7547 BBR_STAT_ADD(bbr_sack_blocks, num_sack_blks);
7548 BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb));
7549 num_sack_blks = new_sb;
7550 if (num_sack_blks < 2) {
7551 goto do_sack_work;
7552 }
7553 /* Sort the sacks */
7554 for (i = 0; i < num_sack_blks; i++) {
7555 for (j = i + 1; j < num_sack_blks; j++) {
7556 if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) {
7557 sack = sack_blocks[i];
7558 sack_blocks[i] = sack_blocks[j];
7559 sack_blocks[j] = sack;
7560 }
7561 }
7562 }
7563 /*
7564 * Now are any of the sack block ends the same (yes some
7565 * implememtations send these)?
7566 */
7567 again:
7568 if (num_sack_blks > 1) {
7569 for (i = 0; i < num_sack_blks; i++) {
7570 for (j = i + 1; j < num_sack_blks; j++) {
7571 if (sack_blocks[i].end == sack_blocks[j].end) {
7572 /*
7573 * Ok these two have the same end we
7574 * want the smallest end and then
7575 * throw away the larger and start
7576 * again.
7577 */
7578 if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) {
7579 /*
7580 * The second block covers
7581 * more area use that
7582 */
7583 sack_blocks[i].start = sack_blocks[j].start;
7584 }
7585 /*
7586 * Now collapse out the dup-sack and
7587 * lower the count
7588 */
7589 for (k = (j + 1); k < num_sack_blks; k++) {
7590 sack_blocks[j].start = sack_blocks[k].start;
7591 sack_blocks[j].end = sack_blocks[k].end;
7592 j++;
7593 }
7594 num_sack_blks--;
7595 goto again;
7596 }
7597 }
7598 }
7599 }
7600 do_sack_work:
7601 rsm = bbr->r_ctl.rc_sacklast;
7602 for (i = 0; i < num_sack_blks; i++) {
7603 acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts);
7604 if (acked) {
7605 bbr->r_wanted_output = 1;
7606 changed += acked;
7607 sack_changed += acked;
7608 }
7609 }
7610 out:
7611 *prev_acked = p_acked;
7612 if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {
7613 /*
7614 * Ok we have a high probability that we need to go in to
7615 * recovery since we have data sack'd
7616 */
7617 struct bbr_sendmap *rsm;
7618
7619 rsm = bbr_check_recovery_mode(tp, bbr, cts);
7620 if (rsm) {
7621 /* Enter recovery */
7622 entered_recovery = 1;
7623 bbr->r_wanted_output = 1;
7624 /*
7625 * When we enter recovery we need to assure we send
7626 * one packet.
7627 */
7628 if (bbr->r_ctl.rc_resend == NULL) {
7629 bbr->r_ctl.rc_resend = rsm;
7630 }
7631 }
7632 }
7633 if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {
7634 /*
7635 * See if we need to rack-retransmit anything if so set it
7636 * up as the thing to resend assuming something else is not
7637 * already in that position.
7638 */
7639 if (bbr->r_ctl.rc_resend == NULL) {
7640 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
7641 }
7642 }
7643 /*
7644 * We return the amount that changed via sack, this is used by the
7645 * ack-received code to augment what was changed between th_ack <->
7646 * snd_una.
7647 */
7648 return (sack_changed);
7649 }
7650
7651 static void
bbr_strike_dupack(struct tcp_bbr * bbr)7652 bbr_strike_dupack(struct tcp_bbr *bbr)
7653 {
7654 struct bbr_sendmap *rsm;
7655
7656 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
7657 if (rsm && (rsm->r_dupack < 0xff)) {
7658 rsm->r_dupack++;
7659 if (rsm->r_dupack >= DUP_ACK_THRESHOLD)
7660 bbr->r_wanted_output = 1;
7661 }
7662 }
7663
7664 /*
7665 * Return value of 1, we do not need to call bbr_process_data().
7666 * return value of 0, bbr_process_data can be called.
7667 * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7668 * its unlocked and probably unsafe to touch the TCB.
7669 */
7670 static int
bbr_process_ack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,uint32_t tiwin,int32_t tlen,int32_t * ofia,int32_t thflags,int32_t * ret_val)7671 bbr_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so,
7672 struct tcpcb *tp, struct tcpopt *to,
7673 uint32_t tiwin, int32_t tlen,
7674 int32_t * ofia, int32_t thflags, int32_t * ret_val)
7675 {
7676 int32_t ourfinisacked = 0;
7677 int32_t acked_amount;
7678 uint16_t nsegs;
7679 int32_t acked;
7680 uint32_t lost, sack_changed = 0;
7681 struct mbuf *mfree;
7682 struct tcp_bbr *bbr;
7683 uint32_t prev_acked = 0;
7684
7685 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7686 lost = bbr->r_ctl.rc_lost;
7687 nsegs = max(1, m->m_pkthdr.lro_nsegs);
7688 if (SEQ_GEQ(tp->snd_una, tp->iss + (65535 << tp->snd_scale))) {
7689 /* Checking SEG.ACK against ISS is definitely redundant. */
7690 tp->t_flags2 |= TF2_NO_ISS_CHECK;
7691 }
7692 if (!V_tcp_insecure_ack) {
7693 tcp_seq seq_min;
7694 bool ghost_ack_check;
7695
7696 if (tp->t_flags2 & TF2_NO_ISS_CHECK) {
7697 /* Check for too old ACKs (RFC 5961, Section 5.2). */
7698 seq_min = tp->snd_una - tp->max_sndwnd;
7699 ghost_ack_check = false;
7700 } else {
7701 if (SEQ_GT(tp->iss + 1, tp->snd_una - tp->max_sndwnd)) {
7702 /* Checking for ghost ACKs is stricter. */
7703 seq_min = tp->iss + 1;
7704 ghost_ack_check = true;
7705 } else {
7706 /*
7707 * Checking for too old ACKs (RFC 5961,
7708 * Section 5.2) is stricter.
7709 */
7710 seq_min = tp->snd_una - tp->max_sndwnd;
7711 ghost_ack_check = false;
7712 }
7713 }
7714 if (SEQ_LT(th->th_ack, seq_min)) {
7715 if (ghost_ack_check)
7716 TCPSTAT_INC(tcps_rcvghostack);
7717 else
7718 TCPSTAT_INC(tcps_rcvacktooold);
7719 /* Send challenge ACK. */
7720 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
7721 bbr->r_wanted_output = 1;
7722 return (1);
7723 }
7724 }
7725 if (SEQ_GT(th->th_ack, tp->snd_max)) {
7726 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
7727 bbr->r_wanted_output = 1;
7728 return (1);
7729 }
7730 if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
7731 /* Process the ack */
7732 if (bbr->rc_in_persist)
7733 tp->t_rxtshift = 0;
7734 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd))
7735 bbr_strike_dupack(bbr);
7736 sack_changed = bbr_log_ack(tp, to, th, &prev_acked);
7737 }
7738 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));
7739 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
7740 /*
7741 * Old ack, behind the last one rcv'd or a duplicate ack
7742 * with SACK info.
7743 */
7744 if (th->th_ack == tp->snd_una) {
7745 bbr_ack_received(tp, bbr, th, 0, sack_changed, prev_acked, __LINE__, 0);
7746 if (bbr->r_state == TCPS_SYN_SENT) {
7747 /*
7748 * Special case on where we sent SYN. When
7749 * the SYN-ACK is processed in syn_sent
7750 * state it bumps the snd_una. This causes
7751 * us to hit here even though we did ack 1
7752 * byte.
7753 *
7754 * Go through the nothing left case so we
7755 * send data.
7756 */
7757 goto nothing_left;
7758 }
7759 }
7760 return (0);
7761 }
7762 /*
7763 * If we reach this point, ACK is not a duplicate, i.e., it ACKs
7764 * something we sent.
7765 */
7766 if (tp->t_flags & TF_NEEDSYN) {
7767 /*
7768 * T/TCP: Connection was half-synchronized, and our SYN has
7769 * been ACK'd (so connection is now fully synchronized). Go
7770 * to non-starred state, increment snd_una for ACK of SYN,
7771 * and check if we can do window scaling.
7772 */
7773 tp->t_flags &= ~TF_NEEDSYN;
7774 tp->snd_una++;
7775 /* Do window scaling? */
7776 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
7777 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
7778 tp->rcv_scale = tp->request_r_scale;
7779 /* Send window already scaled. */
7780 }
7781 }
7782 INP_WLOCK_ASSERT(tptoinpcb(tp));
7783
7784 acked = BYTES_THIS_ACK(tp, th);
7785 KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs);
7786 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
7787
7788 /*
7789 * If we just performed our first retransmit, and the ACK arrives
7790 * within our recovery window, then it was a mistake to do the
7791 * retransmit in the first place. Recover our original cwnd and
7792 * ssthresh, and proceed to transmit where we left off.
7793 */
7794 if (tp->t_flags & TF_PREVVALID) {
7795 tp->t_flags &= ~TF_PREVVALID;
7796 if (tp->t_rxtshift == 1 &&
7797 (int)(ticks - tp->t_badrxtwin) < 0)
7798 bbr_cong_signal(tp, th, CC_RTO_ERR, NULL);
7799 }
7800 SOCK_SENDBUF_LOCK(so);
7801 acked_amount = min(acked, (int)sbavail(&so->so_snd));
7802 tp->snd_wnd -= acked_amount;
7803 mfree = sbcut_locked(&so->so_snd, acked_amount);
7804 /* NB: sowwakeup_locked() does an implicit unlock. */
7805 sowwakeup_locked(so);
7806 m_freem(mfree);
7807 if (SEQ_GT(th->th_ack, tp->snd_una)) {
7808 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
7809 }
7810 tp->snd_una = th->th_ack;
7811 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - lost));
7812 if (IN_RECOVERY(tp->t_flags)) {
7813 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
7814 (SEQ_LT(th->th_ack, tp->snd_max))) {
7815 tcp_bbr_partialack(tp);
7816 } else {
7817 bbr_post_recovery(tp);
7818 }
7819 }
7820 if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
7821 tp->snd_recover = tp->snd_una;
7822 }
7823 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
7824 tp->snd_nxt = tp->snd_max;
7825 }
7826 if (tp->snd_una == tp->snd_max) {
7827 /* Nothing left outstanding */
7828 nothing_left:
7829 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
7830 if (sbavail(&so->so_snd) == 0)
7831 bbr->rc_tp->t_acktime = 0;
7832 if ((sbused(&so->so_snd) == 0) &&
7833 (tp->t_flags & TF_SENTFIN)) {
7834 ourfinisacked = 1;
7835 }
7836 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
7837 if (bbr->rc_in_persist == 0) {
7838 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
7839 }
7840 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
7841 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
7842 /*
7843 * We invalidate the last ack here since we
7844 * don't want to transfer forward the time
7845 * for our sum's calculations.
7846 */
7847 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
7848 (sbavail(&so->so_snd) == 0) &&
7849 (tp->t_flags2 & TF2_DROP_AF_DATA)) {
7850 /*
7851 * The socket was gone and the peer sent data, time
7852 * to reset him.
7853 */
7854 *ret_val = 1;
7855 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
7856 /* tcp_close will kill the inp pre-log the Reset */
7857 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
7858 tp = tcp_close(tp);
7859 ctf_do_dropwithreset(m, tp, th, tlen);
7860 BBR_STAT_INC(bbr_dropped_af_data);
7861 return (1);
7862 }
7863 /* Set need output so persist might get set */
7864 bbr->r_wanted_output = 1;
7865 }
7866 if (ofia)
7867 *ofia = ourfinisacked;
7868 return (0);
7869 }
7870
7871 static void
bbr_enter_persist(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,int32_t line)7872 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
7873 {
7874 if (bbr->rc_in_persist == 0) {
7875 bbr_timer_cancel(bbr, __LINE__, cts);
7876 bbr->r_ctl.rc_last_delay_val = 0;
7877 tp->t_rxtshift = 0;
7878 bbr->rc_in_persist = 1;
7879 bbr->r_ctl.rc_went_idle_time = cts;
7880 /* We should be capped when rw went to 0 but just in case */
7881 bbr_log_type_pesist(bbr, cts, 0, line, 1);
7882 /* Time freezes for the state, so do the accounting now */
7883 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
7884 uint32_t time_in;
7885
7886 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
7887 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7888 int32_t idx;
7889
7890 idx = bbr_state_val(bbr);
7891 counter_u64_add(bbr_state_time[(idx + 5)], time_in);
7892 } else {
7893 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
7894 }
7895 }
7896 bbr->r_ctl.rc_bbr_state_time = cts;
7897 }
7898 }
7899
7900 static void
bbr_restart_after_idle(struct tcp_bbr * bbr,uint32_t cts,uint32_t idle_time)7901 bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time)
7902 {
7903 /*
7904 * Note that if idle time does not exceed our
7905 * threshold, we do nothing continuing the state
7906 * transitions we were last walking through.
7907 */
7908 if (idle_time >= bbr_idle_restart_threshold) {
7909 if (bbr->rc_use_idle_restart) {
7910 bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;
7911 /*
7912 * Set our target using BBR_UNIT, so
7913 * we increase at a dramatic rate but
7914 * we stop when we get the pipe
7915 * full again for our current b/w estimate.
7916 */
7917 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
7918 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
7919 bbr_set_state_target(bbr, __LINE__);
7920 /* Now setup our gains to ramp up */
7921 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
7922 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
7923 bbr_log_type_statechange(bbr, cts, __LINE__);
7924 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
7925 bbr_substate_change(bbr, cts, __LINE__, 1);
7926 }
7927 }
7928 }
7929
7930 static void
bbr_exit_persist(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts,int32_t line)7931 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
7932 {
7933 uint32_t idle_time;
7934
7935 if (bbr->rc_in_persist == 0)
7936 return;
7937 idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);
7938 bbr->rc_in_persist = 0;
7939 bbr->rc_hit_state_1 = 0;
7940 bbr->r_ctl.rc_del_time = cts;
7941 /*
7942 * We invalidate the last ack here since we
7943 * don't want to transfer forward the time
7944 * for our sum's calculations.
7945 */
7946 if (tcp_in_hpts(bbr->rc_tp)) {
7947 tcp_hpts_remove(bbr->rc_tp);
7948 bbr->rc_timer_first = 0;
7949 bbr->r_ctl.rc_hpts_flags = 0;
7950 bbr->r_ctl.rc_last_delay_val = 0;
7951 bbr->r_ctl.rc_hptsi_agg_delay = 0;
7952 bbr->r_agg_early_set = 0;
7953 bbr->r_ctl.rc_agg_early = 0;
7954 }
7955 bbr_log_type_pesist(bbr, cts, idle_time, line, 0);
7956 if (idle_time >= bbr_rtt_probe_time) {
7957 /*
7958 * This qualifies as a RTT_PROBE session since we drop the
7959 * data outstanding to nothing and waited more than
7960 * bbr_rtt_probe_time.
7961 */
7962 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_PERSIST, 0);
7963 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;
7964 }
7965 tp->t_rxtshift = 0;
7966 /*
7967 * If in probeBW and we have persisted more than an RTT lets do
7968 * special handling.
7969 */
7970 /* Force a time based epoch */
7971 bbr_set_epoch(bbr, cts, __LINE__);
7972 /*
7973 * Setup the lost so we don't count anything against the guy
7974 * we have been stuck with during persists.
7975 */
7976 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
7977 /* Time un-freezes for the state */
7978 bbr->r_ctl.rc_bbr_state_time = cts;
7979 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||
7980 (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {
7981 /*
7982 * If we are going back to probe-bw
7983 * or probe_rtt, we may need to possibly
7984 * do a fast restart.
7985 */
7986 bbr_restart_after_idle(bbr, cts, idle_time);
7987 }
7988 }
7989
7990 static void
bbr_collapsed_window(struct tcp_bbr * bbr)7991 bbr_collapsed_window(struct tcp_bbr *bbr)
7992 {
7993 /*
7994 * Now we must walk the
7995 * send map and divide the
7996 * ones left stranded. These
7997 * guys can't cause us to abort
7998 * the connection and are really
7999 * "unsent". However if a buggy
8000 * client actually did keep some
8001 * of the data i.e. collapsed the win
8002 * and refused to ack and then opened
8003 * the win and acked that data. We would
8004 * get into an ack war, the simplier
8005 * method then of just pretending we
8006 * did not send those segments something
8007 * won't work.
8008 */
8009 struct bbr_sendmap *rsm, *nrsm;
8010 tcp_seq max_seq;
8011 uint32_t maxseg;
8012 int can_split = 0;
8013 int fnd = 0;
8014
8015 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
8016 max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;
8017 bbr_log_type_rwnd_collapse(bbr, max_seq, 1, 0);
8018 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
8019 /* Find the first seq past or at maxseq */
8020 if (rsm->r_flags & BBR_RWND_COLLAPSED)
8021 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8022 if (SEQ_GEQ(max_seq, rsm->r_start) &&
8023 SEQ_GEQ(rsm->r_end, max_seq)) {
8024 fnd = 1;
8025 break;
8026 }
8027 }
8028 bbr->rc_has_collapsed = 0;
8029 if (!fnd) {
8030 /* Nothing to do strange */
8031 return;
8032 }
8033 /*
8034 * Now can we split?
8035 *
8036 * We don't want to split if splitting
8037 * would generate too many small segments
8038 * less we let an attacker fragment our
8039 * send_map and leave us out of memory.
8040 */
8041 if ((max_seq != rsm->r_start) &&
8042 (max_seq != rsm->r_end)){
8043 /* can we split? */
8044 int res1, res2;
8045
8046 res1 = max_seq - rsm->r_start;
8047 res2 = rsm->r_end - max_seq;
8048 if ((res1 >= (maxseg/8)) &&
8049 (res2 >= (maxseg/8))) {
8050 /* No small pieces here */
8051 can_split = 1;
8052 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {
8053 /* We are under the limit */
8054 can_split = 1;
8055 }
8056 }
8057 /* Ok do we need to split this rsm? */
8058 if (max_seq == rsm->r_start) {
8059 /* It's this guy no split required */
8060 nrsm = rsm;
8061 } else if (max_seq == rsm->r_end) {
8062 /* It's the next one no split required. */
8063 nrsm = TAILQ_NEXT(rsm, r_next);
8064 if (nrsm == NULL) {
8065 /* Huh? */
8066 return;
8067 }
8068 } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {
8069 /* yep we need to split it */
8070 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
8071 if (nrsm == NULL) {
8072 /* failed XXXrrs what can we do mark the whole? */
8073 nrsm = rsm;
8074 goto no_split;
8075 }
8076 /* Clone it */
8077 bbr_log_type_rwnd_collapse(bbr, max_seq, 3, 0);
8078 bbr_clone_rsm(bbr, nrsm, rsm, max_seq);
8079 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
8080 if (rsm->r_in_tmap) {
8081 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8082 nrsm->r_in_tmap = 1;
8083 }
8084 } else {
8085 /*
8086 * Split not allowed just start here just
8087 * use this guy.
8088 */
8089 nrsm = rsm;
8090 }
8091 no_split:
8092 BBR_STAT_INC(bbr_collapsed_win);
8093 /* reuse fnd as a count */
8094 fnd = 0;
8095 TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {
8096 nrsm->r_flags |= BBR_RWND_COLLAPSED;
8097 fnd++;
8098 bbr->rc_has_collapsed = 1;
8099 }
8100 bbr_log_type_rwnd_collapse(bbr, max_seq, 4, fnd);
8101 }
8102
8103 static void
bbr_un_collapse_window(struct tcp_bbr * bbr)8104 bbr_un_collapse_window(struct tcp_bbr *bbr)
8105 {
8106 struct bbr_sendmap *rsm;
8107 int cleared = 0;
8108
8109 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
8110 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
8111 /* Clear the flag */
8112 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8113 cleared++;
8114 } else
8115 break;
8116 }
8117 bbr_log_type_rwnd_collapse(bbr,
8118 (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);
8119 bbr->rc_has_collapsed = 0;
8120 }
8121
8122 /*
8123 * Return value of 1, the TCB is unlocked and most
8124 * likely gone, return value of 0, the TCB is still
8125 * locked.
8126 */
8127 static int
bbr_process_data(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt)8128 bbr_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so,
8129 struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
8130 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
8131 {
8132 /*
8133 * Update window information. Don't look at window if no ACK: TAC's
8134 * send garbage on first SYN.
8135 */
8136 uint16_t nsegs;
8137 int32_t tfo_syn;
8138 struct tcp_bbr *bbr;
8139
8140 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8141 INP_WLOCK_ASSERT(tptoinpcb(tp));
8142 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8143 if ((thflags & TH_ACK) &&
8144 (SEQ_LT(tp->snd_wl1, th->th_seq) ||
8145 (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
8146 (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
8147 /* keep track of pure window updates */
8148 if (tlen == 0 &&
8149 tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
8150 KMOD_TCPSTAT_INC(tcps_rcvwinupd);
8151 tp->snd_wnd = tiwin;
8152 tp->snd_wl1 = th->th_seq;
8153 tp->snd_wl2 = th->th_ack;
8154 if (tp->snd_wnd > tp->max_sndwnd)
8155 tp->max_sndwnd = tp->snd_wnd;
8156 bbr->r_wanted_output = 1;
8157 } else if (thflags & TH_ACK) {
8158 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
8159 tp->snd_wnd = tiwin;
8160 tp->snd_wl1 = th->th_seq;
8161 tp->snd_wl2 = th->th_ack;
8162 }
8163 }
8164 if (tp->snd_wnd < ctf_outstanding(tp))
8165 /* The peer collapsed its window on us */
8166 bbr_collapsed_window(bbr);
8167 else if (bbr->rc_has_collapsed)
8168 bbr_un_collapse_window(bbr);
8169 /* Was persist timer active and now we have window space? */
8170 if ((bbr->rc_in_persist != 0) &&
8171 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8172 bbr_minseg(bbr)))) {
8173 /*
8174 * Make the rate persist at end of persist mode if idle long
8175 * enough
8176 */
8177 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8178
8179 /* Make sure we output to start the timer */
8180 bbr->r_wanted_output = 1;
8181 }
8182 /* Do we need to enter persist? */
8183 if ((bbr->rc_in_persist == 0) &&
8184 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8185 TCPS_HAVEESTABLISHED(tp->t_state) &&
8186 (tp->snd_max == tp->snd_una) &&
8187 sbavail(&so->so_snd) &&
8188 (sbavail(&so->so_snd) > tp->snd_wnd)) {
8189 /* No send window.. we must enter persist */
8190 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8191 }
8192 if (tp->t_flags2 & TF2_DROP_AF_DATA) {
8193 m_freem(m);
8194 return (0);
8195 }
8196 /*
8197 * We don't support urgent data but
8198 * drag along the up just to make sure
8199 * if there is a stack switch no one
8200 * is surprised.
8201 */
8202 tp->rcv_up = tp->rcv_nxt;
8203
8204 /*
8205 * Process the segment text, merging it into the TCP sequencing
8206 * queue, and arranging for acknowledgment of receipt if necessary.
8207 * This process logically involves adjusting tp->rcv_wnd as data is
8208 * presented to the user (this happens in tcp_usrreq.c, case
8209 * PRU_RCVD). If a FIN has already been received on this connection
8210 * then we just ignore the text.
8211 */
8212 tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
8213 (tp->t_flags & TF_FASTOPEN));
8214 if ((tlen || (thflags & TH_FIN) || (tfo_syn && tlen > 0)) &&
8215 TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8216 tcp_seq save_start = th->th_seq;
8217 tcp_seq save_rnxt = tp->rcv_nxt;
8218 int save_tlen = tlen;
8219
8220 m_adj(m, drop_hdrlen); /* delayed header drop */
8221 /*
8222 * Insert segment which includes th into TCP reassembly
8223 * queue with control block tp. Set thflags to whether
8224 * reassembly now includes a segment with FIN. This handles
8225 * the common case inline (segment is the next to be
8226 * received on an established connection, and the queue is
8227 * empty), avoiding linkage into and removal from the queue
8228 * and repetition of various conversions. Set DELACK for
8229 * segments received in order, but ack immediately when
8230 * segments are out of order (so fast retransmit can work).
8231 */
8232 if (th->th_seq == tp->rcv_nxt &&
8233 SEGQ_EMPTY(tp) &&
8234 (TCPS_HAVEESTABLISHED(tp->t_state) ||
8235 tfo_syn)) {
8236 #ifdef NETFLIX_SB_LIMITS
8237 u_int mcnt, appended;
8238
8239 if (so->so_rcv.sb_shlim) {
8240 mcnt = m_memcnt(m);
8241 appended = 0;
8242 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8243 CFO_NOSLEEP, NULL) == false) {
8244 counter_u64_add(tcp_sb_shlim_fails, 1);
8245 m_freem(m);
8246 return (0);
8247 }
8248 }
8249
8250 #endif
8251 if (DELAY_ACK(tp, bbr, nsegs) || tfo_syn) {
8252 bbr->bbr_segs_rcvd += max(1, nsegs);
8253 tp->t_flags |= TF_DELACK;
8254 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8255 } else {
8256 bbr->r_wanted_output = 1;
8257 tp->t_flags |= TF_ACKNOW;
8258 }
8259 tp->rcv_nxt += tlen;
8260 if (tlen &&
8261 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
8262 (tp->t_fbyte_in == 0)) {
8263 tp->t_fbyte_in = ticks;
8264 if (tp->t_fbyte_in == 0)
8265 tp->t_fbyte_in = 1;
8266 if (tp->t_fbyte_out && tp->t_fbyte_in)
8267 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
8268 }
8269 thflags = tcp_get_flags(th) & TH_FIN;
8270 KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs);
8271 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
8272 SOCK_RECVBUF_LOCK(so);
8273 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
8274 m_freem(m);
8275 else
8276 #ifdef NETFLIX_SB_LIMITS
8277 appended =
8278 #endif
8279 sbappendstream_locked(&so->so_rcv, m, 0);
8280 /* NB: sorwakeup_locked() does an implicit unlock. */
8281 sorwakeup_locked(so);
8282 #ifdef NETFLIX_SB_LIMITS
8283 if (so->so_rcv.sb_shlim && appended != mcnt)
8284 counter_fo_release(so->so_rcv.sb_shlim,
8285 mcnt - appended);
8286 #endif
8287
8288 } else {
8289 /*
8290 * XXX: Due to the header drop above "th" is
8291 * theoretically invalid by now. Fortunately
8292 * m_adj() doesn't actually frees any mbufs when
8293 * trimming from the head.
8294 */
8295 tcp_seq temp = save_start;
8296
8297 thflags = tcp_reass(tp, th, &temp, &tlen, m);
8298 tp->t_flags |= TF_ACKNOW;
8299 if (tp->t_flags & TF_WAKESOR) {
8300 tp->t_flags &= ~TF_WAKESOR;
8301 /* NB: sorwakeup_locked() does an implicit unlock. */
8302 sorwakeup_locked(so);
8303 }
8304 }
8305 if ((tp->t_flags & TF_SACK_PERMIT) &&
8306 (save_tlen > 0) &&
8307 TCPS_HAVEESTABLISHED(tp->t_state)) {
8308 if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) {
8309 /*
8310 * DSACK actually handled in the fastpath
8311 * above.
8312 */
8313 tcp_update_sack_list(tp, save_start,
8314 save_start + save_tlen);
8315 } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
8316 if ((tp->rcv_numsacks >= 1) &&
8317 (tp->sackblks[0].end == save_start)) {
8318 /*
8319 * Partial overlap, recorded at todrop
8320 * above.
8321 */
8322 tcp_update_sack_list(tp,
8323 tp->sackblks[0].start,
8324 tp->sackblks[0].end);
8325 } else {
8326 tcp_update_dsack_list(tp, save_start,
8327 save_start + save_tlen);
8328 }
8329 } else if (tlen >= save_tlen) {
8330 /* Update of sackblks. */
8331 tcp_update_dsack_list(tp, save_start,
8332 save_start + save_tlen);
8333 } else if (tlen > 0) {
8334 tcp_update_dsack_list(tp, save_start,
8335 save_start + tlen);
8336 }
8337 }
8338 } else {
8339 m_freem(m);
8340 thflags &= ~TH_FIN;
8341 }
8342
8343 /*
8344 * If FIN is received ACK the FIN and let the user know that the
8345 * connection is closing.
8346 */
8347 if (thflags & TH_FIN) {
8348 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8349 /* The socket upcall is handled by socantrcvmore. */
8350 socantrcvmore(so);
8351 /*
8352 * If connection is half-synchronized (ie NEEDSYN
8353 * flag on) then delay ACK, so it may be piggybacked
8354 * when SYN is sent. Otherwise, since we received a
8355 * FIN then no more input can be expected, send ACK
8356 * now.
8357 */
8358 if (tp->t_flags & TF_NEEDSYN) {
8359 tp->t_flags |= TF_DELACK;
8360 bbr_timer_cancel(bbr,
8361 __LINE__, bbr->r_ctl.rc_rcvtime);
8362 } else {
8363 tp->t_flags |= TF_ACKNOW;
8364 }
8365 tp->rcv_nxt++;
8366 }
8367 switch (tp->t_state) {
8368 /*
8369 * In SYN_RECEIVED and ESTABLISHED STATES enter the
8370 * CLOSE_WAIT state.
8371 */
8372 case TCPS_SYN_RECEIVED:
8373 tp->t_starttime = ticks;
8374 /* FALLTHROUGH */
8375 case TCPS_ESTABLISHED:
8376 tcp_state_change(tp, TCPS_CLOSE_WAIT);
8377 break;
8378
8379 /*
8380 * If still in FIN_WAIT_1 STATE FIN has not been
8381 * acked so enter the CLOSING state.
8382 */
8383 case TCPS_FIN_WAIT_1:
8384 tcp_state_change(tp, TCPS_CLOSING);
8385 break;
8386
8387 /*
8388 * In FIN_WAIT_2 state enter the TIME_WAIT state,
8389 * starting the time-wait timer, turning off the
8390 * other standard timers.
8391 */
8392 case TCPS_FIN_WAIT_2:
8393 bbr->rc_timer_first = 1;
8394 bbr_timer_cancel(bbr,
8395 __LINE__, bbr->r_ctl.rc_rcvtime);
8396 tcp_twstart(tp);
8397 return (1);
8398 }
8399 }
8400 /*
8401 * Return any desired output.
8402 */
8403 if ((tp->t_flags & TF_ACKNOW) ||
8404 (sbavail(&so->so_snd) > ctf_outstanding(tp))) {
8405 bbr->r_wanted_output = 1;
8406 }
8407 return (0);
8408 }
8409
8410 /*
8411 * Here nothing is really faster, its just that we
8412 * have broken out the fast-data path also just like
8413 * the fast-ack. Return 1 if we processed the packet
8414 * return 0 if you need to take the "slow-path".
8415 */
8416 static int
bbr_do_fastnewdata(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t nxt_pkt)8417 bbr_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so,
8418 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8419 uint32_t tiwin, int32_t nxt_pkt)
8420 {
8421 uint16_t nsegs;
8422 int32_t newsize = 0; /* automatic sockbuf scaling */
8423 struct tcp_bbr *bbr;
8424 #ifdef NETFLIX_SB_LIMITS
8425 u_int mcnt, appended;
8426 #endif
8427
8428 /* On the hpts and we would have called output */
8429 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8430
8431 /*
8432 * If last ACK falls within this segment's sequence numbers, record
8433 * the timestamp. NOTE that the test is modified according to the
8434 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
8435 */
8436 if (bbr->r_ctl.rc_resend != NULL) {
8437 return (0);
8438 }
8439 if (tiwin && tiwin != tp->snd_wnd) {
8440 return (0);
8441 }
8442 if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
8443 return (0);
8444 }
8445 if (__predict_false((to->to_flags & TOF_TS) &&
8446 (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
8447 return (0);
8448 }
8449 if (__predict_false((th->th_ack != tp->snd_una))) {
8450 return (0);
8451 }
8452 if (__predict_false(tlen > sbspace(&so->so_rcv))) {
8453 return (0);
8454 }
8455 if ((to->to_flags & TOF_TS) != 0 &&
8456 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8457 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
8458 tp->ts_recent = to->to_tsval;
8459 }
8460 /*
8461 * This is a pure, in-sequence data packet with nothing on the
8462 * reassembly queue and we have enough buffer space to take it.
8463 */
8464 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8465
8466 #ifdef NETFLIX_SB_LIMITS
8467 if (so->so_rcv.sb_shlim) {
8468 mcnt = m_memcnt(m);
8469 appended = 0;
8470 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8471 CFO_NOSLEEP, NULL) == false) {
8472 counter_u64_add(tcp_sb_shlim_fails, 1);
8473 m_freem(m);
8474 return (1);
8475 }
8476 }
8477 #endif
8478 /* Clean receiver SACK report if present */
8479 if (tp->rcv_numsacks)
8480 tcp_clean_sackreport(tp);
8481 KMOD_TCPSTAT_INC(tcps_preddat);
8482 tp->rcv_nxt += tlen;
8483 if (tlen &&
8484 ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
8485 (tp->t_fbyte_in == 0)) {
8486 tp->t_fbyte_in = ticks;
8487 if (tp->t_fbyte_in == 0)
8488 tp->t_fbyte_in = 1;
8489 if (tp->t_fbyte_out && tp->t_fbyte_in)
8490 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
8491 }
8492 /*
8493 * Pull snd_wl1 up to prevent seq wrap relative to th_seq.
8494 */
8495 tp->snd_wl1 = th->th_seq;
8496 /*
8497 * Pull rcv_up up to prevent seq wrap relative to rcv_nxt.
8498 */
8499 tp->rcv_up = tp->rcv_nxt;
8500 KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs);
8501 KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
8502 newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
8503
8504 /* Add data to socket buffer. */
8505 SOCK_RECVBUF_LOCK(so);
8506 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8507 m_freem(m);
8508 } else {
8509 /*
8510 * Set new socket buffer size. Give up when limit is
8511 * reached.
8512 */
8513 if (newsize)
8514 if (!sbreserve_locked(so, SO_RCV, newsize, NULL))
8515 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
8516 m_adj(m, drop_hdrlen); /* delayed header drop */
8517
8518 #ifdef NETFLIX_SB_LIMITS
8519 appended =
8520 #endif
8521 sbappendstream_locked(&so->so_rcv, m, 0);
8522 ctf_calc_rwin(so, tp);
8523 }
8524 /* NB: sorwakeup_locked() does an implicit unlock. */
8525 sorwakeup_locked(so);
8526 #ifdef NETFLIX_SB_LIMITS
8527 if (so->so_rcv.sb_shlim && mcnt != appended)
8528 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);
8529 #endif
8530 if (DELAY_ACK(tp, bbr, nsegs)) {
8531 bbr->bbr_segs_rcvd += max(1, nsegs);
8532 tp->t_flags |= TF_DELACK;
8533 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8534 } else {
8535 bbr->r_wanted_output = 1;
8536 tp->t_flags |= TF_ACKNOW;
8537 }
8538 return (1);
8539 }
8540
8541 /*
8542 * This subfunction is used to try to highly optimize the
8543 * fast path. We again allow window updates that are
8544 * in sequence to remain in the fast-path. We also add
8545 * in the __predict's to attempt to help the compiler.
8546 * Note that if we return a 0, then we can *not* process
8547 * it and the caller should push the packet into the
8548 * slow-path. If we return 1, then all is well and
8549 * the packet is fully processed.
8550 */
8551 static int
bbr_fastack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t nxt_pkt,uint8_t iptos)8552 bbr_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
8553 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8554 uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos)
8555 {
8556 int32_t acked;
8557 uint16_t nsegs;
8558 uint32_t sack_changed;
8559 uint32_t prev_acked = 0;
8560 struct tcp_bbr *bbr;
8561
8562 if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
8563 /* Old ack, behind (or duplicate to) the last one rcv'd */
8564 return (0);
8565 }
8566 if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
8567 /* Above what we have sent? */
8568 return (0);
8569 }
8570 if (__predict_false(tiwin == 0)) {
8571 /* zero window */
8572 return (0);
8573 }
8574 if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
8575 /* We need a SYN or a FIN, unlikely.. */
8576 return (0);
8577 }
8578 if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
8579 /* Timestamp is behind .. old ack with seq wrap? */
8580 return (0);
8581 }
8582 if (__predict_false(IN_RECOVERY(tp->t_flags))) {
8583 /* Still recovering */
8584 return (0);
8585 }
8586 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8587 if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {
8588 /* We are retransmitting */
8589 return (0);
8590 }
8591 if (__predict_false(bbr->rc_in_persist != 0)) {
8592 /* In persist mode */
8593 return (0);
8594 }
8595 if (bbr->r_ctl.rc_sacked) {
8596 /* We have sack holes on our scoreboard */
8597 return (0);
8598 }
8599 /* Ok if we reach here, we can process a fast-ack */
8600 nsegs = max(1, m->m_pkthdr.lro_nsegs);
8601 sack_changed = bbr_log_ack(tp, to, th, &prev_acked);
8602 /*
8603 * We never detect loss in fast ack [we can't
8604 * have a sack and can't be in recovery so
8605 * we always pass 0 (nothing detected)].
8606 */
8607 bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);
8608 /* Did the window get updated? */
8609 if (tiwin != tp->snd_wnd) {
8610 tp->snd_wnd = tiwin;
8611 tp->snd_wl1 = th->th_seq;
8612 if (tp->snd_wnd > tp->max_sndwnd)
8613 tp->max_sndwnd = tp->snd_wnd;
8614 }
8615 /* Do we need to exit persists? */
8616 if ((bbr->rc_in_persist != 0) &&
8617 (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8618 bbr_minseg(bbr)))) {
8619 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8620 bbr->r_wanted_output = 1;
8621 }
8622 /* Do we need to enter persists? */
8623 if ((bbr->rc_in_persist == 0) &&
8624 (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8625 TCPS_HAVEESTABLISHED(tp->t_state) &&
8626 (tp->snd_max == tp->snd_una) &&
8627 sbavail(&so->so_snd) &&
8628 (sbavail(&so->so_snd) > tp->snd_wnd)) {
8629 /* No send window.. we must enter persist */
8630 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8631 }
8632 /*
8633 * If last ACK falls within this segment's sequence numbers, record
8634 * the timestamp. NOTE that the test is modified according to the
8635 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
8636 */
8637 if ((to->to_flags & TOF_TS) != 0 &&
8638 SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8639 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;
8640 tp->ts_recent = to->to_tsval;
8641 }
8642 /*
8643 * This is a pure ack for outstanding data.
8644 */
8645 KMOD_TCPSTAT_INC(tcps_predack);
8646
8647 /*
8648 * "bad retransmit" recovery.
8649 */
8650 if (tp->t_flags & TF_PREVVALID) {
8651 tp->t_flags &= ~TF_PREVVALID;
8652 if (tp->t_rxtshift == 1 &&
8653 (int)(ticks - tp->t_badrxtwin) < 0)
8654 bbr_cong_signal(tp, th, CC_RTO_ERR, NULL);
8655 }
8656 /*
8657 * Recalculate the transmit timer / rtt.
8658 *
8659 * Some boxes send broken timestamp replies during the SYN+ACK
8660 * phase, ignore timestamps of 0 or we could calculate a huge RTT
8661 * and blow up the retransmit timer.
8662 */
8663 acked = BYTES_THIS_ACK(tp, th);
8664
8665 #ifdef TCP_HHOOK
8666 /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */
8667 hhook_run_tcp_est_in(tp, th, to);
8668 #endif
8669
8670 KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs);
8671 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
8672 sbdrop(&so->so_snd, acked);
8673
8674 if (SEQ_GT(th->th_ack, tp->snd_una))
8675 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
8676 tp->snd_una = th->th_ack;
8677 if (tp->snd_wnd < ctf_outstanding(tp))
8678 /* The peer collapsed its window on us */
8679 bbr_collapsed_window(bbr);
8680 else if (bbr->rc_has_collapsed)
8681 bbr_un_collapse_window(bbr);
8682
8683 if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
8684 tp->snd_recover = tp->snd_una;
8685 }
8686 bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, 0);
8687 /*
8688 * Pull snd_wl2 up to prevent seq wrap relative to th_ack.
8689 */
8690 tp->snd_wl2 = th->th_ack;
8691 m_freem(m);
8692 /*
8693 * If all outstanding data are acked, stop retransmit timer,
8694 * otherwise restart timer using current (possibly backed-off)
8695 * value. If process is waiting for space, wakeup/selwakeup/signal.
8696 * If data are ready to send, let tcp_output decide between more
8697 * output or persist.
8698 * Wake up the socket if we have room to write more.
8699 */
8700 sowwakeup(so);
8701 if (tp->snd_una == tp->snd_max) {
8702 /* Nothing left outstanding */
8703 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
8704 if (sbavail(&so->so_snd) == 0)
8705 bbr->rc_tp->t_acktime = 0;
8706 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8707 if (bbr->rc_in_persist == 0) {
8708 bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
8709 }
8710 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
8711 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
8712 /*
8713 * We invalidate the last ack here since we
8714 * don't want to transfer forward the time
8715 * for our sum's calculations.
8716 */
8717 bbr->r_wanted_output = 1;
8718 }
8719 if (sbavail(&so->so_snd)) {
8720 bbr->r_wanted_output = 1;
8721 }
8722 return (1);
8723 }
8724
8725 /*
8726 * Return value of 1, the TCB is unlocked and most
8727 * likely gone, return value of 0, the TCB is still
8728 * locked.
8729 */
8730 static int
bbr_do_syn_sent(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)8731 bbr_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so,
8732 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8733 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
8734 {
8735 int32_t todrop;
8736 int32_t ourfinisacked = 0;
8737 struct tcp_bbr *bbr;
8738 int32_t ret_val = 0;
8739
8740 INP_WLOCK_ASSERT(tptoinpcb(tp));
8741
8742 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8743 ctf_calc_rwin(so, tp);
8744 /*
8745 * If the state is SYN_SENT: if seg contains an ACK, but not for our
8746 * SYN, drop the input. if seg contains a RST, then drop the
8747 * connection. if seg does not contain SYN, then drop it. Otherwise
8748 * this is an acceptable SYN segment initialize tp->rcv_nxt and
8749 * tp->irs if seg contains ack then advance tp->snd_una. BRR does
8750 * not support ECN so we will not say we are capable. if SYN has
8751 * been acked change to ESTABLISHED else SYN_RCVD state arrange for
8752 * segment to be acked (eventually) continue processing rest of
8753 * data/controls, beginning with URG
8754 */
8755 if ((thflags & TH_ACK) &&
8756 (SEQ_LEQ(th->th_ack, tp->iss) ||
8757 SEQ_GT(th->th_ack, tp->snd_max))) {
8758 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
8759 ctf_do_dropwithreset(m, tp, th, tlen);
8760 return (1);
8761 }
8762 if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) {
8763 TCP_PROBE5(connect__refused, NULL, tp,
8764 mtod(m, const char *), tp, th);
8765 tp = tcp_drop(tp, ECONNREFUSED);
8766 ctf_do_drop(m, tp);
8767 return (1);
8768 }
8769 if (thflags & TH_RST) {
8770 ctf_do_drop(m, tp);
8771 return (1);
8772 }
8773 if (!(thflags & TH_SYN)) {
8774 ctf_do_drop(m, tp);
8775 return (1);
8776 }
8777 tp->irs = th->th_seq;
8778 tcp_rcvseqinit(tp);
8779 if (thflags & TH_ACK) {
8780 int tfo_partial = 0;
8781
8782 KMOD_TCPSTAT_INC(tcps_connects);
8783 soisconnected(so);
8784 #ifdef MAC
8785 mac_socketpeer_set_from_mbuf(m, so);
8786 #endif
8787 /* Do window scaling on this connection? */
8788 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
8789 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
8790 tp->rcv_scale = tp->request_r_scale;
8791 }
8792 tp->rcv_adv += min(tp->rcv_wnd,
8793 TCP_MAXWIN << tp->rcv_scale);
8794 /*
8795 * If not all the data that was sent in the TFO SYN
8796 * has been acked, resend the remainder right away.
8797 */
8798 if ((tp->t_flags & TF_FASTOPEN) &&
8799 (tp->snd_una != tp->snd_max)) {
8800 tp->snd_nxt = th->th_ack;
8801 tfo_partial = 1;
8802 }
8803 /*
8804 * If there's data, delay ACK; if there's also a FIN ACKNOW
8805 * will be turned on later.
8806 */
8807 if (DELAY_ACK(tp, bbr, 1) && tlen != 0 && !tfo_partial) {
8808 bbr->bbr_segs_rcvd += 1;
8809 tp->t_flags |= TF_DELACK;
8810 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8811 } else {
8812 bbr->r_wanted_output = 1;
8813 tp->t_flags |= TF_ACKNOW;
8814 }
8815 if (SEQ_GT(th->th_ack, tp->iss)) {
8816 /*
8817 * The SYN is acked
8818 * handle it specially.
8819 */
8820 bbr_log_syn(tp, to);
8821 }
8822 if (SEQ_GT(th->th_ack, tp->snd_una)) {
8823 /*
8824 * We advance snd_una for the
8825 * fast open case. If th_ack is
8826 * acknowledging data beyond
8827 * snd_una we can't just call
8828 * ack-processing since the
8829 * data stream in our send-map
8830 * will start at snd_una + 1 (one
8831 * beyond the SYN). If its just
8832 * equal we don't need to do that
8833 * and there is no send_map.
8834 */
8835 tp->snd_una++;
8836 }
8837 /*
8838 * Received <SYN,ACK> in SYN_SENT[*] state. Transitions:
8839 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
8840 */
8841 tp->t_starttime = ticks;
8842 if (tp->t_flags & TF_NEEDFIN) {
8843 tcp_state_change(tp, TCPS_FIN_WAIT_1);
8844 tp->t_flags &= ~TF_NEEDFIN;
8845 thflags &= ~TH_SYN;
8846 } else {
8847 tcp_state_change(tp, TCPS_ESTABLISHED);
8848 TCP_PROBE5(connect__established, NULL, tp,
8849 mtod(m, const char *), tp, th);
8850 cc_conn_init(tp);
8851 }
8852 } else {
8853 /*
8854 * Received initial SYN in SYN-SENT[*] state => simultaneous
8855 * open. If segment contains CC option and there is a
8856 * cached CC, apply TAO test. If it succeeds, connection is *
8857 * half-synchronized. Otherwise, do 3-way handshake:
8858 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
8859 * there was no CC option, clear cached CC value.
8860 */
8861 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN | TF_SONOTCONN);
8862 tcp_state_change(tp, TCPS_SYN_RECEIVED);
8863 }
8864 /*
8865 * Advance th->th_seq to correspond to first data byte. If data,
8866 * trim to stay within window, dropping FIN if necessary.
8867 */
8868 th->th_seq++;
8869 if (tlen > tp->rcv_wnd) {
8870 todrop = tlen - tp->rcv_wnd;
8871 m_adj(m, -todrop);
8872 tlen = tp->rcv_wnd;
8873 thflags &= ~TH_FIN;
8874 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin);
8875 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
8876 }
8877 tp->snd_wl1 = th->th_seq - 1;
8878 tp->rcv_up = th->th_seq;
8879 /*
8880 * Client side of transaction: already sent SYN and data. If the
8881 * remote host used T/TCP to validate the SYN, our data will be
8882 * ACK'd; if so, enter normal data segment processing in the middle
8883 * of step 5, ack processing. Otherwise, goto step 6.
8884 */
8885 if (thflags & TH_ACK) {
8886 if ((to->to_flags & TOF_TS) != 0) {
8887 uint32_t t, rtt;
8888
8889 t = tcp_tv_to_msec(&bbr->rc_tv);
8890 if (TSTMP_GEQ(t, to->to_tsecr)) {
8891 rtt = t - to->to_tsecr;
8892 if (rtt == 0) {
8893 rtt = 1;
8894 }
8895 rtt *= MS_IN_USEC;
8896 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
8897 apply_filter_min_small(&bbr->r_ctl.rc_rttprop,
8898 rtt, bbr->r_ctl.rc_rcvtime);
8899 }
8900 }
8901 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val))
8902 return (ret_val);
8903 /* We may have changed to FIN_WAIT_1 above */
8904 if (tp->t_state == TCPS_FIN_WAIT_1) {
8905 /*
8906 * In FIN_WAIT_1 STATE in addition to the processing
8907 * for the ESTABLISHED state if our FIN is now
8908 * acknowledged then enter FIN_WAIT_2.
8909 */
8910 if (ourfinisacked) {
8911 /*
8912 * If we can't receive any more data, then
8913 * closing user can proceed. Starting the
8914 * timer is contrary to the specification,
8915 * but if we don't get a FIN we'll hang
8916 * forever.
8917 *
8918 * XXXjl: we should release the tp also, and
8919 * use a compressed state.
8920 */
8921 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8922 soisdisconnected(so);
8923 tcp_timer_activate(tp, TT_2MSL,
8924 (tcp_fast_finwait2_recycle ?
8925 tcp_finwait2_timeout :
8926 TP_MAXIDLE(tp)));
8927 }
8928 tcp_state_change(tp, TCPS_FIN_WAIT_2);
8929 }
8930 }
8931 }
8932 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
8933 tiwin, thflags, nxt_pkt));
8934 }
8935
8936 /*
8937 * Return value of 1, the TCB is unlocked and most
8938 * likely gone, return value of 0, the TCB is still
8939 * locked.
8940 */
8941 static int
bbr_do_syn_recv(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)8942 bbr_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so,
8943 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8944 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
8945 {
8946 int32_t ourfinisacked = 0;
8947 int32_t ret_val;
8948 struct tcp_bbr *bbr;
8949
8950 INP_WLOCK_ASSERT(tptoinpcb(tp));
8951
8952 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8953 ctf_calc_rwin(so, tp);
8954 if ((thflags & TH_RST) ||
8955 (tp->t_fin_is_rst && (thflags & TH_FIN)))
8956 return (ctf_process_rst(m, th, so, tp));
8957 if ((thflags & TH_ACK) &&
8958 (SEQ_LEQ(th->th_ack, tp->snd_una) ||
8959 SEQ_GT(th->th_ack, tp->snd_max))) {
8960 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
8961 ctf_do_dropwithreset(m, tp, th, tlen);
8962 return (1);
8963 }
8964 if (tp->t_flags & TF_FASTOPEN) {
8965 /*
8966 * When a TFO connection is in SYN_RECEIVED, the only valid
8967 * packets are the initial SYN, a retransmit/copy of the
8968 * initial SYN (possibly with a subset of the original
8969 * data), a valid ACK, a FIN, or a RST.
8970 */
8971 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
8972 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
8973 ctf_do_dropwithreset(m, tp, th, tlen);
8974 return (1);
8975 } else if (thflags & TH_SYN) {
8976 /* non-initial SYN is ignored */
8977 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
8978 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
8979 (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
8980 ctf_do_drop(m, NULL);
8981 return (0);
8982 }
8983 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) {
8984 ctf_do_drop(m, NULL);
8985 return (0);
8986 }
8987 }
8988 /*
8989 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
8990 * it's less than ts_recent, drop it.
8991 */
8992 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
8993 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
8994 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
8995 return (ret_val);
8996 }
8997 /*
8998 * In the SYN-RECEIVED state, validate that the packet belongs to
8999 * this connection before trimming the data to fit the receive
9000 * window. Check the sequence number versus IRS since we know the
9001 * sequence numbers haven't wrapped. This is a partial fix for the
9002 * "LAND" DoS attack.
9003 */
9004 if (SEQ_LT(th->th_seq, tp->irs)) {
9005 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
9006 ctf_do_dropwithreset(m, tp, th, tlen);
9007 return (1);
9008 }
9009 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9010 return (ret_val);
9011 }
9012 /*
9013 * If last ACK falls within this segment's sequence numbers, record
9014 * its timestamp. NOTE: 1) That the test incorporates suggestions
9015 * from the latest proposal of the tcplw@cray.com list (Braden
9016 * 1993/04/26). 2) That updating only on newer timestamps interferes
9017 * with our earlier PAWS tests, so this check should be solely
9018 * predicated on the sequence space of this segment. 3) That we
9019 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9020 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9021 * SEG.Len, This modified check allows us to overcome RFC1323's
9022 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9023 * p.869. In such cases, we can still calculate the RTT correctly
9024 * when RCV.NXT == Last.ACK.Sent.
9025 */
9026 if ((to->to_flags & TOF_TS) != 0 &&
9027 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9028 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9029 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9030 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9031 tp->ts_recent = to->to_tsval;
9032 }
9033 tp->snd_wnd = tiwin;
9034 /*
9035 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9036 * is on (half-synchronized state), then queue data for later
9037 * processing; else drop segment and return.
9038 */
9039 if ((thflags & TH_ACK) == 0) {
9040 if (tp->t_flags & TF_FASTOPEN) {
9041 cc_conn_init(tp);
9042 }
9043 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9044 tiwin, thflags, nxt_pkt));
9045 }
9046 KMOD_TCPSTAT_INC(tcps_connects);
9047 if (tp->t_flags & TF_SONOTCONN) {
9048 tp->t_flags &= ~TF_SONOTCONN;
9049 soisconnected(so);
9050 }
9051 /* Do window scaling? */
9052 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
9053 (TF_RCVD_SCALE | TF_REQ_SCALE)) {
9054 tp->rcv_scale = tp->request_r_scale;
9055 }
9056 /*
9057 * ok for the first time in lets see if we can use the ts to figure
9058 * out what the initial RTT was.
9059 */
9060 if ((to->to_flags & TOF_TS) != 0) {
9061 uint32_t t, rtt;
9062
9063 t = tcp_tv_to_msec(&bbr->rc_tv);
9064 if (TSTMP_GEQ(t, to->to_tsecr)) {
9065 rtt = t - to->to_tsecr;
9066 if (rtt == 0) {
9067 rtt = 1;
9068 }
9069 rtt *= MS_IN_USEC;
9070 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
9071 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);
9072 }
9073 }
9074 /* Drop off any SYN in the send map (probably not there) */
9075 if (thflags & TH_ACK)
9076 bbr_log_syn(tp, to);
9077 if ((tp->t_flags & TF_FASTOPEN) && tp->t_tfo_pending) {
9078 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
9079 tp->t_tfo_pending = NULL;
9080 }
9081 /*
9082 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* ->
9083 * FIN-WAIT-1
9084 */
9085 tp->t_starttime = ticks;
9086 if (tp->t_flags & TF_NEEDFIN) {
9087 tcp_state_change(tp, TCPS_FIN_WAIT_1);
9088 tp->t_flags &= ~TF_NEEDFIN;
9089 } else {
9090 tcp_state_change(tp, TCPS_ESTABLISHED);
9091 TCP_PROBE5(accept__established, NULL, tp,
9092 mtod(m, const char *), tp, th);
9093 /*
9094 * TFO connections call cc_conn_init() during SYN
9095 * processing. Calling it again here for such connections
9096 * is not harmless as it would undo the snd_cwnd reduction
9097 * that occurs when a TFO SYN|ACK is retransmitted.
9098 */
9099 if (!(tp->t_flags & TF_FASTOPEN))
9100 cc_conn_init(tp);
9101 }
9102 /*
9103 * Account for the ACK of our SYN prior to
9104 * regular ACK processing below, except for
9105 * simultaneous SYN, which is handled later.
9106 */
9107 if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))
9108 tp->snd_una++;
9109 /*
9110 * If segment contains data or ACK, will call tcp_reass() later; if
9111 * not, do so now to pass queued data to user.
9112 */
9113 if (tlen == 0 && (thflags & TH_FIN) == 0) {
9114 (void)tcp_reass(tp, (struct tcphdr *)0, NULL, 0,
9115 (struct mbuf *)0);
9116 if (tp->t_flags & TF_WAKESOR) {
9117 tp->t_flags &= ~TF_WAKESOR;
9118 /* NB: sorwakeup_locked() does an implicit unlock. */
9119 sorwakeup_locked(so);
9120 }
9121 }
9122 tp->snd_wl1 = th->th_seq - 1;
9123 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9124 return (ret_val);
9125 }
9126 if (tp->t_state == TCPS_FIN_WAIT_1) {
9127 /* We could have went to FIN_WAIT_1 (or EST) above */
9128 /*
9129 * In FIN_WAIT_1 STATE in addition to the processing for the
9130 * ESTABLISHED state if our FIN is now acknowledged then
9131 * enter FIN_WAIT_2.
9132 */
9133 if (ourfinisacked) {
9134 /*
9135 * If we can't receive any more data, then closing
9136 * user can proceed. Starting the timer is contrary
9137 * to the specification, but if we don't get a FIN
9138 * we'll hang forever.
9139 *
9140 * XXXjl: we should release the tp also, and use a
9141 * compressed state.
9142 */
9143 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9144 soisdisconnected(so);
9145 tcp_timer_activate(tp, TT_2MSL,
9146 (tcp_fast_finwait2_recycle ?
9147 tcp_finwait2_timeout :
9148 TP_MAXIDLE(tp)));
9149 }
9150 tcp_state_change(tp, TCPS_FIN_WAIT_2);
9151 }
9152 }
9153 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9154 tiwin, thflags, nxt_pkt));
9155 }
9156
9157 /*
9158 * Return value of 1, the TCB is unlocked and most
9159 * likely gone, return value of 0, the TCB is still
9160 * locked.
9161 */
9162 static int
bbr_do_established(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)9163 bbr_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so,
9164 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9165 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9166 {
9167 struct tcp_bbr *bbr;
9168 int32_t ret_val;
9169
9170 INP_WLOCK_ASSERT(tptoinpcb(tp));
9171
9172 /*
9173 * Header prediction: check for the two common cases of a
9174 * uni-directional data xfer. If the packet has no control flags,
9175 * is in-sequence, the window didn't change and we're not
9176 * retransmitting, it's a candidate. If the length is zero and the
9177 * ack moved forward, we're the sender side of the xfer. Just free
9178 * the data acked & wake any higher level process that was blocked
9179 * waiting for space. If the length is non-zero and the ack didn't
9180 * move, we're the receiver side. If we're getting packets in-order
9181 * (the reassembly queue is empty), add the data toc The socket
9182 * buffer and note that we need a delayed ack. Make sure that the
9183 * hidden state-flags are also off. Since we check for
9184 * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN.
9185 */
9186 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9187 if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {
9188 /*
9189 * If we have delived under 4 segments increase the initial
9190 * window if raised by the peer. We use this to determine
9191 * dynamic and static rwnd's at the end of a connection.
9192 */
9193 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);
9194 }
9195 if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&
9196 __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_URG | TH_ACK)) == TH_ACK) &&
9197 __predict_true(SEGQ_EMPTY(tp)) &&
9198 __predict_true(th->th_seq == tp->rcv_nxt)) {
9199 if (tlen == 0) {
9200 if (bbr_fastack(m, th, so, tp, to, drop_hdrlen, tlen,
9201 tiwin, nxt_pkt, iptos)) {
9202 return (0);
9203 }
9204 } else {
9205 if (bbr_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen,
9206 tiwin, nxt_pkt)) {
9207 return (0);
9208 }
9209 }
9210 }
9211 ctf_calc_rwin(so, tp);
9212
9213 if ((thflags & TH_RST) ||
9214 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9215 return (ctf_process_rst(m, th, so, tp));
9216 /*
9217 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9218 * synchronized state.
9219 */
9220 if (thflags & TH_SYN) {
9221 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
9222 return (ret_val);
9223 }
9224 /*
9225 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9226 * it's less than ts_recent, drop it.
9227 */
9228 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9229 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9230 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9231 return (ret_val);
9232 }
9233 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9234 return (ret_val);
9235 }
9236 /*
9237 * If last ACK falls within this segment's sequence numbers, record
9238 * its timestamp. NOTE: 1) That the test incorporates suggestions
9239 * from the latest proposal of the tcplw@cray.com list (Braden
9240 * 1993/04/26). 2) That updating only on newer timestamps interferes
9241 * with our earlier PAWS tests, so this check should be solely
9242 * predicated on the sequence space of this segment. 3) That we
9243 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9244 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9245 * SEG.Len, This modified check allows us to overcome RFC1323's
9246 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9247 * p.869. In such cases, we can still calculate the RTT correctly
9248 * when RCV.NXT == Last.ACK.Sent.
9249 */
9250 if ((to->to_flags & TOF_TS) != 0 &&
9251 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9252 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9253 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9254 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9255 tp->ts_recent = to->to_tsval;
9256 }
9257 /*
9258 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9259 * is on (half-synchronized state), then queue data for later
9260 * processing; else drop segment and return.
9261 */
9262 if ((thflags & TH_ACK) == 0) {
9263 if (tp->t_flags & TF_NEEDSYN) {
9264 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9265 tiwin, thflags, nxt_pkt));
9266 } else if (tp->t_flags & TF_ACKNOW) {
9267 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9268 bbr->r_wanted_output = 1;
9269 return (ret_val);
9270 } else {
9271 ctf_do_drop(m, NULL);
9272 return (0);
9273 }
9274 }
9275 /*
9276 * Ack processing.
9277 */
9278 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
9279 return (ret_val);
9280 }
9281 if (sbavail(&so->so_snd)) {
9282 if (ctf_progress_timeout_check(tp, true)) {
9283 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9284 ctf_do_dropwithreset_conn(m, tp, th, tlen);
9285 return (1);
9286 }
9287 }
9288 /* State changes only happen in bbr_process_data() */
9289 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9290 tiwin, thflags, nxt_pkt));
9291 }
9292
9293 /*
9294 * Return value of 1, the TCB is unlocked and most
9295 * likely gone, return value of 0, the TCB is still
9296 * locked.
9297 */
9298 static int
bbr_do_close_wait(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)9299 bbr_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so,
9300 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9301 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9302 {
9303 struct tcp_bbr *bbr;
9304 int32_t ret_val;
9305
9306 INP_WLOCK_ASSERT(tptoinpcb(tp));
9307
9308 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9309 ctf_calc_rwin(so, tp);
9310 if ((thflags & TH_RST) ||
9311 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9312 return (ctf_process_rst(m, th, so, tp));
9313 /*
9314 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9315 * synchronized state.
9316 */
9317 if (thflags & TH_SYN) {
9318 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
9319 return (ret_val);
9320 }
9321 /*
9322 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9323 * it's less than ts_recent, drop it.
9324 */
9325 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9326 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9327 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9328 return (ret_val);
9329 }
9330 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9331 return (ret_val);
9332 }
9333 /*
9334 * If last ACK falls within this segment's sequence numbers, record
9335 * its timestamp. NOTE: 1) That the test incorporates suggestions
9336 * from the latest proposal of the tcplw@cray.com list (Braden
9337 * 1993/04/26). 2) That updating only on newer timestamps interferes
9338 * with our earlier PAWS tests, so this check should be solely
9339 * predicated on the sequence space of this segment. 3) That we
9340 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9341 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9342 * SEG.Len, This modified check allows us to overcome RFC1323's
9343 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9344 * p.869. In such cases, we can still calculate the RTT correctly
9345 * when RCV.NXT == Last.ACK.Sent.
9346 */
9347 if ((to->to_flags & TOF_TS) != 0 &&
9348 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9349 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9350 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9351 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9352 tp->ts_recent = to->to_tsval;
9353 }
9354 /*
9355 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9356 * is on (half-synchronized state), then queue data for later
9357 * processing; else drop segment and return.
9358 */
9359 if ((thflags & TH_ACK) == 0) {
9360 if (tp->t_flags & TF_NEEDSYN) {
9361 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9362 tiwin, thflags, nxt_pkt));
9363 } else if (tp->t_flags & TF_ACKNOW) {
9364 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9365 bbr->r_wanted_output = 1;
9366 return (ret_val);
9367 } else {
9368 ctf_do_drop(m, NULL);
9369 return (0);
9370 }
9371 }
9372 /*
9373 * Ack processing.
9374 */
9375 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
9376 return (ret_val);
9377 }
9378 if (sbavail(&so->so_snd)) {
9379 if (ctf_progress_timeout_check(tp, true)) {
9380 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9381 ctf_do_dropwithreset_conn(m, tp, th, tlen);
9382 return (1);
9383 }
9384 }
9385 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9386 tiwin, thflags, nxt_pkt));
9387 }
9388
9389 static int
bbr_check_data_after_close(struct mbuf * m,struct tcp_bbr * bbr,struct tcpcb * tp,int32_t * tlen,struct tcphdr * th,struct socket * so)9390 bbr_check_data_after_close(struct mbuf *m, struct tcp_bbr *bbr,
9391 struct tcpcb *tp, int32_t * tlen, struct tcphdr *th, struct socket *so)
9392 {
9393
9394 if (bbr->rc_allow_data_af_clo == 0) {
9395 close_now:
9396 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
9397 /* tcp_close will kill the inp pre-log the Reset */
9398 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
9399 tp = tcp_close(tp);
9400 KMOD_TCPSTAT_INC(tcps_rcvafterclose);
9401 ctf_do_dropwithreset(m, tp, th, *tlen);
9402 return (1);
9403 }
9404 if (sbavail(&so->so_snd) == 0)
9405 goto close_now;
9406 /* Ok we allow data that is ignored and a followup reset */
9407 tp->rcv_nxt = th->th_seq + *tlen;
9408 tp->t_flags2 |= TF2_DROP_AF_DATA;
9409 bbr->r_wanted_output = 1;
9410 *tlen = 0;
9411 return (0);
9412 }
9413
9414 /*
9415 * Return value of 1, the TCB is unlocked and most
9416 * likely gone, return value of 0, the TCB is still
9417 * locked.
9418 */
9419 static int
bbr_do_fin_wait_1(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)9420 bbr_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so,
9421 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9422 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9423 {
9424 int32_t ourfinisacked = 0;
9425 int32_t ret_val;
9426 struct tcp_bbr *bbr;
9427
9428 INP_WLOCK_ASSERT(tptoinpcb(tp));
9429
9430 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9431 ctf_calc_rwin(so, tp);
9432 if ((thflags & TH_RST) ||
9433 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9434 return (ctf_process_rst(m, th, so, tp));
9435 /*
9436 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9437 * synchronized state.
9438 */
9439 if (thflags & TH_SYN) {
9440 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
9441 return (ret_val);
9442 }
9443 /*
9444 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9445 * it's less than ts_recent, drop it.
9446 */
9447 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9448 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9449 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9450 return (ret_val);
9451 }
9452 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9453 return (ret_val);
9454 }
9455 /*
9456 * If new data are received on a connection after the user processes
9457 * are gone, then RST the other end.
9458 * We call a new function now so we might continue and setup
9459 * to reset at all data being ack'd.
9460 */
9461 if ((tp->t_flags & TF_CLOSED) && tlen &&
9462 bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9463 return (1);
9464 /*
9465 * If last ACK falls within this segment's sequence numbers, record
9466 * its timestamp. NOTE: 1) That the test incorporates suggestions
9467 * from the latest proposal of the tcplw@cray.com list (Braden
9468 * 1993/04/26). 2) That updating only on newer timestamps interferes
9469 * with our earlier PAWS tests, so this check should be solely
9470 * predicated on the sequence space of this segment. 3) That we
9471 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9472 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9473 * SEG.Len, This modified check allows us to overcome RFC1323's
9474 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9475 * p.869. In such cases, we can still calculate the RTT correctly
9476 * when RCV.NXT == Last.ACK.Sent.
9477 */
9478 if ((to->to_flags & TOF_TS) != 0 &&
9479 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9480 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9481 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9482 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9483 tp->ts_recent = to->to_tsval;
9484 }
9485 /*
9486 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9487 * is on (half-synchronized state), then queue data for later
9488 * processing; else drop segment and return.
9489 */
9490 if ((thflags & TH_ACK) == 0) {
9491 if (tp->t_flags & TF_NEEDSYN) {
9492 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9493 tiwin, thflags, nxt_pkt));
9494 } else if (tp->t_flags & TF_ACKNOW) {
9495 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9496 bbr->r_wanted_output = 1;
9497 return (ret_val);
9498 } else {
9499 ctf_do_drop(m, NULL);
9500 return (0);
9501 }
9502 }
9503 /*
9504 * Ack processing.
9505 */
9506 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9507 return (ret_val);
9508 }
9509 if (ourfinisacked) {
9510 /*
9511 * If we can't receive any more data, then closing user can
9512 * proceed. Starting the timer is contrary to the
9513 * specification, but if we don't get a FIN we'll hang
9514 * forever.
9515 *
9516 * XXXjl: we should release the tp also, and use a
9517 * compressed state.
9518 */
9519 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9520 soisdisconnected(so);
9521 tcp_timer_activate(tp, TT_2MSL,
9522 (tcp_fast_finwait2_recycle ?
9523 tcp_finwait2_timeout :
9524 TP_MAXIDLE(tp)));
9525 }
9526 tcp_state_change(tp, TCPS_FIN_WAIT_2);
9527 }
9528 if (sbavail(&so->so_snd)) {
9529 if (ctf_progress_timeout_check(tp, true)) {
9530 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9531 ctf_do_dropwithreset_conn(m, tp, th, tlen);
9532 return (1);
9533 }
9534 }
9535 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9536 tiwin, thflags, nxt_pkt));
9537 }
9538
9539 /*
9540 * Return value of 1, the TCB is unlocked and most
9541 * likely gone, return value of 0, the TCB is still
9542 * locked.
9543 */
9544 static int
bbr_do_closing(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)9545 bbr_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so,
9546 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9547 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9548 {
9549 int32_t ourfinisacked = 0;
9550 int32_t ret_val;
9551 struct tcp_bbr *bbr;
9552
9553 INP_WLOCK_ASSERT(tptoinpcb(tp));
9554
9555 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9556 ctf_calc_rwin(so, tp);
9557 if ((thflags & TH_RST) ||
9558 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9559 return (ctf_process_rst(m, th, so, tp));
9560 /*
9561 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9562 * synchronized state.
9563 */
9564 if (thflags & TH_SYN) {
9565 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
9566 return (ret_val);
9567 }
9568 /*
9569 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9570 * it's less than ts_recent, drop it.
9571 */
9572 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9573 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9574 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9575 return (ret_val);
9576 }
9577 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9578 return (ret_val);
9579 }
9580 /*
9581 * If last ACK falls within this segment's sequence numbers, record
9582 * its timestamp. NOTE: 1) That the test incorporates suggestions
9583 * from the latest proposal of the tcplw@cray.com list (Braden
9584 * 1993/04/26). 2) That updating only on newer timestamps interferes
9585 * with our earlier PAWS tests, so this check should be solely
9586 * predicated on the sequence space of this segment. 3) That we
9587 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9588 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9589 * SEG.Len, This modified check allows us to overcome RFC1323's
9590 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9591 * p.869. In such cases, we can still calculate the RTT correctly
9592 * when RCV.NXT == Last.ACK.Sent.
9593 */
9594 if ((to->to_flags & TOF_TS) != 0 &&
9595 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9596 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9597 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9598 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9599 tp->ts_recent = to->to_tsval;
9600 }
9601 /*
9602 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9603 * is on (half-synchronized state), then queue data for later
9604 * processing; else drop segment and return.
9605 */
9606 if ((thflags & TH_ACK) == 0) {
9607 if (tp->t_flags & TF_NEEDSYN) {
9608 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9609 tiwin, thflags, nxt_pkt));
9610 } else if (tp->t_flags & TF_ACKNOW) {
9611 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9612 bbr->r_wanted_output = 1;
9613 return (ret_val);
9614 } else {
9615 ctf_do_drop(m, NULL);
9616 return (0);
9617 }
9618 }
9619 /*
9620 * Ack processing.
9621 */
9622 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9623 return (ret_val);
9624 }
9625 if (ourfinisacked) {
9626 tcp_twstart(tp);
9627 m_freem(m);
9628 return (1);
9629 }
9630 if (sbavail(&so->so_snd)) {
9631 if (ctf_progress_timeout_check(tp, true)) {
9632 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9633 ctf_do_dropwithreset_conn(m, tp, th, tlen);
9634 return (1);
9635 }
9636 }
9637 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9638 tiwin, thflags, nxt_pkt));
9639 }
9640
9641 /*
9642 * Return value of 1, the TCB is unlocked and most
9643 * likely gone, return value of 0, the TCB is still
9644 * locked.
9645 */
9646 static int
bbr_do_lastack(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)9647 bbr_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
9648 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9649 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9650 {
9651 int32_t ourfinisacked = 0;
9652 int32_t ret_val;
9653 struct tcp_bbr *bbr;
9654
9655 INP_WLOCK_ASSERT(tptoinpcb(tp));
9656
9657 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9658 ctf_calc_rwin(so, tp);
9659 if ((thflags & TH_RST) ||
9660 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9661 return (ctf_process_rst(m, th, so, tp));
9662 /*
9663 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9664 * synchronized state.
9665 */
9666 if (thflags & TH_SYN) {
9667 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
9668 return (ret_val);
9669 }
9670 /*
9671 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9672 * it's less than ts_recent, drop it.
9673 */
9674 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9675 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9676 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9677 return (ret_val);
9678 }
9679 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9680 return (ret_val);
9681 }
9682 /*
9683 * If last ACK falls within this segment's sequence numbers, record
9684 * its timestamp. NOTE: 1) That the test incorporates suggestions
9685 * from the latest proposal of the tcplw@cray.com list (Braden
9686 * 1993/04/26). 2) That updating only on newer timestamps interferes
9687 * with our earlier PAWS tests, so this check should be solely
9688 * predicated on the sequence space of this segment. 3) That we
9689 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9690 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9691 * SEG.Len, This modified check allows us to overcome RFC1323's
9692 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9693 * p.869. In such cases, we can still calculate the RTT correctly
9694 * when RCV.NXT == Last.ACK.Sent.
9695 */
9696 if ((to->to_flags & TOF_TS) != 0 &&
9697 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9698 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9699 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9700 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9701 tp->ts_recent = to->to_tsval;
9702 }
9703 /*
9704 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9705 * is on (half-synchronized state), then queue data for later
9706 * processing; else drop segment and return.
9707 */
9708 if ((thflags & TH_ACK) == 0) {
9709 if (tp->t_flags & TF_NEEDSYN) {
9710 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9711 tiwin, thflags, nxt_pkt));
9712 } else if (tp->t_flags & TF_ACKNOW) {
9713 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9714 bbr->r_wanted_output = 1;
9715 return (ret_val);
9716 } else {
9717 ctf_do_drop(m, NULL);
9718 return (0);
9719 }
9720 }
9721 /*
9722 * case TCPS_LAST_ACK: Ack processing.
9723 */
9724 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9725 return (ret_val);
9726 }
9727 if (ourfinisacked) {
9728 tp = tcp_close(tp);
9729 ctf_do_drop(m, tp);
9730 return (1);
9731 }
9732 if (sbavail(&so->so_snd)) {
9733 if (ctf_progress_timeout_check(tp, true)) {
9734 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9735 ctf_do_dropwithreset_conn(m, tp, th, tlen);
9736 return (1);
9737 }
9738 }
9739 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9740 tiwin, thflags, nxt_pkt));
9741 }
9742
9743 /*
9744 * Return value of 1, the TCB is unlocked and most
9745 * likely gone, return value of 0, the TCB is still
9746 * locked.
9747 */
9748 static int
bbr_do_fin_wait_2(struct mbuf * m,struct tcphdr * th,struct socket * so,struct tcpcb * tp,struct tcpopt * to,int32_t drop_hdrlen,int32_t tlen,uint32_t tiwin,int32_t thflags,int32_t nxt_pkt,uint8_t iptos)9749 bbr_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so,
9750 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9751 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
9752 {
9753 int32_t ourfinisacked = 0;
9754 int32_t ret_val;
9755 struct tcp_bbr *bbr;
9756
9757 INP_WLOCK_ASSERT(tptoinpcb(tp));
9758
9759 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9760 ctf_calc_rwin(so, tp);
9761 /* Reset receive buffer auto scaling when not in bulk receive mode. */
9762 if ((thflags & TH_RST) ||
9763 (tp->t_fin_is_rst && (thflags & TH_FIN)))
9764 return (ctf_process_rst(m, th, so, tp));
9765
9766 /*
9767 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9768 * synchronized state.
9769 */
9770 if (thflags & TH_SYN) {
9771 ctf_challenge_ack(m, th, tp, iptos, &ret_val);
9772 return (ret_val);
9773 }
9774 /*
9775 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9776 * it's less than ts_recent, drop it.
9777 */
9778 if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9779 TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9780 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9781 return (ret_val);
9782 }
9783 if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9784 return (ret_val);
9785 }
9786 /*
9787 * If new data are received on a connection after the user processes
9788 * are gone, then we may RST the other end depending on the outcome
9789 * of bbr_check_data_after_close.
9790 * We call a new function now so we might continue and setup
9791 * to reset at all data being ack'd.
9792 */
9793 if ((tp->t_flags & TF_CLOSED) && tlen &&
9794 bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9795 return (1);
9796 /*
9797 * If last ACK falls within this segment's sequence numbers, record
9798 * its timestamp. NOTE: 1) That the test incorporates suggestions
9799 * from the latest proposal of the tcplw@cray.com list (Braden
9800 * 1993/04/26). 2) That updating only on newer timestamps interferes
9801 * with our earlier PAWS tests, so this check should be solely
9802 * predicated on the sequence space of this segment. 3) That we
9803 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9804 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9805 * SEG.Len, This modified check allows us to overcome RFC1323's
9806 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9807 * p.869. In such cases, we can still calculate the RTT correctly
9808 * when RCV.NXT == Last.ACK.Sent.
9809 */
9810 if ((to->to_flags & TOF_TS) != 0 &&
9811 SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9812 SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9813 ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9814 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
9815 tp->ts_recent = to->to_tsval;
9816 }
9817 /*
9818 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
9819 * is on (half-synchronized state), then queue data for later
9820 * processing; else drop segment and return.
9821 */
9822 if ((thflags & TH_ACK) == 0) {
9823 if (tp->t_flags & TF_NEEDSYN) {
9824 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9825 tiwin, thflags, nxt_pkt));
9826 } else if (tp->t_flags & TF_ACKNOW) {
9827 ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9828 bbr->r_wanted_output = 1;
9829 return (ret_val);
9830 } else {
9831 ctf_do_drop(m, NULL);
9832 return (0);
9833 }
9834 }
9835 /*
9836 * Ack processing.
9837 */
9838 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9839 return (ret_val);
9840 }
9841 if (sbavail(&so->so_snd)) {
9842 if (ctf_progress_timeout_check(tp, true)) {
9843 bbr_log_progress_event(bbr, tp, tick, PROGRESS_DROP, __LINE__);
9844 ctf_do_dropwithreset_conn(m, tp, th, tlen);
9845 return (1);
9846 }
9847 }
9848 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9849 tiwin, thflags, nxt_pkt));
9850 }
9851
9852 static void
bbr_stop_all_timers(struct tcpcb * tp,struct tcp_bbr * bbr)9853 bbr_stop_all_timers(struct tcpcb *tp, struct tcp_bbr *bbr)
9854 {
9855 /*
9856 * Assure no timers are running.
9857 */
9858 if (tcp_timer_active(tp, TT_PERSIST)) {
9859 /* We enter in persists, set the flag appropriately */
9860 bbr->rc_in_persist = 1;
9861 }
9862 if (tcp_in_hpts(bbr->rc_tp)) {
9863 tcp_hpts_remove(bbr->rc_tp);
9864 }
9865 }
9866
9867 static void
bbr_google_mode_on(struct tcp_bbr * bbr)9868 bbr_google_mode_on(struct tcp_bbr *bbr)
9869 {
9870 bbr->rc_use_google = 1;
9871 bbr->rc_no_pacing = 0;
9872 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
9873 bbr->r_use_policer = bbr_policer_detection_enabled;
9874 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
9875 bbr->bbr_use_rack_cheat = 0;
9876 bbr->r_ctl.rc_incr_tmrs = 0;
9877 bbr->r_ctl.rc_inc_tcp_oh = 0;
9878 bbr->r_ctl.rc_inc_ip_oh = 0;
9879 bbr->r_ctl.rc_inc_enet_oh = 0;
9880 reset_time(&bbr->r_ctl.rc_delrate,
9881 BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT);
9882 reset_time_small(&bbr->r_ctl.rc_rttprop,
9883 (11 * USECS_IN_SECOND));
9884 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9885 }
9886
9887 static void
bbr_google_mode_off(struct tcp_bbr * bbr)9888 bbr_google_mode_off(struct tcp_bbr *bbr)
9889 {
9890 bbr->rc_use_google = 0;
9891 bbr->r_ctl.bbr_google_discount = 0;
9892 bbr->no_pacing_until = bbr_no_pacing_until;
9893 bbr->r_use_policer = 0;
9894 if (bbr->no_pacing_until)
9895 bbr->rc_no_pacing = 1;
9896 else
9897 bbr->rc_no_pacing = 0;
9898 if (bbr_use_rack_resend_cheat)
9899 bbr->bbr_use_rack_cheat = 1;
9900 else
9901 bbr->bbr_use_rack_cheat = 0;
9902 if (bbr_incr_timers)
9903 bbr->r_ctl.rc_incr_tmrs = 1;
9904 else
9905 bbr->r_ctl.rc_incr_tmrs = 0;
9906 if (bbr_include_tcp_oh)
9907 bbr->r_ctl.rc_inc_tcp_oh = 1;
9908 else
9909 bbr->r_ctl.rc_inc_tcp_oh = 0;
9910 if (bbr_include_ip_oh)
9911 bbr->r_ctl.rc_inc_ip_oh = 1;
9912 else
9913 bbr->r_ctl.rc_inc_ip_oh = 0;
9914 if (bbr_include_enet_oh)
9915 bbr->r_ctl.rc_inc_enet_oh = 1;
9916 else
9917 bbr->r_ctl.rc_inc_enet_oh = 0;
9918 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
9919 reset_time(&bbr->r_ctl.rc_delrate,
9920 bbr_num_pktepo_for_del_limit);
9921 reset_time_small(&bbr->r_ctl.rc_rttprop,
9922 (bbr_filter_len_sec * USECS_IN_SECOND));
9923 tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
9924 }
9925 /*
9926 * Return 0 on success, non-zero on failure
9927 * which indicates the error (usually no memory).
9928 */
9929 static int
bbr_init(struct tcpcb * tp,void ** ptr)9930 bbr_init(struct tcpcb *tp, void **ptr)
9931 {
9932 struct inpcb *inp = tptoinpcb(tp);
9933 struct tcp_bbr *bbr = NULL;
9934 uint32_t cts;
9935
9936 tcp_hpts_init(tp);
9937
9938 *ptr = uma_zalloc(bbr_pcb_zone, (M_NOWAIT | M_ZERO));
9939 if (*ptr == NULL) {
9940 /*
9941 * We need to allocate memory but cant. The INP and INP_INFO
9942 * locks and they are recursive (happens during setup. So a
9943 * scheme to drop the locks fails :(
9944 *
9945 */
9946 return (ENOMEM);
9947 }
9948 bbr = (struct tcp_bbr *)*ptr;
9949 bbr->rtt_valid = 0;
9950 tp->t_flags2 |= TF2_CANNOT_DO_ECN;
9951 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
9952 /* Take off any undesired flags */
9953 tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
9954 tp->t_flags2 &= ~TF2_DONT_SACK_QUEUE;
9955 tp->t_flags2 &= ~TF2_MBUF_ACKCMP;
9956 tp->t_flags2 &= ~TF2_MBUF_L_ACKS;
9957
9958 TAILQ_INIT(&bbr->r_ctl.rc_map);
9959 TAILQ_INIT(&bbr->r_ctl.rc_free);
9960 TAILQ_INIT(&bbr->r_ctl.rc_tmap);
9961 bbr->rc_tp = tp;
9962 bbr->rc_inp = inp;
9963 cts = tcp_get_usecs(&bbr->rc_tv);
9964 tp->t_acktime = 0;
9965 bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;
9966 bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;
9967 bbr->rc_tlp_threshold = bbr_tlp_thresh;
9968 bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;
9969 bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;
9970 bbr->r_ctl.rc_min_to = bbr_min_to;
9971 bbr->rc_bbr_state = BBR_STATE_STARTUP;
9972 bbr->r_ctl.bbr_lost_at_state = 0;
9973 bbr->r_ctl.rc_lost_at_startup = 0;
9974 bbr->rc_all_timers_stopped = 0;
9975 bbr->r_ctl.rc_bbr_lastbtlbw = 0;
9976 bbr->r_ctl.rc_pkt_epoch_del = 0;
9977 bbr->r_ctl.rc_pkt_epoch = 0;
9978 bbr->r_ctl.rc_lowest_rtt = 0xffffffff;
9979 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;
9980 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
9981 bbr->r_ctl.rc_went_idle_time = cts;
9982 bbr->rc_pacer_started = cts;
9983 bbr->r_ctl.rc_pkt_epoch_time = cts;
9984 bbr->r_ctl.rc_rcvtime = cts;
9985 bbr->r_ctl.rc_bbr_state_time = cts;
9986 bbr->r_ctl.rc_del_time = cts;
9987 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
9988 bbr->r_ctl.last_in_probertt = cts;
9989 bbr->skip_gain = 0;
9990 bbr->gain_is_limited = 0;
9991 bbr->no_pacing_until = bbr_no_pacing_until;
9992 if (bbr->no_pacing_until)
9993 bbr->rc_no_pacing = 1;
9994 if (bbr_use_google_algo) {
9995 bbr->rc_no_pacing = 0;
9996 bbr->rc_use_google = 1;
9997 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
9998 bbr->r_use_policer = bbr_policer_detection_enabled;
9999 } else {
10000 bbr->rc_use_google = 0;
10001 bbr->r_ctl.bbr_google_discount = 0;
10002 bbr->r_use_policer = 0;
10003 }
10004 if (bbr_ts_limiting)
10005 bbr->rc_use_ts_limit = 1;
10006 else
10007 bbr->rc_use_ts_limit = 0;
10008 if (bbr_ts_can_raise)
10009 bbr->ts_can_raise = 1;
10010 else
10011 bbr->ts_can_raise = 0;
10012 if (V_tcp_delack_enabled == 1)
10013 tp->t_delayed_ack = 2;
10014 else if (V_tcp_delack_enabled == 0)
10015 tp->t_delayed_ack = 0;
10016 else if (V_tcp_delack_enabled < 100)
10017 tp->t_delayed_ack = V_tcp_delack_enabled;
10018 else
10019 tp->t_delayed_ack = 2;
10020 if (bbr->rc_use_google == 0)
10021 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10022 else
10023 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
10024 bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;
10025 bbr->rc_max_rto_sec = bbr_rto_max_sec;
10026 bbr->rc_init_win = bbr_def_init_win;
10027 if (tp->t_flags & TF_REQ_TSTMP)
10028 bbr->rc_last_options = TCP_TS_OVERHEAD;
10029 bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;
10030 bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;
10031 bbr->r_init_rtt = 1;
10032
10033 counter_u64_add(bbr_flows_nohdwr_pacing, 1);
10034 if (bbr_allow_hdwr_pacing)
10035 bbr->bbr_hdw_pace_ena = 1;
10036 else
10037 bbr->bbr_hdw_pace_ena = 0;
10038 if (bbr_sends_full_iwnd)
10039 bbr->bbr_init_win_cheat = 1;
10040 else
10041 bbr->bbr_init_win_cheat = 0;
10042 bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;
10043 bbr->r_ctl.rc_drain_pg = bbr_drain_gain;
10044 bbr->r_ctl.rc_startup_pg = bbr_high_gain;
10045 bbr->rc_loss_exit = bbr_exit_startup_at_loss;
10046 bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;
10047 bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;
10048 bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;
10049 bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;
10050 bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;
10051 bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;
10052 bbr->r_ctl.bbr_cross_over = bbr_cross_over;
10053 bbr->r_ctl.rc_rtt_shrinks = cts;
10054 if (bbr->rc_use_google) {
10055 setup_time_filter(&bbr->r_ctl.rc_delrate,
10056 FILTER_TYPE_MAX,
10057 BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT);
10058 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10059 FILTER_TYPE_MIN, (11 * USECS_IN_SECOND));
10060 } else {
10061 setup_time_filter(&bbr->r_ctl.rc_delrate,
10062 FILTER_TYPE_MAX,
10063 bbr_num_pktepo_for_del_limit);
10064 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10065 FILTER_TYPE_MIN, (bbr_filter_len_sec * USECS_IN_SECOND));
10066 }
10067 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_INIT, 0);
10068 if (bbr_uses_idle_restart)
10069 bbr->rc_use_idle_restart = 1;
10070 else
10071 bbr->rc_use_idle_restart = 0;
10072 bbr->r_ctl.rc_bbr_cur_del_rate = 0;
10073 bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;
10074 if (bbr_resends_use_tso)
10075 bbr->rc_resends_use_tso = 1;
10076 if (tp->snd_una != tp->snd_max) {
10077 /* Create a send map for the current outstanding data */
10078 struct bbr_sendmap *rsm;
10079
10080 rsm = bbr_alloc(bbr);
10081 if (rsm == NULL) {
10082 uma_zfree(bbr_pcb_zone, *ptr);
10083 *ptr = NULL;
10084 return (ENOMEM);
10085 }
10086 rsm->r_rtt_not_allowed = 1;
10087 rsm->r_tim_lastsent[0] = cts;
10088 rsm->r_rtr_cnt = 1;
10089 rsm->r_rtr_bytes = 0;
10090 rsm->r_start = tp->snd_una;
10091 rsm->r_end = tp->snd_max;
10092 rsm->r_dupack = 0;
10093 rsm->r_delivered = bbr->r_ctl.rc_delivered;
10094 rsm->r_ts_valid = 0;
10095 rsm->r_del_ack_ts = tp->ts_recent;
10096 rsm->r_del_time = cts;
10097 if (bbr->r_ctl.r_app_limited_until)
10098 rsm->r_app_limited = 1;
10099 else
10100 rsm->r_app_limited = 0;
10101 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
10102 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
10103 rsm->r_in_tmap = 1;
10104 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
10105 rsm->r_bbr_state = bbr_state_val(bbr);
10106 else
10107 rsm->r_bbr_state = 8;
10108 }
10109 if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))
10110 bbr->bbr_use_rack_cheat = 1;
10111 if (bbr_incr_timers && (bbr->rc_use_google == 0))
10112 bbr->r_ctl.rc_incr_tmrs = 1;
10113 if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))
10114 bbr->r_ctl.rc_inc_tcp_oh = 1;
10115 if (bbr_include_ip_oh && (bbr->rc_use_google == 0))
10116 bbr->r_ctl.rc_inc_ip_oh = 1;
10117 if (bbr_include_enet_oh && (bbr->rc_use_google == 0))
10118 bbr->r_ctl.rc_inc_enet_oh = 1;
10119
10120 bbr_log_type_statechange(bbr, cts, __LINE__);
10121 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
10122 (tp->t_srtt)) {
10123 uint32_t rtt;
10124
10125 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
10126 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
10127 }
10128 /* announce the settings and state */
10129 bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT);
10130 tcp_bbr_tso_size_check(bbr, cts);
10131 /*
10132 * Now call the generic function to start a timer. This will place
10133 * the TCB on the hptsi wheel if a timer is needed with appropriate
10134 * flags.
10135 */
10136 bbr_stop_all_timers(tp, bbr);
10137 /*
10138 * Validate the timers are not in usec, if they are convert.
10139 * BBR should in theory move to USEC and get rid of a
10140 * lot of the TICKS_2 calls.. but for now we stay
10141 * with tick timers.
10142 */
10143 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
10144 TCPT_RANGESET(tp->t_rxtcur,
10145 ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1,
10146 tp->t_rttmin, tcp_rexmit_max);
10147 bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0);
10148 return (0);
10149 }
10150
10151 /*
10152 * Return 0 if we can accept the connection. Return
10153 * non-zero if we can't handle the connection. A EAGAIN
10154 * means you need to wait until the connection is up.
10155 * a EADDRNOTAVAIL means we can never handle the connection
10156 * (no SACK).
10157 */
10158 static int
bbr_handoff_ok(struct tcpcb * tp)10159 bbr_handoff_ok(struct tcpcb *tp)
10160 {
10161 if ((tp->t_state == TCPS_CLOSED) ||
10162 (tp->t_state == TCPS_LISTEN)) {
10163 /* Sure no problem though it may not stick */
10164 return (0);
10165 }
10166 if ((tp->t_state == TCPS_SYN_SENT) ||
10167 (tp->t_state == TCPS_SYN_RECEIVED)) {
10168 /*
10169 * We really don't know you have to get to ESTAB or beyond
10170 * to tell.
10171 */
10172 return (EAGAIN);
10173 }
10174 if (tp->t_flags & TF_SENTFIN)
10175 return (EINVAL);
10176 if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) {
10177 return (0);
10178 }
10179 /*
10180 * If we reach here we don't do SACK on this connection so we can
10181 * never do rack.
10182 */
10183 return (EINVAL);
10184 }
10185
10186 static void
bbr_fini(struct tcpcb * tp,int32_t tcb_is_purged)10187 bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged)
10188 {
10189 if (tp->t_fb_ptr) {
10190 uint32_t calc;
10191 struct tcp_bbr *bbr;
10192 struct bbr_sendmap *rsm;
10193
10194 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10195 if (bbr->r_ctl.crte)
10196 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
10197 bbr_log_flowend(bbr);
10198 bbr->rc_tp = NULL;
10199 if (bbr->bbr_hdrw_pacing)
10200 counter_u64_add(bbr_flows_whdwr_pacing, -1);
10201 else
10202 counter_u64_add(bbr_flows_nohdwr_pacing, -1);
10203 if (bbr->r_ctl.crte != NULL) {
10204 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
10205 bbr->r_ctl.crte = NULL;
10206 }
10207 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10208 while (rsm) {
10209 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
10210 uma_zfree(bbr_zone, rsm);
10211 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10212 }
10213 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10214 while (rsm) {
10215 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
10216 uma_zfree(bbr_zone, rsm);
10217 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10218 }
10219 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;
10220 if (calc > (bbr->r_ctl.rc_init_rwnd / 10))
10221 BBR_STAT_INC(bbr_dynamic_rwnd);
10222 else
10223 BBR_STAT_INC(bbr_static_rwnd);
10224 bbr->r_ctl.rc_free_cnt = 0;
10225 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);
10226 tp->t_fb_ptr = NULL;
10227 }
10228 /* Make sure snd_nxt is correctly set */
10229 tp->snd_nxt = tp->snd_max;
10230 }
10231
10232 static void
bbr_set_state(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t win)10233 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win)
10234 {
10235 switch (tp->t_state) {
10236 case TCPS_SYN_SENT:
10237 bbr->r_state = TCPS_SYN_SENT;
10238 bbr->r_substate = bbr_do_syn_sent;
10239 break;
10240 case TCPS_SYN_RECEIVED:
10241 bbr->r_state = TCPS_SYN_RECEIVED;
10242 bbr->r_substate = bbr_do_syn_recv;
10243 break;
10244 case TCPS_ESTABLISHED:
10245 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);
10246 bbr->r_state = TCPS_ESTABLISHED;
10247 bbr->r_substate = bbr_do_established;
10248 break;
10249 case TCPS_CLOSE_WAIT:
10250 bbr->r_state = TCPS_CLOSE_WAIT;
10251 bbr->r_substate = bbr_do_close_wait;
10252 break;
10253 case TCPS_FIN_WAIT_1:
10254 bbr->r_state = TCPS_FIN_WAIT_1;
10255 bbr->r_substate = bbr_do_fin_wait_1;
10256 break;
10257 case TCPS_CLOSING:
10258 bbr->r_state = TCPS_CLOSING;
10259 bbr->r_substate = bbr_do_closing;
10260 break;
10261 case TCPS_LAST_ACK:
10262 bbr->r_state = TCPS_LAST_ACK;
10263 bbr->r_substate = bbr_do_lastack;
10264 break;
10265 case TCPS_FIN_WAIT_2:
10266 bbr->r_state = TCPS_FIN_WAIT_2;
10267 bbr->r_substate = bbr_do_fin_wait_2;
10268 break;
10269 case TCPS_LISTEN:
10270 case TCPS_CLOSED:
10271 case TCPS_TIME_WAIT:
10272 default:
10273 break;
10274 };
10275 }
10276
10277 static void
bbr_substate_change(struct tcp_bbr * bbr,uint32_t cts,int32_t line,int dolog)10278 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int32_t line, int dolog)
10279 {
10280 /*
10281 * Now what state are we going into now? Is there adjustments
10282 * needed?
10283 */
10284 int32_t old_state;
10285
10286 old_state = bbr_state_val(bbr);
10287 if (bbr_state_val(bbr) == BBR_SUB_LEVEL1) {
10288 /* Save the lowest srtt we saw in our end of the sub-state */
10289 bbr->rc_hit_state_1 = 0;
10290 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)
10291 bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;
10292 }
10293 bbr->rc_bbr_substate++;
10294 if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10295 /*
10296 * We enter the gain(5/4) cycle (possibly less if
10297 * shallow buffer detection is enabled)
10298 */
10299 if (bbr->skip_gain) {
10300 /*
10301 * Hardware pacing has set our rate to
10302 * the max and limited our b/w just
10303 * do level i.e. no gain.
10304 */
10305 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];
10306 } else if (bbr->gain_is_limited &&
10307 bbr->bbr_hdrw_pacing &&
10308 bbr->r_ctl.crte) {
10309 /*
10310 * We can't gain above the hardware pacing
10311 * rate which is less than our rate + the gain
10312 * calculate the gain needed to reach the hardware
10313 * pacing rate..
10314 */
10315 uint64_t bw, rate, gain_calc;
10316
10317 bw = bbr_get_bw(bbr);
10318 rate = bbr->r_ctl.crte->rate;
10319 if ((rate > bw) &&
10320 (((bw * (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN]) / (uint64_t)BBR_UNIT) > rate)) {
10321 gain_calc = (rate * BBR_UNIT) / bw;
10322 if (gain_calc < BBR_UNIT)
10323 gain_calc = BBR_UNIT;
10324 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;
10325 } else {
10326 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10327 }
10328 } else
10329 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10330 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {
10331 bbr->r_ctl.rc_bbr_state_atflight = cts;
10332 } else
10333 bbr->r_ctl.rc_bbr_state_atflight = 0;
10334 } else if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10335 bbr->rc_hit_state_1 = 1;
10336 bbr->r_ctl.rc_exta_time_gd = 0;
10337 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10338 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10339 if (bbr_state_drain_2_tar) {
10340 bbr->r_ctl.rc_bbr_state_atflight = 0;
10341 } else
10342 bbr->r_ctl.rc_bbr_state_atflight = cts;
10343 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];
10344 } else {
10345 /* All other cycles hit here 2-7 */
10346 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {
10347 if (bbr_sub_drain_slam_cwnd &&
10348 (bbr->rc_use_google == 0) &&
10349 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10350 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10351 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10352 }
10353 if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))
10354 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -
10355 bbr_get_rtt(bbr, BBR_RTT_PROP));
10356 else
10357 bbr->r_ctl.rc_exta_time_gd = 0;
10358 if (bbr->r_ctl.rc_exta_time_gd) {
10359 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;
10360 /* Now chop up the time for each state (div by 7) */
10361 bbr->r_ctl.rc_level_state_extra /= 7;
10362 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {
10363 /* Add a randomization */
10364 bbr_randomize_extra_state_time(bbr);
10365 }
10366 }
10367 }
10368 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10369 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];
10370 }
10371 if (bbr->rc_use_google) {
10372 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10373 }
10374 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10375 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10376 if (dolog)
10377 bbr_log_type_statechange(bbr, cts, line);
10378
10379 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10380 uint32_t time_in;
10381
10382 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10383 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
10384 counter_u64_add(bbr_state_time[(old_state + 5)], time_in);
10385 } else {
10386 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10387 }
10388 }
10389 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
10390 bbr_set_state_target(bbr, __LINE__);
10391 if (bbr_sub_drain_slam_cwnd &&
10392 (bbr->rc_use_google == 0) &&
10393 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10394 /* Slam down the cwnd */
10395 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10396 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10397 if (bbr_sub_drain_app_limit) {
10398 /* Go app limited if we are on a long drain */
10399 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +
10400 ctf_flight_size(bbr->rc_tp,
10401 (bbr->r_ctl.rc_sacked +
10402 bbr->r_ctl.rc_lost_bytes)));
10403 }
10404 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10405 }
10406 if (bbr->rc_lt_use_bw) {
10407 /* In policed mode we clamp pacing_gain to BBR_UNIT */
10408 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10409 }
10410 /* Google changes TSO size every cycle */
10411 if (bbr->rc_use_google)
10412 tcp_bbr_tso_size_check(bbr, cts);
10413 bbr->r_ctl.gain_epoch = cts;
10414 bbr->r_ctl.rc_bbr_state_time = cts;
10415 bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;
10416 }
10417
10418 static void
bbr_set_probebw_google_gains(struct tcp_bbr * bbr,uint32_t cts,uint32_t losses)10419 bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10420 {
10421 if ((bbr_state_val(bbr) == BBR_SUB_DRAIN) &&
10422 (google_allow_early_out == 1) &&
10423 (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {
10424 /* We have reached out target flight size possibly early */
10425 goto change_state;
10426 }
10427 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10428 return;
10429 }
10430 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {
10431 /*
10432 * Must be a rttProp movement forward before
10433 * we can change states.
10434 */
10435 return;
10436 }
10437 if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10438 /*
10439 * The needed time has passed but for
10440 * the gain cycle extra rules apply:
10441 * 1) If we have seen loss, we exit
10442 * 2) If we have not reached the target
10443 * we stay in GAIN (gain-to-target).
10444 */
10445 if (google_consider_lost && losses)
10446 goto change_state;
10447 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {
10448 return;
10449 }
10450 }
10451 change_state:
10452 /* For gain we must reach our target, all others last 1 rttProp */
10453 bbr_substate_change(bbr, cts, __LINE__, 1);
10454 }
10455
10456 static void
bbr_set_probebw_gains(struct tcp_bbr * bbr,uint32_t cts,uint32_t losses)10457 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10458 {
10459 uint32_t flight, bbr_cur_cycle_time;
10460
10461 if (bbr->rc_use_google) {
10462 bbr_set_probebw_google_gains(bbr, cts, losses);
10463 return;
10464 }
10465 if (cts == 0) {
10466 /*
10467 * Never alow cts to be 0 we
10468 * do this so we can judge if
10469 * we have set a timestamp.
10470 */
10471 cts = 1;
10472 }
10473 if (bbr_state_is_pkt_epoch)
10474 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
10475 else
10476 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PROP);
10477
10478 if (bbr->r_ctl.rc_bbr_state_atflight == 0) {
10479 if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10480 flight = ctf_flight_size(bbr->rc_tp,
10481 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10482 if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {
10483 /* Keep it slam down */
10484 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {
10485 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10486 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10487 }
10488 if (bbr_sub_drain_app_limit) {
10489 /* Go app limited if we are on a long drain */
10490 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);
10491 }
10492 }
10493 if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&
10494 (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||
10495 (flight >= bbr->r_ctl.flightsize_at_drain))) {
10496 /*
10497 * Still here after the same time as
10498 * the gain. We need to drain harder
10499 * for the next srtt. Reduce by a set amount
10500 * the gain drop is capped at DRAIN states
10501 * value (88).
10502 */
10503 bbr->r_ctl.flightsize_at_drain = flight;
10504 if (bbr_drain_drop_mul &&
10505 bbr_drain_drop_div &&
10506 (bbr_drain_drop_mul < bbr_drain_drop_div)) {
10507 /* Use your specific drop value (def 4/5 = 20%) */
10508 bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;
10509 bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;
10510 } else {
10511 /* You get drop of 20% */
10512 bbr->r_ctl.rc_bbr_hptsi_gain *= 4;
10513 bbr->r_ctl.rc_bbr_hptsi_gain /= 5;
10514 }
10515 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {
10516 /* Reduce our gain again to the bottom */
10517 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
10518 }
10519 bbr_log_exit_gain(bbr, cts, 4);
10520 /*
10521 * Extend out so we wait another
10522 * epoch before dropping again.
10523 */
10524 bbr->r_ctl.gain_epoch = cts;
10525 }
10526 if (flight <= bbr->r_ctl.rc_target_at_state) {
10527 if (bbr_sub_drain_slam_cwnd &&
10528 (bbr->rc_use_google == 0) &&
10529 (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10530 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10531 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10532 }
10533 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10534 bbr_log_exit_gain(bbr, cts, 3);
10535 }
10536 } else {
10537 /* Its a gain */
10538 if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {
10539 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10540 goto change_state;
10541 }
10542 if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||
10543 ((ctf_outstanding(bbr->rc_tp) + bbr->rc_tp->t_maxseg - 1) >=
10544 bbr->rc_tp->snd_wnd)) {
10545 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10546 bbr_log_exit_gain(bbr, cts, 2);
10547 }
10548 }
10549 /**
10550 * We fall through and return always one of two things has
10551 * occurred.
10552 * 1) We are still not at target
10553 * <or>
10554 * 2) We reached the target and set rc_bbr_state_atflight
10555 * which means we no longer hit this block
10556 * next time we are called.
10557 */
10558 return;
10559 }
10560 change_state:
10561 if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))
10562 return;
10563 if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {
10564 /* Less than a full time-period has passed */
10565 return;
10566 }
10567 if (bbr->r_ctl.rc_level_state_extra &&
10568 (bbr_state_val(bbr) > BBR_SUB_DRAIN) &&
10569 ((cts - bbr->r_ctl.rc_bbr_state_time) <
10570 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10571 /* Less than a full time-period + extra has passed */
10572 return;
10573 }
10574 if (bbr_gain_gets_extra_too &&
10575 bbr->r_ctl.rc_level_state_extra &&
10576 (bbr_state_val(bbr) == BBR_SUB_GAIN) &&
10577 ((cts - bbr->r_ctl.rc_bbr_state_time) <
10578 (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10579 /* Less than a full time-period + extra has passed */
10580 return;
10581 }
10582 bbr_substate_change(bbr, cts, __LINE__, 1);
10583 }
10584
10585 static uint32_t
bbr_get_a_state_target(struct tcp_bbr * bbr,uint32_t gain)10586 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain)
10587 {
10588 uint32_t mss, tar;
10589
10590 if (bbr->rc_use_google) {
10591 /* Google just uses the cwnd target */
10592 tar = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), gain);
10593 } else {
10594 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
10595 bbr->r_ctl.rc_pace_max_segs);
10596 /* Get the base cwnd with gain rounded to a mss */
10597 tar = roundup(bbr_get_raw_target_cwnd(bbr, bbr_get_bw(bbr),
10598 gain), mss);
10599 /* Make sure it is within our min */
10600 if (tar < get_min_cwnd(bbr))
10601 return (get_min_cwnd(bbr));
10602 }
10603 return (tar);
10604 }
10605
10606 static void
bbr_set_state_target(struct tcp_bbr * bbr,int line)10607 bbr_set_state_target(struct tcp_bbr *bbr, int line)
10608 {
10609 uint32_t tar, meth;
10610
10611 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
10612 ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
10613 /* Special case using old probe-rtt method */
10614 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10615 meth = 1;
10616 } else {
10617 /* Non-probe-rtt case and reduced probe-rtt */
10618 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
10619 (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {
10620 /* For gain cycle we use the hptsi gain */
10621 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10622 meth = 2;
10623 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {
10624 /*
10625 * If configured, or for google all other states
10626 * get BBR_UNIT.
10627 */
10628 tar = bbr_get_a_state_target(bbr, BBR_UNIT);
10629 meth = 3;
10630 } else {
10631 /*
10632 * Or we set a target based on the pacing gain
10633 * for non-google mode and default (non-configured).
10634 * Note we don't set a target goal below drain (192).
10635 */
10636 if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN]) {
10637 tar = bbr_get_a_state_target(bbr, bbr_hptsi_gain[BBR_SUB_DRAIN]);
10638 meth = 4;
10639 } else {
10640 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10641 meth = 5;
10642 }
10643 }
10644 }
10645 bbr_log_set_of_state_target(bbr, tar, line, meth);
10646 bbr->r_ctl.rc_target_at_state = tar;
10647 }
10648
10649 static void
bbr_enter_probe_rtt(struct tcp_bbr * bbr,uint32_t cts,int32_t line)10650 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
10651 {
10652 /* Change to probe_rtt */
10653 uint32_t time_in;
10654
10655 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10656 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10657 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10658 bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain
10659 + bbr->r_ctl.rc_delivered);
10660 /* Setup so we force feed the filter */
10661 if (bbr->rc_use_google || bbr_probertt_sets_rtt)
10662 bbr->rc_prtt_set_ts = 1;
10663 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10664 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10665 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10666 }
10667 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_ENTERPROBE, 0);
10668 bbr->r_ctl.rc_rtt_shrinks = cts;
10669 bbr->r_ctl.last_in_probertt = cts;
10670 bbr->r_ctl.rc_probertt_srttchktim = cts;
10671 bbr->r_ctl.rc_bbr_state_time = cts;
10672 bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;
10673 /* We need to force the filter to update */
10674
10675 if ((bbr_sub_drain_slam_cwnd) &&
10676 bbr->rc_hit_state_1 &&
10677 (bbr->rc_use_google == 0) &&
10678 (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10679 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)
10680 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10681 } else
10682 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10683 /* Update the lost */
10684 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10685 if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){
10686 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN) */
10687 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10688 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10689 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10690 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10691 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);
10692 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;
10693 } else {
10694 /*
10695 * We bring it down slowly by using a hptsi gain that is
10696 * probably 75%. This will slowly float down our outstanding
10697 * without tampering with the cwnd.
10698 */
10699 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
10700 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10701 bbr_set_state_target(bbr, __LINE__);
10702 if (bbr_prtt_slam_cwnd &&
10703 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
10704 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10705 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10706 }
10707 }
10708 if (ctf_flight_size(bbr->rc_tp,
10709 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
10710 bbr->r_ctl.rc_target_at_state) {
10711 /* We are at target */
10712 bbr->r_ctl.rc_bbr_enters_probertt = cts;
10713 } else {
10714 /* We need to come down to reach target before our time begins */
10715 bbr->r_ctl.rc_bbr_enters_probertt = 0;
10716 }
10717 bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;
10718 BBR_STAT_INC(bbr_enter_probertt);
10719 bbr_log_exit_gain(bbr, cts, 0);
10720 bbr_log_type_statechange(bbr, cts, line);
10721 }
10722
10723 static void
bbr_check_probe_rtt_limits(struct tcp_bbr * bbr,uint32_t cts)10724 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts)
10725 {
10726 /*
10727 * Sanity check on probe-rtt intervals.
10728 * In crazy situations where we are competing
10729 * against new-reno flows with huge buffers
10730 * our rtt-prop interval could come to dominate
10731 * things if we can't get through a full set
10732 * of cycles, we need to adjust it.
10733 */
10734 if (bbr_can_adjust_probertt &&
10735 (bbr->rc_use_google == 0)) {
10736 uint16_t val = 0;
10737 uint32_t cur_rttp, fval, newval, baseval;
10738
10739 /* Are we to small and go into probe-rtt to often? */
10740 baseval = (bbr_get_rtt(bbr, BBR_RTT_PROP) * (BBR_SUBSTATE_COUNT + 1));
10741 cur_rttp = roundup(baseval, USECS_IN_SECOND);
10742 fval = bbr_filter_len_sec * USECS_IN_SECOND;
10743 if (bbr_is_ratio == 0) {
10744 if (fval > bbr_rtt_probe_limit)
10745 newval = cur_rttp + (fval - bbr_rtt_probe_limit);
10746 else
10747 newval = cur_rttp;
10748 } else {
10749 int mul;
10750
10751 mul = fval / bbr_rtt_probe_limit;
10752 newval = cur_rttp * mul;
10753 }
10754 if (cur_rttp > bbr->r_ctl.rc_probertt_int) {
10755 bbr->r_ctl.rc_probertt_int = cur_rttp;
10756 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10757 val = 1;
10758 } else {
10759 /*
10760 * No adjustments were made
10761 * do we need to shrink it?
10762 */
10763 if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {
10764 if (cur_rttp <= bbr_rtt_probe_limit) {
10765 /*
10766 * Things have calmed down lets
10767 * shrink all the way to default
10768 */
10769 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10770 reset_time_small(&bbr->r_ctl.rc_rttprop,
10771 (bbr_filter_len_sec * USECS_IN_SECOND));
10772 cur_rttp = bbr_rtt_probe_limit;
10773 newval = (bbr_filter_len_sec * USECS_IN_SECOND);
10774 val = 2;
10775 } else {
10776 /*
10777 * Well does some adjustment make sense?
10778 */
10779 if (cur_rttp < bbr->r_ctl.rc_probertt_int) {
10780 /* We can reduce interval time some */
10781 bbr->r_ctl.rc_probertt_int = cur_rttp;
10782 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
10783 val = 3;
10784 }
10785 }
10786 }
10787 }
10788 if (val)
10789 bbr_log_rtt_shrinks(bbr, cts, cur_rttp, newval, __LINE__, BBR_RTTS_RESETS_VALUES, val);
10790 }
10791 }
10792
10793 static void
bbr_exit_probe_rtt(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t cts)10794 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
10795 {
10796 /* Exit probe-rtt */
10797
10798 if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {
10799 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10800 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10801 }
10802 bbr_log_exit_gain(bbr, cts, 1);
10803 bbr->rc_hit_state_1 = 0;
10804 bbr->r_ctl.rc_rtt_shrinks = cts;
10805 bbr->r_ctl.last_in_probertt = cts;
10806 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_RTTPROBE, 0);
10807 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10808 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,
10809 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
10810 bbr->r_ctl.rc_delivered);
10811 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10812 uint32_t time_in;
10813
10814 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10815 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10816 }
10817 if (bbr->rc_filled_pipe) {
10818 /* Switch to probe_bw */
10819 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
10820 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
10821 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10822 bbr_substate_change(bbr, cts, __LINE__, 0);
10823 bbr_log_type_statechange(bbr, cts, __LINE__);
10824 } else {
10825 /* Back to startup */
10826 bbr->rc_bbr_state = BBR_STATE_STARTUP;
10827 bbr->r_ctl.rc_bbr_state_time = cts;
10828 /*
10829 * We don't want to give a complete free 3
10830 * measurements until we exit, so we use
10831 * the number of pe's we were in probe-rtt
10832 * to add to the startup_epoch. That way
10833 * we will still retain the old state.
10834 */
10835 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);
10836 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10837 /* Make sure to use the lower pg when shifting back in */
10838 if (bbr->r_ctl.rc_lost &&
10839 bbr_use_lower_gain_in_startup &&
10840 (bbr->rc_use_google == 0))
10841 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10842 else
10843 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
10844 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
10845 /* Probably not needed but set it anyway */
10846 bbr_set_state_target(bbr, __LINE__);
10847 bbr_log_type_statechange(bbr, cts, __LINE__);
10848 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10849 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);
10850 }
10851 bbr_check_probe_rtt_limits(bbr, cts);
10852 }
10853
10854 static int32_t inline
bbr_should_enter_probe_rtt(struct tcp_bbr * bbr,uint32_t cts)10855 bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts)
10856 {
10857 if ((bbr->rc_past_init_win == 1) &&
10858 (bbr->rc_in_persist == 0) &&
10859 (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {
10860 return (1);
10861 }
10862 if (bbr_can_force_probertt &&
10863 (bbr->rc_in_persist == 0) &&
10864 (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
10865 ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
10866 return (1);
10867 }
10868 return (0);
10869 }
10870
10871 static int32_t
bbr_google_startup(struct tcp_bbr * bbr,uint32_t cts,int32_t pkt_epoch)10872 bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t pkt_epoch)
10873 {
10874 uint64_t btlbw, gain;
10875 if (pkt_epoch == 0) {
10876 /*
10877 * Need to be on a pkt-epoch to continue.
10878 */
10879 return (0);
10880 }
10881 btlbw = bbr_get_full_bw(bbr);
10882 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10883 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10884 if (btlbw >= gain) {
10885 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
10886 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10887 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
10888 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
10889 }
10890 if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)
10891 return (1);
10892 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10893 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
10894 return(0);
10895 }
10896
10897 static int32_t inline
bbr_state_startup(struct tcp_bbr * bbr,uint32_t cts,int32_t epoch,int32_t pkt_epoch)10898 bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch)
10899 {
10900 /* Have we gained 25% in the last 3 packet based epoch's? */
10901 uint64_t btlbw, gain;
10902 int do_exit;
10903 int delta, rtt_gain;
10904
10905 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
10906 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
10907 /*
10908 * This qualifies as a RTT_PROBE session since we drop the
10909 * data outstanding to nothing and waited more than
10910 * bbr_rtt_probe_time.
10911 */
10912 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
10913 bbr_set_reduced_rtt(bbr, cts, __LINE__);
10914 }
10915 if (bbr_should_enter_probe_rtt(bbr, cts)) {
10916 bbr_enter_probe_rtt(bbr, cts, __LINE__);
10917 return (0);
10918 }
10919 if (bbr->rc_use_google)
10920 return (bbr_google_startup(bbr, cts, pkt_epoch));
10921
10922 if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
10923 (bbr_use_lower_gain_in_startup)) {
10924 /* Drop to a lower gain 1.5 x since we saw loss */
10925 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
10926 }
10927 if (pkt_epoch == 0) {
10928 /*
10929 * Need to be on a pkt-epoch to continue.
10930 */
10931 return (0);
10932 }
10933 if (bbr_rtt_gain_thresh) {
10934 /*
10935 * Do we allow a flow to stay
10936 * in startup with no loss and no
10937 * gain in rtt over a set threshold?
10938 */
10939 if (bbr->r_ctl.rc_pkt_epoch_rtt &&
10940 bbr->r_ctl.startup_last_srtt &&
10941 (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {
10942 delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;
10943 rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;
10944 } else
10945 rtt_gain = 0;
10946 if ((bbr->r_ctl.startup_last_srtt == 0) ||
10947 (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))
10948 /* First time or new lower value */
10949 bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
10950
10951 if ((bbr->r_ctl.rc_lost == 0) &&
10952 (rtt_gain < bbr_rtt_gain_thresh)) {
10953 /*
10954 * No loss, and we are under
10955 * our gain threhold for
10956 * increasing RTT.
10957 */
10958 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
10959 bbr->r_ctl.rc_bbr_last_startup_epoch++;
10960 bbr_log_startup_event(bbr, cts, rtt_gain,
10961 delta, bbr->r_ctl.startup_last_srtt, 10);
10962 return (0);
10963 }
10964 }
10965 if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&
10966 (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&
10967 (!IN_RECOVERY(bbr->rc_tp->t_flags))) {
10968 /*
10969 * We only assess if we have a new measurement when
10970 * we have no loss and are not in recovery.
10971 * Drag up by one our last_startup epoch so we will hold
10972 * the number of non-gain we have already accumulated.
10973 */
10974 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
10975 bbr->r_ctl.rc_bbr_last_startup_epoch++;
10976 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10977 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);
10978 return (0);
10979 }
10980 /* Case where we reduced the lost (bad retransmit) */
10981 if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)
10982 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10983 bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;
10984 btlbw = bbr_get_full_bw(bbr);
10985 if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)
10986 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10987 (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10988 else
10989 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
10990 (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
10991 do_exit = 0;
10992 if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)
10993 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
10994 if (btlbw >= gain) {
10995 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
10996 /* Update the lost so we won't exit in next set of tests */
10997 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10998 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
10999 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
11000 }
11001 if ((bbr->rc_loss_exit &&
11002 (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
11003 (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&
11004 ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {
11005 /*
11006 * If we had no gain, we had loss and that loss was above
11007 * our threshould, the rwnd is not constrained, and we have
11008 * had at least 3 packet epochs exit. Note that this is
11009 * switched off by sysctl. Google does not do this by the
11010 * way.
11011 */
11012 if ((ctf_flight_size(bbr->rc_tp,
11013 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
11014 (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {
11015 do_exit = 1;
11016 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11017 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);
11018 } else {
11019 /* Just record an updated loss value */
11020 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11021 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11022 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);
11023 }
11024 } else
11025 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11026 if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||
11027 do_exit) {
11028 /* Return 1 to exit the startup state. */
11029 return (1);
11030 }
11031 /* Stay in startup */
11032 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11033 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
11034 return (0);
11035 }
11036
11037 static void
bbr_state_change(struct tcp_bbr * bbr,uint32_t cts,int32_t epoch,int32_t pkt_epoch,uint32_t losses)11038 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t losses)
11039 {
11040 /*
11041 * A tick occurred in the rtt epoch do we need to do anything?
11042 */
11043 #ifdef BBR_INVARIANTS
11044 if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
11045 (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&
11046 (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&
11047 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
11048 (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {
11049 /* Debug code? */
11050 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);
11051 }
11052 #endif
11053 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
11054 /* Do we exit the startup state? */
11055 if (bbr_state_startup(bbr, cts, epoch, pkt_epoch)) {
11056 uint32_t time_in;
11057
11058 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11059 bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);
11060 bbr->rc_filled_pipe = 1;
11061 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11062 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11063 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11064 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11065 } else
11066 time_in = 0;
11067 if (bbr->rc_no_pacing)
11068 bbr->rc_no_pacing = 0;
11069 bbr->r_ctl.rc_bbr_state_time = cts;
11070 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;
11071 bbr->rc_bbr_state = BBR_STATE_DRAIN;
11072 bbr_set_state_target(bbr, __LINE__);
11073 if ((bbr->rc_use_google == 0) &&
11074 bbr_slam_cwnd_in_main_drain) {
11075 /* Here we don't have to worry about probe-rtt */
11076 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
11077 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11078 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11079 }
11080 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
11081 bbr_log_type_statechange(bbr, cts, __LINE__);
11082 if (ctf_flight_size(bbr->rc_tp,
11083 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
11084 bbr->r_ctl.rc_target_at_state) {
11085 /*
11086 * Switch to probe_bw if we are already
11087 * there
11088 */
11089 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11090 bbr_substate_change(bbr, cts, __LINE__, 0);
11091 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11092 bbr_log_type_statechange(bbr, cts, __LINE__);
11093 }
11094 }
11095 } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {
11096 uint32_t inflight;
11097 struct tcpcb *tp;
11098
11099 tp = bbr->rc_tp;
11100 inflight = ctf_flight_size(tp,
11101 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11102 if (inflight >= bbr->r_ctl.rc_target_at_state) {
11103 /* We have reached a flight of the cwnd target */
11104 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11105 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11106 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
11107 bbr_set_state_target(bbr, __LINE__);
11108 /*
11109 * Rig it so we don't do anything crazy and
11110 * start fresh with a new randomization.
11111 */
11112 bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
11113 bbr->rc_bbr_substate = BBR_SUB_LEVEL6;
11114 bbr_substate_change(bbr, cts, __LINE__, 1);
11115 }
11116 } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {
11117 /* Has in-flight reached the bdp (or less)? */
11118 uint32_t inflight;
11119 struct tcpcb *tp;
11120
11121 tp = bbr->rc_tp;
11122 inflight = ctf_flight_size(tp,
11123 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11124 if ((bbr->rc_use_google == 0) &&
11125 bbr_slam_cwnd_in_main_drain &&
11126 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11127 /*
11128 * Here we don't have to worry about probe-rtt
11129 * re-slam it, but keep it slammed down.
11130 */
11131 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11132 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11133 }
11134 if (inflight <= bbr->r_ctl.rc_target_at_state) {
11135 /* We have drained */
11136 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11137 bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11138 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11139 uint32_t time_in;
11140
11141 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11142 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11143 }
11144 if ((bbr->rc_use_google == 0) &&
11145 bbr_slam_cwnd_in_main_drain &&
11146 (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
11147 /* Restore the cwnd */
11148 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
11149 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11150 }
11151 /* Setup probe-rtt has being done now RRS-HERE */
11152 bbr->r_ctl.rc_rtt_shrinks = cts;
11153 bbr->r_ctl.last_in_probertt = cts;
11154 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_LEAVE_DRAIN, 0);
11155 /* Randomly pick a sub-state */
11156 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11157 bbr_substate_change(bbr, cts, __LINE__, 0);
11158 bbr_log_type_statechange(bbr, cts, __LINE__);
11159 }
11160 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {
11161 uint32_t flight;
11162
11163 flight = ctf_flight_size(bbr->rc_tp,
11164 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11165 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);
11166 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&
11167 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11168 /*
11169 * We must keep cwnd at the desired MSS.
11170 */
11171 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
11172 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11173 } else if ((bbr_prtt_slam_cwnd) &&
11174 (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11175 /* Re-slam it */
11176 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11177 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11178 }
11179 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {
11180 /* Has outstanding reached our target? */
11181 if (flight <= bbr->r_ctl.rc_target_at_state) {
11182 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_REACHTAR, 0);
11183 bbr->r_ctl.rc_bbr_enters_probertt = cts;
11184 /* If time is exactly 0, be 1usec off */
11185 if (bbr->r_ctl.rc_bbr_enters_probertt == 0)
11186 bbr->r_ctl.rc_bbr_enters_probertt = 1;
11187 if (bbr->rc_use_google == 0) {
11188 /*
11189 * Restore any lowering that as occurred to
11190 * reach here
11191 */
11192 if (bbr->r_ctl.bbr_rttprobe_gain_val)
11193 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
11194 else
11195 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11196 }
11197 }
11198 if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&
11199 (bbr->rc_use_google == 0) &&
11200 bbr->r_ctl.bbr_rttprobe_gain_val &&
11201 (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||
11202 (flight >= bbr->r_ctl.flightsize_at_drain))) {
11203 /*
11204 * We have doddled with our current hptsi
11205 * gain an srtt and have still not made it
11206 * to target, or we have increased our flight.
11207 * Lets reduce the gain by xx%
11208 * flooring the reduce at DRAIN (based on
11209 * mul/div)
11210 */
11211 int red;
11212
11213 bbr->r_ctl.flightsize_at_drain = flight;
11214 bbr->r_ctl.rc_probertt_srttchktim = cts;
11215 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);
11216 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {
11217 /* Reduce our gain again */
11218 bbr->r_ctl.rc_bbr_hptsi_gain -= red;
11219 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG, 0);
11220 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {
11221 /* one more chance before we give up */
11222 bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
11223 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG_FINAL, 0);
11224 } else {
11225 /* At the very bottom */
11226 bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);
11227 }
11228 }
11229 }
11230 if (bbr->r_ctl.rc_bbr_enters_probertt &&
11231 (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&
11232 ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {
11233 /* Time to exit probe RTT normally */
11234 bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);
11235 }
11236 } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
11237 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
11238 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
11239 /*
11240 * This qualifies as a RTT_PROBE session since we
11241 * drop the data outstanding to nothing and waited
11242 * more than bbr_rtt_probe_time.
11243 */
11244 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
11245 bbr_set_reduced_rtt(bbr, cts, __LINE__);
11246 }
11247 if (bbr_should_enter_probe_rtt(bbr, cts)) {
11248 bbr_enter_probe_rtt(bbr, cts, __LINE__);
11249 } else {
11250 bbr_set_probebw_gains(bbr, cts, losses);
11251 }
11252 }
11253 }
11254
11255 static void
bbr_check_bbr_for_state(struct tcp_bbr * bbr,uint32_t cts,int32_t line,uint32_t losses)11256 bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses)
11257 {
11258 int32_t epoch = 0;
11259
11260 if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
11261 bbr_set_epoch(bbr, cts, line);
11262 /* At each epoch doe lt bw sampling */
11263 epoch = 1;
11264 }
11265 bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);
11266 }
11267
11268 static int
bbr_do_segment_nounlock(struct tcpcb * tp,struct mbuf * m,struct tcphdr * th,int32_t drop_hdrlen,int32_t tlen,uint8_t iptos,int32_t nxt_pkt,struct timeval * tv)11269 bbr_do_segment_nounlock(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
11270 int32_t drop_hdrlen, int32_t tlen, uint8_t iptos, int32_t nxt_pkt,
11271 struct timeval *tv)
11272 {
11273 struct inpcb *inp = tptoinpcb(tp);
11274 struct socket *so = tptosocket(tp);
11275 int32_t thflags, retval;
11276 uint32_t cts, lcts;
11277 uint32_t tiwin;
11278 struct tcpopt to;
11279 struct tcp_bbr *bbr;
11280 struct bbr_sendmap *rsm;
11281 struct timeval ltv;
11282 int32_t did_out = 0;
11283 uint16_t nsegs;
11284 int32_t prev_state;
11285 uint32_t lost;
11286
11287 nsegs = max(1, m->m_pkthdr.lro_nsegs);
11288 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11289 /* add in our stats */
11290 kern_prefetch(bbr, &prev_state);
11291 prev_state = 0;
11292 thflags = tcp_get_flags(th);
11293 /*
11294 * If this is either a state-changing packet or current state isn't
11295 * established, we require a write lock on tcbinfo. Otherwise, we
11296 * allow the tcbinfo to be in either alocked or unlocked, as the
11297 * caller may have unnecessarily acquired a write lock due to a
11298 * race.
11299 */
11300 INP_WLOCK_ASSERT(tptoinpcb(tp));
11301 KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
11302 __func__));
11303 KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
11304 __func__));
11305
11306 tp->t_rcvtime = ticks;
11307 /*
11308 * Unscale the window into a 32-bit value. For the SYN_SENT state
11309 * the scale is zero.
11310 */
11311 tiwin = th->th_win << tp->snd_scale;
11312 #ifdef STATS
11313 stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
11314 #endif
11315
11316 if (m->m_flags & M_TSTMP) {
11317 /* Prefer the hardware timestamp if present */
11318 struct timespec ts;
11319
11320 mbuf_tstmp2timespec(m, &ts);
11321 bbr->rc_tv.tv_sec = ts.tv_sec;
11322 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11323 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usec(&bbr->rc_tv);
11324 } else if (m->m_flags & M_TSTMP_LRO) {
11325 /* Next the arrival timestamp */
11326 struct timespec ts;
11327
11328 mbuf_tstmp2timespec(m, &ts);
11329 bbr->rc_tv.tv_sec = ts.tv_sec;
11330 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11331 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usec(&bbr->rc_tv);
11332 } else {
11333 /*
11334 * Ok just get the current time.
11335 */
11336 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);
11337 }
11338 /*
11339 * Parse options on any incoming segment.
11340 */
11341 tcp_dooptions(&to, (u_char *)(th + 1),
11342 (th->th_off << 2) - sizeof(struct tcphdr),
11343 (thflags & TH_SYN) ? TO_SYN : 0);
11344 if (tp->t_flags2 & TF2_PROC_SACK_PROHIBIT) {
11345 /*
11346 * We don't look at sack's from the
11347 * peer because the MSS is too small which
11348 * can subject us to an attack.
11349 */
11350 to.to_flags &= ~TOF_SACK;
11351 }
11352 /*
11353 * If timestamps were negotiated during SYN/ACK and a
11354 * segment without a timestamp is received, silently drop
11355 * the segment, unless it is a RST segment or missing timestamps are
11356 * tolerated.
11357 * See section 3.2 of RFC 7323.
11358 */
11359 if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&
11360 ((thflags & TH_RST) == 0) && (V_tcp_tolerate_missing_ts == 0)) {
11361 retval = 0;
11362 m_freem(m);
11363 goto done_with_input;
11364 }
11365 /*
11366 * If echoed timestamp is later than the current time, fall back to
11367 * non RFC1323 RTT calculation. Normalize timestamp if syncookies
11368 * were used when this connection was established.
11369 */
11370 if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
11371 to.to_tsecr -= tp->ts_offset;
11372 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_msec(&bbr->rc_tv)))
11373 to.to_tsecr = 0;
11374 }
11375 /*
11376 * If its the first time in we need to take care of options and
11377 * verify we can do SACK for rack!
11378 */
11379 if (bbr->r_state == 0) {
11380 /*
11381 * Process options only when we get SYN/ACK back. The SYN
11382 * case for incoming connections is handled in tcp_syncache.
11383 * According to RFC1323 the window field in a SYN (i.e., a
11384 * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX
11385 * this is traditional behavior, may need to be cleaned up.
11386 */
11387 if (bbr->rc_inp == NULL) {
11388 bbr->rc_inp = inp;
11389 }
11390 /*
11391 * We need to init rc_inp here since its not init'd when
11392 * bbr_init is called
11393 */
11394 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
11395 if ((to.to_flags & TOF_SCALE) &&
11396 (tp->t_flags & TF_REQ_SCALE)) {
11397 tp->t_flags |= TF_RCVD_SCALE;
11398 tp->snd_scale = to.to_wscale;
11399 } else
11400 tp->t_flags &= ~TF_REQ_SCALE;
11401 /*
11402 * Initial send window. It will be updated with the
11403 * next incoming segment to the scaled value.
11404 */
11405 tp->snd_wnd = th->th_win;
11406 if ((to.to_flags & TOF_TS) &&
11407 (tp->t_flags & TF_REQ_TSTMP)) {
11408 tp->t_flags |= TF_RCVD_TSTMP;
11409 tp->ts_recent = to.to_tsval;
11410 tp->ts_recent_age = tcp_tv_to_msec(&bbr->rc_tv);
11411 } else
11412 tp->t_flags &= ~TF_REQ_TSTMP;
11413 if (to.to_flags & TOF_MSS)
11414 tcp_mss(tp, to.to_mss);
11415 if ((tp->t_flags & TF_SACK_PERMIT) &&
11416 (to.to_flags & TOF_SACKPERM) == 0)
11417 tp->t_flags &= ~TF_SACK_PERMIT;
11418 if (tp->t_flags & TF_FASTOPEN) {
11419 if (to.to_flags & TOF_FASTOPEN) {
11420 uint16_t mss;
11421
11422 if (to.to_flags & TOF_MSS)
11423 mss = to.to_mss;
11424 else
11425 if ((inp->inp_vflag & INP_IPV6) != 0)
11426 mss = TCP6_MSS;
11427 else
11428 mss = TCP_MSS;
11429 tcp_fastopen_update_cache(tp, mss,
11430 to.to_tfo_len, to.to_tfo_cookie);
11431 } else
11432 tcp_fastopen_disable_path(tp);
11433 }
11434 }
11435 /*
11436 * At this point we are at the initial call. Here we decide
11437 * if we are doing RACK or not. We do this by seeing if
11438 * TF_SACK_PERMIT is set, if not rack is *not* possible and
11439 * we switch to the default code.
11440 */
11441 if ((tp->t_flags & TF_SACK_PERMIT) == 0) {
11442 /* Bail */
11443 tcp_switch_back_to_default(tp);
11444 (*tp->t_fb->tfb_tcp_do_segment)(tp, m, th, drop_hdrlen,
11445 tlen, iptos);
11446 return (1);
11447 }
11448 /* Set the flag */
11449 bbr->r_is_v6 = (inp->inp_vflag & INP_IPV6) != 0;
11450 tcp_set_hpts(tp);
11451 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);
11452 }
11453 if (thflags & TH_ACK) {
11454 /* Track ack types */
11455 if (to.to_flags & TOF_SACK)
11456 BBR_STAT_INC(bbr_acks_with_sacks);
11457 else
11458 BBR_STAT_INC(bbr_plain_acks);
11459 }
11460 /*
11461 * This is the one exception case where we set the rack state
11462 * always. All other times (timers etc) we must have a rack-state
11463 * set (so we assure we have done the checks above for SACK).
11464 */
11465 if (thflags & TH_FIN)
11466 tcp_log_end_status(tp, TCP_EI_STATUS_CLIENT_FIN);
11467 if (bbr->r_state != tp->t_state)
11468 bbr_set_state(tp, bbr, tiwin);
11469
11470 if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)
11471 kern_prefetch(rsm, &prev_state);
11472 prev_state = bbr->r_state;
11473 bbr->rc_ack_was_delayed = 0;
11474 lost = bbr->r_ctl.rc_lost;
11475 bbr->rc_is_pkt_epoch_now = 0;
11476 if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) {
11477 /* Get the real time into lcts and figure the real delay */
11478 lcts = tcp_get_usecs(<v);
11479 if (TSTMP_GT(lcts, cts)) {
11480 bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;
11481 bbr->rc_ack_was_delayed = 1;
11482 if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,
11483 bbr->r_ctl.highest_hdwr_delay))
11484 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;
11485 } else {
11486 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11487 bbr->rc_ack_was_delayed = 0;
11488 }
11489 } else {
11490 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11491 bbr->rc_ack_was_delayed = 0;
11492 }
11493 bbr_log_ack_event(bbr, th, &to, tlen, nsegs, cts, nxt_pkt, m);
11494 if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) {
11495 retval = 0;
11496 m_freem(m);
11497 goto done_with_input;
11498 }
11499 /*
11500 * If a segment with the ACK-bit set arrives in the SYN-SENT state
11501 * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9.
11502 */
11503 if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&
11504 (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {
11505 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
11506 ctf_do_dropwithreset_conn(m, tp, th, tlen);
11507 return (1);
11508 }
11509 if (tiwin > bbr->r_ctl.rc_high_rwnd)
11510 bbr->r_ctl.rc_high_rwnd = tiwin;
11511 bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,
11512 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11513 bbr->rtt_valid = 0;
11514 if (to.to_flags & TOF_TS) {
11515 bbr->rc_ts_valid = 1;
11516 bbr->r_ctl.last_inbound_ts = to.to_tsval;
11517 } else {
11518 bbr->rc_ts_valid = 0;
11519 bbr->r_ctl.last_inbound_ts = 0;
11520 }
11521 retval = (*bbr->r_substate) (m, th, so,
11522 tp, &to, drop_hdrlen,
11523 tlen, tiwin, thflags, nxt_pkt, iptos);
11524 if (nxt_pkt == 0)
11525 BBR_STAT_INC(bbr_rlock_left_ret0);
11526 else
11527 BBR_STAT_INC(bbr_rlock_left_ret1);
11528 if (retval == 0) {
11529 /*
11530 * If retval is 1 the tcb is unlocked and most likely the tp
11531 * is gone.
11532 */
11533 INP_WLOCK_ASSERT(inp);
11534 tcp_bbr_xmit_timer_commit(bbr, tp, cts);
11535 if (bbr->rc_is_pkt_epoch_now)
11536 bbr_set_pktepoch(bbr, cts, __LINE__);
11537 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));
11538 if (nxt_pkt == 0) {
11539 if ((bbr->r_wanted_output != 0) ||
11540 (tp->t_flags & TF_ACKNOW)) {
11541
11542 bbr->rc_output_starts_timer = 0;
11543 did_out = 1;
11544 if (tcp_output(tp) < 0)
11545 return (1);
11546 } else
11547 bbr_start_hpts_timer(bbr, tp, cts, 6, 0, 0);
11548 }
11549 if ((nxt_pkt == 0) &&
11550 ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
11551 (SEQ_GT(tp->snd_max, tp->snd_una) ||
11552 (tp->t_flags & TF_DELACK) ||
11553 ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
11554 (tp->t_state <= TCPS_CLOSING)))) {
11555 /*
11556 * We could not send (probably in the hpts but
11557 * stopped the timer)?
11558 */
11559 if ((tp->snd_max == tp->snd_una) &&
11560 ((tp->t_flags & TF_DELACK) == 0) &&
11561 (tcp_in_hpts(tp)) &&
11562 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
11563 /*
11564 * keep alive not needed if we are hptsi
11565 * output yet
11566 */
11567 ;
11568 } else {
11569 if (tcp_in_hpts(tp)) {
11570 tcp_hpts_remove(tp);
11571 if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11572 (TSTMP_GT(lcts, bbr->rc_pacer_started))) {
11573 uint32_t del;
11574
11575 del = lcts - bbr->rc_pacer_started;
11576 if (bbr->r_ctl.rc_last_delay_val > del) {
11577 BBR_STAT_INC(bbr_force_timer_start);
11578 bbr->r_ctl.rc_last_delay_val -= del;
11579 bbr->rc_pacer_started = lcts;
11580 } else {
11581 /* We are late */
11582 bbr->r_ctl.rc_last_delay_val = 0;
11583 BBR_STAT_INC(bbr_force_output);
11584 if (tcp_output(tp) < 0)
11585 return (1);
11586 }
11587 }
11588 }
11589 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,
11590 0);
11591 }
11592 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {
11593 /* Do we have the correct timer running? */
11594 bbr_timer_audit(tp, bbr, lcts, &so->so_snd);
11595 }
11596 /* Clear the flag, it may have been cleared by output but we may not have */
11597 if ((nxt_pkt == 0) && (tp->t_flags2 & TF2_HPTS_CALLS))
11598 tp->t_flags2 &= ~TF2_HPTS_CALLS;
11599 /* Do we have a new state */
11600 if (bbr->r_state != tp->t_state)
11601 bbr_set_state(tp, bbr, tiwin);
11602 done_with_input:
11603 bbr_log_doseg_done(bbr, cts, nxt_pkt, did_out);
11604 if (did_out)
11605 bbr->r_wanted_output = 0;
11606 }
11607 return (retval);
11608 }
11609
11610 static void
bbr_do_segment(struct tcpcb * tp,struct mbuf * m,struct tcphdr * th,int32_t drop_hdrlen,int32_t tlen,uint8_t iptos)11611 bbr_do_segment(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
11612 int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
11613 {
11614 struct timeval tv;
11615 int retval;
11616
11617 /* First lets see if we have old packets */
11618 if (!STAILQ_EMPTY(&tp->t_inqueue)) {
11619 if (ctf_do_queued_segments(tp, 1)) {
11620 m_freem(m);
11621 return;
11622 }
11623 }
11624 if (m->m_flags & M_TSTMP_LRO) {
11625 mbuf_tstmp2timeval(m, &tv);
11626 } else {
11627 /* Should not be should we kassert instead? */
11628 tcp_get_usecs(&tv);
11629 }
11630 retval = bbr_do_segment_nounlock(tp, m, th, drop_hdrlen, tlen, iptos,
11631 0, &tv);
11632 if (retval == 0) {
11633 INP_WUNLOCK(tptoinpcb(tp));
11634 }
11635 }
11636
11637 /*
11638 * Return how much data can be sent without violating the
11639 * cwnd or rwnd.
11640 */
11641
11642 static inline uint32_t
bbr_what_can_we_send(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t sendwin,uint32_t avail,int32_t sb_offset,uint32_t cts)11643 bbr_what_can_we_send(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t sendwin,
11644 uint32_t avail, int32_t sb_offset, uint32_t cts)
11645 {
11646 uint32_t len;
11647
11648 if (ctf_outstanding(tp) >= tp->snd_wnd) {
11649 /* We never want to go over our peers rcv-window */
11650 len = 0;
11651 } else {
11652 uint32_t flight;
11653
11654 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11655 if (flight >= sendwin) {
11656 /*
11657 * We have in flight what we are allowed by cwnd (if
11658 * it was rwnd blocking it would have hit above out
11659 * >= tp->snd_wnd).
11660 */
11661 return (0);
11662 }
11663 len = sendwin - flight;
11664 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {
11665 /* We would send too much (beyond the rwnd) */
11666 len = tp->snd_wnd - ctf_outstanding(tp);
11667 }
11668 if ((len + sb_offset) > avail) {
11669 /*
11670 * We don't have that much in the SB, how much is
11671 * there?
11672 */
11673 len = avail - sb_offset;
11674 }
11675 }
11676 return (len);
11677 }
11678
11679 static inline void
bbr_do_send_accounting(struct tcpcb * tp,struct tcp_bbr * bbr,struct bbr_sendmap * rsm,int32_t len,int32_t error)11680 bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11681 {
11682 if (error) {
11683 return;
11684 }
11685 if (rsm) {
11686 if (rsm->r_flags & BBR_TLP) {
11687 /*
11688 * TLP should not count in retran count, but in its
11689 * own bin
11690 */
11691 KMOD_TCPSTAT_INC(tcps_tlpresends);
11692 KMOD_TCPSTAT_ADD(tcps_tlpresend_bytes, len);
11693 } else {
11694 /* Retransmit */
11695 tp->t_sndrexmitpack++;
11696 KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
11697 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
11698 #ifdef STATS
11699 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
11700 len);
11701 #endif
11702 }
11703 /*
11704 * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is
11705 * sub-state
11706 */
11707 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);
11708 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {
11709 /* Non probe_bw log in 1, 2, or 4. */
11710 counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);
11711 } else {
11712 /*
11713 * Log our probe state 3, and log also 5-13 to show
11714 * us the recovery sub-state for the send. This
11715 * means that 3 == (5+6+7+8+9+10+11+12+13)
11716 */
11717 counter_u64_add(bbr_state_resend[BBR_STATE_PROBE_BW], len);
11718 counter_u64_add(bbr_state_resend[(bbr_state_val(bbr) + 5)], len);
11719 }
11720 /* Place in both 16's the totals of retransmitted */
11721 counter_u64_add(bbr_state_lost[16], len);
11722 counter_u64_add(bbr_state_resend[16], len);
11723 /* Place in 17's the total sent */
11724 counter_u64_add(bbr_state_resend[17], len);
11725 counter_u64_add(bbr_state_lost[17], len);
11726
11727 } else {
11728 /* New sends */
11729 KMOD_TCPSTAT_INC(tcps_sndpack);
11730 KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
11731 /* Place in 17's the total sent */
11732 counter_u64_add(bbr_state_resend[17], len);
11733 counter_u64_add(bbr_state_lost[17], len);
11734 #ifdef STATS
11735 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
11736 len);
11737 #endif
11738 }
11739 }
11740
11741 static void
bbr_cwnd_limiting(struct tcpcb * tp,struct tcp_bbr * bbr,uint32_t in_level)11742 bbr_cwnd_limiting(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t in_level)
11743 {
11744 if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {
11745 /*
11746 * Limit the cwnd to not be above N x the target plus whats
11747 * is outstanding. The target is based on the current b/w
11748 * estimate.
11749 */
11750 uint32_t target;
11751
11752 target = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), BBR_UNIT);
11753 target += ctf_outstanding(tp);
11754 target *= bbr_target_cwnd_mult_limit;
11755 if (tp->snd_cwnd > target)
11756 tp->snd_cwnd = target;
11757 bbr_log_type_cwndupd(bbr, 0, 0, 0, 10, 0, 0, __LINE__);
11758 }
11759 }
11760
11761 static int
bbr_window_update_needed(struct tcpcb * tp,struct socket * so,uint32_t recwin,int32_t maxseg)11762 bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, int32_t maxseg)
11763 {
11764 /*
11765 * "adv" is the amount we could increase the window, taking into
11766 * account that we are limited by TCP_MAXWIN << tp->rcv_scale.
11767 */
11768 int32_t adv;
11769 int32_t oldwin;
11770
11771 adv = recwin;
11772 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
11773 oldwin = (tp->rcv_adv - tp->rcv_nxt);
11774 if (adv > oldwin)
11775 adv -= oldwin;
11776 else {
11777 /* We can't increase the window */
11778 adv = 0;
11779 }
11780 } else
11781 oldwin = 0;
11782
11783 /*
11784 * If the new window size ends up being the same as or less
11785 * than the old size when it is scaled, then don't force
11786 * a window update.
11787 */
11788 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)
11789 return (0);
11790
11791 if (adv >= (2 * maxseg) &&
11792 (adv >= (so->so_rcv.sb_hiwat / 4) ||
11793 recwin <= (so->so_rcv.sb_hiwat / 8) ||
11794 so->so_rcv.sb_hiwat <= 8 * maxseg)) {
11795 return (1);
11796 }
11797 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat)
11798 return (1);
11799 return (0);
11800 }
11801
11802 /*
11803 * Return 0 on success and a errno on failure to send.
11804 * Note that a 0 return may not mean we sent anything
11805 * if the TCB was on the hpts. A non-zero return
11806 * does indicate the error we got from ip[6]_output.
11807 */
11808 static int
bbr_output_wtime(struct tcpcb * tp,const struct timeval * tv)11809 bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
11810 {
11811 struct socket *so;
11812 int32_t len;
11813 uint32_t cts;
11814 uint32_t recwin, sendwin;
11815 int32_t sb_offset;
11816 int32_t flags, abandon, error = 0;
11817 struct tcp_log_buffer *lgb;
11818 struct mbuf *m;
11819 struct mbuf *mb;
11820 uint32_t if_hw_tsomaxsegcount = 0;
11821 uint32_t if_hw_tsomaxsegsize = 0;
11822 uint32_t if_hw_tsomax = 0;
11823 struct ip *ip = NULL;
11824 struct tcp_bbr *bbr;
11825 struct tcphdr *th;
11826 struct udphdr *udp = NULL;
11827 u_char opt[TCP_MAXOLEN];
11828 unsigned ipoptlen, optlen, hdrlen;
11829 unsigned ulen;
11830 uint32_t bbr_seq;
11831 uint32_t delay_calc=0;
11832 uint8_t doing_tlp = 0;
11833 uint8_t local_options;
11834 #ifdef BBR_INVARIANTS
11835 uint8_t doing_retran_from = 0;
11836 uint8_t picked_up_retran = 0;
11837 #endif
11838 uint8_t wanted_cookie = 0;
11839 uint8_t more_to_rxt=0;
11840 int32_t prefetch_so_done = 0;
11841 int32_t prefetch_rsm = 0;
11842 uint32_t tot_len = 0;
11843 uint32_t maxseg, pace_max_segs, p_maxseg;
11844 int32_t csum_flags = 0;
11845 int32_t hw_tls;
11846 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
11847 unsigned ipsec_optlen = 0;
11848
11849 #endif
11850 volatile int32_t sack_rxmit;
11851 struct bbr_sendmap *rsm = NULL;
11852 int32_t tso, mtu;
11853 struct tcpopt to;
11854 int32_t pacing_delay = 0;
11855 struct inpcb *inp;
11856 struct sockbuf *sb;
11857 bool hpts_calling;
11858 #ifdef INET6
11859 struct ip6_hdr *ip6 = NULL;
11860 int32_t isipv6;
11861 #endif
11862 uint8_t app_limited = BBR_JR_SENT_DATA;
11863 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11864 /* We take a cache hit here */
11865 memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));
11866 cts = tcp_tv_to_usec(&bbr->rc_tv);
11867 inp = bbr->rc_inp;
11868 hpts_calling = !!(tp->t_flags2 & TF2_HPTS_CALLS);
11869 tp->t_flags2 &= ~TF2_HPTS_CALLS;
11870 so = inp->inp_socket;
11871 sb = &so->so_snd;
11872 if (tp->t_nic_ktls_xmit)
11873 hw_tls = 1;
11874 else
11875 hw_tls = 0;
11876 kern_prefetch(sb, &maxseg);
11877 maxseg = tp->t_maxseg - bbr->rc_last_options;
11878 if (bbr_minseg(bbr) < maxseg) {
11879 tcp_bbr_tso_size_check(bbr, cts);
11880 }
11881 /* Remove any flags that indicate we are pacing on the inp */
11882 pace_max_segs = bbr->r_ctl.rc_pace_max_segs;
11883 p_maxseg = min(maxseg, pace_max_segs);
11884 INP_WLOCK_ASSERT(inp);
11885 #ifdef TCP_OFFLOAD
11886 if (tp->t_flags & TF_TOE)
11887 return (tcp_offload_output(tp));
11888 #endif
11889
11890 #ifdef INET6
11891 if (bbr->r_state) {
11892 /* Use the cache line loaded if possible */
11893 isipv6 = bbr->r_is_v6;
11894 } else {
11895 isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
11896 }
11897 #endif
11898 if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
11899 tcp_in_hpts(tp)) {
11900 /*
11901 * We are on the hpts for some timer but not hptsi output.
11902 * Possibly remove from the hpts so we can send/recv etc.
11903 */
11904 if ((tp->t_flags & TF_ACKNOW) == 0) {
11905 /*
11906 * No immediate demand right now to send an ack, but
11907 * the user may have read, making room for new data
11908 * (a window update). If so we may want to cancel
11909 * whatever timer is running (KEEP/DEL-ACK?) and
11910 * continue to send out a window update. Or we may
11911 * have gotten more data into the socket buffer to
11912 * send.
11913 */
11914 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
11915 (long)TCP_MAXWIN << tp->rcv_scale);
11916 if ((bbr_window_update_needed(tp, so, recwin, maxseg) == 0) &&
11917 ((tcp_outflags[tp->t_state] & TH_RST) == 0) &&
11918 ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <=
11919 (tp->snd_max - tp->snd_una))) {
11920 /*
11921 * Nothing new to send and no window update
11922 * is needed to send. Lets just return and
11923 * let the timer-run off.
11924 */
11925 return (0);
11926 }
11927 }
11928 tcp_hpts_remove(tp);
11929 bbr_timer_cancel(bbr, __LINE__, cts);
11930 }
11931 if (bbr->r_ctl.rc_last_delay_val) {
11932 /* Calculate a rough delay for early escape to sending */
11933 if (SEQ_GT(cts, bbr->rc_pacer_started))
11934 delay_calc = cts - bbr->rc_pacer_started;
11935 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
11936 delay_calc -= bbr->r_ctl.rc_last_delay_val;
11937 else
11938 delay_calc = 0;
11939 }
11940 /* Mark that we have called bbr_output(). */
11941 if ((bbr->r_timer_override) ||
11942 (tp->t_state < TCPS_ESTABLISHED)) {
11943 /* Timeouts or early states are exempt */
11944 if (tcp_in_hpts(tp))
11945 tcp_hpts_remove(tp);
11946 } else if (tcp_in_hpts(tp)) {
11947 if ((bbr->r_ctl.rc_last_delay_val) &&
11948 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11949 delay_calc) {
11950 /*
11951 * We were being paced for output and the delay has
11952 * already exceeded when we were supposed to be
11953 * called, lets go ahead and pull out of the hpts
11954 * and call output.
11955 */
11956 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_LATE], 1);
11957 bbr->r_ctl.rc_last_delay_val = 0;
11958 tcp_hpts_remove(tp);
11959 } else if (tp->t_state == TCPS_CLOSED) {
11960 bbr->r_ctl.rc_last_delay_val = 0;
11961 tcp_hpts_remove(tp);
11962 } else {
11963 /*
11964 * On the hpts, you shall not pass! even if ACKNOW
11965 * is on, we will when the hpts fires, unless of
11966 * course we are overdue.
11967 */
11968 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_INPACE], 1);
11969 return (0);
11970 }
11971 }
11972 bbr->rc_cwnd_limited = 0;
11973 if (bbr->r_ctl.rc_last_delay_val) {
11974 /* recalculate the real delay and deal with over/under */
11975 if (SEQ_GT(cts, bbr->rc_pacer_started))
11976 delay_calc = cts - bbr->rc_pacer_started;
11977 else
11978 delay_calc = 0;
11979 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
11980 /* Setup the delay which will be added in */
11981 delay_calc -= bbr->r_ctl.rc_last_delay_val;
11982 else {
11983 /*
11984 * We are early setup to adjust out pacing delay.
11985 */
11986 uint64_t merged_val;
11987
11988 bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);
11989 bbr->r_agg_early_set = 1;
11990 if (bbr->r_ctl.rc_hptsi_agg_delay) {
11991 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {
11992 /* Nope our previous late cancels out the early */
11993 bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;
11994 bbr->r_agg_early_set = 0;
11995 bbr->r_ctl.rc_agg_early = 0;
11996 } else {
11997 bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;
11998 bbr->r_ctl.rc_hptsi_agg_delay = 0;
11999 }
12000 }
12001 merged_val = bbr->rc_pacer_started;
12002 merged_val <<= 32;
12003 merged_val |= bbr->r_ctl.rc_last_delay_val;
12004 bbr_log_pacing_delay_calc(bbr, hpts_calling,
12005 bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,
12006 bbr->r_agg_early_set, 3);
12007 bbr->r_ctl.rc_last_delay_val = 0;
12008 BBR_STAT_INC(bbr_early);
12009 delay_calc = 0;
12010 }
12011 } else {
12012 /* We were not delayed due to hptsi */
12013 if (bbr->r_agg_early_set)
12014 bbr->r_ctl.rc_agg_early = 0;
12015 bbr->r_agg_early_set = 0;
12016 delay_calc = 0;
12017 }
12018 if (delay_calc) {
12019 /*
12020 * We had a hptsi delay which means we are falling behind on
12021 * sending at the expected rate. Calculate an extra amount
12022 * of data we can send, if any, to put us back on track.
12023 */
12024 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)
12025 bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;
12026 else
12027 bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;
12028 }
12029 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12030 if ((tp->snd_una == tp->snd_max) &&
12031 (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
12032 (sbavail(sb))) {
12033 /*
12034 * Ok we have been idle with nothing outstanding
12035 * we possibly need to start fresh with either a new
12036 * suite of states or a fast-ramp up.
12037 */
12038 bbr_restart_after_idle(bbr,
12039 cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));
12040 }
12041 /*
12042 * Now was there a hptsi delay where we are behind? We only count
12043 * being behind if: a) We are not in recovery. b) There was a delay.
12044 * <and> c) We had room to send something.
12045 *
12046 */
12047 if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
12048 int retval;
12049
12050 retval = bbr_process_timers(tp, bbr, cts, hpts_calling);
12051 if (retval != 0) {
12052 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_ATIMER], 1);
12053 /*
12054 * If timers want tcp_drop(), then pass error out,
12055 * otherwise suppress it.
12056 */
12057 return (retval < 0 ? retval : 0);
12058 }
12059 }
12060 bbr->rc_tp->t_flags2 &= ~TF2_MBUF_QUEUE_READY;
12061 if (hpts_calling &&
12062 (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
12063 bbr->r_ctl.rc_last_delay_val = 0;
12064 }
12065 bbr->r_timer_override = 0;
12066 bbr->r_wanted_output = 0;
12067 /*
12068 * For TFO connections in SYN_RECEIVED, only allow the initial
12069 * SYN|ACK and those sent by the retransmit timer.
12070 */
12071 if ((tp->t_flags & TF_FASTOPEN) &&
12072 ((tp->t_state == TCPS_SYN_RECEIVED) ||
12073 (tp->t_state == TCPS_SYN_SENT)) &&
12074 SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
12075 (tp->t_rxtshift == 0)) { /* not a retransmit */
12076 len = 0;
12077 goto just_return_nolock;
12078 }
12079 /*
12080 * Before sending anything check for a state update. For hpts
12081 * calling without input this is important. If its input calling
12082 * then this was already done.
12083 */
12084 if (bbr->rc_use_google == 0)
12085 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12086 again:
12087 /*
12088 * If we've recently taken a timeout, snd_max will be greater than
12089 * snd_max. BBR in general does not pay much attention to snd_nxt
12090 * for historic reasons the persist timer still uses it. This means
12091 * we have to look at it. All retransmissions that are not persits
12092 * use the rsm that needs to be sent so snd_nxt is ignored. At the
12093 * end of this routine we pull snd_nxt always up to snd_max.
12094 */
12095 doing_tlp = 0;
12096 #ifdef BBR_INVARIANTS
12097 doing_retran_from = picked_up_retran = 0;
12098 #endif
12099 error = 0;
12100 tso = 0;
12101 pacing_delay = 0;
12102 mtu = 0;
12103 sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12104 sb_offset = tp->snd_max - tp->snd_una;
12105 flags = tcp_outflags[tp->t_state];
12106 sack_rxmit = 0;
12107 len = 0;
12108 rsm = NULL;
12109 if (flags & TH_RST) {
12110 SOCK_SENDBUF_LOCK(so);
12111 goto send;
12112 }
12113 recheck_resend:
12114 while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
12115 /* We need to always have one in reserve */
12116 rsm = bbr_alloc(bbr);
12117 if (rsm == NULL) {
12118 error = ENOMEM;
12119 /* Lie to get on the hpts */
12120 tot_len = tp->t_maxseg;
12121 if (hpts_calling)
12122 /* Retry in a ms */
12123 pacing_delay = 1001;
12124 goto just_return_nolock;
12125 }
12126 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
12127 bbr->r_ctl.rc_free_cnt++;
12128 rsm = NULL;
12129 }
12130 /* What do we send, a resend? */
12131 if (bbr->r_ctl.rc_resend == NULL) {
12132 /* Check for rack timeout */
12133 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
12134 if (bbr->r_ctl.rc_resend) {
12135 #ifdef BBR_INVARIANTS
12136 picked_up_retran = 1;
12137 #endif
12138 bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);
12139 }
12140 }
12141 if (bbr->r_ctl.rc_resend) {
12142 rsm = bbr->r_ctl.rc_resend;
12143 #ifdef BBR_INVARIANTS
12144 doing_retran_from = 1;
12145 #endif
12146 /* Remove any TLP flags its a RACK or T-O */
12147 rsm->r_flags &= ~BBR_TLP;
12148 bbr->r_ctl.rc_resend = NULL;
12149 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
12150 #ifdef BBR_INVARIANTS
12151 panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n",
12152 tp, bbr, rsm, rsm->r_start, tp->snd_una);
12153 goto recheck_resend;
12154 #else
12155 /* TSNH */
12156 rsm = NULL;
12157 goto recheck_resend;
12158 #endif
12159 }
12160 if (rsm->r_flags & BBR_HAS_SYN) {
12161 /* Only retransmit a SYN by itself */
12162 len = 0;
12163 if ((flags & TH_SYN) == 0) {
12164 /* Huh something is wrong */
12165 rsm->r_start++;
12166 if (rsm->r_start == rsm->r_end) {
12167 /* Clean it up, somehow we missed the ack? */
12168 bbr_log_syn(tp, NULL);
12169 } else {
12170 /* TFO with data? */
12171 rsm->r_flags &= ~BBR_HAS_SYN;
12172 len = rsm->r_end - rsm->r_start;
12173 }
12174 } else {
12175 /* Retransmitting SYN */
12176 rsm = NULL;
12177 SOCK_SENDBUF_LOCK(so);
12178 goto send;
12179 }
12180 } else
12181 len = rsm->r_end - rsm->r_start;
12182 if ((bbr->rc_resends_use_tso == 0) &&
12183 (len > maxseg)) {
12184 len = maxseg;
12185 more_to_rxt = 1;
12186 }
12187 sb_offset = rsm->r_start - tp->snd_una;
12188 if (len > 0) {
12189 sack_rxmit = 1;
12190 KMOD_TCPSTAT_INC(tcps_sack_rexmits);
12191 KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes,
12192 min(len, maxseg));
12193 } else {
12194 /* I dont think this can happen */
12195 rsm = NULL;
12196 goto recheck_resend;
12197 }
12198 BBR_STAT_INC(bbr_resends_set);
12199 } else if (bbr->r_ctl.rc_tlp_send) {
12200 /*
12201 * Tail loss probe
12202 */
12203 doing_tlp = 1;
12204 rsm = bbr->r_ctl.rc_tlp_send;
12205 bbr->r_ctl.rc_tlp_send = NULL;
12206 sack_rxmit = 1;
12207 len = rsm->r_end - rsm->r_start;
12208 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12209 len = maxseg;
12210
12211 if (SEQ_GT(tp->snd_una, rsm->r_start)) {
12212 #ifdef BBR_INVARIANTS
12213 panic("tp:%p bbc:%p snd_una:%u rsm:%p r_start:%u",
12214 tp, bbr, tp->snd_una, rsm, rsm->r_start);
12215 #else
12216 /* TSNH */
12217 rsm = NULL;
12218 goto recheck_resend;
12219 #endif
12220 }
12221 sb_offset = rsm->r_start - tp->snd_una;
12222 BBR_STAT_INC(bbr_tlp_set);
12223 }
12224 /*
12225 * Enforce a connection sendmap count limit if set
12226 * as long as we are not retransmiting.
12227 */
12228 if ((rsm == NULL) &&
12229 (V_tcp_map_entries_limit > 0) &&
12230 (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
12231 BBR_STAT_INC(bbr_alloc_limited);
12232 if (!bbr->alloc_limit_reported) {
12233 bbr->alloc_limit_reported = 1;
12234 BBR_STAT_INC(bbr_alloc_limited_conns);
12235 }
12236 goto just_return_nolock;
12237 }
12238 #ifdef BBR_INVARIANTS
12239 if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {
12240 panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u",
12241 tp, bbr, rsm, sb_offset, len);
12242 }
12243 #endif
12244 /*
12245 * Get standard flags, and add SYN or FIN if requested by 'hidden'
12246 * state flags.
12247 */
12248 if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))
12249 flags |= TH_FIN;
12250 if (tp->t_flags & TF_NEEDSYN)
12251 flags |= TH_SYN;
12252
12253 if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {
12254 /* we are retransmitting the fin */
12255 len--;
12256 if (len) {
12257 /*
12258 * When retransmitting data do *not* include the
12259 * FIN. This could happen from a TLP probe if we
12260 * allowed data with a FIN.
12261 */
12262 flags &= ~TH_FIN;
12263 }
12264 } else if (rsm) {
12265 if (flags & TH_FIN)
12266 flags &= ~TH_FIN;
12267 }
12268 if ((sack_rxmit == 0) && (prefetch_rsm == 0)) {
12269 void *end_rsm;
12270
12271 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
12272 if (end_rsm)
12273 kern_prefetch(end_rsm, &prefetch_rsm);
12274 prefetch_rsm = 1;
12275 }
12276 SOCK_SENDBUF_LOCK(so);
12277 /*
12278 * If snd_nxt == snd_max and we have transmitted a FIN, the
12279 * sb_offset will be > 0 even if so_snd.sb_cc is 0, resulting in a
12280 * negative length. This can also occur when TCP opens up its
12281 * congestion window while receiving additional duplicate acks after
12282 * fast-retransmit because TCP will reset snd_nxt to snd_max after
12283 * the fast-retransmit.
12284 *
12285 * In the normal retransmit-FIN-only case, however, snd_nxt will be
12286 * set to snd_una, the sb_offset will be 0, and the length may wind
12287 * up 0.
12288 *
12289 * If sack_rxmit is true we are retransmitting from the scoreboard
12290 * in which case len is already set.
12291 */
12292 if (sack_rxmit == 0) {
12293 uint32_t avail;
12294
12295 avail = sbavail(sb);
12296 if (SEQ_GT(tp->snd_max, tp->snd_una))
12297 sb_offset = tp->snd_max - tp->snd_una;
12298 else
12299 sb_offset = 0;
12300 if (bbr->rc_tlp_new_data) {
12301 /* TLP is forcing out new data */
12302 uint32_t tlplen;
12303
12304 doing_tlp = 1;
12305 tlplen = maxseg;
12306
12307 if (tlplen > (uint32_t)(avail - sb_offset)) {
12308 tlplen = (uint32_t)(avail - sb_offset);
12309 }
12310 if (tlplen > tp->snd_wnd) {
12311 len = tp->snd_wnd;
12312 } else {
12313 len = tlplen;
12314 }
12315 bbr->rc_tlp_new_data = 0;
12316 } else {
12317 len = bbr_what_can_we_send(tp, bbr, sendwin, avail, sb_offset, cts);
12318 if ((len < p_maxseg) &&
12319 (bbr->rc_in_persist == 0) &&
12320 (ctf_outstanding(tp) >= (2 * p_maxseg)) &&
12321 ((avail - sb_offset) >= p_maxseg)) {
12322 /*
12323 * We are not completing whats in the socket
12324 * buffer (i.e. there is at least a segment
12325 * waiting to send) and we have 2 or more
12326 * segments outstanding. There is no sense
12327 * of sending a little piece. Lets defer and
12328 * and wait until we can send a whole
12329 * segment.
12330 */
12331 len = 0;
12332 }
12333 if (bbr->rc_in_persist) {
12334 /*
12335 * We are in persists, figure out if
12336 * a retransmit is available (maybe the previous
12337 * persists we sent) or if we have to send new
12338 * data.
12339 */
12340 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
12341 if (rsm) {
12342 len = rsm->r_end - rsm->r_start;
12343 if (rsm->r_flags & BBR_HAS_FIN)
12344 len--;
12345 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12346 len = maxseg;
12347 if (len > 1)
12348 BBR_STAT_INC(bbr_persist_reneg);
12349 /*
12350 * XXXrrs we could force the len to
12351 * 1 byte here to cause the chunk to
12352 * split apart.. but that would then
12353 * mean we always retransmit it as
12354 * one byte even after the window
12355 * opens.
12356 */
12357 sack_rxmit = 1;
12358 sb_offset = rsm->r_start - tp->snd_una;
12359 } else {
12360 /*
12361 * First time through in persists or peer
12362 * acked our one byte. Though we do have
12363 * to have something in the sb.
12364 */
12365 len = 1;
12366 sb_offset = 0;
12367 if (avail == 0)
12368 len = 0;
12369 }
12370 }
12371 }
12372 }
12373 if (prefetch_so_done == 0) {
12374 kern_prefetch(so, &prefetch_so_done);
12375 prefetch_so_done = 1;
12376 }
12377 /*
12378 * Lop off SYN bit if it has already been sent. However, if this is
12379 * SYN-SENT state and if segment contains data and if we don't know
12380 * that foreign host supports TAO, suppress sending segment.
12381 */
12382 if ((flags & TH_SYN) && (rsm == NULL) &&
12383 SEQ_GT(tp->snd_max, tp->snd_una)) {
12384 if (tp->t_state != TCPS_SYN_RECEIVED)
12385 flags &= ~TH_SYN;
12386 /*
12387 * When sending additional segments following a TFO SYN|ACK,
12388 * do not include the SYN bit.
12389 */
12390 if ((tp->t_flags & TF_FASTOPEN) &&
12391 (tp->t_state == TCPS_SYN_RECEIVED))
12392 flags &= ~TH_SYN;
12393 sb_offset--, len++;
12394 if (sbavail(sb) == 0)
12395 len = 0;
12396 } else if ((flags & TH_SYN) && rsm) {
12397 /*
12398 * Subtract one from the len for the SYN being
12399 * retransmitted.
12400 */
12401 len--;
12402 }
12403 /*
12404 * Be careful not to send data and/or FIN on SYN segments. This
12405 * measure is needed to prevent interoperability problems with not
12406 * fully conformant TCP implementations.
12407 */
12408 if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
12409 len = 0;
12410 flags &= ~TH_FIN;
12411 }
12412 /*
12413 * On TFO sockets, ensure no data is sent in the following cases:
12414 *
12415 * - When retransmitting SYN|ACK on a passively-created socket
12416 * - When retransmitting SYN on an actively created socket
12417 * - When sending a zero-length cookie (cookie request) on an
12418 * actively created socket
12419 * - When the socket is in the CLOSED state (RST is being sent)
12420 */
12421 if ((tp->t_flags & TF_FASTOPEN) &&
12422 (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
12423 ((tp->t_state == TCPS_SYN_SENT) &&
12424 (tp->t_tfo_client_cookie_len == 0)) ||
12425 (flags & TH_RST))) {
12426 len = 0;
12427 sack_rxmit = 0;
12428 rsm = NULL;
12429 }
12430 /* Without fast-open there should never be data sent on a SYN */
12431 if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN))
12432 len = 0;
12433 if (len <= 0) {
12434 /*
12435 * If FIN has been sent but not acked, but we haven't been
12436 * called to retransmit, len will be < 0. Otherwise, window
12437 * shrank after we sent into it. If window shrank to 0,
12438 * cancel pending retransmit, pull snd_nxt back to (closed)
12439 * window, and set the persist timer if it isn't already
12440 * going. If the window didn't close completely, just wait
12441 * for an ACK.
12442 *
12443 * We also do a general check here to ensure that we will
12444 * set the persist timer when we have data to send, but a
12445 * 0-byte window. This makes sure the persist timer is set
12446 * even if the packet hits one of the "goto send" lines
12447 * below.
12448 */
12449 len = 0;
12450 if ((tp->snd_wnd == 0) &&
12451 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12452 (tp->snd_una == tp->snd_max) &&
12453 (sb_offset < (int)sbavail(sb))) {
12454 /*
12455 * Not enough room in the rwnd to send
12456 * a paced segment out.
12457 */
12458 bbr_enter_persist(tp, bbr, cts, __LINE__);
12459 }
12460 } else if ((rsm == NULL) &&
12461 (doing_tlp == 0) &&
12462 (len < bbr->r_ctl.rc_pace_max_segs)) {
12463 /*
12464 * We are not sending a full segment for
12465 * some reason. Should we not send anything (think
12466 * sws or persists)?
12467 */
12468 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12469 (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12470 (len < (int)(sbavail(sb) - sb_offset))) {
12471 /*
12472 * Here the rwnd is less than
12473 * the pacing size, this is not a retransmit,
12474 * we are established and
12475 * the send is not the last in the socket buffer
12476 * lets not send, and possibly enter persists.
12477 */
12478 len = 0;
12479 if (tp->snd_max == tp->snd_una)
12480 bbr_enter_persist(tp, bbr, cts, __LINE__);
12481 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&
12482 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12483 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12484 (len < (int)(sbavail(sb) - sb_offset)) &&
12485 (len < bbr_minseg(bbr))) {
12486 /*
12487 * Here we are not retransmitting, and
12488 * the cwnd is not so small that we could
12489 * not send at least a min size (rxt timer
12490 * not having gone off), We have 2 segments or
12491 * more already in flight, its not the tail end
12492 * of the socket buffer and the cwnd is blocking
12493 * us from sending out minimum pacing segment size.
12494 * Lets not send anything.
12495 */
12496 bbr->rc_cwnd_limited = 1;
12497 len = 0;
12498 } else if (((tp->snd_wnd - ctf_outstanding(tp)) <
12499 min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12500 (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12501 bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12502 (len < (int)(sbavail(sb) - sb_offset)) &&
12503 (TCPS_HAVEESTABLISHED(tp->t_state))) {
12504 /*
12505 * Here we have a send window but we have
12506 * filled it up and we can't send another pacing segment.
12507 * We also have in flight more than 2 segments
12508 * and we are not completing the sb i.e. we allow
12509 * the last bytes of the sb to go out even if
12510 * its not a full pacing segment.
12511 */
12512 len = 0;
12513 }
12514 }
12515 /* len will be >= 0 after this point. */
12516 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
12517 tcp_sndbuf_autoscale(tp, so, sendwin);
12518 /*
12519 *
12520 */
12521 if (bbr->rc_in_persist &&
12522 len &&
12523 (rsm == NULL) &&
12524 (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {
12525 /*
12526 * We are in persist, not doing a retransmit and don't have enough space
12527 * yet to send a full TSO. So is it at the end of the sb
12528 * if so we need to send else nuke to 0 and don't send.
12529 */
12530 int sbleft;
12531 if (sbavail(sb) > sb_offset)
12532 sbleft = sbavail(sb) - sb_offset;
12533 else
12534 sbleft = 0;
12535 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {
12536 /* not at end of sb lets not send */
12537 len = 0;
12538 }
12539 }
12540 /*
12541 * Decide if we can use TCP Segmentation Offloading (if supported by
12542 * hardware).
12543 *
12544 * TSO may only be used if we are in a pure bulk sending state. The
12545 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
12546 * options prevent using TSO. With TSO the TCP header is the same
12547 * (except for the sequence number) for all generated packets. This
12548 * makes it impossible to transmit any options which vary per
12549 * generated segment or packet.
12550 *
12551 * IPv4 handling has a clear separation of ip options and ip header
12552 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen()
12553 * does the right thing below to provide length of just ip options
12554 * and thus checking for ipoptlen is enough to decide if ip options
12555 * are present.
12556 */
12557 #ifdef INET6
12558 if (isipv6)
12559 ipoptlen = ip6_optlen(inp);
12560 else
12561 #endif
12562 if (inp->inp_options)
12563 ipoptlen = inp->inp_options->m_len -
12564 offsetof(struct ipoption, ipopt_list);
12565 else
12566 ipoptlen = 0;
12567 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
12568 /*
12569 * Pre-calculate here as we save another lookup into the darknesses
12570 * of IPsec that way and can actually decide if TSO is ok.
12571 */
12572 #ifdef INET6
12573 if (isipv6 && IPSEC_ENABLED(ipv6))
12574 ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp);
12575 #ifdef INET
12576 else
12577 #endif
12578 #endif /* INET6 */
12579 #ifdef INET
12580 if (IPSEC_ENABLED(ipv4))
12581 ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp);
12582 #endif /* INET */
12583 #endif /* IPSEC */
12584 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
12585 ipoptlen += ipsec_optlen;
12586 #endif
12587 if ((tp->t_flags & TF_TSO) && V_tcp_do_tso &&
12588 (len > maxseg) &&
12589 (tp->t_port == 0) &&
12590 ((tp->t_flags & TF_SIGNATURE) == 0) &&
12591 ipoptlen == 0)
12592 tso = 1;
12593
12594 recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
12595 (long)TCP_MAXWIN << tp->rcv_scale);
12596 /*
12597 * Sender silly window avoidance. We transmit under the following
12598 * conditions when len is non-zero:
12599 *
12600 * - We have a full segment (or more with TSO) - This is the last
12601 * buffer in a write()/send() and we are either idle or running
12602 * NODELAY - we've timed out (e.g. persist timer) - we have more
12603 * then 1/2 the maximum send window's worth of data (receiver may be
12604 * limited the window size) - we need to retransmit
12605 */
12606 if (rsm)
12607 goto send;
12608 if (len) {
12609 if (sack_rxmit)
12610 goto send;
12611 if (len >= p_maxseg)
12612 goto send;
12613 /*
12614 * NOTE! on localhost connections an 'ack' from the remote
12615 * end may occur synchronously with the output and cause us
12616 * to flush a buffer queued with moretocome. XXX
12617 *
12618 */
12619 if (((tp->t_flags & TF_MORETOCOME) == 0) && /* normal case */
12620 ((tp->t_flags & TF_NODELAY) ||
12621 ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) &&
12622 (tp->t_flags & TF_NOPUSH) == 0) {
12623 goto send;
12624 }
12625 if ((tp->snd_una == tp->snd_max) && len) { /* Nothing outstanding */
12626 goto send;
12627 }
12628 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
12629 goto send;
12630 }
12631 }
12632 /*
12633 * Sending of standalone window updates.
12634 *
12635 * Window updates are important when we close our window due to a
12636 * full socket buffer and are opening it again after the application
12637 * reads data from it. Once the window has opened again and the
12638 * remote end starts to send again the ACK clock takes over and
12639 * provides the most current window information.
12640 *
12641 * We must avoid the silly window syndrome whereas every read from
12642 * the receive buffer, no matter how small, causes a window update
12643 * to be sent. We also should avoid sending a flurry of window
12644 * updates when the socket buffer had queued a lot of data and the
12645 * application is doing small reads.
12646 *
12647 * Prevent a flurry of pointless window updates by only sending an
12648 * update when we can increase the advertized window by more than
12649 * 1/4th of the socket buffer capacity. When the buffer is getting
12650 * full or is very small be more aggressive and send an update
12651 * whenever we can increase by two mss sized segments. In all other
12652 * situations the ACK's to new incoming data will carry further
12653 * window increases.
12654 *
12655 * Don't send an independent window update if a delayed ACK is
12656 * pending (it will get piggy-backed on it) or the remote side
12657 * already has done a half-close and won't send more data. Skip
12658 * this if the connection is in T/TCP half-open state.
12659 */
12660 if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
12661 !(tp->t_flags & TF_DELACK) &&
12662 !TCPS_HAVERCVDFIN(tp->t_state)) {
12663 /* Check to see if we should do a window update */
12664 if (bbr_window_update_needed(tp, so, recwin, maxseg))
12665 goto send;
12666 }
12667 /*
12668 * Send if we owe the peer an ACK, RST, SYN. ACKNOW
12669 * is also a catch-all for the retransmit timer timeout case.
12670 */
12671 if (tp->t_flags & TF_ACKNOW) {
12672 goto send;
12673 }
12674 if (flags & TH_RST) {
12675 /* Always send a RST if one is due */
12676 goto send;
12677 }
12678 if ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0) {
12679 goto send;
12680 }
12681 /*
12682 * If our state indicates that FIN should be sent and we have not
12683 * yet done so, then we need to send.
12684 */
12685 if (flags & TH_FIN &&
12686 ((tp->t_flags & TF_SENTFIN) == 0)) {
12687 goto send;
12688 }
12689 /*
12690 * No reason to send a segment, just return.
12691 */
12692 just_return:
12693 SOCK_SENDBUF_UNLOCK(so);
12694 just_return_nolock:
12695 if (tot_len)
12696 pacing_delay = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
12697 if (bbr->rc_no_pacing)
12698 pacing_delay = 0;
12699 if (tot_len == 0) {
12700 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=
12701 tp->snd_wnd) {
12702 BBR_STAT_INC(bbr_rwnd_limited);
12703 app_limited = BBR_JR_RWND_LIMITED;
12704 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12705 if ((bbr->rc_in_persist == 0) &&
12706 TCPS_HAVEESTABLISHED(tp->t_state) &&
12707 (tp->snd_max == tp->snd_una) &&
12708 sbavail(&so->so_snd)) {
12709 /* No send window.. we must enter persist */
12710 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
12711 }
12712 } else if (ctf_outstanding(tp) >= sbavail(sb)) {
12713 BBR_STAT_INC(bbr_app_limited);
12714 app_limited = BBR_JR_APP_LIMITED;
12715 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12716 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12717 bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {
12718 BBR_STAT_INC(bbr_cwnd_limited);
12719 app_limited = BBR_JR_CWND_LIMITED;
12720 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12721 bbr->r_ctl.rc_lost_bytes)));
12722 bbr->rc_cwnd_limited = 1;
12723 } else {
12724 BBR_STAT_INC(bbr_app_limited);
12725 app_limited = BBR_JR_APP_LIMITED;
12726 bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
12727 }
12728 bbr->r_ctl.rc_hptsi_agg_delay = 0;
12729 bbr->r_agg_early_set = 0;
12730 bbr->r_ctl.rc_agg_early = 0;
12731 bbr->r_ctl.rc_last_delay_val = 0;
12732 } else if (bbr->rc_use_google == 0)
12733 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12734 /* Are we app limited? */
12735 if ((app_limited == BBR_JR_APP_LIMITED) ||
12736 (app_limited == BBR_JR_RWND_LIMITED)) {
12737 /**
12738 * We are application limited.
12739 */
12740 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12741 bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);
12742 }
12743 if (tot_len == 0)
12744 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_JUSTRET], 1);
12745 /* Dont update the time if we did not send */
12746 bbr->r_ctl.rc_last_delay_val = 0;
12747 bbr->rc_output_starts_timer = 1;
12748 bbr_start_hpts_timer(bbr, tp, cts, 9, pacing_delay, tot_len);
12749 bbr_log_type_just_return(bbr, cts, tot_len, hpts_calling, app_limited, p_maxseg, len);
12750 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
12751 /* Make sure snd_nxt is drug up */
12752 tp->snd_nxt = tp->snd_max;
12753 }
12754 return (error);
12755
12756 send:
12757 if (doing_tlp == 0) {
12758 /*
12759 * Data not a TLP, and its not the rxt firing. If it is the
12760 * rxt firing, we want to leave the tlp_in_progress flag on
12761 * so we don't send another TLP. It has to be a rack timer
12762 * or normal send (response to acked data) to clear the tlp
12763 * in progress flag.
12764 */
12765 bbr->rc_tlp_in_progress = 0;
12766 bbr->rc_tlp_rtx_out = 0;
12767 } else {
12768 /*
12769 * Its a TLP.
12770 */
12771 bbr->rc_tlp_in_progress = 1;
12772 }
12773 bbr_timer_cancel(bbr, __LINE__, cts);
12774 if (rsm == NULL) {
12775 if (sbused(sb) > 0) {
12776 /*
12777 * This is sub-optimal. We only send a stand alone
12778 * FIN on its own segment.
12779 */
12780 if (flags & TH_FIN) {
12781 flags &= ~TH_FIN;
12782 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) {
12783 /* Lets not send this */
12784 pacing_delay = 0;
12785 goto just_return;
12786 }
12787 }
12788 }
12789 } else {
12790 /*
12791 * We do *not* send a FIN on a retransmit if it has data.
12792 * The if clause here where len > 1 should never come true.
12793 */
12794 if ((len > 0) &&
12795 (((rsm->r_flags & BBR_HAS_FIN) == 0) &&
12796 (flags & TH_FIN))) {
12797 flags &= ~TH_FIN;
12798 len--;
12799 }
12800 }
12801 SOCK_SENDBUF_LOCK_ASSERT(so);
12802 if (len > 0) {
12803 if ((tp->snd_una == tp->snd_max) &&
12804 (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
12805 /*
12806 * This qualifies as a RTT_PROBE session since we
12807 * drop the data outstanding to nothing and waited
12808 * more than bbr_rtt_probe_time.
12809 */
12810 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
12811 bbr_set_reduced_rtt(bbr, cts, __LINE__);
12812 }
12813 if (len >= maxseg)
12814 tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
12815 else
12816 tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
12817 }
12818 /*
12819 * Before ESTABLISHED, force sending of initial options unless TCP
12820 * set not to do any options. NOTE: we assume that the IP/TCP header
12821 * plus TCP options always fit in a single mbuf, leaving room for a
12822 * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr)
12823 * + optlen <= MCLBYTES
12824 */
12825 optlen = 0;
12826 #ifdef INET6
12827 if (isipv6)
12828 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
12829 else
12830 #endif
12831 hdrlen = sizeof(struct tcpiphdr);
12832
12833 /*
12834 * Compute options for segment. We only have to care about SYN and
12835 * established connection segments. Options for SYN-ACK segments
12836 * are handled in TCP syncache.
12837 */
12838 to.to_flags = 0;
12839 local_options = 0;
12840 if ((tp->t_flags & TF_NOOPT) == 0) {
12841 /* Maximum segment size. */
12842 if (flags & TH_SYN) {
12843 to.to_mss = tcp_mssopt(&inp->inp_inc);
12844 if (tp->t_port)
12845 to.to_mss -= V_tcp_udp_tunneling_overhead;
12846 to.to_flags |= TOF_MSS;
12847 /*
12848 * On SYN or SYN|ACK transmits on TFO connections,
12849 * only include the TFO option if it is not a
12850 * retransmit, as the presence of the TFO option may
12851 * have caused the original SYN or SYN|ACK to have
12852 * been dropped by a middlebox.
12853 */
12854 if ((tp->t_flags & TF_FASTOPEN) &&
12855 (tp->t_rxtshift == 0)) {
12856 if (tp->t_state == TCPS_SYN_RECEIVED) {
12857 to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
12858 to.to_tfo_cookie =
12859 (u_int8_t *)&tp->t_tfo_cookie.server;
12860 to.to_flags |= TOF_FASTOPEN;
12861 wanted_cookie = 1;
12862 } else if (tp->t_state == TCPS_SYN_SENT) {
12863 to.to_tfo_len =
12864 tp->t_tfo_client_cookie_len;
12865 to.to_tfo_cookie =
12866 tp->t_tfo_cookie.client;
12867 to.to_flags |= TOF_FASTOPEN;
12868 wanted_cookie = 1;
12869 }
12870 }
12871 }
12872 /* Window scaling. */
12873 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
12874 to.to_wscale = tp->request_r_scale;
12875 to.to_flags |= TOF_SCALE;
12876 }
12877 /* Timestamps. */
12878 if ((tp->t_flags & TF_RCVD_TSTMP) ||
12879 ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
12880 to.to_tsval = tcp_tv_to_msec(&bbr->rc_tv) + tp->ts_offset;
12881 to.to_tsecr = tp->ts_recent;
12882 to.to_flags |= TOF_TS;
12883 local_options += TCPOLEN_TIMESTAMP + 2;
12884 }
12885 /* Set receive buffer autosizing timestamp. */
12886 if (tp->rfbuf_ts == 0 &&
12887 (so->so_rcv.sb_flags & SB_AUTOSIZE))
12888 tp->rfbuf_ts = tcp_tv_to_msec(&bbr->rc_tv);
12889 /* Selective ACK's. */
12890 if (flags & TH_SYN)
12891 to.to_flags |= TOF_SACKPERM;
12892 else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
12893 tp->rcv_numsacks > 0) {
12894 to.to_flags |= TOF_SACK;
12895 to.to_nsacks = tp->rcv_numsacks;
12896 to.to_sacks = (u_char *)tp->sackblks;
12897 }
12898 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
12899 /* TCP-MD5 (RFC2385). */
12900 if (tp->t_flags & TF_SIGNATURE)
12901 to.to_flags |= TOF_SIGNATURE;
12902 #endif /* TCP_SIGNATURE */
12903
12904 /* Processing the options. */
12905 hdrlen += (optlen = tcp_addoptions(&to, opt));
12906 /*
12907 * If we wanted a TFO option to be added, but it was unable
12908 * to fit, ensure no data is sent.
12909 */
12910 if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie &&
12911 !(to.to_flags & TOF_FASTOPEN))
12912 len = 0;
12913 }
12914 if (tp->t_port) {
12915 if (V_tcp_udp_tunneling_port == 0) {
12916 /* The port was removed?? */
12917 SOCK_SENDBUF_UNLOCK(so);
12918 return (EHOSTUNREACH);
12919 }
12920 hdrlen += sizeof(struct udphdr);
12921 }
12922 #ifdef INET6
12923 if (isipv6)
12924 ipoptlen = ip6_optlen(inp);
12925 else
12926 #endif
12927 if (inp->inp_options)
12928 ipoptlen = inp->inp_options->m_len -
12929 offsetof(struct ipoption, ipopt_list);
12930 else
12931 ipoptlen = 0;
12932 ipoptlen = 0;
12933 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
12934 ipoptlen += ipsec_optlen;
12935 #endif
12936 if (bbr->rc_last_options != local_options) {
12937 /*
12938 * Cache the options length this generally does not change
12939 * on a connection. We use this to calculate TSO.
12940 */
12941 bbr->rc_last_options = local_options;
12942 }
12943 maxseg = tp->t_maxseg - (ipoptlen + optlen);
12944 p_maxseg = min(maxseg, pace_max_segs);
12945 /*
12946 * Adjust data length if insertion of options will bump the packet
12947 * length beyond the t_maxseg length. Clear the FIN bit because we
12948 * cut off the tail of the segment.
12949 */
12950 if (len > maxseg) {
12951 if (len != 0 && (flags & TH_FIN)) {
12952 flags &= ~TH_FIN;
12953 }
12954 if (tso) {
12955 uint32_t moff;
12956 int32_t max_len;
12957
12958 /* extract TSO information */
12959 if_hw_tsomax = tp->t_tsomax;
12960 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
12961 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
12962 KASSERT(ipoptlen == 0,
12963 ("%s: TSO can't do IP options", __func__));
12964
12965 /*
12966 * Check if we should limit by maximum payload
12967 * length:
12968 */
12969 if (if_hw_tsomax != 0) {
12970 /* compute maximum TSO length */
12971 max_len = (if_hw_tsomax - hdrlen -
12972 max_linkhdr);
12973 if (max_len <= 0) {
12974 len = 0;
12975 } else if (len > max_len) {
12976 len = max_len;
12977 }
12978 }
12979 /*
12980 * Prevent the last segment from being fractional
12981 * unless the send sockbuf can be emptied:
12982 */
12983 if ((sb_offset + len) < sbavail(sb)) {
12984 moff = len % (uint32_t)maxseg;
12985 if (moff != 0) {
12986 len -= moff;
12987 }
12988 }
12989 /*
12990 * In case there are too many small fragments don't
12991 * use TSO:
12992 */
12993 if (len <= maxseg) {
12994 len = maxseg;
12995 tso = 0;
12996 }
12997 } else {
12998 /* Not doing TSO */
12999 if (optlen + ipoptlen >= tp->t_maxseg) {
13000 /*
13001 * Since we don't have enough space to put
13002 * the IP header chain and the TCP header in
13003 * one packet as required by RFC 7112, don't
13004 * send it. Also ensure that at least one
13005 * byte of the payload can be put into the
13006 * TCP segment.
13007 */
13008 SOCK_SENDBUF_UNLOCK(so);
13009 error = EMSGSIZE;
13010 sack_rxmit = 0;
13011 goto out;
13012 }
13013 len = maxseg;
13014 }
13015 } else {
13016 /* Not doing TSO */
13017 if_hw_tsomaxsegcount = 0;
13018 tso = 0;
13019 }
13020 KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
13021 ("%s: len > IP_MAXPACKET", __func__));
13022 #ifdef DIAGNOSTIC
13023 #ifdef INET6
13024 if (max_linkhdr + hdrlen > MCLBYTES)
13025 #else
13026 if (max_linkhdr + hdrlen > MHLEN)
13027 #endif
13028 panic("tcphdr too big");
13029 #endif
13030 /*
13031 * This KASSERT is here to catch edge cases at a well defined place.
13032 * Before, those had triggered (random) panic conditions further
13033 * down.
13034 */
13035 #ifdef BBR_INVARIANTS
13036 if (sack_rxmit) {
13037 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
13038 panic("RSM:%p TP:%p bbr:%p start:%u is < snd_una:%u",
13039 rsm, tp, bbr, rsm->r_start, tp->snd_una);
13040 }
13041 }
13042 #endif
13043 KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
13044 if ((len == 0) &&
13045 (flags & TH_FIN) &&
13046 (sbused(sb))) {
13047 /*
13048 * We have outstanding data, don't send a fin by itself!.
13049 */
13050 pacing_delay = 0;
13051 goto just_return;
13052 }
13053 /*
13054 * Grab a header mbuf, attaching a copy of data to be transmitted,
13055 * and initialize the header from the template for sends on this
13056 * connection.
13057 */
13058 if (len) {
13059 uint32_t moff;
13060
13061 /*
13062 * We place a limit on sending with hptsi.
13063 */
13064 if ((rsm == NULL) && len > pace_max_segs)
13065 len = pace_max_segs;
13066 if (len <= maxseg)
13067 tso = 0;
13068 #ifdef INET6
13069 if (MHLEN < hdrlen + max_linkhdr)
13070 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
13071 else
13072 #endif
13073 m = m_gethdr(M_NOWAIT, MT_DATA);
13074
13075 if (m == NULL) {
13076 BBR_STAT_INC(bbr_failed_mbuf_aloc);
13077 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13078 SOCK_SENDBUF_UNLOCK(so);
13079 error = ENOBUFS;
13080 sack_rxmit = 0;
13081 goto out;
13082 }
13083 m->m_data += max_linkhdr;
13084 m->m_len = hdrlen;
13085 /*
13086 * Start the m_copy functions from the closest mbuf to the
13087 * sb_offset in the socket buffer chain.
13088 */
13089 if ((sb_offset > sbavail(sb)) || ((len + sb_offset) > sbavail(sb))) {
13090 #ifdef BBR_INVARIANTS
13091 if ((len + sb_offset) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0)))
13092 panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u",
13093 tp, bbr, len, sb_offset, sbavail(sb), rsm,
13094 doing_retran_from,
13095 picked_up_retran,
13096 doing_tlp);
13097
13098 #endif
13099 /*
13100 * In this messed up situation we have two choices,
13101 * a) pretend the send worked, and just start timers
13102 * and what not (not good since that may lead us
13103 * back here a lot). <or> b) Send the lowest segment
13104 * in the map. <or> c) Drop the connection. Lets do
13105 * <b> which if it continues to happen will lead to
13106 * <c> via timeouts.
13107 */
13108 BBR_STAT_INC(bbr_offset_recovery);
13109 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
13110 sb_offset = 0;
13111 if (rsm == NULL) {
13112 sack_rxmit = 0;
13113 len = sbavail(sb);
13114 } else {
13115 sack_rxmit = 1;
13116 if (rsm->r_start != tp->snd_una) {
13117 /*
13118 * Things are really messed up, <c>
13119 * is the only thing to do.
13120 */
13121 BBR_STAT_INC(bbr_offset_drop);
13122 SOCK_SENDBUF_UNLOCK(so);
13123 (void)m_free(m);
13124 return (-EFAULT); /* tcp_drop() */
13125 }
13126 len = rsm->r_end - rsm->r_start;
13127 }
13128 if (len > sbavail(sb))
13129 len = sbavail(sb);
13130 if (len > maxseg)
13131 len = maxseg;
13132 }
13133 mb = sbsndptr_noadv(sb, sb_offset, &moff);
13134 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
13135 m_copydata(mb, moff, (int)len,
13136 mtod(m, caddr_t)+hdrlen);
13137 if (rsm == NULL)
13138 sbsndptr_adv(sb, mb, len);
13139 m->m_len += len;
13140 } else {
13141 struct sockbuf *msb;
13142
13143 if (rsm)
13144 msb = NULL;
13145 else
13146 msb = sb;
13147 #ifdef BBR_INVARIANTS
13148 if ((len + moff) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0))) {
13149 if (rsm) {
13150 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d ",
13151 tp, bbr, len, moff,
13152 sbavail(sb), rsm,
13153 tp->snd_una, rsm->r_flags, rsm->r_start,
13154 doing_retran_from,
13155 picked_up_retran,
13156 doing_tlp, sack_rxmit);
13157 } else {
13158 panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u sb_offset:%u snd_una:%u",
13159 tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);
13160 }
13161 }
13162 #endif
13163 m->m_next = tcp_m_copym(
13164 mb, moff, &len,
13165 if_hw_tsomaxsegcount,
13166 if_hw_tsomaxsegsize, msb,
13167 ((rsm == NULL) ? hw_tls : 0));
13168 if (len <= maxseg) {
13169 /*
13170 * Must have ran out of mbufs for the copy
13171 * shorten it to no longer need tso. Lets
13172 * not put on sendalot since we are low on
13173 * mbufs.
13174 */
13175 tso = 0;
13176 }
13177 if (m->m_next == NULL) {
13178 SOCK_SENDBUF_UNLOCK(so);
13179 (void)m_free(m);
13180 error = ENOBUFS;
13181 sack_rxmit = 0;
13182 goto out;
13183 }
13184 }
13185 #ifdef BBR_INVARIANTS
13186 if (tso && len < maxseg) {
13187 panic("tp:%p tso on, but len:%d < maxseg:%d",
13188 tp, len, maxseg);
13189 }
13190 if (tso && if_hw_tsomaxsegcount) {
13191 int32_t seg_cnt = 0;
13192 struct mbuf *foo;
13193
13194 foo = m;
13195 while (foo) {
13196 seg_cnt++;
13197 foo = foo->m_next;
13198 }
13199 if (seg_cnt > if_hw_tsomaxsegcount) {
13200 panic("seg_cnt:%d > max:%d", seg_cnt, if_hw_tsomaxsegcount);
13201 }
13202 }
13203 #endif
13204 /*
13205 * If we're sending everything we've got, set PUSH. (This
13206 * will keep happy those implementations which only give
13207 * data to the user when a buffer fills or a PUSH comes in.)
13208 */
13209 if (sb_offset + len == sbused(sb) &&
13210 sbused(sb) &&
13211 !(flags & TH_SYN)) {
13212 flags |= TH_PUSH;
13213 }
13214 SOCK_SENDBUF_UNLOCK(so);
13215 } else {
13216 SOCK_SENDBUF_UNLOCK(so);
13217 if (tp->t_flags & TF_ACKNOW)
13218 KMOD_TCPSTAT_INC(tcps_sndacks);
13219 else if (flags & (TH_SYN | TH_FIN | TH_RST))
13220 KMOD_TCPSTAT_INC(tcps_sndctrl);
13221 else
13222 KMOD_TCPSTAT_INC(tcps_sndwinup);
13223
13224 m = m_gethdr(M_NOWAIT, MT_DATA);
13225 if (m == NULL) {
13226 BBR_STAT_INC(bbr_failed_mbuf_aloc);
13227 bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13228 error = ENOBUFS;
13229 /* Fudge the send time since we could not send */
13230 sack_rxmit = 0;
13231 goto out;
13232 }
13233 #ifdef INET6
13234 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
13235 MHLEN >= hdrlen) {
13236 M_ALIGN(m, hdrlen);
13237 } else
13238 #endif
13239 m->m_data += max_linkhdr;
13240 m->m_len = hdrlen;
13241 }
13242 SOCK_SENDBUF_UNLOCK_ASSERT(so);
13243 m->m_pkthdr.rcvif = (struct ifnet *)0;
13244 #ifdef MAC
13245 mac_inpcb_create_mbuf(inp, m);
13246 #endif
13247 #ifdef INET6
13248 if (isipv6) {
13249 ip6 = mtod(m, struct ip6_hdr *);
13250 if (tp->t_port) {
13251 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
13252 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13253 udp->uh_dport = tp->t_port;
13254 ulen = hdrlen + len - sizeof(struct ip6_hdr);
13255 udp->uh_ulen = htons(ulen);
13256 th = (struct tcphdr *)(udp + 1);
13257 } else {
13258 th = (struct tcphdr *)(ip6 + 1);
13259 }
13260 tcpip_fillheaders(inp, tp->t_port, ip6, th);
13261 } else
13262 #endif /* INET6 */
13263 {
13264 ip = mtod(m, struct ip *);
13265 if (tp->t_port) {
13266 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
13267 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13268 udp->uh_dport = tp->t_port;
13269 ulen = hdrlen + len - sizeof(struct ip);
13270 udp->uh_ulen = htons(ulen);
13271 th = (struct tcphdr *)(udp + 1);
13272 } else {
13273 th = (struct tcphdr *)(ip + 1);
13274 }
13275 tcpip_fillheaders(inp, tp->t_port, ip, th);
13276 }
13277 /*
13278 * If we are doing retransmissions, then snd_nxt will not reflect
13279 * the first unsent octet. For ACK only packets, we do not want the
13280 * sequence number of the retransmitted packet, we want the sequence
13281 * number of the next unsent octet. So, if there is no data (and no
13282 * SYN or FIN), use snd_max instead of snd_nxt when filling in
13283 * ti_seq. But if we are in persist state, snd_max might reflect
13284 * one byte beyond the right edge of the window, so use snd_nxt in
13285 * that case, since we know we aren't doing a retransmission.
13286 * (retransmit and persist are mutually exclusive...)
13287 */
13288 if (sack_rxmit == 0) {
13289 if (len && ((flags & (TH_FIN | TH_SYN | TH_RST)) == 0)) {
13290 /* New data (including new persists) */
13291 th->th_seq = htonl(tp->snd_max);
13292 bbr_seq = tp->snd_max;
13293 } else if (flags & TH_SYN) {
13294 /* Syn's always send from iss */
13295 th->th_seq = htonl(tp->iss);
13296 bbr_seq = tp->iss;
13297 } else if (flags & TH_FIN) {
13298 if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) {
13299 /*
13300 * If we sent the fin already its 1 minus
13301 * snd_max
13302 */
13303 th->th_seq = (htonl(tp->snd_max - 1));
13304 bbr_seq = (tp->snd_max - 1);
13305 } else {
13306 /* First time FIN use snd_max */
13307 th->th_seq = htonl(tp->snd_max);
13308 bbr_seq = tp->snd_max;
13309 }
13310 } else {
13311 /*
13312 * len == 0 and not persist we use snd_max, sending
13313 * an ack unless we have sent the fin then its 1
13314 * minus.
13315 */
13316 /*
13317 * XXXRRS Question if we are in persists and we have
13318 * nothing outstanding to send and we have not sent
13319 * a FIN, we will send an ACK. In such a case it
13320 * might be better to send (tp->snd_una - 1) which
13321 * would force the peer to ack.
13322 */
13323 if (tp->t_flags & TF_SENTFIN) {
13324 th->th_seq = htonl(tp->snd_max - 1);
13325 bbr_seq = (tp->snd_max - 1);
13326 } else {
13327 th->th_seq = htonl(tp->snd_max);
13328 bbr_seq = tp->snd_max;
13329 }
13330 }
13331 } else {
13332 /* All retransmits use the rsm to guide the send */
13333 th->th_seq = htonl(rsm->r_start);
13334 bbr_seq = rsm->r_start;
13335 }
13336 th->th_ack = htonl(tp->rcv_nxt);
13337 if (optlen) {
13338 bcopy(opt, th + 1, optlen);
13339 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
13340 }
13341 tcp_set_flags(th, flags);
13342 /*
13343 * Calculate receive window. Don't shrink window, but avoid silly
13344 * window syndrome.
13345 */
13346 if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) &&
13347 recwin < maxseg)))
13348 recwin = 0;
13349 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
13350 recwin < (tp->rcv_adv - tp->rcv_nxt))
13351 recwin = (tp->rcv_adv - tp->rcv_nxt);
13352 if (recwin > TCP_MAXWIN << tp->rcv_scale)
13353 recwin = TCP_MAXWIN << tp->rcv_scale;
13354
13355 /*
13356 * According to RFC1323 the window field in a SYN (i.e., a <SYN> or
13357 * <SYN,ACK>) segment itself is never scaled. The <SYN,ACK> case is
13358 * handled in syncache.
13359 */
13360 if (flags & TH_SYN)
13361 th->th_win = htons((u_short)
13362 (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
13363 else {
13364 /* Avoid shrinking window with window scaling. */
13365 recwin = roundup2(recwin, 1 << tp->rcv_scale);
13366 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
13367 }
13368 /*
13369 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
13370 * window. This may cause the remote transmitter to stall. This
13371 * flag tells soreceive() to disable delayed acknowledgements when
13372 * draining the buffer. This can occur if the receiver is
13373 * attempting to read more data than can be buffered prior to
13374 * transmitting on the connection.
13375 */
13376 if (th->th_win == 0) {
13377 tp->t_sndzerowin++;
13378 tp->t_flags |= TF_RXWIN0SENT;
13379 } else
13380 tp->t_flags &= ~TF_RXWIN0SENT;
13381 /*
13382 * We don't support urgent data, but drag along
13383 * the pointer in case of a stack switch.
13384 */
13385 tp->snd_up = tp->snd_una;
13386 /*
13387 * Put TCP length in extended header, and then checksum extended
13388 * header and data.
13389 */
13390 m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
13391
13392 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
13393 if (to.to_flags & TOF_SIGNATURE) {
13394 /*
13395 * Calculate MD5 signature and put it into the place
13396 * determined before. NOTE: since TCP options buffer doesn't
13397 * point into mbuf's data, calculate offset and use it.
13398 */
13399 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
13400 (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
13401 /*
13402 * Do not send segment if the calculation of MD5
13403 * digest has failed.
13404 */
13405 goto out;
13406 }
13407 }
13408 #endif
13409
13410 #ifdef INET6
13411 if (isipv6) {
13412 /*
13413 * ip6_plen is not need to be filled now, and will be filled
13414 * in ip6_output.
13415 */
13416 if (tp->t_port) {
13417 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
13418 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13419 udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
13420 th->th_sum = htons(0);
13421 UDPSTAT_INC(udps_opackets);
13422 } else {
13423 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
13424 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13425 th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) +
13426 optlen + len, IPPROTO_TCP, 0);
13427 }
13428 }
13429 #endif
13430 #if defined(INET6) && defined(INET)
13431 else
13432 #endif
13433 #ifdef INET
13434 {
13435 if (tp->t_port) {
13436 m->m_pkthdr.csum_flags = CSUM_UDP;
13437 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13438 udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
13439 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
13440 th->th_sum = htons(0);
13441 UDPSTAT_INC(udps_opackets);
13442 } else {
13443 csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP;
13444 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13445 th->th_sum = in_pseudo(ip->ip_src.s_addr,
13446 ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
13447 IPPROTO_TCP + len + optlen));
13448 }
13449 /* IP version must be set here for ipv4/ipv6 checking later */
13450 KASSERT(ip->ip_v == IPVERSION,
13451 ("%s: IP version incorrect: %d", __func__, ip->ip_v));
13452 }
13453 #endif
13454
13455 /*
13456 * Enable TSO and specify the size of the segments. The TCP pseudo
13457 * header checksum is always provided. XXX: Fixme: This is currently
13458 * not the case for IPv6.
13459 */
13460 if (tso) {
13461 KASSERT(len > maxseg,
13462 ("%s: len:%d <= tso_segsz:%d", __func__, len, maxseg));
13463 m->m_pkthdr.csum_flags |= CSUM_TSO;
13464 csum_flags |= CSUM_TSO;
13465 m->m_pkthdr.tso_segsz = maxseg;
13466 }
13467 KASSERT(len + hdrlen == m_length(m, NULL),
13468 ("%s: mbuf chain different than expected: %d + %u != %u",
13469 __func__, len, hdrlen, m_length(m, NULL)));
13470
13471 #ifdef TCP_HHOOK
13472 /* Run HHOOK_TC_ESTABLISHED_OUT helper hooks. */
13473 hhook_run_tcp_est_out(tp, th, &to, len, tso);
13474 #endif
13475
13476 /* Log to the black box */
13477 if (tcp_bblogging_on(tp)) {
13478 union tcp_log_stackspecific log;
13479
13480 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
13481 /* Record info on type of transmission */
13482 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;
13483 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);
13484 log.u_bbr.flex3 = maxseg;
13485 log.u_bbr.flex4 = delay_calc;
13486 log.u_bbr.flex5 = bbr->rc_past_init_win;
13487 log.u_bbr.flex5 <<= 1;
13488 log.u_bbr.flex5 |= bbr->rc_no_pacing;
13489 log.u_bbr.flex5 <<= 29;
13490 log.u_bbr.flex5 |= tp->t_maxseg;
13491 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;
13492 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);
13493 /* lets poke in the low and the high here for debugging */
13494 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
13495 if (rsm || sack_rxmit) {
13496 if (doing_tlp)
13497 log.u_bbr.flex8 = 2;
13498 else
13499 log.u_bbr.flex8 = 1;
13500 } else {
13501 log.u_bbr.flex8 = 0;
13502 }
13503 lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
13504 len, &log, false, NULL, NULL, 0, tv);
13505 } else {
13506 lgb = NULL;
13507 }
13508 /*
13509 * Fill in IP length and desired time to live and send to IP level.
13510 * There should be a better way to handle ttl and tos; we could keep
13511 * them in the template, but need a way to checksum without them.
13512 */
13513 /*
13514 * m->m_pkthdr.len should have been set before cksum calcuration,
13515 * because in6_cksum() need it.
13516 */
13517 #ifdef INET6
13518 if (isipv6) {
13519 /*
13520 * we separately set hoplimit for every segment, since the
13521 * user might want to change the value via setsockopt. Also,
13522 * desired default hop limit might be changed via Neighbor
13523 * Discovery.
13524 */
13525 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
13526
13527 /*
13528 * Set the packet size here for the benefit of DTrace
13529 * probes. ip6_output() will set it properly; it's supposed
13530 * to include the option header lengths as well.
13531 */
13532 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
13533
13534 if (V_path_mtu_discovery && maxseg > V_tcp_minmss)
13535 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13536 else
13537 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13538
13539 if (tp->t_state == TCPS_SYN_SENT)
13540 TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
13541
13542 TCP_PROBE5(send, NULL, tp, ip6, tp, th);
13543 /* TODO: IPv6 IP6TOS_ECT bit on */
13544 error = ip6_output(m, inp->in6p_outputopts,
13545 &inp->inp_route6,
13546 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
13547 NULL, NULL, inp);
13548
13549 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
13550 mtu = inp->inp_route6.ro_nh->nh_mtu;
13551 }
13552 #endif /* INET6 */
13553 #if defined(INET) && defined(INET6)
13554 else
13555 #endif
13556 #ifdef INET
13557 {
13558 ip->ip_len = htons(m->m_pkthdr.len);
13559 #ifdef INET6
13560 if (isipv6)
13561 ip->ip_ttl = in6_selecthlim(inp, NULL);
13562 #endif /* INET6 */
13563 /*
13564 * If we do path MTU discovery, then we set DF on every
13565 * packet. This might not be the best thing to do according
13566 * to RFC3390 Section 2. However the tcp hostcache migitates
13567 * the problem so it affects only the first tcp connection
13568 * with a host.
13569 *
13570 * NB: Don't set DF on small MTU/MSS to have a safe
13571 * fallback.
13572 */
13573 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
13574 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13575 if (tp->t_port == 0 || len < V_tcp_minmss) {
13576 ip->ip_off |= htons(IP_DF);
13577 }
13578 } else {
13579 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13580 }
13581
13582 if (tp->t_state == TCPS_SYN_SENT)
13583 TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
13584
13585 TCP_PROBE5(send, NULL, tp, ip, tp, th);
13586
13587 error = ip_output(m, inp->inp_options, &inp->inp_route,
13588 ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
13589 inp);
13590 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
13591 mtu = inp->inp_route.ro_nh->nh_mtu;
13592 }
13593 #endif /* INET */
13594 if (lgb) {
13595 lgb->tlb_errno = error;
13596 lgb = NULL;
13597 }
13598
13599 out:
13600 /*
13601 * In transmit state, time the transmission and arrange for the
13602 * retransmit. In persist state, just set snd_max.
13603 */
13604 if (error == 0) {
13605 tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls);
13606 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
13607 (tp->t_flags & TF_SACK_PERMIT) &&
13608 tp->rcv_numsacks > 0)
13609 tcp_clean_dsack_blocks(tp);
13610 /* We sent an ack clear the bbr_segs_rcvd count */
13611 bbr->output_error_seen = 0;
13612 bbr->oerror_cnt = 0;
13613 bbr->bbr_segs_rcvd = 0;
13614 if (len == 0)
13615 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_SNDACK], 1);
13616 /* Do accounting for new sends */
13617 if ((len > 0) && (rsm == NULL)) {
13618 int idx;
13619 if (tp->snd_una == tp->snd_max) {
13620 /*
13621 * Special case to match google, when
13622 * nothing is in flight the delivered
13623 * time does get updated to the current
13624 * time (see tcp_rate_bsd.c).
13625 */
13626 bbr->r_ctl.rc_del_time = cts;
13627 }
13628 if (len >= maxseg) {
13629 idx = (len / maxseg) + 3;
13630 if (idx >= TCP_MSS_ACCT_ATIMER)
13631 counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1);
13632 else
13633 counter_u64_add(bbr_out_size[idx], 1);
13634 } else {
13635 /* smaller than a MSS */
13636 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);
13637 if (idx >= TCP_MSS_SMALL_MAX_SIZE_DIV)
13638 idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1);
13639 counter_u64_add(bbr_out_size[(idx + TCP_MSS_SMALL_SIZE_OFF)], 1);
13640 }
13641 }
13642 }
13643 abandon = 0;
13644 /*
13645 * We must do the send accounting before we log the output,
13646 * otherwise the state of the rsm could change and we account to the
13647 * wrong bucket.
13648 */
13649 if (len > 0) {
13650 bbr_do_send_accounting(tp, bbr, rsm, len, error);
13651 if (error == 0) {
13652 if (tp->snd_una == tp->snd_max)
13653 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
13654 }
13655 }
13656 bbr_log_output(bbr, tp, &to, len, bbr_seq, (uint8_t) flags, error,
13657 cts, mb, &abandon, rsm, 0, sb);
13658 if (abandon) {
13659 /*
13660 * If bbr_log_output destroys the TCB or sees a TH_RST being
13661 * sent we should hit this condition.
13662 */
13663 return (0);
13664 }
13665 if (bbr->rc_in_persist == 0) {
13666 /*
13667 * Advance snd_nxt over sequence space of this segment.
13668 */
13669 if (error)
13670 /* We don't log or do anything with errors */
13671 goto skip_upd;
13672
13673 if (tp->snd_una == tp->snd_max &&
13674 (len || (flags & (TH_SYN | TH_FIN)))) {
13675 /*
13676 * Update the time we just added data since none was
13677 * outstanding.
13678 */
13679 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
13680 bbr->rc_tp->t_acktime = ticks;
13681 }
13682 if (flags & (TH_SYN | TH_FIN) && (rsm == NULL)) {
13683 if (flags & TH_SYN) {
13684 /*
13685 * Smack the snd_max to iss + 1
13686 * if its a FO we will add len below.
13687 */
13688 tp->snd_max = tp->iss + 1;
13689 }
13690 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
13691 tp->snd_max++;
13692 tp->t_flags |= TF_SENTFIN;
13693 }
13694 }
13695 if (sack_rxmit == 0)
13696 tp->snd_max += len;
13697 skip_upd:
13698 if ((error == 0) && len)
13699 tot_len += len;
13700 } else {
13701 /* Persists case */
13702 int32_t xlen = len;
13703
13704 if (error)
13705 goto nomore;
13706
13707 if (flags & TH_SYN)
13708 ++xlen;
13709 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
13710 ++xlen;
13711 tp->t_flags |= TF_SENTFIN;
13712 }
13713 if (xlen && (tp->snd_una == tp->snd_max)) {
13714 /*
13715 * Update the time we just added data since none was
13716 * outstanding.
13717 */
13718 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
13719 bbr->rc_tp->t_acktime = ticks;
13720 }
13721 if (sack_rxmit == 0)
13722 tp->snd_max += xlen;
13723 tot_len += (len + optlen + ipoptlen);
13724 }
13725 nomore:
13726 if (error) {
13727 /*
13728 * Failures do not advance the seq counter above. For the
13729 * case of ENOBUFS we will fall out and become ack-clocked.
13730 * capping the cwnd at the current flight.
13731 * Everything else will just have to retransmit with the timer
13732 * (no pacer).
13733 */
13734 SOCK_SENDBUF_UNLOCK_ASSERT(so);
13735 BBR_STAT_INC(bbr_saw_oerr);
13736 /* Clear all delay/early tracks */
13737 bbr->r_ctl.rc_hptsi_agg_delay = 0;
13738 bbr->r_ctl.rc_agg_early = 0;
13739 bbr->r_agg_early_set = 0;
13740 bbr->output_error_seen = 1;
13741 if (bbr->oerror_cnt < 0xf)
13742 bbr->oerror_cnt++;
13743 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {
13744 /* drop the session */
13745 return (-ENETDOWN);
13746 }
13747 switch (error) {
13748 case ENOBUFS:
13749 /*
13750 * Make this guy have to get ack's to send
13751 * more but lets make sure we don't
13752 * slam him below a T-O (1MSS).
13753 */
13754 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
13755 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13756 bbr->r_ctl.rc_lost_bytes)) - maxseg;
13757 if (tp->snd_cwnd < maxseg)
13758 tp->snd_cwnd = maxseg;
13759 }
13760 pacing_delay = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;
13761 BBR_STAT_INC(bbr_saw_enobuf);
13762 if (bbr->bbr_hdrw_pacing)
13763 counter_u64_add(bbr_hdwr_pacing_enobuf, 1);
13764 else
13765 counter_u64_add(bbr_nohdwr_pacing_enobuf, 1);
13766 /*
13767 * Here even in the enobuf's case we want to do our
13768 * state update. The reason being we may have been
13769 * called by the input function. If so we have had
13770 * things change.
13771 */
13772 error = 0;
13773 goto enobufs;
13774 case EMSGSIZE:
13775 /*
13776 * For some reason the interface we used initially
13777 * to send segments changed to another or lowered
13778 * its MTU. If TSO was active we either got an
13779 * interface without TSO capabilits or TSO was
13780 * turned off. If we obtained mtu from ip_output()
13781 * then update it and try again.
13782 */
13783 /* Turn on tracing (or try to) */
13784 {
13785 int old_maxseg;
13786
13787 old_maxseg = tp->t_maxseg;
13788 BBR_STAT_INC(bbr_saw_emsgsiz);
13789 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, csum_flags, tso, cts);
13790 if (mtu != 0)
13791 tcp_mss_update(tp, -1, mtu, NULL, NULL);
13792 if (old_maxseg <= tp->t_maxseg) {
13793 /* Huh it did not shrink? */
13794 tp->t_maxseg = old_maxseg - 40;
13795 if (tp->t_maxseg < V_tcp_mssdflt) {
13796 /*
13797 * The MSS is so small we should not
13798 * process incoming SACK's since we are
13799 * subject to attack in such a case.
13800 */
13801 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
13802 } else {
13803 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
13804 }
13805 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, 0, tso, cts);
13806 }
13807 /*
13808 * Nuke all other things that can interfere
13809 * with pacing delay
13810 */
13811 if ((tot_len + len) && (len >= tp->t_maxseg)) {
13812 pacing_delay = bbr_get_pacing_delay(bbr,
13813 bbr->r_ctl.rc_bbr_hptsi_gain,
13814 (tot_len + len), cts, 0);
13815 if (pacing_delay < bbr_error_base_paceout)
13816 pacing_delay = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13817 } else
13818 pacing_delay = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
13819 bbr->rc_output_starts_timer = 1;
13820 bbr_start_hpts_timer(bbr, tp, cts, 10, pacing_delay,
13821 tot_len);
13822 return (error);
13823 }
13824 case EPERM:
13825 case EACCES:
13826 tp->t_softerror = error;
13827 /* FALLTHROUGH */
13828 case EHOSTDOWN:
13829 case EHOSTUNREACH:
13830 case ENETDOWN:
13831 case ENETUNREACH:
13832 if (TCPS_HAVERCVDSYN(tp->t_state)) {
13833 tp->t_softerror = error;
13834 error = 0;
13835 }
13836 /* FALLTHROUGH */
13837 default:
13838 pacing_delay = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;
13839 bbr->rc_output_starts_timer = 1;
13840 bbr_start_hpts_timer(bbr, tp, cts, 11, pacing_delay, 0);
13841 return (error);
13842 }
13843 #ifdef STATS
13844 } else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&
13845 len &&
13846 (rsm == NULL) &&
13847 (bbr->rc_in_persist == 0)) {
13848 tp->gput_seq = bbr_seq;
13849 tp->gput_ack = bbr_seq +
13850 min(sbavail(&so->so_snd) - sb_offset, sendwin);
13851 tp->gput_ts = cts;
13852 tp->t_flags |= TF_GPUTINPROG;
13853 #endif
13854 }
13855 KMOD_TCPSTAT_INC(tcps_sndtotal);
13856 if ((bbr->bbr_hdw_pace_ena) &&
13857 (bbr->bbr_attempt_hdwr_pace == 0) &&
13858 (bbr->rc_past_init_win) &&
13859 (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
13860 (get_filter_value(&bbr->r_ctl.rc_delrate)) &&
13861 (inp->inp_route.ro_nh &&
13862 inp->inp_route.ro_nh->nh_ifp)) {
13863 /*
13864 * We are past the initial window and
13865 * have at least one measurement so we
13866 * could use hardware pacing if its available.
13867 * We have an interface and we have not attempted
13868 * to setup hardware pacing, lets try to now.
13869 */
13870 uint64_t rate_wanted;
13871 int err = 0;
13872
13873 rate_wanted = bbr_get_hardware_rate(bbr);
13874 bbr->bbr_attempt_hdwr_pace = 1;
13875 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,
13876 inp->inp_route.ro_nh->nh_ifp,
13877 rate_wanted,
13878 (RS_PACING_GEQ|RS_PACING_SUB_OK),
13879 &err, NULL);
13880 if (bbr->r_ctl.crte) {
13881 bbr_type_log_hdwr_pacing(bbr,
13882 bbr->r_ctl.crte->ptbl->rs_ifp,
13883 rate_wanted,
13884 bbr->r_ctl.crte->rate,
13885 __LINE__, cts, err);
13886 BBR_STAT_INC(bbr_hdwr_rl_add_ok);
13887 counter_u64_add(bbr_flows_nohdwr_pacing, -1);
13888 counter_u64_add(bbr_flows_whdwr_pacing, 1);
13889 bbr->bbr_hdrw_pacing = 1;
13890 /* Now what is our gain status? */
13891 if (bbr->r_ctl.crte->rate < rate_wanted) {
13892 /* We have a problem */
13893 bbr_setup_less_of_rate(bbr, cts,
13894 bbr->r_ctl.crte->rate, rate_wanted);
13895 } else {
13896 /* We are good */
13897 bbr->gain_is_limited = 0;
13898 bbr->skip_gain = 0;
13899 }
13900 tcp_bbr_tso_size_check(bbr, cts);
13901 } else {
13902 bbr_type_log_hdwr_pacing(bbr,
13903 inp->inp_route.ro_nh->nh_ifp,
13904 rate_wanted,
13905 0,
13906 __LINE__, cts, err);
13907 BBR_STAT_INC(bbr_hdwr_rl_add_fail);
13908 }
13909 }
13910 if (bbr->bbr_hdrw_pacing) {
13911 /*
13912 * Worry about cases where the route
13913 * changes or something happened that we
13914 * lost our hardware pacing possibly during
13915 * the last ip_output call.
13916 */
13917 if (inp->inp_snd_tag == NULL) {
13918 /* A change during ip output disabled hw pacing? */
13919 bbr->bbr_hdrw_pacing = 0;
13920 } else if ((inp->inp_route.ro_nh == NULL) ||
13921 (inp->inp_route.ro_nh->nh_ifp != inp->inp_snd_tag->ifp)) {
13922 /*
13923 * We had an interface or route change,
13924 * detach from the current hdwr pacing
13925 * and setup to re-attempt next go
13926 * round.
13927 */
13928 bbr->bbr_hdrw_pacing = 0;
13929 bbr->bbr_attempt_hdwr_pace = 0;
13930 tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
13931 tcp_bbr_tso_size_check(bbr, cts);
13932 }
13933 }
13934 /*
13935 * Data sent (as far as we can tell). If this advertises a larger
13936 * window than any other segment, then remember the size of the
13937 * advertised window. Any pending ACK has now been sent.
13938 */
13939 if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
13940 tp->rcv_adv = tp->rcv_nxt + recwin;
13941
13942 tp->last_ack_sent = tp->rcv_nxt;
13943 if ((error == 0) &&
13944 (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&
13945 (doing_tlp == 0) &&
13946 (tso == 0) &&
13947 (len > 0) &&
13948 ((flags & TH_RST) == 0) &&
13949 ((flags & TH_SYN) == 0) &&
13950 (IN_RECOVERY(tp->t_flags) == 0) &&
13951 (bbr->rc_in_persist == 0) &&
13952 (tot_len < bbr->r_ctl.rc_pace_max_segs)) {
13953 /*
13954 * For non-tso we need to goto again until we have sent out
13955 * enough data to match what we are hptsi out every hptsi
13956 * interval.
13957 */
13958 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
13959 /* Make sure snd_nxt is drug up */
13960 tp->snd_nxt = tp->snd_max;
13961 }
13962 if (rsm != NULL) {
13963 rsm = NULL;
13964 goto skip_again;
13965 }
13966 rsm = NULL;
13967 sack_rxmit = 0;
13968 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
13969 goto again;
13970 }
13971 skip_again:
13972 if ((error == 0) && (flags & TH_FIN))
13973 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_FIN);
13974 if ((error == 0) && (flags & TH_RST))
13975 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
13976 if (((flags & (TH_RST | TH_SYN | TH_FIN)) == 0) && tot_len) {
13977 /*
13978 * Calculate/Re-Calculate the hptsi timeout in usecs based on
13979 * what we have sent so far
13980 */
13981 pacing_delay = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
13982 if (bbr->rc_no_pacing)
13983 pacing_delay = 0;
13984 }
13985 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
13986 enobufs:
13987 if (bbr->rc_use_google == 0)
13988 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
13989 bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13990 bbr->r_ctl.rc_lost_bytes)));
13991 bbr->rc_output_starts_timer = 1;
13992 if (bbr->bbr_use_rack_cheat &&
13993 (more_to_rxt ||
13994 ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {
13995 /* Rack cheats and shotguns out all rxt's 1ms apart */
13996 if (pacing_delay > 1000)
13997 pacing_delay = 1000;
13998 }
13999 if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {
14000 /*
14001 * We don't change the tso size until some number of sends
14002 * to give the hardware commands time to get down
14003 * to the interface.
14004 */
14005 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;
14006 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {
14007 bbr->hw_pacing_set = 1;
14008 tcp_bbr_tso_size_check(bbr, cts);
14009 }
14010 }
14011 bbr_start_hpts_timer(bbr, tp, cts, 12, pacing_delay, tot_len);
14012 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
14013 /* Make sure snd_nxt is drug up */
14014 tp->snd_nxt = tp->snd_max;
14015 }
14016 return (error);
14017
14018 }
14019
14020 /*
14021 * See bbr_output_wtime() for return values.
14022 */
14023 static int
bbr_output(struct tcpcb * tp)14024 bbr_output(struct tcpcb *tp)
14025 {
14026 int32_t ret;
14027 struct timeval tv;
14028
14029 NET_EPOCH_ASSERT();
14030
14031 INP_WLOCK_ASSERT(tptoinpcb(tp));
14032 (void)tcp_get_usecs(&tv);
14033 ret = bbr_output_wtime(tp, &tv);
14034 return (ret);
14035 }
14036
14037 static void
bbr_mtu_chg(struct tcpcb * tp)14038 bbr_mtu_chg(struct tcpcb *tp)
14039 {
14040 struct tcp_bbr *bbr;
14041 struct bbr_sendmap *rsm, *frsm = NULL;
14042 uint32_t maxseg;
14043
14044 /*
14045 * The MTU has changed. a) Clear the sack filter. b) Mark everything
14046 * over the current size as SACK_PASS so a retransmit will occur.
14047 */
14048
14049 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14050 maxseg = tp->t_maxseg - bbr->rc_last_options;
14051 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
14052 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
14053 /* Don't mess with ones acked (by sack?) */
14054 if (rsm->r_flags & BBR_ACKED)
14055 continue;
14056 if ((rsm->r_end - rsm->r_start) > maxseg) {
14057 /*
14058 * We mark sack-passed on all the previous large
14059 * sends we did. This will force them to retransmit.
14060 */
14061 rsm->r_flags |= BBR_SACK_PASSED;
14062 if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&
14063 bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {
14064 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
14065 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
14066 rsm->r_flags |= BBR_MARKED_LOST;
14067 }
14068 if (frsm == NULL)
14069 frsm = rsm;
14070 }
14071 }
14072 if (frsm) {
14073 bbr->r_ctl.rc_resend = frsm;
14074 }
14075 }
14076
14077 static int
bbr_pru_options(struct tcpcb * tp,int flags)14078 bbr_pru_options(struct tcpcb *tp, int flags)
14079 {
14080 if (flags & PRUS_OOB)
14081 return (EOPNOTSUPP);
14082 return (0);
14083 }
14084
14085 static void
bbr_switch_failed(struct tcpcb * tp)14086 bbr_switch_failed(struct tcpcb *tp)
14087 {
14088 /*
14089 * If a switch fails we only need to
14090 * make sure mbuf_queuing is still in place.
14091 * We also need to make sure we are still in
14092 * ticks granularity (though we should probably
14093 * change bbr to go to USECs).
14094 *
14095 * For timers we need to see if we are still in the
14096 * pacer (if our flags are up) if so we are good, if
14097 * not we need to get back into the pacer.
14098 */
14099 struct timeval tv;
14100 uint32_t cts;
14101 uint32_t toval;
14102 struct tcp_bbr *bbr;
14103 struct hpts_diag diag;
14104
14105 tp->t_flags2 |= TF2_CANNOT_DO_ECN;
14106 tp->t_flags2 |= TF2_SUPPORTS_MBUFQ;
14107 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
14108 if (tp->t_in_hpts > IHPTS_NONE) {
14109 return;
14110 }
14111 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14112 cts = tcp_get_usecs(&tv);
14113 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
14114 if (TSTMP_GT(bbr->rc_pacer_started, cts)) {
14115 toval = bbr->rc_pacer_started - cts;
14116 } else {
14117 /* one slot please */
14118 toval = HPTS_USECS_PER_SLOT;
14119 }
14120 } else if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
14121 if (TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
14122 toval = bbr->r_ctl.rc_timer_exp - cts;
14123 } else {
14124 /* one slot please */
14125 toval = HPTS_USECS_PER_SLOT;
14126 }
14127 } else
14128 toval = HPTS_USECS_PER_SLOT;
14129 tcp_hpts_insert(tp, toval, &diag);
14130 bbr_log_hpts_diag(bbr, cts, &diag);
14131 }
14132
14133 struct tcp_function_block __tcp_bbr = {
14134 .tfb_tcp_block_name = __XSTRING(STACKNAME),
14135 .tfb_tcp_output = bbr_output,
14136 .tfb_do_queued_segments = ctf_do_queued_segments,
14137 .tfb_do_segment_nounlock = bbr_do_segment_nounlock,
14138 .tfb_tcp_do_segment = bbr_do_segment,
14139 .tfb_tcp_ctloutput = bbr_ctloutput,
14140 .tfb_tcp_fb_init = bbr_init,
14141 .tfb_tcp_fb_fini = bbr_fini,
14142 .tfb_tcp_timer_stop_all = bbr_stopall,
14143 .tfb_tcp_rexmit_tmr = bbr_remxt_tmr,
14144 .tfb_tcp_handoff_ok = bbr_handoff_ok,
14145 .tfb_tcp_mtu_chg = bbr_mtu_chg,
14146 .tfb_pru_options = bbr_pru_options,
14147 .tfb_switch_failed = bbr_switch_failed,
14148 .tfb_flags = TCP_FUNC_OUTPUT_CANDROP | TCP_FUNC_DEFAULT_OK,
14149 };
14150
14151 /*
14152 * bbr_ctloutput() must drop the inpcb lock before performing copyin on
14153 * socket option arguments. When it re-acquires the lock after the copy, it
14154 * has to revalidate that the connection is still valid for the socket
14155 * option.
14156 */
14157 static int
bbr_set_sockopt(struct tcpcb * tp,struct sockopt * sopt)14158 bbr_set_sockopt(struct tcpcb *tp, struct sockopt *sopt)
14159 {
14160 struct epoch_tracker et;
14161 struct inpcb *inp = tptoinpcb(tp);
14162 struct tcp_bbr *bbr;
14163 int32_t error = 0, optval;
14164
14165 switch (sopt->sopt_level) {
14166 case IPPROTO_IPV6:
14167 case IPPROTO_IP:
14168 return (tcp_default_ctloutput(tp, sopt));
14169 }
14170
14171 switch (sopt->sopt_name) {
14172 case TCP_RACK_PACE_MAX_SEG:
14173 case TCP_RACK_MIN_TO:
14174 case TCP_RACK_REORD_THRESH:
14175 case TCP_RACK_REORD_FADE:
14176 case TCP_RACK_TLP_THRESH:
14177 case TCP_RACK_PKT_DELAY:
14178 case TCP_BBR_ALGORITHM:
14179 case TCP_BBR_TSLIMITS:
14180 case TCP_BBR_IWINTSO:
14181 case TCP_BBR_STARTUP_PG:
14182 case TCP_BBR_DRAIN_PG:
14183 case TCP_BBR_PROBE_RTT_INT:
14184 case TCP_BBR_PROBE_RTT_GAIN:
14185 case TCP_BBR_PROBE_RTT_LEN:
14186 case TCP_BBR_STARTUP_LOSS_EXIT:
14187 case TCP_BBR_USEDEL_RATE:
14188 case TCP_BBR_MIN_RTO:
14189 case TCP_BBR_MAX_RTO:
14190 case TCP_BBR_PACE_PER_SEC:
14191 case TCP_DELACK:
14192 case TCP_BBR_PACE_DEL_TAR:
14193 case TCP_BBR_SEND_IWND_IN_TSO:
14194 case TCP_BBR_EXTRA_STATE:
14195 case TCP_BBR_UTTER_MAX_TSO:
14196 case TCP_BBR_MIN_TOPACEOUT:
14197 case TCP_BBR_FLOOR_MIN_TSO:
14198 case TCP_BBR_TSTMP_RAISES:
14199 case TCP_BBR_POLICER_DETECT:
14200 case TCP_BBR_USE_RACK_CHEAT:
14201 case TCP_DATA_AFTER_CLOSE:
14202 case TCP_BBR_HDWR_PACE:
14203 case TCP_BBR_PACE_SEG_MAX:
14204 case TCP_BBR_PACE_SEG_MIN:
14205 case TCP_BBR_PACE_CROSS:
14206 case TCP_BBR_PACE_OH:
14207 case TCP_BBR_TMR_PACE_OH:
14208 case TCP_BBR_RACK_RTT_USE:
14209 case TCP_BBR_RETRAN_WTSO:
14210 break;
14211 default:
14212 return (tcp_default_ctloutput(tp, sopt));
14213 break;
14214 }
14215 INP_WUNLOCK(inp);
14216 error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval));
14217 if (error)
14218 return (error);
14219 INP_WLOCK(inp);
14220 if (inp->inp_flags & INP_DROPPED) {
14221 INP_WUNLOCK(inp);
14222 return (ECONNRESET);
14223 }
14224 if (tp->t_fb != &__tcp_bbr) {
14225 INP_WUNLOCK(inp);
14226 return (ENOPROTOOPT);
14227 }
14228 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14229 switch (sopt->sopt_name) {
14230 case TCP_BBR_PACE_PER_SEC:
14231 BBR_OPTS_INC(tcp_bbr_pace_per_sec);
14232 bbr->r_ctl.bbr_hptsi_per_second = optval;
14233 break;
14234 case TCP_BBR_PACE_DEL_TAR:
14235 BBR_OPTS_INC(tcp_bbr_pace_del_tar);
14236 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;
14237 break;
14238 case TCP_BBR_PACE_SEG_MAX:
14239 BBR_OPTS_INC(tcp_bbr_pace_seg_max);
14240 bbr->r_ctl.bbr_hptsi_segments_max = optval;
14241 break;
14242 case TCP_BBR_PACE_SEG_MIN:
14243 BBR_OPTS_INC(tcp_bbr_pace_seg_min);
14244 bbr->r_ctl.bbr_hptsi_bytes_min = optval;
14245 break;
14246 case TCP_BBR_PACE_CROSS:
14247 BBR_OPTS_INC(tcp_bbr_pace_cross);
14248 bbr->r_ctl.bbr_cross_over = optval;
14249 break;
14250 case TCP_BBR_ALGORITHM:
14251 BBR_OPTS_INC(tcp_bbr_algorithm);
14252 if (optval && (bbr->rc_use_google == 0)) {
14253 /* Turn on the google mode */
14254 bbr_google_mode_on(bbr);
14255 if ((optval > 3) && (optval < 500)) {
14256 /*
14257 * Must be at least greater than .3%
14258 * and must be less than 50.0%.
14259 */
14260 bbr->r_ctl.bbr_google_discount = optval;
14261 }
14262 } else if ((optval == 0) && (bbr->rc_use_google == 1)) {
14263 /* Turn off the google mode */
14264 bbr_google_mode_off(bbr);
14265 }
14266 break;
14267 case TCP_BBR_TSLIMITS:
14268 BBR_OPTS_INC(tcp_bbr_tslimits);
14269 if (optval == 1)
14270 bbr->rc_use_ts_limit = 1;
14271 else if (optval == 0)
14272 bbr->rc_use_ts_limit = 0;
14273 else
14274 error = EINVAL;
14275 break;
14276
14277 case TCP_BBR_IWINTSO:
14278 BBR_OPTS_INC(tcp_bbr_iwintso);
14279 if ((optval >= 0) && (optval < 128)) {
14280 uint32_t twin;
14281
14282 bbr->rc_init_win = optval;
14283 twin = bbr_initial_cwnd(bbr, tp);
14284 if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))
14285 tp->snd_cwnd = twin;
14286 else
14287 error = EBUSY;
14288 } else
14289 error = EINVAL;
14290 break;
14291 case TCP_BBR_STARTUP_PG:
14292 BBR_OPTS_INC(tcp_bbr_startup_pg);
14293 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE)) {
14294 bbr->r_ctl.rc_startup_pg = optval;
14295 if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
14296 bbr->r_ctl.rc_bbr_hptsi_gain = optval;
14297 }
14298 } else
14299 error = EINVAL;
14300 break;
14301 case TCP_BBR_DRAIN_PG:
14302 BBR_OPTS_INC(tcp_bbr_drain_pg);
14303 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE))
14304 bbr->r_ctl.rc_drain_pg = optval;
14305 else
14306 error = EINVAL;
14307 break;
14308 case TCP_BBR_PROBE_RTT_LEN:
14309 BBR_OPTS_INC(tcp_bbr_probertt_len);
14310 if (optval <= 1)
14311 reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));
14312 else
14313 error = EINVAL;
14314 break;
14315 case TCP_BBR_PROBE_RTT_GAIN:
14316 BBR_OPTS_INC(tcp_bbr_probertt_gain);
14317 if (optval <= BBR_UNIT)
14318 bbr->r_ctl.bbr_rttprobe_gain_val = optval;
14319 else
14320 error = EINVAL;
14321 break;
14322 case TCP_BBR_PROBE_RTT_INT:
14323 BBR_OPTS_INC(tcp_bbr_probe_rtt_int);
14324 if (optval > 1000)
14325 bbr->r_ctl.rc_probertt_int = optval;
14326 else
14327 error = EINVAL;
14328 break;
14329 case TCP_BBR_MIN_TOPACEOUT:
14330 BBR_OPTS_INC(tcp_bbr_topaceout);
14331 if (optval == 0) {
14332 bbr->no_pacing_until = 0;
14333 bbr->rc_no_pacing = 0;
14334 } else if (optval <= 0x00ff) {
14335 bbr->no_pacing_until = optval;
14336 if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&
14337 (bbr->rc_bbr_state == BBR_STATE_STARTUP)){
14338 /* Turn on no pacing */
14339 bbr->rc_no_pacing = 1;
14340 }
14341 } else
14342 error = EINVAL;
14343 break;
14344 case TCP_BBR_STARTUP_LOSS_EXIT:
14345 BBR_OPTS_INC(tcp_bbr_startup_loss_exit);
14346 bbr->rc_loss_exit = optval;
14347 break;
14348 case TCP_BBR_USEDEL_RATE:
14349 error = EINVAL;
14350 break;
14351 case TCP_BBR_MIN_RTO:
14352 BBR_OPTS_INC(tcp_bbr_min_rto);
14353 bbr->r_ctl.rc_min_rto_ms = optval;
14354 break;
14355 case TCP_BBR_MAX_RTO:
14356 BBR_OPTS_INC(tcp_bbr_max_rto);
14357 bbr->rc_max_rto_sec = optval;
14358 break;
14359 case TCP_RACK_MIN_TO:
14360 /* Minimum time between rack t-o's in ms */
14361 BBR_OPTS_INC(tcp_rack_min_to);
14362 bbr->r_ctl.rc_min_to = optval;
14363 break;
14364 case TCP_RACK_REORD_THRESH:
14365 /* RACK reorder threshold (shift amount) */
14366 BBR_OPTS_INC(tcp_rack_reord_thresh);
14367 if ((optval > 0) && (optval < 31))
14368 bbr->r_ctl.rc_reorder_shift = optval;
14369 else
14370 error = EINVAL;
14371 break;
14372 case TCP_RACK_REORD_FADE:
14373 /* Does reordering fade after ms time */
14374 BBR_OPTS_INC(tcp_rack_reord_fade);
14375 bbr->r_ctl.rc_reorder_fade = optval;
14376 break;
14377 case TCP_RACK_TLP_THRESH:
14378 /* RACK TLP theshold i.e. srtt+(srtt/N) */
14379 BBR_OPTS_INC(tcp_rack_tlp_thresh);
14380 if (optval)
14381 bbr->rc_tlp_threshold = optval;
14382 else
14383 error = EINVAL;
14384 break;
14385 case TCP_BBR_USE_RACK_CHEAT:
14386 BBR_OPTS_INC(tcp_use_rackcheat);
14387 if (bbr->rc_use_google) {
14388 error = EINVAL;
14389 break;
14390 }
14391 BBR_OPTS_INC(tcp_rack_cheat);
14392 if (optval)
14393 bbr->bbr_use_rack_cheat = 1;
14394 else
14395 bbr->bbr_use_rack_cheat = 0;
14396 break;
14397 case TCP_BBR_FLOOR_MIN_TSO:
14398 BBR_OPTS_INC(tcp_utter_max_tso);
14399 if ((optval >= 0) && (optval < 40))
14400 bbr->r_ctl.bbr_hptsi_segments_floor = optval;
14401 else
14402 error = EINVAL;
14403 break;
14404 case TCP_BBR_UTTER_MAX_TSO:
14405 BBR_OPTS_INC(tcp_utter_max_tso);
14406 if ((optval >= 0) && (optval < 0xffff))
14407 bbr->r_ctl.bbr_utter_max = optval;
14408 else
14409 error = EINVAL;
14410 break;
14411
14412 case TCP_BBR_EXTRA_STATE:
14413 BBR_OPTS_INC(tcp_extra_state);
14414 if (optval)
14415 bbr->rc_use_idle_restart = 1;
14416 else
14417 bbr->rc_use_idle_restart = 0;
14418 break;
14419 case TCP_BBR_SEND_IWND_IN_TSO:
14420 BBR_OPTS_INC(tcp_iwnd_tso);
14421 if (optval) {
14422 bbr->bbr_init_win_cheat = 1;
14423 if (bbr->rc_past_init_win == 0) {
14424 uint32_t cts;
14425 cts = tcp_get_usecs(&bbr->rc_tv);
14426 tcp_bbr_tso_size_check(bbr, cts);
14427 }
14428 } else
14429 bbr->bbr_init_win_cheat = 0;
14430 break;
14431 case TCP_BBR_HDWR_PACE:
14432 BBR_OPTS_INC(tcp_hdwr_pacing);
14433 if (optval){
14434 bbr->bbr_hdw_pace_ena = 1;
14435 bbr->bbr_attempt_hdwr_pace = 0;
14436 } else {
14437 bbr->bbr_hdw_pace_ena = 0;
14438 #ifdef RATELIMIT
14439 if (bbr->r_ctl.crte != NULL) {
14440 tcp_rel_pacing_rate(bbr->r_ctl.crte, tp);
14441 bbr->r_ctl.crte = NULL;
14442 }
14443 #endif
14444 }
14445 break;
14446
14447 case TCP_DELACK:
14448 BBR_OPTS_INC(tcp_delack);
14449 if (optval < 100) {
14450 if (optval == 0) /* off */
14451 tp->t_delayed_ack = 0;
14452 else if (optval == 1) /* on which is 2 */
14453 tp->t_delayed_ack = 2;
14454 else /* higher than 2 and less than 100 */
14455 tp->t_delayed_ack = optval;
14456 if (tp->t_flags & TF_DELACK) {
14457 tp->t_flags &= ~TF_DELACK;
14458 tp->t_flags |= TF_ACKNOW;
14459 NET_EPOCH_ENTER(et);
14460 bbr_output(tp);
14461 NET_EPOCH_EXIT(et);
14462 }
14463 } else
14464 error = EINVAL;
14465 break;
14466 case TCP_RACK_PKT_DELAY:
14467 /* RACK added ms i.e. rack-rtt + reord + N */
14468 BBR_OPTS_INC(tcp_rack_pkt_delay);
14469 bbr->r_ctl.rc_pkt_delay = optval;
14470 break;
14471
14472 case TCP_BBR_RETRAN_WTSO:
14473 BBR_OPTS_INC(tcp_retran_wtso);
14474 if (optval)
14475 bbr->rc_resends_use_tso = 1;
14476 else
14477 bbr->rc_resends_use_tso = 0;
14478 break;
14479 case TCP_DATA_AFTER_CLOSE:
14480 BBR_OPTS_INC(tcp_data_ac);
14481 if (optval)
14482 bbr->rc_allow_data_af_clo = 1;
14483 else
14484 bbr->rc_allow_data_af_clo = 0;
14485 break;
14486 case TCP_BBR_POLICER_DETECT:
14487 BBR_OPTS_INC(tcp_policer_det);
14488 if (bbr->rc_use_google == 0)
14489 error = EINVAL;
14490 else if (optval)
14491 bbr->r_use_policer = 1;
14492 else
14493 bbr->r_use_policer = 0;
14494 break;
14495
14496 case TCP_BBR_TSTMP_RAISES:
14497 BBR_OPTS_INC(tcp_ts_raises);
14498 if (optval)
14499 bbr->ts_can_raise = 1;
14500 else
14501 bbr->ts_can_raise = 0;
14502 break;
14503 case TCP_BBR_TMR_PACE_OH:
14504 BBR_OPTS_INC(tcp_pacing_oh_tmr);
14505 if (bbr->rc_use_google) {
14506 error = EINVAL;
14507 } else {
14508 if (optval)
14509 bbr->r_ctl.rc_incr_tmrs = 1;
14510 else
14511 bbr->r_ctl.rc_incr_tmrs = 0;
14512 }
14513 break;
14514 case TCP_BBR_PACE_OH:
14515 BBR_OPTS_INC(tcp_pacing_oh);
14516 if (bbr->rc_use_google) {
14517 error = EINVAL;
14518 } else {
14519 if (optval > (BBR_INCL_TCP_OH|
14520 BBR_INCL_IP_OH|
14521 BBR_INCL_ENET_OH)) {
14522 error = EINVAL;
14523 break;
14524 }
14525 if (optval & BBR_INCL_TCP_OH)
14526 bbr->r_ctl.rc_inc_tcp_oh = 1;
14527 else
14528 bbr->r_ctl.rc_inc_tcp_oh = 0;
14529 if (optval & BBR_INCL_IP_OH)
14530 bbr->r_ctl.rc_inc_ip_oh = 1;
14531 else
14532 bbr->r_ctl.rc_inc_ip_oh = 0;
14533 if (optval & BBR_INCL_ENET_OH)
14534 bbr->r_ctl.rc_inc_enet_oh = 1;
14535 else
14536 bbr->r_ctl.rc_inc_enet_oh = 0;
14537 }
14538 break;
14539 default:
14540 return (tcp_default_ctloutput(tp, sopt));
14541 break;
14542 }
14543 tcp_log_socket_option(tp, sopt->sopt_name, optval, error);
14544 INP_WUNLOCK(inp);
14545 return (error);
14546 }
14547
14548 /*
14549 * return 0 on success, error-num on failure
14550 */
14551 static int
bbr_get_sockopt(struct tcpcb * tp,struct sockopt * sopt)14552 bbr_get_sockopt(struct tcpcb *tp, struct sockopt *sopt)
14553 {
14554 struct inpcb *inp = tptoinpcb(tp);
14555 struct tcp_bbr *bbr;
14556 uint64_t loptval;
14557 int32_t error, optval;
14558
14559 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14560 if (bbr == NULL) {
14561 INP_WUNLOCK(inp);
14562 return (EINVAL);
14563 }
14564 /*
14565 * Because all our options are either boolean or an int, we can just
14566 * pull everything into optval and then unlock and copy. If we ever
14567 * add a option that is not a int, then this will have quite an
14568 * impact to this routine.
14569 */
14570 switch (sopt->sopt_name) {
14571 case TCP_BBR_PACE_PER_SEC:
14572 optval = bbr->r_ctl.bbr_hptsi_per_second;
14573 break;
14574 case TCP_BBR_PACE_DEL_TAR:
14575 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;
14576 break;
14577 case TCP_BBR_PACE_SEG_MAX:
14578 optval = bbr->r_ctl.bbr_hptsi_segments_max;
14579 break;
14580 case TCP_BBR_MIN_TOPACEOUT:
14581 optval = bbr->no_pacing_until;
14582 break;
14583 case TCP_BBR_PACE_SEG_MIN:
14584 optval = bbr->r_ctl.bbr_hptsi_bytes_min;
14585 break;
14586 case TCP_BBR_PACE_CROSS:
14587 optval = bbr->r_ctl.bbr_cross_over;
14588 break;
14589 case TCP_BBR_ALGORITHM:
14590 optval = bbr->rc_use_google;
14591 break;
14592 case TCP_BBR_TSLIMITS:
14593 optval = bbr->rc_use_ts_limit;
14594 break;
14595 case TCP_BBR_IWINTSO:
14596 optval = bbr->rc_init_win;
14597 break;
14598 case TCP_BBR_STARTUP_PG:
14599 optval = bbr->r_ctl.rc_startup_pg;
14600 break;
14601 case TCP_BBR_DRAIN_PG:
14602 optval = bbr->r_ctl.rc_drain_pg;
14603 break;
14604 case TCP_BBR_PROBE_RTT_INT:
14605 optval = bbr->r_ctl.rc_probertt_int;
14606 break;
14607 case TCP_BBR_PROBE_RTT_LEN:
14608 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);
14609 break;
14610 case TCP_BBR_PROBE_RTT_GAIN:
14611 optval = bbr->r_ctl.bbr_rttprobe_gain_val;
14612 break;
14613 case TCP_BBR_STARTUP_LOSS_EXIT:
14614 optval = bbr->rc_loss_exit;
14615 break;
14616 case TCP_BBR_USEDEL_RATE:
14617 loptval = get_filter_value(&bbr->r_ctl.rc_delrate);
14618 break;
14619 case TCP_BBR_MIN_RTO:
14620 optval = bbr->r_ctl.rc_min_rto_ms;
14621 break;
14622 case TCP_BBR_MAX_RTO:
14623 optval = bbr->rc_max_rto_sec;
14624 break;
14625 case TCP_RACK_PACE_MAX_SEG:
14626 /* Max segments in a pace */
14627 optval = bbr->r_ctl.rc_pace_max_segs;
14628 break;
14629 case TCP_RACK_MIN_TO:
14630 /* Minimum time between rack t-o's in ms */
14631 optval = bbr->r_ctl.rc_min_to;
14632 break;
14633 case TCP_RACK_REORD_THRESH:
14634 /* RACK reorder threshold (shift amount) */
14635 optval = bbr->r_ctl.rc_reorder_shift;
14636 break;
14637 case TCP_RACK_REORD_FADE:
14638 /* Does reordering fade after ms time */
14639 optval = bbr->r_ctl.rc_reorder_fade;
14640 break;
14641 case TCP_BBR_USE_RACK_CHEAT:
14642 /* Do we use the rack cheat for rxt */
14643 optval = bbr->bbr_use_rack_cheat;
14644 break;
14645 case TCP_BBR_FLOOR_MIN_TSO:
14646 optval = bbr->r_ctl.bbr_hptsi_segments_floor;
14647 break;
14648 case TCP_BBR_UTTER_MAX_TSO:
14649 optval = bbr->r_ctl.bbr_utter_max;
14650 break;
14651 case TCP_BBR_SEND_IWND_IN_TSO:
14652 /* Do we send TSO size segments initially */
14653 optval = bbr->bbr_init_win_cheat;
14654 break;
14655 case TCP_BBR_EXTRA_STATE:
14656 optval = bbr->rc_use_idle_restart;
14657 break;
14658 case TCP_RACK_TLP_THRESH:
14659 /* RACK TLP theshold i.e. srtt+(srtt/N) */
14660 optval = bbr->rc_tlp_threshold;
14661 break;
14662 case TCP_RACK_PKT_DELAY:
14663 /* RACK added ms i.e. rack-rtt + reord + N */
14664 optval = bbr->r_ctl.rc_pkt_delay;
14665 break;
14666 case TCP_BBR_RETRAN_WTSO:
14667 optval = bbr->rc_resends_use_tso;
14668 break;
14669 case TCP_DATA_AFTER_CLOSE:
14670 optval = bbr->rc_allow_data_af_clo;
14671 break;
14672 case TCP_DELACK:
14673 optval = tp->t_delayed_ack;
14674 break;
14675 case TCP_BBR_HDWR_PACE:
14676 optval = bbr->bbr_hdw_pace_ena;
14677 break;
14678 case TCP_BBR_POLICER_DETECT:
14679 optval = bbr->r_use_policer;
14680 break;
14681 case TCP_BBR_TSTMP_RAISES:
14682 optval = bbr->ts_can_raise;
14683 break;
14684 case TCP_BBR_TMR_PACE_OH:
14685 optval = bbr->r_ctl.rc_incr_tmrs;
14686 break;
14687 case TCP_BBR_PACE_OH:
14688 optval = 0;
14689 if (bbr->r_ctl.rc_inc_tcp_oh)
14690 optval |= BBR_INCL_TCP_OH;
14691 if (bbr->r_ctl.rc_inc_ip_oh)
14692 optval |= BBR_INCL_IP_OH;
14693 if (bbr->r_ctl.rc_inc_enet_oh)
14694 optval |= BBR_INCL_ENET_OH;
14695 break;
14696 default:
14697 return (tcp_default_ctloutput(tp, sopt));
14698 break;
14699 }
14700 INP_WUNLOCK(inp);
14701 if (sopt->sopt_name == TCP_BBR_USEDEL_RATE)
14702 error = sooptcopyout(sopt, &loptval, sizeof loptval);
14703 else
14704 error = sooptcopyout(sopt, &optval, sizeof optval);
14705 return (error);
14706 }
14707
14708 /*
14709 * return 0 on success, error-num on failure
14710 */
14711 static int
bbr_ctloutput(struct tcpcb * tp,struct sockopt * sopt)14712 bbr_ctloutput(struct tcpcb *tp, struct sockopt *sopt)
14713 {
14714 if (sopt->sopt_dir == SOPT_SET) {
14715 return (bbr_set_sockopt(tp, sopt));
14716 } else if (sopt->sopt_dir == SOPT_GET) {
14717 return (bbr_get_sockopt(tp, sopt));
14718 } else {
14719 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
14720 }
14721 }
14722
14723 static const char *bbr_stack_names[] = {
14724 __XSTRING(STACKNAME),
14725 #ifdef STACKALIAS
14726 __XSTRING(STACKALIAS),
14727 #endif
14728 };
14729
14730 static bool bbr_mod_inited = false;
14731
14732 static int
tcp_addbbr(module_t mod,int32_t type,void * data)14733 tcp_addbbr(module_t mod, int32_t type, void *data)
14734 {
14735 int32_t err = 0;
14736 int num_stacks;
14737
14738 switch (type) {
14739 case MOD_LOAD:
14740 printf("Attempting to load " __XSTRING(MODNAME) "\n");
14741 bbr_zone = uma_zcreate(__XSTRING(MODNAME) "_map",
14742 sizeof(struct bbr_sendmap),
14743 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
14744 bbr_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb",
14745 sizeof(struct tcp_bbr),
14746 NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
14747 sysctl_ctx_init(&bbr_sysctl_ctx);
14748 bbr_sysctl_root = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
14749 SYSCTL_STATIC_CHILDREN(_net_inet_tcp),
14750 OID_AUTO,
14751 #ifdef STACKALIAS
14752 __XSTRING(STACKALIAS),
14753 #else
14754 __XSTRING(STACKNAME),
14755 #endif
14756 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
14757 "");
14758 if (bbr_sysctl_root == NULL) {
14759 printf("Failed to add sysctl node\n");
14760 err = EFAULT;
14761 goto free_uma;
14762 }
14763 bbr_init_sysctls();
14764 num_stacks = nitems(bbr_stack_names);
14765 err = register_tcp_functions_as_names(&__tcp_bbr, M_WAITOK,
14766 bbr_stack_names, &num_stacks);
14767 if (err) {
14768 printf("Failed to register %s stack name for "
14769 "%s module\n", bbr_stack_names[num_stacks],
14770 __XSTRING(MODNAME));
14771 sysctl_ctx_free(&bbr_sysctl_ctx);
14772 free_uma:
14773 uma_zdestroy(bbr_zone);
14774 uma_zdestroy(bbr_pcb_zone);
14775 bbr_counter_destroy();
14776 printf("Failed to register " __XSTRING(MODNAME)
14777 " module err:%d\n", err);
14778 return (err);
14779 }
14780 tcp_lro_reg_mbufq();
14781 bbr_mod_inited = true;
14782 printf(__XSTRING(MODNAME) " is now available\n");
14783 break;
14784 case MOD_QUIESCE:
14785 err = deregister_tcp_functions(&__tcp_bbr, true, false);
14786 break;
14787 case MOD_UNLOAD:
14788 err = deregister_tcp_functions(&__tcp_bbr, false, true);
14789 if (err == EBUSY)
14790 break;
14791 if (bbr_mod_inited) {
14792 uma_zdestroy(bbr_zone);
14793 uma_zdestroy(bbr_pcb_zone);
14794 sysctl_ctx_free(&bbr_sysctl_ctx);
14795 bbr_counter_destroy();
14796 printf(__XSTRING(MODNAME)
14797 " is now no longer available\n");
14798 bbr_mod_inited = false;
14799 }
14800 tcp_lro_dereg_mbufq();
14801 err = 0;
14802 break;
14803 default:
14804 return (EOPNOTSUPP);
14805 }
14806 return (err);
14807 }
14808
14809 static moduledata_t tcp_bbr = {
14810 .name = __XSTRING(MODNAME),
14811 .evhand = tcp_addbbr,
14812 .priv = 0
14813 };
14814
14815 MODULE_VERSION(MODNAME, 1);
14816 DECLARE_MODULE(MODNAME, tcp_bbr, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
14817 MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1);
14818