1 /*-
2 * Copyright (c) 2024- Netflix, Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
26
27 #include <sys/cdefs.h>
28 #include "opt_inet.h"
29 #include "opt_inet6.h"
30 #include "opt_ipsec.h"
31 #include "opt_ratelimit.h"
32 #include "opt_kern_tls.h"
33 #if defined(INET) || defined(INET6)
34 #include <sys/param.h>
35 #include <sys/arb.h>
36 #include <sys/module.h>
37 #include <sys/kernel.h>
38 #ifdef TCP_HHOOK
39 #include <sys/hhook.h>
40 #endif
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/mbuf.h>
46 #include <sys/proc.h> /* for proc0 declaration */
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #ifdef STATS
52 #include <sys/qmath.h>
53 #include <sys/tree.h>
54 #include <sys/stats.h> /* Must come after qmath.h and tree.h */
55 #else
56 #include <sys/tree.h>
57 #endif
58 #include <sys/refcount.h>
59 #include <sys/queue.h>
60 #include <sys/tim_filter.h>
61 #include <sys/smp.h>
62 #include <sys/kthread.h>
63 #include <sys/kern_prefetch.h>
64 #include <sys/protosw.h>
65 #ifdef TCP_ACCOUNTING
66 #include <sys/sched.h>
67 #include <machine/cpu.h>
68 #endif
69 #include <vm/uma.h>
70
71 #include <net/route.h>
72 #include <net/route/nhop.h>
73 #include <net/vnet.h>
74
75 #define TCPSTATES /* for logging */
76
77 #include <netinet/in.h>
78 #include <netinet/in_kdtrace.h>
79 #include <netinet/in_pcb.h>
80 #include <netinet/ip.h>
81 #include <netinet/ip_var.h>
82 #include <netinet/ip6.h>
83 #include <netinet6/in6_pcb.h>
84 #include <netinet6/ip6_var.h>
85 #include <netinet/tcp.h>
86 #include <netinet/tcp_fsm.h>
87 #include <netinet/tcp_seq.h>
88 #include <netinet/tcp_timer.h>
89 #include <netinet/tcp_var.h>
90 #include <netinet/tcp_log_buf.h>
91 #include <netinet/tcp_syncache.h>
92 #include <netinet/tcp_hpts.h>
93 #include <netinet/tcp_ratelimit.h>
94 #include <netinet/tcp_accounting.h>
95 #include <netinet/tcpip.h>
96 #include <netinet/cc/cc.h>
97 #include <netinet/cc/cc_newreno.h>
98 #include <netinet/tcp_fastopen.h>
99 #include <netinet/tcp_lro.h>
100 #ifdef NETFLIX_SHARED_CWND
101 #include <netinet/tcp_shared_cwnd.h>
102 #endif
103 #ifdef TCP_OFFLOAD
104 #include <netinet/tcp_offload.h>
105 #endif
106 #ifdef INET6
107 #include <netinet6/tcp6_var.h>
108 #endif
109 #include <netinet/tcp_ecn.h>
110
111 #include <netipsec/ipsec_support.h>
112
113 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
114 #include <netipsec/ipsec.h>
115 #include <netipsec/ipsec6.h>
116 #endif /* IPSEC */
117
118 #include <netinet/udp.h>
119 #include <netinet/udp_var.h>
120 #include <machine/in_cksum.h>
121
122 #ifdef MAC
123 #include <security/mac/mac_framework.h>
124 #endif
125 #include "sack_filter.h"
126 #include "tcp_rack.h"
127 #include "tailq_hash.h"
128 #include "rack_bbr_common.h"
129
130 MALLOC_DECLARE(M_TCPPCM);
131
132 void
rack_update_pcm_ack(struct tcp_rack * rack,int was_cumack,uint32_t start,uint32_t end)133 rack_update_pcm_ack(struct tcp_rack *rack, int was_cumack, uint32_t start, uint32_t end)
134 {
135 struct rack_pcm_stats *e;
136 int i, completed = 0;
137 uint64_t ack_arrival;
138 int segsiz;
139
140 if (rack->pcm_in_progress == 0)
141 return;
142
143 if (SEQ_LEQ(end, rack->r_ctl.pcm_i.sseq)) {
144 /*
145 * Its not in our range of data sent, it
146 * is before our first seq.
147 */
148 return;
149 }
150 /* We take away 1 mss from the end to avoid delayed ack */
151 segsiz = ctf_fixed_maxseg(rack->rc_tp);
152 if (SEQ_GEQ(end, (rack->r_ctl.pcm_i.eseq - segsiz))) {
153 /*
154 * We have reached beyond the end of the
155 * initial send. Even though things may
156 * still be lost and this could be something
157 * from a different send than our burst.
158 */
159 completed = 1;
160 rack->pcm_in_progress = 0;
161 rack->r_ctl.last_pcm_round = rack->r_ctl.current_round;
162 rack->r_ctl.pcm_idle_rounds = 0;
163 }
164 if (SEQ_GEQ(start, rack->r_ctl.pcm_i.eseq)) {
165 /*
166 * This is outside the scope
167 * of the measurement itself and
168 * is likely a sack above our burst.
169 */
170 goto skip_ack_accounting;
171 }
172 /*
173 * Record ACK data.
174 */
175 ack_arrival = tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time);
176 if (SEQ_GT(end, rack->r_ctl.pcm_i.eseq)) {
177 /* Trim the end to the end of our range if it is beyond */
178 end = rack->r_ctl.pcm_i.eseq;
179 }
180 if ((rack->r_ctl.pcm_i.cnt + 1) > rack->r_ctl.pcm_i.cnt_alloc) {
181 /* Need to expand, first is there any present? */
182 size_t sz;
183
184 if (rack->r_ctl.pcm_i.cnt_alloc == 0) {
185 /*
186 * Failed at rack_init I suppose.
187 */
188 rack->r_ctl.pcm_i.cnt_alloc = RACK_DEFAULT_PCM_ARRAY;
189 sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc);
190 rack->r_ctl.pcm_s = malloc(sz, M_TCPPCM, M_NOWAIT);
191 if (rack->r_ctl.pcm_s == NULL) {
192 rack->r_ctl.pcm_i.cnt_alloc = 0;
193 rack->pcm_in_progress = 0;
194 return;
195 }
196 } else {
197 /* Need to expand the array */
198 struct rack_pcm_stats *n;
199 uint16_t new_cnt;
200
201 new_cnt = rack->r_ctl.pcm_i.cnt_alloc * 2;
202 sz = (sizeof(struct rack_pcm_stats) * new_cnt);
203 n = malloc(sz,M_TCPPCM, M_NOWAIT);
204 if (n == NULL) {
205 /* We are dead, no memory */
206 rack->pcm_in_progress = 0;
207 rack->r_ctl.pcm_i.cnt = 0;
208 return;
209 }
210 sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc);
211 memcpy(n, rack->r_ctl.pcm_s, sz);
212 free(rack->r_ctl.pcm_s, M_TCPPCM);
213 rack->r_ctl.pcm_s = n;
214 rack->r_ctl.pcm_i.cnt_alloc = new_cnt;
215 }
216 }
217 e = &rack->r_ctl.pcm_s[rack->r_ctl.pcm_i.cnt];
218 rack->r_ctl.pcm_i.cnt++;
219 e->sseq = start;
220 e->eseq = end;
221 e->ack_time = ack_arrival;
222 skip_ack_accounting:
223 if (completed == 0)
224 return;
225 /*
226 * Ok we are to the point where we can assess what
227 * has happened and make a PCM judgement.
228 */
229
230 if (tcp_bblogging_on(rack->rc_tp)) {
231 union tcp_log_stackspecific log;
232 struct timeval tv;
233 uint64_t prev_time = 0;
234 uint64_t tot_byt = 0;
235 uint32_t tot_lt_12us = 0;
236 uint32_t tot_gt_2mss = 0;
237
238 (void)tcp_get_usecs(&tv);
239 for (i=0; i<rack->r_ctl.pcm_i.cnt; i++) {
240
241 e = &rack->r_ctl.pcm_s[i];
242 memset(&log, 0, sizeof(log));
243 log.u_bbr.timeStamp = tcp_tv_to_usec(&tv);
244 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
245 log.u_bbr.flex8 = 1;
246 log.u_bbr.flex1 = e->sseq;
247 log.u_bbr.flex2 = e->eseq;
248 tot_byt += (e->eseq - e->sseq);
249 if ((i > 0) &&
250 (e->ack_time > prev_time)) {
251 log.u_bbr.flex3 = (uint32_t)(e->ack_time - prev_time);
252 } else {
253 log.u_bbr.flex3 = 0;
254 }
255 if (e->ack_time > rack->r_ctl.pcm_i.send_time) {
256 log.u_bbr.flex4 = (uint32_t)(e->ack_time - rack->r_ctl.pcm_i.send_time);
257 } else {
258 log.u_bbr.flex4 = 0;
259 }
260 if ((e->eseq - e->sseq) > (segsiz * 2)) {
261 tot_gt_2mss++;
262 }
263 if ((i > 0) &&
264 (log.u_bbr.flex3 < 12)) {
265 tot_lt_12us++;
266 }
267 prev_time = e->ack_time;
268 log.u_bbr.cur_del_rate = rack->r_ctl.pcm_i.send_time;
269 if ((i > 0) &&
270 (log.u_bbr.flex3 > 0)) {
271 /*
272 * Calculate a b/w between this chunk and the previous.
273 */
274 log.u_bbr.delRate = (e->eseq - e->sseq);
275 log.u_bbr.delRate *= HPTS_USEC_IN_SEC;
276 log.u_bbr.delRate /= (uint64_t)log.u_bbr.flex3;
277 }
278 log.u_bbr.rttProp = e->ack_time;
279 (void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK,
280 0, &log, false, NULL, NULL, 0, &tv);
281 }
282 if (prev_time > rack->r_ctl.pcm_i.send_time) {
283 /*
284 * Prev time holds the last ack arrival time.
285 */
286 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
287 log.u_bbr.timeStamp = tcp_tv_to_usec(&tv);
288 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
289 log.u_bbr.flex8 = 2;
290 log.u_bbr.flex1 = rack->r_ctl.pcm_i.sseq;
291 log.u_bbr.flex2 = rack->r_ctl.pcm_i.eseq;
292 log.u_bbr.flex3 = tot_byt;
293 log.u_bbr.flex4 = tot_lt_12us; /* How many deltas indicate > 2Gbps */
294 log.u_bbr.flex5 = tot_gt_2mss; /* How many acks represent more than 2MSS */
295 log.u_bbr.flex7 = rack->r_ctl.pcm_i.cnt;
296 log.u_bbr.cwnd_gain = rack->r_ctl.pcm_i.cnt_alloc;
297 log.u_bbr.cur_del_rate = rack->r_ctl.pcm_i.send_time;
298 log.u_bbr.rttProp = prev_time;
299 log.u_bbr.delRate = tot_byt;
300 log.u_bbr.delRate *= HPTS_USEC_IN_SEC;
301 log.u_bbr.delRate /= (prev_time - rack->r_ctl.pcm_i.send_time);
302 (void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK,
303 0, &log, false, NULL, NULL, 0, &tv);
304 }
305 }
306 /*
307 * Here we need a lot to be added including:
308 * 1) Some form of measurement, where if we think the measurement
309 * is valid we iterate over the PCM data and come up with a path
310 * capacity estimate.
311 * 2) We may decide that the PCM is invalid due to ack meddlers and
312 * thus need to increase the PCM size (which defaults to 10mss).
313 * 3) We may need to think about shrinking the PCM size if we are
314 * seeing some sort of presistent loss from making the measurement
315 * (i.e. it got to big and our bursts are causing loss).
316 * 4) If we make a measurement we need to place it somewhere in the
317 * stack to be reported later somehow. Is it a WMA in the stack or
318 * the highest or?
319 * 5) Is there a limit on how big we can go PCM size wise, the code
320 * here will send multiple TSO bursts all at once, but how big
321 * is too big, and does that then put some bound (I think it does)
322 * on the largest capacity we can determine?
323 */
324 /* New code here */
325 /* Clear the cnt we are done */
326 rack->r_ctl.pcm_i.cnt = 0;
327 }
328
329 #endif /* #if !defined(INET) && !defined(INET6) */
330