1 /*- 2 * Copyright (c) 2024- Netflix, Inc. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 */ 26 27 #include <sys/cdefs.h> 28 #include "opt_inet.h" 29 #include "opt_inet6.h" 30 #include "opt_ipsec.h" 31 #include "opt_ratelimit.h" 32 #include "opt_kern_tls.h" 33 #if defined(INET) || defined(INET6) 34 #include <sys/param.h> 35 #include <sys/arb.h> 36 #include <sys/module.h> 37 #include <sys/kernel.h> 38 #ifdef TCP_HHOOK 39 #include <sys/hhook.h> 40 #endif 41 #include <sys/lock.h> 42 #include <sys/malloc.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/mbuf.h> 46 #include <sys/proc.h> /* for proc0 declaration */ 47 #include <sys/socket.h> 48 #include <sys/socketvar.h> 49 #include <sys/sysctl.h> 50 #include <sys/systm.h> 51 #ifdef STATS 52 #include <sys/qmath.h> 53 #include <sys/tree.h> 54 #include <sys/stats.h> /* Must come after qmath.h and tree.h */ 55 #else 56 #include <sys/tree.h> 57 #endif 58 #include <sys/refcount.h> 59 #include <sys/queue.h> 60 #include <sys/tim_filter.h> 61 #include <sys/smp.h> 62 #include <sys/kthread.h> 63 #include <sys/kern_prefetch.h> 64 #include <sys/protosw.h> 65 #ifdef TCP_ACCOUNTING 66 #include <sys/sched.h> 67 #include <machine/cpu.h> 68 #endif 69 #include <vm/uma.h> 70 71 #include <net/route.h> 72 #include <net/route/nhop.h> 73 #include <net/vnet.h> 74 75 #define TCPSTATES /* for logging */ 76 77 #include <netinet/in.h> 78 #include <netinet/in_kdtrace.h> 79 #include <netinet/in_pcb.h> 80 #include <netinet/ip.h> 81 #include <netinet/ip_var.h> 82 #include <netinet/ip6.h> 83 #include <netinet6/in6_pcb.h> 84 #include <netinet6/ip6_var.h> 85 #include <netinet/tcp.h> 86 #include <netinet/tcp_fsm.h> 87 #include <netinet/tcp_seq.h> 88 #include <netinet/tcp_timer.h> 89 #include <netinet/tcp_var.h> 90 #include <netinet/tcp_log_buf.h> 91 #include <netinet/tcp_syncache.h> 92 #include <netinet/tcp_hpts.h> 93 #include <netinet/tcp_ratelimit.h> 94 #include <netinet/tcp_accounting.h> 95 #include <netinet/tcpip.h> 96 #include <netinet/cc/cc.h> 97 #include <netinet/cc/cc_newreno.h> 98 #include <netinet/tcp_fastopen.h> 99 #include <netinet/tcp_lro.h> 100 #ifdef NETFLIX_SHARED_CWND 101 #include <netinet/tcp_shared_cwnd.h> 102 #endif 103 #ifdef TCP_OFFLOAD 104 #include <netinet/tcp_offload.h> 105 #endif 106 #ifdef INET6 107 #include <netinet6/tcp6_var.h> 108 #endif 109 #include <netinet/tcp_ecn.h> 110 111 #include <netipsec/ipsec_support.h> 112 113 #if defined(IPSEC) || defined(IPSEC_SUPPORT) 114 #include <netipsec/ipsec.h> 115 #include <netipsec/ipsec6.h> 116 #endif /* IPSEC */ 117 118 #include <netinet/udp.h> 119 #include <netinet/udp_var.h> 120 #include <machine/in_cksum.h> 121 122 #ifdef MAC 123 #include <security/mac/mac_framework.h> 124 #endif 125 #include "sack_filter.h" 126 #include "tcp_rack.h" 127 #include "tailq_hash.h" 128 #include "rack_bbr_common.h" 129 130 MALLOC_DECLARE(M_TCPPCM); 131 132 void 133 rack_update_pcm_ack(struct tcp_rack *rack, int was_cumack, uint32_t start, uint32_t end) 134 { 135 struct rack_pcm_stats *e; 136 int i, completed = 0; 137 uint64_t ack_arrival; 138 int segsiz; 139 140 if (rack->pcm_in_progress == 0) 141 return; 142 143 if (SEQ_LEQ(end, rack->r_ctl.pcm_i.sseq)) { 144 /* 145 * Its not in our range of data sent, it 146 * is before our first seq. 147 */ 148 return; 149 } 150 /* We take away 1 mss from the end to avoid delayed ack */ 151 segsiz = ctf_fixed_maxseg(rack->rc_tp); 152 if (SEQ_GEQ(end, (rack->r_ctl.pcm_i.eseq - segsiz))) { 153 /* 154 * We have reached beyond the end of the 155 * initial send. Even though things may 156 * still be lost and this could be something 157 * from a different send than our burst. 158 */ 159 completed = 1; 160 rack->pcm_in_progress = 0; 161 rack->r_ctl.last_pcm_round = rack->r_ctl.current_round; 162 rack->r_ctl.pcm_idle_rounds = 0; 163 } 164 if (SEQ_GEQ(start, rack->r_ctl.pcm_i.eseq)) { 165 /* 166 * This is outside the scope 167 * of the measurement itself and 168 * is likely a sack above our burst. 169 */ 170 goto skip_ack_accounting; 171 } 172 /* 173 * Record ACK data. 174 */ 175 ack_arrival = tcp_tv_to_lusec(&rack->r_ctl.act_rcv_time); 176 if (SEQ_GT(end, rack->r_ctl.pcm_i.eseq)) { 177 /* Trim the end to the end of our range if it is beyond */ 178 end = rack->r_ctl.pcm_i.eseq; 179 } 180 if ((rack->r_ctl.pcm_i.cnt + 1) > rack->r_ctl.pcm_i.cnt_alloc) { 181 /* Need to expand, first is there any present? */ 182 size_t sz; 183 184 if (rack->r_ctl.pcm_i.cnt_alloc == 0) { 185 /* 186 * Failed at rack_init I suppose. 187 */ 188 rack->r_ctl.pcm_i.cnt_alloc = RACK_DEFAULT_PCM_ARRAY; 189 sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc); 190 rack->r_ctl.pcm_s = malloc(sz, M_TCPPCM, M_NOWAIT); 191 if (rack->r_ctl.pcm_s == NULL) { 192 rack->r_ctl.pcm_i.cnt_alloc = 0; 193 rack->pcm_in_progress = 0; 194 return; 195 } 196 } else { 197 /* Need to expand the array */ 198 struct rack_pcm_stats *n; 199 uint16_t new_cnt; 200 201 new_cnt = rack->r_ctl.pcm_i.cnt_alloc * 2; 202 sz = (sizeof(struct rack_pcm_stats) * new_cnt); 203 n = malloc(sz,M_TCPPCM, M_NOWAIT); 204 if (n == NULL) { 205 /* We are dead, no memory */ 206 rack->pcm_in_progress = 0; 207 rack->r_ctl.pcm_i.cnt = 0; 208 return; 209 } 210 sz = (sizeof(struct rack_pcm_stats) * rack->r_ctl.pcm_i.cnt_alloc); 211 memcpy(n, rack->r_ctl.pcm_s, sz); 212 free(rack->r_ctl.pcm_s, M_TCPPCM); 213 rack->r_ctl.pcm_s = n; 214 rack->r_ctl.pcm_i.cnt_alloc = new_cnt; 215 } 216 } 217 e = &rack->r_ctl.pcm_s[rack->r_ctl.pcm_i.cnt]; 218 rack->r_ctl.pcm_i.cnt++; 219 e->sseq = start; 220 e->eseq = end; 221 e->ack_time = ack_arrival; 222 skip_ack_accounting: 223 if (completed == 0) 224 return; 225 /* 226 * Ok we are to the point where we can assess what 227 * has happened and make a PCM judgement. 228 */ 229 230 if (tcp_bblogging_on(rack->rc_tp)) { 231 union tcp_log_stackspecific log; 232 struct timeval tv; 233 uint64_t prev_time = 0; 234 uint64_t tot_byt = 0; 235 uint32_t tot_lt_12us = 0; 236 uint32_t tot_gt_2mss = 0; 237 238 (void)tcp_get_usecs(&tv); 239 for (i=0; i<rack->r_ctl.pcm_i.cnt; i++) { 240 241 e = &rack->r_ctl.pcm_s[i]; 242 memset(&log, 0, sizeof(log)); 243 log.u_bbr.timeStamp = tcp_tv_to_usec(&tv); 244 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 245 log.u_bbr.flex8 = 1; 246 log.u_bbr.flex1 = e->sseq; 247 log.u_bbr.flex2 = e->eseq; 248 tot_byt += (e->eseq - e->sseq); 249 if ((i > 0) && 250 (e->ack_time > prev_time)) { 251 log.u_bbr.flex3 = (uint32_t)(e->ack_time - prev_time); 252 } else { 253 log.u_bbr.flex3 = 0; 254 } 255 if (e->ack_time > rack->r_ctl.pcm_i.send_time) { 256 log.u_bbr.flex4 = (uint32_t)(e->ack_time - rack->r_ctl.pcm_i.send_time); 257 } else { 258 log.u_bbr.flex4 = 0; 259 } 260 if ((e->eseq - e->sseq) > (segsiz * 2)) { 261 tot_gt_2mss++; 262 } 263 if ((i > 0) && 264 (log.u_bbr.flex3 < 12)) { 265 tot_lt_12us++; 266 } 267 prev_time = e->ack_time; 268 log.u_bbr.cur_del_rate = rack->r_ctl.pcm_i.send_time; 269 if ((i > 0) && 270 (log.u_bbr.flex3 > 0)) { 271 /* 272 * Calculate a b/w between this chunk and the previous. 273 */ 274 log.u_bbr.delRate = (e->eseq - e->sseq); 275 log.u_bbr.delRate *= HPTS_USEC_IN_SEC; 276 log.u_bbr.delRate /= (uint64_t)log.u_bbr.flex3; 277 } 278 log.u_bbr.rttProp = e->ack_time; 279 (void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK, 280 0, &log, false, NULL, NULL, 0, &tv); 281 } 282 if (prev_time > rack->r_ctl.pcm_i.send_time) { 283 /* 284 * Prev time holds the last ack arrival time. 285 */ 286 memset(&log.u_bbr, 0, sizeof(log.u_bbr)); 287 log.u_bbr.timeStamp = tcp_tv_to_usec(&tv); 288 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked); 289 log.u_bbr.flex8 = 2; 290 log.u_bbr.flex1 = rack->r_ctl.pcm_i.sseq; 291 log.u_bbr.flex2 = rack->r_ctl.pcm_i.eseq; 292 log.u_bbr.flex3 = tot_byt; 293 log.u_bbr.flex4 = tot_lt_12us; /* How many deltas indicate > 2Gbps */ 294 log.u_bbr.flex5 = tot_gt_2mss; /* How many acks represent more than 2MSS */ 295 log.u_bbr.flex7 = rack->r_ctl.pcm_i.cnt; 296 log.u_bbr.cwnd_gain = rack->r_ctl.pcm_i.cnt_alloc; 297 log.u_bbr.cur_del_rate = rack->r_ctl.pcm_i.send_time; 298 log.u_bbr.rttProp = prev_time; 299 log.u_bbr.delRate = tot_byt; 300 log.u_bbr.delRate *= HPTS_USEC_IN_SEC; 301 log.u_bbr.delRate /= (prev_time - rack->r_ctl.pcm_i.send_time); 302 (void)tcp_log_event(rack->rc_tp, NULL, NULL, NULL, TCP_PCM_MEASURE, ERRNO_UNK, 303 0, &log, false, NULL, NULL, 0, &tv); 304 } 305 } 306 /* 307 * Here we need a lot to be added including: 308 * 1) Some form of measurement, where if we think the measurement 309 * is valid we iterate over the PCM data and come up with a path 310 * capacity estimate. 311 * 2) We may decide that the PCM is invalid due to ack meddlers and 312 * thus need to increase the PCM size (which defaults to 10mss). 313 * 3) We may need to think about shrinking the PCM size if we are 314 * seeing some sort of presistent loss from making the measurement 315 * (i.e. it got to big and our bursts are causing loss). 316 * 4) If we make a measurement we need to place it somewhere in the 317 * stack to be reported later somehow. Is it a WMA in the stack or 318 * the highest or? 319 * 5) Is there a limit on how big we can go PCM size wise, the code 320 * here will send multiple TSO bursts all at once, but how big 321 * is too big, and does that then put some bound (I think it does) 322 * on the largest capacity we can determine? 323 */ 324 /* New code here */ 325 /* Clear the cnt we are done */ 326 rack->r_ctl.pcm_i.cnt = 0; 327 } 328 329 #endif /* #if !defined(INET) && !defined(INET6) */ 330