1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
5 * The Regents of the University of California. All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the University nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 #include "opt_inet.h"
34 #include "opt_inet6.h"
35 #include "opt_ipsec.h"
36 #include "opt_kern_tls.h"
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/arb.h>
41 #include <sys/callout.h>
42 #include <sys/eventhandler.h>
43 #ifdef TCP_HHOOK
44 #include <sys/hhook.h>
45 #endif
46 #include <sys/kernel.h>
47 #ifdef TCP_HHOOK
48 #include <sys/khelp.h>
49 #endif
50 #ifdef KERN_TLS
51 #include <sys/ktls.h>
52 #endif
53 #include <sys/qmath.h>
54 #include <sys/stats.h>
55 #include <sys/sysctl.h>
56 #include <sys/jail.h>
57 #include <sys/malloc.h>
58 #include <sys/refcount.h>
59 #include <sys/mbuf.h>
60 #include <sys/priv.h>
61 #include <sys/sdt.h>
62 #include <sys/socket.h>
63 #include <sys/socketvar.h>
64 #include <sys/protosw.h>
65 #include <sys/random.h>
66
67 #include <vm/uma.h>
68
69 #include <net/route.h>
70 #include <net/route/nhop.h>
71 #include <net/if.h>
72 #include <net/if_var.h>
73 #include <net/if_private.h>
74 #include <net/vnet.h>
75
76 #include <netinet/in.h>
77 #include <netinet/in_fib.h>
78 #include <netinet/in_kdtrace.h>
79 #include <netinet/in_pcb.h>
80 #include <netinet/in_systm.h>
81 #include <netinet/in_var.h>
82 #include <netinet/ip.h>
83 #include <netinet/ip_icmp.h>
84 #include <netinet/ip_var.h>
85 #ifdef INET6
86 #include <netinet/icmp6.h>
87 #include <netinet/ip6.h>
88 #include <netinet6/in6_fib.h>
89 #include <netinet6/in6_pcb.h>
90 #include <netinet6/ip6_var.h>
91 #include <netinet6/scope6_var.h>
92 #include <netinet6/nd6.h>
93 #endif
94
95 #include <netinet/tcp.h>
96 #ifdef INVARIANTS
97 #define TCPSTATES
98 #endif
99 #include <netinet/tcp_fsm.h>
100 #include <netinet/tcp_seq.h>
101 #include <netinet/tcp_timer.h>
102 #include <netinet/tcp_var.h>
103 #include <netinet/tcp_ecn.h>
104 #include <netinet/tcp_log_buf.h>
105 #include <netinet/tcp_syncache.h>
106 #include <netinet/tcp_hpts.h>
107 #include <netinet/tcp_lro.h>
108 #include <netinet/cc/cc.h>
109 #include <netinet/tcpip.h>
110 #include <netinet/tcp_fastopen.h>
111 #include <netinet/tcp_accounting.h>
112 #ifdef TCP_OFFLOAD
113 #include <netinet/tcp_offload.h>
114 #endif
115 #include <netinet/udp.h>
116 #include <netinet/udp_var.h>
117 #ifdef INET6
118 #include <netinet6/tcp6_var.h>
119 #endif
120
121 #include <netipsec/ipsec_support.h>
122
123 #include <machine/in_cksum.h>
124 #include <crypto/siphash/siphash.h>
125
126 #include <security/mac/mac_framework.h>
127
128 #ifdef INET6
129 static ip6proto_ctlinput_t tcp6_ctlinput;
130 static udp_tun_icmp_t tcp6_ctlinput_viaudp;
131 #endif
132
133 VNET_DEFINE(int, tcp_mssdflt) = TCP_MSS;
134 #ifdef INET6
135 VNET_DEFINE(int, tcp_v6mssdflt) = TCP6_MSS;
136 #endif
137
138 VNET_DEFINE(uint32_t, tcp_ack_war_time_window) = 1000;
139 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_timewindow,
140 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_ack_war_time_window), 0,
141 "Time interval in ms used to limit the number (ack_war_cnt) of challenge ACKs sent per TCP connection");
142 VNET_DEFINE(uint32_t, tcp_ack_war_cnt) = 5;
143 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_cnt, CTLFLAG_VNET | CTLFLAG_RW,
144 &VNET_NAME(tcp_ack_war_cnt), 0,
145 "Maximum number of challenge ACKs sent per TCP connection during the time interval (ack_war_timewindow)");
146
147 struct rwlock tcp_function_lock;
148
149 static int
sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)150 sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)
151 {
152 int error, new;
153
154 new = V_tcp_mssdflt;
155 error = sysctl_handle_int(oidp, &new, 0, req);
156 if (error == 0 && req->newptr) {
157 if (new < TCP_MINMSS)
158 error = EINVAL;
159 else
160 V_tcp_mssdflt = new;
161 }
162 return (error);
163 }
164
165 SYSCTL_PROC(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt,
166 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
167 &VNET_NAME(tcp_mssdflt), 0, &sysctl_net_inet_tcp_mss_check, "I",
168 "Default TCP Maximum Segment Size");
169
170 #ifdef INET6
171 static int
sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)172 sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)
173 {
174 int error, new;
175
176 new = V_tcp_v6mssdflt;
177 error = sysctl_handle_int(oidp, &new, 0, req);
178 if (error == 0 && req->newptr) {
179 if (new < TCP_MINMSS)
180 error = EINVAL;
181 else
182 V_tcp_v6mssdflt = new;
183 }
184 return (error);
185 }
186
187 SYSCTL_PROC(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt,
188 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
189 &VNET_NAME(tcp_v6mssdflt), 0, &sysctl_net_inet_tcp_mss_v6_check, "I",
190 "Default TCP Maximum Segment Size for IPv6");
191 #endif /* INET6 */
192
193 /*
194 * Minimum MSS we accept and use. This prevents DoS attacks where
195 * we are forced to a ridiculous low MSS like 20 and send hundreds
196 * of packets instead of one. The effect scales with the available
197 * bandwidth and quickly saturates the CPU and network interface
198 * with packet generation and sending. Set to zero to disable MINMSS
199 * checking. This setting prevents us from sending too small packets.
200 */
201 VNET_DEFINE(int, tcp_minmss) = TCP_MINMSS;
202 SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_VNET | CTLFLAG_RW,
203 &VNET_NAME(tcp_minmss), 0,
204 "Minimum TCP Maximum Segment Size");
205
206 VNET_DEFINE(int, tcp_do_rfc1323) = 1;
207 SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, CTLFLAG_VNET | CTLFLAG_RW,
208 &VNET_NAME(tcp_do_rfc1323), 0,
209 "Enable rfc1323 (high performance TCP) extensions");
210
211 /*
212 * As of June 2021, several TCP stacks violate RFC 7323 from September 2014.
213 * Some stacks negotiate TS, but never send them after connection setup. Some
214 * stacks negotiate TS, but don't send them when sending keep-alive segments.
215 * These include modern widely deployed TCP stacks.
216 * Therefore tolerating violations for now...
217 */
218 VNET_DEFINE(int, tcp_tolerate_missing_ts) = 1;
219 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tolerate_missing_ts, CTLFLAG_VNET | CTLFLAG_RW,
220 &VNET_NAME(tcp_tolerate_missing_ts), 0,
221 "Tolerate missing TCP timestamps");
222
223 VNET_DEFINE(int, tcp_ts_offset_per_conn) = 1;
224 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ts_offset_per_conn, CTLFLAG_VNET | CTLFLAG_RW,
225 &VNET_NAME(tcp_ts_offset_per_conn), 0,
226 "Initialize TCP timestamps per connection instead of per host pair");
227
228 /* How many connections are pacing */
229 static volatile uint32_t number_of_tcp_connections_pacing = 0;
230 static uint32_t shadow_num_connections = 0;
231 static counter_u64_t tcp_pacing_failures;
232 static counter_u64_t tcp_dgp_failures;
233 static uint32_t shadow_tcp_pacing_dgp = 0;
234 static volatile uint32_t number_of_dgp_connections = 0;
235
236 static int tcp_pacing_limit = 10000;
237 SYSCTL_INT(_net_inet_tcp, OID_AUTO, pacing_limit, CTLFLAG_RW,
238 &tcp_pacing_limit, 1000,
239 "If the TCP stack does pacing, is there a limit (-1 = no, 0 = no pacing N = number of connections)");
240
241 static int tcp_dgp_limit = -1;
242 SYSCTL_INT(_net_inet_tcp, OID_AUTO, dgp_limit, CTLFLAG_RW,
243 &tcp_dgp_limit, -1,
244 "If the TCP stack does DGP, is there a limit (-1 = no, 0 = no dgp N = number of connections)");
245
246 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pacing_count, CTLFLAG_RD,
247 &shadow_num_connections, 0, "Number of TCP connections being paced");
248
249 SYSCTL_COUNTER_U64(_net_inet_tcp, OID_AUTO, pacing_failures, CTLFLAG_RD,
250 &tcp_pacing_failures, "Number of times we failed to enable pacing to avoid exceeding the limit");
251
252 SYSCTL_COUNTER_U64(_net_inet_tcp, OID_AUTO, dgp_failures, CTLFLAG_RD,
253 &tcp_dgp_failures, "Number of times we failed to enable dgp to avoid exceeding the limit");
254
255 static int tcp_log_debug = 0;
256 SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_debug, CTLFLAG_RW,
257 &tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
258
259 /*
260 * Target size of TCP PCB hash tables. Must be a power of two.
261 *
262 * Note that this can be overridden by the kernel environment
263 * variable net.inet.tcp.tcbhashsize
264 */
265 #ifndef TCBHASHSIZE
266 #define TCBHASHSIZE 0
267 #endif
268 static int tcp_tcbhashsize = TCBHASHSIZE;
269 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN,
270 &tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
271
272 static int do_tcpdrain = 1;
273 SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW, &do_tcpdrain, 0,
274 "Enable tcp_drain routine for extra help when low on mbufs");
275
276 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_VNET | CTLFLAG_RD,
277 &VNET_NAME(tcbinfo.ipi_count), 0, "Number of active PCBs");
278
279 VNET_DEFINE_STATIC(int, icmp_may_rst) = 1;
280 #define V_icmp_may_rst VNET(icmp_may_rst)
281 SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_VNET | CTLFLAG_RW,
282 &VNET_NAME(icmp_may_rst), 0,
283 "Certain ICMP unreachable messages may abort connections in SYN_SENT");
284
285 VNET_DEFINE_STATIC(int, tcp_isn_reseed_interval) = 0;
286 #define V_tcp_isn_reseed_interval VNET(tcp_isn_reseed_interval)
287 SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_VNET | CTLFLAG_RW,
288 &VNET_NAME(tcp_isn_reseed_interval), 0,
289 "Seconds between reseeding of ISN secret");
290
291 static int tcp_soreceive_stream;
292 SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
293 &tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");
294
295 VNET_DEFINE(uma_zone_t, sack_hole_zone);
296 #define V_sack_hole_zone VNET(sack_hole_zone)
297 VNET_DEFINE(uint32_t, tcp_map_entries_limit) = 0; /* unlimited */
298 static int
sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)299 sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)
300 {
301 int error;
302 uint32_t new;
303
304 new = V_tcp_map_entries_limit;
305 error = sysctl_handle_int(oidp, &new, 0, req);
306 if (error == 0 && req->newptr) {
307 /* only allow "0" and value > minimum */
308 if (new > 0 && new < TCP_MIN_MAP_ENTRIES_LIMIT)
309 error = EINVAL;
310 else
311 V_tcp_map_entries_limit = new;
312 }
313 return (error);
314 }
315 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, map_limit,
316 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
317 &VNET_NAME(tcp_map_entries_limit), 0,
318 &sysctl_net_inet_tcp_map_limit_check, "IU",
319 "Total sendmap entries limit");
320
321 VNET_DEFINE(uint32_t, tcp_map_split_limit) = 0; /* unlimited */
322 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, split_limit, CTLFLAG_VNET | CTLFLAG_RW,
323 &VNET_NAME(tcp_map_split_limit), 0,
324 "Total sendmap split entries limit");
325
326 #ifdef TCP_HHOOK
327 VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
328 #endif
329
330 #define TS_OFFSET_SECRET_LENGTH SIPHASH_KEY_LENGTH
331 VNET_DEFINE_STATIC(u_char, ts_offset_secret[TS_OFFSET_SECRET_LENGTH]);
332 #define V_ts_offset_secret VNET(ts_offset_secret)
333
334 static int tcp_default_fb_init(struct tcpcb *tp, void **ptr);
335 static void tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged);
336 static int tcp_default_handoff_ok(struct tcpcb *tp);
337 static struct inpcb *tcp_notify(struct inpcb *, int);
338 static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int);
339 static struct inpcb *tcp_mtudisc(struct inpcb *, int);
340 static struct inpcb *tcp_drop_syn_sent(struct inpcb *, int);
341 static char * tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th,
342 const void *ip4hdr, const void *ip6hdr);
343 static void tcp_default_switch_failed(struct tcpcb *tp);
344 static ipproto_ctlinput_t tcp_ctlinput;
345 static udp_tun_icmp_t tcp_ctlinput_viaudp;
346
347 static struct tcp_function_block tcp_def_funcblk = {
348 .tfb_tcp_block_name = "freebsd",
349 .tfb_tcp_output = tcp_default_output,
350 .tfb_tcp_do_segment = tcp_do_segment,
351 .tfb_tcp_ctloutput = tcp_default_ctloutput,
352 .tfb_tcp_handoff_ok = tcp_default_handoff_ok,
353 .tfb_tcp_fb_init = tcp_default_fb_init,
354 .tfb_tcp_fb_fini = tcp_default_fb_fini,
355 .tfb_switch_failed = tcp_default_switch_failed,
356 .tfb_flags = TCP_FUNC_DEFAULT_OK,
357 };
358
359 static int tcp_fb_cnt = 0;
360 struct tcp_funchead t_functions;
361 VNET_DEFINE_STATIC(struct tcp_function_block *, tcp_func_set_ptr) = &tcp_def_funcblk;
362 #define V_tcp_func_set_ptr VNET(tcp_func_set_ptr)
363
364 void
tcp_record_dsack(struct tcpcb * tp,tcp_seq start,tcp_seq end,int tlp)365 tcp_record_dsack(struct tcpcb *tp, tcp_seq start, tcp_seq end, int tlp)
366 {
367 TCPSTAT_INC(tcps_dsack_count);
368 tp->t_dsack_pack++;
369 if (tlp == 0) {
370 if (SEQ_GT(end, start)) {
371 tp->t_dsack_bytes += (end - start);
372 TCPSTAT_ADD(tcps_dsack_bytes, (end - start));
373 } else {
374 tp->t_dsack_tlp_bytes += (start - end);
375 TCPSTAT_ADD(tcps_dsack_bytes, (start - end));
376 }
377 } else {
378 if (SEQ_GT(end, start)) {
379 tp->t_dsack_bytes += (end - start);
380 TCPSTAT_ADD(tcps_dsack_tlp_bytes, (end - start));
381 } else {
382 tp->t_dsack_tlp_bytes += (start - end);
383 TCPSTAT_ADD(tcps_dsack_tlp_bytes, (start - end));
384 }
385 }
386 }
387
388 static struct tcp_function_block *
find_tcp_functions_locked(struct tcp_function_set * fs)389 find_tcp_functions_locked(struct tcp_function_set *fs)
390 {
391 struct tcp_function *f;
392 struct tcp_function_block *blk = NULL;
393
394 rw_assert(&tcp_function_lock, RA_LOCKED);
395 TAILQ_FOREACH(f, &t_functions, tf_next) {
396 if (strcmp(f->tf_name, fs->function_set_name) == 0) {
397 blk = f->tf_fb;
398 break;
399 }
400 }
401 return (blk);
402 }
403
404 static struct tcp_function_block *
find_tcp_fb_locked(struct tcp_function_block * blk,struct tcp_function ** s)405 find_tcp_fb_locked(struct tcp_function_block *blk, struct tcp_function **s)
406 {
407 struct tcp_function_block *rblk = NULL;
408 struct tcp_function *f;
409
410 rw_assert(&tcp_function_lock, RA_LOCKED);
411 TAILQ_FOREACH(f, &t_functions, tf_next) {
412 if (f->tf_fb == blk) {
413 rblk = blk;
414 if (s) {
415 *s = f;
416 }
417 break;
418 }
419 }
420 return (rblk);
421 }
422
423 struct tcp_function_block *
find_and_ref_tcp_functions(struct tcp_function_set * fs)424 find_and_ref_tcp_functions(struct tcp_function_set *fs)
425 {
426 struct tcp_function_block *blk;
427
428 rw_rlock(&tcp_function_lock);
429 blk = find_tcp_functions_locked(fs);
430 if (blk)
431 refcount_acquire(&blk->tfb_refcnt);
432 rw_runlock(&tcp_function_lock);
433 return (blk);
434 }
435
436 struct tcp_function_block *
find_and_ref_tcp_fb(struct tcp_function_block * blk)437 find_and_ref_tcp_fb(struct tcp_function_block *blk)
438 {
439 struct tcp_function_block *rblk;
440
441 rw_rlock(&tcp_function_lock);
442 rblk = find_tcp_fb_locked(blk, NULL);
443 if (rblk)
444 refcount_acquire(&rblk->tfb_refcnt);
445 rw_runlock(&tcp_function_lock);
446 return (rblk);
447 }
448
449 /* Find a matching alias for the given tcp_function_block. */
450 int
find_tcp_function_alias(struct tcp_function_block * blk,struct tcp_function_set * fs)451 find_tcp_function_alias(struct tcp_function_block *blk,
452 struct tcp_function_set *fs)
453 {
454 struct tcp_function *f;
455 int found;
456
457 found = 0;
458 rw_rlock(&tcp_function_lock);
459 TAILQ_FOREACH(f, &t_functions, tf_next) {
460 if ((f->tf_fb == blk) &&
461 (strncmp(f->tf_name, blk->tfb_tcp_block_name,
462 TCP_FUNCTION_NAME_LEN_MAX) != 0)) {
463 /* Matching function block with different name. */
464 strncpy(fs->function_set_name, f->tf_name,
465 TCP_FUNCTION_NAME_LEN_MAX);
466 found = 1;
467 break;
468 }
469 }
470 /* Null terminate the string appropriately. */
471 if (found) {
472 fs->function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
473 } else {
474 fs->function_set_name[0] = '\0';
475 }
476 rw_runlock(&tcp_function_lock);
477 return (found);
478 }
479
480 static struct tcp_function_block *
find_and_ref_tcp_default_fb(void)481 find_and_ref_tcp_default_fb(void)
482 {
483 struct tcp_function_block *rblk;
484
485 rw_rlock(&tcp_function_lock);
486 rblk = V_tcp_func_set_ptr;
487 refcount_acquire(&rblk->tfb_refcnt);
488 rw_runlock(&tcp_function_lock);
489 return (rblk);
490 }
491
492 void
tcp_switch_back_to_default(struct tcpcb * tp)493 tcp_switch_back_to_default(struct tcpcb *tp)
494 {
495 struct tcp_function_block *tfb;
496 void *ptr = NULL;
497
498 KASSERT(tp->t_fb != &tcp_def_funcblk,
499 ("%s: called by the built-in default stack", __func__));
500
501 if (tp->t_fb->tfb_tcp_timer_stop_all != NULL)
502 tp->t_fb->tfb_tcp_timer_stop_all(tp);
503
504 /*
505 * Now, we'll find a new function block to use.
506 * Start by trying the current user-selected
507 * default, unless this stack is the user-selected
508 * default.
509 */
510 tfb = find_and_ref_tcp_default_fb();
511 if (tfb == tp->t_fb) {
512 refcount_release(&tfb->tfb_refcnt);
513 tfb = NULL;
514 }
515 /* Does the stack accept this connection? */
516 if (tfb != NULL && (*tfb->tfb_tcp_handoff_ok)(tp)) {
517 refcount_release(&tfb->tfb_refcnt);
518 tfb = NULL;
519 }
520 /* Try to use that stack. */
521 if (tfb != NULL) {
522 /* Initialize the new stack. If it succeeds, we are done. */
523 if (tfb->tfb_tcp_fb_init == NULL ||
524 (*tfb->tfb_tcp_fb_init)(tp, &ptr) == 0) {
525 /* Release the old stack */
526 if (tp->t_fb->tfb_tcp_fb_fini != NULL)
527 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
528 refcount_release(&tp->t_fb->tfb_refcnt);
529 /* Now set in all the pointers */
530 tp->t_fb = tfb;
531 tp->t_fb_ptr = ptr;
532 return;
533 }
534 /*
535 * Initialization failed. Release the reference count on
536 * the looked up default stack.
537 */
538 refcount_release(&tfb->tfb_refcnt);
539 }
540
541 /*
542 * If that wasn't feasible, use the built-in default
543 * stack which is not allowed to reject anyone.
544 */
545 tfb = find_and_ref_tcp_fb(&tcp_def_funcblk);
546 if (tfb == NULL) {
547 /* there always should be a default */
548 panic("Can't refer to tcp_def_funcblk");
549 }
550 if ((*tfb->tfb_tcp_handoff_ok)(tp)) {
551 /* The default stack cannot say no */
552 panic("Default stack rejects a new session?");
553 }
554 if (tfb->tfb_tcp_fb_init != NULL &&
555 (*tfb->tfb_tcp_fb_init)(tp, &ptr)) {
556 /* The default stack cannot fail */
557 panic("Default stack initialization failed");
558 }
559 /* Now release the old stack */
560 if (tp->t_fb->tfb_tcp_fb_fini != NULL)
561 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
562 refcount_release(&tp->t_fb->tfb_refcnt);
563 /* And set in the pointers to the new */
564 tp->t_fb = tfb;
565 tp->t_fb_ptr = ptr;
566 }
567
568 static bool
tcp_recv_udp_tunneled_packet(struct mbuf * m,int off,struct inpcb * inp,const struct sockaddr * sa,void * ctx)569 tcp_recv_udp_tunneled_packet(struct mbuf *m, int off, struct inpcb *inp,
570 const struct sockaddr *sa, void *ctx)
571 {
572 struct ip *iph;
573 #ifdef INET6
574 struct ip6_hdr *ip6;
575 #endif
576 struct udphdr *uh;
577 struct tcphdr *th;
578 int thlen;
579 uint16_t port;
580
581 TCPSTAT_INC(tcps_tunneled_pkts);
582 if ((m->m_flags & M_PKTHDR) == 0) {
583 /* Can't handle one that is not a pkt hdr */
584 TCPSTAT_INC(tcps_tunneled_errs);
585 goto out;
586 }
587 thlen = sizeof(struct tcphdr);
588 if (m->m_len < off + sizeof(struct udphdr) + thlen &&
589 (m = m_pullup(m, off + sizeof(struct udphdr) + thlen)) == NULL) {
590 TCPSTAT_INC(tcps_tunneled_errs);
591 goto out;
592 }
593 iph = mtod(m, struct ip *);
594 uh = (struct udphdr *)((caddr_t)iph + off);
595 th = (struct tcphdr *)(uh + 1);
596 thlen = th->th_off << 2;
597 if (m->m_len < off + sizeof(struct udphdr) + thlen) {
598 m = m_pullup(m, off + sizeof(struct udphdr) + thlen);
599 if (m == NULL) {
600 TCPSTAT_INC(tcps_tunneled_errs);
601 goto out;
602 } else {
603 iph = mtod(m, struct ip *);
604 uh = (struct udphdr *)((caddr_t)iph + off);
605 th = (struct tcphdr *)(uh + 1);
606 }
607 }
608 m->m_pkthdr.tcp_tun_port = port = uh->uh_sport;
609 bcopy(th, uh, m->m_len - off);
610 m->m_len -= sizeof(struct udphdr);
611 m->m_pkthdr.len -= sizeof(struct udphdr);
612 /*
613 * We use the same algorithm for
614 * both UDP and TCP for c-sum. So
615 * the code in tcp_input will skip
616 * the checksum. So we do nothing
617 * with the flag (m->m_pkthdr.csum_flags).
618 */
619 switch (iph->ip_v) {
620 #ifdef INET
621 case IPVERSION:
622 iph->ip_len = htons(ntohs(iph->ip_len) - sizeof(struct udphdr));
623 tcp_input_with_port(&m, &off, IPPROTO_TCP, port);
624 break;
625 #endif
626 #ifdef INET6
627 case IPV6_VERSION >> 4:
628 ip6 = mtod(m, struct ip6_hdr *);
629 ip6->ip6_plen = htons(ntohs(ip6->ip6_plen) - sizeof(struct udphdr));
630 tcp6_input_with_port(&m, &off, IPPROTO_TCP, port);
631 break;
632 #endif
633 default:
634 goto out;
635 break;
636 }
637 return (true);
638 out:
639 m_freem(m);
640
641 return (true);
642 }
643
644 static int
sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)645 sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)
646 {
647 int error = ENOENT;
648 struct tcp_function_set fs;
649 struct tcp_function_block *blk;
650
651 memset(&fs, 0, sizeof(fs));
652 rw_rlock(&tcp_function_lock);
653 blk = find_tcp_fb_locked(V_tcp_func_set_ptr, NULL);
654 if (blk) {
655 /* Found him */
656 strcpy(fs.function_set_name, blk->tfb_tcp_block_name);
657 fs.pcbcnt = blk->tfb_refcnt;
658 }
659 rw_runlock(&tcp_function_lock);
660 error = sysctl_handle_string(oidp, fs.function_set_name,
661 sizeof(fs.function_set_name), req);
662
663 /* Check for error or no change */
664 if (error != 0 || req->newptr == NULL)
665 return (error);
666
667 rw_wlock(&tcp_function_lock);
668 blk = find_tcp_functions_locked(&fs);
669 if ((blk == NULL) ||
670 (blk->tfb_flags & TCP_FUNC_BEING_REMOVED)) {
671 error = ENOENT;
672 goto done;
673 }
674 if ((blk->tfb_flags & TCP_FUNC_DEFAULT_OK) == 0) {
675 error = EINVAL;
676 goto done;
677 }
678 V_tcp_func_set_ptr = blk;
679 done:
680 rw_wunlock(&tcp_function_lock);
681 return (error);
682 }
683
684 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_default,
685 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
686 NULL, 0, sysctl_net_inet_default_tcp_functions, "A",
687 "Set/get the default TCP functions");
688
689 static int
sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)690 sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)
691 {
692 int error, cnt, linesz;
693 struct tcp_function *f;
694 char *buffer, *cp;
695 size_t bufsz, outsz;
696 bool alias;
697
698 cnt = 0;
699 rw_rlock(&tcp_function_lock);
700 TAILQ_FOREACH(f, &t_functions, tf_next) {
701 cnt++;
702 }
703 rw_runlock(&tcp_function_lock);
704
705 bufsz = (cnt+2) * ((TCP_FUNCTION_NAME_LEN_MAX * 2) + 13) + 1;
706 buffer = malloc(bufsz, M_TEMP, M_WAITOK);
707
708 error = 0;
709 cp = buffer;
710
711 linesz = snprintf(cp, bufsz, "\n%-32s%c %-32s %s\n", "Stack", 'D',
712 "Alias", "PCB count");
713 cp += linesz;
714 bufsz -= linesz;
715 outsz = linesz;
716
717 rw_rlock(&tcp_function_lock);
718 TAILQ_FOREACH(f, &t_functions, tf_next) {
719 alias = (f->tf_name != f->tf_fb->tfb_tcp_block_name);
720 linesz = snprintf(cp, bufsz, "%-32s%c %-32s %u\n",
721 f->tf_fb->tfb_tcp_block_name,
722 (f->tf_fb == V_tcp_func_set_ptr) ? '*' : ' ',
723 alias ? f->tf_name : "-",
724 f->tf_fb->tfb_refcnt);
725 if (linesz >= bufsz) {
726 error = EOVERFLOW;
727 break;
728 }
729 cp += linesz;
730 bufsz -= linesz;
731 outsz += linesz;
732 }
733 rw_runlock(&tcp_function_lock);
734 if (error == 0)
735 error = sysctl_handle_string(oidp, buffer, outsz + 1, req);
736 free(buffer, M_TEMP);
737 return (error);
738 }
739
740 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available,
741 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
742 NULL, 0, sysctl_net_inet_list_available, "A",
743 "list available TCP Function sets");
744
745 VNET_DEFINE(int, tcp_udp_tunneling_port) = TCP_TUNNELING_PORT_DEFAULT;
746
747 #ifdef INET
748 VNET_DEFINE(struct socket *, udp4_tun_socket) = NULL;
749 #define V_udp4_tun_socket VNET(udp4_tun_socket)
750 #endif
751 #ifdef INET6
752 VNET_DEFINE(struct socket *, udp6_tun_socket) = NULL;
753 #define V_udp6_tun_socket VNET(udp6_tun_socket)
754 #endif
755
756 static struct sx tcpoudp_lock;
757
758 static void
tcp_over_udp_stop(void)759 tcp_over_udp_stop(void)
760 {
761
762 sx_assert(&tcpoudp_lock, SA_XLOCKED);
763
764 #ifdef INET
765 if (V_udp4_tun_socket != NULL) {
766 soclose(V_udp4_tun_socket);
767 V_udp4_tun_socket = NULL;
768 }
769 #endif
770 #ifdef INET6
771 if (V_udp6_tun_socket != NULL) {
772 soclose(V_udp6_tun_socket);
773 V_udp6_tun_socket = NULL;
774 }
775 #endif
776 }
777
778 static int
tcp_over_udp_start(void)779 tcp_over_udp_start(void)
780 {
781 uint16_t port;
782 int ret;
783 #ifdef INET
784 struct sockaddr_in sin;
785 #endif
786 #ifdef INET6
787 struct sockaddr_in6 sin6;
788 #endif
789
790 sx_assert(&tcpoudp_lock, SA_XLOCKED);
791
792 port = V_tcp_udp_tunneling_port;
793 if (ntohs(port) == 0) {
794 /* Must have a port set */
795 return (EINVAL);
796 }
797 #ifdef INET
798 if (V_udp4_tun_socket != NULL) {
799 /* Already running -- must stop first */
800 return (EALREADY);
801 }
802 #endif
803 #ifdef INET6
804 if (V_udp6_tun_socket != NULL) {
805 /* Already running -- must stop first */
806 return (EALREADY);
807 }
808 #endif
809 #ifdef INET
810 if ((ret = socreate(PF_INET, &V_udp4_tun_socket,
811 SOCK_DGRAM, IPPROTO_UDP,
812 curthread->td_ucred, curthread))) {
813 tcp_over_udp_stop();
814 return (ret);
815 }
816 /* Call the special UDP hook. */
817 if ((ret = udp_set_kernel_tunneling(V_udp4_tun_socket,
818 tcp_recv_udp_tunneled_packet,
819 tcp_ctlinput_viaudp,
820 NULL))) {
821 tcp_over_udp_stop();
822 return (ret);
823 }
824 /* Ok, we have a socket, bind it to the port. */
825 memset(&sin, 0, sizeof(struct sockaddr_in));
826 sin.sin_len = sizeof(struct sockaddr_in);
827 sin.sin_family = AF_INET;
828 sin.sin_port = htons(port);
829 if ((ret = sobind(V_udp4_tun_socket,
830 (struct sockaddr *)&sin, curthread))) {
831 tcp_over_udp_stop();
832 return (ret);
833 }
834 #endif
835 #ifdef INET6
836 if ((ret = socreate(PF_INET6, &V_udp6_tun_socket,
837 SOCK_DGRAM, IPPROTO_UDP,
838 curthread->td_ucred, curthread))) {
839 tcp_over_udp_stop();
840 return (ret);
841 }
842 /* Call the special UDP hook. */
843 if ((ret = udp_set_kernel_tunneling(V_udp6_tun_socket,
844 tcp_recv_udp_tunneled_packet,
845 tcp6_ctlinput_viaudp,
846 NULL))) {
847 tcp_over_udp_stop();
848 return (ret);
849 }
850 /* Ok, we have a socket, bind it to the port. */
851 memset(&sin6, 0, sizeof(struct sockaddr_in6));
852 sin6.sin6_len = sizeof(struct sockaddr_in6);
853 sin6.sin6_family = AF_INET6;
854 sin6.sin6_port = htons(port);
855 if ((ret = sobind(V_udp6_tun_socket,
856 (struct sockaddr *)&sin6, curthread))) {
857 tcp_over_udp_stop();
858 return (ret);
859 }
860 #endif
861 return (0);
862 }
863
864 static int
sysctl_net_inet_tcp_udp_tunneling_port_check(SYSCTL_HANDLER_ARGS)865 sysctl_net_inet_tcp_udp_tunneling_port_check(SYSCTL_HANDLER_ARGS)
866 {
867 int error;
868 uint32_t old, new;
869
870 old = V_tcp_udp_tunneling_port;
871 new = old;
872 error = sysctl_handle_int(oidp, &new, 0, req);
873 if ((error == 0) &&
874 (req->newptr != NULL)) {
875 if ((new < TCP_TUNNELING_PORT_MIN) ||
876 (new > TCP_TUNNELING_PORT_MAX)) {
877 error = EINVAL;
878 } else {
879 sx_xlock(&tcpoudp_lock);
880 V_tcp_udp_tunneling_port = new;
881 if (old != 0) {
882 tcp_over_udp_stop();
883 }
884 if (new != 0) {
885 error = tcp_over_udp_start();
886 if (error != 0) {
887 V_tcp_udp_tunneling_port = 0;
888 }
889 }
890 sx_xunlock(&tcpoudp_lock);
891 }
892 }
893 return (error);
894 }
895
896 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_port,
897 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
898 &VNET_NAME(tcp_udp_tunneling_port),
899 0, &sysctl_net_inet_tcp_udp_tunneling_port_check, "IU",
900 "Tunneling port for tcp over udp");
901
902 VNET_DEFINE(int, tcp_udp_tunneling_overhead) = TCP_TUNNELING_OVERHEAD_DEFAULT;
903
904 static int
sysctl_net_inet_tcp_udp_tunneling_overhead_check(SYSCTL_HANDLER_ARGS)905 sysctl_net_inet_tcp_udp_tunneling_overhead_check(SYSCTL_HANDLER_ARGS)
906 {
907 int error, new;
908
909 new = V_tcp_udp_tunneling_overhead;
910 error = sysctl_handle_int(oidp, &new, 0, req);
911 if (error == 0 && req->newptr) {
912 if ((new < TCP_TUNNELING_OVERHEAD_MIN) ||
913 (new > TCP_TUNNELING_OVERHEAD_MAX))
914 error = EINVAL;
915 else
916 V_tcp_udp_tunneling_overhead = new;
917 }
918 return (error);
919 }
920
921 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_overhead,
922 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
923 &VNET_NAME(tcp_udp_tunneling_overhead),
924 0, &sysctl_net_inet_tcp_udp_tunneling_overhead_check, "IU",
925 "MSS reduction when using tcp over udp");
926
927 /*
928 * Exports one (struct tcp_function_info) for each alias/name.
929 */
930 static int
sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)931 sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)
932 {
933 int cnt, error;
934 struct tcp_function *f;
935 struct tcp_function_info tfi;
936
937 /*
938 * We don't allow writes.
939 */
940 if (req->newptr != NULL)
941 return (EINVAL);
942
943 /*
944 * Wire the old buffer so we can directly copy the functions to
945 * user space without dropping the lock.
946 */
947 if (req->oldptr != NULL) {
948 error = sysctl_wire_old_buffer(req, 0);
949 if (error)
950 return (error);
951 }
952
953 /*
954 * Walk the list and copy out matching entries. If INVARIANTS
955 * is compiled in, also walk the list to verify the length of
956 * the list matches what we have recorded.
957 */
958 rw_rlock(&tcp_function_lock);
959
960 cnt = 0;
961 #ifndef INVARIANTS
962 if (req->oldptr == NULL) {
963 cnt = tcp_fb_cnt;
964 goto skip_loop;
965 }
966 #endif
967 TAILQ_FOREACH(f, &t_functions, tf_next) {
968 #ifdef INVARIANTS
969 cnt++;
970 #endif
971 if (req->oldptr != NULL) {
972 bzero(&tfi, sizeof(tfi));
973 tfi.tfi_refcnt = f->tf_fb->tfb_refcnt;
974 tfi.tfi_id = f->tf_fb->tfb_id;
975 (void)strlcpy(tfi.tfi_alias, f->tf_name,
976 sizeof(tfi.tfi_alias));
977 (void)strlcpy(tfi.tfi_name,
978 f->tf_fb->tfb_tcp_block_name, sizeof(tfi.tfi_name));
979 error = SYSCTL_OUT(req, &tfi, sizeof(tfi));
980 /*
981 * Don't stop on error, as that is the
982 * mechanism we use to accumulate length
983 * information if the buffer was too short.
984 */
985 }
986 }
987 KASSERT(cnt == tcp_fb_cnt,
988 ("%s: cnt (%d) != tcp_fb_cnt (%d)", __func__, cnt, tcp_fb_cnt));
989 #ifndef INVARIANTS
990 skip_loop:
991 #endif
992 rw_runlock(&tcp_function_lock);
993 if (req->oldptr == NULL)
994 error = SYSCTL_OUT(req, NULL,
995 (cnt + 1) * sizeof(struct tcp_function_info));
996
997 return (error);
998 }
999
1000 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_info,
1001 CTLTYPE_OPAQUE | CTLFLAG_SKIP | CTLFLAG_RD | CTLFLAG_MPSAFE,
1002 NULL, 0, sysctl_net_inet_list_func_info, "S,tcp_function_info",
1003 "List TCP function block name-to-ID mappings");
1004
1005 /*
1006 * tfb_tcp_handoff_ok() function for the default stack.
1007 * Note that we'll basically try to take all comers.
1008 */
1009 static int
tcp_default_handoff_ok(struct tcpcb * tp)1010 tcp_default_handoff_ok(struct tcpcb *tp)
1011 {
1012
1013 return (0);
1014 }
1015
1016 /*
1017 * tfb_tcp_fb_init() function for the default stack.
1018 *
1019 * This handles making sure we have appropriate timers set if you are
1020 * transitioning a socket that has some amount of setup done.
1021 *
1022 * The init() fuction from the default can *never* return non-zero i.e.
1023 * it is required to always succeed since it is the stack of last resort!
1024 */
1025 static int
tcp_default_fb_init(struct tcpcb * tp,void ** ptr)1026 tcp_default_fb_init(struct tcpcb *tp, void **ptr)
1027 {
1028 struct socket *so = tptosocket(tp);
1029 int rexmt;
1030
1031 INP_WLOCK_ASSERT(tptoinpcb(tp));
1032 /* We don't use the pointer */
1033 *ptr = NULL;
1034
1035 KASSERT(tp->t_state < TCPS_TIME_WAIT,
1036 ("%s: connection %p in unexpected state %d", __func__, tp,
1037 tp->t_state));
1038
1039 /* Make sure we get no interesting mbuf queuing behavior */
1040 /* All mbuf queue/ack compress flags should be off */
1041 tcp_lro_features_off(tp);
1042
1043 /* Cancel the GP measurement in progress */
1044 tp->t_flags &= ~TF_GPUTINPROG;
1045 /* Validate the timers are not in usec, if they are convert */
1046 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
1047 if ((tp->t_state == TCPS_SYN_SENT) ||
1048 (tp->t_state == TCPS_SYN_RECEIVED))
1049 rexmt = tcp_rexmit_initial * tcp_backoff[tp->t_rxtshift];
1050 else
1051 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1052 if (tp->t_rxtshift == 0)
1053 tp->t_rxtcur = rexmt;
1054 else
1055 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX);
1056
1057 /*
1058 * Nothing to do for ESTABLISHED or LISTEN states. And, we don't
1059 * know what to do for unexpected states (which includes TIME_WAIT).
1060 */
1061 if (tp->t_state <= TCPS_LISTEN || tp->t_state >= TCPS_TIME_WAIT)
1062 return (0);
1063
1064 /*
1065 * Make sure some kind of transmission timer is set if there is
1066 * outstanding data.
1067 */
1068 if ((!TCPS_HAVEESTABLISHED(tp->t_state) || sbavail(&so->so_snd) ||
1069 tp->snd_una != tp->snd_max) && !(tcp_timer_active(tp, TT_REXMT) ||
1070 tcp_timer_active(tp, TT_PERSIST))) {
1071 /*
1072 * If the session has established and it looks like it should
1073 * be in the persist state, set the persist timer. Otherwise,
1074 * set the retransmit timer.
1075 */
1076 if (TCPS_HAVEESTABLISHED(tp->t_state) && tp->snd_wnd == 0 &&
1077 (int32_t)(tp->snd_nxt - tp->snd_una) <
1078 (int32_t)sbavail(&so->so_snd))
1079 tcp_setpersist(tp);
1080 else
1081 tcp_timer_activate(tp, TT_REXMT, TP_RXTCUR(tp));
1082 }
1083
1084 /* All non-embryonic sessions get a keepalive timer. */
1085 if (!tcp_timer_active(tp, TT_KEEP))
1086 tcp_timer_activate(tp, TT_KEEP,
1087 TCPS_HAVEESTABLISHED(tp->t_state) ? TP_KEEPIDLE(tp) :
1088 TP_KEEPINIT(tp));
1089
1090 /*
1091 * Make sure critical variables are initialized
1092 * if transitioning while in Recovery.
1093 */
1094 if IN_FASTRECOVERY(tp->t_flags) {
1095 if (tp->sackhint.recover_fs == 0)
1096 tp->sackhint.recover_fs = max(1,
1097 tp->snd_nxt - tp->snd_una);
1098 }
1099
1100 return (0);
1101 }
1102
1103 /*
1104 * tfb_tcp_fb_fini() function for the default stack.
1105 *
1106 * This changes state as necessary (or prudent) to prepare for another stack
1107 * to assume responsibility for the connection.
1108 */
1109 static void
tcp_default_fb_fini(struct tcpcb * tp,int tcb_is_purged)1110 tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged)
1111 {
1112
1113 INP_WLOCK_ASSERT(tptoinpcb(tp));
1114
1115 #ifdef TCP_BLACKBOX
1116 tcp_log_flowend(tp);
1117 #endif
1118 tp->t_acktime = 0;
1119 return;
1120 }
1121
1122 MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
1123 MALLOC_DEFINE(M_TCPFUNCTIONS, "tcpfunc", "TCP function set memory");
1124
1125 static struct mtx isn_mtx;
1126
1127 #define ISN_LOCK_INIT() mtx_init(&isn_mtx, "isn_mtx", NULL, MTX_DEF)
1128 #define ISN_LOCK() mtx_lock(&isn_mtx)
1129 #define ISN_UNLOCK() mtx_unlock(&isn_mtx)
1130
1131 INPCBSTORAGE_DEFINE(tcpcbstor, tcpcb, "tcpinp", "tcp_inpcb", "tcp", "tcphash");
1132
1133 /*
1134 * Take a value and get the next power of 2 that doesn't overflow.
1135 * Used to size the tcp_inpcb hash buckets.
1136 */
1137 static int
maketcp_hashsize(int size)1138 maketcp_hashsize(int size)
1139 {
1140 int hashsize;
1141
1142 /*
1143 * auto tune.
1144 * get the next power of 2 higher than maxsockets.
1145 */
1146 hashsize = 1 << fls(size);
1147 /* catch overflow, and just go one power of 2 smaller */
1148 if (hashsize < size) {
1149 hashsize = 1 << (fls(size) - 1);
1150 }
1151 return (hashsize);
1152 }
1153
1154 static volatile int next_tcp_stack_id = 1;
1155
1156 /*
1157 * Register a TCP function block with the name provided in the names
1158 * array. (Note that this function does NOT automatically register
1159 * blk->tfb_tcp_block_name as a stack name. Therefore, you should
1160 * explicitly include blk->tfb_tcp_block_name in the list of names if
1161 * you wish to register the stack with that name.)
1162 *
1163 * Either all name registrations will succeed or all will fail. If
1164 * a name registration fails, the function will update the num_names
1165 * argument to point to the array index of the name that encountered
1166 * the failure.
1167 *
1168 * Returns 0 on success, or an error code on failure.
1169 */
1170 int
register_tcp_functions_as_names(struct tcp_function_block * blk,int wait,const char * names[],int * num_names)1171 register_tcp_functions_as_names(struct tcp_function_block *blk, int wait,
1172 const char *names[], int *num_names)
1173 {
1174 struct tcp_function *f[TCP_FUNCTION_NAME_NUM_MAX];
1175 struct tcp_function_set fs;
1176 int error, i, num_registered;
1177
1178 KASSERT(names != NULL, ("%s: Called with NULL name list", __func__));
1179 KASSERT(*num_names > 0,
1180 ("%s: Called with non-positive length of name list", __func__));
1181 KASSERT(rw_initialized(&tcp_function_lock),
1182 ("%s: called too early", __func__));
1183
1184 if (*num_names > TCP_FUNCTION_NAME_NUM_MAX) {
1185 /* Too many names. */
1186 *num_names = 0;
1187 return (E2BIG);
1188 }
1189 if ((blk->tfb_tcp_output == NULL) ||
1190 (blk->tfb_tcp_do_segment == NULL) ||
1191 (blk->tfb_tcp_ctloutput == NULL) ||
1192 (blk->tfb_tcp_handoff_ok == NULL) ||
1193 (strlen(blk->tfb_tcp_block_name) == 0)) {
1194 /* These functions are required and a name is needed. */
1195 *num_names = 0;
1196 return (EINVAL);
1197 }
1198
1199 for (i = 0; i < *num_names; i++) {
1200 f[i] = malloc(sizeof(struct tcp_function), M_TCPFUNCTIONS, wait);
1201 if (f[i] == NULL) {
1202 while (--i >= 0)
1203 free(f[i], M_TCPFUNCTIONS);
1204 *num_names = 0;
1205 return (ENOMEM);
1206 }
1207 }
1208
1209 num_registered = 0;
1210 rw_wlock(&tcp_function_lock);
1211 if (find_tcp_fb_locked(blk, NULL) != NULL) {
1212 /* A TCP function block can only be registered once. */
1213 error = EALREADY;
1214 goto cleanup;
1215 }
1216 if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1217 error = EINVAL;
1218 goto cleanup;
1219 }
1220 refcount_init(&blk->tfb_refcnt, 0);
1221 blk->tfb_id = atomic_fetchadd_int(&next_tcp_stack_id, 1);
1222 for (i = 0; i < *num_names; i++) {
1223 (void)strlcpy(fs.function_set_name, names[i],
1224 sizeof(fs.function_set_name));
1225 if (find_tcp_functions_locked(&fs) != NULL) {
1226 /* Duplicate name space not allowed */
1227 error = EALREADY;
1228 goto cleanup;
1229 }
1230 f[i]->tf_fb = blk;
1231 (void)strlcpy(f[i]->tf_name, names[i], sizeof(f[i]->tf_name));
1232 TAILQ_INSERT_TAIL(&t_functions, f[i], tf_next);
1233 tcp_fb_cnt++;
1234 num_registered++;
1235 }
1236 rw_wunlock(&tcp_function_lock);
1237 return (0);
1238
1239 cleanup:
1240 /* Remove the entries just added. */
1241 for (i = 0; i < *num_names; i++) {
1242 if (i < num_registered) {
1243 TAILQ_REMOVE(&t_functions, f[i], tf_next);
1244 tcp_fb_cnt--;
1245 }
1246 f[i]->tf_fb = NULL;
1247 free(f[i], M_TCPFUNCTIONS);
1248 }
1249 rw_wunlock(&tcp_function_lock);
1250 *num_names = num_registered;
1251 return (error);
1252 }
1253
1254 /*
1255 * Register a TCP function block using the name provided in the name
1256 * argument.
1257 *
1258 * Returns 0 on success, or an error code on failure.
1259 */
1260 int
register_tcp_functions_as_name(struct tcp_function_block * blk,const char * name,int wait)1261 register_tcp_functions_as_name(struct tcp_function_block *blk, const char *name,
1262 int wait)
1263 {
1264 const char *name_list[1];
1265 int num_names, rv;
1266
1267 num_names = 1;
1268 if (name != NULL)
1269 name_list[0] = name;
1270 else
1271 name_list[0] = blk->tfb_tcp_block_name;
1272 rv = register_tcp_functions_as_names(blk, wait, name_list, &num_names);
1273 return (rv);
1274 }
1275
1276 /*
1277 * Register a TCP function block using the name defined in
1278 * blk->tfb_tcp_block_name.
1279 *
1280 * Returns 0 on success, or an error code on failure.
1281 */
1282 int
register_tcp_functions(struct tcp_function_block * blk,int wait)1283 register_tcp_functions(struct tcp_function_block *blk, int wait)
1284 {
1285
1286 return (register_tcp_functions_as_name(blk, NULL, wait));
1287 }
1288
1289 /*
1290 * Deregister all names associated with a function block. This
1291 * functionally removes the function block from use within the system.
1292 *
1293 * When called with a true quiesce argument, mark the function block
1294 * as being removed so no more stacks will use it and determine
1295 * whether the removal would succeed.
1296 *
1297 * When called with a false quiesce argument, actually attempt the
1298 * removal.
1299 *
1300 * When called with a force argument, attempt to switch all TCBs to
1301 * use the default stack instead of returning EBUSY.
1302 *
1303 * Returns 0 on success (or if the removal would succeed), or an error
1304 * code on failure.
1305 */
1306 int
deregister_tcp_functions(struct tcp_function_block * blk,bool quiesce,bool force)1307 deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce,
1308 bool force)
1309 {
1310 struct tcp_function *f;
1311 VNET_ITERATOR_DECL(vnet_iter);
1312
1313 if (blk == &tcp_def_funcblk) {
1314 /* You can't un-register the default */
1315 return (EPERM);
1316 }
1317 rw_wlock(&tcp_function_lock);
1318 VNET_LIST_RLOCK_NOSLEEP();
1319 VNET_FOREACH(vnet_iter) {
1320 CURVNET_SET(vnet_iter);
1321 if (blk == V_tcp_func_set_ptr) {
1322 /* You can't free the current default in some vnet. */
1323 CURVNET_RESTORE();
1324 VNET_LIST_RUNLOCK_NOSLEEP();
1325 rw_wunlock(&tcp_function_lock);
1326 return (EBUSY);
1327 }
1328 CURVNET_RESTORE();
1329 }
1330 VNET_LIST_RUNLOCK_NOSLEEP();
1331 /* Mark the block so no more stacks can use it. */
1332 blk->tfb_flags |= TCP_FUNC_BEING_REMOVED;
1333 /*
1334 * If TCBs are still attached to the stack, attempt to switch them
1335 * to the default stack.
1336 */
1337 if (force && blk->tfb_refcnt) {
1338 struct inpcb *inp;
1339 struct tcpcb *tp;
1340 VNET_ITERATOR_DECL(vnet_iter);
1341
1342 rw_wunlock(&tcp_function_lock);
1343
1344 VNET_LIST_RLOCK();
1345 VNET_FOREACH(vnet_iter) {
1346 CURVNET_SET(vnet_iter);
1347 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1348 INPLOOKUP_WLOCKPCB);
1349
1350 while ((inp = inp_next(&inpi)) != NULL) {
1351 tp = intotcpcb(inp);
1352 if (tp == NULL || tp->t_fb != blk)
1353 continue;
1354 tcp_switch_back_to_default(tp);
1355 }
1356 CURVNET_RESTORE();
1357 }
1358 VNET_LIST_RUNLOCK();
1359
1360 rw_wlock(&tcp_function_lock);
1361 }
1362 if (blk->tfb_refcnt) {
1363 /* TCBs still attached. */
1364 rw_wunlock(&tcp_function_lock);
1365 return (EBUSY);
1366 }
1367 if (quiesce) {
1368 /* Skip removal. */
1369 rw_wunlock(&tcp_function_lock);
1370 return (0);
1371 }
1372 /* Remove any function names that map to this function block. */
1373 while (find_tcp_fb_locked(blk, &f) != NULL) {
1374 TAILQ_REMOVE(&t_functions, f, tf_next);
1375 tcp_fb_cnt--;
1376 f->tf_fb = NULL;
1377 free(f, M_TCPFUNCTIONS);
1378 }
1379 rw_wunlock(&tcp_function_lock);
1380 return (0);
1381 }
1382
1383 static void
tcp_drain(void * ctx __unused,int flags __unused)1384 tcp_drain(void *ctx __unused, int flags __unused)
1385 {
1386 struct epoch_tracker et;
1387 VNET_ITERATOR_DECL(vnet_iter);
1388
1389 if (!do_tcpdrain)
1390 return;
1391
1392 NET_EPOCH_ENTER(et);
1393 VNET_LIST_RLOCK_NOSLEEP();
1394 VNET_FOREACH(vnet_iter) {
1395 CURVNET_SET(vnet_iter);
1396 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1397 INPLOOKUP_WLOCKPCB);
1398 struct inpcb *inpb;
1399 struct tcpcb *tcpb;
1400
1401 /*
1402 * Walk the tcpbs, if existing, and flush the reassembly queue,
1403 * if there is one...
1404 * XXX: The "Net/3" implementation doesn't imply that the TCP
1405 * reassembly queue should be flushed, but in a situation
1406 * where we're really low on mbufs, this is potentially
1407 * useful.
1408 */
1409 while ((inpb = inp_next(&inpi)) != NULL) {
1410 if ((tcpb = intotcpcb(inpb)) != NULL) {
1411 tcp_reass_flush(tcpb);
1412 tcp_clean_sackreport(tcpb);
1413 #ifdef TCP_BLACKBOX
1414 tcp_log_drain(tcpb);
1415 #endif
1416 }
1417 }
1418 CURVNET_RESTORE();
1419 }
1420 VNET_LIST_RUNLOCK_NOSLEEP();
1421 NET_EPOCH_EXIT(et);
1422 }
1423
1424 static void
tcp_vnet_init(void * arg __unused)1425 tcp_vnet_init(void *arg __unused)
1426 {
1427
1428 #ifdef TCP_HHOOK
1429 if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN,
1430 &V_tcp_hhh[HHOOK_TCP_EST_IN], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1431 printf("%s: WARNING: unable to register helper hook\n", __func__);
1432 if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT,
1433 &V_tcp_hhh[HHOOK_TCP_EST_OUT], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1434 printf("%s: WARNING: unable to register helper hook\n", __func__);
1435 #endif
1436 #ifdef STATS
1437 if (tcp_stats_init())
1438 printf("%s: WARNING: unable to initialise TCP stats\n",
1439 __func__);
1440 #endif
1441 in_pcbinfo_init(&V_tcbinfo, &tcpcbstor, tcp_tcbhashsize,
1442 tcp_tcbhashsize);
1443
1444 syncache_init();
1445 tcp_hc_init();
1446
1447 TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
1448 V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
1449 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1450
1451 tcp_fastopen_init();
1452
1453 COUNTER_ARRAY_ALLOC(V_tcps_states, TCP_NSTATES, M_WAITOK);
1454 VNET_PCPUSTAT_ALLOC(tcpstat, M_WAITOK);
1455
1456 V_tcp_msl = TCPTV_MSL;
1457 arc4rand(&V_ts_offset_secret, sizeof(V_ts_offset_secret), 0);
1458 }
1459 VNET_SYSINIT(tcp_vnet_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
1460 tcp_vnet_init, NULL);
1461
1462 static void
tcp_init(void * arg __unused)1463 tcp_init(void *arg __unused)
1464 {
1465 int hashsize;
1466
1467 tcp_reass_global_init();
1468
1469 /* XXX virtualize those below? */
1470 tcp_delacktime = TCPTV_DELACK;
1471 tcp_keepinit = TCPTV_KEEP_INIT;
1472 tcp_keepidle = TCPTV_KEEP_IDLE;
1473 tcp_keepintvl = TCPTV_KEEPINTVL;
1474 tcp_maxpersistidle = TCPTV_KEEP_IDLE;
1475 tcp_rexmit_initial = TCPTV_RTOBASE;
1476 if (tcp_rexmit_initial < 1)
1477 tcp_rexmit_initial = 1;
1478 tcp_rexmit_min = TCPTV_MIN;
1479 if (tcp_rexmit_min < 1)
1480 tcp_rexmit_min = 1;
1481 tcp_persmin = TCPTV_PERSMIN;
1482 tcp_persmax = TCPTV_PERSMAX;
1483 tcp_rexmit_slop = TCPTV_CPU_VAR;
1484 tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
1485
1486 /* Setup the tcp function block list */
1487 TAILQ_INIT(&t_functions);
1488 rw_init(&tcp_function_lock, "tcp_func_lock");
1489 register_tcp_functions(&tcp_def_funcblk, M_WAITOK);
1490 sx_init(&tcpoudp_lock, "TCP over UDP configuration");
1491 #ifdef TCP_BLACKBOX
1492 /* Initialize the TCP logging data. */
1493 tcp_log_init();
1494 #endif
1495
1496 if (tcp_soreceive_stream) {
1497 #ifdef INET
1498 tcp_protosw.pr_soreceive = soreceive_stream;
1499 #endif
1500 #ifdef INET6
1501 tcp6_protosw.pr_soreceive = soreceive_stream;
1502 #endif /* INET6 */
1503 }
1504
1505 #ifdef INET6
1506 max_protohdr_grow(sizeof(struct ip6_hdr) + sizeof(struct tcphdr));
1507 #else /* INET6 */
1508 max_protohdr_grow(sizeof(struct tcpiphdr));
1509 #endif /* INET6 */
1510
1511 ISN_LOCK_INIT();
1512 EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
1513 SHUTDOWN_PRI_DEFAULT);
1514 EVENTHANDLER_REGISTER(vm_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1515 EVENTHANDLER_REGISTER(mbuf_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1516
1517 tcp_inp_lro_direct_queue = counter_u64_alloc(M_WAITOK);
1518 tcp_inp_lro_wokeup_queue = counter_u64_alloc(M_WAITOK);
1519 tcp_inp_lro_compressed = counter_u64_alloc(M_WAITOK);
1520 tcp_inp_lro_locks_taken = counter_u64_alloc(M_WAITOK);
1521 tcp_extra_mbuf = counter_u64_alloc(M_WAITOK);
1522 tcp_would_have_but = counter_u64_alloc(M_WAITOK);
1523 tcp_comp_total = counter_u64_alloc(M_WAITOK);
1524 tcp_uncomp_total = counter_u64_alloc(M_WAITOK);
1525 tcp_bad_csums = counter_u64_alloc(M_WAITOK);
1526 tcp_pacing_failures = counter_u64_alloc(M_WAITOK);
1527 tcp_dgp_failures = counter_u64_alloc(M_WAITOK);
1528
1529 hashsize = tcp_tcbhashsize;
1530 if (hashsize == 0) {
1531 /*
1532 * Auto tune the hash size based on maxsockets.
1533 * A perfect hash would have a 1:1 mapping
1534 * (hashsize = maxsockets) however it's been
1535 * suggested that O(2) average is better.
1536 */
1537 hashsize = maketcp_hashsize(maxsockets / 4);
1538 /*
1539 * Our historical default is 512,
1540 * do not autotune lower than this.
1541 */
1542 if (hashsize < 512)
1543 hashsize = 512;
1544 if (bootverbose)
1545 printf("%s: %s auto tuned to %d\n", __func__,
1546 "net.inet.tcp.tcbhashsize", hashsize);
1547 }
1548 /*
1549 * We require a hashsize to be a power of two.
1550 * Previously if it was not a power of two we would just reset it
1551 * back to 512, which could be a nasty surprise if you did not notice
1552 * the error message.
1553 * Instead what we do is clip it to the closest power of two lower
1554 * than the specified hash value.
1555 */
1556 if (!powerof2(hashsize)) {
1557 int oldhashsize = hashsize;
1558
1559 hashsize = maketcp_hashsize(hashsize);
1560 /* prevent absurdly low value */
1561 if (hashsize < 16)
1562 hashsize = 16;
1563 printf("%s: WARNING: TCB hash size not a power of 2, "
1564 "clipped from %d to %d.\n", __func__, oldhashsize,
1565 hashsize);
1566 }
1567 tcp_tcbhashsize = hashsize;
1568
1569 #ifdef INET
1570 IPPROTO_REGISTER(IPPROTO_TCP, tcp_input, tcp_ctlinput);
1571 #endif
1572 #ifdef INET6
1573 IP6PROTO_REGISTER(IPPROTO_TCP, tcp6_input, tcp6_ctlinput);
1574 #endif
1575 }
1576 SYSINIT(tcp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, tcp_init, NULL);
1577
1578 #ifdef VIMAGE
1579 static void
tcp_destroy(void * unused __unused)1580 tcp_destroy(void *unused __unused)
1581 {
1582 #ifdef TCP_HHOOK
1583 int error;
1584 #endif
1585
1586 tcp_hc_destroy();
1587 syncache_destroy();
1588 in_pcbinfo_destroy(&V_tcbinfo);
1589 /* tcp_discardcb() clears the sack_holes up. */
1590 uma_zdestroy(V_sack_hole_zone);
1591
1592 /*
1593 * Cannot free the zone until all tcpcbs are released as we attach
1594 * the allocations to them.
1595 */
1596 tcp_fastopen_destroy();
1597
1598 COUNTER_ARRAY_FREE(V_tcps_states, TCP_NSTATES);
1599 VNET_PCPUSTAT_FREE(tcpstat);
1600
1601 #ifdef TCP_HHOOK
1602 error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]);
1603 if (error != 0) {
1604 printf("%s: WARNING: unable to deregister helper hook "
1605 "type=%d, id=%d: error %d returned\n", __func__,
1606 HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN, error);
1607 }
1608 error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_OUT]);
1609 if (error != 0) {
1610 printf("%s: WARNING: unable to deregister helper hook "
1611 "type=%d, id=%d: error %d returned\n", __func__,
1612 HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT, error);
1613 }
1614 #endif
1615 }
1616 VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_destroy, NULL);
1617 #endif
1618
1619 void
tcp_fini(void * xtp)1620 tcp_fini(void *xtp)
1621 {
1622
1623 }
1624
1625 /*
1626 * Fill in the IP and TCP headers for an outgoing packet, given the tcpcb.
1627 * tcp_template used to store this data in mbufs, but we now recopy it out
1628 * of the tcpcb each time to conserve mbufs.
1629 */
1630 void
tcpip_fillheaders(struct inpcb * inp,uint16_t port,void * ip_ptr,void * tcp_ptr)1631 tcpip_fillheaders(struct inpcb *inp, uint16_t port, void *ip_ptr, void *tcp_ptr)
1632 {
1633 struct tcphdr *th = (struct tcphdr *)tcp_ptr;
1634
1635 INP_WLOCK_ASSERT(inp);
1636
1637 #ifdef INET6
1638 if ((inp->inp_vflag & INP_IPV6) != 0) {
1639 struct ip6_hdr *ip6;
1640
1641 ip6 = (struct ip6_hdr *)ip_ptr;
1642 ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) |
1643 (inp->inp_flow & IPV6_FLOWINFO_MASK);
1644 ip6->ip6_vfc = (ip6->ip6_vfc & ~IPV6_VERSION_MASK) |
1645 (IPV6_VERSION & IPV6_VERSION_MASK);
1646 if (port == 0)
1647 ip6->ip6_nxt = IPPROTO_TCP;
1648 else
1649 ip6->ip6_nxt = IPPROTO_UDP;
1650 ip6->ip6_plen = htons(sizeof(struct tcphdr));
1651 ip6->ip6_src = inp->in6p_laddr;
1652 ip6->ip6_dst = inp->in6p_faddr;
1653 }
1654 #endif /* INET6 */
1655 #if defined(INET6) && defined(INET)
1656 else
1657 #endif
1658 #ifdef INET
1659 {
1660 struct ip *ip;
1661
1662 ip = (struct ip *)ip_ptr;
1663 ip->ip_v = IPVERSION;
1664 ip->ip_hl = 5;
1665 ip->ip_tos = inp->inp_ip_tos;
1666 ip->ip_len = 0;
1667 ip->ip_id = 0;
1668 ip->ip_off = 0;
1669 ip->ip_ttl = inp->inp_ip_ttl;
1670 ip->ip_sum = 0;
1671 if (port == 0)
1672 ip->ip_p = IPPROTO_TCP;
1673 else
1674 ip->ip_p = IPPROTO_UDP;
1675 ip->ip_src = inp->inp_laddr;
1676 ip->ip_dst = inp->inp_faddr;
1677 }
1678 #endif /* INET */
1679 th->th_sport = inp->inp_lport;
1680 th->th_dport = inp->inp_fport;
1681 th->th_seq = 0;
1682 th->th_ack = 0;
1683 th->th_off = 5;
1684 tcp_set_flags(th, 0);
1685 th->th_win = 0;
1686 th->th_urp = 0;
1687 th->th_sum = 0; /* in_pseudo() is called later for ipv4 */
1688 }
1689
1690 /*
1691 * Create template to be used to send tcp packets on a connection.
1692 * Allocates an mbuf and fills in a skeletal tcp/ip header. The only
1693 * use for this function is in keepalives, which use tcp_respond.
1694 */
1695 struct tcptemp *
tcpip_maketemplate(struct inpcb * inp)1696 tcpip_maketemplate(struct inpcb *inp)
1697 {
1698 struct tcptemp *t;
1699
1700 t = malloc(sizeof(*t), M_TEMP, M_NOWAIT);
1701 if (t == NULL)
1702 return (NULL);
1703 tcpip_fillheaders(inp, 0, (void *)&t->tt_ipgen, (void *)&t->tt_t);
1704 return (t);
1705 }
1706
1707 /*
1708 * Send a single message to the TCP at address specified by
1709 * the given TCP/IP header. If m == NULL, then we make a copy
1710 * of the tcpiphdr at th and send directly to the addressed host.
1711 * This is used to force keep alive messages out using the TCP
1712 * template for a connection. If flags are given then we send
1713 * a message back to the TCP which originated the segment th,
1714 * and discard the mbuf containing it and any other attached mbufs.
1715 *
1716 * In any case the ack and sequence number of the transmitted
1717 * segment are as specified by the parameters.
1718 *
1719 * NOTE: If m != NULL, then th must point to *inside* the mbuf.
1720 */
1721
1722 void
tcp_respond(struct tcpcb * tp,void * ipgen,struct tcphdr * th,struct mbuf * m,tcp_seq ack,tcp_seq seq,uint16_t flags)1723 tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
1724 tcp_seq ack, tcp_seq seq, uint16_t flags)
1725 {
1726 struct tcpopt to;
1727 struct inpcb *inp;
1728 struct ip *ip;
1729 struct mbuf *optm;
1730 struct udphdr *uh = NULL;
1731 struct tcphdr *nth;
1732 struct tcp_log_buffer *lgb;
1733 u_char *optp;
1734 #ifdef INET6
1735 struct ip6_hdr *ip6;
1736 int isipv6;
1737 #endif /* INET6 */
1738 int optlen, tlen, win, ulen;
1739 int ect = 0;
1740 bool incl_opts;
1741 uint16_t port;
1742 int output_ret;
1743 #ifdef INVARIANTS
1744 int thflags = tcp_get_flags(th);
1745 #endif
1746
1747 KASSERT(tp != NULL || m != NULL, ("tcp_respond: tp and m both NULL"));
1748 NET_EPOCH_ASSERT();
1749
1750 #ifdef INET6
1751 isipv6 = ((struct ip *)ipgen)->ip_v == (IPV6_VERSION >> 4);
1752 ip6 = ipgen;
1753 #endif /* INET6 */
1754 ip = ipgen;
1755
1756 if (tp != NULL) {
1757 inp = tptoinpcb(tp);
1758 INP_LOCK_ASSERT(inp);
1759 } else
1760 inp = NULL;
1761
1762 if (m != NULL) {
1763 #ifdef INET6
1764 if (isipv6 && ip6 && (ip6->ip6_nxt == IPPROTO_UDP))
1765 port = m->m_pkthdr.tcp_tun_port;
1766 else
1767 #endif
1768 if (ip && (ip->ip_p == IPPROTO_UDP))
1769 port = m->m_pkthdr.tcp_tun_port;
1770 else
1771 port = 0;
1772 } else
1773 port = tp->t_port;
1774
1775 incl_opts = false;
1776 win = 0;
1777 if (tp != NULL) {
1778 if (!(flags & TH_RST)) {
1779 win = sbspace(&inp->inp_socket->so_rcv);
1780 if (win > TCP_MAXWIN << tp->rcv_scale)
1781 win = TCP_MAXWIN << tp->rcv_scale;
1782 }
1783 if ((tp->t_flags & TF_NOOPT) == 0)
1784 incl_opts = true;
1785 }
1786 if (m == NULL) {
1787 m = m_gethdr(M_NOWAIT, MT_DATA);
1788 if (m == NULL)
1789 return;
1790 m->m_data += max_linkhdr;
1791 #ifdef INET6
1792 if (isipv6) {
1793 bcopy((caddr_t)ip6, mtod(m, caddr_t),
1794 sizeof(struct ip6_hdr));
1795 ip6 = mtod(m, struct ip6_hdr *);
1796 nth = (struct tcphdr *)(ip6 + 1);
1797 if (port) {
1798 /* Insert a UDP header */
1799 uh = (struct udphdr *)nth;
1800 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1801 uh->uh_dport = port;
1802 nth = (struct tcphdr *)(uh + 1);
1803 }
1804 } else
1805 #endif /* INET6 */
1806 {
1807 bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip));
1808 ip = mtod(m, struct ip *);
1809 nth = (struct tcphdr *)(ip + 1);
1810 if (port) {
1811 /* Insert a UDP header */
1812 uh = (struct udphdr *)nth;
1813 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1814 uh->uh_dport = port;
1815 nth = (struct tcphdr *)(uh + 1);
1816 }
1817 }
1818 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1819 flags = TH_ACK;
1820 } else if ((!M_WRITABLE(m)) || (port != 0)) {
1821 struct mbuf *n;
1822
1823 /* Can't reuse 'm', allocate a new mbuf. */
1824 n = m_gethdr(M_NOWAIT, MT_DATA);
1825 if (n == NULL) {
1826 m_freem(m);
1827 return;
1828 }
1829
1830 if (!m_dup_pkthdr(n, m, M_NOWAIT)) {
1831 m_freem(m);
1832 m_freem(n);
1833 return;
1834 }
1835
1836 n->m_data += max_linkhdr;
1837 /* m_len is set later */
1838 #define xchg(a,b,type) { type t; t=a; a=b; b=t; }
1839 #ifdef INET6
1840 if (isipv6) {
1841 bcopy((caddr_t)ip6, mtod(n, caddr_t),
1842 sizeof(struct ip6_hdr));
1843 ip6 = mtod(n, struct ip6_hdr *);
1844 xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1845 nth = (struct tcphdr *)(ip6 + 1);
1846 if (port) {
1847 /* Insert a UDP header */
1848 uh = (struct udphdr *)nth;
1849 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1850 uh->uh_dport = port;
1851 nth = (struct tcphdr *)(uh + 1);
1852 }
1853 } else
1854 #endif /* INET6 */
1855 {
1856 bcopy((caddr_t)ip, mtod(n, caddr_t), sizeof(struct ip));
1857 ip = mtod(n, struct ip *);
1858 xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1859 nth = (struct tcphdr *)(ip + 1);
1860 if (port) {
1861 /* Insert a UDP header */
1862 uh = (struct udphdr *)nth;
1863 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1864 uh->uh_dport = port;
1865 nth = (struct tcphdr *)(uh + 1);
1866 }
1867 }
1868 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1869 xchg(nth->th_dport, nth->th_sport, uint16_t);
1870 th = nth;
1871 m_freem(m);
1872 m = n;
1873 } else {
1874 /*
1875 * reuse the mbuf.
1876 * XXX MRT We inherit the FIB, which is lucky.
1877 */
1878 m_freem(m->m_next);
1879 m->m_next = NULL;
1880 m->m_data = (caddr_t)ipgen;
1881 /* clear any receive flags for proper bpf timestamping */
1882 m->m_flags &= ~(M_TSTMP | M_TSTMP_LRO);
1883 /* m_len is set later */
1884 #ifdef INET6
1885 if (isipv6) {
1886 xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1887 nth = (struct tcphdr *)(ip6 + 1);
1888 } else
1889 #endif /* INET6 */
1890 {
1891 xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1892 nth = (struct tcphdr *)(ip + 1);
1893 }
1894 if (th != nth) {
1895 /*
1896 * this is usually a case when an extension header
1897 * exists between the IPv6 header and the
1898 * TCP header.
1899 */
1900 nth->th_sport = th->th_sport;
1901 nth->th_dport = th->th_dport;
1902 }
1903 xchg(nth->th_dport, nth->th_sport, uint16_t);
1904 #undef xchg
1905 }
1906 tlen = 0;
1907 #ifdef INET6
1908 if (isipv6)
1909 tlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1910 #endif
1911 #if defined(INET) && defined(INET6)
1912 else
1913 #endif
1914 #ifdef INET
1915 tlen = sizeof (struct tcpiphdr);
1916 #endif
1917 if (port)
1918 tlen += sizeof (struct udphdr);
1919 #ifdef INVARIANTS
1920 m->m_len = 0;
1921 KASSERT(M_TRAILINGSPACE(m) >= tlen,
1922 ("Not enough trailing space for message (m=%p, need=%d, have=%ld)",
1923 m, tlen, (long)M_TRAILINGSPACE(m)));
1924 #endif
1925 m->m_len = tlen;
1926 to.to_flags = 0;
1927 if (incl_opts) {
1928 ect = tcp_ecn_output_established(tp, &flags, 0, false);
1929 /* Make sure we have room. */
1930 if (M_TRAILINGSPACE(m) < TCP_MAXOLEN) {
1931 m->m_next = m_get(M_NOWAIT, MT_DATA);
1932 if (m->m_next) {
1933 optp = mtod(m->m_next, u_char *);
1934 optm = m->m_next;
1935 } else
1936 incl_opts = false;
1937 } else {
1938 optp = (u_char *) (nth + 1);
1939 optm = m;
1940 }
1941 }
1942 if (incl_opts) {
1943 /* Timestamps. */
1944 if (tp->t_flags & TF_RCVD_TSTMP) {
1945 to.to_tsval = tcp_ts_getticks() + tp->ts_offset;
1946 to.to_tsecr = tp->ts_recent;
1947 to.to_flags |= TOF_TS;
1948 }
1949 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
1950 /* TCP-MD5 (RFC2385). */
1951 if (tp->t_flags & TF_SIGNATURE)
1952 to.to_flags |= TOF_SIGNATURE;
1953 #endif
1954 /* Add the options. */
1955 tlen += optlen = tcp_addoptions(&to, optp);
1956
1957 /* Update m_len in the correct mbuf. */
1958 optm->m_len += optlen;
1959 } else
1960 optlen = 0;
1961 #ifdef INET6
1962 if (isipv6) {
1963 if (uh) {
1964 ulen = tlen - sizeof(struct ip6_hdr);
1965 uh->uh_ulen = htons(ulen);
1966 }
1967 ip6->ip6_flow = htonl(ect << IPV6_FLOWLABEL_LEN);
1968 ip6->ip6_vfc = IPV6_VERSION;
1969 if (port)
1970 ip6->ip6_nxt = IPPROTO_UDP;
1971 else
1972 ip6->ip6_nxt = IPPROTO_TCP;
1973 ip6->ip6_plen = htons(tlen - sizeof(*ip6));
1974 }
1975 #endif
1976 #if defined(INET) && defined(INET6)
1977 else
1978 #endif
1979 #ifdef INET
1980 {
1981 if (uh) {
1982 ulen = tlen - sizeof(struct ip);
1983 uh->uh_ulen = htons(ulen);
1984 }
1985 ip->ip_len = htons(tlen);
1986 if (inp != NULL) {
1987 ip->ip_tos = inp->inp_ip_tos & ~IPTOS_ECN_MASK;
1988 ip->ip_ttl = inp->inp_ip_ttl;
1989 } else {
1990 ip->ip_tos = 0;
1991 ip->ip_ttl = V_ip_defttl;
1992 }
1993 ip->ip_tos |= ect;
1994 if (port) {
1995 ip->ip_p = IPPROTO_UDP;
1996 } else {
1997 ip->ip_p = IPPROTO_TCP;
1998 }
1999 if (V_path_mtu_discovery)
2000 ip->ip_off |= htons(IP_DF);
2001 }
2002 #endif
2003 m->m_pkthdr.len = tlen;
2004 m->m_pkthdr.rcvif = NULL;
2005 #ifdef MAC
2006 if (inp != NULL) {
2007 /*
2008 * Packet is associated with a socket, so allow the
2009 * label of the response to reflect the socket label.
2010 */
2011 INP_LOCK_ASSERT(inp);
2012 mac_inpcb_create_mbuf(inp, m);
2013 } else {
2014 /*
2015 * Packet is not associated with a socket, so possibly
2016 * update the label in place.
2017 */
2018 mac_netinet_tcp_reply(m);
2019 }
2020 #endif
2021 nth->th_seq = htonl(seq);
2022 nth->th_ack = htonl(ack);
2023 nth->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
2024 tcp_set_flags(nth, flags);
2025 if (tp && (flags & TH_RST)) {
2026 /* Log the reset */
2027 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
2028 }
2029 if (tp != NULL)
2030 nth->th_win = htons((u_short) (win >> tp->rcv_scale));
2031 else
2032 nth->th_win = htons((u_short)win);
2033 nth->th_urp = 0;
2034
2035 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2036 if (to.to_flags & TOF_SIGNATURE) {
2037 if (!TCPMD5_ENABLED() ||
2038 TCPMD5_OUTPUT(m, nth, to.to_signature) != 0) {
2039 m_freem(m);
2040 return;
2041 }
2042 }
2043 #endif
2044
2045 #ifdef INET6
2046 if (isipv6) {
2047 if (port) {
2048 m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
2049 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2050 uh->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
2051 nth->th_sum = 0;
2052 } else {
2053 m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
2054 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2055 nth->th_sum = in6_cksum_pseudo(ip6,
2056 tlen - sizeof(struct ip6_hdr), IPPROTO_TCP, 0);
2057 }
2058 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
2059 }
2060 #endif /* INET6 */
2061 #if defined(INET6) && defined(INET)
2062 else
2063 #endif
2064 #ifdef INET
2065 {
2066 if (port) {
2067 uh->uh_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2068 htons(ulen + IPPROTO_UDP));
2069 m->m_pkthdr.csum_flags = CSUM_UDP;
2070 m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2071 nth->th_sum = 0;
2072 } else {
2073 m->m_pkthdr.csum_flags = CSUM_TCP;
2074 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2075 nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2076 htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p)));
2077 }
2078 }
2079 #endif /* INET */
2080 TCP_PROBE3(debug__output, tp, th, m);
2081 if (flags & TH_RST)
2082 TCP_PROBE5(accept__refused, NULL, NULL, m, tp, nth);
2083 lgb = NULL;
2084 if ((tp != NULL) && tcp_bblogging_on(tp)) {
2085 if (INP_WLOCKED(inp)) {
2086 union tcp_log_stackspecific log;
2087 struct timeval tv;
2088
2089 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2090 log.u_bbr.inhpts = tcp_in_hpts(tp);
2091 log.u_bbr.flex8 = 4;
2092 log.u_bbr.pkts_out = tp->t_maxseg;
2093 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2094 log.u_bbr.delivered = 0;
2095 lgb = tcp_log_event(tp, nth, NULL, NULL, TCP_LOG_OUT,
2096 ERRNO_UNK, 0, &log, false, NULL, NULL, 0, &tv);
2097 } else {
2098 /*
2099 * We can not log the packet, since we only own the
2100 * read lock, but a write lock is needed. The read lock
2101 * is not upgraded to a write lock, since only getting
2102 * the read lock was done intentionally to improve the
2103 * handling of SYN flooding attacks.
2104 * This happens only for pure SYN segments received in
2105 * the initial CLOSED state, or received in a more
2106 * advanced state than listen and the UDP encapsulation
2107 * port is unexpected.
2108 * The incoming SYN segments do not really belong to
2109 * the TCP connection and the handling does not change
2110 * the state of the TCP connection. Therefore, the
2111 * sending of the RST segments is not logged. Please
2112 * note that also the incoming SYN segments are not
2113 * logged.
2114 *
2115 * The following code ensures that the above description
2116 * is and stays correct.
2117 */
2118 KASSERT((thflags & (TH_ACK|TH_SYN)) == TH_SYN &&
2119 (tp->t_state == TCPS_CLOSED ||
2120 (tp->t_state > TCPS_LISTEN && tp->t_port != port)),
2121 ("%s: Logging of TCP segment with flags 0x%b and "
2122 "UDP encapsulation port %u skipped in state %s",
2123 __func__, thflags, PRINT_TH_FLAGS,
2124 ntohs(port), tcpstates[tp->t_state]));
2125 }
2126 }
2127
2128 if (flags & TH_ACK)
2129 TCPSTAT_INC(tcps_sndacks);
2130 else if (flags & (TH_SYN|TH_FIN|TH_RST))
2131 TCPSTAT_INC(tcps_sndctrl);
2132 TCPSTAT_INC(tcps_sndtotal);
2133
2134 #ifdef INET6
2135 if (isipv6) {
2136 TCP_PROBE5(send, NULL, tp, ip6, tp, nth);
2137 output_ret = ip6_output(m, inp ? inp->in6p_outputopts : NULL,
2138 NULL, 0, NULL, NULL, inp);
2139 }
2140 #endif /* INET6 */
2141 #if defined(INET) && defined(INET6)
2142 else
2143 #endif
2144 #ifdef INET
2145 {
2146 TCP_PROBE5(send, NULL, tp, ip, tp, nth);
2147 output_ret = ip_output(m, NULL, NULL, 0, NULL, inp);
2148 }
2149 #endif
2150 if (lgb != NULL)
2151 lgb->tlb_errno = output_ret;
2152 }
2153
2154 /*
2155 * Send a challenge ack (no data, no SACK option), but not more than
2156 * V_tcp_ack_war_cnt per V_tcp_ack_war_time_window (per TCP connection).
2157 */
2158 void
tcp_send_challenge_ack(struct tcpcb * tp,struct tcphdr * th,struct mbuf * m)2159 tcp_send_challenge_ack(struct tcpcb *tp, struct tcphdr *th, struct mbuf *m)
2160 {
2161 sbintime_t now;
2162 bool send_challenge_ack;
2163
2164 if (V_tcp_ack_war_time_window == 0 || V_tcp_ack_war_cnt == 0) {
2165 /* ACK war protection is disabled. */
2166 send_challenge_ack = true;
2167 } else {
2168 /* Start new epoch, if the previous one is already over. */
2169 now = getsbinuptime();
2170 if (tp->t_challenge_ack_end < now) {
2171 tp->t_challenge_ack_cnt = 0;
2172 tp->t_challenge_ack_end = now +
2173 V_tcp_ack_war_time_window * SBT_1MS;
2174 }
2175 /*
2176 * Send a challenge ACK, if less than tcp_ack_war_cnt have been
2177 * sent in the current epoch.
2178 */
2179 if (tp->t_challenge_ack_cnt < V_tcp_ack_war_cnt) {
2180 send_challenge_ack = true;
2181 tp->t_challenge_ack_cnt++;
2182 } else {
2183 send_challenge_ack = false;
2184 }
2185 }
2186 if (send_challenge_ack) {
2187 tcp_respond(tp, mtod(m, void *), th, m, tp->rcv_nxt,
2188 tp->snd_nxt, TH_ACK);
2189 tp->last_ack_sent = tp->rcv_nxt;
2190 }
2191 }
2192
2193 /*
2194 * Create a new TCP control block, making an empty reassembly queue and hooking
2195 * it to the argument protocol control block. The `inp' parameter must have
2196 * come from the zone allocator set up by tcpcbstor declaration.
2197 * The caller can provide a pointer to a tcpcb of the listener to inherit the
2198 * TCP function block from the listener.
2199 */
2200 struct tcpcb *
tcp_newtcpcb(struct inpcb * inp,struct tcpcb * listening_tcb)2201 tcp_newtcpcb(struct inpcb *inp, struct tcpcb *listening_tcb)
2202 {
2203 struct tcpcb *tp = intotcpcb(inp);
2204 #ifdef INET6
2205 int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2206 #endif /* INET6 */
2207
2208 /*
2209 * Historically allocation was done with M_ZERO. There is a lot of
2210 * code that rely on that. For now take safe approach and zero whole
2211 * tcpcb. This definitely can be optimized.
2212 */
2213 bzero(&tp->t_start_zero, t_zero_size);
2214
2215 /* Initialise cc_var struct for this tcpcb. */
2216 tp->t_ccv.tp = tp;
2217 rw_rlock(&tcp_function_lock);
2218 if (listening_tcb != NULL) {
2219 INP_LOCK_ASSERT(tptoinpcb(listening_tcb));
2220 KASSERT(listening_tcb->t_fb != NULL,
2221 ("tcp_newtcpcb: listening_tcb->t_fb is NULL"));
2222 if (listening_tcb->t_fb->tfb_flags & TCP_FUNC_BEING_REMOVED) {
2223 rw_runlock(&tcp_function_lock);
2224 return (NULL);
2225 }
2226 tp->t_fb = listening_tcb->t_fb;
2227 } else {
2228 tp->t_fb = V_tcp_func_set_ptr;
2229 }
2230 refcount_acquire(&tp->t_fb->tfb_refcnt);
2231 KASSERT((tp->t_fb->tfb_flags & TCP_FUNC_BEING_REMOVED) == 0,
2232 ("tcp_newtcpcb: using TFB being removed"));
2233 rw_runlock(&tcp_function_lock);
2234 CC_LIST_RLOCK();
2235 if (listening_tcb != NULL) {
2236 if (CC_ALGO(listening_tcb)->flags & CC_MODULE_BEING_REMOVED) {
2237 CC_LIST_RUNLOCK();
2238 if (tp->t_fb->tfb_tcp_fb_fini)
2239 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2240 refcount_release(&tp->t_fb->tfb_refcnt);
2241 return (NULL);
2242 }
2243 CC_ALGO(tp) = CC_ALGO(listening_tcb);
2244 } else
2245 CC_ALGO(tp) = CC_DEFAULT_ALGO();
2246 cc_refer(CC_ALGO(tp));
2247 CC_LIST_RUNLOCK();
2248 if (CC_ALGO(tp)->cb_init != NULL)
2249 if (CC_ALGO(tp)->cb_init(&tp->t_ccv, NULL) > 0) {
2250 cc_detach(tp);
2251 if (tp->t_fb->tfb_tcp_fb_fini)
2252 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2253 refcount_release(&tp->t_fb->tfb_refcnt);
2254 return (NULL);
2255 }
2256
2257 #ifdef TCP_HHOOK
2258 if (khelp_init_osd(HELPER_CLASS_TCP, &tp->t_osd)) {
2259 if (CC_ALGO(tp)->cb_destroy != NULL)
2260 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2261 CC_DATA(tp) = NULL;
2262 cc_detach(tp);
2263 if (tp->t_fb->tfb_tcp_fb_fini)
2264 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2265 refcount_release(&tp->t_fb->tfb_refcnt);
2266 return (NULL);
2267 }
2268 #endif
2269
2270 TAILQ_INIT(&tp->t_segq);
2271 STAILQ_INIT(&tp->t_inqueue);
2272 tp->t_maxseg =
2273 #ifdef INET6
2274 isipv6 ? V_tcp_v6mssdflt :
2275 #endif /* INET6 */
2276 V_tcp_mssdflt;
2277
2278 /* All mbuf queue/ack compress flags should be off */
2279 tcp_lro_features_off(tp);
2280
2281 tp->t_hpts_cpu = HPTS_CPU_NONE;
2282 tp->t_lro_cpu = HPTS_CPU_NONE;
2283
2284 callout_init_rw(&tp->t_callout, &inp->inp_lock,
2285 CALLOUT_TRYLOCK | CALLOUT_RETURNUNLOCKED);
2286 for (int i = 0; i < TT_N; i++)
2287 tp->t_timers[i] = SBT_MAX;
2288
2289 switch (V_tcp_do_rfc1323) {
2290 case 0:
2291 break;
2292 default:
2293 case 1:
2294 tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
2295 break;
2296 case 2:
2297 tp->t_flags = TF_REQ_SCALE;
2298 break;
2299 case 3:
2300 tp->t_flags = TF_REQ_TSTMP;
2301 break;
2302 }
2303 if (V_tcp_do_sack)
2304 tp->t_flags |= TF_SACK_PERMIT;
2305 TAILQ_INIT(&tp->snd_holes);
2306
2307 /*
2308 * Init srtt to TCPTV_SRTTBASE (0), so we can tell that we have no
2309 * rtt estimate. Set rttvar so that srtt + 4 * rttvar gives
2310 * reasonable initial retransmit time.
2311 */
2312 tp->t_srtt = TCPTV_SRTTBASE;
2313 tp->t_rttvar = ((tcp_rexmit_initial - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
2314 tp->t_rttmin = tcp_rexmit_min;
2315 tp->t_rxtcur = tcp_rexmit_initial;
2316 tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2317 tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2318 tp->t_rcvtime = ticks;
2319 /* We always start with ticks granularity */
2320 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
2321 /*
2322 * IPv4 TTL initialization is necessary for an IPv6 socket as well,
2323 * because the socket may be bound to an IPv6 wildcard address,
2324 * which may match an IPv4-mapped IPv6 address.
2325 */
2326 inp->inp_ip_ttl = V_ip_defttl;
2327 #ifdef TCP_BLACKBOX
2328 /* Initialize the per-TCPCB log data. */
2329 tcp_log_tcpcbinit(tp);
2330 #endif
2331 tp->t_pacing_rate = -1;
2332 if (tp->t_fb->tfb_tcp_fb_init) {
2333 if ((*tp->t_fb->tfb_tcp_fb_init)(tp, &tp->t_fb_ptr)) {
2334 if (CC_ALGO(tp)->cb_destroy != NULL)
2335 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2336 CC_DATA(tp) = NULL;
2337 cc_detach(tp);
2338 #ifdef TCP_HHOOK
2339 khelp_destroy_osd(&tp->t_osd);
2340 #endif
2341 refcount_release(&tp->t_fb->tfb_refcnt);
2342 return (NULL);
2343 }
2344 }
2345 #ifdef STATS
2346 if (V_tcp_perconn_stats_enable == 1)
2347 tp->t_stats = stats_blob_alloc(V_tcp_perconn_stats_dflt_tpl, 0);
2348 #endif
2349 if (V_tcp_do_lrd)
2350 tp->t_flags |= TF_LRD;
2351
2352 return (tp);
2353 }
2354
2355 /*
2356 * Drop a TCP connection, reporting
2357 * the specified error. If connection is synchronized,
2358 * then send a RST to peer.
2359 */
2360 struct tcpcb *
tcp_drop(struct tcpcb * tp,int errno)2361 tcp_drop(struct tcpcb *tp, int errno)
2362 {
2363 struct socket *so = tptosocket(tp);
2364
2365 NET_EPOCH_ASSERT();
2366 INP_WLOCK_ASSERT(tptoinpcb(tp));
2367
2368 if (TCPS_HAVERCVDSYN(tp->t_state)) {
2369 tcp_state_change(tp, TCPS_CLOSED);
2370 /* Don't use tcp_output() here due to possible recursion. */
2371 (void)tcp_output_nodrop(tp);
2372 TCPSTAT_INC(tcps_drops);
2373 } else
2374 TCPSTAT_INC(tcps_conndrops);
2375 if (errno == ETIMEDOUT && tp->t_softerror)
2376 errno = tp->t_softerror;
2377 so->so_error = errno;
2378 return (tcp_close(tp));
2379 }
2380
2381 void
tcp_discardcb(struct tcpcb * tp)2382 tcp_discardcb(struct tcpcb *tp)
2383 {
2384 struct inpcb *inp = tptoinpcb(tp);
2385 struct socket *so = tptosocket(tp);
2386 struct mbuf *m;
2387 #ifdef INET6
2388 bool isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2389 #endif
2390
2391 INP_WLOCK_ASSERT(inp);
2392 MPASS(!callout_active(&tp->t_callout));
2393 MPASS(TAILQ_EMPTY(&tp->snd_holes));
2394
2395 /* free the reassembly queue, if any */
2396 tcp_reass_flush(tp);
2397
2398 #ifdef TCP_OFFLOAD
2399 /* Disconnect offload device, if any. */
2400 if (tp->t_flags & TF_TOE)
2401 tcp_offload_detach(tp);
2402 #endif
2403
2404 /* Allow the CC algorithm to clean up after itself. */
2405 if (CC_ALGO(tp)->cb_destroy != NULL)
2406 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2407 CC_DATA(tp) = NULL;
2408 /* Detach from the CC algorithm */
2409 cc_detach(tp);
2410
2411 #ifdef TCP_HHOOK
2412 khelp_destroy_osd(&tp->t_osd);
2413 #endif
2414 #ifdef STATS
2415 stats_blob_destroy(tp->t_stats);
2416 #endif
2417
2418 CC_ALGO(tp) = NULL;
2419 if ((m = STAILQ_FIRST(&tp->t_inqueue)) != NULL) {
2420 struct mbuf *prev;
2421
2422 STAILQ_INIT(&tp->t_inqueue);
2423 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, prev)
2424 m_freem(m);
2425 }
2426 TCPSTATES_DEC(tp->t_state);
2427
2428 if (tp->t_fb->tfb_tcp_fb_fini)
2429 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2430 MPASS(!tcp_in_hpts(tp));
2431 #ifdef TCP_BLACKBOX
2432 tcp_log_tcpcbfini(tp);
2433 #endif
2434
2435 /*
2436 * If we got enough samples through the srtt filter,
2437 * save the rtt and rttvar in the routing entry.
2438 * 'Enough' is arbitrarily defined as 4 rtt samples.
2439 * 4 samples is enough for the srtt filter to converge
2440 * to within enough % of the correct value; fewer samples
2441 * and we could save a bogus rtt. The danger is not high
2442 * as tcp quickly recovers from everything.
2443 * XXX: Works very well but needs some more statistics!
2444 *
2445 * XXXRRS: Updating must be after the stack fini() since
2446 * that may be converting some internal representation of
2447 * say srtt etc into the general one used by other stacks.
2448 */
2449 if (tp->t_rttupdated >= 4) {
2450 struct hc_metrics_lite metrics;
2451 uint32_t ssthresh;
2452
2453 bzero(&metrics, sizeof(metrics));
2454 /*
2455 * Update the ssthresh always when the conditions below
2456 * are satisfied. This gives us better new start value
2457 * for the congestion avoidance for new connections.
2458 * ssthresh is only set if packet loss occurred on a session.
2459 */
2460 ssthresh = tp->snd_ssthresh;
2461 if (ssthresh != 0 && ssthresh < so->so_snd.sb_hiwat / 2) {
2462 /*
2463 * convert the limit from user data bytes to
2464 * packets then to packet data bytes.
2465 */
2466 ssthresh = (ssthresh + tp->t_maxseg / 2) / tp->t_maxseg;
2467 if (ssthresh < 2)
2468 ssthresh = 2;
2469 ssthresh *= (tp->t_maxseg +
2470 #ifdef INET6
2471 (isipv6 ? sizeof (struct ip6_hdr) +
2472 sizeof (struct tcphdr) :
2473 #endif
2474 sizeof (struct tcpiphdr)
2475 #ifdef INET6
2476 )
2477 #endif
2478 );
2479 } else
2480 ssthresh = 0;
2481 metrics.hc_ssthresh = ssthresh;
2482
2483 metrics.hc_rtt = tp->t_srtt;
2484 metrics.hc_rttvar = tp->t_rttvar;
2485 metrics.hc_cwnd = tp->snd_cwnd;
2486 metrics.hc_sendpipe = 0;
2487 metrics.hc_recvpipe = 0;
2488
2489 tcp_hc_update(&inp->inp_inc, &metrics);
2490 }
2491
2492 refcount_release(&tp->t_fb->tfb_refcnt);
2493 }
2494
2495 /*
2496 * Attempt to close a TCP control block, marking it as dropped, and freeing
2497 * the socket if we hold the only reference.
2498 */
2499 struct tcpcb *
tcp_close(struct tcpcb * tp)2500 tcp_close(struct tcpcb *tp)
2501 {
2502 struct inpcb *inp = tptoinpcb(tp);
2503 struct socket *so = tptosocket(tp);
2504
2505 INP_WLOCK_ASSERT(inp);
2506
2507 #ifdef TCP_OFFLOAD
2508 if (tp->t_state == TCPS_LISTEN)
2509 tcp_offload_listen_stop(tp);
2510 #endif
2511 /*
2512 * This releases the TFO pending counter resource for TFO listen
2513 * sockets as well as passively-created TFO sockets that transition
2514 * from SYN_RECEIVED to CLOSED.
2515 */
2516 if (tp->t_tfo_pending) {
2517 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
2518 tp->t_tfo_pending = NULL;
2519 }
2520 tcp_timer_stop(tp);
2521 if (tp->t_fb->tfb_tcp_timer_stop_all != NULL)
2522 tp->t_fb->tfb_tcp_timer_stop_all(tp);
2523 in_pcbdrop(inp);
2524 TCPSTAT_INC(tcps_closed);
2525 if (tp->t_state != TCPS_CLOSED)
2526 tcp_state_change(tp, TCPS_CLOSED);
2527 KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
2528 tcp_free_sackholes(tp);
2529 soisdisconnected(so);
2530 if (inp->inp_flags & INP_SOCKREF) {
2531 inp->inp_flags &= ~INP_SOCKREF;
2532 INP_WUNLOCK(inp);
2533 sorele(so);
2534 return (NULL);
2535 }
2536 return (tp);
2537 }
2538
2539 /*
2540 * Notify a tcp user of an asynchronous error;
2541 * store error as soft error, but wake up user
2542 * (for now, won't do anything until can select for soft error).
2543 *
2544 * Do not wake up user since there currently is no mechanism for
2545 * reporting soft errors (yet - a kqueue filter may be added).
2546 */
2547 static struct inpcb *
tcp_notify(struct inpcb * inp,int error)2548 tcp_notify(struct inpcb *inp, int error)
2549 {
2550 struct tcpcb *tp;
2551
2552 INP_WLOCK_ASSERT(inp);
2553
2554 tp = intotcpcb(inp);
2555 KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
2556
2557 /*
2558 * Ignore some errors if we are hooked up.
2559 * If connection hasn't completed, has retransmitted several times,
2560 * and receives a second error, give up now. This is better
2561 * than waiting a long time to establish a connection that
2562 * can never complete.
2563 */
2564 if (tp->t_state == TCPS_ESTABLISHED &&
2565 (error == EHOSTUNREACH || error == ENETUNREACH ||
2566 error == EHOSTDOWN)) {
2567 if (inp->inp_route.ro_nh) {
2568 NH_FREE(inp->inp_route.ro_nh);
2569 inp->inp_route.ro_nh = (struct nhop_object *)NULL;
2570 }
2571 return (inp);
2572 } else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
2573 tp->t_softerror) {
2574 tp = tcp_drop(tp, error);
2575 if (tp != NULL)
2576 return (inp);
2577 else
2578 return (NULL);
2579 } else {
2580 tp->t_softerror = error;
2581 return (inp);
2582 }
2583 #if 0
2584 wakeup( &so->so_timeo);
2585 sorwakeup(so);
2586 sowwakeup(so);
2587 #endif
2588 }
2589
2590 static int
tcp_pcblist(SYSCTL_HANDLER_ARGS)2591 tcp_pcblist(SYSCTL_HANDLER_ARGS)
2592 {
2593 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
2594 INPLOOKUP_RLOCKPCB);
2595 struct xinpgen xig;
2596 struct inpcb *inp;
2597 int error;
2598
2599 if (req->newptr != NULL)
2600 return (EPERM);
2601
2602 if (req->oldptr == NULL) {
2603 int n;
2604
2605 n = V_tcbinfo.ipi_count +
2606 counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2607 n += imax(n / 8, 10);
2608 req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
2609 return (0);
2610 }
2611
2612 if ((error = sysctl_wire_old_buffer(req, 0)) != 0)
2613 return (error);
2614
2615 bzero(&xig, sizeof(xig));
2616 xig.xig_len = sizeof xig;
2617 xig.xig_count = V_tcbinfo.ipi_count +
2618 counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2619 xig.xig_gen = V_tcbinfo.ipi_gencnt;
2620 xig.xig_sogen = so_gencnt;
2621 error = SYSCTL_OUT(req, &xig, sizeof xig);
2622 if (error)
2623 return (error);
2624
2625 error = syncache_pcblist(req);
2626 if (error)
2627 return (error);
2628
2629 while ((inp = inp_next(&inpi)) != NULL) {
2630 if (inp->inp_gencnt <= xig.xig_gen &&
2631 cr_canseeinpcb(req->td->td_ucred, inp) == 0) {
2632 struct xtcpcb xt;
2633
2634 tcp_inptoxtp(inp, &xt);
2635 error = SYSCTL_OUT(req, &xt, sizeof xt);
2636 if (error) {
2637 INP_RUNLOCK(inp);
2638 break;
2639 } else
2640 continue;
2641 }
2642 }
2643
2644 if (!error) {
2645 /*
2646 * Give the user an updated idea of our state.
2647 * If the generation differs from what we told
2648 * her before, she knows that something happened
2649 * while we were processing this request, and it
2650 * might be necessary to retry.
2651 */
2652 xig.xig_gen = V_tcbinfo.ipi_gencnt;
2653 xig.xig_sogen = so_gencnt;
2654 xig.xig_count = V_tcbinfo.ipi_count +
2655 counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2656 error = SYSCTL_OUT(req, &xig, sizeof xig);
2657 }
2658
2659 return (error);
2660 }
2661
2662 SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
2663 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
2664 NULL, 0, tcp_pcblist, "S,xtcpcb",
2665 "List of active TCP connections");
2666
2667 #ifdef INET
2668 static int
tcp_getcred(SYSCTL_HANDLER_ARGS)2669 tcp_getcred(SYSCTL_HANDLER_ARGS)
2670 {
2671 struct xucred xuc;
2672 struct sockaddr_in addrs[2];
2673 struct epoch_tracker et;
2674 struct inpcb *inp;
2675 int error;
2676
2677 if (req->newptr == NULL)
2678 return (EINVAL);
2679 error = priv_check(req->td, PRIV_NETINET_GETCRED);
2680 if (error)
2681 return (error);
2682 error = SYSCTL_IN(req, addrs, sizeof(addrs));
2683 if (error)
2684 return (error);
2685 NET_EPOCH_ENTER(et);
2686 inp = in_pcblookup(&V_tcbinfo, addrs[1].sin_addr, addrs[1].sin_port,
2687 addrs[0].sin_addr, addrs[0].sin_port, INPLOOKUP_RLOCKPCB, NULL);
2688 NET_EPOCH_EXIT(et);
2689 if (inp != NULL) {
2690 if (error == 0)
2691 error = cr_canseeinpcb(req->td->td_ucred, inp);
2692 if (error == 0)
2693 cru2x(inp->inp_cred, &xuc);
2694 INP_RUNLOCK(inp);
2695 } else
2696 error = ENOENT;
2697 if (error == 0)
2698 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2699 return (error);
2700 }
2701
2702 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, getcred,
2703 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2704 0, 0, tcp_getcred, "S,xucred",
2705 "Get the xucred of a TCP connection");
2706 #endif /* INET */
2707
2708 #ifdef INET6
2709 static int
tcp6_getcred(SYSCTL_HANDLER_ARGS)2710 tcp6_getcred(SYSCTL_HANDLER_ARGS)
2711 {
2712 struct epoch_tracker et;
2713 struct xucred xuc;
2714 struct sockaddr_in6 addrs[2];
2715 struct inpcb *inp;
2716 int error;
2717 #ifdef INET
2718 int mapped = 0;
2719 #endif
2720
2721 if (req->newptr == NULL)
2722 return (EINVAL);
2723 error = priv_check(req->td, PRIV_NETINET_GETCRED);
2724 if (error)
2725 return (error);
2726 error = SYSCTL_IN(req, addrs, sizeof(addrs));
2727 if (error)
2728 return (error);
2729 if ((error = sa6_embedscope(&addrs[0], V_ip6_use_defzone)) != 0 ||
2730 (error = sa6_embedscope(&addrs[1], V_ip6_use_defzone)) != 0) {
2731 return (error);
2732 }
2733 if (IN6_IS_ADDR_V4MAPPED(&addrs[0].sin6_addr)) {
2734 #ifdef INET
2735 if (IN6_IS_ADDR_V4MAPPED(&addrs[1].sin6_addr))
2736 mapped = 1;
2737 else
2738 #endif
2739 return (EINVAL);
2740 }
2741
2742 NET_EPOCH_ENTER(et);
2743 #ifdef INET
2744 if (mapped == 1)
2745 inp = in_pcblookup(&V_tcbinfo,
2746 *(struct in_addr *)&addrs[1].sin6_addr.s6_addr[12],
2747 addrs[1].sin6_port,
2748 *(struct in_addr *)&addrs[0].sin6_addr.s6_addr[12],
2749 addrs[0].sin6_port, INPLOOKUP_RLOCKPCB, NULL);
2750 else
2751 #endif
2752 inp = in6_pcblookup(&V_tcbinfo,
2753 &addrs[1].sin6_addr, addrs[1].sin6_port,
2754 &addrs[0].sin6_addr, addrs[0].sin6_port,
2755 INPLOOKUP_RLOCKPCB, NULL);
2756 NET_EPOCH_EXIT(et);
2757 if (inp != NULL) {
2758 if (error == 0)
2759 error = cr_canseeinpcb(req->td->td_ucred, inp);
2760 if (error == 0)
2761 cru2x(inp->inp_cred, &xuc);
2762 INP_RUNLOCK(inp);
2763 } else
2764 error = ENOENT;
2765 if (error == 0)
2766 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2767 return (error);
2768 }
2769
2770 SYSCTL_PROC(_net_inet6_tcp6, OID_AUTO, getcred,
2771 CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2772 0, 0, tcp6_getcred, "S,xucred",
2773 "Get the xucred of a TCP6 connection");
2774 #endif /* INET6 */
2775
2776 #ifdef INET
2777 /* Path MTU to try next when a fragmentation-needed message is received. */
2778 static inline int
tcp_next_pmtu(const struct icmp * icp,const struct ip * ip)2779 tcp_next_pmtu(const struct icmp *icp, const struct ip *ip)
2780 {
2781 int mtu = ntohs(icp->icmp_nextmtu);
2782
2783 /* If no alternative MTU was proposed, try the next smaller one. */
2784 if (!mtu)
2785 mtu = ip_next_mtu(ntohs(ip->ip_len), 1);
2786 if (mtu < V_tcp_minmss + sizeof(struct tcpiphdr))
2787 mtu = V_tcp_minmss + sizeof(struct tcpiphdr);
2788
2789 return (mtu);
2790 }
2791
2792 static void
tcp_ctlinput_with_port(struct icmp * icp,uint16_t port)2793 tcp_ctlinput_with_port(struct icmp *icp, uint16_t port)
2794 {
2795 struct ip *ip;
2796 struct tcphdr *th;
2797 struct inpcb *inp;
2798 struct tcpcb *tp;
2799 struct inpcb *(*notify)(struct inpcb *, int);
2800 struct in_conninfo inc;
2801 tcp_seq icmp_tcp_seq;
2802 int errno, mtu;
2803
2804 errno = icmp_errmap(icp);
2805 switch (errno) {
2806 case 0:
2807 return;
2808 case EMSGSIZE:
2809 notify = tcp_mtudisc_notify;
2810 break;
2811 case ECONNREFUSED:
2812 if (V_icmp_may_rst)
2813 notify = tcp_drop_syn_sent;
2814 else
2815 notify = tcp_notify;
2816 break;
2817 case EHOSTUNREACH:
2818 if (V_icmp_may_rst && icp->icmp_type == ICMP_TIMXCEED)
2819 notify = tcp_drop_syn_sent;
2820 else
2821 notify = tcp_notify;
2822 break;
2823 default:
2824 notify = tcp_notify;
2825 }
2826
2827 ip = &icp->icmp_ip;
2828 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2829 icmp_tcp_seq = th->th_seq;
2830 inp = in_pcblookup(&V_tcbinfo, ip->ip_dst, th->th_dport, ip->ip_src,
2831 th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
2832 if (inp != NULL) {
2833 tp = intotcpcb(inp);
2834 #ifdef TCP_OFFLOAD
2835 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
2836 /*
2837 * MTU discovery for offloaded connections. Let
2838 * the TOE driver verify seq# and process it.
2839 */
2840 mtu = tcp_next_pmtu(icp, ip);
2841 tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
2842 goto out;
2843 }
2844 #endif
2845 if (tp->t_port != port)
2846 goto out;
2847 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
2848 SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
2849 if (errno == EMSGSIZE) {
2850 /*
2851 * MTU discovery: we got a needfrag and
2852 * will potentially try a lower MTU.
2853 */
2854 mtu = tcp_next_pmtu(icp, ip);
2855
2856 /*
2857 * Only process the offered MTU if it
2858 * is smaller than the current one.
2859 */
2860 if (mtu < tp->t_maxseg +
2861 sizeof(struct tcpiphdr)) {
2862 bzero(&inc, sizeof(inc));
2863 inc.inc_faddr = ip->ip_dst;
2864 inc.inc_fibnum =
2865 inp->inp_inc.inc_fibnum;
2866 tcp_hc_updatemtu(&inc, mtu);
2867 inp = tcp_mtudisc(inp, mtu);
2868 }
2869 } else
2870 inp = (*notify)(inp, errno);
2871 }
2872 } else {
2873 bzero(&inc, sizeof(inc));
2874 inc.inc_fport = th->th_dport;
2875 inc.inc_lport = th->th_sport;
2876 inc.inc_faddr = ip->ip_dst;
2877 inc.inc_laddr = ip->ip_src;
2878 syncache_unreach(&inc, icmp_tcp_seq, port);
2879 }
2880 out:
2881 if (inp != NULL)
2882 INP_WUNLOCK(inp);
2883 }
2884
2885 static void
tcp_ctlinput(struct icmp * icmp)2886 tcp_ctlinput(struct icmp *icmp)
2887 {
2888 tcp_ctlinput_with_port(icmp, htons(0));
2889 }
2890
2891 static void
tcp_ctlinput_viaudp(udp_tun_icmp_param_t param)2892 tcp_ctlinput_viaudp(udp_tun_icmp_param_t param)
2893 {
2894 /* Its a tunneled TCP over UDP icmp */
2895 struct icmp *icmp = param.icmp;
2896 struct ip *outer_ip, *inner_ip;
2897 struct udphdr *udp;
2898 struct tcphdr *th, ttemp;
2899 int i_hlen, o_len;
2900 uint16_t port;
2901
2902 outer_ip = (struct ip *)((caddr_t)icmp - sizeof(struct ip));
2903 inner_ip = &icmp->icmp_ip;
2904 i_hlen = inner_ip->ip_hl << 2;
2905 o_len = ntohs(outer_ip->ip_len);
2906 if (o_len <
2907 (sizeof(struct ip) + 8 + i_hlen + sizeof(struct udphdr) + offsetof(struct tcphdr, th_ack))) {
2908 /* Not enough data present */
2909 return;
2910 }
2911 /* Ok lets strip out the inner udphdr header by copying up on top of it the tcp hdr */
2912 udp = (struct udphdr *)(((caddr_t)inner_ip) + i_hlen);
2913 if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
2914 return;
2915 }
2916 port = udp->uh_dport;
2917 th = (struct tcphdr *)(udp + 1);
2918 memcpy(&ttemp, th, sizeof(struct tcphdr));
2919 memcpy(udp, &ttemp, sizeof(struct tcphdr));
2920 /* Now adjust down the size of the outer IP header */
2921 o_len -= sizeof(struct udphdr);
2922 outer_ip->ip_len = htons(o_len);
2923 /* Now call in to the normal handling code */
2924 tcp_ctlinput_with_port(icmp, port);
2925 }
2926 #endif /* INET */
2927
2928 #ifdef INET6
2929 static inline int
tcp6_next_pmtu(const struct icmp6_hdr * icmp6)2930 tcp6_next_pmtu(const struct icmp6_hdr *icmp6)
2931 {
2932 int mtu = ntohl(icmp6->icmp6_mtu);
2933
2934 /*
2935 * If no alternative MTU was proposed, or the proposed MTU was too
2936 * small, set to the min.
2937 */
2938 if (mtu < IPV6_MMTU)
2939 mtu = IPV6_MMTU - 8; /* XXXNP: what is the adjustment for? */
2940 return (mtu);
2941 }
2942
2943 static void
tcp6_ctlinput_with_port(struct ip6ctlparam * ip6cp,uint16_t port)2944 tcp6_ctlinput_with_port(struct ip6ctlparam *ip6cp, uint16_t port)
2945 {
2946 struct in6_addr *dst;
2947 struct inpcb *(*notify)(struct inpcb *, int);
2948 struct ip6_hdr *ip6;
2949 struct mbuf *m;
2950 struct inpcb *inp;
2951 struct tcpcb *tp;
2952 struct icmp6_hdr *icmp6;
2953 struct in_conninfo inc;
2954 struct tcp_ports {
2955 uint16_t th_sport;
2956 uint16_t th_dport;
2957 } t_ports;
2958 tcp_seq icmp_tcp_seq;
2959 unsigned int mtu;
2960 unsigned int off;
2961 int errno;
2962
2963 icmp6 = ip6cp->ip6c_icmp6;
2964 m = ip6cp->ip6c_m;
2965 ip6 = ip6cp->ip6c_ip6;
2966 off = ip6cp->ip6c_off;
2967 dst = &ip6cp->ip6c_finaldst->sin6_addr;
2968
2969 errno = icmp6_errmap(icmp6);
2970 switch (errno) {
2971 case 0:
2972 return;
2973 case EMSGSIZE:
2974 notify = tcp_mtudisc_notify;
2975 break;
2976 case ECONNREFUSED:
2977 if (V_icmp_may_rst)
2978 notify = tcp_drop_syn_sent;
2979 else
2980 notify = tcp_notify;
2981 break;
2982 case EHOSTUNREACH:
2983 /*
2984 * There are only four ICMPs that may reset connection:
2985 * - administratively prohibited
2986 * - port unreachable
2987 * - time exceeded in transit
2988 * - unknown next header
2989 */
2990 if (V_icmp_may_rst &&
2991 ((icmp6->icmp6_type == ICMP6_DST_UNREACH &&
2992 (icmp6->icmp6_code == ICMP6_DST_UNREACH_ADMIN ||
2993 icmp6->icmp6_code == ICMP6_DST_UNREACH_NOPORT)) ||
2994 (icmp6->icmp6_type == ICMP6_TIME_EXCEEDED &&
2995 icmp6->icmp6_code == ICMP6_TIME_EXCEED_TRANSIT) ||
2996 (icmp6->icmp6_type == ICMP6_PARAM_PROB &&
2997 icmp6->icmp6_code == ICMP6_PARAMPROB_NEXTHEADER)))
2998 notify = tcp_drop_syn_sent;
2999 else
3000 notify = tcp_notify;
3001 break;
3002 default:
3003 notify = tcp_notify;
3004 }
3005
3006 /* Check if we can safely get the ports from the tcp hdr */
3007 if (m == NULL ||
3008 (m->m_pkthdr.len <
3009 (int32_t) (off + sizeof(struct tcp_ports)))) {
3010 return;
3011 }
3012 bzero(&t_ports, sizeof(struct tcp_ports));
3013 m_copydata(m, off, sizeof(struct tcp_ports), (caddr_t)&t_ports);
3014 inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_dst, t_ports.th_dport,
3015 &ip6->ip6_src, t_ports.th_sport, INPLOOKUP_WLOCKPCB, NULL);
3016 off += sizeof(struct tcp_ports);
3017 if (m->m_pkthdr.len < (int32_t) (off + sizeof(tcp_seq))) {
3018 goto out;
3019 }
3020 m_copydata(m, off, sizeof(tcp_seq), (caddr_t)&icmp_tcp_seq);
3021 if (inp != NULL) {
3022 tp = intotcpcb(inp);
3023 #ifdef TCP_OFFLOAD
3024 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
3025 /* MTU discovery for offloaded connections. */
3026 mtu = tcp6_next_pmtu(icmp6);
3027 tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
3028 goto out;
3029 }
3030 #endif
3031 if (tp->t_port != port)
3032 goto out;
3033 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
3034 SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
3035 if (errno == EMSGSIZE) {
3036 /*
3037 * MTU discovery:
3038 * If we got a needfrag set the MTU
3039 * in the route to the suggested new
3040 * value (if given) and then notify.
3041 */
3042 mtu = tcp6_next_pmtu(icmp6);
3043
3044 bzero(&inc, sizeof(inc));
3045 inc.inc_fibnum = M_GETFIB(m);
3046 inc.inc_flags |= INC_ISIPV6;
3047 inc.inc6_faddr = *dst;
3048 if (in6_setscope(&inc.inc6_faddr,
3049 m->m_pkthdr.rcvif, NULL))
3050 goto out;
3051 /*
3052 * Only process the offered MTU if it
3053 * is smaller than the current one.
3054 */
3055 if (mtu < tp->t_maxseg +
3056 sizeof (struct tcphdr) +
3057 sizeof (struct ip6_hdr)) {
3058 tcp_hc_updatemtu(&inc, mtu);
3059 tcp_mtudisc(inp, mtu);
3060 ICMP6STAT_INC(icp6s_pmtuchg);
3061 }
3062 } else
3063 inp = (*notify)(inp, errno);
3064 }
3065 } else {
3066 bzero(&inc, sizeof(inc));
3067 inc.inc_fibnum = M_GETFIB(m);
3068 inc.inc_flags |= INC_ISIPV6;
3069 inc.inc_fport = t_ports.th_dport;
3070 inc.inc_lport = t_ports.th_sport;
3071 inc.inc6_faddr = *dst;
3072 inc.inc6_laddr = ip6->ip6_src;
3073 syncache_unreach(&inc, icmp_tcp_seq, port);
3074 }
3075 out:
3076 if (inp != NULL)
3077 INP_WUNLOCK(inp);
3078 }
3079
3080 static void
tcp6_ctlinput(struct ip6ctlparam * ctl)3081 tcp6_ctlinput(struct ip6ctlparam *ctl)
3082 {
3083 tcp6_ctlinput_with_port(ctl, htons(0));
3084 }
3085
3086 static void
tcp6_ctlinput_viaudp(udp_tun_icmp_param_t param)3087 tcp6_ctlinput_viaudp(udp_tun_icmp_param_t param)
3088 {
3089 struct ip6ctlparam *ip6cp = param.ip6cp;
3090 struct mbuf *m;
3091 struct udphdr *udp;
3092 uint16_t port;
3093
3094 m = m_pulldown(ip6cp->ip6c_m, ip6cp->ip6c_off, sizeof(struct udphdr), NULL);
3095 if (m == NULL) {
3096 return;
3097 }
3098 udp = mtod(m, struct udphdr *);
3099 if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
3100 return;
3101 }
3102 port = udp->uh_dport;
3103 m_adj(m, sizeof(struct udphdr));
3104 if ((m->m_flags & M_PKTHDR) == 0) {
3105 ip6cp->ip6c_m->m_pkthdr.len -= sizeof(struct udphdr);
3106 }
3107 /* Now call in to the normal handling code */
3108 tcp6_ctlinput_with_port(ip6cp, port);
3109 }
3110
3111 #endif /* INET6 */
3112
3113 static uint32_t
tcp_keyed_hash(struct in_conninfo * inc,u_char * key,u_int len)3114 tcp_keyed_hash(struct in_conninfo *inc, u_char *key, u_int len)
3115 {
3116 SIPHASH_CTX ctx;
3117 uint32_t hash[2];
3118
3119 KASSERT(len >= SIPHASH_KEY_LENGTH,
3120 ("%s: keylen %u too short ", __func__, len));
3121 SipHash24_Init(&ctx);
3122 SipHash_SetKey(&ctx, (uint8_t *)key);
3123 SipHash_Update(&ctx, &inc->inc_fport, sizeof(uint16_t));
3124 SipHash_Update(&ctx, &inc->inc_lport, sizeof(uint16_t));
3125 switch (inc->inc_flags & INC_ISIPV6) {
3126 #ifdef INET
3127 case 0:
3128 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(struct in_addr));
3129 SipHash_Update(&ctx, &inc->inc_laddr, sizeof(struct in_addr));
3130 break;
3131 #endif
3132 #ifdef INET6
3133 case INC_ISIPV6:
3134 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(struct in6_addr));
3135 SipHash_Update(&ctx, &inc->inc6_laddr, sizeof(struct in6_addr));
3136 break;
3137 #endif
3138 }
3139 SipHash_Final((uint8_t *)hash, &ctx);
3140
3141 return (hash[0] ^ hash[1]);
3142 }
3143
3144 uint32_t
tcp_new_ts_offset(struct in_conninfo * inc)3145 tcp_new_ts_offset(struct in_conninfo *inc)
3146 {
3147 struct in_conninfo inc_store, *local_inc;
3148
3149 if (!V_tcp_ts_offset_per_conn) {
3150 memcpy(&inc_store, inc, sizeof(struct in_conninfo));
3151 inc_store.inc_lport = 0;
3152 inc_store.inc_fport = 0;
3153 local_inc = &inc_store;
3154 } else {
3155 local_inc = inc;
3156 }
3157 return (tcp_keyed_hash(local_inc, V_ts_offset_secret,
3158 sizeof(V_ts_offset_secret)));
3159 }
3160
3161 /*
3162 * Following is where TCP initial sequence number generation occurs.
3163 *
3164 * There are two places where we must use initial sequence numbers:
3165 * 1. In SYN-ACK packets.
3166 * 2. In SYN packets.
3167 *
3168 * All ISNs for SYN-ACK packets are generated by the syncache. See
3169 * tcp_syncache.c for details.
3170 *
3171 * The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
3172 * depends on this property. In addition, these ISNs should be
3173 * unguessable so as to prevent connection hijacking. To satisfy
3174 * the requirements of this situation, the algorithm outlined in
3175 * RFC 1948 is used, with only small modifications.
3176 *
3177 * Implementation details:
3178 *
3179 * Time is based off the system timer, and is corrected so that it
3180 * increases by one megabyte per second. This allows for proper
3181 * recycling on high speed LANs while still leaving over an hour
3182 * before rollover.
3183 *
3184 * As reading the *exact* system time is too expensive to be done
3185 * whenever setting up a TCP connection, we increment the time
3186 * offset in two ways. First, a small random positive increment
3187 * is added to isn_offset for each connection that is set up.
3188 * Second, the function tcp_isn_tick fires once per clock tick
3189 * and increments isn_offset as necessary so that sequence numbers
3190 * are incremented at approximately ISN_BYTES_PER_SECOND. The
3191 * random positive increments serve only to ensure that the same
3192 * exact sequence number is never sent out twice (as could otherwise
3193 * happen when a port is recycled in less than the system tick
3194 * interval.)
3195 *
3196 * net.inet.tcp.isn_reseed_interval controls the number of seconds
3197 * between seeding of isn_secret. This is normally set to zero,
3198 * as reseeding should not be necessary.
3199 *
3200 * Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
3201 * isn_offset_old, and isn_ctx is performed using the ISN lock. In
3202 * general, this means holding an exclusive (write) lock.
3203 */
3204
3205 #define ISN_BYTES_PER_SECOND 1048576
3206 #define ISN_STATIC_INCREMENT 4096
3207 #define ISN_RANDOM_INCREMENT (4096 - 1)
3208 #define ISN_SECRET_LENGTH SIPHASH_KEY_LENGTH
3209
3210 VNET_DEFINE_STATIC(u_char, isn_secret[ISN_SECRET_LENGTH]);
3211 VNET_DEFINE_STATIC(int, isn_last);
3212 VNET_DEFINE_STATIC(int, isn_last_reseed);
3213 VNET_DEFINE_STATIC(u_int32_t, isn_offset);
3214 VNET_DEFINE_STATIC(u_int32_t, isn_offset_old);
3215
3216 #define V_isn_secret VNET(isn_secret)
3217 #define V_isn_last VNET(isn_last)
3218 #define V_isn_last_reseed VNET(isn_last_reseed)
3219 #define V_isn_offset VNET(isn_offset)
3220 #define V_isn_offset_old VNET(isn_offset_old)
3221
3222 tcp_seq
tcp_new_isn(struct in_conninfo * inc)3223 tcp_new_isn(struct in_conninfo *inc)
3224 {
3225 tcp_seq new_isn;
3226 u_int32_t projected_offset;
3227
3228 ISN_LOCK();
3229 /* Seed if this is the first use, reseed if requested. */
3230 if ((V_isn_last_reseed == 0) || ((V_tcp_isn_reseed_interval > 0) &&
3231 (((u_int)V_isn_last_reseed + (u_int)V_tcp_isn_reseed_interval*hz)
3232 < (u_int)ticks))) {
3233 arc4rand(&V_isn_secret, sizeof(V_isn_secret), 0);
3234 V_isn_last_reseed = ticks;
3235 }
3236
3237 /* Compute the hash and return the ISN. */
3238 new_isn = (tcp_seq)tcp_keyed_hash(inc, V_isn_secret,
3239 sizeof(V_isn_secret));
3240 V_isn_offset += ISN_STATIC_INCREMENT +
3241 (arc4random() & ISN_RANDOM_INCREMENT);
3242 if (ticks != V_isn_last) {
3243 projected_offset = V_isn_offset_old +
3244 ISN_BYTES_PER_SECOND / hz * (ticks - V_isn_last);
3245 if (SEQ_GT(projected_offset, V_isn_offset))
3246 V_isn_offset = projected_offset;
3247 V_isn_offset_old = V_isn_offset;
3248 V_isn_last = ticks;
3249 }
3250 new_isn += V_isn_offset;
3251 ISN_UNLOCK();
3252 return (new_isn);
3253 }
3254
3255 /*
3256 * When a specific ICMP unreachable message is received and the
3257 * connection state is SYN-SENT, drop the connection. This behavior
3258 * is controlled by the icmp_may_rst sysctl.
3259 */
3260 static struct inpcb *
tcp_drop_syn_sent(struct inpcb * inp,int errno)3261 tcp_drop_syn_sent(struct inpcb *inp, int errno)
3262 {
3263 struct tcpcb *tp;
3264
3265 NET_EPOCH_ASSERT();
3266 INP_WLOCK_ASSERT(inp);
3267
3268 tp = intotcpcb(inp);
3269 if (tp->t_state != TCPS_SYN_SENT)
3270 return (inp);
3271
3272 if (tp->t_flags & TF_FASTOPEN)
3273 tcp_fastopen_disable_path(tp);
3274
3275 tp = tcp_drop(tp, errno);
3276 if (tp != NULL)
3277 return (inp);
3278 else
3279 return (NULL);
3280 }
3281
3282 /*
3283 * When `need fragmentation' ICMP is received, update our idea of the MSS
3284 * based on the new value. Also nudge TCP to send something, since we
3285 * know the packet we just sent was dropped.
3286 * This duplicates some code in the tcp_mss() function in tcp_input.c.
3287 */
3288 static struct inpcb *
tcp_mtudisc_notify(struct inpcb * inp,int error)3289 tcp_mtudisc_notify(struct inpcb *inp, int error)
3290 {
3291
3292 return (tcp_mtudisc(inp, -1));
3293 }
3294
3295 static struct inpcb *
tcp_mtudisc(struct inpcb * inp,int mtuoffer)3296 tcp_mtudisc(struct inpcb *inp, int mtuoffer)
3297 {
3298 struct tcpcb *tp;
3299 struct socket *so;
3300
3301 INP_WLOCK_ASSERT(inp);
3302
3303 tp = intotcpcb(inp);
3304 KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));
3305
3306 tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);
3307
3308 so = inp->inp_socket;
3309 SOCK_SENDBUF_LOCK(so);
3310 /* If the mss is larger than the socket buffer, decrease the mss. */
3311 if (so->so_snd.sb_hiwat < tp->t_maxseg) {
3312 tp->t_maxseg = so->so_snd.sb_hiwat;
3313 if (tp->t_maxseg < V_tcp_mssdflt) {
3314 /*
3315 * The MSS is so small we should not process incoming
3316 * SACK's since we are subject to attack in such a
3317 * case.
3318 */
3319 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
3320 } else {
3321 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
3322 }
3323 }
3324 SOCK_SENDBUF_UNLOCK(so);
3325
3326 TCPSTAT_INC(tcps_mturesent);
3327 tp->t_rtttime = 0;
3328 tp->snd_nxt = tp->snd_una;
3329 tcp_free_sackholes(tp);
3330 tp->snd_recover = tp->snd_max;
3331 if (tp->t_flags & TF_SACK_PERMIT)
3332 EXIT_FASTRECOVERY(tp->t_flags);
3333 if (tp->t_fb->tfb_tcp_mtu_chg != NULL) {
3334 /*
3335 * Conceptually the snd_nxt setting
3336 * and freeing sack holes should
3337 * be done by the default stacks
3338 * own tfb_tcp_mtu_chg().
3339 */
3340 tp->t_fb->tfb_tcp_mtu_chg(tp);
3341 }
3342 if (tcp_output(tp) < 0)
3343 return (NULL);
3344 else
3345 return (inp);
3346 }
3347
3348 #ifdef INET
3349 /*
3350 * Look-up the routing entry to the peer of this inpcb. If no route
3351 * is found and it cannot be allocated, then return 0. This routine
3352 * is called by TCP routines that access the rmx structure and by
3353 * tcp_mss_update to get the peer/interface MTU.
3354 */
3355 uint32_t
tcp_maxmtu(struct in_conninfo * inc,struct tcp_ifcap * cap)3356 tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
3357 {
3358 struct nhop_object *nh;
3359 struct ifnet *ifp;
3360 uint32_t maxmtu = 0;
3361
3362 KASSERT(inc != NULL, ("tcp_maxmtu with NULL in_conninfo pointer"));
3363
3364 if (inc->inc_faddr.s_addr != INADDR_ANY) {
3365 nh = fib4_lookup(inc->inc_fibnum, inc->inc_faddr, 0, NHR_NONE, 0);
3366 if (nh == NULL)
3367 return (0);
3368
3369 ifp = nh->nh_ifp;
3370 maxmtu = nh->nh_mtu;
3371
3372 /* Report additional interface capabilities. */
3373 if (cap != NULL) {
3374 if (ifp->if_capenable & IFCAP_TSO4 &&
3375 ifp->if_hwassist & CSUM_TSO) {
3376 cap->ifcap |= CSUM_TSO;
3377 cap->tsomax = ifp->if_hw_tsomax;
3378 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3379 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3380 /* XXXKIB IFCAP2_IPSEC_OFFLOAD_TSO */
3381 cap->ipsec_tso = (ifp->if_capenable2 &
3382 IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0;
3383 }
3384 }
3385 }
3386 return (maxmtu);
3387 }
3388 #endif /* INET */
3389
3390 #ifdef INET6
3391 uint32_t
tcp_maxmtu6(struct in_conninfo * inc,struct tcp_ifcap * cap)3392 tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
3393 {
3394 struct nhop_object *nh;
3395 struct in6_addr dst6;
3396 uint32_t scopeid;
3397 struct ifnet *ifp;
3398 uint32_t maxmtu = 0;
3399
3400 KASSERT(inc != NULL, ("tcp_maxmtu6 with NULL in_conninfo pointer"));
3401
3402 if (inc->inc_flags & INC_IPV6MINMTU)
3403 return (IPV6_MMTU);
3404
3405 if (!IN6_IS_ADDR_UNSPECIFIED(&inc->inc6_faddr)) {
3406 in6_splitscope(&inc->inc6_faddr, &dst6, &scopeid);
3407 nh = fib6_lookup(inc->inc_fibnum, &dst6, scopeid, NHR_NONE, 0);
3408 if (nh == NULL)
3409 return (0);
3410
3411 ifp = nh->nh_ifp;
3412 maxmtu = nh->nh_mtu;
3413
3414 /* Report additional interface capabilities. */
3415 if (cap != NULL) {
3416 if (ifp->if_capenable & IFCAP_TSO6 &&
3417 ifp->if_hwassist & CSUM_TSO) {
3418 cap->ifcap |= CSUM_TSO;
3419 cap->tsomax = ifp->if_hw_tsomax;
3420 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3421 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3422 cap->ipsec_tso = false; /* XXXKIB */
3423 }
3424 }
3425 }
3426
3427 return (maxmtu);
3428 }
3429
3430 /*
3431 * Handle setsockopt(IPV6_USE_MIN_MTU) by a TCP stack.
3432 *
3433 * XXXGL: we are updating inpcb here with INC_IPV6MINMTU flag.
3434 * The right place to do that is ip6_setpktopt() that has just been
3435 * executed. By the way it just filled ip6po_minmtu for us.
3436 */
3437 void
tcp6_use_min_mtu(struct tcpcb * tp)3438 tcp6_use_min_mtu(struct tcpcb *tp)
3439 {
3440 struct inpcb *inp = tptoinpcb(tp);
3441
3442 INP_WLOCK_ASSERT(inp);
3443 /*
3444 * In case of the IPV6_USE_MIN_MTU socket
3445 * option, the INC_IPV6MINMTU flag to announce
3446 * a corresponding MSS during the initial
3447 * handshake. If the TCP connection is not in
3448 * the front states, just reduce the MSS being
3449 * used. This avoids the sending of TCP
3450 * segments which will be fragmented at the
3451 * IPv6 layer.
3452 */
3453 inp->inp_inc.inc_flags |= INC_IPV6MINMTU;
3454 if ((tp->t_state >= TCPS_SYN_SENT) &&
3455 (inp->inp_inc.inc_flags & INC_ISIPV6)) {
3456 struct ip6_pktopts *opt;
3457
3458 opt = inp->in6p_outputopts;
3459 if (opt != NULL && opt->ip6po_minmtu == IP6PO_MINMTU_ALL &&
3460 tp->t_maxseg > TCP6_MSS) {
3461 tp->t_maxseg = TCP6_MSS;
3462 if (tp->t_maxseg < V_tcp_mssdflt) {
3463 /*
3464 * The MSS is so small we should not process incoming
3465 * SACK's since we are subject to attack in such a
3466 * case.
3467 */
3468 tp->t_flags2 |= TF2_PROC_SACK_PROHIBIT;
3469 } else {
3470 tp->t_flags2 &= ~TF2_PROC_SACK_PROHIBIT;
3471 }
3472 }
3473 }
3474 }
3475 #endif /* INET6 */
3476
3477 /*
3478 * Calculate effective SMSS per RFC5681 definition for a given TCP
3479 * connection at its current state, taking into account SACK and etc.
3480 */
3481 u_int
tcp_maxseg(const struct tcpcb * tp)3482 tcp_maxseg(const struct tcpcb *tp)
3483 {
3484 u_int optlen;
3485
3486 if (tp->t_flags & TF_NOOPT)
3487 return (tp->t_maxseg);
3488
3489 /*
3490 * Here we have a simplified code from tcp_addoptions(),
3491 * without a proper loop, and having most of paddings hardcoded.
3492 * We might make mistakes with padding here in some edge cases,
3493 * but this is harmless, since result of tcp_maxseg() is used
3494 * only in cwnd and ssthresh estimations.
3495 */
3496 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3497 if (tp->t_flags & TF_RCVD_TSTMP)
3498 optlen = TCPOLEN_TSTAMP_APPA;
3499 else
3500 optlen = 0;
3501 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3502 if (tp->t_flags & TF_SIGNATURE)
3503 optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3504 #endif
3505 if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0) {
3506 optlen += TCPOLEN_SACKHDR;
3507 optlen += tp->rcv_numsacks * TCPOLEN_SACK;
3508 optlen = PADTCPOLEN(optlen);
3509 }
3510 } else {
3511 if (tp->t_flags & TF_REQ_TSTMP)
3512 optlen = TCPOLEN_TSTAMP_APPA;
3513 else
3514 optlen = PADTCPOLEN(TCPOLEN_MAXSEG);
3515 if (tp->t_flags & TF_REQ_SCALE)
3516 optlen += PADTCPOLEN(TCPOLEN_WINDOW);
3517 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3518 if (tp->t_flags & TF_SIGNATURE)
3519 optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3520 #endif
3521 if (tp->t_flags & TF_SACK_PERMIT)
3522 optlen += PADTCPOLEN(TCPOLEN_SACK_PERMITTED);
3523 }
3524 optlen = min(optlen, TCP_MAXOLEN);
3525 return (tp->t_maxseg - optlen);
3526 }
3527
3528
3529 u_int
tcp_fixed_maxseg(const struct tcpcb * tp)3530 tcp_fixed_maxseg(const struct tcpcb *tp)
3531 {
3532 int optlen;
3533
3534 if (tp->t_flags & TF_NOOPT)
3535 return (tp->t_maxseg);
3536
3537 /*
3538 * Here we have a simplified code from tcp_addoptions(),
3539 * without a proper loop, and having most of paddings hardcoded.
3540 * We only consider fixed options that we would send every
3541 * time I.e. SACK is not considered. This is important
3542 * for cc modules to figure out what the modulo of the
3543 * cwnd should be.
3544 */
3545 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3546 if (tp->t_flags & TF_RCVD_TSTMP)
3547 optlen = TCPOLEN_TSTAMP_APPA;
3548 else
3549 optlen = 0;
3550 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3551 if (tp->t_flags & TF_SIGNATURE)
3552 optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3553 #endif
3554 } else {
3555 if (tp->t_flags & TF_REQ_TSTMP)
3556 optlen = TCPOLEN_TSTAMP_APPA;
3557 else
3558 optlen = PADTCPOLEN(TCPOLEN_MAXSEG);
3559 if (tp->t_flags & TF_REQ_SCALE)
3560 optlen += PADTCPOLEN(TCPOLEN_WINDOW);
3561 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3562 if (tp->t_flags & TF_SIGNATURE)
3563 optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3564 #endif
3565 if (tp->t_flags & TF_SACK_PERMIT)
3566 optlen += PADTCPOLEN(TCPOLEN_SACK_PERMITTED);
3567 }
3568 optlen = min(optlen, TCP_MAXOLEN);
3569 return (tp->t_maxseg - optlen);
3570 }
3571
3572
3573
3574 static int
sysctl_drop(SYSCTL_HANDLER_ARGS)3575 sysctl_drop(SYSCTL_HANDLER_ARGS)
3576 {
3577 /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3578 struct sockaddr_storage addrs[2];
3579 struct inpcb *inp;
3580 struct tcpcb *tp;
3581 #ifdef INET
3582 struct sockaddr_in *fin = NULL, *lin = NULL;
3583 #endif
3584 struct epoch_tracker et;
3585 #ifdef INET6
3586 struct sockaddr_in6 *fin6, *lin6;
3587 #endif
3588 int error;
3589
3590 inp = NULL;
3591 #ifdef INET6
3592 fin6 = lin6 = NULL;
3593 #endif
3594 error = 0;
3595
3596 if (req->oldptr != NULL || req->oldlen != 0)
3597 return (EINVAL);
3598 if (req->newptr == NULL)
3599 return (EPERM);
3600 if (req->newlen < sizeof(addrs))
3601 return (ENOMEM);
3602 error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3603 if (error)
3604 return (error);
3605
3606 switch (addrs[0].ss_family) {
3607 #ifdef INET6
3608 case AF_INET6:
3609 fin6 = (struct sockaddr_in6 *)&addrs[0];
3610 lin6 = (struct sockaddr_in6 *)&addrs[1];
3611 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3612 lin6->sin6_len != sizeof(struct sockaddr_in6))
3613 return (EINVAL);
3614 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3615 if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3616 return (EINVAL);
3617 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3618 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3619 #ifdef INET
3620 fin = (struct sockaddr_in *)&addrs[0];
3621 lin = (struct sockaddr_in *)&addrs[1];
3622 #endif
3623 break;
3624 }
3625 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3626 if (error)
3627 return (error);
3628 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3629 if (error)
3630 return (error);
3631 break;
3632 #endif
3633 #ifdef INET
3634 case AF_INET:
3635 fin = (struct sockaddr_in *)&addrs[0];
3636 lin = (struct sockaddr_in *)&addrs[1];
3637 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3638 lin->sin_len != sizeof(struct sockaddr_in))
3639 return (EINVAL);
3640 break;
3641 #endif
3642 default:
3643 return (EINVAL);
3644 }
3645 NET_EPOCH_ENTER(et);
3646 switch (addrs[0].ss_family) {
3647 #ifdef INET6
3648 case AF_INET6:
3649 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3650 fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3651 INPLOOKUP_WLOCKPCB, NULL);
3652 break;
3653 #endif
3654 #ifdef INET
3655 case AF_INET:
3656 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3657 lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3658 break;
3659 #endif
3660 }
3661 if (inp != NULL) {
3662 if (!SOLISTENING(inp->inp_socket)) {
3663 tp = intotcpcb(inp);
3664 tp = tcp_drop(tp, ECONNABORTED);
3665 if (tp != NULL)
3666 INP_WUNLOCK(inp);
3667 } else
3668 INP_WUNLOCK(inp);
3669 } else
3670 error = ESRCH;
3671 NET_EPOCH_EXIT(et);
3672 return (error);
3673 }
3674
3675 SYSCTL_PROC(_net_inet_tcp, TCPCTL_DROP, drop,
3676 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3677 CTLFLAG_NEEDGIANT, NULL, 0, sysctl_drop, "",
3678 "Drop TCP connection");
3679
3680 static int
tcp_sysctl_setsockopt(SYSCTL_HANDLER_ARGS)3681 tcp_sysctl_setsockopt(SYSCTL_HANDLER_ARGS)
3682 {
3683 return (sysctl_setsockopt(oidp, arg1, arg2, req, &V_tcbinfo,
3684 &tcp_ctloutput_set));
3685 }
3686
3687 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, setsockopt,
3688 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3689 CTLFLAG_MPSAFE, NULL, 0, tcp_sysctl_setsockopt, "",
3690 "Set socket option for TCP endpoint");
3691
3692 #ifdef KERN_TLS
3693 static int
sysctl_switch_tls(SYSCTL_HANDLER_ARGS)3694 sysctl_switch_tls(SYSCTL_HANDLER_ARGS)
3695 {
3696 /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3697 struct sockaddr_storage addrs[2];
3698 struct inpcb *inp;
3699 #ifdef INET
3700 struct sockaddr_in *fin = NULL, *lin = NULL;
3701 #endif
3702 struct epoch_tracker et;
3703 #ifdef INET6
3704 struct sockaddr_in6 *fin6, *lin6;
3705 #endif
3706 int error;
3707
3708 inp = NULL;
3709 #ifdef INET6
3710 fin6 = lin6 = NULL;
3711 #endif
3712 error = 0;
3713
3714 if (req->oldptr != NULL || req->oldlen != 0)
3715 return (EINVAL);
3716 if (req->newptr == NULL)
3717 return (EPERM);
3718 if (req->newlen < sizeof(addrs))
3719 return (ENOMEM);
3720 error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3721 if (error)
3722 return (error);
3723
3724 switch (addrs[0].ss_family) {
3725 #ifdef INET6
3726 case AF_INET6:
3727 fin6 = (struct sockaddr_in6 *)&addrs[0];
3728 lin6 = (struct sockaddr_in6 *)&addrs[1];
3729 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3730 lin6->sin6_len != sizeof(struct sockaddr_in6))
3731 return (EINVAL);
3732 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3733 if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3734 return (EINVAL);
3735 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3736 in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3737 #ifdef INET
3738 fin = (struct sockaddr_in *)&addrs[0];
3739 lin = (struct sockaddr_in *)&addrs[1];
3740 #endif
3741 break;
3742 }
3743 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3744 if (error)
3745 return (error);
3746 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3747 if (error)
3748 return (error);
3749 break;
3750 #endif
3751 #ifdef INET
3752 case AF_INET:
3753 fin = (struct sockaddr_in *)&addrs[0];
3754 lin = (struct sockaddr_in *)&addrs[1];
3755 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3756 lin->sin_len != sizeof(struct sockaddr_in))
3757 return (EINVAL);
3758 break;
3759 #endif
3760 default:
3761 return (EINVAL);
3762 }
3763 NET_EPOCH_ENTER(et);
3764 switch (addrs[0].ss_family) {
3765 #ifdef INET6
3766 case AF_INET6:
3767 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3768 fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3769 INPLOOKUP_WLOCKPCB, NULL);
3770 break;
3771 #endif
3772 #ifdef INET
3773 case AF_INET:
3774 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3775 lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3776 break;
3777 #endif
3778 }
3779 NET_EPOCH_EXIT(et);
3780 if (inp != NULL) {
3781 struct socket *so;
3782
3783 so = inp->inp_socket;
3784 soref(so);
3785 error = ktls_set_tx_mode(so,
3786 arg2 == 0 ? TCP_TLS_MODE_SW : TCP_TLS_MODE_IFNET);
3787 INP_WUNLOCK(inp);
3788 sorele(so);
3789 } else
3790 error = ESRCH;
3791 return (error);
3792 }
3793
3794 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_sw_tls,
3795 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3796 CTLFLAG_NEEDGIANT, NULL, 0, sysctl_switch_tls, "",
3797 "Switch TCP connection to SW TLS");
3798 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_ifnet_tls,
3799 CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3800 CTLFLAG_NEEDGIANT, NULL, 1, sysctl_switch_tls, "",
3801 "Switch TCP connection to ifnet TLS");
3802 #endif
3803
3804 /*
3805 * Generate a standardized TCP log line for use throughout the
3806 * tcp subsystem. Memory allocation is done with M_NOWAIT to
3807 * allow use in the interrupt context.
3808 *
3809 * NB: The caller MUST free(s, M_TCPLOG) the returned string.
3810 * NB: The function may return NULL if memory allocation failed.
3811 *
3812 * Due to header inclusion and ordering limitations the struct ip
3813 * and ip6_hdr pointers have to be passed as void pointers.
3814 */
3815 char *
tcp_log_vain(struct in_conninfo * inc,struct tcphdr * th,const void * ip4hdr,const void * ip6hdr)3816 tcp_log_vain(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3817 const void *ip6hdr)
3818 {
3819
3820 /* Is logging enabled? */
3821 if (V_tcp_log_in_vain == 0)
3822 return (NULL);
3823
3824 return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3825 }
3826
3827 char *
tcp_log_addrs(struct in_conninfo * inc,struct tcphdr * th,const void * ip4hdr,const void * ip6hdr)3828 tcp_log_addrs(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3829 const void *ip6hdr)
3830 {
3831
3832 /* Is logging enabled? */
3833 if (tcp_log_debug == 0)
3834 return (NULL);
3835
3836 return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3837 }
3838
3839 static char *
tcp_log_addr(struct in_conninfo * inc,struct tcphdr * th,const void * ip4hdr,const void * ip6hdr)3840 tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3841 const void *ip6hdr)
3842 {
3843 char *s, *sp;
3844 size_t size;
3845 #ifdef INET
3846 const struct ip *ip = (const struct ip *)ip4hdr;
3847 #endif
3848 #ifdef INET6
3849 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)ip6hdr;
3850 #endif /* INET6 */
3851
3852 /*
3853 * The log line looks like this:
3854 * "TCP: [1.2.3.4]:50332 to [1.2.3.4]:80 tcpflags 0x2<SYN>"
3855 */
3856 size = sizeof("TCP: []:12345 to []:12345 tcpflags 0x2<>") +
3857 sizeof(PRINT_TH_FLAGS) + 1 +
3858 #ifdef INET6
3859 2 * INET6_ADDRSTRLEN;
3860 #else
3861 2 * INET_ADDRSTRLEN;
3862 #endif /* INET6 */
3863
3864 s = malloc(size, M_TCPLOG, M_ZERO|M_NOWAIT);
3865 if (s == NULL)
3866 return (NULL);
3867
3868 strcat(s, "TCP: [");
3869 sp = s + strlen(s);
3870
3871 if (inc && ((inc->inc_flags & INC_ISIPV6) == 0)) {
3872 inet_ntoa_r(inc->inc_faddr, sp);
3873 sp = s + strlen(s);
3874 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3875 sp = s + strlen(s);
3876 inet_ntoa_r(inc->inc_laddr, sp);
3877 sp = s + strlen(s);
3878 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3879 #ifdef INET6
3880 } else if (inc) {
3881 ip6_sprintf(sp, &inc->inc6_faddr);
3882 sp = s + strlen(s);
3883 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3884 sp = s + strlen(s);
3885 ip6_sprintf(sp, &inc->inc6_laddr);
3886 sp = s + strlen(s);
3887 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3888 } else if (ip6 && th) {
3889 ip6_sprintf(sp, &ip6->ip6_src);
3890 sp = s + strlen(s);
3891 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3892 sp = s + strlen(s);
3893 ip6_sprintf(sp, &ip6->ip6_dst);
3894 sp = s + strlen(s);
3895 sprintf(sp, "]:%i", ntohs(th->th_dport));
3896 #endif /* INET6 */
3897 #ifdef INET
3898 } else if (ip && th) {
3899 inet_ntoa_r(ip->ip_src, sp);
3900 sp = s + strlen(s);
3901 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3902 sp = s + strlen(s);
3903 inet_ntoa_r(ip->ip_dst, sp);
3904 sp = s + strlen(s);
3905 sprintf(sp, "]:%i", ntohs(th->th_dport));
3906 #endif /* INET */
3907 } else {
3908 free(s, M_TCPLOG);
3909 return (NULL);
3910 }
3911 sp = s + strlen(s);
3912 if (th)
3913 sprintf(sp, " tcpflags 0x%b", tcp_get_flags(th), PRINT_TH_FLAGS);
3914 if (*(s + size - 1) != '\0')
3915 panic("%s: string too long", __func__);
3916 return (s);
3917 }
3918
3919 /*
3920 * A subroutine which makes it easy to track TCP state changes with DTrace.
3921 * This function shouldn't be called for t_state initializations that don't
3922 * correspond to actual TCP state transitions.
3923 */
3924 void
tcp_state_change(struct tcpcb * tp,int newstate)3925 tcp_state_change(struct tcpcb *tp, int newstate)
3926 {
3927 #if defined(KDTRACE_HOOKS)
3928 int pstate = tp->t_state;
3929 #endif
3930
3931 TCPSTATES_DEC(tp->t_state);
3932 TCPSTATES_INC(newstate);
3933 tp->t_state = newstate;
3934 TCP_PROBE6(state__change, NULL, tp, NULL, tp, NULL, pstate);
3935 }
3936
3937 /*
3938 * Create an external-format (``xtcpcb'') structure using the information in
3939 * the kernel-format tcpcb structure pointed to by tp. This is done to
3940 * reduce the spew of irrelevant information over this interface, to isolate
3941 * user code from changes in the kernel structure, and potentially to provide
3942 * information-hiding if we decide that some of this information should be
3943 * hidden from users.
3944 */
3945 void
tcp_inptoxtp(const struct inpcb * inp,struct xtcpcb * xt)3946 tcp_inptoxtp(const struct inpcb *inp, struct xtcpcb *xt)
3947 {
3948 struct tcpcb *tp = intotcpcb(inp);
3949 sbintime_t now;
3950
3951 bzero(xt, sizeof(*xt));
3952 xt->t_state = tp->t_state;
3953 xt->t_logstate = tcp_get_bblog_state(tp);
3954 xt->t_flags = tp->t_flags;
3955 xt->t_sndzerowin = tp->t_sndzerowin;
3956 xt->t_sndrexmitpack = tp->t_sndrexmitpack;
3957 xt->t_rcvoopack = tp->t_rcvoopack;
3958 xt->t_rcv_wnd = tp->rcv_wnd;
3959 xt->t_snd_wnd = tp->snd_wnd;
3960 xt->t_snd_cwnd = tp->snd_cwnd;
3961 xt->t_snd_ssthresh = tp->snd_ssthresh;
3962 xt->t_dsack_bytes = tp->t_dsack_bytes;
3963 xt->t_dsack_tlp_bytes = tp->t_dsack_tlp_bytes;
3964 xt->t_dsack_pack = tp->t_dsack_pack;
3965 xt->t_maxseg = tp->t_maxseg;
3966 xt->xt_ecn = (tp->t_flags2 & TF2_ECN_PERMIT) ? 1 : 0 +
3967 (tp->t_flags2 & TF2_ACE_PERMIT) ? 2 : 0;
3968
3969 now = getsbinuptime();
3970 #define COPYTIMER(which,where) do { \
3971 if (tp->t_timers[which] != SBT_MAX) \
3972 xt->where = (tp->t_timers[which] - now) / SBT_1MS; \
3973 else \
3974 xt->where = 0; \
3975 } while (0)
3976 COPYTIMER(TT_DELACK, tt_delack);
3977 COPYTIMER(TT_REXMT, tt_rexmt);
3978 COPYTIMER(TT_PERSIST, tt_persist);
3979 COPYTIMER(TT_KEEP, tt_keep);
3980 COPYTIMER(TT_2MSL, tt_2msl);
3981 #undef COPYTIMER
3982 xt->t_rcvtime = 1000 * (ticks - tp->t_rcvtime) / hz;
3983
3984 xt->xt_encaps_port = tp->t_port;
3985 bcopy(tp->t_fb->tfb_tcp_block_name, xt->xt_stack,
3986 TCP_FUNCTION_NAME_LEN_MAX);
3987 bcopy(CC_ALGO(tp)->name, xt->xt_cc, TCP_CA_NAME_MAX);
3988 #ifdef TCP_BLACKBOX
3989 (void)tcp_log_get_id(tp, xt->xt_logid);
3990 #endif
3991
3992 xt->xt_len = sizeof(struct xtcpcb);
3993 in_pcbtoxinpcb(inp, &xt->xt_inp);
3994 }
3995
3996 void
tcp_log_end_status(struct tcpcb * tp,uint8_t status)3997 tcp_log_end_status(struct tcpcb *tp, uint8_t status)
3998 {
3999 uint32_t bit, i;
4000
4001 if ((tp == NULL) ||
4002 (status > TCP_EI_STATUS_MAX_VALUE) ||
4003 (status == 0)) {
4004 /* Invalid */
4005 return;
4006 }
4007 if (status > (sizeof(uint32_t) * 8)) {
4008 /* Should this be a KASSERT? */
4009 return;
4010 }
4011 bit = 1U << (status - 1);
4012 if (bit & tp->t_end_info_status) {
4013 /* already logged */
4014 return;
4015 }
4016 for (i = 0; i < TCP_END_BYTE_INFO; i++) {
4017 if (tp->t_end_info_bytes[i] == TCP_EI_EMPTY_SLOT) {
4018 tp->t_end_info_bytes[i] = status;
4019 tp->t_end_info_status |= bit;
4020 break;
4021 }
4022 }
4023 }
4024
4025 int
tcp_can_enable_pacing(void)4026 tcp_can_enable_pacing(void)
4027 {
4028
4029 if ((tcp_pacing_limit == -1) ||
4030 (tcp_pacing_limit > number_of_tcp_connections_pacing)) {
4031 atomic_fetchadd_int(&number_of_tcp_connections_pacing, 1);
4032 shadow_num_connections = number_of_tcp_connections_pacing;
4033 return (1);
4034 } else {
4035 counter_u64_add(tcp_pacing_failures, 1);
4036 return (0);
4037 }
4038 }
4039
4040 int
tcp_incr_dgp_pacing_cnt(void)4041 tcp_incr_dgp_pacing_cnt(void)
4042 {
4043 if ((tcp_dgp_limit == -1) ||
4044 (tcp_dgp_limit > number_of_dgp_connections)) {
4045 atomic_fetchadd_int(&number_of_dgp_connections, 1);
4046 shadow_tcp_pacing_dgp = number_of_dgp_connections;
4047 return (1);
4048 } else {
4049 counter_u64_add(tcp_dgp_failures, 1);
4050 return (0);
4051 }
4052 }
4053
4054 static uint8_t tcp_dgp_warning = 0;
4055
4056 void
tcp_dec_dgp_pacing_cnt(void)4057 tcp_dec_dgp_pacing_cnt(void)
4058 {
4059 uint32_t ret;
4060
4061 ret = atomic_fetchadd_int(&number_of_dgp_connections, -1);
4062 shadow_tcp_pacing_dgp = number_of_dgp_connections;
4063 KASSERT(ret != 0, ("number_of_dgp_connections -1 would cause wrap?"));
4064 if (ret == 0) {
4065 if (tcp_dgp_limit != -1) {
4066 printf("Warning all DGP is now disabled, count decrements invalidly!\n");
4067 tcp_dgp_limit = 0;
4068 tcp_dgp_warning = 1;
4069 } else if (tcp_dgp_warning == 0) {
4070 printf("Warning DGP pacing is invalid, invalid decrement\n");
4071 tcp_dgp_warning = 1;
4072 }
4073 }
4074
4075 }
4076
4077 static uint8_t tcp_pacing_warning = 0;
4078
4079 void
tcp_decrement_paced_conn(void)4080 tcp_decrement_paced_conn(void)
4081 {
4082 uint32_t ret;
4083
4084 ret = atomic_fetchadd_int(&number_of_tcp_connections_pacing, -1);
4085 shadow_num_connections = number_of_tcp_connections_pacing;
4086 KASSERT(ret != 0, ("tcp_paced_connection_exits -1 would cause wrap?"));
4087 if (ret == 0) {
4088 if (tcp_pacing_limit != -1) {
4089 printf("Warning all pacing is now disabled, count decrements invalidly!\n");
4090 tcp_pacing_limit = 0;
4091 } else if (tcp_pacing_warning == 0) {
4092 printf("Warning pacing count is invalid, invalid decrement\n");
4093 tcp_pacing_warning = 1;
4094 }
4095 }
4096 }
4097
4098 static void
tcp_default_switch_failed(struct tcpcb * tp)4099 tcp_default_switch_failed(struct tcpcb *tp)
4100 {
4101 /*
4102 * If a switch fails we only need to
4103 * care about two things:
4104 * a) The t_flags2
4105 * and
4106 * b) The timer granularity.
4107 * Timeouts, at least for now, don't use the
4108 * old callout system in the other stacks so
4109 * those are hopefully safe.
4110 */
4111 tcp_lro_features_off(tp);
4112 tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
4113 }
4114
4115 #ifdef TCP_ACCOUNTING
4116 int
tcp_do_ack_accounting(struct tcpcb * tp,struct tcphdr * th,struct tcpopt * to,uint32_t tiwin,int mss)4117 tcp_do_ack_accounting(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to, uint32_t tiwin, int mss)
4118 {
4119 if (SEQ_LT(th->th_ack, tp->snd_una)) {
4120 /* Do we have a SACK? */
4121 if (to->to_flags & TOF_SACK) {
4122 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4123 tp->tcp_cnt_counters[ACK_SACK]++;
4124 }
4125 return (ACK_SACK);
4126 } else {
4127 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4128 tp->tcp_cnt_counters[ACK_BEHIND]++;
4129 }
4130 return (ACK_BEHIND);
4131 }
4132 } else if (th->th_ack == tp->snd_una) {
4133 /* Do we have a SACK? */
4134 if (to->to_flags & TOF_SACK) {
4135 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4136 tp->tcp_cnt_counters[ACK_SACK]++;
4137 }
4138 return (ACK_SACK);
4139 } else if (tiwin != tp->snd_wnd) {
4140 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4141 tp->tcp_cnt_counters[ACK_RWND]++;
4142 }
4143 return (ACK_RWND);
4144 } else {
4145 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4146 tp->tcp_cnt_counters[ACK_DUPACK]++;
4147 }
4148 return (ACK_DUPACK);
4149 }
4150 } else {
4151 if (!SEQ_GT(th->th_ack, tp->snd_max)) {
4152 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4153 tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((th->th_ack - tp->snd_una) + mss - 1)/mss);
4154 }
4155 }
4156 if (to->to_flags & TOF_SACK) {
4157 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4158 tp->tcp_cnt_counters[ACK_CUMACK_SACK]++;
4159 }
4160 return (ACK_CUMACK_SACK);
4161 } else {
4162 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4163 tp->tcp_cnt_counters[ACK_CUMACK]++;
4164 }
4165 return (ACK_CUMACK);
4166 }
4167 }
4168 }
4169 #endif
4170
4171 void
tcp_change_time_units(struct tcpcb * tp,int granularity)4172 tcp_change_time_units(struct tcpcb *tp, int granularity)
4173 {
4174 if (tp->t_tmr_granularity == granularity) {
4175 /* We are there */
4176 return;
4177 }
4178 if (granularity == TCP_TMR_GRANULARITY_USEC) {
4179 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS),
4180 ("Granularity is not TICKS its %u in tp:%p",
4181 tp->t_tmr_granularity, tp));
4182 tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
4183 if (tp->t_srtt > 1) {
4184 uint32_t val, frac;
4185
4186 val = tp->t_srtt >> TCP_RTT_SHIFT;
4187 frac = tp->t_srtt & 0x1f;
4188 tp->t_srtt = TICKS_2_USEC(val);
4189 /*
4190 * frac is the fractional part of the srtt (if any)
4191 * but its in ticks and every bit represents
4192 * 1/32nd of a hz.
4193 */
4194 if (frac) {
4195 if (hz == 1000) {
4196 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4197 } else {
4198 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4199 }
4200 tp->t_srtt += frac;
4201 }
4202 }
4203 if (tp->t_rttvar) {
4204 uint32_t val, frac;
4205
4206 val = tp->t_rttvar >> TCP_RTTVAR_SHIFT;
4207 frac = tp->t_rttvar & 0x1f;
4208 tp->t_rttvar = TICKS_2_USEC(val);
4209 /*
4210 * frac is the fractional part of the srtt (if any)
4211 * but its in ticks and every bit represents
4212 * 1/32nd of a hz.
4213 */
4214 if (frac) {
4215 if (hz == 1000) {
4216 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4217 } else {
4218 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4219 }
4220 tp->t_rttvar += frac;
4221 }
4222 }
4223 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_USEC;
4224 } else if (granularity == TCP_TMR_GRANULARITY_TICKS) {
4225 /* Convert back to ticks, with */
4226 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_USEC),
4227 ("Granularity is not USEC its %u in tp:%p",
4228 tp->t_tmr_granularity, tp));
4229 if (tp->t_srtt > 1) {
4230 uint32_t val, frac;
4231
4232 val = USEC_2_TICKS(tp->t_srtt);
4233 frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
4234 tp->t_srtt = val << TCP_RTT_SHIFT;
4235 /*
4236 * frac is the fractional part here is left
4237 * over from converting to hz and shifting.
4238 * We need to convert this to the 5 bit
4239 * remainder.
4240 */
4241 if (frac) {
4242 if (hz == 1000) {
4243 frac = (((uint64_t)frac * (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4244 } else {
4245 frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4246 }
4247 tp->t_srtt += frac;
4248 }
4249 }
4250 if (tp->t_rttvar) {
4251 uint32_t val, frac;
4252
4253 val = USEC_2_TICKS(tp->t_rttvar);
4254 frac = tp->t_rttvar % (HPTS_USEC_IN_SEC / hz);
4255 tp->t_rttvar = val << TCP_RTTVAR_SHIFT;
4256 /*
4257 * frac is the fractional part here is left
4258 * over from converting to hz and shifting.
4259 * We need to convert this to the 4 bit
4260 * remainder.
4261 */
4262 if (frac) {
4263 if (hz == 1000) {
4264 frac = (((uint64_t)frac * (uint64_t)TCP_RTTVAR_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4265 } else {
4266 frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTTVAR_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4267 }
4268 tp->t_rttvar += frac;
4269 }
4270 }
4271 tp->t_rttlow = USEC_2_TICKS(tp->t_rttlow);
4272 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
4273 }
4274 #ifdef INVARIANTS
4275 else {
4276 panic("Unknown granularity:%d tp:%p",
4277 granularity, tp);
4278 }
4279 #endif
4280 }
4281
4282 void
tcp_handle_orphaned_packets(struct tcpcb * tp)4283 tcp_handle_orphaned_packets(struct tcpcb *tp)
4284 {
4285 struct mbuf *save, *m, *prev;
4286 /*
4287 * Called when a stack switch is occuring from the fini()
4288 * of the old stack. We assue the init() as already been
4289 * run of the new stack and it has set the t_flags2 to
4290 * what it supports. This function will then deal with any
4291 * differences i.e. cleanup packets that maybe queued that
4292 * the newstack does not support.
4293 */
4294
4295 if (tp->t_flags2 & TF2_MBUF_L_ACKS)
4296 return;
4297 if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) == 0 &&
4298 !STAILQ_EMPTY(&tp->t_inqueue)) {
4299 /*
4300 * It is unsafe to process the packets since a
4301 * reset may be lurking in them (its rare but it
4302 * can occur). If we were to find a RST, then we
4303 * would end up dropping the connection and the
4304 * INP lock, so when we return the caller (tcp_usrreq)
4305 * will blow up when it trys to unlock the inp.
4306 * This new stack does not do any fancy LRO features
4307 * so all we can do is toss the packets.
4308 */
4309 m = STAILQ_FIRST(&tp->t_inqueue);
4310 STAILQ_INIT(&tp->t_inqueue);
4311 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, save)
4312 m_freem(m);
4313 } else {
4314 /*
4315 * Here we have a stack that does mbuf queuing but
4316 * does not support compressed ack's. We must
4317 * walk all the mbufs and discard any compressed acks.
4318 */
4319 STAILQ_FOREACH_SAFE(m, &tp->t_inqueue, m_stailqpkt, save) {
4320 if (m->m_flags & M_ACKCMP) {
4321 if (m == STAILQ_FIRST(&tp->t_inqueue))
4322 STAILQ_REMOVE_HEAD(&tp->t_inqueue,
4323 m_stailqpkt);
4324 else
4325 STAILQ_REMOVE_AFTER(&tp->t_inqueue,
4326 prev, m_stailqpkt);
4327 m_freem(m);
4328 } else
4329 prev = m;
4330 }
4331 }
4332 }
4333
4334 #ifdef TCP_REQUEST_TRK
4335 uint32_t
tcp_estimate_tls_overhead(struct socket * so,uint64_t tls_usr_bytes)4336 tcp_estimate_tls_overhead(struct socket *so, uint64_t tls_usr_bytes)
4337 {
4338 #ifdef KERN_TLS
4339 struct ktls_session *tls;
4340 uint32_t rec_oh, records;
4341
4342 tls = so->so_snd.sb_tls_info;
4343 if (tls == NULL)
4344 return (0);
4345
4346 rec_oh = tls->params.tls_hlen + tls->params.tls_tlen;
4347 records = ((tls_usr_bytes + tls->params.max_frame_len - 1)/tls->params.max_frame_len);
4348 return (records * rec_oh);
4349 #else
4350 return (0);
4351 #endif
4352 }
4353
4354 extern uint32_t tcp_stale_entry_time;
4355 uint32_t tcp_stale_entry_time = 250000;
4356 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, usrlog_stale, CTLFLAG_RW,
4357 &tcp_stale_entry_time, 250000, "Time that a tcpreq entry without a sendfile ages out");
4358
4359 void
tcp_req_log_req_info(struct tcpcb * tp,struct tcp_sendfile_track * req,uint16_t slot,uint8_t val,uint64_t offset,uint64_t nbytes)4360 tcp_req_log_req_info(struct tcpcb *tp, struct tcp_sendfile_track *req,
4361 uint16_t slot, uint8_t val, uint64_t offset, uint64_t nbytes)
4362 {
4363 if (tcp_bblogging_on(tp)) {
4364 union tcp_log_stackspecific log;
4365 struct timeval tv;
4366
4367 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
4368 log.u_bbr.inhpts = tcp_in_hpts(tp);
4369 log.u_bbr.flex8 = val;
4370 log.u_bbr.rttProp = req->timestamp;
4371 log.u_bbr.delRate = req->start;
4372 log.u_bbr.cur_del_rate = req->end;
4373 log.u_bbr.flex1 = req->start_seq;
4374 log.u_bbr.flex2 = req->end_seq;
4375 log.u_bbr.flex3 = req->flags;
4376 log.u_bbr.flex4 = ((req->localtime >> 32) & 0x00000000ffffffff);
4377 log.u_bbr.flex5 = (req->localtime & 0x00000000ffffffff);
4378 log.u_bbr.flex7 = slot;
4379 log.u_bbr.bw_inuse = offset;
4380 /* nbytes = flex6 | epoch */
4381 log.u_bbr.flex6 = ((nbytes >> 32) & 0x00000000ffffffff);
4382 log.u_bbr.epoch = (nbytes & 0x00000000ffffffff);
4383 /* cspr = lt_epoch | pkts_out */
4384 log.u_bbr.lt_epoch = ((req->cspr >> 32) & 0x00000000ffffffff);
4385 log.u_bbr.pkts_out |= (req->cspr & 0x00000000ffffffff);
4386 log.u_bbr.applimited = tp->t_tcpreq_closed;
4387 log.u_bbr.applimited <<= 8;
4388 log.u_bbr.applimited |= tp->t_tcpreq_open;
4389 log.u_bbr.applimited <<= 8;
4390 log.u_bbr.applimited |= tp->t_tcpreq_req;
4391 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4392 TCP_LOG_EVENTP(tp, NULL,
4393 &tptosocket(tp)->so_rcv,
4394 &tptosocket(tp)->so_snd,
4395 TCP_LOG_REQ_T, 0,
4396 0, &log, false, &tv);
4397 }
4398 }
4399
4400 void
tcp_req_free_a_slot(struct tcpcb * tp,struct tcp_sendfile_track * ent)4401 tcp_req_free_a_slot(struct tcpcb *tp, struct tcp_sendfile_track *ent)
4402 {
4403 if (tp->t_tcpreq_req > 0)
4404 tp->t_tcpreq_req--;
4405 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) {
4406 if (tp->t_tcpreq_open > 0)
4407 tp->t_tcpreq_open--;
4408 } else {
4409 if (tp->t_tcpreq_closed > 0)
4410 tp->t_tcpreq_closed--;
4411 }
4412 ent->flags = TCP_TRK_TRACK_FLG_EMPTY;
4413 }
4414
4415 static void
tcp_req_check_for_stale_entries(struct tcpcb * tp,uint64_t ts,int rm_oldest)4416 tcp_req_check_for_stale_entries(struct tcpcb *tp, uint64_t ts, int rm_oldest)
4417 {
4418 struct tcp_sendfile_track *ent;
4419 uint64_t time_delta, oldest_delta;
4420 int i, oldest, oldest_set = 0, cnt_rm = 0;
4421
4422 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4423 ent = &tp->t_tcpreq_info[i];
4424 if (ent->flags != TCP_TRK_TRACK_FLG_USED) {
4425 /*
4426 * We only care about closed end ranges
4427 * that are allocated and have no sendfile
4428 * ever touching them. They would be in
4429 * state USED.
4430 */
4431 continue;
4432 }
4433 if (ts >= ent->localtime)
4434 time_delta = ts - ent->localtime;
4435 else
4436 time_delta = 0;
4437 if (time_delta &&
4438 ((oldest_delta < time_delta) || (oldest_set == 0))) {
4439 oldest_set = 1;
4440 oldest = i;
4441 oldest_delta = time_delta;
4442 }
4443 if (tcp_stale_entry_time && (time_delta >= tcp_stale_entry_time)) {
4444 /*
4445 * No sendfile in a our time-limit
4446 * time to purge it.
4447 */
4448 cnt_rm++;
4449 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i], i, TCP_TRK_REQ_LOG_STALE,
4450 time_delta, 0);
4451 tcp_req_free_a_slot(tp, ent);
4452 }
4453 }
4454 if ((cnt_rm == 0) && rm_oldest && oldest_set) {
4455 ent = &tp->t_tcpreq_info[oldest];
4456 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i], i, TCP_TRK_REQ_LOG_STALE,
4457 oldest_delta, 1);
4458 tcp_req_free_a_slot(tp, ent);
4459 }
4460 }
4461
4462 int
tcp_req_check_for_comp(struct tcpcb * tp,tcp_seq ack_point)4463 tcp_req_check_for_comp(struct tcpcb *tp, tcp_seq ack_point)
4464 {
4465 int i, ret = 0;
4466 struct tcp_sendfile_track *ent;
4467
4468 /* Clean up any old closed end requests that are now completed */
4469 if (tp->t_tcpreq_req == 0)
4470 return (0);
4471 if (tp->t_tcpreq_closed == 0)
4472 return (0);
4473 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4474 ent = &tp->t_tcpreq_info[i];
4475 /* Skip empty ones */
4476 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4477 continue;
4478 /* Skip open ones */
4479 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN)
4480 continue;
4481 if (SEQ_GEQ(ack_point, ent->end_seq)) {
4482 /* We are past it -- free it */
4483 tcp_req_log_req_info(tp, ent,
4484 i, TCP_TRK_REQ_LOG_FREED, 0, 0);
4485 tcp_req_free_a_slot(tp, ent);
4486 ret++;
4487 }
4488 }
4489 return (ret);
4490 }
4491
4492 int
tcp_req_is_entry_comp(struct tcpcb * tp,struct tcp_sendfile_track * ent,tcp_seq ack_point)4493 tcp_req_is_entry_comp(struct tcpcb *tp, struct tcp_sendfile_track *ent, tcp_seq ack_point)
4494 {
4495 if (tp->t_tcpreq_req == 0)
4496 return (-1);
4497 if (tp->t_tcpreq_closed == 0)
4498 return (-1);
4499 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4500 return (-1);
4501 if (SEQ_GEQ(ack_point, ent->end_seq)) {
4502 return (1);
4503 }
4504 return (0);
4505 }
4506
4507 struct tcp_sendfile_track *
tcp_req_find_a_req_that_is_completed_by(struct tcpcb * tp,tcp_seq th_ack,int * ip)4508 tcp_req_find_a_req_that_is_completed_by(struct tcpcb *tp, tcp_seq th_ack, int *ip)
4509 {
4510 /*
4511 * Given an ack point (th_ack) walk through our entries and
4512 * return the first one found that th_ack goes past the
4513 * end_seq.
4514 */
4515 struct tcp_sendfile_track *ent;
4516 int i;
4517
4518 if (tp->t_tcpreq_req == 0) {
4519 /* none open */
4520 return (NULL);
4521 }
4522 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4523 ent = &tp->t_tcpreq_info[i];
4524 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4525 continue;
4526 if ((ent->flags & TCP_TRK_TRACK_FLG_OPEN) == 0) {
4527 if (SEQ_GEQ(th_ack, ent->end_seq)) {
4528 *ip = i;
4529 return (ent);
4530 }
4531 }
4532 }
4533 return (NULL);
4534 }
4535
4536 struct tcp_sendfile_track *
tcp_req_find_req_for_seq(struct tcpcb * tp,tcp_seq seq)4537 tcp_req_find_req_for_seq(struct tcpcb *tp, tcp_seq seq)
4538 {
4539 struct tcp_sendfile_track *ent;
4540 int i;
4541
4542 if (tp->t_tcpreq_req == 0) {
4543 /* none open */
4544 return (NULL);
4545 }
4546 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4547 ent = &tp->t_tcpreq_info[i];
4548 tcp_req_log_req_info(tp, ent, i, TCP_TRK_REQ_LOG_SEARCH,
4549 (uint64_t)seq, 0);
4550 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) {
4551 continue;
4552 }
4553 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) {
4554 /*
4555 * An open end request only needs to
4556 * match the beginning seq or be
4557 * all we have (once we keep going on
4558 * a open end request we may have a seq
4559 * wrap).
4560 */
4561 if ((SEQ_GEQ(seq, ent->start_seq)) ||
4562 (tp->t_tcpreq_closed == 0))
4563 return (ent);
4564 } else {
4565 /*
4566 * For this one we need to
4567 * be a bit more careful if its
4568 * completed at least.
4569 */
4570 if ((SEQ_GEQ(seq, ent->start_seq)) &&
4571 (SEQ_LT(seq, ent->end_seq))) {
4572 return (ent);
4573 }
4574 }
4575 }
4576 return (NULL);
4577 }
4578
4579 /* Should this be in its own file tcp_req.c ? */
4580 struct tcp_sendfile_track *
tcp_req_alloc_req_full(struct tcpcb * tp,struct tcp_snd_req * req,uint64_t ts,int rec_dups)4581 tcp_req_alloc_req_full(struct tcpcb *tp, struct tcp_snd_req *req, uint64_t ts, int rec_dups)
4582 {
4583 struct tcp_sendfile_track *fil;
4584 int i, allocated;
4585
4586 /* In case the stack does not check for completions do so now */
4587 tcp_req_check_for_comp(tp, tp->snd_una);
4588 /* Check for stale entries */
4589 if (tp->t_tcpreq_req)
4590 tcp_req_check_for_stale_entries(tp, ts,
4591 (tp->t_tcpreq_req >= MAX_TCP_TRK_REQ));
4592 /* Check to see if this is a duplicate of one not started */
4593 if (tp->t_tcpreq_req) {
4594 for (i = 0, allocated = 0; i < MAX_TCP_TRK_REQ; i++) {
4595 fil = &tp->t_tcpreq_info[i];
4596 if ((fil->flags & TCP_TRK_TRACK_FLG_USED) == 0)
4597 continue;
4598 if ((fil->timestamp == req->timestamp) &&
4599 (fil->start == req->start) &&
4600 ((fil->flags & TCP_TRK_TRACK_FLG_OPEN) ||
4601 (fil->end == req->end))) {
4602 /*
4603 * We already have this request
4604 * and it has not been started with sendfile.
4605 * This probably means the user was returned
4606 * a 4xx of some sort and its going to age
4607 * out, lets not duplicate it.
4608 */
4609 return (fil);
4610 }
4611 }
4612 }
4613 /* Ok if there is no room at the inn we are in trouble */
4614 if (tp->t_tcpreq_req >= MAX_TCP_TRK_REQ) {
4615 tcp_trace_point(tp, TCP_TP_REQ_LOG_FAIL);
4616 for (i = 0; i < MAX_TCP_TRK_REQ; i++) {
4617 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i],
4618 i, TCP_TRK_REQ_LOG_ALLOCFAIL, 0, 0);
4619 }
4620 return (NULL);
4621 }
4622 for (i = 0, allocated = 0; i < MAX_TCP_TRK_REQ; i++) {
4623 fil = &tp->t_tcpreq_info[i];
4624 if (fil->flags == TCP_TRK_TRACK_FLG_EMPTY) {
4625 allocated = 1;
4626 fil->flags = TCP_TRK_TRACK_FLG_USED;
4627 fil->timestamp = req->timestamp;
4628 fil->playout_ms = req->playout_ms;
4629 fil->localtime = ts;
4630 fil->start = req->start;
4631 if (req->flags & TCP_LOG_HTTPD_RANGE_END) {
4632 fil->end = req->end;
4633 } else {
4634 fil->end = 0;
4635 fil->flags |= TCP_TRK_TRACK_FLG_OPEN;
4636 }
4637 /*
4638 * We can set the min boundaries to the TCP Sequence space,
4639 * but it might be found to be further up when sendfile
4640 * actually runs on this range (if it ever does).
4641 */
4642 fil->sbcc_at_s = tptosocket(tp)->so_snd.sb_ccc;
4643 fil->start_seq = tp->snd_una +
4644 tptosocket(tp)->so_snd.sb_ccc;
4645 if (req->flags & TCP_LOG_HTTPD_RANGE_END)
4646 fil->end_seq = (fil->start_seq + ((uint32_t)(fil->end - fil->start)));
4647 else
4648 fil->end_seq = 0;
4649 if (tptosocket(tp)->so_snd.sb_tls_info) {
4650 /*
4651 * This session is doing TLS. Take a swag guess
4652 * at the overhead.
4653 */
4654 fil->end_seq += tcp_estimate_tls_overhead(
4655 tptosocket(tp), (fil->end - fil->start));
4656 }
4657 tp->t_tcpreq_req++;
4658 if (fil->flags & TCP_TRK_TRACK_FLG_OPEN)
4659 tp->t_tcpreq_open++;
4660 else
4661 tp->t_tcpreq_closed++;
4662 tcp_req_log_req_info(tp, fil, i,
4663 TCP_TRK_REQ_LOG_NEW, 0, 0);
4664 break;
4665 } else
4666 fil = NULL;
4667 }
4668 return (fil);
4669 }
4670
4671 void
tcp_req_alloc_req(struct tcpcb * tp,union tcp_log_userdata * user,uint64_t ts)4672 tcp_req_alloc_req(struct tcpcb *tp, union tcp_log_userdata *user, uint64_t ts)
4673 {
4674 (void)tcp_req_alloc_req_full(tp, &user->tcp_req, ts, 1);
4675 }
4676 #endif
4677
4678 void
tcp_log_socket_option(struct tcpcb * tp,uint32_t option_num,uint32_t option_val,int err)4679 tcp_log_socket_option(struct tcpcb *tp, uint32_t option_num, uint32_t option_val, int err)
4680 {
4681 if (tcp_bblogging_on(tp)) {
4682 struct tcp_log_buffer *l;
4683
4684 l = tcp_log_event(tp, NULL,
4685 &tptosocket(tp)->so_rcv,
4686 &tptosocket(tp)->so_snd,
4687 TCP_LOG_SOCKET_OPT,
4688 err, 0, NULL, 1,
4689 NULL, NULL, 0, NULL);
4690 if (l) {
4691 l->tlb_flex1 = option_num;
4692 l->tlb_flex2 = option_val;
4693 }
4694 }
4695 }
4696
4697 uint32_t
tcp_get_srtt(struct tcpcb * tp,int granularity)4698 tcp_get_srtt(struct tcpcb *tp, int granularity)
4699 {
4700 uint32_t srtt;
4701
4702 KASSERT(granularity == TCP_TMR_GRANULARITY_USEC ||
4703 granularity == TCP_TMR_GRANULARITY_TICKS,
4704 ("%s: called with unexpected granularity %d", __func__,
4705 granularity));
4706
4707 srtt = tp->t_srtt;
4708
4709 /*
4710 * We only support two granularities. If the stored granularity
4711 * does not match the granularity requested by the caller,
4712 * convert the stored value to the requested unit of granularity.
4713 */
4714 if (tp->t_tmr_granularity != granularity) {
4715 if (granularity == TCP_TMR_GRANULARITY_USEC)
4716 srtt = TICKS_2_USEC(srtt);
4717 else
4718 srtt = USEC_2_TICKS(srtt);
4719 }
4720
4721 /*
4722 * If the srtt is stored with ticks granularity, we need to
4723 * unshift to get the actual value. We do this after the
4724 * conversion above (if one was necessary) in order to maximize
4725 * precision.
4726 */
4727 if (tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS)
4728 srtt = srtt >> TCP_RTT_SHIFT;
4729
4730 return (srtt);
4731 }
4732
4733 void
tcp_account_for_send(struct tcpcb * tp,uint32_t len,uint8_t is_rxt,uint8_t is_tlp,bool hw_tls)4734 tcp_account_for_send(struct tcpcb *tp, uint32_t len, uint8_t is_rxt,
4735 uint8_t is_tlp, bool hw_tls)
4736 {
4737
4738 if (is_tlp) {
4739 tp->t_sndtlppack++;
4740 tp->t_sndtlpbyte += len;
4741 }
4742 /* To get total bytes sent you must add t_snd_rxt_bytes to t_sndbytes */
4743 if (is_rxt)
4744 tp->t_snd_rxt_bytes += len;
4745 else
4746 tp->t_sndbytes += len;
4747
4748 #ifdef KERN_TLS
4749 if (hw_tls && is_rxt && len != 0) {
4750 uint64_t rexmit_percent;
4751
4752 rexmit_percent = (1000ULL * tp->t_snd_rxt_bytes) /
4753 (10ULL * (tp->t_snd_rxt_bytes + tp->t_sndbytes));
4754 if (rexmit_percent > ktls_ifnet_max_rexmit_pct)
4755 ktls_disable_ifnet(tp);
4756 }
4757 #endif
4758 }
4759