1 /*-
2 * Copyright (c) 2015-2017 Patrick Kelsey
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 /*
28 * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include
29 * this code, add the following line to your kernel config:
30 *
31 * options TCP_RFC7413
32 *
33 *
34 * The generated TFO cookies are the 64-bit output of
35 * SipHash24(key=<16-byte-key>, msg=<client-ip>). Multiple concurrent valid
36 * keys are supported so that time-based rolling cookie invalidation
37 * policies can be implemented in the system. The default number of
38 * concurrent keys is 2. This can be adjusted in the kernel config as
39 * follows:
40 *
41 * options TCP_RFC7413_MAX_KEYS=<num-keys>
42 *
43 *
44 * In addition to the facilities defined in RFC7413, this implementation
45 * supports a pre-shared key (PSK) mode of operation in which the TFO server
46 * requires the client to be in possession of a shared secret in order for
47 * the client to be able to successfully open TFO connections with the
48 * server. This is useful, for example, in environments where TFO servers
49 * are exposed to both internal and external clients and only wish to allow
50 * TFO connections from internal clients.
51 *
52 * In the PSK mode of operation, the server generates and sends TFO cookies
53 * to requesting clients as usual. However, when validating cookies
54 * received in TFO SYNs from clients, the server requires the
55 * client-supplied cookie to equal SipHash24(key=<16-byte-psk>,
56 * msg=<cookie-sent-to-client>).
57 *
58 * Multiple concurrent valid pre-shared keys are supported so that
59 * time-based rolling PSK invalidation policies can be implemented in the
60 * system. The default number of concurrent pre-shared keys is 2. This can
61 * be adjusted in the kernel config as follows:
62 *
63 * options TCP_RFC7413_MAX_PSKS=<num-psks>
64 *
65 *
66 * The following TFO-specific sysctls are defined:
67 *
68 * net.inet.tcp.fastopen.acceptany (RW, default 0)
69 * When non-zero, all client-supplied TFO cookies will be considered to
70 * be valid.
71 *
72 * net.inet.tcp.fastopen.autokey (RW, default 120)
73 * When this and net.inet.tcp.fastopen.server_enable are non-zero, a new
74 * key will be automatically generated after this many seconds.
75 *
76 * net.inet.tcp.fastopen.ccache_bucket_limit
77 * (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT)
78 * The maximum number of entries in a client cookie cache bucket.
79 *
80 * net.inet.tcp.fastopen.ccache_buckets
81 * (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT)
82 * The number of client cookie cache buckets.
83 *
84 * net.inet.tcp.fastopen.ccache_list (RO)
85 * Print the client cookie cache.
86 *
87 * net.inet.tcp.fastopen.client_enable (RW, default 1)
88 * When zero, no new active (i.e., client) TFO connections can be
89 * created. On the transition from enabled to disabled, the client
90 * cookie cache is cleared and disabled. The transition from enabled to
91 * disabled does not affect any active TFO connections in progress; it
92 * only prevents new ones from being made.
93 *
94 * net.inet.tcp.fastopen.keylen (RD)
95 * The key length in bytes.
96 *
97 * net.inet.tcp.fastopen.maxkeys (RD)
98 * The maximum number of keys supported.
99 *
100 * net.inet.tcp.fastopen.maxpsks (RD)
101 * The maximum number of pre-shared keys supported.
102 *
103 * net.inet.tcp.fastopen.numkeys (RD)
104 * The current number of keys installed.
105 *
106 * net.inet.tcp.fastopen.numpsks (RD)
107 * The current number of pre-shared keys installed.
108 *
109 * net.inet.tcp.fastopen.path_disable_time
110 * (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT)
111 * When a failure occurs while trying to create a new active (i.e.,
112 * client) TFO connection, new active connections on the same path, as
113 * determined by the tuple {client_ip, server_ip, server_port}, will be
114 * forced to be non-TFO for this many seconds. Note that the path
115 * disable mechanism relies on state stored in client cookie cache
116 * entries, so it is possible for the disable time for a given path to
117 * be reduced if the corresponding client cookie cache entry is reused
118 * due to resource pressure before the disable period has elapsed.
119 *
120 * net.inet.tcp.fastopen.psk_enable (RW, default 0)
121 * When non-zero, pre-shared key (PSK) mode is enabled for all TFO
122 * servers. On the transition from enabled to disabled, all installed
123 * pre-shared keys are removed.
124 *
125 * net.inet.tcp.fastopen.server_enable (RW, default 0)
126 * When zero, no new passive (i.e., server) TFO connections can be
127 * created. On the transition from enabled to disabled, all installed
128 * keys and pre-shared keys are removed. On the transition from
129 * disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and
130 * there are no keys installed, a new key will be generated immediately.
131 * The transition from enabled to disabled does not affect any passive
132 * TFO connections in progress; it only prevents new ones from being
133 * made.
134 *
135 * net.inet.tcp.fastopen.setkey (WR)
136 * Install a new key by writing net.inet.tcp.fastopen.keylen bytes to
137 * this sysctl.
138 *
139 * net.inet.tcp.fastopen.setpsk (WR)
140 * Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen
141 * bytes to this sysctl.
142 *
143 * In order for TFO connections to be created via a listen socket, that
144 * socket must have the TCP_FASTOPEN socket option set on it. This option
145 * can be set on the socket either before or after the listen() is invoked.
146 * Clearing this option on a listen socket after it has been set has no
147 * effect on existing TFO connections or TFO connections in progress; it
148 * only prevents new TFO connections from being made.
149 *
150 * For passively-created sockets, the TCP_FASTOPEN socket option can be
151 * queried to determine whether the connection was established using TFO.
152 * Note that connections that are established via a TFO SYN, but that fall
153 * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option
154 * set.
155 *
156 * Per the RFC, this implementation limits the number of TFO connections
157 * that can be in the SYN_RECEIVED state on a per listen-socket basis.
158 * Whenever this limit is exceeded, requests for new TFO connections are
159 * serviced as non-TFO requests. Without such a limit, given a valid TFO
160 * cookie, an attacker could keep the listen queue in an overflow condition
161 * using a TFO SYN flood. This implementation sets the limit at half the
162 * configured listen backlog.
163 *
164 */
165
166 #include "opt_inet.h"
167
168 #include <sys/param.h>
169 #include <sys/jail.h>
170 #include <sys/kernel.h>
171 #include <sys/hash.h>
172 #include <sys/limits.h>
173 #include <sys/lock.h>
174 #include <sys/proc.h>
175 #include <sys/rmlock.h>
176 #include <sys/sbuf.h>
177 #include <sys/socket.h>
178 #include <sys/socketvar.h>
179 #include <sys/sysctl.h>
180 #include <sys/systm.h>
181
182 #include <crypto/siphash/siphash.h>
183
184 #include <net/vnet.h>
185
186 #include <netinet/in.h>
187 #include <netinet/in_pcb.h>
188 #include <netinet/tcp_var.h>
189 #include <netinet/tcp_fastopen.h>
190
191 #define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH
192
193 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
194 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
195 #endif
196
197 /*
198 * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold
199 * the PSK until the connect occurs.
200 */
201 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
202 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
203 #endif
204
205 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16
206 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048 /* must be power of 2 */
207
208 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900 /* seconds */
209
210 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
211 #define TCP_FASTOPEN_MAX_KEYS 2
212 #else
213 #define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS
214 #endif
215
216 #if TCP_FASTOPEN_MAX_KEYS > 10
217 #undef TCP_FASTOPEN_MAX_KEYS
218 #define TCP_FASTOPEN_MAX_KEYS 10
219 #endif
220
221 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
222 #define TCP_FASTOPEN_MAX_PSKS 2
223 #else
224 #define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS
225 #endif
226
227 #if TCP_FASTOPEN_MAX_PSKS > 10
228 #undef TCP_FASTOPEN_MAX_PSKS
229 #define TCP_FASTOPEN_MAX_PSKS 10
230 #endif
231
232 struct tcp_fastopen_keylist {
233 unsigned int newest;
234 unsigned int newest_psk;
235 uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN];
236 uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN];
237 };
238
239 struct tcp_fastopen_callout {
240 struct callout c;
241 struct vnet *v;
242 };
243
244 union tcp_fastopen_ip_addr {
245 struct in_addr v4;
246 struct in6_addr v6;
247 };
248
249 struct tcp_fastopen_ccache_entry {
250 TAILQ_ENTRY(tcp_fastopen_ccache_entry) cce_link;
251 union tcp_fastopen_ip_addr cce_client_ip; /* network byte order */
252 union tcp_fastopen_ip_addr cce_server_ip; /* network byte order */
253 uint16_t server_port; /* network byte order */
254 uint16_t server_mss; /* host byte order */
255 uint8_t af;
256 uint8_t cookie_len;
257 uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN];
258 sbintime_t disable_time; /* non-zero value means path is disabled */
259 };
260
261 struct tcp_fastopen_ccache;
262
263 struct tcp_fastopen_ccache_bucket {
264 struct mtx ccb_mtx;
265 TAILQ_HEAD(bucket_entries, tcp_fastopen_ccache_entry) ccb_entries;
266 int ccb_num_entries;
267 struct tcp_fastopen_ccache *ccb_ccache;
268 };
269
270 struct tcp_fastopen_ccache {
271 struct tcp_fastopen_ccache_bucket *base;
272 unsigned int bucket_limit;
273 unsigned int buckets;
274 unsigned int mask;
275 uint32_t secret;
276 };
277
278 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
279 struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
280 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
281 struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t,
282 uint8_t *);
283 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *,
284 unsigned int);
285 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *,
286 struct tcp_fastopen_ccache_bucket *);
287
288 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
289 "TCP Fast Open");
290
291 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0;
292 #define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany)
293 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany,
294 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
295 "Accept any non-empty cookie");
296
297 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120;
298 #define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey)
299 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS);
300 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey,
301 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
302 NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU",
303 "Number of seconds between auto-generation of a new key; zero disables");
304
305 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS);
306 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit,
307 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_NEEDGIANT,
308 NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU",
309 "Max entries per bucket in client cookie cache");
310
311 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) =
312 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
313 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
314 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets,
315 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
316 "Client cookie cache number of buckets (power of 2)");
317
318 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1;
319 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS);
320 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable,
321 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
322 NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU",
323 "Enable/disable TCP Fast Open client functionality");
324
325 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen,
326 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN,
327 "Key length in bytes");
328
329 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys,
330 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS,
331 "Maximum number of keys supported");
332
333 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks,
334 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS,
335 "Maximum number of pre-shared keys supported");
336
337 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0;
338 #define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys)
339 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys,
340 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
341 "Number of keys installed");
342
343 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0;
344 #define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks)
345 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks,
346 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
347 "Number of pre-shared keys installed");
348
349 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) =
350 TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT;
351 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
352 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time,
353 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
354 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
355
356 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0;
357 #define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable)
358 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS);
359 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable,
360 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
361 NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU",
362 "Enable/disable TCP Fast Open server pre-shared key mode");
363
364 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0;
365 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS);
366 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable,
367 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
368 NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU",
369 "Enable/disable TCP Fast Open server functionality");
370
371 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS);
372 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey,
373 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
374 NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "",
375 "Install a new key");
376
377 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS);
378 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk,
379 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
380 NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "",
381 "Install a new pre-shared key");
382
383 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS);
384 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list,
385 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
386 NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A",
387 "List of all client cookie cache entries");
388
389 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock);
390 #define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock)
391
392 #define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t))
393 #define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t))
394 #define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock)
395 #define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock)
396
397 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys);
398 #define V_tcp_fastopen_keys VNET(tcp_fastopen_keys)
399
400 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
401 #define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx)
402
403 static uma_zone_t counter_zone;
404 static uma_zone_t ccache_zone;
405
406 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
407
408 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
409 #define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache)
410
411 #define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx)
412 #define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx)
413 #define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
414
415 void
tcp_fastopen_init(void)416 tcp_fastopen_init(void)
417 {
418 counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
419 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
420 ccache_zone = uma_zcreate("tfo_ccache_entries",
421 sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
422 UMA_ALIGN_CACHE, 0);
423 }
424
425 void
tcp_fastopen_vnet_init(void)426 tcp_fastopen_vnet_init(void)
427 {
428 unsigned int i;
429
430 rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
431 callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
432 &V_tcp_fastopen_keylock, 0);
433 V_tcp_fastopen_autokey_ctx.v = curvnet;
434 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
435 V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1;
436
437 TUNABLE_INT_FETCH("net.inet.tcp.fastopen.ccache_bucket_limit",
438 &V_tcp_fastopen_ccache.bucket_limit);
439 if (V_tcp_fastopen_ccache.bucket_limit == 0)
440 V_tcp_fastopen_ccache.bucket_limit =
441 TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT;
442
443 /* May already be non-zero if kernel tunable was set */
444 if ((V_tcp_fastopen_ccache_buckets == 0) ||
445 !powerof2(V_tcp_fastopen_ccache_buckets))
446 V_tcp_fastopen_ccache.buckets =
447 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
448 else
449 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets;
450
451 V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1;
452 V_tcp_fastopen_ccache.secret = arc4random();
453
454 V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets *
455 sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE,
456 M_WAITOK | M_ZERO);
457
458 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
459 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries);
460 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket",
461 NULL, MTX_DEF);
462 if (V_tcp_fastopen_client_enable) {
463 /* enable bucket */
464 V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0;
465 } else {
466 /* disable bucket */
467 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1;
468 }
469 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
470 }
471 }
472
473 void
tcp_fastopen_vnet_destroy(void)474 tcp_fastopen_vnet_destroy(void)
475 {
476 struct tcp_fastopen_ccache_bucket *ccb;
477 unsigned int i;
478
479 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
480 ccb = &V_tcp_fastopen_ccache.base[i];
481 tcp_fastopen_ccache_bucket_trim(ccb, 0);
482 mtx_destroy(&ccb->ccb_mtx);
483 }
484
485 free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
486
487 callout_drain(&V_tcp_fastopen_autokey_ctx.c);
488 rm_destroy(&V_tcp_fastopen_keylock);
489 }
490
491 unsigned int *
tcp_fastopen_alloc_counter(void)492 tcp_fastopen_alloc_counter(void)
493 {
494 unsigned int *counter;
495
496 counter = uma_zalloc(counter_zone, M_NOWAIT);
497 if (counter)
498 *counter = 1;
499 return (counter);
500 }
501
502 void
tcp_fastopen_decrement_counter(unsigned int * counter)503 tcp_fastopen_decrement_counter(unsigned int *counter)
504 {
505 if (*counter == 1)
506 uma_zfree(counter_zone, counter);
507 else
508 atomic_subtract_int(counter, 1);
509 }
510
511 static void
tcp_fastopen_addkey_locked(uint8_t * key)512 tcp_fastopen_addkey_locked(uint8_t *key)
513 {
514
515 V_tcp_fastopen_keys.newest++;
516 if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS)
517 V_tcp_fastopen_keys.newest = 0;
518 memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key,
519 TCP_FASTOPEN_KEY_LEN);
520 if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS)
521 V_tcp_fastopen_numkeys++;
522 }
523
524 static void
tcp_fastopen_addpsk_locked(uint8_t * psk)525 tcp_fastopen_addpsk_locked(uint8_t *psk)
526 {
527
528 V_tcp_fastopen_keys.newest_psk++;
529 if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS)
530 V_tcp_fastopen_keys.newest_psk = 0;
531 memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk,
532 TCP_FASTOPEN_KEY_LEN);
533 if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS)
534 V_tcp_fastopen_numpsks++;
535 }
536
537 static void
tcp_fastopen_autokey_locked(void)538 tcp_fastopen_autokey_locked(void)
539 {
540 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
541
542 arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0);
543 tcp_fastopen_addkey_locked(newkey);
544 }
545
546 static void
tcp_fastopen_autokey_callout(void * arg)547 tcp_fastopen_autokey_callout(void *arg)
548 {
549 struct tcp_fastopen_callout *ctx = arg;
550
551 CURVNET_SET(ctx->v);
552 tcp_fastopen_autokey_locked();
553 callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz,
554 tcp_fastopen_autokey_callout, ctx);
555 CURVNET_RESTORE();
556 }
557
558 static uint64_t
tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH],struct in_conninfo * inc)559 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
560 {
561 SIPHASH_CTX ctx;
562 uint64_t siphash;
563
564 SipHash24_Init(&ctx);
565 SipHash_SetKey(&ctx, key);
566 switch (inc->inc_flags & INC_ISIPV6) {
567 #ifdef INET
568 case 0:
569 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr));
570 break;
571 #endif
572 #ifdef INET6
573 case INC_ISIPV6:
574 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr));
575 break;
576 #endif
577 }
578 SipHash_Final((u_int8_t *)&siphash, &ctx);
579
580 return (siphash);
581 }
582
583 static uint64_t
tcp_fastopen_make_psk_cookie(uint8_t * psk,uint8_t * cookie,uint8_t cookie_len)584 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
585 {
586 SIPHASH_CTX ctx;
587 uint64_t psk_cookie;
588
589 SipHash24_Init(&ctx);
590 SipHash_SetKey(&ctx, psk);
591 SipHash_Update(&ctx, cookie, cookie_len);
592 SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
593
594 return (psk_cookie);
595 }
596
597 static int
tcp_fastopen_find_cookie_match_locked(uint8_t * wire_cookie,uint64_t * cur_cookie)598 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
599 {
600 unsigned int i, psk_index;
601 uint64_t psk_cookie;
602
603 if (V_tcp_fastopen_psk_enable) {
604 psk_index = V_tcp_fastopen_keys.newest_psk;
605 for (i = 0; i < V_tcp_fastopen_numpsks; i++) {
606 psk_cookie =
607 tcp_fastopen_make_psk_cookie(
608 V_tcp_fastopen_keys.psk[psk_index],
609 (uint8_t *)cur_cookie,
610 TCP_FASTOPEN_COOKIE_LEN);
611
612 if (memcmp(wire_cookie, &psk_cookie,
613 TCP_FASTOPEN_COOKIE_LEN) == 0)
614 return (1);
615
616 if (psk_index == 0)
617 psk_index = TCP_FASTOPEN_MAX_PSKS - 1;
618 else
619 psk_index--;
620 }
621 } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0)
622 return (1);
623
624 return (0);
625 }
626
627 /*
628 * Return values:
629 * -1 the cookie is invalid and no valid cookie is available
630 * 0 the cookie is invalid and the latest cookie has been returned
631 * 1 the cookie is valid and the latest cookie has been returned
632 */
633 int
tcp_fastopen_check_cookie(struct in_conninfo * inc,uint8_t * cookie,unsigned int len,uint64_t * latest_cookie)634 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
635 unsigned int len, uint64_t *latest_cookie)
636 {
637 struct rm_priotracker tracker;
638 unsigned int i, key_index;
639 int rv;
640 uint64_t cur_cookie;
641
642 if (V_tcp_fastopen_acceptany) {
643 *latest_cookie = 0;
644 return (1);
645 }
646
647 TCP_FASTOPEN_KEYS_RLOCK(&tracker);
648 if (len != TCP_FASTOPEN_COOKIE_LEN) {
649 if (V_tcp_fastopen_numkeys > 0) {
650 *latest_cookie =
651 tcp_fastopen_make_cookie(
652 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
653 inc);
654 rv = 0;
655 } else
656 rv = -1;
657 goto out;
658 }
659
660 /*
661 * Check against each available key, from newest to oldest.
662 */
663 key_index = V_tcp_fastopen_keys.newest;
664 for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
665 cur_cookie =
666 tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
667 inc);
668 if (i == 0)
669 *latest_cookie = cur_cookie;
670 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie);
671 if (rv)
672 goto out;
673 if (key_index == 0)
674 key_index = TCP_FASTOPEN_MAX_KEYS - 1;
675 else
676 key_index--;
677 }
678 rv = 0;
679
680 out:
681 TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
682 return (rv);
683 }
684
685 static int
sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)686 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
687 {
688 int error;
689 unsigned int new;
690
691 new = V_tcp_fastopen_autokey;
692 error = sysctl_handle_int(oidp, &new, 0, req);
693 if (error == 0 && req->newptr) {
694 if (new > (INT_MAX / hz))
695 return (EINVAL);
696
697 TCP_FASTOPEN_KEYS_WLOCK();
698 if (V_tcp_fastopen_server_enable) {
699 if (V_tcp_fastopen_autokey && !new)
700 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
701 else if (new)
702 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
703 new * hz, tcp_fastopen_autokey_callout,
704 &V_tcp_fastopen_autokey_ctx);
705 }
706 V_tcp_fastopen_autokey = new;
707 TCP_FASTOPEN_KEYS_WUNLOCK();
708 }
709
710 return (error);
711 }
712
713 static int
sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)714 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
715 {
716 int error;
717 unsigned int new;
718
719 new = V_tcp_fastopen_psk_enable;
720 error = sysctl_handle_int(oidp, &new, 0, req);
721 if (error == 0 && req->newptr) {
722 if (V_tcp_fastopen_psk_enable && !new) {
723 /* enabled -> disabled */
724 TCP_FASTOPEN_KEYS_WLOCK();
725 V_tcp_fastopen_numpsks = 0;
726 V_tcp_fastopen_keys.newest_psk =
727 TCP_FASTOPEN_MAX_PSKS - 1;
728 V_tcp_fastopen_psk_enable = 0;
729 TCP_FASTOPEN_KEYS_WUNLOCK();
730 } else if (!V_tcp_fastopen_psk_enable && new) {
731 /* disabled -> enabled */
732 TCP_FASTOPEN_KEYS_WLOCK();
733 V_tcp_fastopen_psk_enable = 1;
734 TCP_FASTOPEN_KEYS_WUNLOCK();
735 }
736 }
737 return (error);
738 }
739
740 static int
sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)741 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
742 {
743 int error;
744 unsigned int new;
745
746 new = V_tcp_fastopen_server_enable;
747 error = sysctl_handle_int(oidp, &new, 0, req);
748 if (error == 0 && req->newptr) {
749 if (V_tcp_fastopen_server_enable && !new) {
750 /* enabled -> disabled */
751 TCP_FASTOPEN_KEYS_WLOCK();
752 V_tcp_fastopen_numkeys = 0;
753 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
754 if (V_tcp_fastopen_autokey)
755 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
756 V_tcp_fastopen_numpsks = 0;
757 V_tcp_fastopen_keys.newest_psk =
758 TCP_FASTOPEN_MAX_PSKS - 1;
759 V_tcp_fastopen_server_enable = 0;
760 TCP_FASTOPEN_KEYS_WUNLOCK();
761 } else if (!V_tcp_fastopen_server_enable && new) {
762 /* disabled -> enabled */
763 TCP_FASTOPEN_KEYS_WLOCK();
764 if (V_tcp_fastopen_autokey &&
765 (V_tcp_fastopen_numkeys == 0)) {
766 tcp_fastopen_autokey_locked();
767 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
768 V_tcp_fastopen_autokey * hz,
769 tcp_fastopen_autokey_callout,
770 &V_tcp_fastopen_autokey_ctx);
771 }
772 V_tcp_fastopen_server_enable = 1;
773 TCP_FASTOPEN_KEYS_WUNLOCK();
774 }
775 }
776 return (error);
777 }
778
779 static int
sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)780 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
781 {
782 int error;
783 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
784
785 if (req->oldptr != NULL || req->oldlen != 0)
786 return (EINVAL);
787 if (req->newptr == NULL)
788 return (EPERM);
789 if (req->newlen != sizeof(newkey))
790 return (EINVAL);
791 error = SYSCTL_IN(req, newkey, sizeof(newkey));
792 if (error)
793 return (error);
794
795 TCP_FASTOPEN_KEYS_WLOCK();
796 tcp_fastopen_addkey_locked(newkey);
797 TCP_FASTOPEN_KEYS_WUNLOCK();
798
799 return (0);
800 }
801
802 static int
sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)803 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
804 {
805 int error;
806 uint8_t newpsk[TCP_FASTOPEN_KEY_LEN];
807
808 if (req->oldptr != NULL || req->oldlen != 0)
809 return (EINVAL);
810 if (req->newptr == NULL)
811 return (EPERM);
812 if (req->newlen != sizeof(newpsk))
813 return (EINVAL);
814 error = SYSCTL_IN(req, newpsk, sizeof(newpsk));
815 if (error)
816 return (error);
817
818 TCP_FASTOPEN_KEYS_WLOCK();
819 tcp_fastopen_addpsk_locked(newpsk);
820 TCP_FASTOPEN_KEYS_WUNLOCK();
821
822 return (0);
823 }
824
825 static int
sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)826 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
827 {
828 struct tcp_fastopen_ccache_bucket *ccb;
829 int error;
830 unsigned int new;
831 unsigned int i;
832
833 new = V_tcp_fastopen_ccache.bucket_limit;
834 error = sysctl_handle_int(oidp, &new, 0, req);
835 if (error == 0 && req->newptr) {
836 if ((new == 0) || (new > INT_MAX))
837 error = EINVAL;
838 else {
839 if (new < V_tcp_fastopen_ccache.bucket_limit) {
840 for (i = 0; i < V_tcp_fastopen_ccache.buckets;
841 i++) {
842 ccb = &V_tcp_fastopen_ccache.base[i];
843 tcp_fastopen_ccache_bucket_trim(ccb, new);
844 }
845 }
846 V_tcp_fastopen_ccache.bucket_limit = new;
847 }
848 }
849 return (error);
850 }
851
852 static int
sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)853 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
854 {
855 struct tcp_fastopen_ccache_bucket *ccb;
856 int error;
857 unsigned int new, i;
858
859 new = V_tcp_fastopen_client_enable;
860 error = sysctl_handle_int(oidp, &new, 0, req);
861 if (error == 0 && req->newptr) {
862 if (V_tcp_fastopen_client_enable && !new) {
863 /* enabled -> disabled */
864 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
865 ccb = &V_tcp_fastopen_ccache.base[i];
866 KASSERT(ccb->ccb_num_entries > -1,
867 ("%s: ccb->ccb_num_entries %d is negative",
868 __func__, ccb->ccb_num_entries));
869 tcp_fastopen_ccache_bucket_trim(ccb, 0);
870 }
871 V_tcp_fastopen_client_enable = 0;
872 } else if (!V_tcp_fastopen_client_enable && new) {
873 /* disabled -> enabled */
874 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
875 ccb = &V_tcp_fastopen_ccache.base[i];
876 CCB_LOCK(ccb);
877 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
878 ("%s: ccb->ccb_entries not empty", __func__));
879 KASSERT(ccb->ccb_num_entries == -1,
880 ("%s: ccb->ccb_num_entries %d not -1", __func__,
881 ccb->ccb_num_entries));
882 ccb->ccb_num_entries = 0; /* enable bucket */
883 CCB_UNLOCK(ccb);
884 }
885 V_tcp_fastopen_client_enable = 1;
886 }
887 }
888 return (error);
889 }
890
891 void
tcp_fastopen_connect(struct tcpcb * tp)892 tcp_fastopen_connect(struct tcpcb *tp)
893 {
894 struct inpcb *inp = tptoinpcb(tp);
895 struct tcp_fastopen_ccache_bucket *ccb;
896 struct tcp_fastopen_ccache_entry *cce;
897 sbintime_t now;
898 uint16_t server_mss;
899 uint64_t psk_cookie;
900
901 psk_cookie = 0;
902 cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb);
903 if (cce) {
904 if (cce->disable_time == 0) {
905 if ((cce->cookie_len > 0) &&
906 (tp->t_tfo_client_cookie_len ==
907 TCP_FASTOPEN_PSK_LEN)) {
908 psk_cookie =
909 tcp_fastopen_make_psk_cookie(
910 tp->t_tfo_cookie.client,
911 cce->cookie, cce->cookie_len);
912 } else {
913 tp->t_tfo_client_cookie_len = cce->cookie_len;
914 memcpy(tp->t_tfo_cookie.client, cce->cookie,
915 cce->cookie_len);
916 }
917 server_mss = cce->server_mss;
918 CCB_UNLOCK(ccb);
919 if (tp->t_tfo_client_cookie_len ==
920 TCP_FASTOPEN_PSK_LEN && psk_cookie) {
921 tp->t_tfo_client_cookie_len =
922 TCP_FASTOPEN_COOKIE_LEN;
923 memcpy(tp->t_tfo_cookie.client, &psk_cookie,
924 TCP_FASTOPEN_COOKIE_LEN);
925 }
926 tcp_mss(tp, server_mss ? server_mss : -1);
927 tp->snd_wnd = tp->t_maxseg;
928 } else {
929 /*
930 * The path is disabled. Check the time and
931 * possibly re-enable.
932 */
933 now = getsbinuptime();
934 if (now - cce->disable_time >
935 ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) {
936 /*
937 * Re-enable path. Force a TFO cookie
938 * request. Forget the old MSS as it may be
939 * bogus now, and we will rediscover it in
940 * the SYN|ACK.
941 */
942 cce->disable_time = 0;
943 cce->server_mss = 0;
944 cce->cookie_len = 0;
945 /*
946 * tp->t_tfo... cookie details are already
947 * zero from the tcpcb init.
948 */
949 } else {
950 /*
951 * Path is disabled, so disable TFO on this
952 * connection.
953 */
954 tp->t_flags &= ~TF_FASTOPEN;
955 }
956 CCB_UNLOCK(ccb);
957 tcp_mss(tp, -1);
958 /*
959 * snd_wnd is irrelevant since we are either forcing
960 * a TFO cookie request or disabling TFO - either
961 * way, no data with the SYN.
962 */
963 }
964 } else {
965 /*
966 * A new entry for this path will be created when a SYN|ACK
967 * comes back, or the attempt otherwise fails.
968 */
969 CCB_UNLOCK(ccb);
970 tcp_mss(tp, -1);
971 /*
972 * snd_wnd is irrelevant since we are forcing a TFO cookie
973 * request.
974 */
975 }
976 }
977
978 void
tcp_fastopen_disable_path(struct tcpcb * tp)979 tcp_fastopen_disable_path(struct tcpcb *tp)
980 {
981 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc;
982 struct tcp_fastopen_ccache_bucket *ccb;
983 struct tcp_fastopen_ccache_entry *cce;
984
985 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
986 if (cce) {
987 cce->server_mss = 0;
988 cce->cookie_len = 0;
989 /*
990 * Preserve the existing disable time if it is already
991 * disabled.
992 */
993 if (cce->disable_time == 0)
994 cce->disable_time = getsbinuptime();
995 } else /* use invalid cookie len to create disabled entry */
996 tcp_fastopen_ccache_create(ccb, inc, 0,
997 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
998
999 CCB_UNLOCK(ccb);
1000 tp->t_flags &= ~TF_FASTOPEN;
1001 }
1002
1003 void
tcp_fastopen_update_cache(struct tcpcb * tp,uint16_t mss,uint8_t cookie_len,uint8_t * cookie)1004 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss,
1005 uint8_t cookie_len, uint8_t *cookie)
1006 {
1007 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc;
1008 struct tcp_fastopen_ccache_bucket *ccb;
1009 struct tcp_fastopen_ccache_entry *cce;
1010
1011 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
1012 if (cce) {
1013 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1014 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1015 ((cookie_len & 0x1) == 0)) {
1016 cce->server_mss = mss;
1017 cce->cookie_len = cookie_len;
1018 memcpy(cce->cookie, cookie, cookie_len);
1019 cce->disable_time = 0;
1020 } else {
1021 /* invalid cookie length, disable entry */
1022 cce->server_mss = 0;
1023 cce->cookie_len = 0;
1024 /*
1025 * Preserve the existing disable time if it is
1026 * already disabled.
1027 */
1028 if (cce->disable_time == 0)
1029 cce->disable_time = getsbinuptime();
1030 }
1031 } else
1032 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie);
1033
1034 CCB_UNLOCK(ccb);
1035 }
1036
1037 static struct tcp_fastopen_ccache_entry *
tcp_fastopen_ccache_lookup(struct in_conninfo * inc,struct tcp_fastopen_ccache_bucket ** ccbp)1038 tcp_fastopen_ccache_lookup(struct in_conninfo *inc,
1039 struct tcp_fastopen_ccache_bucket **ccbp)
1040 {
1041 struct tcp_fastopen_ccache_bucket *ccb;
1042 struct tcp_fastopen_ccache_entry *cce;
1043 uint32_t last_word;
1044 uint32_t hash;
1045
1046 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4,
1047 V_tcp_fastopen_ccache.secret);
1048 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4,
1049 hash);
1050 last_word = inc->inc_fport;
1051 hash = jenkins_hash32(&last_word, 1, hash);
1052 ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask];
1053 *ccbp = ccb;
1054 CCB_LOCK(ccb);
1055
1056 /*
1057 * Always returns with locked bucket.
1058 */
1059 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1060 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) &&
1061 (cce->server_port == inc->inc_ie.ie_fport) &&
1062 (((cce->af == AF_INET) &&
1063 (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) &&
1064 (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) ||
1065 ((cce->af == AF_INET6) &&
1066 IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) &&
1067 IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr))))
1068 break;
1069
1070 return (cce);
1071 }
1072
1073 static struct tcp_fastopen_ccache_entry *
tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket * ccb,struct in_conninfo * inc,uint16_t mss,uint8_t cookie_len,uint8_t * cookie)1074 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
1075 struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
1076 {
1077 struct tcp_fastopen_ccache_entry *cce;
1078
1079 /*
1080 * 1. Create a new entry, or
1081 * 2. Reclaim an existing entry, or
1082 * 3. Fail
1083 */
1084
1085 CCB_LOCK_ASSERT(ccb);
1086
1087 cce = NULL;
1088 if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
1089 cce = uma_zalloc(ccache_zone, M_NOWAIT);
1090
1091 if (cce == NULL) {
1092 /*
1093 * At bucket limit, or out of memory - reclaim last
1094 * entry in bucket.
1095 */
1096 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1097 if (cce == NULL) {
1098 /* XXX count this event */
1099 return (NULL);
1100 }
1101
1102 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1103 } else
1104 ccb->ccb_num_entries++;
1105
1106 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1107 cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET;
1108 if (cce->af == AF_INET) {
1109 cce->cce_client_ip.v4 = inc->inc_laddr;
1110 cce->cce_server_ip.v4 = inc->inc_faddr;
1111 } else {
1112 cce->cce_client_ip.v6 = inc->inc6_laddr;
1113 cce->cce_server_ip.v6 = inc->inc6_faddr;
1114 }
1115 cce->server_port = inc->inc_fport;
1116 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1117 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1118 ((cookie_len & 0x1) == 0)) {
1119 cce->server_mss = mss;
1120 cce->cookie_len = cookie_len;
1121 memcpy(cce->cookie, cookie, cookie_len);
1122 cce->disable_time = 0;
1123 } else {
1124 /* invalid cookie length, disable cce */
1125 cce->server_mss = 0;
1126 cce->cookie_len = 0;
1127 cce->disable_time = getsbinuptime();
1128 }
1129
1130 return (cce);
1131 }
1132
1133 static void
tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket * ccb,unsigned int limit)1134 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb,
1135 unsigned int limit)
1136 {
1137 struct tcp_fastopen_ccache_entry *cce, *cce_tmp;
1138 unsigned int entries;
1139
1140 CCB_LOCK(ccb);
1141 entries = 0;
1142 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1143 entries++;
1144 if (entries > limit)
1145 tcp_fastopen_ccache_entry_drop(cce, ccb);
1146 }
1147 KASSERT(ccb->ccb_num_entries <= (int)limit,
1148 ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1149 ccb->ccb_num_entries, limit));
1150 if (limit == 0) {
1151 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1152 ("%s: ccb->ccb_entries not empty", __func__));
1153 ccb->ccb_num_entries = -1; /* disable bucket */
1154 }
1155 CCB_UNLOCK(ccb);
1156 }
1157
1158 static void
tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry * cce,struct tcp_fastopen_ccache_bucket * ccb)1159 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
1160 struct tcp_fastopen_ccache_bucket *ccb)
1161 {
1162
1163 CCB_LOCK_ASSERT(ccb);
1164
1165 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1166 ccb->ccb_num_entries--;
1167 uma_zfree(ccache_zone, cce);
1168 }
1169
1170 static int
sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)1171 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)
1172 {
1173 struct sbuf sb;
1174 struct tcp_fastopen_ccache_bucket *ccb;
1175 struct tcp_fastopen_ccache_entry *cce;
1176 sbintime_t now, duration, limit;
1177 const int linesize = 128;
1178 int i, error, num_entries;
1179 unsigned int j;
1180 #ifdef INET6
1181 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN];
1182 #else
1183 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN];
1184 #endif
1185
1186 if (jailed_without_vnet(curthread->td_ucred) != 0)
1187 return (EPERM);
1188
1189 /* Only allow root to read the client cookie cache */
1190 if (curthread->td_ucred->cr_uid != 0)
1191 return (EPERM);
1192
1193 num_entries = 0;
1194 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1195 ccb = &V_tcp_fastopen_ccache.base[i];
1196 CCB_LOCK(ccb);
1197 if (ccb->ccb_num_entries > 0)
1198 num_entries += ccb->ccb_num_entries;
1199 CCB_UNLOCK(ccb);
1200 }
1201 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL);
1202
1203 sbuf_printf(&sb,
1204 "\nLocal IP address Remote IP address Port MSS"
1205 " Disabled Cookie\n");
1206
1207 now = getsbinuptime();
1208 limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32;
1209 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1210 ccb = &V_tcp_fastopen_ccache.base[i];
1211 CCB_LOCK(ccb);
1212 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) {
1213 if (cce->disable_time != 0) {
1214 duration = now - cce->disable_time;
1215 if (limit >= duration)
1216 duration = limit - duration;
1217 else
1218 duration = 0;
1219 } else
1220 duration = 0;
1221 sbuf_printf(&sb,
1222 "%-20s %-20s %5u %5u ",
1223 inet_ntop(cce->af, &cce->cce_client_ip,
1224 clt_buf, sizeof(clt_buf)),
1225 inet_ntop(cce->af, &cce->cce_server_ip,
1226 srv_buf, sizeof(srv_buf)),
1227 ntohs(cce->server_port),
1228 cce->server_mss);
1229 if (duration > 0)
1230 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration));
1231 else
1232 sbuf_printf(&sb, "%8s ", "No");
1233 for (j = 0; j < cce->cookie_len; j++)
1234 sbuf_printf(&sb, "%02x", cce->cookie[j]);
1235 sbuf_putc(&sb, '\n');
1236 }
1237 CCB_UNLOCK(ccb);
1238 }
1239 error = sbuf_finish(&sb);
1240 if (error == 0)
1241 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
1242 sbuf_delete(&sb);
1243 return (error);
1244 }
1245