1 /*- 2 * Copyright (c) 2015-2017 Patrick Kelsey 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include 29 * this code, add the following line to your kernel config: 30 * 31 * options TCP_RFC7413 32 * 33 * 34 * The generated TFO cookies are the 64-bit output of 35 * SipHash24(key=<16-byte-key>, msg=<client-ip>). Multiple concurrent valid 36 * keys are supported so that time-based rolling cookie invalidation 37 * policies can be implemented in the system. The default number of 38 * concurrent keys is 2. This can be adjusted in the kernel config as 39 * follows: 40 * 41 * options TCP_RFC7413_MAX_KEYS=<num-keys> 42 * 43 * 44 * In addition to the facilities defined in RFC7413, this implementation 45 * supports a pre-shared key (PSK) mode of operation in which the TFO server 46 * requires the client to be in possession of a shared secret in order for 47 * the client to be able to successfully open TFO connections with the 48 * server. This is useful, for example, in environments where TFO servers 49 * are exposed to both internal and external clients and only wish to allow 50 * TFO connections from internal clients. 51 * 52 * In the PSK mode of operation, the server generates and sends TFO cookies 53 * to requesting clients as usual. However, when validating cookies 54 * received in TFO SYNs from clients, the server requires the 55 * client-supplied cookie to equal SipHash24(key=<16-byte-psk>, 56 * msg=<cookie-sent-to-client>). 57 * 58 * Multiple concurrent valid pre-shared keys are supported so that 59 * time-based rolling PSK invalidation policies can be implemented in the 60 * system. The default number of concurrent pre-shared keys is 2. This can 61 * be adjusted in the kernel config as follows: 62 * 63 * options TCP_RFC7413_MAX_PSKS=<num-psks> 64 * 65 * 66 * The following TFO-specific sysctls are defined: 67 * 68 * net.inet.tcp.fastopen.acceptany (RW, default 0) 69 * When non-zero, all client-supplied TFO cookies will be considered to 70 * be valid. 71 * 72 * net.inet.tcp.fastopen.autokey (RW, default 120) 73 * When this and net.inet.tcp.fastopen.server_enable are non-zero, a new 74 * key will be automatically generated after this many seconds. 75 * 76 * net.inet.tcp.fastopen.ccache_bucket_limit 77 * (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT) 78 * The maximum number of entries in a client cookie cache bucket. 79 * 80 * net.inet.tcp.fastopen.ccache_buckets 81 * (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT) 82 * The number of client cookie cache buckets. 83 * 84 * net.inet.tcp.fastopen.ccache_list (RO) 85 * Print the client cookie cache. 86 * 87 * net.inet.tcp.fastopen.client_enable (RW, default 1) 88 * When zero, no new active (i.e., client) TFO connections can be 89 * created. On the transition from enabled to disabled, the client 90 * cookie cache is cleared and disabled. The transition from enabled to 91 * disabled does not affect any active TFO connections in progress; it 92 * only prevents new ones from being made. 93 * 94 * net.inet.tcp.fastopen.keylen (RD) 95 * The key length in bytes. 96 * 97 * net.inet.tcp.fastopen.maxkeys (RD) 98 * The maximum number of keys supported. 99 * 100 * net.inet.tcp.fastopen.maxpsks (RD) 101 * The maximum number of pre-shared keys supported. 102 * 103 * net.inet.tcp.fastopen.numkeys (RD) 104 * The current number of keys installed. 105 * 106 * net.inet.tcp.fastopen.numpsks (RD) 107 * The current number of pre-shared keys installed. 108 * 109 * net.inet.tcp.fastopen.path_disable_time 110 * (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT) 111 * When a failure occurs while trying to create a new active (i.e., 112 * client) TFO connection, new active connections on the same path, as 113 * determined by the tuple {client_ip, server_ip, server_port}, will be 114 * forced to be non-TFO for this many seconds. Note that the path 115 * disable mechanism relies on state stored in client cookie cache 116 * entries, so it is possible for the disable time for a given path to 117 * be reduced if the corresponding client cookie cache entry is reused 118 * due to resource pressure before the disable period has elapsed. 119 * 120 * net.inet.tcp.fastopen.psk_enable (RW, default 0) 121 * When non-zero, pre-shared key (PSK) mode is enabled for all TFO 122 * servers. On the transition from enabled to disabled, all installed 123 * pre-shared keys are removed. 124 * 125 * net.inet.tcp.fastopen.server_enable (RW, default 0) 126 * When zero, no new passive (i.e., server) TFO connections can be 127 * created. On the transition from enabled to disabled, all installed 128 * keys and pre-shared keys are removed. On the transition from 129 * disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and 130 * there are no keys installed, a new key will be generated immediately. 131 * The transition from enabled to disabled does not affect any passive 132 * TFO connections in progress; it only prevents new ones from being 133 * made. 134 * 135 * net.inet.tcp.fastopen.setkey (WR) 136 * Install a new key by writing net.inet.tcp.fastopen.keylen bytes to 137 * this sysctl. 138 * 139 * net.inet.tcp.fastopen.setpsk (WR) 140 * Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen 141 * bytes to this sysctl. 142 * 143 * In order for TFO connections to be created via a listen socket, that 144 * socket must have the TCP_FASTOPEN socket option set on it. This option 145 * can be set on the socket either before or after the listen() is invoked. 146 * Clearing this option on a listen socket after it has been set has no 147 * effect on existing TFO connections or TFO connections in progress; it 148 * only prevents new TFO connections from being made. 149 * 150 * For passively-created sockets, the TCP_FASTOPEN socket option can be 151 * queried to determine whether the connection was established using TFO. 152 * Note that connections that are established via a TFO SYN, but that fall 153 * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option 154 * set. 155 * 156 * Per the RFC, this implementation limits the number of TFO connections 157 * that can be in the SYN_RECEIVED state on a per listen-socket basis. 158 * Whenever this limit is exceeded, requests for new TFO connections are 159 * serviced as non-TFO requests. Without such a limit, given a valid TFO 160 * cookie, an attacker could keep the listen queue in an overflow condition 161 * using a TFO SYN flood. This implementation sets the limit at half the 162 * configured listen backlog. 163 * 164 */ 165 166 #include "opt_inet.h" 167 168 #include <sys/param.h> 169 #include <sys/jail.h> 170 #include <sys/kernel.h> 171 #include <sys/hash.h> 172 #include <sys/limits.h> 173 #include <sys/lock.h> 174 #include <sys/proc.h> 175 #include <sys/rmlock.h> 176 #include <sys/sbuf.h> 177 #include <sys/socket.h> 178 #include <sys/socketvar.h> 179 #include <sys/sysctl.h> 180 #include <sys/systm.h> 181 182 #include <crypto/siphash/siphash.h> 183 184 #include <net/vnet.h> 185 186 #include <netinet/in.h> 187 #include <netinet/in_pcb.h> 188 #include <netinet/tcp_var.h> 189 #include <netinet/tcp_fastopen.h> 190 191 #define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH 192 193 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN 194 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN 195 #endif 196 197 /* 198 * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold 199 * the PSK until the connect occurs. 200 */ 201 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN 202 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN 203 #endif 204 205 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16 206 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048 /* must be power of 2 */ 207 208 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900 /* seconds */ 209 210 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1) 211 #define TCP_FASTOPEN_MAX_KEYS 2 212 #else 213 #define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS 214 #endif 215 216 #if TCP_FASTOPEN_MAX_KEYS > 10 217 #undef TCP_FASTOPEN_MAX_KEYS 218 #define TCP_FASTOPEN_MAX_KEYS 10 219 #endif 220 221 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1) 222 #define TCP_FASTOPEN_MAX_PSKS 2 223 #else 224 #define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS 225 #endif 226 227 #if TCP_FASTOPEN_MAX_PSKS > 10 228 #undef TCP_FASTOPEN_MAX_PSKS 229 #define TCP_FASTOPEN_MAX_PSKS 10 230 #endif 231 232 struct tcp_fastopen_keylist { 233 unsigned int newest; 234 unsigned int newest_psk; 235 uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN]; 236 uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN]; 237 }; 238 239 struct tcp_fastopen_callout { 240 struct callout c; 241 struct vnet *v; 242 }; 243 244 union tcp_fastopen_ip_addr { 245 struct in_addr v4; 246 struct in6_addr v6; 247 }; 248 249 struct tcp_fastopen_ccache_entry { 250 TAILQ_ENTRY(tcp_fastopen_ccache_entry) cce_link; 251 union tcp_fastopen_ip_addr cce_client_ip; /* network byte order */ 252 union tcp_fastopen_ip_addr cce_server_ip; /* network byte order */ 253 uint16_t server_port; /* network byte order */ 254 uint16_t server_mss; /* host byte order */ 255 uint8_t af; 256 uint8_t cookie_len; 257 uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN]; 258 sbintime_t disable_time; /* non-zero value means path is disabled */ 259 }; 260 261 struct tcp_fastopen_ccache; 262 263 struct tcp_fastopen_ccache_bucket { 264 struct mtx ccb_mtx; 265 TAILQ_HEAD(bucket_entries, tcp_fastopen_ccache_entry) ccb_entries; 266 int ccb_num_entries; 267 struct tcp_fastopen_ccache *ccb_ccache; 268 }; 269 270 struct tcp_fastopen_ccache { 271 struct tcp_fastopen_ccache_bucket *base; 272 unsigned int bucket_limit; 273 unsigned int buckets; 274 unsigned int mask; 275 uint32_t secret; 276 }; 277 278 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup( 279 struct in_conninfo *, struct tcp_fastopen_ccache_bucket **); 280 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create( 281 struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t, 282 uint8_t *); 283 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *, 284 unsigned int); 285 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *, 286 struct tcp_fastopen_ccache_bucket *); 287 288 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 289 "TCP Fast Open"); 290 291 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0; 292 #define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany) 293 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany, 294 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0, 295 "Accept any non-empty cookie"); 296 297 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120; 298 #define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey) 299 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS); 300 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey, 301 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 302 NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU", 303 "Number of seconds between auto-generation of a new key; zero disables"); 304 305 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS); 306 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit, 307 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_NEEDGIANT, 308 NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU", 309 "Max entries per bucket in client cookie cache"); 310 311 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) = 312 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT; 313 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets) 314 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets, 315 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0, 316 "Client cookie cache number of buckets (power of 2)"); 317 318 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1; 319 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS); 320 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable, 321 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 322 NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU", 323 "Enable/disable TCP Fast Open client functionality"); 324 325 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen, 326 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN, 327 "Key length in bytes"); 328 329 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys, 330 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS, 331 "Maximum number of keys supported"); 332 333 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks, 334 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS, 335 "Maximum number of pre-shared keys supported"); 336 337 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0; 338 #define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys) 339 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys, 340 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0, 341 "Number of keys installed"); 342 343 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0; 344 #define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks) 345 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks, 346 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0, 347 "Number of pre-shared keys installed"); 348 349 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) = 350 TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT; 351 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time) 352 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time, 353 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0, 354 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path"); 355 356 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0; 357 #define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable) 358 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS); 359 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable, 360 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 361 NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU", 362 "Enable/disable TCP Fast Open server pre-shared key mode"); 363 364 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0; 365 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS); 366 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable, 367 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 368 NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU", 369 "Enable/disable TCP Fast Open server functionality"); 370 371 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS); 372 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey, 373 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE, 374 NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "", 375 "Install a new key"); 376 377 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS); 378 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk, 379 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE, 380 NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "", 381 "Install a new pre-shared key"); 382 383 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS); 384 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list, 385 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 386 NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A", 387 "List of all client cookie cache entries"); 388 389 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock); 390 #define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock) 391 392 #define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t)) 393 #define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t)) 394 #define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock) 395 #define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock) 396 397 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys); 398 #define V_tcp_fastopen_keys VNET(tcp_fastopen_keys) 399 400 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx); 401 #define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx) 402 403 static uma_zone_t counter_zone; 404 static uma_zone_t ccache_zone; 405 406 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets"); 407 408 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache); 409 #define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache) 410 411 #define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx) 412 #define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx) 413 #define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED) 414 415 void 416 tcp_fastopen_init(void) 417 { 418 counter_zone = uma_zcreate("tfo", sizeof(unsigned int), 419 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 420 ccache_zone = uma_zcreate("tfo_ccache_entries", 421 sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL, 422 UMA_ALIGN_CACHE, 0); 423 } 424 425 void 426 tcp_fastopen_vnet_init(void) 427 { 428 unsigned int i; 429 430 rm_init(&V_tcp_fastopen_keylock, "tfo_keylock"); 431 callout_init_rm(&V_tcp_fastopen_autokey_ctx.c, 432 &V_tcp_fastopen_keylock, 0); 433 V_tcp_fastopen_autokey_ctx.v = curvnet; 434 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1; 435 V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1; 436 437 TUNABLE_INT_FETCH("net.inet.tcp.fastopen.ccache_bucket_limit", 438 &V_tcp_fastopen_ccache.bucket_limit); 439 if (V_tcp_fastopen_ccache.bucket_limit == 0) 440 V_tcp_fastopen_ccache.bucket_limit = 441 TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT; 442 443 /* May already be non-zero if kernel tunable was set */ 444 if ((V_tcp_fastopen_ccache_buckets == 0) || 445 !powerof2(V_tcp_fastopen_ccache_buckets)) 446 V_tcp_fastopen_ccache.buckets = 447 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT; 448 else 449 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets; 450 451 V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1; 452 V_tcp_fastopen_ccache.secret = arc4random(); 453 454 V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets * 455 sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE, 456 M_WAITOK | M_ZERO); 457 458 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 459 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries); 460 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket", 461 NULL, MTX_DEF); 462 if (V_tcp_fastopen_client_enable) { 463 /* enable bucket */ 464 V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0; 465 } else { 466 /* disable bucket */ 467 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1; 468 } 469 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache; 470 } 471 } 472 473 void 474 tcp_fastopen_vnet_destroy(void) 475 { 476 struct tcp_fastopen_ccache_bucket *ccb; 477 unsigned int i; 478 479 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 480 ccb = &V_tcp_fastopen_ccache.base[i]; 481 tcp_fastopen_ccache_bucket_trim(ccb, 0); 482 mtx_destroy(&ccb->ccb_mtx); 483 } 484 485 free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE); 486 487 callout_drain(&V_tcp_fastopen_autokey_ctx.c); 488 rm_destroy(&V_tcp_fastopen_keylock); 489 } 490 491 unsigned int * 492 tcp_fastopen_alloc_counter(void) 493 { 494 unsigned int *counter; 495 496 counter = uma_zalloc(counter_zone, M_NOWAIT); 497 if (counter) 498 *counter = 1; 499 return (counter); 500 } 501 502 void 503 tcp_fastopen_decrement_counter(unsigned int *counter) 504 { 505 if (*counter == 1) 506 uma_zfree(counter_zone, counter); 507 else 508 atomic_subtract_int(counter, 1); 509 } 510 511 static void 512 tcp_fastopen_addkey_locked(uint8_t *key) 513 { 514 515 V_tcp_fastopen_keys.newest++; 516 if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS) 517 V_tcp_fastopen_keys.newest = 0; 518 memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key, 519 TCP_FASTOPEN_KEY_LEN); 520 if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS) 521 V_tcp_fastopen_numkeys++; 522 } 523 524 static void 525 tcp_fastopen_addpsk_locked(uint8_t *psk) 526 { 527 528 V_tcp_fastopen_keys.newest_psk++; 529 if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS) 530 V_tcp_fastopen_keys.newest_psk = 0; 531 memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk, 532 TCP_FASTOPEN_KEY_LEN); 533 if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS) 534 V_tcp_fastopen_numpsks++; 535 } 536 537 static void 538 tcp_fastopen_autokey_locked(void) 539 { 540 uint8_t newkey[TCP_FASTOPEN_KEY_LEN]; 541 542 arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0); 543 tcp_fastopen_addkey_locked(newkey); 544 } 545 546 static void 547 tcp_fastopen_autokey_callout(void *arg) 548 { 549 struct tcp_fastopen_callout *ctx = arg; 550 551 CURVNET_SET(ctx->v); 552 tcp_fastopen_autokey_locked(); 553 callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz, 554 tcp_fastopen_autokey_callout, ctx); 555 CURVNET_RESTORE(); 556 } 557 558 static uint64_t 559 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc) 560 { 561 SIPHASH_CTX ctx; 562 uint64_t siphash; 563 564 SipHash24_Init(&ctx); 565 SipHash_SetKey(&ctx, key); 566 switch (inc->inc_flags & INC_ISIPV6) { 567 #ifdef INET 568 case 0: 569 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr)); 570 break; 571 #endif 572 #ifdef INET6 573 case INC_ISIPV6: 574 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr)); 575 break; 576 #endif 577 } 578 SipHash_Final((u_int8_t *)&siphash, &ctx); 579 580 return (siphash); 581 } 582 583 static uint64_t 584 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len) 585 { 586 SIPHASH_CTX ctx; 587 uint64_t psk_cookie; 588 589 SipHash24_Init(&ctx); 590 SipHash_SetKey(&ctx, psk); 591 SipHash_Update(&ctx, cookie, cookie_len); 592 SipHash_Final((u_int8_t *)&psk_cookie, &ctx); 593 594 return (psk_cookie); 595 } 596 597 static int 598 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie) 599 { 600 unsigned int i, psk_index; 601 uint64_t psk_cookie; 602 603 if (V_tcp_fastopen_psk_enable) { 604 psk_index = V_tcp_fastopen_keys.newest_psk; 605 for (i = 0; i < V_tcp_fastopen_numpsks; i++) { 606 psk_cookie = 607 tcp_fastopen_make_psk_cookie( 608 V_tcp_fastopen_keys.psk[psk_index], 609 (uint8_t *)cur_cookie, 610 TCP_FASTOPEN_COOKIE_LEN); 611 612 if (memcmp(wire_cookie, &psk_cookie, 613 TCP_FASTOPEN_COOKIE_LEN) == 0) 614 return (1); 615 616 if (psk_index == 0) 617 psk_index = TCP_FASTOPEN_MAX_PSKS - 1; 618 else 619 psk_index--; 620 } 621 } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0) 622 return (1); 623 624 return (0); 625 } 626 627 /* 628 * Return values: 629 * -1 the cookie is invalid and no valid cookie is available 630 * 0 the cookie is invalid and the latest cookie has been returned 631 * 1 the cookie is valid and the latest cookie has been returned 632 */ 633 int 634 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie, 635 unsigned int len, uint64_t *latest_cookie) 636 { 637 struct rm_priotracker tracker; 638 unsigned int i, key_index; 639 int rv; 640 uint64_t cur_cookie; 641 642 if (V_tcp_fastopen_acceptany) { 643 *latest_cookie = 0; 644 return (1); 645 } 646 647 TCP_FASTOPEN_KEYS_RLOCK(&tracker); 648 if (len != TCP_FASTOPEN_COOKIE_LEN) { 649 if (V_tcp_fastopen_numkeys > 0) { 650 *latest_cookie = 651 tcp_fastopen_make_cookie( 652 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], 653 inc); 654 rv = 0; 655 } else 656 rv = -1; 657 goto out; 658 } 659 660 /* 661 * Check against each available key, from newest to oldest. 662 */ 663 key_index = V_tcp_fastopen_keys.newest; 664 for (i = 0; i < V_tcp_fastopen_numkeys; i++) { 665 cur_cookie = 666 tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index], 667 inc); 668 if (i == 0) 669 *latest_cookie = cur_cookie; 670 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie); 671 if (rv) 672 goto out; 673 if (key_index == 0) 674 key_index = TCP_FASTOPEN_MAX_KEYS - 1; 675 else 676 key_index--; 677 } 678 rv = 0; 679 680 out: 681 TCP_FASTOPEN_KEYS_RUNLOCK(&tracker); 682 return (rv); 683 } 684 685 static int 686 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS) 687 { 688 int error; 689 unsigned int new; 690 691 new = V_tcp_fastopen_autokey; 692 error = sysctl_handle_int(oidp, &new, 0, req); 693 if (error == 0 && req->newptr) { 694 if (new > (INT_MAX / hz)) 695 return (EINVAL); 696 697 TCP_FASTOPEN_KEYS_WLOCK(); 698 if (V_tcp_fastopen_server_enable) { 699 if (V_tcp_fastopen_autokey && !new) 700 callout_stop(&V_tcp_fastopen_autokey_ctx.c); 701 else if (new) 702 callout_reset(&V_tcp_fastopen_autokey_ctx.c, 703 new * hz, tcp_fastopen_autokey_callout, 704 &V_tcp_fastopen_autokey_ctx); 705 } 706 V_tcp_fastopen_autokey = new; 707 TCP_FASTOPEN_KEYS_WUNLOCK(); 708 } 709 710 return (error); 711 } 712 713 static int 714 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS) 715 { 716 int error; 717 unsigned int new; 718 719 new = V_tcp_fastopen_psk_enable; 720 error = sysctl_handle_int(oidp, &new, 0, req); 721 if (error == 0 && req->newptr) { 722 if (V_tcp_fastopen_psk_enable && !new) { 723 /* enabled -> disabled */ 724 TCP_FASTOPEN_KEYS_WLOCK(); 725 V_tcp_fastopen_numpsks = 0; 726 V_tcp_fastopen_keys.newest_psk = 727 TCP_FASTOPEN_MAX_PSKS - 1; 728 V_tcp_fastopen_psk_enable = 0; 729 TCP_FASTOPEN_KEYS_WUNLOCK(); 730 } else if (!V_tcp_fastopen_psk_enable && new) { 731 /* disabled -> enabled */ 732 TCP_FASTOPEN_KEYS_WLOCK(); 733 V_tcp_fastopen_psk_enable = 1; 734 TCP_FASTOPEN_KEYS_WUNLOCK(); 735 } 736 } 737 return (error); 738 } 739 740 static int 741 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS) 742 { 743 int error; 744 unsigned int new; 745 746 new = V_tcp_fastopen_server_enable; 747 error = sysctl_handle_int(oidp, &new, 0, req); 748 if (error == 0 && req->newptr) { 749 if (V_tcp_fastopen_server_enable && !new) { 750 /* enabled -> disabled */ 751 TCP_FASTOPEN_KEYS_WLOCK(); 752 V_tcp_fastopen_numkeys = 0; 753 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1; 754 if (V_tcp_fastopen_autokey) 755 callout_stop(&V_tcp_fastopen_autokey_ctx.c); 756 V_tcp_fastopen_numpsks = 0; 757 V_tcp_fastopen_keys.newest_psk = 758 TCP_FASTOPEN_MAX_PSKS - 1; 759 V_tcp_fastopen_server_enable = 0; 760 TCP_FASTOPEN_KEYS_WUNLOCK(); 761 } else if (!V_tcp_fastopen_server_enable && new) { 762 /* disabled -> enabled */ 763 TCP_FASTOPEN_KEYS_WLOCK(); 764 if (V_tcp_fastopen_autokey && 765 (V_tcp_fastopen_numkeys == 0)) { 766 tcp_fastopen_autokey_locked(); 767 callout_reset(&V_tcp_fastopen_autokey_ctx.c, 768 V_tcp_fastopen_autokey * hz, 769 tcp_fastopen_autokey_callout, 770 &V_tcp_fastopen_autokey_ctx); 771 } 772 V_tcp_fastopen_server_enable = 1; 773 TCP_FASTOPEN_KEYS_WUNLOCK(); 774 } 775 } 776 return (error); 777 } 778 779 static int 780 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS) 781 { 782 int error; 783 uint8_t newkey[TCP_FASTOPEN_KEY_LEN]; 784 785 if (req->oldptr != NULL || req->oldlen != 0) 786 return (EINVAL); 787 if (req->newptr == NULL) 788 return (EPERM); 789 if (req->newlen != sizeof(newkey)) 790 return (EINVAL); 791 error = SYSCTL_IN(req, newkey, sizeof(newkey)); 792 if (error) 793 return (error); 794 795 TCP_FASTOPEN_KEYS_WLOCK(); 796 tcp_fastopen_addkey_locked(newkey); 797 TCP_FASTOPEN_KEYS_WUNLOCK(); 798 799 return (0); 800 } 801 802 static int 803 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS) 804 { 805 int error; 806 uint8_t newpsk[TCP_FASTOPEN_KEY_LEN]; 807 808 if (req->oldptr != NULL || req->oldlen != 0) 809 return (EINVAL); 810 if (req->newptr == NULL) 811 return (EPERM); 812 if (req->newlen != sizeof(newpsk)) 813 return (EINVAL); 814 error = SYSCTL_IN(req, newpsk, sizeof(newpsk)); 815 if (error) 816 return (error); 817 818 TCP_FASTOPEN_KEYS_WLOCK(); 819 tcp_fastopen_addpsk_locked(newpsk); 820 TCP_FASTOPEN_KEYS_WUNLOCK(); 821 822 return (0); 823 } 824 825 static int 826 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS) 827 { 828 struct tcp_fastopen_ccache_bucket *ccb; 829 int error; 830 unsigned int new; 831 unsigned int i; 832 833 new = V_tcp_fastopen_ccache.bucket_limit; 834 error = sysctl_handle_int(oidp, &new, 0, req); 835 if (error == 0 && req->newptr) { 836 if ((new == 0) || (new > INT_MAX)) 837 error = EINVAL; 838 else { 839 if (new < V_tcp_fastopen_ccache.bucket_limit) { 840 for (i = 0; i < V_tcp_fastopen_ccache.buckets; 841 i++) { 842 ccb = &V_tcp_fastopen_ccache.base[i]; 843 tcp_fastopen_ccache_bucket_trim(ccb, new); 844 } 845 } 846 V_tcp_fastopen_ccache.bucket_limit = new; 847 } 848 } 849 return (error); 850 } 851 852 static int 853 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS) 854 { 855 struct tcp_fastopen_ccache_bucket *ccb; 856 int error; 857 unsigned int new, i; 858 859 new = V_tcp_fastopen_client_enable; 860 error = sysctl_handle_int(oidp, &new, 0, req); 861 if (error == 0 && req->newptr) { 862 if (V_tcp_fastopen_client_enable && !new) { 863 /* enabled -> disabled */ 864 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 865 ccb = &V_tcp_fastopen_ccache.base[i]; 866 KASSERT(ccb->ccb_num_entries > -1, 867 ("%s: ccb->ccb_num_entries %d is negative", 868 __func__, ccb->ccb_num_entries)); 869 tcp_fastopen_ccache_bucket_trim(ccb, 0); 870 } 871 V_tcp_fastopen_client_enable = 0; 872 } else if (!V_tcp_fastopen_client_enable && new) { 873 /* disabled -> enabled */ 874 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 875 ccb = &V_tcp_fastopen_ccache.base[i]; 876 CCB_LOCK(ccb); 877 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries), 878 ("%s: ccb->ccb_entries not empty", __func__)); 879 KASSERT(ccb->ccb_num_entries == -1, 880 ("%s: ccb->ccb_num_entries %d not -1", __func__, 881 ccb->ccb_num_entries)); 882 ccb->ccb_num_entries = 0; /* enable bucket */ 883 CCB_UNLOCK(ccb); 884 } 885 V_tcp_fastopen_client_enable = 1; 886 } 887 } 888 return (error); 889 } 890 891 void 892 tcp_fastopen_connect(struct tcpcb *tp) 893 { 894 struct inpcb *inp = tptoinpcb(tp); 895 struct tcp_fastopen_ccache_bucket *ccb; 896 struct tcp_fastopen_ccache_entry *cce; 897 sbintime_t now; 898 uint16_t server_mss; 899 uint64_t psk_cookie; 900 901 psk_cookie = 0; 902 cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb); 903 if (cce) { 904 if (cce->disable_time == 0) { 905 if ((cce->cookie_len > 0) && 906 (tp->t_tfo_client_cookie_len == 907 TCP_FASTOPEN_PSK_LEN)) { 908 psk_cookie = 909 tcp_fastopen_make_psk_cookie( 910 tp->t_tfo_cookie.client, 911 cce->cookie, cce->cookie_len); 912 } else { 913 tp->t_tfo_client_cookie_len = cce->cookie_len; 914 memcpy(tp->t_tfo_cookie.client, cce->cookie, 915 cce->cookie_len); 916 } 917 server_mss = cce->server_mss; 918 CCB_UNLOCK(ccb); 919 if (tp->t_tfo_client_cookie_len == 920 TCP_FASTOPEN_PSK_LEN && psk_cookie) { 921 tp->t_tfo_client_cookie_len = 922 TCP_FASTOPEN_COOKIE_LEN; 923 memcpy(tp->t_tfo_cookie.client, &psk_cookie, 924 TCP_FASTOPEN_COOKIE_LEN); 925 } 926 tcp_mss(tp, server_mss ? server_mss : -1); 927 tp->snd_wnd = tp->t_maxseg; 928 } else { 929 /* 930 * The path is disabled. Check the time and 931 * possibly re-enable. 932 */ 933 now = getsbinuptime(); 934 if (now - cce->disable_time > 935 ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) { 936 /* 937 * Re-enable path. Force a TFO cookie 938 * request. Forget the old MSS as it may be 939 * bogus now, and we will rediscover it in 940 * the SYN|ACK. 941 */ 942 cce->disable_time = 0; 943 cce->server_mss = 0; 944 cce->cookie_len = 0; 945 /* 946 * tp->t_tfo... cookie details are already 947 * zero from the tcpcb init. 948 */ 949 } else { 950 /* 951 * Path is disabled, so disable TFO on this 952 * connection. 953 */ 954 tp->t_flags &= ~TF_FASTOPEN; 955 } 956 CCB_UNLOCK(ccb); 957 tcp_mss(tp, -1); 958 /* 959 * snd_wnd is irrelevant since we are either forcing 960 * a TFO cookie request or disabling TFO - either 961 * way, no data with the SYN. 962 */ 963 } 964 } else { 965 /* 966 * A new entry for this path will be created when a SYN|ACK 967 * comes back, or the attempt otherwise fails. 968 */ 969 CCB_UNLOCK(ccb); 970 tcp_mss(tp, -1); 971 /* 972 * snd_wnd is irrelevant since we are forcing a TFO cookie 973 * request. 974 */ 975 } 976 } 977 978 void 979 tcp_fastopen_disable_path(struct tcpcb *tp) 980 { 981 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc; 982 struct tcp_fastopen_ccache_bucket *ccb; 983 struct tcp_fastopen_ccache_entry *cce; 984 985 cce = tcp_fastopen_ccache_lookup(inc, &ccb); 986 if (cce) { 987 cce->server_mss = 0; 988 cce->cookie_len = 0; 989 /* 990 * Preserve the existing disable time if it is already 991 * disabled. 992 */ 993 if (cce->disable_time == 0) 994 cce->disable_time = getsbinuptime(); 995 } else /* use invalid cookie len to create disabled entry */ 996 tcp_fastopen_ccache_create(ccb, inc, 0, 997 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL); 998 999 CCB_UNLOCK(ccb); 1000 tp->t_flags &= ~TF_FASTOPEN; 1001 } 1002 1003 void 1004 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss, 1005 uint8_t cookie_len, uint8_t *cookie) 1006 { 1007 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc; 1008 struct tcp_fastopen_ccache_bucket *ccb; 1009 struct tcp_fastopen_ccache_entry *cce; 1010 1011 cce = tcp_fastopen_ccache_lookup(inc, &ccb); 1012 if (cce) { 1013 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) && 1014 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) && 1015 ((cookie_len & 0x1) == 0)) { 1016 cce->server_mss = mss; 1017 cce->cookie_len = cookie_len; 1018 memcpy(cce->cookie, cookie, cookie_len); 1019 cce->disable_time = 0; 1020 } else { 1021 /* invalid cookie length, disable entry */ 1022 cce->server_mss = 0; 1023 cce->cookie_len = 0; 1024 /* 1025 * Preserve the existing disable time if it is 1026 * already disabled. 1027 */ 1028 if (cce->disable_time == 0) 1029 cce->disable_time = getsbinuptime(); 1030 } 1031 } else 1032 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie); 1033 1034 CCB_UNLOCK(ccb); 1035 } 1036 1037 static struct tcp_fastopen_ccache_entry * 1038 tcp_fastopen_ccache_lookup(struct in_conninfo *inc, 1039 struct tcp_fastopen_ccache_bucket **ccbp) 1040 { 1041 struct tcp_fastopen_ccache_bucket *ccb; 1042 struct tcp_fastopen_ccache_entry *cce; 1043 uint32_t last_word; 1044 uint32_t hash; 1045 1046 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4, 1047 V_tcp_fastopen_ccache.secret); 1048 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4, 1049 hash); 1050 last_word = inc->inc_fport; 1051 hash = jenkins_hash32(&last_word, 1, hash); 1052 ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask]; 1053 *ccbp = ccb; 1054 CCB_LOCK(ccb); 1055 1056 /* 1057 * Always returns with locked bucket. 1058 */ 1059 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) 1060 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) && 1061 (cce->server_port == inc->inc_ie.ie_fport) && 1062 (((cce->af == AF_INET) && 1063 (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) && 1064 (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) || 1065 ((cce->af == AF_INET6) && 1066 IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) && 1067 IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr)))) 1068 break; 1069 1070 return (cce); 1071 } 1072 1073 static struct tcp_fastopen_ccache_entry * 1074 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb, 1075 struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie) 1076 { 1077 struct tcp_fastopen_ccache_entry *cce; 1078 1079 /* 1080 * 1. Create a new entry, or 1081 * 2. Reclaim an existing entry, or 1082 * 3. Fail 1083 */ 1084 1085 CCB_LOCK_ASSERT(ccb); 1086 1087 cce = NULL; 1088 if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit) 1089 cce = uma_zalloc(ccache_zone, M_NOWAIT); 1090 1091 if (cce == NULL) { 1092 /* 1093 * At bucket limit, or out of memory - reclaim last 1094 * entry in bucket. 1095 */ 1096 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries); 1097 if (cce == NULL) { 1098 /* XXX count this event */ 1099 return (NULL); 1100 } 1101 1102 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link); 1103 } else 1104 ccb->ccb_num_entries++; 1105 1106 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link); 1107 cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET; 1108 if (cce->af == AF_INET) { 1109 cce->cce_client_ip.v4 = inc->inc_laddr; 1110 cce->cce_server_ip.v4 = inc->inc_faddr; 1111 } else { 1112 cce->cce_client_ip.v6 = inc->inc6_laddr; 1113 cce->cce_server_ip.v6 = inc->inc6_faddr; 1114 } 1115 cce->server_port = inc->inc_fport; 1116 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) && 1117 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) && 1118 ((cookie_len & 0x1) == 0)) { 1119 cce->server_mss = mss; 1120 cce->cookie_len = cookie_len; 1121 memcpy(cce->cookie, cookie, cookie_len); 1122 cce->disable_time = 0; 1123 } else { 1124 /* invalid cookie length, disable cce */ 1125 cce->server_mss = 0; 1126 cce->cookie_len = 0; 1127 cce->disable_time = getsbinuptime(); 1128 } 1129 1130 return (cce); 1131 } 1132 1133 static void 1134 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb, 1135 unsigned int limit) 1136 { 1137 struct tcp_fastopen_ccache_entry *cce, *cce_tmp; 1138 unsigned int entries; 1139 1140 CCB_LOCK(ccb); 1141 entries = 0; 1142 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) { 1143 entries++; 1144 if (entries > limit) 1145 tcp_fastopen_ccache_entry_drop(cce, ccb); 1146 } 1147 KASSERT(ccb->ccb_num_entries <= (int)limit, 1148 ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__, 1149 ccb->ccb_num_entries, limit)); 1150 if (limit == 0) { 1151 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries), 1152 ("%s: ccb->ccb_entries not empty", __func__)); 1153 ccb->ccb_num_entries = -1; /* disable bucket */ 1154 } 1155 CCB_UNLOCK(ccb); 1156 } 1157 1158 static void 1159 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce, 1160 struct tcp_fastopen_ccache_bucket *ccb) 1161 { 1162 1163 CCB_LOCK_ASSERT(ccb); 1164 1165 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link); 1166 ccb->ccb_num_entries--; 1167 uma_zfree(ccache_zone, cce); 1168 } 1169 1170 static int 1171 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS) 1172 { 1173 struct sbuf sb; 1174 struct tcp_fastopen_ccache_bucket *ccb; 1175 struct tcp_fastopen_ccache_entry *cce; 1176 sbintime_t now, duration, limit; 1177 const int linesize = 128; 1178 int i, error, num_entries; 1179 unsigned int j; 1180 #ifdef INET6 1181 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN]; 1182 #else 1183 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN]; 1184 #endif 1185 1186 if (jailed_without_vnet(curthread->td_ucred) != 0) 1187 return (EPERM); 1188 1189 /* Only allow root to read the client cookie cache */ 1190 if (curthread->td_ucred->cr_uid != 0) 1191 return (EPERM); 1192 1193 num_entries = 0; 1194 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 1195 ccb = &V_tcp_fastopen_ccache.base[i]; 1196 CCB_LOCK(ccb); 1197 if (ccb->ccb_num_entries > 0) 1198 num_entries += ccb->ccb_num_entries; 1199 CCB_UNLOCK(ccb); 1200 } 1201 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL); 1202 1203 sbuf_printf(&sb, 1204 "\nLocal IP address Remote IP address Port MSS" 1205 " Disabled Cookie\n"); 1206 1207 now = getsbinuptime(); 1208 limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32; 1209 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 1210 ccb = &V_tcp_fastopen_ccache.base[i]; 1211 CCB_LOCK(ccb); 1212 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) { 1213 if (cce->disable_time != 0) { 1214 duration = now - cce->disable_time; 1215 if (limit >= duration) 1216 duration = limit - duration; 1217 else 1218 duration = 0; 1219 } else 1220 duration = 0; 1221 sbuf_printf(&sb, 1222 "%-20s %-20s %5u %5u ", 1223 inet_ntop(cce->af, &cce->cce_client_ip, 1224 clt_buf, sizeof(clt_buf)), 1225 inet_ntop(cce->af, &cce->cce_server_ip, 1226 srv_buf, sizeof(srv_buf)), 1227 ntohs(cce->server_port), 1228 cce->server_mss); 1229 if (duration > 0) 1230 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration)); 1231 else 1232 sbuf_printf(&sb, "%8s ", "No"); 1233 for (j = 0; j < cce->cookie_len; j++) 1234 sbuf_printf(&sb, "%02x", cce->cookie[j]); 1235 sbuf_putc(&sb, '\n'); 1236 } 1237 CCB_UNLOCK(ccb); 1238 } 1239 error = sbuf_finish(&sb); 1240 if (error == 0) 1241 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb)); 1242 sbuf_delete(&sb); 1243 return (error); 1244 } 1245