1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2002 Andre Oppermann, Internet Business Solutions AG 5 * Copyright (c) 2021 Gleb Smirnoff <glebius@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. The name of the author may not be used to endorse or promote 17 * products derived from this software without specific prior written 18 * permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * The tcp_hostcache moves the tcp-specific cached metrics from the routing 35 * table to a dedicated structure indexed by the remote IP address. It keeps 36 * information on the measured TCP parameters of past TCP sessions to allow 37 * better initial start values to be used with later connections to/from the 38 * same source. Depending on the network parameters (delay, max MTU, 39 * congestion window) between local and remote sites, this can lead to 40 * significant speed-ups for new TCP connections after the first one. 41 * 42 * Due to the tcp_hostcache, all TCP-specific metrics information in the 43 * routing table have been removed. The inpcb no longer keeps a pointer to 44 * the routing entry, and protocol-initiated route cloning has been removed 45 * as well. With these changes, the routing table has gone back to being 46 * more lightwight and only carries information related to packet forwarding. 47 * 48 * tcp_hostcache is designed for multiple concurrent access in SMP 49 * environments and high contention. It is a straight hash. Each bucket row 50 * is protected by its own lock for modification. Readers are protected by 51 * SMR. This puts certain restrictions on writers, e.g. a writer shall only 52 * insert a fully populated entry into a row. Writer can't reuse least used 53 * entry if a hash is full. Value updates for an entry shall be atomic. 54 * 55 * TCP stack(s) communication with tcp_hostcache() is done via KBI functions 56 * tcp_hc_*() and the tcp_hc_metrics structure. 57 * 58 * Since tcp_hostcache is only caching information, there are no fatal 59 * consequences if we either can't allocate a new entry or have to drop 60 * an existing entry, or return somewhat stale information. 61 */ 62 63 /* 64 * Many thanks to jlemon for basic structure of tcp_syncache which is being 65 * followed here. 66 */ 67 68 #include "opt_inet6.h" 69 70 #include <sys/param.h> 71 #include <sys/systm.h> 72 #include <sys/hash.h> 73 #include <sys/jail.h> 74 #include <sys/kernel.h> 75 #include <sys/lock.h> 76 #include <sys/mutex.h> 77 #include <sys/malloc.h> 78 #include <sys/proc.h> 79 #include <sys/sbuf.h> 80 #include <sys/smr.h> 81 #include <sys/socket.h> 82 #include <sys/sysctl.h> 83 84 #include <net/vnet.h> 85 86 #include <netinet/in.h> 87 #include <netinet/in_pcb.h> 88 #include <netinet/tcp.h> 89 #include <netinet/tcp_var.h> 90 91 #include <vm/uma.h> 92 93 struct hc_head { 94 CK_SLIST_HEAD(hc_qhead, hc_metrics) hch_bucket; 95 u_int hch_length; 96 struct mtx hch_mtx; 97 }; 98 99 struct hc_metrics { 100 /* housekeeping */ 101 CK_SLIST_ENTRY(hc_metrics) hc_q; 102 struct in_addr ip4; /* IP address */ 103 struct in6_addr ip6; /* IP6 address */ 104 uint32_t ip6_zoneid; /* IPv6 scope zone id */ 105 /* endpoint specific values for tcp */ 106 uint32_t hc_mtu; /* MTU for this path */ 107 uint32_t hc_ssthresh; /* outbound gateway buffer limit */ 108 uint32_t hc_rtt; /* estimated round trip time */ 109 uint32_t hc_rttvar; /* estimated rtt variance */ 110 uint32_t hc_cwnd; /* congestion window */ 111 uint32_t hc_sendpipe; /* outbound delay-bandwidth product */ 112 uint32_t hc_recvpipe; /* inbound delay-bandwidth product */ 113 /* TCP hostcache internal data */ 114 int hc_expire; /* lifetime for object */ 115 #ifdef TCP_HC_COUNTERS 116 u_long hc_hits; /* number of hits */ 117 u_long hc_updates; /* number of updates */ 118 #endif 119 }; 120 121 struct tcp_hostcache { 122 struct hc_head *hashbase; 123 uma_zone_t zone; 124 smr_t smr; 125 u_int hashsize; 126 u_int hashmask; 127 u_int hashsalt; 128 u_int bucket_limit; 129 u_int cache_count; 130 u_int cache_limit; 131 u_int expire; 132 u_int prune; 133 u_int purgeall; 134 }; 135 136 /* Arbitrary values */ 137 #define TCP_HOSTCACHE_HASHSIZE 512 138 #define TCP_HOSTCACHE_BUCKETLIMIT 30 139 #define TCP_HOSTCACHE_EXPIRE 60*60 /* one hour */ 140 #define TCP_HOSTCACHE_PRUNE 5*60 /* every 5 minutes */ 141 142 VNET_DEFINE_STATIC(struct tcp_hostcache, tcp_hostcache); 143 #define V_tcp_hostcache VNET(tcp_hostcache) 144 145 VNET_DEFINE_STATIC(struct callout, tcp_hc_callout); 146 #define V_tcp_hc_callout VNET(tcp_hc_callout) 147 148 static struct hc_metrics *tcp_hc_lookup(const struct in_conninfo *); 149 static int sysctl_tcp_hc_expire(SYSCTL_HANDLER_ARGS); 150 static int sysctl_tcp_hc_prune(SYSCTL_HANDLER_ARGS); 151 static int sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS); 152 static int sysctl_tcp_hc_histo(SYSCTL_HANDLER_ARGS); 153 static int sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS); 154 static void tcp_hc_purge_internal(int); 155 static void tcp_hc_purge(void *); 156 157 static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hostcache, 158 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 159 "TCP Host cache"); 160 161 VNET_DEFINE(int, tcp_use_hostcache) = 1; 162 #define V_tcp_use_hostcache VNET(tcp_use_hostcache) 163 SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW, 164 &VNET_NAME(tcp_use_hostcache), 0, 165 "Enable the TCP hostcache"); 166 167 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, cachelimit, CTLFLAG_VNET | CTLFLAG_RDTUN, 168 &VNET_NAME(tcp_hostcache.cache_limit), 0, 169 "Overall entry limit for hostcache"); 170 171 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN, 172 &VNET_NAME(tcp_hostcache.hashsize), 0, 173 "Size of TCP hostcache hashtable"); 174 175 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, bucketlimit, 176 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_hostcache.bucket_limit), 0, 177 "Per-bucket hash limit for hostcache"); 178 179 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, count, CTLFLAG_VNET | CTLFLAG_RD, 180 &VNET_NAME(tcp_hostcache.cache_count), 0, 181 "Current number of entries in hostcache"); 182 183 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, expire, 184 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 185 &VNET_NAME(tcp_hostcache.expire), 0, sysctl_tcp_hc_expire, "IU", 186 "Expire time of TCP hostcache entries"); 187 188 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, prune, 189 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 190 &VNET_NAME(tcp_hostcache.prune), 0, sysctl_tcp_hc_prune, "IU", 191 "Time between purge runs"); 192 193 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, purge, CTLFLAG_VNET | CTLFLAG_RW, 194 &VNET_NAME(tcp_hostcache.purgeall), 0, 195 "Expire all entries on next purge run"); 196 197 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, list, 198 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 199 0, 0, sysctl_tcp_hc_list, "A", 200 "List of all hostcache entries"); 201 202 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, histo, 203 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 204 0, 0, sysctl_tcp_hc_histo, "A", 205 "Print a histogram of hostcache hashbucket utilization"); 206 207 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, purgenow, 208 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 209 NULL, 0, sysctl_tcp_hc_purgenow, "IU", 210 "Immediately purge all entries"); 211 212 static MALLOC_DEFINE(M_HOSTCACHE, "hostcache", "TCP hostcache"); 213 214 /* Use jenkins_hash32(), as in other parts of the tcp stack */ 215 #define HOSTCACHE_HASH(inc) \ 216 ((inc)->inc_flags & INC_ISIPV6) ? \ 217 (jenkins_hash32((inc)->inc6_faddr.s6_addr32, 4, \ 218 V_tcp_hostcache.hashsalt) & V_tcp_hostcache.hashmask) \ 219 : \ 220 (jenkins_hash32(&(inc)->inc_faddr.s_addr, 1, \ 221 V_tcp_hostcache.hashsalt) & V_tcp_hostcache.hashmask) 222 223 #define THC_LOCK(h) mtx_lock(&(h)->hch_mtx) 224 #define THC_UNLOCK(h) mtx_unlock(&(h)->hch_mtx) 225 226 void 227 tcp_hc_init(void) 228 { 229 u_int cache_limit; 230 int i; 231 232 /* 233 * Initialize hostcache structures. 234 */ 235 atomic_store_int(&V_tcp_hostcache.cache_count, 0); 236 V_tcp_hostcache.hashsize = TCP_HOSTCACHE_HASHSIZE; 237 V_tcp_hostcache.bucket_limit = TCP_HOSTCACHE_BUCKETLIMIT; 238 V_tcp_hostcache.expire = TCP_HOSTCACHE_EXPIRE; 239 V_tcp_hostcache.prune = TCP_HOSTCACHE_PRUNE; 240 V_tcp_hostcache.hashsalt = arc4random(); 241 242 TUNABLE_INT_FETCH("net.inet.tcp.hostcache.hashsize", 243 &V_tcp_hostcache.hashsize); 244 if (!powerof2(V_tcp_hostcache.hashsize)) { 245 printf("WARNING: hostcache hash size is not a power of 2.\n"); 246 V_tcp_hostcache.hashsize = TCP_HOSTCACHE_HASHSIZE; /* default */ 247 } 248 V_tcp_hostcache.hashmask = V_tcp_hostcache.hashsize - 1; 249 250 TUNABLE_INT_FETCH("net.inet.tcp.hostcache.bucketlimit", 251 &V_tcp_hostcache.bucket_limit); 252 253 cache_limit = V_tcp_hostcache.hashsize * V_tcp_hostcache.bucket_limit; 254 V_tcp_hostcache.cache_limit = cache_limit; 255 TUNABLE_INT_FETCH("net.inet.tcp.hostcache.cachelimit", 256 &V_tcp_hostcache.cache_limit); 257 if (V_tcp_hostcache.cache_limit > cache_limit) 258 V_tcp_hostcache.cache_limit = cache_limit; 259 260 /* 261 * Allocate the hash table. 262 */ 263 V_tcp_hostcache.hashbase = (struct hc_head *) 264 malloc(V_tcp_hostcache.hashsize * sizeof(struct hc_head), 265 M_HOSTCACHE, M_WAITOK | M_ZERO); 266 267 /* 268 * Initialize the hash buckets. 269 */ 270 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 271 CK_SLIST_INIT(&V_tcp_hostcache.hashbase[i].hch_bucket); 272 V_tcp_hostcache.hashbase[i].hch_length = 0; 273 mtx_init(&V_tcp_hostcache.hashbase[i].hch_mtx, "tcp_hc_entry", 274 NULL, MTX_DEF); 275 } 276 277 /* 278 * Allocate the hostcache entries. 279 */ 280 V_tcp_hostcache.zone = 281 uma_zcreate("hostcache", sizeof(struct hc_metrics), 282 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_SMR); 283 uma_zone_set_max(V_tcp_hostcache.zone, V_tcp_hostcache.cache_limit); 284 V_tcp_hostcache.smr = uma_zone_get_smr(V_tcp_hostcache.zone); 285 286 /* 287 * Set up periodic cache cleanup. 288 */ 289 callout_init(&V_tcp_hc_callout, 1); 290 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 291 tcp_hc_purge, curvnet); 292 } 293 294 #ifdef VIMAGE 295 void 296 tcp_hc_destroy(void) 297 { 298 int i; 299 300 callout_drain(&V_tcp_hc_callout); 301 302 /* Purge all hc entries. */ 303 tcp_hc_purge_internal(1); 304 305 /* Free the uma zone and the allocated hash table. */ 306 uma_zdestroy(V_tcp_hostcache.zone); 307 308 for (i = 0; i < V_tcp_hostcache.hashsize; i++) 309 mtx_destroy(&V_tcp_hostcache.hashbase[i].hch_mtx); 310 free(V_tcp_hostcache.hashbase, M_HOSTCACHE); 311 } 312 #endif 313 314 /* 315 * Internal function: compare cache entry to a connection. 316 */ 317 static bool 318 tcp_hc_cmp(struct hc_metrics *hc_entry, const struct in_conninfo *inc) 319 { 320 321 if (inc->inc_flags & INC_ISIPV6) { 322 /* XXX: check ip6_zoneid */ 323 if (memcmp(&inc->inc6_faddr, &hc_entry->ip6, 324 sizeof(inc->inc6_faddr)) == 0) 325 return (true); 326 } else { 327 if (memcmp(&inc->inc_faddr, &hc_entry->ip4, 328 sizeof(inc->inc_faddr)) == 0) 329 return (true); 330 } 331 332 return (false); 333 } 334 335 /* 336 * Internal function: look up an entry in the hostcache for read. 337 * On success returns in SMR section. 338 */ 339 static struct hc_metrics * 340 tcp_hc_lookup(const struct in_conninfo *inc) 341 { 342 struct hc_head *hc_head; 343 struct hc_metrics *hc_entry; 344 345 KASSERT(inc != NULL, ("%s: NULL in_conninfo", __func__)); 346 347 hc_head = &V_tcp_hostcache.hashbase[HOSTCACHE_HASH(inc)]; 348 349 /* 350 * Iterate through entries in bucket row looking for a match. 351 */ 352 smr_enter(V_tcp_hostcache.smr); 353 CK_SLIST_FOREACH(hc_entry, &hc_head->hch_bucket, hc_q) 354 if (tcp_hc_cmp(hc_entry, inc)) 355 break; 356 357 if (hc_entry != NULL) { 358 if (atomic_load_int(&hc_entry->hc_expire) != 359 V_tcp_hostcache.expire) 360 atomic_store_int(&hc_entry->hc_expire, 361 V_tcp_hostcache.expire); 362 #ifdef TCP_HC_COUNTERS 363 hc_entry->hc_hits++; 364 #endif 365 } else 366 smr_exit(V_tcp_hostcache.smr); 367 368 return (hc_entry); 369 } 370 371 /* 372 * External function: look up an entry in the hostcache and fill out the 373 * supplied TCP metrics structure. Fills in NULL when no entry was found or 374 * a value is not set. 375 */ 376 void 377 tcp_hc_get(const struct in_conninfo *inc, 378 struct tcp_hc_metrics *hc_metrics) 379 { 380 struct hc_metrics *hc_entry; 381 382 if (!V_tcp_use_hostcache) { 383 bzero(hc_metrics, sizeof(*hc_metrics)); 384 return; 385 } 386 387 /* 388 * Find the right bucket. 389 */ 390 hc_entry = tcp_hc_lookup(inc); 391 392 /* 393 * If we don't have an existing object. 394 */ 395 if (hc_entry == NULL) { 396 bzero(hc_metrics, sizeof(*hc_metrics)); 397 return; 398 } 399 400 hc_metrics->hc_mtu = atomic_load_32(&hc_entry->hc_mtu); 401 hc_metrics->hc_ssthresh = atomic_load_32(&hc_entry->hc_ssthresh); 402 hc_metrics->hc_rtt = atomic_load_32(&hc_entry->hc_rtt); 403 hc_metrics->hc_rttvar = atomic_load_32(&hc_entry->hc_rttvar); 404 hc_metrics->hc_cwnd = atomic_load_32(&hc_entry->hc_cwnd); 405 hc_metrics->hc_sendpipe = atomic_load_32(&hc_entry->hc_sendpipe); 406 hc_metrics->hc_recvpipe = atomic_load_32(&hc_entry->hc_recvpipe); 407 408 smr_exit(V_tcp_hostcache.smr); 409 } 410 411 /* 412 * External function: look up an entry in the hostcache and return the 413 * discovered path MTU. Returns 0 if no entry is found or value is not 414 * set. 415 */ 416 uint32_t 417 tcp_hc_getmtu(const struct in_conninfo *inc) 418 { 419 struct hc_metrics *hc_entry; 420 uint32_t mtu; 421 422 if (!V_tcp_use_hostcache) 423 return (0); 424 425 hc_entry = tcp_hc_lookup(inc); 426 if (hc_entry == NULL) { 427 return (0); 428 } 429 430 mtu = atomic_load_32(&hc_entry->hc_mtu); 431 smr_exit(V_tcp_hostcache.smr); 432 433 return (mtu); 434 } 435 436 /* 437 * External function: update the MTU value of an entry in the hostcache. 438 * Creates a new entry if none was found. 439 */ 440 void 441 tcp_hc_updatemtu(const struct in_conninfo *inc, uint32_t mtu) 442 { 443 struct tcp_hc_metrics hcm = { .hc_mtu = mtu }; 444 445 return (tcp_hc_update(inc, &hcm)); 446 } 447 448 /* 449 * External function: update the TCP metrics of an entry in the hostcache. 450 * Creates a new entry if none was found. 451 */ 452 void 453 tcp_hc_update(const struct in_conninfo *inc, struct tcp_hc_metrics *hcm) 454 { 455 struct hc_head *hc_head; 456 struct hc_metrics *hc_entry, *hc_prev; 457 uint32_t v; 458 bool new; 459 460 if (!V_tcp_use_hostcache) 461 return; 462 463 hc_head = &V_tcp_hostcache.hashbase[HOSTCACHE_HASH(inc)]; 464 hc_prev = NULL; 465 466 THC_LOCK(hc_head); 467 CK_SLIST_FOREACH(hc_entry, &hc_head->hch_bucket, hc_q) { 468 if (tcp_hc_cmp(hc_entry, inc)) 469 break; 470 if (CK_SLIST_NEXT(hc_entry, hc_q) != NULL) 471 hc_prev = hc_entry; 472 } 473 474 if (hc_entry != NULL) { 475 if (atomic_load_int(&hc_entry->hc_expire) != 476 V_tcp_hostcache.expire) 477 atomic_store_int(&hc_entry->hc_expire, 478 V_tcp_hostcache.expire); 479 #ifdef TCP_HC_COUNTERS 480 hc_entry->hc_updates++; 481 #endif 482 new = false; 483 } else { 484 /* 485 * Try to allocate a new entry. If the bucket limit is 486 * reached, delete the least-used element, located at the end 487 * of the CK_SLIST. During lookup we saved the pointer to 488 * the second to last element, in case if list has at least 2 489 * elements. This will allow to delete last element without 490 * extra traversal. 491 * 492 * Give up if the row is empty. 493 */ 494 if (hc_head->hch_length >= V_tcp_hostcache.bucket_limit || 495 atomic_load_int(&V_tcp_hostcache.cache_count) >= 496 V_tcp_hostcache.cache_limit) { 497 if (hc_prev != NULL) { 498 hc_entry = CK_SLIST_NEXT(hc_prev, hc_q); 499 KASSERT(CK_SLIST_NEXT(hc_entry, hc_q) == NULL, 500 ("%s: %p is not one to last", 501 __func__, hc_prev)); 502 CK_SLIST_REMOVE_AFTER(hc_prev, hc_q); 503 } else if ((hc_entry = 504 CK_SLIST_FIRST(&hc_head->hch_bucket)) != NULL) { 505 KASSERT(CK_SLIST_NEXT(hc_entry, hc_q) == NULL, 506 ("%s: %p is not the only element", 507 __func__, hc_entry)); 508 CK_SLIST_REMOVE_HEAD(&hc_head->hch_bucket, 509 hc_q); 510 } else { 511 THC_UNLOCK(hc_head); 512 return; 513 } 514 KASSERT(hc_head->hch_length > 0 && 515 hc_head->hch_length <= V_tcp_hostcache.bucket_limit, 516 ("tcp_hostcache: bucket length violated at %p", 517 hc_head)); 518 hc_head->hch_length--; 519 atomic_subtract_int(&V_tcp_hostcache.cache_count, 1); 520 TCPSTAT_INC(tcps_hc_bucketoverflow); 521 uma_zfree_smr(V_tcp_hostcache.zone, hc_entry); 522 } 523 524 /* 525 * Allocate a new entry, or balk if not possible. 526 */ 527 hc_entry = uma_zalloc_smr(V_tcp_hostcache.zone, M_NOWAIT); 528 if (hc_entry == NULL) { 529 THC_UNLOCK(hc_head); 530 return; 531 } 532 533 /* 534 * Initialize basic information of hostcache entry. 535 */ 536 bzero(hc_entry, sizeof(*hc_entry)); 537 if (inc->inc_flags & INC_ISIPV6) { 538 hc_entry->ip6 = inc->inc6_faddr; 539 hc_entry->ip6_zoneid = inc->inc6_zoneid; 540 } else 541 hc_entry->ip4 = inc->inc_faddr; 542 hc_entry->hc_expire = V_tcp_hostcache.expire; 543 new = true; 544 } 545 546 /* 547 * Fill in data. Use atomics, since an existing entry is 548 * accessible by readers in SMR section. 549 */ 550 if (hcm->hc_mtu != 0) { 551 atomic_store_32(&hc_entry->hc_mtu, hcm->hc_mtu); 552 } 553 if (hcm->hc_rtt != 0) { 554 if (hc_entry->hc_rtt == 0) 555 v = hcm->hc_rtt; 556 else 557 v = ((uint64_t)hc_entry->hc_rtt + 558 (uint64_t)hcm->hc_rtt) / 2; 559 atomic_store_32(&hc_entry->hc_rtt, v); 560 TCPSTAT_INC(tcps_cachedrtt); 561 } 562 if (hcm->hc_rttvar != 0) { 563 if (hc_entry->hc_rttvar == 0) 564 v = hcm->hc_rttvar; 565 else 566 v = ((uint64_t)hc_entry->hc_rttvar + 567 (uint64_t)hcm->hc_rttvar) / 2; 568 atomic_store_32(&hc_entry->hc_rttvar, v); 569 TCPSTAT_INC(tcps_cachedrttvar); 570 } 571 if (hcm->hc_ssthresh != 0) { 572 if (hc_entry->hc_ssthresh == 0) 573 v = hcm->hc_ssthresh; 574 else 575 v = (hc_entry->hc_ssthresh + 576 hcm->hc_ssthresh) / 2; 577 atomic_store_32(&hc_entry->hc_ssthresh, v); 578 TCPSTAT_INC(tcps_cachedssthresh); 579 } 580 if (hcm->hc_cwnd != 0) { 581 if (hc_entry->hc_cwnd == 0) 582 v = hcm->hc_cwnd; 583 else 584 v = ((uint64_t)hc_entry->hc_cwnd + 585 (uint64_t)hcm->hc_cwnd) / 2; 586 atomic_store_32(&hc_entry->hc_cwnd, v); 587 /* TCPSTAT_INC(tcps_cachedcwnd); */ 588 } 589 if (hcm->hc_sendpipe != 0) { 590 if (hc_entry->hc_sendpipe == 0) 591 v = hcm->hc_sendpipe; 592 else 593 v = ((uint64_t)hc_entry->hc_sendpipe + 594 (uint64_t)hcm->hc_sendpipe) / 2; 595 atomic_store_32(&hc_entry->hc_sendpipe, v); 596 /* TCPSTAT_INC(tcps_cachedsendpipe); */ 597 } 598 if (hcm->hc_recvpipe != 0) { 599 if (hc_entry->hc_recvpipe == 0) 600 v = hcm->hc_recvpipe; 601 else 602 v = ((uint64_t)hc_entry->hc_recvpipe + 603 (uint64_t)hcm->hc_recvpipe) / 2; 604 atomic_store_32(&hc_entry->hc_recvpipe, v); 605 /* TCPSTAT_INC(tcps_cachedrecvpipe); */ 606 } 607 608 /* 609 * Put it upfront. 610 */ 611 if (new) { 612 CK_SLIST_INSERT_HEAD(&hc_head->hch_bucket, hc_entry, hc_q); 613 hc_head->hch_length++; 614 KASSERT(hc_head->hch_length <= V_tcp_hostcache.bucket_limit, 615 ("tcp_hostcache: bucket length too high at %p", hc_head)); 616 atomic_add_int(&V_tcp_hostcache.cache_count, 1); 617 TCPSTAT_INC(tcps_hc_added); 618 } else if (hc_entry != CK_SLIST_FIRST(&hc_head->hch_bucket)) { 619 KASSERT(CK_SLIST_NEXT(hc_prev, hc_q) == hc_entry, 620 ("%s: %p next is not %p", __func__, hc_prev, hc_entry)); 621 CK_SLIST_REMOVE_AFTER(hc_prev, hc_q); 622 CK_SLIST_INSERT_HEAD(&hc_head->hch_bucket, hc_entry, hc_q); 623 } 624 THC_UNLOCK(hc_head); 625 } 626 627 /* 628 * Sysctl function: adjusts the expire timeout and adjusts the prune value accordingly. 629 */ 630 static int 631 sysctl_tcp_hc_expire(SYSCTL_HANDLER_ARGS) 632 { 633 int error, expire; 634 635 expire = V_tcp_hostcache.expire; 636 error = sysctl_handle_int(oidp, &expire, 0, req); 637 if (error != 0 || !req->newptr) 638 return (error); 639 if (expire < V_tcp_hostcache.prune) 640 V_tcp_hostcache.prune = expire; 641 V_tcp_hostcache.expire = expire; 642 return (0); 643 } 644 645 /* 646 * Sysctl function: adjusts the prune time and adjusts the expire timeout accordingly. 647 */ 648 static int 649 sysctl_tcp_hc_prune(SYSCTL_HANDLER_ARGS) 650 { 651 int error, prune; 652 653 prune = V_tcp_hostcache.prune; 654 error = sysctl_handle_int(oidp, &prune, 0, req); 655 if (error != 0 || !req->newptr) 656 return (error); 657 if (prune > V_tcp_hostcache.expire) 658 V_tcp_hostcache.expire = prune; 659 V_tcp_hostcache.prune = prune; 660 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 661 tcp_hc_purge, curvnet); 662 return (0); 663 } 664 665 /* 666 * Sysctl function: prints the list and values of all hostcache entries in 667 * unsorted order. 668 */ 669 static int 670 sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS) 671 { 672 const int linesize = 128; 673 struct sbuf sb; 674 int i, error, len; 675 struct hc_metrics *hc_entry; 676 char ip4buf[INET_ADDRSTRLEN]; 677 #ifdef INET6 678 char ip6buf[INET6_ADDRSTRLEN]; 679 #endif 680 681 if (jailed_without_vnet(curthread->td_ucred) != 0) 682 return (EPERM); 683 684 /* Optimize Buffer length query by sbin/sysctl */ 685 if (req->oldptr == NULL) { 686 len = (atomic_load_int(&V_tcp_hostcache.cache_count) + 1) * 687 linesize; 688 return (SYSCTL_OUT(req, NULL, len)); 689 } 690 691 error = sysctl_wire_old_buffer(req, 0); 692 if (error != 0) { 693 return(error); 694 } 695 696 /* Use a buffer sized for one full bucket */ 697 sbuf_new_for_sysctl(&sb, NULL, V_tcp_hostcache.bucket_limit * 698 linesize, req); 699 700 sbuf_printf(&sb, 701 "\nIP address MTU SSTRESH RTT RTTVAR " 702 " CWND SENDPIPE RECVPIPE " 703 #ifdef TCP_HC_COUNTERS 704 "HITS UPD " 705 #endif 706 "EXP\n"); 707 sbuf_drain(&sb); 708 709 #define msec(u) (((u) + 500) / 1000) 710 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 711 THC_LOCK(&V_tcp_hostcache.hashbase[i]); 712 CK_SLIST_FOREACH(hc_entry, 713 &V_tcp_hostcache.hashbase[i].hch_bucket, hc_q) { 714 sbuf_printf(&sb, 715 "%-15s %5u %8u %6lums %6lums %8u %8u %8u " 716 #ifdef TCP_HC_COUNTERS 717 "%4lu %4lu " 718 #endif 719 "%4i\n", 720 hc_entry->ip4.s_addr ? 721 inet_ntoa_r(hc_entry->ip4, ip4buf) : 722 #ifdef INET6 723 ip6_sprintf(ip6buf, &hc_entry->ip6), 724 #else 725 "IPv6?", 726 #endif 727 hc_entry->hc_mtu, 728 hc_entry->hc_ssthresh, 729 msec((u_long)hc_entry->hc_rtt * 730 (RTM_RTTUNIT / (hz * TCP_RTT_SCALE))), 731 msec((u_long)hc_entry->hc_rttvar * 732 (RTM_RTTUNIT / (hz * TCP_RTTVAR_SCALE))), 733 hc_entry->hc_cwnd, 734 hc_entry->hc_sendpipe, 735 hc_entry->hc_recvpipe, 736 #ifdef TCP_HC_COUNTERS 737 hc_entry->hc_hits, 738 hc_entry->hc_updates, 739 #endif 740 hc_entry->hc_expire); 741 } 742 THC_UNLOCK(&V_tcp_hostcache.hashbase[i]); 743 sbuf_drain(&sb); 744 } 745 #undef msec 746 error = sbuf_finish(&sb); 747 sbuf_delete(&sb); 748 return(error); 749 } 750 751 /* 752 * Sysctl function: prints a histogram of the hostcache hashbucket 753 * utilization. 754 */ 755 static int 756 sysctl_tcp_hc_histo(SYSCTL_HANDLER_ARGS) 757 { 758 const int linesize = 50; 759 struct sbuf sb; 760 int i, error; 761 int *histo; 762 u_int hch_length; 763 764 if (jailed_without_vnet(curthread->td_ucred) != 0) 765 return (EPERM); 766 767 histo = (int *)malloc(sizeof(int) * (V_tcp_hostcache.bucket_limit + 1), 768 M_TEMP, M_NOWAIT|M_ZERO); 769 if (histo == NULL) 770 return(ENOMEM); 771 772 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 773 hch_length = V_tcp_hostcache.hashbase[i].hch_length; 774 KASSERT(hch_length <= V_tcp_hostcache.bucket_limit, 775 ("tcp_hostcache: bucket limit exceeded at %u: %u", 776 i, hch_length)); 777 histo[hch_length]++; 778 } 779 780 /* Use a buffer for 16 lines */ 781 sbuf_new_for_sysctl(&sb, NULL, 16 * linesize, req); 782 783 sbuf_printf(&sb, "\nLength\tCount\n"); 784 for (i = 0; i <= V_tcp_hostcache.bucket_limit; i++) { 785 sbuf_printf(&sb, "%u\t%u\n", i, histo[i]); 786 } 787 error = sbuf_finish(&sb); 788 sbuf_delete(&sb); 789 free(histo, M_TEMP); 790 return(error); 791 } 792 793 /* 794 * Caller has to make sure the curvnet is set properly. 795 */ 796 static void 797 tcp_hc_purge_internal(int all) 798 { 799 struct hc_head *head; 800 struct hc_metrics *hc_entry, *hc_next, *hc_prev; 801 int i; 802 803 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 804 head = &V_tcp_hostcache.hashbase[i]; 805 hc_prev = NULL; 806 THC_LOCK(head); 807 CK_SLIST_FOREACH_SAFE(hc_entry, &head->hch_bucket, hc_q, 808 hc_next) { 809 KASSERT(head->hch_length > 0 && head->hch_length <= 810 V_tcp_hostcache.bucket_limit, ("tcp_hostcache: " 811 "bucket length out of range at %u: %u", i, 812 head->hch_length)); 813 if (all || 814 (int)atomic_load_int(&hc_entry->hc_expire) <= 0) { 815 if (hc_prev != NULL) { 816 KASSERT(hc_entry == 817 CK_SLIST_NEXT(hc_prev, hc_q), 818 ("%s: %p is not next to %p", 819 __func__, hc_entry, hc_prev)); 820 CK_SLIST_REMOVE_AFTER(hc_prev, hc_q); 821 } else { 822 KASSERT(hc_entry == 823 CK_SLIST_FIRST(&head->hch_bucket), 824 ("%s: %p is not first", 825 __func__, hc_entry)); 826 CK_SLIST_REMOVE_HEAD(&head->hch_bucket, 827 hc_q); 828 } 829 uma_zfree_smr(V_tcp_hostcache.zone, hc_entry); 830 head->hch_length--; 831 atomic_subtract_int(&V_tcp_hostcache.cache_count, 1); 832 } else { 833 atomic_subtract_int(&hc_entry->hc_expire, 834 V_tcp_hostcache.prune); 835 hc_prev = hc_entry; 836 } 837 } 838 THC_UNLOCK(head); 839 } 840 } 841 842 /* 843 * Expire and purge (old|all) entries in the tcp_hostcache. Runs 844 * periodically from the callout. 845 */ 846 static void 847 tcp_hc_purge(void *arg) 848 { 849 CURVNET_SET((struct vnet *) arg); 850 int all = 0; 851 852 if (V_tcp_hostcache.purgeall) { 853 if (V_tcp_hostcache.purgeall == 2) 854 V_tcp_hostcache.hashsalt = arc4random(); 855 all = 1; 856 V_tcp_hostcache.purgeall = 0; 857 } 858 859 tcp_hc_purge_internal(all); 860 861 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 862 tcp_hc_purge, arg); 863 CURVNET_RESTORE(); 864 } 865 866 /* 867 * Expire and purge all entries in hostcache immediately. 868 */ 869 static int 870 sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS) 871 { 872 int error, val; 873 874 val = 0; 875 error = sysctl_handle_int(oidp, &val, 0, req); 876 if (error != 0 || !req->newptr) 877 return (error); 878 879 if (val == 2) 880 V_tcp_hostcache.hashsalt = arc4random(); 881 tcp_hc_purge_internal(1); 882 883 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 884 tcp_hc_purge, curvnet); 885 886 return (0); 887 } 888