1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2002 Andre Oppermann, Internet Business Solutions AG 5 * Copyright (c) 2021 Gleb Smirnoff <glebius@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. The name of the author may not be used to endorse or promote 17 * products derived from this software without specific prior written 18 * permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 /* 34 * The tcp_hostcache moves the tcp-specific cached metrics from the routing 35 * table to a dedicated structure indexed by the remote IP address. It keeps 36 * information on the measured TCP parameters of past TCP sessions to allow 37 * better initial start values to be used with later connections to/from the 38 * same source. Depending on the network parameters (delay, max MTU, 39 * congestion window) between local and remote sites, this can lead to 40 * significant speed-ups for new TCP connections after the first one. 41 * 42 * Due to the tcp_hostcache, all TCP-specific metrics information in the 43 * routing table have been removed. The inpcb no longer keeps a pointer to 44 * the routing entry, and protocol-initiated route cloning has been removed 45 * as well. With these changes, the routing table has gone back to being 46 * more lightwight and only carries information related to packet forwarding. 47 * 48 * tcp_hostcache is designed for multiple concurrent access in SMP 49 * environments and high contention. It is a straight hash. Each bucket row 50 * is protected by its own lock for modification. Readers are protected by 51 * SMR. This puts certain restrictions on writers, e.g. a writer shall only 52 * insert a fully populated entry into a row. Writer can't reuse least used 53 * entry if a hash is full. Value updates for an entry shall be atomic. 54 * 55 * TCP stack(s) communication with tcp_hostcache() is done via KBI functions 56 * tcp_hc_*() and the tcp_hc_metrics structure. 57 * 58 * Since tcp_hostcache is only caching information, there are no fatal 59 * consequences if we either can't allocate a new entry or have to drop 60 * an existing entry, or return somewhat stale information. 61 */ 62 63 /* 64 * Many thanks to jlemon for basic structure of tcp_syncache which is being 65 * followed here. 66 */ 67 68 #include "opt_inet6.h" 69 70 #include <sys/param.h> 71 #include <sys/systm.h> 72 #include <sys/hash.h> 73 #include <sys/jail.h> 74 #include <sys/kernel.h> 75 #include <sys/lock.h> 76 #include <sys/mutex.h> 77 #include <sys/malloc.h> 78 #include <sys/proc.h> 79 #include <sys/sbuf.h> 80 #include <sys/smr.h> 81 #include <sys/socket.h> 82 #include <sys/sysctl.h> 83 84 #include <net/vnet.h> 85 86 #include <netinet/in.h> 87 #include <netinet/in_pcb.h> 88 #include <netinet/tcp.h> 89 #include <netinet/tcp_var.h> 90 91 #include <vm/uma.h> 92 93 struct hc_head { 94 CK_SLIST_HEAD(hc_qhead, hc_metrics) hch_bucket; 95 u_int hch_length; 96 struct mtx hch_mtx; 97 }; 98 99 struct hc_metrics { 100 /* housekeeping */ 101 CK_SLIST_ENTRY(hc_metrics) hc_q; 102 struct in_addr ip4; /* IP address */ 103 struct in6_addr ip6; /* IP6 address */ 104 uint32_t ip6_zoneid; /* IPv6 scope zone id */ 105 /* endpoint specific values for tcp */ 106 uint32_t hc_mtu; /* MTU for this path */ 107 uint32_t hc_ssthresh; /* outbound gateway buffer limit */ 108 uint32_t hc_rtt; /* estimated round trip time */ 109 uint32_t hc_rttvar; /* estimated rtt variance */ 110 uint32_t hc_cwnd; /* congestion window */ 111 uint32_t hc_sendpipe; /* outbound delay-bandwidth product */ 112 uint32_t hc_recvpipe; /* inbound delay-bandwidth product */ 113 /* TCP hostcache internal data */ 114 int hc_expire; /* lifetime for object */ 115 #ifdef TCP_HC_COUNTERS 116 u_long hc_hits; /* number of hits */ 117 u_long hc_updates; /* number of updates */ 118 #endif 119 }; 120 121 struct tcp_hostcache { 122 struct hc_head *hashbase; 123 uma_zone_t zone; 124 smr_t smr; 125 u_int hashsize; 126 u_int hashmask; 127 u_int hashsalt; 128 u_int bucket_limit; 129 u_int cache_count; 130 u_int cache_limit; 131 u_int expire; 132 u_int prune; 133 u_int purgeall; 134 }; 135 136 /* Arbitrary values */ 137 #define TCP_HOSTCACHE_HASHSIZE 512 138 #define TCP_HOSTCACHE_BUCKETLIMIT 30 139 #define TCP_HOSTCACHE_EXPIRE 60*60 /* one hour */ 140 #define TCP_HOSTCACHE_PRUNE 5*60 /* every 5 minutes */ 141 142 VNET_DEFINE_STATIC(struct tcp_hostcache, tcp_hostcache); 143 #define V_tcp_hostcache VNET(tcp_hostcache) 144 145 VNET_DEFINE_STATIC(struct callout, tcp_hc_callout); 146 #define V_tcp_hc_callout VNET(tcp_hc_callout) 147 148 static struct hc_metrics *tcp_hc_lookup(const struct in_conninfo *); 149 static int sysctl_tcp_hc_expire(SYSCTL_HANDLER_ARGS); 150 static int sysctl_tcp_hc_prune(SYSCTL_HANDLER_ARGS); 151 static int sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS); 152 static int sysctl_tcp_hc_histo(SYSCTL_HANDLER_ARGS); 153 static int sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS); 154 static void tcp_hc_purge_internal(int); 155 static void tcp_hc_purge(void *); 156 157 static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hostcache, 158 CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 159 "TCP Host cache"); 160 161 VNET_DEFINE(int, tcp_use_hostcache) = 1; 162 #define V_tcp_use_hostcache VNET(tcp_use_hostcache) 163 SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW, 164 &VNET_NAME(tcp_use_hostcache), 0, 165 "Enable the TCP hostcache"); 166 167 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, cachelimit, CTLFLAG_VNET | CTLFLAG_RDTUN, 168 &VNET_NAME(tcp_hostcache.cache_limit), 0, 169 "Overall entry limit for hostcache"); 170 171 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN, 172 &VNET_NAME(tcp_hostcache.hashsize), 0, 173 "Size of TCP hostcache hashtable"); 174 175 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, bucketlimit, 176 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_hostcache.bucket_limit), 0, 177 "Per-bucket hash limit for hostcache"); 178 179 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, count, CTLFLAG_VNET | CTLFLAG_RD, 180 &VNET_NAME(tcp_hostcache.cache_count), 0, 181 "Current number of entries in hostcache"); 182 183 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, expire, 184 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 185 &VNET_NAME(tcp_hostcache.expire), 0, sysctl_tcp_hc_expire, "IU", 186 "Expire time of TCP hostcache entries"); 187 188 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, prune, 189 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 190 &VNET_NAME(tcp_hostcache.prune), 0, sysctl_tcp_hc_prune, "IU", 191 "Time between purge runs"); 192 193 SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, purge, CTLFLAG_VNET | CTLFLAG_RW, 194 &VNET_NAME(tcp_hostcache.purgeall), 0, 195 "Expire all entries on next purge run"); 196 197 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, list, 198 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 199 0, 0, sysctl_tcp_hc_list, "A", 200 "List of all hostcache entries"); 201 202 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, histo, 203 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 204 0, 0, sysctl_tcp_hc_histo, "A", 205 "Print a histogram of hostcache hashbucket utilization"); 206 207 SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, purgenow, 208 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 209 NULL, 0, sysctl_tcp_hc_purgenow, "IU", 210 "Immediately purge all entries"); 211 212 static MALLOC_DEFINE(M_HOSTCACHE, "hostcache", "TCP hostcache"); 213 214 /* Use jenkins_hash32(), as in other parts of the tcp stack */ 215 #define HOSTCACHE_HASH(inc) \ 216 ((inc)->inc_flags & INC_ISIPV6) ? \ 217 (jenkins_hash32((inc)->inc6_faddr.s6_addr32, 4, \ 218 V_tcp_hostcache.hashsalt) & V_tcp_hostcache.hashmask) \ 219 : \ 220 (jenkins_hash32(&(inc)->inc_faddr.s_addr, 1, \ 221 V_tcp_hostcache.hashsalt) & V_tcp_hostcache.hashmask) 222 223 #define THC_LOCK(h) mtx_lock(&(h)->hch_mtx) 224 #define THC_UNLOCK(h) mtx_unlock(&(h)->hch_mtx) 225 226 void 227 tcp_hc_init(void) 228 { 229 u_int cache_limit; 230 int i; 231 232 /* 233 * Initialize hostcache structures. 234 */ 235 atomic_store_int(&V_tcp_hostcache.cache_count, 0); 236 V_tcp_hostcache.hashsize = TCP_HOSTCACHE_HASHSIZE; 237 V_tcp_hostcache.bucket_limit = TCP_HOSTCACHE_BUCKETLIMIT; 238 V_tcp_hostcache.expire = TCP_HOSTCACHE_EXPIRE; 239 V_tcp_hostcache.prune = TCP_HOSTCACHE_PRUNE; 240 V_tcp_hostcache.hashsalt = arc4random(); 241 242 TUNABLE_INT_FETCH("net.inet.tcp.hostcache.hashsize", 243 &V_tcp_hostcache.hashsize); 244 if (!powerof2(V_tcp_hostcache.hashsize)) { 245 printf("WARNING: hostcache hash size is not a power of 2.\n"); 246 V_tcp_hostcache.hashsize = TCP_HOSTCACHE_HASHSIZE; /* default */ 247 } 248 V_tcp_hostcache.hashmask = V_tcp_hostcache.hashsize - 1; 249 250 TUNABLE_INT_FETCH("net.inet.tcp.hostcache.bucketlimit", 251 &V_tcp_hostcache.bucket_limit); 252 253 cache_limit = V_tcp_hostcache.hashsize * V_tcp_hostcache.bucket_limit; 254 V_tcp_hostcache.cache_limit = cache_limit; 255 TUNABLE_INT_FETCH("net.inet.tcp.hostcache.cachelimit", 256 &V_tcp_hostcache.cache_limit); 257 if (V_tcp_hostcache.cache_limit > cache_limit) 258 V_tcp_hostcache.cache_limit = cache_limit; 259 260 /* 261 * Allocate the hash table. 262 */ 263 V_tcp_hostcache.hashbase = (struct hc_head *) 264 malloc(V_tcp_hostcache.hashsize * sizeof(struct hc_head), 265 M_HOSTCACHE, M_WAITOK | M_ZERO); 266 267 /* 268 * Initialize the hash buckets. 269 */ 270 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 271 CK_SLIST_INIT(&V_tcp_hostcache.hashbase[i].hch_bucket); 272 V_tcp_hostcache.hashbase[i].hch_length = 0; 273 mtx_init(&V_tcp_hostcache.hashbase[i].hch_mtx, "tcp_hc_entry", 274 NULL, MTX_DEF); 275 } 276 277 /* 278 * Allocate the hostcache entries. 279 */ 280 V_tcp_hostcache.zone = 281 uma_zcreate("hostcache", sizeof(struct hc_metrics), 282 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_SMR); 283 uma_zone_set_max(V_tcp_hostcache.zone, V_tcp_hostcache.cache_limit); 284 V_tcp_hostcache.smr = uma_zone_get_smr(V_tcp_hostcache.zone); 285 286 /* 287 * Set up periodic cache cleanup. 288 */ 289 callout_init(&V_tcp_hc_callout, 1); 290 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 291 tcp_hc_purge, curvnet); 292 } 293 294 #ifdef VIMAGE 295 void 296 tcp_hc_destroy(void) 297 { 298 int i; 299 300 callout_drain(&V_tcp_hc_callout); 301 302 /* Purge all hc entries. */ 303 tcp_hc_purge_internal(1); 304 305 /* Free the uma zone and the allocated hash table. */ 306 uma_zdestroy(V_tcp_hostcache.zone); 307 308 for (i = 0; i < V_tcp_hostcache.hashsize; i++) 309 mtx_destroy(&V_tcp_hostcache.hashbase[i].hch_mtx); 310 free(V_tcp_hostcache.hashbase, M_HOSTCACHE); 311 } 312 #endif 313 314 /* 315 * Internal function: compare cache entry to a connection. 316 */ 317 static bool 318 tcp_hc_cmp(struct hc_metrics *hc_entry, const struct in_conninfo *inc) 319 { 320 321 if (inc->inc_flags & INC_ISIPV6) { 322 /* XXX: check ip6_zoneid */ 323 if (memcmp(&inc->inc6_faddr, &hc_entry->ip6, 324 sizeof(inc->inc6_faddr)) == 0) 325 return (true); 326 } else { 327 if (memcmp(&inc->inc_faddr, &hc_entry->ip4, 328 sizeof(inc->inc_faddr)) == 0) 329 return (true); 330 } 331 332 return (false); 333 } 334 335 /* 336 * Internal function: look up an entry in the hostcache for read. 337 * On success returns in SMR section. 338 */ 339 static struct hc_metrics * 340 tcp_hc_lookup(const struct in_conninfo *inc) 341 { 342 struct hc_head *hc_head; 343 struct hc_metrics *hc_entry; 344 345 KASSERT(inc != NULL, ("%s: NULL in_conninfo", __func__)); 346 347 hc_head = &V_tcp_hostcache.hashbase[HOSTCACHE_HASH(inc)]; 348 349 /* 350 * Iterate through entries in bucket row looking for a match. 351 */ 352 smr_enter(V_tcp_hostcache.smr); 353 CK_SLIST_FOREACH(hc_entry, &hc_head->hch_bucket, hc_q) 354 if (tcp_hc_cmp(hc_entry, inc)) 355 break; 356 357 if (hc_entry != NULL) { 358 if (atomic_load_int(&hc_entry->hc_expire) != 359 V_tcp_hostcache.expire) 360 atomic_store_int(&hc_entry->hc_expire, 361 V_tcp_hostcache.expire); 362 #ifdef TCP_HC_COUNTERS 363 hc_entry->hc_hits++; 364 #endif 365 TCPSTAT_INC(tcps_hc_hits); 366 } else { 367 smr_exit(V_tcp_hostcache.smr); 368 TCPSTAT_INC(tcps_hc_misses); 369 } 370 371 return (hc_entry); 372 } 373 374 /* 375 * External function: look up an entry in the hostcache and fill out the 376 * supplied TCP metrics structure. Fills in NULL when no entry was found or 377 * a value is not set. 378 */ 379 void 380 tcp_hc_get(const struct in_conninfo *inc, 381 struct tcp_hc_metrics *hc_metrics) 382 { 383 struct hc_metrics *hc_entry; 384 385 if (!V_tcp_use_hostcache) { 386 bzero(hc_metrics, sizeof(*hc_metrics)); 387 return; 388 } 389 390 /* 391 * Find the right bucket. 392 */ 393 hc_entry = tcp_hc_lookup(inc); 394 395 /* 396 * If we don't have an existing object. 397 */ 398 if (hc_entry == NULL) { 399 bzero(hc_metrics, sizeof(*hc_metrics)); 400 return; 401 } 402 403 hc_metrics->hc_mtu = atomic_load_32(&hc_entry->hc_mtu); 404 hc_metrics->hc_ssthresh = atomic_load_32(&hc_entry->hc_ssthresh); 405 hc_metrics->hc_rtt = atomic_load_32(&hc_entry->hc_rtt); 406 hc_metrics->hc_rttvar = atomic_load_32(&hc_entry->hc_rttvar); 407 hc_metrics->hc_cwnd = atomic_load_32(&hc_entry->hc_cwnd); 408 hc_metrics->hc_sendpipe = atomic_load_32(&hc_entry->hc_sendpipe); 409 hc_metrics->hc_recvpipe = atomic_load_32(&hc_entry->hc_recvpipe); 410 411 smr_exit(V_tcp_hostcache.smr); 412 } 413 414 /* 415 * External function: look up an entry in the hostcache and return the 416 * discovered path MTU. Returns 0 if no entry is found or value is not 417 * set. 418 */ 419 uint32_t 420 tcp_hc_getmtu(const struct in_conninfo *inc) 421 { 422 struct hc_metrics *hc_entry; 423 uint32_t mtu; 424 425 if (!V_tcp_use_hostcache) 426 return (0); 427 428 hc_entry = tcp_hc_lookup(inc); 429 if (hc_entry == NULL) { 430 return (0); 431 } 432 433 mtu = atomic_load_32(&hc_entry->hc_mtu); 434 smr_exit(V_tcp_hostcache.smr); 435 436 return (mtu); 437 } 438 439 /* 440 * External function: update the MTU value of an entry in the hostcache. 441 * Creates a new entry if none was found. 442 */ 443 void 444 tcp_hc_updatemtu(const struct in_conninfo *inc, uint32_t mtu) 445 { 446 struct tcp_hc_metrics hcm = { .hc_mtu = mtu }; 447 448 return (tcp_hc_update(inc, &hcm)); 449 } 450 451 /* 452 * External function: update the TCP metrics of an entry in the hostcache. 453 * Creates a new entry if none was found. 454 */ 455 void 456 tcp_hc_update(const struct in_conninfo *inc, struct tcp_hc_metrics *hcm) 457 { 458 struct hc_head *hc_head; 459 struct hc_metrics *hc_entry, *hc_prev; 460 uint32_t v; 461 bool new; 462 463 if (!V_tcp_use_hostcache) 464 return; 465 466 hc_head = &V_tcp_hostcache.hashbase[HOSTCACHE_HASH(inc)]; 467 hc_prev = NULL; 468 469 THC_LOCK(hc_head); 470 CK_SLIST_FOREACH(hc_entry, &hc_head->hch_bucket, hc_q) { 471 if (tcp_hc_cmp(hc_entry, inc)) 472 break; 473 if (CK_SLIST_NEXT(hc_entry, hc_q) != NULL) 474 hc_prev = hc_entry; 475 } 476 477 if (hc_entry != NULL) { 478 if (atomic_load_int(&hc_entry->hc_expire) != 479 V_tcp_hostcache.expire) 480 atomic_store_int(&hc_entry->hc_expire, 481 V_tcp_hostcache.expire); 482 #ifdef TCP_HC_COUNTERS 483 hc_entry->hc_updates++; 484 #endif 485 new = false; 486 } else { 487 /* 488 * Try to allocate a new entry. If the bucket limit is 489 * reached, delete the least-used element, located at the end 490 * of the CK_SLIST. During lookup we saved the pointer to 491 * the second to last element, in case if list has at least 2 492 * elements. This will allow to delete last element without 493 * extra traversal. 494 * 495 * Give up if the row is empty. 496 */ 497 if (hc_head->hch_length >= V_tcp_hostcache.bucket_limit || 498 atomic_load_int(&V_tcp_hostcache.cache_count) >= 499 V_tcp_hostcache.cache_limit) { 500 if (hc_prev != NULL) { 501 hc_entry = CK_SLIST_NEXT(hc_prev, hc_q); 502 KASSERT(CK_SLIST_NEXT(hc_entry, hc_q) == NULL, 503 ("%s: %p is not one to last", 504 __func__, hc_prev)); 505 CK_SLIST_REMOVE_AFTER(hc_prev, hc_q); 506 } else if ((hc_entry = 507 CK_SLIST_FIRST(&hc_head->hch_bucket)) != NULL) { 508 KASSERT(CK_SLIST_NEXT(hc_entry, hc_q) == NULL, 509 ("%s: %p is not the only element", 510 __func__, hc_entry)); 511 CK_SLIST_REMOVE_HEAD(&hc_head->hch_bucket, 512 hc_q); 513 } else { 514 THC_UNLOCK(hc_head); 515 return; 516 } 517 KASSERT(hc_head->hch_length > 0 && 518 hc_head->hch_length <= V_tcp_hostcache.bucket_limit, 519 ("tcp_hostcache: bucket length violated at %p", 520 hc_head)); 521 hc_head->hch_length--; 522 atomic_subtract_int(&V_tcp_hostcache.cache_count, 1); 523 TCPSTAT_INC(tcps_hc_bucketoverflow); 524 uma_zfree_smr(V_tcp_hostcache.zone, hc_entry); 525 } 526 527 /* 528 * Allocate a new entry, or balk if not possible. 529 */ 530 hc_entry = uma_zalloc_smr(V_tcp_hostcache.zone, M_NOWAIT); 531 if (hc_entry == NULL) { 532 THC_UNLOCK(hc_head); 533 TCPSTAT_INC(tcps_hc_allocfail); 534 return; 535 } 536 537 /* 538 * Initialize basic information of hostcache entry. 539 */ 540 bzero(hc_entry, sizeof(*hc_entry)); 541 if (inc->inc_flags & INC_ISIPV6) { 542 hc_entry->ip6 = inc->inc6_faddr; 543 hc_entry->ip6_zoneid = inc->inc6_zoneid; 544 } else 545 hc_entry->ip4 = inc->inc_faddr; 546 hc_entry->hc_expire = V_tcp_hostcache.expire; 547 new = true; 548 } 549 550 /* 551 * Fill in data. Use atomics, since an existing entry is 552 * accessible by readers in SMR section. 553 */ 554 if (hcm->hc_mtu != 0) { 555 atomic_store_32(&hc_entry->hc_mtu, hcm->hc_mtu); 556 } 557 if (hcm->hc_rtt != 0) { 558 if (hc_entry->hc_rtt == 0) 559 v = hcm->hc_rtt; 560 else 561 v = ((uint64_t)hc_entry->hc_rtt + 562 (uint64_t)hcm->hc_rtt) / 2; 563 atomic_store_32(&hc_entry->hc_rtt, v); 564 TCPSTAT_INC(tcps_cachedrtt); 565 } 566 if (hcm->hc_rttvar != 0) { 567 if (hc_entry->hc_rttvar == 0) 568 v = hcm->hc_rttvar; 569 else 570 v = ((uint64_t)hc_entry->hc_rttvar + 571 (uint64_t)hcm->hc_rttvar) / 2; 572 atomic_store_32(&hc_entry->hc_rttvar, v); 573 TCPSTAT_INC(tcps_cachedrttvar); 574 } 575 if (hcm->hc_ssthresh != 0) { 576 if (hc_entry->hc_ssthresh == 0) 577 v = hcm->hc_ssthresh; 578 else 579 v = (hc_entry->hc_ssthresh + 580 hcm->hc_ssthresh) / 2; 581 atomic_store_32(&hc_entry->hc_ssthresh, v); 582 TCPSTAT_INC(tcps_cachedssthresh); 583 } 584 if (hcm->hc_cwnd != 0) { 585 if (hc_entry->hc_cwnd == 0) 586 v = hcm->hc_cwnd; 587 else 588 v = ((uint64_t)hc_entry->hc_cwnd + 589 (uint64_t)hcm->hc_cwnd) / 2; 590 atomic_store_32(&hc_entry->hc_cwnd, v); 591 TCPSTAT_INC(tcps_cachedcwnd); 592 } 593 if (hcm->hc_sendpipe != 0) { 594 if (hc_entry->hc_sendpipe == 0) 595 v = hcm->hc_sendpipe; 596 else 597 v = ((uint64_t)hc_entry->hc_sendpipe + 598 (uint64_t)hcm->hc_sendpipe) / 2; 599 atomic_store_32(&hc_entry->hc_sendpipe, v); 600 TCPSTAT_INC(tcps_cachedsendpipe); 601 } 602 if (hcm->hc_recvpipe != 0) { 603 if (hc_entry->hc_recvpipe == 0) 604 v = hcm->hc_recvpipe; 605 else 606 v = ((uint64_t)hc_entry->hc_recvpipe + 607 (uint64_t)hcm->hc_recvpipe) / 2; 608 atomic_store_32(&hc_entry->hc_recvpipe, v); 609 TCPSTAT_INC(tcps_cachedrecvpipe); 610 } 611 612 /* 613 * Put it upfront. 614 */ 615 if (new) { 616 CK_SLIST_INSERT_HEAD(&hc_head->hch_bucket, hc_entry, hc_q); 617 hc_head->hch_length++; 618 KASSERT(hc_head->hch_length <= V_tcp_hostcache.bucket_limit, 619 ("tcp_hostcache: bucket length too high at %p", hc_head)); 620 atomic_add_int(&V_tcp_hostcache.cache_count, 1); 621 TCPSTAT_INC(tcps_hc_added); 622 } else if (hc_entry != CK_SLIST_FIRST(&hc_head->hch_bucket)) { 623 KASSERT(CK_SLIST_NEXT(hc_prev, hc_q) == hc_entry, 624 ("%s: %p next is not %p", __func__, hc_prev, hc_entry)); 625 CK_SLIST_REMOVE_AFTER(hc_prev, hc_q); 626 CK_SLIST_INSERT_HEAD(&hc_head->hch_bucket, hc_entry, hc_q); 627 } 628 THC_UNLOCK(hc_head); 629 } 630 631 /* 632 * Sysctl function: adjusts the expire timeout and adjusts the prune value accordingly. 633 */ 634 static int 635 sysctl_tcp_hc_expire(SYSCTL_HANDLER_ARGS) 636 { 637 int error, expire; 638 639 expire = V_tcp_hostcache.expire; 640 error = sysctl_handle_int(oidp, &expire, 0, req); 641 if (error != 0 || !req->newptr) 642 return (error); 643 if (expire < V_tcp_hostcache.prune) 644 V_tcp_hostcache.prune = expire; 645 V_tcp_hostcache.expire = expire; 646 return (0); 647 } 648 649 /* 650 * Sysctl function: adjusts the prune time and adjusts the expire timeout accordingly. 651 */ 652 static int 653 sysctl_tcp_hc_prune(SYSCTL_HANDLER_ARGS) 654 { 655 int error, prune; 656 657 prune = V_tcp_hostcache.prune; 658 error = sysctl_handle_int(oidp, &prune, 0, req); 659 if (error != 0 || !req->newptr) 660 return (error); 661 if (prune > V_tcp_hostcache.expire) 662 V_tcp_hostcache.expire = prune; 663 V_tcp_hostcache.prune = prune; 664 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 665 tcp_hc_purge, curvnet); 666 return (0); 667 } 668 669 /* 670 * Sysctl function: prints the list and values of all hostcache entries in 671 * unsorted order. 672 */ 673 static int 674 sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS) 675 { 676 const int linesize = 128; 677 struct sbuf sb; 678 int i, error, len; 679 struct hc_metrics *hc_entry; 680 char ip4buf[INET_ADDRSTRLEN]; 681 #ifdef INET6 682 char ip6buf[INET6_ADDRSTRLEN]; 683 #endif 684 685 if (jailed_without_vnet(curthread->td_ucred) != 0) 686 return (EPERM); 687 688 /* Optimize Buffer length query by sbin/sysctl */ 689 if (req->oldptr == NULL) { 690 len = (atomic_load_int(&V_tcp_hostcache.cache_count) + 1) * 691 linesize; 692 return (SYSCTL_OUT(req, NULL, len)); 693 } 694 695 error = sysctl_wire_old_buffer(req, 0); 696 if (error != 0) { 697 return(error); 698 } 699 700 /* Use a buffer sized for one full bucket */ 701 sbuf_new_for_sysctl(&sb, NULL, V_tcp_hostcache.bucket_limit * 702 linesize, req); 703 704 sbuf_printf(&sb, 705 "\nIP address MTU SSTRESH RTT RTTVAR " 706 " CWND SENDPIPE RECVPIPE " 707 #ifdef TCP_HC_COUNTERS 708 "HITS UPD " 709 #endif 710 "EXP\n"); 711 sbuf_drain(&sb); 712 713 #define msec(u) (((u) + 500) / 1000) 714 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 715 THC_LOCK(&V_tcp_hostcache.hashbase[i]); 716 CK_SLIST_FOREACH(hc_entry, 717 &V_tcp_hostcache.hashbase[i].hch_bucket, hc_q) { 718 sbuf_printf(&sb, 719 "%-15s %5u %8u %6lums %6lums %8u %8u %8u " 720 #ifdef TCP_HC_COUNTERS 721 "%4lu %4lu " 722 #endif 723 "%4i\n", 724 hc_entry->ip4.s_addr ? 725 inet_ntoa_r(hc_entry->ip4, ip4buf) : 726 #ifdef INET6 727 ip6_sprintf(ip6buf, &hc_entry->ip6), 728 #else 729 "IPv6?", 730 #endif 731 hc_entry->hc_mtu, 732 hc_entry->hc_ssthresh, 733 msec((u_long)hc_entry->hc_rtt * 734 (RTM_RTTUNIT / (hz * TCP_RTT_SCALE))), 735 msec((u_long)hc_entry->hc_rttvar * 736 (RTM_RTTUNIT / (hz * TCP_RTTVAR_SCALE))), 737 hc_entry->hc_cwnd, 738 hc_entry->hc_sendpipe, 739 hc_entry->hc_recvpipe, 740 #ifdef TCP_HC_COUNTERS 741 hc_entry->hc_hits, 742 hc_entry->hc_updates, 743 #endif 744 hc_entry->hc_expire); 745 } 746 THC_UNLOCK(&V_tcp_hostcache.hashbase[i]); 747 sbuf_drain(&sb); 748 } 749 #undef msec 750 error = sbuf_finish(&sb); 751 sbuf_delete(&sb); 752 return(error); 753 } 754 755 /* 756 * Sysctl function: prints a histogram of the hostcache hashbucket 757 * utilization. 758 */ 759 static int 760 sysctl_tcp_hc_histo(SYSCTL_HANDLER_ARGS) 761 { 762 const int linesize = 50; 763 struct sbuf sb; 764 int i, error; 765 int *histo; 766 u_int hch_length; 767 768 if (jailed_without_vnet(curthread->td_ucred) != 0) 769 return (EPERM); 770 771 histo = (int *)malloc(sizeof(int) * (V_tcp_hostcache.bucket_limit + 1), 772 M_TEMP, M_NOWAIT|M_ZERO); 773 if (histo == NULL) 774 return(ENOMEM); 775 776 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 777 hch_length = V_tcp_hostcache.hashbase[i].hch_length; 778 KASSERT(hch_length <= V_tcp_hostcache.bucket_limit, 779 ("tcp_hostcache: bucket limit exceeded at %u: %u", 780 i, hch_length)); 781 histo[hch_length]++; 782 } 783 784 /* Use a buffer for 16 lines */ 785 sbuf_new_for_sysctl(&sb, NULL, 16 * linesize, req); 786 787 sbuf_printf(&sb, "\nLength\tCount\n"); 788 for (i = 0; i <= V_tcp_hostcache.bucket_limit; i++) { 789 sbuf_printf(&sb, "%u\t%u\n", i, histo[i]); 790 } 791 error = sbuf_finish(&sb); 792 sbuf_delete(&sb); 793 free(histo, M_TEMP); 794 return(error); 795 } 796 797 /* 798 * Caller has to make sure the curvnet is set properly. 799 */ 800 static void 801 tcp_hc_purge_internal(int all) 802 { 803 struct hc_head *head; 804 struct hc_metrics *hc_entry, *hc_next, *hc_prev; 805 int i; 806 807 for (i = 0; i < V_tcp_hostcache.hashsize; i++) { 808 head = &V_tcp_hostcache.hashbase[i]; 809 hc_prev = NULL; 810 THC_LOCK(head); 811 CK_SLIST_FOREACH_SAFE(hc_entry, &head->hch_bucket, hc_q, 812 hc_next) { 813 KASSERT(head->hch_length > 0 && head->hch_length <= 814 V_tcp_hostcache.bucket_limit, ("tcp_hostcache: " 815 "bucket length out of range at %u: %u", i, 816 head->hch_length)); 817 if (all || 818 (int)atomic_load_int(&hc_entry->hc_expire) <= 0) { 819 if (hc_prev != NULL) { 820 KASSERT(hc_entry == 821 CK_SLIST_NEXT(hc_prev, hc_q), 822 ("%s: %p is not next to %p", 823 __func__, hc_entry, hc_prev)); 824 CK_SLIST_REMOVE_AFTER(hc_prev, hc_q); 825 } else { 826 KASSERT(hc_entry == 827 CK_SLIST_FIRST(&head->hch_bucket), 828 ("%s: %p is not first", 829 __func__, hc_entry)); 830 CK_SLIST_REMOVE_HEAD(&head->hch_bucket, 831 hc_q); 832 } 833 uma_zfree_smr(V_tcp_hostcache.zone, hc_entry); 834 head->hch_length--; 835 atomic_subtract_int(&V_tcp_hostcache.cache_count, 1); 836 } else { 837 atomic_subtract_int(&hc_entry->hc_expire, 838 V_tcp_hostcache.prune); 839 hc_prev = hc_entry; 840 } 841 } 842 THC_UNLOCK(head); 843 } 844 } 845 846 /* 847 * Expire and purge (old|all) entries in the tcp_hostcache. Runs 848 * periodically from the callout. 849 */ 850 static void 851 tcp_hc_purge(void *arg) 852 { 853 CURVNET_SET((struct vnet *) arg); 854 int all = 0; 855 856 if (V_tcp_hostcache.purgeall) { 857 if (V_tcp_hostcache.purgeall == 2) 858 V_tcp_hostcache.hashsalt = arc4random(); 859 all = 1; 860 V_tcp_hostcache.purgeall = 0; 861 } 862 863 tcp_hc_purge_internal(all); 864 865 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 866 tcp_hc_purge, arg); 867 CURVNET_RESTORE(); 868 } 869 870 /* 871 * Expire and purge all entries in hostcache immediately. 872 */ 873 static int 874 sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS) 875 { 876 int error, val; 877 878 val = 0; 879 error = sysctl_handle_int(oidp, &val, 0, req); 880 if (error != 0 || !req->newptr) 881 return (error); 882 883 if (val == 2) 884 V_tcp_hostcache.hashsalt = arc4random(); 885 tcp_hc_purge_internal(1); 886 887 callout_reset(&V_tcp_hc_callout, V_tcp_hostcache.prune * hz, 888 tcp_hc_purge, curvnet); 889 890 return (0); 891 } 892