1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2016-2018 Netflix, Inc. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 * 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/kernel.h> 34 #include <sys/lock.h> 35 #include <sys/malloc.h> 36 #include <sys/mutex.h> 37 #include <sys/queue.h> 38 #include <sys/refcount.h> 39 #include <sys/rwlock.h> 40 #include <sys/socket.h> 41 #include <sys/socketvar.h> 42 #include <sys/sysctl.h> 43 #include <sys/tree.h> 44 #include <sys/counter.h> 45 46 #include <dev/tcp_log/tcp_log_dev.h> 47 48 #include <net/if.h> 49 #include <net/if_var.h> 50 #include <net/vnet.h> 51 52 #include <netinet/in.h> 53 #include <netinet/in_pcb.h> 54 #include <netinet/in_var.h> 55 #include <netinet/tcp_var.h> 56 #include <netinet/tcp_log_buf.h> 57 58 /* Default expiry time */ 59 #define TCP_LOG_EXPIRE_TIME ((sbintime_t)60 * SBT_1S) 60 61 /* Max interval at which to run the expiry timer */ 62 #define TCP_LOG_EXPIRE_INTVL ((sbintime_t)5 * SBT_1S) 63 64 bool tcp_log_verbose; 65 static uma_zone_t tcp_log_bucket_zone, tcp_log_node_zone, tcp_log_zone; 66 static int tcp_log_session_limit = TCP_LOG_BUF_DEFAULT_SESSION_LIMIT; 67 static uint32_t tcp_log_version = TCP_LOG_BUF_VER; 68 RB_HEAD(tcp_log_id_tree, tcp_log_id_bucket); 69 static struct tcp_log_id_tree tcp_log_id_head; 70 static STAILQ_HEAD(, tcp_log_id_node) tcp_log_expireq_head = 71 STAILQ_HEAD_INITIALIZER(tcp_log_expireq_head); 72 static struct mtx tcp_log_expireq_mtx; 73 static struct callout tcp_log_expireq_callout; 74 static u_long tcp_log_auto_ratio = 0; 75 static volatile u_long tcp_log_auto_ratio_cur = 0; 76 static uint32_t tcp_log_auto_mode = TCP_LOG_STATE_TAIL; 77 static bool tcp_log_auto_all = false; 78 79 RB_PROTOTYPE_STATIC(tcp_log_id_tree, tcp_log_id_bucket, tlb_rb, tcp_log_id_cmp) 80 81 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, bb, CTLFLAG_RW, 0, "TCP Black Box controls"); 82 83 SYSCTL_BOOL(_net_inet_tcp_bb, OID_AUTO, log_verbose, CTLFLAG_RW, &tcp_log_verbose, 84 0, "Force verbose logging for TCP traces"); 85 86 SYSCTL_INT(_net_inet_tcp_bb, OID_AUTO, log_session_limit, 87 CTLFLAG_RW, &tcp_log_session_limit, 0, 88 "Maximum number of events maintained for each TCP session"); 89 90 SYSCTL_UMA_MAX(_net_inet_tcp_bb, OID_AUTO, log_global_limit, CTLFLAG_RW, 91 &tcp_log_zone, "Maximum number of events maintained for all TCP sessions"); 92 93 SYSCTL_UMA_CUR(_net_inet_tcp_bb, OID_AUTO, log_global_entries, CTLFLAG_RD, 94 &tcp_log_zone, "Current number of events maintained for all TCP sessions"); 95 96 SYSCTL_UMA_MAX(_net_inet_tcp_bb, OID_AUTO, log_id_limit, CTLFLAG_RW, 97 &tcp_log_bucket_zone, "Maximum number of log IDs"); 98 99 SYSCTL_UMA_CUR(_net_inet_tcp_bb, OID_AUTO, log_id_entries, CTLFLAG_RD, 100 &tcp_log_bucket_zone, "Current number of log IDs"); 101 102 SYSCTL_UMA_MAX(_net_inet_tcp_bb, OID_AUTO, log_id_tcpcb_limit, CTLFLAG_RW, 103 &tcp_log_node_zone, "Maximum number of tcpcbs with log IDs"); 104 105 SYSCTL_UMA_CUR(_net_inet_tcp_bb, OID_AUTO, log_id_tcpcb_entries, CTLFLAG_RD, 106 &tcp_log_node_zone, "Current number of tcpcbs with log IDs"); 107 108 SYSCTL_U32(_net_inet_tcp_bb, OID_AUTO, log_version, CTLFLAG_RD, &tcp_log_version, 109 0, "Version of log formats exported"); 110 111 SYSCTL_ULONG(_net_inet_tcp_bb, OID_AUTO, log_auto_ratio, CTLFLAG_RW, 112 &tcp_log_auto_ratio, 0, "Do auto capturing for 1 out of N sessions"); 113 114 SYSCTL_U32(_net_inet_tcp_bb, OID_AUTO, log_auto_mode, CTLFLAG_RW, 115 &tcp_log_auto_mode, TCP_LOG_STATE_HEAD_AUTO, 116 "Logging mode for auto-selected sessions (default is TCP_LOG_STATE_HEAD_AUTO)"); 117 118 SYSCTL_BOOL(_net_inet_tcp_bb, OID_AUTO, log_auto_all, CTLFLAG_RW, 119 &tcp_log_auto_all, false, 120 "Auto-select from all sessions (rather than just those with IDs)"); 121 122 #ifdef TCPLOG_DEBUG_COUNTERS 123 counter_u64_t tcp_log_queued; 124 counter_u64_t tcp_log_que_fail1; 125 counter_u64_t tcp_log_que_fail2; 126 counter_u64_t tcp_log_que_fail3; 127 counter_u64_t tcp_log_que_fail4; 128 counter_u64_t tcp_log_que_fail5; 129 counter_u64_t tcp_log_que_copyout; 130 counter_u64_t tcp_log_que_read; 131 counter_u64_t tcp_log_que_freed; 132 133 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, queued, CTLFLAG_RD, 134 &tcp_log_queued, "Number of entries queued"); 135 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail1, CTLFLAG_RD, 136 &tcp_log_que_fail1, "Number of entries queued but fail 1"); 137 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail2, CTLFLAG_RD, 138 &tcp_log_que_fail2, "Number of entries queued but fail 2"); 139 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail3, CTLFLAG_RD, 140 &tcp_log_que_fail3, "Number of entries queued but fail 3"); 141 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail4, CTLFLAG_RD, 142 &tcp_log_que_fail4, "Number of entries queued but fail 4"); 143 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, fail5, CTLFLAG_RD, 144 &tcp_log_que_fail5, "Number of entries queued but fail 4"); 145 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, copyout, CTLFLAG_RD, 146 &tcp_log_que_copyout, "Number of entries copied out"); 147 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, read, CTLFLAG_RD, 148 &tcp_log_que_read, "Number of entries read from the queue"); 149 SYSCTL_COUNTER_U64(_net_inet_tcp_bb, OID_AUTO, freed, CTLFLAG_RD, 150 &tcp_log_que_freed, "Number of entries freed after reading"); 151 #endif 152 153 #ifdef INVARIANTS 154 #define TCPLOG_DEBUG_RINGBUF 155 #endif 156 157 struct tcp_log_mem 158 { 159 STAILQ_ENTRY(tcp_log_mem) tlm_queue; 160 struct tcp_log_buffer tlm_buf; 161 struct tcp_log_verbose tlm_v; 162 #ifdef TCPLOG_DEBUG_RINGBUF 163 volatile int tlm_refcnt; 164 #endif 165 }; 166 167 /* 60 bytes for the header, + 16 bytes for padding */ 168 static uint8_t zerobuf[76]; 169 170 /* 171 * Lock order: 172 * 1. TCPID_TREE 173 * 2. TCPID_BUCKET 174 * 3. INP 175 * 176 * Rules: 177 * A. You need a lock on the Tree to add/remove buckets. 178 * B. You need a lock on the bucket to add/remove nodes from the bucket. 179 * C. To change information in a node, you need the INP lock if the tln_closed 180 * field is false. Otherwise, you need the bucket lock. (Note that the 181 * tln_closed field can change at any point, so you need to recheck the 182 * entry after acquiring the INP lock.) 183 * D. To remove a node from the bucket, you must have that entry locked, 184 * according to the criteria of Rule C. Also, the node must not be on 185 * the expiry queue. 186 * E. The exception to C is the expiry queue fields, which are locked by 187 * the TCPLOG_EXPIREQ lock. 188 * 189 * Buckets have a reference count. Each node is a reference. Further, 190 * other callers may add reference counts to keep a bucket from disappearing. 191 * You can add a reference as long as you own a lock sufficient to keep the 192 * bucket from disappearing. For example, a common use is: 193 * a. Have a locked INP, but need to lock the TCPID_BUCKET. 194 * b. Add a refcount on the bucket. (Safe because the INP lock prevents 195 * the TCPID_BUCKET from going away.) 196 * c. Drop the INP lock. 197 * d. Acquire a lock on the TCPID_BUCKET. 198 * e. Acquire a lock on the INP. 199 * f. Drop the refcount on the bucket. 200 * (At this point, the bucket may disappear.) 201 * 202 * Expire queue lock: 203 * You can acquire this with either the bucket or INP lock. Don't reverse it. 204 * When the expire code has committed to freeing a node, it resets the expiry 205 * time to SBT_MAX. That is the signal to everyone else that they should 206 * leave that node alone. 207 */ 208 static struct rwlock tcp_id_tree_lock; 209 #define TCPID_TREE_WLOCK() rw_wlock(&tcp_id_tree_lock) 210 #define TCPID_TREE_RLOCK() rw_rlock(&tcp_id_tree_lock) 211 #define TCPID_TREE_UPGRADE() rw_try_upgrade(&tcp_id_tree_lock) 212 #define TCPID_TREE_WUNLOCK() rw_wunlock(&tcp_id_tree_lock) 213 #define TCPID_TREE_RUNLOCK() rw_runlock(&tcp_id_tree_lock) 214 #define TCPID_TREE_WLOCK_ASSERT() rw_assert(&tcp_id_tree_lock, RA_WLOCKED) 215 #define TCPID_TREE_RLOCK_ASSERT() rw_assert(&tcp_id_tree_lock, RA_RLOCKED) 216 #define TCPID_TREE_UNLOCK_ASSERT() rw_assert(&tcp_id_tree_lock, RA_UNLOCKED) 217 218 #define TCPID_BUCKET_LOCK_INIT(tlb) mtx_init(&((tlb)->tlb_mtx), "tcp log id bucket", NULL, MTX_DEF) 219 #define TCPID_BUCKET_LOCK_DESTROY(tlb) mtx_destroy(&((tlb)->tlb_mtx)) 220 #define TCPID_BUCKET_LOCK(tlb) mtx_lock(&((tlb)->tlb_mtx)) 221 #define TCPID_BUCKET_UNLOCK(tlb) mtx_unlock(&((tlb)->tlb_mtx)) 222 #define TCPID_BUCKET_LOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_OWNED) 223 #define TCPID_BUCKET_UNLOCK_ASSERT(tlb) mtx_assert(&((tlb)->tlb_mtx), MA_NOTOWNED) 224 225 #define TCPID_BUCKET_REF(tlb) refcount_acquire(&((tlb)->tlb_refcnt)) 226 #define TCPID_BUCKET_UNREF(tlb) refcount_release(&((tlb)->tlb_refcnt)) 227 228 #define TCPLOG_EXPIREQ_LOCK() mtx_lock(&tcp_log_expireq_mtx) 229 #define TCPLOG_EXPIREQ_UNLOCK() mtx_unlock(&tcp_log_expireq_mtx) 230 231 SLIST_HEAD(tcp_log_id_head, tcp_log_id_node); 232 233 struct tcp_log_id_bucket 234 { 235 /* 236 * tlb_id must be first. This lets us use strcmp on 237 * (struct tcp_log_id_bucket *) and (char *) interchangeably. 238 */ 239 char tlb_id[TCP_LOG_ID_LEN]; 240 RB_ENTRY(tcp_log_id_bucket) tlb_rb; 241 struct tcp_log_id_head tlb_head; 242 struct mtx tlb_mtx; 243 volatile u_int tlb_refcnt; 244 }; 245 246 struct tcp_log_id_node 247 { 248 SLIST_ENTRY(tcp_log_id_node) tln_list; 249 STAILQ_ENTRY(tcp_log_id_node) tln_expireq; /* Locked by the expireq lock */ 250 sbintime_t tln_expiretime; /* Locked by the expireq lock */ 251 252 /* 253 * If INP is NULL, that means the connection has closed. We've 254 * saved the connection endpoint information and the log entries 255 * in the tln_ie and tln_entries members. We've also saved a pointer 256 * to the enclosing bucket here. If INP is not NULL, the information is 257 * in the PCB and not here. 258 */ 259 struct inpcb *tln_inp; 260 struct tcpcb *tln_tp; 261 struct tcp_log_id_bucket *tln_bucket; 262 struct in_endpoints tln_ie; 263 struct tcp_log_stailq tln_entries; 264 int tln_count; 265 volatile int tln_closed; 266 uint8_t tln_af; 267 }; 268 269 enum tree_lock_state { 270 TREE_UNLOCKED = 0, 271 TREE_RLOCKED, 272 TREE_WLOCKED, 273 }; 274 275 /* Do we want to select this session for auto-logging? */ 276 static __inline bool 277 tcp_log_selectauto(void) 278 { 279 280 /* 281 * If we are doing auto-capturing, figure out whether we will capture 282 * this session. 283 */ 284 if (tcp_log_auto_ratio && 285 (atomic_fetchadd_long(&tcp_log_auto_ratio_cur, 1) % 286 tcp_log_auto_ratio) == 0) 287 return (true); 288 return (false); 289 } 290 291 static __inline int 292 tcp_log_id_cmp(struct tcp_log_id_bucket *a, struct tcp_log_id_bucket *b) 293 { 294 KASSERT(a != NULL, ("tcp_log_id_cmp: argument a is unexpectedly NULL")); 295 KASSERT(b != NULL, ("tcp_log_id_cmp: argument b is unexpectedly NULL")); 296 return strncmp(a->tlb_id, b->tlb_id, TCP_LOG_ID_LEN); 297 } 298 299 RB_GENERATE_STATIC(tcp_log_id_tree, tcp_log_id_bucket, tlb_rb, tcp_log_id_cmp) 300 301 static __inline void 302 tcp_log_id_validate_tree_lock(int tree_locked) 303 { 304 305 #ifdef INVARIANTS 306 switch (tree_locked) { 307 case TREE_WLOCKED: 308 TCPID_TREE_WLOCK_ASSERT(); 309 break; 310 case TREE_RLOCKED: 311 TCPID_TREE_RLOCK_ASSERT(); 312 break; 313 case TREE_UNLOCKED: 314 TCPID_TREE_UNLOCK_ASSERT(); 315 break; 316 default: 317 kassert_panic("%s:%d: unknown tree lock state", __func__, 318 __LINE__); 319 } 320 #endif 321 } 322 323 static __inline void 324 tcp_log_remove_bucket(struct tcp_log_id_bucket *tlb) 325 { 326 327 TCPID_TREE_WLOCK_ASSERT(); 328 KASSERT(SLIST_EMPTY(&tlb->tlb_head), 329 ("%s: Attempt to remove non-empty bucket", __func__)); 330 if (RB_REMOVE(tcp_log_id_tree, &tcp_log_id_head, tlb) == NULL) { 331 #ifdef INVARIANTS 332 kassert_panic("%s:%d: error removing element from tree", 333 __func__, __LINE__); 334 #endif 335 } 336 TCPID_BUCKET_LOCK_DESTROY(tlb); 337 uma_zfree(tcp_log_bucket_zone, tlb); 338 } 339 340 /* 341 * Call with a referenced and locked bucket. 342 * Will return true if the bucket was freed; otherwise, false. 343 * tlb: The bucket to unreference. 344 * tree_locked: A pointer to the state of the tree lock. If the tree lock 345 * state changes, the function will update it. 346 * inp: If not NULL and the function needs to drop the inp lock to relock the 347 * tree, it will do so. (The caller must ensure inp will not become invalid, 348 * probably by holding a reference to it.) 349 */ 350 static bool 351 tcp_log_unref_bucket(struct tcp_log_id_bucket *tlb, int *tree_locked, 352 struct inpcb *inp) 353 { 354 355 KASSERT(tlb != NULL, ("%s: called with NULL tlb", __func__)); 356 KASSERT(tree_locked != NULL, ("%s: called with NULL tree_locked", 357 __func__)); 358 359 tcp_log_id_validate_tree_lock(*tree_locked); 360 361 /* 362 * Did we hold the last reference on the tlb? If so, we may need 363 * to free it. (Note that we can realistically only execute the 364 * loop twice: once without a write lock and once with a write 365 * lock.) 366 */ 367 while (TCPID_BUCKET_UNREF(tlb)) { 368 /* 369 * We need a write lock on the tree to free this. 370 * If we can upgrade the tree lock, this is "easy". If we 371 * can't upgrade the tree lock, we need to do this the 372 * "hard" way: unwind all our locks and relock everything. 373 * In the meantime, anything could have changed. We even 374 * need to validate that we still need to free the bucket. 375 */ 376 if (*tree_locked == TREE_RLOCKED && TCPID_TREE_UPGRADE()) 377 *tree_locked = TREE_WLOCKED; 378 else if (*tree_locked != TREE_WLOCKED) { 379 TCPID_BUCKET_REF(tlb); 380 if (inp != NULL) 381 INP_WUNLOCK(inp); 382 TCPID_BUCKET_UNLOCK(tlb); 383 if (*tree_locked == TREE_RLOCKED) 384 TCPID_TREE_RUNLOCK(); 385 TCPID_TREE_WLOCK(); 386 *tree_locked = TREE_WLOCKED; 387 TCPID_BUCKET_LOCK(tlb); 388 if (inp != NULL) 389 INP_WLOCK(inp); 390 continue; 391 } 392 393 /* 394 * We have an empty bucket and a write lock on the tree. 395 * Remove the empty bucket. 396 */ 397 tcp_log_remove_bucket(tlb); 398 return (true); 399 } 400 return (false); 401 } 402 403 /* 404 * Call with a locked bucket. This function will release the lock on the 405 * bucket before returning. 406 * 407 * The caller is responsible for freeing the tp->t_lin/tln node! 408 * 409 * Note: one of tp or both tlb and tln must be supplied. 410 * 411 * inp: A pointer to the inp. If the function needs to drop the inp lock to 412 * acquire the tree write lock, it will do so. (The caller must ensure inp 413 * will not become invalid, probably by holding a reference to it.) 414 * tp: A pointer to the tcpcb. (optional; if specified, tlb and tln are ignored) 415 * tlb: A pointer to the bucket. (optional; ignored if tp is specified) 416 * tln: A pointer to the node. (optional; ignored if tp is specified) 417 * tree_locked: A pointer to the state of the tree lock. If the tree lock 418 * state changes, the function will update it. 419 * 420 * Will return true if the INP lock was reacquired; otherwise, false. 421 */ 422 static bool 423 tcp_log_remove_id_node(struct inpcb *inp, struct tcpcb *tp, 424 struct tcp_log_id_bucket *tlb, struct tcp_log_id_node *tln, 425 int *tree_locked) 426 { 427 int orig_tree_locked; 428 429 KASSERT(tp != NULL || (tlb != NULL && tln != NULL), 430 ("%s: called with tp=%p, tlb=%p, tln=%p", __func__, 431 tp, tlb, tln)); 432 KASSERT(tree_locked != NULL, ("%s: called with NULL tree_locked", 433 __func__)); 434 435 if (tp != NULL) { 436 tlb = tp->t_lib; 437 tln = tp->t_lin; 438 KASSERT(tlb != NULL, ("%s: unexpectedly NULL tlb", __func__)); 439 KASSERT(tln != NULL, ("%s: unexpectedly NULL tln", __func__)); 440 } 441 442 tcp_log_id_validate_tree_lock(*tree_locked); 443 TCPID_BUCKET_LOCK_ASSERT(tlb); 444 445 /* 446 * Remove the node, clear the log bucket and node from the TCPCB, and 447 * decrement the bucket refcount. In the process, if this is the 448 * last reference, the bucket will be freed. 449 */ 450 SLIST_REMOVE(&tlb->tlb_head, tln, tcp_log_id_node, tln_list); 451 if (tp != NULL) { 452 tp->t_lib = NULL; 453 tp->t_lin = NULL; 454 } 455 orig_tree_locked = *tree_locked; 456 if (!tcp_log_unref_bucket(tlb, tree_locked, inp)) 457 TCPID_BUCKET_UNLOCK(tlb); 458 return (*tree_locked != orig_tree_locked); 459 } 460 461 #define RECHECK_INP_CLEAN(cleanup) do { \ 462 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { \ 463 rv = ECONNRESET; \ 464 cleanup; \ 465 goto done; \ 466 } \ 467 tp = intotcpcb(inp); \ 468 } while (0) 469 470 #define RECHECK_INP() RECHECK_INP_CLEAN(/* noop */) 471 472 static void 473 tcp_log_grow_tlb(char *tlb_id, struct tcpcb *tp) 474 { 475 476 INP_WLOCK_ASSERT(tp->t_inpcb); 477 478 #ifdef NETFLIX 479 if (V_tcp_perconn_stats_enable == 2 && tp->t_stats == NULL) 480 (void)tcp_stats_sample_rollthedice(tp, tlb_id, strlen(tlb_id)); 481 #endif 482 } 483 484 /* 485 * Set the TCP log ID for a TCPCB. 486 * Called with INPCB locked. Returns with it unlocked. 487 */ 488 int 489 tcp_log_set_id(struct tcpcb *tp, char *id) 490 { 491 struct tcp_log_id_bucket *tlb, *tmp_tlb; 492 struct tcp_log_id_node *tln; 493 struct inpcb *inp; 494 int tree_locked, rv; 495 bool bucket_locked; 496 497 tlb = NULL; 498 tln = NULL; 499 inp = tp->t_inpcb; 500 tree_locked = TREE_UNLOCKED; 501 bucket_locked = false; 502 503 restart: 504 INP_WLOCK_ASSERT(inp); 505 506 /* See if the ID is unchanged. */ 507 if ((tp->t_lib != NULL && !strcmp(tp->t_lib->tlb_id, id)) || 508 (tp->t_lib == NULL && *id == 0)) { 509 rv = 0; 510 goto done; 511 } 512 513 /* 514 * If the TCPCB had a previous ID, we need to extricate it from 515 * the previous list. 516 * 517 * Drop the TCPCB lock and lock the tree and the bucket. 518 * Because this is called in the socket context, we (theoretically) 519 * don't need to worry about the INPCB completely going away 520 * while we are gone. 521 */ 522 if (tp->t_lib != NULL) { 523 tlb = tp->t_lib; 524 TCPID_BUCKET_REF(tlb); 525 INP_WUNLOCK(inp); 526 527 if (tree_locked == TREE_UNLOCKED) { 528 TCPID_TREE_RLOCK(); 529 tree_locked = TREE_RLOCKED; 530 } 531 TCPID_BUCKET_LOCK(tlb); 532 bucket_locked = true; 533 INP_WLOCK(inp); 534 535 /* 536 * Unreference the bucket. If our bucket went away, it is no 537 * longer locked or valid. 538 */ 539 if (tcp_log_unref_bucket(tlb, &tree_locked, inp)) { 540 bucket_locked = false; 541 tlb = NULL; 542 } 543 544 /* Validate the INP. */ 545 RECHECK_INP(); 546 547 /* 548 * Evaluate whether the bucket changed while we were unlocked. 549 * 550 * Possible scenarios here: 551 * 1. Bucket is unchanged and the same one we started with. 552 * 2. The TCPCB no longer has a bucket and our bucket was 553 * freed. 554 * 3. The TCPCB has a new bucket, whether ours was freed. 555 * 4. The TCPCB no longer has a bucket and our bucket was 556 * not freed. 557 * 558 * In cases 2-4, we will start over. In case 1, we will 559 * proceed here to remove the bucket. 560 */ 561 if (tlb == NULL || tp->t_lib != tlb) { 562 KASSERT(bucket_locked || tlb == NULL, 563 ("%s: bucket_locked (%d) and tlb (%p) are " 564 "inconsistent", __func__, bucket_locked, tlb)); 565 566 if (bucket_locked) { 567 TCPID_BUCKET_UNLOCK(tlb); 568 bucket_locked = false; 569 tlb = NULL; 570 } 571 goto restart; 572 } 573 574 /* 575 * Store the (struct tcp_log_id_node) for reuse. Then, remove 576 * it from the bucket. In the process, we may end up relocking. 577 * If so, we need to validate that the INP is still valid, and 578 * the TCPCB entries match we expect. 579 * 580 * We will clear tlb and change the bucket_locked state just 581 * before calling tcp_log_remove_id_node(), since that function 582 * will unlock the bucket. 583 */ 584 if (tln != NULL) 585 uma_zfree(tcp_log_node_zone, tln); 586 tln = tp->t_lin; 587 tlb = NULL; 588 bucket_locked = false; 589 if (tcp_log_remove_id_node(inp, tp, NULL, NULL, &tree_locked)) { 590 RECHECK_INP(); 591 592 /* 593 * If the TCPCB moved to a new bucket while we had 594 * dropped the lock, restart. 595 */ 596 if (tp->t_lib != NULL || tp->t_lin != NULL) 597 goto restart; 598 } 599 600 /* 601 * Yay! We successfully removed the TCPCB from its old 602 * bucket. Phew! 603 * 604 * On to bigger and better things... 605 */ 606 } 607 608 /* At this point, the TCPCB should not be in any bucket. */ 609 KASSERT(tp->t_lib == NULL, ("%s: tp->t_lib is not NULL", __func__)); 610 611 /* 612 * If the new ID is not empty, we need to now assign this TCPCB to a 613 * new bucket. 614 */ 615 if (*id) { 616 /* Get a new tln, if we don't already have one to reuse. */ 617 if (tln == NULL) { 618 tln = uma_zalloc(tcp_log_node_zone, M_NOWAIT | M_ZERO); 619 if (tln == NULL) { 620 rv = ENOBUFS; 621 goto done; 622 } 623 tln->tln_inp = inp; 624 tln->tln_tp = tp; 625 } 626 627 /* 628 * Drop the INP lock for a bit. We don't need it, and dropping 629 * it prevents lock order reversals. 630 */ 631 INP_WUNLOCK(inp); 632 633 /* Make sure we have at least a read lock on the tree. */ 634 tcp_log_id_validate_tree_lock(tree_locked); 635 if (tree_locked == TREE_UNLOCKED) { 636 TCPID_TREE_RLOCK(); 637 tree_locked = TREE_RLOCKED; 638 } 639 640 refind: 641 /* 642 * Remember that we constructed (struct tcp_log_id_node) so 643 * we can safely cast the id to it for the purposes of finding. 644 */ 645 KASSERT(tlb == NULL, ("%s:%d tlb unexpectedly non-NULL", 646 __func__, __LINE__)); 647 tmp_tlb = RB_FIND(tcp_log_id_tree, &tcp_log_id_head, 648 (struct tcp_log_id_bucket *) id); 649 650 /* 651 * If we didn't find a matching bucket, we need to add a new 652 * one. This requires a write lock. But, of course, we will 653 * need to recheck some things when we re-acquire the lock. 654 */ 655 if (tmp_tlb == NULL && tree_locked != TREE_WLOCKED) { 656 tree_locked = TREE_WLOCKED; 657 if (!TCPID_TREE_UPGRADE()) { 658 TCPID_TREE_RUNLOCK(); 659 TCPID_TREE_WLOCK(); 660 661 /* 662 * The tree may have changed while we were 663 * unlocked. 664 */ 665 goto refind; 666 } 667 } 668 669 /* If we need to add a new bucket, do it now. */ 670 if (tmp_tlb == NULL) { 671 /* Allocate new bucket. */ 672 tlb = uma_zalloc(tcp_log_bucket_zone, M_NOWAIT); 673 if (tlb == NULL) { 674 rv = ENOBUFS; 675 goto done_noinp; 676 } 677 678 /* 679 * Copy the ID to the bucket. 680 * NB: Don't use strlcpy() unless you are sure 681 * we've always validated NULL termination. 682 * 683 * TODO: When I'm done writing this, see if we 684 * we have correctly validated NULL termination and 685 * can use strlcpy(). :-) 686 */ 687 strncpy(tlb->tlb_id, id, TCP_LOG_ID_LEN - 1); 688 tlb->tlb_id[TCP_LOG_ID_LEN - 1] = '\0'; 689 690 /* 691 * Take the refcount for the first node and go ahead 692 * and lock this. Note that we zero the tlb_mtx 693 * structure, since 0xdeadc0de flips the right bits 694 * for the code to think that this mutex has already 695 * been initialized. :-( 696 */ 697 SLIST_INIT(&tlb->tlb_head); 698 refcount_init(&tlb->tlb_refcnt, 1); 699 memset(&tlb->tlb_mtx, 0, sizeof(struct mtx)); 700 TCPID_BUCKET_LOCK_INIT(tlb); 701 TCPID_BUCKET_LOCK(tlb); 702 bucket_locked = true; 703 704 #define FREE_NEW_TLB() do { \ 705 TCPID_BUCKET_LOCK_DESTROY(tlb); \ 706 uma_zfree(tcp_log_bucket_zone, tlb); \ 707 bucket_locked = false; \ 708 tlb = NULL; \ 709 } while (0) 710 /* 711 * Relock the INP and make sure we are still 712 * unassigned. 713 */ 714 INP_WLOCK(inp); 715 RECHECK_INP_CLEAN(FREE_NEW_TLB()); 716 if (tp->t_lib != NULL) { 717 FREE_NEW_TLB(); 718 goto restart; 719 } 720 721 /* Add the new bucket to the tree. */ 722 tmp_tlb = RB_INSERT(tcp_log_id_tree, &tcp_log_id_head, 723 tlb); 724 KASSERT(tmp_tlb == NULL, 725 ("%s: Unexpected conflicting bucket (%p) while " 726 "adding new bucket (%p)", __func__, tmp_tlb, tlb)); 727 728 /* 729 * If we found a conflicting bucket, free the new 730 * one we made and fall through to use the existing 731 * bucket. 732 */ 733 if (tmp_tlb != NULL) { 734 FREE_NEW_TLB(); 735 INP_WUNLOCK(inp); 736 } 737 #undef FREE_NEW_TLB 738 } 739 740 /* If we found an existing bucket, use it. */ 741 if (tmp_tlb != NULL) { 742 tlb = tmp_tlb; 743 TCPID_BUCKET_LOCK(tlb); 744 bucket_locked = true; 745 746 /* 747 * Relock the INP and make sure we are still 748 * unassigned. 749 */ 750 INP_UNLOCK_ASSERT(inp); 751 INP_WLOCK(inp); 752 RECHECK_INP(); 753 if (tp->t_lib != NULL) { 754 TCPID_BUCKET_UNLOCK(tlb); 755 bucket_locked = false; 756 tlb = NULL; 757 goto restart; 758 } 759 760 /* Take a reference on the bucket. */ 761 TCPID_BUCKET_REF(tlb); 762 } 763 764 tcp_log_grow_tlb(tlb->tlb_id, tp); 765 766 /* Add the new node to the list. */ 767 SLIST_INSERT_HEAD(&tlb->tlb_head, tln, tln_list); 768 tp->t_lib = tlb; 769 tp->t_lin = tln; 770 tln = NULL; 771 } 772 773 rv = 0; 774 775 done: 776 /* Unlock things, as needed, and return. */ 777 INP_WUNLOCK(inp); 778 done_noinp: 779 INP_UNLOCK_ASSERT(inp); 780 if (bucket_locked) { 781 TCPID_BUCKET_LOCK_ASSERT(tlb); 782 TCPID_BUCKET_UNLOCK(tlb); 783 } else if (tlb != NULL) 784 TCPID_BUCKET_UNLOCK_ASSERT(tlb); 785 if (tree_locked == TREE_WLOCKED) { 786 TCPID_TREE_WLOCK_ASSERT(); 787 TCPID_TREE_WUNLOCK(); 788 } else if (tree_locked == TREE_RLOCKED) { 789 TCPID_TREE_RLOCK_ASSERT(); 790 TCPID_TREE_RUNLOCK(); 791 } else 792 TCPID_TREE_UNLOCK_ASSERT(); 793 if (tln != NULL) 794 uma_zfree(tcp_log_node_zone, tln); 795 return (rv); 796 } 797 798 /* 799 * Get the TCP log ID for a TCPCB. 800 * Called with INPCB locked. 801 * 'buf' must point to a buffer that is at least TCP_LOG_ID_LEN bytes long. 802 * Returns number of bytes copied. 803 */ 804 size_t 805 tcp_log_get_id(struct tcpcb *tp, char *buf) 806 { 807 size_t len; 808 809 INP_LOCK_ASSERT(tp->t_inpcb); 810 if (tp->t_lib != NULL) { 811 len = strlcpy(buf, tp->t_lib->tlb_id, TCP_LOG_ID_LEN); 812 KASSERT(len < TCP_LOG_ID_LEN, 813 ("%s:%d: tp->t_lib->tlb_id too long (%zu)", 814 __func__, __LINE__, len)); 815 } else { 816 *buf = '\0'; 817 len = 0; 818 } 819 return (len); 820 } 821 822 /* 823 * Get number of connections with the same log ID. 824 * Log ID is taken from given TCPCB. 825 * Called with INPCB locked. 826 */ 827 u_int 828 tcp_log_get_id_cnt(struct tcpcb *tp) 829 { 830 831 INP_WLOCK_ASSERT(tp->t_inpcb); 832 return ((tp->t_lib == NULL) ? 0 : tp->t_lib->tlb_refcnt); 833 } 834 835 #ifdef TCPLOG_DEBUG_RINGBUF 836 /* 837 * Functions/macros to increment/decrement reference count for a log 838 * entry. This should catch when we do a double-free/double-remove or 839 * a double-add. 840 */ 841 static inline void 842 _tcp_log_entry_refcnt_add(struct tcp_log_mem *log_entry, const char *func, 843 int line) 844 { 845 int refcnt; 846 847 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, 1); 848 if (refcnt != 0) 849 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 0)", 850 func, line, log_entry, refcnt); 851 } 852 #define tcp_log_entry_refcnt_add(l) \ 853 _tcp_log_entry_refcnt_add((l), __func__, __LINE__) 854 855 static inline void 856 _tcp_log_entry_refcnt_rem(struct tcp_log_mem *log_entry, const char *func, 857 int line) 858 { 859 int refcnt; 860 861 refcnt = atomic_fetchadd_int(&log_entry->tlm_refcnt, -1); 862 if (refcnt != 1) 863 panic("%s:%d: log_entry(%p)->tlm_refcnt is %d (expected 1)", 864 func, line, log_entry, refcnt); 865 } 866 #define tcp_log_entry_refcnt_rem(l) \ 867 _tcp_log_entry_refcnt_rem((l), __func__, __LINE__) 868 869 #else /* !TCPLOG_DEBUG_RINGBUF */ 870 871 #define tcp_log_entry_refcnt_add(l) 872 #define tcp_log_entry_refcnt_rem(l) 873 874 #endif 875 876 /* 877 * Cleanup after removing a log entry, but only decrement the count if we 878 * are running INVARIANTS. 879 */ 880 static inline void 881 tcp_log_free_log_common(struct tcp_log_mem *log_entry, int *count __unused) 882 { 883 884 uma_zfree(tcp_log_zone, log_entry); 885 #ifdef INVARIANTS 886 (*count)--; 887 KASSERT(*count >= 0, 888 ("%s: count unexpectedly negative", __func__)); 889 #endif 890 } 891 892 static void 893 tcp_log_free_entries(struct tcp_log_stailq *head, int *count) 894 { 895 struct tcp_log_mem *log_entry; 896 897 /* Free the entries. */ 898 while ((log_entry = STAILQ_FIRST(head)) != NULL) { 899 STAILQ_REMOVE_HEAD(head, tlm_queue); 900 tcp_log_entry_refcnt_rem(log_entry); 901 tcp_log_free_log_common(log_entry, count); 902 } 903 } 904 905 /* Cleanup after removing a log entry. */ 906 static inline void 907 tcp_log_remove_log_cleanup(struct tcpcb *tp, struct tcp_log_mem *log_entry) 908 { 909 uma_zfree(tcp_log_zone, log_entry); 910 tp->t_lognum--; 911 KASSERT(tp->t_lognum >= 0, 912 ("%s: tp->t_lognum unexpectedly negative", __func__)); 913 } 914 915 /* Remove a log entry from the head of a list. */ 916 static inline void 917 tcp_log_remove_log_head(struct tcpcb *tp, struct tcp_log_mem *log_entry) 918 { 919 920 KASSERT(log_entry == STAILQ_FIRST(&tp->t_logs), 921 ("%s: attempt to remove non-HEAD log entry", __func__)); 922 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue); 923 tcp_log_entry_refcnt_rem(log_entry); 924 tcp_log_remove_log_cleanup(tp, log_entry); 925 } 926 927 #ifdef TCPLOG_DEBUG_RINGBUF 928 /* 929 * Initialize the log entry's reference count, which we want to 930 * survive allocations. 931 */ 932 static int 933 tcp_log_zone_init(void *mem, int size, int flags __unused) 934 { 935 struct tcp_log_mem *tlm; 936 937 KASSERT(size >= sizeof(struct tcp_log_mem), 938 ("%s: unexpectedly short (%d) allocation", __func__, size)); 939 tlm = (struct tcp_log_mem *)mem; 940 tlm->tlm_refcnt = 0; 941 return (0); 942 } 943 944 /* 945 * Double check that the refcnt is zero on allocation and return. 946 */ 947 static int 948 tcp_log_zone_ctor(void *mem, int size, void *args __unused, int flags __unused) 949 { 950 struct tcp_log_mem *tlm; 951 952 KASSERT(size >= sizeof(struct tcp_log_mem), 953 ("%s: unexpectedly short (%d) allocation", __func__, size)); 954 tlm = (struct tcp_log_mem *)mem; 955 if (tlm->tlm_refcnt != 0) 956 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)", 957 __func__, __LINE__, tlm, tlm->tlm_refcnt); 958 return (0); 959 } 960 961 static void 962 tcp_log_zone_dtor(void *mem, int size, void *args __unused) 963 { 964 struct tcp_log_mem *tlm; 965 966 KASSERT(size >= sizeof(struct tcp_log_mem), 967 ("%s: unexpectedly short (%d) allocation", __func__, size)); 968 tlm = (struct tcp_log_mem *)mem; 969 if (tlm->tlm_refcnt != 0) 970 panic("%s:%d: tlm(%p)->tlm_refcnt is %d (expected 0)", 971 __func__, __LINE__, tlm, tlm->tlm_refcnt); 972 } 973 #endif /* TCPLOG_DEBUG_RINGBUF */ 974 975 /* Do global initialization. */ 976 void 977 tcp_log_init(void) 978 { 979 980 tcp_log_zone = uma_zcreate("tcp_log", sizeof(struct tcp_log_mem), 981 #ifdef TCPLOG_DEBUG_RINGBUF 982 tcp_log_zone_ctor, tcp_log_zone_dtor, tcp_log_zone_init, 983 #else 984 NULL, NULL, NULL, 985 #endif 986 NULL, UMA_ALIGN_PTR, 0); 987 (void)uma_zone_set_max(tcp_log_zone, TCP_LOG_BUF_DEFAULT_GLOBAL_LIMIT); 988 tcp_log_bucket_zone = uma_zcreate("tcp_log_bucket", 989 sizeof(struct tcp_log_id_bucket), NULL, NULL, NULL, NULL, 990 UMA_ALIGN_PTR, 0); 991 tcp_log_node_zone = uma_zcreate("tcp_log_node", 992 sizeof(struct tcp_log_id_node), NULL, NULL, NULL, NULL, 993 UMA_ALIGN_PTR, 0); 994 #ifdef TCPLOG_DEBUG_COUNTERS 995 tcp_log_queued = counter_u64_alloc(M_WAITOK); 996 tcp_log_que_fail1 = counter_u64_alloc(M_WAITOK); 997 tcp_log_que_fail2 = counter_u64_alloc(M_WAITOK); 998 tcp_log_que_fail3 = counter_u64_alloc(M_WAITOK); 999 tcp_log_que_fail4 = counter_u64_alloc(M_WAITOK); 1000 tcp_log_que_fail5 = counter_u64_alloc(M_WAITOK); 1001 tcp_log_que_copyout = counter_u64_alloc(M_WAITOK); 1002 tcp_log_que_read = counter_u64_alloc(M_WAITOK); 1003 tcp_log_que_freed = counter_u64_alloc(M_WAITOK); 1004 #endif 1005 1006 rw_init_flags(&tcp_id_tree_lock, "TCP ID tree", RW_NEW); 1007 mtx_init(&tcp_log_expireq_mtx, "TCP log expireq", NULL, MTX_DEF); 1008 callout_init(&tcp_log_expireq_callout, 1); 1009 } 1010 1011 /* Do per-TCPCB initialization. */ 1012 void 1013 tcp_log_tcpcbinit(struct tcpcb *tp) 1014 { 1015 1016 /* A new TCPCB should start out zero-initialized. */ 1017 STAILQ_INIT(&tp->t_logs); 1018 1019 /* 1020 * If we are doing auto-capturing, figure out whether we will capture 1021 * this session. 1022 */ 1023 if (tcp_log_selectauto()) { 1024 tp->t_logstate = tcp_log_auto_mode; 1025 tp->t_flags2 |= TF2_LOG_AUTO; 1026 } 1027 } 1028 1029 1030 /* Remove entries */ 1031 static void 1032 tcp_log_expire(void *unused __unused) 1033 { 1034 struct tcp_log_id_bucket *tlb; 1035 struct tcp_log_id_node *tln; 1036 sbintime_t expiry_limit; 1037 int tree_locked; 1038 1039 TCPLOG_EXPIREQ_LOCK(); 1040 if (callout_pending(&tcp_log_expireq_callout)) { 1041 /* Callout was reset. */ 1042 TCPLOG_EXPIREQ_UNLOCK(); 1043 return; 1044 } 1045 1046 /* 1047 * Process entries until we reach one that expires too far in the 1048 * future. Look one second in the future. 1049 */ 1050 expiry_limit = getsbinuptime() + SBT_1S; 1051 tree_locked = TREE_UNLOCKED; 1052 1053 while ((tln = STAILQ_FIRST(&tcp_log_expireq_head)) != NULL && 1054 tln->tln_expiretime <= expiry_limit) { 1055 if (!callout_active(&tcp_log_expireq_callout)) { 1056 /* 1057 * Callout was stopped. I guess we should 1058 * just quit at this point. 1059 */ 1060 TCPLOG_EXPIREQ_UNLOCK(); 1061 return; 1062 } 1063 1064 /* 1065 * Remove the node from the head of the list and unlock 1066 * the list. Change the expiry time to SBT_MAX as a signal 1067 * to other threads that we now own this. 1068 */ 1069 STAILQ_REMOVE_HEAD(&tcp_log_expireq_head, tln_expireq); 1070 tln->tln_expiretime = SBT_MAX; 1071 TCPLOG_EXPIREQ_UNLOCK(); 1072 1073 /* 1074 * Remove the node from the bucket. 1075 */ 1076 tlb = tln->tln_bucket; 1077 TCPID_BUCKET_LOCK(tlb); 1078 if (tcp_log_remove_id_node(NULL, NULL, tlb, tln, &tree_locked)) { 1079 tcp_log_id_validate_tree_lock(tree_locked); 1080 if (tree_locked == TREE_WLOCKED) 1081 TCPID_TREE_WUNLOCK(); 1082 else 1083 TCPID_TREE_RUNLOCK(); 1084 tree_locked = TREE_UNLOCKED; 1085 } 1086 1087 /* Drop the INP reference. */ 1088 INP_WLOCK(tln->tln_inp); 1089 if (!in_pcbrele_wlocked(tln->tln_inp)) 1090 INP_WUNLOCK(tln->tln_inp); 1091 1092 /* Free the log records. */ 1093 tcp_log_free_entries(&tln->tln_entries, &tln->tln_count); 1094 1095 /* Free the node. */ 1096 uma_zfree(tcp_log_node_zone, tln); 1097 1098 /* Relock the expiry queue. */ 1099 TCPLOG_EXPIREQ_LOCK(); 1100 } 1101 1102 /* 1103 * We've expired all the entries we can. Do we need to reschedule 1104 * ourselves? 1105 */ 1106 callout_deactivate(&tcp_log_expireq_callout); 1107 if (tln != NULL) { 1108 /* 1109 * Get max(now + TCP_LOG_EXPIRE_INTVL, tln->tln_expiretime) and 1110 * set the next callout to that. (This helps ensure we generally 1111 * run the callout no more often than desired.) 1112 */ 1113 expiry_limit = getsbinuptime() + TCP_LOG_EXPIRE_INTVL; 1114 if (expiry_limit < tln->tln_expiretime) 1115 expiry_limit = tln->tln_expiretime; 1116 callout_reset_sbt(&tcp_log_expireq_callout, expiry_limit, 1117 SBT_1S, tcp_log_expire, NULL, C_ABSOLUTE); 1118 } 1119 1120 /* We're done. */ 1121 TCPLOG_EXPIREQ_UNLOCK(); 1122 return; 1123 } 1124 1125 /* 1126 * Move log data from the TCPCB to a new node. This will reset the TCPCB log 1127 * entries and log count; however, it will not touch other things from the 1128 * TCPCB (e.g. t_lin, t_lib). 1129 * 1130 * NOTE: Must hold a lock on the INP. 1131 */ 1132 static void 1133 tcp_log_move_tp_to_node(struct tcpcb *tp, struct tcp_log_id_node *tln) 1134 { 1135 1136 INP_WLOCK_ASSERT(tp->t_inpcb); 1137 1138 tln->tln_ie = tp->t_inpcb->inp_inc.inc_ie; 1139 if (tp->t_inpcb->inp_inc.inc_flags & INC_ISIPV6) 1140 tln->tln_af = AF_INET6; 1141 else 1142 tln->tln_af = AF_INET; 1143 tln->tln_entries = tp->t_logs; 1144 tln->tln_count = tp->t_lognum; 1145 tln->tln_bucket = tp->t_lib; 1146 1147 /* Clear information from the PCB. */ 1148 STAILQ_INIT(&tp->t_logs); 1149 tp->t_lognum = 0; 1150 } 1151 1152 /* Do per-TCPCB cleanup */ 1153 void 1154 tcp_log_tcpcbfini(struct tcpcb *tp) 1155 { 1156 struct tcp_log_id_node *tln, *tln_first; 1157 struct tcp_log_mem *log_entry; 1158 sbintime_t callouttime; 1159 1160 INP_WLOCK_ASSERT(tp->t_inpcb); 1161 1162 /* 1163 * If we were gathering packets to be automatically dumped, try to do 1164 * it now. If this succeeds, the log information in the TCPCB will be 1165 * cleared. Otherwise, we'll handle the log information as we do 1166 * for other states. 1167 */ 1168 switch(tp->t_logstate) { 1169 case TCP_LOG_STATE_HEAD_AUTO: 1170 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from head", 1171 M_NOWAIT, false); 1172 break; 1173 case TCP_LOG_STATE_TAIL_AUTO: 1174 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from tail", 1175 M_NOWAIT, false); 1176 break; 1177 case TCP_LOG_STATE_CONTINUAL: 1178 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", 1179 M_NOWAIT, false); 1180 break; 1181 } 1182 1183 /* 1184 * There are two ways we could keep logs: per-socket or per-ID. If 1185 * we are tracking logs with an ID, then the logs survive the 1186 * destruction of the TCPCB. 1187 * 1188 * If the TCPCB is associated with an ID node, move the logs from the 1189 * TCPCB to the ID node. In theory, this is safe, for reasons which I 1190 * will now explain for my own benefit when I next need to figure out 1191 * this code. :-) 1192 * 1193 * We own the INP lock. Therefore, no one else can change the contents 1194 * of this node (Rule C). Further, no one can remove this node from 1195 * the bucket while we hold the lock (Rule D). Basically, no one can 1196 * mess with this node. That leaves two states in which we could be: 1197 * 1198 * 1. Another thread is currently waiting to acquire the INP lock, with 1199 * plans to do something with this node. When we drop the INP lock, 1200 * they will have a chance to do that. They will recheck the 1201 * tln_closed field (see note to Rule C) and then acquire the 1202 * bucket lock before proceeding further. 1203 * 1204 * 2. Another thread will try to acquire a lock at some point in the 1205 * future. If they try to acquire a lock before we set the 1206 * tln_closed field, they will follow state #1. If they try to 1207 * acquire a lock after we set the tln_closed field, they will be 1208 * able to make changes to the node, at will, following Rule C. 1209 * 1210 * Therefore, we currently own this node and can make any changes 1211 * we want. But, as soon as we set the tln_closed field to true, we 1212 * have effectively dropped our lock on the node. (For this reason, we 1213 * also need to make sure our writes are ordered correctly. An atomic 1214 * operation with "release" semantics should be sufficient.) 1215 */ 1216 1217 if (tp->t_lin != NULL) { 1218 /* Copy the relevant information to the log entry. */ 1219 tln = tp->t_lin; 1220 KASSERT(tln->tln_inp == tp->t_inpcb, 1221 ("%s: Mismatched inp (tln->tln_inp=%p, tp->t_inpcb=%p)", 1222 __func__, tln->tln_inp, tp->t_inpcb)); 1223 tcp_log_move_tp_to_node(tp, tln); 1224 1225 /* Clear information from the PCB. */ 1226 tp->t_lin = NULL; 1227 tp->t_lib = NULL; 1228 1229 /* 1230 * Take a reference on the INP. This ensures that the INP 1231 * remains valid while the node is on the expiry queue. This 1232 * ensures the INP is valid for other threads that may be 1233 * racing to lock this node when we move it to the expire 1234 * queue. 1235 */ 1236 in_pcbref(tp->t_inpcb); 1237 1238 /* 1239 * Store the entry on the expiry list. The exact behavior 1240 * depends on whether we have entries to keep. If so, we 1241 * put the entry at the tail of the list and expire in 1242 * TCP_LOG_EXPIRE_TIME. Otherwise, we expire "now" and put 1243 * the entry at the head of the list. (Handling the cleanup 1244 * via the expiry timer lets us avoid locking messy-ness here.) 1245 */ 1246 tln->tln_expiretime = getsbinuptime(); 1247 TCPLOG_EXPIREQ_LOCK(); 1248 if (tln->tln_count) { 1249 tln->tln_expiretime += TCP_LOG_EXPIRE_TIME; 1250 if (STAILQ_EMPTY(&tcp_log_expireq_head) && 1251 !callout_active(&tcp_log_expireq_callout)) { 1252 /* 1253 * We are adding the first entry and a callout 1254 * is not currently scheduled; therefore, we 1255 * need to schedule one. 1256 */ 1257 callout_reset_sbt(&tcp_log_expireq_callout, 1258 tln->tln_expiretime, SBT_1S, tcp_log_expire, 1259 NULL, C_ABSOLUTE); 1260 } 1261 STAILQ_INSERT_TAIL(&tcp_log_expireq_head, tln, 1262 tln_expireq); 1263 } else { 1264 callouttime = tln->tln_expiretime + 1265 TCP_LOG_EXPIRE_INTVL; 1266 tln_first = STAILQ_FIRST(&tcp_log_expireq_head); 1267 1268 if ((tln_first == NULL || 1269 callouttime < tln_first->tln_expiretime) && 1270 (callout_pending(&tcp_log_expireq_callout) || 1271 !callout_active(&tcp_log_expireq_callout))) { 1272 /* 1273 * The list is empty, or we want to run the 1274 * expire code before the first entry's timer 1275 * fires. Also, we are in a case where a callout 1276 * is not actively running. We want to reset 1277 * the callout to occur sooner. 1278 */ 1279 callout_reset_sbt(&tcp_log_expireq_callout, 1280 callouttime, SBT_1S, tcp_log_expire, NULL, 1281 C_ABSOLUTE); 1282 } 1283 1284 /* 1285 * Insert to the head, or just after the head, as 1286 * appropriate. (This might result in small 1287 * mis-orderings as a bunch of "expire now" entries 1288 * gather at the start of the list, but that should 1289 * not produce big problems, since the expire timer 1290 * will walk through all of them.) 1291 */ 1292 if (tln_first == NULL || 1293 tln->tln_expiretime < tln_first->tln_expiretime) 1294 STAILQ_INSERT_HEAD(&tcp_log_expireq_head, tln, 1295 tln_expireq); 1296 else 1297 STAILQ_INSERT_AFTER(&tcp_log_expireq_head, 1298 tln_first, tln, tln_expireq); 1299 } 1300 TCPLOG_EXPIREQ_UNLOCK(); 1301 1302 /* 1303 * We are done messing with the tln. After this point, we 1304 * can't touch it. (Note that the "release" semantics should 1305 * be included with the TCPLOG_EXPIREQ_UNLOCK() call above. 1306 * Therefore, they should be unnecessary here. However, it 1307 * seems like a good idea to include them anyway, since we 1308 * really are releasing a lock here.) 1309 */ 1310 atomic_store_rel_int(&tln->tln_closed, 1); 1311 } else { 1312 /* Remove log entries. */ 1313 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) 1314 tcp_log_remove_log_head(tp, log_entry); 1315 KASSERT(tp->t_lognum == 0, 1316 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", 1317 __func__, tp->t_lognum)); 1318 } 1319 1320 /* 1321 * Change the log state to off (just in case anything tries to sneak 1322 * in a last-minute log). 1323 */ 1324 tp->t_logstate = TCP_LOG_STATE_OFF; 1325 } 1326 1327 /* 1328 * This logs an event for a TCP socket. Normally, this is called via 1329 * TCP_LOG_EVENT or TCP_LOG_EVENT_VERBOSE. See the documentation for 1330 * TCP_LOG_EVENT(). 1331 */ 1332 1333 struct tcp_log_buffer * 1334 tcp_log_event_(struct tcpcb *tp, struct tcphdr *th, struct sockbuf *rxbuf, 1335 struct sockbuf *txbuf, uint8_t eventid, int errornum, uint32_t len, 1336 union tcp_log_stackspecific *stackinfo, int th_hostorder, 1337 const char *output_caller, const char *func, int line, const struct timeval *itv) 1338 { 1339 struct tcp_log_mem *log_entry; 1340 struct tcp_log_buffer *log_buf; 1341 int attempt_count = 0; 1342 struct tcp_log_verbose *log_verbose; 1343 uint32_t logsn; 1344 1345 KASSERT((func == NULL && line == 0) || (func != NULL && line > 0), 1346 ("%s called with inconsistent func (%p) and line (%d) arguments", 1347 __func__, func, line)); 1348 1349 INP_WLOCK_ASSERT(tp->t_inpcb); 1350 1351 KASSERT(tp->t_logstate == TCP_LOG_STATE_HEAD || 1352 tp->t_logstate == TCP_LOG_STATE_TAIL || 1353 tp->t_logstate == TCP_LOG_STATE_CONTINUAL || 1354 tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO || 1355 tp->t_logstate == TCP_LOG_STATE_TAIL_AUTO, 1356 ("%s called with unexpected tp->t_logstate (%d)", __func__, 1357 tp->t_logstate)); 1358 1359 /* 1360 * Get the serial number. We do this early so it will 1361 * increment even if we end up skipping the log entry for some 1362 * reason. 1363 */ 1364 logsn = tp->t_logsn++; 1365 1366 /* 1367 * Can we get a new log entry? If so, increment the lognum counter 1368 * here. 1369 */ 1370 retry: 1371 if (tp->t_lognum < tcp_log_session_limit) { 1372 if ((log_entry = uma_zalloc(tcp_log_zone, M_NOWAIT)) != NULL) 1373 tp->t_lognum++; 1374 } else 1375 log_entry = NULL; 1376 1377 /* Do we need to try to reuse? */ 1378 if (log_entry == NULL) { 1379 /* 1380 * Sacrifice auto-logged sessions without a log ID if 1381 * tcp_log_auto_all is false. (If they don't have a log 1382 * ID by now, it is probable that either they won't get one 1383 * or we are resource-constrained.) 1384 */ 1385 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) && 1386 !tcp_log_auto_all) { 1387 if (tcp_log_state_change(tp, TCP_LOG_STATE_CLEAR)) { 1388 #ifdef INVARIANTS 1389 panic("%s:%d: tcp_log_state_change() failed " 1390 "to set tp %p to TCP_LOG_STATE_CLEAR", 1391 __func__, __LINE__, tp); 1392 #endif 1393 tp->t_logstate = TCP_LOG_STATE_OFF; 1394 } 1395 return (NULL); 1396 } 1397 /* 1398 * If we are in TCP_LOG_STATE_HEAD_AUTO state, try to dump 1399 * the buffers. If successful, deactivate tracing. Otherwise, 1400 * leave it active so we will retry. 1401 */ 1402 if (tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO && 1403 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from head", 1404 M_NOWAIT, false)) { 1405 tp->t_logstate = TCP_LOG_STATE_OFF; 1406 return(NULL); 1407 } else if ((tp->t_logstate == TCP_LOG_STATE_CONTINUAL) && 1408 !tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", 1409 M_NOWAIT, false)) { 1410 if (attempt_count == 0) { 1411 attempt_count++; 1412 goto retry; 1413 } 1414 #ifdef TCPLOG_DEBUG_COUNTERS 1415 counter_u64_add(tcp_log_que_fail4, 1); 1416 #endif 1417 return(NULL); 1418 } else if (tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO) 1419 return(NULL); 1420 1421 /* If in HEAD state, just deactivate the tracing and return. */ 1422 if (tp->t_logstate == TCP_LOG_STATE_HEAD) { 1423 tp->t_logstate = TCP_LOG_STATE_OFF; 1424 return(NULL); 1425 } 1426 1427 /* 1428 * Get a buffer to reuse. If that fails, just give up. 1429 * (We can't log anything without a buffer in which to 1430 * put it.) 1431 * 1432 * Note that we don't change the t_lognum counter 1433 * here. Because we are re-using the buffer, the total 1434 * number won't change. 1435 */ 1436 if ((log_entry = STAILQ_FIRST(&tp->t_logs)) == NULL) 1437 return(NULL); 1438 STAILQ_REMOVE_HEAD(&tp->t_logs, tlm_queue); 1439 tcp_log_entry_refcnt_rem(log_entry); 1440 } 1441 1442 KASSERT(log_entry != NULL, 1443 ("%s: log_entry unexpectedly NULL", __func__)); 1444 1445 /* Extract the log buffer and verbose buffer pointers. */ 1446 log_buf = &log_entry->tlm_buf; 1447 log_verbose = &log_entry->tlm_v; 1448 1449 /* Basic entries. */ 1450 if (itv == NULL) 1451 getmicrouptime(&log_buf->tlb_tv); 1452 else 1453 memcpy(&log_buf->tlb_tv, itv, sizeof(struct timeval)); 1454 log_buf->tlb_ticks = ticks; 1455 log_buf->tlb_sn = logsn; 1456 log_buf->tlb_stackid = tp->t_fb->tfb_id; 1457 log_buf->tlb_eventid = eventid; 1458 log_buf->tlb_eventflags = 0; 1459 log_buf->tlb_errno = errornum; 1460 1461 /* Socket buffers */ 1462 if (rxbuf != NULL) { 1463 log_buf->tlb_eventflags |= TLB_FLAG_RXBUF; 1464 log_buf->tlb_rxbuf.tls_sb_acc = rxbuf->sb_acc; 1465 log_buf->tlb_rxbuf.tls_sb_ccc = rxbuf->sb_ccc; 1466 log_buf->tlb_rxbuf.tls_sb_spare = 0; 1467 } 1468 if (txbuf != NULL) { 1469 log_buf->tlb_eventflags |= TLB_FLAG_TXBUF; 1470 log_buf->tlb_txbuf.tls_sb_acc = txbuf->sb_acc; 1471 log_buf->tlb_txbuf.tls_sb_ccc = txbuf->sb_ccc; 1472 log_buf->tlb_txbuf.tls_sb_spare = 0; 1473 } 1474 /* Copy values from tp to the log entry. */ 1475 #define COPY_STAT(f) log_buf->tlb_ ## f = tp->f 1476 #define COPY_STAT_T(f) log_buf->tlb_ ## f = tp->t_ ## f 1477 COPY_STAT_T(state); 1478 COPY_STAT_T(starttime); 1479 COPY_STAT(iss); 1480 COPY_STAT_T(flags); 1481 COPY_STAT(snd_una); 1482 COPY_STAT(snd_max); 1483 COPY_STAT(snd_cwnd); 1484 COPY_STAT(snd_nxt); 1485 COPY_STAT(snd_recover); 1486 COPY_STAT(snd_wnd); 1487 COPY_STAT(snd_ssthresh); 1488 COPY_STAT_T(srtt); 1489 COPY_STAT_T(rttvar); 1490 COPY_STAT(rcv_up); 1491 COPY_STAT(rcv_adv); 1492 COPY_STAT(rcv_nxt); 1493 COPY_STAT(sack_newdata); 1494 COPY_STAT(rcv_wnd); 1495 COPY_STAT_T(dupacks); 1496 COPY_STAT_T(segqlen); 1497 COPY_STAT(snd_numholes); 1498 COPY_STAT(snd_scale); 1499 COPY_STAT(rcv_scale); 1500 #undef COPY_STAT 1501 #undef COPY_STAT_T 1502 log_buf->tlb_flex1 = 0; 1503 log_buf->tlb_flex2 = 0; 1504 /* Copy stack-specific info. */ 1505 if (stackinfo != NULL) { 1506 memcpy(&log_buf->tlb_stackinfo, stackinfo, 1507 sizeof(log_buf->tlb_stackinfo)); 1508 log_buf->tlb_eventflags |= TLB_FLAG_STACKINFO; 1509 } 1510 1511 /* The packet */ 1512 log_buf->tlb_len = len; 1513 if (th) { 1514 int optlen; 1515 1516 log_buf->tlb_eventflags |= TLB_FLAG_HDR; 1517 log_buf->tlb_th = *th; 1518 if (th_hostorder) 1519 tcp_fields_to_net(&log_buf->tlb_th); 1520 optlen = (th->th_off << 2) - sizeof (struct tcphdr); 1521 if (optlen > 0) 1522 memcpy(log_buf->tlb_opts, th + 1, optlen); 1523 } 1524 1525 /* Verbose information */ 1526 if (func != NULL) { 1527 log_buf->tlb_eventflags |= TLB_FLAG_VERBOSE; 1528 if (output_caller != NULL) 1529 strlcpy(log_verbose->tlv_snd_frm, output_caller, 1530 TCP_FUNC_LEN); 1531 else 1532 *log_verbose->tlv_snd_frm = 0; 1533 strlcpy(log_verbose->tlv_trace_func, func, TCP_FUNC_LEN); 1534 log_verbose->tlv_trace_line = line; 1535 } 1536 1537 /* Insert the new log at the tail. */ 1538 STAILQ_INSERT_TAIL(&tp->t_logs, log_entry, tlm_queue); 1539 tcp_log_entry_refcnt_add(log_entry); 1540 return (log_buf); 1541 } 1542 1543 /* 1544 * Change the logging state for a TCPCB. Returns 0 on success or an 1545 * error code on failure. 1546 */ 1547 int 1548 tcp_log_state_change(struct tcpcb *tp, int state) 1549 { 1550 struct tcp_log_mem *log_entry; 1551 1552 INP_WLOCK_ASSERT(tp->t_inpcb); 1553 switch(state) { 1554 case TCP_LOG_STATE_CLEAR: 1555 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) 1556 tcp_log_remove_log_head(tp, log_entry); 1557 /* Fall through */ 1558 1559 case TCP_LOG_STATE_OFF: 1560 tp->t_logstate = TCP_LOG_STATE_OFF; 1561 break; 1562 1563 case TCP_LOG_STATE_TAIL: 1564 case TCP_LOG_STATE_HEAD: 1565 case TCP_LOG_STATE_CONTINUAL: 1566 case TCP_LOG_STATE_HEAD_AUTO: 1567 case TCP_LOG_STATE_TAIL_AUTO: 1568 tp->t_logstate = state; 1569 break; 1570 1571 default: 1572 return (EINVAL); 1573 } 1574 1575 tp->t_flags2 &= ~(TF2_LOG_AUTO); 1576 1577 return (0); 1578 } 1579 1580 /* If tcp_drain() is called, flush half the log entries. */ 1581 void 1582 tcp_log_drain(struct tcpcb *tp) 1583 { 1584 struct tcp_log_mem *log_entry, *next; 1585 int target, skip; 1586 1587 INP_WLOCK_ASSERT(tp->t_inpcb); 1588 if ((target = tp->t_lognum / 2) == 0) 1589 return; 1590 1591 /* 1592 * If we are logging the "head" packets, we want to discard 1593 * from the tail of the queue. Otherwise, we want to discard 1594 * from the head. 1595 */ 1596 if (tp->t_logstate == TCP_LOG_STATE_HEAD || 1597 tp->t_logstate == TCP_LOG_STATE_HEAD_AUTO) { 1598 skip = tp->t_lognum - target; 1599 STAILQ_FOREACH(log_entry, &tp->t_logs, tlm_queue) 1600 if (!--skip) 1601 break; 1602 KASSERT(log_entry != NULL, 1603 ("%s: skipped through all entries!", __func__)); 1604 if (log_entry == NULL) 1605 return; 1606 while ((next = STAILQ_NEXT(log_entry, tlm_queue)) != NULL) { 1607 STAILQ_REMOVE_AFTER(&tp->t_logs, log_entry, tlm_queue); 1608 tcp_log_entry_refcnt_rem(next); 1609 tcp_log_remove_log_cleanup(tp, next); 1610 #ifdef INVARIANTS 1611 target--; 1612 #endif 1613 } 1614 KASSERT(target == 0, 1615 ("%s: After removing from tail, target was %d", __func__, 1616 target)); 1617 } else if (tp->t_logstate == TCP_LOG_STATE_CONTINUAL) { 1618 (void)tcp_log_dump_tp_logbuf(tp, "auto-dumped from continual", 1619 M_NOWAIT, false); 1620 } else { 1621 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL && 1622 target--) 1623 tcp_log_remove_log_head(tp, log_entry); 1624 KASSERT(target <= 0, 1625 ("%s: After removing from head, target was %d", __func__, 1626 target)); 1627 KASSERT(tp->t_lognum > 0, 1628 ("%s: After removing from head, tp->t_lognum was %d", 1629 __func__, target)); 1630 KASSERT(log_entry != NULL, 1631 ("%s: After removing from head, the tailq was empty", 1632 __func__)); 1633 } 1634 } 1635 1636 static inline int 1637 tcp_log_copyout(struct sockopt *sopt, void *src, void *dst, size_t len) 1638 { 1639 1640 if (sopt->sopt_td != NULL) 1641 return (copyout(src, dst, len)); 1642 bcopy(src, dst, len); 1643 return (0); 1644 } 1645 1646 static int 1647 tcp_log_logs_to_buf(struct sockopt *sopt, struct tcp_log_stailq *log_tailqp, 1648 struct tcp_log_buffer **end, int count) 1649 { 1650 struct tcp_log_buffer *out_entry; 1651 struct tcp_log_mem *log_entry; 1652 size_t entrysize; 1653 int error; 1654 #ifdef INVARIANTS 1655 int orig_count = count; 1656 #endif 1657 1658 /* Copy the data out. */ 1659 error = 0; 1660 out_entry = (struct tcp_log_buffer *) sopt->sopt_val; 1661 STAILQ_FOREACH(log_entry, log_tailqp, tlm_queue) { 1662 count--; 1663 KASSERT(count >= 0, 1664 ("%s:%d: Exceeded expected count (%d) processing list %p", 1665 __func__, __LINE__, orig_count, log_tailqp)); 1666 1667 #ifdef TCPLOG_DEBUG_COUNTERS 1668 counter_u64_add(tcp_log_que_copyout, 1); 1669 #endif 1670 1671 /* 1672 * Skip copying out the header if it isn't present. 1673 * Instead, copy out zeros (to ensure we don't leak info). 1674 * TODO: Make sure we truly do zero everything we don't 1675 * explicitly set. 1676 */ 1677 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR) 1678 entrysize = sizeof(struct tcp_log_buffer); 1679 else 1680 entrysize = offsetof(struct tcp_log_buffer, tlb_th); 1681 error = tcp_log_copyout(sopt, &log_entry->tlm_buf, out_entry, 1682 entrysize); 1683 if (error) 1684 break; 1685 if (!(log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_HDR)) { 1686 error = tcp_log_copyout(sopt, zerobuf, 1687 ((uint8_t *)out_entry) + entrysize, 1688 sizeof(struct tcp_log_buffer) - entrysize); 1689 } 1690 1691 /* 1692 * Copy out the verbose bit, if needed. Either way, 1693 * increment the output pointer the correct amount. 1694 */ 1695 if (log_entry->tlm_buf.tlb_eventflags & TLB_FLAG_VERBOSE) { 1696 error = tcp_log_copyout(sopt, &log_entry->tlm_v, 1697 out_entry->tlb_verbose, 1698 sizeof(struct tcp_log_verbose)); 1699 if (error) 1700 break; 1701 out_entry = (struct tcp_log_buffer *) 1702 (((uint8_t *) (out_entry + 1)) + 1703 sizeof(struct tcp_log_verbose)); 1704 } else 1705 out_entry++; 1706 } 1707 *end = out_entry; 1708 KASSERT(error || count == 0, 1709 ("%s:%d: Less than expected count (%d) processing list %p" 1710 " (%d remain)", __func__, __LINE__, orig_count, 1711 log_tailqp, count)); 1712 1713 return (error); 1714 } 1715 1716 /* 1717 * Copy out the buffer. Note that we do incremental copying, so 1718 * sooptcopyout() won't work. However, the goal is to produce the same 1719 * end result as if we copied in the entire user buffer, updated it, 1720 * and then used sooptcopyout() to copy it out. 1721 * 1722 * NOTE: This should be called with a write lock on the PCB; however, 1723 * the function will drop it after it extracts the data from the TCPCB. 1724 */ 1725 int 1726 tcp_log_getlogbuf(struct sockopt *sopt, struct tcpcb *tp) 1727 { 1728 struct tcp_log_stailq log_tailq; 1729 struct tcp_log_mem *log_entry, *log_next; 1730 struct tcp_log_buffer *out_entry; 1731 struct inpcb *inp; 1732 size_t outsize, entrysize; 1733 int error, outnum; 1734 1735 INP_WLOCK_ASSERT(tp->t_inpcb); 1736 inp = tp->t_inpcb; 1737 1738 /* 1739 * Determine which log entries will fit in the buffer. As an 1740 * optimization, skip this if all the entries will clearly fit 1741 * in the buffer. (However, get an exact size if we are using 1742 * INVARIANTS.) 1743 */ 1744 #ifndef INVARIANTS 1745 if (sopt->sopt_valsize / (sizeof(struct tcp_log_buffer) + 1746 sizeof(struct tcp_log_verbose)) >= tp->t_lognum) { 1747 log_entry = STAILQ_LAST(&tp->t_logs, tcp_log_mem, tlm_queue); 1748 log_next = NULL; 1749 outsize = 0; 1750 outnum = tp->t_lognum; 1751 } else { 1752 #endif 1753 outsize = outnum = 0; 1754 log_entry = NULL; 1755 STAILQ_FOREACH(log_next, &tp->t_logs, tlm_queue) { 1756 entrysize = sizeof(struct tcp_log_buffer); 1757 if (log_next->tlm_buf.tlb_eventflags & 1758 TLB_FLAG_VERBOSE) 1759 entrysize += sizeof(struct tcp_log_verbose); 1760 if ((sopt->sopt_valsize - outsize) < entrysize) 1761 break; 1762 outsize += entrysize; 1763 outnum++; 1764 log_entry = log_next; 1765 } 1766 KASSERT(outsize <= sopt->sopt_valsize, 1767 ("%s: calculated output size (%zu) greater than available" 1768 "space (%zu)", __func__, outsize, sopt->sopt_valsize)); 1769 #ifndef INVARIANTS 1770 } 1771 #endif 1772 1773 /* 1774 * Copy traditional sooptcopyout() behavior: if sopt->sopt_val 1775 * is NULL, silently skip the copy. However, in this case, we 1776 * will leave the list alone and return. Functionally, this 1777 * gives userspace a way to poll for an approximate buffer 1778 * size they will need to get the log entries. 1779 */ 1780 if (sopt->sopt_val == NULL) { 1781 INP_WUNLOCK(inp); 1782 if (outsize == 0) { 1783 outsize = outnum * (sizeof(struct tcp_log_buffer) + 1784 sizeof(struct tcp_log_verbose)); 1785 } 1786 if (sopt->sopt_valsize > outsize) 1787 sopt->sopt_valsize = outsize; 1788 return (0); 1789 } 1790 1791 /* 1792 * Break apart the list. We'll save the ones we want to copy 1793 * out locally and remove them from the TCPCB list. We can 1794 * then drop the INPCB lock while we do the copyout. 1795 * 1796 * There are roughly three cases: 1797 * 1. There was nothing to copy out. That's easy: drop the 1798 * lock and return. 1799 * 2. We are copying out the entire list. Again, that's easy: 1800 * move the whole list. 1801 * 3. We are copying out a partial list. That's harder. We 1802 * need to update the list book-keeping entries. 1803 */ 1804 if (log_entry != NULL && log_next == NULL) { 1805 /* Move entire list. */ 1806 KASSERT(outnum == tp->t_lognum, 1807 ("%s:%d: outnum (%d) should match tp->t_lognum (%d)", 1808 __func__, __LINE__, outnum, tp->t_lognum)); 1809 log_tailq = tp->t_logs; 1810 tp->t_lognum = 0; 1811 STAILQ_INIT(&tp->t_logs); 1812 } else if (log_entry != NULL) { 1813 /* Move partial list. */ 1814 KASSERT(outnum < tp->t_lognum, 1815 ("%s:%d: outnum (%d) not less than tp->t_lognum (%d)", 1816 __func__, __LINE__, outnum, tp->t_lognum)); 1817 STAILQ_FIRST(&log_tailq) = STAILQ_FIRST(&tp->t_logs); 1818 STAILQ_FIRST(&tp->t_logs) = STAILQ_NEXT(log_entry, tlm_queue); 1819 KASSERT(STAILQ_NEXT(log_entry, tlm_queue) != NULL, 1820 ("%s:%d: tp->t_logs is unexpectedly shorter than expected" 1821 "(tp: %p, log_tailq: %p, outnum: %d, tp->t_lognum: %d)", 1822 __func__, __LINE__, tp, &log_tailq, outnum, tp->t_lognum)); 1823 STAILQ_NEXT(log_entry, tlm_queue) = NULL; 1824 log_tailq.stqh_last = &STAILQ_NEXT(log_entry, tlm_queue); 1825 tp->t_lognum -= outnum; 1826 } else 1827 STAILQ_INIT(&log_tailq); 1828 1829 /* Drop the PCB lock. */ 1830 INP_WUNLOCK(inp); 1831 1832 /* Copy the data out. */ 1833 error = tcp_log_logs_to_buf(sopt, &log_tailq, &out_entry, outnum); 1834 1835 if (error) { 1836 /* Restore list */ 1837 INP_WLOCK(inp); 1838 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0) { 1839 tp = intotcpcb(inp); 1840 1841 /* Merge the two lists. */ 1842 STAILQ_CONCAT(&log_tailq, &tp->t_logs); 1843 tp->t_logs = log_tailq; 1844 tp->t_lognum += outnum; 1845 } 1846 INP_WUNLOCK(inp); 1847 } else { 1848 /* Sanity check entries */ 1849 KASSERT(((caddr_t)out_entry - (caddr_t)sopt->sopt_val) == 1850 outsize, ("%s: Actual output size (%zu) != " 1851 "calculated output size (%zu)", __func__, 1852 (size_t)((caddr_t)out_entry - (caddr_t)sopt->sopt_val), 1853 outsize)); 1854 1855 /* Free the entries we just copied out. */ 1856 STAILQ_FOREACH_SAFE(log_entry, &log_tailq, tlm_queue, log_next) { 1857 tcp_log_entry_refcnt_rem(log_entry); 1858 uma_zfree(tcp_log_zone, log_entry); 1859 } 1860 } 1861 1862 sopt->sopt_valsize = (size_t)((caddr_t)out_entry - 1863 (caddr_t)sopt->sopt_val); 1864 return (error); 1865 } 1866 1867 static void 1868 tcp_log_free_queue(struct tcp_log_dev_queue *param) 1869 { 1870 struct tcp_log_dev_log_queue *entry; 1871 1872 KASSERT(param != NULL, ("%s: called with NULL param", __func__)); 1873 if (param == NULL) 1874 return; 1875 1876 entry = (struct tcp_log_dev_log_queue *)param; 1877 1878 /* Free the entries. */ 1879 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count); 1880 1881 /* Free the buffer, if it is allocated. */ 1882 if (entry->tldl_common.tldq_buf != NULL) 1883 free(entry->tldl_common.tldq_buf, M_TCPLOGDEV); 1884 1885 /* Free the queue entry. */ 1886 free(entry, M_TCPLOGDEV); 1887 } 1888 1889 static struct tcp_log_common_header * 1890 tcp_log_expandlogbuf(struct tcp_log_dev_queue *param) 1891 { 1892 struct tcp_log_dev_log_queue *entry; 1893 struct tcp_log_header *hdr; 1894 uint8_t *end; 1895 struct sockopt sopt; 1896 int error; 1897 1898 entry = (struct tcp_log_dev_log_queue *)param; 1899 1900 /* Take a worst-case guess at space needs. */ 1901 sopt.sopt_valsize = sizeof(struct tcp_log_header) + 1902 entry->tldl_count * (sizeof(struct tcp_log_buffer) + 1903 sizeof(struct tcp_log_verbose)); 1904 hdr = malloc(sopt.sopt_valsize, M_TCPLOGDEV, M_NOWAIT); 1905 if (hdr == NULL) { 1906 #ifdef TCPLOG_DEBUG_COUNTERS 1907 counter_u64_add(tcp_log_que_fail5, entry->tldl_count); 1908 #endif 1909 return (NULL); 1910 } 1911 sopt.sopt_val = hdr + 1; 1912 sopt.sopt_valsize -= sizeof(struct tcp_log_header); 1913 sopt.sopt_td = NULL; 1914 1915 error = tcp_log_logs_to_buf(&sopt, &entry->tldl_entries, 1916 (struct tcp_log_buffer **)&end, entry->tldl_count); 1917 if (error) { 1918 free(hdr, M_TCPLOGDEV); 1919 return (NULL); 1920 } 1921 1922 /* Free the entries. */ 1923 tcp_log_free_entries(&entry->tldl_entries, &entry->tldl_count); 1924 entry->tldl_count = 0; 1925 1926 memset(hdr, 0, sizeof(struct tcp_log_header)); 1927 hdr->tlh_version = TCP_LOG_BUF_VER; 1928 hdr->tlh_type = TCP_LOG_DEV_TYPE_BBR; 1929 hdr->tlh_length = end - (uint8_t *)hdr; 1930 hdr->tlh_ie = entry->tldl_ie; 1931 hdr->tlh_af = entry->tldl_af; 1932 getboottime(&hdr->tlh_offset); 1933 strlcpy(hdr->tlh_id, entry->tldl_id, TCP_LOG_ID_LEN); 1934 strlcpy(hdr->tlh_reason, entry->tldl_reason, TCP_LOG_REASON_LEN); 1935 return ((struct tcp_log_common_header *)hdr); 1936 } 1937 1938 /* 1939 * Queue the tcpcb's log buffer for transmission via the log buffer facility. 1940 * 1941 * NOTE: This should be called with a write lock on the PCB. 1942 * 1943 * how should be M_WAITOK or M_NOWAIT. If M_WAITOK, the function will drop 1944 * and reacquire the INP lock if it needs to do so. 1945 * 1946 * If force is false, this will only dump auto-logged sessions if 1947 * tcp_log_auto_all is true or if there is a log ID defined for the session. 1948 */ 1949 int 1950 tcp_log_dump_tp_logbuf(struct tcpcb *tp, char *reason, int how, bool force) 1951 { 1952 struct tcp_log_dev_log_queue *entry; 1953 struct inpcb *inp; 1954 #ifdef TCPLOG_DEBUG_COUNTERS 1955 int num_entries; 1956 #endif 1957 1958 inp = tp->t_inpcb; 1959 INP_WLOCK_ASSERT(inp); 1960 1961 /* If there are no log entries, there is nothing to do. */ 1962 if (tp->t_lognum == 0) 1963 return (0); 1964 1965 /* Check for a log ID. */ 1966 if (tp->t_lib == NULL && (tp->t_flags2 & TF2_LOG_AUTO) && 1967 !tcp_log_auto_all && !force) { 1968 struct tcp_log_mem *log_entry; 1969 1970 /* 1971 * We needed a log ID and none was found. Free the log entries 1972 * and return success. Also, cancel further logging. If the 1973 * session doesn't have a log ID by now, we'll assume it isn't 1974 * going to get one. 1975 */ 1976 while ((log_entry = STAILQ_FIRST(&tp->t_logs)) != NULL) 1977 tcp_log_remove_log_head(tp, log_entry); 1978 KASSERT(tp->t_lognum == 0, 1979 ("%s: After freeing entries, tp->t_lognum=%d (expected 0)", 1980 __func__, tp->t_lognum)); 1981 tp->t_logstate = TCP_LOG_STATE_OFF; 1982 return (0); 1983 } 1984 1985 /* 1986 * Allocate memory. If we must wait, we'll need to drop the locks 1987 * and reacquire them (and do all the related business that goes 1988 * along with that). 1989 */ 1990 entry = malloc(sizeof(struct tcp_log_dev_log_queue), M_TCPLOGDEV, 1991 M_NOWAIT); 1992 if (entry == NULL && (how & M_NOWAIT)) { 1993 #ifdef TCPLOG_DEBUG_COUNTERS 1994 counter_u64_add(tcp_log_que_fail3, 1); 1995 #endif 1996 return (ENOBUFS); 1997 } 1998 if (entry == NULL) { 1999 INP_WUNLOCK(inp); 2000 entry = malloc(sizeof(struct tcp_log_dev_log_queue), 2001 M_TCPLOGDEV, M_WAITOK); 2002 INP_WLOCK(inp); 2003 /* 2004 * Note that this check is slightly overly-restrictive in 2005 * that the TCB can survive either of these events. 2006 * However, there is currently not a good way to ensure 2007 * that is the case. So, if we hit this M_WAIT path, we 2008 * may end up dropping some entries. That seems like a 2009 * small price to pay for safety. 2010 */ 2011 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { 2012 free(entry, M_TCPLOGDEV); 2013 #ifdef TCPLOG_DEBUG_COUNTERS 2014 counter_u64_add(tcp_log_que_fail2, 1); 2015 #endif 2016 return (ECONNRESET); 2017 } 2018 tp = intotcpcb(inp); 2019 if (tp->t_lognum == 0) { 2020 free(entry, M_TCPLOGDEV); 2021 return (0); 2022 } 2023 } 2024 2025 /* Fill in the unique parts of the queue entry. */ 2026 if (tp->t_lib != NULL) 2027 strlcpy(entry->tldl_id, tp->t_lib->tlb_id, TCP_LOG_ID_LEN); 2028 else 2029 strlcpy(entry->tldl_id, "UNKNOWN", TCP_LOG_ID_LEN); 2030 if (reason != NULL) 2031 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN); 2032 else 2033 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_ID_LEN); 2034 entry->tldl_ie = inp->inp_inc.inc_ie; 2035 if (inp->inp_inc.inc_flags & INC_ISIPV6) 2036 entry->tldl_af = AF_INET6; 2037 else 2038 entry->tldl_af = AF_INET; 2039 entry->tldl_entries = tp->t_logs; 2040 entry->tldl_count = tp->t_lognum; 2041 2042 /* Fill in the common parts of the queue entry. */ 2043 entry->tldl_common.tldq_buf = NULL; 2044 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf; 2045 entry->tldl_common.tldq_dtor = tcp_log_free_queue; 2046 2047 /* Clear the log data from the TCPCB. */ 2048 #ifdef TCPLOG_DEBUG_COUNTERS 2049 num_entries = tp->t_lognum; 2050 #endif 2051 tp->t_lognum = 0; 2052 STAILQ_INIT(&tp->t_logs); 2053 2054 /* Add the entry. If no one is listening, free the entry. */ 2055 if (tcp_log_dev_add_log((struct tcp_log_dev_queue *)entry)) { 2056 tcp_log_free_queue((struct tcp_log_dev_queue *)entry); 2057 #ifdef TCPLOG_DEBUG_COUNTERS 2058 counter_u64_add(tcp_log_que_fail1, num_entries); 2059 } else { 2060 counter_u64_add(tcp_log_queued, num_entries); 2061 #endif 2062 } 2063 return (0); 2064 } 2065 2066 /* 2067 * Queue the log_id_node's log buffers for transmission via the log buffer 2068 * facility. 2069 * 2070 * NOTE: This should be called with the bucket locked and referenced. 2071 * 2072 * how should be M_WAITOK or M_NOWAIT. If M_WAITOK, the function will drop 2073 * and reacquire the bucket lock if it needs to do so. (The caller must 2074 * ensure that the tln is no longer on any lists so no one else will mess 2075 * with this while the lock is dropped!) 2076 */ 2077 static int 2078 tcp_log_dump_node_logbuf(struct tcp_log_id_node *tln, char *reason, int how) 2079 { 2080 struct tcp_log_dev_log_queue *entry; 2081 struct tcp_log_id_bucket *tlb; 2082 2083 tlb = tln->tln_bucket; 2084 TCPID_BUCKET_LOCK_ASSERT(tlb); 2085 KASSERT(tlb->tlb_refcnt > 0, 2086 ("%s:%d: Called with unreferenced bucket (tln=%p, tlb=%p)", 2087 __func__, __LINE__, tln, tlb)); 2088 KASSERT(tln->tln_closed, 2089 ("%s:%d: Called for node with tln_closed==false (tln=%p)", 2090 __func__, __LINE__, tln)); 2091 2092 /* If there are no log entries, there is nothing to do. */ 2093 if (tln->tln_count == 0) 2094 return (0); 2095 2096 /* 2097 * Allocate memory. If we must wait, we'll need to drop the locks 2098 * and reacquire them (and do all the related business that goes 2099 * along with that). 2100 */ 2101 entry = malloc(sizeof(struct tcp_log_dev_log_queue), M_TCPLOGDEV, 2102 M_NOWAIT); 2103 if (entry == NULL && (how & M_NOWAIT)) 2104 return (ENOBUFS); 2105 if (entry == NULL) { 2106 TCPID_BUCKET_UNLOCK(tlb); 2107 entry = malloc(sizeof(struct tcp_log_dev_log_queue), 2108 M_TCPLOGDEV, M_WAITOK); 2109 TCPID_BUCKET_LOCK(tlb); 2110 } 2111 2112 /* Fill in the common parts of the queue entry.. */ 2113 entry->tldl_common.tldq_buf = NULL; 2114 entry->tldl_common.tldq_xform = tcp_log_expandlogbuf; 2115 entry->tldl_common.tldq_dtor = tcp_log_free_queue; 2116 2117 /* Fill in the unique parts of the queue entry. */ 2118 strlcpy(entry->tldl_id, tlb->tlb_id, TCP_LOG_ID_LEN); 2119 if (reason != NULL) 2120 strlcpy(entry->tldl_reason, reason, TCP_LOG_REASON_LEN); 2121 else 2122 strlcpy(entry->tldl_reason, "UNKNOWN", TCP_LOG_ID_LEN); 2123 entry->tldl_ie = tln->tln_ie; 2124 entry->tldl_entries = tln->tln_entries; 2125 entry->tldl_count = tln->tln_count; 2126 entry->tldl_af = tln->tln_af; 2127 2128 /* Add the entry. If no one is listening, free the entry. */ 2129 if (tcp_log_dev_add_log((struct tcp_log_dev_queue *)entry)) 2130 tcp_log_free_queue((struct tcp_log_dev_queue *)entry); 2131 2132 return (0); 2133 } 2134 2135 2136 /* 2137 * Queue the log buffers for all sessions in a bucket for transmissions via 2138 * the log buffer facility. 2139 * 2140 * NOTE: This should be called with a locked bucket; however, the function 2141 * will drop the lock. 2142 */ 2143 #define LOCAL_SAVE 10 2144 static void 2145 tcp_log_dumpbucketlogs(struct tcp_log_id_bucket *tlb, char *reason) 2146 { 2147 struct tcp_log_id_node local_entries[LOCAL_SAVE]; 2148 struct inpcb *inp; 2149 struct tcpcb *tp; 2150 struct tcp_log_id_node *cur_tln, *prev_tln, *tmp_tln; 2151 int i, num_local_entries, tree_locked; 2152 bool expireq_locked; 2153 2154 TCPID_BUCKET_LOCK_ASSERT(tlb); 2155 2156 /* 2157 * Take a reference on the bucket to keep it from disappearing until 2158 * we are done. 2159 */ 2160 TCPID_BUCKET_REF(tlb); 2161 2162 /* 2163 * We'll try to create these without dropping locks. However, we 2164 * might very well need to drop locks to get memory. If that's the 2165 * case, we'll save up to 10 on the stack, and sacrifice the rest. 2166 * (Otherwise, we need to worry about finding our place again in a 2167 * potentially changed list. It just doesn't seem worth the trouble 2168 * to do that. 2169 */ 2170 expireq_locked = false; 2171 num_local_entries = 0; 2172 prev_tln = NULL; 2173 tree_locked = TREE_UNLOCKED; 2174 SLIST_FOREACH_SAFE(cur_tln, &tlb->tlb_head, tln_list, tmp_tln) { 2175 /* 2176 * If this isn't associated with a TCPCB, we can pull it off 2177 * the list now. We need to be careful that the expire timer 2178 * hasn't already taken ownership (tln_expiretime == SBT_MAX). 2179 * If so, we let the expire timer code free the data. 2180 */ 2181 if (cur_tln->tln_closed) { 2182 no_inp: 2183 /* 2184 * Get the expireq lock so we can get a consistent 2185 * read of tln_expiretime and so we can remove this 2186 * from the expireq. 2187 */ 2188 if (!expireq_locked) { 2189 TCPLOG_EXPIREQ_LOCK(); 2190 expireq_locked = true; 2191 } 2192 2193 /* 2194 * We ignore entries with tln_expiretime == SBT_MAX. 2195 * The expire timer code already owns those. 2196 */ 2197 KASSERT(cur_tln->tln_expiretime > (sbintime_t) 0, 2198 ("%s:%d: node on the expire queue without positive " 2199 "expire time", __func__, __LINE__)); 2200 if (cur_tln->tln_expiretime == SBT_MAX) { 2201 prev_tln = cur_tln; 2202 continue; 2203 } 2204 2205 /* Remove the entry from the expireq. */ 2206 STAILQ_REMOVE(&tcp_log_expireq_head, cur_tln, 2207 tcp_log_id_node, tln_expireq); 2208 2209 /* Remove the entry from the bucket. */ 2210 if (prev_tln != NULL) 2211 SLIST_REMOVE_AFTER(prev_tln, tln_list); 2212 else 2213 SLIST_REMOVE_HEAD(&tlb->tlb_head, tln_list); 2214 2215 /* 2216 * Drop the INP and bucket reference counts. Due to 2217 * lock-ordering rules, we need to drop the expire 2218 * queue lock. 2219 */ 2220 TCPLOG_EXPIREQ_UNLOCK(); 2221 expireq_locked = false; 2222 2223 /* Drop the INP reference. */ 2224 INP_WLOCK(cur_tln->tln_inp); 2225 if (!in_pcbrele_wlocked(cur_tln->tln_inp)) 2226 INP_WUNLOCK(cur_tln->tln_inp); 2227 2228 if (tcp_log_unref_bucket(tlb, &tree_locked, NULL)) { 2229 #ifdef INVARIANTS 2230 panic("%s: Bucket refcount unexpectedly 0.", 2231 __func__); 2232 #endif 2233 /* 2234 * Recover as best we can: free the entry we 2235 * own. 2236 */ 2237 tcp_log_free_entries(&cur_tln->tln_entries, 2238 &cur_tln->tln_count); 2239 uma_zfree(tcp_log_node_zone, cur_tln); 2240 goto done; 2241 } 2242 2243 if (tcp_log_dump_node_logbuf(cur_tln, reason, 2244 M_NOWAIT)) { 2245 /* 2246 * If we have sapce, save the entries locally. 2247 * Otherwise, free them. 2248 */ 2249 if (num_local_entries < LOCAL_SAVE) { 2250 local_entries[num_local_entries] = 2251 *cur_tln; 2252 num_local_entries++; 2253 } else { 2254 tcp_log_free_entries( 2255 &cur_tln->tln_entries, 2256 &cur_tln->tln_count); 2257 } 2258 } 2259 2260 /* No matter what, we are done with the node now. */ 2261 uma_zfree(tcp_log_node_zone, cur_tln); 2262 2263 /* 2264 * Because we removed this entry from the list, prev_tln 2265 * (which tracks the previous entry still on the tlb 2266 * list) remains unchanged. 2267 */ 2268 continue; 2269 } 2270 2271 /* 2272 * If we get to this point, the session data is still held in 2273 * the TCPCB. So, we need to pull the data out of that. 2274 * 2275 * We will need to drop the expireq lock so we can lock the INP. 2276 * We can then try to extract the data the "easy" way. If that 2277 * fails, we'll save the log entries for later. 2278 */ 2279 if (expireq_locked) { 2280 TCPLOG_EXPIREQ_UNLOCK(); 2281 expireq_locked = false; 2282 } 2283 2284 /* Lock the INP and then re-check the state. */ 2285 inp = cur_tln->tln_inp; 2286 INP_WLOCK(inp); 2287 /* 2288 * If we caught this while it was transitioning, the data 2289 * might have moved from the TCPCB to the tln (signified by 2290 * setting tln_closed to true. If so, treat this like an 2291 * inactive connection. 2292 */ 2293 if (cur_tln->tln_closed) { 2294 /* 2295 * It looks like we may have caught this connection 2296 * while it was transitioning from active to inactive. 2297 * Treat this like an inactive connection. 2298 */ 2299 INP_WUNLOCK(inp); 2300 goto no_inp; 2301 } 2302 2303 /* 2304 * Try to dump the data from the tp without dropping the lock. 2305 * If this fails, try to save off the data locally. 2306 */ 2307 tp = cur_tln->tln_tp; 2308 if (tcp_log_dump_tp_logbuf(tp, reason, M_NOWAIT, true) && 2309 num_local_entries < LOCAL_SAVE) { 2310 tcp_log_move_tp_to_node(tp, 2311 &local_entries[num_local_entries]); 2312 local_entries[num_local_entries].tln_closed = 1; 2313 KASSERT(local_entries[num_local_entries].tln_bucket == 2314 tlb, ("%s: %d: bucket mismatch for node %p", 2315 __func__, __LINE__, cur_tln)); 2316 num_local_entries++; 2317 } 2318 2319 INP_WUNLOCK(inp); 2320 2321 /* 2322 * We are goint to leave the current tln on the list. It will 2323 * become the previous tln. 2324 */ 2325 prev_tln = cur_tln; 2326 } 2327 2328 /* Drop our locks, if any. */ 2329 KASSERT(tree_locked == TREE_UNLOCKED, 2330 ("%s: %d: tree unexpectedly locked", __func__, __LINE__)); 2331 switch (tree_locked) { 2332 case TREE_WLOCKED: 2333 TCPID_TREE_WUNLOCK(); 2334 tree_locked = TREE_UNLOCKED; 2335 break; 2336 case TREE_RLOCKED: 2337 TCPID_TREE_RUNLOCK(); 2338 tree_locked = TREE_UNLOCKED; 2339 break; 2340 } 2341 if (expireq_locked) { 2342 TCPLOG_EXPIREQ_UNLOCK(); 2343 expireq_locked = false; 2344 } 2345 2346 /* 2347 * Try again for any saved entries. tcp_log_dump_node_logbuf() is 2348 * guaranteed to free the log entries within the node. And, since 2349 * the node itself is on our stack, we don't need to free it. 2350 */ 2351 for (i = 0; i < num_local_entries; i++) 2352 tcp_log_dump_node_logbuf(&local_entries[i], reason, M_WAITOK); 2353 2354 /* Drop our reference. */ 2355 if (!tcp_log_unref_bucket(tlb, &tree_locked, NULL)) 2356 TCPID_BUCKET_UNLOCK(tlb); 2357 2358 done: 2359 /* Drop our locks, if any. */ 2360 switch (tree_locked) { 2361 case TREE_WLOCKED: 2362 TCPID_TREE_WUNLOCK(); 2363 break; 2364 case TREE_RLOCKED: 2365 TCPID_TREE_RUNLOCK(); 2366 break; 2367 } 2368 if (expireq_locked) 2369 TCPLOG_EXPIREQ_UNLOCK(); 2370 } 2371 #undef LOCAL_SAVE 2372 2373 2374 /* 2375 * Queue the log buffers for all sessions in a bucket for transmissions via 2376 * the log buffer facility. 2377 * 2378 * NOTE: This should be called with a locked INP; however, the function 2379 * will drop the lock. 2380 */ 2381 void 2382 tcp_log_dump_tp_bucket_logbufs(struct tcpcb *tp, char *reason) 2383 { 2384 struct tcp_log_id_bucket *tlb; 2385 int tree_locked; 2386 2387 /* Figure out our bucket and lock it. */ 2388 INP_WLOCK_ASSERT(tp->t_inpcb); 2389 tlb = tp->t_lib; 2390 if (tlb == NULL) { 2391 /* 2392 * No bucket; treat this like a request to dump a single 2393 * session's traces. 2394 */ 2395 (void)tcp_log_dump_tp_logbuf(tp, reason, M_WAITOK, true); 2396 INP_WUNLOCK(tp->t_inpcb); 2397 return; 2398 } 2399 TCPID_BUCKET_REF(tlb); 2400 INP_WUNLOCK(tp->t_inpcb); 2401 TCPID_BUCKET_LOCK(tlb); 2402 2403 /* If we are the last reference, we have nothing more to do here. */ 2404 tree_locked = TREE_UNLOCKED; 2405 if (tcp_log_unref_bucket(tlb, &tree_locked, NULL)) { 2406 switch (tree_locked) { 2407 case TREE_WLOCKED: 2408 TCPID_TREE_WUNLOCK(); 2409 break; 2410 case TREE_RLOCKED: 2411 TCPID_TREE_RUNLOCK(); 2412 break; 2413 } 2414 return; 2415 } 2416 2417 /* Turn this over to tcp_log_dumpbucketlogs() to finish the work. */ 2418 tcp_log_dumpbucketlogs(tlb, reason); 2419 } 2420 2421 /* 2422 * Mark the end of a flow with the current stack. A stack can add 2423 * stack-specific info to this trace event by overriding this 2424 * function (see bbr_log_flowend() for example). 2425 */ 2426 void 2427 tcp_log_flowend(struct tcpcb *tp) 2428 { 2429 if (tp->t_logstate != TCP_LOG_STATE_OFF) { 2430 struct socket *so = tp->t_inpcb->inp_socket; 2431 TCP_LOG_EVENT(tp, NULL, &so->so_rcv, &so->so_snd, 2432 TCP_LOG_FLOWEND, 0, 0, NULL, false); 2433 } 2434 } 2435 2436