1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2025 Netflix, Inc. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <tests/ktest.h> 29 #include <sys/cdefs.h> 30 #include "opt_inet.h" 31 #include <sys/param.h> 32 #include <sys/bus.h> 33 #include <sys/interrupt.h> 34 #include <sys/errno.h> 35 #include <sys/malloc.h> 36 #include <sys/mutex.h> 37 #include <sys/refcount.h> 38 #include <sys/socket.h> 39 #include <sys/sysctl.h> 40 #include <sys/systm.h> 41 42 #include <netinet/in.h> 43 #include <netinet/tcp.h> 44 #include <netinet/in_pcb.h> 45 #include <netinet/tcp_seq.h> 46 #include <netinet/tcp_var.h> 47 #include <netinet/tcp_hpts.h> 48 #include <netinet/tcp_hpts_internal.h> 49 #include <dev/tcp_log/tcp_log_dev.h> 50 #include <netinet/tcp_log_buf.h> 51 52 #undef tcp_hpts_init 53 #undef tcp_hpts_remove 54 #undef tcp_hpts_insert 55 #undef tcp_set_hpts 56 57 /* Custom definitions that take the tcp_hptsi */ 58 #define tcp_hpts_init(pace, tp) __tcp_hpts_init((pace), (tp)) 59 #define tcp_hpts_remove(pace, tp) __tcp_hpts_remove((pace), (tp)) 60 #define tcp_hpts_insert(pace, tp, usecs, diag) \ 61 __tcp_hpts_insert((pace), (tp), (usecs), (diag)) 62 #define tcp_set_hpts(pace, tp) __tcp_set_hpts((pace), (tp)) 63 64 static MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts_test", "TCP hpts test"); 65 66 static int test_exit_on_failure = true; 67 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts_test, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 68 "TCP HPTS test controls"); 69 SYSCTL_INT(_net_inet_tcp_hpts_test, OID_AUTO, exit_on_failure, CTLFLAG_RW, 70 &test_exit_on_failure, 0, 71 "Exit HPTS test immediately on first failure (1) or continue running all tests (0)"); 72 73 #define KTEST_VERIFY(x) do { \ 74 if (!(x)) { \ 75 KTEST_ERR(ctx, "FAIL: %s", #x); \ 76 if (test_exit_on_failure) \ 77 return (EINVAL); \ 78 } else { \ 79 KTEST_LOG(ctx, "PASS: %s", #x); \ 80 } \ 81 } while (0) 82 83 #define KTEST_EQUAL(x, y) do { \ 84 if ((x) != (y)) { \ 85 KTEST_ERR(ctx, "FAIL: %s != %s (%d != %d)", #x, #y, (x), (y)); \ 86 if (test_exit_on_failure) \ 87 return (EINVAL); \ 88 } else { \ 89 KTEST_LOG(ctx, "PASS: %s == %s", #x, #y); \ 90 } \ 91 } while (0) 92 93 #define KTEST_NEQUAL(x, y) do { \ 94 if ((x) == (y)) { \ 95 KTEST_ERR(ctx, "FAIL: %s == %s (%d == %d)", #x, #y, (x), (y)); \ 96 if (test_exit_on_failure) \ 97 return (EINVAL); \ 98 } else { \ 99 KTEST_LOG(ctx, "PASS: %s != %s", #x, #y); \ 100 } \ 101 } while (0) 102 103 #define KTEST_GREATER_THAN(x, y) do { \ 104 if ((x) <= (y)) { \ 105 KTEST_ERR(ctx, "FAIL: %s <= %s (%d <= %d)", #x, #y, (x), (y)); \ 106 if (test_exit_on_failure) \ 107 return (EINVAL); \ 108 } else { \ 109 KTEST_LOG(ctx, "PASS: %s > %s", #x, #y); \ 110 } \ 111 } while (0) 112 113 #define KTEST_VERIFY_RET(x, y) do { \ 114 if (!(x)) { \ 115 KTEST_ERR(ctx, "FAIL: %s", #x); \ 116 if (test_exit_on_failure) \ 117 return (y); \ 118 } else { \ 119 KTEST_LOG(ctx, "PASS: %s", #x); \ 120 } \ 121 } while (0) 122 123 #ifdef TCP_HPTS_KTEST 124 125 static void 126 dump_hpts_entry(struct ktest_test_context *ctx, struct tcp_hpts_entry *hpts) 127 { 128 KTEST_LOG(ctx, "tcp_hpts_entry(%p)", hpts); 129 KTEST_LOG(ctx, " p_cur_slot: %u", hpts->p_cur_slot); 130 KTEST_LOG(ctx, " p_prev_slot: %u", hpts->p_prev_slot); 131 KTEST_LOG(ctx, " p_nxt_slot: %u", hpts->p_nxt_slot); 132 KTEST_LOG(ctx, " p_runningslot: %u", hpts->p_runningslot); 133 KTEST_LOG(ctx, " p_on_queue_cnt: %d", hpts->p_on_queue_cnt); 134 KTEST_LOG(ctx, " p_hpts_active: %u", hpts->p_hpts_active); 135 KTEST_LOG(ctx, " p_wheel_complete: %u", hpts->p_wheel_complete); 136 KTEST_LOG(ctx, " p_direct_wake: %u", hpts->p_direct_wake); 137 KTEST_LOG(ctx, " p_on_min_sleep: %u", hpts->p_on_min_sleep); 138 KTEST_LOG(ctx, " p_hpts_wake_scheduled: %u", hpts->p_hpts_wake_scheduled); 139 KTEST_LOG(ctx, " hit_callout_thresh: %u", hpts->hit_callout_thresh); 140 KTEST_LOG(ctx, " p_hpts_sleep_time: %u", hpts->p_hpts_sleep_time); 141 KTEST_LOG(ctx, " p_delayed_by: %u", hpts->p_delayed_by); 142 KTEST_LOG(ctx, " overidden_sleep: %u", hpts->overidden_sleep); 143 KTEST_LOG(ctx, " saved_curslot: %u", hpts->saved_curslot); 144 KTEST_LOG(ctx, " saved_prev_slot: %u", hpts->saved_prev_slot); 145 KTEST_LOG(ctx, " syscall_cnt: %lu", hpts->syscall_cnt); 146 KTEST_LOG(ctx, " sleeping: %lu", hpts->sleeping); 147 KTEST_LOG(ctx, " p_cpu: %u", hpts->p_cpu); 148 KTEST_LOG(ctx, " ie_cookie: %p", hpts->ie_cookie); 149 KTEST_LOG(ctx, " p_hptsi: %p", hpts->p_hptsi); 150 KTEST_LOG(ctx, " p_mysleep: %ld.%06ld", hpts->p_mysleep.tv_sec, hpts->p_mysleep.tv_usec); 151 } 152 153 static void 154 dump_tcpcb(struct tcpcb *tp) 155 { 156 struct ktest_test_context *ctx = tp->t_fb_ptr; 157 struct inpcb *inp = &tp->t_inpcb; 158 159 KTEST_LOG(ctx, "tcp_control_block(%p)", tp); 160 161 /* HPTS-specific fields */ 162 KTEST_LOG(ctx, " t_in_hpts: %d", tp->t_in_hpts); 163 KTEST_LOG(ctx, " t_hpts_cpu: %u", tp->t_hpts_cpu); 164 KTEST_LOG(ctx, " t_hpts_slot: %d", tp->t_hpts_slot); 165 KTEST_LOG(ctx, " t_hpts_gencnt: %u", tp->t_hpts_gencnt); 166 KTEST_LOG(ctx, " t_hpts_request: %u", tp->t_hpts_request); 167 168 /* LRO CPU field */ 169 KTEST_LOG(ctx, " t_lro_cpu: %u", tp->t_lro_cpu); 170 171 /* TCP flags that affect HPTS */ 172 KTEST_LOG(ctx, " t_flags2: 0x%x", tp->t_flags2); 173 KTEST_LOG(ctx, " TF2_HPTS_CPU_SET: %s", (tp->t_flags2 & TF2_HPTS_CPU_SET) ? "YES" : "NO"); 174 KTEST_LOG(ctx, " TF2_HPTS_CALLS: %s", (tp->t_flags2 & TF2_HPTS_CALLS) ? "YES" : "NO"); 175 KTEST_LOG(ctx, " TF2_SUPPORTS_MBUFQ: %s", (tp->t_flags2 & TF2_SUPPORTS_MBUFQ) ? "YES" : "NO"); 176 177 /* Input PCB fields that HPTS uses */ 178 KTEST_LOG(ctx, " inp_flags: 0x%x", inp->inp_flags); 179 KTEST_LOG(ctx, " INP_DROPPED: %s", (inp->inp_flags & INP_DROPPED) ? "YES" : "NO"); 180 KTEST_LOG(ctx, " inp_flowid: 0x%x", inp->inp_flowid); 181 KTEST_LOG(ctx, " inp_flowtype: %u", inp->inp_flowtype); 182 KTEST_LOG(ctx, " inp_numa_domain: %d", inp->inp_numa_domain); 183 } 184 185 /* Enum for call counting indices */ 186 enum test_call_counts { 187 CCNT_MICROUPTIME = 0, 188 CCNT_SWI_ADD, 189 CCNT_SWI_REMOVE, 190 CCNT_SWI_SCHED, 191 CCNT_INTR_EVENT_BIND, 192 CCNT_INTR_EVENT_BIND_CPUSET, 193 CCNT_CALLOUT_INIT, 194 CCNT_CALLOUT_RESET_SBT_ON, 195 CCNT_CALLOUT_STOP_SAFE, 196 CCNT_TCP_OUTPUT, 197 CCNT_TCP_TFB_DO_QUEUED_SEGMENTS, 198 CCNT_MAX 199 }; 200 201 static uint32_t call_counts[CCNT_MAX]; 202 203 static uint64_t test_time_usec = 0; 204 205 /* 206 * Reset all test global variables to a clean state. 207 */ 208 static void 209 test_hpts_init(void) 210 { 211 memset(call_counts, 0, sizeof(call_counts)); 212 test_time_usec = 0; 213 } 214 215 static void 216 test_microuptime(struct timeval *tv) 217 { 218 call_counts[CCNT_MICROUPTIME]++; 219 tv->tv_sec = test_time_usec / 1000000; 220 tv->tv_usec = test_time_usec % 1000000; 221 } 222 223 static int 224 test_swi_add(struct intr_event **eventp, const char *name, 225 driver_intr_t handler, void *arg, int pri, enum intr_type flags, 226 void **cookiep) 227 { 228 call_counts[CCNT_SWI_ADD]++; 229 /* Simulate successful SWI creation */ 230 *eventp = (struct intr_event *)0xfeedface; /* Mock event */ 231 *cookiep = (void *)0xdeadbeef; /* Mock cookie */ 232 return (0); 233 } 234 235 static int 236 test_swi_remove(void *cookie) 237 { 238 call_counts[CCNT_SWI_REMOVE]++; 239 /* Simulate successful removal */ 240 return (0); 241 } 242 243 static void 244 test_swi_sched(void *cookie, int flags) 245 { 246 call_counts[CCNT_SWI_SCHED]++; 247 /* Simulate successful SWI scheduling */ 248 } 249 250 static int 251 test_intr_event_bind(struct intr_event *ie, int cpu) 252 { 253 call_counts[CCNT_INTR_EVENT_BIND]++; 254 /* Simulate successful binding */ 255 return (0); 256 } 257 258 static int 259 test_intr_event_bind_ithread_cpuset(struct intr_event *ie, struct _cpuset *mask) 260 { 261 call_counts[CCNT_INTR_EVENT_BIND_CPUSET]++; 262 /* Simulate successful cpuset binding */ 263 return (0); 264 } 265 266 static void 267 test_callout_init(struct callout *c, int mpsafe) 268 { 269 call_counts[CCNT_CALLOUT_INIT]++; 270 memset(c, 0, sizeof(*c)); 271 } 272 273 static int 274 test_callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t precision, 275 void (*func)(void *), void *arg, int cpu, int flags) 276 { 277 call_counts[CCNT_CALLOUT_RESET_SBT_ON]++; 278 /* Return 1 to simulate successful timer scheduling */ 279 return (1); 280 } 281 282 static int 283 test_callout_stop_safe(struct callout *c, int flags) 284 { 285 call_counts[CCNT_CALLOUT_STOP_SAFE]++; 286 /* Return 1 to simulate successful timer stopping */ 287 return (1); 288 } 289 290 static const struct tcp_hptsi_funcs test_funcs = { 291 .microuptime = test_microuptime, 292 .swi_add = test_swi_add, 293 .swi_remove = test_swi_remove, 294 .swi_sched = test_swi_sched, 295 .intr_event_bind = test_intr_event_bind, 296 .intr_event_bind_ithread_cpuset = test_intr_event_bind_ithread_cpuset, 297 .callout_init = test_callout_init, 298 .callout_reset_sbt_on = test_callout_reset_sbt_on, 299 ._callout_stop_safe = test_callout_stop_safe, 300 }; 301 302 #define TP_REMOVE_FROM_HPTS(tp) tp->bits_spare 303 #define TP_LOG_TEST(tp) tp->t_log_state_set 304 305 static int 306 test_tcp_output(struct tcpcb *tp) 307 { 308 struct ktest_test_context *ctx = tp->t_fb_ptr; 309 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending; 310 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu]; 311 312 call_counts[CCNT_TCP_OUTPUT]++; 313 if (TP_LOG_TEST(tp)) { 314 KTEST_LOG(ctx, "=> tcp_output(%p)", tp); 315 dump_tcpcb(tp); 316 dump_hpts_entry(ctx, hpts); 317 } 318 319 if ((TP_REMOVE_FROM_HPTS(tp) & 1) != 0) { 320 if (TP_LOG_TEST(tp)) 321 KTEST_LOG(ctx, "=> tcp_hpts_remove(%p)", tp); 322 tcp_hpts_remove(pace, tp); 323 } 324 325 if ((TP_REMOVE_FROM_HPTS(tp) & 2) != 0) { 326 INP_WUNLOCK(&tp->t_inpcb); /* tcp_output unlocks on error */ 327 return (-1); /* Simulate tcp_output error */ 328 } 329 330 return (0); 331 } 332 333 static int 334 test_tfb_do_queued_segments(struct tcpcb *tp, int flag) 335 { 336 struct ktest_test_context *ctx = tp->t_fb_ptr; 337 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending; 338 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu]; 339 340 call_counts[CCNT_TCP_TFB_DO_QUEUED_SEGMENTS]++; 341 KTEST_LOG(ctx, "=> tfb_do_queued_segments(%p, %d)", tp, flag); 342 dump_tcpcb(tp); 343 dump_hpts_entry(ctx, hpts); 344 345 if ((TP_REMOVE_FROM_HPTS(tp) & 1) != 0) { 346 if (TP_LOG_TEST(tp)) 347 KTEST_LOG(ctx, "=> tcp_hpts_remove(%p)", tp); 348 tcp_hpts_remove(pace, tp); 349 } 350 351 if ((TP_REMOVE_FROM_HPTS(tp) & 2) != 0) { 352 INP_WUNLOCK(&tp->t_inpcb); /* do_queued_segments unlocks on error */ 353 return (-1); /* Simulate do_queued_segments error */ 354 } 355 356 return (0); 357 } 358 359 static struct tcp_function_block test_tcp_fb = { 360 .tfb_tcp_block_name = "hpts_test_tcp", 361 .tfb_tcp_output = test_tcp_output, 362 .tfb_do_queued_segments = test_tfb_do_queued_segments, 363 }; 364 365 /* 366 * Create a minimally initialized tcpcb that can be safely inserted into HPTS. 367 * This function allocates and initializes all the fields that HPTS code 368 * reads or writes. 369 */ 370 static struct tcpcb * 371 test_hpts_create_tcpcb(struct ktest_test_context *ctx, struct tcp_hptsi *pace) 372 { 373 struct tcpcb *tp; 374 375 tp = malloc(sizeof(struct tcpcb), M_TCPHPTS, M_WAITOK | M_ZERO); 376 if (tp) { 377 rw_init_flags(&tp->t_inpcb.inp_lock, "test-inp", 378 RW_RECURSE | RW_DUPOK); 379 refcount_init(&tp->t_inpcb.inp_refcount, 1); 380 tp->t_inpcb.inp_pcbinfo = &V_tcbinfo; 381 tp->t_fb = &test_tcp_fb; 382 tp->t_hpts_cpu = HPTS_CPU_NONE; 383 STAILQ_INIT(&tp->t_inqueue); 384 tcp_hpts_init(pace, tp); 385 386 /* Stuff some pointers in the tcb for test purposes. */ 387 tp->t_fb_ptr = ctx; 388 tp->t_tfo_pending = (unsigned int*)pace; 389 } 390 391 return (tp); 392 } 393 394 /* 395 * Free a test tcpcb created by test_hpts_create_tcpcb() 396 */ 397 static void 398 test_hpts_free_tcpcb(struct tcpcb *tp) 399 { 400 if (tp == NULL) 401 return; 402 403 INP_LOCK_DESTROY(&tp->t_inpcb); 404 free(tp, M_TCPHPTS); 405 } 406 407 /* 408 * *********************************************** 409 * * KTEST functions for testing the HPTS module * 410 * *********************************************** 411 */ 412 413 /* 414 * Validates that the HPTS module is properly loaded and initialized by checking 415 * that the minimum HPTS time is configured. 416 */ 417 KTEST_FUNC(module_load) 418 { 419 test_hpts_init(); 420 KTEST_NEQUAL(tcp_min_hptsi_time, 0); 421 KTEST_VERIFY(tcp_bind_threads >= 0 && tcp_bind_threads <= 2); 422 KTEST_NEQUAL(tcp_hptsi_pace, NULL); 423 return (0); 424 } 425 426 /* 427 * Validates the creation and destruction of tcp_hptsi structures, ensuring 428 * proper initialization of internal fields and clean destruction. 429 */ 430 KTEST_FUNC(hptsi_create_destroy) 431 { 432 struct tcp_hptsi *pace; 433 434 test_hpts_init(); 435 436 pace = tcp_hptsi_create(&test_funcs, false); 437 KTEST_NEQUAL(pace, NULL); 438 KTEST_NEQUAL(pace->rp_ent, NULL); 439 KTEST_NEQUAL(pace->cts_last_ran, NULL); 440 KTEST_VERIFY(pace->rp_num_hptss > 0); 441 KTEST_VERIFY(pace->rp_num_hptss <= MAXCPU); /* Reasonable upper bound */ 442 KTEST_VERIFY(pace->grp_cnt >= 1); /* At least one group */ 443 KTEST_EQUAL(pace->funcs, &test_funcs); /* Verify function pointer was set */ 444 445 /* Verify individual HPTS entries are properly initialized */ 446 for (uint32_t i = 0; i < pace->rp_num_hptss; i++) { 447 KTEST_NEQUAL(pace->rp_ent[i], NULL); 448 KTEST_EQUAL(pace->rp_ent[i]->p_cpu, i); 449 KTEST_EQUAL(pace->rp_ent[i]->p_hptsi, pace); 450 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0); 451 } 452 453 tcp_hptsi_destroy(pace); 454 455 return (0); 456 } 457 458 /* 459 * Validates that tcp_hptsi structures can be started and stopped properly, 460 * including verification that threads are created during start and cleaned up 461 * during stop operations. 462 */ 463 KTEST_FUNC(hptsi_start_stop) 464 { 465 struct tcp_hptsi *pace; 466 467 test_hpts_init(); 468 469 pace = tcp_hptsi_create(&test_funcs, false); 470 KTEST_NEQUAL(pace, NULL); 471 472 tcp_hptsi_start(pace); 473 474 /* Verify that entries have threads started */ 475 struct tcp_hpts_entry *hpts = pace->rp_ent[0]; 476 KTEST_NEQUAL(hpts->ie_cookie, NULL); /* Should have SWI handler */ 477 KTEST_EQUAL(hpts->p_hptsi, pace); /* Should point to our pace */ 478 479 tcp_hptsi_stop(pace); 480 tcp_hptsi_destroy(pace); 481 482 return (0); 483 } 484 485 /* 486 * Validates that multiple tcp_hptsi instances can coexist independently, with 487 * different configurations and CPU assignments without interfering with each 488 * other. 489 */ 490 KTEST_FUNC(hptsi_independence) 491 { 492 struct tcp_hptsi *pace1, *pace2; 493 uint16_t cpu1, cpu2; 494 495 test_hpts_init(); 496 497 pace1 = tcp_hptsi_create(&test_funcs, false); 498 pace2 = tcp_hptsi_create(&test_funcs, false); 499 KTEST_NEQUAL(pace1, NULL); 500 KTEST_NEQUAL(pace2, NULL); 501 KTEST_NEQUAL(pace2->rp_ent, NULL); 502 503 cpu1 = tcp_hptsi_random_cpu(pace1); 504 cpu2 = tcp_hptsi_random_cpu(pace2); 505 KTEST_VERIFY(cpu1 < pace1->rp_num_hptss); 506 KTEST_VERIFY(cpu2 < pace2->rp_num_hptss); 507 508 /* Verify both instances have independent entry arrays */ 509 KTEST_NEQUAL(pace1->rp_ent, pace2->rp_ent); 510 /* Verify they may have different CPU counts but both reasonable */ 511 KTEST_VERIFY(pace1->rp_num_hptss > 0 && pace1->rp_num_hptss <= MAXCPU); 512 KTEST_VERIFY(pace2->rp_num_hptss > 0 && pace2->rp_num_hptss <= MAXCPU); 513 514 tcp_hptsi_destroy(pace1); 515 tcp_hptsi_destroy(pace2); 516 517 return (0); 518 } 519 520 /* 521 * Validates that custom function injection works correctly, ensuring that 522 * test-specific implementations of microuptime and others are properly 523 * called by the HPTS system. 524 */ 525 KTEST_FUNC(function_injection) 526 { 527 struct tcp_hptsi *pace; 528 529 test_hpts_init(); 530 531 pace = tcp_hptsi_create(&test_funcs, false); 532 KTEST_NEQUAL(pace, NULL); 533 KTEST_EQUAL(pace->funcs, &test_funcs); 534 KTEST_VERIFY(call_counts[CCNT_MICROUPTIME] > 0); 535 KTEST_VERIFY(call_counts[CCNT_CALLOUT_INIT] > 0); 536 537 tcp_hptsi_start(pace); 538 KTEST_VERIFY(call_counts[CCNT_SWI_ADD] > 0); 539 KTEST_VERIFY(tcp_bind_threads == 0 || 540 call_counts[CCNT_INTR_EVENT_BIND] > 0 || 541 call_counts[CCNT_INTR_EVENT_BIND_CPUSET] > 0); 542 KTEST_VERIFY(call_counts[CCNT_CALLOUT_RESET_SBT_ON] > 0); 543 544 tcp_hptsi_stop(pace); 545 KTEST_VERIFY(call_counts[CCNT_CALLOUT_STOP_SAFE] > 0); 546 KTEST_VERIFY(call_counts[CCNT_SWI_REMOVE] > 0); 547 548 tcp_hptsi_destroy(pace); 549 550 /* Verify we have a reasonable balance of create/destroy calls */ 551 KTEST_EQUAL(call_counts[CCNT_SWI_ADD], call_counts[CCNT_SWI_REMOVE]); 552 KTEST_VERIFY(call_counts[CCNT_CALLOUT_RESET_SBT_ON] <= call_counts[CCNT_CALLOUT_STOP_SAFE]); 553 554 return (0); 555 } 556 557 /* 558 * Validates that a tcpcb can be properly initialized for HPTS compatibility, 559 * ensuring all required fields are set correctly and function pointers are 560 * valid for safe HPTS operations. 561 */ 562 KTEST_FUNC(tcpcb_initialization) 563 { 564 struct tcp_hptsi *pace; 565 struct tcpcb *tp; 566 567 test_hpts_init(); 568 569 pace = tcp_hptsi_create(&test_funcs, false); 570 KTEST_NEQUAL(pace, NULL); 571 tcp_hptsi_start(pace); 572 573 /* Verify the tcpcb is properly initialized for HPTS */ 574 tp = test_hpts_create_tcpcb(ctx, pace); 575 KTEST_NEQUAL(tp, NULL); 576 KTEST_NEQUAL(tp->t_fb, NULL); 577 KTEST_NEQUAL(tp->t_fb->tfb_tcp_output, NULL); 578 KTEST_NEQUAL(tp->t_fb->tfb_do_queued_segments, NULL); 579 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); 580 KTEST_EQUAL((tp->t_flags2 & (TF2_HPTS_CPU_SET | TF2_HPTS_CALLS)), 0); 581 582 /* Verify that HPTS-specific fields are initialized */ 583 KTEST_EQUAL(tp->t_hpts_gencnt, 0); 584 KTEST_EQUAL(tp->t_hpts_slot, 0); 585 KTEST_EQUAL(tp->t_hpts_request, 0); 586 KTEST_EQUAL(tp->t_lro_cpu, 0); 587 KTEST_VERIFY(tp->t_hpts_cpu < pace->rp_num_hptss); 588 KTEST_EQUAL(tp->t_inpcb.inp_refcount, 1); 589 KTEST_VERIFY(!(tp->t_inpcb.inp_flags & INP_DROPPED)); 590 591 test_hpts_free_tcpcb(tp); 592 tcp_hptsi_stop(pace); 593 tcp_hptsi_destroy(pace); 594 595 return (0); 596 } 597 598 /* 599 * Validates that tcpcb structures can be successfully inserted into and removed 600 * from the HPTS wheel, with proper state tracking and slot assignment during 601 * the process. 602 */ 603 KTEST_FUNC(tcpcb_insertion) 604 { 605 struct tcp_hptsi *pace; 606 struct tcpcb *tp; 607 struct tcp_hpts_entry *hpts; 608 uint32_t timeout_usecs = 10; 609 610 test_hpts_init(); 611 612 pace = tcp_hptsi_create(&test_funcs, false); 613 KTEST_NEQUAL(pace, NULL); 614 tcp_hptsi_start(pace); 615 616 tp = test_hpts_create_tcpcb(ctx, pace); 617 KTEST_NEQUAL(tp, NULL); 618 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); 619 KTEST_EQUAL((tp->t_flags2 & TF2_HPTS_CALLS), 0); 620 621 INP_WLOCK(&tp->t_inpcb); 622 tp->t_flags2 |= TF2_HPTS_CALLS; 623 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 0); 624 tcp_hpts_insert(pace, tp, timeout_usecs, NULL); 625 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 626 INP_WUNLOCK(&tp->t_inpcb); 627 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); 628 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 1); 629 KTEST_VERIFY(tcp_in_hpts(tp)); 630 KTEST_VERIFY(tp->t_hpts_slot >= 0); 631 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS); 632 633 hpts = pace->rp_ent[tp->t_hpts_cpu]; 634 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); 635 KTEST_EQUAL(tp->t_hpts_request, 0); 636 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(timeout_usecs)); 637 //KTEST_EQUAL(tp->t_hpts_gencnt, 1); 638 639 INP_WLOCK(&tp->t_inpcb); 640 tcp_hpts_remove(pace, tp); 641 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); 642 INP_WUNLOCK(&tp->t_inpcb); 643 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); 644 KTEST_VERIFY(!tcp_in_hpts(tp)); 645 646 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); 647 648 test_hpts_free_tcpcb(tp); 649 tcp_hptsi_stop(pace); 650 tcp_hptsi_destroy(pace); 651 652 return (0); 653 } 654 655 /* 656 * Validates the core HPTS timer functionality by verifying that scheduled 657 * tcpcb entries trigger tcp_output calls at appropriate times, simulating 658 * real-world timer-driven TCP processing. 659 */ 660 KTEST_FUNC(timer_functionality) 661 { 662 struct epoch_tracker et; 663 struct tcp_hptsi *pace; 664 struct tcp_hpts_entry *hpts; 665 struct tcpcb *tp; 666 int32_t slots_ran; 667 uint32_t i; 668 669 test_hpts_init(); 670 671 pace = tcp_hptsi_create(&test_funcs, false); 672 KTEST_NEQUAL(pace, NULL); 673 tcp_hptsi_start(pace); 674 675 for (i = 0; i < pace->rp_num_hptss; i++) 676 dump_hpts_entry(ctx, pace->rp_ent[i]); 677 678 /* Create and insert the tcpcb into the HPTS wheel to wait for 500 usec */ 679 tp = test_hpts_create_tcpcb(ctx, pace); 680 KTEST_NEQUAL(tp, NULL); 681 dump_tcpcb(tp); 682 TP_LOG_TEST(tp) = 1; /* Enable logging for this tcpcb */ 683 684 KTEST_LOG(ctx, "=> tcp_hpts_insert(%p)", tp); 685 INP_WLOCK(&tp->t_inpcb); 686 tp->t_flags2 |= TF2_HPTS_CALLS; /* Mark as needing HPTS processing */ 687 tcp_hpts_insert(pace, tp, 500, NULL); 688 INP_WUNLOCK(&tp->t_inpcb); 689 690 dump_tcpcb(tp); 691 for (i = 0; i < pace->rp_num_hptss; i++) 692 dump_hpts_entry(ctx, pace->rp_ent[i]); 693 694 hpts = pace->rp_ent[tp->t_hpts_cpu]; 695 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); 696 KTEST_EQUAL(hpts->p_prev_slot, 0); 697 KTEST_EQUAL(hpts->p_cur_slot, 0); 698 KTEST_EQUAL(hpts->p_runningslot, 0); 699 KTEST_EQUAL(hpts->p_nxt_slot, 1); 700 KTEST_EQUAL(hpts->p_hpts_active, 0); 701 702 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 703 KTEST_EQUAL(tp->t_hpts_request, 0); 704 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500)); 705 706 /* Set our test flag to indicate the tcpcb should be removed from the 707 * wheel when tcp_output is called. */ 708 TP_REMOVE_FROM_HPTS(tp) = 1; 709 710 /* Test early exit condition: advance time by insufficient amount */ 711 KTEST_LOG(ctx, "Testing early exit with insufficient time advancement"); 712 test_time_usec += 1; /* Very small advancement - should cause early exit */ 713 HPTS_LOCK(hpts); 714 NET_EPOCH_ENTER(et); 715 slots_ran = tcp_hptsi(hpts, true); 716 HPTS_UNLOCK(hpts); 717 NET_EPOCH_EXIT(et); 718 719 /* Should return 0 slots due to insufficient time advancement */ 720 KTEST_EQUAL(slots_ran, 0); 721 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); /* No processing should occur */ 722 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); /* Connection still queued */ 723 724 /* Wait for 498 more usecs and trigger the HPTS workers and verify 725 * nothing happens yet (total 499 usec) */ 726 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); 727 test_time_usec += 498; 728 for (i = 0; i < pace->rp_num_hptss; i++) { 729 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]); 730 HPTS_LOCK(pace->rp_ent[i]); 731 NET_EPOCH_ENTER(et); 732 slots_ran = tcp_hptsi(pace->rp_ent[i], true); 733 HPTS_UNLOCK(pace->rp_ent[i]); 734 NET_EPOCH_EXIT(et); 735 736 dump_hpts_entry(ctx, pace->rp_ent[i]); 737 KTEST_VERIFY(slots_ran >= 0); 738 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 49); 739 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 49); 740 } 741 742 dump_tcpcb(tp); 743 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); 744 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 745 KTEST_EQUAL(tp->t_hpts_request, 0); 746 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500)); 747 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); 748 749 /* Wait for 1 more usec and trigger the HPTS workers and verify it 750 * triggers tcp_output this time */ 751 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); 752 test_time_usec += 1; 753 for (i = 0; i < pace->rp_num_hptss; i++) { 754 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]); 755 HPTS_LOCK(pace->rp_ent[i]); 756 NET_EPOCH_ENTER(et); 757 slots_ran = tcp_hptsi(pace->rp_ent[i], true); 758 HPTS_UNLOCK(pace->rp_ent[i]); 759 NET_EPOCH_EXIT(et); 760 761 dump_hpts_entry(ctx, pace->rp_ent[i]); 762 KTEST_VERIFY(slots_ran >= 0); 763 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 50); 764 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 50); 765 } 766 767 dump_tcpcb(tp); 768 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1); 769 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); 770 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); 771 772 test_hpts_free_tcpcb(tp); 773 tcp_hptsi_stop(pace); 774 tcp_hptsi_destroy(pace); 775 776 return (0); 777 } 778 779 /* 780 * Validates HPTS scalability by creating and inserting a LOT of tcpcbs into 781 * the HPTS wheel, testing performance under high load conditions. 782 */ 783 KTEST_FUNC(scalability_tcpcbs) 784 { 785 struct tcp_hptsi *pace; 786 struct tcpcb **tcpcbs; 787 uint32_t i, num_tcpcbs = 100000, total_queued = 0; 788 789 test_hpts_init(); 790 791 pace = tcp_hptsi_create(&test_funcs, false); 792 KTEST_NEQUAL(pace, NULL); 793 tcp_hptsi_start(pace); 794 795 /* Allocate array to hold pointers to all tcpcbs */ 796 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO); 797 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM); 798 799 /* Create a LOT of tcpcbs */ 800 KTEST_LOG(ctx, "Creating %u tcpcbs...", num_tcpcbs); 801 for (i = 0; i < num_tcpcbs; i++) { 802 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace); 803 if (tcpcbs[i] == NULL) { 804 KTEST_ERR(ctx, "FAIL: tcpcbs[i] == NULL"); 805 return (EINVAL); 806 } 807 } 808 809 /* Insert all created tcpcbs into HPTS */ 810 KTEST_LOG(ctx, "Inserting all tcpcbs into HPTS..."); 811 for (i = 0; i < num_tcpcbs; i++) { 812 INP_WLOCK(&tcpcbs[i]->t_inpcb); 813 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS; 814 /* Insert with varying future timeouts to distribute across slots */ 815 tcp_hpts_insert(pace, tcpcbs[i], 100 + (i % 1000), NULL); 816 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); 817 } 818 819 /* Verify total queue counts across all CPUs */ 820 for (i = 0; i < pace->rp_num_hptss; i++) { 821 total_queued += pace->rp_ent[i]->p_on_queue_cnt; 822 } 823 KTEST_EQUAL(total_queued, num_tcpcbs); 824 825 for (i = 0; i < pace->rp_num_hptss; i++) 826 dump_hpts_entry(ctx, pace->rp_ent[i]); 827 828 /* Remove all tcpcbs from HPTS */ 829 KTEST_LOG(ctx, "Removing all tcpcbs from HPTS..."); 830 for (i = 0; i < num_tcpcbs; i++) { 831 INP_WLOCK(&tcpcbs[i]->t_inpcb); 832 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) { 833 tcp_hpts_remove(pace, tcpcbs[i]); 834 } 835 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); 836 } 837 838 /* Verify all queues are now empty */ 839 for (i = 0; i < pace->rp_num_hptss; i++) { 840 if (pace->rp_ent[i]->p_on_queue_cnt != 0) { 841 KTEST_ERR(ctx, "FAIL: pace->rp_ent[i]->p_on_queue_cnt != 0"); 842 return (EINVAL); 843 } 844 } 845 846 for (i = 0; i < num_tcpcbs; i++) { 847 test_hpts_free_tcpcb(tcpcbs[i]); 848 } 849 free(tcpcbs, M_TCPHPTS); 850 tcp_hptsi_stop(pace); 851 tcp_hptsi_destroy(pace); 852 853 return (0); 854 } 855 856 /* 857 * Validates wheel wrap scenarios where the timer falls significantly behind 858 * and needs to process more than one full wheel revolution worth of slots. 859 */ 860 KTEST_FUNC(wheel_wrap_recovery) 861 { 862 struct epoch_tracker et; 863 struct tcp_hptsi *pace; 864 struct tcpcb **tcpcbs; 865 uint32_t i, timeout_usecs, num_tcpcbs = 500; 866 int32_t slots_ran; 867 868 test_hpts_init(); 869 870 pace = tcp_hptsi_create(&test_funcs, false); 871 KTEST_NEQUAL(pace, NULL); 872 tcp_hptsi_start(pace); 873 874 /* Allocate array to hold pointers to tcpcbs */ 875 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO); 876 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM); 877 878 /* Create tcpcbs and insert them across many slots */ 879 for (i = 0; i < num_tcpcbs; i++) { 880 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace); 881 KTEST_NEQUAL(tcpcbs[i], NULL); 882 TP_REMOVE_FROM_HPTS(tcpcbs[i]) = 1; 883 884 timeout_usecs = ((i * NUM_OF_HPTSI_SLOTS) / num_tcpcbs) * HPTS_USECS_PER_SLOT; /* Spread across slots */ 885 886 INP_WLOCK(&tcpcbs[i]->t_inpcb); 887 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS; 888 tcp_hpts_insert(pace, tcpcbs[i], timeout_usecs, NULL); 889 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); 890 } 891 892 /* Fast forward time significantly to trigger wheel wrap */ 893 test_time_usec += (NUM_OF_HPTSI_SLOTS + 5000) * HPTS_USECS_PER_SLOT; 894 895 for (i = 0; i < pace->rp_num_hptss; i++) { 896 KTEST_LOG(ctx, "=> tcp_hptsi(%u)", i); 897 KTEST_NEQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0); 898 899 HPTS_LOCK(pace->rp_ent[i]); 900 NET_EPOCH_ENTER(et); 901 slots_ran = tcp_hptsi(pace->rp_ent[i], true); 902 HPTS_UNLOCK(pace->rp_ent[i]); 903 NET_EPOCH_EXIT(et); 904 905 KTEST_EQUAL(slots_ran, NUM_OF_HPTSI_SLOTS-1); /* Should process all slots */ 906 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0); 907 KTEST_NEQUAL(pace->rp_ent[i]->p_cur_slot, 908 pace->rp_ent[i]->p_prev_slot); 909 } 910 911 /* Cleanup */ 912 for (i = 0; i < num_tcpcbs; i++) { 913 INP_WLOCK(&tcpcbs[i]->t_inpcb); 914 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) { 915 tcp_hpts_remove(pace, tcpcbs[i]); 916 } 917 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); 918 test_hpts_free_tcpcb(tcpcbs[i]); 919 } 920 free(tcpcbs, M_TCPHPTS); 921 tcp_hptsi_stop(pace); 922 tcp_hptsi_destroy(pace); 923 924 return (0); 925 } 926 927 /* 928 * Validates proper handling of tcpcbs in the IHPTS_MOVING state, which occurs 929 * when a tcpcb is being processed by the HPTS thread but gets removed. 930 */ 931 KTEST_FUNC(tcpcb_moving_state) 932 { 933 struct epoch_tracker et; 934 struct tcp_hptsi *pace; 935 struct tcpcb *tp1, *tp2; 936 struct tcp_hpts_entry *hpts; 937 int32_t slots_ran; 938 939 test_hpts_init(); 940 941 pace = tcp_hptsi_create(&test_funcs, false); 942 KTEST_NEQUAL(pace, NULL); 943 tcp_hptsi_start(pace); 944 945 /* Create two tcpcbs on the same CPU/slot */ 946 tp1 = test_hpts_create_tcpcb(ctx, pace); 947 tp2 = test_hpts_create_tcpcb(ctx, pace); 948 KTEST_NEQUAL(tp1, NULL); 949 KTEST_NEQUAL(tp2, NULL); 950 951 /* Force them to the same CPU for predictable testing */ 952 tp1->t_hpts_cpu = 0; 953 tp2->t_hpts_cpu = 0; 954 955 /* Insert both into the same slot */ 956 INP_WLOCK(&tp1->t_inpcb); 957 tp1->t_flags2 |= TF2_HPTS_CALLS; 958 tcp_hpts_insert(pace, tp1, 100, NULL); 959 INP_WUNLOCK(&tp1->t_inpcb); 960 961 INP_WLOCK(&tp2->t_inpcb); 962 tp2->t_flags2 |= TF2_HPTS_CALLS; 963 tcp_hpts_insert(pace, tp2, 100, NULL); 964 INP_WUNLOCK(&tp2->t_inpcb); 965 966 hpts = pace->rp_ent[0]; 967 968 /* Manually transition tp1 to MOVING state to simulate race condition */ 969 HPTS_LOCK(hpts); 970 tp1->t_in_hpts = IHPTS_MOVING; 971 tp1->t_hpts_slot = -1; /* Mark for removal */ 972 HPTS_UNLOCK(hpts); 973 974 /* Set time and run HPTS to process the moving state */ 975 test_time_usec += 100; 976 HPTS_LOCK(hpts); 977 NET_EPOCH_ENTER(et); 978 slots_ran = tcp_hptsi(hpts, true); 979 HPTS_UNLOCK(hpts); 980 NET_EPOCH_EXIT(et); 981 982 KTEST_VERIFY(slots_ran >= 0); 983 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1); /* Shouldn't call on both */ 984 985 /* tp1 should be cleaned up and removed */ 986 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE); 987 /* tp2 should have been processed normally */ 988 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); 989 990 test_hpts_free_tcpcb(tp1); 991 test_hpts_free_tcpcb(tp2); 992 tcp_hptsi_stop(pace); 993 tcp_hptsi_destroy(pace); 994 995 return (0); 996 } 997 998 /* 999 * Validates that tcpcbs with deferred requests (t_hpts_request > 0) are 1000 * properly handled and re-inserted into appropriate future slots after 1001 * the wheel processes enough slots to accommodate the original request. 1002 */ 1003 KTEST_FUNC(deferred_requests) 1004 { 1005 struct epoch_tracker et; 1006 struct tcp_hptsi *pace; 1007 struct tcpcb *tp, *tp2; 1008 struct tcp_hpts_entry *hpts; 1009 uint32_t large_timeout_usecs = (NUM_OF_HPTSI_SLOTS + 5000) * HPTS_USECS_PER_SLOT; /* Beyond wheel capacity */ 1010 uint32_t huge_timeout_usecs = (NUM_OF_HPTSI_SLOTS * 3) * HPTS_USECS_PER_SLOT; /* 3x wheel capacity */ 1011 uint32_t initial_request; 1012 int32_t slots_ran; 1013 1014 test_hpts_init(); 1015 1016 pace = tcp_hptsi_create(&test_funcs, false); 1017 KTEST_NEQUAL(pace, NULL); 1018 tcp_hptsi_start(pace); 1019 1020 tp = test_hpts_create_tcpcb(ctx, pace); 1021 KTEST_NEQUAL(tp, NULL); 1022 1023 /* Insert with a request that exceeds current wheel capacity */ 1024 INP_WLOCK(&tp->t_inpcb); 1025 tp->t_flags2 |= TF2_HPTS_CALLS; 1026 tcp_hpts_insert(pace, tp, large_timeout_usecs, NULL); 1027 INP_WUNLOCK(&tp->t_inpcb); 1028 1029 /* Verify it was inserted with a deferred request */ 1030 dump_tcpcb(tp); 1031 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 1032 KTEST_VERIFY(tp->t_hpts_request > 0); 1033 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS); 1034 1035 hpts = pace->rp_ent[tp->t_hpts_cpu]; 1036 1037 /* Advance time to process deferred requests */ 1038 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT; 1039 1040 /* Process the wheel to handle deferred requests */ 1041 HPTS_LOCK(hpts); 1042 NET_EPOCH_ENTER(et); 1043 slots_ran = tcp_hptsi(hpts, true); 1044 HPTS_UNLOCK(hpts); 1045 NET_EPOCH_EXIT(et); 1046 1047 dump_hpts_entry(ctx, hpts); 1048 KTEST_GREATER_THAN(slots_ran, 0); 1049 dump_tcpcb(tp); 1050 KTEST_EQUAL(tp->t_hpts_request, 0); 1051 1052 /* Test incremental deferred request processing over multiple cycles */ 1053 KTEST_LOG(ctx, "Testing incremental deferred request processing"); 1054 1055 /* Create a new connection with an even larger request */ 1056 tp2 = test_hpts_create_tcpcb(ctx, pace); 1057 KTEST_NEQUAL(tp2, NULL); 1058 tp2->t_hpts_cpu = tp->t_hpts_cpu; /* Same CPU for predictable testing */ 1059 1060 INP_WLOCK(&tp2->t_inpcb); 1061 tp2->t_flags2 |= TF2_HPTS_CALLS; 1062 tcp_hpts_insert(pace, tp2, huge_timeout_usecs, NULL); 1063 INP_WUNLOCK(&tp2->t_inpcb); 1064 1065 /* Verify initial deferred request */ 1066 initial_request = tp2->t_hpts_request; 1067 KTEST_VERIFY(initial_request > NUM_OF_HPTSI_SLOTS); 1068 1069 /* Process one wheel cycle - should reduce but not eliminate request */ 1070 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT; 1071 HPTS_LOCK(hpts); 1072 NET_EPOCH_ENTER(et); 1073 slots_ran = tcp_hptsi(hpts, true); 1074 HPTS_UNLOCK(hpts); 1075 NET_EPOCH_EXIT(et); 1076 1077 /* Request should be reduced but not zero */ 1078 KTEST_GREATER_THAN(initial_request, tp2->t_hpts_request); 1079 KTEST_VERIFY(tp2->t_hpts_request > 0); 1080 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); /* Still queued */ 1081 1082 /* For huge_timeout_usecs = NUM_OF_HPTSI_SLOTS * 3 * HPTS_USECS_PER_SLOT, we need ~3 cycles to complete. 1083 * Each cycle can reduce the request by at most NUM_OF_HPTSI_SLOTS. */ 1084 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT; 1085 HPTS_LOCK(hpts); 1086 NET_EPOCH_ENTER(et); 1087 slots_ran = tcp_hptsi(hpts, true); 1088 HPTS_UNLOCK(hpts); 1089 NET_EPOCH_EXIT(et); 1090 1091 /* After second cycle, request should be reduced significantly (likely by ~NUM_OF_HPTSI_SLOTS) */ 1092 KTEST_VERIFY(tp2->t_hpts_request < initial_request); 1093 KTEST_VERIFY(tp2->t_hpts_request > 0); /* But not yet zero for such a large request */ 1094 1095 /* Clean up second connection */ 1096 INP_WLOCK(&tp2->t_inpcb); 1097 if (tp2->t_in_hpts != IHPTS_NONE) { 1098 tcp_hpts_remove(pace, tp2); 1099 } 1100 INP_WUNLOCK(&tp2->t_inpcb); 1101 test_hpts_free_tcpcb(tp2); 1102 1103 /* Clean up */ 1104 INP_WLOCK(&tp->t_inpcb); 1105 if (tp->t_in_hpts != IHPTS_NONE) { 1106 tcp_hpts_remove(pace, tp); 1107 } 1108 INP_WUNLOCK(&tp->t_inpcb); 1109 test_hpts_free_tcpcb(tp); 1110 tcp_hptsi_stop(pace); 1111 tcp_hptsi_destroy(pace); 1112 1113 return (0); 1114 } 1115 1116 /* 1117 * Validates CPU assignment and affinity mechanisms, including flowid-based 1118 * assignment, random fallback scenarios, and explicit CPU setting. Tests 1119 * the actual cpu assignment logic in hpts_cpuid via tcp_set_hpts. 1120 */ 1121 KTEST_FUNC(cpu_assignment) 1122 { 1123 struct tcp_hptsi *pace; 1124 struct tcpcb *tp1, *tp2, *tp2_dup, *tp3; 1125 1126 test_hpts_init(); 1127 1128 pace = tcp_hptsi_create(&test_funcs, false); 1129 KTEST_NEQUAL(pace, NULL); 1130 1131 /* Test random CPU assignment (no flowid) */ 1132 tp1 = test_hpts_create_tcpcb(ctx, pace); 1133 KTEST_NEQUAL(tp1, NULL); 1134 tp1->t_inpcb.inp_flowtype = M_HASHTYPE_NONE; 1135 INP_WLOCK(&tp1->t_inpcb); 1136 tcp_set_hpts(pace, tp1); 1137 INP_WUNLOCK(&tp1->t_inpcb); 1138 KTEST_VERIFY(tp1->t_hpts_cpu < pace->rp_num_hptss); 1139 KTEST_VERIFY(tp1->t_flags2 & TF2_HPTS_CPU_SET); 1140 1141 /* Test flowid-based assignment */ 1142 tp2 = test_hpts_create_tcpcb(ctx, pace); 1143 KTEST_NEQUAL(tp2, NULL); 1144 tp2->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4; 1145 tp2->t_inpcb.inp_flowid = 12345; 1146 INP_WLOCK(&tp2->t_inpcb); 1147 tcp_set_hpts(pace, tp2); 1148 INP_WUNLOCK(&tp2->t_inpcb); 1149 KTEST_VERIFY(tp2->t_hpts_cpu < pace->rp_num_hptss); 1150 KTEST_VERIFY(tp2->t_flags2 & TF2_HPTS_CPU_SET); 1151 1152 /* With the same flowid, should get same CPU assignment */ 1153 tp2_dup = test_hpts_create_tcpcb(ctx, pace); 1154 KTEST_NEQUAL(tp2_dup, NULL); 1155 tp2_dup->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4; 1156 tp2_dup->t_inpcb.inp_flowid = 12345; 1157 INP_WLOCK(&tp2_dup->t_inpcb); 1158 tcp_set_hpts(pace, tp2_dup); 1159 INP_WUNLOCK(&tp2_dup->t_inpcb); 1160 KTEST_EQUAL(tp2_dup->t_hpts_cpu, tp2->t_hpts_cpu); 1161 1162 /* Test explicit CPU setting */ 1163 tp3 = test_hpts_create_tcpcb(ctx, pace); 1164 KTEST_NEQUAL(tp3, NULL); 1165 tp3->t_hpts_cpu = 1; /* Assume we have at least 2 CPUs */ 1166 tp3->t_flags2 |= TF2_HPTS_CPU_SET; 1167 INP_WLOCK(&tp3->t_inpcb); 1168 tcp_set_hpts(pace, tp3); 1169 INP_WUNLOCK(&tp3->t_inpcb); 1170 KTEST_EQUAL(tp3->t_hpts_cpu, 1); 1171 1172 test_hpts_free_tcpcb(tp1); 1173 test_hpts_free_tcpcb(tp2); 1174 test_hpts_free_tcpcb(tp2_dup); 1175 test_hpts_free_tcpcb(tp3); 1176 tcp_hptsi_destroy(pace); 1177 1178 return (0); 1179 } 1180 1181 /* 1182 * Validates edge cases in slot calculation including boundary conditions 1183 * around slot 0, maximum slots, and slot wrapping arithmetic. 1184 */ 1185 KTEST_FUNC(slot_boundary_conditions) 1186 { 1187 struct tcp_hptsi *pace; 1188 struct tcpcb *tp; 1189 1190 test_hpts_init(); 1191 1192 pace = tcp_hptsi_create(&test_funcs, false); 1193 KTEST_NEQUAL(pace, NULL); 1194 tcp_hptsi_start(pace); 1195 1196 /* Test insertion at slot 0 */ 1197 tp = test_hpts_create_tcpcb(ctx, pace); 1198 KTEST_NEQUAL(tp, NULL); 1199 INP_WLOCK(&tp->t_inpcb); 1200 tp->t_flags2 |= TF2_HPTS_CALLS; 1201 tcp_hpts_insert(pace, tp, 0, NULL); /* Should insert immediately (0 timeout) */ 1202 INP_WUNLOCK(&tp->t_inpcb); 1203 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 1204 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS); 1205 1206 INP_WLOCK(&tp->t_inpcb); 1207 tcp_hpts_remove(pace, tp); 1208 INP_WUNLOCK(&tp->t_inpcb); 1209 1210 /* Test insertion at maximum slot value */ 1211 INP_WLOCK(&tp->t_inpcb); 1212 tp->t_flags2 |= TF2_HPTS_CALLS; 1213 tcp_hpts_insert(pace, tp, (NUM_OF_HPTSI_SLOTS - 1) * HPTS_USECS_PER_SLOT, NULL); 1214 INP_WUNLOCK(&tp->t_inpcb); 1215 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 1216 1217 INP_WLOCK(&tp->t_inpcb); 1218 tcp_hpts_remove(pace, tp); 1219 INP_WUNLOCK(&tp->t_inpcb); 1220 1221 /* Test very small timeout values */ 1222 INP_WLOCK(&tp->t_inpcb); 1223 tp->t_flags2 |= TF2_HPTS_CALLS; 1224 tcp_hpts_insert(pace, tp, 1, NULL); 1225 INP_WUNLOCK(&tp->t_inpcb); 1226 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); 1227 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(1)); /* Should convert 1 usec to slot */ 1228 1229 INP_WLOCK(&tp->t_inpcb); 1230 tcp_hpts_remove(pace, tp); 1231 INP_WUNLOCK(&tp->t_inpcb); 1232 1233 test_hpts_free_tcpcb(tp); 1234 tcp_hptsi_stop(pace); 1235 tcp_hptsi_destroy(pace); 1236 1237 return (0); 1238 } 1239 1240 /* 1241 * Validates HPTS behavior under high load conditions, including proper 1242 * processing of many connections and connection count tracking. 1243 */ 1244 KTEST_FUNC(dynamic_sleep_adjustment) 1245 { 1246 struct epoch_tracker et; 1247 struct tcp_hptsi *pace; 1248 struct tcpcb **tcpcbs; 1249 struct tcp_hpts_entry *hpts; 1250 uint32_t i, num_tcpcbs = DEFAULT_CONNECTION_THRESHOLD + 50; 1251 int32_t slots_ran; 1252 1253 test_hpts_init(); 1254 1255 pace = tcp_hptsi_create(&test_funcs, false); 1256 KTEST_NEQUAL(pace, NULL); 1257 tcp_hptsi_start(pace); 1258 1259 /* Create many connections to exceed threshold */ 1260 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO); 1261 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM); 1262 1263 for (i = 0; i < num_tcpcbs; i++) { 1264 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace); 1265 KTEST_NEQUAL(tcpcbs[i], NULL); 1266 tcpcbs[i]->t_hpts_cpu = 0; /* Force all to CPU 0 */ 1267 INP_WLOCK(&tcpcbs[i]->t_inpcb); 1268 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS; 1269 TP_REMOVE_FROM_HPTS(tcpcbs[i]) = 1; /* Will be removed after output */ 1270 tcp_hpts_insert(pace, tcpcbs[i], 100, NULL); 1271 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); 1272 } 1273 1274 hpts = pace->rp_ent[0]; 1275 dump_hpts_entry(ctx, hpts); 1276 1277 /* Verify we're above threshold */ 1278 KTEST_GREATER_THAN(hpts->p_on_queue_cnt, DEFAULT_CONNECTION_THRESHOLD); 1279 1280 /* Run HPTS to process many connections */ 1281 test_time_usec += 100; 1282 HPTS_LOCK(hpts); 1283 NET_EPOCH_ENTER(et); 1284 slots_ran = tcp_hptsi(hpts, true); 1285 HPTS_UNLOCK(hpts); 1286 NET_EPOCH_EXIT(et); 1287 1288 /* Verify HPTS processed slots and connections correctly */ 1289 KTEST_GREATER_THAN(slots_ran, 0); 1290 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], num_tcpcbs); 1291 1292 /* Verify all connections were removed from queue */ 1293 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); 1294 1295 /* Cleanup */ 1296 for (i = 0; i < num_tcpcbs; i++) { 1297 test_hpts_free_tcpcb(tcpcbs[i]); 1298 } 1299 free(tcpcbs, M_TCPHPTS); 1300 tcp_hptsi_stop(pace); 1301 tcp_hptsi_destroy(pace); 1302 1303 return (0); 1304 } 1305 1306 /* 1307 * Validates handling of concurrent insert/remove operations and race conditions 1308 * between HPTS processing and user operations. 1309 */ 1310 KTEST_FUNC(concurrent_operations) 1311 { 1312 struct tcp_hptsi *pace; 1313 struct tcpcb *tp1, *tp2; 1314 struct tcp_hpts_entry *hpts; 1315 1316 test_hpts_init(); 1317 1318 pace = tcp_hptsi_create(&test_funcs, false); 1319 KTEST_NEQUAL(pace, NULL); 1320 tcp_hptsi_start(pace); 1321 1322 tp1 = test_hpts_create_tcpcb(ctx, pace); 1323 tp2 = test_hpts_create_tcpcb(ctx, pace); 1324 KTEST_NEQUAL(tp1, NULL); 1325 KTEST_NEQUAL(tp2, NULL); 1326 1327 /* Force all to CPU 0 */ 1328 tp1->t_hpts_cpu = 0; 1329 tp2->t_hpts_cpu = 0; 1330 1331 /* Insert tp1 */ 1332 INP_WLOCK(&tp1->t_inpcb); 1333 tp1->t_flags2 |= TF2_HPTS_CALLS; 1334 tcp_hpts_insert(pace, tp1, 100, NULL); 1335 INP_WUNLOCK(&tp1->t_inpcb); 1336 1337 /* Insert tp2 into same slot */ 1338 INP_WLOCK(&tp2->t_inpcb); 1339 tp2->t_flags2 |= TF2_HPTS_CALLS; 1340 tcp_hpts_insert(pace, tp2, 100, NULL); 1341 INP_WUNLOCK(&tp2->t_inpcb); 1342 1343 /* Verify both are inserted */ 1344 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE); 1345 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); 1346 1347 /* Verify they're both assigned to the same slot */ 1348 KTEST_EQUAL(tp1->t_hpts_slot, tp2->t_hpts_slot); 1349 1350 /* Verify queue count reflects both connections */ 1351 KTEST_EQUAL(tp1->t_hpts_cpu, tp2->t_hpts_cpu); /* Should be on same CPU */ 1352 hpts = pace->rp_ent[tp1->t_hpts_cpu]; 1353 KTEST_EQUAL(hpts->p_on_queue_cnt, 2); 1354 1355 /* Remove tp1 while tp2 is still there */ 1356 INP_WLOCK(&tp1->t_inpcb); 1357 tcp_hpts_remove(pace, tp1); 1358 INP_WUNLOCK(&tp1->t_inpcb); 1359 1360 /* Verify tp1 removed, tp2 still there */ 1361 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE); 1362 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); 1363 1364 /* Verify queue count decreased by one */ 1365 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); 1366 1367 /* Remove tp2 */ 1368 INP_WLOCK(&tp2->t_inpcb); 1369 tcp_hpts_remove(pace, tp2); 1370 INP_WUNLOCK(&tp2->t_inpcb); 1371 1372 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); 1373 1374 /* Verify queue is now completely empty */ 1375 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); 1376 1377 test_hpts_free_tcpcb(tp1); 1378 test_hpts_free_tcpcb(tp2); 1379 tcp_hptsi_stop(pace); 1380 tcp_hptsi_destroy(pace); 1381 1382 return (0); 1383 } 1384 1385 /* 1386 * Validates the queued segments processing path via tfb_do_queued_segments, 1387 * which is an alternative to direct tcp_output calls. 1388 */ 1389 KTEST_FUNC(queued_segments_processing) 1390 { 1391 struct epoch_tracker et; 1392 struct tcp_hptsi *pace; 1393 struct tcpcb *tp; 1394 struct tcp_hpts_entry *hpts; 1395 struct mbuf *fake_mbuf; 1396 int32_t slots_ran; 1397 1398 test_hpts_init(); 1399 1400 pace = tcp_hptsi_create(&test_funcs, false); 1401 KTEST_NEQUAL(pace, NULL); 1402 tcp_hptsi_start(pace); 1403 1404 tp = test_hpts_create_tcpcb(ctx, pace); 1405 KTEST_NEQUAL(tp, NULL); 1406 1407 /* Create a minimal fake mbuf that has valid STAILQ pointers */ 1408 fake_mbuf = malloc(sizeof(struct mbuf), M_TCPHPTS, M_WAITOK | M_ZERO); 1409 KTEST_NEQUAL(fake_mbuf, NULL); 1410 1411 /* Set up for queued segments path */ 1412 tp->t_flags2 |= (TF2_HPTS_CALLS | TF2_SUPPORTS_MBUFQ); 1413 STAILQ_INSERT_TAIL(&tp->t_inqueue, fake_mbuf, m_stailqpkt); 1414 1415 INP_WLOCK(&tp->t_inpcb); 1416 tcp_hpts_insert(pace, tp, 100, NULL); 1417 INP_WUNLOCK(&tp->t_inpcb); 1418 1419 hpts = pace->rp_ent[tp->t_hpts_cpu]; 1420 1421 /* Run HPTS and verify queued segments path is taken */ 1422 test_time_usec += 100; 1423 HPTS_LOCK(hpts); 1424 NET_EPOCH_ENTER(et); 1425 slots_ran = tcp_hptsi(hpts, true); 1426 HPTS_UNLOCK(hpts); 1427 NET_EPOCH_EXIT(et); 1428 1429 KTEST_VERIFY(slots_ran >= 0); 1430 KTEST_EQUAL(call_counts[CCNT_TCP_TFB_DO_QUEUED_SEGMENTS], 1); 1431 1432 /* Connection should be removed from HPTS after processing */ 1433 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); 1434 1435 /* Clean up the fake mbuf if it's still in the queue */ 1436 if (!STAILQ_EMPTY(&tp->t_inqueue)) { 1437 struct mbuf *m = STAILQ_FIRST(&tp->t_inqueue); 1438 STAILQ_REMOVE_HEAD(&tp->t_inqueue, m_stailqpkt); 1439 free(m, M_TCPHPTS); 1440 } 1441 1442 test_hpts_free_tcpcb(tp); 1443 tcp_hptsi_stop(pace); 1444 tcp_hptsi_destroy(pace); 1445 1446 return (0); 1447 } 1448 1449 /* 1450 * Validates the direct wake mechanism and wake inhibition logic when 1451 * the connection count exceeds thresholds. 1452 */ 1453 KTEST_FUNC(direct_wake_mechanism) 1454 { 1455 struct tcp_hptsi *pace; 1456 struct tcpcb *tp; 1457 struct tcp_hpts_entry *hpts; 1458 1459 test_hpts_init(); 1460 1461 pace = tcp_hptsi_create(&test_funcs, false); 1462 KTEST_NEQUAL(pace, NULL); 1463 tcp_hptsi_start(pace); 1464 1465 tp = test_hpts_create_tcpcb(ctx, pace); 1466 KTEST_NEQUAL(tp, NULL); 1467 hpts = pace->rp_ent[tp->t_hpts_cpu]; 1468 1469 /* Test direct wake when not over threshold */ 1470 HPTS_LOCK(hpts); 1471 hpts->p_on_queue_cnt = 50; /* Below threshold */ 1472 hpts->p_hpts_wake_scheduled = 0; 1473 tcp_hpts_wake(hpts); 1474 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 1); 1475 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 1); 1476 HPTS_UNLOCK(hpts); 1477 1478 /* Reset for next test */ 1479 hpts->p_hpts_wake_scheduled = 0; 1480 call_counts[CCNT_SWI_SCHED] = 0; 1481 1482 /* Test wake inhibition when over threshold */ 1483 HPTS_LOCK(hpts); 1484 hpts->p_on_queue_cnt = 200; /* Above threshold */ 1485 hpts->p_direct_wake = 1; /* Request direct wake */ 1486 tcp_hpts_wake(hpts); 1487 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 0); /* Should be inhibited */ 1488 KTEST_EQUAL(hpts->p_direct_wake, 0); /* Should be cleared */ 1489 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 0); /* No SWI scheduled */ 1490 HPTS_UNLOCK(hpts); 1491 1492 test_hpts_free_tcpcb(tp); 1493 tcp_hptsi_stop(pace); 1494 tcp_hptsi_destroy(pace); 1495 1496 return (0); 1497 } 1498 1499 /* 1500 * Validates HPTS collision detection when attempting to run HPTS while 1501 * it's already active. 1502 */ 1503 KTEST_FUNC(hpts_collision_detection) 1504 { 1505 struct epoch_tracker et; 1506 struct tcp_hptsi *pace; 1507 struct tcp_hpts_entry *hpts; 1508 int32_t slots_ran; 1509 1510 test_hpts_init(); 1511 1512 pace = tcp_hptsi_create(&test_funcs, false); 1513 KTEST_NEQUAL(pace, NULL); 1514 tcp_hptsi_start(pace); 1515 1516 hpts = pace->rp_ent[0]; 1517 1518 /* Mark HPTS as active */ 1519 HPTS_LOCK(hpts); 1520 hpts->p_hpts_active = 1; 1521 HPTS_UNLOCK(hpts); 1522 1523 /* Attempt to run HPTS again - should detect collision */ 1524 HPTS_LOCK(hpts); 1525 NET_EPOCH_ENTER(et); 1526 slots_ran = tcp_hptsi(hpts, false); /* from_callout = false */ 1527 HPTS_UNLOCK(hpts); 1528 NET_EPOCH_EXIT(et); 1529 1530 /* Should return 0 indicating no work done due to collision */ 1531 KTEST_EQUAL(slots_ran, 0); 1532 1533 tcp_hptsi_stop(pace); 1534 tcp_hptsi_destroy(pace); 1535 1536 return (0); 1537 } 1538 1539 /* 1540 * Validates generation count handling for race condition detection between 1541 * HPTS processing and connection insertion/removal operations. 1542 */ 1543 KTEST_FUNC(generation_count_validation) 1544 { 1545 struct epoch_tracker et; 1546 struct tcp_hptsi *pace; 1547 struct tcp_hpts_entry *hpts; 1548 struct tcpcb *tp1, *tp2; 1549 uint32_t initial_gencnt, slot_to_test = 10; 1550 uint32_t timeout_usecs = slot_to_test * HPTS_USECS_PER_SLOT; 1551 uint32_t tp2_original_gencnt; 1552 int32_t slots_ran; 1553 1554 test_hpts_init(); 1555 1556 pace = tcp_hptsi_create(&test_funcs, false); 1557 KTEST_NEQUAL(pace, NULL); 1558 tcp_hptsi_start(pace); 1559 1560 hpts = pace->rp_ent[0]; 1561 1562 /* Record initial generation count for the test slot */ 1563 initial_gencnt = hpts->p_hptss[slot_to_test].gencnt; 1564 1565 /* Create and insert first connection */ 1566 tp1 = test_hpts_create_tcpcb(ctx, pace); 1567 KTEST_NEQUAL(tp1, NULL); 1568 tp1->t_hpts_cpu = 0; /* Force to CPU 0 */ 1569 1570 INP_WLOCK(&tp1->t_inpcb); 1571 tp1->t_flags2 |= TF2_HPTS_CALLS; 1572 tcp_hpts_insert(pace, tp1, timeout_usecs, NULL); 1573 INP_WUNLOCK(&tp1->t_inpcb); 1574 1575 /* Verify connection stored the generation count */ 1576 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE); 1577 KTEST_EQUAL(tp1->t_hpts_slot, slot_to_test); 1578 KTEST_EQUAL(tp1->t_hpts_gencnt, initial_gencnt); 1579 1580 /* Create second connection but don't insert yet */ 1581 tp2 = test_hpts_create_tcpcb(ctx, pace); 1582 KTEST_NEQUAL(tp2, NULL); 1583 tp2->t_hpts_cpu = 0; /* Force to CPU 0 */ 1584 1585 /* Force generation count increment by processing the slot */ 1586 test_time_usec += (slot_to_test + 1) * HPTS_USECS_PER_SLOT; 1587 HPTS_LOCK(hpts); 1588 NET_EPOCH_ENTER(et); 1589 slots_ran = tcp_hptsi(hpts, true); 1590 HPTS_UNLOCK(hpts); 1591 NET_EPOCH_EXIT(et); 1592 1593 /* Verify processing occurred */ 1594 KTEST_VERIFY(slots_ran > 0); 1595 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1); 1596 1597 /* Verify generation count was incremented */ 1598 KTEST_EQUAL(hpts->p_hptss[slot_to_test].gencnt, initial_gencnt + 1); 1599 1600 /* Verify first connection was processed and removed */ 1601 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE); 1602 1603 /* Insert second connection and record its generation count */ 1604 INP_WLOCK(&tp2->t_inpcb); 1605 tp2->t_flags2 |= TF2_HPTS_CALLS; 1606 tcp_hpts_insert(pace, tp2, timeout_usecs, NULL); 1607 INP_WUNLOCK(&tp2->t_inpcb); 1608 1609 /* Verify connection was inserted successfully */ 1610 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); 1611 1612 /* Record the generation count that tp2 received */ 1613 tp2_original_gencnt = tp2->t_hpts_gencnt; 1614 1615 /* Test generation count mismatch detection during processing */ 1616 /* Manually set stale generation count to simulate race condition */ 1617 tp2->t_hpts_gencnt = tp2_original_gencnt + 100; /* Force a mismatch */ 1618 1619 /* Process the slot to trigger generation count validation */ 1620 test_time_usec += (slot_to_test + 1) * HPTS_USECS_PER_SLOT; 1621 HPTS_LOCK(hpts); 1622 NET_EPOCH_ENTER(et); 1623 slots_ran = tcp_hptsi(hpts, true); 1624 HPTS_UNLOCK(hpts); 1625 NET_EPOCH_EXIT(et); 1626 1627 /* Connection should be processed despite generation count mismatch */ 1628 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); /* Processed and released */ 1629 1630 /* The key test: HPTS should handle mismatched generation counts gracefully */ 1631 KTEST_VERIFY(slots_ran > 0); /* Processing should still occur */ 1632 1633 test_hpts_free_tcpcb(tp1); 1634 test_hpts_free_tcpcb(tp2); 1635 tcp_hptsi_stop(pace); 1636 tcp_hptsi_destroy(pace); 1637 1638 return (0); 1639 } 1640 1641 static const struct ktest_test_info tests[] = { 1642 KTEST_INFO(module_load), 1643 KTEST_INFO(hptsi_create_destroy), 1644 KTEST_INFO(hptsi_start_stop), 1645 KTEST_INFO(hptsi_independence), 1646 KTEST_INFO(function_injection), 1647 KTEST_INFO(tcpcb_initialization), 1648 KTEST_INFO(tcpcb_insertion), 1649 KTEST_INFO(timer_functionality), 1650 KTEST_INFO(scalability_tcpcbs), 1651 KTEST_INFO(wheel_wrap_recovery), 1652 KTEST_INFO(tcpcb_moving_state), 1653 KTEST_INFO(deferred_requests), 1654 KTEST_INFO(cpu_assignment), 1655 KTEST_INFO(slot_boundary_conditions), 1656 KTEST_INFO(dynamic_sleep_adjustment), 1657 KTEST_INFO(concurrent_operations), 1658 KTEST_INFO(queued_segments_processing), 1659 KTEST_INFO(direct_wake_mechanism), 1660 KTEST_INFO(hpts_collision_detection), 1661 KTEST_INFO(generation_count_validation), 1662 }; 1663 1664 #else /* TCP_HPTS_KTEST */ 1665 1666 /* 1667 * Stub to indicate that the TCP HPTS ktest is not enabled. 1668 */ 1669 KTEST_FUNC(module_load_without_tests) 1670 { 1671 KTEST_LOG(ctx, "Warning: TCP HPTS ktest is not enabled"); 1672 return (0); 1673 } 1674 1675 static const struct ktest_test_info tests[] = { 1676 KTEST_INFO(module_load_without_tests), 1677 }; 1678 1679 #endif 1680 1681 KTEST_MODULE_DECLARE(ktest_tcphpts, tests); 1682 KTEST_MODULE_DEPEND(ktest_tcphpts, tcphpts); 1683