1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Netflix, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <tests/ktest.h>
29 #include "opt_inet.h"
30 #include <sys/param.h>
31 #include <sys/bus.h>
32 #include <sys/interrupt.h>
33 #include <sys/errno.h>
34 #include <sys/malloc.h>
35 #include <sys/mutex.h>
36 #include <sys/refcount.h>
37 #include <sys/socket.h>
38 #include <sys/sysctl.h>
39 #include <sys/systm.h>
40
41 #include <netinet/in.h>
42 #include <netinet/tcp.h>
43 #include <netinet/in_pcb.h>
44 #include <netinet/tcp_seq.h>
45 #include <netinet/tcp_var.h>
46 #include <netinet/tcp_hpts.h>
47 #include <netinet/tcp_hpts_internal.h>
48 #include <dev/tcp_log/tcp_log_dev.h>
49 #include <netinet/tcp_log_buf.h>
50
51 #undef tcp_hpts_init
52 #undef tcp_hpts_remove
53 #undef tcp_hpts_insert
54 #undef tcp_set_hpts
55
56 /* Custom definitions that take the tcp_hptsi */
57 #define tcp_hpts_init(pace, tp) __tcp_hpts_init((pace), (tp))
58 #define tcp_hpts_remove(pace, tp) __tcp_hpts_remove((pace), (tp))
59 #define tcp_hpts_insert(pace, tp, usecs, diag) \
60 __tcp_hpts_insert((pace), (tp), (usecs), (diag))
61 #define tcp_set_hpts(pace, tp) __tcp_set_hpts((pace), (tp))
62
63 static MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts_test", "TCP hpts test");
64
65 static int test_exit_on_failure = true;
66 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts_test, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
67 "TCP HPTS test controls");
68 SYSCTL_INT(_net_inet_tcp_hpts_test, OID_AUTO, exit_on_failure, CTLFLAG_RW,
69 &test_exit_on_failure, 0,
70 "Exit HPTS test immediately on first failure (1) or continue running all tests (0)");
71
72 #define KTEST_VERIFY(x) do { \
73 if (!(x)) { \
74 KTEST_ERR(ctx, "FAIL: %s", #x); \
75 if (test_exit_on_failure) \
76 return (EINVAL); \
77 } else { \
78 KTEST_LOG(ctx, "PASS: %s", #x); \
79 } \
80 } while (0)
81
82 #define KTEST_EQUAL(x, y) do { \
83 if ((x) != (y)) { \
84 KTEST_ERR(ctx, "FAIL: %s != %s (%d != %d)", #x, #y, (x), (y)); \
85 if (test_exit_on_failure) \
86 return (EINVAL); \
87 } else { \
88 KTEST_LOG(ctx, "PASS: %s == %s", #x, #y); \
89 } \
90 } while (0)
91
92 #define KTEST_NEQUAL(x, y) do { \
93 if ((x) == (y)) { \
94 KTEST_ERR(ctx, "FAIL: %s == %s (%d == %d)", #x, #y, (x), (y)); \
95 if (test_exit_on_failure) \
96 return (EINVAL); \
97 } else { \
98 KTEST_LOG(ctx, "PASS: %s != %s", #x, #y); \
99 } \
100 } while (0)
101
102 #define KTEST_GREATER_THAN(x, y) do { \
103 if ((x) <= (y)) { \
104 KTEST_ERR(ctx, "FAIL: %s <= %s (%d <= %d)", #x, #y, (x), (y)); \
105 if (test_exit_on_failure) \
106 return (EINVAL); \
107 } else { \
108 KTEST_LOG(ctx, "PASS: %s > %s", #x, #y); \
109 } \
110 } while (0)
111
112 #define KTEST_VERIFY_RET(x, y) do { \
113 if (!(x)) { \
114 KTEST_ERR(ctx, "FAIL: %s", #x); \
115 if (test_exit_on_failure) \
116 return (y); \
117 } else { \
118 KTEST_LOG(ctx, "PASS: %s", #x); \
119 } \
120 } while (0)
121
122 #ifdef TCP_HPTS_KTEST
123
124 static void
dump_hpts_entry(struct ktest_test_context * ctx,struct tcp_hpts_entry * hpts)125 dump_hpts_entry(struct ktest_test_context *ctx, struct tcp_hpts_entry *hpts)
126 {
127 KTEST_LOG(ctx, "tcp_hpts_entry(%p)", hpts);
128 KTEST_LOG(ctx, " p_cur_slot: %u", hpts->p_cur_slot);
129 KTEST_LOG(ctx, " p_prev_slot: %u", hpts->p_prev_slot);
130 KTEST_LOG(ctx, " p_nxt_slot: %u", hpts->p_nxt_slot);
131 KTEST_LOG(ctx, " p_runningslot: %u", hpts->p_runningslot);
132 KTEST_LOG(ctx, " p_on_queue_cnt: %d", hpts->p_on_queue_cnt);
133 KTEST_LOG(ctx, " p_hpts_active: %u", hpts->p_hpts_active);
134 KTEST_LOG(ctx, " p_wheel_complete: %u", hpts->p_wheel_complete);
135 KTEST_LOG(ctx, " p_direct_wake: %u", hpts->p_direct_wake);
136 KTEST_LOG(ctx, " p_on_min_sleep: %u", hpts->p_on_min_sleep);
137 KTEST_LOG(ctx, " p_hpts_wake_scheduled: %u", hpts->p_hpts_wake_scheduled);
138 KTEST_LOG(ctx, " hit_callout_thresh: %u", hpts->hit_callout_thresh);
139 KTEST_LOG(ctx, " p_hpts_sleep_time: %u", hpts->p_hpts_sleep_time);
140 KTEST_LOG(ctx, " p_delayed_by: %u", hpts->p_delayed_by);
141 KTEST_LOG(ctx, " overidden_sleep: %u", hpts->overidden_sleep);
142 KTEST_LOG(ctx, " saved_curslot: %u", hpts->saved_curslot);
143 KTEST_LOG(ctx, " saved_prev_slot: %u", hpts->saved_prev_slot);
144 KTEST_LOG(ctx, " syscall_cnt: %lu", hpts->syscall_cnt);
145 KTEST_LOG(ctx, " sleeping: %lu", hpts->sleeping);
146 KTEST_LOG(ctx, " p_cpu: %u", hpts->p_cpu);
147 KTEST_LOG(ctx, " ie_cookie: %p", hpts->ie_cookie);
148 KTEST_LOG(ctx, " p_hptsi: %p", hpts->p_hptsi);
149 KTEST_LOG(ctx, " p_mysleep: %ld.%06ld", hpts->p_mysleep.tv_sec, hpts->p_mysleep.tv_usec);
150 }
151
152 static void
dump_tcpcb(struct tcpcb * tp)153 dump_tcpcb(struct tcpcb *tp)
154 {
155 struct ktest_test_context *ctx = tp->t_fb_ptr;
156 struct inpcb *inp = &tp->t_inpcb;
157
158 KTEST_LOG(ctx, "tcp_control_block(%p)", tp);
159
160 /* HPTS-specific fields */
161 KTEST_LOG(ctx, " t_in_hpts: %d", tp->t_in_hpts);
162 KTEST_LOG(ctx, " t_hpts_cpu: %u", tp->t_hpts_cpu);
163 KTEST_LOG(ctx, " t_hpts_slot: %d", tp->t_hpts_slot);
164 KTEST_LOG(ctx, " t_hpts_gencnt: %u", tp->t_hpts_gencnt);
165 KTEST_LOG(ctx, " t_hpts_request: %u", tp->t_hpts_request);
166
167 /* LRO CPU field */
168 KTEST_LOG(ctx, " t_lro_cpu: %u", tp->t_lro_cpu);
169
170 /* TCP flags that affect HPTS */
171 KTEST_LOG(ctx, " t_flags2: 0x%x", tp->t_flags2);
172 KTEST_LOG(ctx, " TF2_HPTS_CPU_SET: %s", (tp->t_flags2 & TF2_HPTS_CPU_SET) ? "YES" : "NO");
173 KTEST_LOG(ctx, " TF2_HPTS_CALLS: %s", (tp->t_flags2 & TF2_HPTS_CALLS) ? "YES" : "NO");
174 KTEST_LOG(ctx, " TF2_SUPPORTS_MBUFQ: %s", (tp->t_flags2 & TF2_SUPPORTS_MBUFQ) ? "YES" : "NO");
175
176 /* Input PCB fields that HPTS uses */
177 KTEST_LOG(ctx, " inp_flags: 0x%x", inp->inp_flags);
178 KTEST_LOG(ctx, " INP_DROPPED: %s", (inp->inp_flags & INP_DROPPED) ? "YES" : "NO");
179 KTEST_LOG(ctx, " inp_flowid: 0x%x", inp->inp_flowid);
180 KTEST_LOG(ctx, " inp_flowtype: %u", inp->inp_flowtype);
181 KTEST_LOG(ctx, " inp_numa_domain: %d", inp->inp_numa_domain);
182 }
183
184 /* Enum for call counting indices */
185 enum test_call_counts {
186 CCNT_MICROUPTIME = 0,
187 CCNT_SWI_ADD,
188 CCNT_SWI_REMOVE,
189 CCNT_SWI_SCHED,
190 CCNT_INTR_EVENT_BIND,
191 CCNT_INTR_EVENT_BIND_CPUSET,
192 CCNT_CALLOUT_INIT,
193 CCNT_CALLOUT_RESET_SBT_ON,
194 CCNT_CALLOUT_STOP_SAFE,
195 CCNT_TCP_OUTPUT,
196 CCNT_TCP_TFB_DO_QUEUED_SEGMENTS,
197 CCNT_MAX
198 };
199
200 static uint32_t call_counts[CCNT_MAX];
201
202 static uint64_t test_time_usec = 0;
203
204 /*
205 * Reset all test global variables to a clean state.
206 */
207 static void
test_hpts_init(void)208 test_hpts_init(void)
209 {
210 memset(call_counts, 0, sizeof(call_counts));
211 test_time_usec = 0;
212 }
213
214 static void
test_microuptime(struct timeval * tv)215 test_microuptime(struct timeval *tv)
216 {
217 call_counts[CCNT_MICROUPTIME]++;
218 tv->tv_sec = test_time_usec / 1000000;
219 tv->tv_usec = test_time_usec % 1000000;
220 }
221
222 static int
test_swi_add(struct intr_event ** eventp,const char * name,driver_intr_t handler,void * arg,int pri,enum intr_type flags,void ** cookiep)223 test_swi_add(struct intr_event **eventp, const char *name,
224 driver_intr_t handler, void *arg, int pri, enum intr_type flags,
225 void **cookiep)
226 {
227 call_counts[CCNT_SWI_ADD]++;
228 /* Simulate successful SWI creation */
229 *eventp = (struct intr_event *)0xfeedface; /* Mock event */
230 *cookiep = (void *)0xdeadbeef; /* Mock cookie */
231 return (0);
232 }
233
234 static int
test_swi_remove(void * cookie)235 test_swi_remove(void *cookie)
236 {
237 call_counts[CCNT_SWI_REMOVE]++;
238 /* Simulate successful removal */
239 return (0);
240 }
241
242 static void
test_swi_sched(void * cookie,int flags)243 test_swi_sched(void *cookie, int flags)
244 {
245 call_counts[CCNT_SWI_SCHED]++;
246 /* Simulate successful SWI scheduling */
247 }
248
249 static int
test_intr_event_bind(struct intr_event * ie,int cpu)250 test_intr_event_bind(struct intr_event *ie, int cpu)
251 {
252 call_counts[CCNT_INTR_EVENT_BIND]++;
253 /* Simulate successful binding */
254 return (0);
255 }
256
257 static int
test_intr_event_bind_ithread_cpuset(struct intr_event * ie,struct _cpuset * mask)258 test_intr_event_bind_ithread_cpuset(struct intr_event *ie, struct _cpuset *mask)
259 {
260 call_counts[CCNT_INTR_EVENT_BIND_CPUSET]++;
261 /* Simulate successful cpuset binding */
262 return (0);
263 }
264
265 static void
test_callout_init(struct callout * c,int mpsafe)266 test_callout_init(struct callout *c, int mpsafe)
267 {
268 call_counts[CCNT_CALLOUT_INIT]++;
269 memset(c, 0, sizeof(*c));
270 }
271
272 static int
test_callout_reset_sbt_on(struct callout * c,sbintime_t sbt,sbintime_t precision,void (* func)(void *),void * arg,int cpu,int flags)273 test_callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t precision,
274 void (*func)(void *), void *arg, int cpu, int flags)
275 {
276 call_counts[CCNT_CALLOUT_RESET_SBT_ON]++;
277 /* Return 1 to simulate successful timer scheduling */
278 return (1);
279 }
280
281 static int
test_callout_stop_safe(struct callout * c,int flags)282 test_callout_stop_safe(struct callout *c, int flags)
283 {
284 call_counts[CCNT_CALLOUT_STOP_SAFE]++;
285 /* Return 1 to simulate successful timer stopping */
286 return (1);
287 }
288
289 static const struct tcp_hptsi_funcs test_funcs = {
290 .microuptime = test_microuptime,
291 .swi_add = test_swi_add,
292 .swi_remove = test_swi_remove,
293 .swi_sched = test_swi_sched,
294 .intr_event_bind = test_intr_event_bind,
295 .intr_event_bind_ithread_cpuset = test_intr_event_bind_ithread_cpuset,
296 .callout_init = test_callout_init,
297 .callout_reset_sbt_on = test_callout_reset_sbt_on,
298 ._callout_stop_safe = test_callout_stop_safe,
299 };
300
301 #define TP_REMOVE_FROM_HPTS(tp) tp->bits_spare
302 #define TP_LOG_TEST(tp) tp->t_log_state_set
303
304 static int
test_tcp_output(struct tcpcb * tp)305 test_tcp_output(struct tcpcb *tp)
306 {
307 struct ktest_test_context *ctx = tp->t_fb_ptr;
308 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending;
309 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu];
310
311 call_counts[CCNT_TCP_OUTPUT]++;
312 if (TP_LOG_TEST(tp)) {
313 KTEST_LOG(ctx, "=> tcp_output(%p)", tp);
314 dump_tcpcb(tp);
315 dump_hpts_entry(ctx, hpts);
316 }
317
318 if ((TP_REMOVE_FROM_HPTS(tp) & 1) != 0) {
319 if (TP_LOG_TEST(tp))
320 KTEST_LOG(ctx, "=> tcp_hpts_remove(%p)", tp);
321 tcp_hpts_remove(pace, tp);
322 }
323
324 if ((TP_REMOVE_FROM_HPTS(tp) & 2) != 0) {
325 INP_WUNLOCK(&tp->t_inpcb); /* tcp_output unlocks on error */
326 return (-1); /* Simulate tcp_output error */
327 }
328
329 return (0);
330 }
331
332 static int
test_tfb_do_queued_segments(struct tcpcb * tp,int flag)333 test_tfb_do_queued_segments(struct tcpcb *tp, int flag)
334 {
335 struct ktest_test_context *ctx = tp->t_fb_ptr;
336 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending;
337 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu];
338
339 call_counts[CCNT_TCP_TFB_DO_QUEUED_SEGMENTS]++;
340 KTEST_LOG(ctx, "=> tfb_do_queued_segments(%p, %d)", tp, flag);
341 dump_tcpcb(tp);
342 dump_hpts_entry(ctx, hpts);
343
344 if ((TP_REMOVE_FROM_HPTS(tp) & 1) != 0) {
345 if (TP_LOG_TEST(tp))
346 KTEST_LOG(ctx, "=> tcp_hpts_remove(%p)", tp);
347 tcp_hpts_remove(pace, tp);
348 }
349
350 if ((TP_REMOVE_FROM_HPTS(tp) & 2) != 0) {
351 INP_WUNLOCK(&tp->t_inpcb); /* do_queued_segments unlocks on error */
352 return (-1); /* Simulate do_queued_segments error */
353 }
354
355 return (0);
356 }
357
358 static struct tcp_function_block test_tcp_fb = {
359 .tfb_tcp_block_name = "hpts_test_tcp",
360 .tfb_tcp_output = test_tcp_output,
361 .tfb_do_queued_segments = test_tfb_do_queued_segments,
362 };
363
364 /*
365 * Create a minimally initialized tcpcb that can be safely inserted into HPTS.
366 * This function allocates and initializes all the fields that HPTS code
367 * reads or writes.
368 */
369 static struct tcpcb *
test_hpts_create_tcpcb(struct ktest_test_context * ctx,struct tcp_hptsi * pace)370 test_hpts_create_tcpcb(struct ktest_test_context *ctx, struct tcp_hptsi *pace)
371 {
372 struct tcpcb *tp;
373
374 tp = malloc(sizeof(struct tcpcb), M_TCPHPTS, M_WAITOK | M_ZERO);
375 if (tp) {
376 rw_init_flags(&tp->t_inpcb.inp_lock, "test-inp",
377 RW_RECURSE | RW_DUPOK);
378 refcount_init(&tp->t_inpcb.inp_refcount, 1);
379 tp->t_inpcb.inp_pcbinfo = &V_tcbinfo;
380 tp->t_fb = &test_tcp_fb;
381 tp->t_hpts_cpu = HPTS_CPU_NONE;
382 STAILQ_INIT(&tp->t_inqueue);
383 tcp_hpts_init(pace, tp);
384
385 /* Stuff some pointers in the tcb for test purposes. */
386 tp->t_fb_ptr = ctx;
387 tp->t_tfo_pending = (unsigned int*)pace;
388 }
389
390 return (tp);
391 }
392
393 /*
394 * Free a test tcpcb created by test_hpts_create_tcpcb()
395 */
396 static void
test_hpts_free_tcpcb(struct tcpcb * tp)397 test_hpts_free_tcpcb(struct tcpcb *tp)
398 {
399 if (tp == NULL)
400 return;
401
402 INP_LOCK_DESTROY(&tp->t_inpcb);
403 free(tp, M_TCPHPTS);
404 }
405
406 /*
407 * ***********************************************
408 * * KTEST functions for testing the HPTS module *
409 * ***********************************************
410 */
411
412 /*
413 * Validates that the HPTS module is properly loaded and initialized by checking
414 * that the minimum HPTS time is configured.
415 */
KTEST_FUNC(module_load)416 KTEST_FUNC(module_load)
417 {
418 test_hpts_init();
419 KTEST_NEQUAL(tcp_min_hptsi_time, 0);
420 KTEST_VERIFY(tcp_bind_threads >= 0 && tcp_bind_threads <= 2);
421 KTEST_NEQUAL(tcp_hptsi_pace, NULL);
422 return (0);
423 }
424
425 /*
426 * Validates the creation and destruction of tcp_hptsi structures, ensuring
427 * proper initialization of internal fields and clean destruction.
428 */
KTEST_FUNC(hptsi_create_destroy)429 KTEST_FUNC(hptsi_create_destroy)
430 {
431 struct tcp_hptsi *pace;
432
433 test_hpts_init();
434
435 pace = tcp_hptsi_create(&test_funcs, false);
436 KTEST_NEQUAL(pace, NULL);
437 KTEST_NEQUAL(pace->rp_ent, NULL);
438 KTEST_NEQUAL(pace->cts_last_ran, NULL);
439 KTEST_VERIFY(pace->rp_num_hptss > 0);
440 KTEST_VERIFY(pace->rp_num_hptss <= MAXCPU); /* Reasonable upper bound */
441 KTEST_VERIFY(pace->grp_cnt >= 1); /* At least one group */
442 KTEST_EQUAL(pace->funcs, &test_funcs); /* Verify function pointer was set */
443
444 /* Verify individual HPTS entries are properly initialized */
445 for (uint32_t i = 0; i < pace->rp_num_hptss; i++) {
446 KTEST_NEQUAL(pace->rp_ent[i], NULL);
447 KTEST_EQUAL(pace->rp_ent[i]->p_cpu, i);
448 KTEST_EQUAL(pace->rp_ent[i]->p_hptsi, pace);
449 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0);
450 }
451
452 tcp_hptsi_destroy(pace);
453
454 return (0);
455 }
456
457 /*
458 * Validates that tcp_hptsi structures can be started and stopped properly,
459 * including verification that threads are created during start and cleaned up
460 * during stop operations.
461 */
KTEST_FUNC(hptsi_start_stop)462 KTEST_FUNC(hptsi_start_stop)
463 {
464 struct tcp_hptsi *pace;
465
466 test_hpts_init();
467
468 pace = tcp_hptsi_create(&test_funcs, false);
469 KTEST_NEQUAL(pace, NULL);
470
471 tcp_hptsi_start(pace);
472
473 /* Verify that entries have threads started */
474 struct tcp_hpts_entry *hpts = pace->rp_ent[0];
475 KTEST_NEQUAL(hpts->ie_cookie, NULL); /* Should have SWI handler */
476 KTEST_EQUAL(hpts->p_hptsi, pace); /* Should point to our pace */
477
478 tcp_hptsi_stop(pace);
479 tcp_hptsi_destroy(pace);
480
481 return (0);
482 }
483
484 /*
485 * Validates that multiple tcp_hptsi instances can coexist independently, with
486 * different configurations and CPU assignments without interfering with each
487 * other.
488 */
KTEST_FUNC(hptsi_independence)489 KTEST_FUNC(hptsi_independence)
490 {
491 struct tcp_hptsi *pace1, *pace2;
492 uint16_t cpu1, cpu2;
493
494 test_hpts_init();
495
496 pace1 = tcp_hptsi_create(&test_funcs, false);
497 pace2 = tcp_hptsi_create(&test_funcs, false);
498 KTEST_NEQUAL(pace1, NULL);
499 KTEST_NEQUAL(pace2, NULL);
500 KTEST_NEQUAL(pace2->rp_ent, NULL);
501
502 cpu1 = tcp_hptsi_random_cpu(pace1);
503 cpu2 = tcp_hptsi_random_cpu(pace2);
504 KTEST_VERIFY(cpu1 < pace1->rp_num_hptss);
505 KTEST_VERIFY(cpu2 < pace2->rp_num_hptss);
506
507 /* Verify both instances have independent entry arrays */
508 KTEST_NEQUAL(pace1->rp_ent, pace2->rp_ent);
509 /* Verify they may have different CPU counts but both reasonable */
510 KTEST_VERIFY(pace1->rp_num_hptss > 0 && pace1->rp_num_hptss <= MAXCPU);
511 KTEST_VERIFY(pace2->rp_num_hptss > 0 && pace2->rp_num_hptss <= MAXCPU);
512
513 tcp_hptsi_destroy(pace1);
514 tcp_hptsi_destroy(pace2);
515
516 return (0);
517 }
518
519 /*
520 * Validates that custom function injection works correctly, ensuring that
521 * test-specific implementations of microuptime and others are properly
522 * called by the HPTS system.
523 */
KTEST_FUNC(function_injection)524 KTEST_FUNC(function_injection)
525 {
526 struct tcp_hptsi *pace;
527
528 test_hpts_init();
529
530 pace = tcp_hptsi_create(&test_funcs, false);
531 KTEST_NEQUAL(pace, NULL);
532 KTEST_EQUAL(pace->funcs, &test_funcs);
533 KTEST_VERIFY(call_counts[CCNT_MICROUPTIME] > 0);
534 KTEST_VERIFY(call_counts[CCNT_CALLOUT_INIT] > 0);
535
536 tcp_hptsi_start(pace);
537 KTEST_VERIFY(call_counts[CCNT_SWI_ADD] > 0);
538 KTEST_VERIFY(tcp_bind_threads == 0 ||
539 call_counts[CCNT_INTR_EVENT_BIND] > 0 ||
540 call_counts[CCNT_INTR_EVENT_BIND_CPUSET] > 0);
541 KTEST_VERIFY(call_counts[CCNT_CALLOUT_RESET_SBT_ON] > 0);
542
543 tcp_hptsi_stop(pace);
544 KTEST_VERIFY(call_counts[CCNT_CALLOUT_STOP_SAFE] > 0);
545 KTEST_VERIFY(call_counts[CCNT_SWI_REMOVE] > 0);
546
547 tcp_hptsi_destroy(pace);
548
549 /* Verify we have a reasonable balance of create/destroy calls */
550 KTEST_EQUAL(call_counts[CCNT_SWI_ADD], call_counts[CCNT_SWI_REMOVE]);
551 KTEST_VERIFY(call_counts[CCNT_CALLOUT_RESET_SBT_ON] <= call_counts[CCNT_CALLOUT_STOP_SAFE]);
552
553 return (0);
554 }
555
556 /*
557 * Validates that a tcpcb can be properly initialized for HPTS compatibility,
558 * ensuring all required fields are set correctly and function pointers are
559 * valid for safe HPTS operations.
560 */
KTEST_FUNC(tcpcb_initialization)561 KTEST_FUNC(tcpcb_initialization)
562 {
563 struct tcp_hptsi *pace;
564 struct tcpcb *tp;
565
566 test_hpts_init();
567
568 pace = tcp_hptsi_create(&test_funcs, false);
569 KTEST_NEQUAL(pace, NULL);
570 tcp_hptsi_start(pace);
571
572 /* Verify the tcpcb is properly initialized for HPTS */
573 tp = test_hpts_create_tcpcb(ctx, pace);
574 KTEST_NEQUAL(tp, NULL);
575 KTEST_NEQUAL(tp->t_fb, NULL);
576 KTEST_NEQUAL(tp->t_fb->tfb_tcp_output, NULL);
577 KTEST_NEQUAL(tp->t_fb->tfb_do_queued_segments, NULL);
578 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
579 KTEST_EQUAL((tp->t_flags2 & (TF2_HPTS_CPU_SET | TF2_HPTS_CALLS)), 0);
580
581 /* Verify that HPTS-specific fields are initialized */
582 KTEST_EQUAL(tp->t_hpts_gencnt, 0);
583 KTEST_EQUAL(tp->t_hpts_slot, 0);
584 KTEST_EQUAL(tp->t_hpts_request, 0);
585 KTEST_EQUAL(tp->t_lro_cpu, 0);
586 KTEST_VERIFY(tp->t_hpts_cpu < pace->rp_num_hptss);
587 KTEST_EQUAL(tp->t_inpcb.inp_refcount, 1);
588 KTEST_VERIFY(!(tp->t_inpcb.inp_flags & INP_DROPPED));
589
590 test_hpts_free_tcpcb(tp);
591 tcp_hptsi_stop(pace);
592 tcp_hptsi_destroy(pace);
593
594 return (0);
595 }
596
597 /*
598 * Validates that tcpcb structures can be successfully inserted into and removed
599 * from the HPTS wheel, with proper state tracking and slot assignment during
600 * the process.
601 */
KTEST_FUNC(tcpcb_insertion)602 KTEST_FUNC(tcpcb_insertion)
603 {
604 struct tcp_hptsi *pace;
605 struct tcpcb *tp;
606 struct tcp_hpts_entry *hpts;
607 uint32_t timeout_usecs = 10;
608
609 test_hpts_init();
610
611 pace = tcp_hptsi_create(&test_funcs, false);
612 KTEST_NEQUAL(pace, NULL);
613 tcp_hptsi_start(pace);
614
615 tp = test_hpts_create_tcpcb(ctx, pace);
616 KTEST_NEQUAL(tp, NULL);
617 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
618 KTEST_EQUAL((tp->t_flags2 & TF2_HPTS_CALLS), 0);
619
620 INP_WLOCK(&tp->t_inpcb);
621 tp->t_flags2 |= TF2_HPTS_CALLS;
622 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 0);
623 tcp_hpts_insert(pace, tp, timeout_usecs, NULL);
624 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
625 INP_WUNLOCK(&tp->t_inpcb);
626 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
627 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 1);
628 KTEST_VERIFY(tcp_in_hpts(tp));
629 KTEST_VERIFY(tp->t_hpts_slot >= 0);
630 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS);
631
632 hpts = pace->rp_ent[tp->t_hpts_cpu];
633 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
634 KTEST_EQUAL(tp->t_hpts_request, 0);
635 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(timeout_usecs));
636 //KTEST_EQUAL(tp->t_hpts_gencnt, 1);
637
638 INP_WLOCK(&tp->t_inpcb);
639 tcp_hpts_remove(pace, tp);
640 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
641 INP_WUNLOCK(&tp->t_inpcb);
642 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
643 KTEST_VERIFY(!tcp_in_hpts(tp));
644
645 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
646
647 test_hpts_free_tcpcb(tp);
648 tcp_hptsi_stop(pace);
649 tcp_hptsi_destroy(pace);
650
651 return (0);
652 }
653
654 /*
655 * Validates the core HPTS timer functionality by verifying that scheduled
656 * tcpcb entries trigger tcp_output calls at appropriate times, simulating
657 * real-world timer-driven TCP processing.
658 */
KTEST_FUNC(timer_functionality)659 KTEST_FUNC(timer_functionality)
660 {
661 struct epoch_tracker et;
662 struct tcp_hptsi *pace;
663 struct tcp_hpts_entry *hpts;
664 struct tcpcb *tp;
665 int32_t slots_ran;
666 uint32_t i;
667
668 test_hpts_init();
669
670 pace = tcp_hptsi_create(&test_funcs, false);
671 KTEST_NEQUAL(pace, NULL);
672 tcp_hptsi_start(pace);
673
674 for (i = 0; i < pace->rp_num_hptss; i++)
675 dump_hpts_entry(ctx, pace->rp_ent[i]);
676
677 /* Create and insert the tcpcb into the HPTS wheel to wait for 500 usec */
678 tp = test_hpts_create_tcpcb(ctx, pace);
679 KTEST_NEQUAL(tp, NULL);
680 dump_tcpcb(tp);
681 TP_LOG_TEST(tp) = 1; /* Enable logging for this tcpcb */
682
683 KTEST_LOG(ctx, "=> tcp_hpts_insert(%p)", tp);
684 INP_WLOCK(&tp->t_inpcb);
685 tp->t_flags2 |= TF2_HPTS_CALLS; /* Mark as needing HPTS processing */
686 tcp_hpts_insert(pace, tp, 500, NULL);
687 INP_WUNLOCK(&tp->t_inpcb);
688
689 dump_tcpcb(tp);
690 for (i = 0; i < pace->rp_num_hptss; i++)
691 dump_hpts_entry(ctx, pace->rp_ent[i]);
692
693 hpts = pace->rp_ent[tp->t_hpts_cpu];
694 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
695 KTEST_EQUAL(hpts->p_prev_slot, 0);
696 KTEST_EQUAL(hpts->p_cur_slot, 0);
697 KTEST_EQUAL(hpts->p_runningslot, 0);
698 KTEST_EQUAL(hpts->p_nxt_slot, 1);
699 KTEST_EQUAL(hpts->p_hpts_active, 0);
700
701 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
702 KTEST_EQUAL(tp->t_hpts_request, 0);
703 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500));
704
705 /* Set our test flag to indicate the tcpcb should be removed from the
706 * wheel when tcp_output is called. */
707 TP_REMOVE_FROM_HPTS(tp) = 1;
708
709 /* Test early exit condition: advance time by insufficient amount */
710 KTEST_LOG(ctx, "Testing early exit with insufficient time advancement");
711 test_time_usec += 1; /* Very small advancement - should cause early exit */
712 HPTS_LOCK(hpts);
713 NET_EPOCH_ENTER(et);
714 slots_ran = tcp_hptsi(hpts, true);
715 HPTS_UNLOCK(hpts);
716 NET_EPOCH_EXIT(et);
717
718 /* Should return 0 slots due to insufficient time advancement */
719 KTEST_EQUAL(slots_ran, 0);
720 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); /* No processing should occur */
721 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); /* Connection still queued */
722
723 /* Wait for 498 more usecs and trigger the HPTS workers and verify
724 * nothing happens yet (total 499 usec) */
725 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
726 test_time_usec += 498;
727 for (i = 0; i < pace->rp_num_hptss; i++) {
728 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]);
729 HPTS_LOCK(pace->rp_ent[i]);
730 NET_EPOCH_ENTER(et);
731 slots_ran = tcp_hptsi(pace->rp_ent[i], true);
732 HPTS_UNLOCK(pace->rp_ent[i]);
733 NET_EPOCH_EXIT(et);
734
735 dump_hpts_entry(ctx, pace->rp_ent[i]);
736 KTEST_VERIFY(slots_ran >= 0);
737 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 49);
738 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 49);
739 }
740
741 dump_tcpcb(tp);
742 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
743 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
744 KTEST_EQUAL(tp->t_hpts_request, 0);
745 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500));
746 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
747
748 /* Wait for 1 more usec and trigger the HPTS workers and verify it
749 * triggers tcp_output this time */
750 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
751 test_time_usec += 1;
752 for (i = 0; i < pace->rp_num_hptss; i++) {
753 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]);
754 HPTS_LOCK(pace->rp_ent[i]);
755 NET_EPOCH_ENTER(et);
756 slots_ran = tcp_hptsi(pace->rp_ent[i], true);
757 HPTS_UNLOCK(pace->rp_ent[i]);
758 NET_EPOCH_EXIT(et);
759
760 dump_hpts_entry(ctx, pace->rp_ent[i]);
761 KTEST_VERIFY(slots_ran >= 0);
762 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 50);
763 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 50);
764 }
765
766 dump_tcpcb(tp);
767 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1);
768 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
769 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
770
771 test_hpts_free_tcpcb(tp);
772 tcp_hptsi_stop(pace);
773 tcp_hptsi_destroy(pace);
774
775 return (0);
776 }
777
778 /*
779 * Validates HPTS scalability by creating and inserting a LOT of tcpcbs into
780 * the HPTS wheel, testing performance under high load conditions.
781 */
KTEST_FUNC(scalability_tcpcbs)782 KTEST_FUNC(scalability_tcpcbs)
783 {
784 struct tcp_hptsi *pace;
785 struct tcpcb **tcpcbs;
786 uint32_t i, num_tcpcbs = 100000, total_queued = 0;
787
788 test_hpts_init();
789
790 pace = tcp_hptsi_create(&test_funcs, false);
791 KTEST_NEQUAL(pace, NULL);
792 tcp_hptsi_start(pace);
793
794 /* Allocate array to hold pointers to all tcpcbs */
795 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO);
796 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM);
797
798 /* Create a LOT of tcpcbs */
799 KTEST_LOG(ctx, "Creating %u tcpcbs...", num_tcpcbs);
800 for (i = 0; i < num_tcpcbs; i++) {
801 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace);
802 if (tcpcbs[i] == NULL) {
803 KTEST_ERR(ctx, "FAIL: tcpcbs[i] == NULL");
804 return (EINVAL);
805 }
806 }
807
808 /* Insert all created tcpcbs into HPTS */
809 KTEST_LOG(ctx, "Inserting all tcpcbs into HPTS...");
810 for (i = 0; i < num_tcpcbs; i++) {
811 INP_WLOCK(&tcpcbs[i]->t_inpcb);
812 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS;
813 /* Insert with varying future timeouts to distribute across slots */
814 tcp_hpts_insert(pace, tcpcbs[i], 100 + (i % 1000), NULL);
815 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
816 }
817
818 /* Verify total queue counts across all CPUs */
819 for (i = 0; i < pace->rp_num_hptss; i++) {
820 total_queued += pace->rp_ent[i]->p_on_queue_cnt;
821 }
822 KTEST_EQUAL(total_queued, num_tcpcbs);
823
824 for (i = 0; i < pace->rp_num_hptss; i++)
825 dump_hpts_entry(ctx, pace->rp_ent[i]);
826
827 /* Remove all tcpcbs from HPTS */
828 KTEST_LOG(ctx, "Removing all tcpcbs from HPTS...");
829 for (i = 0; i < num_tcpcbs; i++) {
830 INP_WLOCK(&tcpcbs[i]->t_inpcb);
831 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) {
832 tcp_hpts_remove(pace, tcpcbs[i]);
833 }
834 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
835 }
836
837 /* Verify all queues are now empty */
838 for (i = 0; i < pace->rp_num_hptss; i++) {
839 if (pace->rp_ent[i]->p_on_queue_cnt != 0) {
840 KTEST_ERR(ctx, "FAIL: pace->rp_ent[i]->p_on_queue_cnt != 0");
841 return (EINVAL);
842 }
843 }
844
845 for (i = 0; i < num_tcpcbs; i++) {
846 test_hpts_free_tcpcb(tcpcbs[i]);
847 }
848 free(tcpcbs, M_TCPHPTS);
849 tcp_hptsi_stop(pace);
850 tcp_hptsi_destroy(pace);
851
852 return (0);
853 }
854
855 /*
856 * Validates wheel wrap scenarios where the timer falls significantly behind
857 * and needs to process more than one full wheel revolution worth of slots.
858 */
KTEST_FUNC(wheel_wrap_recovery)859 KTEST_FUNC(wheel_wrap_recovery)
860 {
861 struct epoch_tracker et;
862 struct tcp_hptsi *pace;
863 struct tcpcb **tcpcbs;
864 uint32_t i, timeout_usecs, num_tcpcbs = 500;
865 int32_t slots_ran;
866
867 test_hpts_init();
868
869 pace = tcp_hptsi_create(&test_funcs, false);
870 KTEST_NEQUAL(pace, NULL);
871 tcp_hptsi_start(pace);
872
873 /* Allocate array to hold pointers to tcpcbs */
874 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO);
875 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM);
876
877 /* Create tcpcbs and insert them across many slots */
878 for (i = 0; i < num_tcpcbs; i++) {
879 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace);
880 KTEST_NEQUAL(tcpcbs[i], NULL);
881 TP_REMOVE_FROM_HPTS(tcpcbs[i]) = 1;
882
883 timeout_usecs = ((i * NUM_OF_HPTSI_SLOTS) / num_tcpcbs) * HPTS_USECS_PER_SLOT; /* Spread across slots */
884
885 INP_WLOCK(&tcpcbs[i]->t_inpcb);
886 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS;
887 tcp_hpts_insert(pace, tcpcbs[i], timeout_usecs, NULL);
888 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
889 }
890
891 /* Fast forward time significantly to trigger wheel wrap */
892 test_time_usec += (NUM_OF_HPTSI_SLOTS + 5000) * HPTS_USECS_PER_SLOT;
893
894 for (i = 0; i < pace->rp_num_hptss; i++) {
895 KTEST_LOG(ctx, "=> tcp_hptsi(%u)", i);
896 KTEST_NEQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0);
897
898 HPTS_LOCK(pace->rp_ent[i]);
899 NET_EPOCH_ENTER(et);
900 slots_ran = tcp_hptsi(pace->rp_ent[i], true);
901 HPTS_UNLOCK(pace->rp_ent[i]);
902 NET_EPOCH_EXIT(et);
903
904 KTEST_EQUAL(slots_ran, NUM_OF_HPTSI_SLOTS-1); /* Should process all slots */
905 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0);
906 KTEST_NEQUAL(pace->rp_ent[i]->p_cur_slot,
907 pace->rp_ent[i]->p_prev_slot);
908 }
909
910 /* Cleanup */
911 for (i = 0; i < num_tcpcbs; i++) {
912 INP_WLOCK(&tcpcbs[i]->t_inpcb);
913 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) {
914 tcp_hpts_remove(pace, tcpcbs[i]);
915 }
916 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
917 test_hpts_free_tcpcb(tcpcbs[i]);
918 }
919 free(tcpcbs, M_TCPHPTS);
920 tcp_hptsi_stop(pace);
921 tcp_hptsi_destroy(pace);
922
923 return (0);
924 }
925
926 /*
927 * Validates proper handling of tcpcbs in the IHPTS_MOVING state, which occurs
928 * when a tcpcb is being processed by the HPTS thread but gets removed.
929 */
KTEST_FUNC(tcpcb_moving_state)930 KTEST_FUNC(tcpcb_moving_state)
931 {
932 struct epoch_tracker et;
933 struct tcp_hptsi *pace;
934 struct tcpcb *tp1, *tp2;
935 struct tcp_hpts_entry *hpts;
936 int32_t slots_ran;
937
938 test_hpts_init();
939
940 pace = tcp_hptsi_create(&test_funcs, false);
941 KTEST_NEQUAL(pace, NULL);
942 tcp_hptsi_start(pace);
943
944 /* Create two tcpcbs on the same CPU/slot */
945 tp1 = test_hpts_create_tcpcb(ctx, pace);
946 tp2 = test_hpts_create_tcpcb(ctx, pace);
947 KTEST_NEQUAL(tp1, NULL);
948 KTEST_NEQUAL(tp2, NULL);
949
950 /* Force them to the same CPU for predictable testing */
951 tp1->t_hpts_cpu = 0;
952 tp2->t_hpts_cpu = 0;
953
954 /* Insert both into the same slot */
955 INP_WLOCK(&tp1->t_inpcb);
956 tp1->t_flags2 |= TF2_HPTS_CALLS;
957 tcp_hpts_insert(pace, tp1, 100, NULL);
958 INP_WUNLOCK(&tp1->t_inpcb);
959
960 INP_WLOCK(&tp2->t_inpcb);
961 tp2->t_flags2 |= TF2_HPTS_CALLS;
962 tcp_hpts_insert(pace, tp2, 100, NULL);
963 INP_WUNLOCK(&tp2->t_inpcb);
964
965 hpts = pace->rp_ent[0];
966
967 /* Manually transition tp1 to MOVING state to simulate race condition */
968 HPTS_LOCK(hpts);
969 tp1->t_in_hpts = IHPTS_MOVING;
970 tp1->t_hpts_slot = -1; /* Mark for removal */
971 HPTS_UNLOCK(hpts);
972
973 /* Set time and run HPTS to process the moving state */
974 test_time_usec += 100;
975 HPTS_LOCK(hpts);
976 NET_EPOCH_ENTER(et);
977 slots_ran = tcp_hptsi(hpts, true);
978 HPTS_UNLOCK(hpts);
979 NET_EPOCH_EXIT(et);
980
981 KTEST_VERIFY(slots_ran >= 0);
982 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1); /* Shouldn't call on both */
983
984 /* tp1 should be cleaned up and removed */
985 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE);
986 /* tp2 should have been processed normally */
987 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE);
988
989 test_hpts_free_tcpcb(tp1);
990 test_hpts_free_tcpcb(tp2);
991 tcp_hptsi_stop(pace);
992 tcp_hptsi_destroy(pace);
993
994 return (0);
995 }
996
997 /*
998 * Validates that tcpcbs with deferred requests (t_hpts_request > 0) are
999 * properly handled and re-inserted into appropriate future slots after
1000 * the wheel processes enough slots to accommodate the original request.
1001 */
KTEST_FUNC(deferred_requests)1002 KTEST_FUNC(deferred_requests)
1003 {
1004 struct epoch_tracker et;
1005 struct tcp_hptsi *pace;
1006 struct tcpcb *tp, *tp2;
1007 struct tcp_hpts_entry *hpts;
1008 uint32_t large_timeout_usecs = (NUM_OF_HPTSI_SLOTS + 5000) * HPTS_USECS_PER_SLOT; /* Beyond wheel capacity */
1009 uint32_t huge_timeout_usecs = (NUM_OF_HPTSI_SLOTS * 3) * HPTS_USECS_PER_SLOT; /* 3x wheel capacity */
1010 uint32_t initial_request;
1011 int32_t slots_ran;
1012
1013 test_hpts_init();
1014
1015 pace = tcp_hptsi_create(&test_funcs, false);
1016 KTEST_NEQUAL(pace, NULL);
1017 tcp_hptsi_start(pace);
1018
1019 tp = test_hpts_create_tcpcb(ctx, pace);
1020 KTEST_NEQUAL(tp, NULL);
1021
1022 /* Insert with a request that exceeds current wheel capacity */
1023 INP_WLOCK(&tp->t_inpcb);
1024 tp->t_flags2 |= TF2_HPTS_CALLS;
1025 tcp_hpts_insert(pace, tp, large_timeout_usecs, NULL);
1026 INP_WUNLOCK(&tp->t_inpcb);
1027
1028 /* Verify it was inserted with a deferred request */
1029 dump_tcpcb(tp);
1030 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1031 KTEST_VERIFY(tp->t_hpts_request > 0);
1032 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS);
1033
1034 hpts = pace->rp_ent[tp->t_hpts_cpu];
1035
1036 /* Advance time to process deferred requests */
1037 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT;
1038
1039 /* Process the wheel to handle deferred requests */
1040 HPTS_LOCK(hpts);
1041 NET_EPOCH_ENTER(et);
1042 slots_ran = tcp_hptsi(hpts, true);
1043 HPTS_UNLOCK(hpts);
1044 NET_EPOCH_EXIT(et);
1045
1046 dump_hpts_entry(ctx, hpts);
1047 KTEST_GREATER_THAN(slots_ran, 0);
1048 dump_tcpcb(tp);
1049 KTEST_EQUAL(tp->t_hpts_request, 0);
1050
1051 /* Test incremental deferred request processing over multiple cycles */
1052 KTEST_LOG(ctx, "Testing incremental deferred request processing");
1053
1054 /* Create a new connection with an even larger request */
1055 tp2 = test_hpts_create_tcpcb(ctx, pace);
1056 KTEST_NEQUAL(tp2, NULL);
1057 tp2->t_hpts_cpu = tp->t_hpts_cpu; /* Same CPU for predictable testing */
1058
1059 INP_WLOCK(&tp2->t_inpcb);
1060 tp2->t_flags2 |= TF2_HPTS_CALLS;
1061 tcp_hpts_insert(pace, tp2, huge_timeout_usecs, NULL);
1062 INP_WUNLOCK(&tp2->t_inpcb);
1063
1064 /* Verify initial deferred request */
1065 initial_request = tp2->t_hpts_request;
1066 KTEST_VERIFY(initial_request > NUM_OF_HPTSI_SLOTS);
1067
1068 /* Process one wheel cycle - should reduce but not eliminate request */
1069 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT;
1070 HPTS_LOCK(hpts);
1071 NET_EPOCH_ENTER(et);
1072 slots_ran = tcp_hptsi(hpts, true);
1073 HPTS_UNLOCK(hpts);
1074 NET_EPOCH_EXIT(et);
1075
1076 /* Request should be reduced but not zero */
1077 KTEST_GREATER_THAN(initial_request, tp2->t_hpts_request);
1078 KTEST_VERIFY(tp2->t_hpts_request > 0);
1079 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); /* Still queued */
1080
1081 /* For huge_timeout_usecs = NUM_OF_HPTSI_SLOTS * 3 * HPTS_USECS_PER_SLOT, we need ~3 cycles to complete.
1082 * Each cycle can reduce the request by at most NUM_OF_HPTSI_SLOTS. */
1083 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT;
1084 HPTS_LOCK(hpts);
1085 NET_EPOCH_ENTER(et);
1086 slots_ran = tcp_hptsi(hpts, true);
1087 HPTS_UNLOCK(hpts);
1088 NET_EPOCH_EXIT(et);
1089
1090 /* After second cycle, request should be reduced significantly (likely by ~NUM_OF_HPTSI_SLOTS) */
1091 KTEST_VERIFY(tp2->t_hpts_request < initial_request);
1092 KTEST_VERIFY(tp2->t_hpts_request > 0); /* But not yet zero for such a large request */
1093
1094 /* Clean up second connection */
1095 INP_WLOCK(&tp2->t_inpcb);
1096 if (tp2->t_in_hpts != IHPTS_NONE) {
1097 tcp_hpts_remove(pace, tp2);
1098 }
1099 INP_WUNLOCK(&tp2->t_inpcb);
1100 test_hpts_free_tcpcb(tp2);
1101
1102 /* Clean up */
1103 INP_WLOCK(&tp->t_inpcb);
1104 if (tp->t_in_hpts != IHPTS_NONE) {
1105 tcp_hpts_remove(pace, tp);
1106 }
1107 INP_WUNLOCK(&tp->t_inpcb);
1108 test_hpts_free_tcpcb(tp);
1109 tcp_hptsi_stop(pace);
1110 tcp_hptsi_destroy(pace);
1111
1112 return (0);
1113 }
1114
1115 /*
1116 * Validates CPU assignment and affinity mechanisms, including flowid-based
1117 * assignment, random fallback scenarios, and explicit CPU setting. Tests
1118 * the actual cpu assignment logic in hpts_cpuid via tcp_set_hpts.
1119 */
KTEST_FUNC(cpu_assignment)1120 KTEST_FUNC(cpu_assignment)
1121 {
1122 struct tcp_hptsi *pace;
1123 struct tcpcb *tp1, *tp2, *tp2_dup, *tp3;
1124
1125 test_hpts_init();
1126
1127 pace = tcp_hptsi_create(&test_funcs, false);
1128 KTEST_NEQUAL(pace, NULL);
1129
1130 /* Test random CPU assignment (no flowid) */
1131 tp1 = test_hpts_create_tcpcb(ctx, pace);
1132 KTEST_NEQUAL(tp1, NULL);
1133 tp1->t_inpcb.inp_flowtype = M_HASHTYPE_NONE;
1134 INP_WLOCK(&tp1->t_inpcb);
1135 tcp_set_hpts(pace, tp1);
1136 INP_WUNLOCK(&tp1->t_inpcb);
1137 KTEST_VERIFY(tp1->t_hpts_cpu < pace->rp_num_hptss);
1138 KTEST_VERIFY(tp1->t_flags2 & TF2_HPTS_CPU_SET);
1139
1140 /* Test flowid-based assignment */
1141 tp2 = test_hpts_create_tcpcb(ctx, pace);
1142 KTEST_NEQUAL(tp2, NULL);
1143 tp2->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4;
1144 tp2->t_inpcb.inp_flowid = 12345;
1145 INP_WLOCK(&tp2->t_inpcb);
1146 tcp_set_hpts(pace, tp2);
1147 INP_WUNLOCK(&tp2->t_inpcb);
1148 KTEST_VERIFY(tp2->t_hpts_cpu < pace->rp_num_hptss);
1149 KTEST_VERIFY(tp2->t_flags2 & TF2_HPTS_CPU_SET);
1150
1151 /* With the same flowid, should get same CPU assignment */
1152 tp2_dup = test_hpts_create_tcpcb(ctx, pace);
1153 KTEST_NEQUAL(tp2_dup, NULL);
1154 tp2_dup->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4;
1155 tp2_dup->t_inpcb.inp_flowid = 12345;
1156 INP_WLOCK(&tp2_dup->t_inpcb);
1157 tcp_set_hpts(pace, tp2_dup);
1158 INP_WUNLOCK(&tp2_dup->t_inpcb);
1159 KTEST_EQUAL(tp2_dup->t_hpts_cpu, tp2->t_hpts_cpu);
1160
1161 /* Test explicit CPU setting */
1162 tp3 = test_hpts_create_tcpcb(ctx, pace);
1163 KTEST_NEQUAL(tp3, NULL);
1164 tp3->t_hpts_cpu = 1; /* Assume we have at least 2 CPUs */
1165 tp3->t_flags2 |= TF2_HPTS_CPU_SET;
1166 INP_WLOCK(&tp3->t_inpcb);
1167 tcp_set_hpts(pace, tp3);
1168 INP_WUNLOCK(&tp3->t_inpcb);
1169 KTEST_EQUAL(tp3->t_hpts_cpu, 1);
1170
1171 test_hpts_free_tcpcb(tp1);
1172 test_hpts_free_tcpcb(tp2);
1173 test_hpts_free_tcpcb(tp2_dup);
1174 test_hpts_free_tcpcb(tp3);
1175 tcp_hptsi_destroy(pace);
1176
1177 return (0);
1178 }
1179
1180 /*
1181 * Validates edge cases in slot calculation including boundary conditions
1182 * around slot 0, maximum slots, and slot wrapping arithmetic.
1183 */
KTEST_FUNC(slot_boundary_conditions)1184 KTEST_FUNC(slot_boundary_conditions)
1185 {
1186 struct tcp_hptsi *pace;
1187 struct tcpcb *tp;
1188
1189 test_hpts_init();
1190
1191 pace = tcp_hptsi_create(&test_funcs, false);
1192 KTEST_NEQUAL(pace, NULL);
1193 tcp_hptsi_start(pace);
1194
1195 /* Test insertion at slot 0 */
1196 tp = test_hpts_create_tcpcb(ctx, pace);
1197 KTEST_NEQUAL(tp, NULL);
1198 INP_WLOCK(&tp->t_inpcb);
1199 tp->t_flags2 |= TF2_HPTS_CALLS;
1200 tcp_hpts_insert(pace, tp, 0, NULL); /* Should insert immediately (0 timeout) */
1201 INP_WUNLOCK(&tp->t_inpcb);
1202 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1203 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS);
1204
1205 INP_WLOCK(&tp->t_inpcb);
1206 tcp_hpts_remove(pace, tp);
1207 INP_WUNLOCK(&tp->t_inpcb);
1208
1209 /* Test insertion at maximum slot value */
1210 INP_WLOCK(&tp->t_inpcb);
1211 tp->t_flags2 |= TF2_HPTS_CALLS;
1212 tcp_hpts_insert(pace, tp, (NUM_OF_HPTSI_SLOTS - 1) * HPTS_USECS_PER_SLOT, NULL);
1213 INP_WUNLOCK(&tp->t_inpcb);
1214 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1215
1216 INP_WLOCK(&tp->t_inpcb);
1217 tcp_hpts_remove(pace, tp);
1218 INP_WUNLOCK(&tp->t_inpcb);
1219
1220 /* Test very small timeout values */
1221 INP_WLOCK(&tp->t_inpcb);
1222 tp->t_flags2 |= TF2_HPTS_CALLS;
1223 tcp_hpts_insert(pace, tp, 1, NULL);
1224 INP_WUNLOCK(&tp->t_inpcb);
1225 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1226 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(1)); /* Should convert 1 usec to slot */
1227
1228 INP_WLOCK(&tp->t_inpcb);
1229 tcp_hpts_remove(pace, tp);
1230 INP_WUNLOCK(&tp->t_inpcb);
1231
1232 test_hpts_free_tcpcb(tp);
1233 tcp_hptsi_stop(pace);
1234 tcp_hptsi_destroy(pace);
1235
1236 return (0);
1237 }
1238
1239 /*
1240 * Validates HPTS behavior under high load conditions, including proper
1241 * processing of many connections and connection count tracking.
1242 */
KTEST_FUNC(dynamic_sleep_adjustment)1243 KTEST_FUNC(dynamic_sleep_adjustment)
1244 {
1245 struct epoch_tracker et;
1246 struct tcp_hptsi *pace;
1247 struct tcpcb **tcpcbs;
1248 struct tcp_hpts_entry *hpts;
1249 uint32_t i, num_tcpcbs = DEFAULT_CONNECTION_THRESHOLD + 50;
1250 int32_t slots_ran;
1251
1252 test_hpts_init();
1253
1254 pace = tcp_hptsi_create(&test_funcs, false);
1255 KTEST_NEQUAL(pace, NULL);
1256 tcp_hptsi_start(pace);
1257
1258 /* Create many connections to exceed threshold */
1259 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO);
1260 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM);
1261
1262 for (i = 0; i < num_tcpcbs; i++) {
1263 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace);
1264 KTEST_NEQUAL(tcpcbs[i], NULL);
1265 tcpcbs[i]->t_hpts_cpu = 0; /* Force all to CPU 0 */
1266 INP_WLOCK(&tcpcbs[i]->t_inpcb);
1267 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS;
1268 TP_REMOVE_FROM_HPTS(tcpcbs[i]) = 1; /* Will be removed after output */
1269 tcp_hpts_insert(pace, tcpcbs[i], 100, NULL);
1270 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
1271 }
1272
1273 hpts = pace->rp_ent[0];
1274 dump_hpts_entry(ctx, hpts);
1275
1276 /* Verify we're above threshold */
1277 KTEST_GREATER_THAN(hpts->p_on_queue_cnt, DEFAULT_CONNECTION_THRESHOLD);
1278
1279 /* Run HPTS to process many connections */
1280 test_time_usec += 100;
1281 HPTS_LOCK(hpts);
1282 NET_EPOCH_ENTER(et);
1283 slots_ran = tcp_hptsi(hpts, true);
1284 HPTS_UNLOCK(hpts);
1285 NET_EPOCH_EXIT(et);
1286
1287 /* Verify HPTS processed slots and connections correctly */
1288 KTEST_GREATER_THAN(slots_ran, 0);
1289 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], num_tcpcbs);
1290
1291 /* Verify all connections were removed from queue */
1292 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
1293
1294 /* Cleanup */
1295 for (i = 0; i < num_tcpcbs; i++) {
1296 test_hpts_free_tcpcb(tcpcbs[i]);
1297 }
1298 free(tcpcbs, M_TCPHPTS);
1299 tcp_hptsi_stop(pace);
1300 tcp_hptsi_destroy(pace);
1301
1302 return (0);
1303 }
1304
1305 /*
1306 * Validates handling of concurrent insert/remove operations and race conditions
1307 * between HPTS processing and user operations.
1308 */
KTEST_FUNC(concurrent_operations)1309 KTEST_FUNC(concurrent_operations)
1310 {
1311 struct tcp_hptsi *pace;
1312 struct tcpcb *tp1, *tp2;
1313 struct tcp_hpts_entry *hpts;
1314
1315 test_hpts_init();
1316
1317 pace = tcp_hptsi_create(&test_funcs, false);
1318 KTEST_NEQUAL(pace, NULL);
1319 tcp_hptsi_start(pace);
1320
1321 tp1 = test_hpts_create_tcpcb(ctx, pace);
1322 tp2 = test_hpts_create_tcpcb(ctx, pace);
1323 KTEST_NEQUAL(tp1, NULL);
1324 KTEST_NEQUAL(tp2, NULL);
1325
1326 /* Force all to CPU 0 */
1327 tp1->t_hpts_cpu = 0;
1328 tp2->t_hpts_cpu = 0;
1329
1330 /* Insert tp1 */
1331 INP_WLOCK(&tp1->t_inpcb);
1332 tp1->t_flags2 |= TF2_HPTS_CALLS;
1333 tcp_hpts_insert(pace, tp1, 100, NULL);
1334 INP_WUNLOCK(&tp1->t_inpcb);
1335
1336 /* Insert tp2 into same slot */
1337 INP_WLOCK(&tp2->t_inpcb);
1338 tp2->t_flags2 |= TF2_HPTS_CALLS;
1339 tcp_hpts_insert(pace, tp2, 100, NULL);
1340 INP_WUNLOCK(&tp2->t_inpcb);
1341
1342 /* Verify both are inserted */
1343 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE);
1344 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE);
1345
1346 /* Verify they're both assigned to the same slot */
1347 KTEST_EQUAL(tp1->t_hpts_slot, tp2->t_hpts_slot);
1348
1349 /* Verify queue count reflects both connections */
1350 KTEST_EQUAL(tp1->t_hpts_cpu, tp2->t_hpts_cpu); /* Should be on same CPU */
1351 hpts = pace->rp_ent[tp1->t_hpts_cpu];
1352 KTEST_EQUAL(hpts->p_on_queue_cnt, 2);
1353
1354 /* Remove tp1 while tp2 is still there */
1355 INP_WLOCK(&tp1->t_inpcb);
1356 tcp_hpts_remove(pace, tp1);
1357 INP_WUNLOCK(&tp1->t_inpcb);
1358
1359 /* Verify tp1 removed, tp2 still there */
1360 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE);
1361 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE);
1362
1363 /* Verify queue count decreased by one */
1364 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
1365
1366 /* Remove tp2 */
1367 INP_WLOCK(&tp2->t_inpcb);
1368 tcp_hpts_remove(pace, tp2);
1369 INP_WUNLOCK(&tp2->t_inpcb);
1370
1371 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE);
1372
1373 /* Verify queue is now completely empty */
1374 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
1375
1376 test_hpts_free_tcpcb(tp1);
1377 test_hpts_free_tcpcb(tp2);
1378 tcp_hptsi_stop(pace);
1379 tcp_hptsi_destroy(pace);
1380
1381 return (0);
1382 }
1383
1384 /*
1385 * Validates the queued segments processing path via tfb_do_queued_segments,
1386 * which is an alternative to direct tcp_output calls.
1387 */
KTEST_FUNC(queued_segments_processing)1388 KTEST_FUNC(queued_segments_processing)
1389 {
1390 struct epoch_tracker et;
1391 struct tcp_hptsi *pace;
1392 struct tcpcb *tp;
1393 struct tcp_hpts_entry *hpts;
1394 struct mbuf *fake_mbuf;
1395 int32_t slots_ran;
1396
1397 test_hpts_init();
1398
1399 pace = tcp_hptsi_create(&test_funcs, false);
1400 KTEST_NEQUAL(pace, NULL);
1401 tcp_hptsi_start(pace);
1402
1403 tp = test_hpts_create_tcpcb(ctx, pace);
1404 KTEST_NEQUAL(tp, NULL);
1405
1406 /* Create a minimal fake mbuf that has valid STAILQ pointers */
1407 fake_mbuf = malloc(sizeof(struct mbuf), M_TCPHPTS, M_WAITOK | M_ZERO);
1408 KTEST_NEQUAL(fake_mbuf, NULL);
1409
1410 /* Set up for queued segments path */
1411 tp->t_flags2 |= (TF2_HPTS_CALLS | TF2_SUPPORTS_MBUFQ);
1412 STAILQ_INSERT_TAIL(&tp->t_inqueue, fake_mbuf, m_stailqpkt);
1413
1414 INP_WLOCK(&tp->t_inpcb);
1415 tcp_hpts_insert(pace, tp, 100, NULL);
1416 INP_WUNLOCK(&tp->t_inpcb);
1417
1418 hpts = pace->rp_ent[tp->t_hpts_cpu];
1419
1420 /* Run HPTS and verify queued segments path is taken */
1421 test_time_usec += 100;
1422 HPTS_LOCK(hpts);
1423 NET_EPOCH_ENTER(et);
1424 slots_ran = tcp_hptsi(hpts, true);
1425 HPTS_UNLOCK(hpts);
1426 NET_EPOCH_EXIT(et);
1427
1428 KTEST_VERIFY(slots_ran >= 0);
1429 KTEST_EQUAL(call_counts[CCNT_TCP_TFB_DO_QUEUED_SEGMENTS], 1);
1430
1431 /* Connection should be removed from HPTS after processing */
1432 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
1433
1434 /* Clean up the fake mbuf if it's still in the queue */
1435 if (!STAILQ_EMPTY(&tp->t_inqueue)) {
1436 struct mbuf *m = STAILQ_FIRST(&tp->t_inqueue);
1437 STAILQ_REMOVE_HEAD(&tp->t_inqueue, m_stailqpkt);
1438 free(m, M_TCPHPTS);
1439 }
1440
1441 test_hpts_free_tcpcb(tp);
1442 tcp_hptsi_stop(pace);
1443 tcp_hptsi_destroy(pace);
1444
1445 return (0);
1446 }
1447
1448 /*
1449 * Validates the direct wake mechanism and wake inhibition logic when
1450 * the connection count exceeds thresholds.
1451 */
KTEST_FUNC(direct_wake_mechanism)1452 KTEST_FUNC(direct_wake_mechanism)
1453 {
1454 struct tcp_hptsi *pace;
1455 struct tcpcb *tp;
1456 struct tcp_hpts_entry *hpts;
1457
1458 test_hpts_init();
1459
1460 pace = tcp_hptsi_create(&test_funcs, false);
1461 KTEST_NEQUAL(pace, NULL);
1462 tcp_hptsi_start(pace);
1463
1464 tp = test_hpts_create_tcpcb(ctx, pace);
1465 KTEST_NEQUAL(tp, NULL);
1466 hpts = pace->rp_ent[tp->t_hpts_cpu];
1467
1468 /* Test direct wake when not over threshold */
1469 HPTS_LOCK(hpts);
1470 hpts->p_on_queue_cnt = 50; /* Below threshold */
1471 hpts->p_hpts_wake_scheduled = 0;
1472 tcp_hpts_wake(hpts);
1473 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 1);
1474 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 1);
1475 HPTS_UNLOCK(hpts);
1476
1477 /* Reset for next test */
1478 hpts->p_hpts_wake_scheduled = 0;
1479 call_counts[CCNT_SWI_SCHED] = 0;
1480
1481 /* Test wake inhibition when over threshold */
1482 HPTS_LOCK(hpts);
1483 hpts->p_on_queue_cnt = 200; /* Above threshold */
1484 hpts->p_direct_wake = 1; /* Request direct wake */
1485 tcp_hpts_wake(hpts);
1486 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 0); /* Should be inhibited */
1487 KTEST_EQUAL(hpts->p_direct_wake, 0); /* Should be cleared */
1488 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 0); /* No SWI scheduled */
1489 HPTS_UNLOCK(hpts);
1490
1491 test_hpts_free_tcpcb(tp);
1492 tcp_hptsi_stop(pace);
1493 tcp_hptsi_destroy(pace);
1494
1495 return (0);
1496 }
1497
1498 /*
1499 * Validates HPTS collision detection when attempting to run HPTS while
1500 * it's already active.
1501 */
KTEST_FUNC(hpts_collision_detection)1502 KTEST_FUNC(hpts_collision_detection)
1503 {
1504 struct epoch_tracker et;
1505 struct tcp_hptsi *pace;
1506 struct tcp_hpts_entry *hpts;
1507 int32_t slots_ran;
1508
1509 test_hpts_init();
1510
1511 pace = tcp_hptsi_create(&test_funcs, false);
1512 KTEST_NEQUAL(pace, NULL);
1513 tcp_hptsi_start(pace);
1514
1515 hpts = pace->rp_ent[0];
1516
1517 /* Mark HPTS as active */
1518 HPTS_LOCK(hpts);
1519 hpts->p_hpts_active = 1;
1520 HPTS_UNLOCK(hpts);
1521
1522 /* Attempt to run HPTS again - should detect collision */
1523 HPTS_LOCK(hpts);
1524 NET_EPOCH_ENTER(et);
1525 slots_ran = tcp_hptsi(hpts, false); /* from_callout = false */
1526 HPTS_UNLOCK(hpts);
1527 NET_EPOCH_EXIT(et);
1528
1529 /* Should return 0 indicating no work done due to collision */
1530 KTEST_EQUAL(slots_ran, 0);
1531
1532 tcp_hptsi_stop(pace);
1533 tcp_hptsi_destroy(pace);
1534
1535 return (0);
1536 }
1537
1538 /*
1539 * Validates generation count handling for race condition detection between
1540 * HPTS processing and connection insertion/removal operations.
1541 */
KTEST_FUNC(generation_count_validation)1542 KTEST_FUNC(generation_count_validation)
1543 {
1544 struct epoch_tracker et;
1545 struct tcp_hptsi *pace;
1546 struct tcp_hpts_entry *hpts;
1547 struct tcpcb *tp1, *tp2;
1548 uint32_t initial_gencnt, slot_to_test = 10;
1549 uint32_t timeout_usecs = slot_to_test * HPTS_USECS_PER_SLOT;
1550 uint32_t tp2_original_gencnt;
1551 int32_t slots_ran;
1552
1553 test_hpts_init();
1554
1555 pace = tcp_hptsi_create(&test_funcs, false);
1556 KTEST_NEQUAL(pace, NULL);
1557 tcp_hptsi_start(pace);
1558
1559 hpts = pace->rp_ent[0];
1560
1561 /* Record initial generation count for the test slot */
1562 initial_gencnt = hpts->p_hptss[slot_to_test].gencnt;
1563
1564 /* Create and insert first connection */
1565 tp1 = test_hpts_create_tcpcb(ctx, pace);
1566 KTEST_NEQUAL(tp1, NULL);
1567 tp1->t_hpts_cpu = 0; /* Force to CPU 0 */
1568
1569 INP_WLOCK(&tp1->t_inpcb);
1570 tp1->t_flags2 |= TF2_HPTS_CALLS;
1571 tcp_hpts_insert(pace, tp1, timeout_usecs, NULL);
1572 INP_WUNLOCK(&tp1->t_inpcb);
1573
1574 /* Verify connection stored the generation count */
1575 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE);
1576 KTEST_EQUAL(tp1->t_hpts_slot, slot_to_test);
1577 KTEST_EQUAL(tp1->t_hpts_gencnt, initial_gencnt);
1578
1579 /* Create second connection but don't insert yet */
1580 tp2 = test_hpts_create_tcpcb(ctx, pace);
1581 KTEST_NEQUAL(tp2, NULL);
1582 tp2->t_hpts_cpu = 0; /* Force to CPU 0 */
1583
1584 /* Force generation count increment by processing the slot */
1585 test_time_usec += (slot_to_test + 1) * HPTS_USECS_PER_SLOT;
1586 HPTS_LOCK(hpts);
1587 NET_EPOCH_ENTER(et);
1588 slots_ran = tcp_hptsi(hpts, true);
1589 HPTS_UNLOCK(hpts);
1590 NET_EPOCH_EXIT(et);
1591
1592 /* Verify processing occurred */
1593 KTEST_VERIFY(slots_ran > 0);
1594 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1);
1595
1596 /* Verify generation count was incremented */
1597 KTEST_EQUAL(hpts->p_hptss[slot_to_test].gencnt, initial_gencnt + 1);
1598
1599 /* Verify first connection was processed and removed */
1600 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE);
1601
1602 /* Insert second connection and record its generation count */
1603 INP_WLOCK(&tp2->t_inpcb);
1604 tp2->t_flags2 |= TF2_HPTS_CALLS;
1605 tcp_hpts_insert(pace, tp2, timeout_usecs, NULL);
1606 INP_WUNLOCK(&tp2->t_inpcb);
1607
1608 /* Verify connection was inserted successfully */
1609 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE);
1610
1611 /* Record the generation count that tp2 received */
1612 tp2_original_gencnt = tp2->t_hpts_gencnt;
1613
1614 /* Test generation count mismatch detection during processing */
1615 /* Manually set stale generation count to simulate race condition */
1616 tp2->t_hpts_gencnt = tp2_original_gencnt + 100; /* Force a mismatch */
1617
1618 /* Process the slot to trigger generation count validation */
1619 test_time_usec += (slot_to_test + 1) * HPTS_USECS_PER_SLOT;
1620 HPTS_LOCK(hpts);
1621 NET_EPOCH_ENTER(et);
1622 slots_ran = tcp_hptsi(hpts, true);
1623 HPTS_UNLOCK(hpts);
1624 NET_EPOCH_EXIT(et);
1625
1626 /* Connection should be processed despite generation count mismatch */
1627 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); /* Processed and released */
1628
1629 /* The key test: HPTS should handle mismatched generation counts gracefully */
1630 KTEST_VERIFY(slots_ran > 0); /* Processing should still occur */
1631
1632 test_hpts_free_tcpcb(tp1);
1633 test_hpts_free_tcpcb(tp2);
1634 tcp_hptsi_stop(pace);
1635 tcp_hptsi_destroy(pace);
1636
1637 return (0);
1638 }
1639
1640 static const struct ktest_test_info tests[] = {
1641 KTEST_INFO(module_load),
1642 KTEST_INFO(hptsi_create_destroy),
1643 KTEST_INFO(hptsi_start_stop),
1644 KTEST_INFO(hptsi_independence),
1645 KTEST_INFO(function_injection),
1646 KTEST_INFO(tcpcb_initialization),
1647 KTEST_INFO(tcpcb_insertion),
1648 KTEST_INFO(timer_functionality),
1649 KTEST_INFO(scalability_tcpcbs),
1650 KTEST_INFO(wheel_wrap_recovery),
1651 KTEST_INFO(tcpcb_moving_state),
1652 KTEST_INFO(deferred_requests),
1653 KTEST_INFO(cpu_assignment),
1654 KTEST_INFO(slot_boundary_conditions),
1655 KTEST_INFO(dynamic_sleep_adjustment),
1656 KTEST_INFO(concurrent_operations),
1657 KTEST_INFO(queued_segments_processing),
1658 KTEST_INFO(direct_wake_mechanism),
1659 KTEST_INFO(hpts_collision_detection),
1660 KTEST_INFO(generation_count_validation),
1661 };
1662
1663 #else /* TCP_HPTS_KTEST */
1664
1665 /*
1666 * Stub to indicate that the TCP HPTS ktest is not enabled.
1667 */
KTEST_FUNC(module_load_without_tests)1668 KTEST_FUNC(module_load_without_tests)
1669 {
1670 KTEST_LOG(ctx, "Warning: TCP HPTS ktest is not enabled");
1671 return (0);
1672 }
1673
1674 static const struct ktest_test_info tests[] = {
1675 KTEST_INFO(module_load_without_tests),
1676 };
1677
1678 #endif
1679
1680 KTEST_MODULE_DECLARE(ktest_tcphpts, tests);
1681 KTEST_MODULE_DEPEND(ktest_tcphpts, tcphpts);
1682