1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Netflix, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <tests/ktest.h>
29 #include <sys/cdefs.h>
30 #include "opt_inet.h"
31 #include <sys/param.h>
32 #include <sys/bus.h>
33 #include <sys/interrupt.h>
34 #include <sys/errno.h>
35 #include <sys/malloc.h>
36 #include <sys/mutex.h>
37 #include <sys/refcount.h>
38 #include <sys/socket.h>
39 #include <sys/sysctl.h>
40 #include <sys/systm.h>
41
42 #include <netinet/in.h>
43 #include <netinet/tcp.h>
44 #include <netinet/in_pcb.h>
45 #include <netinet/tcp_seq.h>
46 #include <netinet/tcp_var.h>
47 #include <netinet/tcp_hpts.h>
48 #include <netinet/tcp_hpts_internal.h>
49 #include <dev/tcp_log/tcp_log_dev.h>
50 #include <netinet/tcp_log_buf.h>
51
52 #undef tcp_hpts_init
53 #undef tcp_hpts_remove
54 #undef tcp_hpts_insert
55 #undef tcp_set_hpts
56
57 /* Custom definitions that take the tcp_hptsi */
58 #define tcp_hpts_init(pace, tp) __tcp_hpts_init((pace), (tp))
59 #define tcp_hpts_remove(pace, tp) __tcp_hpts_remove((pace), (tp))
60 #define tcp_hpts_insert(pace, tp, usecs, diag) \
61 __tcp_hpts_insert((pace), (tp), (usecs), (diag))
62 #define tcp_set_hpts(pace, tp) __tcp_set_hpts((pace), (tp))
63
64 static MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts_test", "TCP hpts test");
65
66 static int test_exit_on_failure = true;
67 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts_test, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
68 "TCP HPTS test controls");
69 SYSCTL_INT(_net_inet_tcp_hpts_test, OID_AUTO, exit_on_failure, CTLFLAG_RW,
70 &test_exit_on_failure, 0,
71 "Exit HPTS test immediately on first failure (1) or continue running all tests (0)");
72
73 #define KTEST_VERIFY(x) do { \
74 if (!(x)) { \
75 KTEST_ERR(ctx, "FAIL: %s", #x); \
76 if (test_exit_on_failure) \
77 return (EINVAL); \
78 } else { \
79 KTEST_LOG(ctx, "PASS: %s", #x); \
80 } \
81 } while (0)
82
83 #define KTEST_EQUAL(x, y) do { \
84 if ((x) != (y)) { \
85 KTEST_ERR(ctx, "FAIL: %s != %s (%d != %d)", #x, #y, (x), (y)); \
86 if (test_exit_on_failure) \
87 return (EINVAL); \
88 } else { \
89 KTEST_LOG(ctx, "PASS: %s == %s", #x, #y); \
90 } \
91 } while (0)
92
93 #define KTEST_NEQUAL(x, y) do { \
94 if ((x) == (y)) { \
95 KTEST_ERR(ctx, "FAIL: %s == %s (%d == %d)", #x, #y, (x), (y)); \
96 if (test_exit_on_failure) \
97 return (EINVAL); \
98 } else { \
99 KTEST_LOG(ctx, "PASS: %s != %s", #x, #y); \
100 } \
101 } while (0)
102
103 #define KTEST_GREATER_THAN(x, y) do { \
104 if ((x) <= (y)) { \
105 KTEST_ERR(ctx, "FAIL: %s <= %s (%d <= %d)", #x, #y, (x), (y)); \
106 if (test_exit_on_failure) \
107 return (EINVAL); \
108 } else { \
109 KTEST_LOG(ctx, "PASS: %s > %s", #x, #y); \
110 } \
111 } while (0)
112
113 #define KTEST_VERIFY_RET(x, y) do { \
114 if (!(x)) { \
115 KTEST_ERR(ctx, "FAIL: %s", #x); \
116 if (test_exit_on_failure) \
117 return (y); \
118 } else { \
119 KTEST_LOG(ctx, "PASS: %s", #x); \
120 } \
121 } while (0)
122
123 #ifdef TCP_HPTS_KTEST
124
125 static void
dump_hpts_entry(struct ktest_test_context * ctx,struct tcp_hpts_entry * hpts)126 dump_hpts_entry(struct ktest_test_context *ctx, struct tcp_hpts_entry *hpts)
127 {
128 KTEST_LOG(ctx, "tcp_hpts_entry(%p)", hpts);
129 KTEST_LOG(ctx, " p_cur_slot: %u", hpts->p_cur_slot);
130 KTEST_LOG(ctx, " p_prev_slot: %u", hpts->p_prev_slot);
131 KTEST_LOG(ctx, " p_nxt_slot: %u", hpts->p_nxt_slot);
132 KTEST_LOG(ctx, " p_runningslot: %u", hpts->p_runningslot);
133 KTEST_LOG(ctx, " p_on_queue_cnt: %d", hpts->p_on_queue_cnt);
134 KTEST_LOG(ctx, " p_hpts_active: %u", hpts->p_hpts_active);
135 KTEST_LOG(ctx, " p_wheel_complete: %u", hpts->p_wheel_complete);
136 KTEST_LOG(ctx, " p_direct_wake: %u", hpts->p_direct_wake);
137 KTEST_LOG(ctx, " p_on_min_sleep: %u", hpts->p_on_min_sleep);
138 KTEST_LOG(ctx, " p_hpts_wake_scheduled: %u", hpts->p_hpts_wake_scheduled);
139 KTEST_LOG(ctx, " hit_callout_thresh: %u", hpts->hit_callout_thresh);
140 KTEST_LOG(ctx, " p_hpts_sleep_time: %u", hpts->p_hpts_sleep_time);
141 KTEST_LOG(ctx, " p_delayed_by: %u", hpts->p_delayed_by);
142 KTEST_LOG(ctx, " overidden_sleep: %u", hpts->overidden_sleep);
143 KTEST_LOG(ctx, " saved_curslot: %u", hpts->saved_curslot);
144 KTEST_LOG(ctx, " saved_prev_slot: %u", hpts->saved_prev_slot);
145 KTEST_LOG(ctx, " syscall_cnt: %lu", hpts->syscall_cnt);
146 KTEST_LOG(ctx, " sleeping: %lu", hpts->sleeping);
147 KTEST_LOG(ctx, " p_cpu: %u", hpts->p_cpu);
148 KTEST_LOG(ctx, " ie_cookie: %p", hpts->ie_cookie);
149 KTEST_LOG(ctx, " p_hptsi: %p", hpts->p_hptsi);
150 KTEST_LOG(ctx, " p_mysleep: %ld.%06ld", hpts->p_mysleep.tv_sec, hpts->p_mysleep.tv_usec);
151 }
152
153 static void
dump_tcpcb(struct tcpcb * tp)154 dump_tcpcb(struct tcpcb *tp)
155 {
156 struct ktest_test_context *ctx = tp->t_fb_ptr;
157 struct inpcb *inp = &tp->t_inpcb;
158
159 KTEST_LOG(ctx, "tcp_control_block(%p)", tp);
160
161 /* HPTS-specific fields */
162 KTEST_LOG(ctx, " t_in_hpts: %d", tp->t_in_hpts);
163 KTEST_LOG(ctx, " t_hpts_cpu: %u", tp->t_hpts_cpu);
164 KTEST_LOG(ctx, " t_hpts_slot: %d", tp->t_hpts_slot);
165 KTEST_LOG(ctx, " t_hpts_gencnt: %u", tp->t_hpts_gencnt);
166 KTEST_LOG(ctx, " t_hpts_request: %u", tp->t_hpts_request);
167
168 /* LRO CPU field */
169 KTEST_LOG(ctx, " t_lro_cpu: %u", tp->t_lro_cpu);
170
171 /* TCP flags that affect HPTS */
172 KTEST_LOG(ctx, " t_flags2: 0x%x", tp->t_flags2);
173 KTEST_LOG(ctx, " TF2_HPTS_CPU_SET: %s", (tp->t_flags2 & TF2_HPTS_CPU_SET) ? "YES" : "NO");
174 KTEST_LOG(ctx, " TF2_HPTS_CALLS: %s", (tp->t_flags2 & TF2_HPTS_CALLS) ? "YES" : "NO");
175 KTEST_LOG(ctx, " TF2_SUPPORTS_MBUFQ: %s", (tp->t_flags2 & TF2_SUPPORTS_MBUFQ) ? "YES" : "NO");
176
177 /* Input PCB fields that HPTS uses */
178 KTEST_LOG(ctx, " inp_flags: 0x%x", inp->inp_flags);
179 KTEST_LOG(ctx, " INP_DROPPED: %s", (inp->inp_flags & INP_DROPPED) ? "YES" : "NO");
180 KTEST_LOG(ctx, " inp_flowid: 0x%x", inp->inp_flowid);
181 KTEST_LOG(ctx, " inp_flowtype: %u", inp->inp_flowtype);
182 KTEST_LOG(ctx, " inp_numa_domain: %d", inp->inp_numa_domain);
183 }
184
185 /* Enum for call counting indices */
186 enum test_call_counts {
187 CCNT_MICROUPTIME = 0,
188 CCNT_SWI_ADD,
189 CCNT_SWI_REMOVE,
190 CCNT_SWI_SCHED,
191 CCNT_INTR_EVENT_BIND,
192 CCNT_INTR_EVENT_BIND_CPUSET,
193 CCNT_CALLOUT_INIT,
194 CCNT_CALLOUT_RESET_SBT_ON,
195 CCNT_CALLOUT_STOP_SAFE,
196 CCNT_TCP_OUTPUT,
197 CCNT_TCP_TFB_DO_QUEUED_SEGMENTS,
198 CCNT_MAX
199 };
200
201 static uint32_t call_counts[CCNT_MAX];
202
203 static uint64_t test_time_usec = 0;
204
205 /*
206 * Reset all test global variables to a clean state.
207 */
208 static void
test_hpts_init(void)209 test_hpts_init(void)
210 {
211 memset(call_counts, 0, sizeof(call_counts));
212 test_time_usec = 0;
213 }
214
215 static void
test_microuptime(struct timeval * tv)216 test_microuptime(struct timeval *tv)
217 {
218 call_counts[CCNT_MICROUPTIME]++;
219 tv->tv_sec = test_time_usec / 1000000;
220 tv->tv_usec = test_time_usec % 1000000;
221 }
222
223 static int
test_swi_add(struct intr_event ** eventp,const char * name,driver_intr_t handler,void * arg,int pri,enum intr_type flags,void ** cookiep)224 test_swi_add(struct intr_event **eventp, const char *name,
225 driver_intr_t handler, void *arg, int pri, enum intr_type flags,
226 void **cookiep)
227 {
228 call_counts[CCNT_SWI_ADD]++;
229 /* Simulate successful SWI creation */
230 *eventp = (struct intr_event *)0xfeedface; /* Mock event */
231 *cookiep = (void *)0xdeadbeef; /* Mock cookie */
232 return (0);
233 }
234
235 static int
test_swi_remove(void * cookie)236 test_swi_remove(void *cookie)
237 {
238 call_counts[CCNT_SWI_REMOVE]++;
239 /* Simulate successful removal */
240 return (0);
241 }
242
243 static void
test_swi_sched(void * cookie,int flags)244 test_swi_sched(void *cookie, int flags)
245 {
246 call_counts[CCNT_SWI_SCHED]++;
247 /* Simulate successful SWI scheduling */
248 }
249
250 static int
test_intr_event_bind(struct intr_event * ie,int cpu)251 test_intr_event_bind(struct intr_event *ie, int cpu)
252 {
253 call_counts[CCNT_INTR_EVENT_BIND]++;
254 /* Simulate successful binding */
255 return (0);
256 }
257
258 static int
test_intr_event_bind_ithread_cpuset(struct intr_event * ie,struct _cpuset * mask)259 test_intr_event_bind_ithread_cpuset(struct intr_event *ie, struct _cpuset *mask)
260 {
261 call_counts[CCNT_INTR_EVENT_BIND_CPUSET]++;
262 /* Simulate successful cpuset binding */
263 return (0);
264 }
265
266 static void
test_callout_init(struct callout * c,int mpsafe)267 test_callout_init(struct callout *c, int mpsafe)
268 {
269 call_counts[CCNT_CALLOUT_INIT]++;
270 memset(c, 0, sizeof(*c));
271 }
272
273 static int
test_callout_reset_sbt_on(struct callout * c,sbintime_t sbt,sbintime_t precision,void (* func)(void *),void * arg,int cpu,int flags)274 test_callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t precision,
275 void (*func)(void *), void *arg, int cpu, int flags)
276 {
277 call_counts[CCNT_CALLOUT_RESET_SBT_ON]++;
278 /* Return 1 to simulate successful timer scheduling */
279 return (1);
280 }
281
282 static int
test_callout_stop_safe(struct callout * c,int flags)283 test_callout_stop_safe(struct callout *c, int flags)
284 {
285 call_counts[CCNT_CALLOUT_STOP_SAFE]++;
286 /* Return 1 to simulate successful timer stopping */
287 return (1);
288 }
289
290 static const struct tcp_hptsi_funcs test_funcs = {
291 .microuptime = test_microuptime,
292 .swi_add = test_swi_add,
293 .swi_remove = test_swi_remove,
294 .swi_sched = test_swi_sched,
295 .intr_event_bind = test_intr_event_bind,
296 .intr_event_bind_ithread_cpuset = test_intr_event_bind_ithread_cpuset,
297 .callout_init = test_callout_init,
298 .callout_reset_sbt_on = test_callout_reset_sbt_on,
299 ._callout_stop_safe = test_callout_stop_safe,
300 };
301
302 #define TP_REMOVE_FROM_HPTS(tp) tp->bits_spare
303 #define TP_LOG_TEST(tp) tp->t_log_state_set
304
305 static int
test_tcp_output(struct tcpcb * tp)306 test_tcp_output(struct tcpcb *tp)
307 {
308 struct ktest_test_context *ctx = tp->t_fb_ptr;
309 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending;
310 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu];
311
312 call_counts[CCNT_TCP_OUTPUT]++;
313 if (TP_LOG_TEST(tp)) {
314 KTEST_LOG(ctx, "=> tcp_output(%p)", tp);
315 dump_tcpcb(tp);
316 dump_hpts_entry(ctx, hpts);
317 }
318
319 if ((TP_REMOVE_FROM_HPTS(tp) & 1) != 0) {
320 if (TP_LOG_TEST(tp))
321 KTEST_LOG(ctx, "=> tcp_hpts_remove(%p)", tp);
322 tcp_hpts_remove(pace, tp);
323 }
324
325 if ((TP_REMOVE_FROM_HPTS(tp) & 2) != 0) {
326 INP_WUNLOCK(&tp->t_inpcb); /* tcp_output unlocks on error */
327 return (-1); /* Simulate tcp_output error */
328 }
329
330 return (0);
331 }
332
333 static int
test_tfb_do_queued_segments(struct tcpcb * tp,int flag)334 test_tfb_do_queued_segments(struct tcpcb *tp, int flag)
335 {
336 struct ktest_test_context *ctx = tp->t_fb_ptr;
337 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending;
338 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu];
339
340 call_counts[CCNT_TCP_TFB_DO_QUEUED_SEGMENTS]++;
341 KTEST_LOG(ctx, "=> tfb_do_queued_segments(%p, %d)", tp, flag);
342 dump_tcpcb(tp);
343 dump_hpts_entry(ctx, hpts);
344
345 if ((TP_REMOVE_FROM_HPTS(tp) & 1) != 0) {
346 if (TP_LOG_TEST(tp))
347 KTEST_LOG(ctx, "=> tcp_hpts_remove(%p)", tp);
348 tcp_hpts_remove(pace, tp);
349 }
350
351 if ((TP_REMOVE_FROM_HPTS(tp) & 2) != 0) {
352 INP_WUNLOCK(&tp->t_inpcb); /* do_queued_segments unlocks on error */
353 return (-1); /* Simulate do_queued_segments error */
354 }
355
356 return (0);
357 }
358
359 static struct tcp_function_block test_tcp_fb = {
360 .tfb_tcp_block_name = "hpts_test_tcp",
361 .tfb_tcp_output = test_tcp_output,
362 .tfb_do_queued_segments = test_tfb_do_queued_segments,
363 };
364
365 /*
366 * Create a minimally initialized tcpcb that can be safely inserted into HPTS.
367 * This function allocates and initializes all the fields that HPTS code
368 * reads or writes.
369 */
370 static struct tcpcb *
test_hpts_create_tcpcb(struct ktest_test_context * ctx,struct tcp_hptsi * pace)371 test_hpts_create_tcpcb(struct ktest_test_context *ctx, struct tcp_hptsi *pace)
372 {
373 struct tcpcb *tp;
374
375 tp = malloc(sizeof(struct tcpcb), M_TCPHPTS, M_WAITOK | M_ZERO);
376 if (tp) {
377 rw_init_flags(&tp->t_inpcb.inp_lock, "test-inp",
378 RW_RECURSE | RW_DUPOK);
379 refcount_init(&tp->t_inpcb.inp_refcount, 1);
380 tp->t_inpcb.inp_pcbinfo = &V_tcbinfo;
381 tp->t_fb = &test_tcp_fb;
382 tp->t_hpts_cpu = HPTS_CPU_NONE;
383 STAILQ_INIT(&tp->t_inqueue);
384 tcp_hpts_init(pace, tp);
385
386 /* Stuff some pointers in the tcb for test purposes. */
387 tp->t_fb_ptr = ctx;
388 tp->t_tfo_pending = (unsigned int*)pace;
389 }
390
391 return (tp);
392 }
393
394 /*
395 * Free a test tcpcb created by test_hpts_create_tcpcb()
396 */
397 static void
test_hpts_free_tcpcb(struct tcpcb * tp)398 test_hpts_free_tcpcb(struct tcpcb *tp)
399 {
400 if (tp == NULL)
401 return;
402
403 INP_LOCK_DESTROY(&tp->t_inpcb);
404 free(tp, M_TCPHPTS);
405 }
406
407 /*
408 * ***********************************************
409 * * KTEST functions for testing the HPTS module *
410 * ***********************************************
411 */
412
413 /*
414 * Validates that the HPTS module is properly loaded and initialized by checking
415 * that the minimum HPTS time is configured.
416 */
KTEST_FUNC(module_load)417 KTEST_FUNC(module_load)
418 {
419 test_hpts_init();
420 KTEST_NEQUAL(tcp_min_hptsi_time, 0);
421 KTEST_VERIFY(tcp_bind_threads >= 0 && tcp_bind_threads <= 2);
422 KTEST_NEQUAL(tcp_hptsi_pace, NULL);
423 return (0);
424 }
425
426 /*
427 * Validates the creation and destruction of tcp_hptsi structures, ensuring
428 * proper initialization of internal fields and clean destruction.
429 */
KTEST_FUNC(hptsi_create_destroy)430 KTEST_FUNC(hptsi_create_destroy)
431 {
432 struct tcp_hptsi *pace;
433
434 test_hpts_init();
435
436 pace = tcp_hptsi_create(&test_funcs, false);
437 KTEST_NEQUAL(pace, NULL);
438 KTEST_NEQUAL(pace->rp_ent, NULL);
439 KTEST_NEQUAL(pace->cts_last_ran, NULL);
440 KTEST_VERIFY(pace->rp_num_hptss > 0);
441 KTEST_VERIFY(pace->rp_num_hptss <= MAXCPU); /* Reasonable upper bound */
442 KTEST_VERIFY(pace->grp_cnt >= 1); /* At least one group */
443 KTEST_EQUAL(pace->funcs, &test_funcs); /* Verify function pointer was set */
444
445 /* Verify individual HPTS entries are properly initialized */
446 for (uint32_t i = 0; i < pace->rp_num_hptss; i++) {
447 KTEST_NEQUAL(pace->rp_ent[i], NULL);
448 KTEST_EQUAL(pace->rp_ent[i]->p_cpu, i);
449 KTEST_EQUAL(pace->rp_ent[i]->p_hptsi, pace);
450 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0);
451 }
452
453 tcp_hptsi_destroy(pace);
454
455 return (0);
456 }
457
458 /*
459 * Validates that tcp_hptsi structures can be started and stopped properly,
460 * including verification that threads are created during start and cleaned up
461 * during stop operations.
462 */
KTEST_FUNC(hptsi_start_stop)463 KTEST_FUNC(hptsi_start_stop)
464 {
465 struct tcp_hptsi *pace;
466
467 test_hpts_init();
468
469 pace = tcp_hptsi_create(&test_funcs, false);
470 KTEST_NEQUAL(pace, NULL);
471
472 tcp_hptsi_start(pace);
473
474 /* Verify that entries have threads started */
475 struct tcp_hpts_entry *hpts = pace->rp_ent[0];
476 KTEST_NEQUAL(hpts->ie_cookie, NULL); /* Should have SWI handler */
477 KTEST_EQUAL(hpts->p_hptsi, pace); /* Should point to our pace */
478
479 tcp_hptsi_stop(pace);
480 tcp_hptsi_destroy(pace);
481
482 return (0);
483 }
484
485 /*
486 * Validates that multiple tcp_hptsi instances can coexist independently, with
487 * different configurations and CPU assignments without interfering with each
488 * other.
489 */
KTEST_FUNC(hptsi_independence)490 KTEST_FUNC(hptsi_independence)
491 {
492 struct tcp_hptsi *pace1, *pace2;
493 uint16_t cpu1, cpu2;
494
495 test_hpts_init();
496
497 pace1 = tcp_hptsi_create(&test_funcs, false);
498 pace2 = tcp_hptsi_create(&test_funcs, false);
499 KTEST_NEQUAL(pace1, NULL);
500 KTEST_NEQUAL(pace2, NULL);
501 KTEST_NEQUAL(pace2->rp_ent, NULL);
502
503 cpu1 = tcp_hptsi_random_cpu(pace1);
504 cpu2 = tcp_hptsi_random_cpu(pace2);
505 KTEST_VERIFY(cpu1 < pace1->rp_num_hptss);
506 KTEST_VERIFY(cpu2 < pace2->rp_num_hptss);
507
508 /* Verify both instances have independent entry arrays */
509 KTEST_NEQUAL(pace1->rp_ent, pace2->rp_ent);
510 /* Verify they may have different CPU counts but both reasonable */
511 KTEST_VERIFY(pace1->rp_num_hptss > 0 && pace1->rp_num_hptss <= MAXCPU);
512 KTEST_VERIFY(pace2->rp_num_hptss > 0 && pace2->rp_num_hptss <= MAXCPU);
513
514 tcp_hptsi_destroy(pace1);
515 tcp_hptsi_destroy(pace2);
516
517 return (0);
518 }
519
520 /*
521 * Validates that custom function injection works correctly, ensuring that
522 * test-specific implementations of microuptime and others are properly
523 * called by the HPTS system.
524 */
KTEST_FUNC(function_injection)525 KTEST_FUNC(function_injection)
526 {
527 struct tcp_hptsi *pace;
528
529 test_hpts_init();
530
531 pace = tcp_hptsi_create(&test_funcs, false);
532 KTEST_NEQUAL(pace, NULL);
533 KTEST_EQUAL(pace->funcs, &test_funcs);
534 KTEST_VERIFY(call_counts[CCNT_MICROUPTIME] > 0);
535 KTEST_VERIFY(call_counts[CCNT_CALLOUT_INIT] > 0);
536
537 tcp_hptsi_start(pace);
538 KTEST_VERIFY(call_counts[CCNT_SWI_ADD] > 0);
539 KTEST_VERIFY(tcp_bind_threads == 0 ||
540 call_counts[CCNT_INTR_EVENT_BIND] > 0 ||
541 call_counts[CCNT_INTR_EVENT_BIND_CPUSET] > 0);
542 KTEST_VERIFY(call_counts[CCNT_CALLOUT_RESET_SBT_ON] > 0);
543
544 tcp_hptsi_stop(pace);
545 KTEST_VERIFY(call_counts[CCNT_CALLOUT_STOP_SAFE] > 0);
546 KTEST_VERIFY(call_counts[CCNT_SWI_REMOVE] > 0);
547
548 tcp_hptsi_destroy(pace);
549
550 /* Verify we have a reasonable balance of create/destroy calls */
551 KTEST_EQUAL(call_counts[CCNT_SWI_ADD], call_counts[CCNT_SWI_REMOVE]);
552 KTEST_VERIFY(call_counts[CCNT_CALLOUT_RESET_SBT_ON] <= call_counts[CCNT_CALLOUT_STOP_SAFE]);
553
554 return (0);
555 }
556
557 /*
558 * Validates that a tcpcb can be properly initialized for HPTS compatibility,
559 * ensuring all required fields are set correctly and function pointers are
560 * valid for safe HPTS operations.
561 */
KTEST_FUNC(tcpcb_initialization)562 KTEST_FUNC(tcpcb_initialization)
563 {
564 struct tcp_hptsi *pace;
565 struct tcpcb *tp;
566
567 test_hpts_init();
568
569 pace = tcp_hptsi_create(&test_funcs, false);
570 KTEST_NEQUAL(pace, NULL);
571 tcp_hptsi_start(pace);
572
573 /* Verify the tcpcb is properly initialized for HPTS */
574 tp = test_hpts_create_tcpcb(ctx, pace);
575 KTEST_NEQUAL(tp, NULL);
576 KTEST_NEQUAL(tp->t_fb, NULL);
577 KTEST_NEQUAL(tp->t_fb->tfb_tcp_output, NULL);
578 KTEST_NEQUAL(tp->t_fb->tfb_do_queued_segments, NULL);
579 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
580 KTEST_EQUAL((tp->t_flags2 & (TF2_HPTS_CPU_SET | TF2_HPTS_CALLS)), 0);
581
582 /* Verify that HPTS-specific fields are initialized */
583 KTEST_EQUAL(tp->t_hpts_gencnt, 0);
584 KTEST_EQUAL(tp->t_hpts_slot, 0);
585 KTEST_EQUAL(tp->t_hpts_request, 0);
586 KTEST_EQUAL(tp->t_lro_cpu, 0);
587 KTEST_VERIFY(tp->t_hpts_cpu < pace->rp_num_hptss);
588 KTEST_EQUAL(tp->t_inpcb.inp_refcount, 1);
589 KTEST_VERIFY(!(tp->t_inpcb.inp_flags & INP_DROPPED));
590
591 test_hpts_free_tcpcb(tp);
592 tcp_hptsi_stop(pace);
593 tcp_hptsi_destroy(pace);
594
595 return (0);
596 }
597
598 /*
599 * Validates that tcpcb structures can be successfully inserted into and removed
600 * from the HPTS wheel, with proper state tracking and slot assignment during
601 * the process.
602 */
KTEST_FUNC(tcpcb_insertion)603 KTEST_FUNC(tcpcb_insertion)
604 {
605 struct tcp_hptsi *pace;
606 struct tcpcb *tp;
607 struct tcp_hpts_entry *hpts;
608 uint32_t timeout_usecs = 10;
609
610 test_hpts_init();
611
612 pace = tcp_hptsi_create(&test_funcs, false);
613 KTEST_NEQUAL(pace, NULL);
614 tcp_hptsi_start(pace);
615
616 tp = test_hpts_create_tcpcb(ctx, pace);
617 KTEST_NEQUAL(tp, NULL);
618 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
619 KTEST_EQUAL((tp->t_flags2 & TF2_HPTS_CALLS), 0);
620
621 INP_WLOCK(&tp->t_inpcb);
622 tp->t_flags2 |= TF2_HPTS_CALLS;
623 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 0);
624 tcp_hpts_insert(pace, tp, timeout_usecs, NULL);
625 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
626 INP_WUNLOCK(&tp->t_inpcb);
627 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
628 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 1);
629 KTEST_VERIFY(tcp_in_hpts(tp));
630 KTEST_VERIFY(tp->t_hpts_slot >= 0);
631 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS);
632
633 hpts = pace->rp_ent[tp->t_hpts_cpu];
634 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
635 KTEST_EQUAL(tp->t_hpts_request, 0);
636 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(timeout_usecs));
637 //KTEST_EQUAL(tp->t_hpts_gencnt, 1);
638
639 INP_WLOCK(&tp->t_inpcb);
640 tcp_hpts_remove(pace, tp);
641 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
642 INP_WUNLOCK(&tp->t_inpcb);
643 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
644 KTEST_VERIFY(!tcp_in_hpts(tp));
645
646 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
647
648 test_hpts_free_tcpcb(tp);
649 tcp_hptsi_stop(pace);
650 tcp_hptsi_destroy(pace);
651
652 return (0);
653 }
654
655 /*
656 * Validates the core HPTS timer functionality by verifying that scheduled
657 * tcpcb entries trigger tcp_output calls at appropriate times, simulating
658 * real-world timer-driven TCP processing.
659 */
KTEST_FUNC(timer_functionality)660 KTEST_FUNC(timer_functionality)
661 {
662 struct epoch_tracker et;
663 struct tcp_hptsi *pace;
664 struct tcp_hpts_entry *hpts;
665 struct tcpcb *tp;
666 int32_t slots_ran;
667 uint32_t i;
668
669 test_hpts_init();
670
671 pace = tcp_hptsi_create(&test_funcs, false);
672 KTEST_NEQUAL(pace, NULL);
673 tcp_hptsi_start(pace);
674
675 for (i = 0; i < pace->rp_num_hptss; i++)
676 dump_hpts_entry(ctx, pace->rp_ent[i]);
677
678 /* Create and insert the tcpcb into the HPTS wheel to wait for 500 usec */
679 tp = test_hpts_create_tcpcb(ctx, pace);
680 KTEST_NEQUAL(tp, NULL);
681 dump_tcpcb(tp);
682 TP_LOG_TEST(tp) = 1; /* Enable logging for this tcpcb */
683
684 KTEST_LOG(ctx, "=> tcp_hpts_insert(%p)", tp);
685 INP_WLOCK(&tp->t_inpcb);
686 tp->t_flags2 |= TF2_HPTS_CALLS; /* Mark as needing HPTS processing */
687 tcp_hpts_insert(pace, tp, 500, NULL);
688 INP_WUNLOCK(&tp->t_inpcb);
689
690 dump_tcpcb(tp);
691 for (i = 0; i < pace->rp_num_hptss; i++)
692 dump_hpts_entry(ctx, pace->rp_ent[i]);
693
694 hpts = pace->rp_ent[tp->t_hpts_cpu];
695 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
696 KTEST_EQUAL(hpts->p_prev_slot, 0);
697 KTEST_EQUAL(hpts->p_cur_slot, 0);
698 KTEST_EQUAL(hpts->p_runningslot, 0);
699 KTEST_EQUAL(hpts->p_nxt_slot, 1);
700 KTEST_EQUAL(hpts->p_hpts_active, 0);
701
702 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
703 KTEST_EQUAL(tp->t_hpts_request, 0);
704 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500));
705
706 /* Set our test flag to indicate the tcpcb should be removed from the
707 * wheel when tcp_output is called. */
708 TP_REMOVE_FROM_HPTS(tp) = 1;
709
710 /* Test early exit condition: advance time by insufficient amount */
711 KTEST_LOG(ctx, "Testing early exit with insufficient time advancement");
712 test_time_usec += 1; /* Very small advancement - should cause early exit */
713 HPTS_LOCK(hpts);
714 NET_EPOCH_ENTER(et);
715 slots_ran = tcp_hptsi(hpts, true);
716 HPTS_UNLOCK(hpts);
717 NET_EPOCH_EXIT(et);
718
719 /* Should return 0 slots due to insufficient time advancement */
720 KTEST_EQUAL(slots_ran, 0);
721 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0); /* No processing should occur */
722 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); /* Connection still queued */
723
724 /* Wait for 498 more usecs and trigger the HPTS workers and verify
725 * nothing happens yet (total 499 usec) */
726 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
727 test_time_usec += 498;
728 for (i = 0; i < pace->rp_num_hptss; i++) {
729 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]);
730 HPTS_LOCK(pace->rp_ent[i]);
731 NET_EPOCH_ENTER(et);
732 slots_ran = tcp_hptsi(pace->rp_ent[i], true);
733 HPTS_UNLOCK(pace->rp_ent[i]);
734 NET_EPOCH_EXIT(et);
735
736 dump_hpts_entry(ctx, pace->rp_ent[i]);
737 KTEST_VERIFY(slots_ran >= 0);
738 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 49);
739 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 49);
740 }
741
742 dump_tcpcb(tp);
743 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
744 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
745 KTEST_EQUAL(tp->t_hpts_request, 0);
746 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500));
747 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
748
749 /* Wait for 1 more usec and trigger the HPTS workers and verify it
750 * triggers tcp_output this time */
751 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 0);
752 test_time_usec += 1;
753 for (i = 0; i < pace->rp_num_hptss; i++) {
754 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]);
755 HPTS_LOCK(pace->rp_ent[i]);
756 NET_EPOCH_ENTER(et);
757 slots_ran = tcp_hptsi(pace->rp_ent[i], true);
758 HPTS_UNLOCK(pace->rp_ent[i]);
759 NET_EPOCH_EXIT(et);
760
761 dump_hpts_entry(ctx, pace->rp_ent[i]);
762 KTEST_VERIFY(slots_ran >= 0);
763 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 50);
764 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 50);
765 }
766
767 dump_tcpcb(tp);
768 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1);
769 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
770 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
771
772 test_hpts_free_tcpcb(tp);
773 tcp_hptsi_stop(pace);
774 tcp_hptsi_destroy(pace);
775
776 return (0);
777 }
778
779 /*
780 * Validates HPTS scalability by creating and inserting a LOT of tcpcbs into
781 * the HPTS wheel, testing performance under high load conditions.
782 */
KTEST_FUNC(scalability_tcpcbs)783 KTEST_FUNC(scalability_tcpcbs)
784 {
785 struct tcp_hptsi *pace;
786 struct tcpcb **tcpcbs;
787 uint32_t i, num_tcpcbs = 100000, total_queued = 0;
788
789 test_hpts_init();
790
791 pace = tcp_hptsi_create(&test_funcs, false);
792 KTEST_NEQUAL(pace, NULL);
793 tcp_hptsi_start(pace);
794
795 /* Allocate array to hold pointers to all tcpcbs */
796 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO);
797 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM);
798
799 /* Create a LOT of tcpcbs */
800 KTEST_LOG(ctx, "Creating %u tcpcbs...", num_tcpcbs);
801 for (i = 0; i < num_tcpcbs; i++) {
802 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace);
803 if (tcpcbs[i] == NULL) {
804 KTEST_ERR(ctx, "FAIL: tcpcbs[i] == NULL");
805 return (EINVAL);
806 }
807 }
808
809 /* Insert all created tcpcbs into HPTS */
810 KTEST_LOG(ctx, "Inserting all tcpcbs into HPTS...");
811 for (i = 0; i < num_tcpcbs; i++) {
812 INP_WLOCK(&tcpcbs[i]->t_inpcb);
813 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS;
814 /* Insert with varying future timeouts to distribute across slots */
815 tcp_hpts_insert(pace, tcpcbs[i], 100 + (i % 1000), NULL);
816 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
817 }
818
819 /* Verify total queue counts across all CPUs */
820 for (i = 0; i < pace->rp_num_hptss; i++) {
821 total_queued += pace->rp_ent[i]->p_on_queue_cnt;
822 }
823 KTEST_EQUAL(total_queued, num_tcpcbs);
824
825 for (i = 0; i < pace->rp_num_hptss; i++)
826 dump_hpts_entry(ctx, pace->rp_ent[i]);
827
828 /* Remove all tcpcbs from HPTS */
829 KTEST_LOG(ctx, "Removing all tcpcbs from HPTS...");
830 for (i = 0; i < num_tcpcbs; i++) {
831 INP_WLOCK(&tcpcbs[i]->t_inpcb);
832 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) {
833 tcp_hpts_remove(pace, tcpcbs[i]);
834 }
835 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
836 }
837
838 /* Verify all queues are now empty */
839 for (i = 0; i < pace->rp_num_hptss; i++) {
840 if (pace->rp_ent[i]->p_on_queue_cnt != 0) {
841 KTEST_ERR(ctx, "FAIL: pace->rp_ent[i]->p_on_queue_cnt != 0");
842 return (EINVAL);
843 }
844 }
845
846 for (i = 0; i < num_tcpcbs; i++) {
847 test_hpts_free_tcpcb(tcpcbs[i]);
848 }
849 free(tcpcbs, M_TCPHPTS);
850 tcp_hptsi_stop(pace);
851 tcp_hptsi_destroy(pace);
852
853 return (0);
854 }
855
856 /*
857 * Validates wheel wrap scenarios where the timer falls significantly behind
858 * and needs to process more than one full wheel revolution worth of slots.
859 */
KTEST_FUNC(wheel_wrap_recovery)860 KTEST_FUNC(wheel_wrap_recovery)
861 {
862 struct epoch_tracker et;
863 struct tcp_hptsi *pace;
864 struct tcpcb **tcpcbs;
865 uint32_t i, timeout_usecs, num_tcpcbs = 500;
866 int32_t slots_ran;
867
868 test_hpts_init();
869
870 pace = tcp_hptsi_create(&test_funcs, false);
871 KTEST_NEQUAL(pace, NULL);
872 tcp_hptsi_start(pace);
873
874 /* Allocate array to hold pointers to tcpcbs */
875 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO);
876 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM);
877
878 /* Create tcpcbs and insert them across many slots */
879 for (i = 0; i < num_tcpcbs; i++) {
880 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace);
881 KTEST_NEQUAL(tcpcbs[i], NULL);
882 TP_REMOVE_FROM_HPTS(tcpcbs[i]) = 1;
883
884 timeout_usecs = ((i * NUM_OF_HPTSI_SLOTS) / num_tcpcbs) * HPTS_USECS_PER_SLOT; /* Spread across slots */
885
886 INP_WLOCK(&tcpcbs[i]->t_inpcb);
887 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS;
888 tcp_hpts_insert(pace, tcpcbs[i], timeout_usecs, NULL);
889 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
890 }
891
892 /* Fast forward time significantly to trigger wheel wrap */
893 test_time_usec += (NUM_OF_HPTSI_SLOTS + 5000) * HPTS_USECS_PER_SLOT;
894
895 for (i = 0; i < pace->rp_num_hptss; i++) {
896 KTEST_LOG(ctx, "=> tcp_hptsi(%u)", i);
897 KTEST_NEQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0);
898
899 HPTS_LOCK(pace->rp_ent[i]);
900 NET_EPOCH_ENTER(et);
901 slots_ran = tcp_hptsi(pace->rp_ent[i], true);
902 HPTS_UNLOCK(pace->rp_ent[i]);
903 NET_EPOCH_EXIT(et);
904
905 KTEST_EQUAL(slots_ran, NUM_OF_HPTSI_SLOTS-1); /* Should process all slots */
906 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0);
907 KTEST_NEQUAL(pace->rp_ent[i]->p_cur_slot,
908 pace->rp_ent[i]->p_prev_slot);
909 }
910
911 /* Cleanup */
912 for (i = 0; i < num_tcpcbs; i++) {
913 INP_WLOCK(&tcpcbs[i]->t_inpcb);
914 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) {
915 tcp_hpts_remove(pace, tcpcbs[i]);
916 }
917 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
918 test_hpts_free_tcpcb(tcpcbs[i]);
919 }
920 free(tcpcbs, M_TCPHPTS);
921 tcp_hptsi_stop(pace);
922 tcp_hptsi_destroy(pace);
923
924 return (0);
925 }
926
927 /*
928 * Validates proper handling of tcpcbs in the IHPTS_MOVING state, which occurs
929 * when a tcpcb is being processed by the HPTS thread but gets removed.
930 */
KTEST_FUNC(tcpcb_moving_state)931 KTEST_FUNC(tcpcb_moving_state)
932 {
933 struct epoch_tracker et;
934 struct tcp_hptsi *pace;
935 struct tcpcb *tp1, *tp2;
936 struct tcp_hpts_entry *hpts;
937 int32_t slots_ran;
938
939 test_hpts_init();
940
941 pace = tcp_hptsi_create(&test_funcs, false);
942 KTEST_NEQUAL(pace, NULL);
943 tcp_hptsi_start(pace);
944
945 /* Create two tcpcbs on the same CPU/slot */
946 tp1 = test_hpts_create_tcpcb(ctx, pace);
947 tp2 = test_hpts_create_tcpcb(ctx, pace);
948 KTEST_NEQUAL(tp1, NULL);
949 KTEST_NEQUAL(tp2, NULL);
950
951 /* Force them to the same CPU for predictable testing */
952 tp1->t_hpts_cpu = 0;
953 tp2->t_hpts_cpu = 0;
954
955 /* Insert both into the same slot */
956 INP_WLOCK(&tp1->t_inpcb);
957 tp1->t_flags2 |= TF2_HPTS_CALLS;
958 tcp_hpts_insert(pace, tp1, 100, NULL);
959 INP_WUNLOCK(&tp1->t_inpcb);
960
961 INP_WLOCK(&tp2->t_inpcb);
962 tp2->t_flags2 |= TF2_HPTS_CALLS;
963 tcp_hpts_insert(pace, tp2, 100, NULL);
964 INP_WUNLOCK(&tp2->t_inpcb);
965
966 hpts = pace->rp_ent[0];
967
968 /* Manually transition tp1 to MOVING state to simulate race condition */
969 HPTS_LOCK(hpts);
970 tp1->t_in_hpts = IHPTS_MOVING;
971 tp1->t_hpts_slot = -1; /* Mark for removal */
972 HPTS_UNLOCK(hpts);
973
974 /* Set time and run HPTS to process the moving state */
975 test_time_usec += 100;
976 HPTS_LOCK(hpts);
977 NET_EPOCH_ENTER(et);
978 slots_ran = tcp_hptsi(hpts, true);
979 HPTS_UNLOCK(hpts);
980 NET_EPOCH_EXIT(et);
981
982 KTEST_VERIFY(slots_ran >= 0);
983 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1); /* Shouldn't call on both */
984
985 /* tp1 should be cleaned up and removed */
986 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE);
987 /* tp2 should have been processed normally */
988 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE);
989
990 test_hpts_free_tcpcb(tp1);
991 test_hpts_free_tcpcb(tp2);
992 tcp_hptsi_stop(pace);
993 tcp_hptsi_destroy(pace);
994
995 return (0);
996 }
997
998 /*
999 * Validates that tcpcbs with deferred requests (t_hpts_request > 0) are
1000 * properly handled and re-inserted into appropriate future slots after
1001 * the wheel processes enough slots to accommodate the original request.
1002 */
KTEST_FUNC(deferred_requests)1003 KTEST_FUNC(deferred_requests)
1004 {
1005 struct epoch_tracker et;
1006 struct tcp_hptsi *pace;
1007 struct tcpcb *tp, *tp2;
1008 struct tcp_hpts_entry *hpts;
1009 uint32_t large_timeout_usecs = (NUM_OF_HPTSI_SLOTS + 5000) * HPTS_USECS_PER_SLOT; /* Beyond wheel capacity */
1010 uint32_t huge_timeout_usecs = (NUM_OF_HPTSI_SLOTS * 3) * HPTS_USECS_PER_SLOT; /* 3x wheel capacity */
1011 uint32_t initial_request;
1012 int32_t slots_ran;
1013
1014 test_hpts_init();
1015
1016 pace = tcp_hptsi_create(&test_funcs, false);
1017 KTEST_NEQUAL(pace, NULL);
1018 tcp_hptsi_start(pace);
1019
1020 tp = test_hpts_create_tcpcb(ctx, pace);
1021 KTEST_NEQUAL(tp, NULL);
1022
1023 /* Insert with a request that exceeds current wheel capacity */
1024 INP_WLOCK(&tp->t_inpcb);
1025 tp->t_flags2 |= TF2_HPTS_CALLS;
1026 tcp_hpts_insert(pace, tp, large_timeout_usecs, NULL);
1027 INP_WUNLOCK(&tp->t_inpcb);
1028
1029 /* Verify it was inserted with a deferred request */
1030 dump_tcpcb(tp);
1031 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1032 KTEST_VERIFY(tp->t_hpts_request > 0);
1033 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS);
1034
1035 hpts = pace->rp_ent[tp->t_hpts_cpu];
1036
1037 /* Advance time to process deferred requests */
1038 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT;
1039
1040 /* Process the wheel to handle deferred requests */
1041 HPTS_LOCK(hpts);
1042 NET_EPOCH_ENTER(et);
1043 slots_ran = tcp_hptsi(hpts, true);
1044 HPTS_UNLOCK(hpts);
1045 NET_EPOCH_EXIT(et);
1046
1047 dump_hpts_entry(ctx, hpts);
1048 KTEST_GREATER_THAN(slots_ran, 0);
1049 dump_tcpcb(tp);
1050 KTEST_EQUAL(tp->t_hpts_request, 0);
1051
1052 /* Test incremental deferred request processing over multiple cycles */
1053 KTEST_LOG(ctx, "Testing incremental deferred request processing");
1054
1055 /* Create a new connection with an even larger request */
1056 tp2 = test_hpts_create_tcpcb(ctx, pace);
1057 KTEST_NEQUAL(tp2, NULL);
1058 tp2->t_hpts_cpu = tp->t_hpts_cpu; /* Same CPU for predictable testing */
1059
1060 INP_WLOCK(&tp2->t_inpcb);
1061 tp2->t_flags2 |= TF2_HPTS_CALLS;
1062 tcp_hpts_insert(pace, tp2, huge_timeout_usecs, NULL);
1063 INP_WUNLOCK(&tp2->t_inpcb);
1064
1065 /* Verify initial deferred request */
1066 initial_request = tp2->t_hpts_request;
1067 KTEST_VERIFY(initial_request > NUM_OF_HPTSI_SLOTS);
1068
1069 /* Process one wheel cycle - should reduce but not eliminate request */
1070 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT;
1071 HPTS_LOCK(hpts);
1072 NET_EPOCH_ENTER(et);
1073 slots_ran = tcp_hptsi(hpts, true);
1074 HPTS_UNLOCK(hpts);
1075 NET_EPOCH_EXIT(et);
1076
1077 /* Request should be reduced but not zero */
1078 KTEST_GREATER_THAN(initial_request, tp2->t_hpts_request);
1079 KTEST_VERIFY(tp2->t_hpts_request > 0);
1080 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); /* Still queued */
1081
1082 /* For huge_timeout_usecs = NUM_OF_HPTSI_SLOTS * 3 * HPTS_USECS_PER_SLOT, we need ~3 cycles to complete.
1083 * Each cycle can reduce the request by at most NUM_OF_HPTSI_SLOTS. */
1084 test_time_usec += NUM_OF_HPTSI_SLOTS * HPTS_USECS_PER_SLOT;
1085 HPTS_LOCK(hpts);
1086 NET_EPOCH_ENTER(et);
1087 slots_ran = tcp_hptsi(hpts, true);
1088 HPTS_UNLOCK(hpts);
1089 NET_EPOCH_EXIT(et);
1090
1091 /* After second cycle, request should be reduced significantly (likely by ~NUM_OF_HPTSI_SLOTS) */
1092 KTEST_VERIFY(tp2->t_hpts_request < initial_request);
1093 KTEST_VERIFY(tp2->t_hpts_request > 0); /* But not yet zero for such a large request */
1094
1095 /* Clean up second connection */
1096 INP_WLOCK(&tp2->t_inpcb);
1097 if (tp2->t_in_hpts != IHPTS_NONE) {
1098 tcp_hpts_remove(pace, tp2);
1099 }
1100 INP_WUNLOCK(&tp2->t_inpcb);
1101 test_hpts_free_tcpcb(tp2);
1102
1103 /* Clean up */
1104 INP_WLOCK(&tp->t_inpcb);
1105 if (tp->t_in_hpts != IHPTS_NONE) {
1106 tcp_hpts_remove(pace, tp);
1107 }
1108 INP_WUNLOCK(&tp->t_inpcb);
1109 test_hpts_free_tcpcb(tp);
1110 tcp_hptsi_stop(pace);
1111 tcp_hptsi_destroy(pace);
1112
1113 return (0);
1114 }
1115
1116 /*
1117 * Validates CPU assignment and affinity mechanisms, including flowid-based
1118 * assignment, random fallback scenarios, and explicit CPU setting. Tests
1119 * the actual cpu assignment logic in hpts_cpuid via tcp_set_hpts.
1120 */
KTEST_FUNC(cpu_assignment)1121 KTEST_FUNC(cpu_assignment)
1122 {
1123 struct tcp_hptsi *pace;
1124 struct tcpcb *tp1, *tp2, *tp2_dup, *tp3;
1125
1126 test_hpts_init();
1127
1128 pace = tcp_hptsi_create(&test_funcs, false);
1129 KTEST_NEQUAL(pace, NULL);
1130
1131 /* Test random CPU assignment (no flowid) */
1132 tp1 = test_hpts_create_tcpcb(ctx, pace);
1133 KTEST_NEQUAL(tp1, NULL);
1134 tp1->t_inpcb.inp_flowtype = M_HASHTYPE_NONE;
1135 INP_WLOCK(&tp1->t_inpcb);
1136 tcp_set_hpts(pace, tp1);
1137 INP_WUNLOCK(&tp1->t_inpcb);
1138 KTEST_VERIFY(tp1->t_hpts_cpu < pace->rp_num_hptss);
1139 KTEST_VERIFY(tp1->t_flags2 & TF2_HPTS_CPU_SET);
1140
1141 /* Test flowid-based assignment */
1142 tp2 = test_hpts_create_tcpcb(ctx, pace);
1143 KTEST_NEQUAL(tp2, NULL);
1144 tp2->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4;
1145 tp2->t_inpcb.inp_flowid = 12345;
1146 INP_WLOCK(&tp2->t_inpcb);
1147 tcp_set_hpts(pace, tp2);
1148 INP_WUNLOCK(&tp2->t_inpcb);
1149 KTEST_VERIFY(tp2->t_hpts_cpu < pace->rp_num_hptss);
1150 KTEST_VERIFY(tp2->t_flags2 & TF2_HPTS_CPU_SET);
1151
1152 /* With the same flowid, should get same CPU assignment */
1153 tp2_dup = test_hpts_create_tcpcb(ctx, pace);
1154 KTEST_NEQUAL(tp2_dup, NULL);
1155 tp2_dup->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4;
1156 tp2_dup->t_inpcb.inp_flowid = 12345;
1157 INP_WLOCK(&tp2_dup->t_inpcb);
1158 tcp_set_hpts(pace, tp2_dup);
1159 INP_WUNLOCK(&tp2_dup->t_inpcb);
1160 KTEST_EQUAL(tp2_dup->t_hpts_cpu, tp2->t_hpts_cpu);
1161
1162 /* Test explicit CPU setting */
1163 tp3 = test_hpts_create_tcpcb(ctx, pace);
1164 KTEST_NEQUAL(tp3, NULL);
1165 tp3->t_hpts_cpu = 1; /* Assume we have at least 2 CPUs */
1166 tp3->t_flags2 |= TF2_HPTS_CPU_SET;
1167 INP_WLOCK(&tp3->t_inpcb);
1168 tcp_set_hpts(pace, tp3);
1169 INP_WUNLOCK(&tp3->t_inpcb);
1170 KTEST_EQUAL(tp3->t_hpts_cpu, 1);
1171
1172 test_hpts_free_tcpcb(tp1);
1173 test_hpts_free_tcpcb(tp2);
1174 test_hpts_free_tcpcb(tp2_dup);
1175 test_hpts_free_tcpcb(tp3);
1176 tcp_hptsi_destroy(pace);
1177
1178 return (0);
1179 }
1180
1181 /*
1182 * Validates edge cases in slot calculation including boundary conditions
1183 * around slot 0, maximum slots, and slot wrapping arithmetic.
1184 */
KTEST_FUNC(slot_boundary_conditions)1185 KTEST_FUNC(slot_boundary_conditions)
1186 {
1187 struct tcp_hptsi *pace;
1188 struct tcpcb *tp;
1189
1190 test_hpts_init();
1191
1192 pace = tcp_hptsi_create(&test_funcs, false);
1193 KTEST_NEQUAL(pace, NULL);
1194 tcp_hptsi_start(pace);
1195
1196 /* Test insertion at slot 0 */
1197 tp = test_hpts_create_tcpcb(ctx, pace);
1198 KTEST_NEQUAL(tp, NULL);
1199 INP_WLOCK(&tp->t_inpcb);
1200 tp->t_flags2 |= TF2_HPTS_CALLS;
1201 tcp_hpts_insert(pace, tp, 0, NULL); /* Should insert immediately (0 timeout) */
1202 INP_WUNLOCK(&tp->t_inpcb);
1203 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1204 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS);
1205
1206 INP_WLOCK(&tp->t_inpcb);
1207 tcp_hpts_remove(pace, tp);
1208 INP_WUNLOCK(&tp->t_inpcb);
1209
1210 /* Test insertion at maximum slot value */
1211 INP_WLOCK(&tp->t_inpcb);
1212 tp->t_flags2 |= TF2_HPTS_CALLS;
1213 tcp_hpts_insert(pace, tp, (NUM_OF_HPTSI_SLOTS - 1) * HPTS_USECS_PER_SLOT, NULL);
1214 INP_WUNLOCK(&tp->t_inpcb);
1215 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1216
1217 INP_WLOCK(&tp->t_inpcb);
1218 tcp_hpts_remove(pace, tp);
1219 INP_WUNLOCK(&tp->t_inpcb);
1220
1221 /* Test very small timeout values */
1222 INP_WLOCK(&tp->t_inpcb);
1223 tp->t_flags2 |= TF2_HPTS_CALLS;
1224 tcp_hpts_insert(pace, tp, 1, NULL);
1225 INP_WUNLOCK(&tp->t_inpcb);
1226 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE);
1227 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(1)); /* Should convert 1 usec to slot */
1228
1229 INP_WLOCK(&tp->t_inpcb);
1230 tcp_hpts_remove(pace, tp);
1231 INP_WUNLOCK(&tp->t_inpcb);
1232
1233 test_hpts_free_tcpcb(tp);
1234 tcp_hptsi_stop(pace);
1235 tcp_hptsi_destroy(pace);
1236
1237 return (0);
1238 }
1239
1240 /*
1241 * Validates HPTS behavior under high load conditions, including proper
1242 * processing of many connections and connection count tracking.
1243 */
KTEST_FUNC(dynamic_sleep_adjustment)1244 KTEST_FUNC(dynamic_sleep_adjustment)
1245 {
1246 struct epoch_tracker et;
1247 struct tcp_hptsi *pace;
1248 struct tcpcb **tcpcbs;
1249 struct tcp_hpts_entry *hpts;
1250 uint32_t i, num_tcpcbs = DEFAULT_CONNECTION_THRESHOLD + 50;
1251 int32_t slots_ran;
1252
1253 test_hpts_init();
1254
1255 pace = tcp_hptsi_create(&test_funcs, false);
1256 KTEST_NEQUAL(pace, NULL);
1257 tcp_hptsi_start(pace);
1258
1259 /* Create many connections to exceed threshold */
1260 tcpcbs = malloc(num_tcpcbs * sizeof(struct tcpcb *), M_TCPHPTS, M_WAITOK | M_ZERO);
1261 KTEST_VERIFY_RET(tcpcbs != NULL, ENOMEM);
1262
1263 for (i = 0; i < num_tcpcbs; i++) {
1264 tcpcbs[i] = test_hpts_create_tcpcb(ctx, pace);
1265 KTEST_NEQUAL(tcpcbs[i], NULL);
1266 tcpcbs[i]->t_hpts_cpu = 0; /* Force all to CPU 0 */
1267 INP_WLOCK(&tcpcbs[i]->t_inpcb);
1268 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS;
1269 TP_REMOVE_FROM_HPTS(tcpcbs[i]) = 1; /* Will be removed after output */
1270 tcp_hpts_insert(pace, tcpcbs[i], 100, NULL);
1271 INP_WUNLOCK(&tcpcbs[i]->t_inpcb);
1272 }
1273
1274 hpts = pace->rp_ent[0];
1275 dump_hpts_entry(ctx, hpts);
1276
1277 /* Verify we're above threshold */
1278 KTEST_GREATER_THAN(hpts->p_on_queue_cnt, DEFAULT_CONNECTION_THRESHOLD);
1279
1280 /* Run HPTS to process many connections */
1281 test_time_usec += 100;
1282 HPTS_LOCK(hpts);
1283 NET_EPOCH_ENTER(et);
1284 slots_ran = tcp_hptsi(hpts, true);
1285 HPTS_UNLOCK(hpts);
1286 NET_EPOCH_EXIT(et);
1287
1288 /* Verify HPTS processed slots and connections correctly */
1289 KTEST_GREATER_THAN(slots_ran, 0);
1290 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], num_tcpcbs);
1291
1292 /* Verify all connections were removed from queue */
1293 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
1294
1295 /* Cleanup */
1296 for (i = 0; i < num_tcpcbs; i++) {
1297 test_hpts_free_tcpcb(tcpcbs[i]);
1298 }
1299 free(tcpcbs, M_TCPHPTS);
1300 tcp_hptsi_stop(pace);
1301 tcp_hptsi_destroy(pace);
1302
1303 return (0);
1304 }
1305
1306 /*
1307 * Validates handling of concurrent insert/remove operations and race conditions
1308 * between HPTS processing and user operations.
1309 */
KTEST_FUNC(concurrent_operations)1310 KTEST_FUNC(concurrent_operations)
1311 {
1312 struct tcp_hptsi *pace;
1313 struct tcpcb *tp1, *tp2;
1314 struct tcp_hpts_entry *hpts;
1315
1316 test_hpts_init();
1317
1318 pace = tcp_hptsi_create(&test_funcs, false);
1319 KTEST_NEQUAL(pace, NULL);
1320 tcp_hptsi_start(pace);
1321
1322 tp1 = test_hpts_create_tcpcb(ctx, pace);
1323 tp2 = test_hpts_create_tcpcb(ctx, pace);
1324 KTEST_NEQUAL(tp1, NULL);
1325 KTEST_NEQUAL(tp2, NULL);
1326
1327 /* Force all to CPU 0 */
1328 tp1->t_hpts_cpu = 0;
1329 tp2->t_hpts_cpu = 0;
1330
1331 /* Insert tp1 */
1332 INP_WLOCK(&tp1->t_inpcb);
1333 tp1->t_flags2 |= TF2_HPTS_CALLS;
1334 tcp_hpts_insert(pace, tp1, 100, NULL);
1335 INP_WUNLOCK(&tp1->t_inpcb);
1336
1337 /* Insert tp2 into same slot */
1338 INP_WLOCK(&tp2->t_inpcb);
1339 tp2->t_flags2 |= TF2_HPTS_CALLS;
1340 tcp_hpts_insert(pace, tp2, 100, NULL);
1341 INP_WUNLOCK(&tp2->t_inpcb);
1342
1343 /* Verify both are inserted */
1344 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE);
1345 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE);
1346
1347 /* Verify they're both assigned to the same slot */
1348 KTEST_EQUAL(tp1->t_hpts_slot, tp2->t_hpts_slot);
1349
1350 /* Verify queue count reflects both connections */
1351 KTEST_EQUAL(tp1->t_hpts_cpu, tp2->t_hpts_cpu); /* Should be on same CPU */
1352 hpts = pace->rp_ent[tp1->t_hpts_cpu];
1353 KTEST_EQUAL(hpts->p_on_queue_cnt, 2);
1354
1355 /* Remove tp1 while tp2 is still there */
1356 INP_WLOCK(&tp1->t_inpcb);
1357 tcp_hpts_remove(pace, tp1);
1358 INP_WUNLOCK(&tp1->t_inpcb);
1359
1360 /* Verify tp1 removed, tp2 still there */
1361 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE);
1362 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE);
1363
1364 /* Verify queue count decreased by one */
1365 KTEST_EQUAL(hpts->p_on_queue_cnt, 1);
1366
1367 /* Remove tp2 */
1368 INP_WLOCK(&tp2->t_inpcb);
1369 tcp_hpts_remove(pace, tp2);
1370 INP_WUNLOCK(&tp2->t_inpcb);
1371
1372 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE);
1373
1374 /* Verify queue is now completely empty */
1375 KTEST_EQUAL(hpts->p_on_queue_cnt, 0);
1376
1377 test_hpts_free_tcpcb(tp1);
1378 test_hpts_free_tcpcb(tp2);
1379 tcp_hptsi_stop(pace);
1380 tcp_hptsi_destroy(pace);
1381
1382 return (0);
1383 }
1384
1385 /*
1386 * Validates the queued segments processing path via tfb_do_queued_segments,
1387 * which is an alternative to direct tcp_output calls.
1388 */
KTEST_FUNC(queued_segments_processing)1389 KTEST_FUNC(queued_segments_processing)
1390 {
1391 struct epoch_tracker et;
1392 struct tcp_hptsi *pace;
1393 struct tcpcb *tp;
1394 struct tcp_hpts_entry *hpts;
1395 struct mbuf *fake_mbuf;
1396 int32_t slots_ran;
1397
1398 test_hpts_init();
1399
1400 pace = tcp_hptsi_create(&test_funcs, false);
1401 KTEST_NEQUAL(pace, NULL);
1402 tcp_hptsi_start(pace);
1403
1404 tp = test_hpts_create_tcpcb(ctx, pace);
1405 KTEST_NEQUAL(tp, NULL);
1406
1407 /* Create a minimal fake mbuf that has valid STAILQ pointers */
1408 fake_mbuf = malloc(sizeof(struct mbuf), M_TCPHPTS, M_WAITOK | M_ZERO);
1409 KTEST_NEQUAL(fake_mbuf, NULL);
1410
1411 /* Set up for queued segments path */
1412 tp->t_flags2 |= (TF2_HPTS_CALLS | TF2_SUPPORTS_MBUFQ);
1413 STAILQ_INSERT_TAIL(&tp->t_inqueue, fake_mbuf, m_stailqpkt);
1414
1415 INP_WLOCK(&tp->t_inpcb);
1416 tcp_hpts_insert(pace, tp, 100, NULL);
1417 INP_WUNLOCK(&tp->t_inpcb);
1418
1419 hpts = pace->rp_ent[tp->t_hpts_cpu];
1420
1421 /* Run HPTS and verify queued segments path is taken */
1422 test_time_usec += 100;
1423 HPTS_LOCK(hpts);
1424 NET_EPOCH_ENTER(et);
1425 slots_ran = tcp_hptsi(hpts, true);
1426 HPTS_UNLOCK(hpts);
1427 NET_EPOCH_EXIT(et);
1428
1429 KTEST_VERIFY(slots_ran >= 0);
1430 KTEST_EQUAL(call_counts[CCNT_TCP_TFB_DO_QUEUED_SEGMENTS], 1);
1431
1432 /* Connection should be removed from HPTS after processing */
1433 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE);
1434
1435 /* Clean up the fake mbuf if it's still in the queue */
1436 if (!STAILQ_EMPTY(&tp->t_inqueue)) {
1437 struct mbuf *m = STAILQ_FIRST(&tp->t_inqueue);
1438 STAILQ_REMOVE_HEAD(&tp->t_inqueue, m_stailqpkt);
1439 free(m, M_TCPHPTS);
1440 }
1441
1442 test_hpts_free_tcpcb(tp);
1443 tcp_hptsi_stop(pace);
1444 tcp_hptsi_destroy(pace);
1445
1446 return (0);
1447 }
1448
1449 /*
1450 * Validates the direct wake mechanism and wake inhibition logic when
1451 * the connection count exceeds thresholds.
1452 */
KTEST_FUNC(direct_wake_mechanism)1453 KTEST_FUNC(direct_wake_mechanism)
1454 {
1455 struct tcp_hptsi *pace;
1456 struct tcpcb *tp;
1457 struct tcp_hpts_entry *hpts;
1458
1459 test_hpts_init();
1460
1461 pace = tcp_hptsi_create(&test_funcs, false);
1462 KTEST_NEQUAL(pace, NULL);
1463 tcp_hptsi_start(pace);
1464
1465 tp = test_hpts_create_tcpcb(ctx, pace);
1466 KTEST_NEQUAL(tp, NULL);
1467 hpts = pace->rp_ent[tp->t_hpts_cpu];
1468
1469 /* Test direct wake when not over threshold */
1470 HPTS_LOCK(hpts);
1471 hpts->p_on_queue_cnt = 50; /* Below threshold */
1472 hpts->p_hpts_wake_scheduled = 0;
1473 tcp_hpts_wake(hpts);
1474 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 1);
1475 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 1);
1476 HPTS_UNLOCK(hpts);
1477
1478 /* Reset for next test */
1479 hpts->p_hpts_wake_scheduled = 0;
1480 call_counts[CCNT_SWI_SCHED] = 0;
1481
1482 /* Test wake inhibition when over threshold */
1483 HPTS_LOCK(hpts);
1484 hpts->p_on_queue_cnt = 200; /* Above threshold */
1485 hpts->p_direct_wake = 1; /* Request direct wake */
1486 tcp_hpts_wake(hpts);
1487 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 0); /* Should be inhibited */
1488 KTEST_EQUAL(hpts->p_direct_wake, 0); /* Should be cleared */
1489 KTEST_EQUAL(call_counts[CCNT_SWI_SCHED], 0); /* No SWI scheduled */
1490 HPTS_UNLOCK(hpts);
1491
1492 test_hpts_free_tcpcb(tp);
1493 tcp_hptsi_stop(pace);
1494 tcp_hptsi_destroy(pace);
1495
1496 return (0);
1497 }
1498
1499 /*
1500 * Validates HPTS collision detection when attempting to run HPTS while
1501 * it's already active.
1502 */
KTEST_FUNC(hpts_collision_detection)1503 KTEST_FUNC(hpts_collision_detection)
1504 {
1505 struct epoch_tracker et;
1506 struct tcp_hptsi *pace;
1507 struct tcp_hpts_entry *hpts;
1508 int32_t slots_ran;
1509
1510 test_hpts_init();
1511
1512 pace = tcp_hptsi_create(&test_funcs, false);
1513 KTEST_NEQUAL(pace, NULL);
1514 tcp_hptsi_start(pace);
1515
1516 hpts = pace->rp_ent[0];
1517
1518 /* Mark HPTS as active */
1519 HPTS_LOCK(hpts);
1520 hpts->p_hpts_active = 1;
1521 HPTS_UNLOCK(hpts);
1522
1523 /* Attempt to run HPTS again - should detect collision */
1524 HPTS_LOCK(hpts);
1525 NET_EPOCH_ENTER(et);
1526 slots_ran = tcp_hptsi(hpts, false); /* from_callout = false */
1527 HPTS_UNLOCK(hpts);
1528 NET_EPOCH_EXIT(et);
1529
1530 /* Should return 0 indicating no work done due to collision */
1531 KTEST_EQUAL(slots_ran, 0);
1532
1533 tcp_hptsi_stop(pace);
1534 tcp_hptsi_destroy(pace);
1535
1536 return (0);
1537 }
1538
1539 /*
1540 * Validates generation count handling for race condition detection between
1541 * HPTS processing and connection insertion/removal operations.
1542 */
KTEST_FUNC(generation_count_validation)1543 KTEST_FUNC(generation_count_validation)
1544 {
1545 struct epoch_tracker et;
1546 struct tcp_hptsi *pace;
1547 struct tcp_hpts_entry *hpts;
1548 struct tcpcb *tp1, *tp2;
1549 uint32_t initial_gencnt, slot_to_test = 10;
1550 uint32_t timeout_usecs = slot_to_test * HPTS_USECS_PER_SLOT;
1551 uint32_t tp2_original_gencnt;
1552 int32_t slots_ran;
1553
1554 test_hpts_init();
1555
1556 pace = tcp_hptsi_create(&test_funcs, false);
1557 KTEST_NEQUAL(pace, NULL);
1558 tcp_hptsi_start(pace);
1559
1560 hpts = pace->rp_ent[0];
1561
1562 /* Record initial generation count for the test slot */
1563 initial_gencnt = hpts->p_hptss[slot_to_test].gencnt;
1564
1565 /* Create and insert first connection */
1566 tp1 = test_hpts_create_tcpcb(ctx, pace);
1567 KTEST_NEQUAL(tp1, NULL);
1568 tp1->t_hpts_cpu = 0; /* Force to CPU 0 */
1569
1570 INP_WLOCK(&tp1->t_inpcb);
1571 tp1->t_flags2 |= TF2_HPTS_CALLS;
1572 tcp_hpts_insert(pace, tp1, timeout_usecs, NULL);
1573 INP_WUNLOCK(&tp1->t_inpcb);
1574
1575 /* Verify connection stored the generation count */
1576 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE);
1577 KTEST_EQUAL(tp1->t_hpts_slot, slot_to_test);
1578 KTEST_EQUAL(tp1->t_hpts_gencnt, initial_gencnt);
1579
1580 /* Create second connection but don't insert yet */
1581 tp2 = test_hpts_create_tcpcb(ctx, pace);
1582 KTEST_NEQUAL(tp2, NULL);
1583 tp2->t_hpts_cpu = 0; /* Force to CPU 0 */
1584
1585 /* Force generation count increment by processing the slot */
1586 test_time_usec += (slot_to_test + 1) * HPTS_USECS_PER_SLOT;
1587 HPTS_LOCK(hpts);
1588 NET_EPOCH_ENTER(et);
1589 slots_ran = tcp_hptsi(hpts, true);
1590 HPTS_UNLOCK(hpts);
1591 NET_EPOCH_EXIT(et);
1592
1593 /* Verify processing occurred */
1594 KTEST_VERIFY(slots_ran > 0);
1595 KTEST_EQUAL(call_counts[CCNT_TCP_OUTPUT], 1);
1596
1597 /* Verify generation count was incremented */
1598 KTEST_EQUAL(hpts->p_hptss[slot_to_test].gencnt, initial_gencnt + 1);
1599
1600 /* Verify first connection was processed and removed */
1601 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE);
1602
1603 /* Insert second connection and record its generation count */
1604 INP_WLOCK(&tp2->t_inpcb);
1605 tp2->t_flags2 |= TF2_HPTS_CALLS;
1606 tcp_hpts_insert(pace, tp2, timeout_usecs, NULL);
1607 INP_WUNLOCK(&tp2->t_inpcb);
1608
1609 /* Verify connection was inserted successfully */
1610 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE);
1611
1612 /* Record the generation count that tp2 received */
1613 tp2_original_gencnt = tp2->t_hpts_gencnt;
1614
1615 /* Test generation count mismatch detection during processing */
1616 /* Manually set stale generation count to simulate race condition */
1617 tp2->t_hpts_gencnt = tp2_original_gencnt + 100; /* Force a mismatch */
1618
1619 /* Process the slot to trigger generation count validation */
1620 test_time_usec += (slot_to_test + 1) * HPTS_USECS_PER_SLOT;
1621 HPTS_LOCK(hpts);
1622 NET_EPOCH_ENTER(et);
1623 slots_ran = tcp_hptsi(hpts, true);
1624 HPTS_UNLOCK(hpts);
1625 NET_EPOCH_EXIT(et);
1626
1627 /* Connection should be processed despite generation count mismatch */
1628 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); /* Processed and released */
1629
1630 /* The key test: HPTS should handle mismatched generation counts gracefully */
1631 KTEST_VERIFY(slots_ran > 0); /* Processing should still occur */
1632
1633 test_hpts_free_tcpcb(tp1);
1634 test_hpts_free_tcpcb(tp2);
1635 tcp_hptsi_stop(pace);
1636 tcp_hptsi_destroy(pace);
1637
1638 return (0);
1639 }
1640
1641 static const struct ktest_test_info tests[] = {
1642 KTEST_INFO(module_load),
1643 KTEST_INFO(hptsi_create_destroy),
1644 KTEST_INFO(hptsi_start_stop),
1645 KTEST_INFO(hptsi_independence),
1646 KTEST_INFO(function_injection),
1647 KTEST_INFO(tcpcb_initialization),
1648 KTEST_INFO(tcpcb_insertion),
1649 KTEST_INFO(timer_functionality),
1650 KTEST_INFO(scalability_tcpcbs),
1651 KTEST_INFO(wheel_wrap_recovery),
1652 KTEST_INFO(tcpcb_moving_state),
1653 KTEST_INFO(deferred_requests),
1654 KTEST_INFO(cpu_assignment),
1655 KTEST_INFO(slot_boundary_conditions),
1656 KTEST_INFO(dynamic_sleep_adjustment),
1657 KTEST_INFO(concurrent_operations),
1658 KTEST_INFO(queued_segments_processing),
1659 KTEST_INFO(direct_wake_mechanism),
1660 KTEST_INFO(hpts_collision_detection),
1661 KTEST_INFO(generation_count_validation),
1662 };
1663
1664 #else /* TCP_HPTS_KTEST */
1665
1666 /*
1667 * Stub to indicate that the TCP HPTS ktest is not enabled.
1668 */
KTEST_FUNC(module_load_without_tests)1669 KTEST_FUNC(module_load_without_tests)
1670 {
1671 KTEST_LOG(ctx, "Warning: TCP HPTS ktest is not enabled");
1672 return (0);
1673 }
1674
1675 static const struct ktest_test_info tests[] = {
1676 KTEST_INFO(module_load_without_tests),
1677 };
1678
1679 #endif
1680
1681 KTEST_MODULE_DECLARE(ktest_tcphpts, tests);
1682 KTEST_MODULE_DEPEND(ktest_tcphpts, tcphpts);
1683