Lines Matching +full:test +full:- +full:cpu
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
64 static MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts_test", "TCP hpts test");
68 "TCP HPTS test controls");
71 "Exit HPTS test immediately on first failure (1) or continue running all tests (0)");
129 KTEST_LOG(ctx, " p_cur_slot: %u", hpts->p_cur_slot); in dump_hpts_entry()
130 KTEST_LOG(ctx, " p_prev_slot: %u", hpts->p_prev_slot); in dump_hpts_entry()
131 KTEST_LOG(ctx, " p_nxt_slot: %u", hpts->p_nxt_slot); in dump_hpts_entry()
132 KTEST_LOG(ctx, " p_runningslot: %u", hpts->p_runningslot); in dump_hpts_entry()
133 KTEST_LOG(ctx, " p_on_queue_cnt: %d", hpts->p_on_queue_cnt); in dump_hpts_entry()
134 KTEST_LOG(ctx, " p_hpts_active: %u", hpts->p_hpts_active); in dump_hpts_entry()
135 KTEST_LOG(ctx, " p_wheel_complete: %u", hpts->p_wheel_complete); in dump_hpts_entry()
136 KTEST_LOG(ctx, " p_direct_wake: %u", hpts->p_direct_wake); in dump_hpts_entry()
137 KTEST_LOG(ctx, " p_on_min_sleep: %u", hpts->p_on_min_sleep); in dump_hpts_entry()
138 KTEST_LOG(ctx, " p_hpts_wake_scheduled: %u", hpts->p_hpts_wake_scheduled); in dump_hpts_entry()
139 KTEST_LOG(ctx, " hit_callout_thresh: %u", hpts->hit_callout_thresh); in dump_hpts_entry()
140 KTEST_LOG(ctx, " p_hpts_sleep_time: %u", hpts->p_hpts_sleep_time); in dump_hpts_entry()
141 KTEST_LOG(ctx, " p_delayed_by: %u", hpts->p_delayed_by); in dump_hpts_entry()
142 KTEST_LOG(ctx, " overidden_sleep: %u", hpts->overidden_sleep); in dump_hpts_entry()
143 KTEST_LOG(ctx, " saved_curslot: %u", hpts->saved_curslot); in dump_hpts_entry()
144 KTEST_LOG(ctx, " saved_prev_slot: %u", hpts->saved_prev_slot); in dump_hpts_entry()
145 KTEST_LOG(ctx, " syscall_cnt: %lu", hpts->syscall_cnt); in dump_hpts_entry()
146 KTEST_LOG(ctx, " sleeping: %lu", hpts->sleeping); in dump_hpts_entry()
147 KTEST_LOG(ctx, " p_cpu: %u", hpts->p_cpu); in dump_hpts_entry()
148 KTEST_LOG(ctx, " ie_cookie: %p", hpts->ie_cookie); in dump_hpts_entry()
149 KTEST_LOG(ctx, " p_hptsi: %p", hpts->p_hptsi); in dump_hpts_entry()
150 KTEST_LOG(ctx, " p_mysleep: %ld.%06ld", hpts->p_mysleep.tv_sec, hpts->p_mysleep.tv_usec); in dump_hpts_entry()
156 struct ktest_test_context *ctx = tp->t_fb_ptr; in dump_tcpcb()
157 struct inpcb *inp = &tp->t_inpcb; in dump_tcpcb()
161 /* HPTS-specific fields */ in dump_tcpcb()
162 KTEST_LOG(ctx, " t_in_hpts: %d", tp->t_in_hpts); in dump_tcpcb()
163 KTEST_LOG(ctx, " t_hpts_cpu: %u", tp->t_hpts_cpu); in dump_tcpcb()
164 KTEST_LOG(ctx, " t_hpts_slot: %d", tp->t_hpts_slot); in dump_tcpcb()
165 KTEST_LOG(ctx, " t_hpts_gencnt: %u", tp->t_hpts_gencnt); in dump_tcpcb()
166 KTEST_LOG(ctx, " t_hpts_request: %u", tp->t_hpts_request); in dump_tcpcb()
168 /* LRO CPU field */ in dump_tcpcb()
169 KTEST_LOG(ctx, " t_lro_cpu: %u", tp->t_lro_cpu); in dump_tcpcb()
172 KTEST_LOG(ctx, " t_flags2: 0x%x", tp->t_flags2); in dump_tcpcb()
173 KTEST_LOG(ctx, " TF2_HPTS_CPU_SET: %s", (tp->t_flags2 & TF2_HPTS_CPU_SET) ? "YES" : "NO"); in dump_tcpcb()
174 KTEST_LOG(ctx, " TF2_HPTS_CALLS: %s", (tp->t_flags2 & TF2_HPTS_CALLS) ? "YES" : "NO"); in dump_tcpcb()
175 KTEST_LOG(ctx, " TF2_SUPPORTS_MBUFQ: %s", (tp->t_flags2 & TF2_SUPPORTS_MBUFQ) ? "YES" : "NO"); in dump_tcpcb()
178 KTEST_LOG(ctx, " inp_flags: 0x%x", inp->inp_flags); in dump_tcpcb()
179 KTEST_LOG(ctx, " INP_DROPPED: %s", (inp->inp_flags & INP_DROPPED) ? "YES" : "NO"); in dump_tcpcb()
180 KTEST_LOG(ctx, " inp_flowid: 0x%x", inp->inp_flowid); in dump_tcpcb()
181 KTEST_LOG(ctx, " inp_flowtype: %u", inp->inp_flowtype); in dump_tcpcb()
182 KTEST_LOG(ctx, " inp_numa_domain: %d", inp->inp_numa_domain); in dump_tcpcb()
206 * Reset all test global variables to a clean state.
219 tv->tv_sec = test_time_usec / 1000000; in test_microuptime()
220 tv->tv_usec = test_time_usec % 1000000; in test_microuptime()
251 test_intr_event_bind(struct intr_event *ie, int cpu) in test_intr_event_bind() argument
275 void (*func)(void *), void *arg, int cpu, int flags) in test_callout_reset_sbt_on() argument
302 #define TP_REMOVE_FROM_HPTS(tp) tp->bits_spare
303 #define TP_LOG_TEST(tp) tp->t_log_state_set
308 struct ktest_test_context *ctx = tp->t_fb_ptr; in test_tcp_output()
309 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending; in test_tcp_output()
310 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu]; in test_tcp_output()
326 INP_WUNLOCK(&tp->t_inpcb); /* tcp_output unlocks on error */ in test_tcp_output()
327 return (-1); /* Simulate tcp_output error */ in test_tcp_output()
336 struct ktest_test_context *ctx = tp->t_fb_ptr; in test_tfb_do_queued_segments()
337 struct tcp_hptsi *pace = (struct tcp_hptsi*)tp->t_tfo_pending; in test_tfb_do_queued_segments()
338 struct tcp_hpts_entry *hpts = pace->rp_ent[tp->t_hpts_cpu]; in test_tfb_do_queued_segments()
352 INP_WUNLOCK(&tp->t_inpcb); /* do_queued_segments unlocks on error */ in test_tfb_do_queued_segments()
353 return (-1); /* Simulate do_queued_segments error */ in test_tfb_do_queued_segments()
377 rw_init_flags(&tp->t_inpcb.inp_lock, "test-inp", in test_hpts_create_tcpcb()
379 refcount_init(&tp->t_inpcb.inp_refcount, 1); in test_hpts_create_tcpcb()
380 tp->t_inpcb.inp_pcbinfo = &V_tcbinfo; in test_hpts_create_tcpcb()
381 tp->t_fb = &test_tcp_fb; in test_hpts_create_tcpcb()
382 tp->t_hpts_cpu = HPTS_CPU_NONE; in test_hpts_create_tcpcb()
383 STAILQ_INIT(&tp->t_inqueue); in test_hpts_create_tcpcb()
386 /* Stuff some pointers in the tcb for test purposes. */ in test_hpts_create_tcpcb()
387 tp->t_fb_ptr = ctx; in test_hpts_create_tcpcb()
388 tp->t_tfo_pending = (unsigned int*)pace; in test_hpts_create_tcpcb()
395 * Free a test tcpcb created by test_hpts_create_tcpcb()
403 INP_LOCK_DESTROY(&tp->t_inpcb); in test_hpts_free_tcpcb()
438 KTEST_NEQUAL(pace->rp_ent, NULL); in KTEST_FUNC()
439 KTEST_NEQUAL(pace->cts_last_ran, NULL); in KTEST_FUNC()
440 KTEST_VERIFY(pace->rp_num_hptss > 0); in KTEST_FUNC()
441 KTEST_VERIFY(pace->rp_num_hptss <= MAXCPU); /* Reasonable upper bound */ in KTEST_FUNC()
442 KTEST_VERIFY(pace->grp_cnt >= 1); /* At least one group */ in KTEST_FUNC()
443 KTEST_EQUAL(pace->funcs, &test_funcs); /* Verify function pointer was set */ in KTEST_FUNC()
446 for (uint32_t i = 0; i < pace->rp_num_hptss; i++) { in KTEST_FUNC()
447 KTEST_NEQUAL(pace->rp_ent[i], NULL); in KTEST_FUNC()
448 KTEST_EQUAL(pace->rp_ent[i]->p_cpu, i); in KTEST_FUNC()
449 KTEST_EQUAL(pace->rp_ent[i]->p_hptsi, pace); in KTEST_FUNC()
450 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0); in KTEST_FUNC()
475 struct tcp_hpts_entry *hpts = pace->rp_ent[0]; in KTEST_FUNC()
476 KTEST_NEQUAL(hpts->ie_cookie, NULL); /* Should have SWI handler */ in KTEST_FUNC()
477 KTEST_EQUAL(hpts->p_hptsi, pace); /* Should point to our pace */ in KTEST_FUNC()
487 * different configurations and CPU assignments without interfering with each
501 KTEST_NEQUAL(pace2->rp_ent, NULL); in KTEST_FUNC()
505 KTEST_VERIFY(cpu1 < pace1->rp_num_hptss); in KTEST_FUNC()
506 KTEST_VERIFY(cpu2 < pace2->rp_num_hptss); in KTEST_FUNC()
509 KTEST_NEQUAL(pace1->rp_ent, pace2->rp_ent); in KTEST_FUNC()
510 /* Verify they may have different CPU counts but both reasonable */ in KTEST_FUNC()
511 KTEST_VERIFY(pace1->rp_num_hptss > 0 && pace1->rp_num_hptss <= MAXCPU); in KTEST_FUNC()
512 KTEST_VERIFY(pace2->rp_num_hptss > 0 && pace2->rp_num_hptss <= MAXCPU); in KTEST_FUNC()
522 * test-specific implementations of microuptime and others are properly
533 KTEST_EQUAL(pace->funcs, &test_funcs); in KTEST_FUNC()
576 KTEST_NEQUAL(tp->t_fb, NULL); in KTEST_FUNC()
577 KTEST_NEQUAL(tp->t_fb->tfb_tcp_output, NULL); in KTEST_FUNC()
578 KTEST_NEQUAL(tp->t_fb->tfb_do_queued_segments, NULL); in KTEST_FUNC()
579 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
580 KTEST_EQUAL((tp->t_flags2 & (TF2_HPTS_CPU_SET | TF2_HPTS_CALLS)), 0); in KTEST_FUNC()
582 /* Verify that HPTS-specific fields are initialized */ in KTEST_FUNC()
583 KTEST_EQUAL(tp->t_hpts_gencnt, 0); in KTEST_FUNC()
584 KTEST_EQUAL(tp->t_hpts_slot, 0); in KTEST_FUNC()
585 KTEST_EQUAL(tp->t_hpts_request, 0); in KTEST_FUNC()
586 KTEST_EQUAL(tp->t_lro_cpu, 0); in KTEST_FUNC()
587 KTEST_VERIFY(tp->t_hpts_cpu < pace->rp_num_hptss); in KTEST_FUNC()
588 KTEST_EQUAL(tp->t_inpcb.inp_refcount, 1); in KTEST_FUNC()
589 KTEST_VERIFY(!(tp->t_inpcb.inp_flags & INP_DROPPED)); in KTEST_FUNC()
618 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
619 KTEST_EQUAL((tp->t_flags2 & TF2_HPTS_CALLS), 0); in KTEST_FUNC()
621 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
622 tp->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
625 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
626 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
630 KTEST_VERIFY(tp->t_hpts_slot >= 0); in KTEST_FUNC()
631 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS); in KTEST_FUNC()
633 hpts = pace->rp_ent[tp->t_hpts_cpu]; in KTEST_FUNC()
634 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); in KTEST_FUNC()
635 KTEST_EQUAL(tp->t_hpts_request, 0); in KTEST_FUNC()
636 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(timeout_usecs)); in KTEST_FUNC()
637 //KTEST_EQUAL(tp->t_hpts_gencnt, 1); in KTEST_FUNC()
639 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
641 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
642 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
646 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); in KTEST_FUNC()
658 * real-world timer-driven TCP processing.
675 for (i = 0; i < pace->rp_num_hptss; i++) in KTEST_FUNC()
676 dump_hpts_entry(ctx, pace->rp_ent[i]); in KTEST_FUNC()
685 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
686 tp->t_flags2 |= TF2_HPTS_CALLS; /* Mark as needing HPTS processing */ in KTEST_FUNC()
688 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
691 for (i = 0; i < pace->rp_num_hptss; i++) in KTEST_FUNC()
692 dump_hpts_entry(ctx, pace->rp_ent[i]); in KTEST_FUNC()
694 hpts = pace->rp_ent[tp->t_hpts_cpu]; in KTEST_FUNC()
695 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); in KTEST_FUNC()
696 KTEST_EQUAL(hpts->p_prev_slot, 0); in KTEST_FUNC()
697 KTEST_EQUAL(hpts->p_cur_slot, 0); in KTEST_FUNC()
698 KTEST_EQUAL(hpts->p_runningslot, 0); in KTEST_FUNC()
699 KTEST_EQUAL(hpts->p_nxt_slot, 1); in KTEST_FUNC()
700 KTEST_EQUAL(hpts->p_hpts_active, 0); in KTEST_FUNC()
702 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
703 KTEST_EQUAL(tp->t_hpts_request, 0); in KTEST_FUNC()
704 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500)); in KTEST_FUNC()
706 /* Set our test flag to indicate the tcpcb should be removed from the in KTEST_FUNC()
710 /* Test early exit condition: advance time by insufficient amount */ in KTEST_FUNC()
712 test_time_usec += 1; /* Very small advancement - should cause early exit */ in KTEST_FUNC()
722 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); /* Connection still queued */ in KTEST_FUNC()
728 for (i = 0; i < pace->rp_num_hptss; i++) { in KTEST_FUNC()
729 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]); in KTEST_FUNC()
730 HPTS_LOCK(pace->rp_ent[i]); in KTEST_FUNC()
732 slots_ran = tcp_hptsi(pace->rp_ent[i], true); in KTEST_FUNC()
733 HPTS_UNLOCK(pace->rp_ent[i]); in KTEST_FUNC()
736 dump_hpts_entry(ctx, pace->rp_ent[i]); in KTEST_FUNC()
738 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 49); in KTEST_FUNC()
739 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 49); in KTEST_FUNC()
744 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
745 KTEST_EQUAL(tp->t_hpts_request, 0); in KTEST_FUNC()
746 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(500)); in KTEST_FUNC()
747 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); in KTEST_FUNC()
753 for (i = 0; i < pace->rp_num_hptss; i++) { in KTEST_FUNC()
754 KTEST_LOG(ctx, "=> tcp_hptsi(%p)", pace->rp_ent[i]); in KTEST_FUNC()
755 HPTS_LOCK(pace->rp_ent[i]); in KTEST_FUNC()
757 slots_ran = tcp_hptsi(pace->rp_ent[i], true); in KTEST_FUNC()
758 HPTS_UNLOCK(pace->rp_ent[i]); in KTEST_FUNC()
761 dump_hpts_entry(ctx, pace->rp_ent[i]); in KTEST_FUNC()
763 KTEST_EQUAL(pace->rp_ent[i]->p_prev_slot, 50); in KTEST_FUNC()
764 KTEST_EQUAL(pace->rp_ent[i]->p_cur_slot, 50); in KTEST_FUNC()
769 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
770 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); in KTEST_FUNC()
812 INP_WLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
813 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
816 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
820 for (i = 0; i < pace->rp_num_hptss; i++) { in KTEST_FUNC()
821 total_queued += pace->rp_ent[i]->p_on_queue_cnt; in KTEST_FUNC()
825 for (i = 0; i < pace->rp_num_hptss; i++) in KTEST_FUNC()
826 dump_hpts_entry(ctx, pace->rp_ent[i]); in KTEST_FUNC()
831 INP_WLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
832 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) { in KTEST_FUNC()
835 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
839 for (i = 0; i < pace->rp_num_hptss; i++) { in KTEST_FUNC()
840 if (pace->rp_ent[i]->p_on_queue_cnt != 0) { in KTEST_FUNC()
841 KTEST_ERR(ctx, "FAIL: pace->rp_ent[i]->p_on_queue_cnt != 0"); in KTEST_FUNC()
886 INP_WLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
887 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
889 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
895 for (i = 0; i < pace->rp_num_hptss; i++) { in KTEST_FUNC()
897 KTEST_NEQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0); in KTEST_FUNC()
899 HPTS_LOCK(pace->rp_ent[i]); in KTEST_FUNC()
901 slots_ran = tcp_hptsi(pace->rp_ent[i], true); in KTEST_FUNC()
902 HPTS_UNLOCK(pace->rp_ent[i]); in KTEST_FUNC()
905 KTEST_EQUAL(slots_ran, NUM_OF_HPTSI_SLOTS-1); /* Should process all slots */ in KTEST_FUNC()
906 KTEST_EQUAL(pace->rp_ent[i]->p_on_queue_cnt, 0); in KTEST_FUNC()
907 KTEST_NEQUAL(pace->rp_ent[i]->p_cur_slot, in KTEST_FUNC()
908 pace->rp_ent[i]->p_prev_slot); in KTEST_FUNC()
913 INP_WLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
914 if (tcpcbs[i]->t_in_hpts != IHPTS_NONE) { in KTEST_FUNC()
917 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
945 /* Create two tcpcbs on the same CPU/slot */ in KTEST_FUNC()
951 /* Force them to the same CPU for predictable testing */ in KTEST_FUNC()
952 tp1->t_hpts_cpu = 0; in KTEST_FUNC()
953 tp2->t_hpts_cpu = 0; in KTEST_FUNC()
956 INP_WLOCK(&tp1->t_inpcb); in KTEST_FUNC()
957 tp1->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
959 INP_WUNLOCK(&tp1->t_inpcb); in KTEST_FUNC()
961 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
962 tp2->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
964 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
966 hpts = pace->rp_ent[0]; in KTEST_FUNC()
970 tp1->t_in_hpts = IHPTS_MOVING; in KTEST_FUNC()
971 tp1->t_hpts_slot = -1; /* Mark for removal */ in KTEST_FUNC()
986 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
988 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
1000 * properly handled and re-inserted into appropriate future slots after
1024 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1025 tp->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1027 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1031 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1032 KTEST_VERIFY(tp->t_hpts_request > 0); in KTEST_FUNC()
1033 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS); in KTEST_FUNC()
1035 hpts = pace->rp_ent[tp->t_hpts_cpu]; in KTEST_FUNC()
1050 KTEST_EQUAL(tp->t_hpts_request, 0); in KTEST_FUNC()
1052 /* Test incremental deferred request processing over multiple cycles */ in KTEST_FUNC()
1058 tp2->t_hpts_cpu = tp->t_hpts_cpu; /* Same CPU for predictable testing */ in KTEST_FUNC()
1060 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1061 tp2->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1063 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1066 initial_request = tp2->t_hpts_request; in KTEST_FUNC()
1069 /* Process one wheel cycle - should reduce but not eliminate request */ in KTEST_FUNC()
1078 KTEST_GREATER_THAN(initial_request, tp2->t_hpts_request); in KTEST_FUNC()
1079 KTEST_VERIFY(tp2->t_hpts_request > 0); in KTEST_FUNC()
1080 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); /* Still queued */ in KTEST_FUNC()
1092 KTEST_VERIFY(tp2->t_hpts_request < initial_request); in KTEST_FUNC()
1093 KTEST_VERIFY(tp2->t_hpts_request > 0); /* But not yet zero for such a large request */ in KTEST_FUNC()
1096 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1097 if (tp2->t_in_hpts != IHPTS_NONE) { in KTEST_FUNC()
1100 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1104 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1105 if (tp->t_in_hpts != IHPTS_NONE) { in KTEST_FUNC()
1108 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1117 * Validates CPU assignment and affinity mechanisms, including flowid-based
1118 * assignment, random fallback scenarios, and explicit CPU setting. Tests
1119 * the actual cpu assignment logic in hpts_cpuid via tcp_set_hpts.
1131 /* Test random CPU assignment (no flowid) */ in KTEST_FUNC()
1134 tp1->t_inpcb.inp_flowtype = M_HASHTYPE_NONE; in KTEST_FUNC()
1135 INP_WLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1137 INP_WUNLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1138 KTEST_VERIFY(tp1->t_hpts_cpu < pace->rp_num_hptss); in KTEST_FUNC()
1139 KTEST_VERIFY(tp1->t_flags2 & TF2_HPTS_CPU_SET); in KTEST_FUNC()
1141 /* Test flowid-based assignment */ in KTEST_FUNC()
1144 tp2->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4; in KTEST_FUNC()
1145 tp2->t_inpcb.inp_flowid = 12345; in KTEST_FUNC()
1146 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1148 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1149 KTEST_VERIFY(tp2->t_hpts_cpu < pace->rp_num_hptss); in KTEST_FUNC()
1150 KTEST_VERIFY(tp2->t_flags2 & TF2_HPTS_CPU_SET); in KTEST_FUNC()
1152 /* With the same flowid, should get same CPU assignment */ in KTEST_FUNC()
1155 tp2_dup->t_inpcb.inp_flowtype = M_HASHTYPE_RSS_TCP_IPV4; in KTEST_FUNC()
1156 tp2_dup->t_inpcb.inp_flowid = 12345; in KTEST_FUNC()
1157 INP_WLOCK(&tp2_dup->t_inpcb); in KTEST_FUNC()
1159 INP_WUNLOCK(&tp2_dup->t_inpcb); in KTEST_FUNC()
1160 KTEST_EQUAL(tp2_dup->t_hpts_cpu, tp2->t_hpts_cpu); in KTEST_FUNC()
1162 /* Test explicit CPU setting */ in KTEST_FUNC()
1165 tp3->t_hpts_cpu = 1; /* Assume we have at least 2 CPUs */ in KTEST_FUNC()
1166 tp3->t_flags2 |= TF2_HPTS_CPU_SET; in KTEST_FUNC()
1167 INP_WLOCK(&tp3->t_inpcb); in KTEST_FUNC()
1169 INP_WUNLOCK(&tp3->t_inpcb); in KTEST_FUNC()
1170 KTEST_EQUAL(tp3->t_hpts_cpu, 1); in KTEST_FUNC()
1196 /* Test insertion at slot 0 */ in KTEST_FUNC()
1199 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1200 tp->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1202 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1203 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1204 KTEST_VERIFY(tp->t_hpts_slot < NUM_OF_HPTSI_SLOTS); in KTEST_FUNC()
1206 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1208 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1210 /* Test insertion at maximum slot value */ in KTEST_FUNC()
1211 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1212 tp->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1213 tcp_hpts_insert(pace, tp, (NUM_OF_HPTSI_SLOTS - 1) * HPTS_USECS_PER_SLOT, NULL); in KTEST_FUNC()
1214 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1215 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1217 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1219 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1221 /* Test very small timeout values */ in KTEST_FUNC()
1222 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1223 tp->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1225 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1226 KTEST_EQUAL(tp->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1227 KTEST_EQUAL(tp->t_hpts_slot, HPTS_USEC_TO_SLOTS(1)); /* Should convert 1 usec to slot */ in KTEST_FUNC()
1229 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1231 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1266 tcpcbs[i]->t_hpts_cpu = 0; /* Force all to CPU 0 */ in KTEST_FUNC()
1267 INP_WLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
1268 tcpcbs[i]->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1271 INP_WUNLOCK(&tcpcbs[i]->t_inpcb); in KTEST_FUNC()
1274 hpts = pace->rp_ent[0]; in KTEST_FUNC()
1278 KTEST_GREATER_THAN(hpts->p_on_queue_cnt, DEFAULT_CONNECTION_THRESHOLD); in KTEST_FUNC()
1293 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); in KTEST_FUNC()
1327 /* Force all to CPU 0 */ in KTEST_FUNC()
1328 tp1->t_hpts_cpu = 0; in KTEST_FUNC()
1329 tp2->t_hpts_cpu = 0; in KTEST_FUNC()
1332 INP_WLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1333 tp1->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1335 INP_WUNLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1338 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1339 tp2->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1341 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1344 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1345 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1348 KTEST_EQUAL(tp1->t_hpts_slot, tp2->t_hpts_slot); in KTEST_FUNC()
1351 KTEST_EQUAL(tp1->t_hpts_cpu, tp2->t_hpts_cpu); /* Should be on same CPU */ in KTEST_FUNC()
1352 hpts = pace->rp_ent[tp1->t_hpts_cpu]; in KTEST_FUNC()
1353 KTEST_EQUAL(hpts->p_on_queue_cnt, 2); in KTEST_FUNC()
1356 INP_WLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1358 INP_WUNLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1361 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
1362 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1365 KTEST_EQUAL(hpts->p_on_queue_cnt, 1); in KTEST_FUNC()
1368 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1370 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1372 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
1375 KTEST_EQUAL(hpts->p_on_queue_cnt, 0); in KTEST_FUNC()
1412 tp->t_flags2 |= (TF2_HPTS_CALLS | TF2_SUPPORTS_MBUFQ); in KTEST_FUNC()
1413 STAILQ_INSERT_TAIL(&tp->t_inqueue, fake_mbuf, m_stailqpkt); in KTEST_FUNC()
1415 INP_WLOCK(&tp->t_inpcb); in KTEST_FUNC()
1417 INP_WUNLOCK(&tp->t_inpcb); in KTEST_FUNC()
1419 hpts = pace->rp_ent[tp->t_hpts_cpu]; in KTEST_FUNC()
1433 KTEST_EQUAL(tp->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
1436 if (!STAILQ_EMPTY(&tp->t_inqueue)) { in KTEST_FUNC()
1437 struct mbuf *m = STAILQ_FIRST(&tp->t_inqueue); in KTEST_FUNC()
1438 STAILQ_REMOVE_HEAD(&tp->t_inqueue, m_stailqpkt); in KTEST_FUNC()
1467 hpts = pace->rp_ent[tp->t_hpts_cpu]; in KTEST_FUNC()
1469 /* Test direct wake when not over threshold */ in KTEST_FUNC()
1471 hpts->p_on_queue_cnt = 50; /* Below threshold */ in KTEST_FUNC()
1472 hpts->p_hpts_wake_scheduled = 0; in KTEST_FUNC()
1474 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 1); in KTEST_FUNC()
1478 /* Reset for next test */ in KTEST_FUNC()
1479 hpts->p_hpts_wake_scheduled = 0; in KTEST_FUNC()
1482 /* Test wake inhibition when over threshold */ in KTEST_FUNC()
1484 hpts->p_on_queue_cnt = 200; /* Above threshold */ in KTEST_FUNC()
1485 hpts->p_direct_wake = 1; /* Request direct wake */ in KTEST_FUNC()
1487 KTEST_EQUAL(hpts->p_hpts_wake_scheduled, 0); /* Should be inhibited */ in KTEST_FUNC()
1488 KTEST_EQUAL(hpts->p_direct_wake, 0); /* Should be cleared */ in KTEST_FUNC()
1516 hpts = pace->rp_ent[0]; in KTEST_FUNC()
1520 hpts->p_hpts_active = 1; in KTEST_FUNC()
1523 /* Attempt to run HPTS again - should detect collision */ in KTEST_FUNC()
1560 hpts = pace->rp_ent[0]; in KTEST_FUNC()
1562 /* Record initial generation count for the test slot */ in KTEST_FUNC()
1563 initial_gencnt = hpts->p_hptss[slot_to_test].gencnt; in KTEST_FUNC()
1568 tp1->t_hpts_cpu = 0; /* Force to CPU 0 */ in KTEST_FUNC()
1570 INP_WLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1571 tp1->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1573 INP_WUNLOCK(&tp1->t_inpcb); in KTEST_FUNC()
1576 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1577 KTEST_EQUAL(tp1->t_hpts_slot, slot_to_test); in KTEST_FUNC()
1578 KTEST_EQUAL(tp1->t_hpts_gencnt, initial_gencnt); in KTEST_FUNC()
1583 tp2->t_hpts_cpu = 0; /* Force to CPU 0 */ in KTEST_FUNC()
1598 KTEST_EQUAL(hpts->p_hptss[slot_to_test].gencnt, initial_gencnt + 1); in KTEST_FUNC()
1601 KTEST_EQUAL(tp1->t_in_hpts, IHPTS_NONE); in KTEST_FUNC()
1604 INP_WLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1605 tp2->t_flags2 |= TF2_HPTS_CALLS; in KTEST_FUNC()
1607 INP_WUNLOCK(&tp2->t_inpcb); in KTEST_FUNC()
1610 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_ONQUEUE); in KTEST_FUNC()
1613 tp2_original_gencnt = tp2->t_hpts_gencnt; in KTEST_FUNC()
1615 /* Test generation count mismatch detection during processing */ in KTEST_FUNC()
1617 tp2->t_hpts_gencnt = tp2_original_gencnt + 100; /* Force a mismatch */ in KTEST_FUNC()
1628 KTEST_EQUAL(tp2->t_in_hpts, IHPTS_NONE); /* Processed and released */ in KTEST_FUNC()
1630 /* The key test: HPTS should handle mismatched generation counts gracefully */ in KTEST_FUNC()