Lines Matching refs:hpts
218 #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED) argument
219 #define HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx) argument
220 #define HPTS_TRYLOCK(hpts) mtx_trylock(&(hpts)->p_mtx) argument
221 #define HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx) argument
287 static int32_t tcp_hptsi(struct tcp_hpts_entry *hpts, bool from_callout);
295 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
476 tcp_hpts_log(struct tcp_hpts_entry *hpts, struct tcpcb *tp, struct timeval *tv, in tcp_hpts_log() argument
488 log.u_bbr.flex1 = hpts->p_nxt_slot; in tcp_hpts_log()
489 log.u_bbr.flex2 = hpts->p_cur_slot; in tcp_hpts_log()
490 log.u_bbr.flex3 = hpts->p_prev_slot; in tcp_hpts_log()
492 log.u_bbr.flex5 = hpts->p_curtick; in tcp_hpts_log()
493 log.u_bbr.flex6 = hpts->p_on_queue_cnt; in tcp_hpts_log()
494 log.u_bbr.flex7 = hpts->p_cpu; in tcp_hpts_log()
497 log.u_bbr.applimited = hpts->overidden_sleep; in tcp_hpts_log()
498 log.u_bbr.delivered = hpts->saved_curtick; in tcp_hpts_log()
500 log.u_bbr.epoch = hpts->saved_curslot; in tcp_hpts_log()
501 log.u_bbr.lt_epoch = hpts->saved_prev_slot; in tcp_hpts_log()
502 log.u_bbr.pkts_out = hpts->p_delayed_by; in tcp_hpts_log()
503 log.u_bbr.lost = hpts->p_hpts_sleep_time; in tcp_hpts_log()
504 log.u_bbr.pacing_gain = hpts->p_cpu; in tcp_hpts_log()
505 log.u_bbr.pkt_epoch = hpts->p_runningslot; in tcp_hpts_log()
516 tcp_wakehpts(struct tcp_hpts_entry *hpts) in tcp_wakehpts() argument
518 HPTS_MTX_ASSERT(hpts); in tcp_wakehpts()
520 if (tcp_hpts_no_wake_over_thresh && (hpts->p_on_queue_cnt >= conn_cnt_thresh)) { in tcp_wakehpts()
521 hpts->p_direct_wake = 0; in tcp_wakehpts()
524 if (hpts->p_hpts_wake_scheduled == 0) { in tcp_wakehpts()
525 hpts->p_hpts_wake_scheduled = 1; in tcp_wakehpts()
526 swi_sched(hpts->ie_cookie, 0); in tcp_wakehpts()
533 struct tcp_hpts_entry *hpts; in hpts_timeout_swi() local
535 hpts = (struct tcp_hpts_entry *)arg; in hpts_timeout_swi()
536 swi_sched(hpts->ie_cookie, 0); in hpts_timeout_swi()
540 tcp_hpts_insert_internal(struct tcpcb *tp, struct tcp_hpts_entry *hpts) in tcp_hpts_insert_internal() argument
546 HPTS_MTX_ASSERT(hpts); in tcp_hpts_insert_internal()
547 MPASS(hpts->p_cpu == tp->t_hpts_cpu); in tcp_hpts_insert_internal()
550 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; in tcp_hpts_insert_internal()
563 hpts->p_on_queue_cnt++; in tcp_hpts_insert_internal()
569 struct tcp_hpts_entry *hpts; in tcp_hpts_lock() local
573 hpts = tcp_pace.rp_ent[tp->t_hpts_cpu]; in tcp_hpts_lock()
574 HPTS_LOCK(hpts); in tcp_hpts_lock()
576 return (hpts); in tcp_hpts_lock()
618 struct tcp_hpts_entry *hpts; in tcp_hpts_remove() local
623 hpts = tcp_hpts_lock(tp); in tcp_hpts_remove()
625 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; in tcp_hpts_remove()
631 MPASS(hpts->p_on_queue_cnt > 0); in tcp_hpts_remove()
632 hpts->p_on_queue_cnt--; in tcp_hpts_remove()
659 HPTS_UNLOCK(hpts); in tcp_hpts_remove()
719 max_slots_available(struct tcp_hpts_entry *hpts, uint32_t wheel_slot, uint32_t *target_slot) in max_slots_available() argument
723 if ((hpts->p_hpts_active == 1) && in max_slots_available()
724 (hpts->p_wheel_complete == 0)) { in max_slots_available()
725 end_slot = hpts->p_runningslot; in max_slots_available()
742 end_slot = hpts->p_prev_slot; in max_slots_available()
755 if (hpts->p_prev_slot != wheel_slot) in max_slots_available()
756 dis_to_travel = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); in max_slots_available()
775 if (hpts->p_runningslot == hpts->p_cur_slot) in max_slots_available()
778 dis_to_travel = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); in max_slots_available()
782 if (hpts->p_cur_slot != wheel_slot) { in max_slots_available()
784 pacer_to_now = hpts_slots_diff(hpts->p_cur_slot, wheel_slot); in max_slots_available()
809 *target_slot = hpts->p_nxt_slot; in max_slots_available()
828 check_if_slot_would_be_wrong(struct tcp_hpts_entry *hpts, struct tcpcb *tp, in check_if_slot_would_be_wrong() argument
836 ("hpts:%p tp:%p slot:%d > max", hpts, tp, hptsslot)); in check_if_slot_would_be_wrong()
837 if ((hpts->p_hpts_active) && in check_if_slot_would_be_wrong()
838 (hpts->p_wheel_complete == 0)) { in check_if_slot_would_be_wrong()
847 distance = hpts_slots_diff(hpts->p_runningslot, hptsslot); in check_if_slot_would_be_wrong()
848 if (hpts->p_runningslot != hpts->p_cur_slot) in check_if_slot_would_be_wrong()
849 yet_to_run = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); in check_if_slot_would_be_wrong()
853 "distance:%d yet_to_run:%d rs:%d cs:%d", hpts, tp, in check_if_slot_would_be_wrong()
854 hptsslot, distance, yet_to_run, hpts->p_runningslot, in check_if_slot_would_be_wrong()
855 hpts->p_cur_slot)); in check_if_slot_would_be_wrong()
863 struct tcp_hpts_entry *hpts; in tcp_hpts_insert_diag() local
878 hpts = tcp_hpts_lock(tp); in tcp_hpts_insert_diag()
882 diag->p_hpts_active = hpts->p_hpts_active; in tcp_hpts_insert_diag()
883 diag->p_prev_slot = hpts->p_prev_slot; in tcp_hpts_insert_diag()
884 diag->p_runningslot = hpts->p_runningslot; in tcp_hpts_insert_diag()
885 diag->p_nxt_slot = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
886 diag->p_cur_slot = hpts->p_cur_slot; in tcp_hpts_insert_diag()
887 diag->p_curtick = hpts->p_curtick; in tcp_hpts_insert_diag()
888 diag->p_lasttick = hpts->p_lasttick; in tcp_hpts_insert_diag()
890 diag->p_on_min_sleep = hpts->p_on_min_sleep; in tcp_hpts_insert_diag()
891 diag->hpts_sleep_time = hpts->p_hpts_sleep_time; in tcp_hpts_insert_diag()
896 if ((hpts->p_hpts_active == 0) || (hpts->p_wheel_complete)) { in tcp_hpts_insert_diag()
901 tp->t_hpts_slot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hpts_insert_diag()
902 if ((hpts->p_on_min_sleep == 0) && in tcp_hpts_insert_diag()
903 (hpts->p_hpts_active == 0)) in tcp_hpts_insert_diag()
906 tp->t_hpts_slot = hpts->p_runningslot; in tcp_hpts_insert_diag()
908 tcp_hpts_insert_internal(tp, hpts); in tcp_hpts_insert_diag()
914 hpts->p_direct_wake = 1; in tcp_hpts_insert_diag()
915 tcp_wakehpts(hpts); in tcp_hpts_insert_diag()
917 slot_on = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
918 HPTS_UNLOCK(hpts); in tcp_hpts_insert_diag()
927 maxslots = max_slots_available(hpts, wheel_slot, &last_slot); in tcp_hpts_insert_diag()
961 check_if_slot_would_be_wrong(hpts, tp, tp->t_hpts_slot, line); in tcp_hpts_insert_diag()
964 tcp_hpts_insert_internal(tp, hpts); in tcp_hpts_insert_diag()
965 if ((hpts->p_hpts_active == 0) && in tcp_hpts_insert_diag()
967 (hpts->p_on_min_sleep == 0)) { in tcp_hpts_insert_diag()
977 have_slept = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); in tcp_hpts_insert_diag()
978 if (have_slept < hpts->p_hpts_sleep_time) in tcp_hpts_insert_diag()
979 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; in tcp_hpts_insert_diag()
994 hpts->p_hpts_sleep_time = slot; in tcp_hpts_insert_diag()
1004 hpts->p_direct_wake = 1; in tcp_hpts_insert_diag()
1005 tcp_wakehpts(hpts); in tcp_hpts_insert_diag()
1023 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_insert_diag()
1024 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_insert_diag()
1031 slot_on = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
1032 HPTS_UNLOCK(hpts); in tcp_hpts_insert_diag()
1100 tcp_hpts_set_max_sleep(struct tcp_hpts_entry *hpts, int wrap_loop_cnt) in tcp_hpts_set_max_sleep() argument
1104 if ((hpts->p_on_queue_cnt) && (wrap_loop_cnt < 2)) { in tcp_hpts_set_max_sleep()
1109 for (i = 0, t = hpts_slot(hpts->p_cur_slot, 1); i < NUM_OF_HPTSI_SLOTS; i++) { in tcp_hpts_set_max_sleep()
1110 if (TAILQ_EMPTY(&hpts->p_hptss[t].head) == 0) { in tcp_hpts_set_max_sleep()
1115 KASSERT((i != NUM_OF_HPTSI_SLOTS), ("Hpts:%p cnt:%d but none found", hpts, hpts->p_on_queue_cnt)); in tcp_hpts_set_max_sleep()
1116 hpts->p_hpts_sleep_time = min((i + 1), hpts_sleep_max); in tcp_hpts_set_max_sleep()
1119 hpts->p_hpts_sleep_time = hpts_sleep_max; in tcp_hpts_set_max_sleep()
1124 tcp_hptsi(struct tcp_hpts_entry *hpts, bool from_callout) in tcp_hptsi() argument
1140 HPTS_MTX_ASSERT(hpts); in tcp_hptsi()
1143 hpts->saved_lasttick = hpts->p_lasttick; in tcp_hptsi()
1144 hpts->saved_curtick = hpts->p_curtick; in tcp_hptsi()
1145 hpts->saved_curslot = hpts->p_cur_slot; in tcp_hptsi()
1146 hpts->saved_prev_slot = hpts->p_prev_slot; in tcp_hptsi()
1148 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1149 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1150 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usec(&tv); in tcp_hptsi()
1151 orig_exit_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1152 if ((hpts->p_on_queue_cnt == 0) || in tcp_hptsi()
1153 (hpts->p_lasttick == hpts->p_curtick)) { in tcp_hptsi()
1158 hpts->p_prev_slot = hpts->p_cur_slot; in tcp_hptsi()
1159 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1163 hpts->p_wheel_complete = 0; in tcp_hptsi()
1164 HPTS_MTX_ASSERT(hpts); in tcp_hptsi()
1165 slots_to_run = hpts_slots_diff(hpts->p_prev_slot, hpts->p_cur_slot); in tcp_hptsi()
1166 if (((hpts->p_curtick - hpts->p_lasttick) > (NUM_OF_HPTSI_SLOTS - 1)) && in tcp_hptsi()
1167 (hpts->p_on_queue_cnt != 0)) { in tcp_hptsi()
1185 hpts->p_nxt_slot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hptsi()
1186 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 2); in tcp_hptsi()
1192 hpts->p_cur_slot = hpts->p_prev_slot; in tcp_hptsi()
1200 TAILQ_FOREACH(tp, &hpts->p_hptss[hpts->p_nxt_slot].head, in tcp_hptsi()
1202 MPASS(tp->t_hpts_slot == hpts->p_nxt_slot); in tcp_hptsi()
1204 hpts->p_hptss[hpts->p_nxt_slot].gencnt); in tcp_hptsi()
1214 hpts->p_hptss[hpts->p_runningslot].gencnt; in tcp_hptsi()
1215 tp->t_hpts_slot = hpts->p_runningslot; in tcp_hptsi()
1217 TAILQ_CONCAT(&hpts->p_hptss[hpts->p_runningslot].head, in tcp_hptsi()
1218 &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts); in tcp_hptsi()
1219 hpts->p_hptss[hpts->p_runningslot].count += in tcp_hptsi()
1220 hpts->p_hptss[hpts->p_nxt_slot].count; in tcp_hptsi()
1221 hpts->p_hptss[hpts->p_nxt_slot].count = 0; in tcp_hptsi()
1222 hpts->p_hptss[hpts->p_nxt_slot].gencnt++; in tcp_hptsi()
1230 hpts->p_nxt_slot = hpts->p_prev_slot; in tcp_hptsi()
1231 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hptsi()
1233 if (hpts->p_on_queue_cnt == 0) { in tcp_hptsi()
1246 hpts->p_delayed_by = (slots_to_run - (i + 1)) * in tcp_hptsi()
1249 runningslot = hpts->p_runningslot; in tcp_hptsi()
1250 hptsh = &hpts->p_hptss[runningslot]; in tcp_hptsi()
1252 hpts->p_on_queue_cnt -= hptsh->count; in tcp_hptsi()
1256 HPTS_UNLOCK(hpts); in tcp_hptsi()
1324 HPTS_LOCK(hpts); in tcp_hptsi()
1325 tcp_hpts_insert_internal(tp, hpts); in tcp_hptsi()
1326 HPTS_UNLOCK(hpts); in tcp_hptsi()
1336 hpts, inp, runningslot, tp->t_hpts_slot)); in tcp_hptsi()
1349 HPTS_LOCK(hpts); in tcp_hptsi()
1353 maxslots = max_slots_available(hpts, in tcp_hptsi()
1354 hpts->p_cur_slot, &last_slot); in tcp_hptsi()
1359 hpts->p_runningslot, in tcp_hptsi()
1368 tcp_hpts_insert_internal(tp, hpts); in tcp_hptsi()
1369 HPTS_UNLOCK(hpts); in tcp_hptsi()
1400 tcp_hpts_log(hpts, tp, &tv, slots_to_run, i, from_callout); in tcp_hptsi()
1444 HPTS_LOCK(hpts); in tcp_hptsi()
1445 hpts->p_runningslot++; in tcp_hptsi()
1446 if (hpts->p_runningslot >= NUM_OF_HPTSI_SLOTS) { in tcp_hptsi()
1447 hpts->p_runningslot = 0; in tcp_hptsi()
1451 HPTS_MTX_ASSERT(hpts); in tcp_hptsi()
1452 hpts->p_delayed_by = 0; in tcp_hptsi()
1457 hpts->p_prev_slot = hpts->p_cur_slot; in tcp_hptsi()
1458 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1478 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1479 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1482 orig_exit_slot = hpts->p_cur_slot; in tcp_hptsi()
1485 (hpts->p_lasttick != hpts->p_curtick)) { in tcp_hptsi()
1491 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usec(&tv); in tcp_hptsi()
1497 hpts->p_wheel_complete = 1; in tcp_hptsi()
1506 KASSERT(((hpts->p_prev_slot == hpts->p_cur_slot) || in tcp_hptsi()
1508 ("H:%p p_prev_slot:%u not equal to p_cur_slot:%u", hpts, in tcp_hptsi()
1509 hpts->p_prev_slot, hpts->p_cur_slot)); in tcp_hptsi()
1510 KASSERT(((hpts->p_lasttick == hpts->p_curtick) in tcp_hptsi()
1512 ("H:%p p_lasttick:%u not equal to p_curtick:%u", hpts, in tcp_hptsi()
1513 hpts->p_lasttick, hpts->p_curtick)); in tcp_hptsi()
1514 if (from_callout && (hpts->p_lasttick != hpts->p_curtick)) { in tcp_hptsi()
1515 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1517 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1522 tcp_hpts_set_max_sleep(hpts, wrap_loop_cnt); in tcp_hptsi()
1533 struct tcp_hpts_entry *hpts; in tcp_set_hpts() local
1538 hpts = tcp_hpts_lock(tp); in tcp_set_hpts()
1544 HPTS_UNLOCK(hpts); in tcp_set_hpts()
1592 struct tcp_hpts_entry *hpts; in __tcp_run_hpts() local
1595 hpts = tcp_choose_hpts_to_run(); in __tcp_run_hpts()
1597 if (hpts->p_hpts_active) { in __tcp_run_hpts()
1601 if (!HPTS_TRYLOCK(hpts)) { in __tcp_run_hpts()
1606 if (hpts->p_hpts_active) in __tcp_run_hpts()
1608 hpts->syscall_cnt++; in __tcp_run_hpts()
1610 hpts->p_hpts_active = 1; in __tcp_run_hpts()
1611 ticks_ran = tcp_hptsi(hpts, false); in __tcp_run_hpts()
1613 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in __tcp_run_hpts()
1618 hpts->p_mysleep.tv_usec /= 2; in __tcp_run_hpts()
1619 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) in __tcp_run_hpts()
1620 hpts->p_mysleep.tv_usec = dynamic_min_sleep; in __tcp_run_hpts()
1622 tcp_hpts_set_max_sleep(hpts, 0); in __tcp_run_hpts()
1624 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_USECS_PER_SLOT; in __tcp_run_hpts()
1626 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { in __tcp_run_hpts()
1627 hpts->overidden_sleep = tv.tv_usec; in __tcp_run_hpts()
1628 tv.tv_usec = hpts->p_mysleep.tv_usec; in __tcp_run_hpts()
1631 hpts->overidden_sleep = tv.tv_usec; in __tcp_run_hpts()
1642 hpts->sleeping = tv.tv_usec; in __tcp_run_hpts()
1643 callout_reset_sbt_on(&hpts->co, sb, 0, in __tcp_run_hpts()
1644 hpts_timeout_swi, hpts, hpts->p_cpu, in __tcp_run_hpts()
1648 hpts->p_mysleep.tv_usec *= 2; in __tcp_run_hpts()
1649 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in __tcp_run_hpts()
1650 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in __tcp_run_hpts()
1652 hpts->p_on_min_sleep = 1; in __tcp_run_hpts()
1654 hpts->p_hpts_active = 0; in __tcp_run_hpts()
1656 HPTS_UNLOCK(hpts); in __tcp_run_hpts()
1663 struct tcp_hpts_entry *hpts; in tcp_hpts_thread() local
1669 hpts = (struct tcp_hpts_entry *)ctx; in tcp_hpts_thread()
1670 HPTS_LOCK(hpts); in tcp_hpts_thread()
1671 if (hpts->p_direct_wake) { in tcp_hpts_thread()
1673 callout_stop(&hpts->co); in tcp_hpts_thread()
1678 if (callout_pending(&hpts->co) || in tcp_hpts_thread()
1679 !callout_active(&hpts->co)) { in tcp_hpts_thread()
1680 HPTS_UNLOCK(hpts); in tcp_hpts_thread()
1684 callout_deactivate(&hpts->co); in tcp_hpts_thread()
1685 hpts->p_hpts_wake_scheduled = 0; in tcp_hpts_thread()
1687 if (hpts->p_hpts_active) { in tcp_hpts_thread()
1696 if (hpts->p_direct_wake == 0) { in tcp_hpts_thread()
1698 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in tcp_hpts_thread()
1699 hpts->p_mysleep.tv_usec *= 2; in tcp_hpts_thread()
1700 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in tcp_hpts_thread()
1701 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in tcp_hpts_thread()
1702 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1703 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1712 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1720 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1724 hpts->sleeping = 0; in tcp_hpts_thread()
1725 hpts->p_hpts_active = 1; in tcp_hpts_thread()
1726 ticks_ran = tcp_hptsi(hpts, true); in tcp_hpts_thread()
1728 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_USECS_PER_SLOT; in tcp_hpts_thread()
1729 if ((hpts->p_on_queue_cnt > conn_cnt_thresh) && (hpts->hit_callout_thresh == 0)) { in tcp_hpts_thread()
1730 hpts->hit_callout_thresh = 1; in tcp_hpts_thread()
1732 } else if ((hpts->p_on_queue_cnt <= conn_cnt_thresh) && (hpts->hit_callout_thresh == 1)) { in tcp_hpts_thread()
1733 hpts->hit_callout_thresh = 0; in tcp_hpts_thread()
1736 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in tcp_hpts_thread()
1737 if(hpts->p_direct_wake == 0) { in tcp_hpts_thread()
1743 hpts->p_mysleep.tv_usec *= 2; in tcp_hpts_thread()
1744 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in tcp_hpts_thread()
1745 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in tcp_hpts_thread()
1747 hpts->p_mysleep.tv_usec /= 2; in tcp_hpts_thread()
1748 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) in tcp_hpts_thread()
1749 hpts->p_mysleep.tv_usec = dynamic_min_sleep; in tcp_hpts_thread()
1752 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { in tcp_hpts_thread()
1753 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1754 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1757 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1766 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1767 } else if (hpts->p_on_queue_cnt == 0) { in tcp_hpts_thread()
1772 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1773 hpts->overidden_sleep = 0; in tcp_hpts_thread()
1789 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1791 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1794 hpts->overidden_sleep = 0; in tcp_hpts_thread()
1795 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1798 HPTS_MTX_ASSERT(hpts); in tcp_hpts_thread()
1799 hpts->p_hpts_active = 0; in tcp_hpts_thread()
1801 hpts->p_direct_wake = 0; in tcp_hpts_thread()
1804 hpts->sleeping = tv.tv_usec; in tcp_hpts_thread()
1805 callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_thread()
1806 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_thread()
1809 HPTS_UNLOCK(hpts); in tcp_hpts_thread()
1859 struct tcp_hpts_entry *hpts; in tcp_hpts_mod_load() local
1923 hpts = tcp_pace.rp_ent[i]; in tcp_hpts_mod_load()
1929 mtx_init(&hpts->p_mtx, "tcp_hpts_lck", in tcp_hpts_mod_load()
1932 TAILQ_INIT(&hpts->p_hptss[j].head); in tcp_hpts_mod_load()
1933 hpts->p_hptss[j].count = 0; in tcp_hpts_mod_load()
1934 hpts->p_hptss[j].gencnt = 0; in tcp_hpts_mod_load()
1936 sysctl_ctx_init(&hpts->hpts_ctx); in tcp_hpts_mod_load()
1938 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1944 SYSCTL_ADD_INT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1945 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1947 &hpts->p_on_queue_cnt, 0, in tcp_hpts_mod_load()
1949 SYSCTL_ADD_U16(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1950 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1952 &hpts->p_hpts_active, 0, in tcp_hpts_mod_load()
1954 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1955 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1957 &hpts->p_cur_slot, 0, in tcp_hpts_mod_load()
1959 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1960 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1962 &hpts->p_runningslot, 0, in tcp_hpts_mod_load()
1964 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1965 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1967 &hpts->p_curtick, 0, in tcp_hpts_mod_load()
1969 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1970 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1974 SYSCTL_ADD_LONG(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1975 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1977 &hpts->p_mysleep.tv_usec, in tcp_hpts_mod_load()
1979 SYSCTL_ADD_U64(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1980 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1982 &hpts->sleeping, 0, in tcp_hpts_mod_load()
1984 SYSCTL_ADD_U64(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1985 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1987 &hpts->syscall_cnt, 0, in tcp_hpts_mod_load()
1990 hpts->p_hpts_sleep_time = hpts_sleep_max; in tcp_hpts_mod_load()
1991 hpts->p_num = i; in tcp_hpts_mod_load()
1992 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hpts_mod_load()
1994 hpts->p_prev_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hpts_mod_load()
1995 hpts->p_cpu = 0xffff; in tcp_hpts_mod_load()
1996 hpts->p_nxt_slot = hpts_slot(hpts->p_cur_slot, 1); in tcp_hpts_mod_load()
1997 callout_init(&hpts->co, 1); in tcp_hpts_mod_load()
2007 hpts = tcp_pace.rp_ent[i]; in tcp_hpts_mod_load()
2008 hpts->p_cpu = i; in tcp_hpts_mod_load()
2010 error = swi_add(&hpts->ie, "hpts", in tcp_hpts_mod_load()
2011 tcp_hpts_thread, (void *)hpts, in tcp_hpts_mod_load()
2012 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); in tcp_hpts_mod_load()
2015 hpts, i, error)); in tcp_hpts_mod_load()
2017 hpts->p_mysleep.tv_sec = 0; in tcp_hpts_mod_load()
2018 hpts->p_mysleep.tv_usec = tcp_min_hptsi_time; in tcp_hpts_mod_load()
2020 if (intr_event_bind(hpts->ie, i) == 0) in tcp_hpts_mod_load()
2026 if (intr_event_bind_ithread_cpuset(hpts->ie, in tcp_hpts_mod_load()
2040 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_USECS_PER_SLOT; in tcp_hpts_mod_load()
2041 hpts->sleeping = tv.tv_usec; in tcp_hpts_mod_load()
2043 callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_mod_load()
2044 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_mod_load()
2073 struct tcp_hpts_entry *hpts = tcp_pace.rp_ent[i]; in tcp_hpts_mod_unload() local
2075 rv = callout_drain(&hpts->co); in tcp_hpts_mod_unload()
2078 rv = swi_remove(hpts->ie_cookie); in tcp_hpts_mod_unload()
2081 rv = sysctl_ctx_free(&hpts->hpts_ctx); in tcp_hpts_mod_unload()
2084 mtx_destroy(&hpts->p_mtx); in tcp_hpts_mod_unload()
2085 free(hpts->p_hptss, M_TCPHPTS); in tcp_hpts_mod_unload()
2086 free(hpts, M_TCPHPTS); in tcp_hpts_mod_unload()