Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 /*-
2 * Copyright (c) 2016-2018 Netflix, Inc.
34 * The tcp_hpts system is designed to provide a high precision timer
84 * need to wake up the hpts system to get immediate
87 * the current hpts timer run (this is usually set when
88 * a rack timer is up so we know SACK's are happening
93 * knows how to take the input queue of packets from tp->t_inqueue
163 * This gives a range of 10usec - 1024ms to place
166 * when seeing the remainder will re-insert the
174 #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED)
175 #define HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx)
176 #define HPTS_TRYLOCK(hpts) mtx_trylock(&(hpts)->p_mtx)
177 #define HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx)
208 uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */
219 uint16_t p_num; /* The hpts number one per cpu */
220 uint16_t p_cpu; /* The hpts CPU */
259 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \
260 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \
261 if ((vvp)->tv_usec < 0) { \
262 (vvp)->tv_sec--; \
263 (vvp)->tv_usec += 1000000; \
271 int cpu[MAXCPU]; member
334 "Use of irq CPU tunable");
368 if (error == 0 && req->newptr) { in sysctl_net_inet_tcp_hpts_max_sleep()
386 if (error == 0 && req->newptr) { in sysctl_net_inet_tcp_hpts_min_sleep()
439 * 64 bit - delRate, rttProp, bw_inuse in tcp_hpts_log()
440 * 16 bit - cwnd_gain in tcp_hpts_log()
441 * 8 bit - bbr_state, bbr_substate, inhpts; in tcp_hpts_log()
444 log.u_bbr.flex1 = hpts->p_nxt_slot; in tcp_hpts_log()
445 log.u_bbr.flex2 = hpts->p_cur_slot; in tcp_hpts_log()
446 log.u_bbr.flex3 = hpts->p_prev_slot; in tcp_hpts_log()
448 log.u_bbr.flex5 = hpts->p_curtick; in tcp_hpts_log()
449 log.u_bbr.flex6 = hpts->p_on_queue_cnt; in tcp_hpts_log()
450 log.u_bbr.flex7 = hpts->p_cpu; in tcp_hpts_log()
453 log.u_bbr.applimited = hpts->overidden_sleep; in tcp_hpts_log()
454 log.u_bbr.delivered = hpts->saved_curtick; in tcp_hpts_log()
456 log.u_bbr.epoch = hpts->saved_curslot; in tcp_hpts_log()
457 log.u_bbr.lt_epoch = hpts->saved_prev_slot; in tcp_hpts_log()
458 log.u_bbr.pkts_out = hpts->p_delayed_by; in tcp_hpts_log()
459 log.u_bbr.lost = hpts->p_hpts_sleep_time; in tcp_hpts_log()
460 log.u_bbr.pacing_gain = hpts->p_cpu; in tcp_hpts_log()
461 log.u_bbr.pkt_epoch = hpts->p_runningslot; in tcp_hpts_log()
464 &tptosocket(tp)->so_rcv, in tcp_hpts_log()
465 &tptosocket(tp)->so_snd, in tcp_hpts_log()
475 if (tcp_hpts_no_wake_over_thresh && (hpts->p_on_queue_cnt >= conn_cnt_thresh)) { in tcp_wakehpts()
476 hpts->p_direct_wake = 0; in tcp_wakehpts()
479 if (hpts->p_hpts_wake_scheduled == 0) { in tcp_wakehpts()
480 hpts->p_hpts_wake_scheduled = 1; in tcp_wakehpts()
481 swi_sched(hpts->ie_cookie, 0); in tcp_wakehpts()
491 swi_sched(hpts->ie_cookie, 0); in hpts_timeout_swi()
502 MPASS(hpts->p_cpu == tp->t_hpts_cpu); in tcp_hpts_insert_internal()
503 MPASS(!(inp->inp_flags & INP_DROPPED)); in tcp_hpts_insert_internal()
505 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; in tcp_hpts_insert_internal()
507 if (tp->t_in_hpts == IHPTS_NONE) { in tcp_hpts_insert_internal()
508 tp->t_in_hpts = IHPTS_ONQUEUE; in tcp_hpts_insert_internal()
510 } else if (tp->t_in_hpts == IHPTS_MOVING) { in tcp_hpts_insert_internal()
511 tp->t_in_hpts = IHPTS_ONQUEUE; in tcp_hpts_insert_internal()
513 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); in tcp_hpts_insert_internal()
514 tp->t_hpts_gencnt = hptsh->gencnt; in tcp_hpts_insert_internal()
516 TAILQ_INSERT_TAIL(&hptsh->head, tp, t_hpts); in tcp_hpts_insert_internal()
517 hptsh->count++; in tcp_hpts_insert_internal()
518 hpts->p_on_queue_cnt++; in tcp_hpts_insert_internal()
528 hpts = tcp_pace.rp_ent[tp->t_hpts_cpu]; in tcp_hpts_lock()
539 tp->t_in_hpts = IHPTS_NONE; in tcp_hpts_release()
545 * Initialize tcpcb to get ready for use with HPTS. We will know which CPU
547 * a single CPU with newborn connections and use a random one.
549 * can be called once again if stack is switched. In that case we inherit CPU
558 if (__predict_true(tp->t_hpts_cpu == HPTS_CPU_NONE)) { in tcp_hpts_init()
559 tp->t_hpts_cpu = hpts_random_cpu(); in tcp_hpts_init()
560 MPASS(!(tp->t_flags2 & TF2_HPTS_CPU_SET)); in tcp_hpts_init()
579 if (tp->t_in_hpts == IHPTS_ONQUEUE) { in tcp_hpts_remove()
580 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; in tcp_hpts_remove()
581 tp->t_hpts_request = 0; in tcp_hpts_remove()
582 if (__predict_true(tp->t_hpts_gencnt == hptsh->gencnt)) { in tcp_hpts_remove()
583 TAILQ_REMOVE(&hptsh->head, tp, t_hpts); in tcp_hpts_remove()
584 MPASS(hptsh->count > 0); in tcp_hpts_remove()
585 hptsh->count--; in tcp_hpts_remove()
586 MPASS(hpts->p_on_queue_cnt > 0); in tcp_hpts_remove()
587 hpts->p_on_queue_cnt--; in tcp_hpts_remove()
597 TAILQ_FOREACH(tmp, &hptsh->head, t_hpts) in tcp_hpts_remove()
600 tp->t_in_hpts = IHPTS_MOVING; in tcp_hpts_remove()
601 tp->t_hpts_slot = -1; in tcp_hpts_remove()
603 } else if (tp->t_in_hpts == IHPTS_MOVING) { in tcp_hpts_remove()
607 * tcp_hpts_remove() marks as IHPTS_MOVING, slot = -1 in tcp_hpts_remove()
612 tp->t_hpts_slot = -1; in tcp_hpts_remove()
649 return (slot_now - prev_slot); in hpts_slots_diff()
655 return (NUM_OF_HPTSI_SLOTS - 1); in hpts_slots_diff()
657 return ((NUM_OF_HPTSI_SLOTS - prev_slot) + slot_now); in hpts_slots_diff()
678 if ((hpts->p_hpts_active == 1) && in max_slots_available()
679 (hpts->p_wheel_complete == 0)) { in max_slots_available()
680 end_slot = hpts->p_runningslot; in max_slots_available()
683 end_slot = NUM_OF_HPTSI_SLOTS - 1; in max_slots_available()
685 end_slot--; in max_slots_available()
697 end_slot = hpts->p_prev_slot; in max_slots_available()
699 end_slot = NUM_OF_HPTSI_SLOTS - 1; in max_slots_available()
701 end_slot--; in max_slots_available()
710 if (hpts->p_prev_slot != wheel_slot) in max_slots_available()
711 dis_to_travel = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); in max_slots_available()
722 return (NUM_OF_HPTSI_SLOTS - dis_to_travel); in max_slots_available()
725 * So how many slots are open between p_runningslot -> p_cur_slot in max_slots_available()
726 * that is what is currently un-available for insertion. Special in max_slots_available()
730 if (hpts->p_runningslot == hpts->p_cur_slot) in max_slots_available()
733 dis_to_travel = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); in max_slots_available()
737 if (hpts->p_cur_slot != wheel_slot) { in max_slots_available()
739 pacer_to_now = hpts_slots_diff(hpts->p_cur_slot, wheel_slot); in max_slots_available()
749 avail_on_wheel = NUM_OF_HPTSI_SLOTS - dis_to_travel; in max_slots_available()
764 *target_slot = hpts->p_nxt_slot; in max_slots_available()
776 return (avail_on_wheel - pacer_to_now); in max_slots_available()
792 if ((hpts->p_hpts_active) && in check_if_slot_would_be_wrong()
793 (hpts->p_wheel_complete == 0)) { in check_if_slot_would_be_wrong()
802 distance = hpts_slots_diff(hpts->p_runningslot, hptsslot); in check_if_slot_would_be_wrong()
803 if (hpts->p_runningslot != hpts->p_cur_slot) in check_if_slot_would_be_wrong()
804 yet_to_run = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); in check_if_slot_would_be_wrong()
809 hptsslot, distance, yet_to_run, hpts->p_runningslot, in check_if_slot_would_be_wrong()
810 hpts->p_cur_slot)); in check_if_slot_would_be_wrong()
825 MPASS(!(tptoinpcb(tp)->inp_flags & INP_DROPPED)); in tcp_hpts_insert_diag()
826 MPASS(!(tp->t_in_hpts == IHPTS_ONQUEUE)); in tcp_hpts_insert_diag()
829 * We now return the next-slot the hpts will be on, beyond its in tcp_hpts_insert_diag()
837 diag->p_hpts_active = hpts->p_hpts_active; in tcp_hpts_insert_diag()
838 diag->p_prev_slot = hpts->p_prev_slot; in tcp_hpts_insert_diag()
839 diag->p_runningslot = hpts->p_runningslot; in tcp_hpts_insert_diag()
840 diag->p_nxt_slot = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
841 diag->p_cur_slot = hpts->p_cur_slot; in tcp_hpts_insert_diag()
842 diag->p_curtick = hpts->p_curtick; in tcp_hpts_insert_diag()
843 diag->p_lasttick = hpts->p_lasttick; in tcp_hpts_insert_diag()
844 diag->slot_req = slot; in tcp_hpts_insert_diag()
845 diag->p_on_min_sleep = hpts->p_on_min_sleep; in tcp_hpts_insert_diag()
846 diag->hpts_sleep_time = hpts->p_hpts_sleep_time; in tcp_hpts_insert_diag()
850 tp->t_hpts_request = 0; in tcp_hpts_insert_diag()
851 if ((hpts->p_hpts_active == 0) || (hpts->p_wheel_complete)) { in tcp_hpts_insert_diag()
856 tp->t_hpts_slot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hpts_insert_diag()
857 if ((hpts->p_on_min_sleep == 0) && in tcp_hpts_insert_diag()
858 (hpts->p_hpts_active == 0)) in tcp_hpts_insert_diag()
861 tp->t_hpts_slot = hpts->p_runningslot; in tcp_hpts_insert_diag()
862 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) in tcp_hpts_insert_diag()
869 hpts->p_direct_wake = 1; in tcp_hpts_insert_diag()
872 slot_on = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
884 diag->wheel_slot = wheel_slot; in tcp_hpts_insert_diag()
885 diag->maxslots = maxslots; in tcp_hpts_insert_diag()
886 diag->wheel_cts = wheel_cts; in tcp_hpts_insert_diag()
898 slot--; in tcp_hpts_insert_diag()
900 tp->t_hpts_slot = last_slot; in tcp_hpts_insert_diag()
901 tp->t_hpts_request = slot; in tcp_hpts_insert_diag()
904 tp->t_hpts_request = 0; in tcp_hpts_insert_diag()
905 tp->t_hpts_slot = hpts_slot(wheel_slot, slot); in tcp_hpts_insert_diag()
908 tp->t_hpts_request = slot - maxslots; in tcp_hpts_insert_diag()
909 tp->t_hpts_slot = last_slot; in tcp_hpts_insert_diag()
912 diag->slot_remaining = tp->t_hpts_request; in tcp_hpts_insert_diag()
913 diag->inp_hptsslot = tp->t_hpts_slot; in tcp_hpts_insert_diag()
916 check_if_slot_would_be_wrong(hpts, tp, tp->t_hpts_slot, line); in tcp_hpts_insert_diag()
918 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) in tcp_hpts_insert_diag()
920 if ((hpts->p_hpts_active == 0) && in tcp_hpts_insert_diag()
921 (tp->t_hpts_request == 0) && in tcp_hpts_insert_diag()
922 (hpts->p_on_min_sleep == 0)) { in tcp_hpts_insert_diag()
926 * it will wake up at and if we need to reschedule in tcp_hpts_insert_diag()
927 * its time-out. in tcp_hpts_insert_diag()
931 /* Now do we need to restart the hpts's timer? */ in tcp_hpts_insert_diag()
932 have_slept = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); in tcp_hpts_insert_diag()
933 if (have_slept < hpts->p_hpts_sleep_time) in tcp_hpts_insert_diag()
934 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; in tcp_hpts_insert_diag()
936 /* We are over-due */ in tcp_hpts_insert_diag()
941 diag->have_slept = have_slept; in tcp_hpts_insert_diag()
942 diag->yet_to_sleep = yet_to_sleep; in tcp_hpts_insert_diag()
947 * We need to reschedule the hpts's time-out. in tcp_hpts_insert_diag()
949 hpts->p_hpts_sleep_time = slot; in tcp_hpts_insert_diag()
959 hpts->p_direct_wake = 1; in tcp_hpts_insert_diag()
962 diag->need_new_to = 0; in tcp_hpts_insert_diag()
963 diag->co_ret = 0xffff0000; in tcp_hpts_insert_diag()
974 need_new_to -= HPTS_USEC_IN_SEC; in tcp_hpts_insert_diag()
978 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_insert_diag()
979 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_insert_diag()
982 diag->need_new_to = need_new_to; in tcp_hpts_insert_diag()
983 diag->co_ret = co_ret; in tcp_hpts_insert_diag()
986 slot_on = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
1002 if (tp->t_flags2 & TF2_HPTS_CPU_SET) { in hpts_cpuid()
1003 return (tp->t_hpts_cpu); in hpts_cpuid()
1006 * If we are using the irq cpu set by LRO or in hpts_cpuid()
1010 if (tp->t_lro_cpu == HPTS_CPU_NONE) { in hpts_cpuid()
1014 return (tp->t_lro_cpu); in hpts_cpuid()
1018 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); in hpts_cpuid()
1025 * We don't have a flowid -> cpuid mapping, so cheat and just map in hpts_cpuid()
1029 if (inp->inp_flowtype == M_HASHTYPE_NONE) { in hpts_cpuid()
1040 (inp->inp_numa_domain == M_NODOM)) { in hpts_cpuid()
1042 cpuid = inp->inp_flowid % mp_ncpus; in hpts_cpuid()
1045 /* Hash into the cpu's that use that domain */ in hpts_cpuid()
1046 di = &hpts_domains[inp->inp_numa_domain]; in hpts_cpuid()
1047 cpuid = di->cpu[inp->inp_flowid % di->count]; in hpts_cpuid()
1059 if ((hpts->p_on_queue_cnt) && (wrap_loop_cnt < 2)) { in tcp_hpts_set_max_sleep()
1064 for (i = 0, t = hpts_slot(hpts->p_cur_slot, 1); i < NUM_OF_HPTSI_SLOTS; i++) { in tcp_hpts_set_max_sleep()
1065 if (TAILQ_EMPTY(&hpts->p_hptss[t].head) == 0) { in tcp_hpts_set_max_sleep()
1070 KASSERT((i != NUM_OF_HPTSI_SLOTS), ("Hpts:%p cnt:%d but none found", hpts, hpts->p_on_queue_cnt)); in tcp_hpts_set_max_sleep()
1071 hpts->p_hpts_sleep_time = min((i + 1), hpts_sleep_max); in tcp_hpts_set_max_sleep()
1074 hpts->p_hpts_sleep_time = hpts_sleep_max; in tcp_hpts_set_max_sleep()
1098 hpts->saved_lasttick = hpts->p_lasttick; in tcp_hptsi()
1099 hpts->saved_curtick = hpts->p_curtick; in tcp_hptsi()
1100 hpts->saved_curslot = hpts->p_cur_slot; in tcp_hptsi()
1101 hpts->saved_prev_slot = hpts->p_prev_slot; in tcp_hptsi()
1103 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1104 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1105 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv); in tcp_hptsi()
1106 orig_exit_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1107 if ((hpts->p_on_queue_cnt == 0) || in tcp_hptsi()
1108 (hpts->p_lasttick == hpts->p_curtick)) { in tcp_hptsi()
1113 hpts->p_prev_slot = hpts->p_cur_slot; in tcp_hptsi()
1114 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1118 hpts->p_wheel_complete = 0; in tcp_hptsi()
1120 slots_to_run = hpts_slots_diff(hpts->p_prev_slot, hpts->p_cur_slot); in tcp_hptsi()
1121 if (((hpts->p_curtick - hpts->p_lasttick) > in tcp_hptsi()
1122 ((NUM_OF_HPTSI_SLOTS-1) * HPTS_TICKS_PER_SLOT)) && in tcp_hptsi()
1123 (hpts->p_on_queue_cnt != 0)) { in tcp_hptsi()
1141 hpts->p_nxt_slot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hptsi()
1142 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 2); in tcp_hptsi()
1148 hpts->p_cur_slot = hpts->p_prev_slot; in tcp_hptsi()
1156 TAILQ_FOREACH(tp, &hpts->p_hptss[hpts->p_nxt_slot].head, in tcp_hptsi()
1158 MPASS(tp->t_hpts_slot == hpts->p_nxt_slot); in tcp_hptsi()
1159 MPASS(tp->t_hpts_gencnt == in tcp_hptsi()
1160 hpts->p_hptss[hpts->p_nxt_slot].gencnt); in tcp_hptsi()
1161 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); in tcp_hptsi()
1169 tp->t_hpts_gencnt = in tcp_hptsi()
1170 hpts->p_hptss[hpts->p_runningslot].gencnt; in tcp_hptsi()
1171 tp->t_hpts_slot = hpts->p_runningslot; in tcp_hptsi()
1173 TAILQ_CONCAT(&hpts->p_hptss[hpts->p_runningslot].head, in tcp_hptsi()
1174 &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts); in tcp_hptsi()
1175 hpts->p_hptss[hpts->p_runningslot].count += in tcp_hptsi()
1176 hpts->p_hptss[hpts->p_nxt_slot].count; in tcp_hptsi()
1177 hpts->p_hptss[hpts->p_nxt_slot].count = 0; in tcp_hptsi()
1178 hpts->p_hptss[hpts->p_nxt_slot].gencnt++; in tcp_hptsi()
1179 slots_to_run = NUM_OF_HPTSI_SLOTS - 1; in tcp_hptsi()
1186 hpts->p_nxt_slot = hpts->p_prev_slot; in tcp_hptsi()
1187 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hptsi()
1189 if (hpts->p_on_queue_cnt == 0) { in tcp_hptsi()
1202 hpts->p_delayed_by = (slots_to_run - (i + 1)) * in tcp_hptsi()
1205 runningslot = hpts->p_runningslot; in tcp_hptsi()
1206 hptsh = &hpts->p_hptss[runningslot]; in tcp_hptsi()
1207 TAILQ_SWAP(&head, &hptsh->head, tcpcb, t_hpts); in tcp_hptsi()
1208 hpts->p_on_queue_cnt -= hptsh->count; in tcp_hptsi()
1209 hptsh->count = 0; in tcp_hptsi()
1210 hptsh->gencnt++; in tcp_hptsi()
1233 * re-used and something else (not an in tcp_hptsi()
1237 * and we tried to pre-fetch it). in tcp_hptsi()
1268 if ((tp->t_flags2 & TF2_HPTS_CPU_SET) == 0) { in tcp_hptsi()
1274 if (__predict_false(tp->t_in_hpts == IHPTS_MOVING)) { in tcp_hptsi()
1275 if (tp->t_hpts_slot == -1) { in tcp_hptsi()
1276 tp->t_in_hpts = IHPTS_NONE; in tcp_hptsi()
1288 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); in tcp_hptsi()
1289 MPASS(!(inp->inp_flags & INP_DROPPED)); in tcp_hptsi()
1290 KASSERT(runningslot == tp->t_hpts_slot, in tcp_hptsi()
1291 ("Hpts:%p inp:%p slot mis-aligned %u vs %u", in tcp_hptsi()
1292 hpts, inp, runningslot, tp->t_hpts_slot)); in tcp_hptsi()
1294 if (tp->t_hpts_request) { in tcp_hptsi()
1303 remaining_slots = slots_to_run - (i + 1); in tcp_hptsi()
1304 if (tp->t_hpts_request > remaining_slots) { in tcp_hptsi()
1310 hpts->p_cur_slot, &last_slot); in tcp_hptsi()
1311 if (maxslots >= tp->t_hpts_request) { in tcp_hptsi()
1314 tp->t_hpts_slot = hpts_slot( in tcp_hptsi()
1315 hpts->p_runningslot, in tcp_hptsi()
1316 tp->t_hpts_request); in tcp_hptsi()
1317 tp->t_hpts_request = 0; in tcp_hptsi()
1320 tp->t_hpts_slot = last_slot; in tcp_hptsi()
1321 tp->t_hpts_request -= in tcp_hptsi()
1329 tp->t_hpts_request = 0; in tcp_hptsi()
1337 * the right CPU. This should be a rare in tcp_hptsi()
1346 * goes off and sees the mis-match. We in tcp_hptsi()
1347 * simply correct it here and the CPU will in tcp_hptsi()
1350 * :-) in tcp_hptsi()
1354 CURVNET_SET(inp->inp_vnet); in tcp_hptsi()
1361 if (tp->t_fb_ptr != NULL) { in tcp_hptsi()
1362 kern_prefetch(tp->t_fb_ptr, &did_prefetch); in tcp_hptsi()
1382 tp->t_flags2 |= TF2_HPTS_CALLS; in tcp_hptsi()
1383 if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) && in tcp_hptsi()
1384 !STAILQ_EMPTY(&tp->t_inqueue)) in tcp_hptsi()
1385 error = -(*tp->t_fb->tfb_do_queued_segments)(tp, in tcp_hptsi()
1396 * slot_pos_of_endpoint <-> orig_exit_slot in tcp_hptsi()
1404 hpts->p_runningslot++; in tcp_hptsi()
1405 if (hpts->p_runningslot >= NUM_OF_HPTSI_SLOTS) { in tcp_hptsi()
1406 hpts->p_runningslot = 0; in tcp_hptsi()
1411 hpts->p_delayed_by = 0; in tcp_hptsi()
1414 * more ticks (if we did not hit eno-bufs). in tcp_hptsi()
1416 hpts->p_prev_slot = hpts->p_cur_slot; in tcp_hptsi()
1417 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1437 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1438 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1441 orig_exit_slot = hpts->p_cur_slot; in tcp_hptsi()
1444 (hpts->p_lasttick != hpts->p_curtick)) { in tcp_hptsi()
1450 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usectick(&tv); in tcp_hptsi()
1456 hpts->p_wheel_complete = 1; in tcp_hptsi()
1465 KASSERT(((hpts->p_prev_slot == hpts->p_cur_slot) || in tcp_hptsi()
1468 hpts->p_prev_slot, hpts->p_cur_slot)); in tcp_hptsi()
1469 KASSERT(((hpts->p_lasttick == hpts->p_curtick) in tcp_hptsi()
1472 hpts->p_lasttick, hpts->p_curtick)); in tcp_hptsi()
1473 if (from_callout && (hpts->p_lasttick != hpts->p_curtick)) { in tcp_hptsi()
1474 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1476 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1498 if (tp->t_in_hpts == IHPTS_NONE && !(tp->t_flags2 & TF2_HPTS_CPU_SET)) { in __tcp_set_hpts()
1499 tp->t_hpts_cpu = hpts_cpuid(tp, &failed); in __tcp_set_hpts()
1501 tp->t_flags2 |= TF2_HPTS_CPU_SET; in __tcp_set_hpts()
1519 * this CPU is in. in tcp_choose_hpts_to_run()
1523 if (CPU_ISSET(curcpu, &tcp_pace.grps[i]->cg_mask)) { in tcp_choose_hpts_to_run()
1524 start = tcp_pace.grps[i]->cg_first; in tcp_choose_hpts_to_run()
1525 end = (tcp_pace.grps[i]->cg_last + 1); in tcp_choose_hpts_to_run()
1530 oldest_idx = -1; in tcp_choose_hpts_to_run()
1533 calc = cts - tcp_pace.cts_last_ran[i]; in tcp_choose_hpts_to_run()
1556 if (hpts->p_hpts_active) { in __tcp_run_hpts()
1565 if (hpts->p_hpts_active) in __tcp_run_hpts()
1567 hpts->syscall_cnt++; in __tcp_run_hpts()
1569 hpts->p_hpts_active = 1; in __tcp_run_hpts()
1572 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in __tcp_run_hpts()
1577 hpts->p_mysleep.tv_usec /= 2; in __tcp_run_hpts()
1578 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) in __tcp_run_hpts()
1579 hpts->p_mysleep.tv_usec = dynamic_min_sleep; in __tcp_run_hpts()
1583 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; in __tcp_run_hpts()
1585 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { in __tcp_run_hpts()
1586 hpts->overidden_sleep = tv.tv_usec; in __tcp_run_hpts()
1587 tv.tv_usec = hpts->p_mysleep.tv_usec; in __tcp_run_hpts()
1590 hpts->overidden_sleep = tv.tv_usec; in __tcp_run_hpts()
1594 * In this mode the timer is a backstop to in __tcp_run_hpts()
1601 hpts->sleeping = tv.tv_usec; in __tcp_run_hpts()
1602 callout_reset_sbt_on(&hpts->co, sb, 0, in __tcp_run_hpts()
1603 hpts_timeout_swi, hpts, hpts->p_cpu, in __tcp_run_hpts()
1607 hpts->p_mysleep.tv_usec *= 2; in __tcp_run_hpts()
1608 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in __tcp_run_hpts()
1609 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in __tcp_run_hpts()
1611 hpts->p_on_min_sleep = 1; in __tcp_run_hpts()
1613 hpts->p_hpts_active = 0; in __tcp_run_hpts()
1630 if (hpts->p_direct_wake) { in tcp_hpts_thread()
1632 callout_stop(&hpts->co); in tcp_hpts_thread()
1637 if (callout_pending(&hpts->co) || in tcp_hpts_thread()
1638 !callout_active(&hpts->co)) { in tcp_hpts_thread()
1643 callout_deactivate(&hpts->co); in tcp_hpts_thread()
1644 hpts->p_hpts_wake_scheduled = 0; in tcp_hpts_thread()
1646 if (hpts->p_hpts_active) { in tcp_hpts_thread()
1655 if (hpts->p_direct_wake == 0) { in tcp_hpts_thread()
1657 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in tcp_hpts_thread()
1658 hpts->p_mysleep.tv_usec *= 2; in tcp_hpts_thread()
1659 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in tcp_hpts_thread()
1660 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in tcp_hpts_thread()
1661 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1662 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1671 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1679 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1683 hpts->sleeping = 0; in tcp_hpts_thread()
1684 hpts->p_hpts_active = 1; in tcp_hpts_thread()
1687 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; in tcp_hpts_thread()
1688 if ((hpts->p_on_queue_cnt > conn_cnt_thresh) && (hpts->hit_callout_thresh == 0)) { in tcp_hpts_thread()
1689 hpts->hit_callout_thresh = 1; in tcp_hpts_thread()
1691 } else if ((hpts->p_on_queue_cnt <= conn_cnt_thresh) && (hpts->hit_callout_thresh == 1)) { in tcp_hpts_thread()
1692 hpts->hit_callout_thresh = 0; in tcp_hpts_thread()
1695 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in tcp_hpts_thread()
1696 if(hpts->p_direct_wake == 0) { in tcp_hpts_thread()
1702 hpts->p_mysleep.tv_usec *= 2; in tcp_hpts_thread()
1703 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in tcp_hpts_thread()
1704 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in tcp_hpts_thread()
1706 hpts->p_mysleep.tv_usec /= 2; in tcp_hpts_thread()
1707 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) in tcp_hpts_thread()
1708 hpts->p_mysleep.tv_usec = dynamic_min_sleep; in tcp_hpts_thread()
1711 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { in tcp_hpts_thread()
1712 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1713 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1716 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1720 * In this mode the timer is a backstop to in tcp_hpts_thread()
1725 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1726 } else if (hpts->p_on_queue_cnt == 0) { in tcp_hpts_thread()
1728 * No one on the wheel, please wake us up in tcp_hpts_thread()
1731 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1732 hpts->overidden_sleep = 0; in tcp_hpts_thread()
1741 * where the timer is what runs hpts mainly. in tcp_hpts_thread()
1746 * we cannot be awoken. in tcp_hpts_thread()
1748 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1750 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1753 hpts->overidden_sleep = 0; in tcp_hpts_thread()
1754 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1758 hpts->p_hpts_active = 0; in tcp_hpts_thread()
1760 hpts->p_direct_wake = 0; in tcp_hpts_thread()
1763 hpts->sleeping = tv.tv_usec; in tcp_hpts_thread()
1764 callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_thread()
1765 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_thread()
1779 if (cg->cg_level == CG_SHARE_L3) in hpts_count_level()
1782 for (i = 0; i < cg->cg_children; i++) { in hpts_count_level()
1783 count_l3 += hpts_count_level(&cg->cg_child[i]); in hpts_count_level()
1794 if (cg->cg_level == CG_SHARE_L3) { in hpts_gather_grps()
1804 for (i = 0; i < cg->cg_children; i++) { in hpts_gather_grps()
1805 hpts_gather_grps(grps, at, max, &cg->cg_child[i]); in hpts_gather_grps()
1861 * All we need is the top level all cpu's are in in tcp_hpts_mod_load()
1862 * the same cache so when we use grp[0]->cg_mask in tcp_hpts_mod_load()
1863 * with the cg_first <-> cg_last it will include in tcp_hpts_mod_load()
1864 * all cpu's in it. The level here is probably in tcp_hpts_mod_load()
1881 tcp_pace.rp_ent[i]->p_hptss = malloc(asz, M_TCPHPTS, M_WAITOK); in tcp_hpts_mod_load()
1888 mtx_init(&hpts->p_mtx, "tcp_hpts_lck", in tcp_hpts_mod_load()
1891 TAILQ_INIT(&hpts->p_hptss[j].head); in tcp_hpts_mod_load()
1892 hpts->p_hptss[j].count = 0; in tcp_hpts_mod_load()
1893 hpts->p_hptss[j].gencnt = 0; in tcp_hpts_mod_load()
1895 sysctl_ctx_init(&hpts->hpts_ctx); in tcp_hpts_mod_load()
1897 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1903 SYSCTL_ADD_INT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1904 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1906 &hpts->p_on_queue_cnt, 0, in tcp_hpts_mod_load()
1908 SYSCTL_ADD_U16(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1909 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1911 &hpts->p_hpts_active, 0, in tcp_hpts_mod_load()
1913 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1914 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1916 &hpts->p_cur_slot, 0, in tcp_hpts_mod_load()
1918 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1919 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1921 &hpts->p_runningslot, 0, in tcp_hpts_mod_load()
1923 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1924 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1926 &hpts->p_curtick, 0, in tcp_hpts_mod_load()
1928 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1929 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1933 SYSCTL_ADD_LONG(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1934 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1936 &hpts->p_mysleep.tv_usec, in tcp_hpts_mod_load()
1938 SYSCTL_ADD_U64(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1939 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1941 &hpts->sleeping, 0, in tcp_hpts_mod_load()
1943 SYSCTL_ADD_U64(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1944 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1946 &hpts->syscall_cnt, 0, in tcp_hpts_mod_load()
1949 hpts->p_hpts_sleep_time = hpts_sleep_max; in tcp_hpts_mod_load()
1950 hpts->p_num = i; in tcp_hpts_mod_load()
1951 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hpts_mod_load()
1953 hpts->p_prev_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hpts_mod_load()
1954 hpts->p_cpu = 0xffff; in tcp_hpts_mod_load()
1955 hpts->p_nxt_slot = hpts_slot(hpts->p_cur_slot, 1); in tcp_hpts_mod_load()
1956 callout_init(&hpts->co, 1); in tcp_hpts_mod_load()
1967 hpts->p_cpu = i; in tcp_hpts_mod_load()
1969 error = swi_add(&hpts->ie, "hpts", in tcp_hpts_mod_load()
1971 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); in tcp_hpts_mod_load()
1976 hpts->p_mysleep.tv_sec = 0; in tcp_hpts_mod_load()
1977 hpts->p_mysleep.tv_usec = tcp_min_hptsi_time; in tcp_hpts_mod_load()
1979 if (intr_event_bind(hpts->ie, i) == 0) in tcp_hpts_mod_load()
1982 /* Find the group for this CPU (i) and bind into it */ in tcp_hpts_mod_load()
1984 if (CPU_ISSET(i, &tcp_pace.grps[j]->cg_mask)) { in tcp_hpts_mod_load()
1985 if (intr_event_bind_ithread_cpuset(hpts->ie, in tcp_hpts_mod_load()
1986 &tcp_pace.grps[j]->cg_mask) == 0) { in tcp_hpts_mod_load()
1989 domain = pc->pc_domain; in tcp_hpts_mod_load()
1991 hpts_domains[domain].cpu[count] = i; in tcp_hpts_mod_load()
1999 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_SLOT; in tcp_hpts_mod_load()
2000 hpts->sleeping = tv.tv_usec; in tcp_hpts_mod_load()
2002 callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_mod_load()
2003 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_mod_load()
2034 rv = callout_drain(&hpts->co); in tcp_hpts_mod_unload()
2037 rv = swi_remove(hpts->ie_cookie); in tcp_hpts_mod_unload()
2040 rv = sysctl_ctx_free(&hpts->hpts_ctx); in tcp_hpts_mod_unload()
2043 mtx_destroy(&hpts->p_mtx); in tcp_hpts_mod_unload()
2044 free(hpts->p_hptss, M_TCPHPTS); in tcp_hpts_mod_unload()