Lines Matching +full:timer +full:- +full:cannot +full:- +full:wake +full:- +full:cpu
1 /*-
2 * Copyright (c) 2016-2018 Netflix, Inc.
34 * The tcp_hpts system is designed to provide a high precision timer
84 * need to wake up the hpts system to get immediate
87 * the current hpts timer run (this is usually set when
88 * a rack timer is up so we know SACK's are happening
93 * knows how to take the input queue of packets from tp->t_inqueue
163 * This gives a range of 10usec - 1024ms to place
166 * when seeing the remainder will re-insert the
192 * the cost of more CPU (context switching).
205 * to push hpts forward mainly and the timer is only a backstop.
218 #define HPTS_MTX_ASSERT(hpts) mtx_assert(&(hpts)->p_mtx, MA_OWNED)
219 #define HPTS_LOCK(hpts) mtx_lock(&(hpts)->p_mtx)
220 #define HPTS_TRYLOCK(hpts) mtx_trylock(&(hpts)->p_mtx)
221 #define HPTS_UNLOCK(hpts) mtx_unlock(&(hpts)->p_mtx)
252 uint32_t overidden_sleep; /* what was overrided by min-sleep for logging */
263 uint16_t p_num; /* The hpts number one per cpu */
264 uint16_t p_cpu; /* The hpts CPU */
302 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \
303 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec; \
304 if ((vvp)->tv_usec < 0) { \
305 (vvp)->tv_sec--; \
306 (vvp)->tv_usec += 1000000; \
314 int cpu[MAXCPU]; member
377 "Use of irq CPU tunable");
411 if (error == 0 && req->newptr) { in sysctl_net_inet_tcp_hpts_max_sleep()
429 if (error == 0 && req->newptr) { in sysctl_net_inet_tcp_hpts_min_sleep()
483 * 64 bit - delRate, rttProp, bw_inuse in tcp_hpts_log()
484 * 16 bit - cwnd_gain in tcp_hpts_log()
485 * 8 bit - bbr_state, bbr_substate, inhpts; in tcp_hpts_log()
488 log.u_bbr.flex1 = hpts->p_nxt_slot; in tcp_hpts_log()
489 log.u_bbr.flex2 = hpts->p_cur_slot; in tcp_hpts_log()
490 log.u_bbr.flex3 = hpts->p_prev_slot; in tcp_hpts_log()
492 log.u_bbr.flex5 = hpts->p_curtick; in tcp_hpts_log()
493 log.u_bbr.flex6 = hpts->p_on_queue_cnt; in tcp_hpts_log()
494 log.u_bbr.flex7 = hpts->p_cpu; in tcp_hpts_log()
497 log.u_bbr.applimited = hpts->overidden_sleep; in tcp_hpts_log()
498 log.u_bbr.delivered = hpts->saved_curtick; in tcp_hpts_log()
500 log.u_bbr.epoch = hpts->saved_curslot; in tcp_hpts_log()
501 log.u_bbr.lt_epoch = hpts->saved_prev_slot; in tcp_hpts_log()
502 log.u_bbr.pkts_out = hpts->p_delayed_by; in tcp_hpts_log()
503 log.u_bbr.lost = hpts->p_hpts_sleep_time; in tcp_hpts_log()
504 log.u_bbr.pacing_gain = hpts->p_cpu; in tcp_hpts_log()
505 log.u_bbr.pkt_epoch = hpts->p_runningslot; in tcp_hpts_log()
508 &tptosocket(tp)->so_rcv, in tcp_hpts_log()
509 &tptosocket(tp)->so_snd, in tcp_hpts_log()
520 if (tcp_hpts_no_wake_over_thresh && (hpts->p_on_queue_cnt >= conn_cnt_thresh)) { in tcp_wakehpts()
521 hpts->p_direct_wake = 0; in tcp_wakehpts()
524 if (hpts->p_hpts_wake_scheduled == 0) { in tcp_wakehpts()
525 hpts->p_hpts_wake_scheduled = 1; in tcp_wakehpts()
526 swi_sched(hpts->ie_cookie, 0); in tcp_wakehpts()
536 swi_sched(hpts->ie_cookie, 0); in hpts_timeout_swi()
547 MPASS(hpts->p_cpu == tp->t_hpts_cpu); in tcp_hpts_insert_internal()
548 MPASS(!(inp->inp_flags & INP_DROPPED)); in tcp_hpts_insert_internal()
550 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; in tcp_hpts_insert_internal()
552 if (tp->t_in_hpts == IHPTS_NONE) { in tcp_hpts_insert_internal()
553 tp->t_in_hpts = IHPTS_ONQUEUE; in tcp_hpts_insert_internal()
555 } else if (tp->t_in_hpts == IHPTS_MOVING) { in tcp_hpts_insert_internal()
556 tp->t_in_hpts = IHPTS_ONQUEUE; in tcp_hpts_insert_internal()
558 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); in tcp_hpts_insert_internal()
559 tp->t_hpts_gencnt = hptsh->gencnt; in tcp_hpts_insert_internal()
561 TAILQ_INSERT_TAIL(&hptsh->head, tp, t_hpts); in tcp_hpts_insert_internal()
562 hptsh->count++; in tcp_hpts_insert_internal()
563 hpts->p_on_queue_cnt++; in tcp_hpts_insert_internal()
573 hpts = tcp_pace.rp_ent[tp->t_hpts_cpu]; in tcp_hpts_lock()
584 tp->t_in_hpts = IHPTS_NONE; in tcp_hpts_release()
590 * Initialize tcpcb to get ready for use with HPTS. We will know which CPU
592 * a single CPU with newborn connections and use a random one.
594 * can be called once again if stack is switched. In that case we inherit CPU
603 if (__predict_true(tp->t_hpts_cpu == HPTS_CPU_NONE)) { in tcp_hpts_init()
604 tp->t_hpts_cpu = hpts_random_cpu(); in tcp_hpts_init()
605 MPASS(!(tp->t_flags2 & TF2_HPTS_CPU_SET)); in tcp_hpts_init()
624 if (tp->t_in_hpts == IHPTS_ONQUEUE) { in tcp_hpts_remove()
625 hptsh = &hpts->p_hptss[tp->t_hpts_slot]; in tcp_hpts_remove()
626 tp->t_hpts_request = 0; in tcp_hpts_remove()
627 if (__predict_true(tp->t_hpts_gencnt == hptsh->gencnt)) { in tcp_hpts_remove()
628 TAILQ_REMOVE(&hptsh->head, tp, t_hpts); in tcp_hpts_remove()
629 MPASS(hptsh->count > 0); in tcp_hpts_remove()
630 hptsh->count--; in tcp_hpts_remove()
631 MPASS(hpts->p_on_queue_cnt > 0); in tcp_hpts_remove()
632 hpts->p_on_queue_cnt--; in tcp_hpts_remove()
642 TAILQ_FOREACH(tmp, &hptsh->head, t_hpts) in tcp_hpts_remove()
645 tp->t_in_hpts = IHPTS_MOVING; in tcp_hpts_remove()
646 tp->t_hpts_slot = -1; in tcp_hpts_remove()
648 } else if (tp->t_in_hpts == IHPTS_MOVING) { in tcp_hpts_remove()
652 * tcp_hpts_remove() marks as IHPTS_MOVING, slot = -1 in tcp_hpts_remove()
657 tp->t_hpts_slot = -1; in tcp_hpts_remove()
694 return (slot_now - prev_slot); in hpts_slots_diff()
700 return (NUM_OF_HPTSI_SLOTS - 1); in hpts_slots_diff()
702 return ((NUM_OF_HPTSI_SLOTS - prev_slot) + slot_now); in hpts_slots_diff()
723 if ((hpts->p_hpts_active == 1) && in max_slots_available()
724 (hpts->p_wheel_complete == 0)) { in max_slots_available()
725 end_slot = hpts->p_runningslot; in max_slots_available()
728 end_slot = NUM_OF_HPTSI_SLOTS - 1; in max_slots_available()
730 end_slot--; in max_slots_available()
742 end_slot = hpts->p_prev_slot; in max_slots_available()
744 end_slot = NUM_OF_HPTSI_SLOTS - 1; in max_slots_available()
746 end_slot--; in max_slots_available()
755 if (hpts->p_prev_slot != wheel_slot) in max_slots_available()
756 dis_to_travel = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); in max_slots_available()
767 return (NUM_OF_HPTSI_SLOTS - dis_to_travel); in max_slots_available()
770 * So how many slots are open between p_runningslot -> p_cur_slot in max_slots_available()
771 * that is what is currently un-available for insertion. Special in max_slots_available()
775 if (hpts->p_runningslot == hpts->p_cur_slot) in max_slots_available()
778 dis_to_travel = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); in max_slots_available()
782 if (hpts->p_cur_slot != wheel_slot) { in max_slots_available()
784 pacer_to_now = hpts_slots_diff(hpts->p_cur_slot, wheel_slot); in max_slots_available()
794 avail_on_wheel = NUM_OF_HPTSI_SLOTS - dis_to_travel; in max_slots_available()
809 *target_slot = hpts->p_nxt_slot; in max_slots_available()
821 return (avail_on_wheel - pacer_to_now); in max_slots_available()
837 if ((hpts->p_hpts_active) && in check_if_slot_would_be_wrong()
838 (hpts->p_wheel_complete == 0)) { in check_if_slot_would_be_wrong()
847 distance = hpts_slots_diff(hpts->p_runningslot, hptsslot); in check_if_slot_would_be_wrong()
848 if (hpts->p_runningslot != hpts->p_cur_slot) in check_if_slot_would_be_wrong()
849 yet_to_run = hpts_slots_diff(hpts->p_runningslot, hpts->p_cur_slot); in check_if_slot_would_be_wrong()
854 hptsslot, distance, yet_to_run, hpts->p_runningslot, in check_if_slot_would_be_wrong()
855 hpts->p_cur_slot)); in check_if_slot_would_be_wrong()
870 MPASS(!(tptoinpcb(tp)->inp_flags & INP_DROPPED)); in tcp_hpts_insert_diag()
871 MPASS(!(tp->t_in_hpts == IHPTS_ONQUEUE)); in tcp_hpts_insert_diag()
874 * We now return the next-slot the hpts will be on, beyond its in tcp_hpts_insert_diag()
882 diag->p_hpts_active = hpts->p_hpts_active; in tcp_hpts_insert_diag()
883 diag->p_prev_slot = hpts->p_prev_slot; in tcp_hpts_insert_diag()
884 diag->p_runningslot = hpts->p_runningslot; in tcp_hpts_insert_diag()
885 diag->p_nxt_slot = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
886 diag->p_cur_slot = hpts->p_cur_slot; in tcp_hpts_insert_diag()
887 diag->p_curtick = hpts->p_curtick; in tcp_hpts_insert_diag()
888 diag->p_lasttick = hpts->p_lasttick; in tcp_hpts_insert_diag()
889 diag->slot_req = slot; in tcp_hpts_insert_diag()
890 diag->p_on_min_sleep = hpts->p_on_min_sleep; in tcp_hpts_insert_diag()
891 diag->hpts_sleep_time = hpts->p_hpts_sleep_time; in tcp_hpts_insert_diag()
895 tp->t_hpts_request = 0; in tcp_hpts_insert_diag()
896 if ((hpts->p_hpts_active == 0) || (hpts->p_wheel_complete)) { in tcp_hpts_insert_diag()
901 tp->t_hpts_slot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hpts_insert_diag()
902 if ((hpts->p_on_min_sleep == 0) && in tcp_hpts_insert_diag()
903 (hpts->p_hpts_active == 0)) in tcp_hpts_insert_diag()
906 tp->t_hpts_slot = hpts->p_runningslot; in tcp_hpts_insert_diag()
907 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) in tcp_hpts_insert_diag()
914 hpts->p_direct_wake = 1; in tcp_hpts_insert_diag()
917 slot_on = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
929 diag->wheel_slot = wheel_slot; in tcp_hpts_insert_diag()
930 diag->maxslots = maxslots; in tcp_hpts_insert_diag()
931 diag->wheel_cts = wheel_cts; in tcp_hpts_insert_diag()
943 slot--; in tcp_hpts_insert_diag()
945 tp->t_hpts_slot = last_slot; in tcp_hpts_insert_diag()
946 tp->t_hpts_request = slot; in tcp_hpts_insert_diag()
949 tp->t_hpts_request = 0; in tcp_hpts_insert_diag()
950 tp->t_hpts_slot = hpts_slot(wheel_slot, slot); in tcp_hpts_insert_diag()
953 tp->t_hpts_request = slot - maxslots; in tcp_hpts_insert_diag()
954 tp->t_hpts_slot = last_slot; in tcp_hpts_insert_diag()
957 diag->slot_remaining = tp->t_hpts_request; in tcp_hpts_insert_diag()
958 diag->inp_hptsslot = tp->t_hpts_slot; in tcp_hpts_insert_diag()
961 check_if_slot_would_be_wrong(hpts, tp, tp->t_hpts_slot, line); in tcp_hpts_insert_diag()
963 if (__predict_true(tp->t_in_hpts != IHPTS_MOVING)) in tcp_hpts_insert_diag()
965 if ((hpts->p_hpts_active == 0) && in tcp_hpts_insert_diag()
966 (tp->t_hpts_request == 0) && in tcp_hpts_insert_diag()
967 (hpts->p_on_min_sleep == 0)) { in tcp_hpts_insert_diag()
971 * it will wake up at and if we need to reschedule in tcp_hpts_insert_diag()
972 * its time-out. in tcp_hpts_insert_diag()
976 /* Now do we need to restart the hpts's timer? */ in tcp_hpts_insert_diag()
977 have_slept = hpts_slots_diff(hpts->p_prev_slot, wheel_slot); in tcp_hpts_insert_diag()
978 if (have_slept < hpts->p_hpts_sleep_time) in tcp_hpts_insert_diag()
979 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept; in tcp_hpts_insert_diag()
981 /* We are over-due */ in tcp_hpts_insert_diag()
986 diag->have_slept = have_slept; in tcp_hpts_insert_diag()
987 diag->yet_to_sleep = yet_to_sleep; in tcp_hpts_insert_diag()
992 * We need to reschedule the hpts's time-out. in tcp_hpts_insert_diag()
994 hpts->p_hpts_sleep_time = slot; in tcp_hpts_insert_diag()
1004 hpts->p_direct_wake = 1; in tcp_hpts_insert_diag()
1007 diag->need_new_to = 0; in tcp_hpts_insert_diag()
1008 diag->co_ret = 0xffff0000; in tcp_hpts_insert_diag()
1019 need_new_to -= HPTS_USEC_IN_SEC; in tcp_hpts_insert_diag()
1023 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_insert_diag()
1024 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_insert_diag()
1027 diag->need_new_to = need_new_to; in tcp_hpts_insert_diag()
1028 diag->co_ret = co_ret; in tcp_hpts_insert_diag()
1031 slot_on = hpts->p_nxt_slot; in tcp_hpts_insert_diag()
1047 if (tp->t_flags2 & TF2_HPTS_CPU_SET) { in hpts_cpuid()
1048 return (tp->t_hpts_cpu); in hpts_cpuid()
1051 * If we are using the irq cpu set by LRO or in hpts_cpuid()
1055 if (tp->t_lro_cpu == HPTS_CPU_NONE) { in hpts_cpuid()
1059 return (tp->t_lro_cpu); in hpts_cpuid()
1063 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype); in hpts_cpuid()
1070 * We don't have a flowid -> cpuid mapping, so cheat and just map in hpts_cpuid()
1074 if (inp->inp_flowtype == M_HASHTYPE_NONE) { in hpts_cpuid()
1085 (inp->inp_numa_domain == M_NODOM)) { in hpts_cpuid()
1087 cpuid = inp->inp_flowid % mp_ncpus; in hpts_cpuid()
1090 /* Hash into the cpu's that use that domain */ in hpts_cpuid()
1091 di = &hpts_domains[inp->inp_numa_domain]; in hpts_cpuid()
1092 cpuid = di->cpu[inp->inp_flowid % di->count]; in hpts_cpuid()
1104 if ((hpts->p_on_queue_cnt) && (wrap_loop_cnt < 2)) { in tcp_hpts_set_max_sleep()
1109 for (i = 0, t = hpts_slot(hpts->p_cur_slot, 1); i < NUM_OF_HPTSI_SLOTS; i++) { in tcp_hpts_set_max_sleep()
1110 if (TAILQ_EMPTY(&hpts->p_hptss[t].head) == 0) { in tcp_hpts_set_max_sleep()
1115 KASSERT((i != NUM_OF_HPTSI_SLOTS), ("Hpts:%p cnt:%d but none found", hpts, hpts->p_on_queue_cnt)); in tcp_hpts_set_max_sleep()
1116 hpts->p_hpts_sleep_time = min((i + 1), hpts_sleep_max); in tcp_hpts_set_max_sleep()
1119 hpts->p_hpts_sleep_time = hpts_sleep_max; in tcp_hpts_set_max_sleep()
1143 hpts->saved_lasttick = hpts->p_lasttick; in tcp_hptsi()
1144 hpts->saved_curtick = hpts->p_curtick; in tcp_hptsi()
1145 hpts->saved_curslot = hpts->p_cur_slot; in tcp_hptsi()
1146 hpts->saved_prev_slot = hpts->p_prev_slot; in tcp_hptsi()
1148 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1149 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1150 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usec(&tv); in tcp_hptsi()
1151 orig_exit_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1152 if ((hpts->p_on_queue_cnt == 0) || in tcp_hptsi()
1153 (hpts->p_lasttick == hpts->p_curtick)) { in tcp_hptsi()
1158 hpts->p_prev_slot = hpts->p_cur_slot; in tcp_hptsi()
1159 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1163 hpts->p_wheel_complete = 0; in tcp_hptsi()
1165 slots_to_run = hpts_slots_diff(hpts->p_prev_slot, hpts->p_cur_slot); in tcp_hptsi()
1166 if (((hpts->p_curtick - hpts->p_lasttick) > (NUM_OF_HPTSI_SLOTS - 1)) && in tcp_hptsi()
1167 (hpts->p_on_queue_cnt != 0)) { in tcp_hptsi()
1185 hpts->p_nxt_slot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hptsi()
1186 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 2); in tcp_hptsi()
1192 hpts->p_cur_slot = hpts->p_prev_slot; in tcp_hptsi()
1200 TAILQ_FOREACH(tp, &hpts->p_hptss[hpts->p_nxt_slot].head, in tcp_hptsi()
1202 MPASS(tp->t_hpts_slot == hpts->p_nxt_slot); in tcp_hptsi()
1203 MPASS(tp->t_hpts_gencnt == in tcp_hptsi()
1204 hpts->p_hptss[hpts->p_nxt_slot].gencnt); in tcp_hptsi()
1205 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); in tcp_hptsi()
1213 tp->t_hpts_gencnt = in tcp_hptsi()
1214 hpts->p_hptss[hpts->p_runningslot].gencnt; in tcp_hptsi()
1215 tp->t_hpts_slot = hpts->p_runningslot; in tcp_hptsi()
1217 TAILQ_CONCAT(&hpts->p_hptss[hpts->p_runningslot].head, in tcp_hptsi()
1218 &hpts->p_hptss[hpts->p_nxt_slot].head, t_hpts); in tcp_hptsi()
1219 hpts->p_hptss[hpts->p_runningslot].count += in tcp_hptsi()
1220 hpts->p_hptss[hpts->p_nxt_slot].count; in tcp_hptsi()
1221 hpts->p_hptss[hpts->p_nxt_slot].count = 0; in tcp_hptsi()
1222 hpts->p_hptss[hpts->p_nxt_slot].gencnt++; in tcp_hptsi()
1223 slots_to_run = NUM_OF_HPTSI_SLOTS - 1; in tcp_hptsi()
1230 hpts->p_nxt_slot = hpts->p_prev_slot; in tcp_hptsi()
1231 hpts->p_runningslot = hpts_slot(hpts->p_prev_slot, 1); in tcp_hptsi()
1233 if (hpts->p_on_queue_cnt == 0) { in tcp_hptsi()
1246 hpts->p_delayed_by = (slots_to_run - (i + 1)) * in tcp_hptsi()
1249 runningslot = hpts->p_runningslot; in tcp_hptsi()
1250 hptsh = &hpts->p_hptss[runningslot]; in tcp_hptsi()
1251 TAILQ_SWAP(&head, &hptsh->head, tcpcb, t_hpts); in tcp_hptsi()
1252 hpts->p_on_queue_cnt -= hptsh->count; in tcp_hptsi()
1253 hptsh->count = 0; in tcp_hptsi()
1254 hptsh->gencnt++; in tcp_hptsi()
1277 * re-used and something else (not an in tcp_hptsi()
1281 * and we tried to pre-fetch it). in tcp_hptsi()
1312 if ((tp->t_flags2 & TF2_HPTS_CPU_SET) == 0) { in tcp_hptsi()
1318 if (__predict_false(tp->t_in_hpts == IHPTS_MOVING)) { in tcp_hptsi()
1319 if (tp->t_hpts_slot == -1) { in tcp_hptsi()
1320 tp->t_in_hpts = IHPTS_NONE; in tcp_hptsi()
1332 MPASS(tp->t_in_hpts == IHPTS_ONQUEUE); in tcp_hptsi()
1333 MPASS(!(inp->inp_flags & INP_DROPPED)); in tcp_hptsi()
1334 KASSERT(runningslot == tp->t_hpts_slot, in tcp_hptsi()
1335 ("Hpts:%p inp:%p slot mis-aligned %u vs %u", in tcp_hptsi()
1336 hpts, inp, runningslot, tp->t_hpts_slot)); in tcp_hptsi()
1338 if (tp->t_hpts_request) { in tcp_hptsi()
1347 remaining_slots = slots_to_run - (i + 1); in tcp_hptsi()
1348 if (tp->t_hpts_request > remaining_slots) { in tcp_hptsi()
1354 hpts->p_cur_slot, &last_slot); in tcp_hptsi()
1355 if (maxslots >= tp->t_hpts_request) { in tcp_hptsi()
1358 tp->t_hpts_slot = hpts_slot( in tcp_hptsi()
1359 hpts->p_runningslot, in tcp_hptsi()
1360 tp->t_hpts_request); in tcp_hptsi()
1361 tp->t_hpts_request = 0; in tcp_hptsi()
1364 tp->t_hpts_slot = last_slot; in tcp_hptsi()
1365 tp->t_hpts_request -= in tcp_hptsi()
1373 tp->t_hpts_request = 0; in tcp_hptsi()
1381 * the right CPU. This should be a rare in tcp_hptsi()
1390 * goes off and sees the mis-match. We in tcp_hptsi()
1391 * simply correct it here and the CPU will in tcp_hptsi()
1394 * :-) in tcp_hptsi()
1398 CURVNET_SET(inp->inp_vnet); in tcp_hptsi()
1402 if (tp->t_fb_ptr != NULL) { in tcp_hptsi()
1403 kern_prefetch(tp->t_fb_ptr, &did_prefetch); in tcp_hptsi()
1423 tp->t_flags2 |= TF2_HPTS_CALLS; in tcp_hptsi()
1424 if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) && in tcp_hptsi()
1425 !STAILQ_EMPTY(&tp->t_inqueue)) in tcp_hptsi()
1426 error = -(*tp->t_fb->tfb_do_queued_segments)(tp, in tcp_hptsi()
1437 * slot_pos_of_endpoint <-> orig_exit_slot in tcp_hptsi()
1445 hpts->p_runningslot++; in tcp_hptsi()
1446 if (hpts->p_runningslot >= NUM_OF_HPTSI_SLOTS) { in tcp_hptsi()
1447 hpts->p_runningslot = 0; in tcp_hptsi()
1452 hpts->p_delayed_by = 0; in tcp_hptsi()
1455 * more ticks (if we did not hit eno-bufs). in tcp_hptsi()
1457 hpts->p_prev_slot = hpts->p_cur_slot; in tcp_hptsi()
1458 hpts->p_lasttick = hpts->p_curtick; in tcp_hptsi()
1478 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1479 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1482 orig_exit_slot = hpts->p_cur_slot; in tcp_hptsi()
1485 (hpts->p_lasttick != hpts->p_curtick)) { in tcp_hptsi()
1491 tcp_pace.cts_last_ran[hpts->p_num] = tcp_tv_to_usec(&tv); in tcp_hptsi()
1497 hpts->p_wheel_complete = 1; in tcp_hptsi()
1506 KASSERT(((hpts->p_prev_slot == hpts->p_cur_slot) || in tcp_hptsi()
1509 hpts->p_prev_slot, hpts->p_cur_slot)); in tcp_hptsi()
1510 KASSERT(((hpts->p_lasttick == hpts->p_curtick) in tcp_hptsi()
1513 hpts->p_lasttick, hpts->p_curtick)); in tcp_hptsi()
1514 if (from_callout && (hpts->p_lasttick != hpts->p_curtick)) { in tcp_hptsi()
1515 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hptsi()
1517 hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hptsi()
1539 if (tp->t_in_hpts == IHPTS_NONE && !(tp->t_flags2 & TF2_HPTS_CPU_SET)) { in tcp_set_hpts()
1540 tp->t_hpts_cpu = hpts_cpuid(tp, &failed); in tcp_set_hpts()
1542 tp->t_flags2 |= TF2_HPTS_CPU_SET; in tcp_set_hpts()
1560 * this CPU is in. in tcp_choose_hpts_to_run()
1564 if (CPU_ISSET(curcpu, &tcp_pace.grps[i]->cg_mask)) { in tcp_choose_hpts_to_run()
1565 start = tcp_pace.grps[i]->cg_first; in tcp_choose_hpts_to_run()
1566 end = (tcp_pace.grps[i]->cg_last + 1); in tcp_choose_hpts_to_run()
1571 oldest_idx = -1; in tcp_choose_hpts_to_run()
1574 calc = cts - tcp_pace.cts_last_ran[i]; in tcp_choose_hpts_to_run()
1597 if (hpts->p_hpts_active) { in __tcp_run_hpts()
1606 if (hpts->p_hpts_active) in __tcp_run_hpts()
1608 hpts->syscall_cnt++; in __tcp_run_hpts()
1610 hpts->p_hpts_active = 1; in __tcp_run_hpts()
1613 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in __tcp_run_hpts()
1618 hpts->p_mysleep.tv_usec /= 2; in __tcp_run_hpts()
1619 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) in __tcp_run_hpts()
1620 hpts->p_mysleep.tv_usec = dynamic_min_sleep; in __tcp_run_hpts()
1624 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_USECS_PER_SLOT; in __tcp_run_hpts()
1626 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { in __tcp_run_hpts()
1627 hpts->overidden_sleep = tv.tv_usec; in __tcp_run_hpts()
1628 tv.tv_usec = hpts->p_mysleep.tv_usec; in __tcp_run_hpts()
1631 hpts->overidden_sleep = tv.tv_usec; in __tcp_run_hpts()
1635 * In this mode the timer is a backstop to in __tcp_run_hpts()
1642 hpts->sleeping = tv.tv_usec; in __tcp_run_hpts()
1643 callout_reset_sbt_on(&hpts->co, sb, 0, in __tcp_run_hpts()
1644 hpts_timeout_swi, hpts, hpts->p_cpu, in __tcp_run_hpts()
1648 hpts->p_mysleep.tv_usec *= 2; in __tcp_run_hpts()
1649 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in __tcp_run_hpts()
1650 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in __tcp_run_hpts()
1652 hpts->p_on_min_sleep = 1; in __tcp_run_hpts()
1654 hpts->p_hpts_active = 0; in __tcp_run_hpts()
1671 if (hpts->p_direct_wake) { in tcp_hpts_thread()
1673 callout_stop(&hpts->co); in tcp_hpts_thread()
1678 if (callout_pending(&hpts->co) || in tcp_hpts_thread()
1679 !callout_active(&hpts->co)) { in tcp_hpts_thread()
1684 callout_deactivate(&hpts->co); in tcp_hpts_thread()
1685 hpts->p_hpts_wake_scheduled = 0; in tcp_hpts_thread()
1687 if (hpts->p_hpts_active) { in tcp_hpts_thread()
1696 if (hpts->p_direct_wake == 0) { in tcp_hpts_thread()
1698 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in tcp_hpts_thread()
1699 hpts->p_mysleep.tv_usec *= 2; in tcp_hpts_thread()
1700 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in tcp_hpts_thread()
1701 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in tcp_hpts_thread()
1702 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1703 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1712 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1720 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1724 hpts->sleeping = 0; in tcp_hpts_thread()
1725 hpts->p_hpts_active = 1; in tcp_hpts_thread()
1728 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_USECS_PER_SLOT; in tcp_hpts_thread()
1729 if ((hpts->p_on_queue_cnt > conn_cnt_thresh) && (hpts->hit_callout_thresh == 0)) { in tcp_hpts_thread()
1730 hpts->hit_callout_thresh = 1; in tcp_hpts_thread()
1732 } else if ((hpts->p_on_queue_cnt <= conn_cnt_thresh) && (hpts->hit_callout_thresh == 1)) { in tcp_hpts_thread()
1733 hpts->hit_callout_thresh = 0; in tcp_hpts_thread()
1736 if (hpts->p_on_queue_cnt >= conn_cnt_thresh) { in tcp_hpts_thread()
1737 if(hpts->p_direct_wake == 0) { in tcp_hpts_thread()
1743 hpts->p_mysleep.tv_usec *= 2; in tcp_hpts_thread()
1744 if (hpts->p_mysleep.tv_usec > dynamic_max_sleep) in tcp_hpts_thread()
1745 hpts->p_mysleep.tv_usec = dynamic_max_sleep; in tcp_hpts_thread()
1747 hpts->p_mysleep.tv_usec /= 2; in tcp_hpts_thread()
1748 if (hpts->p_mysleep.tv_usec < dynamic_min_sleep) in tcp_hpts_thread()
1749 hpts->p_mysleep.tv_usec = dynamic_min_sleep; in tcp_hpts_thread()
1752 if (tv.tv_usec < hpts->p_mysleep.tv_usec) { in tcp_hpts_thread()
1753 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1754 tv.tv_usec = hpts->p_mysleep.tv_usec; in tcp_hpts_thread()
1757 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1761 * In this mode the timer is a backstop to in tcp_hpts_thread()
1766 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1767 } else if (hpts->p_on_queue_cnt == 0) { in tcp_hpts_thread()
1769 * No one on the wheel, please wake us up in tcp_hpts_thread()
1772 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1773 hpts->overidden_sleep = 0; in tcp_hpts_thread()
1782 * where the timer is what runs hpts mainly. in tcp_hpts_thread()
1787 * we cannot be awoken. in tcp_hpts_thread()
1789 hpts->overidden_sleep = tv.tv_usec; in tcp_hpts_thread()
1791 hpts->p_on_min_sleep = 1; in tcp_hpts_thread()
1794 hpts->overidden_sleep = 0; in tcp_hpts_thread()
1795 hpts->p_on_min_sleep = 0; in tcp_hpts_thread()
1799 hpts->p_hpts_active = 0; in tcp_hpts_thread()
1801 hpts->p_direct_wake = 0; in tcp_hpts_thread()
1804 hpts->sleeping = tv.tv_usec; in tcp_hpts_thread()
1805 callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_thread()
1806 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_thread()
1820 if (cg->cg_level == CG_SHARE_L3) in hpts_count_level()
1823 for (i = 0; i < cg->cg_children; i++) { in hpts_count_level()
1824 count_l3 += hpts_count_level(&cg->cg_child[i]); in hpts_count_level()
1835 if (cg->cg_level == CG_SHARE_L3) { in hpts_gather_grps()
1845 for (i = 0; i < cg->cg_children; i++) { in hpts_gather_grps()
1846 hpts_gather_grps(grps, at, max, &cg->cg_child[i]); in hpts_gather_grps()
1902 * All we need is the top level all cpu's are in in tcp_hpts_mod_load()
1903 * the same cache so when we use grp[0]->cg_mask in tcp_hpts_mod_load()
1904 * with the cg_first <-> cg_last it will include in tcp_hpts_mod_load()
1905 * all cpu's in it. The level here is probably in tcp_hpts_mod_load()
1922 tcp_pace.rp_ent[i]->p_hptss = malloc(asz, M_TCPHPTS, M_WAITOK); in tcp_hpts_mod_load()
1929 mtx_init(&hpts->p_mtx, "tcp_hpts_lck", in tcp_hpts_mod_load()
1932 TAILQ_INIT(&hpts->p_hptss[j].head); in tcp_hpts_mod_load()
1933 hpts->p_hptss[j].count = 0; in tcp_hpts_mod_load()
1934 hpts->p_hptss[j].gencnt = 0; in tcp_hpts_mod_load()
1936 sysctl_ctx_init(&hpts->hpts_ctx); in tcp_hpts_mod_load()
1938 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1944 SYSCTL_ADD_INT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1945 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1947 &hpts->p_on_queue_cnt, 0, in tcp_hpts_mod_load()
1949 SYSCTL_ADD_U16(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1950 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1952 &hpts->p_hpts_active, 0, in tcp_hpts_mod_load()
1954 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1955 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1957 &hpts->p_cur_slot, 0, in tcp_hpts_mod_load()
1959 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1960 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1962 &hpts->p_runningslot, 0, in tcp_hpts_mod_load()
1964 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1965 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1967 &hpts->p_curtick, 0, in tcp_hpts_mod_load()
1969 SYSCTL_ADD_UINT(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1970 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1974 SYSCTL_ADD_LONG(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1975 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1977 &hpts->p_mysleep.tv_usec, in tcp_hpts_mod_load()
1979 SYSCTL_ADD_U64(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1980 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1982 &hpts->sleeping, 0, in tcp_hpts_mod_load()
1984 SYSCTL_ADD_U64(&hpts->hpts_ctx, in tcp_hpts_mod_load()
1985 SYSCTL_CHILDREN(hpts->hpts_root), in tcp_hpts_mod_load()
1987 &hpts->syscall_cnt, 0, in tcp_hpts_mod_load()
1990 hpts->p_hpts_sleep_time = hpts_sleep_max; in tcp_hpts_mod_load()
1991 hpts->p_num = i; in tcp_hpts_mod_load()
1992 hpts->p_curtick = tcp_gethptstick(&tv); in tcp_hpts_mod_load()
1994 hpts->p_prev_slot = hpts->p_cur_slot = tick_to_wheel(hpts->p_curtick); in tcp_hpts_mod_load()
1995 hpts->p_cpu = 0xffff; in tcp_hpts_mod_load()
1996 hpts->p_nxt_slot = hpts_slot(hpts->p_cur_slot, 1); in tcp_hpts_mod_load()
1997 callout_init(&hpts->co, 1); in tcp_hpts_mod_load()
2008 hpts->p_cpu = i; in tcp_hpts_mod_load()
2010 error = swi_add(&hpts->ie, "hpts", in tcp_hpts_mod_load()
2012 SWI_NET, INTR_MPSAFE, &hpts->ie_cookie); in tcp_hpts_mod_load()
2017 hpts->p_mysleep.tv_sec = 0; in tcp_hpts_mod_load()
2018 hpts->p_mysleep.tv_usec = tcp_min_hptsi_time; in tcp_hpts_mod_load()
2020 if (intr_event_bind(hpts->ie, i) == 0) in tcp_hpts_mod_load()
2023 /* Find the group for this CPU (i) and bind into it */ in tcp_hpts_mod_load()
2025 if (CPU_ISSET(i, &tcp_pace.grps[j]->cg_mask)) { in tcp_hpts_mod_load()
2026 if (intr_event_bind_ithread_cpuset(hpts->ie, in tcp_hpts_mod_load()
2027 &tcp_pace.grps[j]->cg_mask) == 0) { in tcp_hpts_mod_load()
2030 domain = pc->pc_domain; in tcp_hpts_mod_load()
2032 hpts_domains[domain].cpu[count] = i; in tcp_hpts_mod_load()
2040 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_USECS_PER_SLOT; in tcp_hpts_mod_load()
2041 hpts->sleeping = tv.tv_usec; in tcp_hpts_mod_load()
2043 callout_reset_sbt_on(&hpts->co, sb, 0, in tcp_hpts_mod_load()
2044 hpts_timeout_swi, hpts, hpts->p_cpu, in tcp_hpts_mod_load()
2075 rv = callout_drain(&hpts->co); in tcp_hpts_mod_unload()
2078 rv = swi_remove(hpts->ie_cookie); in tcp_hpts_mod_unload()
2081 rv = sysctl_ctx_free(&hpts->hpts_ctx); in tcp_hpts_mod_unload()
2084 mtx_destroy(&hpts->p_mtx); in tcp_hpts_mod_unload()
2085 free(hpts->p_hptss, M_TCPHPTS); in tcp_hpts_mod_unload()