1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org>
57 int			cpu_disable_c2_sleep = 0; /* Timer dies in C2. */
58 int			cpu_disable_c3_sleep = 0; /* Timer dies in C3. */
75 		if (timer->et_flags & ET_FLAGS_PERCPU)			\
76 			mtx_lock_spin(&(state)->et_hw_mtx);		\
83 		if (timer->et_flags & ET_FLAGS_PERCPU)			\
84 			mtx_unlock_spin(&(state)->et_hw_mtx);		\
89 static struct eventtimer *timer = NULL;  variable
90 static sbintime_t	timerperiod;	/* Timer period for periodic mode. */
93 static sbintime_t	nexttick;	/* Next global timer tick time. */
97 static char		timername[32];	/* Wanted timer. */
98 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
108 static int		periodic;	/* Periodic or one-shot mode. */
113 	struct mtx	et_hw_mtx;	/* Per-CPU timer mutex. */
118 	sbintime_t	nexttick;	/* Next timer tick time. */
132  * Timer broadcast IPI handler.
144 	now = state->now;  in hardclockintr()
170 		frame = curthread->td_intr_frame;  in handleevents()
177 	while (now >= state->nexthard) {  in handleevents()
178 		state->nexthard += tick_sbt;  in handleevents()
183 		*hct = state->nexthard - tick_sbt;  in handleevents()
190 	while (now >= state->nextstat) {  in handleevents()
191 		state->nextstat += statperiod;  in handleevents()
200 		while (now >= state->nextprof) {  in handleevents()
201 			state->nextprof += profperiod;  in handleevents()
209 		state->nextprof = state->nextstat;  in handleevents()
210 	if (now >= state->nextcallopt || now >= state->nextcall) {  in handleevents()
211 		state->nextcall = state->nextcallopt = SBT_MAX;  in handleevents()
218 		state->idle = 0;  in handleevents()
219 		state->nextevent = t;  in handleevents()
221 		    (timer->et_flags & ET_FLAGS_PERCPU));  in handleevents()
237 	event = state->nexthard;  in getnextcpuevent()
248 			event += tick_sbt * (hardfreq - 1);  in getnextcpuevent()
251 	if (event > state->nextcall)  in getnextcpuevent()
252 		event = state->nextcall;  in getnextcpuevent()
253 	if (!idle) { /* If CPU is active - handle other types of events. */  in getnextcpuevent()
254 		if (event > state->nextstat)  in getnextcpuevent()
255 			event = state->nextstat;  in getnextcpuevent()
256 		if (profiling && event > state->nextprof)  in getnextcpuevent()
257 			event = state->nextprof;  in getnextcpuevent()
275 	c = -1;  in getnextevent()
277 	event = state->nextevent;  in getnextevent()
279 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {  in getnextevent()
282 			if (event > state->nextevent) {  in getnextevent()
283 				event = state->nextevent;  in getnextevent()
296 /* Hardware timer callback function. */
312 	if (et->et_flags & ET_FLAGS_PERCPU) {
313 		next = &state->nexttick;
320 		*next = -1;	/* Next tick is not scheduled yet. */
321 	state->now = now;
329 	/* Prepare broadcasting to other CPUs for non-per-CPU timers. */
332 	if ((et->et_flags & ET_FLAGS_PERCPU) == 0) {
334 	if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
339 			state->now = now;
340 			if (now >= state->nextevent) {
341 				state->nextevent += SBT_1S;
343 					state->ipi = 1;
356 	/* Broadcast interrupt to other CPUs for non-per-CPU timers. */
362 			if (state->ipi) {
363 				state->ipi = 0;
372  * Load new value into hardware timer.
384 	if (timer->et_flags & ET_FLAGS_PERCPU)
385 		next = &state->nexttick;
395 			new = timerperiod - tmp;
402 			et_start(timer, new, timerperiod);
411 			et_start(timer, new - now, 0);
417  * Prepare event timer parameters after configuration changes.
424 	if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
426 	else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
432 	freq = round_freq(timer, freq);
437  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
446 	switch (atomic_load_acq_int(&state->action)) {
452 		state->handle = 0;
453 		atomic_store_rel_int(&state->action, 0);
457 		et_stop(timer);
459 		state->handle = 0;
460 		atomic_store_rel_int(&state->action, 0);
463 	if (atomic_readandclear_int(&state->handle) && !busy) {
472  * Reconfigure specified timer.
473  * For per-CPU timers use IPI to make other CPUs to reconfigure.
495 			nexttick = -1;
501 			state->now = now;
504 				state->nextevent = SBT_MAX;
507 				state->nextevent = next;
509 				state->nexttick = next;
511 				state->nexttick = -1;
512 			state->nexthard = next;
513 			state->nextstat = next;
514 			state->nextprof = next;
515 			state->nextcall = next;
516 			state->nextcallopt = next;
520 		/* Start global timer or per-CPU timer of this CPU. */
524 		/* Stop global timer or per-CPU timer of this CPU. */
525 		et_stop(timer);
530 	/* If timer is global we are done. */
531 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
533 	/* If timer is global or there is no other CPUs yet - we are done. */
534 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
542 		atomic_store_rel_int(&state->action,
554 		if (atomic_load_acq_int(&state->action))
562  * Calculate nearest frequency supported by hardware timer.
569 	if (et->et_frequency != 0) {
570 		div = lmax((et->et_frequency + freq / 2) / freq, 1);
571 		if (et->et_flags & ET_FLAGS_POW2DIV)
572 			div = 1 << (flsl(div + div / 2) - 1);
573 		freq = (et->et_frequency + div / 2) / div;
575 	if (et->et_min_period > SBT_1S)
576 		panic("Event timer \"%s\" doesn't support sub-second periods!",
577 		    et->et_name);
578 	else if (et->et_min_period != 0)
579 		freq = min(freq, SBT2FREQ(et->et_min_period));
580 	if (et->et_max_period < SBT_1S && et->et_max_period != 0)
581 		freq = max(freq, SBT2FREQ(et->et_max_period));
597 		mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
598 		state->nextcall = SBT_MAX;
599 		state->nextcallopt = SBT_MAX;
602 	/* Grab requested timer or the best of present. */
604 		timer = et_find(timername, 0, 0);
605 	if (timer == NULL && periodic) {
606 		timer = et_find(NULL,
609 	if (timer == NULL) {
610 		timer = et_find(NULL,
613 	if (timer == NULL && !periodic) {
614 		timer = et_find(NULL,
617 	if (timer == NULL)
618 		panic("No usable event timer found!");
619 	et_init(timer, timercb, NULL, NULL);
621 	/* Adapt to timer capabilities. */
622 	if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
624 	else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
626 	if (timer->et_flags & ET_FLAGS_C3STOP)
643 		base = round_freq(timer, hz * singlemul);
657 		profhz = round_freq(timer, profhz);
659 		hz = round_freq(timer, hz);
660 		stathz = round_freq(timer, 127);
661 		profhz = round_freq(timer, stathz * 64);
674  * Start per-CPU event timers on APs.
684 	state->now = sbinuptime();
689 	td->td_intr_nesting_level++;
690 	handleevents(state->now, 2);
691 	td->td_intr_nesting_level--;
747 		profiling--;
761 	    (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
766 		return (-1);
770 		now = state->now;
776 	state->idle = 1;
777 	state->nextevent = t;
781 	return (MAX(t - now, 0));
795 	if (atomic_load_int(&state->idle) == 0 || busy)
799 		now = state->now;
805 	td->td_intr_nesting_level++;
807 	td->td_intr_nesting_level--;
812  * Change the frequency of the given timer.  This changes et->et_frequency and
813  * if et is the active timer it reconfigures the timer on all CPUs.  This is
821 	if (et == timer) {
823 		et->et_frequency = newfreq;
826 		et->et_frequency = newfreq;
839 	CTR5(KTR_SPARE2, "new co:  on %d at %d.%08x - %d.%08x",
848 	 * If there is callout time already set earlier -- do nothing.
854 	state->nextcallopt = bt_opt;
855 	if (bt >= state->nextcall)
857 	state->nextcall = bt;
858 	/* If there is some other event set earlier -- do nothing. */
859 	if (bt >= state->nextevent)
861 	state->nextevent = bt;
862 	/* If timer is periodic -- there is nothing to reprogram. */
865 	/* If timer is global or of the current CPU -- reprogram it. */
866 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
873 	state->handle = 1;
891 	et = timer;
892 	snprintf(buf, sizeof(buf), "%s", et->et_name);
896 	et = timer;
897 	if (error != 0 || req->newptr == NULL ||
898 	    strcasecmp(buf, et->et_name) == 0) {
908 	et_free(timer);
909 	if (et->et_flags & ET_FLAGS_C3STOP)
911 	if (timer->et_flags & ET_FLAGS_C3STOP)
912 		cpu_disable_c3_sleep--;
914 	timer = et;
915 	et_init(timer, timercb, NULL, NULL);
920 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
922     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
925  * Report or change the active event timer periodicity.
934 	if (error != 0 || req->newptr == NULL)
945     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
965 		    c, st->action, st->handle, st->ipi, st->idle,
966 		    (uintmax_t)st->now,
967 		    (uintmax_t)st->nextevent,
968 		    (uintmax_t)(st->nextevent - st->now) / tick_sbt,
969 		    (uintmax_t)st->nexttick,
970 		    (uintmax_t)(st->nexttick - st->now) / tick_sbt,
971 		    (uintmax_t)st->nexthard,
972 		    (uintmax_t)(st->nexthard - st->now) / tick_sbt,
973 		    (uintmax_t)st->nextstat,
974 		    (uintmax_t)(st->nextstat - st->now) / tick_sbt,
975 		    (uintmax_t)st->nextprof,
976 		    (uintmax_t)(st->nextprof - st->now) / tick_sbt,
977 		    (uintmax_t)st->nextcall,
978 		    (uintmax_t)(st->nextcall - st->now) / tick_sbt,
979 		    (uintmax_t)st->nextcallopt,
980 		    (uintmax_t)(st->nextcallopt - st->now) / tick_sbt);