xref: /freebsd/sys/kern/kern_clocksource.c (revision a812392203d7c4c3f0db9d8a0f3391374c49c71f)
1 /*-
2  * Copyright (c) 2010-2013 Alexander Motin <mav@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer,
10  *    without modification, immediately at the beginning of the file.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
29 
30 /*
31  * Common routines to manage event timers hardware.
32  */
33 
34 #include "opt_device_polling.h"
35 
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/bus.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/kdb.h>
42 #include <sys/ktr.h>
43 #include <sys/mutex.h>
44 #include <sys/proc.h>
45 #include <sys/kernel.h>
46 #include <sys/sched.h>
47 #include <sys/smp.h>
48 #include <sys/sysctl.h>
49 #include <sys/timeet.h>
50 #include <sys/timetc.h>
51 
52 #include <machine/atomic.h>
53 #include <machine/clock.h>
54 #include <machine/cpu.h>
55 #include <machine/smp.h>
56 
57 int			cpu_can_deep_sleep = 0;	/* C3 state is available. */
58 int			cpu_disable_deep_sleep = 0; /* Timer dies in C3. */
59 
60 static void		setuptimer(void);
61 static void		loadtimer(sbintime_t now, int first);
62 static int		doconfigtimer(void);
63 static void		configtimer(int start);
64 static int		round_freq(struct eventtimer *et, int freq);
65 
66 static sbintime_t	getnextcpuevent(int idle);
67 static sbintime_t	getnextevent(void);
68 static int		handleevents(sbintime_t now, int fake);
69 
70 static struct mtx	et_hw_mtx;
71 
72 #define	ET_HW_LOCK(state)						\
73 	{								\
74 		if (timer->et_flags & ET_FLAGS_PERCPU)			\
75 			mtx_lock_spin(&(state)->et_hw_mtx);		\
76 		else							\
77 			mtx_lock_spin(&et_hw_mtx);			\
78 	}
79 
80 #define	ET_HW_UNLOCK(state)						\
81 	{								\
82 		if (timer->et_flags & ET_FLAGS_PERCPU)			\
83 			mtx_unlock_spin(&(state)->et_hw_mtx);		\
84 		else							\
85 			mtx_unlock_spin(&et_hw_mtx);			\
86 	}
87 
88 static struct eventtimer *timer = NULL;
89 static sbintime_t	timerperiod;	/* Timer period for periodic mode. */
90 static sbintime_t	statperiod;	/* statclock() events period. */
91 static sbintime_t	profperiod;	/* profclock() events period. */
92 static sbintime_t	nexttick;	/* Next global timer tick time. */
93 static u_int		busy = 1;	/* Reconfiguration is in progress. */
94 static int		profiling;	/* Profiling events enabled. */
95 
96 static char		timername[32];	/* Wanted timer. */
97 TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
98 
99 static int		singlemul;	/* Multiplier for periodic mode. */
100 SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RWTUN, &singlemul,
101     0, "Multiplier for periodic mode");
102 
103 static u_int		idletick;	/* Run periodic events when idle. */
104 SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick,
105     0, "Run periodic events when idle");
106 
107 static int		periodic;	/* Periodic or one-shot mode. */
108 static int		want_periodic;	/* What mode to prefer. */
109 TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
110 
111 struct pcpu_state {
112 	struct mtx	et_hw_mtx;	/* Per-CPU timer mutex. */
113 	u_int		action;		/* Reconfiguration requests. */
114 	u_int		handle;		/* Immediate handle resuests. */
115 	sbintime_t	now;		/* Last tick time. */
116 	sbintime_t	nextevent;	/* Next scheduled event on this CPU. */
117 	sbintime_t	nexttick;	/* Next timer tick time. */
118 	sbintime_t	nexthard;	/* Next hardlock() event. */
119 	sbintime_t	nextstat;	/* Next statclock() event. */
120 	sbintime_t	nextprof;	/* Next profclock() event. */
121 	sbintime_t	nextcall;	/* Next callout event. */
122 	sbintime_t	nextcallopt;	/* Next optional callout event. */
123 	int		ipi;		/* This CPU needs IPI. */
124 	int		idle;		/* This CPU is in idle mode. */
125 };
126 
127 static DPCPU_DEFINE(struct pcpu_state, timerstate);
128 DPCPU_DEFINE(sbintime_t, hardclocktime);
129 
130 /*
131  * Timer broadcast IPI handler.
132  */
133 int
134 hardclockintr(void)
135 {
136 	sbintime_t now;
137 	struct pcpu_state *state;
138 	int done;
139 
140 	if (doconfigtimer() || busy)
141 		return (FILTER_HANDLED);
142 	state = DPCPU_PTR(timerstate);
143 	now = state->now;
144 	CTR3(KTR_SPARE2, "ipi  at %d:    now  %d.%08x",
145 	    curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
146 	done = handleevents(now, 0);
147 	return (done ? FILTER_HANDLED : FILTER_STRAY);
148 }
149 
150 /*
151  * Handle all events for specified time on this CPU
152  */
153 static int
154 handleevents(sbintime_t now, int fake)
155 {
156 	sbintime_t t, *hct;
157 	struct trapframe *frame;
158 	struct pcpu_state *state;
159 	int usermode;
160 	int done, runs;
161 
162 	CTR3(KTR_SPARE2, "handle at %d:  now  %d.%08x",
163 	    curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
164 	done = 0;
165 	if (fake) {
166 		frame = NULL;
167 		usermode = 0;
168 	} else {
169 		frame = curthread->td_intr_frame;
170 		usermode = TRAPF_USERMODE(frame);
171 	}
172 
173 	state = DPCPU_PTR(timerstate);
174 
175 	runs = 0;
176 	while (now >= state->nexthard) {
177 		state->nexthard += tick_sbt;
178 		runs++;
179 	}
180 	if (runs) {
181 		hct = DPCPU_PTR(hardclocktime);
182 		*hct = state->nexthard - tick_sbt;
183 		if (fake < 2) {
184 			hardclock_cnt(runs, usermode);
185 			done = 1;
186 		}
187 	}
188 	runs = 0;
189 	while (now >= state->nextstat) {
190 		state->nextstat += statperiod;
191 		runs++;
192 	}
193 	if (runs && fake < 2) {
194 		statclock_cnt(runs, usermode);
195 		done = 1;
196 	}
197 	if (profiling) {
198 		runs = 0;
199 		while (now >= state->nextprof) {
200 			state->nextprof += profperiod;
201 			runs++;
202 		}
203 		if (runs && !fake) {
204 			profclock_cnt(runs, usermode, TRAPF_PC(frame));
205 			done = 1;
206 		}
207 	} else
208 		state->nextprof = state->nextstat;
209 	if (now >= state->nextcallopt) {
210 		state->nextcall = state->nextcallopt = SBT_MAX;
211 		callout_process(now);
212 	}
213 
214 	t = getnextcpuevent(0);
215 	ET_HW_LOCK(state);
216 	if (!busy) {
217 		state->idle = 0;
218 		state->nextevent = t;
219 		loadtimer(now, (fake == 2) &&
220 		    (timer->et_flags & ET_FLAGS_PERCPU));
221 	}
222 	ET_HW_UNLOCK(state);
223 	return (done);
224 }
225 
226 /*
227  * Schedule binuptime of the next event on current CPU.
228  */
229 static sbintime_t
230 getnextcpuevent(int idle)
231 {
232 	sbintime_t event;
233 	struct pcpu_state *state;
234 	u_int hardfreq;
235 
236 	state = DPCPU_PTR(timerstate);
237 	/* Handle hardclock() events, skipping some if CPU is idle. */
238 	event = state->nexthard;
239 	if (idle) {
240 		hardfreq = (u_int)hz / 2;
241 		if (tc_min_ticktock_freq > 2
242 #ifdef SMP
243 		    && curcpu == CPU_FIRST()
244 #endif
245 		    )
246 			hardfreq = hz / tc_min_ticktock_freq;
247 		if (hardfreq > 1)
248 			event += tick_sbt * (hardfreq - 1);
249 	}
250 	/* Handle callout events. */
251 	if (event > state->nextcall)
252 		event = state->nextcall;
253 	if (!idle) { /* If CPU is active - handle other types of events. */
254 		if (event > state->nextstat)
255 			event = state->nextstat;
256 		if (profiling && event > state->nextprof)
257 			event = state->nextprof;
258 	}
259 	return (event);
260 }
261 
262 /*
263  * Schedule binuptime of the next event on all CPUs.
264  */
265 static sbintime_t
266 getnextevent(void)
267 {
268 	struct pcpu_state *state;
269 	sbintime_t event;
270 #ifdef SMP
271 	int	cpu;
272 #endif
273 	int	c;
274 
275 	state = DPCPU_PTR(timerstate);
276 	event = state->nextevent;
277 	c = -1;
278 #ifdef SMP
279 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
280 		CPU_FOREACH(cpu) {
281 			state = DPCPU_ID_PTR(cpu, timerstate);
282 			if (event > state->nextevent) {
283 				event = state->nextevent;
284 				c = cpu;
285 			}
286 		}
287 	}
288 #endif
289 	CTR4(KTR_SPARE2, "next at %d:    next %d.%08x by %d",
290 	    curcpu, (int)(event >> 32), (u_int)(event & 0xffffffff), c);
291 	return (event);
292 }
293 
294 /* Hardware timer callback function. */
295 static void
296 timercb(struct eventtimer *et, void *arg)
297 {
298 	sbintime_t now;
299 	sbintime_t *next;
300 	struct pcpu_state *state;
301 #ifdef SMP
302 	int cpu, bcast;
303 #endif
304 
305 	/* Do not touch anything if somebody reconfiguring timers. */
306 	if (busy)
307 		return;
308 	/* Update present and next tick times. */
309 	state = DPCPU_PTR(timerstate);
310 	if (et->et_flags & ET_FLAGS_PERCPU) {
311 		next = &state->nexttick;
312 	} else
313 		next = &nexttick;
314 	now = sbinuptime();
315 	if (periodic)
316 		*next = now + timerperiod;
317 	else
318 		*next = -1;	/* Next tick is not scheduled yet. */
319 	state->now = now;
320 	CTR3(KTR_SPARE2, "intr at %d:    now  %d.%08x",
321 	    curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
322 
323 #ifdef SMP
324 	/* Prepare broadcasting to other CPUs for non-per-CPU timers. */
325 	bcast = 0;
326 	if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
327 		CPU_FOREACH(cpu) {
328 			state = DPCPU_ID_PTR(cpu, timerstate);
329 			ET_HW_LOCK(state);
330 			state->now = now;
331 			if (now >= state->nextevent) {
332 				state->nextevent += SBT_1S;
333 				if (curcpu != cpu) {
334 					state->ipi = 1;
335 					bcast = 1;
336 				}
337 			}
338 			ET_HW_UNLOCK(state);
339 		}
340 	}
341 #endif
342 
343 	/* Handle events for this time on this CPU. */
344 	handleevents(now, 0);
345 
346 #ifdef SMP
347 	/* Broadcast interrupt to other CPUs for non-per-CPU timers. */
348 	if (bcast) {
349 		CPU_FOREACH(cpu) {
350 			if (curcpu == cpu)
351 				continue;
352 			state = DPCPU_ID_PTR(cpu, timerstate);
353 			if (state->ipi) {
354 				state->ipi = 0;
355 				ipi_cpu(cpu, IPI_HARDCLOCK);
356 			}
357 		}
358 	}
359 #endif
360 }
361 
362 /*
363  * Load new value into hardware timer.
364  */
365 static void
366 loadtimer(sbintime_t now, int start)
367 {
368 	struct pcpu_state *state;
369 	sbintime_t new;
370 	sbintime_t *next;
371 	uint64_t tmp;
372 	int eq;
373 
374 	if (timer->et_flags & ET_FLAGS_PERCPU) {
375 		state = DPCPU_PTR(timerstate);
376 		next = &state->nexttick;
377 	} else
378 		next = &nexttick;
379 	if (periodic) {
380 		if (start) {
381 			/*
382 			 * Try to start all periodic timers aligned
383 			 * to period to make events synchronous.
384 			 */
385 			tmp = now % timerperiod;
386 			new = timerperiod - tmp;
387 			if (new < tmp)		/* Left less then passed. */
388 				new += timerperiod;
389 			CTR5(KTR_SPARE2, "load p at %d:   now %d.%08x first in %d.%08x",
390 			    curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff),
391 			    (int)(new >> 32), (u_int)(new & 0xffffffff));
392 			*next = new + now;
393 			et_start(timer, new, timerperiod);
394 		}
395 	} else {
396 		new = getnextevent();
397 		eq = (new == *next);
398 		CTR4(KTR_SPARE2, "load at %d:    next %d.%08x eq %d",
399 		    curcpu, (int)(new >> 32), (u_int)(new & 0xffffffff), eq);
400 		if (!eq) {
401 			*next = new;
402 			et_start(timer, new - now, 0);
403 		}
404 	}
405 }
406 
407 /*
408  * Prepare event timer parameters after configuration changes.
409  */
410 static void
411 setuptimer(void)
412 {
413 	int freq;
414 
415 	if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
416 		periodic = 0;
417 	else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
418 		periodic = 1;
419 	singlemul = MIN(MAX(singlemul, 1), 20);
420 	freq = hz * singlemul;
421 	while (freq < (profiling ? profhz : stathz))
422 		freq += hz;
423 	freq = round_freq(timer, freq);
424 	timerperiod = SBT_1S / freq;
425 }
426 
427 /*
428  * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
429  */
430 static int
431 doconfigtimer(void)
432 {
433 	sbintime_t now;
434 	struct pcpu_state *state;
435 
436 	state = DPCPU_PTR(timerstate);
437 	switch (atomic_load_acq_int(&state->action)) {
438 	case 1:
439 		now = sbinuptime();
440 		ET_HW_LOCK(state);
441 		loadtimer(now, 1);
442 		ET_HW_UNLOCK(state);
443 		state->handle = 0;
444 		atomic_store_rel_int(&state->action, 0);
445 		return (1);
446 	case 2:
447 		ET_HW_LOCK(state);
448 		et_stop(timer);
449 		ET_HW_UNLOCK(state);
450 		state->handle = 0;
451 		atomic_store_rel_int(&state->action, 0);
452 		return (1);
453 	}
454 	if (atomic_readandclear_int(&state->handle) && !busy) {
455 		now = sbinuptime();
456 		handleevents(now, 0);
457 		return (1);
458 	}
459 	return (0);
460 }
461 
462 /*
463  * Reconfigure specified timer.
464  * For per-CPU timers use IPI to make other CPUs to reconfigure.
465  */
466 static void
467 configtimer(int start)
468 {
469 	sbintime_t now, next;
470 	struct pcpu_state *state;
471 	int cpu;
472 
473 	if (start) {
474 		setuptimer();
475 		now = sbinuptime();
476 	} else
477 		now = 0;
478 	critical_enter();
479 	ET_HW_LOCK(DPCPU_PTR(timerstate));
480 	if (start) {
481 		/* Initialize time machine parameters. */
482 		next = now + timerperiod;
483 		if (periodic)
484 			nexttick = next;
485 		else
486 			nexttick = -1;
487 		CPU_FOREACH(cpu) {
488 			state = DPCPU_ID_PTR(cpu, timerstate);
489 			state->now = now;
490 			if (!smp_started && cpu != CPU_FIRST())
491 				state->nextevent = SBT_MAX;
492 			else
493 				state->nextevent = next;
494 			if (periodic)
495 				state->nexttick = next;
496 			else
497 				state->nexttick = -1;
498 			state->nexthard = next;
499 			state->nextstat = next;
500 			state->nextprof = next;
501 			state->nextcall = next;
502 			state->nextcallopt = next;
503 			hardclock_sync(cpu);
504 		}
505 		busy = 0;
506 		/* Start global timer or per-CPU timer of this CPU. */
507 		loadtimer(now, 1);
508 	} else {
509 		busy = 1;
510 		/* Stop global timer or per-CPU timer of this CPU. */
511 		et_stop(timer);
512 	}
513 	ET_HW_UNLOCK(DPCPU_PTR(timerstate));
514 #ifdef SMP
515 	/* If timer is global or there is no other CPUs yet - we are done. */
516 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
517 		critical_exit();
518 		return;
519 	}
520 	/* Set reconfigure flags for other CPUs. */
521 	CPU_FOREACH(cpu) {
522 		state = DPCPU_ID_PTR(cpu, timerstate);
523 		atomic_store_rel_int(&state->action,
524 		    (cpu == curcpu) ? 0 : ( start ? 1 : 2));
525 	}
526 	/* Broadcast reconfigure IPI. */
527 	ipi_all_but_self(IPI_HARDCLOCK);
528 	/* Wait for reconfiguration completed. */
529 restart:
530 	cpu_spinwait();
531 	CPU_FOREACH(cpu) {
532 		if (cpu == curcpu)
533 			continue;
534 		state = DPCPU_ID_PTR(cpu, timerstate);
535 		if (atomic_load_acq_int(&state->action))
536 			goto restart;
537 	}
538 #endif
539 	critical_exit();
540 }
541 
542 /*
543  * Calculate nearest frequency supported by hardware timer.
544  */
545 static int
546 round_freq(struct eventtimer *et, int freq)
547 {
548 	uint64_t div;
549 
550 	if (et->et_frequency != 0) {
551 		div = lmax((et->et_frequency + freq / 2) / freq, 1);
552 		if (et->et_flags & ET_FLAGS_POW2DIV)
553 			div = 1 << (flsl(div + div / 2) - 1);
554 		freq = (et->et_frequency + div / 2) / div;
555 	}
556 	if (et->et_min_period > SBT_1S)
557 		panic("Event timer \"%s\" doesn't support sub-second periods!",
558 		    et->et_name);
559 	else if (et->et_min_period != 0)
560 		freq = min(freq, SBT2FREQ(et->et_min_period));
561 	if (et->et_max_period < SBT_1S && et->et_max_period != 0)
562 		freq = max(freq, SBT2FREQ(et->et_max_period));
563 	return (freq);
564 }
565 
566 /*
567  * Configure and start event timers (BSP part).
568  */
569 void
570 cpu_initclocks_bsp(void)
571 {
572 	struct pcpu_state *state;
573 	int base, div, cpu;
574 
575 	mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
576 	CPU_FOREACH(cpu) {
577 		state = DPCPU_ID_PTR(cpu, timerstate);
578 		mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
579 		state->nextcall = SBT_MAX;
580 		state->nextcallopt = SBT_MAX;
581 	}
582 	periodic = want_periodic;
583 	/* Grab requested timer or the best of present. */
584 	if (timername[0])
585 		timer = et_find(timername, 0, 0);
586 	if (timer == NULL && periodic) {
587 		timer = et_find(NULL,
588 		    ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
589 	}
590 	if (timer == NULL) {
591 		timer = et_find(NULL,
592 		    ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
593 	}
594 	if (timer == NULL && !periodic) {
595 		timer = et_find(NULL,
596 		    ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
597 	}
598 	if (timer == NULL)
599 		panic("No usable event timer found!");
600 	et_init(timer, timercb, NULL, NULL);
601 
602 	/* Adapt to timer capabilities. */
603 	if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
604 		periodic = 0;
605 	else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
606 		periodic = 1;
607 	if (timer->et_flags & ET_FLAGS_C3STOP)
608 		cpu_disable_deep_sleep++;
609 
610 	/*
611 	 * We honor the requested 'hz' value.
612 	 * We want to run stathz in the neighborhood of 128hz.
613 	 * We would like profhz to run as often as possible.
614 	 */
615 	if (singlemul <= 0 || singlemul > 20) {
616 		if (hz >= 1500 || (hz % 128) == 0)
617 			singlemul = 1;
618 		else if (hz >= 750)
619 			singlemul = 2;
620 		else
621 			singlemul = 4;
622 	}
623 	if (periodic) {
624 		base = round_freq(timer, hz * singlemul);
625 		singlemul = max((base + hz / 2) / hz, 1);
626 		hz = (base + singlemul / 2) / singlemul;
627 		if (base <= 128)
628 			stathz = base;
629 		else {
630 			div = base / 128;
631 			if (div >= singlemul && (div % singlemul) == 0)
632 				div++;
633 			stathz = base / div;
634 		}
635 		profhz = stathz;
636 		while ((profhz + stathz) <= 128 * 64)
637 			profhz += stathz;
638 		profhz = round_freq(timer, profhz);
639 	} else {
640 		hz = round_freq(timer, hz);
641 		stathz = round_freq(timer, 127);
642 		profhz = round_freq(timer, stathz * 64);
643 	}
644 	tick = 1000000 / hz;
645 	tick_sbt = SBT_1S / hz;
646 	tick_bt = sbttobt(tick_sbt);
647 	statperiod = SBT_1S / stathz;
648 	profperiod = SBT_1S / profhz;
649 	ET_LOCK();
650 	configtimer(1);
651 	ET_UNLOCK();
652 }
653 
654 /*
655  * Start per-CPU event timers on APs.
656  */
657 void
658 cpu_initclocks_ap(void)
659 {
660 	sbintime_t now;
661 	struct pcpu_state *state;
662 	struct thread *td;
663 
664 	state = DPCPU_PTR(timerstate);
665 	now = sbinuptime();
666 	ET_HW_LOCK(state);
667 	state->now = now;
668 	hardclock_sync(curcpu);
669 	spinlock_enter();
670 	ET_HW_UNLOCK(state);
671 	td = curthread;
672 	td->td_intr_nesting_level++;
673 	handleevents(state->now, 2);
674 	td->td_intr_nesting_level--;
675 	spinlock_exit();
676 }
677 
678 /*
679  * Switch to profiling clock rates.
680  */
681 void
682 cpu_startprofclock(void)
683 {
684 
685 	ET_LOCK();
686 	if (profiling == 0) {
687 		if (periodic) {
688 			configtimer(0);
689 			profiling = 1;
690 			configtimer(1);
691 		} else
692 			profiling = 1;
693 	} else
694 		profiling++;
695 	ET_UNLOCK();
696 }
697 
698 /*
699  * Switch to regular clock rates.
700  */
701 void
702 cpu_stopprofclock(void)
703 {
704 
705 	ET_LOCK();
706 	if (profiling == 1) {
707 		if (periodic) {
708 			configtimer(0);
709 			profiling = 0;
710 			configtimer(1);
711 		} else
712 		profiling = 0;
713 	} else
714 		profiling--;
715 	ET_UNLOCK();
716 }
717 
718 /*
719  * Switch to idle mode (all ticks handled).
720  */
721 sbintime_t
722 cpu_idleclock(void)
723 {
724 	sbintime_t now, t;
725 	struct pcpu_state *state;
726 
727 	if (idletick || busy ||
728 	    (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
729 #ifdef DEVICE_POLLING
730 	    || curcpu == CPU_FIRST()
731 #endif
732 	    )
733 		return (-1);
734 	state = DPCPU_PTR(timerstate);
735 	if (periodic)
736 		now = state->now;
737 	else
738 		now = sbinuptime();
739 	CTR3(KTR_SPARE2, "idle at %d:    now  %d.%08x",
740 	    curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
741 	t = getnextcpuevent(1);
742 	ET_HW_LOCK(state);
743 	state->idle = 1;
744 	state->nextevent = t;
745 	if (!periodic)
746 		loadtimer(now, 0);
747 	ET_HW_UNLOCK(state);
748 	return (MAX(t - now, 0));
749 }
750 
751 /*
752  * Switch to active mode (skip empty ticks).
753  */
754 void
755 cpu_activeclock(void)
756 {
757 	sbintime_t now;
758 	struct pcpu_state *state;
759 	struct thread *td;
760 
761 	state = DPCPU_PTR(timerstate);
762 	if (state->idle == 0 || busy)
763 		return;
764 	if (periodic)
765 		now = state->now;
766 	else
767 		now = sbinuptime();
768 	CTR3(KTR_SPARE2, "active at %d:  now  %d.%08x",
769 	    curcpu, (int)(now >> 32), (u_int)(now & 0xffffffff));
770 	spinlock_enter();
771 	td = curthread;
772 	td->td_intr_nesting_level++;
773 	handleevents(now, 1);
774 	td->td_intr_nesting_level--;
775 	spinlock_exit();
776 }
777 
778 /*
779  * Change the frequency of the given timer.  This changes et->et_frequency and
780  * if et is the active timer it reconfigures the timer on all CPUs.  This is
781  * intended to be a private interface for the use of et_change_frequency() only.
782  */
783 void
784 cpu_et_frequency(struct eventtimer *et, uint64_t newfreq)
785 {
786 
787 	ET_LOCK();
788 	if (et == timer) {
789 		configtimer(0);
790 		et->et_frequency = newfreq;
791 		configtimer(1);
792 	} else
793 		et->et_frequency = newfreq;
794 	ET_UNLOCK();
795 }
796 
797 void
798 cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt)
799 {
800 	struct pcpu_state *state;
801 
802 	/* Do not touch anything if somebody reconfiguring timers. */
803 	if (busy)
804 		return;
805 	CTR6(KTR_SPARE2, "new co at %d:    on %d at %d.%08x - %d.%08x",
806 	    curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff),
807 	    (int)(bt >> 32), (u_int)(bt & 0xffffffff));
808 	state = DPCPU_ID_PTR(cpu, timerstate);
809 	ET_HW_LOCK(state);
810 
811 	/*
812 	 * If there is callout time already set earlier -- do nothing.
813 	 * This check may appear redundant because we check already in
814 	 * callout_process() but this double check guarantees we're safe
815 	 * with respect to race conditions between interrupts execution
816 	 * and scheduling.
817 	 */
818 	state->nextcallopt = bt_opt;
819 	if (bt >= state->nextcall)
820 		goto done;
821 	state->nextcall = bt;
822 	/* If there is some other event set earlier -- do nothing. */
823 	if (bt >= state->nextevent)
824 		goto done;
825 	state->nextevent = bt;
826 	/* If timer is periodic -- there is nothing to reprogram. */
827 	if (periodic)
828 		goto done;
829 	/* If timer is global or of the current CPU -- reprogram it. */
830 	if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || cpu == curcpu) {
831 		loadtimer(sbinuptime(), 0);
832 done:
833 		ET_HW_UNLOCK(state);
834 		return;
835 	}
836 	/* Otherwise make other CPU to reprogram it. */
837 	state->handle = 1;
838 	ET_HW_UNLOCK(state);
839 #ifdef SMP
840 	ipi_cpu(cpu, IPI_HARDCLOCK);
841 #endif
842 }
843 
844 /*
845  * Report or change the active event timers hardware.
846  */
847 static int
848 sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
849 {
850 	char buf[32];
851 	struct eventtimer *et;
852 	int error;
853 
854 	ET_LOCK();
855 	et = timer;
856 	snprintf(buf, sizeof(buf), "%s", et->et_name);
857 	ET_UNLOCK();
858 	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
859 	ET_LOCK();
860 	et = timer;
861 	if (error != 0 || req->newptr == NULL ||
862 	    strcasecmp(buf, et->et_name) == 0) {
863 		ET_UNLOCK();
864 		return (error);
865 	}
866 	et = et_find(buf, 0, 0);
867 	if (et == NULL) {
868 		ET_UNLOCK();
869 		return (ENOENT);
870 	}
871 	configtimer(0);
872 	et_free(timer);
873 	if (et->et_flags & ET_FLAGS_C3STOP)
874 		cpu_disable_deep_sleep++;
875 	if (timer->et_flags & ET_FLAGS_C3STOP)
876 		cpu_disable_deep_sleep--;
877 	periodic = want_periodic;
878 	timer = et;
879 	et_init(timer, timercb, NULL, NULL);
880 	configtimer(1);
881 	ET_UNLOCK();
882 	return (error);
883 }
884 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
885     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
886     0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
887 
888 /*
889  * Report or change the active event timer periodicity.
890  */
891 static int
892 sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
893 {
894 	int error, val;
895 
896 	val = periodic;
897 	error = sysctl_handle_int(oidp, &val, 0, req);
898 	if (error != 0 || req->newptr == NULL)
899 		return (error);
900 	ET_LOCK();
901 	configtimer(0);
902 	periodic = want_periodic = val;
903 	configtimer(1);
904 	ET_UNLOCK();
905 	return (error);
906 }
907 SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
908     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
909     0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
910