xref: /freebsd/sys/kern/kern_tc.c (revision 26a222dc0c048fc071b548eadad7b80405a1b126)
1 /*-
2  * ----------------------------------------------------------------------------
3  * "THE BEER-WARE LICENSE" (Revision 42):
4  * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
5  * can do whatever you want with this stuff. If we meet some day, and you think
6  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
7  * ----------------------------------------------------------------------------
8  *
9  * Copyright (c) 2011 The FreeBSD Foundation
10  * All rights reserved.
11  *
12  * Portions of this software were developed by Julien Ridoux at the University
13  * of Melbourne under sponsorship from the FreeBSD Foundation.
14  */
15 
16 #include <sys/cdefs.h>
17 __FBSDID("$FreeBSD$");
18 
19 #include "opt_compat.h"
20 #include "opt_ntp.h"
21 #include "opt_ffclock.h"
22 
23 #include <sys/param.h>
24 #include <sys/kernel.h>
25 #include <sys/limits.h>
26 #include <sys/lock.h>
27 #include <sys/mutex.h>
28 #include <sys/sysctl.h>
29 #include <sys/syslog.h>
30 #include <sys/systm.h>
31 #include <sys/timeffc.h>
32 #include <sys/timepps.h>
33 #include <sys/timetc.h>
34 #include <sys/timex.h>
35 #include <sys/vdso.h>
36 
37 /*
38  * A large step happens on boot.  This constant detects such steps.
39  * It is relatively small so that ntp_update_second gets called enough
40  * in the typical 'missed a couple of seconds' case, but doesn't loop
41  * forever when the time step is large.
42  */
43 #define LARGE_STEP	200
44 
45 /*
46  * Implement a dummy timecounter which we can use until we get a real one
47  * in the air.  This allows the console and other early stuff to use
48  * time services.
49  */
50 
51 static u_int
52 dummy_get_timecount(struct timecounter *tc)
53 {
54 	static u_int now;
55 
56 	return (++now);
57 }
58 
59 static struct timecounter dummy_timecounter = {
60 	dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000
61 };
62 
63 struct timehands {
64 	/* These fields must be initialized by the driver. */
65 	struct timecounter	*th_counter;
66 	int64_t			th_adjustment;
67 	uint64_t		th_scale;
68 	u_int	 		th_offset_count;
69 	struct bintime		th_offset;
70 	struct timeval		th_microtime;
71 	struct timespec		th_nanotime;
72 	/* Fields not to be copied in tc_windup start with th_generation. */
73 	volatile u_int		th_generation;
74 	struct timehands	*th_next;
75 };
76 
77 static struct timehands th0;
78 static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0};
79 static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9};
80 static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8};
81 static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7};
82 static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6};
83 static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5};
84 static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4};
85 static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3};
86 static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2};
87 static struct timehands th0 = {
88 	&dummy_timecounter,
89 	0,
90 	(uint64_t)-1 / 1000000,
91 	0,
92 	{1, 0},
93 	{0, 0},
94 	{0, 0},
95 	1,
96 	&th1
97 };
98 
99 static struct timehands *volatile timehands = &th0;
100 struct timecounter *timecounter = &dummy_timecounter;
101 static struct timecounter *timecounters = &dummy_timecounter;
102 
103 int tc_min_ticktock_freq = 1;
104 
105 volatile time_t time_second = 1;
106 volatile time_t time_uptime = 1;
107 
108 struct bintime boottimebin;
109 struct timeval boottime;
110 static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS);
111 SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD,
112     NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime");
113 
114 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, "");
115 static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
116 
117 static int timestepwarnings;
118 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
119     &timestepwarnings, 0, "Log time steps");
120 
121 struct bintime bt_timethreshold;
122 struct bintime bt_tickthreshold;
123 sbintime_t sbt_timethreshold;
124 sbintime_t sbt_tickthreshold;
125 struct bintime tc_tick_bt;
126 sbintime_t tc_tick_sbt;
127 int tc_precexp;
128 int tc_timepercentage = TC_DEFAULTPERC;
129 static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS);
130 SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation,
131     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0,
132     sysctl_kern_timecounter_adjprecision, "I",
133     "Allowed time interval deviation in percents");
134 
135 static void tc_windup(void);
136 static void cpu_tick_calibrate(int);
137 
138 void dtrace_getnanotime(struct timespec *tsp);
139 
140 static int
141 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
142 {
143 #ifndef __mips__
144 #ifdef SCTL_MASK32
145 	int tv[2];
146 
147 	if (req->flags & SCTL_MASK32) {
148 		tv[0] = boottime.tv_sec;
149 		tv[1] = boottime.tv_usec;
150 		return SYSCTL_OUT(req, tv, sizeof(tv));
151 	} else
152 #endif
153 #endif
154 		return SYSCTL_OUT(req, &boottime, sizeof(boottime));
155 }
156 
157 static int
158 sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS)
159 {
160 	u_int ncount;
161 	struct timecounter *tc = arg1;
162 
163 	ncount = tc->tc_get_timecount(tc);
164 	return sysctl_handle_int(oidp, &ncount, 0, req);
165 }
166 
167 static int
168 sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS)
169 {
170 	uint64_t freq;
171 	struct timecounter *tc = arg1;
172 
173 	freq = tc->tc_frequency;
174 	return sysctl_handle_64(oidp, &freq, 0, req);
175 }
176 
177 /*
178  * Return the difference between the timehands' counter value now and what
179  * was when we copied it to the timehands' offset_count.
180  */
181 static __inline u_int
182 tc_delta(struct timehands *th)
183 {
184 	struct timecounter *tc;
185 
186 	tc = th->th_counter;
187 	return ((tc->tc_get_timecount(tc) - th->th_offset_count) &
188 	    tc->tc_counter_mask);
189 }
190 
191 /*
192  * Functions for reading the time.  We have to loop until we are sure that
193  * the timehands that we operated on was not updated under our feet.  See
194  * the comment in <sys/time.h> for a description of these 12 functions.
195  */
196 
197 #ifdef FFCLOCK
198 void
199 fbclock_binuptime(struct bintime *bt)
200 {
201 	struct timehands *th;
202 	unsigned int gen;
203 
204 	do {
205 		th = timehands;
206 		gen = th->th_generation;
207 		*bt = th->th_offset;
208 		bintime_addx(bt, th->th_scale * tc_delta(th));
209 	} while (gen == 0 || gen != th->th_generation);
210 }
211 
212 void
213 fbclock_nanouptime(struct timespec *tsp)
214 {
215 	struct bintime bt;
216 
217 	fbclock_binuptime(&bt);
218 	bintime2timespec(&bt, tsp);
219 }
220 
221 void
222 fbclock_microuptime(struct timeval *tvp)
223 {
224 	struct bintime bt;
225 
226 	fbclock_binuptime(&bt);
227 	bintime2timeval(&bt, tvp);
228 }
229 
230 void
231 fbclock_bintime(struct bintime *bt)
232 {
233 
234 	fbclock_binuptime(bt);
235 	bintime_add(bt, &boottimebin);
236 }
237 
238 void
239 fbclock_nanotime(struct timespec *tsp)
240 {
241 	struct bintime bt;
242 
243 	fbclock_bintime(&bt);
244 	bintime2timespec(&bt, tsp);
245 }
246 
247 void
248 fbclock_microtime(struct timeval *tvp)
249 {
250 	struct bintime bt;
251 
252 	fbclock_bintime(&bt);
253 	bintime2timeval(&bt, tvp);
254 }
255 
256 void
257 fbclock_getbinuptime(struct bintime *bt)
258 {
259 	struct timehands *th;
260 	unsigned int gen;
261 
262 	do {
263 		th = timehands;
264 		gen = th->th_generation;
265 		*bt = th->th_offset;
266 	} while (gen == 0 || gen != th->th_generation);
267 }
268 
269 void
270 fbclock_getnanouptime(struct timespec *tsp)
271 {
272 	struct timehands *th;
273 	unsigned int gen;
274 
275 	do {
276 		th = timehands;
277 		gen = th->th_generation;
278 		bintime2timespec(&th->th_offset, tsp);
279 	} while (gen == 0 || gen != th->th_generation);
280 }
281 
282 void
283 fbclock_getmicrouptime(struct timeval *tvp)
284 {
285 	struct timehands *th;
286 	unsigned int gen;
287 
288 	do {
289 		th = timehands;
290 		gen = th->th_generation;
291 		bintime2timeval(&th->th_offset, tvp);
292 	} while (gen == 0 || gen != th->th_generation);
293 }
294 
295 void
296 fbclock_getbintime(struct bintime *bt)
297 {
298 	struct timehands *th;
299 	unsigned int gen;
300 
301 	do {
302 		th = timehands;
303 		gen = th->th_generation;
304 		*bt = th->th_offset;
305 	} while (gen == 0 || gen != th->th_generation);
306 	bintime_add(bt, &boottimebin);
307 }
308 
309 void
310 fbclock_getnanotime(struct timespec *tsp)
311 {
312 	struct timehands *th;
313 	unsigned int gen;
314 
315 	do {
316 		th = timehands;
317 		gen = th->th_generation;
318 		*tsp = th->th_nanotime;
319 	} while (gen == 0 || gen != th->th_generation);
320 }
321 
322 void
323 fbclock_getmicrotime(struct timeval *tvp)
324 {
325 	struct timehands *th;
326 	unsigned int gen;
327 
328 	do {
329 		th = timehands;
330 		gen = th->th_generation;
331 		*tvp = th->th_microtime;
332 	} while (gen == 0 || gen != th->th_generation);
333 }
334 #else /* !FFCLOCK */
335 void
336 binuptime(struct bintime *bt)
337 {
338 	struct timehands *th;
339 	u_int gen;
340 
341 	do {
342 		th = timehands;
343 		gen = th->th_generation;
344 		*bt = th->th_offset;
345 		bintime_addx(bt, th->th_scale * tc_delta(th));
346 	} while (gen == 0 || gen != th->th_generation);
347 }
348 
349 void
350 nanouptime(struct timespec *tsp)
351 {
352 	struct bintime bt;
353 
354 	binuptime(&bt);
355 	bintime2timespec(&bt, tsp);
356 }
357 
358 void
359 microuptime(struct timeval *tvp)
360 {
361 	struct bintime bt;
362 
363 	binuptime(&bt);
364 	bintime2timeval(&bt, tvp);
365 }
366 
367 void
368 bintime(struct bintime *bt)
369 {
370 
371 	binuptime(bt);
372 	bintime_add(bt, &boottimebin);
373 }
374 
375 void
376 nanotime(struct timespec *tsp)
377 {
378 	struct bintime bt;
379 
380 	bintime(&bt);
381 	bintime2timespec(&bt, tsp);
382 }
383 
384 void
385 microtime(struct timeval *tvp)
386 {
387 	struct bintime bt;
388 
389 	bintime(&bt);
390 	bintime2timeval(&bt, tvp);
391 }
392 
393 void
394 getbinuptime(struct bintime *bt)
395 {
396 	struct timehands *th;
397 	u_int gen;
398 
399 	do {
400 		th = timehands;
401 		gen = th->th_generation;
402 		*bt = th->th_offset;
403 	} while (gen == 0 || gen != th->th_generation);
404 }
405 
406 void
407 getnanouptime(struct timespec *tsp)
408 {
409 	struct timehands *th;
410 	u_int gen;
411 
412 	do {
413 		th = timehands;
414 		gen = th->th_generation;
415 		bintime2timespec(&th->th_offset, tsp);
416 	} while (gen == 0 || gen != th->th_generation);
417 }
418 
419 void
420 getmicrouptime(struct timeval *tvp)
421 {
422 	struct timehands *th;
423 	u_int gen;
424 
425 	do {
426 		th = timehands;
427 		gen = th->th_generation;
428 		bintime2timeval(&th->th_offset, tvp);
429 	} while (gen == 0 || gen != th->th_generation);
430 }
431 
432 void
433 getbintime(struct bintime *bt)
434 {
435 	struct timehands *th;
436 	u_int gen;
437 
438 	do {
439 		th = timehands;
440 		gen = th->th_generation;
441 		*bt = th->th_offset;
442 	} while (gen == 0 || gen != th->th_generation);
443 	bintime_add(bt, &boottimebin);
444 }
445 
446 void
447 getnanotime(struct timespec *tsp)
448 {
449 	struct timehands *th;
450 	u_int gen;
451 
452 	do {
453 		th = timehands;
454 		gen = th->th_generation;
455 		*tsp = th->th_nanotime;
456 	} while (gen == 0 || gen != th->th_generation);
457 }
458 
459 void
460 getmicrotime(struct timeval *tvp)
461 {
462 	struct timehands *th;
463 	u_int gen;
464 
465 	do {
466 		th = timehands;
467 		gen = th->th_generation;
468 		*tvp = th->th_microtime;
469 	} while (gen == 0 || gen != th->th_generation);
470 }
471 #endif /* FFCLOCK */
472 
473 #ifdef FFCLOCK
474 /*
475  * Support for feed-forward synchronization algorithms. This is heavily inspired
476  * by the timehands mechanism but kept independent from it. *_windup() functions
477  * have some connection to avoid accessing the timecounter hardware more than
478  * necessary.
479  */
480 
481 /* Feed-forward clock estimates kept updated by the synchronization daemon. */
482 struct ffclock_estimate ffclock_estimate;
483 struct bintime ffclock_boottime;	/* Feed-forward boot time estimate. */
484 uint32_t ffclock_status;		/* Feed-forward clock status. */
485 int8_t ffclock_updated;			/* New estimates are available. */
486 struct mtx ffclock_mtx;			/* Mutex on ffclock_estimate. */
487 
488 struct fftimehands {
489 	struct ffclock_estimate	cest;
490 	struct bintime		tick_time;
491 	struct bintime		tick_time_lerp;
492 	ffcounter		tick_ffcount;
493 	uint64_t		period_lerp;
494 	volatile uint8_t	gen;
495 	struct fftimehands	*next;
496 };
497 
498 #define	NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
499 
500 static struct fftimehands ffth[10];
501 static struct fftimehands *volatile fftimehands = ffth;
502 
503 static void
504 ffclock_init(void)
505 {
506 	struct fftimehands *cur;
507 	struct fftimehands *last;
508 
509 	memset(ffth, 0, sizeof(ffth));
510 
511 	last = ffth + NUM_ELEMENTS(ffth) - 1;
512 	for (cur = ffth; cur < last; cur++)
513 		cur->next = cur + 1;
514 	last->next = ffth;
515 
516 	ffclock_updated = 0;
517 	ffclock_status = FFCLOCK_STA_UNSYNC;
518 	mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
519 }
520 
521 /*
522  * Reset the feed-forward clock estimates. Called from inittodr() to get things
523  * kick started and uses the timecounter nominal frequency as a first period
524  * estimate. Note: this function may be called several time just after boot.
525  * Note: this is the only function that sets the value of boot time for the
526  * monotonic (i.e. uptime) version of the feed-forward clock.
527  */
528 void
529 ffclock_reset_clock(struct timespec *ts)
530 {
531 	struct timecounter *tc;
532 	struct ffclock_estimate cest;
533 
534 	tc = timehands->th_counter;
535 	memset(&cest, 0, sizeof(struct ffclock_estimate));
536 
537 	timespec2bintime(ts, &ffclock_boottime);
538 	timespec2bintime(ts, &(cest.update_time));
539 	ffclock_read_counter(&cest.update_ffcount);
540 	cest.leapsec_next = 0;
541 	cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
542 	cest.errb_abs = 0;
543 	cest.errb_rate = 0;
544 	cest.status = FFCLOCK_STA_UNSYNC;
545 	cest.leapsec_total = 0;
546 	cest.leapsec = 0;
547 
548 	mtx_lock(&ffclock_mtx);
549 	bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
550 	ffclock_updated = INT8_MAX;
551 	mtx_unlock(&ffclock_mtx);
552 
553 	printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
554 	    (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
555 	    (unsigned long)ts->tv_nsec);
556 }
557 
558 /*
559  * Sub-routine to convert a time interval measured in RAW counter units to time
560  * in seconds stored in bintime format.
561  * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
562  * larger than the max value of u_int (on 32 bit architecture). Loop to consume
563  * extra cycles.
564  */
565 static void
566 ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
567 {
568 	struct bintime bt2;
569 	ffcounter delta, delta_max;
570 
571 	delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
572 	bintime_clear(bt);
573 	do {
574 		if (ffdelta > delta_max)
575 			delta = delta_max;
576 		else
577 			delta = ffdelta;
578 		bt2.sec = 0;
579 		bt2.frac = period;
580 		bintime_mul(&bt2, (unsigned int)delta);
581 		bintime_add(bt, &bt2);
582 		ffdelta -= delta;
583 	} while (ffdelta > 0);
584 }
585 
586 /*
587  * Update the fftimehands.
588  * Push the tick ffcount and time(s) forward based on current clock estimate.
589  * The conversion from ffcounter to bintime relies on the difference clock
590  * principle, whose accuracy relies on computing small time intervals. If a new
591  * clock estimate has been passed by the synchronisation daemon, make it
592  * current, and compute the linear interpolation for monotonic time if needed.
593  */
594 static void
595 ffclock_windup(unsigned int delta)
596 {
597 	struct ffclock_estimate *cest;
598 	struct fftimehands *ffth;
599 	struct bintime bt, gap_lerp;
600 	ffcounter ffdelta;
601 	uint64_t frac;
602 	unsigned int polling;
603 	uint8_t forward_jump, ogen;
604 
605 	/*
606 	 * Pick the next timehand, copy current ffclock estimates and move tick
607 	 * times and counter forward.
608 	 */
609 	forward_jump = 0;
610 	ffth = fftimehands->next;
611 	ogen = ffth->gen;
612 	ffth->gen = 0;
613 	cest = &ffth->cest;
614 	bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
615 	ffdelta = (ffcounter)delta;
616 	ffth->period_lerp = fftimehands->period_lerp;
617 
618 	ffth->tick_time = fftimehands->tick_time;
619 	ffclock_convert_delta(ffdelta, cest->period, &bt);
620 	bintime_add(&ffth->tick_time, &bt);
621 
622 	ffth->tick_time_lerp = fftimehands->tick_time_lerp;
623 	ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
624 	bintime_add(&ffth->tick_time_lerp, &bt);
625 
626 	ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
627 
628 	/*
629 	 * Assess the status of the clock, if the last update is too old, it is
630 	 * likely the synchronisation daemon is dead and the clock is free
631 	 * running.
632 	 */
633 	if (ffclock_updated == 0) {
634 		ffdelta = ffth->tick_ffcount - cest->update_ffcount;
635 		ffclock_convert_delta(ffdelta, cest->period, &bt);
636 		if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
637 			ffclock_status |= FFCLOCK_STA_UNSYNC;
638 	}
639 
640 	/*
641 	 * If available, grab updated clock estimates and make them current.
642 	 * Recompute time at this tick using the updated estimates. The clock
643 	 * estimates passed the feed-forward synchronisation daemon may result
644 	 * in time conversion that is not monotonically increasing (just after
645 	 * the update). time_lerp is a particular linear interpolation over the
646 	 * synchronisation algo polling period that ensures monotonicity for the
647 	 * clock ids requesting it.
648 	 */
649 	if (ffclock_updated > 0) {
650 		bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
651 		ffdelta = ffth->tick_ffcount - cest->update_ffcount;
652 		ffth->tick_time = cest->update_time;
653 		ffclock_convert_delta(ffdelta, cest->period, &bt);
654 		bintime_add(&ffth->tick_time, &bt);
655 
656 		/* ffclock_reset sets ffclock_updated to INT8_MAX */
657 		if (ffclock_updated == INT8_MAX)
658 			ffth->tick_time_lerp = ffth->tick_time;
659 
660 		if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
661 			forward_jump = 1;
662 		else
663 			forward_jump = 0;
664 
665 		bintime_clear(&gap_lerp);
666 		if (forward_jump) {
667 			gap_lerp = ffth->tick_time;
668 			bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
669 		} else {
670 			gap_lerp = ffth->tick_time_lerp;
671 			bintime_sub(&gap_lerp, &ffth->tick_time);
672 		}
673 
674 		/*
675 		 * The reset from the RTC clock may be far from accurate, and
676 		 * reducing the gap between real time and interpolated time
677 		 * could take a very long time if the interpolated clock insists
678 		 * on strict monotonicity. The clock is reset under very strict
679 		 * conditions (kernel time is known to be wrong and
680 		 * synchronization daemon has been restarted recently.
681 		 * ffclock_boottime absorbs the jump to ensure boot time is
682 		 * correct and uptime functions stay consistent.
683 		 */
684 		if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
685 		    ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
686 		    ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
687 			if (forward_jump)
688 				bintime_add(&ffclock_boottime, &gap_lerp);
689 			else
690 				bintime_sub(&ffclock_boottime, &gap_lerp);
691 			ffth->tick_time_lerp = ffth->tick_time;
692 			bintime_clear(&gap_lerp);
693 		}
694 
695 		ffclock_status = cest->status;
696 		ffth->period_lerp = cest->period;
697 
698 		/*
699 		 * Compute corrected period used for the linear interpolation of
700 		 * time. The rate of linear interpolation is capped to 5000PPM
701 		 * (5ms/s).
702 		 */
703 		if (bintime_isset(&gap_lerp)) {
704 			ffdelta = cest->update_ffcount;
705 			ffdelta -= fftimehands->cest.update_ffcount;
706 			ffclock_convert_delta(ffdelta, cest->period, &bt);
707 			polling = bt.sec;
708 			bt.sec = 0;
709 			bt.frac = 5000000 * (uint64_t)18446744073LL;
710 			bintime_mul(&bt, polling);
711 			if (bintime_cmp(&gap_lerp, &bt, >))
712 				gap_lerp = bt;
713 
714 			/* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
715 			frac = 0;
716 			if (gap_lerp.sec > 0) {
717 				frac -= 1;
718 				frac /= ffdelta / gap_lerp.sec;
719 			}
720 			frac += gap_lerp.frac / ffdelta;
721 
722 			if (forward_jump)
723 				ffth->period_lerp += frac;
724 			else
725 				ffth->period_lerp -= frac;
726 		}
727 
728 		ffclock_updated = 0;
729 	}
730 	if (++ogen == 0)
731 		ogen = 1;
732 	ffth->gen = ogen;
733 	fftimehands = ffth;
734 }
735 
736 /*
737  * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
738  * the old and new hardware counter cannot be read simultaneously. tc_windup()
739  * does read the two counters 'back to back', but a few cycles are effectively
740  * lost, and not accumulated in tick_ffcount. This is a fairly radical
741  * operation for a feed-forward synchronization daemon, and it is its job to not
742  * pushing irrelevant data to the kernel. Because there is no locking here,
743  * simply force to ignore pending or next update to give daemon a chance to
744  * realize the counter has changed.
745  */
746 static void
747 ffclock_change_tc(struct timehands *th)
748 {
749 	struct fftimehands *ffth;
750 	struct ffclock_estimate *cest;
751 	struct timecounter *tc;
752 	uint8_t ogen;
753 
754 	tc = th->th_counter;
755 	ffth = fftimehands->next;
756 	ogen = ffth->gen;
757 	ffth->gen = 0;
758 
759 	cest = &ffth->cest;
760 	bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
761 	cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
762 	cest->errb_abs = 0;
763 	cest->errb_rate = 0;
764 	cest->status |= FFCLOCK_STA_UNSYNC;
765 
766 	ffth->tick_ffcount = fftimehands->tick_ffcount;
767 	ffth->tick_time_lerp = fftimehands->tick_time_lerp;
768 	ffth->tick_time = fftimehands->tick_time;
769 	ffth->period_lerp = cest->period;
770 
771 	/* Do not lock but ignore next update from synchronization daemon. */
772 	ffclock_updated--;
773 
774 	if (++ogen == 0)
775 		ogen = 1;
776 	ffth->gen = ogen;
777 	fftimehands = ffth;
778 }
779 
780 /*
781  * Retrieve feed-forward counter and time of last kernel tick.
782  */
783 void
784 ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
785 {
786 	struct fftimehands *ffth;
787 	uint8_t gen;
788 
789 	/*
790 	 * No locking but check generation has not changed. Also need to make
791 	 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
792 	 */
793 	do {
794 		ffth = fftimehands;
795 		gen = ffth->gen;
796 		if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
797 			*bt = ffth->tick_time_lerp;
798 		else
799 			*bt = ffth->tick_time;
800 		*ffcount = ffth->tick_ffcount;
801 	} while (gen == 0 || gen != ffth->gen);
802 }
803 
804 /*
805  * Absolute clock conversion. Low level function to convert ffcounter to
806  * bintime. The ffcounter is converted using the current ffclock period estimate
807  * or the "interpolated period" to ensure monotonicity.
808  * NOTE: this conversion may have been deferred, and the clock updated since the
809  * hardware counter has been read.
810  */
811 void
812 ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
813 {
814 	struct fftimehands *ffth;
815 	struct bintime bt2;
816 	ffcounter ffdelta;
817 	uint8_t gen;
818 
819 	/*
820 	 * No locking but check generation has not changed. Also need to make
821 	 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
822 	 */
823 	do {
824 		ffth = fftimehands;
825 		gen = ffth->gen;
826 		if (ffcount > ffth->tick_ffcount)
827 			ffdelta = ffcount - ffth->tick_ffcount;
828 		else
829 			ffdelta = ffth->tick_ffcount - ffcount;
830 
831 		if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
832 			*bt = ffth->tick_time_lerp;
833 			ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
834 		} else {
835 			*bt = ffth->tick_time;
836 			ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
837 		}
838 
839 		if (ffcount > ffth->tick_ffcount)
840 			bintime_add(bt, &bt2);
841 		else
842 			bintime_sub(bt, &bt2);
843 	} while (gen == 0 || gen != ffth->gen);
844 }
845 
846 /*
847  * Difference clock conversion.
848  * Low level function to Convert a time interval measured in RAW counter units
849  * into bintime. The difference clock allows measuring small intervals much more
850  * reliably than the absolute clock.
851  */
852 void
853 ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
854 {
855 	struct fftimehands *ffth;
856 	uint8_t gen;
857 
858 	/* No locking but check generation has not changed. */
859 	do {
860 		ffth = fftimehands;
861 		gen = ffth->gen;
862 		ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
863 	} while (gen == 0 || gen != ffth->gen);
864 }
865 
866 /*
867  * Access to current ffcounter value.
868  */
869 void
870 ffclock_read_counter(ffcounter *ffcount)
871 {
872 	struct timehands *th;
873 	struct fftimehands *ffth;
874 	unsigned int gen, delta;
875 
876 	/*
877 	 * ffclock_windup() called from tc_windup(), safe to rely on
878 	 * th->th_generation only, for correct delta and ffcounter.
879 	 */
880 	do {
881 		th = timehands;
882 		gen = th->th_generation;
883 		ffth = fftimehands;
884 		delta = tc_delta(th);
885 		*ffcount = ffth->tick_ffcount;
886 	} while (gen == 0 || gen != th->th_generation);
887 
888 	*ffcount += delta;
889 }
890 
891 void
892 binuptime(struct bintime *bt)
893 {
894 
895 	binuptime_fromclock(bt, sysclock_active);
896 }
897 
898 void
899 nanouptime(struct timespec *tsp)
900 {
901 
902 	nanouptime_fromclock(tsp, sysclock_active);
903 }
904 
905 void
906 microuptime(struct timeval *tvp)
907 {
908 
909 	microuptime_fromclock(tvp, sysclock_active);
910 }
911 
912 void
913 bintime(struct bintime *bt)
914 {
915 
916 	bintime_fromclock(bt, sysclock_active);
917 }
918 
919 void
920 nanotime(struct timespec *tsp)
921 {
922 
923 	nanotime_fromclock(tsp, sysclock_active);
924 }
925 
926 void
927 microtime(struct timeval *tvp)
928 {
929 
930 	microtime_fromclock(tvp, sysclock_active);
931 }
932 
933 void
934 getbinuptime(struct bintime *bt)
935 {
936 
937 	getbinuptime_fromclock(bt, sysclock_active);
938 }
939 
940 void
941 getnanouptime(struct timespec *tsp)
942 {
943 
944 	getnanouptime_fromclock(tsp, sysclock_active);
945 }
946 
947 void
948 getmicrouptime(struct timeval *tvp)
949 {
950 
951 	getmicrouptime_fromclock(tvp, sysclock_active);
952 }
953 
954 void
955 getbintime(struct bintime *bt)
956 {
957 
958 	getbintime_fromclock(bt, sysclock_active);
959 }
960 
961 void
962 getnanotime(struct timespec *tsp)
963 {
964 
965 	getnanotime_fromclock(tsp, sysclock_active);
966 }
967 
968 void
969 getmicrotime(struct timeval *tvp)
970 {
971 
972 	getmicrouptime_fromclock(tvp, sysclock_active);
973 }
974 
975 #endif /* FFCLOCK */
976 
977 /*
978  * This is a clone of getnanotime and used for walltimestamps.
979  * The dtrace_ prefix prevents fbt from creating probes for
980  * it so walltimestamp can be safely used in all fbt probes.
981  */
982 void
983 dtrace_getnanotime(struct timespec *tsp)
984 {
985 	struct timehands *th;
986 	u_int gen;
987 
988 	do {
989 		th = timehands;
990 		gen = th->th_generation;
991 		*tsp = th->th_nanotime;
992 	} while (gen == 0 || gen != th->th_generation);
993 }
994 
995 /*
996  * System clock currently providing time to the system. Modifiable via sysctl
997  * when the FFCLOCK option is defined.
998  */
999 int sysclock_active = SYSCLOCK_FBCK;
1000 
1001 /* Internal NTP status and error estimates. */
1002 extern int time_status;
1003 extern long time_esterror;
1004 
1005 /*
1006  * Take a snapshot of sysclock data which can be used to compare system clocks
1007  * and generate timestamps after the fact.
1008  */
1009 void
1010 sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
1011 {
1012 	struct fbclock_info *fbi;
1013 	struct timehands *th;
1014 	struct bintime bt;
1015 	unsigned int delta, gen;
1016 #ifdef FFCLOCK
1017 	ffcounter ffcount;
1018 	struct fftimehands *ffth;
1019 	struct ffclock_info *ffi;
1020 	struct ffclock_estimate cest;
1021 
1022 	ffi = &clock_snap->ff_info;
1023 #endif
1024 
1025 	fbi = &clock_snap->fb_info;
1026 	delta = 0;
1027 
1028 	do {
1029 		th = timehands;
1030 		gen = th->th_generation;
1031 		fbi->th_scale = th->th_scale;
1032 		fbi->tick_time = th->th_offset;
1033 #ifdef FFCLOCK
1034 		ffth = fftimehands;
1035 		ffi->tick_time = ffth->tick_time_lerp;
1036 		ffi->tick_time_lerp = ffth->tick_time_lerp;
1037 		ffi->period = ffth->cest.period;
1038 		ffi->period_lerp = ffth->period_lerp;
1039 		clock_snap->ffcount = ffth->tick_ffcount;
1040 		cest = ffth->cest;
1041 #endif
1042 		if (!fast)
1043 			delta = tc_delta(th);
1044 	} while (gen == 0 || gen != th->th_generation);
1045 
1046 	clock_snap->delta = delta;
1047 	clock_snap->sysclock_active = sysclock_active;
1048 
1049 	/* Record feedback clock status and error. */
1050 	clock_snap->fb_info.status = time_status;
1051 	/* XXX: Very crude estimate of feedback clock error. */
1052 	bt.sec = time_esterror / 1000000;
1053 	bt.frac = ((time_esterror - bt.sec) * 1000000) *
1054 	    (uint64_t)18446744073709ULL;
1055 	clock_snap->fb_info.error = bt;
1056 
1057 #ifdef FFCLOCK
1058 	if (!fast)
1059 		clock_snap->ffcount += delta;
1060 
1061 	/* Record feed-forward clock leap second adjustment. */
1062 	ffi->leapsec_adjustment = cest.leapsec_total;
1063 	if (clock_snap->ffcount > cest.leapsec_next)
1064 		ffi->leapsec_adjustment -= cest.leapsec;
1065 
1066 	/* Record feed-forward clock status and error. */
1067 	clock_snap->ff_info.status = cest.status;
1068 	ffcount = clock_snap->ffcount - cest.update_ffcount;
1069 	ffclock_convert_delta(ffcount, cest.period, &bt);
1070 	/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */
1071 	bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL);
1072 	/* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */
1073 	bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL);
1074 	clock_snap->ff_info.error = bt;
1075 #endif
1076 }
1077 
1078 /*
1079  * Convert a sysclock snapshot into a struct bintime based on the specified
1080  * clock source and flags.
1081  */
1082 int
1083 sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt,
1084     int whichclock, uint32_t flags)
1085 {
1086 #ifdef FFCLOCK
1087 	struct bintime bt2;
1088 	uint64_t period;
1089 #endif
1090 
1091 	switch (whichclock) {
1092 	case SYSCLOCK_FBCK:
1093 		*bt = cs->fb_info.tick_time;
1094 
1095 		/* If snapshot was created with !fast, delta will be >0. */
1096 		if (cs->delta > 0)
1097 			bintime_addx(bt, cs->fb_info.th_scale * cs->delta);
1098 
1099 		if ((flags & FBCLOCK_UPTIME) == 0)
1100 			bintime_add(bt, &boottimebin);
1101 		break;
1102 #ifdef FFCLOCK
1103 	case SYSCLOCK_FFWD:
1104 		if (flags & FFCLOCK_LERP) {
1105 			*bt = cs->ff_info.tick_time_lerp;
1106 			period = cs->ff_info.period_lerp;
1107 		} else {
1108 			*bt = cs->ff_info.tick_time;
1109 			period = cs->ff_info.period;
1110 		}
1111 
1112 		/* If snapshot was created with !fast, delta will be >0. */
1113 		if (cs->delta > 0) {
1114 			ffclock_convert_delta(cs->delta, period, &bt2);
1115 			bintime_add(bt, &bt2);
1116 		}
1117 
1118 		/* Leap second adjustment. */
1119 		if (flags & FFCLOCK_LEAPSEC)
1120 			bt->sec -= cs->ff_info.leapsec_adjustment;
1121 
1122 		/* Boot time adjustment, for uptime/monotonic clocks. */
1123 		if (flags & FFCLOCK_UPTIME)
1124 			bintime_sub(bt, &ffclock_boottime);
1125 		break;
1126 #endif
1127 	default:
1128 		return (EINVAL);
1129 		break;
1130 	}
1131 
1132 	return (0);
1133 }
1134 
1135 /*
1136  * Initialize a new timecounter and possibly use it.
1137  */
1138 void
1139 tc_init(struct timecounter *tc)
1140 {
1141 	u_int u;
1142 	struct sysctl_oid *tc_root;
1143 
1144 	u = tc->tc_frequency / tc->tc_counter_mask;
1145 	/* XXX: We need some margin here, 10% is a guess */
1146 	u *= 11;
1147 	u /= 10;
1148 	if (u > hz && tc->tc_quality >= 0) {
1149 		tc->tc_quality = -2000;
1150 		if (bootverbose) {
1151 			printf("Timecounter \"%s\" frequency %ju Hz",
1152 			    tc->tc_name, (uintmax_t)tc->tc_frequency);
1153 			printf(" -- Insufficient hz, needs at least %u\n", u);
1154 		}
1155 	} else if (tc->tc_quality >= 0 || bootverbose) {
1156 		printf("Timecounter \"%s\" frequency %ju Hz quality %d\n",
1157 		    tc->tc_name, (uintmax_t)tc->tc_frequency,
1158 		    tc->tc_quality);
1159 	}
1160 
1161 	tc->tc_next = timecounters;
1162 	timecounters = tc;
1163 	/*
1164 	 * Set up sysctl tree for this counter.
1165 	 */
1166 	tc_root = SYSCTL_ADD_NODE(NULL,
1167 	    SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name,
1168 	    CTLFLAG_RW, 0, "timecounter description");
1169 	SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1170 	    "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0,
1171 	    "mask for implemented bits");
1172 	SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1173 	    "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc),
1174 	    sysctl_kern_timecounter_get, "IU", "current timecounter value");
1175 	SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1176 	    "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc),
1177 	     sysctl_kern_timecounter_freq, "QU", "timecounter frequency");
1178 	SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1179 	    "quality", CTLFLAG_RD, &(tc->tc_quality), 0,
1180 	    "goodness of time counter");
1181 	/*
1182 	 * Never automatically use a timecounter with negative quality.
1183 	 * Even though we run on the dummy counter, switching here may be
1184 	 * worse since this timecounter may not be monotonous.
1185 	 */
1186 	if (tc->tc_quality < 0)
1187 		return;
1188 	if (tc->tc_quality < timecounter->tc_quality)
1189 		return;
1190 	if (tc->tc_quality == timecounter->tc_quality &&
1191 	    tc->tc_frequency < timecounter->tc_frequency)
1192 		return;
1193 	(void)tc->tc_get_timecount(tc);
1194 	(void)tc->tc_get_timecount(tc);
1195 	timecounter = tc;
1196 }
1197 
1198 /* Report the frequency of the current timecounter. */
1199 uint64_t
1200 tc_getfrequency(void)
1201 {
1202 
1203 	return (timehands->th_counter->tc_frequency);
1204 }
1205 
1206 /*
1207  * Step our concept of UTC.  This is done by modifying our estimate of
1208  * when we booted.
1209  * XXX: not locked.
1210  */
1211 void
1212 tc_setclock(struct timespec *ts)
1213 {
1214 	struct timespec tbef, taft;
1215 	struct bintime bt, bt2;
1216 
1217 	cpu_tick_calibrate(1);
1218 	nanotime(&tbef);
1219 	timespec2bintime(ts, &bt);
1220 	binuptime(&bt2);
1221 	bintime_sub(&bt, &bt2);
1222 	bintime_add(&bt2, &boottimebin);
1223 	boottimebin = bt;
1224 	bintime2timeval(&bt, &boottime);
1225 
1226 	/* XXX fiddle all the little crinkly bits around the fiords... */
1227 	tc_windup();
1228 	nanotime(&taft);
1229 	if (timestepwarnings) {
1230 		log(LOG_INFO,
1231 		    "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
1232 		    (intmax_t)tbef.tv_sec, tbef.tv_nsec,
1233 		    (intmax_t)taft.tv_sec, taft.tv_nsec,
1234 		    (intmax_t)ts->tv_sec, ts->tv_nsec);
1235 	}
1236 	cpu_tick_calibrate(1);
1237 }
1238 
1239 /*
1240  * Initialize the next struct timehands in the ring and make
1241  * it the active timehands.  Along the way we might switch to a different
1242  * timecounter and/or do seconds processing in NTP.  Slightly magic.
1243  */
1244 static void
1245 tc_windup(void)
1246 {
1247 	struct bintime bt;
1248 	struct timehands *th, *tho;
1249 	uint64_t scale;
1250 	u_int delta, ncount, ogen;
1251 	int i;
1252 	time_t t;
1253 
1254 	/*
1255 	 * Make the next timehands a copy of the current one, but do not
1256 	 * overwrite the generation or next pointer.  While we update
1257 	 * the contents, the generation must be zero.
1258 	 */
1259 	tho = timehands;
1260 	th = tho->th_next;
1261 	ogen = th->th_generation;
1262 	th->th_generation = 0;
1263 	bcopy(tho, th, offsetof(struct timehands, th_generation));
1264 
1265 	/*
1266 	 * Capture a timecounter delta on the current timecounter and if
1267 	 * changing timecounters, a counter value from the new timecounter.
1268 	 * Update the offset fields accordingly.
1269 	 */
1270 	delta = tc_delta(th);
1271 	if (th->th_counter != timecounter)
1272 		ncount = timecounter->tc_get_timecount(timecounter);
1273 	else
1274 		ncount = 0;
1275 #ifdef FFCLOCK
1276 	ffclock_windup(delta);
1277 #endif
1278 	th->th_offset_count += delta;
1279 	th->th_offset_count &= th->th_counter->tc_counter_mask;
1280 	while (delta > th->th_counter->tc_frequency) {
1281 		/* Eat complete unadjusted seconds. */
1282 		delta -= th->th_counter->tc_frequency;
1283 		th->th_offset.sec++;
1284 	}
1285 	if ((delta > th->th_counter->tc_frequency / 2) &&
1286 	    (th->th_scale * delta < ((uint64_t)1 << 63))) {
1287 		/* The product th_scale * delta just barely overflows. */
1288 		th->th_offset.sec++;
1289 	}
1290 	bintime_addx(&th->th_offset, th->th_scale * delta);
1291 
1292 	/*
1293 	 * Hardware latching timecounters may not generate interrupts on
1294 	 * PPS events, so instead we poll them.  There is a finite risk that
1295 	 * the hardware might capture a count which is later than the one we
1296 	 * got above, and therefore possibly in the next NTP second which might
1297 	 * have a different rate than the current NTP second.  It doesn't
1298 	 * matter in practice.
1299 	 */
1300 	if (tho->th_counter->tc_poll_pps)
1301 		tho->th_counter->tc_poll_pps(tho->th_counter);
1302 
1303 	/*
1304 	 * Deal with NTP second processing.  The for loop normally
1305 	 * iterates at most once, but in extreme situations it might
1306 	 * keep NTP sane if timeouts are not run for several seconds.
1307 	 * At boot, the time step can be large when the TOD hardware
1308 	 * has been read, so on really large steps, we call
1309 	 * ntp_update_second only twice.  We need to call it twice in
1310 	 * case we missed a leap second.
1311 	 */
1312 	bt = th->th_offset;
1313 	bintime_add(&bt, &boottimebin);
1314 	i = bt.sec - tho->th_microtime.tv_sec;
1315 	if (i > LARGE_STEP)
1316 		i = 2;
1317 	for (; i > 0; i--) {
1318 		t = bt.sec;
1319 		ntp_update_second(&th->th_adjustment, &bt.sec);
1320 		if (bt.sec != t)
1321 			boottimebin.sec += bt.sec - t;
1322 	}
1323 	/* Update the UTC timestamps used by the get*() functions. */
1324 	/* XXX shouldn't do this here.  Should force non-`get' versions. */
1325 	bintime2timeval(&bt, &th->th_microtime);
1326 	bintime2timespec(&bt, &th->th_nanotime);
1327 
1328 	/* Now is a good time to change timecounters. */
1329 	if (th->th_counter != timecounter) {
1330 #ifndef __arm__
1331 		if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0)
1332 			cpu_disable_c2_sleep++;
1333 		if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0)
1334 			cpu_disable_c2_sleep--;
1335 #endif
1336 		th->th_counter = timecounter;
1337 		th->th_offset_count = ncount;
1338 		tc_min_ticktock_freq = max(1, timecounter->tc_frequency /
1339 		    (((uint64_t)timecounter->tc_counter_mask + 1) / 3));
1340 #ifdef FFCLOCK
1341 		ffclock_change_tc(th);
1342 #endif
1343 	}
1344 
1345 	/*-
1346 	 * Recalculate the scaling factor.  We want the number of 1/2^64
1347 	 * fractions of a second per period of the hardware counter, taking
1348 	 * into account the th_adjustment factor which the NTP PLL/adjtime(2)
1349 	 * processing provides us with.
1350 	 *
1351 	 * The th_adjustment is nanoseconds per second with 32 bit binary
1352 	 * fraction and we want 64 bit binary fraction of second:
1353 	 *
1354 	 *	 x = a * 2^32 / 10^9 = a * 4.294967296
1355 	 *
1356 	 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
1357 	 * we can only multiply by about 850 without overflowing, that
1358 	 * leaves no suitably precise fractions for multiply before divide.
1359 	 *
1360 	 * Divide before multiply with a fraction of 2199/512 results in a
1361 	 * systematic undercompensation of 10PPM of th_adjustment.  On a
1362 	 * 5000PPM adjustment this is a 0.05PPM error.  This is acceptable.
1363  	 *
1364 	 * We happily sacrifice the lowest of the 64 bits of our result
1365 	 * to the goddess of code clarity.
1366 	 *
1367 	 */
1368 	scale = (uint64_t)1 << 63;
1369 	scale += (th->th_adjustment / 1024) * 2199;
1370 	scale /= th->th_counter->tc_frequency;
1371 	th->th_scale = scale * 2;
1372 
1373 	/*
1374 	 * Now that the struct timehands is again consistent, set the new
1375 	 * generation number, making sure to not make it zero.
1376 	 */
1377 	if (++ogen == 0)
1378 		ogen = 1;
1379 	th->th_generation = ogen;
1380 
1381 	/* Go live with the new struct timehands. */
1382 #ifdef FFCLOCK
1383 	switch (sysclock_active) {
1384 	case SYSCLOCK_FBCK:
1385 #endif
1386 		time_second = th->th_microtime.tv_sec;
1387 		time_uptime = th->th_offset.sec;
1388 #ifdef FFCLOCK
1389 		break;
1390 	case SYSCLOCK_FFWD:
1391 		time_second = fftimehands->tick_time_lerp.sec;
1392 		time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
1393 		break;
1394 	}
1395 #endif
1396 
1397 	timehands = th;
1398 	timekeep_push_vdso();
1399 }
1400 
1401 /* Report or change the active timecounter hardware. */
1402 static int
1403 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
1404 {
1405 	char newname[32];
1406 	struct timecounter *newtc, *tc;
1407 	int error;
1408 
1409 	tc = timecounter;
1410 	strlcpy(newname, tc->tc_name, sizeof(newname));
1411 
1412 	error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req);
1413 	if (error != 0 || req->newptr == NULL ||
1414 	    strcmp(newname, tc->tc_name) == 0)
1415 		return (error);
1416 	for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
1417 		if (strcmp(newname, newtc->tc_name) != 0)
1418 			continue;
1419 
1420 		/* Warm up new timecounter. */
1421 		(void)newtc->tc_get_timecount(newtc);
1422 		(void)newtc->tc_get_timecount(newtc);
1423 
1424 		timecounter = newtc;
1425 
1426 		/*
1427 		 * The vdso timehands update is deferred until the next
1428 		 * 'tc_windup()'.
1429 		 *
1430 		 * This is prudent given that 'timekeep_push_vdso()' does not
1431 		 * use any locking and that it can be called in hard interrupt
1432 		 * context via 'tc_windup()'.
1433 		 */
1434 		return (0);
1435 	}
1436 	return (EINVAL);
1437 }
1438 
1439 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW,
1440     0, 0, sysctl_kern_timecounter_hardware, "A",
1441     "Timecounter hardware selected");
1442 
1443 
1444 /* Report or change the active timecounter hardware. */
1445 static int
1446 sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS)
1447 {
1448 	char buf[32], *spc;
1449 	struct timecounter *tc;
1450 	int error;
1451 
1452 	spc = "";
1453 	error = 0;
1454 	for (tc = timecounters; error == 0 && tc != NULL; tc = tc->tc_next) {
1455 		sprintf(buf, "%s%s(%d)",
1456 		    spc, tc->tc_name, tc->tc_quality);
1457 		error = SYSCTL_OUT(req, buf, strlen(buf));
1458 		spc = " ";
1459 	}
1460 	return (error);
1461 }
1462 
1463 SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
1464     0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected");
1465 
1466 /*
1467  * RFC 2783 PPS-API implementation.
1468  */
1469 
1470 static int
1471 pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps)
1472 {
1473 	int err, timo;
1474 	pps_seq_t aseq, cseq;
1475 	struct timeval tv;
1476 
1477 	if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
1478 		return (EINVAL);
1479 
1480 	/*
1481 	 * If no timeout is requested, immediately return whatever values were
1482 	 * most recently captured.  If timeout seconds is -1, that's a request
1483 	 * to block without a timeout.  WITNESS won't let us sleep forever
1484 	 * without a lock (we really don't need a lock), so just repeatedly
1485 	 * sleep a long time.
1486 	 */
1487 	if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) {
1488 		if (fapi->timeout.tv_sec == -1)
1489 			timo = 0x7fffffff;
1490 		else {
1491 			tv.tv_sec = fapi->timeout.tv_sec;
1492 			tv.tv_usec = fapi->timeout.tv_nsec / 1000;
1493 			timo = tvtohz(&tv);
1494 		}
1495 		aseq = pps->ppsinfo.assert_sequence;
1496 		cseq = pps->ppsinfo.clear_sequence;
1497 		while (aseq == pps->ppsinfo.assert_sequence &&
1498 		    cseq == pps->ppsinfo.clear_sequence) {
1499 			if (pps->mtx != NULL)
1500 				err = msleep(pps, pps->mtx, PCATCH, "ppsfch", timo);
1501 			else
1502 				err = tsleep(pps, PCATCH, "ppsfch", timo);
1503 			if (err == EWOULDBLOCK && fapi->timeout.tv_sec == -1) {
1504 				continue;
1505 			} else if (err != 0) {
1506 				return (err);
1507 			}
1508 		}
1509 	}
1510 
1511 	pps->ppsinfo.current_mode = pps->ppsparam.mode;
1512 	fapi->pps_info_buf = pps->ppsinfo;
1513 
1514 	return (0);
1515 }
1516 
1517 int
1518 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
1519 {
1520 	pps_params_t *app;
1521 	struct pps_fetch_args *fapi;
1522 #ifdef FFCLOCK
1523 	struct pps_fetch_ffc_args *fapi_ffc;
1524 #endif
1525 #ifdef PPS_SYNC
1526 	struct pps_kcbind_args *kapi;
1527 #endif
1528 
1529 	KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl"));
1530 	switch (cmd) {
1531 	case PPS_IOC_CREATE:
1532 		return (0);
1533 	case PPS_IOC_DESTROY:
1534 		return (0);
1535 	case PPS_IOC_SETPARAMS:
1536 		app = (pps_params_t *)data;
1537 		if (app->mode & ~pps->ppscap)
1538 			return (EINVAL);
1539 #ifdef FFCLOCK
1540 		/* Ensure only a single clock is selected for ffc timestamp. */
1541 		if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
1542 			return (EINVAL);
1543 #endif
1544 		pps->ppsparam = *app;
1545 		return (0);
1546 	case PPS_IOC_GETPARAMS:
1547 		app = (pps_params_t *)data;
1548 		*app = pps->ppsparam;
1549 		app->api_version = PPS_API_VERS_1;
1550 		return (0);
1551 	case PPS_IOC_GETCAP:
1552 		*(int*)data = pps->ppscap;
1553 		return (0);
1554 	case PPS_IOC_FETCH:
1555 		fapi = (struct pps_fetch_args *)data;
1556 		return (pps_fetch(fapi, pps));
1557 #ifdef FFCLOCK
1558 	case PPS_IOC_FETCH_FFCOUNTER:
1559 		fapi_ffc = (struct pps_fetch_ffc_args *)data;
1560 		if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
1561 		    PPS_TSFMT_TSPEC)
1562 			return (EINVAL);
1563 		if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
1564 			return (EOPNOTSUPP);
1565 		pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
1566 		fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
1567 		/* Overwrite timestamps if feedback clock selected. */
1568 		switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
1569 		case PPS_TSCLK_FBCK:
1570 			fapi_ffc->pps_info_buf_ffc.assert_timestamp =
1571 			    pps->ppsinfo.assert_timestamp;
1572 			fapi_ffc->pps_info_buf_ffc.clear_timestamp =
1573 			    pps->ppsinfo.clear_timestamp;
1574 			break;
1575 		case PPS_TSCLK_FFWD:
1576 			break;
1577 		default:
1578 			break;
1579 		}
1580 		return (0);
1581 #endif /* FFCLOCK */
1582 	case PPS_IOC_KCBIND:
1583 #ifdef PPS_SYNC
1584 		kapi = (struct pps_kcbind_args *)data;
1585 		/* XXX Only root should be able to do this */
1586 		if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
1587 			return (EINVAL);
1588 		if (kapi->kernel_consumer != PPS_KC_HARDPPS)
1589 			return (EINVAL);
1590 		if (kapi->edge & ~pps->ppscap)
1591 			return (EINVAL);
1592 		pps->kcmode = kapi->edge;
1593 		return (0);
1594 #else
1595 		return (EOPNOTSUPP);
1596 #endif
1597 	default:
1598 		return (ENOIOCTL);
1599 	}
1600 }
1601 
1602 void
1603 pps_init(struct pps_state *pps)
1604 {
1605 	pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT;
1606 	if (pps->ppscap & PPS_CAPTUREASSERT)
1607 		pps->ppscap |= PPS_OFFSETASSERT;
1608 	if (pps->ppscap & PPS_CAPTURECLEAR)
1609 		pps->ppscap |= PPS_OFFSETCLEAR;
1610 #ifdef FFCLOCK
1611 	pps->ppscap |= PPS_TSCLK_MASK;
1612 #endif
1613 }
1614 
1615 void
1616 pps_capture(struct pps_state *pps)
1617 {
1618 	struct timehands *th;
1619 
1620 	KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
1621 	th = timehands;
1622 	pps->capgen = th->th_generation;
1623 	pps->capth = th;
1624 #ifdef FFCLOCK
1625 	pps->capffth = fftimehands;
1626 #endif
1627 	pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
1628 	if (pps->capgen != th->th_generation)
1629 		pps->capgen = 0;
1630 }
1631 
1632 void
1633 pps_event(struct pps_state *pps, int event)
1634 {
1635 	struct bintime bt;
1636 	struct timespec ts, *tsp, *osp;
1637 	u_int tcount, *pcount;
1638 	int foff, fhard;
1639 	pps_seq_t *pseq;
1640 #ifdef FFCLOCK
1641 	struct timespec *tsp_ffc;
1642 	pps_seq_t *pseq_ffc;
1643 	ffcounter *ffcount;
1644 #endif
1645 
1646 	KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
1647 	/* If the timecounter was wound up underneath us, bail out. */
1648 	if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation)
1649 		return;
1650 
1651 	/* Things would be easier with arrays. */
1652 	if (event == PPS_CAPTUREASSERT) {
1653 		tsp = &pps->ppsinfo.assert_timestamp;
1654 		osp = &pps->ppsparam.assert_offset;
1655 		foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
1656 		fhard = pps->kcmode & PPS_CAPTUREASSERT;
1657 		pcount = &pps->ppscount[0];
1658 		pseq = &pps->ppsinfo.assert_sequence;
1659 #ifdef FFCLOCK
1660 		ffcount = &pps->ppsinfo_ffc.assert_ffcount;
1661 		tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
1662 		pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
1663 #endif
1664 	} else {
1665 		tsp = &pps->ppsinfo.clear_timestamp;
1666 		osp = &pps->ppsparam.clear_offset;
1667 		foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
1668 		fhard = pps->kcmode & PPS_CAPTURECLEAR;
1669 		pcount = &pps->ppscount[1];
1670 		pseq = &pps->ppsinfo.clear_sequence;
1671 #ifdef FFCLOCK
1672 		ffcount = &pps->ppsinfo_ffc.clear_ffcount;
1673 		tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
1674 		pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
1675 #endif
1676 	}
1677 
1678 	/*
1679 	 * If the timecounter changed, we cannot compare the count values, so
1680 	 * we have to drop the rest of the PPS-stuff until the next event.
1681 	 */
1682 	if (pps->ppstc != pps->capth->th_counter) {
1683 		pps->ppstc = pps->capth->th_counter;
1684 		*pcount = pps->capcount;
1685 		pps->ppscount[2] = pps->capcount;
1686 		return;
1687 	}
1688 
1689 	/* Convert the count to a timespec. */
1690 	tcount = pps->capcount - pps->capth->th_offset_count;
1691 	tcount &= pps->capth->th_counter->tc_counter_mask;
1692 	bt = pps->capth->th_offset;
1693 	bintime_addx(&bt, pps->capth->th_scale * tcount);
1694 	bintime_add(&bt, &boottimebin);
1695 	bintime2timespec(&bt, &ts);
1696 
1697 	/* If the timecounter was wound up underneath us, bail out. */
1698 	if (pps->capgen != pps->capth->th_generation)
1699 		return;
1700 
1701 	*pcount = pps->capcount;
1702 	(*pseq)++;
1703 	*tsp = ts;
1704 
1705 	if (foff) {
1706 		timespecadd(tsp, osp);
1707 		if (tsp->tv_nsec < 0) {
1708 			tsp->tv_nsec += 1000000000;
1709 			tsp->tv_sec -= 1;
1710 		}
1711 	}
1712 
1713 #ifdef FFCLOCK
1714 	*ffcount = pps->capffth->tick_ffcount + tcount;
1715 	bt = pps->capffth->tick_time;
1716 	ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
1717 	bintime_add(&bt, &pps->capffth->tick_time);
1718 	bintime2timespec(&bt, &ts);
1719 	(*pseq_ffc)++;
1720 	*tsp_ffc = ts;
1721 #endif
1722 
1723 #ifdef PPS_SYNC
1724 	if (fhard) {
1725 		uint64_t scale;
1726 
1727 		/*
1728 		 * Feed the NTP PLL/FLL.
1729 		 * The FLL wants to know how many (hardware) nanoseconds
1730 		 * elapsed since the previous event.
1731 		 */
1732 		tcount = pps->capcount - pps->ppscount[2];
1733 		pps->ppscount[2] = pps->capcount;
1734 		tcount &= pps->capth->th_counter->tc_counter_mask;
1735 		scale = (uint64_t)1 << 63;
1736 		scale /= pps->capth->th_counter->tc_frequency;
1737 		scale *= 2;
1738 		bt.sec = 0;
1739 		bt.frac = 0;
1740 		bintime_addx(&bt, scale * tcount);
1741 		bintime2timespec(&bt, &ts);
1742 		hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
1743 	}
1744 #endif
1745 
1746 	/* Wakeup anyone sleeping in pps_fetch().  */
1747 	wakeup(pps);
1748 }
1749 
1750 /*
1751  * Timecounters need to be updated every so often to prevent the hardware
1752  * counter from overflowing.  Updating also recalculates the cached values
1753  * used by the get*() family of functions, so their precision depends on
1754  * the update frequency.
1755  */
1756 
1757 static int tc_tick;
1758 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0,
1759     "Approximate number of hardclock ticks in a millisecond");
1760 
1761 void
1762 tc_ticktock(int cnt)
1763 {
1764 	static int count;
1765 
1766 	count += cnt;
1767 	if (count < tc_tick)
1768 		return;
1769 	count = 0;
1770 	tc_windup();
1771 }
1772 
1773 static void __inline
1774 tc_adjprecision(void)
1775 {
1776 	int t;
1777 
1778 	if (tc_timepercentage > 0) {
1779 		t = (99 + tc_timepercentage) / tc_timepercentage;
1780 		tc_precexp = fls(t + (t >> 1)) - 1;
1781 		FREQ2BT(hz / tc_tick, &bt_timethreshold);
1782 		FREQ2BT(hz, &bt_tickthreshold);
1783 		bintime_shift(&bt_timethreshold, tc_precexp);
1784 		bintime_shift(&bt_tickthreshold, tc_precexp);
1785 	} else {
1786 		tc_precexp = 31;
1787 		bt_timethreshold.sec = INT_MAX;
1788 		bt_timethreshold.frac = ~(uint64_t)0;
1789 		bt_tickthreshold = bt_timethreshold;
1790 	}
1791 	sbt_timethreshold = bttosbt(bt_timethreshold);
1792 	sbt_tickthreshold = bttosbt(bt_tickthreshold);
1793 }
1794 
1795 static int
1796 sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS)
1797 {
1798 	int error, val;
1799 
1800 	val = tc_timepercentage;
1801 	error = sysctl_handle_int(oidp, &val, 0, req);
1802 	if (error != 0 || req->newptr == NULL)
1803 		return (error);
1804 	tc_timepercentage = val;
1805 	if (cold)
1806 		goto done;
1807 	tc_adjprecision();
1808 done:
1809 	return (0);
1810 }
1811 
1812 static void
1813 inittimecounter(void *dummy)
1814 {
1815 	u_int p;
1816 	int tick_rate;
1817 
1818 	/*
1819 	 * Set the initial timeout to
1820 	 * max(1, <approx. number of hardclock ticks in a millisecond>).
1821 	 * People should probably not use the sysctl to set the timeout
1822 	 * to smaller than its inital value, since that value is the
1823 	 * smallest reasonable one.  If they want better timestamps they
1824 	 * should use the non-"get"* functions.
1825 	 */
1826 	if (hz > 1000)
1827 		tc_tick = (hz + 500) / 1000;
1828 	else
1829 		tc_tick = 1;
1830 	tc_adjprecision();
1831 	FREQ2BT(hz, &tick_bt);
1832 	tick_sbt = bttosbt(tick_bt);
1833 	tick_rate = hz / tc_tick;
1834 	FREQ2BT(tick_rate, &tc_tick_bt);
1835 	tc_tick_sbt = bttosbt(tc_tick_bt);
1836 	p = (tc_tick * 1000000) / hz;
1837 	printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
1838 
1839 #ifdef FFCLOCK
1840 	ffclock_init();
1841 #endif
1842 	/* warm up new timecounter (again) and get rolling. */
1843 	(void)timecounter->tc_get_timecount(timecounter);
1844 	(void)timecounter->tc_get_timecount(timecounter);
1845 	tc_windup();
1846 }
1847 
1848 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
1849 
1850 /* Cpu tick handling -------------------------------------------------*/
1851 
1852 static int cpu_tick_variable;
1853 static uint64_t	cpu_tick_frequency;
1854 
1855 static uint64_t
1856 tc_cpu_ticks(void)
1857 {
1858 	static uint64_t base;
1859 	static unsigned last;
1860 	unsigned u;
1861 	struct timecounter *tc;
1862 
1863 	tc = timehands->th_counter;
1864 	u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
1865 	if (u < last)
1866 		base += (uint64_t)tc->tc_counter_mask + 1;
1867 	last = u;
1868 	return (u + base);
1869 }
1870 
1871 void
1872 cpu_tick_calibration(void)
1873 {
1874 	static time_t last_calib;
1875 
1876 	if (time_uptime != last_calib && !(time_uptime & 0xf)) {
1877 		cpu_tick_calibrate(0);
1878 		last_calib = time_uptime;
1879 	}
1880 }
1881 
1882 /*
1883  * This function gets called every 16 seconds on only one designated
1884  * CPU in the system from hardclock() via cpu_tick_calibration()().
1885  *
1886  * Whenever the real time clock is stepped we get called with reset=1
1887  * to make sure we handle suspend/resume and similar events correctly.
1888  */
1889 
1890 static void
1891 cpu_tick_calibrate(int reset)
1892 {
1893 	static uint64_t c_last;
1894 	uint64_t c_this, c_delta;
1895 	static struct bintime  t_last;
1896 	struct bintime t_this, t_delta;
1897 	uint32_t divi;
1898 
1899 	if (reset) {
1900 		/* The clock was stepped, abort & reset */
1901 		t_last.sec = 0;
1902 		return;
1903 	}
1904 
1905 	/* we don't calibrate fixed rate cputicks */
1906 	if (!cpu_tick_variable)
1907 		return;
1908 
1909 	getbinuptime(&t_this);
1910 	c_this = cpu_ticks();
1911 	if (t_last.sec != 0) {
1912 		c_delta = c_this - c_last;
1913 		t_delta = t_this;
1914 		bintime_sub(&t_delta, &t_last);
1915 		/*
1916 		 * Headroom:
1917 		 * 	2^(64-20) / 16[s] =
1918 		 * 	2^(44) / 16[s] =
1919 		 * 	17.592.186.044.416 / 16 =
1920 		 * 	1.099.511.627.776 [Hz]
1921 		 */
1922 		divi = t_delta.sec << 20;
1923 		divi |= t_delta.frac >> (64 - 20);
1924 		c_delta <<= 20;
1925 		c_delta /= divi;
1926 		if (c_delta > cpu_tick_frequency) {
1927 			if (0 && bootverbose)
1928 				printf("cpu_tick increased to %ju Hz\n",
1929 				    c_delta);
1930 			cpu_tick_frequency = c_delta;
1931 		}
1932 	}
1933 	c_last = c_this;
1934 	t_last = t_this;
1935 }
1936 
1937 void
1938 set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var)
1939 {
1940 
1941 	if (func == NULL) {
1942 		cpu_ticks = tc_cpu_ticks;
1943 	} else {
1944 		cpu_tick_frequency = freq;
1945 		cpu_tick_variable = var;
1946 		cpu_ticks = func;
1947 	}
1948 }
1949 
1950 uint64_t
1951 cpu_tickrate(void)
1952 {
1953 
1954 	if (cpu_ticks == tc_cpu_ticks)
1955 		return (tc_getfrequency());
1956 	return (cpu_tick_frequency);
1957 }
1958 
1959 /*
1960  * We need to be slightly careful converting cputicks to microseconds.
1961  * There is plenty of margin in 64 bits of microseconds (half a million
1962  * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply
1963  * before divide conversion (to retain precision) we find that the
1964  * margin shrinks to 1.5 hours (one millionth of 146y).
1965  * With a three prong approach we never lose significant bits, no
1966  * matter what the cputick rate and length of timeinterval is.
1967  */
1968 
1969 uint64_t
1970 cputick2usec(uint64_t tick)
1971 {
1972 
1973 	if (tick > 18446744073709551LL)		/* floor(2^64 / 1000) */
1974 		return (tick / (cpu_tickrate() / 1000000LL));
1975 	else if (tick > 18446744073709LL)	/* floor(2^64 / 1000000) */
1976 		return ((tick * 1000LL) / (cpu_tickrate() / 1000LL));
1977 	else
1978 		return ((tick * 1000000LL) / cpu_tickrate());
1979 }
1980 
1981 cpu_tick_f	*cpu_ticks = tc_cpu_ticks;
1982 
1983 static int vdso_th_enable = 1;
1984 static int
1985 sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
1986 {
1987 	int old_vdso_th_enable, error;
1988 
1989 	old_vdso_th_enable = vdso_th_enable;
1990 	error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
1991 	if (error != 0)
1992 		return (error);
1993 	vdso_th_enable = old_vdso_th_enable;
1994 	return (0);
1995 }
1996 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
1997     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1998     NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
1999 
2000 uint32_t
2001 tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
2002 {
2003 	struct timehands *th;
2004 	uint32_t enabled;
2005 
2006 	th = timehands;
2007 	vdso_th->th_algo = VDSO_TH_ALGO_1;
2008 	vdso_th->th_scale = th->th_scale;
2009 	vdso_th->th_offset_count = th->th_offset_count;
2010 	vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
2011 	vdso_th->th_offset = th->th_offset;
2012 	vdso_th->th_boottime = boottimebin;
2013 	enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter);
2014 	if (!vdso_th_enable)
2015 		enabled = 0;
2016 	return (enabled);
2017 }
2018 
2019 #ifdef COMPAT_FREEBSD32
2020 uint32_t
2021 tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
2022 {
2023 	struct timehands *th;
2024 	uint32_t enabled;
2025 
2026 	th = timehands;
2027 	vdso_th32->th_algo = VDSO_TH_ALGO_1;
2028 	*(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
2029 	vdso_th32->th_offset_count = th->th_offset_count;
2030 	vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
2031 	vdso_th32->th_offset.sec = th->th_offset.sec;
2032 	*(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
2033 	vdso_th32->th_boottime.sec = boottimebin.sec;
2034 	*(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac;
2035 	enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter);
2036 	if (!vdso_th_enable)
2037 		enabled = 0;
2038 	return (enabled);
2039 }
2040 #endif
2041