xref: /freebsd/sys/kern/kern_tc.c (revision a2aef24aa3c8458e4036735dd6928b4ef77294e5)
1 /*-
2  * ----------------------------------------------------------------------------
3  * "THE BEER-WARE LICENSE" (Revision 42):
4  * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
5  * can do whatever you want with this stuff. If we meet some day, and you think
6  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
7  * ----------------------------------------------------------------------------
8  *
9  * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation
10  * All rights reserved.
11  *
12  * Portions of this software were developed by Julien Ridoux at the University
13  * of Melbourne under sponsorship from the FreeBSD Foundation.
14  *
15  * Portions of this software were developed by Konstantin Belousov
16  * under sponsorship from the FreeBSD Foundation.
17  */
18 
19 #include <sys/cdefs.h>
20 __FBSDID("$FreeBSD$");
21 
22 #include "opt_compat.h"
23 #include "opt_ntp.h"
24 #include "opt_ffclock.h"
25 
26 #include <sys/param.h>
27 #include <sys/kernel.h>
28 #include <sys/limits.h>
29 #include <sys/lock.h>
30 #include <sys/mutex.h>
31 #include <sys/proc.h>
32 #include <sys/sbuf.h>
33 #include <sys/sleepqueue.h>
34 #include <sys/sysctl.h>
35 #include <sys/syslog.h>
36 #include <sys/systm.h>
37 #include <sys/timeffc.h>
38 #include <sys/timepps.h>
39 #include <sys/timetc.h>
40 #include <sys/timex.h>
41 #include <sys/vdso.h>
42 
43 /*
44  * A large step happens on boot.  This constant detects such steps.
45  * It is relatively small so that ntp_update_second gets called enough
46  * in the typical 'missed a couple of seconds' case, but doesn't loop
47  * forever when the time step is large.
48  */
49 #define LARGE_STEP	200
50 
51 /*
52  * Implement a dummy timecounter which we can use until we get a real one
53  * in the air.  This allows the console and other early stuff to use
54  * time services.
55  */
56 
57 static u_int
58 dummy_get_timecount(struct timecounter *tc)
59 {
60 	static u_int now;
61 
62 	return (++now);
63 }
64 
65 static struct timecounter dummy_timecounter = {
66 	dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000
67 };
68 
69 struct timehands {
70 	/* These fields must be initialized by the driver. */
71 	struct timecounter	*th_counter;
72 	int64_t			th_adjustment;
73 	uint64_t		th_scale;
74 	u_int	 		th_offset_count;
75 	struct bintime		th_offset;
76 	struct bintime		th_bintime;
77 	struct timeval		th_microtime;
78 	struct timespec		th_nanotime;
79 	struct bintime		th_boottime;
80 	/* Fields not to be copied in tc_windup start with th_generation. */
81 	u_int			th_generation;
82 	struct timehands	*th_next;
83 };
84 
85 static struct timehands th0;
86 static struct timehands th1 = {
87 	.th_next = &th0
88 };
89 static struct timehands th0 = {
90 	.th_counter = &dummy_timecounter,
91 	.th_scale = (uint64_t)-1 / 1000000,
92 	.th_offset = { .sec = 1 },
93 	.th_generation = 1,
94 	.th_next = &th1
95 };
96 
97 static struct timehands *volatile timehands = &th0;
98 struct timecounter *timecounter = &dummy_timecounter;
99 static struct timecounter *timecounters = &dummy_timecounter;
100 
101 int tc_min_ticktock_freq = 1;
102 
103 volatile time_t time_second = 1;
104 volatile time_t time_uptime = 1;
105 
106 static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS);
107 SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD,
108     NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime");
109 
110 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, "");
111 static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, "");
112 
113 static int timestepwarnings;
114 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
115     &timestepwarnings, 0, "Log time steps");
116 
117 struct bintime bt_timethreshold;
118 struct bintime bt_tickthreshold;
119 sbintime_t sbt_timethreshold;
120 sbintime_t sbt_tickthreshold;
121 struct bintime tc_tick_bt;
122 sbintime_t tc_tick_sbt;
123 int tc_precexp;
124 int tc_timepercentage = TC_DEFAULTPERC;
125 static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS);
126 SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation,
127     CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0,
128     sysctl_kern_timecounter_adjprecision, "I",
129     "Allowed time interval deviation in percents");
130 
131 volatile int rtc_generation = 1;
132 
133 static int tc_chosen;	/* Non-zero if a specific tc was chosen via sysctl. */
134 
135 static void tc_windup(struct bintime *new_boottimebin);
136 static void cpu_tick_calibrate(int);
137 
138 void dtrace_getnanotime(struct timespec *tsp);
139 
140 static int
141 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
142 {
143 	struct timeval boottime;
144 
145 	getboottime(&boottime);
146 
147 #ifndef __mips__
148 #ifdef SCTL_MASK32
149 	int tv[2];
150 
151 	if (req->flags & SCTL_MASK32) {
152 		tv[0] = boottime.tv_sec;
153 		tv[1] = boottime.tv_usec;
154 		return (SYSCTL_OUT(req, tv, sizeof(tv)));
155 	}
156 #endif
157 #endif
158 	return (SYSCTL_OUT(req, &boottime, sizeof(boottime)));
159 }
160 
161 static int
162 sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS)
163 {
164 	u_int ncount;
165 	struct timecounter *tc = arg1;
166 
167 	ncount = tc->tc_get_timecount(tc);
168 	return (sysctl_handle_int(oidp, &ncount, 0, req));
169 }
170 
171 static int
172 sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS)
173 {
174 	uint64_t freq;
175 	struct timecounter *tc = arg1;
176 
177 	freq = tc->tc_frequency;
178 	return (sysctl_handle_64(oidp, &freq, 0, req));
179 }
180 
181 /*
182  * Return the difference between the timehands' counter value now and what
183  * was when we copied it to the timehands' offset_count.
184  */
185 static __inline u_int
186 tc_delta(struct timehands *th)
187 {
188 	struct timecounter *tc;
189 
190 	tc = th->th_counter;
191 	return ((tc->tc_get_timecount(tc) - th->th_offset_count) &
192 	    tc->tc_counter_mask);
193 }
194 
195 /*
196  * Functions for reading the time.  We have to loop until we are sure that
197  * the timehands that we operated on was not updated under our feet.  See
198  * the comment in <sys/time.h> for a description of these 12 functions.
199  */
200 
201 #ifdef FFCLOCK
202 void
203 fbclock_binuptime(struct bintime *bt)
204 {
205 	struct timehands *th;
206 	unsigned int gen;
207 
208 	do {
209 		th = timehands;
210 		gen = atomic_load_acq_int(&th->th_generation);
211 		*bt = th->th_offset;
212 		bintime_addx(bt, th->th_scale * tc_delta(th));
213 		atomic_thread_fence_acq();
214 	} while (gen == 0 || gen != th->th_generation);
215 }
216 
217 void
218 fbclock_nanouptime(struct timespec *tsp)
219 {
220 	struct bintime bt;
221 
222 	fbclock_binuptime(&bt);
223 	bintime2timespec(&bt, tsp);
224 }
225 
226 void
227 fbclock_microuptime(struct timeval *tvp)
228 {
229 	struct bintime bt;
230 
231 	fbclock_binuptime(&bt);
232 	bintime2timeval(&bt, tvp);
233 }
234 
235 void
236 fbclock_bintime(struct bintime *bt)
237 {
238 	struct timehands *th;
239 	unsigned int gen;
240 
241 	do {
242 		th = timehands;
243 		gen = atomic_load_acq_int(&th->th_generation);
244 		*bt = th->th_bintime;
245 		bintime_addx(bt, th->th_scale * tc_delta(th));
246 		atomic_thread_fence_acq();
247 	} while (gen == 0 || gen != th->th_generation);
248 }
249 
250 void
251 fbclock_nanotime(struct timespec *tsp)
252 {
253 	struct bintime bt;
254 
255 	fbclock_bintime(&bt);
256 	bintime2timespec(&bt, tsp);
257 }
258 
259 void
260 fbclock_microtime(struct timeval *tvp)
261 {
262 	struct bintime bt;
263 
264 	fbclock_bintime(&bt);
265 	bintime2timeval(&bt, tvp);
266 }
267 
268 void
269 fbclock_getbinuptime(struct bintime *bt)
270 {
271 	struct timehands *th;
272 	unsigned int gen;
273 
274 	do {
275 		th = timehands;
276 		gen = atomic_load_acq_int(&th->th_generation);
277 		*bt = th->th_offset;
278 		atomic_thread_fence_acq();
279 	} while (gen == 0 || gen != th->th_generation);
280 }
281 
282 void
283 fbclock_getnanouptime(struct timespec *tsp)
284 {
285 	struct timehands *th;
286 	unsigned int gen;
287 
288 	do {
289 		th = timehands;
290 		gen = atomic_load_acq_int(&th->th_generation);
291 		bintime2timespec(&th->th_offset, tsp);
292 		atomic_thread_fence_acq();
293 	} while (gen == 0 || gen != th->th_generation);
294 }
295 
296 void
297 fbclock_getmicrouptime(struct timeval *tvp)
298 {
299 	struct timehands *th;
300 	unsigned int gen;
301 
302 	do {
303 		th = timehands;
304 		gen = atomic_load_acq_int(&th->th_generation);
305 		bintime2timeval(&th->th_offset, tvp);
306 		atomic_thread_fence_acq();
307 	} while (gen == 0 || gen != th->th_generation);
308 }
309 
310 void
311 fbclock_getbintime(struct bintime *bt)
312 {
313 	struct timehands *th;
314 	unsigned int gen;
315 
316 	do {
317 		th = timehands;
318 		gen = atomic_load_acq_int(&th->th_generation);
319 		*bt = th->th_bintime;
320 		atomic_thread_fence_acq();
321 	} while (gen == 0 || gen != th->th_generation);
322 }
323 
324 void
325 fbclock_getnanotime(struct timespec *tsp)
326 {
327 	struct timehands *th;
328 	unsigned int gen;
329 
330 	do {
331 		th = timehands;
332 		gen = atomic_load_acq_int(&th->th_generation);
333 		*tsp = th->th_nanotime;
334 		atomic_thread_fence_acq();
335 	} while (gen == 0 || gen != th->th_generation);
336 }
337 
338 void
339 fbclock_getmicrotime(struct timeval *tvp)
340 {
341 	struct timehands *th;
342 	unsigned int gen;
343 
344 	do {
345 		th = timehands;
346 		gen = atomic_load_acq_int(&th->th_generation);
347 		*tvp = th->th_microtime;
348 		atomic_thread_fence_acq();
349 	} while (gen == 0 || gen != th->th_generation);
350 }
351 #else /* !FFCLOCK */
352 void
353 binuptime(struct bintime *bt)
354 {
355 	struct timehands *th;
356 	u_int gen;
357 
358 	do {
359 		th = timehands;
360 		gen = atomic_load_acq_int(&th->th_generation);
361 		*bt = th->th_offset;
362 		bintime_addx(bt, th->th_scale * tc_delta(th));
363 		atomic_thread_fence_acq();
364 	} while (gen == 0 || gen != th->th_generation);
365 }
366 
367 void
368 nanouptime(struct timespec *tsp)
369 {
370 	struct bintime bt;
371 
372 	binuptime(&bt);
373 	bintime2timespec(&bt, tsp);
374 }
375 
376 void
377 microuptime(struct timeval *tvp)
378 {
379 	struct bintime bt;
380 
381 	binuptime(&bt);
382 	bintime2timeval(&bt, tvp);
383 }
384 
385 void
386 bintime(struct bintime *bt)
387 {
388 	struct timehands *th;
389 	u_int gen;
390 
391 	do {
392 		th = timehands;
393 		gen = atomic_load_acq_int(&th->th_generation);
394 		*bt = th->th_bintime;
395 		bintime_addx(bt, th->th_scale * tc_delta(th));
396 		atomic_thread_fence_acq();
397 	} while (gen == 0 || gen != th->th_generation);
398 }
399 
400 void
401 nanotime(struct timespec *tsp)
402 {
403 	struct bintime bt;
404 
405 	bintime(&bt);
406 	bintime2timespec(&bt, tsp);
407 }
408 
409 void
410 microtime(struct timeval *tvp)
411 {
412 	struct bintime bt;
413 
414 	bintime(&bt);
415 	bintime2timeval(&bt, tvp);
416 }
417 
418 void
419 getbinuptime(struct bintime *bt)
420 {
421 	struct timehands *th;
422 	u_int gen;
423 
424 	do {
425 		th = timehands;
426 		gen = atomic_load_acq_int(&th->th_generation);
427 		*bt = th->th_offset;
428 		atomic_thread_fence_acq();
429 	} while (gen == 0 || gen != th->th_generation);
430 }
431 
432 void
433 getnanouptime(struct timespec *tsp)
434 {
435 	struct timehands *th;
436 	u_int gen;
437 
438 	do {
439 		th = timehands;
440 		gen = atomic_load_acq_int(&th->th_generation);
441 		bintime2timespec(&th->th_offset, tsp);
442 		atomic_thread_fence_acq();
443 	} while (gen == 0 || gen != th->th_generation);
444 }
445 
446 void
447 getmicrouptime(struct timeval *tvp)
448 {
449 	struct timehands *th;
450 	u_int gen;
451 
452 	do {
453 		th = timehands;
454 		gen = atomic_load_acq_int(&th->th_generation);
455 		bintime2timeval(&th->th_offset, tvp);
456 		atomic_thread_fence_acq();
457 	} while (gen == 0 || gen != th->th_generation);
458 }
459 
460 void
461 getbintime(struct bintime *bt)
462 {
463 	struct timehands *th;
464 	u_int gen;
465 
466 	do {
467 		th = timehands;
468 		gen = atomic_load_acq_int(&th->th_generation);
469 		*bt = th->th_bintime;
470 		atomic_thread_fence_acq();
471 	} while (gen == 0 || gen != th->th_generation);
472 }
473 
474 void
475 getnanotime(struct timespec *tsp)
476 {
477 	struct timehands *th;
478 	u_int gen;
479 
480 	do {
481 		th = timehands;
482 		gen = atomic_load_acq_int(&th->th_generation);
483 		*tsp = th->th_nanotime;
484 		atomic_thread_fence_acq();
485 	} while (gen == 0 || gen != th->th_generation);
486 }
487 
488 void
489 getmicrotime(struct timeval *tvp)
490 {
491 	struct timehands *th;
492 	u_int gen;
493 
494 	do {
495 		th = timehands;
496 		gen = atomic_load_acq_int(&th->th_generation);
497 		*tvp = th->th_microtime;
498 		atomic_thread_fence_acq();
499 	} while (gen == 0 || gen != th->th_generation);
500 }
501 #endif /* FFCLOCK */
502 
503 void
504 getboottime(struct timeval *boottime)
505 {
506 	struct bintime boottimebin;
507 
508 	getboottimebin(&boottimebin);
509 	bintime2timeval(&boottimebin, boottime);
510 }
511 
512 void
513 getboottimebin(struct bintime *boottimebin)
514 {
515 	struct timehands *th;
516 	u_int gen;
517 
518 	do {
519 		th = timehands;
520 		gen = atomic_load_acq_int(&th->th_generation);
521 		*boottimebin = th->th_boottime;
522 		atomic_thread_fence_acq();
523 	} while (gen == 0 || gen != th->th_generation);
524 }
525 
526 #ifdef FFCLOCK
527 /*
528  * Support for feed-forward synchronization algorithms. This is heavily inspired
529  * by the timehands mechanism but kept independent from it. *_windup() functions
530  * have some connection to avoid accessing the timecounter hardware more than
531  * necessary.
532  */
533 
534 /* Feed-forward clock estimates kept updated by the synchronization daemon. */
535 struct ffclock_estimate ffclock_estimate;
536 struct bintime ffclock_boottime;	/* Feed-forward boot time estimate. */
537 uint32_t ffclock_status;		/* Feed-forward clock status. */
538 int8_t ffclock_updated;			/* New estimates are available. */
539 struct mtx ffclock_mtx;			/* Mutex on ffclock_estimate. */
540 
541 struct fftimehands {
542 	struct ffclock_estimate	cest;
543 	struct bintime		tick_time;
544 	struct bintime		tick_time_lerp;
545 	ffcounter		tick_ffcount;
546 	uint64_t		period_lerp;
547 	volatile uint8_t	gen;
548 	struct fftimehands	*next;
549 };
550 
551 #define	NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x))
552 
553 static struct fftimehands ffth[10];
554 static struct fftimehands *volatile fftimehands = ffth;
555 
556 static void
557 ffclock_init(void)
558 {
559 	struct fftimehands *cur;
560 	struct fftimehands *last;
561 
562 	memset(ffth, 0, sizeof(ffth));
563 
564 	last = ffth + NUM_ELEMENTS(ffth) - 1;
565 	for (cur = ffth; cur < last; cur++)
566 		cur->next = cur + 1;
567 	last->next = ffth;
568 
569 	ffclock_updated = 0;
570 	ffclock_status = FFCLOCK_STA_UNSYNC;
571 	mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF);
572 }
573 
574 /*
575  * Reset the feed-forward clock estimates. Called from inittodr() to get things
576  * kick started and uses the timecounter nominal frequency as a first period
577  * estimate. Note: this function may be called several time just after boot.
578  * Note: this is the only function that sets the value of boot time for the
579  * monotonic (i.e. uptime) version of the feed-forward clock.
580  */
581 void
582 ffclock_reset_clock(struct timespec *ts)
583 {
584 	struct timecounter *tc;
585 	struct ffclock_estimate cest;
586 
587 	tc = timehands->th_counter;
588 	memset(&cest, 0, sizeof(struct ffclock_estimate));
589 
590 	timespec2bintime(ts, &ffclock_boottime);
591 	timespec2bintime(ts, &(cest.update_time));
592 	ffclock_read_counter(&cest.update_ffcount);
593 	cest.leapsec_next = 0;
594 	cest.period = ((1ULL << 63) / tc->tc_frequency) << 1;
595 	cest.errb_abs = 0;
596 	cest.errb_rate = 0;
597 	cest.status = FFCLOCK_STA_UNSYNC;
598 	cest.leapsec_total = 0;
599 	cest.leapsec = 0;
600 
601 	mtx_lock(&ffclock_mtx);
602 	bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
603 	ffclock_updated = INT8_MAX;
604 	mtx_unlock(&ffclock_mtx);
605 
606 	printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name,
607 	    (unsigned long long)tc->tc_frequency, (long)ts->tv_sec,
608 	    (unsigned long)ts->tv_nsec);
609 }
610 
611 /*
612  * Sub-routine to convert a time interval measured in RAW counter units to time
613  * in seconds stored in bintime format.
614  * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be
615  * larger than the max value of u_int (on 32 bit architecture). Loop to consume
616  * extra cycles.
617  */
618 static void
619 ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt)
620 {
621 	struct bintime bt2;
622 	ffcounter delta, delta_max;
623 
624 	delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1;
625 	bintime_clear(bt);
626 	do {
627 		if (ffdelta > delta_max)
628 			delta = delta_max;
629 		else
630 			delta = ffdelta;
631 		bt2.sec = 0;
632 		bt2.frac = period;
633 		bintime_mul(&bt2, (unsigned int)delta);
634 		bintime_add(bt, &bt2);
635 		ffdelta -= delta;
636 	} while (ffdelta > 0);
637 }
638 
639 /*
640  * Update the fftimehands.
641  * Push the tick ffcount and time(s) forward based on current clock estimate.
642  * The conversion from ffcounter to bintime relies on the difference clock
643  * principle, whose accuracy relies on computing small time intervals. If a new
644  * clock estimate has been passed by the synchronisation daemon, make it
645  * current, and compute the linear interpolation for monotonic time if needed.
646  */
647 static void
648 ffclock_windup(unsigned int delta)
649 {
650 	struct ffclock_estimate *cest;
651 	struct fftimehands *ffth;
652 	struct bintime bt, gap_lerp;
653 	ffcounter ffdelta;
654 	uint64_t frac;
655 	unsigned int polling;
656 	uint8_t forward_jump, ogen;
657 
658 	/*
659 	 * Pick the next timehand, copy current ffclock estimates and move tick
660 	 * times and counter forward.
661 	 */
662 	forward_jump = 0;
663 	ffth = fftimehands->next;
664 	ogen = ffth->gen;
665 	ffth->gen = 0;
666 	cest = &ffth->cest;
667 	bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate));
668 	ffdelta = (ffcounter)delta;
669 	ffth->period_lerp = fftimehands->period_lerp;
670 
671 	ffth->tick_time = fftimehands->tick_time;
672 	ffclock_convert_delta(ffdelta, cest->period, &bt);
673 	bintime_add(&ffth->tick_time, &bt);
674 
675 	ffth->tick_time_lerp = fftimehands->tick_time_lerp;
676 	ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt);
677 	bintime_add(&ffth->tick_time_lerp, &bt);
678 
679 	ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta;
680 
681 	/*
682 	 * Assess the status of the clock, if the last update is too old, it is
683 	 * likely the synchronisation daemon is dead and the clock is free
684 	 * running.
685 	 */
686 	if (ffclock_updated == 0) {
687 		ffdelta = ffth->tick_ffcount - cest->update_ffcount;
688 		ffclock_convert_delta(ffdelta, cest->period, &bt);
689 		if (bt.sec > 2 * FFCLOCK_SKM_SCALE)
690 			ffclock_status |= FFCLOCK_STA_UNSYNC;
691 	}
692 
693 	/*
694 	 * If available, grab updated clock estimates and make them current.
695 	 * Recompute time at this tick using the updated estimates. The clock
696 	 * estimates passed the feed-forward synchronisation daemon may result
697 	 * in time conversion that is not monotonically increasing (just after
698 	 * the update). time_lerp is a particular linear interpolation over the
699 	 * synchronisation algo polling period that ensures monotonicity for the
700 	 * clock ids requesting it.
701 	 */
702 	if (ffclock_updated > 0) {
703 		bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate));
704 		ffdelta = ffth->tick_ffcount - cest->update_ffcount;
705 		ffth->tick_time = cest->update_time;
706 		ffclock_convert_delta(ffdelta, cest->period, &bt);
707 		bintime_add(&ffth->tick_time, &bt);
708 
709 		/* ffclock_reset sets ffclock_updated to INT8_MAX */
710 		if (ffclock_updated == INT8_MAX)
711 			ffth->tick_time_lerp = ffth->tick_time;
712 
713 		if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >))
714 			forward_jump = 1;
715 		else
716 			forward_jump = 0;
717 
718 		bintime_clear(&gap_lerp);
719 		if (forward_jump) {
720 			gap_lerp = ffth->tick_time;
721 			bintime_sub(&gap_lerp, &ffth->tick_time_lerp);
722 		} else {
723 			gap_lerp = ffth->tick_time_lerp;
724 			bintime_sub(&gap_lerp, &ffth->tick_time);
725 		}
726 
727 		/*
728 		 * The reset from the RTC clock may be far from accurate, and
729 		 * reducing the gap between real time and interpolated time
730 		 * could take a very long time if the interpolated clock insists
731 		 * on strict monotonicity. The clock is reset under very strict
732 		 * conditions (kernel time is known to be wrong and
733 		 * synchronization daemon has been restarted recently.
734 		 * ffclock_boottime absorbs the jump to ensure boot time is
735 		 * correct and uptime functions stay consistent.
736 		 */
737 		if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) &&
738 		    ((cest->status & FFCLOCK_STA_UNSYNC) == 0) &&
739 		    ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) {
740 			if (forward_jump)
741 				bintime_add(&ffclock_boottime, &gap_lerp);
742 			else
743 				bintime_sub(&ffclock_boottime, &gap_lerp);
744 			ffth->tick_time_lerp = ffth->tick_time;
745 			bintime_clear(&gap_lerp);
746 		}
747 
748 		ffclock_status = cest->status;
749 		ffth->period_lerp = cest->period;
750 
751 		/*
752 		 * Compute corrected period used for the linear interpolation of
753 		 * time. The rate of linear interpolation is capped to 5000PPM
754 		 * (5ms/s).
755 		 */
756 		if (bintime_isset(&gap_lerp)) {
757 			ffdelta = cest->update_ffcount;
758 			ffdelta -= fftimehands->cest.update_ffcount;
759 			ffclock_convert_delta(ffdelta, cest->period, &bt);
760 			polling = bt.sec;
761 			bt.sec = 0;
762 			bt.frac = 5000000 * (uint64_t)18446744073LL;
763 			bintime_mul(&bt, polling);
764 			if (bintime_cmp(&gap_lerp, &bt, >))
765 				gap_lerp = bt;
766 
767 			/* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */
768 			frac = 0;
769 			if (gap_lerp.sec > 0) {
770 				frac -= 1;
771 				frac /= ffdelta / gap_lerp.sec;
772 			}
773 			frac += gap_lerp.frac / ffdelta;
774 
775 			if (forward_jump)
776 				ffth->period_lerp += frac;
777 			else
778 				ffth->period_lerp -= frac;
779 		}
780 
781 		ffclock_updated = 0;
782 	}
783 	if (++ogen == 0)
784 		ogen = 1;
785 	ffth->gen = ogen;
786 	fftimehands = ffth;
787 }
788 
789 /*
790  * Adjust the fftimehands when the timecounter is changed. Stating the obvious,
791  * the old and new hardware counter cannot be read simultaneously. tc_windup()
792  * does read the two counters 'back to back', but a few cycles are effectively
793  * lost, and not accumulated in tick_ffcount. This is a fairly radical
794  * operation for a feed-forward synchronization daemon, and it is its job to not
795  * pushing irrelevant data to the kernel. Because there is no locking here,
796  * simply force to ignore pending or next update to give daemon a chance to
797  * realize the counter has changed.
798  */
799 static void
800 ffclock_change_tc(struct timehands *th)
801 {
802 	struct fftimehands *ffth;
803 	struct ffclock_estimate *cest;
804 	struct timecounter *tc;
805 	uint8_t ogen;
806 
807 	tc = th->th_counter;
808 	ffth = fftimehands->next;
809 	ogen = ffth->gen;
810 	ffth->gen = 0;
811 
812 	cest = &ffth->cest;
813 	bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate));
814 	cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1;
815 	cest->errb_abs = 0;
816 	cest->errb_rate = 0;
817 	cest->status |= FFCLOCK_STA_UNSYNC;
818 
819 	ffth->tick_ffcount = fftimehands->tick_ffcount;
820 	ffth->tick_time_lerp = fftimehands->tick_time_lerp;
821 	ffth->tick_time = fftimehands->tick_time;
822 	ffth->period_lerp = cest->period;
823 
824 	/* Do not lock but ignore next update from synchronization daemon. */
825 	ffclock_updated--;
826 
827 	if (++ogen == 0)
828 		ogen = 1;
829 	ffth->gen = ogen;
830 	fftimehands = ffth;
831 }
832 
833 /*
834  * Retrieve feed-forward counter and time of last kernel tick.
835  */
836 void
837 ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags)
838 {
839 	struct fftimehands *ffth;
840 	uint8_t gen;
841 
842 	/*
843 	 * No locking but check generation has not changed. Also need to make
844 	 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
845 	 */
846 	do {
847 		ffth = fftimehands;
848 		gen = ffth->gen;
849 		if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP)
850 			*bt = ffth->tick_time_lerp;
851 		else
852 			*bt = ffth->tick_time;
853 		*ffcount = ffth->tick_ffcount;
854 	} while (gen == 0 || gen != ffth->gen);
855 }
856 
857 /*
858  * Absolute clock conversion. Low level function to convert ffcounter to
859  * bintime. The ffcounter is converted using the current ffclock period estimate
860  * or the "interpolated period" to ensure monotonicity.
861  * NOTE: this conversion may have been deferred, and the clock updated since the
862  * hardware counter has been read.
863  */
864 void
865 ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags)
866 {
867 	struct fftimehands *ffth;
868 	struct bintime bt2;
869 	ffcounter ffdelta;
870 	uint8_t gen;
871 
872 	/*
873 	 * No locking but check generation has not changed. Also need to make
874 	 * sure ffdelta is positive, i.e. ffcount > tick_ffcount.
875 	 */
876 	do {
877 		ffth = fftimehands;
878 		gen = ffth->gen;
879 		if (ffcount > ffth->tick_ffcount)
880 			ffdelta = ffcount - ffth->tick_ffcount;
881 		else
882 			ffdelta = ffth->tick_ffcount - ffcount;
883 
884 		if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) {
885 			*bt = ffth->tick_time_lerp;
886 			ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2);
887 		} else {
888 			*bt = ffth->tick_time;
889 			ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2);
890 		}
891 
892 		if (ffcount > ffth->tick_ffcount)
893 			bintime_add(bt, &bt2);
894 		else
895 			bintime_sub(bt, &bt2);
896 	} while (gen == 0 || gen != ffth->gen);
897 }
898 
899 /*
900  * Difference clock conversion.
901  * Low level function to Convert a time interval measured in RAW counter units
902  * into bintime. The difference clock allows measuring small intervals much more
903  * reliably than the absolute clock.
904  */
905 void
906 ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt)
907 {
908 	struct fftimehands *ffth;
909 	uint8_t gen;
910 
911 	/* No locking but check generation has not changed. */
912 	do {
913 		ffth = fftimehands;
914 		gen = ffth->gen;
915 		ffclock_convert_delta(ffdelta, ffth->cest.period, bt);
916 	} while (gen == 0 || gen != ffth->gen);
917 }
918 
919 /*
920  * Access to current ffcounter value.
921  */
922 void
923 ffclock_read_counter(ffcounter *ffcount)
924 {
925 	struct timehands *th;
926 	struct fftimehands *ffth;
927 	unsigned int gen, delta;
928 
929 	/*
930 	 * ffclock_windup() called from tc_windup(), safe to rely on
931 	 * th->th_generation only, for correct delta and ffcounter.
932 	 */
933 	do {
934 		th = timehands;
935 		gen = atomic_load_acq_int(&th->th_generation);
936 		ffth = fftimehands;
937 		delta = tc_delta(th);
938 		*ffcount = ffth->tick_ffcount;
939 		atomic_thread_fence_acq();
940 	} while (gen == 0 || gen != th->th_generation);
941 
942 	*ffcount += delta;
943 }
944 
945 void
946 binuptime(struct bintime *bt)
947 {
948 
949 	binuptime_fromclock(bt, sysclock_active);
950 }
951 
952 void
953 nanouptime(struct timespec *tsp)
954 {
955 
956 	nanouptime_fromclock(tsp, sysclock_active);
957 }
958 
959 void
960 microuptime(struct timeval *tvp)
961 {
962 
963 	microuptime_fromclock(tvp, sysclock_active);
964 }
965 
966 void
967 bintime(struct bintime *bt)
968 {
969 
970 	bintime_fromclock(bt, sysclock_active);
971 }
972 
973 void
974 nanotime(struct timespec *tsp)
975 {
976 
977 	nanotime_fromclock(tsp, sysclock_active);
978 }
979 
980 void
981 microtime(struct timeval *tvp)
982 {
983 
984 	microtime_fromclock(tvp, sysclock_active);
985 }
986 
987 void
988 getbinuptime(struct bintime *bt)
989 {
990 
991 	getbinuptime_fromclock(bt, sysclock_active);
992 }
993 
994 void
995 getnanouptime(struct timespec *tsp)
996 {
997 
998 	getnanouptime_fromclock(tsp, sysclock_active);
999 }
1000 
1001 void
1002 getmicrouptime(struct timeval *tvp)
1003 {
1004 
1005 	getmicrouptime_fromclock(tvp, sysclock_active);
1006 }
1007 
1008 void
1009 getbintime(struct bintime *bt)
1010 {
1011 
1012 	getbintime_fromclock(bt, sysclock_active);
1013 }
1014 
1015 void
1016 getnanotime(struct timespec *tsp)
1017 {
1018 
1019 	getnanotime_fromclock(tsp, sysclock_active);
1020 }
1021 
1022 void
1023 getmicrotime(struct timeval *tvp)
1024 {
1025 
1026 	getmicrouptime_fromclock(tvp, sysclock_active);
1027 }
1028 
1029 #endif /* FFCLOCK */
1030 
1031 /*
1032  * This is a clone of getnanotime and used for walltimestamps.
1033  * The dtrace_ prefix prevents fbt from creating probes for
1034  * it so walltimestamp can be safely used in all fbt probes.
1035  */
1036 void
1037 dtrace_getnanotime(struct timespec *tsp)
1038 {
1039 	struct timehands *th;
1040 	u_int gen;
1041 
1042 	do {
1043 		th = timehands;
1044 		gen = atomic_load_acq_int(&th->th_generation);
1045 		*tsp = th->th_nanotime;
1046 		atomic_thread_fence_acq();
1047 	} while (gen == 0 || gen != th->th_generation);
1048 }
1049 
1050 /*
1051  * System clock currently providing time to the system. Modifiable via sysctl
1052  * when the FFCLOCK option is defined.
1053  */
1054 int sysclock_active = SYSCLOCK_FBCK;
1055 
1056 /* Internal NTP status and error estimates. */
1057 extern int time_status;
1058 extern long time_esterror;
1059 
1060 /*
1061  * Take a snapshot of sysclock data which can be used to compare system clocks
1062  * and generate timestamps after the fact.
1063  */
1064 void
1065 sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast)
1066 {
1067 	struct fbclock_info *fbi;
1068 	struct timehands *th;
1069 	struct bintime bt;
1070 	unsigned int delta, gen;
1071 #ifdef FFCLOCK
1072 	ffcounter ffcount;
1073 	struct fftimehands *ffth;
1074 	struct ffclock_info *ffi;
1075 	struct ffclock_estimate cest;
1076 
1077 	ffi = &clock_snap->ff_info;
1078 #endif
1079 
1080 	fbi = &clock_snap->fb_info;
1081 	delta = 0;
1082 
1083 	do {
1084 		th = timehands;
1085 		gen = atomic_load_acq_int(&th->th_generation);
1086 		fbi->th_scale = th->th_scale;
1087 		fbi->tick_time = th->th_offset;
1088 #ifdef FFCLOCK
1089 		ffth = fftimehands;
1090 		ffi->tick_time = ffth->tick_time_lerp;
1091 		ffi->tick_time_lerp = ffth->tick_time_lerp;
1092 		ffi->period = ffth->cest.period;
1093 		ffi->period_lerp = ffth->period_lerp;
1094 		clock_snap->ffcount = ffth->tick_ffcount;
1095 		cest = ffth->cest;
1096 #endif
1097 		if (!fast)
1098 			delta = tc_delta(th);
1099 		atomic_thread_fence_acq();
1100 	} while (gen == 0 || gen != th->th_generation);
1101 
1102 	clock_snap->delta = delta;
1103 	clock_snap->sysclock_active = sysclock_active;
1104 
1105 	/* Record feedback clock status and error. */
1106 	clock_snap->fb_info.status = time_status;
1107 	/* XXX: Very crude estimate of feedback clock error. */
1108 	bt.sec = time_esterror / 1000000;
1109 	bt.frac = ((time_esterror - bt.sec) * 1000000) *
1110 	    (uint64_t)18446744073709ULL;
1111 	clock_snap->fb_info.error = bt;
1112 
1113 #ifdef FFCLOCK
1114 	if (!fast)
1115 		clock_snap->ffcount += delta;
1116 
1117 	/* Record feed-forward clock leap second adjustment. */
1118 	ffi->leapsec_adjustment = cest.leapsec_total;
1119 	if (clock_snap->ffcount > cest.leapsec_next)
1120 		ffi->leapsec_adjustment -= cest.leapsec;
1121 
1122 	/* Record feed-forward clock status and error. */
1123 	clock_snap->ff_info.status = cest.status;
1124 	ffcount = clock_snap->ffcount - cest.update_ffcount;
1125 	ffclock_convert_delta(ffcount, cest.period, &bt);
1126 	/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */
1127 	bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL);
1128 	/* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */
1129 	bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL);
1130 	clock_snap->ff_info.error = bt;
1131 #endif
1132 }
1133 
1134 /*
1135  * Convert a sysclock snapshot into a struct bintime based on the specified
1136  * clock source and flags.
1137  */
1138 int
1139 sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt,
1140     int whichclock, uint32_t flags)
1141 {
1142 	struct bintime boottimebin;
1143 #ifdef FFCLOCK
1144 	struct bintime bt2;
1145 	uint64_t period;
1146 #endif
1147 
1148 	switch (whichclock) {
1149 	case SYSCLOCK_FBCK:
1150 		*bt = cs->fb_info.tick_time;
1151 
1152 		/* If snapshot was created with !fast, delta will be >0. */
1153 		if (cs->delta > 0)
1154 			bintime_addx(bt, cs->fb_info.th_scale * cs->delta);
1155 
1156 		if ((flags & FBCLOCK_UPTIME) == 0) {
1157 			getboottimebin(&boottimebin);
1158 			bintime_add(bt, &boottimebin);
1159 		}
1160 		break;
1161 #ifdef FFCLOCK
1162 	case SYSCLOCK_FFWD:
1163 		if (flags & FFCLOCK_LERP) {
1164 			*bt = cs->ff_info.tick_time_lerp;
1165 			period = cs->ff_info.period_lerp;
1166 		} else {
1167 			*bt = cs->ff_info.tick_time;
1168 			period = cs->ff_info.period;
1169 		}
1170 
1171 		/* If snapshot was created with !fast, delta will be >0. */
1172 		if (cs->delta > 0) {
1173 			ffclock_convert_delta(cs->delta, period, &bt2);
1174 			bintime_add(bt, &bt2);
1175 		}
1176 
1177 		/* Leap second adjustment. */
1178 		if (flags & FFCLOCK_LEAPSEC)
1179 			bt->sec -= cs->ff_info.leapsec_adjustment;
1180 
1181 		/* Boot time adjustment, for uptime/monotonic clocks. */
1182 		if (flags & FFCLOCK_UPTIME)
1183 			bintime_sub(bt, &ffclock_boottime);
1184 		break;
1185 #endif
1186 	default:
1187 		return (EINVAL);
1188 		break;
1189 	}
1190 
1191 	return (0);
1192 }
1193 
1194 /*
1195  * Initialize a new timecounter and possibly use it.
1196  */
1197 void
1198 tc_init(struct timecounter *tc)
1199 {
1200 	u_int u;
1201 	struct sysctl_oid *tc_root;
1202 
1203 	u = tc->tc_frequency / tc->tc_counter_mask;
1204 	/* XXX: We need some margin here, 10% is a guess */
1205 	u *= 11;
1206 	u /= 10;
1207 	if (u > hz && tc->tc_quality >= 0) {
1208 		tc->tc_quality = -2000;
1209 		if (bootverbose) {
1210 			printf("Timecounter \"%s\" frequency %ju Hz",
1211 			    tc->tc_name, (uintmax_t)tc->tc_frequency);
1212 			printf(" -- Insufficient hz, needs at least %u\n", u);
1213 		}
1214 	} else if (tc->tc_quality >= 0 || bootverbose) {
1215 		printf("Timecounter \"%s\" frequency %ju Hz quality %d\n",
1216 		    tc->tc_name, (uintmax_t)tc->tc_frequency,
1217 		    tc->tc_quality);
1218 	}
1219 
1220 	tc->tc_next = timecounters;
1221 	timecounters = tc;
1222 	/*
1223 	 * Set up sysctl tree for this counter.
1224 	 */
1225 	tc_root = SYSCTL_ADD_NODE_WITH_LABEL(NULL,
1226 	    SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name,
1227 	    CTLFLAG_RW, 0, "timecounter description", "timecounter");
1228 	SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1229 	    "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0,
1230 	    "mask for implemented bits");
1231 	SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1232 	    "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc),
1233 	    sysctl_kern_timecounter_get, "IU", "current timecounter value");
1234 	SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1235 	    "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc),
1236 	     sysctl_kern_timecounter_freq, "QU", "timecounter frequency");
1237 	SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO,
1238 	    "quality", CTLFLAG_RD, &(tc->tc_quality), 0,
1239 	    "goodness of time counter");
1240 	/*
1241 	 * Do not automatically switch if the current tc was specifically
1242 	 * chosen.  Never automatically use a timecounter with negative quality.
1243 	 * Even though we run on the dummy counter, switching here may be
1244 	 * worse since this timecounter may not be monotonic.
1245 	 */
1246 	if (tc_chosen)
1247 		return;
1248 	if (tc->tc_quality < 0)
1249 		return;
1250 	if (tc->tc_quality < timecounter->tc_quality)
1251 		return;
1252 	if (tc->tc_quality == timecounter->tc_quality &&
1253 	    tc->tc_frequency < timecounter->tc_frequency)
1254 		return;
1255 	(void)tc->tc_get_timecount(tc);
1256 	(void)tc->tc_get_timecount(tc);
1257 	timecounter = tc;
1258 }
1259 
1260 /* Report the frequency of the current timecounter. */
1261 uint64_t
1262 tc_getfrequency(void)
1263 {
1264 
1265 	return (timehands->th_counter->tc_frequency);
1266 }
1267 
1268 static bool
1269 sleeping_on_old_rtc(struct thread *td)
1270 {
1271 
1272 	/*
1273 	 * td_rtcgen is modified by curthread when it is running,
1274 	 * and by other threads in this function.  By finding the thread
1275 	 * on a sleepqueue and holding the lock on the sleepqueue
1276 	 * chain, we guarantee that the thread is not running and that
1277 	 * modifying td_rtcgen is safe.  Setting td_rtcgen to zero informs
1278 	 * the thread that it was woken due to a real-time clock adjustment.
1279 	 * (The declaration of td_rtcgen refers to this comment.)
1280 	 */
1281 	if (td->td_rtcgen != 0 && td->td_rtcgen != rtc_generation) {
1282 		td->td_rtcgen = 0;
1283 		return (true);
1284 	}
1285 	return (false);
1286 }
1287 
1288 static struct mtx tc_setclock_mtx;
1289 MTX_SYSINIT(tc_setclock_init, &tc_setclock_mtx, "tcsetc", MTX_SPIN);
1290 
1291 /*
1292  * Step our concept of UTC.  This is done by modifying our estimate of
1293  * when we booted.
1294  */
1295 void
1296 tc_setclock(struct timespec *ts)
1297 {
1298 	struct timespec tbef, taft;
1299 	struct bintime bt, bt2;
1300 
1301 	timespec2bintime(ts, &bt);
1302 	nanotime(&tbef);
1303 	mtx_lock_spin(&tc_setclock_mtx);
1304 	cpu_tick_calibrate(1);
1305 	binuptime(&bt2);
1306 	bintime_sub(&bt, &bt2);
1307 
1308 	/* XXX fiddle all the little crinkly bits around the fiords... */
1309 	tc_windup(&bt);
1310 	mtx_unlock_spin(&tc_setclock_mtx);
1311 
1312 	/* Avoid rtc_generation == 0, since td_rtcgen == 0 is special. */
1313 	atomic_add_rel_int(&rtc_generation, 2);
1314 	sleepq_chains_remove_matching(sleeping_on_old_rtc);
1315 	if (timestepwarnings) {
1316 		nanotime(&taft);
1317 		log(LOG_INFO,
1318 		    "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n",
1319 		    (intmax_t)tbef.tv_sec, tbef.tv_nsec,
1320 		    (intmax_t)taft.tv_sec, taft.tv_nsec,
1321 		    (intmax_t)ts->tv_sec, ts->tv_nsec);
1322 	}
1323 }
1324 
1325 /*
1326  * Initialize the next struct timehands in the ring and make
1327  * it the active timehands.  Along the way we might switch to a different
1328  * timecounter and/or do seconds processing in NTP.  Slightly magic.
1329  */
1330 static void
1331 tc_windup(struct bintime *new_boottimebin)
1332 {
1333 	struct bintime bt;
1334 	struct timehands *th, *tho;
1335 	uint64_t scale;
1336 	u_int delta, ncount, ogen;
1337 	int i;
1338 	time_t t;
1339 
1340 	/*
1341 	 * Make the next timehands a copy of the current one, but do
1342 	 * not overwrite the generation or next pointer.  While we
1343 	 * update the contents, the generation must be zero.  We need
1344 	 * to ensure that the zero generation is visible before the
1345 	 * data updates become visible, which requires release fence.
1346 	 * For similar reasons, re-reading of the generation after the
1347 	 * data is read should use acquire fence.
1348 	 */
1349 	tho = timehands;
1350 	th = tho->th_next;
1351 	ogen = th->th_generation;
1352 	th->th_generation = 0;
1353 	atomic_thread_fence_rel();
1354 	bcopy(tho, th, offsetof(struct timehands, th_generation));
1355 	if (new_boottimebin != NULL)
1356 		th->th_boottime = *new_boottimebin;
1357 
1358 	/*
1359 	 * Capture a timecounter delta on the current timecounter and if
1360 	 * changing timecounters, a counter value from the new timecounter.
1361 	 * Update the offset fields accordingly.
1362 	 */
1363 	delta = tc_delta(th);
1364 	if (th->th_counter != timecounter)
1365 		ncount = timecounter->tc_get_timecount(timecounter);
1366 	else
1367 		ncount = 0;
1368 #ifdef FFCLOCK
1369 	ffclock_windup(delta);
1370 #endif
1371 	th->th_offset_count += delta;
1372 	th->th_offset_count &= th->th_counter->tc_counter_mask;
1373 	while (delta > th->th_counter->tc_frequency) {
1374 		/* Eat complete unadjusted seconds. */
1375 		delta -= th->th_counter->tc_frequency;
1376 		th->th_offset.sec++;
1377 	}
1378 	if ((delta > th->th_counter->tc_frequency / 2) &&
1379 	    (th->th_scale * delta < ((uint64_t)1 << 63))) {
1380 		/* The product th_scale * delta just barely overflows. */
1381 		th->th_offset.sec++;
1382 	}
1383 	bintime_addx(&th->th_offset, th->th_scale * delta);
1384 
1385 	/*
1386 	 * Hardware latching timecounters may not generate interrupts on
1387 	 * PPS events, so instead we poll them.  There is a finite risk that
1388 	 * the hardware might capture a count which is later than the one we
1389 	 * got above, and therefore possibly in the next NTP second which might
1390 	 * have a different rate than the current NTP second.  It doesn't
1391 	 * matter in practice.
1392 	 */
1393 	if (tho->th_counter->tc_poll_pps)
1394 		tho->th_counter->tc_poll_pps(tho->th_counter);
1395 
1396 	/*
1397 	 * Deal with NTP second processing.  The for loop normally
1398 	 * iterates at most once, but in extreme situations it might
1399 	 * keep NTP sane if timeouts are not run for several seconds.
1400 	 * At boot, the time step can be large when the TOD hardware
1401 	 * has been read, so on really large steps, we call
1402 	 * ntp_update_second only twice.  We need to call it twice in
1403 	 * case we missed a leap second.
1404 	 */
1405 	bt = th->th_offset;
1406 	bintime_add(&bt, &th->th_boottime);
1407 	i = bt.sec - tho->th_microtime.tv_sec;
1408 	if (i > LARGE_STEP)
1409 		i = 2;
1410 	for (; i > 0; i--) {
1411 		t = bt.sec;
1412 		ntp_update_second(&th->th_adjustment, &bt.sec);
1413 		if (bt.sec != t)
1414 			th->th_boottime.sec += bt.sec - t;
1415 	}
1416 	/* Update the UTC timestamps used by the get*() functions. */
1417 	th->th_bintime = bt;
1418 	bintime2timeval(&bt, &th->th_microtime);
1419 	bintime2timespec(&bt, &th->th_nanotime);
1420 
1421 	/* Now is a good time to change timecounters. */
1422 	if (th->th_counter != timecounter) {
1423 #ifndef __arm__
1424 		if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0)
1425 			cpu_disable_c2_sleep++;
1426 		if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0)
1427 			cpu_disable_c2_sleep--;
1428 #endif
1429 		th->th_counter = timecounter;
1430 		th->th_offset_count = ncount;
1431 		tc_min_ticktock_freq = max(1, timecounter->tc_frequency /
1432 		    (((uint64_t)timecounter->tc_counter_mask + 1) / 3));
1433 #ifdef FFCLOCK
1434 		ffclock_change_tc(th);
1435 #endif
1436 	}
1437 
1438 	/*-
1439 	 * Recalculate the scaling factor.  We want the number of 1/2^64
1440 	 * fractions of a second per period of the hardware counter, taking
1441 	 * into account the th_adjustment factor which the NTP PLL/adjtime(2)
1442 	 * processing provides us with.
1443 	 *
1444 	 * The th_adjustment is nanoseconds per second with 32 bit binary
1445 	 * fraction and we want 64 bit binary fraction of second:
1446 	 *
1447 	 *	 x = a * 2^32 / 10^9 = a * 4.294967296
1448 	 *
1449 	 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int
1450 	 * we can only multiply by about 850 without overflowing, that
1451 	 * leaves no suitably precise fractions for multiply before divide.
1452 	 *
1453 	 * Divide before multiply with a fraction of 2199/512 results in a
1454 	 * systematic undercompensation of 10PPM of th_adjustment.  On a
1455 	 * 5000PPM adjustment this is a 0.05PPM error.  This is acceptable.
1456  	 *
1457 	 * We happily sacrifice the lowest of the 64 bits of our result
1458 	 * to the goddess of code clarity.
1459 	 *
1460 	 */
1461 	scale = (uint64_t)1 << 63;
1462 	scale += (th->th_adjustment / 1024) * 2199;
1463 	scale /= th->th_counter->tc_frequency;
1464 	th->th_scale = scale * 2;
1465 
1466 	/*
1467 	 * Now that the struct timehands is again consistent, set the new
1468 	 * generation number, making sure to not make it zero.
1469 	 */
1470 	if (++ogen == 0)
1471 		ogen = 1;
1472 	atomic_store_rel_int(&th->th_generation, ogen);
1473 
1474 	/* Go live with the new struct timehands. */
1475 #ifdef FFCLOCK
1476 	switch (sysclock_active) {
1477 	case SYSCLOCK_FBCK:
1478 #endif
1479 		time_second = th->th_microtime.tv_sec;
1480 		time_uptime = th->th_offset.sec;
1481 #ifdef FFCLOCK
1482 		break;
1483 	case SYSCLOCK_FFWD:
1484 		time_second = fftimehands->tick_time_lerp.sec;
1485 		time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec;
1486 		break;
1487 	}
1488 #endif
1489 
1490 	timehands = th;
1491 	timekeep_push_vdso();
1492 }
1493 
1494 /* Report or change the active timecounter hardware. */
1495 static int
1496 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
1497 {
1498 	char newname[32];
1499 	struct timecounter *newtc, *tc;
1500 	int error;
1501 
1502 	tc = timecounter;
1503 	strlcpy(newname, tc->tc_name, sizeof(newname));
1504 
1505 	error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req);
1506 	if (error != 0 || req->newptr == NULL)
1507 		return (error);
1508 	/* Record that the tc in use now was specifically chosen. */
1509 	tc_chosen = 1;
1510 	if (strcmp(newname, tc->tc_name) == 0)
1511 		return (0);
1512 	for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) {
1513 		if (strcmp(newname, newtc->tc_name) != 0)
1514 			continue;
1515 
1516 		/* Warm up new timecounter. */
1517 		(void)newtc->tc_get_timecount(newtc);
1518 		(void)newtc->tc_get_timecount(newtc);
1519 
1520 		timecounter = newtc;
1521 
1522 		/*
1523 		 * The vdso timehands update is deferred until the next
1524 		 * 'tc_windup()'.
1525 		 *
1526 		 * This is prudent given that 'timekeep_push_vdso()' does not
1527 		 * use any locking and that it can be called in hard interrupt
1528 		 * context via 'tc_windup()'.
1529 		 */
1530 		return (0);
1531 	}
1532 	return (EINVAL);
1533 }
1534 
1535 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW,
1536     0, 0, sysctl_kern_timecounter_hardware, "A",
1537     "Timecounter hardware selected");
1538 
1539 
1540 /* Report the available timecounter hardware. */
1541 static int
1542 sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS)
1543 {
1544 	struct sbuf sb;
1545 	struct timecounter *tc;
1546 	int error;
1547 
1548 	sbuf_new_for_sysctl(&sb, NULL, 0, req);
1549 	for (tc = timecounters; tc != NULL; tc = tc->tc_next) {
1550 		if (tc != timecounters)
1551 			sbuf_putc(&sb, ' ');
1552 		sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality);
1553 	}
1554 	error = sbuf_finish(&sb);
1555 	sbuf_delete(&sb);
1556 	return (error);
1557 }
1558 
1559 SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
1560     0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected");
1561 
1562 /*
1563  * RFC 2783 PPS-API implementation.
1564  */
1565 
1566 /*
1567  *  Return true if the driver is aware of the abi version extensions in the
1568  *  pps_state structure, and it supports at least the given abi version number.
1569  */
1570 static inline int
1571 abi_aware(struct pps_state *pps, int vers)
1572 {
1573 
1574 	return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers);
1575 }
1576 
1577 static int
1578 pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps)
1579 {
1580 	int err, timo;
1581 	pps_seq_t aseq, cseq;
1582 	struct timeval tv;
1583 
1584 	if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC)
1585 		return (EINVAL);
1586 
1587 	/*
1588 	 * If no timeout is requested, immediately return whatever values were
1589 	 * most recently captured.  If timeout seconds is -1, that's a request
1590 	 * to block without a timeout.  WITNESS won't let us sleep forever
1591 	 * without a lock (we really don't need a lock), so just repeatedly
1592 	 * sleep a long time.
1593 	 */
1594 	if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) {
1595 		if (fapi->timeout.tv_sec == -1)
1596 			timo = 0x7fffffff;
1597 		else {
1598 			tv.tv_sec = fapi->timeout.tv_sec;
1599 			tv.tv_usec = fapi->timeout.tv_nsec / 1000;
1600 			timo = tvtohz(&tv);
1601 		}
1602 		aseq = pps->ppsinfo.assert_sequence;
1603 		cseq = pps->ppsinfo.clear_sequence;
1604 		while (aseq == pps->ppsinfo.assert_sequence &&
1605 		    cseq == pps->ppsinfo.clear_sequence) {
1606 			if (abi_aware(pps, 1) && pps->driver_mtx != NULL) {
1607 				if (pps->flags & PPSFLAG_MTX_SPIN) {
1608 					err = msleep_spin(pps, pps->driver_mtx,
1609 					    "ppsfch", timo);
1610 				} else {
1611 					err = msleep(pps, pps->driver_mtx, PCATCH,
1612 					    "ppsfch", timo);
1613 				}
1614 			} else {
1615 				err = tsleep(pps, PCATCH, "ppsfch", timo);
1616 			}
1617 			if (err == EWOULDBLOCK) {
1618 				if (fapi->timeout.tv_sec == -1) {
1619 					continue;
1620 				} else {
1621 					return (ETIMEDOUT);
1622 				}
1623 			} else if (err != 0) {
1624 				return (err);
1625 			}
1626 		}
1627 	}
1628 
1629 	pps->ppsinfo.current_mode = pps->ppsparam.mode;
1630 	fapi->pps_info_buf = pps->ppsinfo;
1631 
1632 	return (0);
1633 }
1634 
1635 int
1636 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps)
1637 {
1638 	pps_params_t *app;
1639 	struct pps_fetch_args *fapi;
1640 #ifdef FFCLOCK
1641 	struct pps_fetch_ffc_args *fapi_ffc;
1642 #endif
1643 #ifdef PPS_SYNC
1644 	struct pps_kcbind_args *kapi;
1645 #endif
1646 
1647 	KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl"));
1648 	switch (cmd) {
1649 	case PPS_IOC_CREATE:
1650 		return (0);
1651 	case PPS_IOC_DESTROY:
1652 		return (0);
1653 	case PPS_IOC_SETPARAMS:
1654 		app = (pps_params_t *)data;
1655 		if (app->mode & ~pps->ppscap)
1656 			return (EINVAL);
1657 #ifdef FFCLOCK
1658 		/* Ensure only a single clock is selected for ffc timestamp. */
1659 		if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK)
1660 			return (EINVAL);
1661 #endif
1662 		pps->ppsparam = *app;
1663 		return (0);
1664 	case PPS_IOC_GETPARAMS:
1665 		app = (pps_params_t *)data;
1666 		*app = pps->ppsparam;
1667 		app->api_version = PPS_API_VERS_1;
1668 		return (0);
1669 	case PPS_IOC_GETCAP:
1670 		*(int*)data = pps->ppscap;
1671 		return (0);
1672 	case PPS_IOC_FETCH:
1673 		fapi = (struct pps_fetch_args *)data;
1674 		return (pps_fetch(fapi, pps));
1675 #ifdef FFCLOCK
1676 	case PPS_IOC_FETCH_FFCOUNTER:
1677 		fapi_ffc = (struct pps_fetch_ffc_args *)data;
1678 		if (fapi_ffc->tsformat && fapi_ffc->tsformat !=
1679 		    PPS_TSFMT_TSPEC)
1680 			return (EINVAL);
1681 		if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec)
1682 			return (EOPNOTSUPP);
1683 		pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode;
1684 		fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc;
1685 		/* Overwrite timestamps if feedback clock selected. */
1686 		switch (pps->ppsparam.mode & PPS_TSCLK_MASK) {
1687 		case PPS_TSCLK_FBCK:
1688 			fapi_ffc->pps_info_buf_ffc.assert_timestamp =
1689 			    pps->ppsinfo.assert_timestamp;
1690 			fapi_ffc->pps_info_buf_ffc.clear_timestamp =
1691 			    pps->ppsinfo.clear_timestamp;
1692 			break;
1693 		case PPS_TSCLK_FFWD:
1694 			break;
1695 		default:
1696 			break;
1697 		}
1698 		return (0);
1699 #endif /* FFCLOCK */
1700 	case PPS_IOC_KCBIND:
1701 #ifdef PPS_SYNC
1702 		kapi = (struct pps_kcbind_args *)data;
1703 		/* XXX Only root should be able to do this */
1704 		if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC)
1705 			return (EINVAL);
1706 		if (kapi->kernel_consumer != PPS_KC_HARDPPS)
1707 			return (EINVAL);
1708 		if (kapi->edge & ~pps->ppscap)
1709 			return (EINVAL);
1710 		pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) |
1711 		    (pps->kcmode & KCMODE_ABIFLAG);
1712 		return (0);
1713 #else
1714 		return (EOPNOTSUPP);
1715 #endif
1716 	default:
1717 		return (ENOIOCTL);
1718 	}
1719 }
1720 
1721 void
1722 pps_init(struct pps_state *pps)
1723 {
1724 	pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT;
1725 	if (pps->ppscap & PPS_CAPTUREASSERT)
1726 		pps->ppscap |= PPS_OFFSETASSERT;
1727 	if (pps->ppscap & PPS_CAPTURECLEAR)
1728 		pps->ppscap |= PPS_OFFSETCLEAR;
1729 #ifdef FFCLOCK
1730 	pps->ppscap |= PPS_TSCLK_MASK;
1731 #endif
1732 	pps->kcmode &= ~KCMODE_ABIFLAG;
1733 }
1734 
1735 void
1736 pps_init_abi(struct pps_state *pps)
1737 {
1738 
1739 	pps_init(pps);
1740 	if (pps->driver_abi > 0) {
1741 		pps->kcmode |= KCMODE_ABIFLAG;
1742 		pps->kernel_abi = PPS_ABI_VERSION;
1743 	}
1744 }
1745 
1746 void
1747 pps_capture(struct pps_state *pps)
1748 {
1749 	struct timehands *th;
1750 
1751 	KASSERT(pps != NULL, ("NULL pps pointer in pps_capture"));
1752 	th = timehands;
1753 	pps->capgen = atomic_load_acq_int(&th->th_generation);
1754 	pps->capth = th;
1755 #ifdef FFCLOCK
1756 	pps->capffth = fftimehands;
1757 #endif
1758 	pps->capcount = th->th_counter->tc_get_timecount(th->th_counter);
1759 	atomic_thread_fence_acq();
1760 	if (pps->capgen != th->th_generation)
1761 		pps->capgen = 0;
1762 }
1763 
1764 void
1765 pps_event(struct pps_state *pps, int event)
1766 {
1767 	struct bintime bt;
1768 	struct timespec ts, *tsp, *osp;
1769 	u_int tcount, *pcount;
1770 	int foff;
1771 	pps_seq_t *pseq;
1772 #ifdef FFCLOCK
1773 	struct timespec *tsp_ffc;
1774 	pps_seq_t *pseq_ffc;
1775 	ffcounter *ffcount;
1776 #endif
1777 #ifdef PPS_SYNC
1778 	int fhard;
1779 #endif
1780 
1781 	KASSERT(pps != NULL, ("NULL pps pointer in pps_event"));
1782 	/* Nothing to do if not currently set to capture this event type. */
1783 	if ((event & pps->ppsparam.mode) == 0)
1784 		return;
1785 	/* If the timecounter was wound up underneath us, bail out. */
1786 	if (pps->capgen == 0 || pps->capgen !=
1787 	    atomic_load_acq_int(&pps->capth->th_generation))
1788 		return;
1789 
1790 	/* Things would be easier with arrays. */
1791 	if (event == PPS_CAPTUREASSERT) {
1792 		tsp = &pps->ppsinfo.assert_timestamp;
1793 		osp = &pps->ppsparam.assert_offset;
1794 		foff = pps->ppsparam.mode & PPS_OFFSETASSERT;
1795 #ifdef PPS_SYNC
1796 		fhard = pps->kcmode & PPS_CAPTUREASSERT;
1797 #endif
1798 		pcount = &pps->ppscount[0];
1799 		pseq = &pps->ppsinfo.assert_sequence;
1800 #ifdef FFCLOCK
1801 		ffcount = &pps->ppsinfo_ffc.assert_ffcount;
1802 		tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp;
1803 		pseq_ffc = &pps->ppsinfo_ffc.assert_sequence;
1804 #endif
1805 	} else {
1806 		tsp = &pps->ppsinfo.clear_timestamp;
1807 		osp = &pps->ppsparam.clear_offset;
1808 		foff = pps->ppsparam.mode & PPS_OFFSETCLEAR;
1809 #ifdef PPS_SYNC
1810 		fhard = pps->kcmode & PPS_CAPTURECLEAR;
1811 #endif
1812 		pcount = &pps->ppscount[1];
1813 		pseq = &pps->ppsinfo.clear_sequence;
1814 #ifdef FFCLOCK
1815 		ffcount = &pps->ppsinfo_ffc.clear_ffcount;
1816 		tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp;
1817 		pseq_ffc = &pps->ppsinfo_ffc.clear_sequence;
1818 #endif
1819 	}
1820 
1821 	/*
1822 	 * If the timecounter changed, we cannot compare the count values, so
1823 	 * we have to drop the rest of the PPS-stuff until the next event.
1824 	 */
1825 	if (pps->ppstc != pps->capth->th_counter) {
1826 		pps->ppstc = pps->capth->th_counter;
1827 		*pcount = pps->capcount;
1828 		pps->ppscount[2] = pps->capcount;
1829 		return;
1830 	}
1831 
1832 	/* Convert the count to a timespec. */
1833 	tcount = pps->capcount - pps->capth->th_offset_count;
1834 	tcount &= pps->capth->th_counter->tc_counter_mask;
1835 	bt = pps->capth->th_bintime;
1836 	bintime_addx(&bt, pps->capth->th_scale * tcount);
1837 	bintime2timespec(&bt, &ts);
1838 
1839 	/* If the timecounter was wound up underneath us, bail out. */
1840 	atomic_thread_fence_acq();
1841 	if (pps->capgen != pps->capth->th_generation)
1842 		return;
1843 
1844 	*pcount = pps->capcount;
1845 	(*pseq)++;
1846 	*tsp = ts;
1847 
1848 	if (foff) {
1849 		timespecadd(tsp, osp);
1850 		if (tsp->tv_nsec < 0) {
1851 			tsp->tv_nsec += 1000000000;
1852 			tsp->tv_sec -= 1;
1853 		}
1854 	}
1855 
1856 #ifdef FFCLOCK
1857 	*ffcount = pps->capffth->tick_ffcount + tcount;
1858 	bt = pps->capffth->tick_time;
1859 	ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt);
1860 	bintime_add(&bt, &pps->capffth->tick_time);
1861 	bintime2timespec(&bt, &ts);
1862 	(*pseq_ffc)++;
1863 	*tsp_ffc = ts;
1864 #endif
1865 
1866 #ifdef PPS_SYNC
1867 	if (fhard) {
1868 		uint64_t scale;
1869 
1870 		/*
1871 		 * Feed the NTP PLL/FLL.
1872 		 * The FLL wants to know how many (hardware) nanoseconds
1873 		 * elapsed since the previous event.
1874 		 */
1875 		tcount = pps->capcount - pps->ppscount[2];
1876 		pps->ppscount[2] = pps->capcount;
1877 		tcount &= pps->capth->th_counter->tc_counter_mask;
1878 		scale = (uint64_t)1 << 63;
1879 		scale /= pps->capth->th_counter->tc_frequency;
1880 		scale *= 2;
1881 		bt.sec = 0;
1882 		bt.frac = 0;
1883 		bintime_addx(&bt, scale * tcount);
1884 		bintime2timespec(&bt, &ts);
1885 		hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec);
1886 	}
1887 #endif
1888 
1889 	/* Wakeup anyone sleeping in pps_fetch().  */
1890 	wakeup(pps);
1891 }
1892 
1893 /*
1894  * Timecounters need to be updated every so often to prevent the hardware
1895  * counter from overflowing.  Updating also recalculates the cached values
1896  * used by the get*() family of functions, so their precision depends on
1897  * the update frequency.
1898  */
1899 
1900 static int tc_tick;
1901 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0,
1902     "Approximate number of hardclock ticks in a millisecond");
1903 
1904 void
1905 tc_ticktock(int cnt)
1906 {
1907 	static int count;
1908 
1909 	if (mtx_trylock_spin(&tc_setclock_mtx)) {
1910 		count += cnt;
1911 		if (count >= tc_tick) {
1912 			count = 0;
1913 			tc_windup(NULL);
1914 		}
1915 		mtx_unlock_spin(&tc_setclock_mtx);
1916 	}
1917 }
1918 
1919 static void __inline
1920 tc_adjprecision(void)
1921 {
1922 	int t;
1923 
1924 	if (tc_timepercentage > 0) {
1925 		t = (99 + tc_timepercentage) / tc_timepercentage;
1926 		tc_precexp = fls(t + (t >> 1)) - 1;
1927 		FREQ2BT(hz / tc_tick, &bt_timethreshold);
1928 		FREQ2BT(hz, &bt_tickthreshold);
1929 		bintime_shift(&bt_timethreshold, tc_precexp);
1930 		bintime_shift(&bt_tickthreshold, tc_precexp);
1931 	} else {
1932 		tc_precexp = 31;
1933 		bt_timethreshold.sec = INT_MAX;
1934 		bt_timethreshold.frac = ~(uint64_t)0;
1935 		bt_tickthreshold = bt_timethreshold;
1936 	}
1937 	sbt_timethreshold = bttosbt(bt_timethreshold);
1938 	sbt_tickthreshold = bttosbt(bt_tickthreshold);
1939 }
1940 
1941 static int
1942 sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS)
1943 {
1944 	int error, val;
1945 
1946 	val = tc_timepercentage;
1947 	error = sysctl_handle_int(oidp, &val, 0, req);
1948 	if (error != 0 || req->newptr == NULL)
1949 		return (error);
1950 	tc_timepercentage = val;
1951 	if (cold)
1952 		goto done;
1953 	tc_adjprecision();
1954 done:
1955 	return (0);
1956 }
1957 
1958 static void
1959 inittimecounter(void *dummy)
1960 {
1961 	u_int p;
1962 	int tick_rate;
1963 
1964 	/*
1965 	 * Set the initial timeout to
1966 	 * max(1, <approx. number of hardclock ticks in a millisecond>).
1967 	 * People should probably not use the sysctl to set the timeout
1968 	 * to smaller than its initial value, since that value is the
1969 	 * smallest reasonable one.  If they want better timestamps they
1970 	 * should use the non-"get"* functions.
1971 	 */
1972 	if (hz > 1000)
1973 		tc_tick = (hz + 500) / 1000;
1974 	else
1975 		tc_tick = 1;
1976 	tc_adjprecision();
1977 	FREQ2BT(hz, &tick_bt);
1978 	tick_sbt = bttosbt(tick_bt);
1979 	tick_rate = hz / tc_tick;
1980 	FREQ2BT(tick_rate, &tc_tick_bt);
1981 	tc_tick_sbt = bttosbt(tc_tick_bt);
1982 	p = (tc_tick * 1000000) / hz;
1983 	printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000);
1984 
1985 #ifdef FFCLOCK
1986 	ffclock_init();
1987 #endif
1988 	/* warm up new timecounter (again) and get rolling. */
1989 	(void)timecounter->tc_get_timecount(timecounter);
1990 	(void)timecounter->tc_get_timecount(timecounter);
1991 	mtx_lock_spin(&tc_setclock_mtx);
1992 	tc_windup(NULL);
1993 	mtx_unlock_spin(&tc_setclock_mtx);
1994 }
1995 
1996 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
1997 
1998 /* Cpu tick handling -------------------------------------------------*/
1999 
2000 static int cpu_tick_variable;
2001 static uint64_t	cpu_tick_frequency;
2002 
2003 static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base);
2004 static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last);
2005 
2006 static uint64_t
2007 tc_cpu_ticks(void)
2008 {
2009 	struct timecounter *tc;
2010 	uint64_t res, *base;
2011 	unsigned u, *last;
2012 
2013 	critical_enter();
2014 	base = DPCPU_PTR(tc_cpu_ticks_base);
2015 	last = DPCPU_PTR(tc_cpu_ticks_last);
2016 	tc = timehands->th_counter;
2017 	u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
2018 	if (u < *last)
2019 		*base += (uint64_t)tc->tc_counter_mask + 1;
2020 	*last = u;
2021 	res = u + *base;
2022 	critical_exit();
2023 	return (res);
2024 }
2025 
2026 void
2027 cpu_tick_calibration(void)
2028 {
2029 	static time_t last_calib;
2030 
2031 	if (time_uptime != last_calib && !(time_uptime & 0xf)) {
2032 		cpu_tick_calibrate(0);
2033 		last_calib = time_uptime;
2034 	}
2035 }
2036 
2037 /*
2038  * This function gets called every 16 seconds on only one designated
2039  * CPU in the system from hardclock() via cpu_tick_calibration()().
2040  *
2041  * Whenever the real time clock is stepped we get called with reset=1
2042  * to make sure we handle suspend/resume and similar events correctly.
2043  */
2044 
2045 static void
2046 cpu_tick_calibrate(int reset)
2047 {
2048 	static uint64_t c_last;
2049 	uint64_t c_this, c_delta;
2050 	static struct bintime  t_last;
2051 	struct bintime t_this, t_delta;
2052 	uint32_t divi;
2053 
2054 	if (reset) {
2055 		/* The clock was stepped, abort & reset */
2056 		t_last.sec = 0;
2057 		return;
2058 	}
2059 
2060 	/* we don't calibrate fixed rate cputicks */
2061 	if (!cpu_tick_variable)
2062 		return;
2063 
2064 	getbinuptime(&t_this);
2065 	c_this = cpu_ticks();
2066 	if (t_last.sec != 0) {
2067 		c_delta = c_this - c_last;
2068 		t_delta = t_this;
2069 		bintime_sub(&t_delta, &t_last);
2070 		/*
2071 		 * Headroom:
2072 		 * 	2^(64-20) / 16[s] =
2073 		 * 	2^(44) / 16[s] =
2074 		 * 	17.592.186.044.416 / 16 =
2075 		 * 	1.099.511.627.776 [Hz]
2076 		 */
2077 		divi = t_delta.sec << 20;
2078 		divi |= t_delta.frac >> (64 - 20);
2079 		c_delta <<= 20;
2080 		c_delta /= divi;
2081 		if (c_delta > cpu_tick_frequency) {
2082 			if (0 && bootverbose)
2083 				printf("cpu_tick increased to %ju Hz\n",
2084 				    c_delta);
2085 			cpu_tick_frequency = c_delta;
2086 		}
2087 	}
2088 	c_last = c_this;
2089 	t_last = t_this;
2090 }
2091 
2092 void
2093 set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var)
2094 {
2095 
2096 	if (func == NULL) {
2097 		cpu_ticks = tc_cpu_ticks;
2098 	} else {
2099 		cpu_tick_frequency = freq;
2100 		cpu_tick_variable = var;
2101 		cpu_ticks = func;
2102 	}
2103 }
2104 
2105 uint64_t
2106 cpu_tickrate(void)
2107 {
2108 
2109 	if (cpu_ticks == tc_cpu_ticks)
2110 		return (tc_getfrequency());
2111 	return (cpu_tick_frequency);
2112 }
2113 
2114 /*
2115  * We need to be slightly careful converting cputicks to microseconds.
2116  * There is plenty of margin in 64 bits of microseconds (half a million
2117  * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply
2118  * before divide conversion (to retain precision) we find that the
2119  * margin shrinks to 1.5 hours (one millionth of 146y).
2120  * With a three prong approach we never lose significant bits, no
2121  * matter what the cputick rate and length of timeinterval is.
2122  */
2123 
2124 uint64_t
2125 cputick2usec(uint64_t tick)
2126 {
2127 
2128 	if (tick > 18446744073709551LL)		/* floor(2^64 / 1000) */
2129 		return (tick / (cpu_tickrate() / 1000000LL));
2130 	else if (tick > 18446744073709LL)	/* floor(2^64 / 1000000) */
2131 		return ((tick * 1000LL) / (cpu_tickrate() / 1000LL));
2132 	else
2133 		return ((tick * 1000000LL) / cpu_tickrate());
2134 }
2135 
2136 cpu_tick_f	*cpu_ticks = tc_cpu_ticks;
2137 
2138 static int vdso_th_enable = 1;
2139 static int
2140 sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
2141 {
2142 	int old_vdso_th_enable, error;
2143 
2144 	old_vdso_th_enable = vdso_th_enable;
2145 	error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req);
2146 	if (error != 0)
2147 		return (error);
2148 	vdso_th_enable = old_vdso_th_enable;
2149 	return (0);
2150 }
2151 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
2152     CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
2153     NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day");
2154 
2155 uint32_t
2156 tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
2157 {
2158 	struct timehands *th;
2159 	uint32_t enabled;
2160 
2161 	th = timehands;
2162 	vdso_th->th_scale = th->th_scale;
2163 	vdso_th->th_offset_count = th->th_offset_count;
2164 	vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
2165 	vdso_th->th_offset = th->th_offset;
2166 	vdso_th->th_boottime = th->th_boottime;
2167 	if (th->th_counter->tc_fill_vdso_timehands != NULL) {
2168 		enabled = th->th_counter->tc_fill_vdso_timehands(vdso_th,
2169 		    th->th_counter);
2170 	} else
2171 		enabled = 0;
2172 	if (!vdso_th_enable)
2173 		enabled = 0;
2174 	return (enabled);
2175 }
2176 
2177 #ifdef COMPAT_FREEBSD32
2178 uint32_t
2179 tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
2180 {
2181 	struct timehands *th;
2182 	uint32_t enabled;
2183 
2184 	th = timehands;
2185 	*(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale;
2186 	vdso_th32->th_offset_count = th->th_offset_count;
2187 	vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask;
2188 	vdso_th32->th_offset.sec = th->th_offset.sec;
2189 	*(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
2190 	vdso_th32->th_boottime.sec = th->th_boottime.sec;
2191 	*(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac;
2192 	if (th->th_counter->tc_fill_vdso_timehands32 != NULL) {
2193 		enabled = th->th_counter->tc_fill_vdso_timehands32(vdso_th32,
2194 		    th->th_counter);
2195 	} else
2196 		enabled = 0;
2197 	if (!vdso_th_enable)
2198 		enabled = 0;
2199 	return (enabled);
2200 }
2201 #endif
2202