xref: /freebsd/sys/kern/kern_ffclock.c (revision cf13a585108b17af3458834fe79d608c6a425649)
1 /*-
2  * Copyright (c) 2011 The University of Melbourne
3  * All rights reserved.
4  *
5  * This software was developed by Julien Ridoux at the University of Melbourne
6  * under sponsorship from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include "opt_ffclock.h"
34 
35 #include <sys/param.h>
36 #include <sys/bus.h>
37 #include <sys/kernel.h>
38 #include <sys/lock.h>
39 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <sys/priv.h>
42 #include <sys/proc.h>
43 #include <sys/sbuf.h>
44 #include <sys/sysent.h>
45 #include <sys/sysproto.h>
46 #include <sys/sysctl.h>
47 #include <sys/systm.h>
48 #include <sys/timeffc.h>
49 
50 #ifdef FFCLOCK
51 
52 extern struct ffclock_estimate ffclock_estimate;
53 extern struct bintime ffclock_boottime;
54 extern int8_t ffclock_updated;
55 extern struct mtx ffclock_mtx;
56 
57 /*
58  * Feed-forward clock absolute time. This should be the preferred way to read
59  * the feed-forward clock for "wall-clock" type time. The flags allow to compose
60  * various flavours of absolute time (e.g. with or without leap seconds taken
61  * into account). If valid pointers are provided, the ffcounter value and an
62  * upper bound on clock error associated with the bintime are provided.
63  * NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value
64  * read earlier.
65  */
66 void
67 ffclock_abstime(ffcounter *ffcount, struct bintime *bt,
68     struct bintime *error_bound, uint32_t flags)
69 {
70 	struct ffclock_estimate cest;
71 	ffcounter ffc;
72 	ffcounter update_ffcount;
73 	ffcounter ffdelta_error;
74 
75 	/* Get counter and corresponding time. */
76 	if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST)
77 		ffclock_last_tick(&ffc, bt, flags);
78 	else {
79 		ffclock_read_counter(&ffc);
80 		ffclock_convert_abs(ffc, bt, flags);
81 	}
82 
83 	/* Current ffclock estimate, use update_ffcount as generation number. */
84 	do {
85 		update_ffcount = ffclock_estimate.update_ffcount;
86 		bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
87 	} while (update_ffcount != ffclock_estimate.update_ffcount);
88 
89 	/*
90 	 * Leap second adjustment. Total as seen by synchronisation algorithm
91 	 * since it started. cest.leapsec_next is the ffcounter prediction of
92 	 * when the next leapsecond occurs.
93 	 */
94 	if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) {
95 		bt->sec -= cest.leapsec_total;
96 		if (ffc > cest.leapsec_next)
97 			bt->sec -= cest.leapsec;
98 	}
99 
100 	/* Boot time adjustment, for uptime/monotonic clocks. */
101 	if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) {
102 		bintime_sub(bt, &ffclock_boottime);
103 	}
104 
105 	/* Compute error bound if a valid pointer has been passed. */
106 	if (error_bound) {
107 		ffdelta_error = ffc - cest.update_ffcount;
108 		ffclock_convert_diff(ffdelta_error, error_bound);
109 		/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
110 		bintime_mul(error_bound, cest.errb_rate *
111 		    (uint64_t)18446744073709LL);
112 		/* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */
113 		bintime_addx(error_bound, cest.errb_abs *
114 		    (uint64_t)18446744073LL);
115 	}
116 
117 	if (ffcount)
118 		*ffcount = ffc;
119 }
120 
121 /*
122  * Feed-forward difference clock. This should be the preferred way to convert a
123  * time interval in ffcounter values into a time interval in seconds. If a valid
124  * pointer is passed, an upper bound on the error in computing the time interval
125  * in seconds is provided.
126  */
127 void
128 ffclock_difftime(ffcounter ffdelta, struct bintime *bt,
129     struct bintime *error_bound)
130 {
131 	ffcounter update_ffcount;
132 	uint32_t err_rate;
133 
134 	ffclock_convert_diff(ffdelta, bt);
135 
136 	if (error_bound) {
137 		do {
138 			update_ffcount = ffclock_estimate.update_ffcount;
139 			err_rate = ffclock_estimate.errb_rate;
140 		} while (update_ffcount != ffclock_estimate.update_ffcount);
141 
142 		ffclock_convert_diff(ffdelta, error_bound);
143 		/* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */
144 		bintime_mul(error_bound, err_rate * (uint64_t)18446744073709LL);
145 	}
146 }
147 
148 /*
149  * Sysctl for the Feed-Forward Clock.
150  */
151 
152 static int ffclock_version = 2;
153 SYSCTL_NODE(_kern, OID_AUTO, ffclock, CTLFLAG_RW, 0,
154     "Feed-Forward Clock Support");
155 SYSCTL_INT(_kern_ffclock, OID_AUTO, version, CTLFLAG_RD, &ffclock_version, 0,
156     "Version of Feed-Forward Clock Support");
157 
158 /*
159  * Sysctl to select which clock is read when calling any of the
160  * [get]{bin,nano,micro}[up]time() functions.
161  */
162 char *sysclocks[] = {"feedback", "feed-forward"};
163 
164 #define	NUM_SYSCLOCKS (sizeof(sysclocks) / sizeof(*sysclocks))
165 
166 /* Report or change the active timecounter hardware. */
167 static int
168 sysctl_kern_ffclock_choice(SYSCTL_HANDLER_ARGS)
169 {
170 	struct sbuf *s;
171 	int clk, error;
172 
173 	s = sbuf_new_for_sysctl(NULL, NULL, 16 * NUM_SYSCLOCKS, req);
174 	if (s == NULL)
175 		return (ENOMEM);
176 
177 	for (clk = 0; clk < NUM_SYSCLOCKS; clk++) {
178 		sbuf_cat(s, sysclocks[clk]);
179 		if (clk + 1 < NUM_SYSCLOCKS)
180 			sbuf_cat(s, " ");
181 	}
182 	error = sbuf_finish(s);
183 	sbuf_delete(s);
184 
185 	return (error);
186 }
187 
188 SYSCTL_PROC(_kern_ffclock, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD,
189     0, 0, sysctl_kern_ffclock_choice, "A", "Clock paradigms available");
190 
191 extern int sysclock_active;
192 
193 static int
194 sysctl_kern_ffclock_active(SYSCTL_HANDLER_ARGS)
195 {
196 	char newclock[32];
197 	int error;
198 
199 	switch (sysclock_active) {
200 	case SYSCLOCK_FBCK:
201 		strlcpy(newclock, sysclocks[SYSCLOCK_FBCK], sizeof(newclock));
202 		break;
203 	case SYSCLOCK_FFWD:
204 		strlcpy(newclock, sysclocks[SYSCLOCK_FFWD], sizeof(newclock));
205 		break;
206 	}
207 
208 	error = sysctl_handle_string(oidp, &newclock[0], sizeof(newclock), req);
209 	if (error != 0 || req->newptr == NULL)
210 		return (error);
211 	if (strncmp(newclock, sysclocks[SYSCLOCK_FBCK],
212 	    sizeof(sysclocks[SYSCLOCK_FBCK])) == 0)
213 		sysclock_active = SYSCLOCK_FBCK;
214 	else if (strncmp(newclock, sysclocks[SYSCLOCK_FFWD],
215 	    sizeof(sysclocks[SYSCLOCK_FFWD])) == 0)
216 		sysclock_active = SYSCLOCK_FFWD;
217 	else
218 		return (EINVAL);
219 
220 	return (error);
221 }
222 
223 SYSCTL_PROC(_kern_ffclock, OID_AUTO, active, CTLTYPE_STRING | CTLFLAG_RW,
224     0, 0, sysctl_kern_ffclock_active, "A", "Kernel clock selected");
225 
226 int sysctl_kern_ffclock_ffcounter_bypass = 0;
227 
228 SYSCTL_INT(_kern_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW,
229     &sysctl_kern_ffclock_ffcounter_bypass, 0,
230     "Use reliable hardware timecounter as the Feed-Forward Counter");
231 
232 /*
233  * High level functions to access the Feed-Forward Clock.
234  */
235 void
236 ffclock_bintime(struct bintime *bt)
237 {
238 
239 	ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
240 }
241 
242 void
243 ffclock_nanotime(struct timespec *tsp)
244 {
245 	struct bintime bt;
246 
247 	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
248 	bintime2timespec(&bt, tsp);
249 }
250 
251 void
252 ffclock_microtime(struct timeval *tvp)
253 {
254 	struct bintime bt;
255 
256 	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC);
257 	bintime2timeval(&bt, tvp);
258 }
259 
260 void
261 ffclock_getbintime(struct bintime *bt)
262 {
263 
264 	ffclock_abstime(NULL, bt, NULL,
265 	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
266 }
267 
268 void
269 ffclock_getnanotime(struct timespec *tsp)
270 {
271 	struct bintime bt;
272 
273 	ffclock_abstime(NULL, &bt, NULL,
274 	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
275 	bintime2timespec(&bt, tsp);
276 }
277 
278 void
279 ffclock_getmicrotime(struct timeval *tvp)
280 {
281 	struct bintime bt;
282 
283 	ffclock_abstime(NULL, &bt, NULL,
284 	    FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST);
285 	bintime2timeval(&bt, tvp);
286 }
287 
288 void
289 ffclock_binuptime(struct bintime *bt)
290 {
291 
292 	ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
293 }
294 
295 void
296 ffclock_nanouptime(struct timespec *tsp)
297 {
298 	struct bintime bt;
299 
300 	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
301 	bintime2timespec(&bt, tsp);
302 }
303 
304 void
305 ffclock_microuptime(struct timeval *tvp)
306 {
307 	struct bintime bt;
308 
309 	ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME);
310 	bintime2timeval(&bt, tvp);
311 }
312 
313 void
314 ffclock_getbinuptime(struct bintime *bt)
315 {
316 
317 	ffclock_abstime(NULL, bt, NULL,
318 	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
319 }
320 
321 void
322 ffclock_getnanouptime(struct timespec *tsp)
323 {
324 	struct bintime bt;
325 
326 	ffclock_abstime(NULL, &bt, NULL,
327 	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
328 	bintime2timespec(&bt, tsp);
329 }
330 
331 void
332 ffclock_getmicrouptime(struct timeval *tvp)
333 {
334 	struct bintime bt;
335 
336 	ffclock_abstime(NULL, &bt, NULL,
337 	    FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST);
338 	bintime2timeval(&bt, tvp);
339 }
340 
341 void
342 ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt)
343 {
344 
345 	ffclock_difftime(ffdelta, bt, NULL);
346 }
347 
348 void
349 ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp)
350 {
351 	struct bintime bt;
352 
353 	ffclock_difftime(ffdelta, &bt, NULL);
354 	bintime2timespec(&bt, tsp);
355 }
356 
357 void
358 ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp)
359 {
360 	struct bintime bt;
361 
362 	ffclock_difftime(ffdelta, &bt, NULL);
363 	bintime2timeval(&bt, tvp);
364 }
365 
366 /*
367  * System call allowing userland applications to retrieve the current value of
368  * the Feed-Forward Clock counter.
369  */
370 #ifndef _SYS_SYSPROTO_H_
371 struct ffclock_getcounter_args {
372 	ffcounter *ffcount;
373 };
374 #endif
375 /* ARGSUSED */
376 int
377 sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap)
378 {
379 	ffcounter ffcount;
380 	int error;
381 
382 	ffcount = 0;
383 	ffclock_read_counter(&ffcount);
384 	if (ffcount == 0)
385 		return (EAGAIN);
386 	error = copyout(&ffcount, uap->ffcount, sizeof(ffcounter));
387 
388 	return (error);
389 }
390 
391 /*
392  * System call allowing the synchronisation daemon to push new feed-foward clock
393  * estimates to the kernel. Acquire ffclock_mtx to prevent concurrent updates
394  * and ensure data consistency.
395  * NOTE: ffclock_updated signals the fftimehands that new estimates are
396  * available. The updated estimates are picked up by the fftimehands on next
397  * tick, which could take as long as 1/hz seconds (if ticks are not missed).
398  */
399 #ifndef _SYS_SYSPROTO_H_
400 struct ffclock_setestimate_args {
401 	struct ffclock_estimate *cest;
402 };
403 #endif
404 /* ARGSUSED */
405 int
406 sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap)
407 {
408 	struct ffclock_estimate cest;
409 	int error;
410 
411 	/* Reuse of PRIV_CLOCK_SETTIME. */
412 	if ((error = priv_check(td, PRIV_CLOCK_SETTIME)) != 0)
413 		return (error);
414 
415 	if ((error = copyin(uap->cest, &cest, sizeof(struct ffclock_estimate)))
416 	    != 0)
417 		return (error);
418 
419 	mtx_lock(&ffclock_mtx);
420 	memcpy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate));
421 	ffclock_updated++;
422 	mtx_unlock(&ffclock_mtx);
423 	return (error);
424 }
425 
426 /*
427  * System call allowing userland applications to retrieve the clock estimates
428  * stored within the kernel. It is useful to kickstart the synchronisation
429  * daemon with the kernel's knowledge of hardware timecounter.
430  */
431 #ifndef _SYS_SYSPROTO_H_
432 struct ffclock_getestimate_args {
433 	struct ffclock_estimate *cest;
434 };
435 #endif
436 /* ARGSUSED */
437 int
438 sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap)
439 {
440 	struct ffclock_estimate cest;
441 	int error;
442 
443 	mtx_lock(&ffclock_mtx);
444 	memcpy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate));
445 	mtx_unlock(&ffclock_mtx);
446 	error = copyout(&cest, uap->cest, sizeof(struct ffclock_estimate));
447 	return (error);
448 }
449 
450 #else /* !FFCLOCK */
451 
452 int
453 sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap)
454 {
455 
456 	return (ENOSYS);
457 }
458 
459 int
460 sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap)
461 {
462 
463 	return (ENOSYS);
464 }
465 
466 int
467 sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap)
468 {
469 
470 	return (ENOSYS);
471 }
472 
473 #endif /* FFCLOCK */
474