1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2011 The University of Melbourne 5 * All rights reserved. 6 * 7 * This software was developed by Julien Ridoux at the University of Melbourne 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_ffclock.h" 36 37 #include <sys/param.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/lock.h> 41 #include <sys/module.h> 42 #include <sys/mutex.h> 43 #include <sys/priv.h> 44 #include <sys/proc.h> 45 #include <sys/sbuf.h> 46 #include <sys/sysproto.h> 47 #include <sys/sysctl.h> 48 #include <sys/systm.h> 49 #include <sys/timeffc.h> 50 51 #ifdef FFCLOCK 52 53 FEATURE(ffclock, "Feed-forward clock support"); 54 55 extern struct ffclock_estimate ffclock_estimate; 56 extern struct bintime ffclock_boottime; 57 extern int8_t ffclock_updated; 58 extern struct mtx ffclock_mtx; 59 60 /* 61 * Feed-forward clock absolute time. This should be the preferred way to read 62 * the feed-forward clock for "wall-clock" type time. The flags allow to compose 63 * various flavours of absolute time (e.g. with or without leap seconds taken 64 * into account). If valid pointers are provided, the ffcounter value and an 65 * upper bound on clock error associated with the bintime are provided. 66 * NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value 67 * read earlier. 68 */ 69 void 70 ffclock_abstime(ffcounter *ffcount, struct bintime *bt, 71 struct bintime *error_bound, uint32_t flags) 72 { 73 struct ffclock_estimate cest; 74 ffcounter ffc; 75 ffcounter update_ffcount; 76 ffcounter ffdelta_error; 77 78 /* Get counter and corresponding time. */ 79 if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST) 80 ffclock_last_tick(&ffc, bt, flags); 81 else { 82 ffclock_read_counter(&ffc); 83 ffclock_convert_abs(ffc, bt, flags); 84 } 85 86 /* Current ffclock estimate, use update_ffcount as generation number. */ 87 do { 88 update_ffcount = ffclock_estimate.update_ffcount; 89 bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate)); 90 } while (update_ffcount != ffclock_estimate.update_ffcount); 91 92 /* 93 * Leap second adjustment. Total as seen by synchronisation algorithm 94 * since it started. cest.leapsec_next is the ffcounter prediction of 95 * when the next leapsecond occurs. 96 */ 97 if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) { 98 bt->sec -= cest.leapsec_total; 99 if (ffc > cest.leapsec_next) 100 bt->sec -= cest.leapsec; 101 } 102 103 /* Boot time adjustment, for uptime/monotonic clocks. */ 104 if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) { 105 bintime_sub(bt, &ffclock_boottime); 106 } 107 108 /* Compute error bound if a valid pointer has been passed. */ 109 if (error_bound) { 110 ffdelta_error = ffc - cest.update_ffcount; 111 ffclock_convert_diff(ffdelta_error, error_bound); 112 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */ 113 bintime_mul(error_bound, cest.errb_rate * 114 (uint64_t)18446744073709LL); 115 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */ 116 bintime_addx(error_bound, cest.errb_abs * 117 (uint64_t)18446744073LL); 118 } 119 120 if (ffcount) 121 *ffcount = ffc; 122 } 123 124 /* 125 * Feed-forward difference clock. This should be the preferred way to convert a 126 * time interval in ffcounter values into a time interval in seconds. If a valid 127 * pointer is passed, an upper bound on the error in computing the time interval 128 * in seconds is provided. 129 */ 130 void 131 ffclock_difftime(ffcounter ffdelta, struct bintime *bt, 132 struct bintime *error_bound) 133 { 134 ffcounter update_ffcount; 135 uint32_t err_rate; 136 137 ffclock_convert_diff(ffdelta, bt); 138 139 if (error_bound) { 140 do { 141 update_ffcount = ffclock_estimate.update_ffcount; 142 err_rate = ffclock_estimate.errb_rate; 143 } while (update_ffcount != ffclock_estimate.update_ffcount); 144 145 ffclock_convert_diff(ffdelta, error_bound); 146 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */ 147 bintime_mul(error_bound, err_rate * (uint64_t)18446744073709LL); 148 } 149 } 150 151 /* 152 * Create a new kern.sysclock sysctl node, which will be home to some generic 153 * sysclock configuration variables. Feed-forward clock specific variables will 154 * live under the ffclock subnode. 155 */ 156 157 SYSCTL_NODE(_kern, OID_AUTO, sysclock, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 158 "System clock related configuration"); 159 SYSCTL_NODE(_kern_sysclock, OID_AUTO, ffclock, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 160 "Feed-forward clock configuration"); 161 162 static char *sysclocks[] = {"feedback", "feed-forward"}; 163 #define MAX_SYSCLOCK_NAME_LEN 16 164 #define NUM_SYSCLOCKS nitems(sysclocks) 165 166 static int ffclock_version = 2; 167 SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, version, CTLFLAG_RD, 168 &ffclock_version, 0, "Feed-forward clock kernel version"); 169 170 /* List available sysclocks. */ 171 static int 172 sysctl_kern_sysclock_available(SYSCTL_HANDLER_ARGS) 173 { 174 struct sbuf *s; 175 int clk, error; 176 177 s = sbuf_new_for_sysctl(NULL, NULL, 178 MAX_SYSCLOCK_NAME_LEN * NUM_SYSCLOCKS, req); 179 if (s == NULL) 180 return (ENOMEM); 181 182 for (clk = 0; clk < NUM_SYSCLOCKS; clk++) { 183 sbuf_cat(s, sysclocks[clk]); 184 if (clk + 1 < NUM_SYSCLOCKS) 185 sbuf_cat(s, " "); 186 } 187 error = sbuf_finish(s); 188 sbuf_delete(s); 189 190 return (error); 191 } 192 193 SYSCTL_PROC(_kern_sysclock, OID_AUTO, available, 194 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, 0, 0, 195 sysctl_kern_sysclock_available, "A", 196 "List of available system clocks"); 197 198 /* 199 * Return the name of the active system clock if read, or attempt to change 200 * the active system clock to the user specified one if written to. The active 201 * system clock is read when calling any of the [get]{bin,nano,micro}[up]time() 202 * functions. 203 */ 204 static int 205 sysctl_kern_sysclock_active(SYSCTL_HANDLER_ARGS) 206 { 207 char newclock[MAX_SYSCLOCK_NAME_LEN]; 208 int error; 209 int clk; 210 211 /* Return the name of the current active sysclock. */ 212 strlcpy(newclock, sysclocks[sysclock_active], sizeof(newclock)); 213 error = sysctl_handle_string(oidp, newclock, sizeof(newclock), req); 214 215 /* Check for error or no change */ 216 if (error != 0 || req->newptr == NULL) 217 goto done; 218 219 /* Change the active sysclock to the user specified one: */ 220 error = EINVAL; 221 for (clk = 0; clk < NUM_SYSCLOCKS; clk++) { 222 if (strncmp(newclock, sysclocks[clk], 223 MAX_SYSCLOCK_NAME_LEN - 1)) { 224 continue; 225 } 226 sysclock_active = clk; 227 error = 0; 228 break; 229 } 230 done: 231 return (error); 232 } 233 234 SYSCTL_PROC(_kern_sysclock, OID_AUTO, active, 235 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, 0, 236 sysctl_kern_sysclock_active, "A", 237 "Name of the active system clock which is currently serving time"); 238 239 static int sysctl_kern_ffclock_ffcounter_bypass = 0; 240 SYSCTL_INT(_kern_sysclock_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW, 241 &sysctl_kern_ffclock_ffcounter_bypass, 0, 242 "Use reliable hardware timecounter as the feed-forward counter"); 243 244 /* 245 * High level functions to access the Feed-Forward Clock. 246 */ 247 void 248 ffclock_bintime(struct bintime *bt) 249 { 250 251 ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 252 } 253 254 void 255 ffclock_nanotime(struct timespec *tsp) 256 { 257 struct bintime bt; 258 259 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 260 bintime2timespec(&bt, tsp); 261 } 262 263 void 264 ffclock_microtime(struct timeval *tvp) 265 { 266 struct bintime bt; 267 268 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 269 bintime2timeval(&bt, tvp); 270 } 271 272 void 273 ffclock_getbintime(struct bintime *bt) 274 { 275 276 ffclock_abstime(NULL, bt, NULL, 277 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 278 } 279 280 void 281 ffclock_getnanotime(struct timespec *tsp) 282 { 283 struct bintime bt; 284 285 ffclock_abstime(NULL, &bt, NULL, 286 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 287 bintime2timespec(&bt, tsp); 288 } 289 290 void 291 ffclock_getmicrotime(struct timeval *tvp) 292 { 293 struct bintime bt; 294 295 ffclock_abstime(NULL, &bt, NULL, 296 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 297 bintime2timeval(&bt, tvp); 298 } 299 300 void 301 ffclock_binuptime(struct bintime *bt) 302 { 303 304 ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 305 } 306 307 void 308 ffclock_nanouptime(struct timespec *tsp) 309 { 310 struct bintime bt; 311 312 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 313 bintime2timespec(&bt, tsp); 314 } 315 316 void 317 ffclock_microuptime(struct timeval *tvp) 318 { 319 struct bintime bt; 320 321 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 322 bintime2timeval(&bt, tvp); 323 } 324 325 void 326 ffclock_getbinuptime(struct bintime *bt) 327 { 328 329 ffclock_abstime(NULL, bt, NULL, 330 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 331 } 332 333 void 334 ffclock_getnanouptime(struct timespec *tsp) 335 { 336 struct bintime bt; 337 338 ffclock_abstime(NULL, &bt, NULL, 339 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 340 bintime2timespec(&bt, tsp); 341 } 342 343 void 344 ffclock_getmicrouptime(struct timeval *tvp) 345 { 346 struct bintime bt; 347 348 ffclock_abstime(NULL, &bt, NULL, 349 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 350 bintime2timeval(&bt, tvp); 351 } 352 353 void 354 ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt) 355 { 356 357 ffclock_difftime(ffdelta, bt, NULL); 358 } 359 360 void 361 ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp) 362 { 363 struct bintime bt; 364 365 ffclock_difftime(ffdelta, &bt, NULL); 366 bintime2timespec(&bt, tsp); 367 } 368 369 void 370 ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp) 371 { 372 struct bintime bt; 373 374 ffclock_difftime(ffdelta, &bt, NULL); 375 bintime2timeval(&bt, tvp); 376 } 377 378 /* 379 * System call allowing userland applications to retrieve the current value of 380 * the Feed-Forward Clock counter. 381 */ 382 #ifndef _SYS_SYSPROTO_H_ 383 struct ffclock_getcounter_args { 384 ffcounter *ffcount; 385 }; 386 #endif 387 /* ARGSUSED */ 388 int 389 sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap) 390 { 391 ffcounter ffcount; 392 int error; 393 394 ffcount = 0; 395 ffclock_read_counter(&ffcount); 396 if (ffcount == 0) 397 return (EAGAIN); 398 error = copyout(&ffcount, uap->ffcount, sizeof(ffcounter)); 399 400 return (error); 401 } 402 403 /* 404 * System call allowing the synchronisation daemon to push new feed-forward clock 405 * estimates to the kernel. Acquire ffclock_mtx to prevent concurrent updates 406 * and ensure data consistency. 407 * NOTE: ffclock_updated signals the fftimehands that new estimates are 408 * available. The updated estimates are picked up by the fftimehands on next 409 * tick, which could take as long as 1/hz seconds (if ticks are not missed). 410 */ 411 #ifndef _SYS_SYSPROTO_H_ 412 struct ffclock_setestimate_args { 413 struct ffclock_estimate *cest; 414 }; 415 #endif 416 /* ARGSUSED */ 417 int 418 sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap) 419 { 420 struct ffclock_estimate cest; 421 int error; 422 423 /* Reuse of PRIV_CLOCK_SETTIME. */ 424 if ((error = priv_check(td, PRIV_CLOCK_SETTIME)) != 0) 425 return (error); 426 427 if ((error = copyin(uap->cest, &cest, sizeof(struct ffclock_estimate))) 428 != 0) 429 return (error); 430 431 mtx_lock(&ffclock_mtx); 432 memcpy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate)); 433 ffclock_updated++; 434 mtx_unlock(&ffclock_mtx); 435 return (error); 436 } 437 438 /* 439 * System call allowing userland applications to retrieve the clock estimates 440 * stored within the kernel. It is useful to kickstart the synchronisation 441 * daemon with the kernel's knowledge of hardware timecounter. 442 */ 443 #ifndef _SYS_SYSPROTO_H_ 444 struct ffclock_getestimate_args { 445 struct ffclock_estimate *cest; 446 }; 447 #endif 448 /* ARGSUSED */ 449 int 450 sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap) 451 { 452 struct ffclock_estimate cest; 453 int error; 454 455 mtx_lock(&ffclock_mtx); 456 memcpy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 457 mtx_unlock(&ffclock_mtx); 458 error = copyout(&cest, uap->cest, sizeof(struct ffclock_estimate)); 459 return (error); 460 } 461 462 #else /* !FFCLOCK */ 463 464 int 465 sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap) 466 { 467 468 return (ENOSYS); 469 } 470 471 int 472 sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap) 473 { 474 475 return (ENOSYS); 476 } 477 478 int 479 sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap) 480 { 481 482 return (ENOSYS); 483 } 484 485 #endif /* FFCLOCK */ 486