1 /*- 2 * Copyright (c) 2011 The University of Melbourne 3 * All rights reserved. 4 * 5 * This software was developed by Julien Ridoux at the University of Melbourne 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_ffclock.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/module.h> 40 #include <sys/mutex.h> 41 #include <sys/priv.h> 42 #include <sys/proc.h> 43 #include <sys/sbuf.h> 44 #include <sys/sysent.h> 45 #include <sys/sysproto.h> 46 #include <sys/sysctl.h> 47 #include <sys/systm.h> 48 #include <sys/timeffc.h> 49 50 #ifdef FFCLOCK 51 52 extern struct ffclock_estimate ffclock_estimate; 53 extern struct bintime ffclock_boottime; 54 extern int8_t ffclock_updated; 55 extern struct mtx ffclock_mtx; 56 57 /* 58 * Feed-forward clock absolute time. This should be the preferred way to read 59 * the feed-forward clock for "wall-clock" type time. The flags allow to compose 60 * various flavours of absolute time (e.g. with or without leap seconds taken 61 * into account). If valid pointers are provided, the ffcounter value and an 62 * upper bound on clock error associated with the bintime are provided. 63 * NOTE: use ffclock_convert_abs() to differ the conversion of a ffcounter value 64 * read earlier. 65 */ 66 void 67 ffclock_abstime(ffcounter *ffcount, struct bintime *bt, 68 struct bintime *error_bound, uint32_t flags) 69 { 70 struct ffclock_estimate cest; 71 ffcounter ffc; 72 ffcounter update_ffcount; 73 ffcounter ffdelta_error; 74 75 /* Get counter and corresponding time. */ 76 if ((flags & FFCLOCK_FAST) == FFCLOCK_FAST) 77 ffclock_last_tick(&ffc, bt, flags); 78 else { 79 ffclock_read_counter(&ffc); 80 ffclock_convert_abs(ffc, bt, flags); 81 } 82 83 /* Current ffclock estimate, use update_ffcount as generation number. */ 84 do { 85 update_ffcount = ffclock_estimate.update_ffcount; 86 bcopy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate)); 87 } while (update_ffcount != ffclock_estimate.update_ffcount); 88 89 /* 90 * Leap second adjustment. Total as seen by synchronisation algorithm 91 * since it started. cest.leapsec_next is the ffcounter prediction of 92 * when the next leapsecond occurs. 93 */ 94 if ((flags & FFCLOCK_LEAPSEC) == FFCLOCK_LEAPSEC) { 95 bt->sec -= cest.leapsec_total; 96 if (ffc > cest.leapsec_next) 97 bt->sec -= cest.leapsec; 98 } 99 100 /* Boot time adjustment, for uptime/monotonic clocks. */ 101 if ((flags & FFCLOCK_UPTIME) == FFCLOCK_UPTIME) { 102 bintime_sub(bt, &ffclock_boottime); 103 } 104 105 /* Compute error bound if a valid pointer has been passed. */ 106 if (error_bound) { 107 ffdelta_error = ffc - cest.update_ffcount; 108 ffclock_convert_diff(ffdelta_error, error_bound); 109 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */ 110 bintime_mul(error_bound, cest.errb_rate * 111 (uint64_t)18446744073709LL); 112 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns] */ 113 bintime_addx(error_bound, cest.errb_abs * 114 (uint64_t)18446744073LL); 115 } 116 117 if (ffcount) 118 *ffcount = ffc; 119 } 120 121 /* 122 * Feed-forward difference clock. This should be the preferred way to convert a 123 * time interval in ffcounter values into a time interval in seconds. If a valid 124 * pointer is passed, an upper bound on the error in computing the time interval 125 * in seconds is provided. 126 */ 127 void 128 ffclock_difftime(ffcounter ffdelta, struct bintime *bt, 129 struct bintime *error_bound) 130 { 131 ffcounter update_ffcount; 132 uint32_t err_rate; 133 134 ffclock_convert_diff(ffdelta, bt); 135 136 if (error_bound) { 137 do { 138 update_ffcount = ffclock_estimate.update_ffcount; 139 err_rate = ffclock_estimate.errb_rate; 140 } while (update_ffcount != ffclock_estimate.update_ffcount); 141 142 ffclock_convert_diff(ffdelta, error_bound); 143 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s] */ 144 bintime_mul(error_bound, err_rate * (uint64_t)18446744073709LL); 145 } 146 } 147 148 /* 149 * Sysctl for the Feed-Forward Clock. 150 */ 151 152 static int ffclock_version = 2; 153 SYSCTL_NODE(_kern, OID_AUTO, ffclock, CTLFLAG_RW, 0, 154 "Feed-Forward Clock Support"); 155 SYSCTL_INT(_kern_ffclock, OID_AUTO, version, CTLFLAG_RD, &ffclock_version, 0, 156 "Version of Feed-Forward Clock Support"); 157 158 /* 159 * Sysctl to select which clock is read when calling any of the 160 * [get]{bin,nano,micro}[up]time() functions. 161 */ 162 char *sysclocks[] = {"feedback", "feed-forward"}; 163 164 #define NUM_SYSCLOCKS (sizeof(sysclocks) / sizeof(*sysclocks)) 165 166 /* Report or change the active timecounter hardware. */ 167 static int 168 sysctl_kern_ffclock_choice(SYSCTL_HANDLER_ARGS) 169 { 170 struct sbuf *s; 171 int clk, error; 172 173 s = sbuf_new_for_sysctl(NULL, NULL, 16 * NUM_SYSCLOCKS, req); 174 if (s == NULL) 175 return (ENOMEM); 176 177 for (clk = 0; clk < NUM_SYSCLOCKS; clk++) { 178 sbuf_cat(s, sysclocks[clk]); 179 if (clk + 1 < NUM_SYSCLOCKS) 180 sbuf_cat(s, " "); 181 } 182 error = sbuf_finish(s); 183 sbuf_delete(s); 184 185 return (error); 186 } 187 188 SYSCTL_PROC(_kern_ffclock, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, 189 0, 0, sysctl_kern_ffclock_choice, "A", "Clock paradigms available"); 190 191 extern int sysclock_active; 192 193 static int 194 sysctl_kern_ffclock_active(SYSCTL_HANDLER_ARGS) 195 { 196 char newclock[32]; 197 int error; 198 199 switch (sysclock_active) { 200 case SYSCLOCK_FBCK: 201 strlcpy(newclock, sysclocks[SYSCLOCK_FBCK], sizeof(newclock)); 202 break; 203 case SYSCLOCK_FFWD: 204 strlcpy(newclock, sysclocks[SYSCLOCK_FFWD], sizeof(newclock)); 205 break; 206 } 207 208 error = sysctl_handle_string(oidp, &newclock[0], sizeof(newclock), req); 209 if (error != 0 || req->newptr == NULL) 210 return (error); 211 if (strncmp(newclock, sysclocks[SYSCLOCK_FBCK], 212 sizeof(sysclocks[SYSCLOCK_FBCK])) == 0) 213 sysclock_active = SYSCLOCK_FBCK; 214 else if (strncmp(newclock, sysclocks[SYSCLOCK_FFWD], 215 sizeof(sysclocks[SYSCLOCK_FFWD])) == 0) 216 sysclock_active = SYSCLOCK_FFWD; 217 else 218 return (EINVAL); 219 220 return (error); 221 } 222 223 SYSCTL_PROC(_kern_ffclock, OID_AUTO, active, CTLTYPE_STRING | CTLFLAG_RW, 224 0, 0, sysctl_kern_ffclock_active, "A", "Kernel clock selected"); 225 226 int sysctl_kern_ffclock_ffcounter_bypass = 0; 227 228 SYSCTL_INT(_kern_ffclock, OID_AUTO, ffcounter_bypass, CTLFLAG_RW, 229 &sysctl_kern_ffclock_ffcounter_bypass, 0, 230 "Use reliable hardware timecounter as the Feed-Forward Counter"); 231 232 /* 233 * High level functions to access the Feed-Forward Clock. 234 */ 235 void 236 ffclock_bintime(struct bintime *bt) 237 { 238 239 ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 240 } 241 242 void 243 ffclock_nanotime(struct timespec *tsp) 244 { 245 struct bintime bt; 246 247 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 248 bintime2timespec(&bt, tsp); 249 } 250 251 void 252 ffclock_microtime(struct timeval *tvp) 253 { 254 struct bintime bt; 255 256 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_LEAPSEC); 257 bintime2timeval(&bt, tvp); 258 } 259 260 void 261 ffclock_getbintime(struct bintime *bt) 262 { 263 264 ffclock_abstime(NULL, bt, NULL, 265 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 266 } 267 268 void 269 ffclock_getnanotime(struct timespec *tsp) 270 { 271 struct bintime bt; 272 273 ffclock_abstime(NULL, &bt, NULL, 274 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 275 bintime2timespec(&bt, tsp); 276 } 277 278 void 279 ffclock_getmicrotime(struct timeval *tvp) 280 { 281 struct bintime bt; 282 283 ffclock_abstime(NULL, &bt, NULL, 284 FFCLOCK_LERP | FFCLOCK_LEAPSEC | FFCLOCK_FAST); 285 bintime2timeval(&bt, tvp); 286 } 287 288 void 289 ffclock_binuptime(struct bintime *bt) 290 { 291 292 ffclock_abstime(NULL, bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 293 } 294 295 void 296 ffclock_nanouptime(struct timespec *tsp) 297 { 298 struct bintime bt; 299 300 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 301 bintime2timespec(&bt, tsp); 302 } 303 304 void 305 ffclock_microuptime(struct timeval *tvp) 306 { 307 struct bintime bt; 308 309 ffclock_abstime(NULL, &bt, NULL, FFCLOCK_LERP | FFCLOCK_UPTIME); 310 bintime2timeval(&bt, tvp); 311 } 312 313 void 314 ffclock_getbinuptime(struct bintime *bt) 315 { 316 317 ffclock_abstime(NULL, bt, NULL, 318 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 319 } 320 321 void 322 ffclock_getnanouptime(struct timespec *tsp) 323 { 324 struct bintime bt; 325 326 ffclock_abstime(NULL, &bt, NULL, 327 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 328 bintime2timespec(&bt, tsp); 329 } 330 331 void 332 ffclock_getmicrouptime(struct timeval *tvp) 333 { 334 struct bintime bt; 335 336 ffclock_abstime(NULL, &bt, NULL, 337 FFCLOCK_LERP | FFCLOCK_UPTIME | FFCLOCK_FAST); 338 bintime2timeval(&bt, tvp); 339 } 340 341 void 342 ffclock_bindifftime(ffcounter ffdelta, struct bintime *bt) 343 { 344 345 ffclock_difftime(ffdelta, bt, NULL); 346 } 347 348 void 349 ffclock_nanodifftime(ffcounter ffdelta, struct timespec *tsp) 350 { 351 struct bintime bt; 352 353 ffclock_difftime(ffdelta, &bt, NULL); 354 bintime2timespec(&bt, tsp); 355 } 356 357 void 358 ffclock_microdifftime(ffcounter ffdelta, struct timeval *tvp) 359 { 360 struct bintime bt; 361 362 ffclock_difftime(ffdelta, &bt, NULL); 363 bintime2timeval(&bt, tvp); 364 } 365 366 /* 367 * System call allowing userland applications to retrieve the current value of 368 * the Feed-Forward Clock counter. 369 */ 370 #ifndef _SYS_SYSPROTO_H_ 371 struct ffclock_getcounter_args { 372 ffcounter *ffcount; 373 }; 374 #endif 375 /* ARGSUSED */ 376 int 377 sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap) 378 { 379 ffcounter ffcount; 380 int error; 381 382 ffcount = 0; 383 ffclock_read_counter(&ffcount); 384 if (ffcount == 0) 385 return (EAGAIN); 386 error = copyout(&ffcount, uap->ffcount, sizeof(ffcounter)); 387 388 return (error); 389 } 390 391 /* 392 * System call allowing the synchronisation daemon to push new feed-foward clock 393 * estimates to the kernel. Acquire ffclock_mtx to prevent concurrent updates 394 * and ensure data consistency. 395 * NOTE: ffclock_updated signals the fftimehands that new estimates are 396 * available. The updated estimates are picked up by the fftimehands on next 397 * tick, which could take as long as 1/hz seconds (if ticks are not missed). 398 */ 399 #ifndef _SYS_SYSPROTO_H_ 400 struct ffclock_setestimate_args { 401 struct ffclock_estimate *cest; 402 }; 403 #endif 404 /* ARGSUSED */ 405 int 406 sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap) 407 { 408 struct ffclock_estimate cest; 409 int error; 410 411 /* Reuse of PRIV_CLOCK_SETTIME. */ 412 if ((error = priv_check(td, PRIV_CLOCK_SETTIME)) != 0) 413 return (error); 414 415 if ((error = copyin(uap->cest, &cest, sizeof(struct ffclock_estimate))) 416 != 0) 417 return (error); 418 419 mtx_lock(&ffclock_mtx); 420 memcpy(&ffclock_estimate, &cest, sizeof(struct ffclock_estimate)); 421 ffclock_updated++; 422 mtx_unlock(&ffclock_mtx); 423 return (error); 424 } 425 426 /* 427 * System call allowing userland applications to retrieve the clock estimates 428 * stored within the kernel. It is useful to kickstart the synchronisation 429 * daemon with the kernel's knowledge of hardware timecounter. 430 */ 431 #ifndef _SYS_SYSPROTO_H_ 432 struct ffclock_getestimate_args { 433 struct ffclock_estimate *cest; 434 }; 435 #endif 436 /* ARGSUSED */ 437 int 438 sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap) 439 { 440 struct ffclock_estimate cest; 441 int error; 442 443 mtx_lock(&ffclock_mtx); 444 memcpy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 445 mtx_unlock(&ffclock_mtx); 446 error = copyout(&cest, uap->cest, sizeof(struct ffclock_estimate)); 447 return (error); 448 } 449 450 #else /* !FFCLOCK */ 451 452 int 453 sys_ffclock_getcounter(struct thread *td, struct ffclock_getcounter_args *uap) 454 { 455 456 return (ENOSYS); 457 } 458 459 int 460 sys_ffclock_setestimate(struct thread *td, struct ffclock_setestimate_args *uap) 461 { 462 463 return (ENOSYS); 464 } 465 466 int 467 sys_ffclock_getestimate(struct thread *td, struct ffclock_getestimate_args *uap) 468 { 469 470 return (ENOSYS); 471 } 472 473 #endif /* FFCLOCK */ 474