1 /*- 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * Copyright (c) 2011 The FreeBSD Foundation 10 * All rights reserved. 11 * 12 * Portions of this software were developed by Julien Ridoux at the University 13 * of Melbourne under sponsorship from the FreeBSD Foundation. 14 */ 15 16 #include <sys/cdefs.h> 17 __FBSDID("$FreeBSD$"); 18 19 #include "opt_compat.h" 20 #include "opt_ntp.h" 21 #include "opt_ffclock.h" 22 23 #include <sys/param.h> 24 #include <sys/kernel.h> 25 #include <sys/limits.h> 26 #include <sys/lock.h> 27 #include <sys/mutex.h> 28 #include <sys/sbuf.h> 29 #include <sys/sysctl.h> 30 #include <sys/syslog.h> 31 #include <sys/systm.h> 32 #include <sys/timeffc.h> 33 #include <sys/timepps.h> 34 #include <sys/timetc.h> 35 #include <sys/timex.h> 36 #include <sys/vdso.h> 37 38 /* 39 * A large step happens on boot. This constant detects such steps. 40 * It is relatively small so that ntp_update_second gets called enough 41 * in the typical 'missed a couple of seconds' case, but doesn't loop 42 * forever when the time step is large. 43 */ 44 #define LARGE_STEP 200 45 46 /* 47 * Implement a dummy timecounter which we can use until we get a real one 48 * in the air. This allows the console and other early stuff to use 49 * time services. 50 */ 51 52 static u_int 53 dummy_get_timecount(struct timecounter *tc) 54 { 55 static u_int now; 56 57 return (++now); 58 } 59 60 static struct timecounter dummy_timecounter = { 61 dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000 62 }; 63 64 struct timehands { 65 /* These fields must be initialized by the driver. */ 66 struct timecounter *th_counter; 67 int64_t th_adjustment; 68 uint64_t th_scale; 69 u_int th_offset_count; 70 struct bintime th_offset; 71 struct timeval th_microtime; 72 struct timespec th_nanotime; 73 /* Fields not to be copied in tc_windup start with th_generation. */ 74 u_int th_generation; 75 struct timehands *th_next; 76 }; 77 78 static struct timehands th0; 79 static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0}; 80 static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9}; 81 static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8}; 82 static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7}; 83 static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6}; 84 static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5}; 85 static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4}; 86 static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3}; 87 static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2}; 88 static struct timehands th0 = { 89 &dummy_timecounter, 90 0, 91 (uint64_t)-1 / 1000000, 92 0, 93 {1, 0}, 94 {0, 0}, 95 {0, 0}, 96 1, 97 &th1 98 }; 99 100 static struct timehands *volatile timehands = &th0; 101 struct timecounter *timecounter = &dummy_timecounter; 102 static struct timecounter *timecounters = &dummy_timecounter; 103 104 int tc_min_ticktock_freq = 1; 105 106 volatile time_t time_second = 1; 107 volatile time_t time_uptime = 1; 108 109 struct bintime boottimebin; 110 struct timeval boottime; 111 static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); 112 SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD, 113 NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime"); 114 115 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); 116 static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, ""); 117 118 static int timestepwarnings; 119 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, 120 ×tepwarnings, 0, "Log time steps"); 121 122 struct bintime bt_timethreshold; 123 struct bintime bt_tickthreshold; 124 sbintime_t sbt_timethreshold; 125 sbintime_t sbt_tickthreshold; 126 struct bintime tc_tick_bt; 127 sbintime_t tc_tick_sbt; 128 int tc_precexp; 129 int tc_timepercentage = TC_DEFAULTPERC; 130 static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS); 131 SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation, 132 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0, 133 sysctl_kern_timecounter_adjprecision, "I", 134 "Allowed time interval deviation in percents"); 135 136 static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */ 137 138 static void tc_windup(void); 139 static void cpu_tick_calibrate(int); 140 141 void dtrace_getnanotime(struct timespec *tsp); 142 143 static int 144 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) 145 { 146 #ifndef __mips__ 147 #ifdef SCTL_MASK32 148 int tv[2]; 149 150 if (req->flags & SCTL_MASK32) { 151 tv[0] = boottime.tv_sec; 152 tv[1] = boottime.tv_usec; 153 return SYSCTL_OUT(req, tv, sizeof(tv)); 154 } else 155 #endif 156 #endif 157 return SYSCTL_OUT(req, &boottime, sizeof(boottime)); 158 } 159 160 static int 161 sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS) 162 { 163 u_int ncount; 164 struct timecounter *tc = arg1; 165 166 ncount = tc->tc_get_timecount(tc); 167 return sysctl_handle_int(oidp, &ncount, 0, req); 168 } 169 170 static int 171 sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS) 172 { 173 uint64_t freq; 174 struct timecounter *tc = arg1; 175 176 freq = tc->tc_frequency; 177 return sysctl_handle_64(oidp, &freq, 0, req); 178 } 179 180 /* 181 * Return the difference between the timehands' counter value now and what 182 * was when we copied it to the timehands' offset_count. 183 */ 184 static __inline u_int 185 tc_delta(struct timehands *th) 186 { 187 struct timecounter *tc; 188 189 tc = th->th_counter; 190 return ((tc->tc_get_timecount(tc) - th->th_offset_count) & 191 tc->tc_counter_mask); 192 } 193 194 /* 195 * Functions for reading the time. We have to loop until we are sure that 196 * the timehands that we operated on was not updated under our feet. See 197 * the comment in <sys/time.h> for a description of these 12 functions. 198 */ 199 200 #ifdef FFCLOCK 201 void 202 fbclock_binuptime(struct bintime *bt) 203 { 204 struct timehands *th; 205 unsigned int gen; 206 207 do { 208 th = timehands; 209 gen = atomic_load_acq_int(&th->th_generation); 210 *bt = th->th_offset; 211 bintime_addx(bt, th->th_scale * tc_delta(th)); 212 atomic_thread_fence_acq(); 213 } while (gen == 0 || gen != th->th_generation); 214 } 215 216 void 217 fbclock_nanouptime(struct timespec *tsp) 218 { 219 struct bintime bt; 220 221 fbclock_binuptime(&bt); 222 bintime2timespec(&bt, tsp); 223 } 224 225 void 226 fbclock_microuptime(struct timeval *tvp) 227 { 228 struct bintime bt; 229 230 fbclock_binuptime(&bt); 231 bintime2timeval(&bt, tvp); 232 } 233 234 void 235 fbclock_bintime(struct bintime *bt) 236 { 237 238 fbclock_binuptime(bt); 239 bintime_add(bt, &boottimebin); 240 } 241 242 void 243 fbclock_nanotime(struct timespec *tsp) 244 { 245 struct bintime bt; 246 247 fbclock_bintime(&bt); 248 bintime2timespec(&bt, tsp); 249 } 250 251 void 252 fbclock_microtime(struct timeval *tvp) 253 { 254 struct bintime bt; 255 256 fbclock_bintime(&bt); 257 bintime2timeval(&bt, tvp); 258 } 259 260 void 261 fbclock_getbinuptime(struct bintime *bt) 262 { 263 struct timehands *th; 264 unsigned int gen; 265 266 do { 267 th = timehands; 268 gen = atomic_load_acq_int(&th->th_generation); 269 *bt = th->th_offset; 270 atomic_thread_fence_acq(); 271 } while (gen == 0 || gen != th->th_generation); 272 } 273 274 void 275 fbclock_getnanouptime(struct timespec *tsp) 276 { 277 struct timehands *th; 278 unsigned int gen; 279 280 do { 281 th = timehands; 282 gen = atomic_load_acq_int(&th->th_generation); 283 bintime2timespec(&th->th_offset, tsp); 284 atomic_thread_fence_acq(); 285 } while (gen == 0 || gen != th->th_generation); 286 } 287 288 void 289 fbclock_getmicrouptime(struct timeval *tvp) 290 { 291 struct timehands *th; 292 unsigned int gen; 293 294 do { 295 th = timehands; 296 gen = atomic_load_acq_int(&th->th_generation); 297 bintime2timeval(&th->th_offset, tvp); 298 atomic_thread_fence_acq(); 299 } while (gen == 0 || gen != th->th_generation); 300 } 301 302 void 303 fbclock_getbintime(struct bintime *bt) 304 { 305 struct timehands *th; 306 unsigned int gen; 307 308 do { 309 th = timehands; 310 gen = atomic_load_acq_int(&th->th_generation); 311 *bt = th->th_offset; 312 atomic_thread_fence_acq(); 313 } while (gen == 0 || gen != th->th_generation); 314 bintime_add(bt, &boottimebin); 315 } 316 317 void 318 fbclock_getnanotime(struct timespec *tsp) 319 { 320 struct timehands *th; 321 unsigned int gen; 322 323 do { 324 th = timehands; 325 gen = atomic_load_acq_int(&th->th_generation); 326 *tsp = th->th_nanotime; 327 atomic_thread_fence_acq(); 328 } while (gen == 0 || gen != th->th_generation); 329 } 330 331 void 332 fbclock_getmicrotime(struct timeval *tvp) 333 { 334 struct timehands *th; 335 unsigned int gen; 336 337 do { 338 th = timehands; 339 gen = atomic_load_acq_int(&th->th_generation); 340 *tvp = th->th_microtime; 341 atomic_thread_fence_acq(); 342 } while (gen == 0 || gen != th->th_generation); 343 } 344 #else /* !FFCLOCK */ 345 void 346 binuptime(struct bintime *bt) 347 { 348 struct timehands *th; 349 u_int gen; 350 351 do { 352 th = timehands; 353 gen = atomic_load_acq_int(&th->th_generation); 354 *bt = th->th_offset; 355 bintime_addx(bt, th->th_scale * tc_delta(th)); 356 atomic_thread_fence_acq(); 357 } while (gen == 0 || gen != th->th_generation); 358 } 359 360 void 361 nanouptime(struct timespec *tsp) 362 { 363 struct bintime bt; 364 365 binuptime(&bt); 366 bintime2timespec(&bt, tsp); 367 } 368 369 void 370 microuptime(struct timeval *tvp) 371 { 372 struct bintime bt; 373 374 binuptime(&bt); 375 bintime2timeval(&bt, tvp); 376 } 377 378 void 379 bintime(struct bintime *bt) 380 { 381 382 binuptime(bt); 383 bintime_add(bt, &boottimebin); 384 } 385 386 void 387 nanotime(struct timespec *tsp) 388 { 389 struct bintime bt; 390 391 bintime(&bt); 392 bintime2timespec(&bt, tsp); 393 } 394 395 void 396 microtime(struct timeval *tvp) 397 { 398 struct bintime bt; 399 400 bintime(&bt); 401 bintime2timeval(&bt, tvp); 402 } 403 404 void 405 getbinuptime(struct bintime *bt) 406 { 407 struct timehands *th; 408 u_int gen; 409 410 do { 411 th = timehands; 412 gen = atomic_load_acq_int(&th->th_generation); 413 *bt = th->th_offset; 414 atomic_thread_fence_acq(); 415 } while (gen == 0 || gen != th->th_generation); 416 } 417 418 void 419 getnanouptime(struct timespec *tsp) 420 { 421 struct timehands *th; 422 u_int gen; 423 424 do { 425 th = timehands; 426 gen = atomic_load_acq_int(&th->th_generation); 427 bintime2timespec(&th->th_offset, tsp); 428 atomic_thread_fence_acq(); 429 } while (gen == 0 || gen != th->th_generation); 430 } 431 432 void 433 getmicrouptime(struct timeval *tvp) 434 { 435 struct timehands *th; 436 u_int gen; 437 438 do { 439 th = timehands; 440 gen = atomic_load_acq_int(&th->th_generation); 441 bintime2timeval(&th->th_offset, tvp); 442 atomic_thread_fence_acq(); 443 } while (gen == 0 || gen != th->th_generation); 444 } 445 446 void 447 getbintime(struct bintime *bt) 448 { 449 struct timehands *th; 450 u_int gen; 451 452 do { 453 th = timehands; 454 gen = atomic_load_acq_int(&th->th_generation); 455 *bt = th->th_offset; 456 atomic_thread_fence_acq(); 457 } while (gen == 0 || gen != th->th_generation); 458 bintime_add(bt, &boottimebin); 459 } 460 461 void 462 getnanotime(struct timespec *tsp) 463 { 464 struct timehands *th; 465 u_int gen; 466 467 do { 468 th = timehands; 469 gen = atomic_load_acq_int(&th->th_generation); 470 *tsp = th->th_nanotime; 471 atomic_thread_fence_acq(); 472 } while (gen == 0 || gen != th->th_generation); 473 } 474 475 void 476 getmicrotime(struct timeval *tvp) 477 { 478 struct timehands *th; 479 u_int gen; 480 481 do { 482 th = timehands; 483 gen = atomic_load_acq_int(&th->th_generation); 484 *tvp = th->th_microtime; 485 atomic_thread_fence_acq(); 486 } while (gen == 0 || gen != th->th_generation); 487 } 488 #endif /* FFCLOCK */ 489 490 #ifdef FFCLOCK 491 /* 492 * Support for feed-forward synchronization algorithms. This is heavily inspired 493 * by the timehands mechanism but kept independent from it. *_windup() functions 494 * have some connection to avoid accessing the timecounter hardware more than 495 * necessary. 496 */ 497 498 /* Feed-forward clock estimates kept updated by the synchronization daemon. */ 499 struct ffclock_estimate ffclock_estimate; 500 struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */ 501 uint32_t ffclock_status; /* Feed-forward clock status. */ 502 int8_t ffclock_updated; /* New estimates are available. */ 503 struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ 504 505 struct fftimehands { 506 struct ffclock_estimate cest; 507 struct bintime tick_time; 508 struct bintime tick_time_lerp; 509 ffcounter tick_ffcount; 510 uint64_t period_lerp; 511 volatile uint8_t gen; 512 struct fftimehands *next; 513 }; 514 515 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) 516 517 static struct fftimehands ffth[10]; 518 static struct fftimehands *volatile fftimehands = ffth; 519 520 static void 521 ffclock_init(void) 522 { 523 struct fftimehands *cur; 524 struct fftimehands *last; 525 526 memset(ffth, 0, sizeof(ffth)); 527 528 last = ffth + NUM_ELEMENTS(ffth) - 1; 529 for (cur = ffth; cur < last; cur++) 530 cur->next = cur + 1; 531 last->next = ffth; 532 533 ffclock_updated = 0; 534 ffclock_status = FFCLOCK_STA_UNSYNC; 535 mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); 536 } 537 538 /* 539 * Reset the feed-forward clock estimates. Called from inittodr() to get things 540 * kick started and uses the timecounter nominal frequency as a first period 541 * estimate. Note: this function may be called several time just after boot. 542 * Note: this is the only function that sets the value of boot time for the 543 * monotonic (i.e. uptime) version of the feed-forward clock. 544 */ 545 void 546 ffclock_reset_clock(struct timespec *ts) 547 { 548 struct timecounter *tc; 549 struct ffclock_estimate cest; 550 551 tc = timehands->th_counter; 552 memset(&cest, 0, sizeof(struct ffclock_estimate)); 553 554 timespec2bintime(ts, &ffclock_boottime); 555 timespec2bintime(ts, &(cest.update_time)); 556 ffclock_read_counter(&cest.update_ffcount); 557 cest.leapsec_next = 0; 558 cest.period = ((1ULL << 63) / tc->tc_frequency) << 1; 559 cest.errb_abs = 0; 560 cest.errb_rate = 0; 561 cest.status = FFCLOCK_STA_UNSYNC; 562 cest.leapsec_total = 0; 563 cest.leapsec = 0; 564 565 mtx_lock(&ffclock_mtx); 566 bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 567 ffclock_updated = INT8_MAX; 568 mtx_unlock(&ffclock_mtx); 569 570 printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name, 571 (unsigned long long)tc->tc_frequency, (long)ts->tv_sec, 572 (unsigned long)ts->tv_nsec); 573 } 574 575 /* 576 * Sub-routine to convert a time interval measured in RAW counter units to time 577 * in seconds stored in bintime format. 578 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be 579 * larger than the max value of u_int (on 32 bit architecture). Loop to consume 580 * extra cycles. 581 */ 582 static void 583 ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) 584 { 585 struct bintime bt2; 586 ffcounter delta, delta_max; 587 588 delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; 589 bintime_clear(bt); 590 do { 591 if (ffdelta > delta_max) 592 delta = delta_max; 593 else 594 delta = ffdelta; 595 bt2.sec = 0; 596 bt2.frac = period; 597 bintime_mul(&bt2, (unsigned int)delta); 598 bintime_add(bt, &bt2); 599 ffdelta -= delta; 600 } while (ffdelta > 0); 601 } 602 603 /* 604 * Update the fftimehands. 605 * Push the tick ffcount and time(s) forward based on current clock estimate. 606 * The conversion from ffcounter to bintime relies on the difference clock 607 * principle, whose accuracy relies on computing small time intervals. If a new 608 * clock estimate has been passed by the synchronisation daemon, make it 609 * current, and compute the linear interpolation for monotonic time if needed. 610 */ 611 static void 612 ffclock_windup(unsigned int delta) 613 { 614 struct ffclock_estimate *cest; 615 struct fftimehands *ffth; 616 struct bintime bt, gap_lerp; 617 ffcounter ffdelta; 618 uint64_t frac; 619 unsigned int polling; 620 uint8_t forward_jump, ogen; 621 622 /* 623 * Pick the next timehand, copy current ffclock estimates and move tick 624 * times and counter forward. 625 */ 626 forward_jump = 0; 627 ffth = fftimehands->next; 628 ogen = ffth->gen; 629 ffth->gen = 0; 630 cest = &ffth->cest; 631 bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate)); 632 ffdelta = (ffcounter)delta; 633 ffth->period_lerp = fftimehands->period_lerp; 634 635 ffth->tick_time = fftimehands->tick_time; 636 ffclock_convert_delta(ffdelta, cest->period, &bt); 637 bintime_add(&ffth->tick_time, &bt); 638 639 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 640 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); 641 bintime_add(&ffth->tick_time_lerp, &bt); 642 643 ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; 644 645 /* 646 * Assess the status of the clock, if the last update is too old, it is 647 * likely the synchronisation daemon is dead and the clock is free 648 * running. 649 */ 650 if (ffclock_updated == 0) { 651 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 652 ffclock_convert_delta(ffdelta, cest->period, &bt); 653 if (bt.sec > 2 * FFCLOCK_SKM_SCALE) 654 ffclock_status |= FFCLOCK_STA_UNSYNC; 655 } 656 657 /* 658 * If available, grab updated clock estimates and make them current. 659 * Recompute time at this tick using the updated estimates. The clock 660 * estimates passed the feed-forward synchronisation daemon may result 661 * in time conversion that is not monotonically increasing (just after 662 * the update). time_lerp is a particular linear interpolation over the 663 * synchronisation algo polling period that ensures monotonicity for the 664 * clock ids requesting it. 665 */ 666 if (ffclock_updated > 0) { 667 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate)); 668 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 669 ffth->tick_time = cest->update_time; 670 ffclock_convert_delta(ffdelta, cest->period, &bt); 671 bintime_add(&ffth->tick_time, &bt); 672 673 /* ffclock_reset sets ffclock_updated to INT8_MAX */ 674 if (ffclock_updated == INT8_MAX) 675 ffth->tick_time_lerp = ffth->tick_time; 676 677 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) 678 forward_jump = 1; 679 else 680 forward_jump = 0; 681 682 bintime_clear(&gap_lerp); 683 if (forward_jump) { 684 gap_lerp = ffth->tick_time; 685 bintime_sub(&gap_lerp, &ffth->tick_time_lerp); 686 } else { 687 gap_lerp = ffth->tick_time_lerp; 688 bintime_sub(&gap_lerp, &ffth->tick_time); 689 } 690 691 /* 692 * The reset from the RTC clock may be far from accurate, and 693 * reducing the gap between real time and interpolated time 694 * could take a very long time if the interpolated clock insists 695 * on strict monotonicity. The clock is reset under very strict 696 * conditions (kernel time is known to be wrong and 697 * synchronization daemon has been restarted recently. 698 * ffclock_boottime absorbs the jump to ensure boot time is 699 * correct and uptime functions stay consistent. 700 */ 701 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && 702 ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && 703 ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { 704 if (forward_jump) 705 bintime_add(&ffclock_boottime, &gap_lerp); 706 else 707 bintime_sub(&ffclock_boottime, &gap_lerp); 708 ffth->tick_time_lerp = ffth->tick_time; 709 bintime_clear(&gap_lerp); 710 } 711 712 ffclock_status = cest->status; 713 ffth->period_lerp = cest->period; 714 715 /* 716 * Compute corrected period used for the linear interpolation of 717 * time. The rate of linear interpolation is capped to 5000PPM 718 * (5ms/s). 719 */ 720 if (bintime_isset(&gap_lerp)) { 721 ffdelta = cest->update_ffcount; 722 ffdelta -= fftimehands->cest.update_ffcount; 723 ffclock_convert_delta(ffdelta, cest->period, &bt); 724 polling = bt.sec; 725 bt.sec = 0; 726 bt.frac = 5000000 * (uint64_t)18446744073LL; 727 bintime_mul(&bt, polling); 728 if (bintime_cmp(&gap_lerp, &bt, >)) 729 gap_lerp = bt; 730 731 /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ 732 frac = 0; 733 if (gap_lerp.sec > 0) { 734 frac -= 1; 735 frac /= ffdelta / gap_lerp.sec; 736 } 737 frac += gap_lerp.frac / ffdelta; 738 739 if (forward_jump) 740 ffth->period_lerp += frac; 741 else 742 ffth->period_lerp -= frac; 743 } 744 745 ffclock_updated = 0; 746 } 747 if (++ogen == 0) 748 ogen = 1; 749 ffth->gen = ogen; 750 fftimehands = ffth; 751 } 752 753 /* 754 * Adjust the fftimehands when the timecounter is changed. Stating the obvious, 755 * the old and new hardware counter cannot be read simultaneously. tc_windup() 756 * does read the two counters 'back to back', but a few cycles are effectively 757 * lost, and not accumulated in tick_ffcount. This is a fairly radical 758 * operation for a feed-forward synchronization daemon, and it is its job to not 759 * pushing irrelevant data to the kernel. Because there is no locking here, 760 * simply force to ignore pending or next update to give daemon a chance to 761 * realize the counter has changed. 762 */ 763 static void 764 ffclock_change_tc(struct timehands *th) 765 { 766 struct fftimehands *ffth; 767 struct ffclock_estimate *cest; 768 struct timecounter *tc; 769 uint8_t ogen; 770 771 tc = th->th_counter; 772 ffth = fftimehands->next; 773 ogen = ffth->gen; 774 ffth->gen = 0; 775 776 cest = &ffth->cest; 777 bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate)); 778 cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; 779 cest->errb_abs = 0; 780 cest->errb_rate = 0; 781 cest->status |= FFCLOCK_STA_UNSYNC; 782 783 ffth->tick_ffcount = fftimehands->tick_ffcount; 784 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 785 ffth->tick_time = fftimehands->tick_time; 786 ffth->period_lerp = cest->period; 787 788 /* Do not lock but ignore next update from synchronization daemon. */ 789 ffclock_updated--; 790 791 if (++ogen == 0) 792 ogen = 1; 793 ffth->gen = ogen; 794 fftimehands = ffth; 795 } 796 797 /* 798 * Retrieve feed-forward counter and time of last kernel tick. 799 */ 800 void 801 ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags) 802 { 803 struct fftimehands *ffth; 804 uint8_t gen; 805 806 /* 807 * No locking but check generation has not changed. Also need to make 808 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 809 */ 810 do { 811 ffth = fftimehands; 812 gen = ffth->gen; 813 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) 814 *bt = ffth->tick_time_lerp; 815 else 816 *bt = ffth->tick_time; 817 *ffcount = ffth->tick_ffcount; 818 } while (gen == 0 || gen != ffth->gen); 819 } 820 821 /* 822 * Absolute clock conversion. Low level function to convert ffcounter to 823 * bintime. The ffcounter is converted using the current ffclock period estimate 824 * or the "interpolated period" to ensure monotonicity. 825 * NOTE: this conversion may have been deferred, and the clock updated since the 826 * hardware counter has been read. 827 */ 828 void 829 ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags) 830 { 831 struct fftimehands *ffth; 832 struct bintime bt2; 833 ffcounter ffdelta; 834 uint8_t gen; 835 836 /* 837 * No locking but check generation has not changed. Also need to make 838 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 839 */ 840 do { 841 ffth = fftimehands; 842 gen = ffth->gen; 843 if (ffcount > ffth->tick_ffcount) 844 ffdelta = ffcount - ffth->tick_ffcount; 845 else 846 ffdelta = ffth->tick_ffcount - ffcount; 847 848 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { 849 *bt = ffth->tick_time_lerp; 850 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); 851 } else { 852 *bt = ffth->tick_time; 853 ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); 854 } 855 856 if (ffcount > ffth->tick_ffcount) 857 bintime_add(bt, &bt2); 858 else 859 bintime_sub(bt, &bt2); 860 } while (gen == 0 || gen != ffth->gen); 861 } 862 863 /* 864 * Difference clock conversion. 865 * Low level function to Convert a time interval measured in RAW counter units 866 * into bintime. The difference clock allows measuring small intervals much more 867 * reliably than the absolute clock. 868 */ 869 void 870 ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt) 871 { 872 struct fftimehands *ffth; 873 uint8_t gen; 874 875 /* No locking but check generation has not changed. */ 876 do { 877 ffth = fftimehands; 878 gen = ffth->gen; 879 ffclock_convert_delta(ffdelta, ffth->cest.period, bt); 880 } while (gen == 0 || gen != ffth->gen); 881 } 882 883 /* 884 * Access to current ffcounter value. 885 */ 886 void 887 ffclock_read_counter(ffcounter *ffcount) 888 { 889 struct timehands *th; 890 struct fftimehands *ffth; 891 unsigned int gen, delta; 892 893 /* 894 * ffclock_windup() called from tc_windup(), safe to rely on 895 * th->th_generation only, for correct delta and ffcounter. 896 */ 897 do { 898 th = timehands; 899 gen = atomic_load_acq_int(&th->th_generation); 900 ffth = fftimehands; 901 delta = tc_delta(th); 902 *ffcount = ffth->tick_ffcount; 903 atomic_thread_fence_acq(); 904 } while (gen == 0 || gen != th->th_generation); 905 906 *ffcount += delta; 907 } 908 909 void 910 binuptime(struct bintime *bt) 911 { 912 913 binuptime_fromclock(bt, sysclock_active); 914 } 915 916 void 917 nanouptime(struct timespec *tsp) 918 { 919 920 nanouptime_fromclock(tsp, sysclock_active); 921 } 922 923 void 924 microuptime(struct timeval *tvp) 925 { 926 927 microuptime_fromclock(tvp, sysclock_active); 928 } 929 930 void 931 bintime(struct bintime *bt) 932 { 933 934 bintime_fromclock(bt, sysclock_active); 935 } 936 937 void 938 nanotime(struct timespec *tsp) 939 { 940 941 nanotime_fromclock(tsp, sysclock_active); 942 } 943 944 void 945 microtime(struct timeval *tvp) 946 { 947 948 microtime_fromclock(tvp, sysclock_active); 949 } 950 951 void 952 getbinuptime(struct bintime *bt) 953 { 954 955 getbinuptime_fromclock(bt, sysclock_active); 956 } 957 958 void 959 getnanouptime(struct timespec *tsp) 960 { 961 962 getnanouptime_fromclock(tsp, sysclock_active); 963 } 964 965 void 966 getmicrouptime(struct timeval *tvp) 967 { 968 969 getmicrouptime_fromclock(tvp, sysclock_active); 970 } 971 972 void 973 getbintime(struct bintime *bt) 974 { 975 976 getbintime_fromclock(bt, sysclock_active); 977 } 978 979 void 980 getnanotime(struct timespec *tsp) 981 { 982 983 getnanotime_fromclock(tsp, sysclock_active); 984 } 985 986 void 987 getmicrotime(struct timeval *tvp) 988 { 989 990 getmicrouptime_fromclock(tvp, sysclock_active); 991 } 992 993 #endif /* FFCLOCK */ 994 995 /* 996 * This is a clone of getnanotime and used for walltimestamps. 997 * The dtrace_ prefix prevents fbt from creating probes for 998 * it so walltimestamp can be safely used in all fbt probes. 999 */ 1000 void 1001 dtrace_getnanotime(struct timespec *tsp) 1002 { 1003 struct timehands *th; 1004 u_int gen; 1005 1006 do { 1007 th = timehands; 1008 gen = atomic_load_acq_int(&th->th_generation); 1009 *tsp = th->th_nanotime; 1010 atomic_thread_fence_acq(); 1011 } while (gen == 0 || gen != th->th_generation); 1012 } 1013 1014 /* 1015 * System clock currently providing time to the system. Modifiable via sysctl 1016 * when the FFCLOCK option is defined. 1017 */ 1018 int sysclock_active = SYSCLOCK_FBCK; 1019 1020 /* Internal NTP status and error estimates. */ 1021 extern int time_status; 1022 extern long time_esterror; 1023 1024 /* 1025 * Take a snapshot of sysclock data which can be used to compare system clocks 1026 * and generate timestamps after the fact. 1027 */ 1028 void 1029 sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) 1030 { 1031 struct fbclock_info *fbi; 1032 struct timehands *th; 1033 struct bintime bt; 1034 unsigned int delta, gen; 1035 #ifdef FFCLOCK 1036 ffcounter ffcount; 1037 struct fftimehands *ffth; 1038 struct ffclock_info *ffi; 1039 struct ffclock_estimate cest; 1040 1041 ffi = &clock_snap->ff_info; 1042 #endif 1043 1044 fbi = &clock_snap->fb_info; 1045 delta = 0; 1046 1047 do { 1048 th = timehands; 1049 gen = atomic_load_acq_int(&th->th_generation); 1050 fbi->th_scale = th->th_scale; 1051 fbi->tick_time = th->th_offset; 1052 #ifdef FFCLOCK 1053 ffth = fftimehands; 1054 ffi->tick_time = ffth->tick_time_lerp; 1055 ffi->tick_time_lerp = ffth->tick_time_lerp; 1056 ffi->period = ffth->cest.period; 1057 ffi->period_lerp = ffth->period_lerp; 1058 clock_snap->ffcount = ffth->tick_ffcount; 1059 cest = ffth->cest; 1060 #endif 1061 if (!fast) 1062 delta = tc_delta(th); 1063 atomic_thread_fence_acq(); 1064 } while (gen == 0 || gen != th->th_generation); 1065 1066 clock_snap->delta = delta; 1067 clock_snap->sysclock_active = sysclock_active; 1068 1069 /* Record feedback clock status and error. */ 1070 clock_snap->fb_info.status = time_status; 1071 /* XXX: Very crude estimate of feedback clock error. */ 1072 bt.sec = time_esterror / 1000000; 1073 bt.frac = ((time_esterror - bt.sec) * 1000000) * 1074 (uint64_t)18446744073709ULL; 1075 clock_snap->fb_info.error = bt; 1076 1077 #ifdef FFCLOCK 1078 if (!fast) 1079 clock_snap->ffcount += delta; 1080 1081 /* Record feed-forward clock leap second adjustment. */ 1082 ffi->leapsec_adjustment = cest.leapsec_total; 1083 if (clock_snap->ffcount > cest.leapsec_next) 1084 ffi->leapsec_adjustment -= cest.leapsec; 1085 1086 /* Record feed-forward clock status and error. */ 1087 clock_snap->ff_info.status = cest.status; 1088 ffcount = clock_snap->ffcount - cest.update_ffcount; 1089 ffclock_convert_delta(ffcount, cest.period, &bt); 1090 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */ 1091 bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL); 1092 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */ 1093 bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL); 1094 clock_snap->ff_info.error = bt; 1095 #endif 1096 } 1097 1098 /* 1099 * Convert a sysclock snapshot into a struct bintime based on the specified 1100 * clock source and flags. 1101 */ 1102 int 1103 sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt, 1104 int whichclock, uint32_t flags) 1105 { 1106 #ifdef FFCLOCK 1107 struct bintime bt2; 1108 uint64_t period; 1109 #endif 1110 1111 switch (whichclock) { 1112 case SYSCLOCK_FBCK: 1113 *bt = cs->fb_info.tick_time; 1114 1115 /* If snapshot was created with !fast, delta will be >0. */ 1116 if (cs->delta > 0) 1117 bintime_addx(bt, cs->fb_info.th_scale * cs->delta); 1118 1119 if ((flags & FBCLOCK_UPTIME) == 0) 1120 bintime_add(bt, &boottimebin); 1121 break; 1122 #ifdef FFCLOCK 1123 case SYSCLOCK_FFWD: 1124 if (flags & FFCLOCK_LERP) { 1125 *bt = cs->ff_info.tick_time_lerp; 1126 period = cs->ff_info.period_lerp; 1127 } else { 1128 *bt = cs->ff_info.tick_time; 1129 period = cs->ff_info.period; 1130 } 1131 1132 /* If snapshot was created with !fast, delta will be >0. */ 1133 if (cs->delta > 0) { 1134 ffclock_convert_delta(cs->delta, period, &bt2); 1135 bintime_add(bt, &bt2); 1136 } 1137 1138 /* Leap second adjustment. */ 1139 if (flags & FFCLOCK_LEAPSEC) 1140 bt->sec -= cs->ff_info.leapsec_adjustment; 1141 1142 /* Boot time adjustment, for uptime/monotonic clocks. */ 1143 if (flags & FFCLOCK_UPTIME) 1144 bintime_sub(bt, &ffclock_boottime); 1145 break; 1146 #endif 1147 default: 1148 return (EINVAL); 1149 break; 1150 } 1151 1152 return (0); 1153 } 1154 1155 /* 1156 * Initialize a new timecounter and possibly use it. 1157 */ 1158 void 1159 tc_init(struct timecounter *tc) 1160 { 1161 u_int u; 1162 struct sysctl_oid *tc_root; 1163 1164 u = tc->tc_frequency / tc->tc_counter_mask; 1165 /* XXX: We need some margin here, 10% is a guess */ 1166 u *= 11; 1167 u /= 10; 1168 if (u > hz && tc->tc_quality >= 0) { 1169 tc->tc_quality = -2000; 1170 if (bootverbose) { 1171 printf("Timecounter \"%s\" frequency %ju Hz", 1172 tc->tc_name, (uintmax_t)tc->tc_frequency); 1173 printf(" -- Insufficient hz, needs at least %u\n", u); 1174 } 1175 } else if (tc->tc_quality >= 0 || bootverbose) { 1176 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n", 1177 tc->tc_name, (uintmax_t)tc->tc_frequency, 1178 tc->tc_quality); 1179 } 1180 1181 tc->tc_next = timecounters; 1182 timecounters = tc; 1183 /* 1184 * Set up sysctl tree for this counter. 1185 */ 1186 tc_root = SYSCTL_ADD_NODE(NULL, 1187 SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, 1188 CTLFLAG_RW, 0, "timecounter description"); 1189 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1190 "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, 1191 "mask for implemented bits"); 1192 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1193 "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc), 1194 sysctl_kern_timecounter_get, "IU", "current timecounter value"); 1195 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1196 "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc), 1197 sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); 1198 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1199 "quality", CTLFLAG_RD, &(tc->tc_quality), 0, 1200 "goodness of time counter"); 1201 /* 1202 * Do not automatically switch if the current tc was specifically 1203 * chosen. Never automatically use a timecounter with negative quality. 1204 * Even though we run on the dummy counter, switching here may be 1205 * worse since this timecounter may not be monotonic. 1206 */ 1207 if (tc_chosen) 1208 return; 1209 if (tc->tc_quality < 0) 1210 return; 1211 if (tc->tc_quality < timecounter->tc_quality) 1212 return; 1213 if (tc->tc_quality == timecounter->tc_quality && 1214 tc->tc_frequency < timecounter->tc_frequency) 1215 return; 1216 (void)tc->tc_get_timecount(tc); 1217 (void)tc->tc_get_timecount(tc); 1218 timecounter = tc; 1219 } 1220 1221 /* Report the frequency of the current timecounter. */ 1222 uint64_t 1223 tc_getfrequency(void) 1224 { 1225 1226 return (timehands->th_counter->tc_frequency); 1227 } 1228 1229 /* 1230 * Step our concept of UTC. This is done by modifying our estimate of 1231 * when we booted. 1232 * XXX: not locked. 1233 */ 1234 void 1235 tc_setclock(struct timespec *ts) 1236 { 1237 struct timespec tbef, taft; 1238 struct bintime bt, bt2; 1239 1240 cpu_tick_calibrate(1); 1241 nanotime(&tbef); 1242 timespec2bintime(ts, &bt); 1243 binuptime(&bt2); 1244 bintime_sub(&bt, &bt2); 1245 bintime_add(&bt2, &boottimebin); 1246 boottimebin = bt; 1247 bintime2timeval(&bt, &boottime); 1248 1249 /* XXX fiddle all the little crinkly bits around the fiords... */ 1250 tc_windup(); 1251 nanotime(&taft); 1252 if (timestepwarnings) { 1253 log(LOG_INFO, 1254 "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n", 1255 (intmax_t)tbef.tv_sec, tbef.tv_nsec, 1256 (intmax_t)taft.tv_sec, taft.tv_nsec, 1257 (intmax_t)ts->tv_sec, ts->tv_nsec); 1258 } 1259 cpu_tick_calibrate(1); 1260 } 1261 1262 /* 1263 * Initialize the next struct timehands in the ring and make 1264 * it the active timehands. Along the way we might switch to a different 1265 * timecounter and/or do seconds processing in NTP. Slightly magic. 1266 */ 1267 static void 1268 tc_windup(void) 1269 { 1270 struct bintime bt; 1271 struct timehands *th, *tho; 1272 uint64_t scale; 1273 u_int delta, ncount, ogen; 1274 int i; 1275 time_t t; 1276 1277 /* 1278 * Make the next timehands a copy of the current one, but do 1279 * not overwrite the generation or next pointer. While we 1280 * update the contents, the generation must be zero. We need 1281 * to ensure that the zero generation is visible before the 1282 * data updates become visible, which requires release fence. 1283 * For similar reasons, re-reading of the generation after the 1284 * data is read should use acquire fence. 1285 */ 1286 tho = timehands; 1287 th = tho->th_next; 1288 ogen = th->th_generation; 1289 th->th_generation = 0; 1290 atomic_thread_fence_rel(); 1291 bcopy(tho, th, offsetof(struct timehands, th_generation)); 1292 1293 /* 1294 * Capture a timecounter delta on the current timecounter and if 1295 * changing timecounters, a counter value from the new timecounter. 1296 * Update the offset fields accordingly. 1297 */ 1298 delta = tc_delta(th); 1299 if (th->th_counter != timecounter) 1300 ncount = timecounter->tc_get_timecount(timecounter); 1301 else 1302 ncount = 0; 1303 #ifdef FFCLOCK 1304 ffclock_windup(delta); 1305 #endif 1306 th->th_offset_count += delta; 1307 th->th_offset_count &= th->th_counter->tc_counter_mask; 1308 while (delta > th->th_counter->tc_frequency) { 1309 /* Eat complete unadjusted seconds. */ 1310 delta -= th->th_counter->tc_frequency; 1311 th->th_offset.sec++; 1312 } 1313 if ((delta > th->th_counter->tc_frequency / 2) && 1314 (th->th_scale * delta < ((uint64_t)1 << 63))) { 1315 /* The product th_scale * delta just barely overflows. */ 1316 th->th_offset.sec++; 1317 } 1318 bintime_addx(&th->th_offset, th->th_scale * delta); 1319 1320 /* 1321 * Hardware latching timecounters may not generate interrupts on 1322 * PPS events, so instead we poll them. There is a finite risk that 1323 * the hardware might capture a count which is later than the one we 1324 * got above, and therefore possibly in the next NTP second which might 1325 * have a different rate than the current NTP second. It doesn't 1326 * matter in practice. 1327 */ 1328 if (tho->th_counter->tc_poll_pps) 1329 tho->th_counter->tc_poll_pps(tho->th_counter); 1330 1331 /* 1332 * Deal with NTP second processing. The for loop normally 1333 * iterates at most once, but in extreme situations it might 1334 * keep NTP sane if timeouts are not run for several seconds. 1335 * At boot, the time step can be large when the TOD hardware 1336 * has been read, so on really large steps, we call 1337 * ntp_update_second only twice. We need to call it twice in 1338 * case we missed a leap second. 1339 */ 1340 bt = th->th_offset; 1341 bintime_add(&bt, &boottimebin); 1342 i = bt.sec - tho->th_microtime.tv_sec; 1343 if (i > LARGE_STEP) 1344 i = 2; 1345 for (; i > 0; i--) { 1346 t = bt.sec; 1347 ntp_update_second(&th->th_adjustment, &bt.sec); 1348 if (bt.sec != t) 1349 boottimebin.sec += bt.sec - t; 1350 } 1351 /* Update the UTC timestamps used by the get*() functions. */ 1352 /* XXX shouldn't do this here. Should force non-`get' versions. */ 1353 bintime2timeval(&bt, &th->th_microtime); 1354 bintime2timespec(&bt, &th->th_nanotime); 1355 1356 /* Now is a good time to change timecounters. */ 1357 if (th->th_counter != timecounter) { 1358 #ifndef __arm__ 1359 if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0) 1360 cpu_disable_c2_sleep++; 1361 if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0) 1362 cpu_disable_c2_sleep--; 1363 #endif 1364 th->th_counter = timecounter; 1365 th->th_offset_count = ncount; 1366 tc_min_ticktock_freq = max(1, timecounter->tc_frequency / 1367 (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); 1368 #ifdef FFCLOCK 1369 ffclock_change_tc(th); 1370 #endif 1371 } 1372 1373 /*- 1374 * Recalculate the scaling factor. We want the number of 1/2^64 1375 * fractions of a second per period of the hardware counter, taking 1376 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 1377 * processing provides us with. 1378 * 1379 * The th_adjustment is nanoseconds per second with 32 bit binary 1380 * fraction and we want 64 bit binary fraction of second: 1381 * 1382 * x = a * 2^32 / 10^9 = a * 4.294967296 1383 * 1384 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 1385 * we can only multiply by about 850 without overflowing, that 1386 * leaves no suitably precise fractions for multiply before divide. 1387 * 1388 * Divide before multiply with a fraction of 2199/512 results in a 1389 * systematic undercompensation of 10PPM of th_adjustment. On a 1390 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 1391 * 1392 * We happily sacrifice the lowest of the 64 bits of our result 1393 * to the goddess of code clarity. 1394 * 1395 */ 1396 scale = (uint64_t)1 << 63; 1397 scale += (th->th_adjustment / 1024) * 2199; 1398 scale /= th->th_counter->tc_frequency; 1399 th->th_scale = scale * 2; 1400 1401 /* 1402 * Now that the struct timehands is again consistent, set the new 1403 * generation number, making sure to not make it zero. 1404 */ 1405 if (++ogen == 0) 1406 ogen = 1; 1407 atomic_store_rel_int(&th->th_generation, ogen); 1408 1409 /* Go live with the new struct timehands. */ 1410 #ifdef FFCLOCK 1411 switch (sysclock_active) { 1412 case SYSCLOCK_FBCK: 1413 #endif 1414 time_second = th->th_microtime.tv_sec; 1415 time_uptime = th->th_offset.sec; 1416 #ifdef FFCLOCK 1417 break; 1418 case SYSCLOCK_FFWD: 1419 time_second = fftimehands->tick_time_lerp.sec; 1420 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec; 1421 break; 1422 } 1423 #endif 1424 1425 timehands = th; 1426 timekeep_push_vdso(); 1427 } 1428 1429 /* Report or change the active timecounter hardware. */ 1430 static int 1431 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) 1432 { 1433 char newname[32]; 1434 struct timecounter *newtc, *tc; 1435 int error; 1436 1437 tc = timecounter; 1438 strlcpy(newname, tc->tc_name, sizeof(newname)); 1439 1440 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); 1441 if (error != 0 || req->newptr == NULL) 1442 return (error); 1443 /* Record that the tc in use now was specifically chosen. */ 1444 tc_chosen = 1; 1445 if (strcmp(newname, tc->tc_name) == 0) 1446 return (0); 1447 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 1448 if (strcmp(newname, newtc->tc_name) != 0) 1449 continue; 1450 1451 /* Warm up new timecounter. */ 1452 (void)newtc->tc_get_timecount(newtc); 1453 (void)newtc->tc_get_timecount(newtc); 1454 1455 timecounter = newtc; 1456 1457 /* 1458 * The vdso timehands update is deferred until the next 1459 * 'tc_windup()'. 1460 * 1461 * This is prudent given that 'timekeep_push_vdso()' does not 1462 * use any locking and that it can be called in hard interrupt 1463 * context via 'tc_windup()'. 1464 */ 1465 return (0); 1466 } 1467 return (EINVAL); 1468 } 1469 1470 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, 1471 0, 0, sysctl_kern_timecounter_hardware, "A", 1472 "Timecounter hardware selected"); 1473 1474 1475 /* Report the available timecounter hardware. */ 1476 static int 1477 sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) 1478 { 1479 struct sbuf sb; 1480 struct timecounter *tc; 1481 int error; 1482 1483 sbuf_new_for_sysctl(&sb, NULL, 0, req); 1484 for (tc = timecounters; tc != NULL; tc = tc->tc_next) { 1485 if (tc != timecounters) 1486 sbuf_putc(&sb, ' '); 1487 sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality); 1488 } 1489 error = sbuf_finish(&sb); 1490 sbuf_delete(&sb); 1491 return (error); 1492 } 1493 1494 SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, 1495 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); 1496 1497 /* 1498 * RFC 2783 PPS-API implementation. 1499 */ 1500 1501 /* 1502 * Return true if the driver is aware of the abi version extensions in the 1503 * pps_state structure, and it supports at least the given abi version number. 1504 */ 1505 static inline int 1506 abi_aware(struct pps_state *pps, int vers) 1507 { 1508 1509 return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers); 1510 } 1511 1512 static int 1513 pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps) 1514 { 1515 int err, timo; 1516 pps_seq_t aseq, cseq; 1517 struct timeval tv; 1518 1519 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) 1520 return (EINVAL); 1521 1522 /* 1523 * If no timeout is requested, immediately return whatever values were 1524 * most recently captured. If timeout seconds is -1, that's a request 1525 * to block without a timeout. WITNESS won't let us sleep forever 1526 * without a lock (we really don't need a lock), so just repeatedly 1527 * sleep a long time. 1528 */ 1529 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) { 1530 if (fapi->timeout.tv_sec == -1) 1531 timo = 0x7fffffff; 1532 else { 1533 tv.tv_sec = fapi->timeout.tv_sec; 1534 tv.tv_usec = fapi->timeout.tv_nsec / 1000; 1535 timo = tvtohz(&tv); 1536 } 1537 aseq = pps->ppsinfo.assert_sequence; 1538 cseq = pps->ppsinfo.clear_sequence; 1539 while (aseq == pps->ppsinfo.assert_sequence && 1540 cseq == pps->ppsinfo.clear_sequence) { 1541 if (abi_aware(pps, 1) && pps->driver_mtx != NULL) { 1542 if (pps->flags & PPSFLAG_MTX_SPIN) { 1543 err = msleep_spin(pps, pps->driver_mtx, 1544 "ppsfch", timo); 1545 } else { 1546 err = msleep(pps, pps->driver_mtx, PCATCH, 1547 "ppsfch", timo); 1548 } 1549 } else { 1550 err = tsleep(pps, PCATCH, "ppsfch", timo); 1551 } 1552 if (err == EWOULDBLOCK) { 1553 if (fapi->timeout.tv_sec == -1) { 1554 continue; 1555 } else { 1556 return (ETIMEDOUT); 1557 } 1558 } else if (err != 0) { 1559 return (err); 1560 } 1561 } 1562 } 1563 1564 pps->ppsinfo.current_mode = pps->ppsparam.mode; 1565 fapi->pps_info_buf = pps->ppsinfo; 1566 1567 return (0); 1568 } 1569 1570 int 1571 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) 1572 { 1573 pps_params_t *app; 1574 struct pps_fetch_args *fapi; 1575 #ifdef FFCLOCK 1576 struct pps_fetch_ffc_args *fapi_ffc; 1577 #endif 1578 #ifdef PPS_SYNC 1579 struct pps_kcbind_args *kapi; 1580 #endif 1581 1582 KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl")); 1583 switch (cmd) { 1584 case PPS_IOC_CREATE: 1585 return (0); 1586 case PPS_IOC_DESTROY: 1587 return (0); 1588 case PPS_IOC_SETPARAMS: 1589 app = (pps_params_t *)data; 1590 if (app->mode & ~pps->ppscap) 1591 return (EINVAL); 1592 #ifdef FFCLOCK 1593 /* Ensure only a single clock is selected for ffc timestamp. */ 1594 if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK) 1595 return (EINVAL); 1596 #endif 1597 pps->ppsparam = *app; 1598 return (0); 1599 case PPS_IOC_GETPARAMS: 1600 app = (pps_params_t *)data; 1601 *app = pps->ppsparam; 1602 app->api_version = PPS_API_VERS_1; 1603 return (0); 1604 case PPS_IOC_GETCAP: 1605 *(int*)data = pps->ppscap; 1606 return (0); 1607 case PPS_IOC_FETCH: 1608 fapi = (struct pps_fetch_args *)data; 1609 return (pps_fetch(fapi, pps)); 1610 #ifdef FFCLOCK 1611 case PPS_IOC_FETCH_FFCOUNTER: 1612 fapi_ffc = (struct pps_fetch_ffc_args *)data; 1613 if (fapi_ffc->tsformat && fapi_ffc->tsformat != 1614 PPS_TSFMT_TSPEC) 1615 return (EINVAL); 1616 if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec) 1617 return (EOPNOTSUPP); 1618 pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode; 1619 fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc; 1620 /* Overwrite timestamps if feedback clock selected. */ 1621 switch (pps->ppsparam.mode & PPS_TSCLK_MASK) { 1622 case PPS_TSCLK_FBCK: 1623 fapi_ffc->pps_info_buf_ffc.assert_timestamp = 1624 pps->ppsinfo.assert_timestamp; 1625 fapi_ffc->pps_info_buf_ffc.clear_timestamp = 1626 pps->ppsinfo.clear_timestamp; 1627 break; 1628 case PPS_TSCLK_FFWD: 1629 break; 1630 default: 1631 break; 1632 } 1633 return (0); 1634 #endif /* FFCLOCK */ 1635 case PPS_IOC_KCBIND: 1636 #ifdef PPS_SYNC 1637 kapi = (struct pps_kcbind_args *)data; 1638 /* XXX Only root should be able to do this */ 1639 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) 1640 return (EINVAL); 1641 if (kapi->kernel_consumer != PPS_KC_HARDPPS) 1642 return (EINVAL); 1643 if (kapi->edge & ~pps->ppscap) 1644 return (EINVAL); 1645 pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) | 1646 (pps->kcmode & KCMODE_ABIFLAG); 1647 return (0); 1648 #else 1649 return (EOPNOTSUPP); 1650 #endif 1651 default: 1652 return (ENOIOCTL); 1653 } 1654 } 1655 1656 void 1657 pps_init(struct pps_state *pps) 1658 { 1659 pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT; 1660 if (pps->ppscap & PPS_CAPTUREASSERT) 1661 pps->ppscap |= PPS_OFFSETASSERT; 1662 if (pps->ppscap & PPS_CAPTURECLEAR) 1663 pps->ppscap |= PPS_OFFSETCLEAR; 1664 #ifdef FFCLOCK 1665 pps->ppscap |= PPS_TSCLK_MASK; 1666 #endif 1667 pps->kcmode &= ~KCMODE_ABIFLAG; 1668 } 1669 1670 void 1671 pps_init_abi(struct pps_state *pps) 1672 { 1673 1674 pps_init(pps); 1675 if (pps->driver_abi > 0) { 1676 pps->kcmode |= KCMODE_ABIFLAG; 1677 pps->kernel_abi = PPS_ABI_VERSION; 1678 } 1679 } 1680 1681 void 1682 pps_capture(struct pps_state *pps) 1683 { 1684 struct timehands *th; 1685 1686 KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); 1687 th = timehands; 1688 pps->capgen = atomic_load_acq_int(&th->th_generation); 1689 pps->capth = th; 1690 #ifdef FFCLOCK 1691 pps->capffth = fftimehands; 1692 #endif 1693 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); 1694 atomic_thread_fence_acq(); 1695 if (pps->capgen != th->th_generation) 1696 pps->capgen = 0; 1697 } 1698 1699 void 1700 pps_event(struct pps_state *pps, int event) 1701 { 1702 struct bintime bt; 1703 struct timespec ts, *tsp, *osp; 1704 u_int tcount, *pcount; 1705 int foff, fhard; 1706 pps_seq_t *pseq; 1707 #ifdef FFCLOCK 1708 struct timespec *tsp_ffc; 1709 pps_seq_t *pseq_ffc; 1710 ffcounter *ffcount; 1711 #endif 1712 1713 KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); 1714 /* Nothing to do if not currently set to capture this event type. */ 1715 if ((event & pps->ppsparam.mode) == 0) 1716 return; 1717 /* If the timecounter was wound up underneath us, bail out. */ 1718 if (pps->capgen == 0 || pps->capgen != 1719 atomic_load_acq_int(&pps->capth->th_generation)) 1720 return; 1721 1722 /* Things would be easier with arrays. */ 1723 if (event == PPS_CAPTUREASSERT) { 1724 tsp = &pps->ppsinfo.assert_timestamp; 1725 osp = &pps->ppsparam.assert_offset; 1726 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 1727 fhard = pps->kcmode & PPS_CAPTUREASSERT; 1728 pcount = &pps->ppscount[0]; 1729 pseq = &pps->ppsinfo.assert_sequence; 1730 #ifdef FFCLOCK 1731 ffcount = &pps->ppsinfo_ffc.assert_ffcount; 1732 tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp; 1733 pseq_ffc = &pps->ppsinfo_ffc.assert_sequence; 1734 #endif 1735 } else { 1736 tsp = &pps->ppsinfo.clear_timestamp; 1737 osp = &pps->ppsparam.clear_offset; 1738 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 1739 fhard = pps->kcmode & PPS_CAPTURECLEAR; 1740 pcount = &pps->ppscount[1]; 1741 pseq = &pps->ppsinfo.clear_sequence; 1742 #ifdef FFCLOCK 1743 ffcount = &pps->ppsinfo_ffc.clear_ffcount; 1744 tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp; 1745 pseq_ffc = &pps->ppsinfo_ffc.clear_sequence; 1746 #endif 1747 } 1748 1749 /* 1750 * If the timecounter changed, we cannot compare the count values, so 1751 * we have to drop the rest of the PPS-stuff until the next event. 1752 */ 1753 if (pps->ppstc != pps->capth->th_counter) { 1754 pps->ppstc = pps->capth->th_counter; 1755 *pcount = pps->capcount; 1756 pps->ppscount[2] = pps->capcount; 1757 return; 1758 } 1759 1760 /* Convert the count to a timespec. */ 1761 tcount = pps->capcount - pps->capth->th_offset_count; 1762 tcount &= pps->capth->th_counter->tc_counter_mask; 1763 bt = pps->capth->th_offset; 1764 bintime_addx(&bt, pps->capth->th_scale * tcount); 1765 bintime_add(&bt, &boottimebin); 1766 bintime2timespec(&bt, &ts); 1767 1768 /* If the timecounter was wound up underneath us, bail out. */ 1769 atomic_thread_fence_acq(); 1770 if (pps->capgen != pps->capth->th_generation) 1771 return; 1772 1773 *pcount = pps->capcount; 1774 (*pseq)++; 1775 *tsp = ts; 1776 1777 if (foff) { 1778 timespecadd(tsp, osp); 1779 if (tsp->tv_nsec < 0) { 1780 tsp->tv_nsec += 1000000000; 1781 tsp->tv_sec -= 1; 1782 } 1783 } 1784 1785 #ifdef FFCLOCK 1786 *ffcount = pps->capffth->tick_ffcount + tcount; 1787 bt = pps->capffth->tick_time; 1788 ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt); 1789 bintime_add(&bt, &pps->capffth->tick_time); 1790 bintime2timespec(&bt, &ts); 1791 (*pseq_ffc)++; 1792 *tsp_ffc = ts; 1793 #endif 1794 1795 #ifdef PPS_SYNC 1796 if (fhard) { 1797 uint64_t scale; 1798 1799 /* 1800 * Feed the NTP PLL/FLL. 1801 * The FLL wants to know how many (hardware) nanoseconds 1802 * elapsed since the previous event. 1803 */ 1804 tcount = pps->capcount - pps->ppscount[2]; 1805 pps->ppscount[2] = pps->capcount; 1806 tcount &= pps->capth->th_counter->tc_counter_mask; 1807 scale = (uint64_t)1 << 63; 1808 scale /= pps->capth->th_counter->tc_frequency; 1809 scale *= 2; 1810 bt.sec = 0; 1811 bt.frac = 0; 1812 bintime_addx(&bt, scale * tcount); 1813 bintime2timespec(&bt, &ts); 1814 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); 1815 } 1816 #endif 1817 1818 /* Wakeup anyone sleeping in pps_fetch(). */ 1819 wakeup(pps); 1820 } 1821 1822 /* 1823 * Timecounters need to be updated every so often to prevent the hardware 1824 * counter from overflowing. Updating also recalculates the cached values 1825 * used by the get*() family of functions, so their precision depends on 1826 * the update frequency. 1827 */ 1828 1829 static int tc_tick; 1830 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0, 1831 "Approximate number of hardclock ticks in a millisecond"); 1832 1833 void 1834 tc_ticktock(int cnt) 1835 { 1836 static int count; 1837 1838 count += cnt; 1839 if (count < tc_tick) 1840 return; 1841 count = 0; 1842 tc_windup(); 1843 } 1844 1845 static void __inline 1846 tc_adjprecision(void) 1847 { 1848 int t; 1849 1850 if (tc_timepercentage > 0) { 1851 t = (99 + tc_timepercentage) / tc_timepercentage; 1852 tc_precexp = fls(t + (t >> 1)) - 1; 1853 FREQ2BT(hz / tc_tick, &bt_timethreshold); 1854 FREQ2BT(hz, &bt_tickthreshold); 1855 bintime_shift(&bt_timethreshold, tc_precexp); 1856 bintime_shift(&bt_tickthreshold, tc_precexp); 1857 } else { 1858 tc_precexp = 31; 1859 bt_timethreshold.sec = INT_MAX; 1860 bt_timethreshold.frac = ~(uint64_t)0; 1861 bt_tickthreshold = bt_timethreshold; 1862 } 1863 sbt_timethreshold = bttosbt(bt_timethreshold); 1864 sbt_tickthreshold = bttosbt(bt_tickthreshold); 1865 } 1866 1867 static int 1868 sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS) 1869 { 1870 int error, val; 1871 1872 val = tc_timepercentage; 1873 error = sysctl_handle_int(oidp, &val, 0, req); 1874 if (error != 0 || req->newptr == NULL) 1875 return (error); 1876 tc_timepercentage = val; 1877 if (cold) 1878 goto done; 1879 tc_adjprecision(); 1880 done: 1881 return (0); 1882 } 1883 1884 static void 1885 inittimecounter(void *dummy) 1886 { 1887 u_int p; 1888 int tick_rate; 1889 1890 /* 1891 * Set the initial timeout to 1892 * max(1, <approx. number of hardclock ticks in a millisecond>). 1893 * People should probably not use the sysctl to set the timeout 1894 * to smaller than its inital value, since that value is the 1895 * smallest reasonable one. If they want better timestamps they 1896 * should use the non-"get"* functions. 1897 */ 1898 if (hz > 1000) 1899 tc_tick = (hz + 500) / 1000; 1900 else 1901 tc_tick = 1; 1902 tc_adjprecision(); 1903 FREQ2BT(hz, &tick_bt); 1904 tick_sbt = bttosbt(tick_bt); 1905 tick_rate = hz / tc_tick; 1906 FREQ2BT(tick_rate, &tc_tick_bt); 1907 tc_tick_sbt = bttosbt(tc_tick_bt); 1908 p = (tc_tick * 1000000) / hz; 1909 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); 1910 1911 #ifdef FFCLOCK 1912 ffclock_init(); 1913 #endif 1914 /* warm up new timecounter (again) and get rolling. */ 1915 (void)timecounter->tc_get_timecount(timecounter); 1916 (void)timecounter->tc_get_timecount(timecounter); 1917 tc_windup(); 1918 } 1919 1920 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); 1921 1922 /* Cpu tick handling -------------------------------------------------*/ 1923 1924 static int cpu_tick_variable; 1925 static uint64_t cpu_tick_frequency; 1926 1927 static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base); 1928 static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last); 1929 1930 static uint64_t 1931 tc_cpu_ticks(void) 1932 { 1933 struct timecounter *tc; 1934 uint64_t res, *base; 1935 unsigned u, *last; 1936 1937 critical_enter(); 1938 base = DPCPU_PTR(tc_cpu_ticks_base); 1939 last = DPCPU_PTR(tc_cpu_ticks_last); 1940 tc = timehands->th_counter; 1941 u = tc->tc_get_timecount(tc) & tc->tc_counter_mask; 1942 if (u < *last) 1943 *base += (uint64_t)tc->tc_counter_mask + 1; 1944 *last = u; 1945 res = u + *base; 1946 critical_exit(); 1947 return (res); 1948 } 1949 1950 void 1951 cpu_tick_calibration(void) 1952 { 1953 static time_t last_calib; 1954 1955 if (time_uptime != last_calib && !(time_uptime & 0xf)) { 1956 cpu_tick_calibrate(0); 1957 last_calib = time_uptime; 1958 } 1959 } 1960 1961 /* 1962 * This function gets called every 16 seconds on only one designated 1963 * CPU in the system from hardclock() via cpu_tick_calibration()(). 1964 * 1965 * Whenever the real time clock is stepped we get called with reset=1 1966 * to make sure we handle suspend/resume and similar events correctly. 1967 */ 1968 1969 static void 1970 cpu_tick_calibrate(int reset) 1971 { 1972 static uint64_t c_last; 1973 uint64_t c_this, c_delta; 1974 static struct bintime t_last; 1975 struct bintime t_this, t_delta; 1976 uint32_t divi; 1977 1978 if (reset) { 1979 /* The clock was stepped, abort & reset */ 1980 t_last.sec = 0; 1981 return; 1982 } 1983 1984 /* we don't calibrate fixed rate cputicks */ 1985 if (!cpu_tick_variable) 1986 return; 1987 1988 getbinuptime(&t_this); 1989 c_this = cpu_ticks(); 1990 if (t_last.sec != 0) { 1991 c_delta = c_this - c_last; 1992 t_delta = t_this; 1993 bintime_sub(&t_delta, &t_last); 1994 /* 1995 * Headroom: 1996 * 2^(64-20) / 16[s] = 1997 * 2^(44) / 16[s] = 1998 * 17.592.186.044.416 / 16 = 1999 * 1.099.511.627.776 [Hz] 2000 */ 2001 divi = t_delta.sec << 20; 2002 divi |= t_delta.frac >> (64 - 20); 2003 c_delta <<= 20; 2004 c_delta /= divi; 2005 if (c_delta > cpu_tick_frequency) { 2006 if (0 && bootverbose) 2007 printf("cpu_tick increased to %ju Hz\n", 2008 c_delta); 2009 cpu_tick_frequency = c_delta; 2010 } 2011 } 2012 c_last = c_this; 2013 t_last = t_this; 2014 } 2015 2016 void 2017 set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var) 2018 { 2019 2020 if (func == NULL) { 2021 cpu_ticks = tc_cpu_ticks; 2022 } else { 2023 cpu_tick_frequency = freq; 2024 cpu_tick_variable = var; 2025 cpu_ticks = func; 2026 } 2027 } 2028 2029 uint64_t 2030 cpu_tickrate(void) 2031 { 2032 2033 if (cpu_ticks == tc_cpu_ticks) 2034 return (tc_getfrequency()); 2035 return (cpu_tick_frequency); 2036 } 2037 2038 /* 2039 * We need to be slightly careful converting cputicks to microseconds. 2040 * There is plenty of margin in 64 bits of microseconds (half a million 2041 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply 2042 * before divide conversion (to retain precision) we find that the 2043 * margin shrinks to 1.5 hours (one millionth of 146y). 2044 * With a three prong approach we never lose significant bits, no 2045 * matter what the cputick rate and length of timeinterval is. 2046 */ 2047 2048 uint64_t 2049 cputick2usec(uint64_t tick) 2050 { 2051 2052 if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */ 2053 return (tick / (cpu_tickrate() / 1000000LL)); 2054 else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */ 2055 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL)); 2056 else 2057 return ((tick * 1000000LL) / cpu_tickrate()); 2058 } 2059 2060 cpu_tick_f *cpu_ticks = tc_cpu_ticks; 2061 2062 static int vdso_th_enable = 1; 2063 static int 2064 sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) 2065 { 2066 int old_vdso_th_enable, error; 2067 2068 old_vdso_th_enable = vdso_th_enable; 2069 error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); 2070 if (error != 0) 2071 return (error); 2072 vdso_th_enable = old_vdso_th_enable; 2073 return (0); 2074 } 2075 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, 2076 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 2077 NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); 2078 2079 uint32_t 2080 tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) 2081 { 2082 struct timehands *th; 2083 uint32_t enabled; 2084 2085 th = timehands; 2086 vdso_th->th_algo = VDSO_TH_ALGO_1; 2087 vdso_th->th_scale = th->th_scale; 2088 vdso_th->th_offset_count = th->th_offset_count; 2089 vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; 2090 vdso_th->th_offset = th->th_offset; 2091 vdso_th->th_boottime = boottimebin; 2092 enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter); 2093 if (!vdso_th_enable) 2094 enabled = 0; 2095 return (enabled); 2096 } 2097 2098 #ifdef COMPAT_FREEBSD32 2099 uint32_t 2100 tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) 2101 { 2102 struct timehands *th; 2103 uint32_t enabled; 2104 2105 th = timehands; 2106 vdso_th32->th_algo = VDSO_TH_ALGO_1; 2107 *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; 2108 vdso_th32->th_offset_count = th->th_offset_count; 2109 vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; 2110 vdso_th32->th_offset.sec = th->th_offset.sec; 2111 *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; 2112 vdso_th32->th_boottime.sec = boottimebin.sec; 2113 *(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac; 2114 enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter); 2115 if (!vdso_th_enable) 2116 enabled = 0; 2117 return (enabled); 2118 } 2119 #endif 2120