1 /*- 2 * SPDX-License-Identifier: Beerware 3 * 4 * ---------------------------------------------------------------------------- 5 * "THE BEER-WARE LICENSE" (Revision 42): 6 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 7 * can do whatever you want with this stuff. If we meet some day, and you think 8 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 9 * ---------------------------------------------------------------------------- 10 * 11 * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation 12 * All rights reserved. 13 * 14 * Portions of this software were developed by Julien Ridoux at the University 15 * of Melbourne under sponsorship from the FreeBSD Foundation. 16 * 17 * Portions of this software were developed by Konstantin Belousov 18 * under sponsorship from the FreeBSD Foundation. 19 */ 20 21 #include <sys/cdefs.h> 22 __FBSDID("$FreeBSD$"); 23 24 #include "opt_ntp.h" 25 #include "opt_ffclock.h" 26 27 #include <sys/param.h> 28 #include <sys/kernel.h> 29 #include <sys/limits.h> 30 #include <sys/lock.h> 31 #include <sys/mutex.h> 32 #include <sys/proc.h> 33 #include <sys/sbuf.h> 34 #include <sys/sleepqueue.h> 35 #include <sys/sysctl.h> 36 #include <sys/syslog.h> 37 #include <sys/systm.h> 38 #include <sys/timeffc.h> 39 #include <sys/timepps.h> 40 #include <sys/timetc.h> 41 #include <sys/timex.h> 42 #include <sys/vdso.h> 43 44 /* 45 * A large step happens on boot. This constant detects such steps. 46 * It is relatively small so that ntp_update_second gets called enough 47 * in the typical 'missed a couple of seconds' case, but doesn't loop 48 * forever when the time step is large. 49 */ 50 #define LARGE_STEP 200 51 52 /* 53 * Implement a dummy timecounter which we can use until we get a real one 54 * in the air. This allows the console and other early stuff to use 55 * time services. 56 */ 57 58 static u_int 59 dummy_get_timecount(struct timecounter *tc) 60 { 61 static u_int now; 62 63 return (++now); 64 } 65 66 static struct timecounter dummy_timecounter = { 67 dummy_get_timecount, 0, ~0u, 1000000, "dummy", -1000000 68 }; 69 70 struct timehands { 71 /* These fields must be initialized by the driver. */ 72 struct timecounter *th_counter; 73 int64_t th_adjustment; 74 uint64_t th_scale; 75 u_int th_offset_count; 76 struct bintime th_offset; 77 struct bintime th_bintime; 78 struct timeval th_microtime; 79 struct timespec th_nanotime; 80 struct bintime th_boottime; 81 /* Fields not to be copied in tc_windup start with th_generation. */ 82 u_int th_generation; 83 struct timehands *th_next; 84 }; 85 86 static struct timehands ths[16] = { 87 [0] = { 88 .th_counter = &dummy_timecounter, 89 .th_scale = (uint64_t)-1 / 1000000, 90 .th_offset = { .sec = 1 }, 91 .th_generation = 1, 92 }, 93 }; 94 95 static struct timehands *volatile timehands = &ths[0]; 96 struct timecounter *timecounter = &dummy_timecounter; 97 static struct timecounter *timecounters = &dummy_timecounter; 98 99 int tc_min_ticktock_freq = 1; 100 101 volatile time_t time_second = 1; 102 volatile time_t time_uptime = 1; 103 104 static int sysctl_kern_boottime(SYSCTL_HANDLER_ARGS); 105 SYSCTL_PROC(_kern, KERN_BOOTTIME, boottime, CTLTYPE_STRUCT|CTLFLAG_RD, 106 NULL, 0, sysctl_kern_boottime, "S,timeval", "System boottime"); 107 108 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); 109 static SYSCTL_NODE(_kern_timecounter, OID_AUTO, tc, CTLFLAG_RW, 0, ""); 110 111 static int timestepwarnings; 112 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW, 113 ×tepwarnings, 0, "Log time steps"); 114 115 static int timehands_count = 2; 116 SYSCTL_INT(_kern_timecounter, OID_AUTO, timehands_count, 117 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, 118 &timehands_count, 0, "Count of timehands in rotation"); 119 120 struct bintime bt_timethreshold; 121 struct bintime bt_tickthreshold; 122 sbintime_t sbt_timethreshold; 123 sbintime_t sbt_tickthreshold; 124 struct bintime tc_tick_bt; 125 sbintime_t tc_tick_sbt; 126 int tc_precexp; 127 int tc_timepercentage = TC_DEFAULTPERC; 128 static int sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS); 129 SYSCTL_PROC(_kern_timecounter, OID_AUTO, alloweddeviation, 130 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 0, 0, 131 sysctl_kern_timecounter_adjprecision, "I", 132 "Allowed time interval deviation in percents"); 133 134 volatile int rtc_generation = 1; 135 136 static int tc_chosen; /* Non-zero if a specific tc was chosen via sysctl. */ 137 138 static void tc_windup(struct bintime *new_boottimebin); 139 static void cpu_tick_calibrate(int); 140 141 void dtrace_getnanotime(struct timespec *tsp); 142 143 static int 144 sysctl_kern_boottime(SYSCTL_HANDLER_ARGS) 145 { 146 struct timeval boottime; 147 148 getboottime(&boottime); 149 150 /* i386 is the only arch which uses a 32bits time_t */ 151 #ifdef __amd64__ 152 #ifdef SCTL_MASK32 153 int tv[2]; 154 155 if (req->flags & SCTL_MASK32) { 156 tv[0] = boottime.tv_sec; 157 tv[1] = boottime.tv_usec; 158 return (SYSCTL_OUT(req, tv, sizeof(tv))); 159 } 160 #endif 161 #endif 162 return (SYSCTL_OUT(req, &boottime, sizeof(boottime))); 163 } 164 165 static int 166 sysctl_kern_timecounter_get(SYSCTL_HANDLER_ARGS) 167 { 168 u_int ncount; 169 struct timecounter *tc = arg1; 170 171 ncount = tc->tc_get_timecount(tc); 172 return (sysctl_handle_int(oidp, &ncount, 0, req)); 173 } 174 175 static int 176 sysctl_kern_timecounter_freq(SYSCTL_HANDLER_ARGS) 177 { 178 uint64_t freq; 179 struct timecounter *tc = arg1; 180 181 freq = tc->tc_frequency; 182 return (sysctl_handle_64(oidp, &freq, 0, req)); 183 } 184 185 /* 186 * Return the difference between the timehands' counter value now and what 187 * was when we copied it to the timehands' offset_count. 188 */ 189 static __inline u_int 190 tc_delta(struct timehands *th) 191 { 192 struct timecounter *tc; 193 194 tc = th->th_counter; 195 return ((tc->tc_get_timecount(tc) - th->th_offset_count) & 196 tc->tc_counter_mask); 197 } 198 199 /* 200 * Functions for reading the time. We have to loop until we are sure that 201 * the timehands that we operated on was not updated under our feet. See 202 * the comment in <sys/time.h> for a description of these 12 functions. 203 */ 204 205 #ifdef FFCLOCK 206 void 207 fbclock_binuptime(struct bintime *bt) 208 { 209 struct timehands *th; 210 unsigned int gen; 211 212 do { 213 th = timehands; 214 gen = atomic_load_acq_int(&th->th_generation); 215 *bt = th->th_offset; 216 bintime_addx(bt, th->th_scale * tc_delta(th)); 217 atomic_thread_fence_acq(); 218 } while (gen == 0 || gen != th->th_generation); 219 } 220 221 void 222 fbclock_nanouptime(struct timespec *tsp) 223 { 224 struct bintime bt; 225 226 fbclock_binuptime(&bt); 227 bintime2timespec(&bt, tsp); 228 } 229 230 void 231 fbclock_microuptime(struct timeval *tvp) 232 { 233 struct bintime bt; 234 235 fbclock_binuptime(&bt); 236 bintime2timeval(&bt, tvp); 237 } 238 239 void 240 fbclock_bintime(struct bintime *bt) 241 { 242 struct timehands *th; 243 unsigned int gen; 244 245 do { 246 th = timehands; 247 gen = atomic_load_acq_int(&th->th_generation); 248 *bt = th->th_bintime; 249 bintime_addx(bt, th->th_scale * tc_delta(th)); 250 atomic_thread_fence_acq(); 251 } while (gen == 0 || gen != th->th_generation); 252 } 253 254 void 255 fbclock_nanotime(struct timespec *tsp) 256 { 257 struct bintime bt; 258 259 fbclock_bintime(&bt); 260 bintime2timespec(&bt, tsp); 261 } 262 263 void 264 fbclock_microtime(struct timeval *tvp) 265 { 266 struct bintime bt; 267 268 fbclock_bintime(&bt); 269 bintime2timeval(&bt, tvp); 270 } 271 272 void 273 fbclock_getbinuptime(struct bintime *bt) 274 { 275 struct timehands *th; 276 unsigned int gen; 277 278 do { 279 th = timehands; 280 gen = atomic_load_acq_int(&th->th_generation); 281 *bt = th->th_offset; 282 atomic_thread_fence_acq(); 283 } while (gen == 0 || gen != th->th_generation); 284 } 285 286 void 287 fbclock_getnanouptime(struct timespec *tsp) 288 { 289 struct timehands *th; 290 unsigned int gen; 291 292 do { 293 th = timehands; 294 gen = atomic_load_acq_int(&th->th_generation); 295 bintime2timespec(&th->th_offset, tsp); 296 atomic_thread_fence_acq(); 297 } while (gen == 0 || gen != th->th_generation); 298 } 299 300 void 301 fbclock_getmicrouptime(struct timeval *tvp) 302 { 303 struct timehands *th; 304 unsigned int gen; 305 306 do { 307 th = timehands; 308 gen = atomic_load_acq_int(&th->th_generation); 309 bintime2timeval(&th->th_offset, tvp); 310 atomic_thread_fence_acq(); 311 } while (gen == 0 || gen != th->th_generation); 312 } 313 314 void 315 fbclock_getbintime(struct bintime *bt) 316 { 317 struct timehands *th; 318 unsigned int gen; 319 320 do { 321 th = timehands; 322 gen = atomic_load_acq_int(&th->th_generation); 323 *bt = th->th_bintime; 324 atomic_thread_fence_acq(); 325 } while (gen == 0 || gen != th->th_generation); 326 } 327 328 void 329 fbclock_getnanotime(struct timespec *tsp) 330 { 331 struct timehands *th; 332 unsigned int gen; 333 334 do { 335 th = timehands; 336 gen = atomic_load_acq_int(&th->th_generation); 337 *tsp = th->th_nanotime; 338 atomic_thread_fence_acq(); 339 } while (gen == 0 || gen != th->th_generation); 340 } 341 342 void 343 fbclock_getmicrotime(struct timeval *tvp) 344 { 345 struct timehands *th; 346 unsigned int gen; 347 348 do { 349 th = timehands; 350 gen = atomic_load_acq_int(&th->th_generation); 351 *tvp = th->th_microtime; 352 atomic_thread_fence_acq(); 353 } while (gen == 0 || gen != th->th_generation); 354 } 355 #else /* !FFCLOCK */ 356 void 357 binuptime(struct bintime *bt) 358 { 359 struct timehands *th; 360 u_int gen; 361 362 do { 363 th = timehands; 364 gen = atomic_load_acq_int(&th->th_generation); 365 *bt = th->th_offset; 366 bintime_addx(bt, th->th_scale * tc_delta(th)); 367 atomic_thread_fence_acq(); 368 } while (gen == 0 || gen != th->th_generation); 369 } 370 371 void 372 nanouptime(struct timespec *tsp) 373 { 374 struct bintime bt; 375 376 binuptime(&bt); 377 bintime2timespec(&bt, tsp); 378 } 379 380 void 381 microuptime(struct timeval *tvp) 382 { 383 struct bintime bt; 384 385 binuptime(&bt); 386 bintime2timeval(&bt, tvp); 387 } 388 389 void 390 bintime(struct bintime *bt) 391 { 392 struct timehands *th; 393 u_int gen; 394 395 do { 396 th = timehands; 397 gen = atomic_load_acq_int(&th->th_generation); 398 *bt = th->th_bintime; 399 bintime_addx(bt, th->th_scale * tc_delta(th)); 400 atomic_thread_fence_acq(); 401 } while (gen == 0 || gen != th->th_generation); 402 } 403 404 void 405 nanotime(struct timespec *tsp) 406 { 407 struct bintime bt; 408 409 bintime(&bt); 410 bintime2timespec(&bt, tsp); 411 } 412 413 void 414 microtime(struct timeval *tvp) 415 { 416 struct bintime bt; 417 418 bintime(&bt); 419 bintime2timeval(&bt, tvp); 420 } 421 422 void 423 getbinuptime(struct bintime *bt) 424 { 425 struct timehands *th; 426 u_int gen; 427 428 do { 429 th = timehands; 430 gen = atomic_load_acq_int(&th->th_generation); 431 *bt = th->th_offset; 432 atomic_thread_fence_acq(); 433 } while (gen == 0 || gen != th->th_generation); 434 } 435 436 void 437 getnanouptime(struct timespec *tsp) 438 { 439 struct timehands *th; 440 u_int gen; 441 442 do { 443 th = timehands; 444 gen = atomic_load_acq_int(&th->th_generation); 445 bintime2timespec(&th->th_offset, tsp); 446 atomic_thread_fence_acq(); 447 } while (gen == 0 || gen != th->th_generation); 448 } 449 450 void 451 getmicrouptime(struct timeval *tvp) 452 { 453 struct timehands *th; 454 u_int gen; 455 456 do { 457 th = timehands; 458 gen = atomic_load_acq_int(&th->th_generation); 459 bintime2timeval(&th->th_offset, tvp); 460 atomic_thread_fence_acq(); 461 } while (gen == 0 || gen != th->th_generation); 462 } 463 464 void 465 getbintime(struct bintime *bt) 466 { 467 struct timehands *th; 468 u_int gen; 469 470 do { 471 th = timehands; 472 gen = atomic_load_acq_int(&th->th_generation); 473 *bt = th->th_bintime; 474 atomic_thread_fence_acq(); 475 } while (gen == 0 || gen != th->th_generation); 476 } 477 478 void 479 getnanotime(struct timespec *tsp) 480 { 481 struct timehands *th; 482 u_int gen; 483 484 do { 485 th = timehands; 486 gen = atomic_load_acq_int(&th->th_generation); 487 *tsp = th->th_nanotime; 488 atomic_thread_fence_acq(); 489 } while (gen == 0 || gen != th->th_generation); 490 } 491 492 void 493 getmicrotime(struct timeval *tvp) 494 { 495 struct timehands *th; 496 u_int gen; 497 498 do { 499 th = timehands; 500 gen = atomic_load_acq_int(&th->th_generation); 501 *tvp = th->th_microtime; 502 atomic_thread_fence_acq(); 503 } while (gen == 0 || gen != th->th_generation); 504 } 505 #endif /* FFCLOCK */ 506 507 void 508 getboottime(struct timeval *boottime) 509 { 510 struct bintime boottimebin; 511 512 getboottimebin(&boottimebin); 513 bintime2timeval(&boottimebin, boottime); 514 } 515 516 void 517 getboottimebin(struct bintime *boottimebin) 518 { 519 struct timehands *th; 520 u_int gen; 521 522 do { 523 th = timehands; 524 gen = atomic_load_acq_int(&th->th_generation); 525 *boottimebin = th->th_boottime; 526 atomic_thread_fence_acq(); 527 } while (gen == 0 || gen != th->th_generation); 528 } 529 530 #ifdef FFCLOCK 531 /* 532 * Support for feed-forward synchronization algorithms. This is heavily inspired 533 * by the timehands mechanism but kept independent from it. *_windup() functions 534 * have some connection to avoid accessing the timecounter hardware more than 535 * necessary. 536 */ 537 538 /* Feed-forward clock estimates kept updated by the synchronization daemon. */ 539 struct ffclock_estimate ffclock_estimate; 540 struct bintime ffclock_boottime; /* Feed-forward boot time estimate. */ 541 uint32_t ffclock_status; /* Feed-forward clock status. */ 542 int8_t ffclock_updated; /* New estimates are available. */ 543 struct mtx ffclock_mtx; /* Mutex on ffclock_estimate. */ 544 545 struct fftimehands { 546 struct ffclock_estimate cest; 547 struct bintime tick_time; 548 struct bintime tick_time_lerp; 549 ffcounter tick_ffcount; 550 uint64_t period_lerp; 551 volatile uint8_t gen; 552 struct fftimehands *next; 553 }; 554 555 #define NUM_ELEMENTS(x) (sizeof(x) / sizeof(*x)) 556 557 static struct fftimehands ffth[10]; 558 static struct fftimehands *volatile fftimehands = ffth; 559 560 static void 561 ffclock_init(void) 562 { 563 struct fftimehands *cur; 564 struct fftimehands *last; 565 566 memset(ffth, 0, sizeof(ffth)); 567 568 last = ffth + NUM_ELEMENTS(ffth) - 1; 569 for (cur = ffth; cur < last; cur++) 570 cur->next = cur + 1; 571 last->next = ffth; 572 573 ffclock_updated = 0; 574 ffclock_status = FFCLOCK_STA_UNSYNC; 575 mtx_init(&ffclock_mtx, "ffclock lock", NULL, MTX_DEF); 576 } 577 578 /* 579 * Reset the feed-forward clock estimates. Called from inittodr() to get things 580 * kick started and uses the timecounter nominal frequency as a first period 581 * estimate. Note: this function may be called several time just after boot. 582 * Note: this is the only function that sets the value of boot time for the 583 * monotonic (i.e. uptime) version of the feed-forward clock. 584 */ 585 void 586 ffclock_reset_clock(struct timespec *ts) 587 { 588 struct timecounter *tc; 589 struct ffclock_estimate cest; 590 591 tc = timehands->th_counter; 592 memset(&cest, 0, sizeof(struct ffclock_estimate)); 593 594 timespec2bintime(ts, &ffclock_boottime); 595 timespec2bintime(ts, &(cest.update_time)); 596 ffclock_read_counter(&cest.update_ffcount); 597 cest.leapsec_next = 0; 598 cest.period = ((1ULL << 63) / tc->tc_frequency) << 1; 599 cest.errb_abs = 0; 600 cest.errb_rate = 0; 601 cest.status = FFCLOCK_STA_UNSYNC; 602 cest.leapsec_total = 0; 603 cest.leapsec = 0; 604 605 mtx_lock(&ffclock_mtx); 606 bcopy(&cest, &ffclock_estimate, sizeof(struct ffclock_estimate)); 607 ffclock_updated = INT8_MAX; 608 mtx_unlock(&ffclock_mtx); 609 610 printf("ffclock reset: %s (%llu Hz), time = %ld.%09lu\n", tc->tc_name, 611 (unsigned long long)tc->tc_frequency, (long)ts->tv_sec, 612 (unsigned long)ts->tv_nsec); 613 } 614 615 /* 616 * Sub-routine to convert a time interval measured in RAW counter units to time 617 * in seconds stored in bintime format. 618 * NOTE: bintime_mul requires u_int, but the value of the ffcounter may be 619 * larger than the max value of u_int (on 32 bit architecture). Loop to consume 620 * extra cycles. 621 */ 622 static void 623 ffclock_convert_delta(ffcounter ffdelta, uint64_t period, struct bintime *bt) 624 { 625 struct bintime bt2; 626 ffcounter delta, delta_max; 627 628 delta_max = (1ULL << (8 * sizeof(unsigned int))) - 1; 629 bintime_clear(bt); 630 do { 631 if (ffdelta > delta_max) 632 delta = delta_max; 633 else 634 delta = ffdelta; 635 bt2.sec = 0; 636 bt2.frac = period; 637 bintime_mul(&bt2, (unsigned int)delta); 638 bintime_add(bt, &bt2); 639 ffdelta -= delta; 640 } while (ffdelta > 0); 641 } 642 643 /* 644 * Update the fftimehands. 645 * Push the tick ffcount and time(s) forward based on current clock estimate. 646 * The conversion from ffcounter to bintime relies on the difference clock 647 * principle, whose accuracy relies on computing small time intervals. If a new 648 * clock estimate has been passed by the synchronisation daemon, make it 649 * current, and compute the linear interpolation for monotonic time if needed. 650 */ 651 static void 652 ffclock_windup(unsigned int delta) 653 { 654 struct ffclock_estimate *cest; 655 struct fftimehands *ffth; 656 struct bintime bt, gap_lerp; 657 ffcounter ffdelta; 658 uint64_t frac; 659 unsigned int polling; 660 uint8_t forward_jump, ogen; 661 662 /* 663 * Pick the next timehand, copy current ffclock estimates and move tick 664 * times and counter forward. 665 */ 666 forward_jump = 0; 667 ffth = fftimehands->next; 668 ogen = ffth->gen; 669 ffth->gen = 0; 670 cest = &ffth->cest; 671 bcopy(&fftimehands->cest, cest, sizeof(struct ffclock_estimate)); 672 ffdelta = (ffcounter)delta; 673 ffth->period_lerp = fftimehands->period_lerp; 674 675 ffth->tick_time = fftimehands->tick_time; 676 ffclock_convert_delta(ffdelta, cest->period, &bt); 677 bintime_add(&ffth->tick_time, &bt); 678 679 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 680 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt); 681 bintime_add(&ffth->tick_time_lerp, &bt); 682 683 ffth->tick_ffcount = fftimehands->tick_ffcount + ffdelta; 684 685 /* 686 * Assess the status of the clock, if the last update is too old, it is 687 * likely the synchronisation daemon is dead and the clock is free 688 * running. 689 */ 690 if (ffclock_updated == 0) { 691 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 692 ffclock_convert_delta(ffdelta, cest->period, &bt); 693 if (bt.sec > 2 * FFCLOCK_SKM_SCALE) 694 ffclock_status |= FFCLOCK_STA_UNSYNC; 695 } 696 697 /* 698 * If available, grab updated clock estimates and make them current. 699 * Recompute time at this tick using the updated estimates. The clock 700 * estimates passed the feed-forward synchronisation daemon may result 701 * in time conversion that is not monotonically increasing (just after 702 * the update). time_lerp is a particular linear interpolation over the 703 * synchronisation algo polling period that ensures monotonicity for the 704 * clock ids requesting it. 705 */ 706 if (ffclock_updated > 0) { 707 bcopy(&ffclock_estimate, cest, sizeof(struct ffclock_estimate)); 708 ffdelta = ffth->tick_ffcount - cest->update_ffcount; 709 ffth->tick_time = cest->update_time; 710 ffclock_convert_delta(ffdelta, cest->period, &bt); 711 bintime_add(&ffth->tick_time, &bt); 712 713 /* ffclock_reset sets ffclock_updated to INT8_MAX */ 714 if (ffclock_updated == INT8_MAX) 715 ffth->tick_time_lerp = ffth->tick_time; 716 717 if (bintime_cmp(&ffth->tick_time, &ffth->tick_time_lerp, >)) 718 forward_jump = 1; 719 else 720 forward_jump = 0; 721 722 bintime_clear(&gap_lerp); 723 if (forward_jump) { 724 gap_lerp = ffth->tick_time; 725 bintime_sub(&gap_lerp, &ffth->tick_time_lerp); 726 } else { 727 gap_lerp = ffth->tick_time_lerp; 728 bintime_sub(&gap_lerp, &ffth->tick_time); 729 } 730 731 /* 732 * The reset from the RTC clock may be far from accurate, and 733 * reducing the gap between real time and interpolated time 734 * could take a very long time if the interpolated clock insists 735 * on strict monotonicity. The clock is reset under very strict 736 * conditions (kernel time is known to be wrong and 737 * synchronization daemon has been restarted recently. 738 * ffclock_boottime absorbs the jump to ensure boot time is 739 * correct and uptime functions stay consistent. 740 */ 741 if (((ffclock_status & FFCLOCK_STA_UNSYNC) == FFCLOCK_STA_UNSYNC) && 742 ((cest->status & FFCLOCK_STA_UNSYNC) == 0) && 743 ((cest->status & FFCLOCK_STA_WARMUP) == FFCLOCK_STA_WARMUP)) { 744 if (forward_jump) 745 bintime_add(&ffclock_boottime, &gap_lerp); 746 else 747 bintime_sub(&ffclock_boottime, &gap_lerp); 748 ffth->tick_time_lerp = ffth->tick_time; 749 bintime_clear(&gap_lerp); 750 } 751 752 ffclock_status = cest->status; 753 ffth->period_lerp = cest->period; 754 755 /* 756 * Compute corrected period used for the linear interpolation of 757 * time. The rate of linear interpolation is capped to 5000PPM 758 * (5ms/s). 759 */ 760 if (bintime_isset(&gap_lerp)) { 761 ffdelta = cest->update_ffcount; 762 ffdelta -= fftimehands->cest.update_ffcount; 763 ffclock_convert_delta(ffdelta, cest->period, &bt); 764 polling = bt.sec; 765 bt.sec = 0; 766 bt.frac = 5000000 * (uint64_t)18446744073LL; 767 bintime_mul(&bt, polling); 768 if (bintime_cmp(&gap_lerp, &bt, >)) 769 gap_lerp = bt; 770 771 /* Approximate 1 sec by 1-(1/2^64) to ease arithmetic */ 772 frac = 0; 773 if (gap_lerp.sec > 0) { 774 frac -= 1; 775 frac /= ffdelta / gap_lerp.sec; 776 } 777 frac += gap_lerp.frac / ffdelta; 778 779 if (forward_jump) 780 ffth->period_lerp += frac; 781 else 782 ffth->period_lerp -= frac; 783 } 784 785 ffclock_updated = 0; 786 } 787 if (++ogen == 0) 788 ogen = 1; 789 ffth->gen = ogen; 790 fftimehands = ffth; 791 } 792 793 /* 794 * Adjust the fftimehands when the timecounter is changed. Stating the obvious, 795 * the old and new hardware counter cannot be read simultaneously. tc_windup() 796 * does read the two counters 'back to back', but a few cycles are effectively 797 * lost, and not accumulated in tick_ffcount. This is a fairly radical 798 * operation for a feed-forward synchronization daemon, and it is its job to not 799 * pushing irrelevant data to the kernel. Because there is no locking here, 800 * simply force to ignore pending or next update to give daemon a chance to 801 * realize the counter has changed. 802 */ 803 static void 804 ffclock_change_tc(struct timehands *th) 805 { 806 struct fftimehands *ffth; 807 struct ffclock_estimate *cest; 808 struct timecounter *tc; 809 uint8_t ogen; 810 811 tc = th->th_counter; 812 ffth = fftimehands->next; 813 ogen = ffth->gen; 814 ffth->gen = 0; 815 816 cest = &ffth->cest; 817 bcopy(&(fftimehands->cest), cest, sizeof(struct ffclock_estimate)); 818 cest->period = ((1ULL << 63) / tc->tc_frequency ) << 1; 819 cest->errb_abs = 0; 820 cest->errb_rate = 0; 821 cest->status |= FFCLOCK_STA_UNSYNC; 822 823 ffth->tick_ffcount = fftimehands->tick_ffcount; 824 ffth->tick_time_lerp = fftimehands->tick_time_lerp; 825 ffth->tick_time = fftimehands->tick_time; 826 ffth->period_lerp = cest->period; 827 828 /* Do not lock but ignore next update from synchronization daemon. */ 829 ffclock_updated--; 830 831 if (++ogen == 0) 832 ogen = 1; 833 ffth->gen = ogen; 834 fftimehands = ffth; 835 } 836 837 /* 838 * Retrieve feed-forward counter and time of last kernel tick. 839 */ 840 void 841 ffclock_last_tick(ffcounter *ffcount, struct bintime *bt, uint32_t flags) 842 { 843 struct fftimehands *ffth; 844 uint8_t gen; 845 846 /* 847 * No locking but check generation has not changed. Also need to make 848 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 849 */ 850 do { 851 ffth = fftimehands; 852 gen = ffth->gen; 853 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) 854 *bt = ffth->tick_time_lerp; 855 else 856 *bt = ffth->tick_time; 857 *ffcount = ffth->tick_ffcount; 858 } while (gen == 0 || gen != ffth->gen); 859 } 860 861 /* 862 * Absolute clock conversion. Low level function to convert ffcounter to 863 * bintime. The ffcounter is converted using the current ffclock period estimate 864 * or the "interpolated period" to ensure monotonicity. 865 * NOTE: this conversion may have been deferred, and the clock updated since the 866 * hardware counter has been read. 867 */ 868 void 869 ffclock_convert_abs(ffcounter ffcount, struct bintime *bt, uint32_t flags) 870 { 871 struct fftimehands *ffth; 872 struct bintime bt2; 873 ffcounter ffdelta; 874 uint8_t gen; 875 876 /* 877 * No locking but check generation has not changed. Also need to make 878 * sure ffdelta is positive, i.e. ffcount > tick_ffcount. 879 */ 880 do { 881 ffth = fftimehands; 882 gen = ffth->gen; 883 if (ffcount > ffth->tick_ffcount) 884 ffdelta = ffcount - ffth->tick_ffcount; 885 else 886 ffdelta = ffth->tick_ffcount - ffcount; 887 888 if ((flags & FFCLOCK_LERP) == FFCLOCK_LERP) { 889 *bt = ffth->tick_time_lerp; 890 ffclock_convert_delta(ffdelta, ffth->period_lerp, &bt2); 891 } else { 892 *bt = ffth->tick_time; 893 ffclock_convert_delta(ffdelta, ffth->cest.period, &bt2); 894 } 895 896 if (ffcount > ffth->tick_ffcount) 897 bintime_add(bt, &bt2); 898 else 899 bintime_sub(bt, &bt2); 900 } while (gen == 0 || gen != ffth->gen); 901 } 902 903 /* 904 * Difference clock conversion. 905 * Low level function to Convert a time interval measured in RAW counter units 906 * into bintime. The difference clock allows measuring small intervals much more 907 * reliably than the absolute clock. 908 */ 909 void 910 ffclock_convert_diff(ffcounter ffdelta, struct bintime *bt) 911 { 912 struct fftimehands *ffth; 913 uint8_t gen; 914 915 /* No locking but check generation has not changed. */ 916 do { 917 ffth = fftimehands; 918 gen = ffth->gen; 919 ffclock_convert_delta(ffdelta, ffth->cest.period, bt); 920 } while (gen == 0 || gen != ffth->gen); 921 } 922 923 /* 924 * Access to current ffcounter value. 925 */ 926 void 927 ffclock_read_counter(ffcounter *ffcount) 928 { 929 struct timehands *th; 930 struct fftimehands *ffth; 931 unsigned int gen, delta; 932 933 /* 934 * ffclock_windup() called from tc_windup(), safe to rely on 935 * th->th_generation only, for correct delta and ffcounter. 936 */ 937 do { 938 th = timehands; 939 gen = atomic_load_acq_int(&th->th_generation); 940 ffth = fftimehands; 941 delta = tc_delta(th); 942 *ffcount = ffth->tick_ffcount; 943 atomic_thread_fence_acq(); 944 } while (gen == 0 || gen != th->th_generation); 945 946 *ffcount += delta; 947 } 948 949 void 950 binuptime(struct bintime *bt) 951 { 952 953 binuptime_fromclock(bt, sysclock_active); 954 } 955 956 void 957 nanouptime(struct timespec *tsp) 958 { 959 960 nanouptime_fromclock(tsp, sysclock_active); 961 } 962 963 void 964 microuptime(struct timeval *tvp) 965 { 966 967 microuptime_fromclock(tvp, sysclock_active); 968 } 969 970 void 971 bintime(struct bintime *bt) 972 { 973 974 bintime_fromclock(bt, sysclock_active); 975 } 976 977 void 978 nanotime(struct timespec *tsp) 979 { 980 981 nanotime_fromclock(tsp, sysclock_active); 982 } 983 984 void 985 microtime(struct timeval *tvp) 986 { 987 988 microtime_fromclock(tvp, sysclock_active); 989 } 990 991 void 992 getbinuptime(struct bintime *bt) 993 { 994 995 getbinuptime_fromclock(bt, sysclock_active); 996 } 997 998 void 999 getnanouptime(struct timespec *tsp) 1000 { 1001 1002 getnanouptime_fromclock(tsp, sysclock_active); 1003 } 1004 1005 void 1006 getmicrouptime(struct timeval *tvp) 1007 { 1008 1009 getmicrouptime_fromclock(tvp, sysclock_active); 1010 } 1011 1012 void 1013 getbintime(struct bintime *bt) 1014 { 1015 1016 getbintime_fromclock(bt, sysclock_active); 1017 } 1018 1019 void 1020 getnanotime(struct timespec *tsp) 1021 { 1022 1023 getnanotime_fromclock(tsp, sysclock_active); 1024 } 1025 1026 void 1027 getmicrotime(struct timeval *tvp) 1028 { 1029 1030 getmicrouptime_fromclock(tvp, sysclock_active); 1031 } 1032 1033 #endif /* FFCLOCK */ 1034 1035 /* 1036 * This is a clone of getnanotime and used for walltimestamps. 1037 * The dtrace_ prefix prevents fbt from creating probes for 1038 * it so walltimestamp can be safely used in all fbt probes. 1039 */ 1040 void 1041 dtrace_getnanotime(struct timespec *tsp) 1042 { 1043 struct timehands *th; 1044 u_int gen; 1045 1046 do { 1047 th = timehands; 1048 gen = atomic_load_acq_int(&th->th_generation); 1049 *tsp = th->th_nanotime; 1050 atomic_thread_fence_acq(); 1051 } while (gen == 0 || gen != th->th_generation); 1052 } 1053 1054 /* 1055 * System clock currently providing time to the system. Modifiable via sysctl 1056 * when the FFCLOCK option is defined. 1057 */ 1058 int sysclock_active = SYSCLOCK_FBCK; 1059 1060 /* Internal NTP status and error estimates. */ 1061 extern int time_status; 1062 extern long time_esterror; 1063 1064 /* 1065 * Take a snapshot of sysclock data which can be used to compare system clocks 1066 * and generate timestamps after the fact. 1067 */ 1068 void 1069 sysclock_getsnapshot(struct sysclock_snap *clock_snap, int fast) 1070 { 1071 struct fbclock_info *fbi; 1072 struct timehands *th; 1073 struct bintime bt; 1074 unsigned int delta, gen; 1075 #ifdef FFCLOCK 1076 ffcounter ffcount; 1077 struct fftimehands *ffth; 1078 struct ffclock_info *ffi; 1079 struct ffclock_estimate cest; 1080 1081 ffi = &clock_snap->ff_info; 1082 #endif 1083 1084 fbi = &clock_snap->fb_info; 1085 delta = 0; 1086 1087 do { 1088 th = timehands; 1089 gen = atomic_load_acq_int(&th->th_generation); 1090 fbi->th_scale = th->th_scale; 1091 fbi->tick_time = th->th_offset; 1092 #ifdef FFCLOCK 1093 ffth = fftimehands; 1094 ffi->tick_time = ffth->tick_time_lerp; 1095 ffi->tick_time_lerp = ffth->tick_time_lerp; 1096 ffi->period = ffth->cest.period; 1097 ffi->period_lerp = ffth->period_lerp; 1098 clock_snap->ffcount = ffth->tick_ffcount; 1099 cest = ffth->cest; 1100 #endif 1101 if (!fast) 1102 delta = tc_delta(th); 1103 atomic_thread_fence_acq(); 1104 } while (gen == 0 || gen != th->th_generation); 1105 1106 clock_snap->delta = delta; 1107 clock_snap->sysclock_active = sysclock_active; 1108 1109 /* Record feedback clock status and error. */ 1110 clock_snap->fb_info.status = time_status; 1111 /* XXX: Very crude estimate of feedback clock error. */ 1112 bt.sec = time_esterror / 1000000; 1113 bt.frac = ((time_esterror - bt.sec) * 1000000) * 1114 (uint64_t)18446744073709ULL; 1115 clock_snap->fb_info.error = bt; 1116 1117 #ifdef FFCLOCK 1118 if (!fast) 1119 clock_snap->ffcount += delta; 1120 1121 /* Record feed-forward clock leap second adjustment. */ 1122 ffi->leapsec_adjustment = cest.leapsec_total; 1123 if (clock_snap->ffcount > cest.leapsec_next) 1124 ffi->leapsec_adjustment -= cest.leapsec; 1125 1126 /* Record feed-forward clock status and error. */ 1127 clock_snap->ff_info.status = cest.status; 1128 ffcount = clock_snap->ffcount - cest.update_ffcount; 1129 ffclock_convert_delta(ffcount, cest.period, &bt); 1130 /* 18446744073709 = int(2^64/1e12), err_bound_rate in [ps/s]. */ 1131 bintime_mul(&bt, cest.errb_rate * (uint64_t)18446744073709ULL); 1132 /* 18446744073 = int(2^64 / 1e9), since err_abs in [ns]. */ 1133 bintime_addx(&bt, cest.errb_abs * (uint64_t)18446744073ULL); 1134 clock_snap->ff_info.error = bt; 1135 #endif 1136 } 1137 1138 /* 1139 * Convert a sysclock snapshot into a struct bintime based on the specified 1140 * clock source and flags. 1141 */ 1142 int 1143 sysclock_snap2bintime(struct sysclock_snap *cs, struct bintime *bt, 1144 int whichclock, uint32_t flags) 1145 { 1146 struct bintime boottimebin; 1147 #ifdef FFCLOCK 1148 struct bintime bt2; 1149 uint64_t period; 1150 #endif 1151 1152 switch (whichclock) { 1153 case SYSCLOCK_FBCK: 1154 *bt = cs->fb_info.tick_time; 1155 1156 /* If snapshot was created with !fast, delta will be >0. */ 1157 if (cs->delta > 0) 1158 bintime_addx(bt, cs->fb_info.th_scale * cs->delta); 1159 1160 if ((flags & FBCLOCK_UPTIME) == 0) { 1161 getboottimebin(&boottimebin); 1162 bintime_add(bt, &boottimebin); 1163 } 1164 break; 1165 #ifdef FFCLOCK 1166 case SYSCLOCK_FFWD: 1167 if (flags & FFCLOCK_LERP) { 1168 *bt = cs->ff_info.tick_time_lerp; 1169 period = cs->ff_info.period_lerp; 1170 } else { 1171 *bt = cs->ff_info.tick_time; 1172 period = cs->ff_info.period; 1173 } 1174 1175 /* If snapshot was created with !fast, delta will be >0. */ 1176 if (cs->delta > 0) { 1177 ffclock_convert_delta(cs->delta, period, &bt2); 1178 bintime_add(bt, &bt2); 1179 } 1180 1181 /* Leap second adjustment. */ 1182 if (flags & FFCLOCK_LEAPSEC) 1183 bt->sec -= cs->ff_info.leapsec_adjustment; 1184 1185 /* Boot time adjustment, for uptime/monotonic clocks. */ 1186 if (flags & FFCLOCK_UPTIME) 1187 bintime_sub(bt, &ffclock_boottime); 1188 break; 1189 #endif 1190 default: 1191 return (EINVAL); 1192 break; 1193 } 1194 1195 return (0); 1196 } 1197 1198 /* 1199 * Initialize a new timecounter and possibly use it. 1200 */ 1201 void 1202 tc_init(struct timecounter *tc) 1203 { 1204 u_int u; 1205 struct sysctl_oid *tc_root; 1206 1207 u = tc->tc_frequency / tc->tc_counter_mask; 1208 /* XXX: We need some margin here, 10% is a guess */ 1209 u *= 11; 1210 u /= 10; 1211 if (u > hz && tc->tc_quality >= 0) { 1212 tc->tc_quality = -2000; 1213 if (bootverbose) { 1214 printf("Timecounter \"%s\" frequency %ju Hz", 1215 tc->tc_name, (uintmax_t)tc->tc_frequency); 1216 printf(" -- Insufficient hz, needs at least %u\n", u); 1217 } 1218 } else if (tc->tc_quality >= 0 || bootverbose) { 1219 printf("Timecounter \"%s\" frequency %ju Hz quality %d\n", 1220 tc->tc_name, (uintmax_t)tc->tc_frequency, 1221 tc->tc_quality); 1222 } 1223 1224 tc->tc_next = timecounters; 1225 timecounters = tc; 1226 /* 1227 * Set up sysctl tree for this counter. 1228 */ 1229 tc_root = SYSCTL_ADD_NODE_WITH_LABEL(NULL, 1230 SYSCTL_STATIC_CHILDREN(_kern_timecounter_tc), OID_AUTO, tc->tc_name, 1231 CTLFLAG_RW, 0, "timecounter description", "timecounter"); 1232 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1233 "mask", CTLFLAG_RD, &(tc->tc_counter_mask), 0, 1234 "mask for implemented bits"); 1235 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1236 "counter", CTLTYPE_UINT | CTLFLAG_RD, tc, sizeof(*tc), 1237 sysctl_kern_timecounter_get, "IU", "current timecounter value"); 1238 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1239 "frequency", CTLTYPE_U64 | CTLFLAG_RD, tc, sizeof(*tc), 1240 sysctl_kern_timecounter_freq, "QU", "timecounter frequency"); 1241 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(tc_root), OID_AUTO, 1242 "quality", CTLFLAG_RD, &(tc->tc_quality), 0, 1243 "goodness of time counter"); 1244 /* 1245 * Do not automatically switch if the current tc was specifically 1246 * chosen. Never automatically use a timecounter with negative quality. 1247 * Even though we run on the dummy counter, switching here may be 1248 * worse since this timecounter may not be monotonic. 1249 */ 1250 if (tc_chosen) 1251 return; 1252 if (tc->tc_quality < 0) 1253 return; 1254 if (tc->tc_quality < timecounter->tc_quality) 1255 return; 1256 if (tc->tc_quality == timecounter->tc_quality && 1257 tc->tc_frequency < timecounter->tc_frequency) 1258 return; 1259 (void)tc->tc_get_timecount(tc); 1260 (void)tc->tc_get_timecount(tc); 1261 timecounter = tc; 1262 } 1263 1264 /* Report the frequency of the current timecounter. */ 1265 uint64_t 1266 tc_getfrequency(void) 1267 { 1268 1269 return (timehands->th_counter->tc_frequency); 1270 } 1271 1272 static bool 1273 sleeping_on_old_rtc(struct thread *td) 1274 { 1275 1276 /* 1277 * td_rtcgen is modified by curthread when it is running, 1278 * and by other threads in this function. By finding the thread 1279 * on a sleepqueue and holding the lock on the sleepqueue 1280 * chain, we guarantee that the thread is not running and that 1281 * modifying td_rtcgen is safe. Setting td_rtcgen to zero informs 1282 * the thread that it was woken due to a real-time clock adjustment. 1283 * (The declaration of td_rtcgen refers to this comment.) 1284 */ 1285 if (td->td_rtcgen != 0 && td->td_rtcgen != rtc_generation) { 1286 td->td_rtcgen = 0; 1287 return (true); 1288 } 1289 return (false); 1290 } 1291 1292 static struct mtx tc_setclock_mtx; 1293 MTX_SYSINIT(tc_setclock_init, &tc_setclock_mtx, "tcsetc", MTX_SPIN); 1294 1295 /* 1296 * Step our concept of UTC. This is done by modifying our estimate of 1297 * when we booted. 1298 */ 1299 void 1300 tc_setclock(struct timespec *ts) 1301 { 1302 struct timespec tbef, taft; 1303 struct bintime bt, bt2; 1304 1305 timespec2bintime(ts, &bt); 1306 nanotime(&tbef); 1307 mtx_lock_spin(&tc_setclock_mtx); 1308 cpu_tick_calibrate(1); 1309 binuptime(&bt2); 1310 bintime_sub(&bt, &bt2); 1311 1312 /* XXX fiddle all the little crinkly bits around the fiords... */ 1313 tc_windup(&bt); 1314 mtx_unlock_spin(&tc_setclock_mtx); 1315 1316 /* Avoid rtc_generation == 0, since td_rtcgen == 0 is special. */ 1317 atomic_add_rel_int(&rtc_generation, 2); 1318 sleepq_chains_remove_matching(sleeping_on_old_rtc); 1319 if (timestepwarnings) { 1320 nanotime(&taft); 1321 log(LOG_INFO, 1322 "Time stepped from %jd.%09ld to %jd.%09ld (%jd.%09ld)\n", 1323 (intmax_t)tbef.tv_sec, tbef.tv_nsec, 1324 (intmax_t)taft.tv_sec, taft.tv_nsec, 1325 (intmax_t)ts->tv_sec, ts->tv_nsec); 1326 } 1327 } 1328 1329 /* 1330 * Initialize the next struct timehands in the ring and make 1331 * it the active timehands. Along the way we might switch to a different 1332 * timecounter and/or do seconds processing in NTP. Slightly magic. 1333 */ 1334 static void 1335 tc_windup(struct bintime *new_boottimebin) 1336 { 1337 struct bintime bt; 1338 struct timehands *th, *tho; 1339 uint64_t scale; 1340 u_int delta, ncount, ogen; 1341 int i; 1342 time_t t; 1343 1344 /* 1345 * Make the next timehands a copy of the current one, but do 1346 * not overwrite the generation or next pointer. While we 1347 * update the contents, the generation must be zero. We need 1348 * to ensure that the zero generation is visible before the 1349 * data updates become visible, which requires release fence. 1350 * For similar reasons, re-reading of the generation after the 1351 * data is read should use acquire fence. 1352 */ 1353 tho = timehands; 1354 th = tho->th_next; 1355 ogen = th->th_generation; 1356 th->th_generation = 0; 1357 atomic_thread_fence_rel(); 1358 memcpy(th, tho, offsetof(struct timehands, th_generation)); 1359 if (new_boottimebin != NULL) 1360 th->th_boottime = *new_boottimebin; 1361 1362 /* 1363 * Capture a timecounter delta on the current timecounter and if 1364 * changing timecounters, a counter value from the new timecounter. 1365 * Update the offset fields accordingly. 1366 */ 1367 delta = tc_delta(th); 1368 if (th->th_counter != timecounter) 1369 ncount = timecounter->tc_get_timecount(timecounter); 1370 else 1371 ncount = 0; 1372 #ifdef FFCLOCK 1373 ffclock_windup(delta); 1374 #endif 1375 th->th_offset_count += delta; 1376 th->th_offset_count &= th->th_counter->tc_counter_mask; 1377 while (delta > th->th_counter->tc_frequency) { 1378 /* Eat complete unadjusted seconds. */ 1379 delta -= th->th_counter->tc_frequency; 1380 th->th_offset.sec++; 1381 } 1382 if ((delta > th->th_counter->tc_frequency / 2) && 1383 (th->th_scale * delta < ((uint64_t)1 << 63))) { 1384 /* The product th_scale * delta just barely overflows. */ 1385 th->th_offset.sec++; 1386 } 1387 bintime_addx(&th->th_offset, th->th_scale * delta); 1388 1389 /* 1390 * Hardware latching timecounters may not generate interrupts on 1391 * PPS events, so instead we poll them. There is a finite risk that 1392 * the hardware might capture a count which is later than the one we 1393 * got above, and therefore possibly in the next NTP second which might 1394 * have a different rate than the current NTP second. It doesn't 1395 * matter in practice. 1396 */ 1397 if (tho->th_counter->tc_poll_pps) 1398 tho->th_counter->tc_poll_pps(tho->th_counter); 1399 1400 /* 1401 * Deal with NTP second processing. The for loop normally 1402 * iterates at most once, but in extreme situations it might 1403 * keep NTP sane if timeouts are not run for several seconds. 1404 * At boot, the time step can be large when the TOD hardware 1405 * has been read, so on really large steps, we call 1406 * ntp_update_second only twice. We need to call it twice in 1407 * case we missed a leap second. 1408 */ 1409 bt = th->th_offset; 1410 bintime_add(&bt, &th->th_boottime); 1411 i = bt.sec - tho->th_microtime.tv_sec; 1412 if (i > LARGE_STEP) 1413 i = 2; 1414 for (; i > 0; i--) { 1415 t = bt.sec; 1416 ntp_update_second(&th->th_adjustment, &bt.sec); 1417 if (bt.sec != t) 1418 th->th_boottime.sec += bt.sec - t; 1419 } 1420 /* Update the UTC timestamps used by the get*() functions. */ 1421 th->th_bintime = bt; 1422 bintime2timeval(&bt, &th->th_microtime); 1423 bintime2timespec(&bt, &th->th_nanotime); 1424 1425 /* Now is a good time to change timecounters. */ 1426 if (th->th_counter != timecounter) { 1427 #ifndef __arm__ 1428 if ((timecounter->tc_flags & TC_FLAGS_C2STOP) != 0) 1429 cpu_disable_c2_sleep++; 1430 if ((th->th_counter->tc_flags & TC_FLAGS_C2STOP) != 0) 1431 cpu_disable_c2_sleep--; 1432 #endif 1433 th->th_counter = timecounter; 1434 th->th_offset_count = ncount; 1435 tc_min_ticktock_freq = max(1, timecounter->tc_frequency / 1436 (((uint64_t)timecounter->tc_counter_mask + 1) / 3)); 1437 #ifdef FFCLOCK 1438 ffclock_change_tc(th); 1439 #endif 1440 } 1441 1442 /*- 1443 * Recalculate the scaling factor. We want the number of 1/2^64 1444 * fractions of a second per period of the hardware counter, taking 1445 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 1446 * processing provides us with. 1447 * 1448 * The th_adjustment is nanoseconds per second with 32 bit binary 1449 * fraction and we want 64 bit binary fraction of second: 1450 * 1451 * x = a * 2^32 / 10^9 = a * 4.294967296 1452 * 1453 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 1454 * we can only multiply by about 850 without overflowing, that 1455 * leaves no suitably precise fractions for multiply before divide. 1456 * 1457 * Divide before multiply with a fraction of 2199/512 results in a 1458 * systematic undercompensation of 10PPM of th_adjustment. On a 1459 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 1460 * 1461 * We happily sacrifice the lowest of the 64 bits of our result 1462 * to the goddess of code clarity. 1463 * 1464 */ 1465 scale = (uint64_t)1 << 63; 1466 scale += (th->th_adjustment / 1024) * 2199; 1467 scale /= th->th_counter->tc_frequency; 1468 th->th_scale = scale * 2; 1469 1470 /* 1471 * Now that the struct timehands is again consistent, set the new 1472 * generation number, making sure to not make it zero. 1473 */ 1474 if (++ogen == 0) 1475 ogen = 1; 1476 atomic_store_rel_int(&th->th_generation, ogen); 1477 1478 /* Go live with the new struct timehands. */ 1479 #ifdef FFCLOCK 1480 switch (sysclock_active) { 1481 case SYSCLOCK_FBCK: 1482 #endif 1483 time_second = th->th_microtime.tv_sec; 1484 time_uptime = th->th_offset.sec; 1485 #ifdef FFCLOCK 1486 break; 1487 case SYSCLOCK_FFWD: 1488 time_second = fftimehands->tick_time_lerp.sec; 1489 time_uptime = fftimehands->tick_time_lerp.sec - ffclock_boottime.sec; 1490 break; 1491 } 1492 #endif 1493 1494 timehands = th; 1495 timekeep_push_vdso(); 1496 } 1497 1498 /* Report or change the active timecounter hardware. */ 1499 static int 1500 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) 1501 { 1502 char newname[32]; 1503 struct timecounter *newtc, *tc; 1504 int error; 1505 1506 tc = timecounter; 1507 strlcpy(newname, tc->tc_name, sizeof(newname)); 1508 1509 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); 1510 if (error != 0 || req->newptr == NULL) 1511 return (error); 1512 /* Record that the tc in use now was specifically chosen. */ 1513 tc_chosen = 1; 1514 if (strcmp(newname, tc->tc_name) == 0) 1515 return (0); 1516 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 1517 if (strcmp(newname, newtc->tc_name) != 0) 1518 continue; 1519 1520 /* Warm up new timecounter. */ 1521 (void)newtc->tc_get_timecount(newtc); 1522 (void)newtc->tc_get_timecount(newtc); 1523 1524 timecounter = newtc; 1525 1526 /* 1527 * The vdso timehands update is deferred until the next 1528 * 'tc_windup()'. 1529 * 1530 * This is prudent given that 'timekeep_push_vdso()' does not 1531 * use any locking and that it can be called in hard interrupt 1532 * context via 'tc_windup()'. 1533 */ 1534 return (0); 1535 } 1536 return (EINVAL); 1537 } 1538 1539 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, 1540 0, 0, sysctl_kern_timecounter_hardware, "A", 1541 "Timecounter hardware selected"); 1542 1543 /* Report the available timecounter hardware. */ 1544 static int 1545 sysctl_kern_timecounter_choice(SYSCTL_HANDLER_ARGS) 1546 { 1547 struct sbuf sb; 1548 struct timecounter *tc; 1549 int error; 1550 1551 sbuf_new_for_sysctl(&sb, NULL, 0, req); 1552 for (tc = timecounters; tc != NULL; tc = tc->tc_next) { 1553 if (tc != timecounters) 1554 sbuf_putc(&sb, ' '); 1555 sbuf_printf(&sb, "%s(%d)", tc->tc_name, tc->tc_quality); 1556 } 1557 error = sbuf_finish(&sb); 1558 sbuf_delete(&sb); 1559 return (error); 1560 } 1561 1562 SYSCTL_PROC(_kern_timecounter, OID_AUTO, choice, CTLTYPE_STRING | CTLFLAG_RD, 1563 0, 0, sysctl_kern_timecounter_choice, "A", "Timecounter hardware detected"); 1564 1565 /* 1566 * RFC 2783 PPS-API implementation. 1567 */ 1568 1569 /* 1570 * Return true if the driver is aware of the abi version extensions in the 1571 * pps_state structure, and it supports at least the given abi version number. 1572 */ 1573 static inline int 1574 abi_aware(struct pps_state *pps, int vers) 1575 { 1576 1577 return ((pps->kcmode & KCMODE_ABIFLAG) && pps->driver_abi >= vers); 1578 } 1579 1580 static int 1581 pps_fetch(struct pps_fetch_args *fapi, struct pps_state *pps) 1582 { 1583 int err, timo; 1584 pps_seq_t aseq, cseq; 1585 struct timeval tv; 1586 1587 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) 1588 return (EINVAL); 1589 1590 /* 1591 * If no timeout is requested, immediately return whatever values were 1592 * most recently captured. If timeout seconds is -1, that's a request 1593 * to block without a timeout. WITNESS won't let us sleep forever 1594 * without a lock (we really don't need a lock), so just repeatedly 1595 * sleep a long time. 1596 */ 1597 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) { 1598 if (fapi->timeout.tv_sec == -1) 1599 timo = 0x7fffffff; 1600 else { 1601 tv.tv_sec = fapi->timeout.tv_sec; 1602 tv.tv_usec = fapi->timeout.tv_nsec / 1000; 1603 timo = tvtohz(&tv); 1604 } 1605 aseq = atomic_load_int(&pps->ppsinfo.assert_sequence); 1606 cseq = atomic_load_int(&pps->ppsinfo.clear_sequence); 1607 while (aseq == atomic_load_int(&pps->ppsinfo.assert_sequence) && 1608 cseq == atomic_load_int(&pps->ppsinfo.clear_sequence)) { 1609 if (abi_aware(pps, 1) && pps->driver_mtx != NULL) { 1610 if (pps->flags & PPSFLAG_MTX_SPIN) { 1611 err = msleep_spin(pps, pps->driver_mtx, 1612 "ppsfch", timo); 1613 } else { 1614 err = msleep(pps, pps->driver_mtx, PCATCH, 1615 "ppsfch", timo); 1616 } 1617 } else { 1618 err = tsleep(pps, PCATCH, "ppsfch", timo); 1619 } 1620 if (err == EWOULDBLOCK) { 1621 if (fapi->timeout.tv_sec == -1) { 1622 continue; 1623 } else { 1624 return (ETIMEDOUT); 1625 } 1626 } else if (err != 0) { 1627 return (err); 1628 } 1629 } 1630 } 1631 1632 pps->ppsinfo.current_mode = pps->ppsparam.mode; 1633 fapi->pps_info_buf = pps->ppsinfo; 1634 1635 return (0); 1636 } 1637 1638 int 1639 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) 1640 { 1641 pps_params_t *app; 1642 struct pps_fetch_args *fapi; 1643 #ifdef FFCLOCK 1644 struct pps_fetch_ffc_args *fapi_ffc; 1645 #endif 1646 #ifdef PPS_SYNC 1647 struct pps_kcbind_args *kapi; 1648 #endif 1649 1650 KASSERT(pps != NULL, ("NULL pps pointer in pps_ioctl")); 1651 switch (cmd) { 1652 case PPS_IOC_CREATE: 1653 return (0); 1654 case PPS_IOC_DESTROY: 1655 return (0); 1656 case PPS_IOC_SETPARAMS: 1657 app = (pps_params_t *)data; 1658 if (app->mode & ~pps->ppscap) 1659 return (EINVAL); 1660 #ifdef FFCLOCK 1661 /* Ensure only a single clock is selected for ffc timestamp. */ 1662 if ((app->mode & PPS_TSCLK_MASK) == PPS_TSCLK_MASK) 1663 return (EINVAL); 1664 #endif 1665 pps->ppsparam = *app; 1666 return (0); 1667 case PPS_IOC_GETPARAMS: 1668 app = (pps_params_t *)data; 1669 *app = pps->ppsparam; 1670 app->api_version = PPS_API_VERS_1; 1671 return (0); 1672 case PPS_IOC_GETCAP: 1673 *(int*)data = pps->ppscap; 1674 return (0); 1675 case PPS_IOC_FETCH: 1676 fapi = (struct pps_fetch_args *)data; 1677 return (pps_fetch(fapi, pps)); 1678 #ifdef FFCLOCK 1679 case PPS_IOC_FETCH_FFCOUNTER: 1680 fapi_ffc = (struct pps_fetch_ffc_args *)data; 1681 if (fapi_ffc->tsformat && fapi_ffc->tsformat != 1682 PPS_TSFMT_TSPEC) 1683 return (EINVAL); 1684 if (fapi_ffc->timeout.tv_sec || fapi_ffc->timeout.tv_nsec) 1685 return (EOPNOTSUPP); 1686 pps->ppsinfo_ffc.current_mode = pps->ppsparam.mode; 1687 fapi_ffc->pps_info_buf_ffc = pps->ppsinfo_ffc; 1688 /* Overwrite timestamps if feedback clock selected. */ 1689 switch (pps->ppsparam.mode & PPS_TSCLK_MASK) { 1690 case PPS_TSCLK_FBCK: 1691 fapi_ffc->pps_info_buf_ffc.assert_timestamp = 1692 pps->ppsinfo.assert_timestamp; 1693 fapi_ffc->pps_info_buf_ffc.clear_timestamp = 1694 pps->ppsinfo.clear_timestamp; 1695 break; 1696 case PPS_TSCLK_FFWD: 1697 break; 1698 default: 1699 break; 1700 } 1701 return (0); 1702 #endif /* FFCLOCK */ 1703 case PPS_IOC_KCBIND: 1704 #ifdef PPS_SYNC 1705 kapi = (struct pps_kcbind_args *)data; 1706 /* XXX Only root should be able to do this */ 1707 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) 1708 return (EINVAL); 1709 if (kapi->kernel_consumer != PPS_KC_HARDPPS) 1710 return (EINVAL); 1711 if (kapi->edge & ~pps->ppscap) 1712 return (EINVAL); 1713 pps->kcmode = (kapi->edge & KCMODE_EDGEMASK) | 1714 (pps->kcmode & KCMODE_ABIFLAG); 1715 return (0); 1716 #else 1717 return (EOPNOTSUPP); 1718 #endif 1719 default: 1720 return (ENOIOCTL); 1721 } 1722 } 1723 1724 void 1725 pps_init(struct pps_state *pps) 1726 { 1727 pps->ppscap |= PPS_TSFMT_TSPEC | PPS_CANWAIT; 1728 if (pps->ppscap & PPS_CAPTUREASSERT) 1729 pps->ppscap |= PPS_OFFSETASSERT; 1730 if (pps->ppscap & PPS_CAPTURECLEAR) 1731 pps->ppscap |= PPS_OFFSETCLEAR; 1732 #ifdef FFCLOCK 1733 pps->ppscap |= PPS_TSCLK_MASK; 1734 #endif 1735 pps->kcmode &= ~KCMODE_ABIFLAG; 1736 } 1737 1738 void 1739 pps_init_abi(struct pps_state *pps) 1740 { 1741 1742 pps_init(pps); 1743 if (pps->driver_abi > 0) { 1744 pps->kcmode |= KCMODE_ABIFLAG; 1745 pps->kernel_abi = PPS_ABI_VERSION; 1746 } 1747 } 1748 1749 void 1750 pps_capture(struct pps_state *pps) 1751 { 1752 struct timehands *th; 1753 1754 KASSERT(pps != NULL, ("NULL pps pointer in pps_capture")); 1755 th = timehands; 1756 pps->capgen = atomic_load_acq_int(&th->th_generation); 1757 pps->capth = th; 1758 #ifdef FFCLOCK 1759 pps->capffth = fftimehands; 1760 #endif 1761 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); 1762 atomic_thread_fence_acq(); 1763 if (pps->capgen != th->th_generation) 1764 pps->capgen = 0; 1765 } 1766 1767 void 1768 pps_event(struct pps_state *pps, int event) 1769 { 1770 struct bintime bt; 1771 struct timespec ts, *tsp, *osp; 1772 u_int tcount, *pcount; 1773 int foff; 1774 pps_seq_t *pseq; 1775 #ifdef FFCLOCK 1776 struct timespec *tsp_ffc; 1777 pps_seq_t *pseq_ffc; 1778 ffcounter *ffcount; 1779 #endif 1780 #ifdef PPS_SYNC 1781 int fhard; 1782 #endif 1783 1784 KASSERT(pps != NULL, ("NULL pps pointer in pps_event")); 1785 /* Nothing to do if not currently set to capture this event type. */ 1786 if ((event & pps->ppsparam.mode) == 0) 1787 return; 1788 /* If the timecounter was wound up underneath us, bail out. */ 1789 if (pps->capgen == 0 || pps->capgen != 1790 atomic_load_acq_int(&pps->capth->th_generation)) 1791 return; 1792 1793 /* Things would be easier with arrays. */ 1794 if (event == PPS_CAPTUREASSERT) { 1795 tsp = &pps->ppsinfo.assert_timestamp; 1796 osp = &pps->ppsparam.assert_offset; 1797 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 1798 #ifdef PPS_SYNC 1799 fhard = pps->kcmode & PPS_CAPTUREASSERT; 1800 #endif 1801 pcount = &pps->ppscount[0]; 1802 pseq = &pps->ppsinfo.assert_sequence; 1803 #ifdef FFCLOCK 1804 ffcount = &pps->ppsinfo_ffc.assert_ffcount; 1805 tsp_ffc = &pps->ppsinfo_ffc.assert_timestamp; 1806 pseq_ffc = &pps->ppsinfo_ffc.assert_sequence; 1807 #endif 1808 } else { 1809 tsp = &pps->ppsinfo.clear_timestamp; 1810 osp = &pps->ppsparam.clear_offset; 1811 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 1812 #ifdef PPS_SYNC 1813 fhard = pps->kcmode & PPS_CAPTURECLEAR; 1814 #endif 1815 pcount = &pps->ppscount[1]; 1816 pseq = &pps->ppsinfo.clear_sequence; 1817 #ifdef FFCLOCK 1818 ffcount = &pps->ppsinfo_ffc.clear_ffcount; 1819 tsp_ffc = &pps->ppsinfo_ffc.clear_timestamp; 1820 pseq_ffc = &pps->ppsinfo_ffc.clear_sequence; 1821 #endif 1822 } 1823 1824 /* 1825 * If the timecounter changed, we cannot compare the count values, so 1826 * we have to drop the rest of the PPS-stuff until the next event. 1827 */ 1828 if (pps->ppstc != pps->capth->th_counter) { 1829 pps->ppstc = pps->capth->th_counter; 1830 *pcount = pps->capcount; 1831 pps->ppscount[2] = pps->capcount; 1832 return; 1833 } 1834 1835 /* Convert the count to a timespec. */ 1836 tcount = pps->capcount - pps->capth->th_offset_count; 1837 tcount &= pps->capth->th_counter->tc_counter_mask; 1838 bt = pps->capth->th_bintime; 1839 bintime_addx(&bt, pps->capth->th_scale * tcount); 1840 bintime2timespec(&bt, &ts); 1841 1842 /* If the timecounter was wound up underneath us, bail out. */ 1843 atomic_thread_fence_acq(); 1844 if (pps->capgen != pps->capth->th_generation) 1845 return; 1846 1847 *pcount = pps->capcount; 1848 (*pseq)++; 1849 *tsp = ts; 1850 1851 if (foff) { 1852 timespecadd(tsp, osp, tsp); 1853 if (tsp->tv_nsec < 0) { 1854 tsp->tv_nsec += 1000000000; 1855 tsp->tv_sec -= 1; 1856 } 1857 } 1858 1859 #ifdef FFCLOCK 1860 *ffcount = pps->capffth->tick_ffcount + tcount; 1861 bt = pps->capffth->tick_time; 1862 ffclock_convert_delta(tcount, pps->capffth->cest.period, &bt); 1863 bintime_add(&bt, &pps->capffth->tick_time); 1864 bintime2timespec(&bt, &ts); 1865 (*pseq_ffc)++; 1866 *tsp_ffc = ts; 1867 #endif 1868 1869 #ifdef PPS_SYNC 1870 if (fhard) { 1871 uint64_t scale; 1872 1873 /* 1874 * Feed the NTP PLL/FLL. 1875 * The FLL wants to know how many (hardware) nanoseconds 1876 * elapsed since the previous event. 1877 */ 1878 tcount = pps->capcount - pps->ppscount[2]; 1879 pps->ppscount[2] = pps->capcount; 1880 tcount &= pps->capth->th_counter->tc_counter_mask; 1881 scale = (uint64_t)1 << 63; 1882 scale /= pps->capth->th_counter->tc_frequency; 1883 scale *= 2; 1884 bt.sec = 0; 1885 bt.frac = 0; 1886 bintime_addx(&bt, scale * tcount); 1887 bintime2timespec(&bt, &ts); 1888 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); 1889 } 1890 #endif 1891 1892 /* Wakeup anyone sleeping in pps_fetch(). */ 1893 wakeup(pps); 1894 } 1895 1896 /* 1897 * Timecounters need to be updated every so often to prevent the hardware 1898 * counter from overflowing. Updating also recalculates the cached values 1899 * used by the get*() family of functions, so their precision depends on 1900 * the update frequency. 1901 */ 1902 1903 static int tc_tick; 1904 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tc_tick, 0, 1905 "Approximate number of hardclock ticks in a millisecond"); 1906 1907 void 1908 tc_ticktock(int cnt) 1909 { 1910 static int count; 1911 1912 if (mtx_trylock_spin(&tc_setclock_mtx)) { 1913 count += cnt; 1914 if (count >= tc_tick) { 1915 count = 0; 1916 tc_windup(NULL); 1917 } 1918 mtx_unlock_spin(&tc_setclock_mtx); 1919 } 1920 } 1921 1922 static void __inline 1923 tc_adjprecision(void) 1924 { 1925 int t; 1926 1927 if (tc_timepercentage > 0) { 1928 t = (99 + tc_timepercentage) / tc_timepercentage; 1929 tc_precexp = fls(t + (t >> 1)) - 1; 1930 FREQ2BT(hz / tc_tick, &bt_timethreshold); 1931 FREQ2BT(hz, &bt_tickthreshold); 1932 bintime_shift(&bt_timethreshold, tc_precexp); 1933 bintime_shift(&bt_tickthreshold, tc_precexp); 1934 } else { 1935 tc_precexp = 31; 1936 bt_timethreshold.sec = INT_MAX; 1937 bt_timethreshold.frac = ~(uint64_t)0; 1938 bt_tickthreshold = bt_timethreshold; 1939 } 1940 sbt_timethreshold = bttosbt(bt_timethreshold); 1941 sbt_tickthreshold = bttosbt(bt_tickthreshold); 1942 } 1943 1944 static int 1945 sysctl_kern_timecounter_adjprecision(SYSCTL_HANDLER_ARGS) 1946 { 1947 int error, val; 1948 1949 val = tc_timepercentage; 1950 error = sysctl_handle_int(oidp, &val, 0, req); 1951 if (error != 0 || req->newptr == NULL) 1952 return (error); 1953 tc_timepercentage = val; 1954 if (cold) 1955 goto done; 1956 tc_adjprecision(); 1957 done: 1958 return (0); 1959 } 1960 1961 /* Set up the requested number of timehands. */ 1962 static void 1963 inittimehands(void *dummy) 1964 { 1965 struct timehands *thp; 1966 int i; 1967 1968 TUNABLE_INT_FETCH("kern.timecounter.timehands_count", 1969 &timehands_count); 1970 if (timehands_count < 1) 1971 timehands_count = 1; 1972 if (timehands_count > nitems(ths)) 1973 timehands_count = nitems(ths); 1974 for (i = 1, thp = &ths[0]; i < timehands_count; thp = &ths[i++]) 1975 thp->th_next = &ths[i]; 1976 thp->th_next = &ths[0]; 1977 } 1978 SYSINIT(timehands, SI_SUB_TUNABLES, SI_ORDER_ANY, inittimehands, NULL); 1979 1980 static void 1981 inittimecounter(void *dummy) 1982 { 1983 u_int p; 1984 int tick_rate; 1985 1986 /* 1987 * Set the initial timeout to 1988 * max(1, <approx. number of hardclock ticks in a millisecond>). 1989 * People should probably not use the sysctl to set the timeout 1990 * to smaller than its initial value, since that value is the 1991 * smallest reasonable one. If they want better timestamps they 1992 * should use the non-"get"* functions. 1993 */ 1994 if (hz > 1000) 1995 tc_tick = (hz + 500) / 1000; 1996 else 1997 tc_tick = 1; 1998 tc_adjprecision(); 1999 FREQ2BT(hz, &tick_bt); 2000 tick_sbt = bttosbt(tick_bt); 2001 tick_rate = hz / tc_tick; 2002 FREQ2BT(tick_rate, &tc_tick_bt); 2003 tc_tick_sbt = bttosbt(tc_tick_bt); 2004 p = (tc_tick * 1000000) / hz; 2005 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); 2006 2007 #ifdef FFCLOCK 2008 ffclock_init(); 2009 #endif 2010 2011 /* warm up new timecounter (again) and get rolling. */ 2012 (void)timecounter->tc_get_timecount(timecounter); 2013 (void)timecounter->tc_get_timecount(timecounter); 2014 mtx_lock_spin(&tc_setclock_mtx); 2015 tc_windup(NULL); 2016 mtx_unlock_spin(&tc_setclock_mtx); 2017 } 2018 2019 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL); 2020 2021 /* Cpu tick handling -------------------------------------------------*/ 2022 2023 static int cpu_tick_variable; 2024 static uint64_t cpu_tick_frequency; 2025 2026 DPCPU_DEFINE_STATIC(uint64_t, tc_cpu_ticks_base); 2027 DPCPU_DEFINE_STATIC(unsigned, tc_cpu_ticks_last); 2028 2029 static uint64_t 2030 tc_cpu_ticks(void) 2031 { 2032 struct timecounter *tc; 2033 uint64_t res, *base; 2034 unsigned u, *last; 2035 2036 critical_enter(); 2037 base = DPCPU_PTR(tc_cpu_ticks_base); 2038 last = DPCPU_PTR(tc_cpu_ticks_last); 2039 tc = timehands->th_counter; 2040 u = tc->tc_get_timecount(tc) & tc->tc_counter_mask; 2041 if (u < *last) 2042 *base += (uint64_t)tc->tc_counter_mask + 1; 2043 *last = u; 2044 res = u + *base; 2045 critical_exit(); 2046 return (res); 2047 } 2048 2049 void 2050 cpu_tick_calibration(void) 2051 { 2052 static time_t last_calib; 2053 2054 if (time_uptime != last_calib && !(time_uptime & 0xf)) { 2055 cpu_tick_calibrate(0); 2056 last_calib = time_uptime; 2057 } 2058 } 2059 2060 /* 2061 * This function gets called every 16 seconds on only one designated 2062 * CPU in the system from hardclock() via cpu_tick_calibration()(). 2063 * 2064 * Whenever the real time clock is stepped we get called with reset=1 2065 * to make sure we handle suspend/resume and similar events correctly. 2066 */ 2067 2068 static void 2069 cpu_tick_calibrate(int reset) 2070 { 2071 static uint64_t c_last; 2072 uint64_t c_this, c_delta; 2073 static struct bintime t_last; 2074 struct bintime t_this, t_delta; 2075 uint32_t divi; 2076 2077 if (reset) { 2078 /* The clock was stepped, abort & reset */ 2079 t_last.sec = 0; 2080 return; 2081 } 2082 2083 /* we don't calibrate fixed rate cputicks */ 2084 if (!cpu_tick_variable) 2085 return; 2086 2087 getbinuptime(&t_this); 2088 c_this = cpu_ticks(); 2089 if (t_last.sec != 0) { 2090 c_delta = c_this - c_last; 2091 t_delta = t_this; 2092 bintime_sub(&t_delta, &t_last); 2093 /* 2094 * Headroom: 2095 * 2^(64-20) / 16[s] = 2096 * 2^(44) / 16[s] = 2097 * 17.592.186.044.416 / 16 = 2098 * 1.099.511.627.776 [Hz] 2099 */ 2100 divi = t_delta.sec << 20; 2101 divi |= t_delta.frac >> (64 - 20); 2102 c_delta <<= 20; 2103 c_delta /= divi; 2104 if (c_delta > cpu_tick_frequency) { 2105 if (0 && bootverbose) 2106 printf("cpu_tick increased to %ju Hz\n", 2107 c_delta); 2108 cpu_tick_frequency = c_delta; 2109 } 2110 } 2111 c_last = c_this; 2112 t_last = t_this; 2113 } 2114 2115 void 2116 set_cputicker(cpu_tick_f *func, uint64_t freq, unsigned var) 2117 { 2118 2119 if (func == NULL) { 2120 cpu_ticks = tc_cpu_ticks; 2121 } else { 2122 cpu_tick_frequency = freq; 2123 cpu_tick_variable = var; 2124 cpu_ticks = func; 2125 } 2126 } 2127 2128 uint64_t 2129 cpu_tickrate(void) 2130 { 2131 2132 if (cpu_ticks == tc_cpu_ticks) 2133 return (tc_getfrequency()); 2134 return (cpu_tick_frequency); 2135 } 2136 2137 /* 2138 * We need to be slightly careful converting cputicks to microseconds. 2139 * There is plenty of margin in 64 bits of microseconds (half a million 2140 * years) and in 64 bits at 4 GHz (146 years), but if we do a multiply 2141 * before divide conversion (to retain precision) we find that the 2142 * margin shrinks to 1.5 hours (one millionth of 146y). 2143 * With a three prong approach we never lose significant bits, no 2144 * matter what the cputick rate and length of timeinterval is. 2145 */ 2146 2147 uint64_t 2148 cputick2usec(uint64_t tick) 2149 { 2150 2151 if (tick > 18446744073709551LL) /* floor(2^64 / 1000) */ 2152 return (tick / (cpu_tickrate() / 1000000LL)); 2153 else if (tick > 18446744073709LL) /* floor(2^64 / 1000000) */ 2154 return ((tick * 1000LL) / (cpu_tickrate() / 1000LL)); 2155 else 2156 return ((tick * 1000000LL) / cpu_tickrate()); 2157 } 2158 2159 cpu_tick_f *cpu_ticks = tc_cpu_ticks; 2160 2161 static int vdso_th_enable = 1; 2162 static int 2163 sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) 2164 { 2165 int old_vdso_th_enable, error; 2166 2167 old_vdso_th_enable = vdso_th_enable; 2168 error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); 2169 if (error != 0) 2170 return (error); 2171 vdso_th_enable = old_vdso_th_enable; 2172 return (0); 2173 } 2174 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, 2175 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, 2176 NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); 2177 2178 uint32_t 2179 tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) 2180 { 2181 struct timehands *th; 2182 uint32_t enabled; 2183 2184 th = timehands; 2185 vdso_th->th_scale = th->th_scale; 2186 vdso_th->th_offset_count = th->th_offset_count; 2187 vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; 2188 vdso_th->th_offset = th->th_offset; 2189 vdso_th->th_boottime = th->th_boottime; 2190 if (th->th_counter->tc_fill_vdso_timehands != NULL) { 2191 enabled = th->th_counter->tc_fill_vdso_timehands(vdso_th, 2192 th->th_counter); 2193 } else 2194 enabled = 0; 2195 if (!vdso_th_enable) 2196 enabled = 0; 2197 return (enabled); 2198 } 2199 2200 #ifdef COMPAT_FREEBSD32 2201 uint32_t 2202 tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) 2203 { 2204 struct timehands *th; 2205 uint32_t enabled; 2206 2207 th = timehands; 2208 *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; 2209 vdso_th32->th_offset_count = th->th_offset_count; 2210 vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; 2211 vdso_th32->th_offset.sec = th->th_offset.sec; 2212 *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; 2213 vdso_th32->th_boottime.sec = th->th_boottime.sec; 2214 *(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac; 2215 if (th->th_counter->tc_fill_vdso_timehands32 != NULL) { 2216 enabled = th->th_counter->tc_fill_vdso_timehands32(vdso_th32, 2217 th->th_counter); 2218 } else 2219 enabled = 0; 2220 if (!vdso_th_enable) 2221 enabled = 0; 2222 return (enabled); 2223 } 2224 #endif 2225