1 /*- 2 * ---------------------------------------------------------------------------- 3 * "THE BEER-WARE LICENSE" (Revision 42): 4 * <phk@FreeBSD.ORG> wrote this file. As long as you retain this notice you 5 * can do whatever you want with this stuff. If we meet some day, and you think 6 * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp 7 * ---------------------------------------------------------------------------- 8 * 9 * $FreeBSD$ 10 */ 11 12 #include "opt_ntp.h" 13 14 #include <sys/param.h> 15 #include <sys/kernel.h> 16 #include <sys/sysctl.h> 17 #include <sys/systm.h> 18 #include <sys/timepps.h> 19 #include <sys/timetc.h> 20 #include <sys/timex.h> 21 22 /* 23 * Implement a dummy timecounter which we can use until we get a real one 24 * in the air. This allows the console and other early stuff to use 25 * time services. 26 */ 27 28 static u_int 29 dummy_get_timecount(struct timecounter *tc) 30 { 31 static u_int now; 32 33 return (++now); 34 } 35 36 static struct timecounter dummy_timecounter = { 37 dummy_get_timecount, 0, ~0u, 1000000, "dummy", 38 }; 39 40 struct timehands { 41 /* These fields must be initialized by the driver. */ 42 struct timecounter *th_counter; 43 int64_t th_adjustment; 44 u_int64_t th_scale; 45 u_int th_offset_count; 46 struct bintime th_offset; 47 struct timeval th_microtime; 48 struct timespec th_nanotime; 49 /* Fields not to be copied in tc_windup start with th_generation. */ 50 volatile u_int th_generation; 51 struct timehands *th_next; 52 }; 53 54 extern struct timehands th0; 55 static struct timehands th9 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th0}; 56 static struct timehands th8 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th9}; 57 static struct timehands th7 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th8}; 58 static struct timehands th6 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th7}; 59 static struct timehands th5 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th6}; 60 static struct timehands th4 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th5}; 61 static struct timehands th3 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th4}; 62 static struct timehands th2 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th3}; 63 static struct timehands th1 = { NULL, 0, 0, 0, {0, 0}, {0, 0}, {0, 0}, 0, &th2}; 64 static struct timehands th0 = { 65 &dummy_timecounter, 66 0, 67 (uint64_t)-1 / 1000000, 68 0, 69 {1, 0}, 70 {0, 0}, 71 {0, 0}, 72 1, 73 &th1 74 }; 75 76 static struct timehands *volatile timehands = &th0; 77 struct timecounter *timecounter = &dummy_timecounter; 78 static struct timecounter *timecounters = &dummy_timecounter; 79 80 time_t time_second = 1; 81 82 static struct bintime boottimebin; 83 struct timeval boottime; 84 SYSCTL_STRUCT(_kern, KERN_BOOTTIME, boottime, CTLFLAG_RD, 85 &boottime, timeval, "System boottime"); 86 87 SYSCTL_NODE(_kern, OID_AUTO, timecounter, CTLFLAG_RW, 0, ""); 88 89 #define TC_STATS(foo) \ 90 static u_int foo; \ 91 SYSCTL_UINT(_kern_timecounter, OID_AUTO, foo, CTLFLAG_RD, &foo, 0, "");\ 92 struct __hack 93 94 TC_STATS(nbinuptime); TC_STATS(nnanouptime); TC_STATS(nmicrouptime); 95 TC_STATS(nbintime); TC_STATS(nnanotime); TC_STATS(nmicrotime); 96 TC_STATS(ngetbinuptime); TC_STATS(ngetnanouptime); TC_STATS(ngetmicrouptime); 97 TC_STATS(ngetbintime); TC_STATS(ngetnanotime); TC_STATS(ngetmicrotime); 98 99 #undef TC_STATS 100 101 static void tc_windup(void); 102 103 /* 104 * Return the difference between the timehands' counter value now and what 105 * was when we copied it to the timehands' offset_count. 106 */ 107 static __inline u_int 108 tc_delta(struct timehands *th) 109 { 110 struct timecounter *tc; 111 112 tc = th->th_counter; 113 return ((tc->tc_get_timecount(tc) - th->th_offset_count) & 114 tc->tc_counter_mask); 115 } 116 117 /* 118 * Functions for reading the time. We have to loop until we are sure that 119 * the timehands that we operated on was not updated under our feet. See 120 * the comment in <sys/time.h> for a description of these 12 functions. 121 */ 122 123 void 124 binuptime(struct bintime *bt) 125 { 126 struct timehands *th; 127 u_int gen; 128 129 nbinuptime++; 130 do { 131 th = timehands; 132 gen = th->th_generation; 133 *bt = th->th_offset; 134 bintime_addx(bt, th->th_scale * tc_delta(th)); 135 } while (gen == 0 || gen != th->th_generation); 136 } 137 138 void 139 nanouptime(struct timespec *tsp) 140 { 141 struct bintime bt; 142 143 nnanouptime++; 144 binuptime(&bt); 145 bintime2timespec(&bt, tsp); 146 } 147 148 void 149 microuptime(struct timeval *tvp) 150 { 151 struct bintime bt; 152 153 nmicrouptime++; 154 binuptime(&bt); 155 bintime2timeval(&bt, tvp); 156 } 157 158 void 159 bintime(struct bintime *bt) 160 { 161 162 nbintime++; 163 binuptime(bt); 164 bintime_add(bt, &boottimebin); 165 } 166 167 void 168 nanotime(struct timespec *tsp) 169 { 170 struct bintime bt; 171 172 nnanotime++; 173 bintime(&bt); 174 bintime2timespec(&bt, tsp); 175 } 176 177 void 178 microtime(struct timeval *tvp) 179 { 180 struct bintime bt; 181 182 nmicrotime++; 183 bintime(&bt); 184 bintime2timeval(&bt, tvp); 185 } 186 187 void 188 getbinuptime(struct bintime *bt) 189 { 190 struct timehands *th; 191 u_int gen; 192 193 ngetbinuptime++; 194 do { 195 th = timehands; 196 gen = th->th_generation; 197 *bt = th->th_offset; 198 } while (gen == 0 || gen != th->th_generation); 199 } 200 201 void 202 getnanouptime(struct timespec *tsp) 203 { 204 struct timehands *th; 205 u_int gen; 206 207 ngetnanouptime++; 208 do { 209 th = timehands; 210 gen = th->th_generation; 211 bintime2timespec(&th->th_offset, tsp); 212 } while (gen == 0 || gen != th->th_generation); 213 } 214 215 void 216 getmicrouptime(struct timeval *tvp) 217 { 218 struct timehands *th; 219 u_int gen; 220 221 ngetmicrouptime++; 222 do { 223 th = timehands; 224 gen = th->th_generation; 225 bintime2timeval(&th->th_offset, tvp); 226 } while (gen == 0 || gen != th->th_generation); 227 } 228 229 void 230 getbintime(struct bintime *bt) 231 { 232 struct timehands *th; 233 u_int gen; 234 235 ngetbintime++; 236 do { 237 th = timehands; 238 gen = th->th_generation; 239 *bt = th->th_offset; 240 } while (gen == 0 || gen != th->th_generation); 241 bintime_add(bt, &boottimebin); 242 } 243 244 void 245 getnanotime(struct timespec *tsp) 246 { 247 struct timehands *th; 248 u_int gen; 249 250 ngetnanotime++; 251 do { 252 th = timehands; 253 gen = th->th_generation; 254 *tsp = th->th_nanotime; 255 } while (gen == 0 || gen != th->th_generation); 256 } 257 258 void 259 getmicrotime(struct timeval *tvp) 260 { 261 struct timehands *th; 262 u_int gen; 263 264 ngetmicrotime++; 265 do { 266 th = timehands; 267 gen = th->th_generation; 268 *tvp = th->th_microtime; 269 } while (gen == 0 || gen != th->th_generation); 270 } 271 272 /* 273 * Initialize a new timecounter. 274 * We should really try to rank the timecounters and intelligently determine 275 * if the new timecounter is better than the current one. This is subject 276 * to further study. For now always use the new timecounter. 277 */ 278 void 279 tc_init(struct timecounter *tc) 280 { 281 282 tc->tc_next = timecounters; 283 timecounters = tc; 284 printf("Timecounter \"%s\" frequency %lu Hz\n", 285 tc->tc_name, (u_long)tc->tc_frequency); 286 (void)tc->tc_get_timecount(tc); 287 (void)tc->tc_get_timecount(tc); 288 timecounter = tc; 289 } 290 291 /* Report the frequency of the current timecounter. */ 292 u_int32_t 293 tc_getfrequency(void) 294 { 295 296 return (timehands->th_counter->tc_frequency); 297 } 298 299 /* 300 * Step our concept of GMT. This is done by modifying our estimate of 301 * when we booted. XXX: needs futher work. 302 */ 303 void 304 tc_setclock(struct timespec *ts) 305 { 306 struct timespec ts2; 307 308 nanouptime(&ts2); 309 boottime.tv_sec = ts->tv_sec - ts2.tv_sec; 310 /* XXX boottime should probably be a timespec. */ 311 boottime.tv_usec = (ts->tv_nsec - ts2.tv_nsec) / 1000; 312 if (boottime.tv_usec < 0) { 313 boottime.tv_usec += 1000000; 314 boottime.tv_sec--; 315 } 316 timeval2bintime(&boottime, &boottimebin); 317 318 /* XXX fiddle all the little crinkly bits around the fiords... */ 319 tc_windup(); 320 } 321 322 /* 323 * Initialize the next struct timehands in the ring and make 324 * it the active timehands. Along the way we might switch to a different 325 * timecounter and/or do seconds processing in NTP. Slightly magic. 326 */ 327 static void 328 tc_windup(void) 329 { 330 struct bintime bt; 331 struct timehands *th, *tho; 332 u_int64_t scale; 333 u_int delta, ncount, ogen; 334 int i; 335 336 /* 337 * Make the next timehands a copy of the current one, but do not 338 * overwrite the generation or next pointer. While we update 339 * the contents, the generation must be zero. 340 */ 341 tho = timehands; 342 th = tho->th_next; 343 ogen = th->th_generation; 344 th->th_generation = 0; 345 bcopy(tho, th, offsetof(struct timehands, th_generation)); 346 347 /* 348 * Capture a timecounter delta on the current timecounter and if 349 * changing timecounters, a counter value from the new timecounter. 350 * Update the offset fields accordingly. 351 */ 352 delta = tc_delta(th); 353 if (th->th_counter != timecounter) 354 ncount = timecounter->tc_get_timecount(timecounter); 355 else 356 ncount = 0; 357 th->th_offset_count += delta; 358 th->th_offset_count &= th->th_counter->tc_counter_mask; 359 bintime_addx(&th->th_offset, th->th_scale * delta); 360 361 /* 362 * Hardware latching timecounters may not generate interrupts on 363 * PPS events, so instead we poll them. There is a finite risk that 364 * the hardware might capture a count which is later than the one we 365 * got above, and therefore possibly in the next NTP second which might 366 * have a different rate than the current NTP second. It doesn't 367 * matter in practice. 368 */ 369 if (tho->th_counter->tc_poll_pps) 370 tho->th_counter->tc_poll_pps(tho->th_counter); 371 372 /* 373 * Deal with NTP second processing. The for loop normally only 374 * iterates once, but in extreme situations it might keep NTP sane 375 * if timeouts are not run for several seconds. 376 */ 377 for (i = th->th_offset.sec - tho->th_offset.sec; i > 0; i--) 378 ntp_update_second(&th->th_adjustment, &th->th_offset.sec); 379 380 /* Now is a good time to change timecounters. */ 381 if (th->th_counter != timecounter) { 382 th->th_counter = timecounter; 383 th->th_offset_count = ncount; 384 } 385 386 /*- 387 * Recalculate the scaling factor. We want the number of 1/2^64 388 * fractions of a second per period of the hardware counter, taking 389 * into account the th_adjustment factor which the NTP PLL/adjtime(2) 390 * processing provides us with. 391 * 392 * The th_adjustment is nanoseconds per second with 32 bit binary 393 * fraction and want 64 bit binary fraction of second: 394 * 395 * x = a * 2^32 / 10^9 = a * 4.294967296 396 * 397 * The range of th_adjustment is +/- 5000PPM so inside a 64bit int 398 * we can only multiply by about 850 without overflowing, but that 399 * leaves suitably precise fractions for multiply before divide. 400 * 401 * Divide before multiply with a fraction of 2199/512 results in a 402 * systematic undercompensation of 10PPM of th_adjustment. On a 403 * 5000PPM adjustment this is a 0.05PPM error. This is acceptable. 404 * 405 * We happily sacrifice the lowest of the 64 bits of our result 406 * to the goddess of code clarity. 407 * 408 */ 409 scale = (u_int64_t)1 << 63; 410 scale += (th->th_adjustment / 1024) * 2199; 411 scale /= th->th_counter->tc_frequency; 412 th->th_scale = scale * 2; 413 414 /* Update the GMT timestamps used for the get*() functions. */ 415 bt = th->th_offset; 416 bintime_add(&bt, &boottimebin); 417 bintime2timeval(&bt, &th->th_microtime); 418 bintime2timespec(&bt, &th->th_nanotime); 419 420 /* 421 * Now that the struct timehands is again consistent, set the new 422 * generation number, making sure to not make it zero. 423 */ 424 if (++ogen == 0) 425 ogen = 1; 426 th->th_generation = ogen; 427 428 /* Go live with the new struct timehands. */ 429 time_second = th->th_microtime.tv_sec; 430 timehands = th; 431 } 432 433 /* Report or change the active timecounter hardware. */ 434 static int 435 sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) 436 { 437 char newname[32]; 438 struct timecounter *newtc, *tc; 439 int error; 440 441 tc = timecounter; 442 strncpy(newname, tc->tc_name, sizeof(newname)); 443 newname[sizeof(newname) - 1] = '\0'; 444 error = sysctl_handle_string(oidp, &newname[0], sizeof(newname), req); 445 if (error != 0 || req->newptr == NULL || 446 strcmp(newname, tc->tc_name) == 0) 447 return (error); 448 for (newtc = timecounters; newtc != NULL; newtc = newtc->tc_next) { 449 if (strcmp(newname, newtc->tc_name) != 0) 450 continue; 451 452 /* Warm up new timecounter. */ 453 (void)newtc->tc_get_timecount(newtc); 454 (void)newtc->tc_get_timecount(newtc); 455 456 timecounter = newtc; 457 return (0); 458 } 459 return (EINVAL); 460 } 461 462 SYSCTL_PROC(_kern_timecounter, OID_AUTO, hardware, CTLTYPE_STRING | CTLFLAG_RW, 463 0, 0, sysctl_kern_timecounter_hardware, "A", ""); 464 465 /* 466 * RFC 2783 PPS-API implementation. 467 */ 468 469 int 470 pps_ioctl(u_long cmd, caddr_t data, struct pps_state *pps) 471 { 472 pps_params_t *app; 473 struct pps_fetch_args *fapi; 474 #ifdef PPS_SYNC 475 struct pps_kcbind_args *kapi; 476 #endif 477 478 switch (cmd) { 479 case PPS_IOC_CREATE: 480 return (0); 481 case PPS_IOC_DESTROY: 482 return (0); 483 case PPS_IOC_SETPARAMS: 484 app = (pps_params_t *)data; 485 if (app->mode & ~pps->ppscap) 486 return (EINVAL); 487 pps->ppsparam = *app; 488 return (0); 489 case PPS_IOC_GETPARAMS: 490 app = (pps_params_t *)data; 491 *app = pps->ppsparam; 492 app->api_version = PPS_API_VERS_1; 493 return (0); 494 case PPS_IOC_GETCAP: 495 *(int*)data = pps->ppscap; 496 return (0); 497 case PPS_IOC_FETCH: 498 fapi = (struct pps_fetch_args *)data; 499 if (fapi->tsformat && fapi->tsformat != PPS_TSFMT_TSPEC) 500 return (EINVAL); 501 if (fapi->timeout.tv_sec || fapi->timeout.tv_nsec) 502 return (EOPNOTSUPP); 503 pps->ppsinfo.current_mode = pps->ppsparam.mode; 504 fapi->pps_info_buf = pps->ppsinfo; 505 return (0); 506 case PPS_IOC_KCBIND: 507 #ifdef PPS_SYNC 508 kapi = (struct pps_kcbind_args *)data; 509 /* XXX Only root should be able to do this */ 510 if (kapi->tsformat && kapi->tsformat != PPS_TSFMT_TSPEC) 511 return (EINVAL); 512 if (kapi->kernel_consumer != PPS_KC_HARDPPS) 513 return (EINVAL); 514 if (kapi->edge & ~pps->ppscap) 515 return (EINVAL); 516 pps->kcmode = kapi->edge; 517 return (0); 518 #else 519 return (EOPNOTSUPP); 520 #endif 521 default: 522 return (ENOTTY); 523 } 524 } 525 526 void 527 pps_init(struct pps_state *pps) 528 { 529 pps->ppscap |= PPS_TSFMT_TSPEC; 530 if (pps->ppscap & PPS_CAPTUREASSERT) 531 pps->ppscap |= PPS_OFFSETASSERT; 532 if (pps->ppscap & PPS_CAPTURECLEAR) 533 pps->ppscap |= PPS_OFFSETCLEAR; 534 } 535 536 void 537 pps_capture(struct pps_state *pps) 538 { 539 struct timehands *th; 540 541 th = timehands; 542 pps->capgen = th->th_generation; 543 pps->capth = th; 544 pps->capcount = th->th_counter->tc_get_timecount(th->th_counter); 545 if (pps->capgen != th->th_generation) 546 pps->capgen = 0; 547 } 548 549 void 550 pps_event(struct pps_state *pps, int event) 551 { 552 struct bintime bt; 553 struct timespec ts, *tsp, *osp; 554 u_int tcount, *pcount; 555 int foff, fhard; 556 pps_seq_t *pseq; 557 558 /* If the timecounter was wound up underneath us, bail out. */ 559 if (pps->capgen == 0 || pps->capgen != pps->capth->th_generation) 560 return; 561 562 /* Things would be easier with arrays. */ 563 if (event == PPS_CAPTUREASSERT) { 564 tsp = &pps->ppsinfo.assert_timestamp; 565 osp = &pps->ppsparam.assert_offset; 566 foff = pps->ppsparam.mode & PPS_OFFSETASSERT; 567 fhard = pps->kcmode & PPS_CAPTUREASSERT; 568 pcount = &pps->ppscount[0]; 569 pseq = &pps->ppsinfo.assert_sequence; 570 } else { 571 tsp = &pps->ppsinfo.clear_timestamp; 572 osp = &pps->ppsparam.clear_offset; 573 foff = pps->ppsparam.mode & PPS_OFFSETCLEAR; 574 fhard = pps->kcmode & PPS_CAPTURECLEAR; 575 pcount = &pps->ppscount[1]; 576 pseq = &pps->ppsinfo.clear_sequence; 577 } 578 579 /* 580 * If the timecounter changed, we cannot compare the count values, so 581 * we have to drop the rest of the PPS-stuff until the next event. 582 */ 583 if (pps->ppstc != pps->capth->th_counter) { 584 pps->ppstc = pps->capth->th_counter; 585 *pcount = pps->capcount; 586 pps->ppscount[2] = pps->capcount; 587 return; 588 } 589 590 /* Return if nothing really happened. */ 591 if (*pcount == pps->capcount) 592 return; 593 594 /* Convert the count to a timespec. */ 595 tcount = pps->capcount - pps->capth->th_offset_count; 596 tcount &= pps->capth->th_counter->tc_counter_mask; 597 bt = pps->capth->th_offset; 598 bintime_addx(&bt, pps->capth->th_scale * tcount); 599 bintime_add(&bt, &boottimebin); 600 bintime2timespec(&bt, &ts); 601 602 /* If the timecounter was wound up underneath us, bail out. */ 603 if (pps->capgen != pps->capth->th_generation) 604 return; 605 606 *pcount = pps->capcount; 607 (*pseq)++; 608 *tsp = ts; 609 610 if (foff) { 611 timespecadd(tsp, osp); 612 if (tsp->tv_nsec < 0) { 613 tsp->tv_nsec += 1000000000; 614 tsp->tv_sec -= 1; 615 } 616 } 617 #ifdef PPS_SYNC 618 if (fhard) { 619 /* 620 * Feed the NTP PLL/FLL. 621 * The FLL wants to know how many nanoseconds elapsed since 622 * the previous event. 623 * I have never been able to convince myself that this code 624 * is actually correct: Using th_scale is bound to contain 625 * a phase correction component from userland, when running 626 * as FLL, so the number hardpps() gets is not meaningful IMO. 627 */ 628 tcount = pps->capcount - pps->ppscount[2]; 629 pps->ppscount[2] = pps->capcount; 630 tcount &= pps->capth->th_counter->tc_counter_mask; 631 bt.sec = 0; 632 bt.frac = 0; 633 bintime_addx(&bt, pps->capth->th_scale * tcount); 634 bintime2timespec(&bt, &ts); 635 hardpps(tsp, ts.tv_nsec + 1000000000 * ts.tv_sec); 636 } 637 #endif 638 } 639 640 /* 641 * Timecounters need to be updated every so often to prevent the hardware 642 * counter from overflowing. Updating also recalculates the cached values 643 * used by the get*() family of functions, so their precision depends on 644 * the update frequency. 645 */ 646 647 static int tc_tick; 648 SYSCTL_INT(_kern_timecounter, OID_AUTO, tick, CTLFLAG_RD, &tick, 0, ""); 649 650 static void 651 tc_ticktock(void *dummy) 652 { 653 654 tc_windup(); 655 timeout(tc_ticktock, NULL, tc_tick); 656 } 657 658 static void 659 inittimecounter(void *dummy) 660 { 661 u_int p; 662 663 /* 664 * Set the initial timeout to 665 * max(1, <approx. number of hardclock ticks in a millisecond>). 666 * People should probably not use the sysctl to set the timeout 667 * to smaller than its inital value, since that value is the 668 * smallest reasonable one. If they want better timestamps they 669 * should use the non-"get"* functions. 670 */ 671 if (hz > 1000) 672 tc_tick = (hz + 500) / 1000; 673 else 674 tc_tick = 1; 675 p = (tc_tick * 1000000) / hz; 676 printf("Timecounters tick every %d.%03u msec\n", p / 1000, p % 1000); 677 678 /* warm up new timecounter (again) and get rolling. */ 679 (void)timecounter->tc_get_timecount(timecounter); 680 (void)timecounter->tc_get_timecount(timecounter); 681 tc_ticktock(NULL); 682 } 683 684 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_FIRST, inittimecounter, NULL) 685