1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Kernel timekeeping code and accessor functions. Based on code from 4 * timer.c, moved in commit 8524070b7982. 5 */ 6 #include <linux/audit.h> 7 #include <linux/clocksource.h> 8 #include <linux/compiler.h> 9 #include <linux/jiffies.h> 10 #include <linux/kobject.h> 11 #include <linux/module.h> 12 #include <linux/nmi.h> 13 #include <linux/pvclock_gtod.h> 14 #include <linux/random.h> 15 #include <linux/sched/clock.h> 16 #include <linux/sched/loadavg.h> 17 #include <linux/static_key.h> 18 #include <linux/stop_machine.h> 19 #include <linux/syscore_ops.h> 20 #include <linux/tick.h> 21 #include <linux/time.h> 22 #include <linux/timex.h> 23 #include <linux/timekeeper_internal.h> 24 25 #include <vdso/auxclock.h> 26 27 #include "tick-internal.h" 28 #include "timekeeping_internal.h" 29 #include "ntp_internal.h" 30 31 #define TK_CLEAR_NTP (1 << 0) 32 #define TK_CLOCK_WAS_SET (1 << 1) 33 34 #define TK_UPDATE_ALL (TK_CLEAR_NTP | TK_CLOCK_WAS_SET) 35 36 enum timekeeping_adv_mode { 37 /* Update timekeeper when a tick has passed */ 38 TK_ADV_TICK, 39 40 /* Update timekeeper on a direct frequency change */ 41 TK_ADV_FREQ 42 }; 43 44 /* 45 * The most important data for readout fits into a single 64 byte 46 * cache line. 47 */ 48 struct tk_data { 49 seqcount_raw_spinlock_t seq; 50 struct timekeeper timekeeper; 51 struct timekeeper shadow_timekeeper; 52 raw_spinlock_t lock; 53 } ____cacheline_aligned; 54 55 static struct tk_data timekeeper_data[TIMEKEEPERS_MAX]; 56 57 /* The core timekeeper */ 58 #define tk_core (timekeeper_data[TIMEKEEPER_CORE]) 59 60 #ifdef CONFIG_POSIX_AUX_CLOCKS 61 static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts) 62 { 63 return ktime_get_aux_ts64(CLOCK_AUX + tkid - TIMEKEEPER_AUX_FIRST, ts); 64 } 65 66 static inline bool tk_is_aux(const struct timekeeper *tk) 67 { 68 return tk->id >= TIMEKEEPER_AUX_FIRST && tk->id <= TIMEKEEPER_AUX_LAST; 69 } 70 static inline struct tk_data *aux_get_tk_data(clockid_t id); 71 #else 72 static inline bool tk_get_aux_ts64(unsigned int tkid, struct timespec64 *ts) 73 { 74 return false; 75 } 76 77 static inline bool tk_is_aux(const struct timekeeper *tk) 78 { 79 return false; 80 } 81 static inline struct tk_data *aux_get_tk_data(clockid_t id) 82 { 83 return NULL; 84 } 85 #endif 86 87 static inline void tk_update_aux_offs(struct timekeeper *tk, ktime_t offs) 88 { 89 tk->offs_aux = offs; 90 tk->monotonic_to_aux = ktime_to_timespec64(offs); 91 } 92 93 /* flag for if timekeeping is suspended */ 94 int __read_mostly timekeeping_suspended; 95 96 /** 97 * struct tk_fast - NMI safe timekeeper 98 * @seq: Sequence counter for protecting updates. The lowest bit 99 * is the index for the tk_read_base array 100 * @base: tk_read_base array. Access is indexed by the lowest bit of 101 * @seq. 102 * 103 * See @update_fast_timekeeper() below. 104 */ 105 struct tk_fast { 106 seqcount_latch_t seq; 107 struct tk_read_base base[2]; 108 }; 109 110 /* Suspend-time cycles value for halted fast timekeeper. */ 111 static u64 cycles_at_suspend; 112 113 static u64 dummy_clock_read(struct clocksource *cs) 114 { 115 if (timekeeping_suspended) 116 return cycles_at_suspend; 117 return local_clock(); 118 } 119 120 static struct clocksource dummy_clock = { 121 .read = dummy_clock_read, 122 }; 123 124 /* 125 * Boot time initialization which allows local_clock() to be utilized 126 * during early boot when clocksources are not available. local_clock() 127 * returns nanoseconds already so no conversion is required, hence mult=1 128 * and shift=0. When the first proper clocksource is installed then 129 * the fast time keepers are updated with the correct values. 130 */ 131 #define FAST_TK_INIT \ 132 { \ 133 .clock = &dummy_clock, \ 134 .mask = CLOCKSOURCE_MASK(64), \ 135 .mult = 1, \ 136 .shift = 0, \ 137 } 138 139 static struct tk_fast tk_fast_mono ____cacheline_aligned = { 140 .seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq), 141 .base[0] = FAST_TK_INIT, 142 .base[1] = FAST_TK_INIT, 143 }; 144 145 static struct tk_fast tk_fast_raw ____cacheline_aligned = { 146 .seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq), 147 .base[0] = FAST_TK_INIT, 148 .base[1] = FAST_TK_INIT, 149 }; 150 151 #ifdef CONFIG_POSIX_AUX_CLOCKS 152 static __init void tk_aux_setup(void); 153 static void tk_aux_update_clocksource(void); 154 static void tk_aux_advance(void); 155 #else 156 static inline void tk_aux_setup(void) { } 157 static inline void tk_aux_update_clocksource(void) { } 158 static inline void tk_aux_advance(void) { } 159 #endif 160 161 unsigned long timekeeper_lock_irqsave(void) 162 { 163 unsigned long flags; 164 165 raw_spin_lock_irqsave(&tk_core.lock, flags); 166 return flags; 167 } 168 169 void timekeeper_unlock_irqrestore(unsigned long flags) 170 { 171 raw_spin_unlock_irqrestore(&tk_core.lock, flags); 172 } 173 174 /* 175 * Multigrain timestamps require tracking the latest fine-grained timestamp 176 * that has been issued, and never returning a coarse-grained timestamp that is 177 * earlier than that value. 178 * 179 * mg_floor represents the latest fine-grained time that has been handed out as 180 * a file timestamp on the system. This is tracked as a monotonic ktime_t, and 181 * converted to a realtime clock value on an as-needed basis. 182 * 183 * Maintaining mg_floor ensures the multigrain interfaces never issue a 184 * timestamp earlier than one that has been previously issued. 185 * 186 * The exception to this rule is when there is a backward realtime clock jump. If 187 * such an event occurs, a timestamp can appear to be earlier than a previous one. 188 */ 189 static __cacheline_aligned_in_smp atomic64_t mg_floor; 190 191 static inline void tk_normalize_xtime(struct timekeeper *tk) 192 { 193 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { 194 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 195 tk->xtime_sec++; 196 } 197 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) { 198 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 199 tk->raw_sec++; 200 } 201 } 202 203 static inline struct timespec64 tk_xtime(const struct timekeeper *tk) 204 { 205 struct timespec64 ts; 206 207 ts.tv_sec = tk->xtime_sec; 208 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 209 return ts; 210 } 211 212 static inline struct timespec64 tk_xtime_coarse(const struct timekeeper *tk) 213 { 214 struct timespec64 ts; 215 216 ts.tv_sec = tk->xtime_sec; 217 ts.tv_nsec = tk->coarse_nsec; 218 return ts; 219 } 220 221 /* 222 * Update the nanoseconds part for the coarse time keepers. They can't rely 223 * on xtime_nsec because xtime_nsec could be adjusted by a small negative 224 * amount when the multiplication factor of the clock is adjusted, which 225 * could cause the coarse clocks to go slightly backwards. See 226 * timekeeping_apply_adjustment(). Thus we keep a separate copy for the coarse 227 * clockids which only is updated when the clock has been set or we have 228 * accumulated time. 229 */ 230 static inline void tk_update_coarse_nsecs(struct timekeeper *tk) 231 { 232 tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift; 233 } 234 235 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) 236 { 237 tk->xtime_sec = ts->tv_sec; 238 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; 239 tk_update_coarse_nsecs(tk); 240 } 241 242 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) 243 { 244 tk->xtime_sec += ts->tv_sec; 245 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; 246 tk_normalize_xtime(tk); 247 tk_update_coarse_nsecs(tk); 248 } 249 250 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) 251 { 252 struct timespec64 tmp; 253 254 /* 255 * Verify consistency of: offset_real = -wall_to_monotonic 256 * before modifying anything 257 */ 258 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, 259 -tk->wall_to_monotonic.tv_nsec); 260 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp)); 261 tk->wall_to_monotonic = wtm; 262 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec); 263 /* Paired with READ_ONCE() in ktime_mono_to_any() */ 264 WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp)); 265 WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0))); 266 } 267 268 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) 269 { 270 /* Paired with READ_ONCE() in ktime_mono_to_any() */ 271 WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta)); 272 /* 273 * Timespec representation for VDSO update to avoid 64bit division 274 * on every update. 275 */ 276 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot); 277 } 278 279 #ifdef CONFIG_ARCH_WANTS_CLOCKSOURCE_READ_INLINE 280 #include <asm/clock_inlined.h> 281 282 static DEFINE_STATIC_KEY_FALSE(clocksource_read_inlined); 283 284 /* 285 * tk_clock_read - atomic clocksource read() helper 286 * 287 * This helper is necessary to use in the read paths because, while the 288 * seqcount ensures we don't return a bad value while structures are updated, 289 * it doesn't protect from potential crashes. There is the possibility that 290 * the tkr's clocksource may change between the read reference, and the 291 * clock reference passed to the read function. This can cause crashes if 292 * the wrong clocksource is passed to the wrong read function. 293 * This isn't necessary to use when holding the tk_core.lock or doing 294 * a read of the fast-timekeeper tkrs (which is protected by its own locking 295 * and update logic). 296 */ 297 static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr) 298 { 299 struct clocksource *clock = READ_ONCE(tkr->clock); 300 301 if (static_branch_likely(&clocksource_read_inlined)) 302 return arch_inlined_clocksource_read(clock); 303 304 return clock->read(clock); 305 } 306 307 static inline void clocksource_disable_inline_read(void) 308 { 309 static_branch_disable(&clocksource_read_inlined); 310 } 311 312 static inline void clocksource_enable_inline_read(void) 313 { 314 static_branch_enable(&clocksource_read_inlined); 315 } 316 #else 317 static __always_inline u64 tk_clock_read(const struct tk_read_base *tkr) 318 { 319 struct clocksource *clock = READ_ONCE(tkr->clock); 320 321 return clock->read(clock); 322 } 323 324 static inline void clocksource_disable_inline_read(void) { } 325 static inline void clocksource_enable_inline_read(void) { } 326 #endif 327 328 /** 329 * tk_setup_internals - Set up internals to use clocksource clock. 330 * 331 * @tk: The target timekeeper to setup. 332 * @clock: Pointer to clocksource. 333 * 334 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment 335 * pair and interval request. 336 * 337 * Unless you're the timekeeping code, you should not be using this! 338 */ 339 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) 340 { 341 u64 interval; 342 struct clocksource *old_clock; 343 344 ++tk->cs_was_changed_seq; 345 old_clock = tk->tkr_mono.clock; 346 tk->tkr_mono.clock = clock; 347 tk->tkr_mono.mask = clock->mask; 348 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono); 349 350 tk->tkr_raw.clock = clock; 351 tk->tkr_raw.mask = clock->mask; 352 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; 353 354 /* Do the ns -> cycle conversion first, using original mult */ 355 interval = (u64)NTP_INTERVAL_LENGTH << clock->shift; 356 interval += clock->mult / 2; 357 do_div(interval, clock->mult); 358 if (interval == 0) 359 interval = 1; 360 361 tk->cycle_interval = interval; 362 363 /* Go back from cycles -> shifted ns */ 364 tk->xtime_interval = interval * clock->mult; 365 tk->raw_interval = interval * clock->mult; 366 367 /* if changing clocks, convert xtime_nsec shift units */ 368 if (old_clock) { 369 int shift_change = clock->shift - old_clock->shift; 370 if (shift_change < 0) { 371 tk->tkr_mono.xtime_nsec >>= -shift_change; 372 tk->tkr_raw.xtime_nsec >>= -shift_change; 373 } else { 374 tk->tkr_mono.xtime_nsec <<= shift_change; 375 tk->tkr_raw.xtime_nsec <<= shift_change; 376 } 377 } 378 379 tk->tkr_mono.shift = clock->shift; 380 tk->tkr_raw.shift = clock->shift; 381 382 tk->ntp_error = 0; 383 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; 384 385 /* 386 * ntp_tick is the tick length that NTP disciplines (its ±500 PPM 387 * scales only this part), in NTP-shifted ns: the real interval of 388 * a whole number of counter cycles. Because cycle_interval is 389 * rounded to an integer number of cycles, this ntp_tick differs 390 * from the true intended 1/HZ tick length by up to half a cycle 391 * period. 392 */ 393 tk->ntp_tick = (u64)tk->xtime_interval << tk->ntp_error_shift; 394 395 /* 396 * cs_tick_adj is the constant difference between the disciplined 397 * ntp_tick above and the true 1/HZ tick, expressed per-second to 398 * match the ntp_update_frequency() addends and handed to NTP via 399 * ntp_clear() to be explicitly included in its tick_length. 400 * 401 * Worked example: HZ=1000, ACPI PM timer at 3.579545 MHz, which 402 * has 3579.545 cycles in 1ms, rounded to cycle_interval = 3580. 403 * 404 * So ntp_tick is actually 1.000127ms, as that is the amount of 405 * time that 3580 cycles will take at the nominal frequency. This 406 * is the part that NTP disciplines, causing each 3580 counts to 407 * advance the clock by up to NTP's ±500PPM of that amount. 408 * 409 * The "extra" 127ns/tick is what's stored in cs_tick_adj and 410 * applied as a constant correction by ntp_update_frequency() so 411 * that NTP *believes* it's disciplining a 1ms tick. 412 */ 413 tk->cs_tick_adj = (s64)tk->ntp_tick - 414 ((s64)NTP_INTERVAL_LENGTH << NTP_SCALE_SHIFT); 415 tk->cs_tick_adj *= NTP_INTERVAL_FREQ; 416 417 /* 418 * The timekeeper keeps its own mult values for the currently 419 * active clocksource. These value will be adjusted via NTP 420 * to counteract clock drifting. 421 */ 422 tk->tkr_mono.mult = clock->mult; 423 tk->tkr_raw.mult = clock->mult; 424 tk->ntp_err_mult = 0; 425 tk->skip_second_overflow = 0; 426 tk->skew_delta = 0; 427 428 tk->cs_id = clock->id; 429 430 /* Coupled clockevent data */ 431 if (IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) && 432 clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT) { 433 /* 434 * Aim for an one hour maximum delta and use KHz to handle 435 * clocksources with a frequency above 4GHz correctly as 436 * the frequency argument of clocks_calc_mult_shift() is u32. 437 */ 438 clocks_calc_mult_shift(&tk->cs_ns_to_cyc_mult, &tk->cs_ns_to_cyc_shift, 439 NSEC_PER_MSEC, clock->freq_khz, 3600 * 1000); 440 /* 441 * Initialize the conversion limit as the previous clocksource 442 * might have the same shift/mult pair so the quick check in 443 * tk_update_ns_to_cyc() fails to update it after a clocksource 444 * change leaving it effectivly zero. 445 */ 446 tk->cs_ns_to_cyc_maxns = div_u64(clock->mask, tk->cs_ns_to_cyc_mult); 447 } 448 } 449 450 /* Timekeeper helper functions. */ 451 static noinline u64 delta_to_ns_safe(const struct tk_read_base *tkr, u64 delta) 452 { 453 return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift); 454 } 455 456 static __always_inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles) 457 { 458 /* Calculate the delta since the last update_wall_time() */ 459 u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask; 460 461 /* 462 * This detects both negative motion and the case where the delta 463 * overflows the multiplication with tkr->mult. 464 */ 465 if (unlikely(delta > tkr->clock->max_cycles)) { 466 /* 467 * Handle clocksource inconsistency between CPUs to prevent 468 * time from going backwards by checking for the MSB of the 469 * mask being set in the delta. 470 */ 471 if (delta & ~(mask >> 1)) 472 return tkr->xtime_nsec >> tkr->shift; 473 474 return delta_to_ns_safe(tkr, delta); 475 } 476 477 return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift; 478 } 479 480 static __always_inline u64 timekeeping_get_ns(const struct tk_read_base *tkr) 481 { 482 return timekeeping_cycles_to_ns(tkr, tk_clock_read(tkr)); 483 } 484 485 /** 486 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper. 487 * @tkr: Timekeeping readout base from which we take the update 488 * @tkf: Pointer to NMI safe timekeeper 489 * 490 * We want to use this from any context including NMI and tracing / 491 * instrumenting the timekeeping code itself. 492 * 493 * Employ the latch technique; see @write_seqcount_latch. 494 * 495 * So if a NMI hits the update of base[0] then it will use base[1] 496 * which is still consistent. In the worst case this can result is a 497 * slightly wrong timestamp (a few nanoseconds). See 498 * @ktime_get_mono_fast_ns. 499 */ 500 static void update_fast_timekeeper(const struct tk_read_base *tkr, 501 struct tk_fast *tkf) 502 { 503 struct tk_read_base *base = tkf->base; 504 505 /* Force readers off to base[1] */ 506 write_seqcount_latch_begin(&tkf->seq); 507 508 /* Update base[0] */ 509 memcpy(base, tkr, sizeof(*base)); 510 511 /* Force readers back to base[0] */ 512 write_seqcount_latch(&tkf->seq); 513 514 /* Update base[1] */ 515 memcpy(base + 1, base, sizeof(*base)); 516 517 write_seqcount_latch_end(&tkf->seq); 518 } 519 520 static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf) 521 { 522 struct tk_read_base *tkr; 523 unsigned int seq; 524 u64 now; 525 526 do { 527 seq = read_seqcount_latch(&tkf->seq); 528 tkr = tkf->base + (seq & 0x01); 529 now = ktime_to_ns(tkr->base); 530 now += timekeeping_get_ns(tkr); 531 } while (read_seqcount_latch_retry(&tkf->seq, seq)); 532 533 return now; 534 } 535 536 /** 537 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic 538 * 539 * This timestamp is not guaranteed to be monotonic across an update. 540 * The timestamp is calculated by: 541 * 542 * now = base_mono + clock_delta * slope 543 * 544 * So if the update lowers the slope, readers who are forced to the 545 * not yet updated second array are still using the old steeper slope. 546 * 547 * tmono 548 * ^ 549 * | o n 550 * | o n 551 * | u 552 * | o 553 * |o 554 * |12345678---> reader order 555 * 556 * o = old slope 557 * u = update 558 * n = new slope 559 * 560 * So reader 6 will observe time going backwards versus reader 5. 561 * 562 * While other CPUs are likely to be able to observe that, the only way 563 * for a CPU local observation is when an NMI hits in the middle of 564 * the update. Timestamps taken from that NMI context might be ahead 565 * of the following timestamps. Callers need to be aware of that and 566 * deal with it. 567 */ 568 u64 notrace ktime_get_mono_fast_ns(void) 569 { 570 return __ktime_get_fast_ns(&tk_fast_mono); 571 } 572 EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns); 573 574 /** 575 * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw 576 * 577 * Contrary to ktime_get_mono_fast_ns() this is always correct because the 578 * conversion factor is not affected by NTP/PTP correction. 579 */ 580 u64 notrace ktime_get_raw_fast_ns(void) 581 { 582 return __ktime_get_fast_ns(&tk_fast_raw); 583 } 584 EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns); 585 586 /** 587 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock. 588 * 589 * To keep it NMI safe since we're accessing from tracing, we're not using a 590 * separate timekeeper with updates to monotonic clock and boot offset 591 * protected with seqcounts. This has the following minor side effects: 592 * 593 * (1) Its possible that a timestamp be taken after the boot offset is updated 594 * but before the timekeeper is updated. If this happens, the new boot offset 595 * is added to the old timekeeping making the clock appear to update slightly 596 * earlier: 597 * CPU 0 CPU 1 598 * timekeeping_inject_sleeptime64() 599 * __timekeeping_inject_sleeptime(tk, delta); 600 * timestamp(); 601 * timekeeping_update_staged(tkd, TK_CLEAR_NTP...); 602 * 603 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be 604 * partially updated. Since the tk->offs_boot update is a rare event, this 605 * should be a rare occurrence which postprocessing should be able to handle. 606 * 607 * The caveats vs. timestamp ordering as documented for ktime_get_mono_fast_ns() 608 * apply as well. 609 */ 610 u64 notrace ktime_get_boot_fast_ns(void) 611 { 612 struct timekeeper *tk = &tk_core.timekeeper; 613 614 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot))); 615 } 616 EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns); 617 618 /** 619 * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock. 620 * 621 * The same limitations as described for ktime_get_boot_fast_ns() apply. The 622 * mono time and the TAI offset are not read atomically which may yield wrong 623 * readouts. However, an update of the TAI offset is an rare event e.g., caused 624 * by settime or adjtimex with an offset. The user of this function has to deal 625 * with the possibility of wrong timestamps in post processing. 626 */ 627 u64 notrace ktime_get_tai_fast_ns(void) 628 { 629 struct timekeeper *tk = &tk_core.timekeeper; 630 631 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai))); 632 } 633 EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns); 634 635 /** 636 * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime. 637 * 638 * See ktime_get_mono_fast_ns() for documentation of the time stamp ordering. 639 */ 640 u64 ktime_get_real_fast_ns(void) 641 { 642 struct tk_fast *tkf = &tk_fast_mono; 643 struct tk_read_base *tkr; 644 u64 baser, delta; 645 unsigned int seq; 646 647 do { 648 seq = raw_read_seqcount_latch(&tkf->seq); 649 tkr = tkf->base + (seq & 0x01); 650 baser = ktime_to_ns(tkr->base_real); 651 delta = timekeeping_get_ns(tkr); 652 } while (raw_read_seqcount_latch_retry(&tkf->seq, seq)); 653 654 return baser + delta; 655 } 656 EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns); 657 658 /** 659 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource. 660 * @tk: Timekeeper to snapshot. 661 * 662 * It generally is unsafe to access the clocksource after timekeeping has been 663 * suspended, so take a snapshot of the readout base of @tk and use it as the 664 * fast timekeeper's readout base while suspended. It will return the same 665 * number of cycles every time until timekeeping is resumed at which time the 666 * proper readout base for the fast timekeeper will be restored automatically. 667 */ 668 static void halt_fast_timekeeper(const struct timekeeper *tk) 669 { 670 static struct tk_read_base tkr_dummy; 671 const struct tk_read_base *tkr = &tk->tkr_mono; 672 673 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy)); 674 cycles_at_suspend = tk_clock_read(tkr); 675 tkr_dummy.clock = &dummy_clock; 676 tkr_dummy.base_real = tkr->base + tk->offs_real; 677 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono); 678 679 tkr = &tk->tkr_raw; 680 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy)); 681 tkr_dummy.clock = &dummy_clock; 682 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw); 683 } 684 685 static RAW_NOTIFIER_HEAD(pvclock_gtod_chain); 686 687 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) 688 { 689 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); 690 } 691 692 /** 693 * pvclock_gtod_register_notifier - register a pvclock timedata update listener 694 * @nb: Pointer to the notifier block to register 695 */ 696 int pvclock_gtod_register_notifier(struct notifier_block *nb) 697 { 698 struct timekeeper *tk = &tk_core.timekeeper; 699 int ret; 700 701 guard(raw_spinlock_irqsave)(&tk_core.lock); 702 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb); 703 update_pvclock_gtod(tk, true); 704 705 return ret; 706 } 707 EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier); 708 709 /** 710 * pvclock_gtod_unregister_notifier - unregister a pvclock 711 * timedata update listener 712 * @nb: Pointer to the notifier block to unregister 713 */ 714 int pvclock_gtod_unregister_notifier(struct notifier_block *nb) 715 { 716 guard(raw_spinlock_irqsave)(&tk_core.lock); 717 return raw_notifier_chain_unregister(&pvclock_gtod_chain, nb); 718 } 719 EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier); 720 721 /* 722 * tk_update_leap_state - helper to update the next_leap_ktime 723 */ 724 static inline void tk_update_leap_state(struct timekeeper *tk) 725 { 726 tk->next_leap_ktime = ntp_get_next_leap(tk->id); 727 if (tk->next_leap_ktime != KTIME_MAX) 728 /* Convert to monotonic time */ 729 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); 730 } 731 732 /* 733 * Leap state update for both shadow and the real timekeeper 734 * Separate to spare a full memcpy() of the timekeeper. 735 */ 736 static void tk_update_leap_state_all(struct tk_data *tkd) 737 { 738 write_seqcount_begin(&tkd->seq); 739 tk_update_leap_state(&tkd->shadow_timekeeper); 740 tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime; 741 write_seqcount_end(&tkd->seq); 742 } 743 744 /* 745 * Update the ktime_t based scalar nsec members of the timekeeper 746 */ 747 static inline void tk_update_ktime_data(struct timekeeper *tk) 748 { 749 u64 seconds; 750 u32 nsec; 751 752 /* 753 * The xtime based monotonic readout is: 754 * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now(); 755 * The ktime based monotonic readout is: 756 * nsec = base_mono + now(); 757 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec 758 */ 759 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); 760 nsec = (u32) tk->wall_to_monotonic.tv_nsec; 761 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); 762 763 /* 764 * The sum of the nanoseconds portions of xtime and 765 * wall_to_monotonic can be greater/equal one second. Take 766 * this into account before updating tk->ktime_sec. 767 */ 768 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); 769 if (nsec >= NSEC_PER_SEC) 770 seconds++; 771 tk->ktime_sec = seconds; 772 773 /* Update the monotonic raw base */ 774 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC); 775 } 776 777 static inline void tk_update_ns_to_cyc(struct timekeeper *tks, struct timekeeper *tkc) 778 { 779 struct tk_read_base *tkrs = &tks->tkr_mono; 780 struct tk_read_base *tkrc = &tkc->tkr_mono; 781 unsigned int shift; 782 783 if (!IS_ENABLED(CONFIG_GENERIC_CLOCKEVENTS_COUPLED) || 784 !(tkrs->clock->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT)) 785 return; 786 787 if (tkrs->mult == tkrc->mult && tkrs->shift == tkrc->shift) 788 return; 789 /* 790 * The conversion math is simple: 791 * 792 * CS::MULT (1 << NS_TO_CYC_SHIFT) 793 * --------------- = ---------------------- 794 * (1 << CS:SHIFT) NS_TO_CYC_MULT 795 * 796 * Ergo: 797 * 798 * NS_TO_CYC_MULT = (1 << (CS::SHIFT + NS_TO_CYC_SHIFT)) / CS::MULT 799 * 800 * NS_TO_CYC_SHIFT has been set up in tk_setup_internals() 801 */ 802 shift = tkrs->shift + tks->cs_ns_to_cyc_shift; 803 tks->cs_ns_to_cyc_mult = (u32)div_u64(1ULL << shift, tkrs->mult); 804 tks->cs_ns_to_cyc_maxns = div_u64(tkrs->clock->mask, tks->cs_ns_to_cyc_mult); 805 } 806 807 /* 808 * Restore the shadow timekeeper from the real timekeeper. 809 */ 810 static void timekeeping_restore_shadow(struct tk_data *tkd) 811 { 812 lockdep_assert_held(&tkd->lock); 813 memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper)); 814 } 815 816 static void timekeeping_update_from_shadow(struct tk_data *tkd, unsigned int action) 817 { 818 struct timekeeper *tk = &tkd->shadow_timekeeper; 819 820 lockdep_assert_held(&tkd->lock); 821 822 /* 823 * Block out readers before running the updates below because that 824 * updates VDSO and other time related infrastructure. Not blocking 825 * the readers might let a reader see time going backwards when 826 * reading from the VDSO after the VDSO update and then reading in 827 * the kernel from the timekeeper before that got updated. 828 */ 829 write_seqcount_begin(&tkd->seq); 830 831 if (action & TK_CLEAR_NTP) { 832 tk->ntp_error = 0; 833 ntp_clear(tk->id, tk->cs_tick_adj); 834 } 835 836 tk_update_leap_state(tk); 837 tk_update_ktime_data(tk); 838 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real; 839 840 if (tk->id == TIMEKEEPER_CORE) { 841 tk_update_ns_to_cyc(tk, &tkd->timekeeper); 842 update_vsyscall(tk); 843 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); 844 845 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); 846 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); 847 } else if (tk_is_aux(tk)) { 848 vdso_time_update_aux(tk); 849 } 850 851 if (action & TK_CLOCK_WAS_SET) 852 tk->clock_was_set_seq++; 853 854 /* 855 * Update the real timekeeper. 856 * 857 * We could avoid this memcpy() by switching pointers, but that has 858 * the downside that the reader side does not longer benefit from 859 * the cacheline optimized data layout of the timekeeper and requires 860 * another indirection. 861 * 862 * Write xtime_sec first so that even if the memcpy() tears the store 863 * data integrity is provided for ktime_get_real_seconds(). 864 * The same goes for ktime_sec and ktime_get_seconds(). 865 */ 866 WRITE_ONCE(tkd->timekeeper.xtime_sec, tk->xtime_sec); 867 WRITE_ONCE(tkd->timekeeper.ktime_sec, tk->ktime_sec); 868 memcpy(&tkd->timekeeper, tk, sizeof(*tk)); 869 write_seqcount_end(&tkd->seq); 870 } 871 872 /** 873 * timekeeping_forward_now - update clock to the current time 874 * @tk: Pointer to the timekeeper to update 875 * 876 * Forward the current clock to update its state since the last call to 877 * update_wall_time(). This is useful before significant clock changes, 878 * as it avoids having to deal with this time offset explicitly. 879 */ 880 static void timekeeping_forward_now(struct timekeeper *tk) 881 { 882 u64 cycle_now, delta; 883 884 cycle_now = tk_clock_read(&tk->tkr_mono); 885 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask, 886 tk->tkr_mono.clock->max_raw_delta); 887 tk->tkr_mono.cycle_last = cycle_now; 888 tk->tkr_raw.cycle_last = cycle_now; 889 890 while (delta > 0) { 891 u64 max = tk->tkr_mono.clock->max_cycles; 892 u64 incr = delta < max ? delta : max; 893 894 tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult; 895 tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult; 896 tk_normalize_xtime(tk); 897 delta -= incr; 898 } 899 tk_update_coarse_nsecs(tk); 900 } 901 902 /* 903 * ktime_expiry_to_cycles - Convert a expiry time to clocksource cycles 904 * @id: Clocksource ID which is required for validity 905 * @expires_ns: Absolute CLOCK_MONOTONIC expiry time (nsecs) to be converted 906 * @cycles: Pointer to storage for corresponding absolute cycles value 907 * 908 * Convert a CLOCK_MONOTONIC based absolute expiry time to a cycles value 909 * based on the correlated clocksource of the clockevent device by using 910 * the base nanoseconds and cycles values of the last timekeeper update and 911 * converting the delta between @expires_ns and base nanoseconds to cycles. 912 * 913 * This only works for clockevent devices which are using a less than or 914 * equal comparator against the clocksource. 915 * 916 * Utilizing this avoids two clocksource reads for such devices, the 917 * ktime_get() in clockevents_program_event() to calculate the delta expiry 918 * value and the readout in the device::set_next_event() callback to 919 * convert the delta back to a absolute comparator value. 920 * 921 * Returns: True if @id matches the current clocksource ID, false otherwise 922 */ 923 bool ktime_expiry_to_cycles(enum clocksource_ids id, ktime_t expires_ns, u64 *cycles) 924 { 925 struct timekeeper *tk = &tk_core.timekeeper; 926 struct tk_read_base *tkrm = &tk->tkr_mono; 927 ktime_t base_ns, delta_ns, max_ns; 928 u64 base_cycles, delta_cycles; 929 unsigned int seq; 930 u32 mult, shift; 931 932 /* 933 * Racy check to avoid the seqcount overhead when ID does not match. If 934 * the relevant clocksource is installed concurrently, then this will 935 * just delay the switch over to this mechanism until the next event is 936 * programmed. If the ID is not matching the clock events code will use 937 * the regular relative set_next_event() callback as before. 938 */ 939 if (data_race(tk->cs_id) != id) 940 return false; 941 942 do { 943 seq = read_seqcount_begin(&tk_core.seq); 944 945 if (tk->cs_id != id) 946 return false; 947 948 base_cycles = tkrm->cycle_last; 949 base_ns = tkrm->base + (tkrm->xtime_nsec >> tkrm->shift); 950 951 mult = tk->cs_ns_to_cyc_mult; 952 shift = tk->cs_ns_to_cyc_shift; 953 max_ns = tk->cs_ns_to_cyc_maxns; 954 955 } while (read_seqcount_retry(&tk_core.seq, seq)); 956 957 /* Prevent negative deltas and multiplication overflows */ 958 delta_ns = min(expires_ns - base_ns, max_ns); 959 delta_ns = max(delta_ns, 0); 960 961 /* Convert to cycles */ 962 delta_cycles = ((u64)delta_ns * mult) >> shift; 963 *cycles = base_cycles + delta_cycles; 964 return true; 965 } 966 967 /** 968 * ktime_get_real_ts64 - Returns the time of day in a timespec64. 969 * @ts: pointer to the timespec to be set 970 * 971 * Returns the time of day in a timespec64 (WARN if suspended). 972 */ 973 void ktime_get_real_ts64(struct timespec64 *ts) 974 { 975 struct timekeeper *tk = &tk_core.timekeeper; 976 unsigned int seq; 977 u64 nsecs; 978 979 WARN_ON(timekeeping_suspended); 980 981 do { 982 seq = read_seqcount_begin(&tk_core.seq); 983 984 ts->tv_sec = tk->xtime_sec; 985 nsecs = timekeeping_get_ns(&tk->tkr_mono); 986 987 } while (read_seqcount_retry(&tk_core.seq, seq)); 988 989 ts->tv_nsec = 0; 990 timespec64_add_ns(ts, nsecs); 991 } 992 EXPORT_SYMBOL(ktime_get_real_ts64); 993 994 ktime_t ktime_get(void) 995 { 996 struct timekeeper *tk = &tk_core.timekeeper; 997 unsigned int seq; 998 ktime_t base; 999 u64 nsecs; 1000 1001 WARN_ON(timekeeping_suspended); 1002 1003 do { 1004 seq = read_seqcount_begin(&tk_core.seq); 1005 base = tk->tkr_mono.base; 1006 nsecs = timekeeping_get_ns(&tk->tkr_mono); 1007 1008 } while (read_seqcount_retry(&tk_core.seq, seq)); 1009 1010 return ktime_add_ns(base, nsecs); 1011 } 1012 EXPORT_SYMBOL_GPL(ktime_get); 1013 1014 u32 ktime_get_resolution_ns(void) 1015 { 1016 struct timekeeper *tk = &tk_core.timekeeper; 1017 unsigned int seq; 1018 u32 nsecs; 1019 1020 WARN_ON(timekeeping_suspended); 1021 1022 do { 1023 seq = read_seqcount_begin(&tk_core.seq); 1024 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; 1025 } while (read_seqcount_retry(&tk_core.seq, seq)); 1026 1027 return nsecs; 1028 } 1029 EXPORT_SYMBOL_GPL(ktime_get_resolution_ns); 1030 1031 static const ktime_t *const offsets[TK_OFFS_MAX] = { 1032 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real, 1033 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot, 1034 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai, 1035 }; 1036 1037 ktime_t ktime_get_with_offset(enum tk_offsets offs) 1038 { 1039 struct timekeeper *tk = &tk_core.timekeeper; 1040 const ktime_t *offset = offsets[offs]; 1041 unsigned int seq; 1042 ktime_t base; 1043 u64 nsecs; 1044 1045 WARN_ON(timekeeping_suspended); 1046 1047 do { 1048 seq = read_seqcount_begin(&tk_core.seq); 1049 base = ktime_add(tk->tkr_mono.base, *offset); 1050 nsecs = timekeeping_get_ns(&tk->tkr_mono); 1051 1052 } while (read_seqcount_retry(&tk_core.seq, seq)); 1053 1054 return ktime_add_ns(base, nsecs); 1055 1056 } 1057 EXPORT_SYMBOL_GPL(ktime_get_with_offset); 1058 1059 ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs) 1060 { 1061 struct timekeeper *tk = &tk_core.timekeeper; 1062 const ktime_t *offset = offsets[offs]; 1063 unsigned int seq; 1064 ktime_t base; 1065 u64 nsecs; 1066 1067 WARN_ON(timekeeping_suspended); 1068 1069 do { 1070 seq = read_seqcount_begin(&tk_core.seq); 1071 base = ktime_add(tk->tkr_mono.base, *offset); 1072 nsecs = tk->coarse_nsec; 1073 1074 } while (read_seqcount_retry(&tk_core.seq, seq)); 1075 1076 return ktime_add_ns(base, nsecs); 1077 } 1078 EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset); 1079 1080 /** 1081 * ktime_mono_to_any() - convert monotonic time to any other time 1082 * @tmono: time to convert. 1083 * @offs: which offset to use 1084 */ 1085 ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs) 1086 { 1087 const ktime_t *offset = offsets[offs]; 1088 unsigned int seq; 1089 ktime_t tconv; 1090 1091 if (IS_ENABLED(CONFIG_64BIT)) { 1092 /* 1093 * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and 1094 * tk_update_sleep_time(). 1095 */ 1096 return ktime_add(tmono, READ_ONCE(*offset)); 1097 } 1098 1099 do { 1100 seq = read_seqcount_begin(&tk_core.seq); 1101 tconv = ktime_add(tmono, *offset); 1102 } while (read_seqcount_retry(&tk_core.seq, seq)); 1103 1104 return tconv; 1105 } 1106 EXPORT_SYMBOL_GPL(ktime_mono_to_any); 1107 1108 /** 1109 * ktime_get_raw - Returns the raw monotonic time in ktime_t format 1110 */ 1111 ktime_t ktime_get_raw(void) 1112 { 1113 struct timekeeper *tk = &tk_core.timekeeper; 1114 unsigned int seq; 1115 ktime_t base; 1116 u64 nsecs; 1117 1118 do { 1119 seq = read_seqcount_begin(&tk_core.seq); 1120 base = tk->tkr_raw.base; 1121 nsecs = timekeeping_get_ns(&tk->tkr_raw); 1122 1123 } while (read_seqcount_retry(&tk_core.seq, seq)); 1124 1125 return ktime_add_ns(base, nsecs); 1126 } 1127 EXPORT_SYMBOL_GPL(ktime_get_raw); 1128 1129 /** 1130 * ktime_get_ts64 - get the monotonic clock in timespec64 format 1131 * @ts: pointer to timespec variable 1132 * 1133 * The function calculates the monotonic clock from the realtime 1134 * clock and the wall_to_monotonic offset and stores the result 1135 * in normalized timespec64 format in the variable pointed to by @ts. 1136 */ 1137 void ktime_get_ts64(struct timespec64 *ts) 1138 { 1139 struct timekeeper *tk = &tk_core.timekeeper; 1140 struct timespec64 tomono; 1141 unsigned int seq; 1142 u64 nsec; 1143 1144 WARN_ON(timekeeping_suspended); 1145 1146 do { 1147 seq = read_seqcount_begin(&tk_core.seq); 1148 ts->tv_sec = tk->xtime_sec; 1149 nsec = timekeeping_get_ns(&tk->tkr_mono); 1150 tomono = tk->wall_to_monotonic; 1151 1152 } while (read_seqcount_retry(&tk_core.seq, seq)); 1153 1154 ts->tv_sec += tomono.tv_sec; 1155 ts->tv_nsec = 0; 1156 timespec64_add_ns(ts, nsec + tomono.tv_nsec); 1157 } 1158 EXPORT_SYMBOL_GPL(ktime_get_ts64); 1159 1160 /** 1161 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC 1162 * 1163 * Returns the seconds portion of CLOCK_MONOTONIC with a single non 1164 * serialized read. tk->ktime_sec is of type 'unsigned long' so this 1165 * works on both 32 and 64 bit systems. On 32 bit systems the readout 1166 * covers ~136 years of uptime which should be enough to prevent 1167 * premature wrap arounds. 1168 */ 1169 time64_t ktime_get_seconds(void) 1170 { 1171 struct timekeeper *tk = &tk_core.timekeeper; 1172 1173 WARN_ON(timekeeping_suspended); 1174 return READ_ONCE(tk->ktime_sec); 1175 } 1176 EXPORT_SYMBOL_GPL(ktime_get_seconds); 1177 1178 /** 1179 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME 1180 * 1181 * Returns the wall clock seconds since 1970. 1182 * 1183 * For 64bit systems the fast access to tk->xtime_sec is preserved. On 1184 * 32bit systems the access must be protected with the sequence 1185 * counter to provide "atomic" access to the 64bit tk->xtime_sec 1186 * value. 1187 */ 1188 time64_t ktime_get_real_seconds(void) 1189 { 1190 struct timekeeper *tk = &tk_core.timekeeper; 1191 time64_t seconds; 1192 unsigned int seq; 1193 1194 if (IS_ENABLED(CONFIG_64BIT)) 1195 return READ_ONCE(tk->xtime_sec); 1196 1197 do { 1198 seq = read_seqcount_begin(&tk_core.seq); 1199 seconds = READ_ONCE(tk->xtime_sec); 1200 1201 } while (read_seqcount_retry(&tk_core.seq, seq)); 1202 1203 return seconds; 1204 } 1205 EXPORT_SYMBOL_GPL(ktime_get_real_seconds); 1206 1207 /** 1208 * __ktime_get_real_seconds - Unprotected access to CLOCK_REALTIME seconds 1209 * 1210 * The same as ktime_get_real_seconds() but without the sequence counter 1211 * protection. This function is used in restricted contexts like the x86 MCE 1212 * handler and in KGDB. It's unprotected on 32-bit vs. concurrent half 1213 * completed modification and only to be used for such critical contexts. 1214 * 1215 * Returns: Racy snapshot of the CLOCK_REALTIME seconds value 1216 */ 1217 noinstr time64_t __ktime_get_real_seconds(void) 1218 { 1219 struct timekeeper *tk = &tk_core.timekeeper; 1220 1221 return READ_ONCE(tk->xtime_sec); 1222 } 1223 1224 static inline u64 tk_clock_read_snapshot(const struct tk_read_base *tkr, 1225 struct clocksource_hw_snapshot *chs) 1226 { 1227 struct clocksource *clock = READ_ONCE(tkr->clock); 1228 1229 if (unlikely(clock->read_snapshot)) 1230 return clock->read_snapshot(clock, chs); 1231 1232 return clock->read(clock); 1233 } 1234 1235 1236 /** 1237 * ktime_get_snapshot_id - Simultaneously snapshot a given clock ID with 1238 * the corresponding monotonic raw and the underlying 1239 * clocksource counter value. 1240 * @clock_id: The clock ID to snapshot 1241 * @systime_snapshot: Pointer to struct receiving the system time snapshot 1242 * 1243 * For the system time keeping clocks (REALTIME, MONOTONIC and BOOTTIME) the 1244 * monotonic raw clock is CLOCK_MONOTONIC_RAW. For AUX clocks this is the 1245 * monotonic raw clock related to the AUX clock. These AUX clock related 1246 * monotonic raw clocks have a strict linear offset to the system time 1247 * CLOCK_MONOTONIC_RAW: 1248 * 1249 * MONOTONIC_RAW(AUX$N) = CLOCK_MONOTONIC_RAW(system) + offset(AUX$N) 1250 * 1251 * The offset is established when a AUX clock is initialized, but it is 1252 * currently not accessible. 1253 */ 1254 void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot) 1255 { 1256 ktime_t base_raw, base_sys, offs_sys, *offs, offs_zero = 0; 1257 u64 nsec_raw, nsec_sys, now; 1258 struct timekeeper *tk; 1259 struct tk_data *tkd; 1260 unsigned int seq; 1261 1262 /* Invalidate the snapshot for all failure cases */ 1263 systime_snapshot->valid = false; 1264 1265 if (WARN_ON_ONCE(timekeeping_suspended)) 1266 return; 1267 1268 switch (clock_id) { 1269 case CLOCK_REALTIME: 1270 tkd = &tk_core; 1271 offs = &tk_core.timekeeper.offs_real; 1272 break; 1273 /* Map RAW to MONOTONIC so the loop below is trivial */ 1274 case CLOCK_MONOTONIC_RAW: 1275 case CLOCK_MONOTONIC: 1276 tkd = &tk_core; 1277 offs = &offs_zero; 1278 break; 1279 case CLOCK_BOOTTIME: 1280 tkd = &tk_core; 1281 offs = &tk_core.timekeeper.offs_boot; 1282 break; 1283 case CLOCK_AUX ... CLOCK_AUX_LAST: 1284 tkd = aux_get_tk_data(clock_id); 1285 if (!tkd) 1286 return; 1287 offs = &tkd->timekeeper.offs_aux; 1288 break; 1289 default: 1290 WARN_ON_ONCE(1); 1291 return; 1292 } 1293 1294 tk = &tkd->timekeeper; 1295 1296 do { 1297 struct clocksource_hw_snapshot chs = { }; 1298 1299 seq = read_seqcount_begin(&tkd->seq); 1300 1301 /* Aux clocks can be invalid */ 1302 if (!tk->clock_valid) 1303 return; 1304 1305 now = tk_clock_read_snapshot(&tk->tkr_mono, &chs); 1306 systime_snapshot->cs_id = tk->tkr_mono.clock->id; 1307 1308 systime_snapshot->hw_cycles = chs.hw_cycles; 1309 systime_snapshot->hw_csid = chs.hw_csid; 1310 1311 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq; 1312 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq; 1313 1314 base_sys = tk->tkr_mono.base; 1315 offs_sys = *offs; 1316 base_raw = tk->tkr_raw.base; 1317 1318 nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, now); 1319 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now); 1320 } while (read_seqcount_retry(&tkd->seq, seq)); 1321 1322 systime_snapshot->cycles = now; 1323 systime_snapshot->systime = ktime_add_ns(base_sys, offs_sys + nsec_sys); 1324 systime_snapshot->monoraw = ktime_add_ns(base_raw, nsec_raw); 1325 1326 /* 1327 * Special case for PTP. Just transfer the raw time into sys, 1328 * so the call sites can consistently use snap::systime. 1329 */ 1330 if (clock_id == CLOCK_MONOTONIC_RAW) 1331 systime_snapshot->systime = systime_snapshot->monoraw; 1332 /* Tell the consumer that this snapshot is valid */ 1333 systime_snapshot->valid = true; 1334 } 1335 EXPORT_SYMBOL_GPL(ktime_get_snapshot_id); 1336 1337 /* Scale base by mult/div checking for overflow */ 1338 static int scale64_check_overflow(u64 mult, u64 div, u64 *base) 1339 { 1340 u64 tmp, rem; 1341 1342 tmp = div64_u64_rem(*base, div, &rem); 1343 1344 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) || 1345 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem))) 1346 return -EOVERFLOW; 1347 tmp *= mult; 1348 1349 rem = div64_u64(rem * mult, div); 1350 *base = tmp + rem; 1351 return 0; 1352 } 1353 1354 /** 1355 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval 1356 * @history: Snapshot representing start of history 1357 * @partial_history_cycles: Cycle offset into history (fractional part) 1358 * @total_history_cycles: Total history length in cycles 1359 * @discontinuity: True indicates clock was set on history period 1360 * @ts: Cross timestamp that should be adjusted using 1361 * partial/total ratio 1362 * 1363 * Helper function used by get_device_system_crosststamp() to correct the 1364 * crosstimestamp corresponding to the start of the current interval to the 1365 * system counter value (timestamp point) provided by the driver. The 1366 * total_history_* quantities are the total history starting at the provided 1367 * reference point and ending at the start of the current interval. The cycle 1368 * count between the driver timestamp point and the start of the current 1369 * interval is partial_history_cycles. 1370 */ 1371 static int adjust_historical_crosststamp(struct system_time_snapshot *history, 1372 u64 partial_history_cycles, 1373 u64 total_history_cycles, 1374 bool discontinuity, 1375 struct system_device_crosststamp *ts) 1376 { 1377 struct timekeeper *tk = &tk_core.timekeeper; 1378 u64 corr_raw, corr_sys; 1379 bool interp_forward; 1380 int ret; 1381 1382 if (total_history_cycles == 0 || partial_history_cycles == 0) 1383 return 0; 1384 1385 /* Interpolate shortest distance from beginning or end of history */ 1386 interp_forward = partial_history_cycles > total_history_cycles / 2; 1387 partial_history_cycles = interp_forward ? 1388 total_history_cycles - partial_history_cycles : 1389 partial_history_cycles; 1390 1391 /* 1392 * Scale the monotonic raw time delta by: 1393 * partial_history_cycles / total_history_cycles 1394 */ 1395 corr_raw = (u64)ktime_to_ns(ktime_sub(ts->sys_monoraw, history->monoraw)); 1396 ret = scale64_check_overflow(partial_history_cycles, 1397 total_history_cycles, &corr_raw); 1398 if (ret) 1399 return ret; 1400 1401 /* 1402 * If there is a discontinuity in the history, scale monotonic raw 1403 * correction by: 1404 * mult(sys)/mult(raw) yielding the system time correction 1405 * 1406 * Otherwise, calculate the system time correction similar to monotonic 1407 * raw calculation 1408 */ 1409 if (discontinuity) { 1410 corr_sys = mul_u64_u32_div(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult); 1411 } else { 1412 corr_sys = (u64)ktime_to_ns(ktime_sub(ts->sys_systime, history->systime)); 1413 ret = scale64_check_overflow(partial_history_cycles, total_history_cycles, 1414 &corr_sys); 1415 if (ret) 1416 return ret; 1417 } 1418 1419 /* Fixup monotonic raw and system time time values */ 1420 if (interp_forward) { 1421 ts->sys_monoraw = ktime_add_ns(history->monoraw, corr_raw); 1422 ts->sys_systime = ktime_add_ns(history->systime, corr_sys); 1423 } else { 1424 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw); 1425 ts->sys_systime = ktime_sub_ns(ts->sys_systime, corr_sys); 1426 } 1427 1428 return 0; 1429 } 1430 1431 /* 1432 * timestamp_in_interval - true if ts is chronologically in [start, end] 1433 * 1434 * True if ts occurs chronologically at or after start, and before or at end. 1435 */ 1436 static bool timestamp_in_interval(u64 start, u64 end, u64 ts) 1437 { 1438 if (ts >= start && ts <= end) 1439 return true; 1440 if (start > end && (ts >= start || ts <= end)) 1441 return true; 1442 return false; 1443 } 1444 1445 static bool convert_clock(u64 *val, u32 numerator, u32 denominator) 1446 { 1447 u64 rem, res; 1448 1449 if (!numerator || !denominator) 1450 return false; 1451 1452 res = div64_u64_rem(*val, denominator, &rem) * numerator; 1453 *val = res + div_u64(rem * numerator, denominator); 1454 return true; 1455 } 1456 1457 static bool convert_base_to_cs(struct system_counterval_t *scv) 1458 { 1459 struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock; 1460 struct clocksource_base *base; 1461 u32 num, den; 1462 1463 /* The timestamp was taken from the time keeper clock source */ 1464 if (cs->id == scv->cs_id) 1465 return true; 1466 1467 /* 1468 * Check whether cs_id matches the base clock. Prevent the compiler from 1469 * re-evaluating @base as the clocksource might change concurrently. 1470 */ 1471 base = READ_ONCE(cs->base); 1472 if (!base || base->id != scv->cs_id) 1473 return false; 1474 1475 num = scv->use_nsecs ? cs->freq_khz : base->numerator; 1476 den = scv->use_nsecs ? USEC_PER_SEC : base->denominator; 1477 1478 if (!convert_clock(&scv->cycles, num, den)) 1479 return false; 1480 1481 scv->cycles += base->offset; 1482 /* Set the clocksource ID as scv::cycles is now clocksource based */ 1483 scv->cs_id = cs->id; 1484 return true; 1485 } 1486 1487 static bool convert_cs_to_base(u64 *cycles, enum clocksource_ids base_id) 1488 { 1489 struct clocksource *cs = tk_core.timekeeper.tkr_mono.clock; 1490 struct clocksource_base *base; 1491 1492 /* 1493 * Check whether base_id matches the base clock. Prevent the compiler from 1494 * re-evaluating @base as the clocksource might change concurrently. 1495 */ 1496 base = READ_ONCE(cs->base); 1497 if (!base || base->id != base_id) 1498 return false; 1499 1500 *cycles -= base->offset; 1501 if (!convert_clock(cycles, base->denominator, base->numerator)) 1502 return false; 1503 return true; 1504 } 1505 1506 static bool convert_ns_to_cs(u64 *delta) 1507 { 1508 struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono; 1509 1510 if (BITS_TO_BYTES(fls64(*delta) + tkr->shift) >= sizeof(*delta)) 1511 return false; 1512 1513 *delta = div_u64((*delta << tkr->shift) - tkr->xtime_nsec, tkr->mult); 1514 return true; 1515 } 1516 1517 /** 1518 * ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp 1519 * @treal: CLOCK_REALTIME timestamp to convert 1520 * @base_id: base clocksource id 1521 * @cycles: pointer to store the converted base clock timestamp 1522 * 1523 * Converts a supplied, future realtime clock value to the corresponding base clock value. 1524 * 1525 * Return: true if the conversion is successful, false otherwise. 1526 */ 1527 bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles) 1528 { 1529 struct timekeeper *tk = &tk_core.timekeeper; 1530 unsigned int seq; 1531 u64 delta; 1532 1533 do { 1534 seq = read_seqcount_begin(&tk_core.seq); 1535 if ((u64)treal < tk->tkr_mono.base_real) 1536 return false; 1537 delta = (u64)treal - tk->tkr_mono.base_real; 1538 if (!convert_ns_to_cs(&delta)) 1539 return false; 1540 *cycles = tk->tkr_mono.cycle_last + delta; 1541 if (!convert_cs_to_base(cycles, base_id)) 1542 return false; 1543 } while (read_seqcount_retry(&tk_core.seq, seq)); 1544 1545 return true; 1546 } 1547 EXPORT_SYMBOL_GPL(ktime_real_to_base_clock); 1548 1549 /** 1550 * get_device_system_crosststamp - Synchronously capture system/device timestamp 1551 * @get_time_fn: Callback to get simultaneous device time and system counter 1552 * from the device driver 1553 * @ctx: Context passed to get_time_fn() 1554 * @history_begin: Historical reference point used to interpolate system time when 1555 * the counter value provided by the driver is before the current interval 1556 * @xtstamp: Receives simultaneously captured system and device time 1557 * 1558 * Reads a timestamp from a device and correlates it to system time 1559 * 1560 * See documentation for ktime_get_snapshot_id() for information about the raw 1561 * monotonic time stamp which is used here. 1562 */ 1563 int get_device_system_crosststamp(int (*get_time_fn) 1564 (ktime_t *device_time, 1565 struct system_counterval_t *sys_counterval, 1566 void *ctx), 1567 void *ctx, 1568 struct system_time_snapshot *history_begin, 1569 struct system_device_crosststamp *xtstamp) 1570 { 1571 u64 syscnt_cycles, cycles, now, interval_start; 1572 ktime_t base_sys, base_raw, *offs; 1573 u32 clock_was_set_seq = 0; 1574 u64 nsec_sys, nsec_raw; 1575 u8 cs_was_changed_seq; 1576 unsigned int seq; 1577 bool do_interp; 1578 struct timekeeper *tk; 1579 struct tk_data *tkd; 1580 int ret; 1581 1582 switch (xtstamp->clock_id) { 1583 case CLOCK_REALTIME: 1584 tkd = &tk_core; 1585 offs = &tk_core.timekeeper.offs_real; 1586 break; 1587 case CLOCK_AUX ... CLOCK_AUX_LAST: 1588 tkd = aux_get_tk_data(xtstamp->clock_id); 1589 if (!tkd) 1590 return -ENODEV; 1591 offs = &tkd->timekeeper.offs_aux; 1592 break; 1593 default: 1594 WARN_ON_ONCE(1); 1595 return -ENODEV; 1596 } 1597 1598 tk = &tkd->timekeeper; 1599 1600 do { 1601 seq = read_seqcount_begin(&tkd->seq); 1602 /* 1603 * Try to synchronously capture device time and a system 1604 * counter value calling back into the device driver 1605 */ 1606 ret = get_time_fn(&xtstamp->device, &xtstamp->sys_counter, ctx); 1607 if (ret) 1608 return ret; 1609 1610 /* 1611 * Verify that the clocksource ID associated with the captured 1612 * system counter value is the same as for the currently 1613 * installed timekeeper clocksource and convert to it. 1614 */ 1615 if (xtstamp->sys_counter.cs_id == CSID_GENERIC || 1616 !convert_base_to_cs(&xtstamp->sys_counter)) 1617 return -ENODEV; 1618 1619 cycles = syscnt_cycles = xtstamp->sys_counter.cycles; 1620 1621 /* 1622 * Check whether the system counter value provided by the 1623 * device driver is on the current timekeeping interval. 1624 */ 1625 now = tk_clock_read(&tk->tkr_mono); 1626 interval_start = tk->tkr_mono.cycle_last; 1627 if (!timestamp_in_interval(interval_start, now, cycles)) { 1628 clock_was_set_seq = tk->clock_was_set_seq; 1629 cs_was_changed_seq = tk->cs_was_changed_seq; 1630 cycles = interval_start; 1631 do_interp = true; 1632 } else { 1633 do_interp = false; 1634 } 1635 1636 base_sys = ktime_add(tk->tkr_mono.base, *offs); 1637 base_raw = tk->tkr_raw.base; 1638 1639 nsec_sys = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles); 1640 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles); 1641 } while (read_seqcount_retry(&tkd->seq, seq)); 1642 1643 xtstamp->sys_systime = ktime_add_ns(base_sys, nsec_sys); 1644 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw); 1645 1646 /* 1647 * Interpolate if necessary, adjusting back from the start of the 1648 * current interval 1649 */ 1650 if (do_interp) { 1651 u64 partial_history_cycles, total_history_cycles; 1652 bool discontinuity; 1653 1654 /* 1655 * Check that the counter value is not before the provided 1656 * history reference and that the history doesn't cross a 1657 * clocksource change 1658 */ 1659 if (!history_begin || 1660 !timestamp_in_interval(history_begin->cycles, cycles, syscnt_cycles) || 1661 history_begin->cs_was_changed_seq != cs_was_changed_seq) 1662 return -EINVAL; 1663 1664 partial_history_cycles = cycles - syscnt_cycles; 1665 total_history_cycles = cycles - history_begin->cycles; 1666 discontinuity = history_begin->clock_was_set_seq != clock_was_set_seq; 1667 1668 ret = adjust_historical_crosststamp(history_begin, partial_history_cycles, 1669 total_history_cycles, discontinuity, xtstamp); 1670 } 1671 1672 return ret; 1673 } 1674 EXPORT_SYMBOL_GPL(get_device_system_crosststamp); 1675 1676 /** 1677 * timekeeping_clocksource_has_base - Check whether the current clocksource 1678 * is based on given a base clock 1679 * @id: base clocksource ID 1680 * 1681 * Note: The return value is a snapshot which can become invalid right 1682 * after the function returns. 1683 * 1684 * Return: true if the timekeeper clocksource has a base clock with @id, 1685 * false otherwise 1686 */ 1687 bool timekeeping_clocksource_has_base(enum clocksource_ids id) 1688 { 1689 /* 1690 * This is a snapshot, so no point in using the sequence 1691 * count. Just prevent the compiler from re-evaluating @base as the 1692 * clocksource might change concurrently. 1693 */ 1694 struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base); 1695 1696 return base ? base->id == id : false; 1697 } 1698 EXPORT_SYMBOL_GPL(timekeeping_clocksource_has_base); 1699 1700 /** 1701 * do_settimeofday64 - Sets the time of day. 1702 * @ts: pointer to the timespec64 variable containing the new time 1703 * 1704 * Sets the time of day to the new time and update NTP and notify hrtimers 1705 */ 1706 int do_settimeofday64(const struct timespec64 *ts) 1707 { 1708 struct timespec64 ts_delta, xt; 1709 1710 if (!timespec64_valid_settod(ts)) 1711 return -EINVAL; 1712 1713 scoped_guard (raw_spinlock_irqsave, &tk_core.lock) { 1714 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1715 1716 timekeeping_forward_now(tks); 1717 1718 xt = tk_xtime(tks); 1719 ts_delta = timespec64_sub(*ts, xt); 1720 1721 if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) { 1722 timekeeping_restore_shadow(&tk_core); 1723 return -EINVAL; 1724 } 1725 1726 tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta)); 1727 tk_set_xtime(tks, ts); 1728 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 1729 } 1730 1731 /* Signal hrtimers about time change */ 1732 clock_was_set(CLOCK_SET_WALL); 1733 1734 audit_tk_injoffset(ts_delta); 1735 add_device_randomness(ts, sizeof(*ts)); 1736 return 0; 1737 } 1738 EXPORT_SYMBOL(do_settimeofday64); 1739 1740 static inline bool timekeeper_is_core_tk(struct timekeeper *tk) 1741 { 1742 return !IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) || tk->id == TIMEKEEPER_CORE; 1743 } 1744 1745 /** 1746 * __timekeeping_inject_offset - Adds or subtracts from the current time. 1747 * @tkd: Pointer to the timekeeper to modify 1748 * @ts: Pointer to the timespec variable containing the offset 1749 * 1750 * Adds or subtracts an offset value from the current time. 1751 */ 1752 static int __timekeeping_inject_offset(struct tk_data *tkd, const struct timespec64 *ts) 1753 { 1754 struct timekeeper *tks = &tkd->shadow_timekeeper; 1755 struct timespec64 tmp; 1756 1757 if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC) 1758 return -EINVAL; 1759 1760 timekeeping_forward_now(tks); 1761 1762 if (timekeeper_is_core_tk(tks)) { 1763 /* Make sure the proposed value is valid */ 1764 tmp = timespec64_add(tk_xtime(tks), *ts); 1765 if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 || 1766 !timespec64_valid_settod(&tmp)) { 1767 timekeeping_restore_shadow(tkd); 1768 return -EINVAL; 1769 } 1770 1771 tk_xtime_add(tks, ts); 1772 tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts)); 1773 } else { 1774 struct tk_read_base *tkr_mono = &tks->tkr_mono; 1775 ktime_t now, offs; 1776 1777 /* Get the current time */ 1778 now = ktime_add_ns(tkr_mono->base, timekeeping_get_ns(tkr_mono)); 1779 /* Add the relative offset change */ 1780 offs = ktime_add(tks->offs_aux, timespec64_to_ktime(*ts)); 1781 1782 /* Prevent that the resulting time becomes negative */ 1783 if (ktime_add(now, offs) < 0) { 1784 timekeeping_restore_shadow(tkd); 1785 return -EINVAL; 1786 } 1787 tk_update_aux_offs(tks, offs); 1788 } 1789 1790 timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL); 1791 return 0; 1792 } 1793 1794 static int timekeeping_inject_offset(const struct timespec64 *ts) 1795 { 1796 int ret; 1797 1798 scoped_guard (raw_spinlock_irqsave, &tk_core.lock) 1799 ret = __timekeeping_inject_offset(&tk_core, ts); 1800 1801 /* Signal hrtimers about time change */ 1802 if (!ret) 1803 clock_was_set(CLOCK_SET_WALL); 1804 return ret; 1805 } 1806 1807 /* 1808 * Indicates if there is an offset between the system clock and the hardware 1809 * clock/persistent clock/rtc. 1810 */ 1811 int persistent_clock_is_local; 1812 1813 /* 1814 * Adjust the time obtained from the CMOS to be UTC time instead of 1815 * local time. 1816 * 1817 * This is ugly, but preferable to the alternatives. Otherwise we 1818 * would either need to write a program to do it in /etc/rc (and risk 1819 * confusion if the program gets run more than once; it would also be 1820 * hard to make the program warp the clock precisely n hours) or 1821 * compile in the timezone information into the kernel. Bad, bad.... 1822 * 1823 * - TYT, 1992-01-01 1824 * 1825 * The best thing to do is to keep the CMOS clock in universal time (UTC) 1826 * as real UNIX machines always do it. This avoids all headaches about 1827 * daylight saving times and warping kernel clocks. 1828 */ 1829 void timekeeping_warp_clock(void) 1830 { 1831 if (sys_tz.tz_minuteswest != 0) { 1832 struct timespec64 adjust; 1833 1834 persistent_clock_is_local = 1; 1835 adjust.tv_sec = sys_tz.tz_minuteswest * 60; 1836 adjust.tv_nsec = 0; 1837 timekeeping_inject_offset(&adjust); 1838 } 1839 } 1840 1841 /* 1842 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic 1843 */ 1844 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) 1845 { 1846 tk->tai_offset = tai_offset; 1847 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); 1848 } 1849 1850 /* 1851 * change_clocksource - Swaps clocksources if a new one is available 1852 * 1853 * Accumulates current time interval and initializes new clocksource 1854 */ 1855 static int change_clocksource(void *data) 1856 { 1857 struct clocksource *new = data, *old = NULL; 1858 1859 /* 1860 * If the clocksource is in a module, get a module reference. 1861 * Succeeds for built-in code (owner == NULL) as well. Abort if the 1862 * reference can't be acquired. 1863 */ 1864 if (!try_module_get(new->owner)) 1865 return 0; 1866 1867 /* Abort if the device can't be enabled */ 1868 if (new->enable && new->enable(new) != 0) { 1869 module_put(new->owner); 1870 return 0; 1871 } 1872 1873 scoped_guard (raw_spinlock_irqsave, &tk_core.lock) { 1874 struct timekeeper *tks = &tk_core.shadow_timekeeper; 1875 1876 timekeeping_forward_now(tks); 1877 old = tks->tkr_mono.clock; 1878 tk_setup_internals(tks, new); 1879 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 1880 } 1881 1882 tk_aux_update_clocksource(); 1883 1884 if (old) { 1885 if (old->disable) 1886 old->disable(old); 1887 module_put(old->owner); 1888 } 1889 1890 return 0; 1891 } 1892 1893 /** 1894 * timekeeping_notify - Install a new clock source 1895 * @clock: pointer to the clock source 1896 * 1897 * This function is called from clocksource.c after a new, better clock 1898 * source has been registered. The caller holds the clocksource_mutex. 1899 */ 1900 int timekeeping_notify(struct clocksource *clock) 1901 { 1902 struct timekeeper *tk = &tk_core.timekeeper; 1903 1904 if (tk->tkr_mono.clock == clock) 1905 return 0; 1906 1907 /* Disable inlined reads accross the clocksource switch */ 1908 clocksource_disable_inline_read(); 1909 1910 stop_machine(change_clocksource, clock, NULL); 1911 1912 /* 1913 * If the clocksource has been selected and supports inlined reads 1914 * enable the branch. 1915 */ 1916 if (tk->tkr_mono.clock == clock && clock->flags & CLOCK_SOURCE_CAN_INLINE_READ) 1917 clocksource_enable_inline_read(); 1918 1919 tick_clock_notify(); 1920 return tk->tkr_mono.clock == clock ? 0 : -1; 1921 } 1922 1923 /** 1924 * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec 1925 * @ts: pointer to the timespec64 to be set 1926 * 1927 * Returns the raw monotonic time (completely un-modified by ntp) 1928 */ 1929 void ktime_get_raw_ts64(struct timespec64 *ts) 1930 { 1931 struct timekeeper *tk = &tk_core.timekeeper; 1932 unsigned int seq; 1933 u64 nsecs; 1934 1935 do { 1936 seq = read_seqcount_begin(&tk_core.seq); 1937 ts->tv_sec = tk->raw_sec; 1938 nsecs = timekeeping_get_ns(&tk->tkr_raw); 1939 1940 } while (read_seqcount_retry(&tk_core.seq, seq)); 1941 1942 ts->tv_nsec = 0; 1943 timespec64_add_ns(ts, nsecs); 1944 } 1945 EXPORT_SYMBOL(ktime_get_raw_ts64); 1946 1947 /** 1948 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres 1949 */ 1950 int timekeeping_valid_for_hres(void) 1951 { 1952 struct timekeeper *tk = &tk_core.timekeeper; 1953 unsigned int seq; 1954 int ret; 1955 1956 do { 1957 seq = read_seqcount_begin(&tk_core.seq); 1958 1959 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; 1960 1961 } while (read_seqcount_retry(&tk_core.seq, seq)); 1962 1963 return ret; 1964 } 1965 1966 /** 1967 * timekeeping_max_deferment - Returns max time the clocksource can be deferred 1968 */ 1969 u64 timekeeping_max_deferment(void) 1970 { 1971 struct timekeeper *tk = &tk_core.timekeeper; 1972 unsigned int seq; 1973 u64 ret; 1974 1975 do { 1976 seq = read_seqcount_begin(&tk_core.seq); 1977 1978 ret = tk->tkr_mono.clock->max_idle_ns; 1979 1980 } while (read_seqcount_retry(&tk_core.seq, seq)); 1981 1982 return ret; 1983 } 1984 1985 /** 1986 * read_persistent_clock64 - Return time from the persistent clock. 1987 * @ts: Pointer to the storage for the readout value 1988 * 1989 * Weak dummy function for arches that do not yet support it. 1990 * Reads the time from the battery backed persistent clock. 1991 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported. 1992 * 1993 * XXX - Do be sure to remove it once all arches implement it. 1994 */ 1995 void __weak read_persistent_clock64(struct timespec64 *ts) 1996 { 1997 ts->tv_sec = 0; 1998 ts->tv_nsec = 0; 1999 } 2000 2001 /** 2002 * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset 2003 * from the boot. 2004 * @wall_time: current time as returned by persistent clock 2005 * @boot_offset: offset that is defined as wall_time - boot_time 2006 * 2007 * Weak dummy function for arches that do not yet support it. 2008 * 2009 * The default function calculates offset based on the current value of 2010 * local_clock(). This way architectures that support sched_clock() but don't 2011 * support dedicated boot time clock will provide the best estimate of the 2012 * boot time. 2013 */ 2014 void __weak __init 2015 read_persistent_wall_and_boot_offset(struct timespec64 *wall_time, 2016 struct timespec64 *boot_offset) 2017 { 2018 read_persistent_clock64(wall_time); 2019 *boot_offset = ns_to_timespec64(local_clock()); 2020 } 2021 2022 static __init void tkd_basic_setup(struct tk_data *tkd, enum timekeeper_ids tk_id, bool valid) 2023 { 2024 raw_spin_lock_init(&tkd->lock); 2025 seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock); 2026 tkd->timekeeper.id = tkd->shadow_timekeeper.id = tk_id; 2027 tkd->timekeeper.clock_valid = tkd->shadow_timekeeper.clock_valid = valid; 2028 } 2029 2030 /* 2031 * Flag reflecting whether timekeeping_resume() has injected sleeptime. 2032 * 2033 * The flag starts of false and is only set when a suspend reaches 2034 * timekeeping_suspend(), timekeeping_resume() sets it to false when the 2035 * timekeeper clocksource is not stopping across suspend and has been 2036 * used to update sleep time. If the timekeeper clocksource has stopped 2037 * then the flag stays true and is used by the RTC resume code to decide 2038 * whether sleeptime must be injected and if so the flag gets false then. 2039 * 2040 * If a suspend fails before reaching timekeeping_resume() then the flag 2041 * stays false and prevents erroneous sleeptime injection. 2042 */ 2043 static bool suspend_timing_needed; 2044 2045 /* Flag for if there is a persistent clock on this platform */ 2046 static bool persistent_clock_exists; 2047 2048 /* 2049 * timekeeping_init - Initializes the clocksource and common timekeeping values 2050 */ 2051 void __init timekeeping_init(void) 2052 { 2053 struct timespec64 wall_time, boot_offset, wall_to_mono; 2054 struct timekeeper *tks = &tk_core.shadow_timekeeper; 2055 struct clocksource *clock; 2056 2057 tkd_basic_setup(&tk_core, TIMEKEEPER_CORE, true); 2058 tk_aux_setup(); 2059 2060 read_persistent_wall_and_boot_offset(&wall_time, &boot_offset); 2061 if (timespec64_valid_settod(&wall_time) && 2062 timespec64_to_ns(&wall_time) > 0) { 2063 persistent_clock_exists = true; 2064 } else if (timespec64_to_ns(&wall_time) != 0) { 2065 pr_warn("Persistent clock returned invalid value"); 2066 wall_time = (struct timespec64){0}; 2067 } 2068 2069 if (timespec64_compare(&wall_time, &boot_offset) < 0) 2070 boot_offset = (struct timespec64){0}; 2071 2072 /* 2073 * We want set wall_to_mono, so the following is true: 2074 * wall time + wall_to_mono = boot time 2075 */ 2076 wall_to_mono = timespec64_sub(boot_offset, wall_time); 2077 2078 clock = clocksource_default_clock(); 2079 if (clock->enable) 2080 clock->enable(clock); 2081 2082 guard(raw_spinlock_irqsave)(&tk_core.lock); 2083 2084 ntp_init(); 2085 2086 tk_setup_internals(tks, clock); 2087 2088 tk_set_xtime(tks, &wall_time); 2089 tks->raw_sec = 0; 2090 2091 tk_set_wall_to_mono(tks, wall_to_mono); 2092 2093 /* 2094 * Use TK_UPDATE_ALL so the NTP layer picks up the clocksource's 2095 * cs_tick_adj via ntp_clear(). Clearing NTP here is otherwise 2096 * redundant as ntp_init() already initialised it above. 2097 */ 2098 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 2099 } 2100 2101 /* time in seconds when suspend began for persistent clock */ 2102 static struct timespec64 timekeeping_suspend_time; 2103 2104 /** 2105 * __timekeeping_inject_sleeptime - Internal function to add sleep interval 2106 * @tk: Pointer to the timekeeper to be updated 2107 * @delta: Pointer to the delta value in timespec64 format 2108 * 2109 * Takes a timespec offset measuring a suspend interval and properly 2110 * adds the sleep offset to the timekeeping variables. 2111 */ 2112 static void __timekeeping_inject_sleeptime(struct timekeeper *tk, 2113 const struct timespec64 *delta) 2114 { 2115 if (!timespec64_valid_strict(delta)) { 2116 printk_deferred(KERN_WARNING 2117 "__timekeeping_inject_sleeptime: Invalid " 2118 "sleep delta value!\n"); 2119 return; 2120 } 2121 tk_xtime_add(tk, delta); 2122 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); 2123 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); 2124 tk_debug_account_sleep_time(delta); 2125 } 2126 2127 #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE) 2128 /* 2129 * We have three kinds of time sources to use for sleep time 2130 * injection, the preference order is: 2131 * 1) non-stop clocksource 2132 * 2) persistent clock (ie: RTC accessible when irqs are off) 2133 * 3) RTC 2134 * 2135 * 1) and 2) are used by timekeeping, 3) by RTC subsystem. 2136 * If system has neither 1) nor 2), 3) will be used finally. 2137 * 2138 * 2139 * If timekeeping has injected sleeptime via either 1) or 2), 2140 * 3) becomes needless, so in this case we don't need to call 2141 * rtc_resume(), and this is what timekeeping_rtc_skipresume() 2142 * means. 2143 */ 2144 bool timekeeping_rtc_skipresume(void) 2145 { 2146 return !suspend_timing_needed; 2147 } 2148 2149 /* 2150 * 1) can be determined whether to use or not only when doing 2151 * timekeeping_resume() which is invoked after rtc_suspend(), 2152 * so we can't skip rtc_suspend() surely if system has 1). 2153 * 2154 * But if system has 2), 2) will definitely be used, so in this 2155 * case we don't need to call rtc_suspend(), and this is what 2156 * timekeeping_rtc_skipsuspend() means. 2157 */ 2158 bool timekeeping_rtc_skipsuspend(void) 2159 { 2160 return persistent_clock_exists; 2161 } 2162 2163 /** 2164 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values 2165 * @delta: pointer to a timespec64 delta value 2166 * 2167 * This hook is for architectures that cannot support read_persistent_clock64 2168 * because their RTC/persistent clock is only accessible when irqs are enabled. 2169 * and also don't have an effective nonstop clocksource. 2170 * 2171 * This function should only be called by rtc_resume(), and allows 2172 * a suspend offset to be injected into the timekeeping values. 2173 */ 2174 void timekeeping_inject_sleeptime64(const struct timespec64 *delta) 2175 { 2176 scoped_guard(raw_spinlock_irqsave, &tk_core.lock) { 2177 struct timekeeper *tks = &tk_core.shadow_timekeeper; 2178 2179 suspend_timing_needed = false; 2180 timekeeping_forward_now(tks); 2181 __timekeeping_inject_sleeptime(tks, delta); 2182 timekeeping_update_from_shadow(&tk_core, TK_UPDATE_ALL); 2183 } 2184 2185 /* Signal hrtimers about time change */ 2186 clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT); 2187 } 2188 #endif 2189 2190 /** 2191 * timekeeping_resume - Resumes the generic timekeeping subsystem. 2192 */ 2193 void timekeeping_resume(void) 2194 { 2195 struct timekeeper *tks = &tk_core.shadow_timekeeper; 2196 struct clocksource *clock = tks->tkr_mono.clock; 2197 struct timespec64 ts_new, ts_delta; 2198 bool inject_sleeptime = false; 2199 u64 cycle_now, nsec; 2200 unsigned long flags; 2201 2202 read_persistent_clock64(&ts_new); 2203 2204 clockevents_resume(); 2205 clocksource_resume(); 2206 2207 raw_spin_lock_irqsave(&tk_core.lock, flags); 2208 2209 /* 2210 * After system resumes, we need to calculate the suspended time and 2211 * compensate it for the OS time. There are 3 sources that could be 2212 * used: Nonstop clocksource during suspend, persistent clock and rtc 2213 * device. 2214 * 2215 * One specific platform may have 1 or 2 or all of them, and the 2216 * preference will be: 2217 * suspend-nonstop clocksource -> persistent clock -> rtc 2218 * The less preferred source will only be tried if there is no better 2219 * usable source. The rtc part is handled separately in rtc core code. 2220 */ 2221 cycle_now = tk_clock_read(&tks->tkr_mono); 2222 nsec = clocksource_stop_suspend_timing(clock, cycle_now); 2223 if (nsec > 0) { 2224 ts_delta = ns_to_timespec64(nsec); 2225 inject_sleeptime = true; 2226 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) { 2227 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time); 2228 inject_sleeptime = true; 2229 } 2230 2231 if (inject_sleeptime) { 2232 suspend_timing_needed = false; 2233 __timekeeping_inject_sleeptime(tks, &ts_delta); 2234 } 2235 2236 /* Re-base the last cycle value */ 2237 tks->tkr_mono.cycle_last = cycle_now; 2238 tks->tkr_raw.cycle_last = cycle_now; 2239 2240 tks->ntp_error = 0; 2241 timekeeping_suspended = 0; 2242 timekeeping_update_from_shadow(&tk_core, TK_CLOCK_WAS_SET); 2243 raw_spin_unlock_irqrestore(&tk_core.lock, flags); 2244 2245 touch_softlockup_watchdog(); 2246 2247 /* Resume the clockevent device(s) and hrtimers */ 2248 tick_resume(); 2249 /* Notify timerfd as resume is equivalent to clock_was_set() */ 2250 timerfd_resume(); 2251 } 2252 2253 static void timekeeping_syscore_resume(void *data) 2254 { 2255 timekeeping_resume(); 2256 } 2257 2258 int timekeeping_suspend(void) 2259 { 2260 struct timekeeper *tks = &tk_core.shadow_timekeeper; 2261 struct timespec64 delta, delta_delta; 2262 static struct timespec64 old_delta; 2263 struct clocksource *curr_clock; 2264 unsigned long flags; 2265 u64 cycle_now; 2266 2267 read_persistent_clock64(&timekeeping_suspend_time); 2268 2269 /* 2270 * On some systems the persistent_clock can not be detected at 2271 * timekeeping_init by its return value, so if we see a valid 2272 * value returned, update the persistent_clock_exists flag. 2273 */ 2274 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec) 2275 persistent_clock_exists = true; 2276 2277 suspend_timing_needed = true; 2278 2279 raw_spin_lock_irqsave(&tk_core.lock, flags); 2280 timekeeping_forward_now(tks); 2281 timekeeping_suspended = 1; 2282 2283 /* 2284 * Since we've called forward_now, cycle_last stores the value 2285 * just read from the current clocksource. Save this to potentially 2286 * use in suspend timing. 2287 */ 2288 curr_clock = tks->tkr_mono.clock; 2289 cycle_now = tks->tkr_mono.cycle_last; 2290 clocksource_start_suspend_timing(curr_clock, cycle_now); 2291 2292 if (persistent_clock_exists) { 2293 /* 2294 * To avoid drift caused by repeated suspend/resumes, 2295 * which each can add ~1 second drift error, 2296 * try to compensate so the difference in system time 2297 * and persistent_clock time stays close to constant. 2298 */ 2299 delta = timespec64_sub(tk_xtime(tks), timekeeping_suspend_time); 2300 delta_delta = timespec64_sub(delta, old_delta); 2301 if (abs(delta_delta.tv_sec) >= 2) { 2302 /* 2303 * if delta_delta is too large, assume time correction 2304 * has occurred and set old_delta to the current delta. 2305 */ 2306 old_delta = delta; 2307 } else { 2308 /* Otherwise try to adjust old_system to compensate */ 2309 timekeeping_suspend_time = 2310 timespec64_add(timekeeping_suspend_time, delta_delta); 2311 } 2312 } 2313 2314 timekeeping_update_from_shadow(&tk_core, 0); 2315 halt_fast_timekeeper(tks); 2316 raw_spin_unlock_irqrestore(&tk_core.lock, flags); 2317 2318 tick_suspend(); 2319 clocksource_suspend(); 2320 clockevents_suspend(); 2321 2322 return 0; 2323 } 2324 2325 static int timekeeping_syscore_suspend(void *data) 2326 { 2327 return timekeeping_suspend(); 2328 } 2329 2330 /* sysfs resume/suspend bits for timekeeping */ 2331 static const struct syscore_ops timekeeping_syscore_ops = { 2332 .resume = timekeeping_syscore_resume, 2333 .suspend = timekeeping_syscore_suspend, 2334 }; 2335 2336 static struct syscore timekeeping_syscore = { 2337 .ops = &timekeeping_syscore_ops, 2338 }; 2339 2340 static int __init timekeeping_init_ops(void) 2341 { 2342 register_syscore(&timekeeping_syscore); 2343 return 0; 2344 } 2345 device_initcall(timekeeping_init_ops); 2346 2347 /* 2348 * Apply a multiplier adjustment to the timekeeper 2349 */ 2350 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, 2351 s64 offset, 2352 s32 mult_adj) 2353 { 2354 s64 interval = tk->cycle_interval; 2355 2356 if (mult_adj == 0) { 2357 return; 2358 } else if (mult_adj == -1) { 2359 interval = -interval; 2360 offset = -offset; 2361 } else if (mult_adj != 1) { 2362 interval *= mult_adj; 2363 offset *= mult_adj; 2364 } 2365 2366 /* 2367 * So the following can be confusing. 2368 * 2369 * To keep things simple, lets assume mult_adj == 1 for now. 2370 * 2371 * When mult_adj != 1, remember that the interval and offset values 2372 * have been appropriately scaled so the math is the same. 2373 * 2374 * The basic idea here is that we're increasing the multiplier 2375 * by one, this causes the xtime_interval to be incremented by 2376 * one cycle_interval. This is because: 2377 * xtime_interval = cycle_interval * mult 2378 * So if mult is being incremented by one: 2379 * xtime_interval = cycle_interval * (mult + 1) 2380 * Its the same as: 2381 * xtime_interval = (cycle_interval * mult) + cycle_interval 2382 * Which can be shortened to: 2383 * xtime_interval += cycle_interval 2384 * 2385 * So offset stores the non-accumulated cycles. Thus the current 2386 * time (in shifted nanoseconds) is: 2387 * now = (offset * adj) + xtime_nsec 2388 * Now, even though we're adjusting the clock frequency, we have 2389 * to keep time consistent. In other words, we can't jump back 2390 * in time, and we also want to avoid jumping forward in time. 2391 * 2392 * So given the same offset value, we need the time to be the same 2393 * both before and after the freq adjustment. 2394 * now = (offset * adj_1) + xtime_nsec_1 2395 * now = (offset * adj_2) + xtime_nsec_2 2396 * So: 2397 * (offset * adj_1) + xtime_nsec_1 = 2398 * (offset * adj_2) + xtime_nsec_2 2399 * And we know: 2400 * adj_2 = adj_1 + 1 2401 * So: 2402 * (offset * adj_1) + xtime_nsec_1 = 2403 * (offset * (adj_1+1)) + xtime_nsec_2 2404 * (offset * adj_1) + xtime_nsec_1 = 2405 * (offset * adj_1) + offset + xtime_nsec_2 2406 * Canceling the sides: 2407 * xtime_nsec_1 = offset + xtime_nsec_2 2408 * Which gives us: 2409 * xtime_nsec_2 = xtime_nsec_1 - offset 2410 * Which simplifies to: 2411 * xtime_nsec -= offset 2412 * 2413 * When subtracting offset from xtime_nsec, the same amount 2414 * (in appropriate units) has to be added to ntp_error, in 2415 * order to correctly track the delta between the time 2416 * reported in xtime_nsec, and the intended time. 2417 */ 2418 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { 2419 /* NTP adjustment caused clocksource mult overflow */ 2420 WARN_ON_ONCE(1); 2421 return; 2422 } 2423 2424 tk->tkr_mono.mult += mult_adj; 2425 tk->xtime_interval += interval; 2426 tk->tkr_mono.xtime_nsec -= offset; 2427 tk->ntp_error += offset << tk->ntp_error_shift; 2428 } 2429 2430 /* 2431 * Adjust the timekeeper's multiplier to the correct frequency 2432 * and also to reduce the accumulated error value. 2433 */ 2434 static void timekeeping_adjust(struct timekeeper *tk, s64 offset) 2435 { 2436 u64 ntp_tl = ntp_tick_length(tk->id); 2437 s64 skew = ntp_get_skew_delta(tk->id); 2438 u32 mult; 2439 2440 /* 2441 * Determine the multiplier from the current NTP tick length plus 2442 * skew_delta. The skew biases mult so that ±1 dithering can deliver 2443 * the time_offset slew rate. Recompute when either changes. 2444 */ 2445 if (likely(tk->ntp_tick == ntp_tl && tk->skew_delta == skew)) { 2446 /* Revert to the base mult rate. */ 2447 mult = tk->tkr_mono.mult - tk->ntp_err_mult; 2448 } else { 2449 tk->ntp_tick = ntp_tl; 2450 tk->skew_delta = skew; 2451 /* 2452 * skew_delta is stored pre-divided by HZ (matching time_offset); 2453 * scale it back up to the full per-tick rate for the mult bias. 2454 */ 2455 skew *= NTP_INTERVAL_FREQ; 2456 mult = div64_u64((tk->ntp_tick + skew) >> tk->ntp_error_shift, 2457 tk->cycle_interval); 2458 } 2459 2460 /* 2461 * If the clock is behind the NTP time, increase the multiplier by 1 2462 * to catch up with it. If it's ahead and there was a remainder in the 2463 * tick division, the clock will slow down. Otherwise it will stay 2464 * ahead until the tick length changes to a non-divisible value. 2465 */ 2466 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0; 2467 mult += tk->ntp_err_mult; 2468 2469 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult); 2470 2471 if (unlikely(tk->tkr_mono.clock->maxadj && 2472 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) 2473 > tk->tkr_mono.clock->maxadj))) { 2474 printk_once(KERN_WARNING 2475 "Adjusting %s more than 11%% (%ld vs %ld)\n", 2476 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, 2477 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); 2478 } 2479 2480 /* 2481 * It may be possible that when we entered this function, xtime_nsec 2482 * was very small. Further, if we're slightly speeding the clocksource 2483 * in the code above, its possible the required corrective factor to 2484 * xtime_nsec could cause it to underflow. 2485 * 2486 * Now, since we have already accumulated the second and the NTP 2487 * subsystem has been notified via second_overflow(), we need to skip 2488 * the next update. 2489 */ 2490 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { 2491 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC << 2492 tk->tkr_mono.shift; 2493 tk->xtime_sec--; 2494 tk->skip_second_overflow = 1; 2495 } 2496 } 2497 2498 /* 2499 * accumulate_nsecs_to_secs - Accumulates nsecs into secs 2500 * 2501 * Helper function that accumulates the nsecs greater than a second 2502 * from the xtime_nsec field to the xtime_secs field. 2503 * It also calls into the NTP code to handle leapsecond processing. 2504 */ 2505 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) 2506 { 2507 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; 2508 unsigned int clock_set = 0; 2509 2510 while (tk->tkr_mono.xtime_nsec >= nsecps) { 2511 int leap; 2512 2513 tk->tkr_mono.xtime_nsec -= nsecps; 2514 tk->xtime_sec++; 2515 2516 /* 2517 * Skip NTP update if this second was accumulated before, 2518 * i.e. xtime_nsec underflowed in timekeeping_adjust() 2519 */ 2520 if (unlikely(tk->skip_second_overflow)) { 2521 tk->skip_second_overflow = 0; 2522 continue; 2523 } 2524 2525 /* Figure out if its a leap sec and apply if needed */ 2526 leap = second_overflow(tk->id, tk->xtime_sec); 2527 if (unlikely(leap)) { 2528 struct timespec64 ts; 2529 2530 tk->xtime_sec += leap; 2531 2532 ts.tv_sec = leap; 2533 ts.tv_nsec = 0; 2534 tk_set_wall_to_mono(tk, 2535 timespec64_sub(tk->wall_to_monotonic, ts)); 2536 2537 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); 2538 2539 clock_set = TK_CLOCK_WAS_SET; 2540 } 2541 } 2542 return clock_set; 2543 } 2544 2545 /* 2546 * logarithmic_accumulation - shifted accumulation of cycles 2547 * 2548 * This functions accumulates a shifted interval of cycles into 2549 * a shifted interval nanoseconds. Allows for O(log) accumulation 2550 * loop. 2551 * 2552 * Returns the unconsumed cycles. 2553 */ 2554 static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset, 2555 u32 shift, unsigned int *clock_set) 2556 { 2557 u64 interval = tk->cycle_interval << shift; 2558 u64 snsec_per_sec; 2559 2560 /* If the offset is smaller than a shifted interval, do nothing */ 2561 if (offset < interval) 2562 return offset; 2563 2564 /* Accumulate one shifted interval */ 2565 offset -= interval; 2566 tk->tkr_mono.cycle_last += interval; 2567 tk->tkr_raw.cycle_last += interval; 2568 2569 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; 2570 *clock_set |= accumulate_nsecs_to_secs(tk); 2571 2572 /* Accumulate raw time */ 2573 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift; 2574 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift; 2575 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) { 2576 tk->tkr_raw.xtime_nsec -= snsec_per_sec; 2577 tk->raw_sec++; 2578 } 2579 2580 /* Accumulate error between NTP and clock interval */ 2581 tk->ntp_error += tk->ntp_tick << shift; 2582 tk->ntp_error -= tk->xtime_interval << (tk->ntp_error_shift + shift); 2583 2584 /* 2585 * When skewing, do so by adjusting ntp_error to impart an extra 2586 * target delta into ntp_error per tick, limited to what can be 2587 * drained from time_offset / time_adjust to avoid overshoot. 2588 * 2589 * The base 'mult' value was calculated with the skew taken into 2590 * account, such that the per-tick choice of 'mult' vs. 'mult+1' 2591 * allows for the desired effective rate and ntp_error does not 2592 * grow unbounded. 2593 * 2594 * Once the full desired phase offset is delivered, any remaining 2595 * skew imparted by the adjusted 'mult', accounted above, remains 2596 * in ntp_error and will be compensated by the dithering over time. 2597 */ 2598 if (tk->skew_delta) 2599 tk->ntp_error += ntp_drain_skew(tk->id, tk->skew_delta << shift, 2600 shift) * NTP_INTERVAL_FREQ; 2601 2602 return offset; 2603 } 2604 2605 /* 2606 * timekeeping_advance - Updates the timekeeper to the current time and 2607 * current NTP tick length 2608 */ 2609 static bool __timekeeping_advance(struct tk_data *tkd, enum timekeeping_adv_mode mode) 2610 { 2611 struct timekeeper *tk = &tkd->shadow_timekeeper; 2612 struct timekeeper *real_tk = &tkd->timekeeper; 2613 unsigned int clock_set = 0; 2614 int shift = 0, maxshift; 2615 u64 offset, orig_offset; 2616 2617 /* Make sure we're fully resumed: */ 2618 if (unlikely(timekeeping_suspended)) 2619 return false; 2620 2621 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono), 2622 tk->tkr_mono.cycle_last, tk->tkr_mono.mask, 2623 tk->tkr_mono.clock->max_raw_delta); 2624 orig_offset = offset; 2625 /* Check if there's really nothing to do */ 2626 if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK) 2627 return false; 2628 2629 /* 2630 * With NO_HZ we may have to accumulate many cycle_intervals 2631 * (think "ticks") worth of time at once. To do this efficiently, 2632 * we calculate the largest doubling multiple of cycle_intervals 2633 * that is smaller than the offset. We then accumulate that 2634 * chunk in one go, and then try to consume the next smaller 2635 * doubled multiple. 2636 */ 2637 shift = ilog2(offset) - ilog2(tk->cycle_interval); 2638 shift = max(0, shift); 2639 /* Bound shift to one less than what overflows tick_length */ 2640 maxshift = (64 - (ilog2(ntp_tick_length(tk->id)) + 1)) - 1; 2641 shift = min(shift, maxshift); 2642 while (offset >= tk->cycle_interval) { 2643 offset = logarithmic_accumulation(tk, offset, shift, &clock_set); 2644 if (offset < tk->cycle_interval<<shift) 2645 shift--; 2646 } 2647 2648 /* Adjust the multiplier to correct NTP error */ 2649 timekeeping_adjust(tk, offset); 2650 2651 /* 2652 * Finally, make sure that after the rounding 2653 * xtime_nsec isn't larger than NSEC_PER_SEC 2654 */ 2655 clock_set |= accumulate_nsecs_to_secs(tk); 2656 2657 /* 2658 * To avoid inconsistencies caused adjtimex TK_ADV_FREQ calls 2659 * making small negative adjustments to the base xtime_nsec 2660 * value, only update the coarse clocks if we accumulated time 2661 */ 2662 if (orig_offset != offset) 2663 tk_update_coarse_nsecs(tk); 2664 2665 timekeeping_update_from_shadow(tkd, clock_set); 2666 2667 return !!clock_set; 2668 } 2669 2670 static bool timekeeping_advance(enum timekeeping_adv_mode mode) 2671 { 2672 guard(raw_spinlock_irqsave)(&tk_core.lock); 2673 return __timekeeping_advance(&tk_core, mode); 2674 } 2675 2676 /** 2677 * update_wall_time - Uses the current clocksource to increment the wall time 2678 * 2679 * It also updates the enabled auxiliary clock timekeepers 2680 */ 2681 void update_wall_time(void) 2682 { 2683 if (timekeeping_advance(TK_ADV_TICK)) 2684 clock_was_set_delayed(); 2685 tk_aux_advance(); 2686 } 2687 2688 /** 2689 * getboottime64 - Return the real time of system boot. 2690 * @ts: pointer to the timespec64 to be set 2691 * 2692 * Returns the wall-time of boot in a timespec64. 2693 * 2694 * This is based on the wall_to_monotonic offset and the total suspend 2695 * time. Calls to settimeofday will affect the value returned (which 2696 * basically means that however wrong your real time clock is at boot time, 2697 * you get the right time here). 2698 */ 2699 void getboottime64(struct timespec64 *ts) 2700 { 2701 struct timekeeper *tk = &tk_core.timekeeper; 2702 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); 2703 2704 *ts = ktime_to_timespec64(t); 2705 } 2706 EXPORT_SYMBOL_GPL(getboottime64); 2707 2708 void ktime_get_coarse_real_ts64(struct timespec64 *ts) 2709 { 2710 struct timekeeper *tk = &tk_core.timekeeper; 2711 unsigned int seq; 2712 2713 do { 2714 seq = read_seqcount_begin(&tk_core.seq); 2715 2716 *ts = tk_xtime_coarse(tk); 2717 } while (read_seqcount_retry(&tk_core.seq, seq)); 2718 } 2719 EXPORT_SYMBOL(ktime_get_coarse_real_ts64); 2720 2721 /** 2722 * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor 2723 * @ts: timespec64 to be filled 2724 * 2725 * Fetch the global mg_floor value, convert it to realtime and compare it 2726 * to the current coarse-grained time. Fill @ts with whichever is 2727 * latest. Note that this is a filesystem-specific interface and should be 2728 * avoided outside of that context. 2729 */ 2730 void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts) 2731 { 2732 struct timekeeper *tk = &tk_core.timekeeper; 2733 u64 floor = atomic64_read(&mg_floor); 2734 ktime_t f_real, offset, coarse; 2735 unsigned int seq; 2736 2737 do { 2738 seq = read_seqcount_begin(&tk_core.seq); 2739 *ts = tk_xtime_coarse(tk); 2740 offset = tk_core.timekeeper.offs_real; 2741 } while (read_seqcount_retry(&tk_core.seq, seq)); 2742 2743 coarse = timespec64_to_ktime(*ts); 2744 f_real = ktime_add(floor, offset); 2745 if (ktime_after(f_real, coarse)) 2746 *ts = ktime_to_timespec64(f_real); 2747 } 2748 2749 /** 2750 * ktime_get_real_ts64_mg - attempt to update floor value and return result 2751 * @ts: pointer to the timespec to be set 2752 * 2753 * Get a monotonic fine-grained time value and attempt to swap it into 2754 * mg_floor. If that succeeds then accept the new floor value. If it fails 2755 * then another task raced in during the interim time and updated the 2756 * floor. Since any update to the floor must be later than the previous 2757 * floor, either outcome is acceptable. 2758 * 2759 * Typically this will be called after calling ktime_get_coarse_real_ts64_mg(), 2760 * and determining that the resulting coarse-grained timestamp did not effect 2761 * a change in ctime. Any more recent floor value would effect a change to 2762 * ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure. 2763 * 2764 * @ts will be filled with the latest floor value, regardless of the outcome of 2765 * the cmpxchg. Note that this is a filesystem specific interface and should be 2766 * avoided outside of that context. 2767 */ 2768 void ktime_get_real_ts64_mg(struct timespec64 *ts) 2769 { 2770 struct timekeeper *tk = &tk_core.timekeeper; 2771 ktime_t old = atomic64_read(&mg_floor); 2772 ktime_t offset, mono; 2773 unsigned int seq; 2774 u64 nsecs; 2775 2776 do { 2777 seq = read_seqcount_begin(&tk_core.seq); 2778 2779 ts->tv_sec = tk->xtime_sec; 2780 mono = tk->tkr_mono.base; 2781 nsecs = timekeeping_get_ns(&tk->tkr_mono); 2782 offset = tk_core.timekeeper.offs_real; 2783 } while (read_seqcount_retry(&tk_core.seq, seq)); 2784 2785 mono = ktime_add_ns(mono, nsecs); 2786 2787 /* 2788 * Attempt to update the floor with the new time value. As any 2789 * update must be later then the existing floor, and would effect 2790 * a change to ctime from the perspective of the current task, 2791 * accept the resulting floor value regardless of the outcome of 2792 * the swap. 2793 */ 2794 if (atomic64_try_cmpxchg(&mg_floor, &old, mono)) { 2795 ts->tv_nsec = 0; 2796 timespec64_add_ns(ts, nsecs); 2797 timekeeping_inc_mg_floor_swaps(); 2798 } else { 2799 /* 2800 * Another task changed mg_floor since "old" was fetched. 2801 * "old" has been updated with the latest value of "mg_floor". 2802 * That value is newer than the previous floor value, which 2803 * is enough to effect a change to ctime. Accept it. 2804 */ 2805 *ts = ktime_to_timespec64(ktime_add(old, offset)); 2806 } 2807 } 2808 2809 void ktime_get_coarse_ts64(struct timespec64 *ts) 2810 { 2811 struct timekeeper *tk = &tk_core.timekeeper; 2812 struct timespec64 now, mono; 2813 unsigned int seq; 2814 2815 do { 2816 seq = read_seqcount_begin(&tk_core.seq); 2817 2818 now = tk_xtime_coarse(tk); 2819 mono = tk->wall_to_monotonic; 2820 } while (read_seqcount_retry(&tk_core.seq, seq)); 2821 2822 set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec, 2823 now.tv_nsec + mono.tv_nsec); 2824 } 2825 EXPORT_SYMBOL(ktime_get_coarse_ts64); 2826 2827 /* 2828 * Must hold jiffies_lock 2829 */ 2830 void do_timer(unsigned long ticks) 2831 { 2832 jiffies_64 += ticks; 2833 calc_global_load(); 2834 } 2835 2836 /** 2837 * ktime_get_update_offsets_now - hrtimer helper 2838 * @cwsseq: pointer to check and store the clock was set sequence number 2839 * @offs_real: pointer to storage for monotonic -> realtime offset 2840 * @offs_boot: pointer to storage for monotonic -> boottime offset 2841 * @offs_tai: pointer to storage for monotonic -> clock tai offset 2842 * 2843 * Returns current monotonic time and updates the offsets if the 2844 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are 2845 * different. 2846 * 2847 * Called from hrtimer_interrupt() or retrigger_next_event() 2848 */ 2849 ktime_t ktime_get_update_offsets_now(u32 *cwsseq, ktime_t *offs_real, 2850 ktime_t *offs_boot, ktime_t *offs_tai) 2851 { 2852 struct timekeeper *tk = &tk_core.timekeeper; 2853 unsigned int seq; 2854 ktime_t base; 2855 u64 nsecs; 2856 2857 do { 2858 seq = read_seqcount_begin(&tk_core.seq); 2859 2860 base = tk->tkr_mono.base; 2861 nsecs = timekeeping_get_ns(&tk->tkr_mono); 2862 base = ktime_add_ns(base, nsecs); 2863 2864 if (*cwsseq != tk->clock_was_set_seq) { 2865 *cwsseq = tk->clock_was_set_seq; 2866 *offs_real = tk->offs_real; 2867 *offs_boot = tk->offs_boot; 2868 *offs_tai = tk->offs_tai; 2869 } 2870 2871 /* Handle leapsecond insertion adjustments */ 2872 if (unlikely(base >= tk->next_leap_ktime)) 2873 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); 2874 2875 } while (read_seqcount_retry(&tk_core.seq, seq)); 2876 2877 return base; 2878 } 2879 2880 /* 2881 * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex 2882 */ 2883 static int timekeeping_validate_timex(const struct __kernel_timex *txc, bool aux_clock) 2884 { 2885 if (txc->modes & ADJ_ADJTIME) { 2886 /* singleshot must not be used with any other mode bits */ 2887 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT)) 2888 return -EINVAL; 2889 if (!(txc->modes & ADJ_OFFSET_READONLY) && 2890 !capable(CAP_SYS_TIME)) 2891 return -EPERM; 2892 } else { 2893 /* In order to modify anything, you gotta be super-user! */ 2894 if (txc->modes && !capable(CAP_SYS_TIME)) 2895 return -EPERM; 2896 /* 2897 * if the quartz is off by more than 10% then 2898 * something is VERY wrong! 2899 */ 2900 if (txc->modes & ADJ_TICK && 2901 (txc->tick < 900000/USER_HZ || 2902 txc->tick > 1100000/USER_HZ)) 2903 return -EINVAL; 2904 } 2905 2906 if (txc->modes & ADJ_SETOFFSET) { 2907 /* In order to inject time, you gotta be super-user! */ 2908 if (!capable(CAP_SYS_TIME)) 2909 return -EPERM; 2910 2911 /* 2912 * Validate if a timespec/timeval used to inject a time 2913 * offset is valid. Offsets can be positive or negative, so 2914 * we don't check tv_sec. The value of the timeval/timespec 2915 * is the sum of its fields,but *NOTE*: 2916 * The field tv_usec/tv_nsec must always be non-negative and 2917 * we can't have more nanoseconds/microseconds than a second. 2918 */ 2919 if (txc->time.tv_usec < 0) 2920 return -EINVAL; 2921 2922 if (txc->modes & ADJ_NANO) { 2923 if (txc->time.tv_usec >= NSEC_PER_SEC) 2924 return -EINVAL; 2925 } else { 2926 if (txc->time.tv_usec >= USEC_PER_SEC) 2927 return -EINVAL; 2928 } 2929 } 2930 2931 /* 2932 * Check for potential multiplication overflows that can 2933 * only happen on 64-bit systems: 2934 */ 2935 if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) { 2936 if (LLONG_MIN / PPM_SCALE > txc->freq) 2937 return -EINVAL; 2938 if (LLONG_MAX / PPM_SCALE < txc->freq) 2939 return -EINVAL; 2940 } 2941 2942 if (aux_clock) { 2943 /* Auxiliary clocks are similar to TAI and do not have leap seconds */ 2944 if (txc->modes & ADJ_STATUS && 2945 txc->status & (STA_INS | STA_DEL)) 2946 return -EINVAL; 2947 2948 /* No TAI offset setting */ 2949 if (txc->modes & ADJ_TAI) 2950 return -EINVAL; 2951 2952 /* No PPS support either */ 2953 if (txc->modes & ADJ_STATUS && 2954 txc->status & (STA_PPSFREQ | STA_PPSTIME)) 2955 return -EINVAL; 2956 } 2957 2958 return 0; 2959 } 2960 2961 /** 2962 * random_get_entropy_fallback - Returns the raw clock source value, 2963 * used by random.c for platforms with no valid random_get_entropy(). 2964 */ 2965 unsigned long random_get_entropy_fallback(void) 2966 { 2967 struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono; 2968 struct clocksource *clock = READ_ONCE(tkr->clock); 2969 2970 if (unlikely(timekeeping_suspended || !clock)) 2971 return 0; 2972 return clock->read(clock); 2973 } 2974 EXPORT_SYMBOL_GPL(random_get_entropy_fallback); 2975 2976 struct adjtimex_result { 2977 struct audit_ntp_data ad; 2978 struct timespec64 delta; 2979 bool clock_set; 2980 }; 2981 2982 static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc, 2983 struct adjtimex_result *result) 2984 { 2985 struct timekeeper *tks = &tkd->shadow_timekeeper; 2986 bool aux_clock = !timekeeper_is_core_tk(tks); 2987 struct timespec64 ts; 2988 s32 orig_tai, tai; 2989 int ret; 2990 2991 /* Validate the data before disabling interrupts */ 2992 ret = timekeeping_validate_timex(txc, aux_clock); 2993 if (ret) 2994 return ret; 2995 add_device_randomness(txc, sizeof(*txc)); 2996 2997 if (!aux_clock) { 2998 ktime_get_real_ts64(&ts); 2999 } else { 3000 if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts)) 3001 return -ENODEV; 3002 } 3003 3004 add_device_randomness(&ts, sizeof(ts)); 3005 3006 guard(raw_spinlock_irqsave)(&tkd->lock); 3007 3008 if (!tks->clock_valid) 3009 return -ENODEV; 3010 3011 if (txc->modes & ADJ_SETOFFSET) { 3012 result->delta.tv_sec = txc->time.tv_sec; 3013 result->delta.tv_nsec = txc->time.tv_usec; 3014 if (!(txc->modes & ADJ_NANO)) 3015 result->delta.tv_nsec *= 1000; 3016 ret = __timekeeping_inject_offset(tkd, &result->delta); 3017 if (ret) 3018 return ret; 3019 result->clock_set = true; 3020 } 3021 3022 orig_tai = tai = tks->tai_offset; 3023 ret = ntp_adjtimex(tks->id, txc, &ts, &tai, &result->ad); 3024 3025 if (tai != orig_tai) { 3026 __timekeeping_set_tai_offset(tks, tai); 3027 timekeeping_update_from_shadow(tkd, TK_CLOCK_WAS_SET); 3028 result->clock_set = true; 3029 } else { 3030 tk_update_leap_state_all(tkd); 3031 } 3032 3033 /* Update the multiplier immediately if frequency was set directly */ 3034 if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK)) 3035 result->clock_set |= __timekeeping_advance(tkd, TK_ADV_FREQ); 3036 3037 return ret; 3038 } 3039 3040 /** 3041 * do_adjtimex() - Accessor function to NTP __do_adjtimex function 3042 * @txc: Pointer to kernel_timex structure containing NTP parameters 3043 */ 3044 int do_adjtimex(struct __kernel_timex *txc) 3045 { 3046 struct adjtimex_result result = { }; 3047 int ret; 3048 3049 ret = __do_adjtimex(&tk_core, txc, &result); 3050 if (ret < 0) 3051 return ret; 3052 3053 if (txc->modes & ADJ_SETOFFSET) 3054 audit_tk_injoffset(result.delta); 3055 3056 audit_ntp_log(&result.ad); 3057 3058 if (result.clock_set) 3059 clock_was_set(CLOCK_SET_WALL); 3060 3061 ntp_notify_cmos_timer(result.delta.tv_sec != 0); 3062 3063 return ret; 3064 } 3065 3066 /* 3067 * Invoked from NTP with the time keeper lock held, so lockless access is 3068 * fine. 3069 */ 3070 long ktime_get_ntp_seconds(unsigned int id) 3071 { 3072 return timekeeper_data[id].timekeeper.xtime_sec; 3073 } 3074 3075 #ifdef CONFIG_NTP_PPS 3076 /** 3077 * hardpps() - Accessor function to NTP __hardpps function 3078 * @phase_ts: Pointer to timespec64 structure representing phase timestamp 3079 * @raw_ts: Pointer to timespec64 structure representing raw timestamp 3080 */ 3081 void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts) 3082 { 3083 guard(raw_spinlock_irqsave)(&tk_core.lock); 3084 __hardpps(phase_ts, raw_ts); 3085 } 3086 EXPORT_SYMBOL(hardpps); 3087 #endif /* CONFIG_NTP_PPS */ 3088 3089 #ifdef CONFIG_POSIX_AUX_CLOCKS 3090 #include "posix-timers.h" 3091 3092 /* 3093 * Bitmap for the activated auxiliary timekeepers to allow lockless quick 3094 * checks in the hot paths without touching extra cache lines. If set, then 3095 * the state of the corresponding timekeeper has to be re-checked under 3096 * timekeeper::lock. 3097 */ 3098 static unsigned long aux_timekeepers; 3099 3100 static inline unsigned int clockid_to_tkid(unsigned int id) 3101 { 3102 return TIMEKEEPER_AUX_FIRST + id - CLOCK_AUX; 3103 } 3104 3105 static inline struct tk_data *aux_get_tk_data(clockid_t id) 3106 { 3107 if (!clockid_is_aux_clock(id)) 3108 return NULL; 3109 return &timekeeper_data[clockid_to_tkid(id)]; 3110 } 3111 3112 /* Invoked from timekeeping after a clocksource change */ 3113 static void tk_aux_update_clocksource(void) 3114 { 3115 unsigned long active = READ_ONCE(aux_timekeepers); 3116 unsigned int id; 3117 3118 for_each_set_bit(id, &active, BITS_PER_LONG) { 3119 struct tk_data *tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST]; 3120 struct timekeeper *tks = &tkd->shadow_timekeeper; 3121 3122 guard(raw_spinlock_irqsave)(&tkd->lock); 3123 if (!tks->clock_valid) 3124 continue; 3125 3126 timekeeping_forward_now(tks); 3127 tk_setup_internals(tks, tk_core.timekeeper.tkr_raw.clock); 3128 timekeeping_update_from_shadow(tkd, TK_UPDATE_ALL); 3129 } 3130 } 3131 3132 static void tk_aux_advance(void) 3133 { 3134 unsigned long active = READ_ONCE(aux_timekeepers); 3135 unsigned int id; 3136 3137 /* Lockless quick check to avoid extra cache lines */ 3138 for_each_set_bit(id, &active, BITS_PER_LONG) { 3139 struct tk_data *aux_tkd = &timekeeper_data[id + TIMEKEEPER_AUX_FIRST]; 3140 3141 guard(raw_spinlock)(&aux_tkd->lock); 3142 if (aux_tkd->shadow_timekeeper.clock_valid) 3143 __timekeeping_advance(aux_tkd, TK_ADV_TICK); 3144 } 3145 } 3146 3147 /** 3148 * ktime_get_aux - Get time for a AUX clock 3149 * @id: ID of the clock to read (CLOCK_AUX...) 3150 * @kt: Pointer to ktime_t to store the time stamp 3151 * 3152 * Returns: True if the timestamp is valid, false otherwise 3153 */ 3154 bool ktime_get_aux(clockid_t id, ktime_t *kt) 3155 { 3156 struct tk_data *aux_tkd = aux_get_tk_data(id); 3157 struct timekeeper *aux_tk; 3158 unsigned int seq; 3159 ktime_t base; 3160 u64 nsecs; 3161 3162 WARN_ON(timekeeping_suspended); 3163 3164 if (!aux_tkd) 3165 return false; 3166 3167 aux_tk = &aux_tkd->timekeeper; 3168 do { 3169 seq = read_seqcount_begin(&aux_tkd->seq); 3170 if (!aux_tk->clock_valid) 3171 return false; 3172 3173 base = ktime_add(aux_tk->tkr_mono.base, aux_tk->offs_aux); 3174 nsecs = timekeeping_get_ns(&aux_tk->tkr_mono); 3175 } while (read_seqcount_retry(&aux_tkd->seq, seq)); 3176 3177 *kt = ktime_add_ns(base, nsecs); 3178 return true; 3179 } 3180 EXPORT_SYMBOL_GPL(ktime_get_aux); 3181 3182 /** 3183 * ktime_get_aux_ts64 - Get time for a AUX clock 3184 * @id: ID of the clock to read (CLOCK_AUX...) 3185 * @ts: Pointer to timespec64 to store the time stamp 3186 * 3187 * Returns: True if the timestamp is valid, false otherwise 3188 */ 3189 bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *ts) 3190 { 3191 ktime_t now; 3192 3193 if (!ktime_get_aux(id, &now)) 3194 return false; 3195 *ts = ktime_to_timespec64(now); 3196 return true; 3197 } 3198 EXPORT_SYMBOL_GPL(ktime_get_aux_ts64); 3199 3200 static int aux_get_res(clockid_t id, struct timespec64 *tp) 3201 { 3202 if (!clockid_is_aux_clock(id)) 3203 return -ENODEV; 3204 3205 tp->tv_sec = aux_clock_resolution_ns() / NSEC_PER_SEC; 3206 tp->tv_nsec = aux_clock_resolution_ns() % NSEC_PER_SEC; 3207 return 0; 3208 } 3209 3210 static int aux_get_timespec(clockid_t id, struct timespec64 *tp) 3211 { 3212 return ktime_get_aux_ts64(id, tp) ? 0 : -ENODEV; 3213 } 3214 3215 static int aux_clock_set(const clockid_t id, const struct timespec64 *tnew) 3216 { 3217 struct tk_data *aux_tkd = aux_get_tk_data(id); 3218 struct timekeeper *aux_tks; 3219 ktime_t tnow, nsecs; 3220 3221 if (!timespec64_valid_settod(tnew)) 3222 return -EINVAL; 3223 if (!aux_tkd) 3224 return -ENODEV; 3225 3226 aux_tks = &aux_tkd->shadow_timekeeper; 3227 3228 guard(raw_spinlock_irq)(&aux_tkd->lock); 3229 if (!aux_tks->clock_valid) 3230 return -ENODEV; 3231 3232 /* Forward the timekeeper base time */ 3233 timekeeping_forward_now(aux_tks); 3234 /* 3235 * Get the updated base time. tkr_mono.base has not been 3236 * updated yet, so do that first. That makes the update 3237 * in timekeeping_update_from_shadow() redundant, but 3238 * that's harmless. After that @tnow can be calculated 3239 * by using tkr_mono::cycle_last, which has been set 3240 * by timekeeping_forward_now(). 3241 */ 3242 tk_update_ktime_data(aux_tks); 3243 nsecs = timekeeping_cycles_to_ns(&aux_tks->tkr_mono, aux_tks->tkr_mono.cycle_last); 3244 tnow = ktime_add(aux_tks->tkr_mono.base, nsecs); 3245 3246 /* 3247 * Calculate the new AUX offset as delta to @tnow ("monotonic"). 3248 * That avoids all the tk::xtime back and forth conversions as 3249 * xtime ("realtime") is not applicable for auxiliary clocks and 3250 * kept in sync with "monotonic". 3251 */ 3252 tk_update_aux_offs(aux_tks, ktime_sub(timespec64_to_ktime(*tnew), tnow)); 3253 3254 timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL); 3255 return 0; 3256 } 3257 3258 static int aux_clock_adj(const clockid_t id, struct __kernel_timex *txc) 3259 { 3260 struct tk_data *aux_tkd = aux_get_tk_data(id); 3261 struct adjtimex_result result = { }; 3262 3263 if (!aux_tkd) 3264 return -ENODEV; 3265 3266 /* 3267 * @result is ignored for now as there are neither hrtimers nor a 3268 * RTC related to auxiliary clocks for now. 3269 */ 3270 return __do_adjtimex(aux_tkd, txc, &result); 3271 } 3272 3273 const struct k_clock clock_aux = { 3274 .clock_getres = aux_get_res, 3275 .clock_get_timespec = aux_get_timespec, 3276 .clock_set = aux_clock_set, 3277 .clock_adj = aux_clock_adj, 3278 }; 3279 3280 static void aux_clock_enable(clockid_t id) 3281 { 3282 struct tk_read_base *tkr_raw = &tk_core.timekeeper.tkr_raw; 3283 struct tk_data *aux_tkd = aux_get_tk_data(id); 3284 struct timekeeper *aux_tks = &aux_tkd->shadow_timekeeper; 3285 3286 /* Prevent the core timekeeper from changing. */ 3287 guard(raw_spinlock_irq)(&tk_core.lock); 3288 3289 /* 3290 * Setup the auxiliary clock assuming that the raw core timekeeper 3291 * clock frequency conversion is close enough. Userspace has to 3292 * adjust for the deviation via clock_adjtime(2). 3293 */ 3294 guard(raw_spinlock_nested)(&aux_tkd->lock); 3295 3296 /* Remove leftovers of a previous registration */ 3297 memset(aux_tks, 0, sizeof(*aux_tks)); 3298 /* Restore the timekeeper id */ 3299 aux_tks->id = aux_tkd->timekeeper.id; 3300 /* Setup the timekeeper based on the current system clocksource */ 3301 tk_setup_internals(aux_tks, tkr_raw->clock); 3302 3303 /* Mark it valid and set it live */ 3304 aux_tks->clock_valid = true; 3305 timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL); 3306 } 3307 3308 static void aux_clock_disable(clockid_t id) 3309 { 3310 struct tk_data *aux_tkd = aux_get_tk_data(id); 3311 3312 guard(raw_spinlock_irq)(&aux_tkd->lock); 3313 aux_tkd->shadow_timekeeper.clock_valid = false; 3314 timekeeping_update_from_shadow(aux_tkd, TK_UPDATE_ALL); 3315 } 3316 3317 static DEFINE_MUTEX(aux_clock_mutex); 3318 3319 static ssize_t aux_clock_enable_store(struct kobject *kobj, struct kobj_attribute *attr, 3320 const char *buf, size_t count) 3321 { 3322 /* Lazy atoi() as name is "0..7" */ 3323 int id = kobj->name[0] & 0x7; 3324 bool enable; 3325 3326 if (!capable(CAP_SYS_TIME)) 3327 return -EPERM; 3328 3329 if (kstrtobool(buf, &enable) < 0) 3330 return -EINVAL; 3331 3332 guard(mutex)(&aux_clock_mutex); 3333 if (enable == test_bit(id, &aux_timekeepers)) 3334 return count; 3335 3336 if (enable) { 3337 aux_clock_enable(CLOCK_AUX + id); 3338 set_bit(id, &aux_timekeepers); 3339 } else { 3340 aux_clock_disable(CLOCK_AUX + id); 3341 clear_bit(id, &aux_timekeepers); 3342 } 3343 return count; 3344 } 3345 3346 static ssize_t aux_clock_enable_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 3347 { 3348 unsigned long active = READ_ONCE(aux_timekeepers); 3349 /* Lazy atoi() as name is "0..7" */ 3350 int id = kobj->name[0] & 0x7; 3351 3352 return sysfs_emit(buf, "%d\n", test_bit(id, &active)); 3353 } 3354 3355 static struct kobj_attribute aux_clock_enable_attr = __ATTR_RW(aux_clock_enable); 3356 3357 static struct attribute *aux_clock_enable_attrs[] = { 3358 &aux_clock_enable_attr.attr, 3359 NULL 3360 }; 3361 3362 static const struct attribute_group aux_clock_enable_attr_group = { 3363 .attrs = aux_clock_enable_attrs, 3364 }; 3365 3366 static int __init tk_aux_sysfs_init(void) 3367 { 3368 struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj); 3369 struct kobject *clks[MAX_AUX_CLOCKS]; 3370 int ret = -ENOMEM; 3371 int i; 3372 3373 if (!tko) 3374 return ret; 3375 3376 auxo = kobject_create_and_add("aux_clocks", tko); 3377 if (!auxo) 3378 goto err_clean; 3379 3380 for (i = 0; i < MAX_AUX_CLOCKS; i++) { 3381 char id[2] = { [0] = '0' + i, }; 3382 clks[i] = kobject_create_and_add(id, auxo); 3383 3384 if (!clks[i]) { 3385 ret = -ENOMEM; 3386 goto err_clks; 3387 } 3388 3389 ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group); 3390 if (ret) 3391 goto err_clk; 3392 } 3393 return 0; 3394 3395 err_clk: 3396 kobject_put(clks[i]); 3397 err_clks: 3398 while (--i >= 0) { 3399 sysfs_remove_group(clks[i], &aux_clock_enable_attr_group); 3400 kobject_put(clks[i]); 3401 } 3402 err_clean: 3403 kobject_put(auxo); 3404 kobject_put(tko); 3405 return ret; 3406 } 3407 late_initcall(tk_aux_sysfs_init); 3408 3409 static __init void tk_aux_setup(void) 3410 { 3411 for (int i = TIMEKEEPER_AUX_FIRST; i <= TIMEKEEPER_AUX_LAST; i++) 3412 tkd_basic_setup(&timekeeper_data[i], i, false); 3413 } 3414 #endif /* CONFIG_POSIX_AUX_CLOCKS */ 3415