1 // SPDX-License-Identifier: GPL-2.0
4  *  timer.c, moved in commit 8524070b7982.
21 #include <linux/tick.h>
28 #include "tick-internal.h"
38 	/* Update timekeeper when a tick has passed */
62  * struct tk_fast - NMI safe timekeeper
75 /* Suspend-time cycles value for halted fast timekeeper. */
92  * returns nanoseconds already so no conversion is required, hence mult=1
130  * Multigrain timestamps require tracking the latest fine-grained timestamp
131  * that has been issued, and never returning a coarse-grained timestamp that is
134  * mg_floor represents the latest fine-grained time that has been handed out as
136  * converted to a realtime clock value on an as-needed basis.
148 	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {  in tk_normalize_xtime()
149 		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;  in tk_normalize_xtime()
150 		tk->xtime_sec++;  in tk_normalize_xtime()
152 	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {  in tk_normalize_xtime()
153 		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;  in tk_normalize_xtime()
154 		tk->raw_sec++;  in tk_normalize_xtime()
162 	ts.tv_sec = tk->xtime_sec;  in tk_xtime()
163 	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);  in tk_xtime()
169 	tk->xtime_sec = ts->tv_sec;  in tk_set_xtime()
170 	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;  in tk_set_xtime()
175 	tk->xtime_sec += ts->tv_sec;  in tk_xtime_add()
176 	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;  in tk_xtime_add()
185 	 * Verify consistency of: offset_real = -wall_to_monotonic  in tk_set_wall_to_mono()
188 	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,  in tk_set_wall_to_mono()
189 					-tk->wall_to_monotonic.tv_nsec);  in tk_set_wall_to_mono()
190 	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));  in tk_set_wall_to_mono()
191 	tk->wall_to_monotonic = wtm;  in tk_set_wall_to_mono()
192 	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);  in tk_set_wall_to_mono()
193 	/* Paired with READ_ONCE() in ktime_mono_to_any() */  in tk_set_wall_to_mono()
194 	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));  in tk_set_wall_to_mono()
195 	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));  in tk_set_wall_to_mono()
200 	/* Paired with READ_ONCE() in ktime_mono_to_any() */  in tk_update_sleep_time()
201 	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));  in tk_update_sleep_time()
206 	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);  in tk_update_sleep_time()
210  * tk_clock_read - atomic clocksource read() helper
212  * This helper is necessary to use in the read paths because, while the
219  * a read of the fast-timekeeper tkrs (which is protected by its own locking
224 	struct clocksource *clock = READ_ONCE(tkr->clock);  in tk_clock_read()
226 	return clock->read(clock);  in tk_clock_read()
230  * tk_setup_internals - Set up internals to use clocksource clock.
246 	++tk->cs_was_changed_seq;  in tk_setup_internals()
247 	old_clock = tk->tkr_mono.clock;  in tk_setup_internals()
248 	tk->tkr_mono.clock = clock;  in tk_setup_internals()
249 	tk->tkr_mono.mask = clock->mask;  in tk_setup_internals()
250 	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);  in tk_setup_internals()
252 	tk->tkr_raw.clock = clock;  in tk_setup_internals()
253 	tk->tkr_raw.mask = clock->mask;  in tk_setup_internals()
254 	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;  in tk_setup_internals()
256 	/* Do the ns -> cycle conversion first, using original mult */  in tk_setup_internals()
258 	tmp <<= clock->shift;  in tk_setup_internals()
260 	tmp += clock->mult/2;  in tk_setup_internals()
261 	do_div(tmp, clock->mult);  in tk_setup_internals()
266 	tk->cycle_interval = interval;  in tk_setup_internals()
268 	/* Go back from cycles -> shifted ns */  in tk_setup_internals()
269 	tk->xtime_interval = interval * clock->mult;  in tk_setup_internals()
270 	tk->xtime_remainder = ntpinterval - tk->xtime_interval;  in tk_setup_internals()
271 	tk->raw_interval = interval * clock->mult;  in tk_setup_internals()
275 		int shift_change = clock->shift - old_clock->shift;  in tk_setup_internals()
277 			tk->tkr_mono.xtime_nsec >>= -shift_change;  in tk_setup_internals()
278 			tk->tkr_raw.xtime_nsec >>= -shift_change;  in tk_setup_internals()
280 			tk->tkr_mono.xtime_nsec <<= shift_change;  in tk_setup_internals()
281 			tk->tkr_raw.xtime_nsec <<= shift_change;  in tk_setup_internals()
285 	tk->tkr_mono.shift = clock->shift;  in tk_setup_internals()
286 	tk->tkr_raw.shift = clock->shift;  in tk_setup_internals()
288 	tk->ntp_error = 0;  in tk_setup_internals()
289 	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;  in tk_setup_internals()
290 	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;  in tk_setup_internals()
297 	tk->tkr_mono.mult = clock->mult;  in tk_setup_internals()
298 	tk->tkr_raw.mult = clock->mult;  in tk_setup_internals()
299 	tk->ntp_err_mult = 0;  in tk_setup_internals()
300 	tk->skip_second_overflow = 0;  in tk_setup_internals()
306 	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);  in delta_to_ns_safe()
312 	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;  in timekeeping_cycles_to_ns()
316 	 * overflows the multiplication with tkr->mult.  in timekeeping_cycles_to_ns()
318 	if (unlikely(delta > tkr->clock->max_cycles)) {  in timekeeping_cycles_to_ns()
322 		 * mask being set in the delta.  in timekeeping_cycles_to_ns()
325 			return tkr->xtime_nsec >> tkr->shift;  in timekeeping_cycles_to_ns()
330 	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;  in timekeeping_cycles_to_ns()
339  * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
349  * which is still consistent. In the worst case this can result is a
356 	struct tk_read_base *base = tkf->base;  in update_fast_timekeeper()
359 	write_seqcount_latch_begin(&tkf->seq);  in update_fast_timekeeper()
365 	write_seqcount_latch(&tkf->seq);  in update_fast_timekeeper()
370 	write_seqcount_latch_end(&tkf->seq);  in update_fast_timekeeper()
380 		seq = read_seqcount_latch(&tkf->seq);  in __ktime_get_fast_ns()
381 		tkr = tkf->base + (seq & 0x01);  in __ktime_get_fast_ns()
382 		now = ktime_to_ns(tkr->base);  in __ktime_get_fast_ns()
384 	} while (read_seqcount_latch_retry(&tkf->seq, seq));  in __ktime_get_fast_ns()
390  * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
407  * |12345678---> reader order
416  * for a CPU local observation is when an NMI hits in the middle of
428  * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
440  * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
456  * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
457  * partially updated.  Since the tk->offs_boot update is a rare event, this
467 	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));  in ktime_get_boot_fast_ns()
472  * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
478  * with the possibility of wrong timestamps in post processing.
484 	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));  in ktime_get_tai_fast_ns()
495 		seq = raw_read_seqcount_latch(&tkf->seq);  in __ktime_get_real_fast()
496 		tkr = tkf->base + (seq & 0x01);  in __ktime_get_real_fast()
497 		basem = ktime_to_ns(tkr->base);  in __ktime_get_real_fast()
498 		baser = ktime_to_ns(tkr->base_real);  in __ktime_get_real_fast()
500 	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));  in __ktime_get_real_fast()
508  * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
519  * ktime_get_fast_timestamps: - NMI safe timestamps
525  * happens late during resume and not in timekeeping_resume(). That could
543  * real time does not. On the next readout both are in sync again.
550  * steps of suspend/resume. The accessors still work, but the timestamps
553  * For regular suspend/resume there is no observable difference vs. sched
556  * OTOH, access to sched clock is not guaranteed across suspend/resume on
557  * all systems either so it depends on the hardware in use.
560  * using sched clock in a similar way as during early boot. But it's not as
568 	snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono);  in ktime_get_fast_timestamps()
569 	snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot));  in ktime_get_fast_timestamps()
573  * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
585 	const struct tk_read_base *tkr = &tk->tkr_mono;  in halt_fast_timekeeper()
590 	tkr_dummy.base_real = tkr->base + tk->offs_real;  in halt_fast_timekeeper()
593 	tkr = &tk->tkr_raw;  in halt_fast_timekeeper()
607  * pvclock_gtod_register_notifier - register a pvclock timedata update listener
624  * pvclock_gtod_unregister_notifier - unregister a pvclock
636  * tk_update_leap_state - helper to update the next_leap_ktime
640 	tk->next_leap_ktime = ntp_get_next_leap();  in tk_update_leap_state()
641 	if (tk->next_leap_ktime != KTIME_MAX)  in tk_update_leap_state()
643 		tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);  in tk_update_leap_state()
652 	write_seqcount_begin(&tkd->seq);  in tk_update_leap_state_all()
653 	tk_update_leap_state(&tkd->shadow_timekeeper);  in tk_update_leap_state_all()
654 	tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime;  in tk_update_leap_state_all()
655 	write_seqcount_end(&tkd->seq);  in tk_update_leap_state_all()
673 	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);  in tk_update_ktime_data()
674 	nsec = (u32) tk->wall_to_monotonic.tv_nsec;  in tk_update_ktime_data()
675 	tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);  in tk_update_ktime_data()
680 	 * this into account before updating tk->ktime_sec.  in tk_update_ktime_data()
682 	nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);  in tk_update_ktime_data()
685 	tk->ktime_sec = seconds;  in tk_update_ktime_data()
688 	tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);  in tk_update_ktime_data()
696 	lockdep_assert_held(&tkd->lock);  in timekeeping_restore_shadow()
697 	memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper));  in timekeeping_restore_shadow()
704 	lockdep_assert_held(&tkd->lock);  in timekeeping_update_from_shadow()
710 	 * reading from the VDSO after the VDSO update and then reading in  in timekeeping_update_from_shadow()
713 	write_seqcount_begin(&tkd->seq);  in timekeeping_update_from_shadow()
716 		tk->ntp_error = 0;  in timekeeping_update_from_shadow()
726 	tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;  in timekeeping_update_from_shadow()
727 	update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);  in timekeeping_update_from_shadow()
728 	update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);  in timekeeping_update_from_shadow()
731 		tk->clock_was_set_seq++;  in timekeeping_update_from_shadow()
741 	memcpy(&tkd->timekeeper, tk, sizeof(*tk));  in timekeeping_update_from_shadow()
742 	write_seqcount_end(&tkd->seq);  in timekeeping_update_from_shadow()
746  * timekeeping_forward_now - update clock to the current time
757 	cycle_now = tk_clock_read(&tk->tkr_mono);  in timekeeping_forward_now()
758 	delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask,  in timekeeping_forward_now()
759 				  tk->tkr_mono.clock->max_raw_delta);  in timekeeping_forward_now()
760 	tk->tkr_mono.cycle_last = cycle_now;  in timekeeping_forward_now()
761 	tk->tkr_raw.cycle_last  = cycle_now;  in timekeeping_forward_now()
764 		u64 max = tk->tkr_mono.clock->max_cycles;  in timekeeping_forward_now()
767 		tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;  in timekeeping_forward_now()
768 		tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;  in timekeeping_forward_now()
770 		delta -= incr;  in timekeeping_forward_now()
775  * ktime_get_real_ts64 - Returns the time of day in a timespec64.
778  * Returns the time of day in a timespec64 (WARN if suspended).
791 		ts->tv_sec = tk->xtime_sec;  in ktime_get_real_ts64()
792 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_real_ts64()
796 	ts->tv_nsec = 0;  in ktime_get_real_ts64()
812 		base = tk->tkr_mono.base;  in ktime_get()
813 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get()
831 		nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;  in ktime_get_resolution_ns()
855 		base = ktime_add(tk->tkr_mono.base, *offset);  in ktime_get_with_offset()
856 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_with_offset()
876 		base = ktime_add(tk->tkr_mono.base, *offset);  in ktime_get_coarse_with_offset()
877 		nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;  in ktime_get_coarse_with_offset()
886  * ktime_mono_to_any() - convert monotonic time to any other time
898 		 * Paired with WRITE_ONCE()s in tk_set_wall_to_mono() and  in ktime_mono_to_any()
914  * ktime_get_raw - Returns the raw monotonic time in ktime_t format
925 		base = tk->tkr_raw.base;  in ktime_get_raw()
926 		nsecs = timekeeping_get_ns(&tk->tkr_raw);  in ktime_get_raw()
935  * ktime_get_ts64 - get the monotonic clock in timespec64 format
940  * in normalized timespec64 format in the variable pointed to by @ts.
953 		ts->tv_sec = tk->xtime_sec;  in ktime_get_ts64()
954 		nsec = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_ts64()
955 		tomono = tk->wall_to_monotonic;  in ktime_get_ts64()
959 	ts->tv_sec += tomono.tv_sec;  in ktime_get_ts64()
960 	ts->tv_nsec = 0;  in ktime_get_ts64()
966  * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
969  * serialized read. tk->ktime_sec is of type 'unsigned long' so this
979 	return tk->ktime_sec;  in ktime_get_seconds()
984  * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
988  * For 64bit systems the fast access to tk->xtime_sec is preserved. On
990  * counter to provide "atomic" access to the 64bit tk->xtime_sec
1000 		return tk->xtime_sec;  in ktime_get_real_seconds()
1004 		seconds = tk->xtime_sec;  in ktime_get_real_seconds()
1013  * __ktime_get_real_seconds - The same as ktime_get_real_seconds
1021 	return tk->xtime_sec;  in __ktime_get_real_seconds()
1025  * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1043 		now = tk_clock_read(&tk->tkr_mono);  in ktime_get_snapshot()
1044 		systime_snapshot->cs_id = tk->tkr_mono.clock->id;  in ktime_get_snapshot()
1045 		systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;  in ktime_get_snapshot()
1046 		systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;  in ktime_get_snapshot()
1047 		base_real = ktime_add(tk->tkr_mono.base,  in ktime_get_snapshot()
1049 		base_boot = ktime_add(tk->tkr_mono.base,  in ktime_get_snapshot()
1051 		base_raw = tk->tkr_raw.base;  in ktime_get_snapshot()
1052 		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);  in ktime_get_snapshot()
1053 		nsec_raw  = timekeeping_cycles_to_ns(&tk->tkr_raw, now);  in ktime_get_snapshot()
1056 	systime_snapshot->cycles = now;  in ktime_get_snapshot()
1057 	systime_snapshot->real = ktime_add_ns(base_real, nsec_real);  in ktime_get_snapshot()
1058 	systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real);  in ktime_get_snapshot()
1059 	systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);  in ktime_get_snapshot()
1070 	if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||  in scale64_check_overflow()
1071 	    ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))  in scale64_check_overflow()
1072 		return -EOVERFLOW;  in scale64_check_overflow()
1081  * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1084  * @total_history_cycles:	Total history length in cycles
1114 		total_history_cycles - partial_history_cycles :  in adjust_historical_crosststamp()
1122 		ktime_sub(ts->sys_monoraw, history->raw));  in adjust_historical_crosststamp()
1129 	 * If there is a discontinuity in the history, scale monotonic raw  in adjust_historical_crosststamp()
1137 			(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);  in adjust_historical_crosststamp()
1140 			ktime_sub(ts->sys_realtime, history->real));  in adjust_historical_crosststamp()
1149 		ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);  in adjust_historical_crosststamp()
1150 		ts->sys_realtime = ktime_add_ns(history->real, corr_real);  in adjust_historical_crosststamp()
1152 		ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);  in adjust_historical_crosststamp()
1153 		ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);  in adjust_historical_crosststamp()
1160  * timestamp_in_interval - true if ts is chronologically in [start, end]
1192 	if (cs->id == scv->cs_id)  in convert_base_to_cs()
1197 	 * re-evaluating @base as the clocksource might change concurrently.  in convert_base_to_cs()
1199 	base = READ_ONCE(cs->base);  in convert_base_to_cs()
1200 	if (!base || base->id != scv->cs_id)  in convert_base_to_cs()
1203 	num = scv->use_nsecs ? cs->freq_khz : base->numerator;  in convert_base_to_cs()
1204 	den = scv->use_nsecs ? USEC_PER_SEC : base->denominator;  in convert_base_to_cs()
1206 	if (!convert_clock(&scv->cycles, num, den))  in convert_base_to_cs()
1209 	scv->cycles += base->offset;  in convert_base_to_cs()
1220 	 * re-evaluating @base as the clocksource might change concurrently.  in convert_cs_to_base()
1222 	base = READ_ONCE(cs->base);  in convert_cs_to_base()
1223 	if (!base || base->id != base_id)  in convert_cs_to_base()
1226 	*cycles -= base->offset;  in convert_cs_to_base()
1227 	if (!convert_clock(cycles, base->denominator, base->numerator))  in convert_cs_to_base()
1236 	if (BITS_TO_BYTES(fls64(*delta) + tkr->shift) >= sizeof(*delta))  in convert_ns_to_cs()
1239 	*delta = div_u64((*delta << tkr->shift) - tkr->xtime_nsec, tkr->mult);  in convert_ns_to_cs()
1244  * ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp
1261 		if ((u64)treal < tk->tkr_mono.base_real)  in ktime_real_to_base_clock()
1263 		delta = (u64)treal - tk->tkr_mono.base_real;  in ktime_real_to_base_clock()
1266 		*cycles = tk->tkr_mono.cycle_last + delta;  in ktime_real_to_base_clock()
1276  * get_device_system_crosststamp - Synchronously capture system/device timestamp
1311 		ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);  in get_device_system_crosststamp()
1322 			return -ENODEV;  in get_device_system_crosststamp()
1329 		now = tk_clock_read(&tk->tkr_mono);  in get_device_system_crosststamp()
1330 		interval_start = tk->tkr_mono.cycle_last;  in get_device_system_crosststamp()
1332 			clock_was_set_seq = tk->clock_was_set_seq;  in get_device_system_crosststamp()
1333 			cs_was_changed_seq = tk->cs_was_changed_seq;  in get_device_system_crosststamp()
1340 		base_real = ktime_add(tk->tkr_mono.base,  in get_device_system_crosststamp()
1342 		base_raw = tk->tkr_raw.base;  in get_device_system_crosststamp()
1344 		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);  in get_device_system_crosststamp()
1345 		nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles);  in get_device_system_crosststamp()
1348 	xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);  in get_device_system_crosststamp()
1349 	xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);  in get_device_system_crosststamp()
1365 		    !timestamp_in_interval(history_begin->cycles,  in get_device_system_crosststamp()
1367 		    history_begin->cs_was_changed_seq != cs_was_changed_seq)  in get_device_system_crosststamp()
1368 			return -EINVAL;  in get_device_system_crosststamp()
1369 		partial_history_cycles = cycles - system_counterval.cycles;  in get_device_system_crosststamp()
1370 		total_history_cycles = cycles - history_begin->cycles;  in get_device_system_crosststamp()
1372 			history_begin->clock_was_set_seq != clock_was_set_seq;  in get_device_system_crosststamp()
1387  * timekeeping_clocksource_has_base - Check whether the current clocksource
1400 	 * This is a snapshot, so no point in using the sequence  in timekeeping_clocksource_has_base()
1401 	 * count. Just prevent the compiler from re-evaluating @base as the  in timekeeping_clocksource_has_base()
1404 	struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base);  in timekeeping_clocksource_has_base()
1406 	return base ? base->id == id : false;  in timekeeping_clocksource_has_base()
1411  * do_settimeofday64 - Sets the time of day.
1421 		return -EINVAL;  in do_settimeofday64()
1431 		if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) {  in do_settimeofday64()
1433 			return -EINVAL;  in do_settimeofday64()
1436 		tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta));  in do_settimeofday64()
1451  * timekeeping_inject_offset - Adds or subtracts from the current time.
1458 	if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)  in timekeeping_inject_offset()
1459 		return -EINVAL;  in timekeeping_inject_offset()
1469 		if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 ||  in timekeeping_inject_offset()
1472 			return -EINVAL;  in timekeeping_inject_offset()
1476 		tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts));  in timekeeping_inject_offset()
1496  * would either need to write a program to do it in /etc/rc (and risk
1499  * compile in the timezone information into the kernel.  Bad, bad....
1501  *						- TYT, 1992-01-01
1503  * The best thing to do is to keep the CMOS clock in universal time (UTC)
1520  * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1524 	tk->tai_offset = tai_offset;  in __timekeeping_set_tai_offset()
1525 	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));  in __timekeeping_set_tai_offset()
1529  * change_clocksource - Swaps clocksources if a new one is available
1538 	 * If the clocksource is in a module, get a module reference.  in change_clocksource()
1539 	 * Succeeds for built-in code (owner == NULL) as well. Abort if the  in change_clocksource()
1542 	if (!try_module_get(new->owner))  in change_clocksource()
1546 	if (new->enable && new->enable(new) != 0) {  in change_clocksource()
1547 		module_put(new->owner);  in change_clocksource()
1555 		old = tks->tkr_mono.clock;  in change_clocksource()
1561 		if (old->disable)  in change_clocksource()
1562 			old->disable(old);  in change_clocksource()
1563 		module_put(old->owner);  in change_clocksource()
1570  * timekeeping_notify - Install a new clock source
1580 	if (tk->tkr_mono.clock == clock)  in timekeeping_notify()
1584 	return tk->tkr_mono.clock == clock ? 0 : -1;  in timekeeping_notify()
1588  * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1591  * Returns the raw monotonic time (completely un-modified by ntp)
1601 		ts->tv_sec = tk->raw_sec;  in ktime_get_raw_ts64()
1602 		nsecs = timekeeping_get_ns(&tk->tkr_raw);  in ktime_get_raw_ts64()
1606 	ts->tv_nsec = 0;  in ktime_get_raw_ts64()
1613  * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1624 		ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;  in timekeeping_valid_for_hres()
1632  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1643 		ret = tk->tkr_mono.clock->max_idle_ns;  in timekeeping_max_deferment()
1651  * read_persistent_clock64 -  Return time from the persistent clock.
1658  *  XXX - Do be sure to remove it once all arches implement it.
1662 	ts->tv_sec = 0;  in read_persistent_clock64()
1663 	ts->tv_nsec = 0;  in read_persistent_clock64()
1667  * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1670  * @boot_offset:  offset that is defined as wall_time - boot_time
1689 	raw_spin_lock_init(&tkd->lock);  in tkd_basic_setup()
1690 	seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);  in tkd_basic_setup()
1696  * The flag starts of false and is only set when a suspend reaches
1698  * timekeeper clocksource is not stopping across suspend and has been
1703  * If a suspend fails before reaching timekeeping_resume() then the flag
1712  * timekeeping_init - Initializes the clocksource and common timekeeping values
1745 	if (clock->enable)  in timekeeping_init()
1746 		clock->enable(clock);  in timekeeping_init()
1750 	tks->raw_sec = 0;  in timekeeping_init()
1757 /* time in seconds when suspend began for persistent clock */
1761  * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1763  * @delta:	Pointer to the delta value in timespec64 format
1765  * Takes a timespec offset measuring a suspend interval and properly
1778 	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));  in __timekeeping_inject_sleeptime()
1787  * 1) non-stop clocksource
1796  * 3) becomes needless, so in this case we don't need to call
1810  * But if system has 2), 2) will definitely be used, so in this
1820  * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1828  * a suspend offset to be injected into the timekeeping values.
1847  * timekeeping_resume - Resumes the generic timekeeping subsystem.
1852 	struct clocksource *clock = tks->tkr_mono.clock;  in timekeeping_resume()
1868 	 * used: Nonstop clocksource during suspend, persistent clock and rtc  in timekeeping_resume()
1873 	 *	suspend-nonstop clocksource -> persistent clock -> rtc  in timekeeping_resume()
1874 	 * The less preferred source will only be tried if there is no better  in timekeeping_resume()
1875 	 * usable source. The rtc part is handled separately in rtc core code.  in timekeeping_resume()
1877 	cycle_now = tk_clock_read(&tks->tkr_mono);  in timekeeping_resume()
1892 	/* Re-base the last cycle value */  in timekeeping_resume()
1893 	tks->tkr_mono.cycle_last = cycle_now;  in timekeeping_resume()
1894 	tks->tkr_raw.cycle_last  = cycle_now;  in timekeeping_resume()
1896 	tks->ntp_error = 0;  in timekeeping_resume()
1937 	 * use in suspend timing.  in timekeeping_suspend()
1939 	curr_clock = tks->tkr_mono.clock;  in timekeeping_suspend()
1940 	cycle_now = tks->tkr_mono.cycle_last;  in timekeeping_suspend()
1945 		 * To avoid drift caused by repeated suspend/resumes,  in timekeeping_suspend()
1947 		 * try to compensate so the difference in system time  in timekeeping_suspend()
1976 /* sysfs resume/suspend bits for timekeeping */
1979 	.suspend	= timekeeping_suspend,
1996 	s64 interval = tk->cycle_interval;  in timekeeping_apply_adjustment()
2000 	} else if (mult_adj == -1) {  in timekeeping_apply_adjustment()
2001 		interval = -interval;  in timekeeping_apply_adjustment()
2002 		offset = -offset;  in timekeeping_apply_adjustment()
2027 	 * So offset stores the non-accumulated cycles. Thus the current  in timekeeping_apply_adjustment()
2028 	 * time (in shifted nanoseconds) is:  in timekeeping_apply_adjustment()
2031 	 * to keep time consistent. In other words, we can't jump back  in timekeeping_apply_adjustment()
2032 	 * in time, and we also want to avoid jumping forward in time.  in timekeeping_apply_adjustment()
2051 	 *	xtime_nsec_2 = xtime_nsec_1 - offset  in timekeeping_apply_adjustment()
2053 	 *	xtime_nsec -= offset  in timekeeping_apply_adjustment()
2055 	if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {  in timekeeping_apply_adjustment()
2061 	tk->tkr_mono.mult += mult_adj;  in timekeeping_apply_adjustment()
2062 	tk->xtime_interval += interval;  in timekeeping_apply_adjustment()
2063 	tk->tkr_mono.xtime_nsec -= offset;  in timekeeping_apply_adjustment()
2076 	 * Determine the multiplier from the current NTP tick length.  in timekeeping_adjust()
2077 	 * Avoid expensive division when the tick length doesn't change.  in timekeeping_adjust()
2079 	if (likely(tk->ntp_tick == ntp_tl)) {  in timekeeping_adjust()
2080 		mult = tk->tkr_mono.mult - tk->ntp_err_mult;  in timekeeping_adjust()
2082 		tk->ntp_tick = ntp_tl;  in timekeeping_adjust()
2083 		mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -  in timekeeping_adjust()
2084 				 tk->xtime_remainder, tk->cycle_interval);  in timekeeping_adjust()
2089 	 * to catch up with it. If it's ahead and there was a remainder in the  in timekeeping_adjust()
2090 	 * tick division, the clock will slow down. Otherwise it will stay  in timekeeping_adjust()
2091 	 * ahead until the tick length changes to a non-divisible value.  in timekeeping_adjust()
2093 	tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;  in timekeeping_adjust()
2094 	mult += tk->ntp_err_mult;  in timekeeping_adjust()
2096 	timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);  in timekeeping_adjust()
2098 	if (unlikely(tk->tkr_mono.clock->maxadj &&  in timekeeping_adjust()
2099 		(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)  in timekeeping_adjust()
2100 			> tk->tkr_mono.clock->maxadj))) {  in timekeeping_adjust()
2103 			tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,  in timekeeping_adjust()
2104 			(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);  in timekeeping_adjust()
2110 	 * in the code above, its possible the required corrective factor to  in timekeeping_adjust()
2117 	if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {  in timekeeping_adjust()
2118 		tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<  in timekeeping_adjust()
2119 							tk->tkr_mono.shift;  in timekeeping_adjust()
2120 		tk->xtime_sec--;  in timekeeping_adjust()
2121 		tk->skip_second_overflow = 1;  in timekeeping_adjust()
2126  * accumulate_nsecs_to_secs - Accumulates nsecs into secs
2134 	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;  in accumulate_nsecs_to_secs()
2137 	while (tk->tkr_mono.xtime_nsec >= nsecps) {  in accumulate_nsecs_to_secs()
2140 		tk->tkr_mono.xtime_nsec -= nsecps;  in accumulate_nsecs_to_secs()
2141 		tk->xtime_sec++;  in accumulate_nsecs_to_secs()
2145 		 * i.e. xtime_nsec underflowed in timekeeping_adjust()  in accumulate_nsecs_to_secs()
2147 		if (unlikely(tk->skip_second_overflow)) {  in accumulate_nsecs_to_secs()
2148 			tk->skip_second_overflow = 0;  in accumulate_nsecs_to_secs()
2153 		leap = second_overflow(tk->xtime_sec);  in accumulate_nsecs_to_secs()
2157 			tk->xtime_sec += leap;  in accumulate_nsecs_to_secs()
2162 				timespec64_sub(tk->wall_to_monotonic, ts));  in accumulate_nsecs_to_secs()
2164 			__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);  in accumulate_nsecs_to_secs()
2173  * logarithmic_accumulation - shifted accumulation of cycles
2184 	u64 interval = tk->cycle_interval << shift;  in logarithmic_accumulation()
2192 	offset -= interval;  in logarithmic_accumulation()
2193 	tk->tkr_mono.cycle_last += interval;  in logarithmic_accumulation()
2194 	tk->tkr_raw.cycle_last  += interval;  in logarithmic_accumulation()
2196 	tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;  in logarithmic_accumulation()
2200 	tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;  in logarithmic_accumulation()
2201 	snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;  in logarithmic_accumulation()
2202 	while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {  in logarithmic_accumulation()
2203 		tk->tkr_raw.xtime_nsec -= snsec_per_sec;  in logarithmic_accumulation()
2204 		tk->raw_sec++;  in logarithmic_accumulation()
2208 	tk->ntp_error += tk->ntp_tick << shift;  in logarithmic_accumulation()
2209 	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<  in logarithmic_accumulation()
2210 						(tk->ntp_error_shift + shift);  in logarithmic_accumulation()
2216  * timekeeping_advance - Updates the timekeeper to the current time and
2217  * current NTP tick length
2233 	offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),  in timekeeping_advance()
2234 				   tk->tkr_mono.cycle_last, tk->tkr_mono.mask,  in timekeeping_advance()
2235 				   tk->tkr_mono.clock->max_raw_delta);  in timekeeping_advance()
2238 	if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)  in timekeeping_advance()
2246 	 * chunk in one go, and then try to consume the next smaller  in timekeeping_advance()
2249 	shift = ilog2(offset) - ilog2(tk->cycle_interval);  in timekeeping_advance()
2252 	maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;  in timekeeping_advance()
2254 	while (offset >= tk->cycle_interval) {  in timekeeping_advance()
2256 		if (offset < tk->cycle_interval<<shift)  in timekeeping_advance()
2257 			shift--;  in timekeeping_advance()
2275  * update_wall_time - Uses the current clocksource to increment the wall time
2285  * getboottime64 - Return the real time of system boot.
2288  * Returns the wall-time of boot in a timespec64.
2290  * This is based on the wall_to_monotonic offset and the total suspend
2298 	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);  in getboottime64()
2318  * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
2322  * to the current coarse-grained time. Fill @ts with whichever is
2323  * latest. Note that this is a filesystem-specific interface and should be
2346  * ktime_get_real_ts64_mg - attempt to update floor value and return result
2349  * Get a monotonic fine-grained time value and attempt to swap it into
2351  * then another task raced in during the interim time and updated the
2356  * and determining that the resulting coarse-grained timestamp did not effect
2357  * a change in ctime. Any more recent floor value would effect a change to
2358  * ctime, so there is no need to retry the atomic64_try_cmpxchg() on failure.
2375 		ts->tv_sec = tk->xtime_sec;  in ktime_get_real_ts64_mg()
2376 		mono = tk->tkr_mono.base;  in ktime_get_real_ts64_mg()
2377 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_real_ts64_mg()
2391 		ts->tv_nsec = 0;  in ktime_get_real_ts64_mg()
2415 		mono = tk->wall_to_monotonic;  in ktime_get_coarse_ts64()
2433  * ktime_get_update_offsets_now - hrtimer helper
2435  * @offs_real:	pointer to storage for monotonic -> realtime offset
2436  * @offs_boot:	pointer to storage for monotonic -> boottime offset
2437  * @offs_tai:	pointer to storage for monotonic -> clock tai offset
2440  * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2456 		base = tk->tkr_mono.base;  in ktime_get_update_offsets_now()
2457 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_update_offsets_now()
2460 		if (*cwsseq != tk->clock_was_set_seq) {  in ktime_get_update_offsets_now()
2461 			*cwsseq = tk->clock_was_set_seq;  in ktime_get_update_offsets_now()
2462 			*offs_real = tk->offs_real;  in ktime_get_update_offsets_now()
2463 			*offs_boot = tk->offs_boot;  in ktime_get_update_offsets_now()
2464 			*offs_tai = tk->offs_tai;  in ktime_get_update_offsets_now()
2468 		if (unlikely(base >= tk->next_leap_ktime))  in ktime_get_update_offsets_now()
2469 			*offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));  in ktime_get_update_offsets_now()
2477  * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2481 	if (txc->modes & ADJ_ADJTIME) {  in timekeeping_validate_timex()
2483 		if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))  in timekeeping_validate_timex()
2484 			return -EINVAL;  in timekeeping_validate_timex()
2485 		if (!(txc->modes & ADJ_OFFSET_READONLY) &&  in timekeeping_validate_timex()
2487 			return -EPERM;  in timekeeping_validate_timex()
2489 		/* In order to modify anything, you gotta be super-user! */  in timekeeping_validate_timex()
2490 		if (txc->modes && !capable(CAP_SYS_TIME))  in timekeeping_validate_timex()
2491 			return -EPERM;  in timekeeping_validate_timex()
2496 		if (txc->modes & ADJ_TICK &&  in timekeeping_validate_timex()
2497 		    (txc->tick <  900000/USER_HZ ||  in timekeeping_validate_timex()
2498 		     txc->tick > 1100000/USER_HZ))  in timekeeping_validate_timex()
2499 			return -EINVAL;  in timekeeping_validate_timex()
2502 	if (txc->modes & ADJ_SETOFFSET) {  in timekeeping_validate_timex()
2503 		/* In order to inject time, you gotta be super-user! */  in timekeeping_validate_timex()
2505 			return -EPERM;  in timekeeping_validate_timex()
2512 		 * The field tv_usec/tv_nsec must always be non-negative and  in timekeeping_validate_timex()
2515 		if (txc->time.tv_usec < 0)  in timekeeping_validate_timex()
2516 			return -EINVAL;  in timekeeping_validate_timex()
2518 		if (txc->modes & ADJ_NANO) {  in timekeeping_validate_timex()
2519 			if (txc->time.tv_usec >= NSEC_PER_SEC)  in timekeeping_validate_timex()
2520 				return -EINVAL;  in timekeeping_validate_timex()
2522 			if (txc->time.tv_usec >= USEC_PER_SEC)  in timekeeping_validate_timex()
2523 				return -EINVAL;  in timekeeping_validate_timex()
2529 	 * only happen on 64-bit systems:  in timekeeping_validate_timex()
2531 	if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {  in timekeeping_validate_timex()
2532 		if (LLONG_MIN / PPM_SCALE > txc->freq)  in timekeeping_validate_timex()
2533 			return -EINVAL;  in timekeeping_validate_timex()
2534 		if (LLONG_MAX / PPM_SCALE < txc->freq)  in timekeeping_validate_timex()
2535 			return -EINVAL;  in timekeeping_validate_timex()
2542  * random_get_entropy_fallback - Returns the raw clock source value,
2543  * used by random.c for platforms with no valid random_get_entropy().
2548 	struct clocksource *clock = READ_ONCE(tkr->clock);  in random_get_entropy_fallback()
2552 	return clock->read(clock);  in random_get_entropy_fallback()
2557  * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2574 	if (txc->modes & ADJ_SETOFFSET) {  in do_adjtimex()
2577 		delta.tv_sec  = txc->time.tv_sec;  in do_adjtimex()
2578 		delta.tv_nsec = txc->time.tv_usec;  in do_adjtimex()
2579 		if (!(txc->modes & ADJ_NANO))  in do_adjtimex()
2598 		orig_tai = tai = tks->tai_offset;  in do_adjtimex()
2613 	if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))  in do_adjtimex()
2626  * hardpps() - Accessor function to NTP __hardpps function