1 // SPDX-License-Identifier: GPL-2.0
28 #include "tick-internal.h"
62  * struct tk_fast - NMI safe timekeeper
75 /* Suspend-time cycles value for halted fast timekeeper. */
130  * Multigrain timestamps require tracking the latest fine-grained timestamp
131  * that has been issued, and never returning a coarse-grained timestamp that is
134  * mg_floor represents the latest fine-grained time that has been handed out as
136  * converted to a realtime clock value on an as-needed basis.
148 	while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {  in tk_normalize_xtime()
149 		tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;  in tk_normalize_xtime()
150 		tk->xtime_sec++;  in tk_normalize_xtime()
152 	while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {  in tk_normalize_xtime()
153 		tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;  in tk_normalize_xtime()
154 		tk->raw_sec++;  in tk_normalize_xtime()
162 	ts.tv_sec = tk->xtime_sec;  in tk_xtime()
163 	ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);  in tk_xtime()
171 	ts.tv_sec = tk->xtime_sec;  in tk_xtime_coarse()
172 	ts.tv_nsec = tk->coarse_nsec;  in tk_xtime_coarse()
179  * amount when the multiplication factor of the clock is adjusted, which
187 	tk->coarse_nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;  in tk_update_coarse_nsecs()
192 	tk->xtime_sec = ts->tv_sec;  in tk_set_xtime()
193 	tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;  in tk_set_xtime()
199 	tk->xtime_sec += ts->tv_sec;  in tk_xtime_add()
200 	tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;  in tk_xtime_add()
210 	 * Verify consistency of: offset_real = -wall_to_monotonic  in tk_set_wall_to_mono()
213 	set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,  in tk_set_wall_to_mono()
214 					-tk->wall_to_monotonic.tv_nsec);  in tk_set_wall_to_mono()
215 	WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));  in tk_set_wall_to_mono()
216 	tk->wall_to_monotonic = wtm;  in tk_set_wall_to_mono()
217 	set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);  in tk_set_wall_to_mono()
219 	WRITE_ONCE(tk->offs_real, timespec64_to_ktime(tmp));  in tk_set_wall_to_mono()
220 	WRITE_ONCE(tk->offs_tai, ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)));  in tk_set_wall_to_mono()
226 	WRITE_ONCE(tk->offs_boot, ktime_add(tk->offs_boot, delta));  in tk_update_sleep_time()
231 	tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);  in tk_update_sleep_time()
235  * tk_clock_read - atomic clocksource read() helper
240  * the tkr's clocksource may change between the read reference, and the
241  * clock reference passed to the read function.  This can cause crashes if
244  * a read of the fast-timekeeper tkrs (which is protected by its own locking
249 	struct clocksource *clock = READ_ONCE(tkr->clock);  in tk_clock_read()
251 	return clock->read(clock);  in tk_clock_read()
255  * tk_setup_internals - Set up internals to use clocksource clock.
271 	++tk->cs_was_changed_seq;  in tk_setup_internals()
272 	old_clock = tk->tkr_mono.clock;  in tk_setup_internals()
273 	tk->tkr_mono.clock = clock;  in tk_setup_internals()
274 	tk->tkr_mono.mask = clock->mask;  in tk_setup_internals()
275 	tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);  in tk_setup_internals()
277 	tk->tkr_raw.clock = clock;  in tk_setup_internals()
278 	tk->tkr_raw.mask = clock->mask;  in tk_setup_internals()
279 	tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;  in tk_setup_internals()
281 	/* Do the ns -> cycle conversion first, using original mult */  in tk_setup_internals()
283 	tmp <<= clock->shift;  in tk_setup_internals()
285 	tmp += clock->mult/2;  in tk_setup_internals()
286 	do_div(tmp, clock->mult);  in tk_setup_internals()
291 	tk->cycle_interval = interval;  in tk_setup_internals()
293 	/* Go back from cycles -> shifted ns */  in tk_setup_internals()
294 	tk->xtime_interval = interval * clock->mult;  in tk_setup_internals()
295 	tk->xtime_remainder = ntpinterval - tk->xtime_interval;  in tk_setup_internals()
296 	tk->raw_interval = interval * clock->mult;  in tk_setup_internals()
300 		int shift_change = clock->shift - old_clock->shift;  in tk_setup_internals()
302 			tk->tkr_mono.xtime_nsec >>= -shift_change;  in tk_setup_internals()
303 			tk->tkr_raw.xtime_nsec >>= -shift_change;  in tk_setup_internals()
305 			tk->tkr_mono.xtime_nsec <<= shift_change;  in tk_setup_internals()
306 			tk->tkr_raw.xtime_nsec <<= shift_change;  in tk_setup_internals()
310 	tk->tkr_mono.shift = clock->shift;  in tk_setup_internals()
311 	tk->tkr_raw.shift = clock->shift;  in tk_setup_internals()
313 	tk->ntp_error = 0;  in tk_setup_internals()
314 	tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;  in tk_setup_internals()
315 	tk->ntp_tick = ntpinterval << tk->ntp_error_shift;  in tk_setup_internals()
322 	tk->tkr_mono.mult = clock->mult;  in tk_setup_internals()
323 	tk->tkr_raw.mult = clock->mult;  in tk_setup_internals()
324 	tk->ntp_err_mult = 0;  in tk_setup_internals()
325 	tk->skip_second_overflow = 0;  in tk_setup_internals()
331 	return mul_u64_u32_add_u64_shr(delta, tkr->mult, tkr->xtime_nsec, tkr->shift);  in delta_to_ns_safe()
337 	u64 mask = tkr->mask, delta = (cycles - tkr->cycle_last) & mask;  in timekeeping_cycles_to_ns()
341 	 * overflows the multiplication with tkr->mult.  in timekeeping_cycles_to_ns()
343 	if (unlikely(delta > tkr->clock->max_cycles)) {  in timekeeping_cycles_to_ns()
350 			return tkr->xtime_nsec >> tkr->shift;  in timekeeping_cycles_to_ns()
355 	return ((delta * tkr->mult) + tkr->xtime_nsec) >> tkr->shift;  in timekeeping_cycles_to_ns()
364  * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
381 	struct tk_read_base *base = tkf->base;  in update_fast_timekeeper()
384 	write_seqcount_latch_begin(&tkf->seq);  in update_fast_timekeeper()
390 	write_seqcount_latch(&tkf->seq);  in update_fast_timekeeper()
395 	write_seqcount_latch_end(&tkf->seq);  in update_fast_timekeeper()
405 		seq = read_seqcount_latch(&tkf->seq);  in __ktime_get_fast_ns()
406 		tkr = tkf->base + (seq & 0x01);  in __ktime_get_fast_ns()
407 		now = ktime_to_ns(tkr->base);  in __ktime_get_fast_ns()
409 	} while (read_seqcount_latch_retry(&tkf->seq, seq));  in __ktime_get_fast_ns()
415  * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
432  * |12345678---> reader order
453  * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
456  * conversion factor is not affected by NTP/PTP correction.
465  * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
481  * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
482  * partially updated.  Since the tk->offs_boot update is a rare event, this
492 	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));  in ktime_get_boot_fast_ns()
497  * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
509 	return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));  in ktime_get_tai_fast_ns()
514  * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
526 		seq = raw_read_seqcount_latch(&tkf->seq);  in ktime_get_real_fast_ns()
527 		tkr = tkf->base + (seq & 0x01);  in ktime_get_real_fast_ns()
528 		baser = ktime_to_ns(tkr->base_real);  in ktime_get_real_fast_ns()
530 	} while (raw_read_seqcount_latch_retry(&tkf->seq, seq));  in ktime_get_real_fast_ns()
537  * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
549 	const struct tk_read_base *tkr = &tk->tkr_mono;  in halt_fast_timekeeper()
554 	tkr_dummy.base_real = tkr->base + tk->offs_real;  in halt_fast_timekeeper()
557 	tkr = &tk->tkr_raw;  in halt_fast_timekeeper()
571  * pvclock_gtod_register_notifier - register a pvclock timedata update listener
588  * pvclock_gtod_unregister_notifier - unregister a pvclock
600  * tk_update_leap_state - helper to update the next_leap_ktime
604 	tk->next_leap_ktime = ntp_get_next_leap();  in tk_update_leap_state()
605 	if (tk->next_leap_ktime != KTIME_MAX)  in tk_update_leap_state()
607 		tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);  in tk_update_leap_state()
616 	write_seqcount_begin(&tkd->seq);  in tk_update_leap_state_all()
617 	tk_update_leap_state(&tkd->shadow_timekeeper);  in tk_update_leap_state_all()
618 	tkd->timekeeper.next_leap_ktime = tkd->shadow_timekeeper.next_leap_ktime;  in tk_update_leap_state_all()
619 	write_seqcount_end(&tkd->seq);  in tk_update_leap_state_all()
637 	seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);  in tk_update_ktime_data()
638 	nsec = (u32) tk->wall_to_monotonic.tv_nsec;  in tk_update_ktime_data()
639 	tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);  in tk_update_ktime_data()
644 	 * this into account before updating tk->ktime_sec.  in tk_update_ktime_data()
646 	nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);  in tk_update_ktime_data()
649 	tk->ktime_sec = seconds;  in tk_update_ktime_data()
652 	tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);  in tk_update_ktime_data()
660 	lockdep_assert_held(&tkd->lock);  in timekeeping_restore_shadow()
661 	memcpy(&tkd->shadow_timekeeper, &tkd->timekeeper, sizeof(tkd->timekeeper));  in timekeeping_restore_shadow()
668 	lockdep_assert_held(&tkd->lock);  in timekeeping_update_from_shadow()
677 	write_seqcount_begin(&tkd->seq);  in timekeeping_update_from_shadow()
680 		tk->ntp_error = 0;  in timekeeping_update_from_shadow()
690 	tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;  in timekeeping_update_from_shadow()
691 	update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);  in timekeeping_update_from_shadow()
692 	update_fast_timekeeper(&tk->tkr_raw,  &tk_fast_raw);  in timekeeping_update_from_shadow()
695 		tk->clock_was_set_seq++;  in timekeeping_update_from_shadow()
705 	memcpy(&tkd->timekeeper, tk, sizeof(*tk));  in timekeeping_update_from_shadow()
706 	write_seqcount_end(&tkd->seq);  in timekeeping_update_from_shadow()
710  * timekeeping_forward_now - update clock to the current time
721 	cycle_now = tk_clock_read(&tk->tkr_mono);  in timekeeping_forward_now()
722 	delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask,  in timekeeping_forward_now()
723 				  tk->tkr_mono.clock->max_raw_delta);  in timekeeping_forward_now()
724 	tk->tkr_mono.cycle_last = cycle_now;  in timekeeping_forward_now()
725 	tk->tkr_raw.cycle_last  = cycle_now;  in timekeeping_forward_now()
728 		u64 max = tk->tkr_mono.clock->max_cycles;  in timekeeping_forward_now()
731 		tk->tkr_mono.xtime_nsec += incr * tk->tkr_mono.mult;  in timekeeping_forward_now()
732 		tk->tkr_raw.xtime_nsec += incr * tk->tkr_raw.mult;  in timekeeping_forward_now()
734 		delta -= incr;  in timekeeping_forward_now()
740  * ktime_get_real_ts64 - Returns the time of day in a timespec64.
756 		ts->tv_sec = tk->xtime_sec;  in ktime_get_real_ts64()
757 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_real_ts64()
761 	ts->tv_nsec = 0;  in ktime_get_real_ts64()
777 		base = tk->tkr_mono.base;  in ktime_get()
778 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get()
796 		nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;  in ktime_get_resolution_ns()
820 		base = ktime_add(tk->tkr_mono.base, *offset);  in ktime_get_with_offset()
821 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_with_offset()
841 		base = ktime_add(tk->tkr_mono.base, *offset);  in ktime_get_coarse_with_offset()
842 		nsecs = tk->coarse_nsec;  in ktime_get_coarse_with_offset()
851  * ktime_mono_to_any() - convert monotonic time to any other time
879  * ktime_get_raw - Returns the raw monotonic time in ktime_t format
890 		base = tk->tkr_raw.base;  in ktime_get_raw()
891 		nsecs = timekeeping_get_ns(&tk->tkr_raw);  in ktime_get_raw()
900  * ktime_get_ts64 - get the monotonic clock in timespec64 format
918 		ts->tv_sec = tk->xtime_sec;  in ktime_get_ts64()
919 		nsec = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_ts64()
920 		tomono = tk->wall_to_monotonic;  in ktime_get_ts64()
924 	ts->tv_sec += tomono.tv_sec;  in ktime_get_ts64()
925 	ts->tv_nsec = 0;  in ktime_get_ts64()
931  * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
934  * serialized read. tk->ktime_sec is of type 'unsigned long' so this
944 	return tk->ktime_sec;  in ktime_get_seconds()
949  * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
953  * For 64bit systems the fast access to tk->xtime_sec is preserved. On
955  * counter to provide "atomic" access to the 64bit tk->xtime_sec
965 		return tk->xtime_sec;  in ktime_get_real_seconds()
969 		seconds = tk->xtime_sec;  in ktime_get_real_seconds()
978  * __ktime_get_real_seconds - The same as ktime_get_real_seconds
986 	return tk->xtime_sec;  in __ktime_get_real_seconds()
990  * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1008 		now = tk_clock_read(&tk->tkr_mono);  in ktime_get_snapshot()
1009 		systime_snapshot->cs_id = tk->tkr_mono.clock->id;  in ktime_get_snapshot()
1010 		systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;  in ktime_get_snapshot()
1011 		systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;  in ktime_get_snapshot()
1012 		base_real = ktime_add(tk->tkr_mono.base,  in ktime_get_snapshot()
1014 		base_boot = ktime_add(tk->tkr_mono.base,  in ktime_get_snapshot()
1016 		base_raw = tk->tkr_raw.base;  in ktime_get_snapshot()
1017 		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);  in ktime_get_snapshot()
1018 		nsec_raw  = timekeeping_cycles_to_ns(&tk->tkr_raw, now);  in ktime_get_snapshot()
1021 	systime_snapshot->cycles = now;  in ktime_get_snapshot()
1022 	systime_snapshot->real = ktime_add_ns(base_real, nsec_real);  in ktime_get_snapshot()
1023 	systime_snapshot->boot = ktime_add_ns(base_boot, nsec_real);  in ktime_get_snapshot()
1024 	systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);  in ktime_get_snapshot()
1028 /* Scale base by mult/div checking for overflow */
1029 static int scale64_check_overflow(u64 mult, u64 div, u64 *base)  in scale64_check_overflow()  argument
1033 	tmp = div64_u64_rem(*base, div, &rem);  in scale64_check_overflow()
1035 	if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||  in scale64_check_overflow()
1036 	    ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))  in scale64_check_overflow()
1037 		return -EOVERFLOW;  in scale64_check_overflow()
1040 	rem = div64_u64(rem * mult, div);  in scale64_check_overflow()
1046  * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1058  * reference point and ending at the start of the current interval. The cycle
1079 		total_history_cycles - partial_history_cycles :  in adjust_historical_crosststamp()
1087 		ktime_sub(ts->sys_monoraw, history->raw));  in adjust_historical_crosststamp()
1102 			(corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);  in adjust_historical_crosststamp()
1105 			ktime_sub(ts->sys_realtime, history->real));  in adjust_historical_crosststamp()
1114 		ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);  in adjust_historical_crosststamp()
1115 		ts->sys_realtime = ktime_add_ns(history->real, corr_real);  in adjust_historical_crosststamp()
1117 		ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);  in adjust_historical_crosststamp()
1118 		ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);  in adjust_historical_crosststamp()
1125  * timestamp_in_interval - true if ts is chronologically in [start, end]
1157 	if (cs->id == scv->cs_id)  in convert_base_to_cs()
1162 	 * re-evaluating @base as the clocksource might change concurrently.  in convert_base_to_cs()
1164 	base = READ_ONCE(cs->base);  in convert_base_to_cs()
1165 	if (!base || base->id != scv->cs_id)  in convert_base_to_cs()
1168 	num = scv->use_nsecs ? cs->freq_khz : base->numerator;  in convert_base_to_cs()
1169 	den = scv->use_nsecs ? USEC_PER_SEC : base->denominator;  in convert_base_to_cs()
1171 	if (!convert_clock(&scv->cycles, num, den))  in convert_base_to_cs()
1174 	scv->cycles += base->offset;  in convert_base_to_cs()
1185 	 * re-evaluating @base as the clocksource might change concurrently.  in convert_cs_to_base()
1187 	base = READ_ONCE(cs->base);  in convert_cs_to_base()
1188 	if (!base || base->id != base_id)  in convert_cs_to_base()
1191 	*cycles -= base->offset;  in convert_cs_to_base()
1192 	if (!convert_clock(cycles, base->denominator, base->numerator))  in convert_cs_to_base()
1201 	if (BITS_TO_BYTES(fls64(*delta) + tkr->shift) >= sizeof(*delta))  in convert_ns_to_cs()
1204 	*delta = div_u64((*delta << tkr->shift) - tkr->xtime_nsec, tkr->mult);  in convert_ns_to_cs()
1209  * ktime_real_to_base_clock() - Convert CLOCK_REALTIME timestamp to a base clock timestamp
1226 		if ((u64)treal < tk->tkr_mono.base_real)  in ktime_real_to_base_clock()
1228 		delta = (u64)treal - tk->tkr_mono.base_real;  in ktime_real_to_base_clock()
1231 		*cycles = tk->tkr_mono.cycle_last + delta;  in ktime_real_to_base_clock()
1241  * get_device_system_crosststamp - Synchronously capture system/device timestamp
1245  * @history_begin:	Historical reference point used to interpolate system
1276 		ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);  in get_device_system_crosststamp()
1287 			return -ENODEV;  in get_device_system_crosststamp()
1294 		now = tk_clock_read(&tk->tkr_mono);  in get_device_system_crosststamp()
1295 		interval_start = tk->tkr_mono.cycle_last;  in get_device_system_crosststamp()
1297 			clock_was_set_seq = tk->clock_was_set_seq;  in get_device_system_crosststamp()
1298 			cs_was_changed_seq = tk->cs_was_changed_seq;  in get_device_system_crosststamp()
1305 		base_real = ktime_add(tk->tkr_mono.base,  in get_device_system_crosststamp()
1307 		base_raw = tk->tkr_raw.base;  in get_device_system_crosststamp()
1309 		nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, cycles);  in get_device_system_crosststamp()
1310 		nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, cycles);  in get_device_system_crosststamp()
1313 	xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);  in get_device_system_crosststamp()
1314 	xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);  in get_device_system_crosststamp()
1326 		 * history reference and that the history doesn't cross a  in get_device_system_crosststamp()
1330 		    !timestamp_in_interval(history_begin->cycles,  in get_device_system_crosststamp()
1332 		    history_begin->cs_was_changed_seq != cs_was_changed_seq)  in get_device_system_crosststamp()
1333 			return -EINVAL;  in get_device_system_crosststamp()
1334 		partial_history_cycles = cycles - system_counterval.cycles;  in get_device_system_crosststamp()
1335 		total_history_cycles = cycles - history_begin->cycles;  in get_device_system_crosststamp()
1337 			history_begin->clock_was_set_seq != clock_was_set_seq;  in get_device_system_crosststamp()
1352  * timekeeping_clocksource_has_base - Check whether the current clocksource
1366 	 * count. Just prevent the compiler from re-evaluating @base as the  in timekeeping_clocksource_has_base()
1369 	struct clocksource_base *base = READ_ONCE(tk_core.timekeeper.tkr_mono.clock->base);  in timekeeping_clocksource_has_base()
1371 	return base ? base->id == id : false;  in timekeeping_clocksource_has_base()
1376  * do_settimeofday64 - Sets the time of day.
1386 		return -EINVAL;  in do_settimeofday64()
1396 		if (timespec64_compare(&tks->wall_to_monotonic, &ts_delta) > 0) {  in do_settimeofday64()
1398 			return -EINVAL;  in do_settimeofday64()
1401 		tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, ts_delta));  in do_settimeofday64()
1416  * timekeeping_inject_offset - Adds or subtracts from the current time.
1423 	if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)  in timekeeping_inject_offset()
1424 		return -EINVAL;  in timekeeping_inject_offset()
1434 		if (timespec64_compare(&tks->wall_to_monotonic, ts) > 0 ||  in timekeeping_inject_offset()
1437 			return -EINVAL;  in timekeeping_inject_offset()
1441 		tk_set_wall_to_mono(tks, timespec64_sub(tks->wall_to_monotonic, *ts));  in timekeeping_inject_offset()
1466  *						- TYT, 1992-01-01
1485  * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
1489 	tk->tai_offset = tai_offset;  in __timekeeping_set_tai_offset()
1490 	tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));  in __timekeeping_set_tai_offset()
1494  * change_clocksource - Swaps clocksources if a new one is available
1503 	 * If the clocksource is in a module, get a module reference.  in change_clocksource()
1504 	 * Succeeds for built-in code (owner == NULL) as well. Abort if the  in change_clocksource()
1505 	 * reference can't be acquired.  in change_clocksource()
1507 	if (!try_module_get(new->owner))  in change_clocksource()
1511 	if (new->enable && new->enable(new) != 0) {  in change_clocksource()
1512 		module_put(new->owner);  in change_clocksource()
1520 		old = tks->tkr_mono.clock;  in change_clocksource()
1526 		if (old->disable)  in change_clocksource()
1527 			old->disable(old);  in change_clocksource()
1528 		module_put(old->owner);  in change_clocksource()
1535  * timekeeping_notify - Install a new clock source
1545 	if (tk->tkr_mono.clock == clock)  in timekeeping_notify()
1549 	return tk->tkr_mono.clock == clock ? 0 : -1;  in timekeeping_notify()
1553  * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
1556  * Returns the raw monotonic time (completely un-modified by ntp)
1566 		ts->tv_sec = tk->raw_sec;  in ktime_get_raw_ts64()
1567 		nsecs = timekeeping_get_ns(&tk->tkr_raw);  in ktime_get_raw_ts64()
1571 	ts->tv_nsec = 0;  in ktime_get_raw_ts64()
1578  * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
1589 		ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;  in timekeeping_valid_for_hres()
1597  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
1608 		ret = tk->tkr_mono.clock->max_idle_ns;  in timekeeping_max_deferment()
1616  * read_persistent_clock64 -  Return time from the persistent clock.
1623  *  XXX - Do be sure to remove it once all arches implement it.
1627 	ts->tv_sec = 0;  in read_persistent_clock64()
1628 	ts->tv_nsec = 0;  in read_persistent_clock64()
1632  * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1635  * @boot_offset:  offset that is defined as wall_time - boot_time
1654 	raw_spin_lock_init(&tkd->lock);  in tkd_basic_setup()
1655 	seqcount_raw_spinlock_init(&tkd->seq, &tkd->lock);  in tkd_basic_setup()
1677  * timekeeping_init - Initializes the clocksource and common timekeeping values
1710 	if (clock->enable)  in timekeeping_init()
1711 		clock->enable(clock);  in timekeeping_init()
1715 	tks->raw_sec = 0;  in timekeeping_init()
1726  * __timekeeping_inject_sleeptime - Internal function to add sleep interval
1743 	tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));  in __timekeeping_inject_sleeptime()
1752  * 1) non-stop clocksource
1785  * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1812  * timekeeping_resume - Resumes the generic timekeeping subsystem.
1817 	struct clocksource *clock = tks->tkr_mono.clock;  in timekeeping_resume()
1838 	 *	suspend-nonstop clocksource -> persistent clock -> rtc  in timekeeping_resume()
1842 	cycle_now = tk_clock_read(&tks->tkr_mono);  in timekeeping_resume()
1857 	/* Re-base the last cycle value */  in timekeeping_resume()
1858 	tks->tkr_mono.cycle_last = cycle_now;  in timekeeping_resume()
1859 	tks->tkr_raw.cycle_last  = cycle_now;  in timekeeping_resume()
1861 	tks->ntp_error = 0;  in timekeeping_resume()
1904 	curr_clock = tks->tkr_mono.clock;  in timekeeping_suspend()
1905 	cycle_now = tks->tkr_mono.cycle_last;  in timekeeping_suspend()
1961 	s64 interval = tk->cycle_interval;  in timekeeping_apply_adjustment()
1965 	} else if (mult_adj == -1) {  in timekeeping_apply_adjustment()
1966 		interval = -interval;  in timekeeping_apply_adjustment()
1967 		offset = -offset;  in timekeeping_apply_adjustment()
1992 	 * So offset stores the non-accumulated cycles. Thus the current  in timekeeping_apply_adjustment()
2016 	 *	xtime_nsec_2 = xtime_nsec_1 - offset  in timekeeping_apply_adjustment()
2018 	 *	xtime_nsec -= offset  in timekeeping_apply_adjustment()
2020 	if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {  in timekeeping_apply_adjustment()
2026 	tk->tkr_mono.mult += mult_adj;  in timekeeping_apply_adjustment()
2027 	tk->xtime_interval += interval;  in timekeeping_apply_adjustment()
2028 	tk->tkr_mono.xtime_nsec -= offset;  in timekeeping_apply_adjustment()
2044 	if (likely(tk->ntp_tick == ntp_tl)) {  in timekeeping_adjust()
2045 		mult = tk->tkr_mono.mult - tk->ntp_err_mult;  in timekeeping_adjust()
2047 		tk->ntp_tick = ntp_tl;  in timekeeping_adjust()
2048 		mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -  in timekeeping_adjust()
2049 				 tk->xtime_remainder, tk->cycle_interval);  in timekeeping_adjust()
2056 	 * ahead until the tick length changes to a non-divisible value.  in timekeeping_adjust()
2058 	tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;  in timekeeping_adjust()
2059 	mult += tk->ntp_err_mult;  in timekeeping_adjust()
2061 	timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);  in timekeeping_adjust()
2063 	if (unlikely(tk->tkr_mono.clock->maxadj &&  in timekeeping_adjust()
2064 		(abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)  in timekeeping_adjust()
2065 			> tk->tkr_mono.clock->maxadj))) {  in timekeeping_adjust()
2068 			tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,  in timekeeping_adjust()
2069 			(long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);  in timekeeping_adjust()
2075 	 * in the code above, its possible the required corrective factor to  in timekeeping_adjust()
2082 	if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {  in timekeeping_adjust()
2083 		tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<  in timekeeping_adjust()
2084 							tk->tkr_mono.shift;  in timekeeping_adjust()
2085 		tk->xtime_sec--;  in timekeeping_adjust()
2086 		tk->skip_second_overflow = 1;  in timekeeping_adjust()
2091  * accumulate_nsecs_to_secs - Accumulates nsecs into secs
2099 	u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;  in accumulate_nsecs_to_secs()
2102 	while (tk->tkr_mono.xtime_nsec >= nsecps) {  in accumulate_nsecs_to_secs()
2105 		tk->tkr_mono.xtime_nsec -= nsecps;  in accumulate_nsecs_to_secs()
2106 		tk->xtime_sec++;  in accumulate_nsecs_to_secs()
2112 		if (unlikely(tk->skip_second_overflow)) {  in accumulate_nsecs_to_secs()
2113 			tk->skip_second_overflow = 0;  in accumulate_nsecs_to_secs()
2118 		leap = second_overflow(tk->xtime_sec);  in accumulate_nsecs_to_secs()
2122 			tk->xtime_sec += leap;  in accumulate_nsecs_to_secs()
2127 				timespec64_sub(tk->wall_to_monotonic, ts));  in accumulate_nsecs_to_secs()
2129 			__timekeeping_set_tai_offset(tk, tk->tai_offset - leap);  in accumulate_nsecs_to_secs()
2138  * logarithmic_accumulation - shifted accumulation of cycles
2149 	u64 interval = tk->cycle_interval << shift;  in logarithmic_accumulation()
2157 	offset -= interval;  in logarithmic_accumulation()
2158 	tk->tkr_mono.cycle_last += interval;  in logarithmic_accumulation()
2159 	tk->tkr_raw.cycle_last  += interval;  in logarithmic_accumulation()
2161 	tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;  in logarithmic_accumulation()
2165 	tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;  in logarithmic_accumulation()
2166 	snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;  in logarithmic_accumulation()
2167 	while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {  in logarithmic_accumulation()
2168 		tk->tkr_raw.xtime_nsec -= snsec_per_sec;  in logarithmic_accumulation()
2169 		tk->raw_sec++;  in logarithmic_accumulation()
2173 	tk->ntp_error += tk->ntp_tick << shift;  in logarithmic_accumulation()
2174 	tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<  in logarithmic_accumulation()
2175 						(tk->ntp_error_shift + shift);  in logarithmic_accumulation()
2181  * timekeeping_advance - Updates the timekeeper to the current time and
2198 	offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),  in timekeeping_advance()
2199 				   tk->tkr_mono.cycle_last, tk->tkr_mono.mask,  in timekeeping_advance()
2200 				   tk->tkr_mono.clock->max_raw_delta);  in timekeeping_advance()
2203 	if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)  in timekeeping_advance()
2214 	shift = ilog2(offset) - ilog2(tk->cycle_interval);  in timekeeping_advance()
2217 	maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;  in timekeeping_advance()
2219 	while (offset >= tk->cycle_interval) {  in timekeeping_advance()
2221 		if (offset < tk->cycle_interval<<shift)  in timekeeping_advance()
2222 			shift--;  in timekeeping_advance()
2248  * update_wall_time - Uses the current clocksource to increment the wall time
2258  * getboottime64 - Return the real time of system boot.
2261  * Returns the wall-time of boot in a timespec64.
2271 	ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);  in getboottime64()
2291  * ktime_get_coarse_real_ts64_mg - return latter of coarse grained time or floor
2295  * to the current coarse-grained time. Fill @ts with whichever is
2296  * latest. Note that this is a filesystem-specific interface and should be
2319  * ktime_get_real_ts64_mg - attempt to update floor value and return result
2322  * Get a monotonic fine-grained time value and attempt to swap it into
2329  * and determining that the resulting coarse-grained timestamp did not effect
2348 		ts->tv_sec = tk->xtime_sec;  in ktime_get_real_ts64_mg()
2349 		mono = tk->tkr_mono.base;  in ktime_get_real_ts64_mg()
2350 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_real_ts64_mg()
2364 		ts->tv_nsec = 0;  in ktime_get_real_ts64_mg()
2388 		mono = tk->wall_to_monotonic;  in ktime_get_coarse_ts64()
2406  * ktime_get_update_offsets_now - hrtimer helper
2408  * @offs_real:	pointer to storage for monotonic -> realtime offset
2409  * @offs_boot:	pointer to storage for monotonic -> boottime offset
2410  * @offs_tai:	pointer to storage for monotonic -> clock tai offset
2429 		base = tk->tkr_mono.base;  in ktime_get_update_offsets_now()
2430 		nsecs = timekeeping_get_ns(&tk->tkr_mono);  in ktime_get_update_offsets_now()
2433 		if (*cwsseq != tk->clock_was_set_seq) {  in ktime_get_update_offsets_now()
2434 			*cwsseq = tk->clock_was_set_seq;  in ktime_get_update_offsets_now()
2435 			*offs_real = tk->offs_real;  in ktime_get_update_offsets_now()
2436 			*offs_boot = tk->offs_boot;  in ktime_get_update_offsets_now()
2437 			*offs_tai = tk->offs_tai;  in ktime_get_update_offsets_now()
2441 		if (unlikely(base >= tk->next_leap_ktime))  in ktime_get_update_offsets_now()
2442 			*offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));  in ktime_get_update_offsets_now()
2450  * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
2454 	if (txc->modes & ADJ_ADJTIME) {  in timekeeping_validate_timex()
2456 		if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))  in timekeeping_validate_timex()
2457 			return -EINVAL;  in timekeeping_validate_timex()
2458 		if (!(txc->modes & ADJ_OFFSET_READONLY) &&  in timekeeping_validate_timex()
2460 			return -EPERM;  in timekeeping_validate_timex()
2462 		/* In order to modify anything, you gotta be super-user! */  in timekeeping_validate_timex()
2463 		if (txc->modes && !capable(CAP_SYS_TIME))  in timekeeping_validate_timex()
2464 			return -EPERM;  in timekeeping_validate_timex()
2469 		if (txc->modes & ADJ_TICK &&  in timekeeping_validate_timex()
2470 		    (txc->tick <  900000/USER_HZ ||  in timekeeping_validate_timex()
2471 		     txc->tick > 1100000/USER_HZ))  in timekeeping_validate_timex()
2472 			return -EINVAL;  in timekeeping_validate_timex()
2475 	if (txc->modes & ADJ_SETOFFSET) {  in timekeeping_validate_timex()
2476 		/* In order to inject time, you gotta be super-user! */  in timekeeping_validate_timex()
2478 			return -EPERM;  in timekeeping_validate_timex()
2485 		 * The field tv_usec/tv_nsec must always be non-negative and  in timekeeping_validate_timex()
2488 		if (txc->time.tv_usec < 0)  in timekeeping_validate_timex()
2489 			return -EINVAL;  in timekeeping_validate_timex()
2491 		if (txc->modes & ADJ_NANO) {  in timekeeping_validate_timex()
2492 			if (txc->time.tv_usec >= NSEC_PER_SEC)  in timekeeping_validate_timex()
2493 				return -EINVAL;  in timekeeping_validate_timex()
2495 			if (txc->time.tv_usec >= USEC_PER_SEC)  in timekeeping_validate_timex()
2496 				return -EINVAL;  in timekeeping_validate_timex()
2502 	 * only happen on 64-bit systems:  in timekeeping_validate_timex()
2504 	if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {  in timekeeping_validate_timex()
2505 		if (LLONG_MIN / PPM_SCALE > txc->freq)  in timekeeping_validate_timex()
2506 			return -EINVAL;  in timekeeping_validate_timex()
2507 		if (LLONG_MAX / PPM_SCALE < txc->freq)  in timekeeping_validate_timex()
2508 			return -EINVAL;  in timekeeping_validate_timex()
2515  * random_get_entropy_fallback - Returns the raw clock source value,
2521 	struct clocksource *clock = READ_ONCE(tkr->clock);  in random_get_entropy_fallback()
2525 	return clock->read(clock);  in random_get_entropy_fallback()
2530  * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2547 	if (txc->modes & ADJ_SETOFFSET) {  in do_adjtimex()
2550 		delta.tv_sec  = txc->time.tv_sec;  in do_adjtimex()
2551 		delta.tv_nsec = txc->time.tv_usec;  in do_adjtimex()
2552 		if (!(txc->modes & ADJ_NANO))  in do_adjtimex()
2571 		orig_tai = tai = tks->tai_offset;  in do_adjtimex()
2586 	if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))  in do_adjtimex()
2599  * hardpps() - Accessor function to NTP __hardpps function