21 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)  in vdso_delta_ok()  argument
23 	return delta < vc->max_cycles;  in vdso_delta_ok()
26 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)  in vdso_delta_ok()  argument
43 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)  in vdso_calc_ns()  argument
45 	u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc);  in vdso_calc_ns()
47 	if (likely(vdso_delta_ok(vc, delta)))  in vdso_calc_ns()
48 		return vdso_shift_ns((delta * vc->mult) + base, vc->shift);  in vdso_calc_ns()
50 	return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift);  in vdso_calc_ns()
62 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)  in vdso_clocksource_ok()  argument
64 	return vc->clock_mode != VDSO_CLOCKMODE_NONE;  in vdso_clocksource_ok()
92 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc,  in vdso_get_timestamp()  argument
95 	const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx];  in vdso_get_timestamp()
98 	if (unlikely(!vdso_clocksource_ok(vc)))  in vdso_get_timestamp()
101 	cycles = __arch_get_hw_counter(vc->clock_mode, vd);  in vdso_get_timestamp()
105 	*ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);  in vdso_get_timestamp()
123 	const struct vdso_clock *vc = vd->clock_data;  in do_hres_timens()  local
129 		vc = &vc[CS_HRES_COARSE];  in do_hres_timens()
131 		vc = &vc[CS_RAW];  in do_hres_timens()
134 		seq = vdso_read_begin(vc);  in do_hres_timens()
136 		if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))  in do_hres_timens()
138 	} while (unlikely(vdso_read_retry(vc, seq)));  in do_hres_timens()
150 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,  in do_hres()  argument
164 		 * special VVAR page installed which has vc->seq set to 1 and  in do_hres()
165 		 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time  in do_hres()
167 		 * because if vc->seq is odd, i.e. a concurrent update is in  in do_hres()
168 		 * progress the extra check for vc->clock_mode is just a few  in do_hres()
169 		 * extra instructions while spin waiting for vc->seq to become  in do_hres()
172 		while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) {  in do_hres()
174 			    vc->clock_mode == VDSO_CLOCKMODE_TIMENS)  in do_hres()
175 				return do_hres_timens(vd, vc, clk, ts);  in do_hres()
180 		if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))  in do_hres()
182 	} while (unlikely(vdso_read_retry(vc, seq)));  in do_hres()
195 	const struct vdso_clock *vc = vd->clock_data;  in do_coarse_timens()  local
201 	vdso_ts = &vc->basetime[clk];  in do_coarse_timens()
204 		seq = vdso_read_begin(vc);  in do_coarse_timens()
207 	} while (unlikely(vdso_read_retry(vc, seq)));  in do_coarse_timens()
219 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,  in do_coarse()  argument
222 	const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];  in do_coarse()
230 		while ((seq = READ_ONCE(vc->seq)) & 1) {  in do_coarse()
232 			    vc->clock_mode == VDSO_CLOCKMODE_TIMENS)  in do_coarse()
233 				return do_coarse_timens(vd, vc, clk, ts);  in do_coarse()
240 	} while (unlikely(vdso_read_retry(vc, seq)));  in do_coarse()
248 	const struct vdso_clock *vc;  in do_aux()  local
256 	vc = &vd->aux_clock_data[idx];  in do_aux()
263 		while ((seq = READ_ONCE(vc->seq)) & 1) {  in do_aux()
264 			if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {  in do_aux()
266 				vc = &vd->aux_clock_data[idx];  in do_aux()
275 		if (vc->clock_mode == VDSO_CLOCKMODE_NONE)  in do_aux()
278 		if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns))  in do_aux()
280 	} while (unlikely(vdso_read_retry(vc, seq)));  in do_aux()
291 	const struct vdso_clock *vc = vd->clock_data;  in __cvdso_clock_gettime_common()  local
303 		vc = &vc[CS_HRES_COARSE];  in __cvdso_clock_gettime_common()
305 		return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts);  in __cvdso_clock_gettime_common()
307 		vc = &vc[CS_RAW];  in __cvdso_clock_gettime_common()
313 	return do_hres(vd, vc, clock, ts);  in __cvdso_clock_gettime_common()
366 	const struct vdso_clock *vc = vd->clock_data;  in __cvdso_gettimeofday_data()  local
371 		if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts))  in __cvdso_gettimeofday_data()
380 		    vc->clock_mode == VDSO_CLOCKMODE_TIMENS)  in __cvdso_gettimeofday_data()
400 	const struct vdso_clock *vc = vd->clock_data;  in __cvdso_time_data()  local
404 	    vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {  in __cvdso_time_data()
406 		vc = vd->clock_data;  in __cvdso_time_data()
409 	t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);  in __cvdso_time_data()
428 	const struct vdso_clock *vc = vd->clock_data;  in __cvdso_clock_getres_common()  local
436 	    vc->clock_mode == VDSO_CLOCKMODE_TIMENS)  in __cvdso_clock_getres_common()