xref: /linux/lib/vdso/gettimeofday.c (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Generic userspace implementations of gettimeofday() and similar.
4  */
5 #include <vdso/auxclock.h>
6 #include <vdso/clocksource.h>
7 #include <vdso/datapage.h>
8 #include <vdso/helpers.h>
9 #include <vdso/ktime.h>
10 #include <vdso/limits.h>
11 #include <vdso/math64.h>
12 #include <vdso/time32.h>
13 #include <vdso/time64.h>
14 
15 #include <uapi/linux/unistd.h>
16 
17 /*
18  * The generic vDSO implementation requires that gettimeofday.h
19  * provides:
20  * - __arch_get_hw_counter(): to get the hw counter based on the
21  *   clock_mode.
22  * - gettimeofday_fallback(): fallback for gettimeofday.
23  * - clock_gettime_fallback(): fallback for clock_gettime.
24  * - clock_getres_fallback(): fallback for clock_getres.
25  */
26 #include <asm/vdso/gettimeofday.h>
27 
28 #include <linux/build_bug.h>
29 
30 #ifndef vdso_calc_ns
31 
32 #ifdef VDSO_DELTA_NOMASK
33 # define VDSO_DELTA_MASK(vd)	ULLONG_MAX
34 #else
35 # define VDSO_DELTA_MASK(vd)	(vd->mask)
36 #endif
37 
38 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
39 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
40 {
41 	return delta < vc->max_cycles;
42 }
43 #else
44 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
45 {
46 	return true;
47 }
48 #endif
49 
50 #ifndef vdso_shift_ns
51 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
52 {
53 	return ns >> shift;
54 }
55 #endif
56 
57 /*
58  * Default implementation which works for all sane clocksources. That
59  * obviously excludes x86/TSC.
60  */
61 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
62 {
63 	u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc);
64 
65 	if (likely(vdso_delta_ok(vc, delta)))
66 		return vdso_shift_ns((delta * vc->mult) + base, vc->shift);
67 
68 	return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift);
69 }
70 #endif /* vdso_calc_ns */
71 
72 #ifndef __arch_vdso_hres_capable
73 static inline bool __arch_vdso_hres_capable(void)
74 {
75 	return true;
76 }
77 #endif
78 
79 #ifndef vdso_clocksource_ok
80 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
81 {
82 	return vc->clock_mode != VDSO_CLOCKMODE_NONE;
83 }
84 #endif
85 
86 #ifndef vdso_cycles_ok
87 static inline bool vdso_cycles_ok(u64 cycles)
88 {
89 	return true;
90 }
91 #endif
92 
93 #ifndef __arch_get_vdso_u_time_data
94 static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void)
95 {
96 	return &vdso_u_time_data;
97 }
98 #endif
99 
100 static __always_inline bool vdso_clockid_valid(clockid_t clock)
101 {
102 	/* Check for negative values or invalid clocks */
103 	return likely((u32) clock <= CLOCK_AUX_LAST);
104 }
105 
106 /*
107  * Must not be invoked within the sequence read section as a race inside
108  * that loop could result in __iter_div_u64_rem() being extremely slow.
109  */
110 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns)
111 {
112 	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
113 	ts->tv_nsec = ns;
114 }
115 
116 static __always_inline
117 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc,
118 			unsigned int clkidx, u64 *sec, u64 *ns)
119 {
120 	const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx];
121 	u64 cycles;
122 
123 	if (unlikely(!vdso_clocksource_ok(vc)))
124 		return false;
125 
126 	cycles = __arch_get_hw_counter(vc->clock_mode, vd);
127 	if (unlikely(!vdso_cycles_ok(cycles)))
128 		return false;
129 
130 	*ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);
131 	*sec = vdso_ts->sec;
132 
133 	return true;
134 }
135 
136 static __always_inline
137 const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd)
138 {
139 	return (void *)vd + PAGE_SIZE;
140 }
141 
142 static __always_inline
143 const struct vdso_clock *vdso_timens_clock(const struct vdso_clock *vc)
144 {
145 	return (void *)vc + PAGE_SIZE;
146 }
147 
148 static __always_inline
149 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
150 		    clockid_t clk, struct __kernel_timespec *ts)
151 {
152 	const struct vdso_time_data *vd = vdso_timens_data(vdns);
153 	const struct vdso_clock *vc = vdso_timens_clock(vcns);
154 	const struct timens_offset *offs = &vcns->offset[clk];
155 	u32 seq;
156 	s64 sec;
157 	u64 ns;
158 
159 	do {
160 		seq = vdso_read_begin(vc);
161 
162 		if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
163 			return false;
164 	} while (vdso_read_retry(vc, seq));
165 
166 	/* Add the namespace offset */
167 	sec += offs->sec;
168 	ns += offs->nsec;
169 
170 	vdso_set_timespec(ts, sec, ns);
171 
172 	return true;
173 }
174 
175 static __always_inline
176 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,
177 	     clockid_t clk, struct __kernel_timespec *ts)
178 {
179 	u64 sec, ns;
180 	u32 seq;
181 
182 	/* Allows to compile the high resolution parts out */
183 	if (!__arch_vdso_hres_capable())
184 		return false;
185 
186 	do {
187 		if (vdso_read_begin_timens(vc, &seq))
188 			return do_hres_timens(vd, vc, clk, ts);
189 
190 		if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
191 			return false;
192 	} while (vdso_read_retry(vc, seq));
193 
194 	vdso_set_timespec(ts, sec, ns);
195 
196 	return true;
197 }
198 
199 static __always_inline
200 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
201 		      clockid_t clk, struct __kernel_timespec *ts)
202 {
203 	const struct vdso_time_data *vd = vdso_timens_data(vdns);
204 	const struct timens_offset *offs = &vcns->offset[clk];
205 	const struct vdso_clock *vc = vd->clock_data;
206 	const struct vdso_timestamp *vdso_ts;
207 	u64 nsec;
208 	s64 sec;
209 	s32 seq;
210 
211 	vdso_ts = &vc->basetime[clk];
212 
213 	do {
214 		seq = vdso_read_begin(vc);
215 		sec = vdso_ts->sec;
216 		nsec = vdso_ts->nsec;
217 	} while (vdso_read_retry(vc, seq));
218 
219 	/* Add the namespace offset */
220 	sec += offs->sec;
221 	nsec += offs->nsec;
222 
223 	vdso_set_timespec(ts, sec, nsec);
224 
225 	return true;
226 }
227 
228 static __always_inline
229 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,
230 	       clockid_t clk, struct __kernel_timespec *ts)
231 {
232 	const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];
233 	u32 seq;
234 
235 	do {
236 		if (vdso_read_begin_timens(vc, &seq))
237 			return do_coarse_timens(vd, vc, clk, ts);
238 
239 		ts->tv_sec = vdso_ts->sec;
240 		ts->tv_nsec = vdso_ts->nsec;
241 	} while (vdso_read_retry(vc, seq));
242 
243 	return true;
244 }
245 
246 static __always_inline
247 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts)
248 {
249 	const struct vdso_clock *vc;
250 	u32 seq, idx;
251 	u64 sec, ns;
252 
253 	if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
254 		return false;
255 
256 	idx = clock - CLOCK_AUX;
257 	vc = &vd->aux_clock_data[idx];
258 
259 	do {
260 		while (vdso_read_begin_timens(vc, &seq)) {
261 			/* Re-read from the real time data page, reload seq by looping */
262 			vd = vdso_timens_data(vd);
263 			vc = &vd->aux_clock_data[idx];
264 		}
265 
266 		/* Auxclock disabled? */
267 		if (vc->clock_mode == VDSO_CLOCKMODE_NONE)
268 			return false;
269 
270 		if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns))
271 			return false;
272 	} while (vdso_read_retry(vc, seq));
273 
274 	vdso_set_timespec(ts, sec, ns);
275 
276 	return true;
277 }
278 
279 static __always_inline bool
280 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
281 			     struct __kernel_timespec *ts)
282 {
283 	const struct vdso_clock *vc = vd->clock_data;
284 	u32 msk;
285 
286 	if (!vdso_clockid_valid(clock))
287 		return false;
288 
289 	/*
290 	 * Convert the clockid to a bitmask and use it to check which
291 	 * clocks are handled in the VDSO directly.
292 	 */
293 	msk = 1U << clock;
294 	if (likely(msk & VDSO_HRES))
295 		vc = &vc[CS_HRES_COARSE];
296 	else if (msk & VDSO_COARSE)
297 		return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts);
298 	else if (msk & VDSO_RAW)
299 		vc = &vc[CS_RAW];
300 	else if (msk & VDSO_AUX)
301 		return do_aux(vd, clock, ts);
302 	else
303 		return false;
304 
305 	return do_hres(vd, vc, clock, ts);
306 }
307 
308 static int
309 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
310 			   struct __kernel_timespec *ts)
311 {
312 	bool ok;
313 
314 	ok = __cvdso_clock_gettime_common(vd, clock, ts);
315 
316 	if (unlikely(!ok))
317 		return clock_gettime_fallback(clock, ts);
318 	return 0;
319 }
320 
321 static __maybe_unused int
322 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
323 {
324 	return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts);
325 }
326 
327 #ifdef BUILD_VDSO32
328 static int
329 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
330 			     struct old_timespec32 *res)
331 {
332 	struct __kernel_timespec ts;
333 	bool ok;
334 
335 	BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
336 
337 	ok = __cvdso_clock_gettime_common(vd, clock, &ts);
338 
339 	if (unlikely(!ok))
340 		return clock_gettime32_fallback(clock, res);
341 
342 	/* For ok == true */
343 	res->tv_sec = ts.tv_sec;
344 	res->tv_nsec = ts.tv_nsec;
345 
346 	return 0;
347 }
348 
349 static __maybe_unused int
350 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
351 {
352 	return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res);
353 }
354 #endif /* BUILD_VDSO32 */
355 
356 static int
357 __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
358 			  struct __kernel_old_timeval *tv, struct timezone *tz)
359 {
360 	const struct vdso_clock *vc = vd->clock_data;
361 
362 #ifndef __NR_gettimeofday
363 	BUILD_BUG();
364 #endif
365 
366 	BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
367 
368 	if (likely(tv != NULL)) {
369 		struct __kernel_timespec ts;
370 
371 		if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
372 			return gettimeofday_fallback(tv, tz);
373 
374 		tv->tv_sec = ts.tv_sec;
375 		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
376 	}
377 
378 	if (unlikely(tz != NULL)) {
379 		if (vdso_is_timens_clock(vc))
380 			vd = vdso_timens_data(vd);
381 
382 		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
383 		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
384 	}
385 
386 	return 0;
387 }
388 
389 static __maybe_unused int
390 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
391 {
392 	return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz);
393 }
394 
395 #ifdef VDSO_HAS_TIME
396 static __kernel_old_time_t
397 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
398 {
399 	const struct vdso_clock *vc = vd->clock_data;
400 	__kernel_old_time_t t;
401 
402 #ifndef __NR_time
403 	BUILD_BUG();
404 #endif
405 
406 	BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
407 
408 	if (vdso_is_timens_clock(vc)) {
409 		vd = vdso_timens_data(vd);
410 		vc = vd->clock_data;
411 	}
412 
413 	t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
414 
415 	if (time)
416 		*time = t;
417 
418 	return t;
419 }
420 
421 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
422 {
423 	return __cvdso_time_data(__arch_get_vdso_u_time_data(), time);
424 }
425 #endif /* VDSO_HAS_TIME */
426 
427 #ifdef VDSO_HAS_CLOCK_GETRES
428 static __always_inline
429 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
430 				 struct __kernel_timespec *res)
431 {
432 	const struct vdso_clock *vc = vd->clock_data;
433 	u32 msk;
434 	u64 ns;
435 
436 	if (!vdso_clockid_valid(clock))
437 		return false;
438 
439 	if (vdso_is_timens_clock(vc))
440 		vd = vdso_timens_data(vd);
441 
442 	/*
443 	 * Convert the clockid to a bitmask and use it to check which
444 	 * clocks are handled in the VDSO directly.
445 	 */
446 	msk = 1U << clock;
447 	if (msk & (VDSO_HRES | VDSO_RAW)) {
448 		/*
449 		 * Preserves the behaviour of posix_get_hrtimer_res().
450 		 */
451 		ns = READ_ONCE(vd->hrtimer_res);
452 	} else if (msk & VDSO_COARSE) {
453 		/*
454 		 * Preserves the behaviour of posix_get_coarse_res().
455 		 */
456 		ns = LOW_RES_NSEC;
457 	} else if (msk & VDSO_AUX) {
458 		ns = aux_clock_resolution_ns();
459 	} else {
460 		return false;
461 	}
462 
463 	if (likely(res)) {
464 		res->tv_sec = 0;
465 		res->tv_nsec = ns;
466 	}
467 	return true;
468 }
469 
470 static
471 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
472 			      struct __kernel_timespec *res)
473 {
474 	bool ok;
475 
476 	ok =  __cvdso_clock_getres_common(vd, clock, res);
477 
478 	if (unlikely(!ok))
479 		return clock_getres_fallback(clock, res);
480 	return 0;
481 }
482 
483 static __maybe_unused
484 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
485 {
486 	return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res);
487 }
488 
489 #ifdef BUILD_VDSO32
490 static int
491 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
492 				 struct old_timespec32 *res)
493 {
494 	struct __kernel_timespec ts;
495 	bool ok;
496 
497 	BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME));
498 
499 	ok = __cvdso_clock_getres_common(vd, clock, &ts);
500 
501 	if (unlikely(!ok))
502 		return clock_getres32_fallback(clock, res);
503 
504 	if (likely(res)) {
505 		res->tv_sec = ts.tv_sec;
506 		res->tv_nsec = ts.tv_nsec;
507 	}
508 	return 0;
509 }
510 
511 static __maybe_unused int
512 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
513 {
514 	return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(),
515 						clock, res);
516 }
517 #endif /* BUILD_VDSO32 */
518 #endif /* VDSO_HAS_CLOCK_GETRES */
519