1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5 #include <vdso/auxclock.h>
6 #include <vdso/clocksource.h>
7 #include <vdso/datapage.h>
8 #include <vdso/helpers.h>
9 #include <vdso/ktime.h>
10 #include <vdso/limits.h>
11 #include <vdso/math64.h>
12 #include <vdso/time32.h>
13 #include <vdso/time64.h>
14
15 /*
16 * The generic vDSO implementation requires that gettimeofday.h
17 * provides:
18 * - __arch_get_hw_counter(): to get the hw counter based on the
19 * clock_mode.
20 * - gettimeofday_fallback(): fallback for gettimeofday.
21 * - clock_gettime_fallback(): fallback for clock_gettime.
22 * - clock_getres_fallback(): fallback for clock_getres.
23 */
24 #include <asm/vdso/gettimeofday.h>
25
26 /* Bring in default accessors */
27 #include <vdso/vsyscall.h>
28
29 #ifndef vdso_calc_ns
30
31 #ifdef VDSO_DELTA_NOMASK
32 # define VDSO_DELTA_MASK(vd) ULLONG_MAX
33 #else
34 # define VDSO_DELTA_MASK(vd) (vd->mask)
35 #endif
36
37 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
vdso_delta_ok(const struct vdso_clock * vc,u64 delta)38 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
39 {
40 return delta < vc->max_cycles;
41 }
42 #else
vdso_delta_ok(const struct vdso_clock * vc,u64 delta)43 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
44 {
45 return true;
46 }
47 #endif
48
49 #ifndef vdso_shift_ns
vdso_shift_ns(u64 ns,u32 shift)50 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
51 {
52 return ns >> shift;
53 }
54 #endif
55
56 /*
57 * Default implementation which works for all sane clocksources. That
58 * obviously excludes x86/TSC.
59 */
vdso_calc_ns(const struct vdso_clock * vc,u64 cycles,u64 base)60 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
61 {
62 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc);
63
64 if (likely(vdso_delta_ok(vc, delta)))
65 return vdso_shift_ns((delta * vc->mult) + base, vc->shift);
66
67 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift);
68 }
69 #endif /* vdso_calc_ns */
70
71 #ifndef __arch_vdso_hres_capable
__arch_vdso_hres_capable(void)72 static inline bool __arch_vdso_hres_capable(void)
73 {
74 return true;
75 }
76 #endif
77
78 #ifndef vdso_clocksource_ok
vdso_clocksource_ok(const struct vdso_clock * vc)79 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
80 {
81 return vc->clock_mode != VDSO_CLOCKMODE_NONE;
82 }
83 #endif
84
85 #ifndef vdso_cycles_ok
vdso_cycles_ok(u64 cycles)86 static inline bool vdso_cycles_ok(u64 cycles)
87 {
88 return true;
89 }
90 #endif
91
vdso_clockid_valid(clockid_t clock)92 static __always_inline bool vdso_clockid_valid(clockid_t clock)
93 {
94 /* Check for negative values or invalid clocks */
95 return likely((u32) clock <= CLOCK_AUX_LAST);
96 }
97
98 /*
99 * Must not be invoked within the sequence read section as a race inside
100 * that loop could result in __iter_div_u64_rem() being extremely slow.
101 */
vdso_set_timespec(struct __kernel_timespec * ts,u64 sec,u64 ns)102 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns)
103 {
104 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
105 ts->tv_nsec = ns;
106 }
107
108 static __always_inline
vdso_get_timestamp(const struct vdso_time_data * vd,const struct vdso_clock * vc,unsigned int clkidx,u64 * sec,u64 * ns)109 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc,
110 unsigned int clkidx, u64 *sec, u64 *ns)
111 {
112 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx];
113 u64 cycles;
114
115 if (unlikely(!vdso_clocksource_ok(vc)))
116 return false;
117
118 cycles = __arch_get_hw_counter(vc->clock_mode, vd);
119 if (unlikely(!vdso_cycles_ok(cycles)))
120 return false;
121
122 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);
123 *sec = vdso_ts->sec;
124
125 return true;
126 }
127
128 static __always_inline
__arch_get_vdso_u_timens_data(const struct vdso_time_data * vd)129 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
130 {
131 return (void *)vd + PAGE_SIZE;
132 }
133
134 static __always_inline
do_hres_timens(const struct vdso_time_data * vdns,const struct vdso_clock * vcns,clockid_t clk,struct __kernel_timespec * ts)135 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
136 clockid_t clk, struct __kernel_timespec *ts)
137 {
138 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
139 const struct timens_offset *offs = &vcns->offset[clk];
140 const struct vdso_clock *vc = vd->clock_data;
141 u32 seq;
142 s64 sec;
143 u64 ns;
144
145 if (clk != CLOCK_MONOTONIC_RAW)
146 vc = &vc[CS_HRES_COARSE];
147 else
148 vc = &vc[CS_RAW];
149
150 do {
151 seq = vdso_read_begin(vc);
152
153 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
154 return false;
155 } while (vdso_read_retry(vc, seq));
156
157 /* Add the namespace offset */
158 sec += offs->sec;
159 ns += offs->nsec;
160
161 vdso_set_timespec(ts, sec, ns);
162
163 return true;
164 }
165
166 static __always_inline
do_hres(const struct vdso_time_data * vd,const struct vdso_clock * vc,clockid_t clk,struct __kernel_timespec * ts)167 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,
168 clockid_t clk, struct __kernel_timespec *ts)
169 {
170 u64 sec, ns;
171 u32 seq;
172
173 /* Allows to compile the high resolution parts out */
174 if (!__arch_vdso_hres_capable())
175 return false;
176
177 do {
178 if (vdso_read_begin_timens(vc, &seq))
179 return do_hres_timens(vd, vc, clk, ts);
180
181 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
182 return false;
183 } while (vdso_read_retry(vc, seq));
184
185 vdso_set_timespec(ts, sec, ns);
186
187 return true;
188 }
189
190 static __always_inline
do_coarse_timens(const struct vdso_time_data * vdns,const struct vdso_clock * vcns,clockid_t clk,struct __kernel_timespec * ts)191 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
192 clockid_t clk, struct __kernel_timespec *ts)
193 {
194 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
195 const struct timens_offset *offs = &vcns->offset[clk];
196 const struct vdso_clock *vc = vd->clock_data;
197 const struct vdso_timestamp *vdso_ts;
198 u64 nsec;
199 s64 sec;
200 s32 seq;
201
202 vdso_ts = &vc->basetime[clk];
203
204 do {
205 seq = vdso_read_begin(vc);
206 sec = vdso_ts->sec;
207 nsec = vdso_ts->nsec;
208 } while (vdso_read_retry(vc, seq));
209
210 /* Add the namespace offset */
211 sec += offs->sec;
212 nsec += offs->nsec;
213
214 vdso_set_timespec(ts, sec, nsec);
215
216 return true;
217 }
218
219 static __always_inline
do_coarse(const struct vdso_time_data * vd,const struct vdso_clock * vc,clockid_t clk,struct __kernel_timespec * ts)220 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,
221 clockid_t clk, struct __kernel_timespec *ts)
222 {
223 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];
224 u32 seq;
225
226 do {
227 if (vdso_read_begin_timens(vc, &seq))
228 return do_coarse_timens(vd, vc, clk, ts);
229
230 ts->tv_sec = vdso_ts->sec;
231 ts->tv_nsec = vdso_ts->nsec;
232 } while (vdso_read_retry(vc, seq));
233
234 return true;
235 }
236
237 static __always_inline
do_aux(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * ts)238 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts)
239 {
240 const struct vdso_clock *vc;
241 u32 seq, idx;
242 u64 sec, ns;
243
244 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
245 return false;
246
247 idx = clock - CLOCK_AUX;
248 vc = &vd->aux_clock_data[idx];
249
250 do {
251 while (vdso_read_begin_timens(vc, &seq)) {
252 /* Re-read from the real time data page, reload seq by looping */
253 vd = __arch_get_vdso_u_timens_data(vd);
254 vc = &vd->aux_clock_data[idx];
255 }
256
257 /* Auxclock disabled? */
258 if (vc->clock_mode == VDSO_CLOCKMODE_NONE)
259 return false;
260
261 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns))
262 return false;
263 } while (vdso_read_retry(vc, seq));
264
265 vdso_set_timespec(ts, sec, ns);
266
267 return true;
268 }
269
270 static __always_inline bool
__cvdso_clock_gettime_common(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * ts)271 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
272 struct __kernel_timespec *ts)
273 {
274 const struct vdso_clock *vc = vd->clock_data;
275 u32 msk;
276
277 if (!vdso_clockid_valid(clock))
278 return false;
279
280 /*
281 * Convert the clockid to a bitmask and use it to check which
282 * clocks are handled in the VDSO directly.
283 */
284 msk = 1U << clock;
285 if (likely(msk & VDSO_HRES))
286 vc = &vc[CS_HRES_COARSE];
287 else if (msk & VDSO_COARSE)
288 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts);
289 else if (msk & VDSO_RAW)
290 vc = &vc[CS_RAW];
291 else if (msk & VDSO_AUX)
292 return do_aux(vd, clock, ts);
293 else
294 return false;
295
296 return do_hres(vd, vc, clock, ts);
297 }
298
299 static int
__cvdso_clock_gettime_data(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * ts)300 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
301 struct __kernel_timespec *ts)
302 {
303 bool ok;
304
305 ok = __cvdso_clock_gettime_common(vd, clock, ts);
306
307 if (unlikely(!ok))
308 return clock_gettime_fallback(clock, ts);
309 return 0;
310 }
311
312 static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock,struct __kernel_timespec * ts)313 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
314 {
315 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts);
316 }
317
318 #ifdef BUILD_VDSO32
319 static int
__cvdso_clock_gettime32_data(const struct vdso_time_data * vd,clockid_t clock,struct old_timespec32 * res)320 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
321 struct old_timespec32 *res)
322 {
323 struct __kernel_timespec ts;
324 bool ok;
325
326 ok = __cvdso_clock_gettime_common(vd, clock, &ts);
327
328 if (unlikely(!ok))
329 return clock_gettime32_fallback(clock, res);
330
331 /* For ok == true */
332 res->tv_sec = ts.tv_sec;
333 res->tv_nsec = ts.tv_nsec;
334
335 return 0;
336 }
337
338 static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock,struct old_timespec32 * res)339 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
340 {
341 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res);
342 }
343 #endif /* BUILD_VDSO32 */
344
345 static int
__cvdso_gettimeofday_data(const struct vdso_time_data * vd,struct __kernel_old_timeval * tv,struct timezone * tz)346 __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
347 struct __kernel_old_timeval *tv, struct timezone *tz)
348 {
349 const struct vdso_clock *vc = vd->clock_data;
350
351 if (likely(tv != NULL)) {
352 struct __kernel_timespec ts;
353
354 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
355 return gettimeofday_fallback(tv, tz);
356
357 tv->tv_sec = ts.tv_sec;
358 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
359 }
360
361 if (unlikely(tz != NULL)) {
362 if (vdso_is_timens_clock(vc))
363 vd = __arch_get_vdso_u_timens_data(vd);
364
365 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
366 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
367 }
368
369 return 0;
370 }
371
372 static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval * tv,struct timezone * tz)373 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
374 {
375 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz);
376 }
377
378 #ifdef VDSO_HAS_TIME
379 static __kernel_old_time_t
__cvdso_time_data(const struct vdso_time_data * vd,__kernel_old_time_t * time)380 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
381 {
382 const struct vdso_clock *vc = vd->clock_data;
383 __kernel_old_time_t t;
384
385 if (vdso_is_timens_clock(vc)) {
386 vd = __arch_get_vdso_u_timens_data(vd);
387 vc = vd->clock_data;
388 }
389
390 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
391
392 if (time)
393 *time = t;
394
395 return t;
396 }
397
__cvdso_time(__kernel_old_time_t * time)398 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
399 {
400 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time);
401 }
402 #endif /* VDSO_HAS_TIME */
403
404 #ifdef VDSO_HAS_CLOCK_GETRES
405 static __always_inline
__cvdso_clock_getres_common(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * res)406 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
407 struct __kernel_timespec *res)
408 {
409 const struct vdso_clock *vc = vd->clock_data;
410 u32 msk;
411 u64 ns;
412
413 if (!vdso_clockid_valid(clock))
414 return false;
415
416 if (vdso_is_timens_clock(vc))
417 vd = __arch_get_vdso_u_timens_data(vd);
418
419 /*
420 * Convert the clockid to a bitmask and use it to check which
421 * clocks are handled in the VDSO directly.
422 */
423 msk = 1U << clock;
424 if (msk & (VDSO_HRES | VDSO_RAW)) {
425 /*
426 * Preserves the behaviour of posix_get_hrtimer_res().
427 */
428 ns = READ_ONCE(vd->hrtimer_res);
429 } else if (msk & VDSO_COARSE) {
430 /*
431 * Preserves the behaviour of posix_get_coarse_res().
432 */
433 ns = LOW_RES_NSEC;
434 } else if (msk & VDSO_AUX) {
435 ns = aux_clock_resolution_ns();
436 } else {
437 return false;
438 }
439
440 if (likely(res)) {
441 res->tv_sec = 0;
442 res->tv_nsec = ns;
443 }
444 return true;
445 }
446
447 static
__cvdso_clock_getres_data(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * res)448 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
449 struct __kernel_timespec *res)
450 {
451 bool ok;
452
453 ok = __cvdso_clock_getres_common(vd, clock, res);
454
455 if (unlikely(!ok))
456 return clock_getres_fallback(clock, res);
457 return 0;
458 }
459
460 static __maybe_unused
__cvdso_clock_getres(clockid_t clock,struct __kernel_timespec * res)461 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
462 {
463 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res);
464 }
465
466 #ifdef BUILD_VDSO32
467 static int
__cvdso_clock_getres_time32_data(const struct vdso_time_data * vd,clockid_t clock,struct old_timespec32 * res)468 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
469 struct old_timespec32 *res)
470 {
471 struct __kernel_timespec ts;
472 bool ok;
473
474 ok = __cvdso_clock_getres_common(vd, clock, &ts);
475
476 if (unlikely(!ok))
477 return clock_getres32_fallback(clock, res);
478
479 if (likely(res)) {
480 res->tv_sec = ts.tv_sec;
481 res->tv_nsec = ts.tv_nsec;
482 }
483 return 0;
484 }
485
486 static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock,struct old_timespec32 * res)487 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
488 {
489 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(),
490 clock, res);
491 }
492 #endif /* BUILD_VDSO32 */
493 #endif /* VDSO_HAS_CLOCK_GETRES */
494