1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5 #include <vdso/auxclock.h>
6 #include <vdso/datapage.h>
7 #include <vdso/helpers.h>
8
9 /* Bring in default accessors */
10 #include <vdso/vsyscall.h>
11
12 #ifndef vdso_calc_ns
13
14 #ifdef VDSO_DELTA_NOMASK
15 # define VDSO_DELTA_MASK(vd) ULLONG_MAX
16 #else
17 # define VDSO_DELTA_MASK(vd) (vd->mask)
18 #endif
19
20 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
vdso_delta_ok(const struct vdso_clock * vc,u64 delta)21 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
22 {
23 return delta < vc->max_cycles;
24 }
25 #else
vdso_delta_ok(const struct vdso_clock * vc,u64 delta)26 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta)
27 {
28 return true;
29 }
30 #endif
31
32 #ifndef vdso_shift_ns
vdso_shift_ns(u64 ns,u32 shift)33 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
34 {
35 return ns >> shift;
36 }
37 #endif
38
39 /*
40 * Default implementation which works for all sane clocksources. That
41 * obviously excludes x86/TSC.
42 */
vdso_calc_ns(const struct vdso_clock * vc,u64 cycles,u64 base)43 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base)
44 {
45 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc);
46
47 if (likely(vdso_delta_ok(vc, delta)))
48 return vdso_shift_ns((delta * vc->mult) + base, vc->shift);
49
50 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift);
51 }
52 #endif /* vdso_calc_ns */
53
54 #ifndef __arch_vdso_hres_capable
__arch_vdso_hres_capable(void)55 static inline bool __arch_vdso_hres_capable(void)
56 {
57 return true;
58 }
59 #endif
60
61 #ifndef vdso_clocksource_ok
vdso_clocksource_ok(const struct vdso_clock * vc)62 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc)
63 {
64 return vc->clock_mode != VDSO_CLOCKMODE_NONE;
65 }
66 #endif
67
68 #ifndef vdso_cycles_ok
vdso_cycles_ok(u64 cycles)69 static inline bool vdso_cycles_ok(u64 cycles)
70 {
71 return true;
72 }
73 #endif
74
vdso_clockid_valid(clockid_t clock)75 static __always_inline bool vdso_clockid_valid(clockid_t clock)
76 {
77 /* Check for negative values or invalid clocks */
78 return likely((u32) clock <= CLOCK_AUX_LAST);
79 }
80
81 /*
82 * Must not be invoked within the sequence read section as a race inside
83 * that loop could result in __iter_div_u64_rem() being extremely slow.
84 */
vdso_set_timespec(struct __kernel_timespec * ts,u64 sec,u64 ns)85 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns)
86 {
87 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
88 ts->tv_nsec = ns;
89 }
90
91 static __always_inline
vdso_get_timestamp(const struct vdso_time_data * vd,const struct vdso_clock * vc,unsigned int clkidx,u64 * sec,u64 * ns)92 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc,
93 unsigned int clkidx, u64 *sec, u64 *ns)
94 {
95 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx];
96 u64 cycles;
97
98 if (unlikely(!vdso_clocksource_ok(vc)))
99 return false;
100
101 cycles = __arch_get_hw_counter(vc->clock_mode, vd);
102 if (unlikely(!vdso_cycles_ok(cycles)))
103 return false;
104
105 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec);
106 *sec = vdso_ts->sec;
107
108 return true;
109 }
110
111 #ifdef CONFIG_TIME_NS
112
113 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE
114 static __always_inline
__arch_get_vdso_u_timens_data(const struct vdso_time_data * vd)115 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
116 {
117 return (void *)vd + PAGE_SIZE;
118 }
119 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */
120
121 static __always_inline
do_hres_timens(const struct vdso_time_data * vdns,const struct vdso_clock * vcns,clockid_t clk,struct __kernel_timespec * ts)122 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
123 clockid_t clk, struct __kernel_timespec *ts)
124 {
125 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
126 const struct timens_offset *offs = &vcns->offset[clk];
127 const struct vdso_clock *vc = vd->clock_data;
128 u32 seq;
129 s64 sec;
130 u64 ns;
131
132 if (clk != CLOCK_MONOTONIC_RAW)
133 vc = &vc[CS_HRES_COARSE];
134 else
135 vc = &vc[CS_RAW];
136
137 do {
138 seq = vdso_read_begin(vc);
139
140 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
141 return false;
142 } while (unlikely(vdso_read_retry(vc, seq)));
143
144 /* Add the namespace offset */
145 sec += offs->sec;
146 ns += offs->nsec;
147
148 vdso_set_timespec(ts, sec, ns);
149
150 return true;
151 }
152 #else
153 static __always_inline
__arch_get_vdso_u_timens_data(const struct vdso_time_data * vd)154 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd)
155 {
156 return NULL;
157 }
158
159 static __always_inline
do_hres_timens(const struct vdso_time_data * vdns,const struct vdso_clock * vcns,clockid_t clk,struct __kernel_timespec * ts)160 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
161 clockid_t clk, struct __kernel_timespec *ts)
162 {
163 return false;
164 }
165 #endif
166
167 static __always_inline
do_hres(const struct vdso_time_data * vd,const struct vdso_clock * vc,clockid_t clk,struct __kernel_timespec * ts)168 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc,
169 clockid_t clk, struct __kernel_timespec *ts)
170 {
171 u64 sec, ns;
172 u32 seq;
173
174 /* Allows to compile the high resolution parts out */
175 if (!__arch_vdso_hres_capable())
176 return false;
177
178 do {
179 /*
180 * Open coded function vdso_read_begin() to handle
181 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a
182 * special VVAR page installed which has vc->seq set to 1 and
183 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time
184 * namespace affected tasks this does not affect performance
185 * because if vc->seq is odd, i.e. a concurrent update is in
186 * progress the extra check for vc->clock_mode is just a few
187 * extra instructions while spin waiting for vc->seq to become
188 * even again.
189 */
190 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) {
191 if (IS_ENABLED(CONFIG_TIME_NS) &&
192 vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
193 return do_hres_timens(vd, vc, clk, ts);
194 cpu_relax();
195 }
196 smp_rmb();
197
198 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns))
199 return false;
200 } while (unlikely(vdso_read_retry(vc, seq)));
201
202 vdso_set_timespec(ts, sec, ns);
203
204 return true;
205 }
206
207 #ifdef CONFIG_TIME_NS
208 static __always_inline
do_coarse_timens(const struct vdso_time_data * vdns,const struct vdso_clock * vcns,clockid_t clk,struct __kernel_timespec * ts)209 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
210 clockid_t clk, struct __kernel_timespec *ts)
211 {
212 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns);
213 const struct timens_offset *offs = &vcns->offset[clk];
214 const struct vdso_clock *vc = vd->clock_data;
215 const struct vdso_timestamp *vdso_ts;
216 u64 nsec;
217 s64 sec;
218 s32 seq;
219
220 vdso_ts = &vc->basetime[clk];
221
222 do {
223 seq = vdso_read_begin(vc);
224 sec = vdso_ts->sec;
225 nsec = vdso_ts->nsec;
226 } while (unlikely(vdso_read_retry(vc, seq)));
227
228 /* Add the namespace offset */
229 sec += offs->sec;
230 nsec += offs->nsec;
231
232 vdso_set_timespec(ts, sec, nsec);
233
234 return true;
235 }
236 #else
237 static __always_inline
do_coarse_timens(const struct vdso_time_data * vdns,const struct vdso_clock * vcns,clockid_t clk,struct __kernel_timespec * ts)238 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns,
239 clockid_t clk, struct __kernel_timespec *ts)
240 {
241 return false;
242 }
243 #endif
244
245 static __always_inline
do_coarse(const struct vdso_time_data * vd,const struct vdso_clock * vc,clockid_t clk,struct __kernel_timespec * ts)246 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc,
247 clockid_t clk, struct __kernel_timespec *ts)
248 {
249 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk];
250 u32 seq;
251
252 do {
253 /*
254 * Open coded function vdso_read_begin() to handle
255 * VDSO_CLOCK_TIMENS. See comment in do_hres().
256 */
257 while ((seq = READ_ONCE(vc->seq)) & 1) {
258 if (IS_ENABLED(CONFIG_TIME_NS) &&
259 vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
260 return do_coarse_timens(vd, vc, clk, ts);
261 cpu_relax();
262 }
263 smp_rmb();
264
265 ts->tv_sec = vdso_ts->sec;
266 ts->tv_nsec = vdso_ts->nsec;
267 } while (unlikely(vdso_read_retry(vc, seq)));
268
269 return true;
270 }
271
272 static __always_inline
do_aux(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * ts)273 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts)
274 {
275 const struct vdso_clock *vc;
276 u32 seq, idx;
277 u64 sec, ns;
278
279 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS))
280 return false;
281
282 idx = clock - CLOCK_AUX;
283 vc = &vd->aux_clock_data[idx];
284
285 do {
286 /*
287 * Open coded function vdso_read_begin() to handle
288 * VDSO_CLOCK_TIMENS. See comment in do_hres().
289 */
290 while ((seq = READ_ONCE(vc->seq)) & 1) {
291 if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {
292 vd = __arch_get_vdso_u_timens_data(vd);
293 vc = &vd->aux_clock_data[idx];
294 /* Re-read from the real time data page */
295 continue;
296 }
297 cpu_relax();
298 }
299 smp_rmb();
300
301 /* Auxclock disabled? */
302 if (vc->clock_mode == VDSO_CLOCKMODE_NONE)
303 return false;
304
305 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns))
306 return false;
307 } while (unlikely(vdso_read_retry(vc, seq)));
308
309 vdso_set_timespec(ts, sec, ns);
310
311 return true;
312 }
313
314 static __always_inline bool
__cvdso_clock_gettime_common(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * ts)315 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock,
316 struct __kernel_timespec *ts)
317 {
318 const struct vdso_clock *vc = vd->clock_data;
319 u32 msk;
320
321 if (!vdso_clockid_valid(clock))
322 return false;
323
324 /*
325 * Convert the clockid to a bitmask and use it to check which
326 * clocks are handled in the VDSO directly.
327 */
328 msk = 1U << clock;
329 if (likely(msk & VDSO_HRES))
330 vc = &vc[CS_HRES_COARSE];
331 else if (msk & VDSO_COARSE)
332 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts);
333 else if (msk & VDSO_RAW)
334 vc = &vc[CS_RAW];
335 else if (msk & VDSO_AUX)
336 return do_aux(vd, clock, ts);
337 else
338 return false;
339
340 return do_hres(vd, vc, clock, ts);
341 }
342
343 static __maybe_unused int
__cvdso_clock_gettime_data(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * ts)344 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock,
345 struct __kernel_timespec *ts)
346 {
347 bool ok;
348
349 ok = __cvdso_clock_gettime_common(vd, clock, ts);
350
351 if (unlikely(!ok))
352 return clock_gettime_fallback(clock, ts);
353 return 0;
354 }
355
356 static __maybe_unused int
__cvdso_clock_gettime(clockid_t clock,struct __kernel_timespec * ts)357 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
358 {
359 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts);
360 }
361
362 #ifdef BUILD_VDSO32
363 static __maybe_unused int
__cvdso_clock_gettime32_data(const struct vdso_time_data * vd,clockid_t clock,struct old_timespec32 * res)364 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock,
365 struct old_timespec32 *res)
366 {
367 struct __kernel_timespec ts;
368 bool ok;
369
370 ok = __cvdso_clock_gettime_common(vd, clock, &ts);
371
372 if (unlikely(!ok))
373 return clock_gettime32_fallback(clock, res);
374
375 /* For ok == true */
376 res->tv_sec = ts.tv_sec;
377 res->tv_nsec = ts.tv_nsec;
378
379 return 0;
380 }
381
382 static __maybe_unused int
__cvdso_clock_gettime32(clockid_t clock,struct old_timespec32 * res)383 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
384 {
385 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res);
386 }
387 #endif /* BUILD_VDSO32 */
388
389 static __maybe_unused int
__cvdso_gettimeofday_data(const struct vdso_time_data * vd,struct __kernel_old_timeval * tv,struct timezone * tz)390 __cvdso_gettimeofday_data(const struct vdso_time_data *vd,
391 struct __kernel_old_timeval *tv, struct timezone *tz)
392 {
393 const struct vdso_clock *vc = vd->clock_data;
394
395 if (likely(tv != NULL)) {
396 struct __kernel_timespec ts;
397
398 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
399 return gettimeofday_fallback(tv, tz);
400
401 tv->tv_sec = ts.tv_sec;
402 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
403 }
404
405 if (unlikely(tz != NULL)) {
406 if (IS_ENABLED(CONFIG_TIME_NS) &&
407 vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
408 vd = __arch_get_vdso_u_timens_data(vd);
409
410 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
411 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
412 }
413
414 return 0;
415 }
416
417 static __maybe_unused int
__cvdso_gettimeofday(struct __kernel_old_timeval * tv,struct timezone * tz)418 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
419 {
420 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz);
421 }
422
423 #ifdef VDSO_HAS_TIME
424 static __maybe_unused __kernel_old_time_t
__cvdso_time_data(const struct vdso_time_data * vd,__kernel_old_time_t * time)425 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time)
426 {
427 const struct vdso_clock *vc = vd->clock_data;
428 __kernel_old_time_t t;
429
430 if (IS_ENABLED(CONFIG_TIME_NS) &&
431 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) {
432 vd = __arch_get_vdso_u_timens_data(vd);
433 vc = vd->clock_data;
434 }
435
436 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
437
438 if (time)
439 *time = t;
440
441 return t;
442 }
443
__cvdso_time(__kernel_old_time_t * time)444 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
445 {
446 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time);
447 }
448 #endif /* VDSO_HAS_TIME */
449
450 #ifdef VDSO_HAS_CLOCK_GETRES
451 static __maybe_unused
__cvdso_clock_getres_common(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * res)452 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock,
453 struct __kernel_timespec *res)
454 {
455 const struct vdso_clock *vc = vd->clock_data;
456 u32 msk;
457 u64 ns;
458
459 if (!vdso_clockid_valid(clock))
460 return false;
461
462 if (IS_ENABLED(CONFIG_TIME_NS) &&
463 vc->clock_mode == VDSO_CLOCKMODE_TIMENS)
464 vd = __arch_get_vdso_u_timens_data(vd);
465
466 /*
467 * Convert the clockid to a bitmask and use it to check which
468 * clocks are handled in the VDSO directly.
469 */
470 msk = 1U << clock;
471 if (msk & (VDSO_HRES | VDSO_RAW)) {
472 /*
473 * Preserves the behaviour of posix_get_hrtimer_res().
474 */
475 ns = READ_ONCE(vd->hrtimer_res);
476 } else if (msk & VDSO_COARSE) {
477 /*
478 * Preserves the behaviour of posix_get_coarse_res().
479 */
480 ns = LOW_RES_NSEC;
481 } else if (msk & VDSO_AUX) {
482 ns = aux_clock_resolution_ns();
483 } else {
484 return false;
485 }
486
487 if (likely(res)) {
488 res->tv_sec = 0;
489 res->tv_nsec = ns;
490 }
491 return true;
492 }
493
494 static __maybe_unused
__cvdso_clock_getres_data(const struct vdso_time_data * vd,clockid_t clock,struct __kernel_timespec * res)495 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock,
496 struct __kernel_timespec *res)
497 {
498 bool ok;
499
500 ok = __cvdso_clock_getres_common(vd, clock, res);
501
502 if (unlikely(!ok))
503 return clock_getres_fallback(clock, res);
504 return 0;
505 }
506
507 static __maybe_unused
__cvdso_clock_getres(clockid_t clock,struct __kernel_timespec * res)508 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
509 {
510 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res);
511 }
512
513 #ifdef BUILD_VDSO32
514 static __maybe_unused int
__cvdso_clock_getres_time32_data(const struct vdso_time_data * vd,clockid_t clock,struct old_timespec32 * res)515 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock,
516 struct old_timespec32 *res)
517 {
518 struct __kernel_timespec ts;
519 bool ok;
520
521 ok = __cvdso_clock_getres_common(vd, clock, &ts);
522
523 if (unlikely(!ok))
524 return clock_getres32_fallback(clock, res);
525
526 if (likely(res)) {
527 res->tv_sec = ts.tv_sec;
528 res->tv_nsec = ts.tv_nsec;
529 }
530 return 0;
531 }
532
533 static __maybe_unused int
__cvdso_clock_getres_time32(clockid_t clock,struct old_timespec32 * res)534 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
535 {
536 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(),
537 clock, res);
538 }
539 #endif /* BUILD_VDSO32 */
540 #endif /* VDSO_HAS_CLOCK_GETRES */
541