1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/auxclock.h> 6 #include <vdso/clocksource.h> 7 #include <vdso/datapage.h> 8 #include <vdso/helpers.h> 9 #include <vdso/ktime.h> 10 #include <vdso/limits.h> 11 #include <vdso/math64.h> 12 #include <vdso/time32.h> 13 #include <vdso/time64.h> 14 15 /* 16 * The generic vDSO implementation requires that gettimeofday.h 17 * provides: 18 * - __arch_get_hw_counter(): to get the hw counter based on the 19 * clock_mode. 20 * - gettimeofday_fallback(): fallback for gettimeofday. 21 * - clock_gettime_fallback(): fallback for clock_gettime. 22 * - clock_getres_fallback(): fallback for clock_getres. 23 */ 24 #include <asm/vdso/gettimeofday.h> 25 26 /* Bring in default accessors */ 27 #include <vdso/vsyscall.h> 28 29 #ifndef vdso_calc_ns 30 31 #ifdef VDSO_DELTA_NOMASK 32 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 33 #else 34 # define VDSO_DELTA_MASK(vd) (vd->mask) 35 #endif 36 37 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 38 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 39 { 40 return delta < vc->max_cycles; 41 } 42 #else 43 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 44 { 45 return true; 46 } 47 #endif 48 49 #ifndef vdso_shift_ns 50 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 51 { 52 return ns >> shift; 53 } 54 #endif 55 56 /* 57 * Default implementation which works for all sane clocksources. That 58 * obviously excludes x86/TSC. 59 */ 60 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 61 { 62 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 63 64 if (likely(vdso_delta_ok(vc, delta))) 65 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 66 67 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 68 } 69 #endif /* vdso_calc_ns */ 70 71 #ifndef __arch_vdso_hres_capable 72 static inline bool __arch_vdso_hres_capable(void) 73 { 74 return true; 75 } 76 #endif 77 78 #ifndef vdso_clocksource_ok 79 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 80 { 81 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 82 } 83 #endif 84 85 #ifndef vdso_cycles_ok 86 static inline bool vdso_cycles_ok(u64 cycles) 87 { 88 return true; 89 } 90 #endif 91 92 static __always_inline bool vdso_clockid_valid(clockid_t clock) 93 { 94 /* Check for negative values or invalid clocks */ 95 return likely((u32) clock <= CLOCK_AUX_LAST); 96 } 97 98 /* 99 * Must not be invoked within the sequence read section as a race inside 100 * that loop could result in __iter_div_u64_rem() being extremely slow. 101 */ 102 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 103 { 104 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 105 ts->tv_nsec = ns; 106 } 107 108 static __always_inline 109 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 110 unsigned int clkidx, u64 *sec, u64 *ns) 111 { 112 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 113 u64 cycles; 114 115 if (unlikely(!vdso_clocksource_ok(vc))) 116 return false; 117 118 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 119 if (unlikely(!vdso_cycles_ok(cycles))) 120 return false; 121 122 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 123 *sec = vdso_ts->sec; 124 125 return true; 126 } 127 128 static __always_inline 129 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 130 { 131 return (void *)vd + PAGE_SIZE; 132 } 133 134 static __always_inline 135 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 136 clockid_t clk, struct __kernel_timespec *ts) 137 { 138 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 139 const struct timens_offset *offs = &vcns->offset[clk]; 140 const struct vdso_clock *vc = vd->clock_data; 141 u32 seq; 142 s64 sec; 143 u64 ns; 144 145 if (clk != CLOCK_MONOTONIC_RAW) 146 vc = &vc[CS_HRES_COARSE]; 147 else 148 vc = &vc[CS_RAW]; 149 150 do { 151 seq = vdso_read_begin(vc); 152 153 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 154 return false; 155 } while (vdso_read_retry(vc, seq)); 156 157 /* Add the namespace offset */ 158 sec += offs->sec; 159 ns += offs->nsec; 160 161 vdso_set_timespec(ts, sec, ns); 162 163 return true; 164 } 165 166 static __always_inline 167 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 168 clockid_t clk, struct __kernel_timespec *ts) 169 { 170 u64 sec, ns; 171 u32 seq; 172 173 /* Allows to compile the high resolution parts out */ 174 if (!__arch_vdso_hres_capable()) 175 return false; 176 177 do { 178 if (vdso_read_begin_timens(vc, &seq)) 179 return do_hres_timens(vd, vc, clk, ts); 180 181 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 182 return false; 183 } while (vdso_read_retry(vc, seq)); 184 185 vdso_set_timespec(ts, sec, ns); 186 187 return true; 188 } 189 190 static __always_inline 191 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 192 clockid_t clk, struct __kernel_timespec *ts) 193 { 194 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 195 const struct timens_offset *offs = &vcns->offset[clk]; 196 const struct vdso_clock *vc = vd->clock_data; 197 const struct vdso_timestamp *vdso_ts; 198 u64 nsec; 199 s64 sec; 200 s32 seq; 201 202 vdso_ts = &vc->basetime[clk]; 203 204 do { 205 seq = vdso_read_begin(vc); 206 sec = vdso_ts->sec; 207 nsec = vdso_ts->nsec; 208 } while (vdso_read_retry(vc, seq)); 209 210 /* Add the namespace offset */ 211 sec += offs->sec; 212 nsec += offs->nsec; 213 214 vdso_set_timespec(ts, sec, nsec); 215 216 return true; 217 } 218 219 static __always_inline 220 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 221 clockid_t clk, struct __kernel_timespec *ts) 222 { 223 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 224 u32 seq; 225 226 do { 227 if (vdso_read_begin_timens(vc, &seq)) 228 return do_coarse_timens(vd, vc, clk, ts); 229 230 ts->tv_sec = vdso_ts->sec; 231 ts->tv_nsec = vdso_ts->nsec; 232 } while (vdso_read_retry(vc, seq)); 233 234 return true; 235 } 236 237 static __always_inline 238 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) 239 { 240 const struct vdso_clock *vc; 241 u32 seq, idx; 242 u64 sec, ns; 243 244 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 245 return false; 246 247 idx = clock - CLOCK_AUX; 248 vc = &vd->aux_clock_data[idx]; 249 250 do { 251 while (vdso_read_begin_timens(vc, &seq)) { 252 /* Re-read from the real time data page, reload seq by looping */ 253 vd = __arch_get_vdso_u_timens_data(vd); 254 vc = &vd->aux_clock_data[idx]; 255 } 256 257 /* Auxclock disabled? */ 258 if (vc->clock_mode == VDSO_CLOCKMODE_NONE) 259 return false; 260 261 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) 262 return false; 263 } while (vdso_read_retry(vc, seq)); 264 265 vdso_set_timespec(ts, sec, ns); 266 267 return true; 268 } 269 270 static __always_inline bool 271 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 272 struct __kernel_timespec *ts) 273 { 274 const struct vdso_clock *vc = vd->clock_data; 275 u32 msk; 276 277 if (!vdso_clockid_valid(clock)) 278 return false; 279 280 /* 281 * Convert the clockid to a bitmask and use it to check which 282 * clocks are handled in the VDSO directly. 283 */ 284 msk = 1U << clock; 285 if (likely(msk & VDSO_HRES)) 286 vc = &vc[CS_HRES_COARSE]; 287 else if (msk & VDSO_COARSE) 288 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 289 else if (msk & VDSO_RAW) 290 vc = &vc[CS_RAW]; 291 else if (msk & VDSO_AUX) 292 return do_aux(vd, clock, ts); 293 else 294 return false; 295 296 return do_hres(vd, vc, clock, ts); 297 } 298 299 static int 300 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 301 struct __kernel_timespec *ts) 302 { 303 bool ok; 304 305 ok = __cvdso_clock_gettime_common(vd, clock, ts); 306 307 if (unlikely(!ok)) 308 return clock_gettime_fallback(clock, ts); 309 return 0; 310 } 311 312 static __maybe_unused int 313 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 314 { 315 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 316 } 317 318 #ifdef BUILD_VDSO32 319 static int 320 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 321 struct old_timespec32 *res) 322 { 323 struct __kernel_timespec ts; 324 bool ok; 325 326 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 327 328 if (unlikely(!ok)) 329 return clock_gettime32_fallback(clock, res); 330 331 /* For ok == true */ 332 res->tv_sec = ts.tv_sec; 333 res->tv_nsec = ts.tv_nsec; 334 335 return 0; 336 } 337 338 static __maybe_unused int 339 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 340 { 341 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 342 } 343 #endif /* BUILD_VDSO32 */ 344 345 static int 346 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 347 struct __kernel_old_timeval *tv, struct timezone *tz) 348 { 349 const struct vdso_clock *vc = vd->clock_data; 350 351 if (likely(tv != NULL)) { 352 struct __kernel_timespec ts; 353 354 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 355 return gettimeofday_fallback(tv, tz); 356 357 tv->tv_sec = ts.tv_sec; 358 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 359 } 360 361 if (unlikely(tz != NULL)) { 362 if (vdso_is_timens_clock(vc)) 363 vd = __arch_get_vdso_u_timens_data(vd); 364 365 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 366 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 367 } 368 369 return 0; 370 } 371 372 static __maybe_unused int 373 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 374 { 375 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 376 } 377 378 #ifdef VDSO_HAS_TIME 379 static __kernel_old_time_t 380 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 381 { 382 const struct vdso_clock *vc = vd->clock_data; 383 __kernel_old_time_t t; 384 385 if (vdso_is_timens_clock(vc)) { 386 vd = __arch_get_vdso_u_timens_data(vd); 387 vc = vd->clock_data; 388 } 389 390 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 391 392 if (time) 393 *time = t; 394 395 return t; 396 } 397 398 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 399 { 400 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 401 } 402 #endif /* VDSO_HAS_TIME */ 403 404 #ifdef VDSO_HAS_CLOCK_GETRES 405 static __always_inline 406 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 407 struct __kernel_timespec *res) 408 { 409 const struct vdso_clock *vc = vd->clock_data; 410 u32 msk; 411 u64 ns; 412 413 if (!vdso_clockid_valid(clock)) 414 return false; 415 416 if (vdso_is_timens_clock(vc)) 417 vd = __arch_get_vdso_u_timens_data(vd); 418 419 /* 420 * Convert the clockid to a bitmask and use it to check which 421 * clocks are handled in the VDSO directly. 422 */ 423 msk = 1U << clock; 424 if (msk & (VDSO_HRES | VDSO_RAW)) { 425 /* 426 * Preserves the behaviour of posix_get_hrtimer_res(). 427 */ 428 ns = READ_ONCE(vd->hrtimer_res); 429 } else if (msk & VDSO_COARSE) { 430 /* 431 * Preserves the behaviour of posix_get_coarse_res(). 432 */ 433 ns = LOW_RES_NSEC; 434 } else if (msk & VDSO_AUX) { 435 ns = aux_clock_resolution_ns(); 436 } else { 437 return false; 438 } 439 440 if (likely(res)) { 441 res->tv_sec = 0; 442 res->tv_nsec = ns; 443 } 444 return true; 445 } 446 447 static 448 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 449 struct __kernel_timespec *res) 450 { 451 bool ok; 452 453 ok = __cvdso_clock_getres_common(vd, clock, res); 454 455 if (unlikely(!ok)) 456 return clock_getres_fallback(clock, res); 457 return 0; 458 } 459 460 static __maybe_unused 461 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 462 { 463 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 464 } 465 466 #ifdef BUILD_VDSO32 467 static int 468 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 469 struct old_timespec32 *res) 470 { 471 struct __kernel_timespec ts; 472 bool ok; 473 474 ok = __cvdso_clock_getres_common(vd, clock, &ts); 475 476 if (unlikely(!ok)) 477 return clock_getres32_fallback(clock, res); 478 479 if (likely(res)) { 480 res->tv_sec = ts.tv_sec; 481 res->tv_nsec = ts.tv_nsec; 482 } 483 return 0; 484 } 485 486 static __maybe_unused int 487 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 488 { 489 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 490 clock, res); 491 } 492 #endif /* BUILD_VDSO32 */ 493 #endif /* VDSO_HAS_CLOCK_GETRES */ 494