1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/auxclock.h> 6 #include <vdso/clocksource.h> 7 #include <vdso/datapage.h> 8 #include <vdso/helpers.h> 9 #include <vdso/ktime.h> 10 #include <vdso/limits.h> 11 #include <vdso/math64.h> 12 #include <vdso/time32.h> 13 #include <vdso/time64.h> 14 15 #include <uapi/linux/unistd.h> 16 17 /* 18 * The generic vDSO implementation requires that gettimeofday.h 19 * provides: 20 * - __arch_get_hw_counter(): to get the hw counter based on the 21 * clock_mode. 22 * - gettimeofday_fallback(): fallback for gettimeofday. 23 * - clock_gettime_fallback(): fallback for clock_gettime. 24 * - clock_getres_fallback(): fallback for clock_getres. 25 */ 26 #include <asm/vdso/gettimeofday.h> 27 28 #include <linux/build_bug.h> 29 30 /* Bring in default accessors */ 31 #include <vdso/vsyscall.h> 32 33 #ifndef vdso_calc_ns 34 35 #ifdef VDSO_DELTA_NOMASK 36 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 37 #else 38 # define VDSO_DELTA_MASK(vd) (vd->mask) 39 #endif 40 41 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 42 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 43 { 44 return delta < vc->max_cycles; 45 } 46 #else 47 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 48 { 49 return true; 50 } 51 #endif 52 53 #ifndef vdso_shift_ns 54 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 55 { 56 return ns >> shift; 57 } 58 #endif 59 60 /* 61 * Default implementation which works for all sane clocksources. That 62 * obviously excludes x86/TSC. 63 */ 64 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 65 { 66 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 67 68 if (likely(vdso_delta_ok(vc, delta))) 69 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 70 71 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 72 } 73 #endif /* vdso_calc_ns */ 74 75 #ifndef __arch_vdso_hres_capable 76 static inline bool __arch_vdso_hres_capable(void) 77 { 78 return true; 79 } 80 #endif 81 82 #ifndef vdso_clocksource_ok 83 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 84 { 85 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 86 } 87 #endif 88 89 #ifndef vdso_cycles_ok 90 static inline bool vdso_cycles_ok(u64 cycles) 91 { 92 return true; 93 } 94 #endif 95 96 static __always_inline bool vdso_clockid_valid(clockid_t clock) 97 { 98 /* Check for negative values or invalid clocks */ 99 return likely((u32) clock <= CLOCK_AUX_LAST); 100 } 101 102 /* 103 * Must not be invoked within the sequence read section as a race inside 104 * that loop could result in __iter_div_u64_rem() being extremely slow. 105 */ 106 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 107 { 108 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 109 ts->tv_nsec = ns; 110 } 111 112 static __always_inline 113 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 114 unsigned int clkidx, u64 *sec, u64 *ns) 115 { 116 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 117 u64 cycles; 118 119 if (unlikely(!vdso_clocksource_ok(vc))) 120 return false; 121 122 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 123 if (unlikely(!vdso_cycles_ok(cycles))) 124 return false; 125 126 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 127 *sec = vdso_ts->sec; 128 129 return true; 130 } 131 132 static __always_inline 133 const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd) 134 { 135 return (void *)vd + PAGE_SIZE; 136 } 137 138 static __always_inline 139 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 140 clockid_t clk, struct __kernel_timespec *ts) 141 { 142 const struct vdso_time_data *vd = vdso_timens_data(vdns); 143 const struct timens_offset *offs = &vcns->offset[clk]; 144 const struct vdso_clock *vc = vd->clock_data; 145 u32 seq; 146 s64 sec; 147 u64 ns; 148 149 if (clk != CLOCK_MONOTONIC_RAW) 150 vc = &vc[CS_HRES_COARSE]; 151 else 152 vc = &vc[CS_RAW]; 153 154 do { 155 seq = vdso_read_begin(vc); 156 157 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 158 return false; 159 } while (vdso_read_retry(vc, seq)); 160 161 /* Add the namespace offset */ 162 sec += offs->sec; 163 ns += offs->nsec; 164 165 vdso_set_timespec(ts, sec, ns); 166 167 return true; 168 } 169 170 static __always_inline 171 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 172 clockid_t clk, struct __kernel_timespec *ts) 173 { 174 u64 sec, ns; 175 u32 seq; 176 177 /* Allows to compile the high resolution parts out */ 178 if (!__arch_vdso_hres_capable()) 179 return false; 180 181 do { 182 if (vdso_read_begin_timens(vc, &seq)) 183 return do_hres_timens(vd, vc, clk, ts); 184 185 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 186 return false; 187 } while (vdso_read_retry(vc, seq)); 188 189 vdso_set_timespec(ts, sec, ns); 190 191 return true; 192 } 193 194 static __always_inline 195 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 196 clockid_t clk, struct __kernel_timespec *ts) 197 { 198 const struct vdso_time_data *vd = vdso_timens_data(vdns); 199 const struct timens_offset *offs = &vcns->offset[clk]; 200 const struct vdso_clock *vc = vd->clock_data; 201 const struct vdso_timestamp *vdso_ts; 202 u64 nsec; 203 s64 sec; 204 s32 seq; 205 206 vdso_ts = &vc->basetime[clk]; 207 208 do { 209 seq = vdso_read_begin(vc); 210 sec = vdso_ts->sec; 211 nsec = vdso_ts->nsec; 212 } while (vdso_read_retry(vc, seq)); 213 214 /* Add the namespace offset */ 215 sec += offs->sec; 216 nsec += offs->nsec; 217 218 vdso_set_timespec(ts, sec, nsec); 219 220 return true; 221 } 222 223 static __always_inline 224 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 225 clockid_t clk, struct __kernel_timespec *ts) 226 { 227 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 228 u32 seq; 229 230 do { 231 if (vdso_read_begin_timens(vc, &seq)) 232 return do_coarse_timens(vd, vc, clk, ts); 233 234 ts->tv_sec = vdso_ts->sec; 235 ts->tv_nsec = vdso_ts->nsec; 236 } while (vdso_read_retry(vc, seq)); 237 238 return true; 239 } 240 241 static __always_inline 242 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) 243 { 244 const struct vdso_clock *vc; 245 u32 seq, idx; 246 u64 sec, ns; 247 248 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 249 return false; 250 251 idx = clock - CLOCK_AUX; 252 vc = &vd->aux_clock_data[idx]; 253 254 do { 255 while (vdso_read_begin_timens(vc, &seq)) { 256 /* Re-read from the real time data page, reload seq by looping */ 257 vd = vdso_timens_data(vd); 258 vc = &vd->aux_clock_data[idx]; 259 } 260 261 /* Auxclock disabled? */ 262 if (vc->clock_mode == VDSO_CLOCKMODE_NONE) 263 return false; 264 265 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) 266 return false; 267 } while (vdso_read_retry(vc, seq)); 268 269 vdso_set_timespec(ts, sec, ns); 270 271 return true; 272 } 273 274 static __always_inline bool 275 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 276 struct __kernel_timespec *ts) 277 { 278 const struct vdso_clock *vc = vd->clock_data; 279 u32 msk; 280 281 if (!vdso_clockid_valid(clock)) 282 return false; 283 284 /* 285 * Convert the clockid to a bitmask and use it to check which 286 * clocks are handled in the VDSO directly. 287 */ 288 msk = 1U << clock; 289 if (likely(msk & VDSO_HRES)) 290 vc = &vc[CS_HRES_COARSE]; 291 else if (msk & VDSO_COARSE) 292 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 293 else if (msk & VDSO_RAW) 294 vc = &vc[CS_RAW]; 295 else if (msk & VDSO_AUX) 296 return do_aux(vd, clock, ts); 297 else 298 return false; 299 300 return do_hres(vd, vc, clock, ts); 301 } 302 303 static int 304 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 305 struct __kernel_timespec *ts) 306 { 307 bool ok; 308 309 ok = __cvdso_clock_gettime_common(vd, clock, ts); 310 311 if (unlikely(!ok)) 312 return clock_gettime_fallback(clock, ts); 313 return 0; 314 } 315 316 static __maybe_unused int 317 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 318 { 319 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 320 } 321 322 #ifdef BUILD_VDSO32 323 static int 324 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 325 struct old_timespec32 *res) 326 { 327 struct __kernel_timespec ts; 328 bool ok; 329 330 BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 331 332 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 333 334 if (unlikely(!ok)) 335 return clock_gettime32_fallback(clock, res); 336 337 /* For ok == true */ 338 res->tv_sec = ts.tv_sec; 339 res->tv_nsec = ts.tv_nsec; 340 341 return 0; 342 } 343 344 static __maybe_unused int 345 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 346 { 347 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 348 } 349 #endif /* BUILD_VDSO32 */ 350 351 static int 352 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 353 struct __kernel_old_timeval *tv, struct timezone *tz) 354 { 355 const struct vdso_clock *vc = vd->clock_data; 356 357 #ifndef __NR_gettimeofday 358 BUILD_BUG(); 359 #endif 360 361 BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 362 363 if (likely(tv != NULL)) { 364 struct __kernel_timespec ts; 365 366 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 367 return gettimeofday_fallback(tv, tz); 368 369 tv->tv_sec = ts.tv_sec; 370 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 371 } 372 373 if (unlikely(tz != NULL)) { 374 if (vdso_is_timens_clock(vc)) 375 vd = vdso_timens_data(vd); 376 377 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 378 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 379 } 380 381 return 0; 382 } 383 384 static __maybe_unused int 385 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 386 { 387 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 388 } 389 390 #ifdef VDSO_HAS_TIME 391 static __kernel_old_time_t 392 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 393 { 394 const struct vdso_clock *vc = vd->clock_data; 395 __kernel_old_time_t t; 396 397 #ifndef __NR_time 398 BUILD_BUG(); 399 #endif 400 401 BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 402 403 if (vdso_is_timens_clock(vc)) { 404 vd = vdso_timens_data(vd); 405 vc = vd->clock_data; 406 } 407 408 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 409 410 if (time) 411 *time = t; 412 413 return t; 414 } 415 416 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 417 { 418 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 419 } 420 #endif /* VDSO_HAS_TIME */ 421 422 #ifdef VDSO_HAS_CLOCK_GETRES 423 static __always_inline 424 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 425 struct __kernel_timespec *res) 426 { 427 const struct vdso_clock *vc = vd->clock_data; 428 u32 msk; 429 u64 ns; 430 431 if (!vdso_clockid_valid(clock)) 432 return false; 433 434 if (vdso_is_timens_clock(vc)) 435 vd = vdso_timens_data(vd); 436 437 /* 438 * Convert the clockid to a bitmask and use it to check which 439 * clocks are handled in the VDSO directly. 440 */ 441 msk = 1U << clock; 442 if (msk & (VDSO_HRES | VDSO_RAW)) { 443 /* 444 * Preserves the behaviour of posix_get_hrtimer_res(). 445 */ 446 ns = READ_ONCE(vd->hrtimer_res); 447 } else if (msk & VDSO_COARSE) { 448 /* 449 * Preserves the behaviour of posix_get_coarse_res(). 450 */ 451 ns = LOW_RES_NSEC; 452 } else if (msk & VDSO_AUX) { 453 ns = aux_clock_resolution_ns(); 454 } else { 455 return false; 456 } 457 458 if (likely(res)) { 459 res->tv_sec = 0; 460 res->tv_nsec = ns; 461 } 462 return true; 463 } 464 465 static 466 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 467 struct __kernel_timespec *res) 468 { 469 bool ok; 470 471 ok = __cvdso_clock_getres_common(vd, clock, res); 472 473 if (unlikely(!ok)) 474 return clock_getres_fallback(clock, res); 475 return 0; 476 } 477 478 static __maybe_unused 479 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 480 { 481 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 482 } 483 484 #ifdef BUILD_VDSO32 485 static int 486 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 487 struct old_timespec32 *res) 488 { 489 struct __kernel_timespec ts; 490 bool ok; 491 492 BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 493 494 ok = __cvdso_clock_getres_common(vd, clock, &ts); 495 496 if (unlikely(!ok)) 497 return clock_getres32_fallback(clock, res); 498 499 if (likely(res)) { 500 res->tv_sec = ts.tv_sec; 501 res->tv_nsec = ts.tv_nsec; 502 } 503 return 0; 504 } 505 506 static __maybe_unused int 507 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 508 { 509 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 510 clock, res); 511 } 512 #endif /* BUILD_VDSO32 */ 513 #endif /* VDSO_HAS_CLOCK_GETRES */ 514