1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/auxclock.h> 6 #include <vdso/datapage.h> 7 #include <vdso/helpers.h> 8 9 /* Bring in default accessors */ 10 #include <vdso/vsyscall.h> 11 12 #ifndef vdso_calc_ns 13 14 #ifdef VDSO_DELTA_NOMASK 15 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 16 #else 17 # define VDSO_DELTA_MASK(vd) (vd->mask) 18 #endif 19 20 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 21 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 22 { 23 return delta < vc->max_cycles; 24 } 25 #else 26 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 27 { 28 return true; 29 } 30 #endif 31 32 #ifndef vdso_shift_ns 33 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 34 { 35 return ns >> shift; 36 } 37 #endif 38 39 /* 40 * Default implementation which works for all sane clocksources. That 41 * obviously excludes x86/TSC. 42 */ 43 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 44 { 45 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 46 47 if (likely(vdso_delta_ok(vc, delta))) 48 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 49 50 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 51 } 52 #endif /* vdso_calc_ns */ 53 54 #ifndef __arch_vdso_hres_capable 55 static inline bool __arch_vdso_hres_capable(void) 56 { 57 return true; 58 } 59 #endif 60 61 #ifndef vdso_clocksource_ok 62 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 63 { 64 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 65 } 66 #endif 67 68 #ifndef vdso_cycles_ok 69 static inline bool vdso_cycles_ok(u64 cycles) 70 { 71 return true; 72 } 73 #endif 74 75 static __always_inline bool vdso_clockid_valid(clockid_t clock) 76 { 77 /* Check for negative values or invalid clocks */ 78 return likely((u32) clock <= CLOCK_AUX_LAST); 79 } 80 81 /* 82 * Must not be invoked within the sequence read section as a race inside 83 * that loop could result in __iter_div_u64_rem() being extremely slow. 84 */ 85 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 86 { 87 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 88 ts->tv_nsec = ns; 89 } 90 91 static __always_inline 92 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 93 unsigned int clkidx, u64 *sec, u64 *ns) 94 { 95 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 96 u64 cycles; 97 98 if (unlikely(!vdso_clocksource_ok(vc))) 99 return false; 100 101 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 102 if (unlikely(!vdso_cycles_ok(cycles))) 103 return false; 104 105 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 106 *sec = vdso_ts->sec; 107 108 return true; 109 } 110 111 #ifdef CONFIG_TIME_NS 112 113 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE 114 static __always_inline 115 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 116 { 117 return (void *)vd + PAGE_SIZE; 118 } 119 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ 120 121 static __always_inline 122 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 123 clockid_t clk, struct __kernel_timespec *ts) 124 { 125 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 126 const struct timens_offset *offs = &vcns->offset[clk]; 127 const struct vdso_clock *vc = vd->clock_data; 128 u32 seq; 129 s64 sec; 130 u64 ns; 131 132 if (clk != CLOCK_MONOTONIC_RAW) 133 vc = &vc[CS_HRES_COARSE]; 134 else 135 vc = &vc[CS_RAW]; 136 137 do { 138 seq = vdso_read_begin(vc); 139 140 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 141 return false; 142 } while (unlikely(vdso_read_retry(vc, seq))); 143 144 /* Add the namespace offset */ 145 sec += offs->sec; 146 ns += offs->nsec; 147 148 vdso_set_timespec(ts, sec, ns); 149 150 return true; 151 } 152 #else 153 static __always_inline 154 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 155 { 156 return NULL; 157 } 158 159 static __always_inline 160 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 161 clockid_t clk, struct __kernel_timespec *ts) 162 { 163 return false; 164 } 165 #endif 166 167 static __always_inline 168 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 169 clockid_t clk, struct __kernel_timespec *ts) 170 { 171 u64 sec, ns; 172 u32 seq; 173 174 /* Allows to compile the high resolution parts out */ 175 if (!__arch_vdso_hres_capable()) 176 return false; 177 178 do { 179 /* 180 * Open coded function vdso_read_begin() to handle 181 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 182 * special VVAR page installed which has vc->seq set to 1 and 183 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 184 * namespace affected tasks this does not affect performance 185 * because if vc->seq is odd, i.e. a concurrent update is in 186 * progress the extra check for vc->clock_mode is just a few 187 * extra instructions while spin waiting for vc->seq to become 188 * even again. 189 */ 190 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) { 191 if (IS_ENABLED(CONFIG_TIME_NS) && 192 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 193 return do_hres_timens(vd, vc, clk, ts); 194 cpu_relax(); 195 } 196 smp_rmb(); 197 198 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 199 return false; 200 } while (unlikely(vdso_read_retry(vc, seq))); 201 202 vdso_set_timespec(ts, sec, ns); 203 204 return true; 205 } 206 207 #ifdef CONFIG_TIME_NS 208 static __always_inline 209 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 210 clockid_t clk, struct __kernel_timespec *ts) 211 { 212 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 213 const struct timens_offset *offs = &vcns->offset[clk]; 214 const struct vdso_clock *vc = vd->clock_data; 215 const struct vdso_timestamp *vdso_ts; 216 u64 nsec; 217 s64 sec; 218 s32 seq; 219 220 vdso_ts = &vc->basetime[clk]; 221 222 do { 223 seq = vdso_read_begin(vc); 224 sec = vdso_ts->sec; 225 nsec = vdso_ts->nsec; 226 } while (unlikely(vdso_read_retry(vc, seq))); 227 228 /* Add the namespace offset */ 229 sec += offs->sec; 230 nsec += offs->nsec; 231 232 vdso_set_timespec(ts, sec, nsec); 233 234 return true; 235 } 236 #else 237 static __always_inline 238 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 239 clockid_t clk, struct __kernel_timespec *ts) 240 { 241 return false; 242 } 243 #endif 244 245 static __always_inline 246 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 247 clockid_t clk, struct __kernel_timespec *ts) 248 { 249 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 250 u32 seq; 251 252 do { 253 /* 254 * Open coded function vdso_read_begin() to handle 255 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 256 */ 257 while ((seq = READ_ONCE(vc->seq)) & 1) { 258 if (IS_ENABLED(CONFIG_TIME_NS) && 259 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 260 return do_coarse_timens(vd, vc, clk, ts); 261 cpu_relax(); 262 } 263 smp_rmb(); 264 265 ts->tv_sec = vdso_ts->sec; 266 ts->tv_nsec = vdso_ts->nsec; 267 } while (unlikely(vdso_read_retry(vc, seq))); 268 269 return true; 270 } 271 272 static __always_inline 273 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) 274 { 275 const struct vdso_clock *vc; 276 u32 seq, idx; 277 u64 sec, ns; 278 279 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 280 return false; 281 282 idx = clock - CLOCK_AUX; 283 vc = &vd->aux_clock_data[idx]; 284 285 do { 286 /* 287 * Open coded function vdso_read_begin() to handle 288 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 289 */ 290 while ((seq = READ_ONCE(vc->seq)) & 1) { 291 if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 292 vd = __arch_get_vdso_u_timens_data(vd); 293 vc = &vd->aux_clock_data[idx]; 294 /* Re-read from the real time data page */ 295 continue; 296 } 297 cpu_relax(); 298 } 299 smp_rmb(); 300 301 /* Auxclock disabled? */ 302 if (vc->clock_mode == VDSO_CLOCKMODE_NONE) 303 return false; 304 305 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) 306 return false; 307 } while (unlikely(vdso_read_retry(vc, seq))); 308 309 vdso_set_timespec(ts, sec, ns); 310 311 return true; 312 } 313 314 static __always_inline bool 315 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 316 struct __kernel_timespec *ts) 317 { 318 const struct vdso_clock *vc = vd->clock_data; 319 u32 msk; 320 321 if (!vdso_clockid_valid(clock)) 322 return false; 323 324 /* 325 * Convert the clockid to a bitmask and use it to check which 326 * clocks are handled in the VDSO directly. 327 */ 328 msk = 1U << clock; 329 if (likely(msk & VDSO_HRES)) 330 vc = &vc[CS_HRES_COARSE]; 331 else if (msk & VDSO_COARSE) 332 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 333 else if (msk & VDSO_RAW) 334 vc = &vc[CS_RAW]; 335 else if (msk & VDSO_AUX) 336 return do_aux(vd, clock, ts); 337 else 338 return false; 339 340 return do_hres(vd, vc, clock, ts); 341 } 342 343 static __maybe_unused int 344 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 345 struct __kernel_timespec *ts) 346 { 347 bool ok; 348 349 ok = __cvdso_clock_gettime_common(vd, clock, ts); 350 351 if (unlikely(!ok)) 352 return clock_gettime_fallback(clock, ts); 353 return 0; 354 } 355 356 static __maybe_unused int 357 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 358 { 359 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 360 } 361 362 #ifdef BUILD_VDSO32 363 static __maybe_unused int 364 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 365 struct old_timespec32 *res) 366 { 367 struct __kernel_timespec ts; 368 bool ok; 369 370 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 371 372 if (unlikely(!ok)) 373 return clock_gettime32_fallback(clock, res); 374 375 /* For ok == true */ 376 res->tv_sec = ts.tv_sec; 377 res->tv_nsec = ts.tv_nsec; 378 379 return 0; 380 } 381 382 static __maybe_unused int 383 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 384 { 385 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 386 } 387 #endif /* BUILD_VDSO32 */ 388 389 static __maybe_unused int 390 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 391 struct __kernel_old_timeval *tv, struct timezone *tz) 392 { 393 const struct vdso_clock *vc = vd->clock_data; 394 395 if (likely(tv != NULL)) { 396 struct __kernel_timespec ts; 397 398 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 399 return gettimeofday_fallback(tv, tz); 400 401 tv->tv_sec = ts.tv_sec; 402 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 403 } 404 405 if (unlikely(tz != NULL)) { 406 if (IS_ENABLED(CONFIG_TIME_NS) && 407 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 408 vd = __arch_get_vdso_u_timens_data(vd); 409 410 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 411 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 412 } 413 414 return 0; 415 } 416 417 static __maybe_unused int 418 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 419 { 420 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 421 } 422 423 #ifdef VDSO_HAS_TIME 424 static __maybe_unused __kernel_old_time_t 425 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 426 { 427 const struct vdso_clock *vc = vd->clock_data; 428 __kernel_old_time_t t; 429 430 if (IS_ENABLED(CONFIG_TIME_NS) && 431 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 432 vd = __arch_get_vdso_u_timens_data(vd); 433 vc = vd->clock_data; 434 } 435 436 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 437 438 if (time) 439 *time = t; 440 441 return t; 442 } 443 444 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 445 { 446 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 447 } 448 #endif /* VDSO_HAS_TIME */ 449 450 #ifdef VDSO_HAS_CLOCK_GETRES 451 static __maybe_unused 452 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 453 struct __kernel_timespec *res) 454 { 455 const struct vdso_clock *vc = vd->clock_data; 456 u32 msk; 457 u64 ns; 458 459 if (!vdso_clockid_valid(clock)) 460 return false; 461 462 if (IS_ENABLED(CONFIG_TIME_NS) && 463 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 464 vd = __arch_get_vdso_u_timens_data(vd); 465 466 /* 467 * Convert the clockid to a bitmask and use it to check which 468 * clocks are handled in the VDSO directly. 469 */ 470 msk = 1U << clock; 471 if (msk & (VDSO_HRES | VDSO_RAW)) { 472 /* 473 * Preserves the behaviour of posix_get_hrtimer_res(). 474 */ 475 ns = READ_ONCE(vd->hrtimer_res); 476 } else if (msk & VDSO_COARSE) { 477 /* 478 * Preserves the behaviour of posix_get_coarse_res(). 479 */ 480 ns = LOW_RES_NSEC; 481 } else if (msk & VDSO_AUX) { 482 ns = aux_clock_resolution_ns(); 483 } else { 484 return false; 485 } 486 487 if (likely(res)) { 488 res->tv_sec = 0; 489 res->tv_nsec = ns; 490 } 491 return true; 492 } 493 494 static __maybe_unused 495 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 496 struct __kernel_timespec *res) 497 { 498 bool ok; 499 500 ok = __cvdso_clock_getres_common(vd, clock, res); 501 502 if (unlikely(!ok)) 503 return clock_getres_fallback(clock, res); 504 return 0; 505 } 506 507 static __maybe_unused 508 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 509 { 510 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 511 } 512 513 #ifdef BUILD_VDSO32 514 static __maybe_unused int 515 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 516 struct old_timespec32 *res) 517 { 518 struct __kernel_timespec ts; 519 bool ok; 520 521 ok = __cvdso_clock_getres_common(vd, clock, &ts); 522 523 if (unlikely(!ok)) 524 return clock_getres32_fallback(clock, res); 525 526 if (likely(res)) { 527 res->tv_sec = ts.tv_sec; 528 res->tv_nsec = ts.tv_nsec; 529 } 530 return 0; 531 } 532 533 static __maybe_unused int 534 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 535 { 536 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 537 clock, res); 538 } 539 #endif /* BUILD_VDSO32 */ 540 #endif /* VDSO_HAS_CLOCK_GETRES */ 541