1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/auxclock.h> 6 #include <vdso/datapage.h> 7 #include <vdso/helpers.h> 8 9 /* Bring in default accessors */ 10 #include <vdso/vsyscall.h> 11 12 #ifndef vdso_calc_ns 13 14 #ifdef VDSO_DELTA_NOMASK 15 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 16 #else 17 # define VDSO_DELTA_MASK(vd) (vd->mask) 18 #endif 19 20 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 21 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 22 { 23 return delta < vc->max_cycles; 24 } 25 #else 26 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 27 { 28 return true; 29 } 30 #endif 31 32 #ifndef vdso_shift_ns 33 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 34 { 35 return ns >> shift; 36 } 37 #endif 38 39 /* 40 * Default implementation which works for all sane clocksources. That 41 * obviously excludes x86/TSC. 42 */ 43 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 44 { 45 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 46 47 if (likely(vdso_delta_ok(vc, delta))) 48 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 49 50 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 51 } 52 #endif /* vdso_calc_ns */ 53 54 #ifndef __arch_vdso_hres_capable 55 static inline bool __arch_vdso_hres_capable(void) 56 { 57 return true; 58 } 59 #endif 60 61 #ifndef vdso_clocksource_ok 62 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 63 { 64 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 65 } 66 #endif 67 68 #ifndef vdso_cycles_ok 69 static inline bool vdso_cycles_ok(u64 cycles) 70 { 71 return true; 72 } 73 #endif 74 75 static __always_inline bool vdso_clockid_valid(clockid_t clock) 76 { 77 /* Check for negative values or invalid clocks */ 78 return likely((u32) clock <= CLOCK_AUX_LAST); 79 } 80 81 /* 82 * Must not be invoked within the sequence read section as a race inside 83 * that loop could result in __iter_div_u64_rem() being extremely slow. 84 */ 85 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 86 { 87 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 88 ts->tv_nsec = ns; 89 } 90 91 static __always_inline 92 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 93 unsigned int clkidx, u64 *sec, u64 *ns) 94 { 95 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 96 u64 cycles; 97 98 if (unlikely(!vdso_clocksource_ok(vc))) 99 return false; 100 101 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 102 if (unlikely(!vdso_cycles_ok(cycles))) 103 return false; 104 105 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 106 *sec = vdso_ts->sec; 107 108 return true; 109 } 110 111 static __always_inline 112 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 113 { 114 return (void *)vd + PAGE_SIZE; 115 } 116 117 static __always_inline 118 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 119 clockid_t clk, struct __kernel_timespec *ts) 120 { 121 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 122 const struct timens_offset *offs = &vcns->offset[clk]; 123 const struct vdso_clock *vc = vd->clock_data; 124 u32 seq; 125 s64 sec; 126 u64 ns; 127 128 if (clk != CLOCK_MONOTONIC_RAW) 129 vc = &vc[CS_HRES_COARSE]; 130 else 131 vc = &vc[CS_RAW]; 132 133 do { 134 seq = vdso_read_begin(vc); 135 136 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 137 return false; 138 } while (unlikely(vdso_read_retry(vc, seq))); 139 140 /* Add the namespace offset */ 141 sec += offs->sec; 142 ns += offs->nsec; 143 144 vdso_set_timespec(ts, sec, ns); 145 146 return true; 147 } 148 149 static __always_inline 150 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 151 clockid_t clk, struct __kernel_timespec *ts) 152 { 153 u64 sec, ns; 154 u32 seq; 155 156 /* Allows to compile the high resolution parts out */ 157 if (!__arch_vdso_hres_capable()) 158 return false; 159 160 do { 161 /* 162 * Open coded function vdso_read_begin() to handle 163 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 164 * special VVAR page installed which has vc->seq set to 1 and 165 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 166 * namespace affected tasks this does not affect performance 167 * because if vc->seq is odd, i.e. a concurrent update is in 168 * progress the extra check for vc->clock_mode is just a few 169 * extra instructions while spin waiting for vc->seq to become 170 * even again. 171 */ 172 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) { 173 if (IS_ENABLED(CONFIG_TIME_NS) && 174 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 175 return do_hres_timens(vd, vc, clk, ts); 176 cpu_relax(); 177 } 178 smp_rmb(); 179 180 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 181 return false; 182 } while (unlikely(vdso_read_retry(vc, seq))); 183 184 vdso_set_timespec(ts, sec, ns); 185 186 return true; 187 } 188 189 static __always_inline 190 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 191 clockid_t clk, struct __kernel_timespec *ts) 192 { 193 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 194 const struct timens_offset *offs = &vcns->offset[clk]; 195 const struct vdso_clock *vc = vd->clock_data; 196 const struct vdso_timestamp *vdso_ts; 197 u64 nsec; 198 s64 sec; 199 s32 seq; 200 201 vdso_ts = &vc->basetime[clk]; 202 203 do { 204 seq = vdso_read_begin(vc); 205 sec = vdso_ts->sec; 206 nsec = vdso_ts->nsec; 207 } while (unlikely(vdso_read_retry(vc, seq))); 208 209 /* Add the namespace offset */ 210 sec += offs->sec; 211 nsec += offs->nsec; 212 213 vdso_set_timespec(ts, sec, nsec); 214 215 return true; 216 } 217 218 static __always_inline 219 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 220 clockid_t clk, struct __kernel_timespec *ts) 221 { 222 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 223 u32 seq; 224 225 do { 226 /* 227 * Open coded function vdso_read_begin() to handle 228 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 229 */ 230 while ((seq = READ_ONCE(vc->seq)) & 1) { 231 if (IS_ENABLED(CONFIG_TIME_NS) && 232 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 233 return do_coarse_timens(vd, vc, clk, ts); 234 cpu_relax(); 235 } 236 smp_rmb(); 237 238 ts->tv_sec = vdso_ts->sec; 239 ts->tv_nsec = vdso_ts->nsec; 240 } while (unlikely(vdso_read_retry(vc, seq))); 241 242 return true; 243 } 244 245 static __always_inline 246 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) 247 { 248 const struct vdso_clock *vc; 249 u32 seq, idx; 250 u64 sec, ns; 251 252 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 253 return false; 254 255 idx = clock - CLOCK_AUX; 256 vc = &vd->aux_clock_data[idx]; 257 258 do { 259 /* 260 * Open coded function vdso_read_begin() to handle 261 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 262 */ 263 while ((seq = READ_ONCE(vc->seq)) & 1) { 264 if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 265 vd = __arch_get_vdso_u_timens_data(vd); 266 vc = &vd->aux_clock_data[idx]; 267 /* Re-read from the real time data page */ 268 continue; 269 } 270 cpu_relax(); 271 } 272 smp_rmb(); 273 274 /* Auxclock disabled? */ 275 if (vc->clock_mode == VDSO_CLOCKMODE_NONE) 276 return false; 277 278 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) 279 return false; 280 } while (unlikely(vdso_read_retry(vc, seq))); 281 282 vdso_set_timespec(ts, sec, ns); 283 284 return true; 285 } 286 287 static __always_inline bool 288 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 289 struct __kernel_timespec *ts) 290 { 291 const struct vdso_clock *vc = vd->clock_data; 292 u32 msk; 293 294 if (!vdso_clockid_valid(clock)) 295 return false; 296 297 /* 298 * Convert the clockid to a bitmask and use it to check which 299 * clocks are handled in the VDSO directly. 300 */ 301 msk = 1U << clock; 302 if (likely(msk & VDSO_HRES)) 303 vc = &vc[CS_HRES_COARSE]; 304 else if (msk & VDSO_COARSE) 305 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 306 else if (msk & VDSO_RAW) 307 vc = &vc[CS_RAW]; 308 else if (msk & VDSO_AUX) 309 return do_aux(vd, clock, ts); 310 else 311 return false; 312 313 return do_hres(vd, vc, clock, ts); 314 } 315 316 static __maybe_unused int 317 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 318 struct __kernel_timespec *ts) 319 { 320 bool ok; 321 322 ok = __cvdso_clock_gettime_common(vd, clock, ts); 323 324 if (unlikely(!ok)) 325 return clock_gettime_fallback(clock, ts); 326 return 0; 327 } 328 329 static __maybe_unused int 330 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 331 { 332 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 333 } 334 335 #ifdef BUILD_VDSO32 336 static __maybe_unused int 337 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 338 struct old_timespec32 *res) 339 { 340 struct __kernel_timespec ts; 341 bool ok; 342 343 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 344 345 if (unlikely(!ok)) 346 return clock_gettime32_fallback(clock, res); 347 348 /* For ok == true */ 349 res->tv_sec = ts.tv_sec; 350 res->tv_nsec = ts.tv_nsec; 351 352 return 0; 353 } 354 355 static __maybe_unused int 356 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 357 { 358 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 359 } 360 #endif /* BUILD_VDSO32 */ 361 362 static __maybe_unused int 363 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 364 struct __kernel_old_timeval *tv, struct timezone *tz) 365 { 366 const struct vdso_clock *vc = vd->clock_data; 367 368 if (likely(tv != NULL)) { 369 struct __kernel_timespec ts; 370 371 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 372 return gettimeofday_fallback(tv, tz); 373 374 tv->tv_sec = ts.tv_sec; 375 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 376 } 377 378 if (unlikely(tz != NULL)) { 379 if (IS_ENABLED(CONFIG_TIME_NS) && 380 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 381 vd = __arch_get_vdso_u_timens_data(vd); 382 383 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 384 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 385 } 386 387 return 0; 388 } 389 390 static __maybe_unused int 391 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 392 { 393 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 394 } 395 396 #ifdef VDSO_HAS_TIME 397 static __maybe_unused __kernel_old_time_t 398 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 399 { 400 const struct vdso_clock *vc = vd->clock_data; 401 __kernel_old_time_t t; 402 403 if (IS_ENABLED(CONFIG_TIME_NS) && 404 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 405 vd = __arch_get_vdso_u_timens_data(vd); 406 vc = vd->clock_data; 407 } 408 409 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 410 411 if (time) 412 *time = t; 413 414 return t; 415 } 416 417 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 418 { 419 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 420 } 421 #endif /* VDSO_HAS_TIME */ 422 423 #ifdef VDSO_HAS_CLOCK_GETRES 424 static __maybe_unused 425 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 426 struct __kernel_timespec *res) 427 { 428 const struct vdso_clock *vc = vd->clock_data; 429 u32 msk; 430 u64 ns; 431 432 if (!vdso_clockid_valid(clock)) 433 return false; 434 435 if (IS_ENABLED(CONFIG_TIME_NS) && 436 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 437 vd = __arch_get_vdso_u_timens_data(vd); 438 439 /* 440 * Convert the clockid to a bitmask and use it to check which 441 * clocks are handled in the VDSO directly. 442 */ 443 msk = 1U << clock; 444 if (msk & (VDSO_HRES | VDSO_RAW)) { 445 /* 446 * Preserves the behaviour of posix_get_hrtimer_res(). 447 */ 448 ns = READ_ONCE(vd->hrtimer_res); 449 } else if (msk & VDSO_COARSE) { 450 /* 451 * Preserves the behaviour of posix_get_coarse_res(). 452 */ 453 ns = LOW_RES_NSEC; 454 } else if (msk & VDSO_AUX) { 455 ns = aux_clock_resolution_ns(); 456 } else { 457 return false; 458 } 459 460 if (likely(res)) { 461 res->tv_sec = 0; 462 res->tv_nsec = ns; 463 } 464 return true; 465 } 466 467 static __maybe_unused 468 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 469 struct __kernel_timespec *res) 470 { 471 bool ok; 472 473 ok = __cvdso_clock_getres_common(vd, clock, res); 474 475 if (unlikely(!ok)) 476 return clock_getres_fallback(clock, res); 477 return 0; 478 } 479 480 static __maybe_unused 481 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 482 { 483 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 484 } 485 486 #ifdef BUILD_VDSO32 487 static __maybe_unused int 488 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 489 struct old_timespec32 *res) 490 { 491 struct __kernel_timespec ts; 492 bool ok; 493 494 ok = __cvdso_clock_getres_common(vd, clock, &ts); 495 496 if (unlikely(!ok)) 497 return clock_getres32_fallback(clock, res); 498 499 if (likely(res)) { 500 res->tv_sec = ts.tv_sec; 501 res->tv_nsec = ts.tv_nsec; 502 } 503 return 0; 504 } 505 506 static __maybe_unused int 507 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 508 { 509 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 510 clock, res); 511 } 512 #endif /* BUILD_VDSO32 */ 513 #endif /* VDSO_HAS_CLOCK_GETRES */ 514