1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/auxclock.h> 6 #include <vdso/datapage.h> 7 #include <vdso/helpers.h> 8 9 /* Bring in default accessors */ 10 #include <vdso/vsyscall.h> 11 12 #ifndef vdso_calc_ns 13 14 #ifdef VDSO_DELTA_NOMASK 15 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 16 #else 17 # define VDSO_DELTA_MASK(vd) (vd->mask) 18 #endif 19 20 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 21 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 22 { 23 return delta < vc->max_cycles; 24 } 25 #else 26 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 27 { 28 return true; 29 } 30 #endif 31 32 #ifndef vdso_shift_ns 33 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 34 { 35 return ns >> shift; 36 } 37 #endif 38 39 /* 40 * Default implementation which works for all sane clocksources. That 41 * obviously excludes x86/TSC. 42 */ 43 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 44 { 45 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 46 47 if (likely(vdso_delta_ok(vc, delta))) 48 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 49 50 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 51 } 52 #endif /* vdso_calc_ns */ 53 54 #ifndef __arch_vdso_hres_capable 55 static inline bool __arch_vdso_hres_capable(void) 56 { 57 return true; 58 } 59 #endif 60 61 #ifndef vdso_clocksource_ok 62 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 63 { 64 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 65 } 66 #endif 67 68 #ifndef vdso_cycles_ok 69 static inline bool vdso_cycles_ok(u64 cycles) 70 { 71 return true; 72 } 73 #endif 74 75 static __always_inline bool vdso_clockid_valid(clockid_t clock) 76 { 77 /* Check for negative values or invalid clocks */ 78 return likely((u32) clock <= CLOCK_AUX_LAST); 79 } 80 81 /* 82 * Must not be invoked within the sequence read section as a race inside 83 * that loop could result in __iter_div_u64_rem() being extremely slow. 84 */ 85 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 86 { 87 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 88 ts->tv_nsec = ns; 89 } 90 91 static __always_inline 92 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 93 unsigned int clkidx, u64 *sec, u64 *ns) 94 { 95 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 96 u64 cycles; 97 98 if (unlikely(!vdso_clocksource_ok(vc))) 99 return false; 100 101 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 102 if (unlikely(!vdso_cycles_ok(cycles))) 103 return false; 104 105 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 106 *sec = vdso_ts->sec; 107 108 return true; 109 } 110 111 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE 112 static __always_inline 113 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 114 { 115 return (void *)vd + PAGE_SIZE; 116 } 117 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ 118 119 static __always_inline 120 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 121 clockid_t clk, struct __kernel_timespec *ts) 122 { 123 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 124 const struct timens_offset *offs = &vcns->offset[clk]; 125 const struct vdso_clock *vc = vd->clock_data; 126 u32 seq; 127 s64 sec; 128 u64 ns; 129 130 if (clk != CLOCK_MONOTONIC_RAW) 131 vc = &vc[CS_HRES_COARSE]; 132 else 133 vc = &vc[CS_RAW]; 134 135 do { 136 seq = vdso_read_begin(vc); 137 138 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 139 return false; 140 } while (unlikely(vdso_read_retry(vc, seq))); 141 142 /* Add the namespace offset */ 143 sec += offs->sec; 144 ns += offs->nsec; 145 146 vdso_set_timespec(ts, sec, ns); 147 148 return true; 149 } 150 151 static __always_inline 152 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 153 clockid_t clk, struct __kernel_timespec *ts) 154 { 155 u64 sec, ns; 156 u32 seq; 157 158 /* Allows to compile the high resolution parts out */ 159 if (!__arch_vdso_hres_capable()) 160 return false; 161 162 do { 163 /* 164 * Open coded function vdso_read_begin() to handle 165 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 166 * special VVAR page installed which has vc->seq set to 1 and 167 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 168 * namespace affected tasks this does not affect performance 169 * because if vc->seq is odd, i.e. a concurrent update is in 170 * progress the extra check for vc->clock_mode is just a few 171 * extra instructions while spin waiting for vc->seq to become 172 * even again. 173 */ 174 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) { 175 if (IS_ENABLED(CONFIG_TIME_NS) && 176 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 177 return do_hres_timens(vd, vc, clk, ts); 178 cpu_relax(); 179 } 180 smp_rmb(); 181 182 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 183 return false; 184 } while (unlikely(vdso_read_retry(vc, seq))); 185 186 vdso_set_timespec(ts, sec, ns); 187 188 return true; 189 } 190 191 static __always_inline 192 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 193 clockid_t clk, struct __kernel_timespec *ts) 194 { 195 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 196 const struct timens_offset *offs = &vcns->offset[clk]; 197 const struct vdso_clock *vc = vd->clock_data; 198 const struct vdso_timestamp *vdso_ts; 199 u64 nsec; 200 s64 sec; 201 s32 seq; 202 203 vdso_ts = &vc->basetime[clk]; 204 205 do { 206 seq = vdso_read_begin(vc); 207 sec = vdso_ts->sec; 208 nsec = vdso_ts->nsec; 209 } while (unlikely(vdso_read_retry(vc, seq))); 210 211 /* Add the namespace offset */ 212 sec += offs->sec; 213 nsec += offs->nsec; 214 215 vdso_set_timespec(ts, sec, nsec); 216 217 return true; 218 } 219 220 static __always_inline 221 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 222 clockid_t clk, struct __kernel_timespec *ts) 223 { 224 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 225 u32 seq; 226 227 do { 228 /* 229 * Open coded function vdso_read_begin() to handle 230 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 231 */ 232 while ((seq = READ_ONCE(vc->seq)) & 1) { 233 if (IS_ENABLED(CONFIG_TIME_NS) && 234 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 235 return do_coarse_timens(vd, vc, clk, ts); 236 cpu_relax(); 237 } 238 smp_rmb(); 239 240 ts->tv_sec = vdso_ts->sec; 241 ts->tv_nsec = vdso_ts->nsec; 242 } while (unlikely(vdso_read_retry(vc, seq))); 243 244 return true; 245 } 246 247 static __always_inline 248 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) 249 { 250 const struct vdso_clock *vc; 251 u32 seq, idx; 252 u64 sec, ns; 253 254 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 255 return false; 256 257 idx = clock - CLOCK_AUX; 258 vc = &vd->aux_clock_data[idx]; 259 260 do { 261 /* 262 * Open coded function vdso_read_begin() to handle 263 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 264 */ 265 while ((seq = READ_ONCE(vc->seq)) & 1) { 266 if (IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 267 vd = __arch_get_vdso_u_timens_data(vd); 268 vc = &vd->aux_clock_data[idx]; 269 /* Re-read from the real time data page */ 270 continue; 271 } 272 cpu_relax(); 273 } 274 smp_rmb(); 275 276 /* Auxclock disabled? */ 277 if (vc->clock_mode == VDSO_CLOCKMODE_NONE) 278 return false; 279 280 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) 281 return false; 282 } while (unlikely(vdso_read_retry(vc, seq))); 283 284 vdso_set_timespec(ts, sec, ns); 285 286 return true; 287 } 288 289 static __always_inline bool 290 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 291 struct __kernel_timespec *ts) 292 { 293 const struct vdso_clock *vc = vd->clock_data; 294 u32 msk; 295 296 if (!vdso_clockid_valid(clock)) 297 return false; 298 299 /* 300 * Convert the clockid to a bitmask and use it to check which 301 * clocks are handled in the VDSO directly. 302 */ 303 msk = 1U << clock; 304 if (likely(msk & VDSO_HRES)) 305 vc = &vc[CS_HRES_COARSE]; 306 else if (msk & VDSO_COARSE) 307 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 308 else if (msk & VDSO_RAW) 309 vc = &vc[CS_RAW]; 310 else if (msk & VDSO_AUX) 311 return do_aux(vd, clock, ts); 312 else 313 return false; 314 315 return do_hres(vd, vc, clock, ts); 316 } 317 318 static __maybe_unused int 319 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 320 struct __kernel_timespec *ts) 321 { 322 bool ok; 323 324 ok = __cvdso_clock_gettime_common(vd, clock, ts); 325 326 if (unlikely(!ok)) 327 return clock_gettime_fallback(clock, ts); 328 return 0; 329 } 330 331 static __maybe_unused int 332 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 333 { 334 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 335 } 336 337 #ifdef BUILD_VDSO32 338 static __maybe_unused int 339 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 340 struct old_timespec32 *res) 341 { 342 struct __kernel_timespec ts; 343 bool ok; 344 345 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 346 347 if (unlikely(!ok)) 348 return clock_gettime32_fallback(clock, res); 349 350 /* For ok == true */ 351 res->tv_sec = ts.tv_sec; 352 res->tv_nsec = ts.tv_nsec; 353 354 return 0; 355 } 356 357 static __maybe_unused int 358 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 359 { 360 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 361 } 362 #endif /* BUILD_VDSO32 */ 363 364 static __maybe_unused int 365 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 366 struct __kernel_old_timeval *tv, struct timezone *tz) 367 { 368 const struct vdso_clock *vc = vd->clock_data; 369 370 if (likely(tv != NULL)) { 371 struct __kernel_timespec ts; 372 373 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 374 return gettimeofday_fallback(tv, tz); 375 376 tv->tv_sec = ts.tv_sec; 377 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 378 } 379 380 if (unlikely(tz != NULL)) { 381 if (IS_ENABLED(CONFIG_TIME_NS) && 382 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 383 vd = __arch_get_vdso_u_timens_data(vd); 384 385 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 386 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 387 } 388 389 return 0; 390 } 391 392 static __maybe_unused int 393 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 394 { 395 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 396 } 397 398 #ifdef VDSO_HAS_TIME 399 static __maybe_unused __kernel_old_time_t 400 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 401 { 402 const struct vdso_clock *vc = vd->clock_data; 403 __kernel_old_time_t t; 404 405 if (IS_ENABLED(CONFIG_TIME_NS) && 406 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 407 vd = __arch_get_vdso_u_timens_data(vd); 408 vc = vd->clock_data; 409 } 410 411 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 412 413 if (time) 414 *time = t; 415 416 return t; 417 } 418 419 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 420 { 421 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 422 } 423 #endif /* VDSO_HAS_TIME */ 424 425 #ifdef VDSO_HAS_CLOCK_GETRES 426 static __maybe_unused 427 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 428 struct __kernel_timespec *res) 429 { 430 const struct vdso_clock *vc = vd->clock_data; 431 u32 msk; 432 u64 ns; 433 434 if (!vdso_clockid_valid(clock)) 435 return false; 436 437 if (IS_ENABLED(CONFIG_TIME_NS) && 438 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 439 vd = __arch_get_vdso_u_timens_data(vd); 440 441 /* 442 * Convert the clockid to a bitmask and use it to check which 443 * clocks are handled in the VDSO directly. 444 */ 445 msk = 1U << clock; 446 if (msk & (VDSO_HRES | VDSO_RAW)) { 447 /* 448 * Preserves the behaviour of posix_get_hrtimer_res(). 449 */ 450 ns = READ_ONCE(vd->hrtimer_res); 451 } else if (msk & VDSO_COARSE) { 452 /* 453 * Preserves the behaviour of posix_get_coarse_res(). 454 */ 455 ns = LOW_RES_NSEC; 456 } else if (msk & VDSO_AUX) { 457 ns = aux_clock_resolution_ns(); 458 } else { 459 return false; 460 } 461 462 if (likely(res)) { 463 res->tv_sec = 0; 464 res->tv_nsec = ns; 465 } 466 return true; 467 } 468 469 static __maybe_unused 470 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 471 struct __kernel_timespec *res) 472 { 473 bool ok; 474 475 ok = __cvdso_clock_getres_common(vd, clock, res); 476 477 if (unlikely(!ok)) 478 return clock_getres_fallback(clock, res); 479 return 0; 480 } 481 482 static __maybe_unused 483 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 484 { 485 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 486 } 487 488 #ifdef BUILD_VDSO32 489 static __maybe_unused int 490 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 491 struct old_timespec32 *res) 492 { 493 struct __kernel_timespec ts; 494 bool ok; 495 496 ok = __cvdso_clock_getres_common(vd, clock, &ts); 497 498 if (unlikely(!ok)) 499 return clock_getres32_fallback(clock, res); 500 501 if (likely(res)) { 502 res->tv_sec = ts.tv_sec; 503 res->tv_nsec = ts.tv_nsec; 504 } 505 return 0; 506 } 507 508 static __maybe_unused int 509 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 510 { 511 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 512 clock, res); 513 } 514 #endif /* BUILD_VDSO32 */ 515 #endif /* VDSO_HAS_CLOCK_GETRES */ 516