1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/datapage.h> 6 #include <vdso/helpers.h> 7 8 /* Bring in default accessors */ 9 #include <vdso/vsyscall.h> 10 11 #ifndef vdso_calc_ns 12 13 #ifdef VDSO_DELTA_NOMASK 14 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 15 #else 16 # define VDSO_DELTA_MASK(vd) (vd->mask) 17 #endif 18 19 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 20 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 21 { 22 return delta < vc->max_cycles; 23 } 24 #else 25 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 26 { 27 return true; 28 } 29 #endif 30 31 #ifndef vdso_shift_ns 32 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 33 { 34 return ns >> shift; 35 } 36 #endif 37 38 /* 39 * Default implementation which works for all sane clocksources. That 40 * obviously excludes x86/TSC. 41 */ 42 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 43 { 44 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 45 46 if (likely(vdso_delta_ok(vc, delta))) 47 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 48 49 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 50 } 51 #endif /* vdso_calc_ns */ 52 53 #ifndef __arch_vdso_hres_capable 54 static inline bool __arch_vdso_hres_capable(void) 55 { 56 return true; 57 } 58 #endif 59 60 #ifndef vdso_clocksource_ok 61 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 62 { 63 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 64 } 65 #endif 66 67 #ifndef vdso_cycles_ok 68 static inline bool vdso_cycles_ok(u64 cycles) 69 { 70 return true; 71 } 72 #endif 73 74 static __always_inline bool vdso_clockid_valid(clockid_t clock) 75 { 76 /* Check for negative values or invalid clocks */ 77 return likely((u32) clock < MAX_CLOCKS); 78 } 79 80 #ifdef CONFIG_TIME_NS 81 82 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE 83 static __always_inline 84 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 85 { 86 return (void *)vd + PAGE_SIZE; 87 } 88 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ 89 90 static __always_inline 91 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 92 clockid_t clk, struct __kernel_timespec *ts) 93 { 94 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 95 const struct timens_offset *offs = &vcns->offset[clk]; 96 const struct vdso_clock *vc = vd->clock_data; 97 const struct vdso_timestamp *vdso_ts; 98 u64 cycles, ns; 99 u32 seq; 100 s64 sec; 101 102 if (clk != CLOCK_MONOTONIC_RAW) 103 vc = &vc[CS_HRES_COARSE]; 104 else 105 vc = &vc[CS_RAW]; 106 vdso_ts = &vc->basetime[clk]; 107 108 do { 109 seq = vdso_read_begin(vc); 110 111 if (unlikely(!vdso_clocksource_ok(vc))) 112 return false; 113 114 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 115 if (unlikely(!vdso_cycles_ok(cycles))) 116 return false; 117 ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 118 sec = vdso_ts->sec; 119 } while (unlikely(vdso_read_retry(vc, seq))); 120 121 /* Add the namespace offset */ 122 sec += offs->sec; 123 ns += offs->nsec; 124 125 /* 126 * Do this outside the loop: a race inside the loop could result 127 * in __iter_div_u64_rem() being extremely slow. 128 */ 129 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 130 ts->tv_nsec = ns; 131 132 return true; 133 } 134 #else 135 static __always_inline 136 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 137 { 138 return NULL; 139 } 140 141 static __always_inline 142 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 143 clockid_t clk, struct __kernel_timespec *ts) 144 { 145 return false; 146 } 147 #endif 148 149 static __always_inline 150 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 151 clockid_t clk, struct __kernel_timespec *ts) 152 { 153 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 154 u64 cycles, sec, ns; 155 u32 seq; 156 157 /* Allows to compile the high resolution parts out */ 158 if (!__arch_vdso_hres_capable()) 159 return false; 160 161 do { 162 /* 163 * Open coded function vdso_read_begin() to handle 164 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 165 * special VVAR page installed which has vc->seq set to 1 and 166 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 167 * namespace affected tasks this does not affect performance 168 * because if vc->seq is odd, i.e. a concurrent update is in 169 * progress the extra check for vc->clock_mode is just a few 170 * extra instructions while spin waiting for vc->seq to become 171 * even again. 172 */ 173 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) { 174 if (IS_ENABLED(CONFIG_TIME_NS) && 175 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 176 return do_hres_timens(vd, vc, clk, ts); 177 cpu_relax(); 178 } 179 smp_rmb(); 180 181 if (unlikely(!vdso_clocksource_ok(vc))) 182 return false; 183 184 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 185 if (unlikely(!vdso_cycles_ok(cycles))) 186 return false; 187 ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 188 sec = vdso_ts->sec; 189 } while (unlikely(vdso_read_retry(vc, seq))); 190 191 /* 192 * Do this outside the loop: a race inside the loop could result 193 * in __iter_div_u64_rem() being extremely slow. 194 */ 195 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 196 ts->tv_nsec = ns; 197 198 return true; 199 } 200 201 #ifdef CONFIG_TIME_NS 202 static __always_inline 203 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 204 clockid_t clk, struct __kernel_timespec *ts) 205 { 206 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 207 const struct timens_offset *offs = &vcns->offset[clk]; 208 const struct vdso_clock *vc = vd->clock_data; 209 const struct vdso_timestamp *vdso_ts; 210 u64 nsec; 211 s64 sec; 212 s32 seq; 213 214 vdso_ts = &vc->basetime[clk]; 215 216 do { 217 seq = vdso_read_begin(vc); 218 sec = vdso_ts->sec; 219 nsec = vdso_ts->nsec; 220 } while (unlikely(vdso_read_retry(vc, seq))); 221 222 /* Add the namespace offset */ 223 sec += offs->sec; 224 nsec += offs->nsec; 225 226 /* 227 * Do this outside the loop: a race inside the loop could result 228 * in __iter_div_u64_rem() being extremely slow. 229 */ 230 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 231 ts->tv_nsec = nsec; 232 return true; 233 } 234 #else 235 static __always_inline 236 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 237 clockid_t clk, struct __kernel_timespec *ts) 238 { 239 return false; 240 } 241 #endif 242 243 static __always_inline 244 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 245 clockid_t clk, struct __kernel_timespec *ts) 246 { 247 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 248 u32 seq; 249 250 do { 251 /* 252 * Open coded function vdso_read_begin() to handle 253 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 254 */ 255 while ((seq = READ_ONCE(vc->seq)) & 1) { 256 if (IS_ENABLED(CONFIG_TIME_NS) && 257 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 258 return do_coarse_timens(vd, vc, clk, ts); 259 cpu_relax(); 260 } 261 smp_rmb(); 262 263 ts->tv_sec = vdso_ts->sec; 264 ts->tv_nsec = vdso_ts->nsec; 265 } while (unlikely(vdso_read_retry(vc, seq))); 266 267 return true; 268 } 269 270 static __always_inline bool 271 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 272 struct __kernel_timespec *ts) 273 { 274 const struct vdso_clock *vc = vd->clock_data; 275 u32 msk; 276 277 if (!vdso_clockid_valid(clock)) 278 return false; 279 280 /* 281 * Convert the clockid to a bitmask and use it to check which 282 * clocks are handled in the VDSO directly. 283 */ 284 msk = 1U << clock; 285 if (likely(msk & VDSO_HRES)) 286 vc = &vc[CS_HRES_COARSE]; 287 else if (msk & VDSO_COARSE) 288 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 289 else if (msk & VDSO_RAW) 290 vc = &vc[CS_RAW]; 291 else 292 return false; 293 294 return do_hres(vd, vc, clock, ts); 295 } 296 297 static __maybe_unused int 298 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 299 struct __kernel_timespec *ts) 300 { 301 bool ok; 302 303 ok = __cvdso_clock_gettime_common(vd, clock, ts); 304 305 if (unlikely(!ok)) 306 return clock_gettime_fallback(clock, ts); 307 return 0; 308 } 309 310 static __maybe_unused int 311 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 312 { 313 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 314 } 315 316 #ifdef BUILD_VDSO32 317 static __maybe_unused int 318 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 319 struct old_timespec32 *res) 320 { 321 struct __kernel_timespec ts; 322 bool ok; 323 324 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 325 326 if (unlikely(!ok)) 327 return clock_gettime32_fallback(clock, res); 328 329 /* For ok == true */ 330 res->tv_sec = ts.tv_sec; 331 res->tv_nsec = ts.tv_nsec; 332 333 return 0; 334 } 335 336 static __maybe_unused int 337 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 338 { 339 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 340 } 341 #endif /* BUILD_VDSO32 */ 342 343 static __maybe_unused int 344 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 345 struct __kernel_old_timeval *tv, struct timezone *tz) 346 { 347 const struct vdso_clock *vc = vd->clock_data; 348 349 if (likely(tv != NULL)) { 350 struct __kernel_timespec ts; 351 352 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 353 return gettimeofday_fallback(tv, tz); 354 355 tv->tv_sec = ts.tv_sec; 356 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 357 } 358 359 if (unlikely(tz != NULL)) { 360 if (IS_ENABLED(CONFIG_TIME_NS) && 361 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 362 vd = __arch_get_vdso_u_timens_data(vd); 363 364 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 365 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 366 } 367 368 return 0; 369 } 370 371 static __maybe_unused int 372 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 373 { 374 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 375 } 376 377 #ifdef VDSO_HAS_TIME 378 static __maybe_unused __kernel_old_time_t 379 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 380 { 381 const struct vdso_clock *vc = vd->clock_data; 382 __kernel_old_time_t t; 383 384 if (IS_ENABLED(CONFIG_TIME_NS) && 385 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 386 vd = __arch_get_vdso_u_timens_data(vd); 387 vc = vd->clock_data; 388 } 389 390 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 391 392 if (time) 393 *time = t; 394 395 return t; 396 } 397 398 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 399 { 400 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 401 } 402 #endif /* VDSO_HAS_TIME */ 403 404 #ifdef VDSO_HAS_CLOCK_GETRES 405 static __maybe_unused 406 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 407 struct __kernel_timespec *res) 408 { 409 const struct vdso_clock *vc = vd->clock_data; 410 u32 msk; 411 u64 ns; 412 413 if (!vdso_clockid_valid(clock)) 414 return false; 415 416 if (IS_ENABLED(CONFIG_TIME_NS) && 417 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 418 vd = __arch_get_vdso_u_timens_data(vd); 419 420 /* 421 * Convert the clockid to a bitmask and use it to check which 422 * clocks are handled in the VDSO directly. 423 */ 424 msk = 1U << clock; 425 if (msk & (VDSO_HRES | VDSO_RAW)) { 426 /* 427 * Preserves the behaviour of posix_get_hrtimer_res(). 428 */ 429 ns = READ_ONCE(vd->hrtimer_res); 430 } else if (msk & VDSO_COARSE) { 431 /* 432 * Preserves the behaviour of posix_get_coarse_res(). 433 */ 434 ns = LOW_RES_NSEC; 435 } else { 436 return false; 437 } 438 439 if (likely(res)) { 440 res->tv_sec = 0; 441 res->tv_nsec = ns; 442 } 443 return true; 444 } 445 446 static __maybe_unused 447 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 448 struct __kernel_timespec *res) 449 { 450 bool ok; 451 452 ok = __cvdso_clock_getres_common(vd, clock, res); 453 454 if (unlikely(!ok)) 455 return clock_getres_fallback(clock, res); 456 return 0; 457 } 458 459 static __maybe_unused 460 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 461 { 462 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 463 } 464 465 #ifdef BUILD_VDSO32 466 static __maybe_unused int 467 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 468 struct old_timespec32 *res) 469 { 470 struct __kernel_timespec ts; 471 bool ok; 472 473 ok = __cvdso_clock_getres_common(vd, clock, &ts); 474 475 if (unlikely(!ok)) 476 return clock_getres32_fallback(clock, res); 477 478 if (likely(res)) { 479 res->tv_sec = ts.tv_sec; 480 res->tv_nsec = ts.tv_nsec; 481 } 482 return 0; 483 } 484 485 static __maybe_unused int 486 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 487 { 488 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 489 clock, res); 490 } 491 #endif /* BUILD_VDSO32 */ 492 #endif /* VDSO_HAS_CLOCK_GETRES */ 493