1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/datapage.h> 6 #include <vdso/helpers.h> 7 8 /* Bring in default accessors */ 9 #include <vdso/vsyscall.h> 10 11 #ifndef vdso_calc_ns 12 13 #ifdef VDSO_DELTA_NOMASK 14 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 15 #else 16 # define VDSO_DELTA_MASK(vd) (vd->mask) 17 #endif 18 19 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 20 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 21 { 22 return delta < vc->max_cycles; 23 } 24 #else 25 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 26 { 27 return true; 28 } 29 #endif 30 31 #ifndef vdso_shift_ns 32 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 33 { 34 return ns >> shift; 35 } 36 #endif 37 38 /* 39 * Default implementation which works for all sane clocksources. That 40 * obviously excludes x86/TSC. 41 */ 42 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 43 { 44 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 45 46 if (likely(vdso_delta_ok(vc, delta))) 47 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 48 49 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 50 } 51 #endif /* vdso_calc_ns */ 52 53 #ifndef __arch_vdso_hres_capable 54 static inline bool __arch_vdso_hres_capable(void) 55 { 56 return true; 57 } 58 #endif 59 60 #ifndef vdso_clocksource_ok 61 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 62 { 63 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 64 } 65 #endif 66 67 #ifndef vdso_cycles_ok 68 static inline bool vdso_cycles_ok(u64 cycles) 69 { 70 return true; 71 } 72 #endif 73 74 static __always_inline bool vdso_clockid_valid(clockid_t clock) 75 { 76 /* Check for negative values or invalid clocks */ 77 return likely((u32) clock < MAX_CLOCKS); 78 } 79 80 /* 81 * Must not be invoked within the sequence read section as a race inside 82 * that loop could result in __iter_div_u64_rem() being extremely slow. 83 */ 84 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 85 { 86 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 87 ts->tv_nsec = ns; 88 } 89 90 static __always_inline 91 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 92 unsigned int clkidx, u64 *sec, u64 *ns) 93 { 94 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 95 u64 cycles; 96 97 if (unlikely(!vdso_clocksource_ok(vc))) 98 return false; 99 100 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 101 if (unlikely(!vdso_cycles_ok(cycles))) 102 return false; 103 104 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 105 *sec = vdso_ts->sec; 106 107 return true; 108 } 109 110 #ifdef CONFIG_TIME_NS 111 112 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE 113 static __always_inline 114 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 115 { 116 return (void *)vd + PAGE_SIZE; 117 } 118 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ 119 120 static __always_inline 121 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 122 clockid_t clk, struct __kernel_timespec *ts) 123 { 124 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 125 const struct timens_offset *offs = &vcns->offset[clk]; 126 const struct vdso_clock *vc = vd->clock_data; 127 u32 seq; 128 s64 sec; 129 u64 ns; 130 131 if (clk != CLOCK_MONOTONIC_RAW) 132 vc = &vc[CS_HRES_COARSE]; 133 else 134 vc = &vc[CS_RAW]; 135 136 do { 137 seq = vdso_read_begin(vc); 138 139 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 140 return false; 141 } while (unlikely(vdso_read_retry(vc, seq))); 142 143 /* Add the namespace offset */ 144 sec += offs->sec; 145 ns += offs->nsec; 146 147 vdso_set_timespec(ts, sec, ns); 148 149 return true; 150 } 151 #else 152 static __always_inline 153 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 154 { 155 return NULL; 156 } 157 158 static __always_inline 159 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 160 clockid_t clk, struct __kernel_timespec *ts) 161 { 162 return false; 163 } 164 #endif 165 166 static __always_inline 167 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 168 clockid_t clk, struct __kernel_timespec *ts) 169 { 170 u64 sec, ns; 171 u32 seq; 172 173 /* Allows to compile the high resolution parts out */ 174 if (!__arch_vdso_hres_capable()) 175 return false; 176 177 do { 178 /* 179 * Open coded function vdso_read_begin() to handle 180 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 181 * special VVAR page installed which has vc->seq set to 1 and 182 * vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 183 * namespace affected tasks this does not affect performance 184 * because if vc->seq is odd, i.e. a concurrent update is in 185 * progress the extra check for vc->clock_mode is just a few 186 * extra instructions while spin waiting for vc->seq to become 187 * even again. 188 */ 189 while (unlikely((seq = READ_ONCE(vc->seq)) & 1)) { 190 if (IS_ENABLED(CONFIG_TIME_NS) && 191 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 192 return do_hres_timens(vd, vc, clk, ts); 193 cpu_relax(); 194 } 195 smp_rmb(); 196 197 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 198 return false; 199 } while (unlikely(vdso_read_retry(vc, seq))); 200 201 vdso_set_timespec(ts, sec, ns); 202 203 return true; 204 } 205 206 #ifdef CONFIG_TIME_NS 207 static __always_inline 208 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 209 clockid_t clk, struct __kernel_timespec *ts) 210 { 211 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 212 const struct timens_offset *offs = &vcns->offset[clk]; 213 const struct vdso_clock *vc = vd->clock_data; 214 const struct vdso_timestamp *vdso_ts; 215 u64 nsec; 216 s64 sec; 217 s32 seq; 218 219 vdso_ts = &vc->basetime[clk]; 220 221 do { 222 seq = vdso_read_begin(vc); 223 sec = vdso_ts->sec; 224 nsec = vdso_ts->nsec; 225 } while (unlikely(vdso_read_retry(vc, seq))); 226 227 /* Add the namespace offset */ 228 sec += offs->sec; 229 nsec += offs->nsec; 230 231 vdso_set_timespec(ts, sec, nsec); 232 233 return true; 234 } 235 #else 236 static __always_inline 237 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 238 clockid_t clk, struct __kernel_timespec *ts) 239 { 240 return false; 241 } 242 #endif 243 244 static __always_inline 245 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 246 clockid_t clk, struct __kernel_timespec *ts) 247 { 248 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 249 u32 seq; 250 251 do { 252 /* 253 * Open coded function vdso_read_begin() to handle 254 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 255 */ 256 while ((seq = READ_ONCE(vc->seq)) & 1) { 257 if (IS_ENABLED(CONFIG_TIME_NS) && 258 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 259 return do_coarse_timens(vd, vc, clk, ts); 260 cpu_relax(); 261 } 262 smp_rmb(); 263 264 ts->tv_sec = vdso_ts->sec; 265 ts->tv_nsec = vdso_ts->nsec; 266 } while (unlikely(vdso_read_retry(vc, seq))); 267 268 return true; 269 } 270 271 static __always_inline bool 272 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 273 struct __kernel_timespec *ts) 274 { 275 const struct vdso_clock *vc = vd->clock_data; 276 u32 msk; 277 278 if (!vdso_clockid_valid(clock)) 279 return false; 280 281 /* 282 * Convert the clockid to a bitmask and use it to check which 283 * clocks are handled in the VDSO directly. 284 */ 285 msk = 1U << clock; 286 if (likely(msk & VDSO_HRES)) 287 vc = &vc[CS_HRES_COARSE]; 288 else if (msk & VDSO_COARSE) 289 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 290 else if (msk & VDSO_RAW) 291 vc = &vc[CS_RAW]; 292 else 293 return false; 294 295 return do_hres(vd, vc, clock, ts); 296 } 297 298 static __maybe_unused int 299 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 300 struct __kernel_timespec *ts) 301 { 302 bool ok; 303 304 ok = __cvdso_clock_gettime_common(vd, clock, ts); 305 306 if (unlikely(!ok)) 307 return clock_gettime_fallback(clock, ts); 308 return 0; 309 } 310 311 static __maybe_unused int 312 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 313 { 314 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 315 } 316 317 #ifdef BUILD_VDSO32 318 static __maybe_unused int 319 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 320 struct old_timespec32 *res) 321 { 322 struct __kernel_timespec ts; 323 bool ok; 324 325 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 326 327 if (unlikely(!ok)) 328 return clock_gettime32_fallback(clock, res); 329 330 /* For ok == true */ 331 res->tv_sec = ts.tv_sec; 332 res->tv_nsec = ts.tv_nsec; 333 334 return 0; 335 } 336 337 static __maybe_unused int 338 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 339 { 340 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 341 } 342 #endif /* BUILD_VDSO32 */ 343 344 static __maybe_unused int 345 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 346 struct __kernel_old_timeval *tv, struct timezone *tz) 347 { 348 const struct vdso_clock *vc = vd->clock_data; 349 350 if (likely(tv != NULL)) { 351 struct __kernel_timespec ts; 352 353 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 354 return gettimeofday_fallback(tv, tz); 355 356 tv->tv_sec = ts.tv_sec; 357 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 358 } 359 360 if (unlikely(tz != NULL)) { 361 if (IS_ENABLED(CONFIG_TIME_NS) && 362 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 363 vd = __arch_get_vdso_u_timens_data(vd); 364 365 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 366 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 367 } 368 369 return 0; 370 } 371 372 static __maybe_unused int 373 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 374 { 375 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 376 } 377 378 #ifdef VDSO_HAS_TIME 379 static __maybe_unused __kernel_old_time_t 380 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 381 { 382 const struct vdso_clock *vc = vd->clock_data; 383 __kernel_old_time_t t; 384 385 if (IS_ENABLED(CONFIG_TIME_NS) && 386 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) { 387 vd = __arch_get_vdso_u_timens_data(vd); 388 vc = vd->clock_data; 389 } 390 391 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 392 393 if (time) 394 *time = t; 395 396 return t; 397 } 398 399 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 400 { 401 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 402 } 403 #endif /* VDSO_HAS_TIME */ 404 405 #ifdef VDSO_HAS_CLOCK_GETRES 406 static __maybe_unused 407 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 408 struct __kernel_timespec *res) 409 { 410 const struct vdso_clock *vc = vd->clock_data; 411 u32 msk; 412 u64 ns; 413 414 if (!vdso_clockid_valid(clock)) 415 return false; 416 417 if (IS_ENABLED(CONFIG_TIME_NS) && 418 vc->clock_mode == VDSO_CLOCKMODE_TIMENS) 419 vd = __arch_get_vdso_u_timens_data(vd); 420 421 /* 422 * Convert the clockid to a bitmask and use it to check which 423 * clocks are handled in the VDSO directly. 424 */ 425 msk = 1U << clock; 426 if (msk & (VDSO_HRES | VDSO_RAW)) { 427 /* 428 * Preserves the behaviour of posix_get_hrtimer_res(). 429 */ 430 ns = READ_ONCE(vd->hrtimer_res); 431 } else if (msk & VDSO_COARSE) { 432 /* 433 * Preserves the behaviour of posix_get_coarse_res(). 434 */ 435 ns = LOW_RES_NSEC; 436 } else { 437 return false; 438 } 439 440 if (likely(res)) { 441 res->tv_sec = 0; 442 res->tv_nsec = ns; 443 } 444 return true; 445 } 446 447 static __maybe_unused 448 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 449 struct __kernel_timespec *res) 450 { 451 bool ok; 452 453 ok = __cvdso_clock_getres_common(vd, clock, res); 454 455 if (unlikely(!ok)) 456 return clock_getres_fallback(clock, res); 457 return 0; 458 } 459 460 static __maybe_unused 461 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 462 { 463 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 464 } 465 466 #ifdef BUILD_VDSO32 467 static __maybe_unused int 468 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 469 struct old_timespec32 *res) 470 { 471 struct __kernel_timespec ts; 472 bool ok; 473 474 ok = __cvdso_clock_getres_common(vd, clock, &ts); 475 476 if (unlikely(!ok)) 477 return clock_getres32_fallback(clock, res); 478 479 if (likely(res)) { 480 res->tv_sec = ts.tv_sec; 481 res->tv_nsec = ts.tv_nsec; 482 } 483 return 0; 484 } 485 486 static __maybe_unused int 487 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 488 { 489 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 490 clock, res); 491 } 492 #endif /* BUILD_VDSO32 */ 493 #endif /* VDSO_HAS_CLOCK_GETRES */ 494