1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/auxclock.h> 6 #include <vdso/clocksource.h> 7 #include <vdso/datapage.h> 8 #include <vdso/helpers.h> 9 #include <vdso/ktime.h> 10 #include <vdso/limits.h> 11 #include <vdso/math64.h> 12 #include <vdso/time32.h> 13 #include <vdso/time64.h> 14 15 #include <uapi/linux/unistd.h> 16 17 /* 18 * The generic vDSO implementation requires that gettimeofday.h 19 * provides: 20 * - __arch_get_hw_counter(): to get the hw counter based on the 21 * clock_mode. 22 * - gettimeofday_fallback(): fallback for gettimeofday. 23 * - clock_gettime_fallback(): fallback for clock_gettime. 24 * - clock_getres_fallback(): fallback for clock_getres. 25 */ 26 #include <asm/vdso/gettimeofday.h> 27 28 #include <linux/build_bug.h> 29 30 #ifndef vdso_calc_ns 31 32 #ifdef VDSO_DELTA_NOMASK 33 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 34 #else 35 # define VDSO_DELTA_MASK(vd) (vd->mask) 36 #endif 37 38 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 39 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 40 { 41 return delta < vc->max_cycles; 42 } 43 #else 44 static __always_inline bool vdso_delta_ok(const struct vdso_clock *vc, u64 delta) 45 { 46 return true; 47 } 48 #endif 49 50 #ifndef vdso_shift_ns 51 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 52 { 53 return ns >> shift; 54 } 55 #endif 56 57 /* 58 * Default implementation which works for all sane clocksources. That 59 * obviously excludes x86/TSC. 60 */ 61 static __always_inline u64 vdso_calc_ns(const struct vdso_clock *vc, u64 cycles, u64 base) 62 { 63 u64 delta = (cycles - vc->cycle_last) & VDSO_DELTA_MASK(vc); 64 65 if (likely(vdso_delta_ok(vc, delta))) 66 return vdso_shift_ns((delta * vc->mult) + base, vc->shift); 67 68 return mul_u64_u32_add_u64_shr(delta, vc->mult, base, vc->shift); 69 } 70 #endif /* vdso_calc_ns */ 71 72 #ifndef __arch_vdso_hres_capable 73 static inline bool __arch_vdso_hres_capable(void) 74 { 75 return true; 76 } 77 #endif 78 79 #ifndef vdso_clocksource_ok 80 static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) 81 { 82 return vc->clock_mode != VDSO_CLOCKMODE_NONE; 83 } 84 #endif 85 86 #ifndef vdso_cycles_ok 87 static inline bool vdso_cycles_ok(u64 cycles) 88 { 89 return true; 90 } 91 #endif 92 93 #ifndef __arch_get_vdso_u_time_data 94 static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) 95 { 96 return &vdso_u_time_data; 97 } 98 #endif 99 100 static __always_inline bool vdso_clockid_valid(clockid_t clock) 101 { 102 /* Check for negative values or invalid clocks */ 103 return likely((u32) clock <= CLOCK_AUX_LAST); 104 } 105 106 /* 107 * Must not be invoked within the sequence read section as a race inside 108 * that loop could result in __iter_div_u64_rem() being extremely slow. 109 */ 110 static __always_inline void vdso_set_timespec(struct __kernel_timespec *ts, u64 sec, u64 ns) 111 { 112 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 113 ts->tv_nsec = ns; 114 } 115 116 static __always_inline 117 bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock *vc, 118 unsigned int clkidx, u64 *sec, u64 *ns) 119 { 120 const struct vdso_timestamp *vdso_ts = &vc->basetime[clkidx]; 121 u64 cycles; 122 123 if (unlikely(!vdso_clocksource_ok(vc))) 124 return false; 125 126 cycles = __arch_get_hw_counter(vc->clock_mode, vd); 127 if (unlikely(!vdso_cycles_ok(cycles))) 128 return false; 129 130 *ns = vdso_calc_ns(vc, cycles, vdso_ts->nsec); 131 *sec = vdso_ts->sec; 132 133 return true; 134 } 135 136 static __always_inline 137 const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd) 138 { 139 return (void *)vd + PAGE_SIZE; 140 } 141 142 static __always_inline 143 const struct vdso_clock *vdso_timens_clock(const struct vdso_clock *vc) 144 { 145 return (void *)vc + PAGE_SIZE; 146 } 147 148 static __always_inline 149 bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 150 clockid_t clk, struct __kernel_timespec *ts) 151 { 152 const struct vdso_time_data *vd = vdso_timens_data(vdns); 153 const struct vdso_clock *vc = vdso_timens_clock(vcns); 154 const struct timens_offset *offs = &vcns->offset[clk]; 155 u32 seq; 156 s64 sec; 157 u64 ns; 158 159 do { 160 seq = vdso_read_begin(vc); 161 162 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 163 return false; 164 } while (vdso_read_retry(vc, seq)); 165 166 /* Add the namespace offset */ 167 sec += offs->sec; 168 ns += offs->nsec; 169 170 vdso_set_timespec(ts, sec, ns); 171 172 return true; 173 } 174 175 static __always_inline 176 bool do_hres(const struct vdso_time_data *vd, const struct vdso_clock *vc, 177 clockid_t clk, struct __kernel_timespec *ts) 178 { 179 u64 sec, ns; 180 u32 seq; 181 182 /* Allows to compile the high resolution parts out */ 183 if (!__arch_vdso_hres_capable()) 184 return false; 185 186 do { 187 if (vdso_read_begin_timens(vc, &seq)) 188 return do_hres_timens(vd, vc, clk, ts); 189 190 if (!vdso_get_timestamp(vd, vc, clk, &sec, &ns)) 191 return false; 192 } while (vdso_read_retry(vc, seq)); 193 194 vdso_set_timespec(ts, sec, ns); 195 196 return true; 197 } 198 199 static __always_inline 200 bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, 201 clockid_t clk, struct __kernel_timespec *ts) 202 { 203 const struct vdso_time_data *vd = vdso_timens_data(vdns); 204 const struct timens_offset *offs = &vcns->offset[clk]; 205 const struct vdso_clock *vc = vd->clock_data; 206 const struct vdso_timestamp *vdso_ts; 207 u64 nsec; 208 s64 sec; 209 s32 seq; 210 211 vdso_ts = &vc->basetime[clk]; 212 213 do { 214 seq = vdso_read_begin(vc); 215 sec = vdso_ts->sec; 216 nsec = vdso_ts->nsec; 217 } while (vdso_read_retry(vc, seq)); 218 219 /* Add the namespace offset */ 220 sec += offs->sec; 221 nsec += offs->nsec; 222 223 vdso_set_timespec(ts, sec, nsec); 224 225 return true; 226 } 227 228 static __always_inline 229 bool do_coarse(const struct vdso_time_data *vd, const struct vdso_clock *vc, 230 clockid_t clk, struct __kernel_timespec *ts) 231 { 232 const struct vdso_timestamp *vdso_ts = &vc->basetime[clk]; 233 u32 seq; 234 235 do { 236 if (vdso_read_begin_timens(vc, &seq)) 237 return do_coarse_timens(vd, vc, clk, ts); 238 239 ts->tv_sec = vdso_ts->sec; 240 ts->tv_nsec = vdso_ts->nsec; 241 } while (vdso_read_retry(vc, seq)); 242 243 return true; 244 } 245 246 static __always_inline 247 bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_timespec *ts) 248 { 249 const struct vdso_clock *vc; 250 u32 seq, idx; 251 u64 sec, ns; 252 253 if (!IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) 254 return false; 255 256 idx = clock - CLOCK_AUX; 257 vc = &vd->aux_clock_data[idx]; 258 259 do { 260 while (vdso_read_begin_timens(vc, &seq)) { 261 /* Re-read from the real time data page, reload seq by looping */ 262 vd = vdso_timens_data(vd); 263 vc = &vd->aux_clock_data[idx]; 264 } 265 266 /* Auxclock disabled? */ 267 if (vc->clock_mode == VDSO_CLOCKMODE_NONE) 268 return false; 269 270 if (!vdso_get_timestamp(vd, vc, VDSO_BASE_AUX, &sec, &ns)) 271 return false; 272 } while (vdso_read_retry(vc, seq)); 273 274 vdso_set_timespec(ts, sec, ns); 275 276 return true; 277 } 278 279 static __always_inline bool 280 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 281 struct __kernel_timespec *ts) 282 { 283 const struct vdso_clock *vc = vd->clock_data; 284 u32 msk; 285 286 if (!vdso_clockid_valid(clock)) 287 return false; 288 289 /* 290 * Convert the clockid to a bitmask and use it to check which 291 * clocks are handled in the VDSO directly. 292 */ 293 msk = 1U << clock; 294 if (likely(msk & VDSO_HRES)) 295 vc = &vc[CS_HRES_COARSE]; 296 else if (msk & VDSO_COARSE) 297 return do_coarse(vd, &vc[CS_HRES_COARSE], clock, ts); 298 else if (msk & VDSO_RAW) 299 vc = &vc[CS_RAW]; 300 else if (msk & VDSO_AUX) 301 return do_aux(vd, clock, ts); 302 else 303 return false; 304 305 return do_hres(vd, vc, clock, ts); 306 } 307 308 static int 309 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 310 struct __kernel_timespec *ts) 311 { 312 bool ok; 313 314 ok = __cvdso_clock_gettime_common(vd, clock, ts); 315 316 if (unlikely(!ok)) 317 return clock_gettime_fallback(clock, ts); 318 return 0; 319 } 320 321 static __maybe_unused int 322 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 323 { 324 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 325 } 326 327 #ifdef BUILD_VDSO32 328 static int 329 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 330 struct old_timespec32 *res) 331 { 332 struct __kernel_timespec ts; 333 bool ok; 334 335 BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 336 337 ok = __cvdso_clock_gettime_common(vd, clock, &ts); 338 339 if (unlikely(!ok)) 340 return clock_gettime32_fallback(clock, res); 341 342 /* For ok == true */ 343 res->tv_sec = ts.tv_sec; 344 res->tv_nsec = ts.tv_nsec; 345 346 return 0; 347 } 348 349 static __maybe_unused int 350 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 351 { 352 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 353 } 354 #endif /* BUILD_VDSO32 */ 355 356 static int 357 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 358 struct __kernel_old_timeval *tv, struct timezone *tz) 359 { 360 const struct vdso_clock *vc = vd->clock_data; 361 362 #ifndef __NR_gettimeofday 363 BUILD_BUG(); 364 #endif 365 366 BUILD_BUG_ON(sizeof(tv->tv_sec) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 367 368 if (likely(tv != NULL)) { 369 struct __kernel_timespec ts; 370 371 if (!do_hres(vd, &vc[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 372 return gettimeofday_fallback(tv, tz); 373 374 tv->tv_sec = ts.tv_sec; 375 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 376 } 377 378 if (unlikely(tz != NULL)) { 379 if (vdso_is_timens_clock(vc)) 380 vd = vdso_timens_data(vd); 381 382 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 383 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 384 } 385 386 return 0; 387 } 388 389 static __maybe_unused int 390 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 391 { 392 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 393 } 394 395 #ifdef VDSO_HAS_TIME 396 static __kernel_old_time_t 397 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 398 { 399 const struct vdso_clock *vc = vd->clock_data; 400 __kernel_old_time_t t; 401 402 #ifndef __NR_time 403 BUILD_BUG(); 404 #endif 405 406 BUILD_BUG_ON(sizeof(*time) != 8 && !IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 407 408 if (vdso_is_timens_clock(vc)) { 409 vd = vdso_timens_data(vd); 410 vc = vd->clock_data; 411 } 412 413 t = READ_ONCE(vc[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 414 415 if (time) 416 *time = t; 417 418 return t; 419 } 420 421 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 422 { 423 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 424 } 425 #endif /* VDSO_HAS_TIME */ 426 427 #ifdef VDSO_HAS_CLOCK_GETRES 428 static __always_inline 429 bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 430 struct __kernel_timespec *res) 431 { 432 const struct vdso_clock *vc = vd->clock_data; 433 u32 msk; 434 u64 ns; 435 436 if (!vdso_clockid_valid(clock)) 437 return false; 438 439 if (vdso_is_timens_clock(vc)) 440 vd = vdso_timens_data(vd); 441 442 /* 443 * Convert the clockid to a bitmask and use it to check which 444 * clocks are handled in the VDSO directly. 445 */ 446 msk = 1U << clock; 447 if (msk & (VDSO_HRES | VDSO_RAW)) { 448 /* 449 * Preserves the behaviour of posix_get_hrtimer_res(). 450 */ 451 ns = READ_ONCE(vd->hrtimer_res); 452 } else if (msk & VDSO_COARSE) { 453 /* 454 * Preserves the behaviour of posix_get_coarse_res(). 455 */ 456 ns = LOW_RES_NSEC; 457 } else if (msk & VDSO_AUX) { 458 ns = aux_clock_resolution_ns(); 459 } else { 460 return false; 461 } 462 463 if (likely(res)) { 464 res->tv_sec = 0; 465 res->tv_nsec = ns; 466 } 467 return true; 468 } 469 470 static 471 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 472 struct __kernel_timespec *res) 473 { 474 bool ok; 475 476 ok = __cvdso_clock_getres_common(vd, clock, res); 477 478 if (unlikely(!ok)) 479 return clock_getres_fallback(clock, res); 480 return 0; 481 } 482 483 static __maybe_unused 484 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 485 { 486 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 487 } 488 489 #ifdef BUILD_VDSO32 490 static int 491 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 492 struct old_timespec32 *res) 493 { 494 struct __kernel_timespec ts; 495 bool ok; 496 497 BUILD_BUG_ON(!IS_ENABLED(CONFIG_COMPAT_32BIT_TIME)); 498 499 ok = __cvdso_clock_getres_common(vd, clock, &ts); 500 501 if (unlikely(!ok)) 502 return clock_getres32_fallback(clock, res); 503 504 if (likely(res)) { 505 res->tv_sec = ts.tv_sec; 506 res->tv_nsec = ts.tv_nsec; 507 } 508 return 0; 509 } 510 511 static __maybe_unused int 512 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 513 { 514 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 515 clock, res); 516 } 517 #endif /* BUILD_VDSO32 */ 518 #endif /* VDSO_HAS_CLOCK_GETRES */ 519