1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/datapage.h> 6 #include <vdso/helpers.h> 7 8 #ifndef vdso_calc_ns 9 10 #ifdef VDSO_DELTA_NOMASK 11 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 12 #else 13 # define VDSO_DELTA_MASK(vd) (vd->mask) 14 #endif 15 16 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 17 static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) 18 { 19 return delta < vd->max_cycles; 20 } 21 #else 22 static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) 23 { 24 return true; 25 } 26 #endif 27 28 #ifndef vdso_shift_ns 29 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 30 { 31 return ns >> shift; 32 } 33 #endif 34 35 /* 36 * Default implementation which works for all sane clocksources. That 37 * obviously excludes x86/TSC. 38 */ 39 static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base) 40 { 41 u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd); 42 43 if (likely(vdso_delta_ok(vd, delta))) 44 return vdso_shift_ns((delta * vd->mult) + base, vd->shift); 45 46 return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift); 47 } 48 #endif /* vdso_calc_ns */ 49 50 #ifndef __arch_vdso_hres_capable 51 static inline bool __arch_vdso_hres_capable(void) 52 { 53 return true; 54 } 55 #endif 56 57 #ifndef vdso_clocksource_ok 58 static inline bool vdso_clocksource_ok(const struct vdso_data *vd) 59 { 60 return vd->clock_mode != VDSO_CLOCKMODE_NONE; 61 } 62 #endif 63 64 #ifndef vdso_cycles_ok 65 static inline bool vdso_cycles_ok(u64 cycles) 66 { 67 return true; 68 } 69 #endif 70 71 #ifdef CONFIG_TIME_NS 72 static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 73 struct __kernel_timespec *ts) 74 { 75 const struct timens_offset *offs = &vdns->offset[clk]; 76 const struct vdso_timestamp *vdso_ts; 77 const struct vdso_data *vd; 78 u64 cycles, ns; 79 u32 seq; 80 s64 sec; 81 82 vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); 83 vd = __arch_get_timens_vdso_data(vd); 84 if (clk != CLOCK_MONOTONIC_RAW) 85 vd = &vd[CS_HRES_COARSE]; 86 else 87 vd = &vd[CS_RAW]; 88 vdso_ts = &vd->basetime[clk]; 89 90 do { 91 seq = vdso_read_begin(vd); 92 93 if (unlikely(!vdso_clocksource_ok(vd))) 94 return -1; 95 96 cycles = __arch_get_hw_counter(vd->clock_mode, vd); 97 if (unlikely(!vdso_cycles_ok(cycles))) 98 return -1; 99 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); 100 sec = vdso_ts->sec; 101 } while (unlikely(vdso_read_retry(vd, seq))); 102 103 /* Add the namespace offset */ 104 sec += offs->sec; 105 ns += offs->nsec; 106 107 /* 108 * Do this outside the loop: a race inside the loop could result 109 * in __iter_div_u64_rem() being extremely slow. 110 */ 111 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 112 ts->tv_nsec = ns; 113 114 return 0; 115 } 116 #else 117 static __always_inline 118 const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 119 { 120 return NULL; 121 } 122 123 static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 124 struct __kernel_timespec *ts) 125 { 126 return -EINVAL; 127 } 128 #endif 129 130 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 131 struct __kernel_timespec *ts) 132 { 133 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 134 u64 cycles, sec, ns; 135 u32 seq; 136 137 /* Allows to compile the high resolution parts out */ 138 if (!__arch_vdso_hres_capable()) 139 return -1; 140 141 do { 142 /* 143 * Open coded to handle VDSO_CLOCKMODE_TIMENS. Time namespace 144 * enabled tasks have a special VVAR page installed which 145 * has vd->seq set to 1 and vd->clock_mode set to 146 * VDSO_CLOCKMODE_TIMENS. For non time namespace affected tasks 147 * this does not affect performance because if vd->seq is 148 * odd, i.e. a concurrent update is in progress the extra 149 * check for vd->clock_mode is just a few extra 150 * instructions while spin waiting for vd->seq to become 151 * even again. 152 */ 153 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 154 if (IS_ENABLED(CONFIG_TIME_NS) && 155 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 156 return do_hres_timens(vd, clk, ts); 157 cpu_relax(); 158 } 159 smp_rmb(); 160 161 if (unlikely(!vdso_clocksource_ok(vd))) 162 return -1; 163 164 cycles = __arch_get_hw_counter(vd->clock_mode, vd); 165 if (unlikely(!vdso_cycles_ok(cycles))) 166 return -1; 167 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); 168 sec = vdso_ts->sec; 169 } while (unlikely(vdso_read_retry(vd, seq))); 170 171 /* 172 * Do this outside the loop: a race inside the loop could result 173 * in __iter_div_u64_rem() being extremely slow. 174 */ 175 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 176 ts->tv_nsec = ns; 177 178 return 0; 179 } 180 181 #ifdef CONFIG_TIME_NS 182 static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 183 struct __kernel_timespec *ts) 184 { 185 const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns); 186 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 187 const struct timens_offset *offs = &vdns->offset[clk]; 188 u64 nsec; 189 s64 sec; 190 s32 seq; 191 192 do { 193 seq = vdso_read_begin(vd); 194 sec = vdso_ts->sec; 195 nsec = vdso_ts->nsec; 196 } while (unlikely(vdso_read_retry(vd, seq))); 197 198 /* Add the namespace offset */ 199 sec += offs->sec; 200 nsec += offs->nsec; 201 202 /* 203 * Do this outside the loop: a race inside the loop could result 204 * in __iter_div_u64_rem() being extremely slow. 205 */ 206 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 207 ts->tv_nsec = nsec; 208 return 0; 209 } 210 #else 211 static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 212 struct __kernel_timespec *ts) 213 { 214 return -1; 215 } 216 #endif 217 218 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 219 struct __kernel_timespec *ts) 220 { 221 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 222 u32 seq; 223 224 do { 225 /* 226 * Open coded to handle VDSO_CLOCK_TIMENS. See comment in 227 * do_hres(). 228 */ 229 while ((seq = READ_ONCE(vd->seq)) & 1) { 230 if (IS_ENABLED(CONFIG_TIME_NS) && 231 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 232 return do_coarse_timens(vd, clk, ts); 233 cpu_relax(); 234 } 235 smp_rmb(); 236 237 ts->tv_sec = vdso_ts->sec; 238 ts->tv_nsec = vdso_ts->nsec; 239 } while (unlikely(vdso_read_retry(vd, seq))); 240 241 return 0; 242 } 243 244 static __always_inline int 245 __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, 246 struct __kernel_timespec *ts) 247 { 248 u32 msk; 249 250 /* Check for negative values or invalid clocks */ 251 if (unlikely((u32) clock >= MAX_CLOCKS)) 252 return -1; 253 254 /* 255 * Convert the clockid to a bitmask and use it to check which 256 * clocks are handled in the VDSO directly. 257 */ 258 msk = 1U << clock; 259 if (likely(msk & VDSO_HRES)) 260 vd = &vd[CS_HRES_COARSE]; 261 else if (msk & VDSO_COARSE) 262 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 263 else if (msk & VDSO_RAW) 264 vd = &vd[CS_RAW]; 265 else 266 return -1; 267 268 return do_hres(vd, clock, ts); 269 } 270 271 static __maybe_unused int 272 __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, 273 struct __kernel_timespec *ts) 274 { 275 int ret = __cvdso_clock_gettime_common(vd, clock, ts); 276 277 if (unlikely(ret)) 278 return clock_gettime_fallback(clock, ts); 279 return 0; 280 } 281 282 static __maybe_unused int 283 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 284 { 285 return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts); 286 } 287 288 #ifdef BUILD_VDSO32 289 static __maybe_unused int 290 __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, 291 struct old_timespec32 *res) 292 { 293 struct __kernel_timespec ts; 294 int ret; 295 296 ret = __cvdso_clock_gettime_common(vd, clock, &ts); 297 298 if (unlikely(ret)) 299 return clock_gettime32_fallback(clock, res); 300 301 /* For ret == 0 */ 302 res->tv_sec = ts.tv_sec; 303 res->tv_nsec = ts.tv_nsec; 304 305 return ret; 306 } 307 308 static __maybe_unused int 309 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 310 { 311 return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res); 312 } 313 #endif /* BUILD_VDSO32 */ 314 315 static __maybe_unused int 316 __cvdso_gettimeofday_data(const struct vdso_data *vd, 317 struct __kernel_old_timeval *tv, struct timezone *tz) 318 { 319 320 if (likely(tv != NULL)) { 321 struct __kernel_timespec ts; 322 323 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 324 return gettimeofday_fallback(tv, tz); 325 326 tv->tv_sec = ts.tv_sec; 327 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 328 } 329 330 if (unlikely(tz != NULL)) { 331 if (IS_ENABLED(CONFIG_TIME_NS) && 332 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 333 vd = __arch_get_timens_vdso_data(vd); 334 335 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 336 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 337 } 338 339 return 0; 340 } 341 342 static __maybe_unused int 343 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 344 { 345 return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz); 346 } 347 348 #ifdef VDSO_HAS_TIME 349 static __maybe_unused __kernel_old_time_t 350 __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) 351 { 352 __kernel_old_time_t t; 353 354 if (IS_ENABLED(CONFIG_TIME_NS) && 355 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 356 vd = __arch_get_timens_vdso_data(vd); 357 358 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 359 360 if (time) 361 *time = t; 362 363 return t; 364 } 365 366 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 367 { 368 return __cvdso_time_data(__arch_get_vdso_data(), time); 369 } 370 #endif /* VDSO_HAS_TIME */ 371 372 #ifdef VDSO_HAS_CLOCK_GETRES 373 static __maybe_unused 374 int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, 375 struct __kernel_timespec *res) 376 { 377 u32 msk; 378 u64 ns; 379 380 /* Check for negative values or invalid clocks */ 381 if (unlikely((u32) clock >= MAX_CLOCKS)) 382 return -1; 383 384 if (IS_ENABLED(CONFIG_TIME_NS) && 385 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 386 vd = __arch_get_timens_vdso_data(vd); 387 388 /* 389 * Convert the clockid to a bitmask and use it to check which 390 * clocks are handled in the VDSO directly. 391 */ 392 msk = 1U << clock; 393 if (msk & (VDSO_HRES | VDSO_RAW)) { 394 /* 395 * Preserves the behaviour of posix_get_hrtimer_res(). 396 */ 397 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 398 } else if (msk & VDSO_COARSE) { 399 /* 400 * Preserves the behaviour of posix_get_coarse_res(). 401 */ 402 ns = LOW_RES_NSEC; 403 } else { 404 return -1; 405 } 406 407 if (likely(res)) { 408 res->tv_sec = 0; 409 res->tv_nsec = ns; 410 } 411 return 0; 412 } 413 414 static __maybe_unused 415 int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, 416 struct __kernel_timespec *res) 417 { 418 int ret = __cvdso_clock_getres_common(vd, clock, res); 419 420 if (unlikely(ret)) 421 return clock_getres_fallback(clock, res); 422 return 0; 423 } 424 425 static __maybe_unused 426 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 427 { 428 return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res); 429 } 430 431 #ifdef BUILD_VDSO32 432 static __maybe_unused int 433 __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, 434 struct old_timespec32 *res) 435 { 436 struct __kernel_timespec ts; 437 int ret; 438 439 ret = __cvdso_clock_getres_common(vd, clock, &ts); 440 441 if (unlikely(ret)) 442 return clock_getres32_fallback(clock, res); 443 444 if (likely(res)) { 445 res->tv_sec = ts.tv_sec; 446 res->tv_nsec = ts.tv_nsec; 447 } 448 return ret; 449 } 450 451 static __maybe_unused int 452 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 453 { 454 return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(), 455 clock, res); 456 } 457 #endif /* BUILD_VDSO32 */ 458 #endif /* VDSO_HAS_CLOCK_GETRES */ 459