1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/datapage.h> 6 #include <vdso/helpers.h> 7 8 /* Bring in default accessors */ 9 #include <vdso/vsyscall.h> 10 11 #ifndef vdso_calc_ns 12 13 #ifdef VDSO_DELTA_NOMASK 14 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 15 #else 16 # define VDSO_DELTA_MASK(vd) (vd->mask) 17 #endif 18 19 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 20 static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) 21 { 22 return delta < vd->max_cycles; 23 } 24 #else 25 static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta) 26 { 27 return true; 28 } 29 #endif 30 31 #ifndef vdso_shift_ns 32 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 33 { 34 return ns >> shift; 35 } 36 #endif 37 38 /* 39 * Default implementation which works for all sane clocksources. That 40 * obviously excludes x86/TSC. 41 */ 42 static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base) 43 { 44 u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd); 45 46 if (likely(vdso_delta_ok(vd, delta))) 47 return vdso_shift_ns((delta * vd->mult) + base, vd->shift); 48 49 return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift); 50 } 51 #endif /* vdso_calc_ns */ 52 53 #ifndef __arch_vdso_hres_capable 54 static inline bool __arch_vdso_hres_capable(void) 55 { 56 return true; 57 } 58 #endif 59 60 #ifndef vdso_clocksource_ok 61 static inline bool vdso_clocksource_ok(const struct vdso_data *vd) 62 { 63 return vd->clock_mode != VDSO_CLOCKMODE_NONE; 64 } 65 #endif 66 67 #ifndef vdso_cycles_ok 68 static inline bool vdso_cycles_ok(u64 cycles) 69 { 70 return true; 71 } 72 #endif 73 74 #ifdef CONFIG_TIME_NS 75 76 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE 77 static __always_inline 78 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 79 { 80 return (void *)vd + PAGE_SIZE; 81 } 82 #define __arch_get_timens_vdso_data(vd) __arch_get_vdso_u_timens_data(vd) 83 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ 84 85 static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 86 struct __kernel_timespec *ts) 87 { 88 const struct timens_offset *offs = &vdns->offset[clk]; 89 const struct vdso_timestamp *vdso_ts; 90 const struct vdso_data *vd; 91 u64 cycles, ns; 92 u32 seq; 93 s64 sec; 94 95 vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); 96 vd = __arch_get_timens_vdso_data(vd); 97 if (clk != CLOCK_MONOTONIC_RAW) 98 vd = &vd[CS_HRES_COARSE]; 99 else 100 vd = &vd[CS_RAW]; 101 vdso_ts = &vd->basetime[clk]; 102 103 do { 104 seq = vdso_read_begin(vd); 105 106 if (unlikely(!vdso_clocksource_ok(vd))) 107 return -1; 108 109 cycles = __arch_get_hw_counter(vd->clock_mode, vd); 110 if (unlikely(!vdso_cycles_ok(cycles))) 111 return -1; 112 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); 113 sec = vdso_ts->sec; 114 } while (unlikely(vdso_read_retry(vd, seq))); 115 116 /* Add the namespace offset */ 117 sec += offs->sec; 118 ns += offs->nsec; 119 120 /* 121 * Do this outside the loop: a race inside the loop could result 122 * in __iter_div_u64_rem() being extremely slow. 123 */ 124 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 125 ts->tv_nsec = ns; 126 127 return 0; 128 } 129 #else 130 static __always_inline 131 const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) 132 { 133 return NULL; 134 } 135 136 static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, 137 struct __kernel_timespec *ts) 138 { 139 return -EINVAL; 140 } 141 #endif 142 143 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, 144 struct __kernel_timespec *ts) 145 { 146 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 147 u64 cycles, sec, ns; 148 u32 seq; 149 150 /* Allows to compile the high resolution parts out */ 151 if (!__arch_vdso_hres_capable()) 152 return -1; 153 154 do { 155 /* 156 * Open coded function vdso_read_begin() to handle 157 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 158 * special VVAR page installed which has vd->seq set to 1 and 159 * vd->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 160 * namespace affected tasks this does not affect performance 161 * because if vd->seq is odd, i.e. a concurrent update is in 162 * progress the extra check for vd->clock_mode is just a few 163 * extra instructions while spin waiting for vd->seq to become 164 * even again. 165 */ 166 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 167 if (IS_ENABLED(CONFIG_TIME_NS) && 168 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 169 return do_hres_timens(vd, clk, ts); 170 cpu_relax(); 171 } 172 smp_rmb(); 173 174 if (unlikely(!vdso_clocksource_ok(vd))) 175 return -1; 176 177 cycles = __arch_get_hw_counter(vd->clock_mode, vd); 178 if (unlikely(!vdso_cycles_ok(cycles))) 179 return -1; 180 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); 181 sec = vdso_ts->sec; 182 } while (unlikely(vdso_read_retry(vd, seq))); 183 184 /* 185 * Do this outside the loop: a race inside the loop could result 186 * in __iter_div_u64_rem() being extremely slow. 187 */ 188 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 189 ts->tv_nsec = ns; 190 191 return 0; 192 } 193 194 #ifdef CONFIG_TIME_NS 195 static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 196 struct __kernel_timespec *ts) 197 { 198 const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns); 199 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 200 const struct timens_offset *offs = &vdns->offset[clk]; 201 u64 nsec; 202 s64 sec; 203 s32 seq; 204 205 do { 206 seq = vdso_read_begin(vd); 207 sec = vdso_ts->sec; 208 nsec = vdso_ts->nsec; 209 } while (unlikely(vdso_read_retry(vd, seq))); 210 211 /* Add the namespace offset */ 212 sec += offs->sec; 213 nsec += offs->nsec; 214 215 /* 216 * Do this outside the loop: a race inside the loop could result 217 * in __iter_div_u64_rem() being extremely slow. 218 */ 219 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 220 ts->tv_nsec = nsec; 221 return 0; 222 } 223 #else 224 static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk, 225 struct __kernel_timespec *ts) 226 { 227 return -1; 228 } 229 #endif 230 231 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, 232 struct __kernel_timespec *ts) 233 { 234 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 235 u32 seq; 236 237 do { 238 /* 239 * Open coded function vdso_read_begin() to handle 240 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 241 */ 242 while ((seq = READ_ONCE(vd->seq)) & 1) { 243 if (IS_ENABLED(CONFIG_TIME_NS) && 244 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 245 return do_coarse_timens(vd, clk, ts); 246 cpu_relax(); 247 } 248 smp_rmb(); 249 250 ts->tv_sec = vdso_ts->sec; 251 ts->tv_nsec = vdso_ts->nsec; 252 } while (unlikely(vdso_read_retry(vd, seq))); 253 254 return 0; 255 } 256 257 static __always_inline int 258 __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, 259 struct __kernel_timespec *ts) 260 { 261 u32 msk; 262 263 /* Check for negative values or invalid clocks */ 264 if (unlikely((u32) clock >= MAX_CLOCKS)) 265 return -1; 266 267 /* 268 * Convert the clockid to a bitmask and use it to check which 269 * clocks are handled in the VDSO directly. 270 */ 271 msk = 1U << clock; 272 if (likely(msk & VDSO_HRES)) 273 vd = &vd[CS_HRES_COARSE]; 274 else if (msk & VDSO_COARSE) 275 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 276 else if (msk & VDSO_RAW) 277 vd = &vd[CS_RAW]; 278 else 279 return -1; 280 281 return do_hres(vd, clock, ts); 282 } 283 284 static __maybe_unused int 285 __cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock, 286 struct __kernel_timespec *ts) 287 { 288 int ret = __cvdso_clock_gettime_common(vd, clock, ts); 289 290 if (unlikely(ret)) 291 return clock_gettime_fallback(clock, ts); 292 return 0; 293 } 294 295 static __maybe_unused int 296 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 297 { 298 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 299 } 300 301 #ifdef BUILD_VDSO32 302 static __maybe_unused int 303 __cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock, 304 struct old_timespec32 *res) 305 { 306 struct __kernel_timespec ts; 307 int ret; 308 309 ret = __cvdso_clock_gettime_common(vd, clock, &ts); 310 311 if (unlikely(ret)) 312 return clock_gettime32_fallback(clock, res); 313 314 /* For ret == 0 */ 315 res->tv_sec = ts.tv_sec; 316 res->tv_nsec = ts.tv_nsec; 317 318 return ret; 319 } 320 321 static __maybe_unused int 322 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 323 { 324 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 325 } 326 #endif /* BUILD_VDSO32 */ 327 328 static __maybe_unused int 329 __cvdso_gettimeofday_data(const struct vdso_data *vd, 330 struct __kernel_old_timeval *tv, struct timezone *tz) 331 { 332 333 if (likely(tv != NULL)) { 334 struct __kernel_timespec ts; 335 336 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 337 return gettimeofday_fallback(tv, tz); 338 339 tv->tv_sec = ts.tv_sec; 340 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 341 } 342 343 if (unlikely(tz != NULL)) { 344 if (IS_ENABLED(CONFIG_TIME_NS) && 345 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 346 vd = __arch_get_timens_vdso_data(vd); 347 348 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 349 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 350 } 351 352 return 0; 353 } 354 355 static __maybe_unused int 356 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 357 { 358 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 359 } 360 361 #ifdef VDSO_HAS_TIME 362 static __maybe_unused __kernel_old_time_t 363 __cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time) 364 { 365 __kernel_old_time_t t; 366 367 if (IS_ENABLED(CONFIG_TIME_NS) && 368 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 369 vd = __arch_get_timens_vdso_data(vd); 370 371 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 372 373 if (time) 374 *time = t; 375 376 return t; 377 } 378 379 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 380 { 381 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 382 } 383 #endif /* VDSO_HAS_TIME */ 384 385 #ifdef VDSO_HAS_CLOCK_GETRES 386 static __maybe_unused 387 int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock, 388 struct __kernel_timespec *res) 389 { 390 u32 msk; 391 u64 ns; 392 393 /* Check for negative values or invalid clocks */ 394 if (unlikely((u32) clock >= MAX_CLOCKS)) 395 return -1; 396 397 if (IS_ENABLED(CONFIG_TIME_NS) && 398 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 399 vd = __arch_get_timens_vdso_data(vd); 400 401 /* 402 * Convert the clockid to a bitmask and use it to check which 403 * clocks are handled in the VDSO directly. 404 */ 405 msk = 1U << clock; 406 if (msk & (VDSO_HRES | VDSO_RAW)) { 407 /* 408 * Preserves the behaviour of posix_get_hrtimer_res(). 409 */ 410 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 411 } else if (msk & VDSO_COARSE) { 412 /* 413 * Preserves the behaviour of posix_get_coarse_res(). 414 */ 415 ns = LOW_RES_NSEC; 416 } else { 417 return -1; 418 } 419 420 if (likely(res)) { 421 res->tv_sec = 0; 422 res->tv_nsec = ns; 423 } 424 return 0; 425 } 426 427 static __maybe_unused 428 int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock, 429 struct __kernel_timespec *res) 430 { 431 int ret = __cvdso_clock_getres_common(vd, clock, res); 432 433 if (unlikely(ret)) 434 return clock_getres_fallback(clock, res); 435 return 0; 436 } 437 438 static __maybe_unused 439 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 440 { 441 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 442 } 443 444 #ifdef BUILD_VDSO32 445 static __maybe_unused int 446 __cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock, 447 struct old_timespec32 *res) 448 { 449 struct __kernel_timespec ts; 450 int ret; 451 452 ret = __cvdso_clock_getres_common(vd, clock, &ts); 453 454 if (unlikely(ret)) 455 return clock_getres32_fallback(clock, res); 456 457 if (likely(res)) { 458 res->tv_sec = ts.tv_sec; 459 res->tv_nsec = ts.tv_nsec; 460 } 461 return ret; 462 } 463 464 static __maybe_unused int 465 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 466 { 467 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 468 clock, res); 469 } 470 #endif /* BUILD_VDSO32 */ 471 #endif /* VDSO_HAS_CLOCK_GETRES */ 472