1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <vdso/datapage.h> 6 #include <vdso/helpers.h> 7 8 /* Bring in default accessors */ 9 #include <vdso/vsyscall.h> 10 11 #ifndef vdso_calc_ns 12 13 #ifdef VDSO_DELTA_NOMASK 14 # define VDSO_DELTA_MASK(vd) ULLONG_MAX 15 #else 16 # define VDSO_DELTA_MASK(vd) (vd->mask) 17 #endif 18 19 #ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT 20 static __always_inline bool vdso_delta_ok(const struct vdso_time_data *vd, u64 delta) 21 { 22 return delta < vd->max_cycles; 23 } 24 #else 25 static __always_inline bool vdso_delta_ok(const struct vdso_time_data *vd, u64 delta) 26 { 27 return true; 28 } 29 #endif 30 31 #ifndef vdso_shift_ns 32 static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift) 33 { 34 return ns >> shift; 35 } 36 #endif 37 38 /* 39 * Default implementation which works for all sane clocksources. That 40 * obviously excludes x86/TSC. 41 */ 42 static __always_inline u64 vdso_calc_ns(const struct vdso_time_data *vd, u64 cycles, u64 base) 43 { 44 u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd); 45 46 if (likely(vdso_delta_ok(vd, delta))) 47 return vdso_shift_ns((delta * vd->mult) + base, vd->shift); 48 49 return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift); 50 } 51 #endif /* vdso_calc_ns */ 52 53 #ifndef __arch_vdso_hres_capable 54 static inline bool __arch_vdso_hres_capable(void) 55 { 56 return true; 57 } 58 #endif 59 60 #ifndef vdso_clocksource_ok 61 static inline bool vdso_clocksource_ok(const struct vdso_time_data *vd) 62 { 63 return vd->clock_mode != VDSO_CLOCKMODE_NONE; 64 } 65 #endif 66 67 #ifndef vdso_cycles_ok 68 static inline bool vdso_cycles_ok(u64 cycles) 69 { 70 return true; 71 } 72 #endif 73 74 #ifdef CONFIG_TIME_NS 75 76 #ifdef CONFIG_GENERIC_VDSO_DATA_STORE 77 static __always_inline 78 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 79 { 80 return (void *)vd + PAGE_SIZE; 81 } 82 #endif /* CONFIG_GENERIC_VDSO_DATA_STORE */ 83 84 static __always_inline int do_hres_timens(const struct vdso_time_data *vdns, clockid_t clk, 85 struct __kernel_timespec *ts) 86 { 87 const struct timens_offset *offs = &vdns->offset[clk]; 88 const struct vdso_timestamp *vdso_ts; 89 const struct vdso_time_data *vd; 90 u64 cycles, ns; 91 u32 seq; 92 s64 sec; 93 94 vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE); 95 vd = __arch_get_vdso_u_timens_data(vd); 96 if (clk != CLOCK_MONOTONIC_RAW) 97 vd = &vd[CS_HRES_COARSE]; 98 else 99 vd = &vd[CS_RAW]; 100 vdso_ts = &vd->basetime[clk]; 101 102 do { 103 seq = vdso_read_begin(vd); 104 105 if (unlikely(!vdso_clocksource_ok(vd))) 106 return -1; 107 108 cycles = __arch_get_hw_counter(vd->clock_mode, vd); 109 if (unlikely(!vdso_cycles_ok(cycles))) 110 return -1; 111 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); 112 sec = vdso_ts->sec; 113 } while (unlikely(vdso_read_retry(vd, seq))); 114 115 /* Add the namespace offset */ 116 sec += offs->sec; 117 ns += offs->nsec; 118 119 /* 120 * Do this outside the loop: a race inside the loop could result 121 * in __iter_div_u64_rem() being extremely slow. 122 */ 123 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 124 ts->tv_nsec = ns; 125 126 return 0; 127 } 128 #else 129 static __always_inline 130 const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) 131 { 132 return NULL; 133 } 134 135 static __always_inline int do_hres_timens(const struct vdso_time_data *vdns, clockid_t clk, 136 struct __kernel_timespec *ts) 137 { 138 return -EINVAL; 139 } 140 #endif 141 142 static __always_inline int do_hres(const struct vdso_time_data *vd, clockid_t clk, 143 struct __kernel_timespec *ts) 144 { 145 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 146 u64 cycles, sec, ns; 147 u32 seq; 148 149 /* Allows to compile the high resolution parts out */ 150 if (!__arch_vdso_hres_capable()) 151 return -1; 152 153 do { 154 /* 155 * Open coded function vdso_read_begin() to handle 156 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a 157 * special VVAR page installed which has vd->seq set to 1 and 158 * vd->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time 159 * namespace affected tasks this does not affect performance 160 * because if vd->seq is odd, i.e. a concurrent update is in 161 * progress the extra check for vd->clock_mode is just a few 162 * extra instructions while spin waiting for vd->seq to become 163 * even again. 164 */ 165 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) { 166 if (IS_ENABLED(CONFIG_TIME_NS) && 167 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 168 return do_hres_timens(vd, clk, ts); 169 cpu_relax(); 170 } 171 smp_rmb(); 172 173 if (unlikely(!vdso_clocksource_ok(vd))) 174 return -1; 175 176 cycles = __arch_get_hw_counter(vd->clock_mode, vd); 177 if (unlikely(!vdso_cycles_ok(cycles))) 178 return -1; 179 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec); 180 sec = vdso_ts->sec; 181 } while (unlikely(vdso_read_retry(vd, seq))); 182 183 /* 184 * Do this outside the loop: a race inside the loop could result 185 * in __iter_div_u64_rem() being extremely slow. 186 */ 187 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 188 ts->tv_nsec = ns; 189 190 return 0; 191 } 192 193 #ifdef CONFIG_TIME_NS 194 static __always_inline int do_coarse_timens(const struct vdso_time_data *vdns, clockid_t clk, 195 struct __kernel_timespec *ts) 196 { 197 const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); 198 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 199 const struct timens_offset *offs = &vdns->offset[clk]; 200 u64 nsec; 201 s64 sec; 202 s32 seq; 203 204 do { 205 seq = vdso_read_begin(vd); 206 sec = vdso_ts->sec; 207 nsec = vdso_ts->nsec; 208 } while (unlikely(vdso_read_retry(vd, seq))); 209 210 /* Add the namespace offset */ 211 sec += offs->sec; 212 nsec += offs->nsec; 213 214 /* 215 * Do this outside the loop: a race inside the loop could result 216 * in __iter_div_u64_rem() being extremely slow. 217 */ 218 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); 219 ts->tv_nsec = nsec; 220 return 0; 221 } 222 #else 223 static __always_inline int do_coarse_timens(const struct vdso_time_data *vdns, clockid_t clk, 224 struct __kernel_timespec *ts) 225 { 226 return -1; 227 } 228 #endif 229 230 static __always_inline int do_coarse(const struct vdso_time_data *vd, clockid_t clk, 231 struct __kernel_timespec *ts) 232 { 233 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 234 u32 seq; 235 236 do { 237 /* 238 * Open coded function vdso_read_begin() to handle 239 * VDSO_CLOCK_TIMENS. See comment in do_hres(). 240 */ 241 while ((seq = READ_ONCE(vd->seq)) & 1) { 242 if (IS_ENABLED(CONFIG_TIME_NS) && 243 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 244 return do_coarse_timens(vd, clk, ts); 245 cpu_relax(); 246 } 247 smp_rmb(); 248 249 ts->tv_sec = vdso_ts->sec; 250 ts->tv_nsec = vdso_ts->nsec; 251 } while (unlikely(vdso_read_retry(vd, seq))); 252 253 return 0; 254 } 255 256 static __always_inline int 257 __cvdso_clock_gettime_common(const struct vdso_time_data *vd, clockid_t clock, 258 struct __kernel_timespec *ts) 259 { 260 u32 msk; 261 262 /* Check for negative values or invalid clocks */ 263 if (unlikely((u32) clock >= MAX_CLOCKS)) 264 return -1; 265 266 /* 267 * Convert the clockid to a bitmask and use it to check which 268 * clocks are handled in the VDSO directly. 269 */ 270 msk = 1U << clock; 271 if (likely(msk & VDSO_HRES)) 272 vd = &vd[CS_HRES_COARSE]; 273 else if (msk & VDSO_COARSE) 274 return do_coarse(&vd[CS_HRES_COARSE], clock, ts); 275 else if (msk & VDSO_RAW) 276 vd = &vd[CS_RAW]; 277 else 278 return -1; 279 280 return do_hres(vd, clock, ts); 281 } 282 283 static __maybe_unused int 284 __cvdso_clock_gettime_data(const struct vdso_time_data *vd, clockid_t clock, 285 struct __kernel_timespec *ts) 286 { 287 int ret = __cvdso_clock_gettime_common(vd, clock, ts); 288 289 if (unlikely(ret)) 290 return clock_gettime_fallback(clock, ts); 291 return 0; 292 } 293 294 static __maybe_unused int 295 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 296 { 297 return __cvdso_clock_gettime_data(__arch_get_vdso_u_time_data(), clock, ts); 298 } 299 300 #ifdef BUILD_VDSO32 301 static __maybe_unused int 302 __cvdso_clock_gettime32_data(const struct vdso_time_data *vd, clockid_t clock, 303 struct old_timespec32 *res) 304 { 305 struct __kernel_timespec ts; 306 int ret; 307 308 ret = __cvdso_clock_gettime_common(vd, clock, &ts); 309 310 if (unlikely(ret)) 311 return clock_gettime32_fallback(clock, res); 312 313 /* For ret == 0 */ 314 res->tv_sec = ts.tv_sec; 315 res->tv_nsec = ts.tv_nsec; 316 317 return ret; 318 } 319 320 static __maybe_unused int 321 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 322 { 323 return __cvdso_clock_gettime32_data(__arch_get_vdso_u_time_data(), clock, res); 324 } 325 #endif /* BUILD_VDSO32 */ 326 327 static __maybe_unused int 328 __cvdso_gettimeofday_data(const struct vdso_time_data *vd, 329 struct __kernel_old_timeval *tv, struct timezone *tz) 330 { 331 332 if (likely(tv != NULL)) { 333 struct __kernel_timespec ts; 334 335 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 336 return gettimeofday_fallback(tv, tz); 337 338 tv->tv_sec = ts.tv_sec; 339 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 340 } 341 342 if (unlikely(tz != NULL)) { 343 if (IS_ENABLED(CONFIG_TIME_NS) && 344 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 345 vd = __arch_get_vdso_u_timens_data(vd); 346 347 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 348 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 349 } 350 351 return 0; 352 } 353 354 static __maybe_unused int 355 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 356 { 357 return __cvdso_gettimeofday_data(__arch_get_vdso_u_time_data(), tv, tz); 358 } 359 360 #ifdef VDSO_HAS_TIME 361 static __maybe_unused __kernel_old_time_t 362 __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) 363 { 364 __kernel_old_time_t t; 365 366 if (IS_ENABLED(CONFIG_TIME_NS) && 367 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 368 vd = __arch_get_vdso_u_timens_data(vd); 369 370 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 371 372 if (time) 373 *time = t; 374 375 return t; 376 } 377 378 static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time) 379 { 380 return __cvdso_time_data(__arch_get_vdso_u_time_data(), time); 381 } 382 #endif /* VDSO_HAS_TIME */ 383 384 #ifdef VDSO_HAS_CLOCK_GETRES 385 static __maybe_unused 386 int __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t clock, 387 struct __kernel_timespec *res) 388 { 389 u32 msk; 390 u64 ns; 391 392 /* Check for negative values or invalid clocks */ 393 if (unlikely((u32) clock >= MAX_CLOCKS)) 394 return -1; 395 396 if (IS_ENABLED(CONFIG_TIME_NS) && 397 vd->clock_mode == VDSO_CLOCKMODE_TIMENS) 398 vd = __arch_get_vdso_u_timens_data(vd); 399 400 /* 401 * Convert the clockid to a bitmask and use it to check which 402 * clocks are handled in the VDSO directly. 403 */ 404 msk = 1U << clock; 405 if (msk & (VDSO_HRES | VDSO_RAW)) { 406 /* 407 * Preserves the behaviour of posix_get_hrtimer_res(). 408 */ 409 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 410 } else if (msk & VDSO_COARSE) { 411 /* 412 * Preserves the behaviour of posix_get_coarse_res(). 413 */ 414 ns = LOW_RES_NSEC; 415 } else { 416 return -1; 417 } 418 419 if (likely(res)) { 420 res->tv_sec = 0; 421 res->tv_nsec = ns; 422 } 423 return 0; 424 } 425 426 static __maybe_unused 427 int __cvdso_clock_getres_data(const struct vdso_time_data *vd, clockid_t clock, 428 struct __kernel_timespec *res) 429 { 430 int ret = __cvdso_clock_getres_common(vd, clock, res); 431 432 if (unlikely(ret)) 433 return clock_getres_fallback(clock, res); 434 return 0; 435 } 436 437 static __maybe_unused 438 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 439 { 440 return __cvdso_clock_getres_data(__arch_get_vdso_u_time_data(), clock, res); 441 } 442 443 #ifdef BUILD_VDSO32 444 static __maybe_unused int 445 __cvdso_clock_getres_time32_data(const struct vdso_time_data *vd, clockid_t clock, 446 struct old_timespec32 *res) 447 { 448 struct __kernel_timespec ts; 449 int ret; 450 451 ret = __cvdso_clock_getres_common(vd, clock, &ts); 452 453 if (unlikely(ret)) 454 return clock_getres32_fallback(clock, res); 455 456 if (likely(res)) { 457 res->tv_sec = ts.tv_sec; 458 res->tv_nsec = ts.tv_nsec; 459 } 460 return ret; 461 } 462 463 static __maybe_unused int 464 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 465 { 466 return __cvdso_clock_getres_time32_data(__arch_get_vdso_u_time_data(), 467 clock, res); 468 } 469 #endif /* BUILD_VDSO32 */ 470 #endif /* VDSO_HAS_CLOCK_GETRES */ 471