1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> 5 * Copyright (c) 2021 Dmitry Chagin <dchagin@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 30static int 31__vdso_native_to_linux_timespec(struct l_timespec *lts, 32 struct timespec *nts) 33{ 34 35#ifdef COMPAT_LINUX32 36 if (nts->tv_sec > INT_MAX || nts->tv_sec < INT_MIN) 37 return (LINUX_EOVERFLOW); 38#endif 39 lts->tv_sec = nts->tv_sec; 40 lts->tv_nsec = nts->tv_nsec; 41 return (0); 42} 43 44static int 45__vdso_native_to_linux_timeval(l_timeval *ltv, 46 struct timeval *ntv) 47{ 48 49#ifdef COMPAT_LINUX32 50 if (ntv->tv_sec > INT_MAX || ntv->tv_sec < INT_MIN) 51 return (LINUX_EOVERFLOW); 52#endif 53 ltv->tv_sec = ntv->tv_sec; 54 ltv->tv_usec = ntv->tv_usec; 55 return (0); 56} 57 58 59#if defined(__i386__) || (defined(__amd64__) && defined(COMPAT_LINUX32)) 60static int 61__vdso_native_to_linux_timespec64(struct l_timespec64 *lts, 62 struct timespec *nts) 63{ 64 65 lts->tv_sec = nts->tv_sec; 66 lts->tv_nsec = nts->tv_nsec; 67 return (0); 68} 69#endif 70 71static int 72__vdso_linux_to_native_clockid(clockid_t *n, clockid_t l) 73{ 74 75 switch (l) { 76 case LINUX_CLOCK_REALTIME: 77 *n = CLOCK_REALTIME; 78 break; 79 case LINUX_CLOCK_MONOTONIC: 80 *n = CLOCK_MONOTONIC; 81 break; 82 case LINUX_CLOCK_REALTIME_COARSE: 83 *n = CLOCK_REALTIME_FAST; 84 break; 85 case LINUX_CLOCK_MONOTONIC_COARSE: 86 case LINUX_CLOCK_MONOTONIC_RAW: 87 *n = CLOCK_MONOTONIC_FAST; 88 break; 89 case LINUX_CLOCK_BOOTTIME: 90 *n = CLOCK_UPTIME; 91 break; 92 default: 93 return (LINUX_EINVAL); 94 } 95 return (0); 96} 97 98/* 99 * The code below adapted from 100 * lib/libc/sys/__vdso_gettimeofday.c 101 */ 102 103static inline void 104__vdso_gettimekeep(struct vdso_timekeep **tk) 105{ 106 107 *tk = (struct vdso_timekeep *)kern_timekeep_base; 108} 109 110static int 111tc_delta(const struct vdso_timehands *th, u_int *delta) 112{ 113 int error; 114 u_int tc; 115 116 error = __vdso_gettc(th, &tc); 117 if (error == 0) 118 *delta = (tc - th->th_offset_count) & th->th_counter_mask; 119 return (error); 120} 121 122/* 123 * Calculate the absolute or boot-relative time from the 124 * machine-specific fast timecounter and the published timehands 125 * structure read from the shared page. 126 * 127 * The lockless reading scheme is similar to the one used to read the 128 * in-kernel timehands, see sys/kern/kern_tc.c:binuptime(). This code 129 * is based on the kernel implementation. 130 */ 131static int 132freebsd_binuptime(struct bintime *bt, struct vdso_timekeep *tk, bool abs) 133{ 134 struct vdso_timehands *th; 135 uint32_t curr, gen; 136 uint64_t scale, x; 137 u_int delta, scale_bits; 138 int error; 139 140 do { 141 if (!tk->tk_enabled) 142 return (ENOSYS); 143 144 curr = atomic_load_acq_32(&tk->tk_current); 145 th = &tk->tk_th[curr]; 146 gen = atomic_load_acq_32(&th->th_gen); 147 *bt = th->th_offset; 148 error = tc_delta(th, &delta); 149 if (error == EAGAIN) 150 continue; 151 if (error != 0) 152 return (error); 153 scale = th->th_scale; 154#ifdef _LP64 155 scale_bits = ffsl(scale); 156#else 157 scale_bits = ffsll(scale); 158#endif 159 if (__predict_false(scale_bits + fls(delta) > 63)) { 160 x = (scale >> 32) * delta; 161 scale &= 0xffffffff; 162 bt->sec += x >> 32; 163 bintime_addx(bt, x << 32); 164 } 165 bintime_addx(bt, scale * delta); 166 if (abs) 167 bintime_add(bt, &th->th_boottime); 168 169 /* 170 * Ensure that the load of th_offset is completed 171 * before the load of th_gen. 172 */ 173 atomic_thread_fence_acq(); 174 } while (curr != tk->tk_current || gen == 0 || gen != th->th_gen); 175 return (0); 176} 177 178static int 179freebsd_getnanouptime(struct bintime *bt, struct vdso_timekeep *tk) 180{ 181 struct vdso_timehands *th; 182 uint32_t curr, gen; 183 184 do { 185 if (!tk->tk_enabled) 186 return (ENOSYS); 187 188 curr = atomic_load_acq_32(&tk->tk_current); 189 th = &tk->tk_th[curr]; 190 gen = atomic_load_acq_32(&th->th_gen); 191 *bt = th->th_offset; 192 193 /* 194 * Ensure that the load of th_offset is completed 195 * before the load of th_gen. 196 */ 197 atomic_thread_fence_acq(); 198 } while (curr != tk->tk_current || gen == 0 || gen != th->th_gen); 199 return (0); 200} 201 202static int 203freebsd_gettimeofday(struct timeval *tv, struct timezone *tz) 204{ 205 struct vdso_timekeep *tk; 206 struct bintime bt; 207 int error; 208 209 if (tz != NULL) 210 return (ENOSYS); 211 __vdso_gettimekeep(&tk); 212 if (tk == NULL) 213 return (ENOSYS); 214 if (tk->tk_ver != VDSO_TK_VER_CURR) 215 return (ENOSYS); 216 error = freebsd_binuptime(&bt, tk, true); 217 if (error == 0) 218 bintime2timeval(&bt, tv); 219 return (error); 220} 221 222static int 223freebsd_clock_gettime(clockid_t clock_id, struct timespec *ts) 224{ 225 struct vdso_timekeep *tk; 226 struct bintime bt; 227 int error; 228 229 __vdso_gettimekeep(&tk); 230 if (tk == NULL) 231 return (ENOSYS); 232 if (tk->tk_ver != VDSO_TK_VER_CURR) 233 return (ENOSYS); 234 switch (clock_id) { 235 case CLOCK_REALTIME: 236 case CLOCK_REALTIME_PRECISE: 237 case CLOCK_REALTIME_FAST: 238 error = freebsd_binuptime(&bt, tk, true); 239 break; 240 case CLOCK_MONOTONIC: 241 case CLOCK_MONOTONIC_PRECISE: 242 case CLOCK_UPTIME: 243 case CLOCK_UPTIME_PRECISE: 244 error = freebsd_binuptime(&bt, tk, false); 245 break; 246 case CLOCK_MONOTONIC_FAST: 247 case CLOCK_UPTIME_FAST: 248 error = freebsd_getnanouptime(&bt, tk); 249 break; 250 default: 251 error = ENOSYS; 252 break; 253 } 254 if (error == 0) 255 bintime2timespec(&bt, ts); 256 return (error); 257} 258 259/* 260 * Linux vDSO interfaces 261 * 262 */ 263int 264__vdso_clock_gettime(clockid_t clock_id, struct l_timespec *lts) 265{ 266 struct timespec ts; 267 clockid_t which; 268 int error; 269 270 error = __vdso_linux_to_native_clockid(&which, clock_id); 271 if (error != 0) 272 return (__vdso_clock_gettime_fallback(clock_id, lts)); 273 error = freebsd_clock_gettime(which, &ts); 274 if (error == 0) 275 return (-__vdso_native_to_linux_timespec(lts, &ts)); 276 else 277 return (__vdso_clock_gettime_fallback(clock_id, lts)); 278} 279 280int 281__vdso_gettimeofday(l_timeval *ltv, struct timezone *tz) 282{ 283 struct timeval tv; 284 int error; 285 286 error = freebsd_gettimeofday(&tv, tz); 287 if (error != 0) 288 return (__vdso_gettimeofday_fallback(ltv, tz)); 289 return (-__vdso_native_to_linux_timeval(ltv, &tv)); 290} 291 292int 293__vdso_clock_getres(clockid_t clock_id, struct l_timespec *lts) 294{ 295 296 return (__vdso_clock_getres_fallback(clock_id, lts)); 297} 298 299#if defined(__i386__) || defined(COMPAT_LINUX32) 300int 301__vdso_clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts) 302{ 303 struct timespec ts; 304 clockid_t which; 305 int error; 306 307 error = __vdso_linux_to_native_clockid(&which, clock_id); 308 if (error != 0) 309 return (__vdso_clock_gettime64_fallback(clock_id, lts)); 310 error = freebsd_clock_gettime(which, &ts); 311 if (error == 0) 312 return(-__vdso_native_to_linux_timespec64(lts, &ts)); 313 else 314 return(__vdso_clock_gettime64_fallback(clock_id, lts)); 315} 316 317int clock_gettime64(clockid_t clock_id, struct l_timespec64 *lts) 318 __attribute__((weak, alias("__vdso_clock_gettime64"))); 319#endif 320 321#if defined(__amd64__) && !defined(COMPAT_LINUX32) 322int 323__vdso_getcpu(uint32_t *cpu, uint32_t *node, void *cache) 324{ 325 326 return (__vdso_getcpu_fallback(cpu, node, cache)); 327} 328#endif 329 330#if defined(__i386__) || defined(__amd64__) 331int 332__vdso_time(long *tm) 333{ 334 335 return (__vdso_time_fallback(tm)); 336} 337#endif 338