1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Generic userspace implementations of gettimeofday() and similar. 4 */ 5 #include <linux/compiler.h> 6 #include <linux/math64.h> 7 #include <linux/time.h> 8 #include <linux/kernel.h> 9 #include <linux/hrtimer_defs.h> 10 #include <vdso/datapage.h> 11 #include <vdso/helpers.h> 12 13 /* 14 * The generic vDSO implementation requires that gettimeofday.h 15 * provides: 16 * - __arch_get_vdso_data(): to get the vdso datapage. 17 * - __arch_get_hw_counter(): to get the hw counter based on the 18 * clock_mode. 19 * - gettimeofday_fallback(): fallback for gettimeofday. 20 * - clock_gettime_fallback(): fallback for clock_gettime. 21 * - clock_getres_fallback(): fallback for clock_getres. 22 */ 23 #ifdef ENABLE_COMPAT_VDSO 24 #include <asm/vdso/compat_gettimeofday.h> 25 #else 26 #include <asm/vdso/gettimeofday.h> 27 #endif /* ENABLE_COMPAT_VDSO */ 28 29 static int do_hres(const struct vdso_data *vd, clockid_t clk, 30 struct __kernel_timespec *ts) 31 { 32 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 33 u64 cycles, last, sec, ns; 34 u32 seq; 35 36 do { 37 seq = vdso_read_begin(vd); 38 cycles = __arch_get_hw_counter(vd->clock_mode) & 39 vd->mask; 40 ns = vdso_ts->nsec; 41 last = vd->cycle_last; 42 if (unlikely((s64)cycles < 0)) 43 return clock_gettime_fallback(clk, ts); 44 if (cycles > last) 45 ns += (cycles - last) * vd->mult; 46 ns >>= vd->shift; 47 sec = vdso_ts->sec; 48 } while (unlikely(vdso_read_retry(vd, seq))); 49 50 /* 51 * Do this outside the loop: a race inside the loop could result 52 * in __iter_div_u64_rem() being extremely slow. 53 */ 54 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); 55 ts->tv_nsec = ns; 56 57 return 0; 58 } 59 60 static void do_coarse(const struct vdso_data *vd, clockid_t clk, 61 struct __kernel_timespec *ts) 62 { 63 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk]; 64 u32 seq; 65 66 do { 67 seq = vdso_read_begin(vd); 68 ts->tv_sec = vdso_ts->sec; 69 ts->tv_nsec = vdso_ts->nsec; 70 } while (unlikely(vdso_read_retry(vd, seq))); 71 } 72 73 static __maybe_unused int 74 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) 75 { 76 const struct vdso_data *vd = __arch_get_vdso_data(); 77 u32 msk; 78 79 /* Check for negative values or invalid clocks */ 80 if (unlikely((u32) clock >= MAX_CLOCKS)) 81 goto fallback; 82 83 /* 84 * Convert the clockid to a bitmask and use it to check which 85 * clocks are handled in the VDSO directly. 86 */ 87 msk = 1U << clock; 88 if (likely(msk & VDSO_HRES)) { 89 return do_hres(&vd[CS_HRES_COARSE], clock, ts); 90 } else if (msk & VDSO_COARSE) { 91 do_coarse(&vd[CS_HRES_COARSE], clock, ts); 92 return 0; 93 } else if (msk & VDSO_RAW) { 94 return do_hres(&vd[CS_RAW], clock, ts); 95 } 96 97 fallback: 98 return clock_gettime_fallback(clock, ts); 99 } 100 101 static __maybe_unused int 102 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res) 103 { 104 struct __kernel_timespec ts; 105 int ret; 106 107 if (res == NULL) 108 goto fallback; 109 110 ret = __cvdso_clock_gettime(clock, &ts); 111 112 if (ret == 0) { 113 res->tv_sec = ts.tv_sec; 114 res->tv_nsec = ts.tv_nsec; 115 } 116 117 return ret; 118 119 fallback: 120 return clock_gettime_fallback(clock, (struct __kernel_timespec *)res); 121 } 122 123 static __maybe_unused int 124 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) 125 { 126 const struct vdso_data *vd = __arch_get_vdso_data(); 127 128 if (likely(tv != NULL)) { 129 struct __kernel_timespec ts; 130 131 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts)) 132 return gettimeofday_fallback(tv, tz); 133 134 tv->tv_sec = ts.tv_sec; 135 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC; 136 } 137 138 if (unlikely(tz != NULL)) { 139 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; 140 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; 141 } 142 143 return 0; 144 } 145 146 #ifdef VDSO_HAS_TIME 147 static __maybe_unused time_t __cvdso_time(time_t *time) 148 { 149 const struct vdso_data *vd = __arch_get_vdso_data(); 150 time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec); 151 152 if (time) 153 *time = t; 154 155 return t; 156 } 157 #endif /* VDSO_HAS_TIME */ 158 159 #ifdef VDSO_HAS_CLOCK_GETRES 160 static __maybe_unused 161 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res) 162 { 163 const struct vdso_data *vd = __arch_get_vdso_data(); 164 u64 ns; 165 u32 msk; 166 u64 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res); 167 168 /* Check for negative values or invalid clocks */ 169 if (unlikely((u32) clock >= MAX_CLOCKS)) 170 goto fallback; 171 172 /* 173 * Convert the clockid to a bitmask and use it to check which 174 * clocks are handled in the VDSO directly. 175 */ 176 msk = 1U << clock; 177 if (msk & VDSO_HRES) { 178 /* 179 * Preserves the behaviour of posix_get_hrtimer_res(). 180 */ 181 ns = hrtimer_res; 182 } else if (msk & VDSO_COARSE) { 183 /* 184 * Preserves the behaviour of posix_get_coarse_res(). 185 */ 186 ns = LOW_RES_NSEC; 187 } else if (msk & VDSO_RAW) { 188 /* 189 * Preserves the behaviour of posix_get_hrtimer_res(). 190 */ 191 ns = hrtimer_res; 192 } else { 193 goto fallback; 194 } 195 196 if (res) { 197 res->tv_sec = 0; 198 res->tv_nsec = ns; 199 } 200 201 return 0; 202 203 fallback: 204 return clock_getres_fallback(clock, res); 205 } 206 207 static __maybe_unused int 208 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res) 209 { 210 struct __kernel_timespec ts; 211 int ret; 212 213 if (res == NULL) 214 goto fallback; 215 216 ret = __cvdso_clock_getres(clock, &ts); 217 218 if (ret == 0) { 219 res->tv_sec = ts.tv_sec; 220 res->tv_nsec = ts.tv_nsec; 221 } 222 223 return ret; 224 225 fallback: 226 return clock_getres_fallback(clock, (struct __kernel_timespec *)res); 227 } 228 #endif /* VDSO_HAS_CLOCK_GETRES */ 229