1 /* 2 * Copyright 2006 Andi Kleen, SUSE Labs. 3 * Subject to the GNU Public License, v.2 4 * 5 * Fast user context implementation of clock_gettime, gettimeofday, and time. 6 * 7 * The code should have no internal unresolved relocations. 8 * Check with readelf after changing. 9 * Also alternative() doesn't work. 10 */ 11 /* 12 * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. 13 */ 14 15 /* Disable profiling for userspace code: */ 16 #ifndef DISABLE_BRANCH_PROFILING 17 #define DISABLE_BRANCH_PROFILING 18 #endif 19 20 #include <linux/kernel.h> 21 #include <linux/time.h> 22 #include <linux/string.h> 23 #include <asm/io.h> 24 #include <asm/unistd.h> 25 #include <asm/timex.h> 26 #include <asm/clocksource.h> 27 #include <asm/vvar.h> 28 29 #undef TICK_PRIV_BIT 30 #ifdef CONFIG_SPARC64 31 #define TICK_PRIV_BIT (1UL << 63) 32 #else 33 #define TICK_PRIV_BIT (1ULL << 63) 34 #endif 35 36 #ifdef CONFIG_SPARC64 37 #define SYSCALL_STRING \ 38 "ta 0x6d;" \ 39 "bcs,a 1f;" \ 40 " sub %%g0, %%o0, %%o0;" \ 41 "1:" 42 #else 43 #define SYSCALL_STRING \ 44 "ta 0x10;" \ 45 "bcs,a 1f;" \ 46 " sub %%g0, %%o0, %%o0;" \ 47 "1:" 48 #endif 49 50 #define SYSCALL_CLOBBERS \ 51 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ 52 "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ 53 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ 54 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ 55 "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ 56 "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ 57 "cc", "memory" 58 59 /* 60 * Compute the vvar page's address in the process address space, and return it 61 * as a pointer to the vvar_data. 62 */ 63 static notrace noinline struct vvar_data * 64 get_vvar_data(void) 65 { 66 unsigned long ret; 67 68 /* 69 * vdso data page is the first vDSO page so grab the return address 70 * and move up a page to get to the data page. 71 */ 72 ret = (unsigned long)__builtin_return_address(0); 73 ret &= ~(8192 - 1); 74 ret -= 8192; 75 76 return (struct vvar_data *) ret; 77 } 78 79 static notrace long 80 vdso_fallback_gettime(long clock, struct timespec *ts) 81 { 82 register long num __asm__("g1") = __NR_clock_gettime; 83 register long o0 __asm__("o0") = clock; 84 register long o1 __asm__("o1") = (long) ts; 85 86 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), 87 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); 88 return o0; 89 } 90 91 static notrace __always_inline long 92 vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz) 93 { 94 register long num __asm__("g1") = __NR_gettimeofday; 95 register long o0 __asm__("o0") = (long) tv; 96 register long o1 __asm__("o1") = (long) tz; 97 98 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), 99 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); 100 return o0; 101 } 102 103 #ifdef CONFIG_SPARC64 104 static notrace noinline u64 105 vread_tick(void) { 106 u64 ret; 107 108 __asm__ __volatile__("rd %%asr24, %0 \n" 109 ".section .vread_tick_patch, \"ax\" \n" 110 "rd %%tick, %0 \n" 111 ".previous \n" 112 : "=&r" (ret)); 113 return ret & ~TICK_PRIV_BIT; 114 } 115 #else 116 static notrace noinline u64 117 vread_tick(void) 118 { 119 unsigned int lo, hi; 120 121 __asm__ __volatile__("rd %%asr24, %%g1\n\t" 122 "srlx %%g1, 32, %1\n\t" 123 "srl %%g1, 0, %0\n" 124 ".section .vread_tick_patch, \"ax\" \n" 125 "rd %%tick, %%g1\n" 126 ".previous \n" 127 : "=&r" (lo), "=&r" (hi) 128 : 129 : "g1"); 130 return lo | ((u64)hi << 32); 131 } 132 #endif 133 134 static notrace inline u64 135 vgetsns(struct vvar_data *vvar) 136 { 137 u64 v; 138 u64 cycles; 139 140 cycles = vread_tick(); 141 v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; 142 return v * vvar->clock.mult; 143 } 144 145 static notrace noinline int 146 do_realtime(struct vvar_data *vvar, struct timespec *ts) 147 { 148 unsigned long seq; 149 u64 ns; 150 151 ts->tv_nsec = 0; 152 do { 153 seq = vvar_read_begin(vvar); 154 ts->tv_sec = vvar->wall_time_sec; 155 ns = vvar->wall_time_snsec; 156 ns += vgetsns(vvar); 157 ns >>= vvar->clock.shift; 158 } while (unlikely(vvar_read_retry(vvar, seq))); 159 160 timespec_add_ns(ts, ns); 161 162 return 0; 163 } 164 165 static notrace noinline int 166 do_monotonic(struct vvar_data *vvar, struct timespec *ts) 167 { 168 unsigned long seq; 169 u64 ns; 170 171 ts->tv_nsec = 0; 172 do { 173 seq = vvar_read_begin(vvar); 174 ts->tv_sec = vvar->monotonic_time_sec; 175 ns = vvar->monotonic_time_snsec; 176 ns += vgetsns(vvar); 177 ns >>= vvar->clock.shift; 178 } while (unlikely(vvar_read_retry(vvar, seq))); 179 180 timespec_add_ns(ts, ns); 181 182 return 0; 183 } 184 185 static notrace noinline int 186 do_realtime_coarse(struct vvar_data *vvar, struct timespec *ts) 187 { 188 unsigned long seq; 189 190 do { 191 seq = vvar_read_begin(vvar); 192 ts->tv_sec = vvar->wall_time_coarse_sec; 193 ts->tv_nsec = vvar->wall_time_coarse_nsec; 194 } while (unlikely(vvar_read_retry(vvar, seq))); 195 return 0; 196 } 197 198 static notrace noinline int 199 do_monotonic_coarse(struct vvar_data *vvar, struct timespec *ts) 200 { 201 unsigned long seq; 202 203 do { 204 seq = vvar_read_begin(vvar); 205 ts->tv_sec = vvar->monotonic_time_coarse_sec; 206 ts->tv_nsec = vvar->monotonic_time_coarse_nsec; 207 } while (unlikely(vvar_read_retry(vvar, seq))); 208 209 return 0; 210 } 211 212 notrace int 213 __vdso_clock_gettime(clockid_t clock, struct timespec *ts) 214 { 215 struct vvar_data *vvd = get_vvar_data(); 216 217 switch (clock) { 218 case CLOCK_REALTIME: 219 if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) 220 break; 221 return do_realtime(vvd, ts); 222 case CLOCK_MONOTONIC: 223 if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) 224 break; 225 return do_monotonic(vvd, ts); 226 case CLOCK_REALTIME_COARSE: 227 return do_realtime_coarse(vvd, ts); 228 case CLOCK_MONOTONIC_COARSE: 229 return do_monotonic_coarse(vvd, ts); 230 } 231 /* 232 * Unknown clock ID ? Fall back to the syscall. 233 */ 234 return vdso_fallback_gettime(clock, ts); 235 } 236 int 237 clock_gettime(clockid_t, struct timespec *) 238 __attribute__((weak, alias("__vdso_clock_gettime"))); 239 240 notrace int 241 __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) 242 { 243 struct vvar_data *vvd = get_vvar_data(); 244 245 if (likely(vvd->vclock_mode != VCLOCK_NONE)) { 246 if (likely(tv != NULL)) { 247 union tstv_t { 248 struct timespec ts; 249 struct timeval tv; 250 } *tstv = (union tstv_t *) tv; 251 do_realtime(vvd, &tstv->ts); 252 /* 253 * Assign before dividing to ensure that the division is 254 * done in the type of tv_usec, not tv_nsec. 255 * 256 * There cannot be > 1 billion usec in a second: 257 * do_realtime() has already distributed such overflow 258 * into tv_sec. So we can assign it to an int safely. 259 */ 260 tstv->tv.tv_usec = tstv->ts.tv_nsec; 261 tstv->tv.tv_usec /= 1000; 262 } 263 if (unlikely(tz != NULL)) { 264 /* Avoid memcpy. Some old compilers fail to inline it */ 265 tz->tz_minuteswest = vvd->tz_minuteswest; 266 tz->tz_dsttime = vvd->tz_dsttime; 267 } 268 return 0; 269 } 270 return vdso_fallback_gettimeofday(tv, tz); 271 } 272 int 273 gettimeofday(struct timeval *, struct timezone *) 274 __attribute__((weak, alias("__vdso_gettimeofday"))); 275