1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/nstime.h" 5 6 #include "jemalloc/internal/assert.h" 7 8 #define BILLION UINT64_C(1000000000) 9 #define MILLION UINT64_C(1000000) 10 11 static void 12 nstime_set_initialized(nstime_t *time) { 13 #ifdef JEMALLOC_DEBUG 14 time->magic = NSTIME_MAGIC; 15 #endif 16 } 17 18 static void 19 nstime_assert_initialized(const nstime_t *time) { 20 #ifdef JEMALLOC_DEBUG 21 /* 22 * Some parts (e.g. stats) rely on memset to zero initialize. Treat 23 * these as valid initialization. 24 */ 25 assert(time->magic == NSTIME_MAGIC || 26 (time->magic == 0 && time->ns == 0)); 27 #endif 28 } 29 30 static void 31 nstime_pair_assert_initialized(const nstime_t *t1, const nstime_t *t2) { 32 nstime_assert_initialized(t1); 33 nstime_assert_initialized(t2); 34 } 35 36 static void 37 nstime_initialize_operand(nstime_t *time) { 38 /* 39 * Operations like nstime_add may have the initial operand being zero 40 * initialized (covered by the assert below). Full-initialize needed 41 * before changing it to non-zero. 42 */ 43 nstime_assert_initialized(time); 44 nstime_set_initialized(time); 45 } 46 47 void 48 nstime_init(nstime_t *time, uint64_t ns) { 49 nstime_set_initialized(time); 50 time->ns = ns; 51 } 52 53 void 54 nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec) { 55 nstime_set_initialized(time); 56 time->ns = sec * BILLION + nsec; 57 } 58 59 uint64_t 60 nstime_ns(const nstime_t *time) { 61 nstime_assert_initialized(time); 62 return time->ns; 63 } 64 65 uint64_t 66 nstime_msec(const nstime_t *time) { 67 nstime_assert_initialized(time); 68 return time->ns / MILLION; 69 } 70 71 uint64_t 72 nstime_sec(const nstime_t *time) { 73 nstime_assert_initialized(time); 74 return time->ns / BILLION; 75 } 76 77 uint64_t 78 nstime_nsec(const nstime_t *time) { 79 nstime_assert_initialized(time); 80 return time->ns % BILLION; 81 } 82 83 void 84 nstime_copy(nstime_t *time, const nstime_t *source) { 85 /* Source is required to be initialized. */ 86 nstime_assert_initialized(source); 87 *time = *source; 88 nstime_assert_initialized(time); 89 } 90 91 int 92 nstime_compare(const nstime_t *a, const nstime_t *b) { 93 nstime_pair_assert_initialized(a, b); 94 return (a->ns > b->ns) - (a->ns < b->ns); 95 } 96 97 void 98 nstime_add(nstime_t *time, const nstime_t *addend) { 99 nstime_pair_assert_initialized(time, addend); 100 assert(UINT64_MAX - time->ns >= addend->ns); 101 102 nstime_initialize_operand(time); 103 time->ns += addend->ns; 104 } 105 106 void 107 nstime_iadd(nstime_t *time, uint64_t addend) { 108 nstime_assert_initialized(time); 109 assert(UINT64_MAX - time->ns >= addend); 110 111 nstime_initialize_operand(time); 112 time->ns += addend; 113 } 114 115 void 116 nstime_subtract(nstime_t *time, const nstime_t *subtrahend) { 117 nstime_pair_assert_initialized(time, subtrahend); 118 assert(nstime_compare(time, subtrahend) >= 0); 119 120 /* No initialize operand -- subtraction must be initialized. */ 121 time->ns -= subtrahend->ns; 122 } 123 124 void 125 nstime_isubtract(nstime_t *time, uint64_t subtrahend) { 126 nstime_assert_initialized(time); 127 assert(time->ns >= subtrahend); 128 129 /* No initialize operand -- subtraction must be initialized. */ 130 time->ns -= subtrahend; 131 } 132 133 void 134 nstime_imultiply(nstime_t *time, uint64_t multiplier) { 135 nstime_assert_initialized(time); 136 assert((((time->ns | multiplier) & (UINT64_MAX << (sizeof(uint64_t) << 137 2))) == 0) || ((time->ns * multiplier) / multiplier == time->ns)); 138 139 nstime_initialize_operand(time); 140 time->ns *= multiplier; 141 } 142 143 void 144 nstime_idivide(nstime_t *time, uint64_t divisor) { 145 nstime_assert_initialized(time); 146 assert(divisor != 0); 147 148 nstime_initialize_operand(time); 149 time->ns /= divisor; 150 } 151 152 uint64_t 153 nstime_divide(const nstime_t *time, const nstime_t *divisor) { 154 nstime_pair_assert_initialized(time, divisor); 155 assert(divisor->ns != 0); 156 157 /* No initialize operand -- *time itself remains unchanged. */ 158 return time->ns / divisor->ns; 159 } 160 161 /* Returns time since *past, w/o updating *past. */ 162 uint64_t 163 nstime_ns_since(const nstime_t *past) { 164 nstime_assert_initialized(past); 165 166 nstime_t now; 167 nstime_copy(&now, past); 168 nstime_update(&now); 169 170 assert(nstime_compare(&now, past) >= 0); 171 return now.ns - past->ns; 172 } 173 174 #ifdef _WIN32 175 # define NSTIME_MONOTONIC true 176 static void 177 nstime_get(nstime_t *time) { 178 FILETIME ft; 179 uint64_t ticks_100ns; 180 181 GetSystemTimeAsFileTime(&ft); 182 ticks_100ns = (((uint64_t)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; 183 184 nstime_init(time, ticks_100ns * 100); 185 } 186 #elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE) 187 # define NSTIME_MONOTONIC true 188 static void 189 nstime_get(nstime_t *time) { 190 struct timespec ts; 191 192 clock_gettime(CLOCK_MONOTONIC_COARSE, &ts); 193 nstime_init2(time, ts.tv_sec, ts.tv_nsec); 194 } 195 #elif defined(JEMALLOC_HAVE_CLOCK_MONOTONIC) 196 # define NSTIME_MONOTONIC true 197 static void 198 nstime_get(nstime_t *time) { 199 struct timespec ts; 200 201 clock_gettime(CLOCK_MONOTONIC, &ts); 202 nstime_init2(time, ts.tv_sec, ts.tv_nsec); 203 } 204 #elif defined(JEMALLOC_HAVE_MACH_ABSOLUTE_TIME) 205 # define NSTIME_MONOTONIC true 206 static void 207 nstime_get(nstime_t *time) { 208 nstime_init(time, mach_absolute_time()); 209 } 210 #else 211 # define NSTIME_MONOTONIC false 212 static void 213 nstime_get(nstime_t *time) { 214 struct timeval tv; 215 216 gettimeofday(&tv, NULL); 217 nstime_init2(time, tv.tv_sec, tv.tv_usec * 1000); 218 } 219 #endif 220 221 static bool 222 nstime_monotonic_impl(void) { 223 return NSTIME_MONOTONIC; 224 #undef NSTIME_MONOTONIC 225 } 226 nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl; 227 228 prof_time_res_t opt_prof_time_res = 229 prof_time_res_default; 230 231 const char *prof_time_res_mode_names[] = { 232 "default", 233 "high", 234 }; 235 236 237 static void 238 nstime_get_realtime(nstime_t *time) { 239 #if defined(JEMALLOC_HAVE_CLOCK_REALTIME) && !defined(_WIN32) 240 struct timespec ts; 241 242 clock_gettime(CLOCK_REALTIME, &ts); 243 nstime_init2(time, ts.tv_sec, ts.tv_nsec); 244 #else 245 unreachable(); 246 #endif 247 } 248 249 static void 250 nstime_prof_update_impl(nstime_t *time) { 251 nstime_t old_time; 252 253 nstime_copy(&old_time, time); 254 255 if (opt_prof_time_res == prof_time_res_high) { 256 nstime_get_realtime(time); 257 } else { 258 nstime_get(time); 259 } 260 } 261 nstime_prof_update_t *JET_MUTABLE nstime_prof_update = nstime_prof_update_impl; 262 263 static void 264 nstime_update_impl(nstime_t *time) { 265 nstime_t old_time; 266 267 nstime_copy(&old_time, time); 268 nstime_get(time); 269 270 /* Handle non-monotonic clocks. */ 271 if (unlikely(nstime_compare(&old_time, time) > 0)) { 272 nstime_copy(time, &old_time); 273 } 274 } 275 nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl; 276 277 void 278 nstime_init_update(nstime_t *time) { 279 nstime_init_zero(time); 280 nstime_update(time); 281 } 282 283 void 284 nstime_prof_init_update(nstime_t *time) { 285 nstime_init_zero(time); 286 nstime_prof_update(time); 287 } 288 289 290