1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #ifndef _SYS_CLOCK_H 27 #define _SYS_CLOCK_H 28 29 #ifdef __cplusplus 30 extern "C" { 31 #endif 32 33 #include <sys/spl.h> 34 #include <sys/time.h> 35 #include <sys/machclock.h> 36 37 #ifndef _ASM 38 39 #ifdef _KERNEL 40 41 extern void setcpudelay(void); 42 43 extern uint_t nsec_scale; 44 extern uint_t nsec_shift; 45 extern uint_t nsec_per_sys_tick; 46 extern uint64_t sys_tick_freq; 47 48 extern int traptrace_use_stick; 49 extern uint64_t system_clock_freq; 50 extern uint_t sys_clock_mhz; 51 52 extern void mon_clock_init(void); 53 extern void mon_clock_start(void); 54 extern void mon_clock_stop(void); 55 extern void mon_clock_share(void); 56 extern void mon_clock_unshare(void); 57 58 extern hrtime_t hrtime_base; 59 extern void hres_tick(void); 60 extern void clkstart(void); 61 extern void cbe_level14(); 62 extern hrtime_t tick2ns(hrtime_t, uint_t); 63 64 typedef struct { 65 uint64_t cbe_level1_inum; 66 uint64_t cbe_level10_inum; 67 } cbe_data_t; 68 69 #endif /* _KERNEL */ 70 71 #endif /* _ASM */ 72 73 74 #define CBE_LOW_PIL 1 75 #define CBE_LOCK_PIL LOCK_LEVEL 76 #define CBE_HIGH_PIL 14 77 78 #define ADJ_SHIFT 4 /* used in get_hrestime and _level10 */ 79 80 /* 81 * Locking strategy for high-resolution timing services 82 * 83 * We generally construct timestamps from two or more components: 84 * a hardware time source and one or more software time sources. 85 * These components cannot all be loaded simultaneously, so we need 86 * some sort of locking strategy to generate consistent timestamps. 87 * 88 * To minimize lock contention and cache thrashing we employ the 89 * weakest possible synchronization model: writers (rare) serialize 90 * on an acquisition-counting mutex, described below; readers (common) 91 * execute in parallel with no synchronization at all -- they don't 92 * exclude other readers, and they don't even exclude writers. Instead, 93 * readers just examine the writer lock's value before and after loading 94 * all the components of a timestamp to detect writer intervention. 95 * In the rare case when a writer does intervene, the reader will 96 * detect it, discard the timestamp and try again. 97 * 98 * The writer lock, hres_lock, is a 32-bit integer consisting of an 99 * 8-bit lock and a 24-bit acquisition count. To acquire the lock we 100 * set the lock field with ldstub, which sets the low-order 8 bits to 101 * 0xff; to clear the lock, we increment it, which simultaneously clears 102 * the lock field (0xff --> 0x00) and increments the acquisition count 103 * (due to carry into bit 8). Thus each acquisition transforms hres_lock 104 * from N:0 to N:ff, and each release transforms N:ff into (N+1):0. 105 * 106 * Readers can detect writer intervention by loading hres_lock before 107 * and after loading the time components they need; if either lock value 108 * contains 0xff in the low-order bits (lock held), or if the lock values 109 * are not equal (lock was acquired and released), a writer intervened 110 * and the reader must try again. If the lock values are equal and the 111 * low-order 8 bits are clear, the timestamp must be valid. We can check 112 * both of these conditions with a single compare instruction by checking 113 * whether old_hres_lock & ~1 == new_hres_lock, as illustrated by the 114 * following table of all possible lock states: 115 * 116 * initial & ~1 final result of compare 117 * ------------ ----- ----------------- 118 * now:00 now:00 valid 119 * now:00 now:ff invalid 120 * now:00 later:00 invalid 121 * now:00 later:ff invalid 122 * now:fe now:ff invalid 123 * now:fe later:00 invalid 124 * now:fe later:ff invalid 125 * 126 * Implementation considerations: 127 * 128 * (1) Load buffering. 129 * 130 * On a CPU that does load buffering we must ensure that the load of 131 * hres_lock completes before the load of any timestamp components. 132 * This is essential *even on a CPU that does in-order loads* because 133 * accessing the hardware time source may not involve a memory reference 134 * (e.g. rd %tick). A convenient way to address this is to clear the 135 * lower bit (andn with 1) of the old lock value right away, since this 136 * generates a dependency on the load of hres_lock. We have to do this 137 * anyway to perform the lock comparison described above. 138 * 139 * (2) Out-of-order loads. 140 * 141 * On a CPU that does out-of-order loads we must ensure that the loads 142 * of all timestamp components have completed before we load the final 143 * value of hres_lock. This can be done either by generating load 144 * dependencies on the timestamp components or by membar #LoadLoad. 145 * 146 * (3) Interaction with the high level cyclic handler, hres_tick(). 147 * 148 * One unusual property of hres_lock is that it's acquired in a high 149 * level cyclic handler, hres_tick(). Thus, hres_lock must be acquired at 150 * CBE_HIGH_PIL or higher to prevent single-CPU deadlock. 151 * 152 * (4) Cross-calls. 153 * 154 * If a cross-call happens while one CPU has hres_lock and another is 155 * trying to acquire it in the clock interrupt path, the system will 156 * deadlock: the first CPU will never release hres_lock since it's 157 * waiting to be released from the cross-call, and the cross-call can't 158 * complete because the second CPU is spinning on hres_lock with traps 159 * disabled. Thus cross-calls must be blocked while holding hres_lock. 160 * 161 * Together, (3) and (4) imply that hres_lock should only be acquired 162 * at PIL >= max(XCALL_PIL, CBE_HIGH_PIL), or while traps are disabled. 163 */ 164 #define HRES_LOCK_OFFSET 3 165 166 #define CLOCK_LOCK(oldsplp) \ 167 lock_set_spl((lock_t *)&hres_lock + HRES_LOCK_OFFSET, \ 168 ipltospl(CBE_HIGH_PIL), oldsplp) 169 170 #define CLOCK_UNLOCK(spl) \ 171 membar_ldst_stst(); \ 172 hres_lock++; \ 173 splx(spl); \ 174 LOCKSTAT_RECORD0(LS_CLOCK_UNLOCK_RELEASE, \ 175 (lock_t *)&hres_lock + HRES_LOCK_OFFSET); 176 177 /* 178 * NATIVE_TIME_TO_NSEC_SCALE is called with NSEC_SHIFT to convert hi-res 179 * timestamps into nanoseconds. On systems that have a %stick register, 180 * hi-res timestamps are in %stick units. On systems that do not have a 181 * %stick register, hi-res timestamps are in %tick units. 182 * 183 * NATIVE_TIME_TO_NSEC_SCALE is called with TICK_NSEC_SHIFT to convert from 184 * %tick units to nanoseconds on all implementations whether %stick is 185 * available or not. 186 */ 187 188 /* 189 * At least 62.5 MHz CPU %tick frequency 190 */ 191 192 #define TICK_NSEC_SHIFT 4 193 194 /* 195 * Convert hi-res native time (V9's %tick in our case) into nanoseconds. 196 * 197 * The challenge is to multiply a %tick value by (NANOSEC / sys_tick_freq) 198 * without using floating point and without overflowing 64-bit integers. 199 * We assume that all sun4u systems will have a 16 nsec or better clock 200 * (i.e. faster than 62.5 MHz), which means that (ticks << 4) has units 201 * greater than one nanosecond, so converting from (ticks << 4) to nsec 202 * requires multiplication by a rational number, R, between 0 and 1. 203 * To avoid floating-point we precompute (R * 2^32) during boot and 204 * stash this away in nsec_scale. Thus we can compute (tick * R) as 205 * (tick * nsec_scale) >> 32, which is accurate to about 1 part per billion. 206 * 207 * To avoid 64-bit overflow when multiplying (tick << 4) by nsec_scale, 208 * we split (tick << 4) into its high and low 32-bit pieces, H and L, 209 * multiply each piece separately, and add up the relevant bits of the 210 * partial products. Putting it all together we have: 211 * 212 * nsec = (tick << 4) * R 213 * = ((tick << 4) * nsec_scale) >> 32 214 * = ((H << 32) + L) * nsec_scale) >> 32 215 * = (H * nsec_scale) + ((L * nsec_scale) >> 32) 216 * 217 * The last line is the computation we actually perform: it requires no 218 * floating point and all intermediate results fit in 64-bit registers. 219 * 220 * Note that we require that tick is less than (1 << (64 - NSEC_SHIFT)); 221 * greater values will result in overflow and misbehavior (not that this 222 * is a serious problem; (1 << (64 - NSEC_SHIFT)) nanoseconds is over 223 * thirty-six years). Nonetheless, clients may wish to be aware of this 224 * limitation; NATIVE_TIME_MAX() returns this maximum native time. 225 * 226 * We provide two versions of this macro: a "full-service" version that 227 * just converts ticks to nanoseconds and a higher-performance version that 228 * expects the scaling factor nsec_scale as its second argument (so that 229 * callers can distance the load of nsec_scale from its use). Note that 230 * we take a fast path if we determine the ticks to be less than 32 bits 231 * (as it often is for the delta between %tick values for successive 232 * firings of the hres_tick() cyclic). 233 * 234 * Note that in the 32-bit path we don't even bother clearing NPT. 235 * We get away with this by making hardclk.c ensure than nsec_scale 236 * is even, so we can take advantage of the associativity of modular 237 * arithmetic: multiplying %tick by any even number, say 2*n, is 238 * equivalent to multiplying %tick by 2, then by n. Multiplication 239 * by 2 is equivalent to shifting left by one, which clears NPT. 240 * 241 * Finally, note that the macros use the labels "6:" and "7:"; these 242 * labels must not be used across an invocation of either macro. 243 */ 244 #define NATIVE_TIME_TO_NSEC_SCALE(out, scr1, scr2, shift) \ 245 srlx out, 32, scr2; /* check high 32 bits */ \ 246 /* CSTYLED */ \ 247 brz,a,pt scr2, 6f; /* if clear, 32-bit fast path */\ 248 mulx out, scr1, out; /* delay: 32-bit fast path */ \ 249 sllx out, shift, out; /* clear NPT and pre-scale */ \ 250 srlx out, 32, scr2; /* scr2 = hi32(tick<<4) = H */ \ 251 mulx scr2, scr1, scr2; /* scr2 = (H*F) */ \ 252 srl out, 0, out; /* out = lo32(tick<<4) = L */ \ 253 mulx out, scr1, scr1; /* scr1 = (L*F) */ \ 254 srlx scr1, 32, scr1; /* scr1 = (L*F) >> 32 */ \ 255 ba 7f; /* branch over 32-bit path */ \ 256 add scr1, scr2, out; /* out = (H*F) + ((L*F) >> 32) */\ 257 6: \ 258 srlx out, 32 - shift, out; \ 259 7: 260 261 #define NATIVE_TIME_TO_NSEC(out, scr1, scr2) \ 262 sethi %hi(nsec_scale), scr1; /* load scaling factor */ \ 263 ld [scr1 + %lo(nsec_scale)], scr1; \ 264 NATIVE_TIME_TO_NSEC_SCALE(out, scr1, scr2, NSEC_SHIFT); 265 266 #define NATIVE_TIME_MAX(out) \ 267 mov -1, out; \ 268 srlx out, NSEC_SHIFT, out 269 270 /* 271 * NSEC_SHIFT and VTRACE_SHIFT constants are defined in 272 * <sys/machclock.h> file. 273 */ 274 275 #ifdef __cplusplus 276 } 277 #endif 278 279 #endif /* !_SYS_CLOCK_H */ 280