1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/dtrace.h> 30 #include <sys/cmn_err.h> 31 #include <sys/tnf.h> 32 #include <sys/atomic.h> 33 #include <sys/prsystm.h> 34 #include <sys/modctl.h> 35 #include <sys/aio_impl.h> 36 37 #ifdef __sparc 38 #include <sys/privregs.h> 39 #endif 40 41 void (*dtrace_cpu_init)(processorid_t); 42 void (*dtrace_modload)(struct modctl *); 43 void (*dtrace_modunload)(struct modctl *); 44 void (*dtrace_helpers_cleanup)(void); 45 void (*dtrace_helpers_fork)(proc_t *, proc_t *); 46 void (*dtrace_cpustart_init)(void); 47 void (*dtrace_cpustart_fini)(void); 48 49 void (*dtrace_debugger_init)(void); 50 void (*dtrace_debugger_fini)(void); 51 52 dtrace_vtime_state_t dtrace_vtime_active = 0; 53 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1; 54 55 typedef struct dtrace_hrestime { 56 lock_t dthr_lock; /* lock for this element */ 57 timestruc_t dthr_hrestime; /* hrestime value */ 58 int64_t dthr_adj; /* hrestime_adj value */ 59 hrtime_t dthr_hrtime; /* hrtime value */ 60 } dtrace_hrestime_t; 61 62 static dtrace_hrestime_t dtrace_hrestime[2]; 63 64 /* 65 * Making available adjustable high-resolution time in DTrace is regrettably 66 * more complicated than one might think it should be. The problem is that 67 * the variables related to adjusted high-resolution time (hrestime, 68 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may 69 * be held when we enter probe context. One might think that we could address 70 * this by having a single snapshot copy that is stored under a different lock 71 * from hres_tick(), using the snapshot iff hres_lock is locked in probe 72 * context. Unfortunately, this too won't work: because hres_lock is grabbed 73 * in more than just hres_tick() context, we could enter probe context 74 * concurrently on two different CPUs with both locks (hres_lock and the 75 * snapshot lock) held. As this implies, the fundamental problem is that we 76 * need to have access to a snapshot of these variables that we _know_ will 77 * not be locked in probe context. To effect this, we have two snapshots 78 * protected by two different locks, and we mandate that these snapshots are 79 * recorded in succession by a single thread calling dtrace_hres_tick(). (We 80 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls 81 * hres_tick().) A single thread can't be in two places at once: one of the 82 * snapshot locks is guaranteed to be unheld at all times. The 83 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then 84 * the other to find the unlocked snapshot. 85 */ 86 void 87 dtrace_hres_tick(void) 88 { 89 int i; 90 ushort_t spl; 91 92 for (i = 0; i < 2; i++) { 93 dtrace_hrestime_t tmp; 94 95 spl = hr_clock_lock(); 96 tmp.dthr_hrestime = hrestime; 97 tmp.dthr_adj = hrestime_adj; 98 tmp.dthr_hrtime = dtrace_gethrtime(); 99 hr_clock_unlock(spl); 100 101 lock_set(&dtrace_hrestime[i].dthr_lock); 102 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime; 103 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj; 104 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime; 105 dtrace_membar_producer(); 106 107 /* 108 * To allow for lock-free examination of this lock, we use 109 * the same trick that is used hres_lock; for more details, 110 * see the description of this technique in sun4u/sys/clock.h. 111 */ 112 dtrace_hrestime[i].dthr_lock++; 113 } 114 } 115 116 hrtime_t 117 dtrace_gethrestime(void) 118 { 119 dtrace_hrestime_t snap; 120 hrtime_t now; 121 int i = 0, adj, nslt; 122 123 for (;;) { 124 snap.dthr_lock = dtrace_hrestime[i].dthr_lock; 125 dtrace_membar_consumer(); 126 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime; 127 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime; 128 snap.dthr_adj = dtrace_hrestime[i].dthr_adj; 129 dtrace_membar_consumer(); 130 131 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock) 132 break; 133 134 /* 135 * If we're here, the lock was either locked, or it 136 * transitioned while we were taking the snapshot. Either 137 * way, we're going to try the other dtrace_hrestime element; 138 * we know that it isn't possible for both to be locked 139 * simultaneously, so we will ultimately get a good snapshot. 140 */ 141 i ^= 1; 142 } 143 144 /* 145 * We have a good snapshot. Now perform any necessary adjustments. 146 */ 147 nslt = dtrace_gethrtime() - snap.dthr_hrtime; 148 ASSERT(nslt >= 0); 149 150 now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) + 151 snap.dthr_hrestime.tv_nsec; 152 153 if (snap.dthr_adj != 0) { 154 if (snap.dthr_adj > 0) { 155 adj = (nslt >> adj_shift); 156 if (adj > snap.dthr_adj) 157 adj = (int)snap.dthr_adj; 158 } else { 159 adj = -(nslt >> adj_shift); 160 if (adj < snap.dthr_adj) 161 adj = (int)snap.dthr_adj; 162 } 163 now += adj; 164 } 165 166 return (now); 167 } 168 169 void 170 dtrace_vtime_enable(void) 171 { 172 dtrace_vtime_state_t state, nstate; 173 174 do { 175 state = dtrace_vtime_active; 176 177 switch (state) { 178 case DTRACE_VTIME_INACTIVE: 179 nstate = DTRACE_VTIME_ACTIVE; 180 break; 181 182 case DTRACE_VTIME_INACTIVE_TNF: 183 nstate = DTRACE_VTIME_ACTIVE_TNF; 184 break; 185 186 case DTRACE_VTIME_ACTIVE: 187 case DTRACE_VTIME_ACTIVE_TNF: 188 panic("DTrace virtual time already enabled"); 189 /*NOTREACHED*/ 190 } 191 192 } while (cas32((uint32_t *)&dtrace_vtime_active, 193 state, nstate) != state); 194 } 195 196 void 197 dtrace_vtime_disable(void) 198 { 199 dtrace_vtime_state_t state, nstate; 200 201 do { 202 state = dtrace_vtime_active; 203 204 switch (state) { 205 case DTRACE_VTIME_ACTIVE: 206 nstate = DTRACE_VTIME_INACTIVE; 207 break; 208 209 case DTRACE_VTIME_ACTIVE_TNF: 210 nstate = DTRACE_VTIME_INACTIVE_TNF; 211 break; 212 213 case DTRACE_VTIME_INACTIVE: 214 case DTRACE_VTIME_INACTIVE_TNF: 215 panic("DTrace virtual time already disabled"); 216 /*NOTREACHED*/ 217 } 218 219 } while (cas32((uint32_t *)&dtrace_vtime_active, 220 state, nstate) != state); 221 } 222 223 void 224 dtrace_vtime_enable_tnf(void) 225 { 226 dtrace_vtime_state_t state, nstate; 227 228 do { 229 state = dtrace_vtime_active; 230 231 switch (state) { 232 case DTRACE_VTIME_ACTIVE: 233 nstate = DTRACE_VTIME_ACTIVE_TNF; 234 break; 235 236 case DTRACE_VTIME_INACTIVE: 237 nstate = DTRACE_VTIME_INACTIVE_TNF; 238 break; 239 240 case DTRACE_VTIME_ACTIVE_TNF: 241 case DTRACE_VTIME_INACTIVE_TNF: 242 panic("TNF already active"); 243 /*NOTREACHED*/ 244 } 245 246 } while (cas32((uint32_t *)&dtrace_vtime_active, 247 state, nstate) != state); 248 } 249 250 void 251 dtrace_vtime_disable_tnf(void) 252 { 253 dtrace_vtime_state_t state, nstate; 254 255 do { 256 state = dtrace_vtime_active; 257 258 switch (state) { 259 case DTRACE_VTIME_ACTIVE_TNF: 260 nstate = DTRACE_VTIME_ACTIVE; 261 break; 262 263 case DTRACE_VTIME_INACTIVE_TNF: 264 nstate = DTRACE_VTIME_INACTIVE; 265 break; 266 267 case DTRACE_VTIME_ACTIVE: 268 case DTRACE_VTIME_INACTIVE: 269 panic("TNF already inactive"); 270 /*NOTREACHED*/ 271 } 272 273 } while (cas32((uint32_t *)&dtrace_vtime_active, 274 state, nstate) != state); 275 } 276 277 void 278 dtrace_vtime_switch(kthread_t *next) 279 { 280 dtrace_icookie_t cookie; 281 hrtime_t ts; 282 283 if (tnf_tracing_active) { 284 tnf_thread_switch(next); 285 286 if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF) 287 return; 288 } 289 290 cookie = dtrace_interrupt_disable(); 291 ts = dtrace_gethrtime(); 292 293 if (curthread->t_dtrace_start != 0) { 294 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start; 295 curthread->t_dtrace_start = 0; 296 } 297 298 next->t_dtrace_start = ts; 299 300 dtrace_interrupt_enable(cookie); 301 } 302 303 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *); 304 void (*dtrace_fasttrap_exec_ptr)(proc_t *); 305 void (*dtrace_fasttrap_exit_ptr)(proc_t *); 306 307 /* 308 * This function is called by cfork() in the event that it appears that 309 * there may be dtrace tracepoints active in the parent process's address 310 * space. This first confirms the existence of dtrace tracepoints in the 311 * parent process and calls into the fasttrap module to remove the 312 * corresponding tracepoints from the child. By knowing that there are 313 * existing tracepoints, and ensuring they can't be removed, we can rely 314 * on the fasttrap module remaining loaded. 315 */ 316 void 317 dtrace_fasttrap_fork(proc_t *p, proc_t *cp) 318 { 319 ASSERT(p->p_proc_flag & P_PR_LOCK); 320 ASSERT(p->p_dtrace_count > 0); 321 ASSERT(dtrace_fasttrap_fork_ptr != NULL); 322 323 dtrace_fasttrap_fork_ptr(p, cp); 324 } 325