1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 #include <sys/dtrace.h> 30 #include <sys/cmn_err.h> 31 #include <sys/tnf.h> 32 #include <sys/atomic.h> 33 #include <sys/prsystm.h> 34 #include <sys/modctl.h> 35 #include <sys/aio_impl.h> 36 37 #ifdef __sparc 38 #include <sys/privregs.h> 39 #endif 40 41 void (*dtrace_cpu_init)(processorid_t); 42 void (*dtrace_modload)(struct modctl *); 43 void (*dtrace_modunload)(struct modctl *); 44 void (*dtrace_helpers_cleanup)(void); 45 void (*dtrace_helpers_fork)(proc_t *, proc_t *); 46 void (*dtrace_cpustart_init)(void); 47 void (*dtrace_cpustart_fini)(void); 48 49 void (*dtrace_kreloc_init)(void); 50 void (*dtrace_kreloc_fini)(void); 51 52 void (*dtrace_debugger_init)(void); 53 void (*dtrace_debugger_fini)(void); 54 55 dtrace_vtime_state_t dtrace_vtime_active = 0; 56 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1; 57 58 typedef struct dtrace_hrestime { 59 lock_t dthr_lock; /* lock for this element */ 60 timestruc_t dthr_hrestime; /* hrestime value */ 61 int64_t dthr_adj; /* hrestime_adj value */ 62 hrtime_t dthr_hrtime; /* hrtime value */ 63 } dtrace_hrestime_t; 64 65 static dtrace_hrestime_t dtrace_hrestime[2]; 66 67 /* 68 * Making available adjustable high-resolution time in DTrace is regrettably 69 * more complicated than one might think it should be. The problem is that 70 * the variables related to adjusted high-resolution time (hrestime, 71 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may 72 * be held when we enter probe context. One might think that we could address 73 * this by having a single snapshot copy that is stored under a different lock 74 * from hres_tick(), using the snapshot iff hres_lock is locked in probe 75 * context. Unfortunately, this too won't work: because hres_lock is grabbed 76 * in more than just hres_tick() context, we could enter probe context 77 * concurrently on two different CPUs with both locks (hres_lock and the 78 * snapshot lock) held. As this implies, the fundamental problem is that we 79 * need to have access to a snapshot of these variables that we _know_ will 80 * not be locked in probe context. To effect this, we have two snapshots 81 * protected by two different locks, and we mandate that these snapshots are 82 * recorded in succession by a single thread calling dtrace_hres_tick(). (We 83 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls 84 * hres_tick().) A single thread can't be in two places at once: one of the 85 * snapshot locks is guaranteed to be unheld at all times. The 86 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then 87 * the other to find the unlocked snapshot. 88 */ 89 void 90 dtrace_hres_tick(void) 91 { 92 int i; 93 ushort_t spl; 94 95 for (i = 0; i < 2; i++) { 96 dtrace_hrestime_t tmp; 97 98 spl = hr_clock_lock(); 99 tmp.dthr_hrestime = hrestime; 100 tmp.dthr_adj = hrestime_adj; 101 tmp.dthr_hrtime = dtrace_gethrtime(); 102 hr_clock_unlock(spl); 103 104 lock_set(&dtrace_hrestime[i].dthr_lock); 105 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime; 106 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj; 107 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime; 108 dtrace_membar_producer(); 109 110 /* 111 * To allow for lock-free examination of this lock, we use 112 * the same trick that is used hres_lock; for more details, 113 * see the description of this technique in sun4u/sys/clock.h. 114 */ 115 dtrace_hrestime[i].dthr_lock++; 116 } 117 } 118 119 hrtime_t 120 dtrace_gethrestime(void) 121 { 122 dtrace_hrestime_t snap; 123 hrtime_t now; 124 int i = 0, adj, nslt; 125 126 for (;;) { 127 snap.dthr_lock = dtrace_hrestime[i].dthr_lock; 128 dtrace_membar_consumer(); 129 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime; 130 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime; 131 snap.dthr_adj = dtrace_hrestime[i].dthr_adj; 132 dtrace_membar_consumer(); 133 134 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock) 135 break; 136 137 /* 138 * If we're here, the lock was either locked, or it 139 * transitioned while we were taking the snapshot. Either 140 * way, we're going to try the other dtrace_hrestime element; 141 * we know that it isn't possible for both to be locked 142 * simultaneously, so we will ultimately get a good snapshot. 143 */ 144 i ^= 1; 145 } 146 147 /* 148 * We have a good snapshot. Now perform any necessary adjustments. 149 */ 150 nslt = dtrace_gethrtime() - snap.dthr_hrtime; 151 ASSERT(nslt >= 0); 152 153 now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) + 154 snap.dthr_hrestime.tv_nsec; 155 156 if (snap.dthr_adj != 0) { 157 if (snap.dthr_adj > 0) { 158 adj = (nslt >> adj_shift); 159 if (adj > snap.dthr_adj) 160 adj = (int)snap.dthr_adj; 161 } else { 162 adj = -(nslt >> adj_shift); 163 if (adj < snap.dthr_adj) 164 adj = (int)snap.dthr_adj; 165 } 166 now += adj; 167 } 168 169 return (now); 170 } 171 172 void 173 dtrace_vtime_enable(void) 174 { 175 dtrace_vtime_state_t state, nstate; 176 177 do { 178 state = dtrace_vtime_active; 179 180 switch (state) { 181 case DTRACE_VTIME_INACTIVE: 182 nstate = DTRACE_VTIME_ACTIVE; 183 break; 184 185 case DTRACE_VTIME_INACTIVE_TNF: 186 nstate = DTRACE_VTIME_ACTIVE_TNF; 187 break; 188 189 case DTRACE_VTIME_ACTIVE: 190 case DTRACE_VTIME_ACTIVE_TNF: 191 panic("DTrace virtual time already enabled"); 192 /*NOTREACHED*/ 193 } 194 195 } while (cas32((uint32_t *)&dtrace_vtime_active, 196 state, nstate) != state); 197 } 198 199 void 200 dtrace_vtime_disable(void) 201 { 202 dtrace_vtime_state_t state, nstate; 203 204 do { 205 state = dtrace_vtime_active; 206 207 switch (state) { 208 case DTRACE_VTIME_ACTIVE: 209 nstate = DTRACE_VTIME_INACTIVE; 210 break; 211 212 case DTRACE_VTIME_ACTIVE_TNF: 213 nstate = DTRACE_VTIME_INACTIVE_TNF; 214 break; 215 216 case DTRACE_VTIME_INACTIVE: 217 case DTRACE_VTIME_INACTIVE_TNF: 218 panic("DTrace virtual time already disabled"); 219 /*NOTREACHED*/ 220 } 221 222 } while (cas32((uint32_t *)&dtrace_vtime_active, 223 state, nstate) != state); 224 } 225 226 void 227 dtrace_vtime_enable_tnf(void) 228 { 229 dtrace_vtime_state_t state, nstate; 230 231 do { 232 state = dtrace_vtime_active; 233 234 switch (state) { 235 case DTRACE_VTIME_ACTIVE: 236 nstate = DTRACE_VTIME_ACTIVE_TNF; 237 break; 238 239 case DTRACE_VTIME_INACTIVE: 240 nstate = DTRACE_VTIME_INACTIVE_TNF; 241 break; 242 243 case DTRACE_VTIME_ACTIVE_TNF: 244 case DTRACE_VTIME_INACTIVE_TNF: 245 panic("TNF already active"); 246 /*NOTREACHED*/ 247 } 248 249 } while (cas32((uint32_t *)&dtrace_vtime_active, 250 state, nstate) != state); 251 } 252 253 void 254 dtrace_vtime_disable_tnf(void) 255 { 256 dtrace_vtime_state_t state, nstate; 257 258 do { 259 state = dtrace_vtime_active; 260 261 switch (state) { 262 case DTRACE_VTIME_ACTIVE_TNF: 263 nstate = DTRACE_VTIME_ACTIVE; 264 break; 265 266 case DTRACE_VTIME_INACTIVE_TNF: 267 nstate = DTRACE_VTIME_INACTIVE; 268 break; 269 270 case DTRACE_VTIME_ACTIVE: 271 case DTRACE_VTIME_INACTIVE: 272 panic("TNF already inactive"); 273 /*NOTREACHED*/ 274 } 275 276 } while (cas32((uint32_t *)&dtrace_vtime_active, 277 state, nstate) != state); 278 } 279 280 void 281 dtrace_vtime_switch(kthread_t *next) 282 { 283 dtrace_icookie_t cookie; 284 hrtime_t ts; 285 286 if (tnf_tracing_active) { 287 tnf_thread_switch(next); 288 289 if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF) 290 return; 291 } 292 293 cookie = dtrace_interrupt_disable(); 294 ts = dtrace_gethrtime(); 295 296 if (curthread->t_dtrace_start != 0) { 297 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start; 298 curthread->t_dtrace_start = 0; 299 } 300 301 next->t_dtrace_start = ts; 302 303 dtrace_interrupt_enable(cookie); 304 } 305 306 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *); 307 void (*dtrace_fasttrap_exec_ptr)(proc_t *); 308 void (*dtrace_fasttrap_exit_ptr)(proc_t *); 309 310 /* 311 * This function is called by cfork() in the event that it appears that 312 * there may be dtrace tracepoints active in the parent process's address 313 * space. This first confirms the existence of dtrace tracepoints in the 314 * parent process and calls into the fasttrap module to remove the 315 * corresponding tracepoints from the child. By knowing that there are 316 * existing tracepoints, and ensuring they can't be removed, we can rely 317 * on the fasttrap module remaining loaded. 318 */ 319 void 320 dtrace_fasttrap_fork(proc_t *p, proc_t *cp) 321 { 322 ASSERT(MUTEX_HELD(&p->p_lock)); 323 ASSERT(p->p_proc_flag & P_PR_LOCK); 324 mutex_exit(&p->p_lock); 325 326 /* 327 * If any tracepoints exist, call into the fasttrap module to remove 328 * those tracepoints. We can rely on that module being loaded since 329 * the extant tracepoints can't be removed while we hold P_PR_LOCK. 330 */ 331 if (p->p_dtrace_count > 0) { 332 ASSERT(dtrace_fasttrap_fork_ptr != NULL); 333 dtrace_fasttrap_fork_ptr(p, cp); 334 } 335 336 mutex_enter(&p->p_lock); 337 } 338