1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/dtrace.h> 28 #include <sys/cmn_err.h> 29 #include <sys/tnf.h> 30 #include <sys/atomic.h> 31 #include <sys/prsystm.h> 32 #include <sys/modctl.h> 33 #include <sys/aio_impl.h> 34 35 #ifdef __sparc 36 #include <sys/privregs.h> 37 #endif 38 39 void (*dtrace_cpu_init)(processorid_t); 40 void (*dtrace_modload)(struct modctl *); 41 void (*dtrace_modunload)(struct modctl *); 42 void (*dtrace_helpers_cleanup)(void); 43 void (*dtrace_helpers_fork)(proc_t *, proc_t *); 44 void (*dtrace_cpustart_init)(void); 45 void (*dtrace_cpustart_fini)(void); 46 void (*dtrace_cpc_fire)(uint64_t); 47 void (*dtrace_closef)(void); 48 49 void (*dtrace_debugger_init)(void); 50 void (*dtrace_debugger_fini)(void); 51 52 dtrace_vtime_state_t dtrace_vtime_active = 0; 53 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1; 54 55 /* 56 * dtrace_cpc_in_use usage statement: this global variable is used by the cpc 57 * hardware overflow interrupt handler and the kernel cpc framework to check 58 * whether or not the DTrace cpc provider is currently in use. The variable is 59 * set before counters are enabled with the first enabling and cleared when 60 * the last enabling is disabled. Its value at any given time indicates the 61 * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock 62 * is held during initial setting to protect races between kcpc_open() and the 63 * first enabling. The locking provided by the DTrace subsystem, the kernel 64 * cpc framework and the cpu management framework protect consumers from race 65 * conditions on enabling and disabling probes. 66 */ 67 uint32_t dtrace_cpc_in_use = 0; 68 69 typedef struct dtrace_hrestime { 70 lock_t dthr_lock; /* lock for this element */ 71 timestruc_t dthr_hrestime; /* hrestime value */ 72 int64_t dthr_adj; /* hrestime_adj value */ 73 hrtime_t dthr_hrtime; /* hrtime value */ 74 } dtrace_hrestime_t; 75 76 static dtrace_hrestime_t dtrace_hrestime[2]; 77 78 /* 79 * Making available adjustable high-resolution time in DTrace is regrettably 80 * more complicated than one might think it should be. The problem is that 81 * the variables related to adjusted high-resolution time (hrestime, 82 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may 83 * be held when we enter probe context. One might think that we could address 84 * this by having a single snapshot copy that is stored under a different lock 85 * from hres_tick(), using the snapshot iff hres_lock is locked in probe 86 * context. Unfortunately, this too won't work: because hres_lock is grabbed 87 * in more than just hres_tick() context, we could enter probe context 88 * concurrently on two different CPUs with both locks (hres_lock and the 89 * snapshot lock) held. As this implies, the fundamental problem is that we 90 * need to have access to a snapshot of these variables that we _know_ will 91 * not be locked in probe context. To effect this, we have two snapshots 92 * protected by two different locks, and we mandate that these snapshots are 93 * recorded in succession by a single thread calling dtrace_hres_tick(). (We 94 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls 95 * hres_tick().) A single thread can't be in two places at once: one of the 96 * snapshot locks is guaranteed to be unheld at all times. The 97 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then 98 * the other to find the unlocked snapshot. 99 */ 100 void 101 dtrace_hres_tick(void) 102 { 103 int i; 104 ushort_t spl; 105 106 for (i = 0; i < 2; i++) { 107 dtrace_hrestime_t tmp; 108 109 spl = hr_clock_lock(); 110 tmp.dthr_hrestime = hrestime; 111 tmp.dthr_adj = hrestime_adj; 112 tmp.dthr_hrtime = dtrace_gethrtime(); 113 hr_clock_unlock(spl); 114 115 lock_set(&dtrace_hrestime[i].dthr_lock); 116 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime; 117 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj; 118 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime; 119 dtrace_membar_producer(); 120 121 /* 122 * To allow for lock-free examination of this lock, we use 123 * the same trick that is used hres_lock; for more details, 124 * see the description of this technique in sun4u/sys/clock.h. 125 */ 126 dtrace_hrestime[i].dthr_lock++; 127 } 128 } 129 130 hrtime_t 131 dtrace_gethrestime(void) 132 { 133 dtrace_hrestime_t snap; 134 hrtime_t now; 135 int i = 0, adj, nslt; 136 137 for (;;) { 138 snap.dthr_lock = dtrace_hrestime[i].dthr_lock; 139 dtrace_membar_consumer(); 140 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime; 141 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime; 142 snap.dthr_adj = dtrace_hrestime[i].dthr_adj; 143 dtrace_membar_consumer(); 144 145 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock) 146 break; 147 148 /* 149 * If we're here, the lock was either locked, or it 150 * transitioned while we were taking the snapshot. Either 151 * way, we're going to try the other dtrace_hrestime element; 152 * we know that it isn't possible for both to be locked 153 * simultaneously, so we will ultimately get a good snapshot. 154 */ 155 i ^= 1; 156 } 157 158 /* 159 * We have a good snapshot. Now perform any necessary adjustments. 160 */ 161 nslt = dtrace_gethrtime() - snap.dthr_hrtime; 162 ASSERT(nslt >= 0); 163 164 now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) + 165 snap.dthr_hrestime.tv_nsec; 166 167 if (snap.dthr_adj != 0) { 168 if (snap.dthr_adj > 0) { 169 adj = (nslt >> adj_shift); 170 if (adj > snap.dthr_adj) 171 adj = (int)snap.dthr_adj; 172 } else { 173 adj = -(nslt >> adj_shift); 174 if (adj < snap.dthr_adj) 175 adj = (int)snap.dthr_adj; 176 } 177 now += adj; 178 } 179 180 return (now); 181 } 182 183 void 184 dtrace_vtime_enable(void) 185 { 186 dtrace_vtime_state_t state, nstate; 187 188 do { 189 state = dtrace_vtime_active; 190 191 switch (state) { 192 case DTRACE_VTIME_INACTIVE: 193 nstate = DTRACE_VTIME_ACTIVE; 194 break; 195 196 case DTRACE_VTIME_INACTIVE_TNF: 197 nstate = DTRACE_VTIME_ACTIVE_TNF; 198 break; 199 200 case DTRACE_VTIME_ACTIVE: 201 case DTRACE_VTIME_ACTIVE_TNF: 202 panic("DTrace virtual time already enabled"); 203 /*NOTREACHED*/ 204 } 205 206 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active, 207 state, nstate) != state); 208 } 209 210 void 211 dtrace_vtime_disable(void) 212 { 213 dtrace_vtime_state_t state, nstate; 214 215 do { 216 state = dtrace_vtime_active; 217 218 switch (state) { 219 case DTRACE_VTIME_ACTIVE: 220 nstate = DTRACE_VTIME_INACTIVE; 221 break; 222 223 case DTRACE_VTIME_ACTIVE_TNF: 224 nstate = DTRACE_VTIME_INACTIVE_TNF; 225 break; 226 227 case DTRACE_VTIME_INACTIVE: 228 case DTRACE_VTIME_INACTIVE_TNF: 229 panic("DTrace virtual time already disabled"); 230 /*NOTREACHED*/ 231 } 232 233 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active, 234 state, nstate) != state); 235 } 236 237 void 238 dtrace_vtime_enable_tnf(void) 239 { 240 dtrace_vtime_state_t state, nstate; 241 242 do { 243 state = dtrace_vtime_active; 244 245 switch (state) { 246 case DTRACE_VTIME_ACTIVE: 247 nstate = DTRACE_VTIME_ACTIVE_TNF; 248 break; 249 250 case DTRACE_VTIME_INACTIVE: 251 nstate = DTRACE_VTIME_INACTIVE_TNF; 252 break; 253 254 case DTRACE_VTIME_ACTIVE_TNF: 255 case DTRACE_VTIME_INACTIVE_TNF: 256 panic("TNF already active"); 257 /*NOTREACHED*/ 258 } 259 260 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active, 261 state, nstate) != state); 262 } 263 264 void 265 dtrace_vtime_disable_tnf(void) 266 { 267 dtrace_vtime_state_t state, nstate; 268 269 do { 270 state = dtrace_vtime_active; 271 272 switch (state) { 273 case DTRACE_VTIME_ACTIVE_TNF: 274 nstate = DTRACE_VTIME_ACTIVE; 275 break; 276 277 case DTRACE_VTIME_INACTIVE_TNF: 278 nstate = DTRACE_VTIME_INACTIVE; 279 break; 280 281 case DTRACE_VTIME_ACTIVE: 282 case DTRACE_VTIME_INACTIVE: 283 panic("TNF already inactive"); 284 /*NOTREACHED*/ 285 } 286 287 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active, 288 state, nstate) != state); 289 } 290 291 void 292 dtrace_vtime_switch(kthread_t *next) 293 { 294 dtrace_icookie_t cookie; 295 hrtime_t ts; 296 297 if (tnf_tracing_active) { 298 tnf_thread_switch(next); 299 300 if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF) 301 return; 302 } 303 304 cookie = dtrace_interrupt_disable(); 305 ts = dtrace_gethrtime(); 306 307 if (curthread->t_dtrace_start != 0) { 308 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start; 309 curthread->t_dtrace_start = 0; 310 } 311 312 next->t_dtrace_start = ts; 313 314 dtrace_interrupt_enable(cookie); 315 } 316 317 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *); 318 void (*dtrace_fasttrap_exec_ptr)(proc_t *); 319 void (*dtrace_fasttrap_exit_ptr)(proc_t *); 320 321 /* 322 * This function is called by cfork() in the event that it appears that 323 * there may be dtrace tracepoints active in the parent process's address 324 * space. This first confirms the existence of dtrace tracepoints in the 325 * parent process and calls into the fasttrap module to remove the 326 * corresponding tracepoints from the child. By knowing that there are 327 * existing tracepoints, and ensuring they can't be removed, we can rely 328 * on the fasttrap module remaining loaded. 329 */ 330 void 331 dtrace_fasttrap_fork(proc_t *p, proc_t *cp) 332 { 333 ASSERT(p->p_proc_flag & P_PR_LOCK); 334 ASSERT(p->p_dtrace_count > 0); 335 ASSERT(dtrace_fasttrap_fork_ptr != NULL); 336 337 dtrace_fasttrap_fork_ptr(p, cp); 338 } 339