1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/dtrace.h> 28 #include <sys/cmn_err.h> 29 #include <sys/tnf.h> 30 #include <sys/atomic.h> 31 #include <sys/prsystm.h> 32 #include <sys/modctl.h> 33 #include <sys/aio_impl.h> 34 35 #ifdef __sparc 36 #include <sys/privregs.h> 37 #endif 38 39 void (*dtrace_cpu_init)(processorid_t); 40 void (*dtrace_modload)(struct modctl *); 41 void (*dtrace_modunload)(struct modctl *); 42 void (*dtrace_helpers_cleanup)(void); 43 void (*dtrace_helpers_fork)(proc_t *, proc_t *); 44 void (*dtrace_cpustart_init)(void); 45 void (*dtrace_cpustart_fini)(void); 46 void (*dtrace_cpc_fire)(uint64_t); 47 48 void (*dtrace_debugger_init)(void); 49 void (*dtrace_debugger_fini)(void); 50 51 dtrace_vtime_state_t dtrace_vtime_active = 0; 52 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1; 53 54 /* 55 * dtrace_cpc_in_use usage statement: this global variable is used by the cpc 56 * hardware overflow interrupt handler and the kernel cpc framework to check 57 * whether or not the DTrace cpc provider is currently in use. The variable is 58 * set before counters are enabled with the first enabling and cleared when 59 * the last enabling is disabled. Its value at any given time indicates the 60 * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock 61 * is held during initial setting to protect races between kcpc_open() and the 62 * first enabling. The locking provided by the DTrace subsystem, the kernel 63 * cpc framework and the cpu management framework protect consumers from race 64 * conditions on enabling and disabling probes. 65 */ 66 uint32_t dtrace_cpc_in_use = 0; 67 68 typedef struct dtrace_hrestime { 69 lock_t dthr_lock; /* lock for this element */ 70 timestruc_t dthr_hrestime; /* hrestime value */ 71 int64_t dthr_adj; /* hrestime_adj value */ 72 hrtime_t dthr_hrtime; /* hrtime value */ 73 } dtrace_hrestime_t; 74 75 static dtrace_hrestime_t dtrace_hrestime[2]; 76 77 /* 78 * Making available adjustable high-resolution time in DTrace is regrettably 79 * more complicated than one might think it should be. The problem is that 80 * the variables related to adjusted high-resolution time (hrestime, 81 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may 82 * be held when we enter probe context. One might think that we could address 83 * this by having a single snapshot copy that is stored under a different lock 84 * from hres_tick(), using the snapshot iff hres_lock is locked in probe 85 * context. Unfortunately, this too won't work: because hres_lock is grabbed 86 * in more than just hres_tick() context, we could enter probe context 87 * concurrently on two different CPUs with both locks (hres_lock and the 88 * snapshot lock) held. As this implies, the fundamental problem is that we 89 * need to have access to a snapshot of these variables that we _know_ will 90 * not be locked in probe context. To effect this, we have two snapshots 91 * protected by two different locks, and we mandate that these snapshots are 92 * recorded in succession by a single thread calling dtrace_hres_tick(). (We 93 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls 94 * hres_tick().) A single thread can't be in two places at once: one of the 95 * snapshot locks is guaranteed to be unheld at all times. The 96 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then 97 * the other to find the unlocked snapshot. 98 */ 99 void 100 dtrace_hres_tick(void) 101 { 102 int i; 103 ushort_t spl; 104 105 for (i = 0; i < 2; i++) { 106 dtrace_hrestime_t tmp; 107 108 spl = hr_clock_lock(); 109 tmp.dthr_hrestime = hrestime; 110 tmp.dthr_adj = hrestime_adj; 111 tmp.dthr_hrtime = dtrace_gethrtime(); 112 hr_clock_unlock(spl); 113 114 lock_set(&dtrace_hrestime[i].dthr_lock); 115 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime; 116 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj; 117 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime; 118 dtrace_membar_producer(); 119 120 /* 121 * To allow for lock-free examination of this lock, we use 122 * the same trick that is used hres_lock; for more details, 123 * see the description of this technique in sun4u/sys/clock.h. 124 */ 125 dtrace_hrestime[i].dthr_lock++; 126 } 127 } 128 129 hrtime_t 130 dtrace_gethrestime(void) 131 { 132 dtrace_hrestime_t snap; 133 hrtime_t now; 134 int i = 0, adj, nslt; 135 136 for (;;) { 137 snap.dthr_lock = dtrace_hrestime[i].dthr_lock; 138 dtrace_membar_consumer(); 139 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime; 140 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime; 141 snap.dthr_adj = dtrace_hrestime[i].dthr_adj; 142 dtrace_membar_consumer(); 143 144 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock) 145 break; 146 147 /* 148 * If we're here, the lock was either locked, or it 149 * transitioned while we were taking the snapshot. Either 150 * way, we're going to try the other dtrace_hrestime element; 151 * we know that it isn't possible for both to be locked 152 * simultaneously, so we will ultimately get a good snapshot. 153 */ 154 i ^= 1; 155 } 156 157 /* 158 * We have a good snapshot. Now perform any necessary adjustments. 159 */ 160 nslt = dtrace_gethrtime() - snap.dthr_hrtime; 161 ASSERT(nslt >= 0); 162 163 now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) + 164 snap.dthr_hrestime.tv_nsec; 165 166 if (snap.dthr_adj != 0) { 167 if (snap.dthr_adj > 0) { 168 adj = (nslt >> adj_shift); 169 if (adj > snap.dthr_adj) 170 adj = (int)snap.dthr_adj; 171 } else { 172 adj = -(nslt >> adj_shift); 173 if (adj < snap.dthr_adj) 174 adj = (int)snap.dthr_adj; 175 } 176 now += adj; 177 } 178 179 return (now); 180 } 181 182 void 183 dtrace_vtime_enable(void) 184 { 185 dtrace_vtime_state_t state, nstate; 186 187 do { 188 state = dtrace_vtime_active; 189 190 switch (state) { 191 case DTRACE_VTIME_INACTIVE: 192 nstate = DTRACE_VTIME_ACTIVE; 193 break; 194 195 case DTRACE_VTIME_INACTIVE_TNF: 196 nstate = DTRACE_VTIME_ACTIVE_TNF; 197 break; 198 199 case DTRACE_VTIME_ACTIVE: 200 case DTRACE_VTIME_ACTIVE_TNF: 201 panic("DTrace virtual time already enabled"); 202 /*NOTREACHED*/ 203 } 204 205 } while (cas32((uint32_t *)&dtrace_vtime_active, 206 state, nstate) != state); 207 } 208 209 void 210 dtrace_vtime_disable(void) 211 { 212 dtrace_vtime_state_t state, nstate; 213 214 do { 215 state = dtrace_vtime_active; 216 217 switch (state) { 218 case DTRACE_VTIME_ACTIVE: 219 nstate = DTRACE_VTIME_INACTIVE; 220 break; 221 222 case DTRACE_VTIME_ACTIVE_TNF: 223 nstate = DTRACE_VTIME_INACTIVE_TNF; 224 break; 225 226 case DTRACE_VTIME_INACTIVE: 227 case DTRACE_VTIME_INACTIVE_TNF: 228 panic("DTrace virtual time already disabled"); 229 /*NOTREACHED*/ 230 } 231 232 } while (cas32((uint32_t *)&dtrace_vtime_active, 233 state, nstate) != state); 234 } 235 236 void 237 dtrace_vtime_enable_tnf(void) 238 { 239 dtrace_vtime_state_t state, nstate; 240 241 do { 242 state = dtrace_vtime_active; 243 244 switch (state) { 245 case DTRACE_VTIME_ACTIVE: 246 nstate = DTRACE_VTIME_ACTIVE_TNF; 247 break; 248 249 case DTRACE_VTIME_INACTIVE: 250 nstate = DTRACE_VTIME_INACTIVE_TNF; 251 break; 252 253 case DTRACE_VTIME_ACTIVE_TNF: 254 case DTRACE_VTIME_INACTIVE_TNF: 255 panic("TNF already active"); 256 /*NOTREACHED*/ 257 } 258 259 } while (cas32((uint32_t *)&dtrace_vtime_active, 260 state, nstate) != state); 261 } 262 263 void 264 dtrace_vtime_disable_tnf(void) 265 { 266 dtrace_vtime_state_t state, nstate; 267 268 do { 269 state = dtrace_vtime_active; 270 271 switch (state) { 272 case DTRACE_VTIME_ACTIVE_TNF: 273 nstate = DTRACE_VTIME_ACTIVE; 274 break; 275 276 case DTRACE_VTIME_INACTIVE_TNF: 277 nstate = DTRACE_VTIME_INACTIVE; 278 break; 279 280 case DTRACE_VTIME_ACTIVE: 281 case DTRACE_VTIME_INACTIVE: 282 panic("TNF already inactive"); 283 /*NOTREACHED*/ 284 } 285 286 } while (cas32((uint32_t *)&dtrace_vtime_active, 287 state, nstate) != state); 288 } 289 290 void 291 dtrace_vtime_switch(kthread_t *next) 292 { 293 dtrace_icookie_t cookie; 294 hrtime_t ts; 295 296 if (tnf_tracing_active) { 297 tnf_thread_switch(next); 298 299 if (dtrace_vtime_active == DTRACE_VTIME_INACTIVE_TNF) 300 return; 301 } 302 303 cookie = dtrace_interrupt_disable(); 304 ts = dtrace_gethrtime(); 305 306 if (curthread->t_dtrace_start != 0) { 307 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start; 308 curthread->t_dtrace_start = 0; 309 } 310 311 next->t_dtrace_start = ts; 312 313 dtrace_interrupt_enable(cookie); 314 } 315 316 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *); 317 void (*dtrace_fasttrap_exec_ptr)(proc_t *); 318 void (*dtrace_fasttrap_exit_ptr)(proc_t *); 319 320 /* 321 * This function is called by cfork() in the event that it appears that 322 * there may be dtrace tracepoints active in the parent process's address 323 * space. This first confirms the existence of dtrace tracepoints in the 324 * parent process and calls into the fasttrap module to remove the 325 * corresponding tracepoints from the child. By knowing that there are 326 * existing tracepoints, and ensuring they can't be removed, we can rely 327 * on the fasttrap module remaining loaded. 328 */ 329 void 330 dtrace_fasttrap_fork(proc_t *p, proc_t *cp) 331 { 332 ASSERT(p->p_proc_flag & P_PR_LOCK); 333 ASSERT(p->p_dtrace_count > 0); 334 ASSERT(dtrace_fasttrap_fork_ptr != NULL); 335 336 dtrace_fasttrap_fork_ptr(p, cp); 337 } 338