1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * Copyright 2016 Joyent, Inc. 26 */ 27 28 #include <sys/dtrace.h> 29 #include <sys/cmn_err.h> 30 #include <sys/atomic.h> 31 #include <sys/prsystm.h> 32 #include <sys/modctl.h> 33 #include <sys/aio_impl.h> 34 35 #ifdef __sparc 36 #include <sys/privregs.h> 37 #endif 38 39 void (*dtrace_cpu_init)(processorid_t); 40 void (*dtrace_modload)(struct modctl *); 41 void (*dtrace_modunload)(struct modctl *); 42 void (*dtrace_helpers_cleanup)(proc_t *); 43 void (*dtrace_helpers_fork)(proc_t *, proc_t *); 44 void (*dtrace_cpustart_init)(void); 45 void (*dtrace_cpustart_fini)(void); 46 void (*dtrace_cpc_fire)(uint64_t); 47 void (*dtrace_closef)(void); 48 49 void (*dtrace_debugger_init)(void); 50 void (*dtrace_debugger_fini)(void); 51 52 dtrace_vtime_state_t dtrace_vtime_active = 0; 53 dtrace_cacheid_t dtrace_predcache_id = DTRACE_CACHEIDNONE + 1; 54 55 /* 56 * dtrace_cpc_in_use usage statement: this global variable is used by the cpc 57 * hardware overflow interrupt handler and the kernel cpc framework to check 58 * whether or not the DTrace cpc provider is currently in use. The variable is 59 * set before counters are enabled with the first enabling and cleared when 60 * the last enabling is disabled. Its value at any given time indicates the 61 * number of active dcpc based enablings. The global 'kcpc_cpuctx_lock' rwlock 62 * is held during initial setting to protect races between kcpc_open() and the 63 * first enabling. The locking provided by the DTrace subsystem, the kernel 64 * cpc framework and the cpu management framework protect consumers from race 65 * conditions on enabling and disabling probes. 66 */ 67 uint32_t dtrace_cpc_in_use = 0; 68 69 typedef struct dtrace_hrestime { 70 lock_t dthr_lock; /* lock for this element */ 71 timestruc_t dthr_hrestime; /* hrestime value */ 72 int64_t dthr_adj; /* hrestime_adj value */ 73 hrtime_t dthr_hrtime; /* hrtime value */ 74 } dtrace_hrestime_t; 75 76 static dtrace_hrestime_t dtrace_hrestime[2]; 77 78 /* 79 * Making available adjustable high-resolution time in DTrace is regrettably 80 * more complicated than one might think it should be. The problem is that 81 * the variables related to adjusted high-resolution time (hrestime, 82 * hrestime_adj and friends) are adjusted under hres_lock -- and this lock may 83 * be held when we enter probe context. One might think that we could address 84 * this by having a single snapshot copy that is stored under a different lock 85 * from hres_tick(), using the snapshot iff hres_lock is locked in probe 86 * context. Unfortunately, this too won't work: because hres_lock is grabbed 87 * in more than just hres_tick() context, we could enter probe context 88 * concurrently on two different CPUs with both locks (hres_lock and the 89 * snapshot lock) held. As this implies, the fundamental problem is that we 90 * need to have access to a snapshot of these variables that we _know_ will 91 * not be locked in probe context. To effect this, we have two snapshots 92 * protected by two different locks, and we mandate that these snapshots are 93 * recorded in succession by a single thread calling dtrace_hres_tick(). (We 94 * assure this by calling it out of the same CY_HIGH_LEVEL cyclic that calls 95 * hres_tick().) A single thread can't be in two places at once: one of the 96 * snapshot locks is guaranteed to be unheld at all times. The 97 * dtrace_gethrestime() algorithm is thus to check first one snapshot and then 98 * the other to find the unlocked snapshot. 99 */ 100 void 101 dtrace_hres_tick(void) 102 { 103 int i; 104 ushort_t spl; 105 106 for (i = 0; i < 2; i++) { 107 dtrace_hrestime_t tmp; 108 109 spl = hr_clock_lock(); 110 tmp.dthr_hrestime = hrestime; 111 tmp.dthr_adj = hrestime_adj; 112 tmp.dthr_hrtime = dtrace_gethrtime(); 113 hr_clock_unlock(spl); 114 115 lock_set(&dtrace_hrestime[i].dthr_lock); 116 dtrace_hrestime[i].dthr_hrestime = tmp.dthr_hrestime; 117 dtrace_hrestime[i].dthr_adj = tmp.dthr_adj; 118 dtrace_hrestime[i].dthr_hrtime = tmp.dthr_hrtime; 119 dtrace_membar_producer(); 120 121 /* 122 * To allow for lock-free examination of this lock, we use 123 * the same trick that is used hres_lock; for more details, 124 * see the description of this technique in sun4u/sys/clock.h. 125 */ 126 dtrace_hrestime[i].dthr_lock++; 127 } 128 } 129 130 hrtime_t 131 dtrace_gethrestime(void) 132 { 133 dtrace_hrestime_t snap; 134 hrtime_t now; 135 int i = 0, adj, nslt; 136 137 for (;;) { 138 snap.dthr_lock = dtrace_hrestime[i].dthr_lock; 139 dtrace_membar_consumer(); 140 snap.dthr_hrestime = dtrace_hrestime[i].dthr_hrestime; 141 snap.dthr_hrtime = dtrace_hrestime[i].dthr_hrtime; 142 snap.dthr_adj = dtrace_hrestime[i].dthr_adj; 143 dtrace_membar_consumer(); 144 145 if ((snap.dthr_lock & ~1) == dtrace_hrestime[i].dthr_lock) 146 break; 147 148 /* 149 * If we're here, the lock was either locked, or it 150 * transitioned while we were taking the snapshot. Either 151 * way, we're going to try the other dtrace_hrestime element; 152 * we know that it isn't possible for both to be locked 153 * simultaneously, so we will ultimately get a good snapshot. 154 */ 155 i ^= 1; 156 } 157 158 /* 159 * We have a good snapshot. Now perform any necessary adjustments. 160 */ 161 nslt = dtrace_gethrtime() - snap.dthr_hrtime; 162 ASSERT(nslt >= 0); 163 164 now = ((hrtime_t)snap.dthr_hrestime.tv_sec * (hrtime_t)NANOSEC) + 165 snap.dthr_hrestime.tv_nsec; 166 167 if (snap.dthr_adj != 0) { 168 if (snap.dthr_adj > 0) { 169 adj = (nslt >> adj_shift); 170 if (adj > snap.dthr_adj) 171 adj = (int)snap.dthr_adj; 172 } else { 173 adj = -(nslt >> adj_shift); 174 if (adj < snap.dthr_adj) 175 adj = (int)snap.dthr_adj; 176 } 177 now += adj; 178 } 179 180 return (now); 181 } 182 183 void 184 dtrace_vtime_enable(void) 185 { 186 dtrace_vtime_state_t state, nstate; 187 188 nstate = DTRACE_VTIME_INACTIVE; 189 do { 190 state = dtrace_vtime_active; 191 192 switch (state) { 193 case DTRACE_VTIME_INACTIVE: 194 nstate = DTRACE_VTIME_ACTIVE; 195 break; 196 197 case DTRACE_VTIME_ACTIVE: 198 panic("DTrace virtual time already enabled"); 199 /*NOTREACHED*/ 200 } 201 202 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active, 203 state, nstate) != state); 204 } 205 206 void 207 dtrace_vtime_disable(void) 208 { 209 dtrace_vtime_state_t state, nstate; 210 211 nstate = DTRACE_VTIME_INACTIVE; 212 do { 213 state = dtrace_vtime_active; 214 215 switch (state) { 216 case DTRACE_VTIME_ACTIVE: 217 nstate = DTRACE_VTIME_INACTIVE; 218 break; 219 220 case DTRACE_VTIME_INACTIVE: 221 panic("DTrace virtual time already disabled"); 222 /*NOTREACHED*/ 223 } 224 225 } while (atomic_cas_32((uint32_t *)&dtrace_vtime_active, 226 state, nstate) != state); 227 } 228 229 void 230 dtrace_vtime_switch(kthread_t *next) 231 { 232 dtrace_icookie_t cookie; 233 hrtime_t ts; 234 235 cookie = dtrace_interrupt_disable(); 236 ts = dtrace_gethrtime(); 237 238 if (curthread->t_dtrace_start != 0) { 239 curthread->t_dtrace_vtime += ts - curthread->t_dtrace_start; 240 curthread->t_dtrace_start = 0; 241 } 242 243 next->t_dtrace_start = ts; 244 245 dtrace_interrupt_enable(cookie); 246 } 247 248 void (*dtrace_fasttrap_fork_ptr)(proc_t *, proc_t *); 249 void (*dtrace_fasttrap_exec_ptr)(proc_t *); 250 void (*dtrace_fasttrap_exit_ptr)(proc_t *); 251 252 /* 253 * This function is called by cfork() in the event that it appears that 254 * there may be dtrace tracepoints active in the parent process's address 255 * space. This first confirms the existence of dtrace tracepoints in the 256 * parent process and calls into the fasttrap module to remove the 257 * corresponding tracepoints from the child. By knowing that there are 258 * existing tracepoints, and ensuring they can't be removed, we can rely 259 * on the fasttrap module remaining loaded. 260 */ 261 void 262 dtrace_fasttrap_fork(proc_t *p, proc_t *cp) 263 { 264 ASSERT(p->p_proc_flag & P_PR_LOCK); 265 ASSERT(p->p_dtrace_count > 0); 266 ASSERT(dtrace_fasttrap_fork_ptr != NULL); 267 268 dtrace_fasttrap_fork_ptr(p, cp); 269 } 270