1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2016, Joyent, Inc. All rights reserved. 25 * Copyright (c) 2012, 2014 by Delphix. All rights reserved. 26 */ 27 28 /* 29 * DTrace - Dynamic Tracing for Solaris 30 * 31 * This is the implementation of the Solaris Dynamic Tracing framework 32 * (DTrace). The user-visible interface to DTrace is described at length in 33 * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace 34 * library, the in-kernel DTrace framework, and the DTrace providers are 35 * described in the block comments in the <sys/dtrace.h> header file. The 36 * internal architecture of DTrace is described in the block comments in the 37 * <sys/dtrace_impl.h> header file. The comments contained within the DTrace 38 * implementation very much assume mastery of all of these sources; if one has 39 * an unanswered question about the implementation, one should consult them 40 * first. 41 * 42 * The functions here are ordered roughly as follows: 43 * 44 * - Probe context functions 45 * - Probe hashing functions 46 * - Non-probe context utility functions 47 * - Matching functions 48 * - Provider-to-Framework API functions 49 * - Probe management functions 50 * - DIF object functions 51 * - Format functions 52 * - Predicate functions 53 * - ECB functions 54 * - Buffer functions 55 * - Enabling functions 56 * - DOF functions 57 * - Anonymous enabling functions 58 * - Consumer state functions 59 * - Helper functions 60 * - Hook functions 61 * - Driver cookbook functions 62 * 63 * Each group of functions begins with a block comment labelled the "DTrace 64 * [Group] Functions", allowing one to find each block by searching forward 65 * on capital-f functions. 66 */ 67 #include <sys/errno.h> 68 #include <sys/stat.h> 69 #include <sys/modctl.h> 70 #include <sys/conf.h> 71 #include <sys/systm.h> 72 #include <sys/ddi.h> 73 #include <sys/sunddi.h> 74 #include <sys/cpuvar.h> 75 #include <sys/kmem.h> 76 #include <sys/strsubr.h> 77 #include <sys/sysmacros.h> 78 #include <sys/dtrace_impl.h> 79 #include <sys/atomic.h> 80 #include <sys/cmn_err.h> 81 #include <sys/mutex_impl.h> 82 #include <sys/rwlock_impl.h> 83 #include <sys/ctf_api.h> 84 #include <sys/panic.h> 85 #include <sys/priv_impl.h> 86 #include <sys/policy.h> 87 #include <sys/cred_impl.h> 88 #include <sys/procfs_isa.h> 89 #include <sys/taskq.h> 90 #include <sys/mkdev.h> 91 #include <sys/kdi.h> 92 #include <sys/zone.h> 93 #include <sys/socket.h> 94 #include <netinet/in.h> 95 #include "strtolctype.h" 96 97 /* 98 * DTrace Tunable Variables 99 * 100 * The following variables may be tuned by adding a line to /etc/system that 101 * includes both the name of the DTrace module ("dtrace") and the name of the 102 * variable. For example: 103 * 104 * set dtrace:dtrace_destructive_disallow = 1 105 * 106 * In general, the only variables that one should be tuning this way are those 107 * that affect system-wide DTrace behavior, and for which the default behavior 108 * is undesirable. Most of these variables are tunable on a per-consumer 109 * basis using DTrace options, and need not be tuned on a system-wide basis. 110 * When tuning these variables, avoid pathological values; while some attempt 111 * is made to verify the integrity of these variables, they are not considered 112 * part of the supported interface to DTrace, and they are therefore not 113 * checked comprehensively. Further, these variables should not be tuned 114 * dynamically via "mdb -kw" or other means; they should only be tuned via 115 * /etc/system. 116 */ 117 int dtrace_destructive_disallow = 0; 118 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024); 119 size_t dtrace_difo_maxsize = (256 * 1024); 120 dtrace_optval_t dtrace_dof_maxsize = (8 * 1024 * 1024); 121 size_t dtrace_statvar_maxsize = (16 * 1024); 122 size_t dtrace_actions_max = (16 * 1024); 123 size_t dtrace_retain_max = 1024; 124 dtrace_optval_t dtrace_helper_actions_max = 1024; 125 dtrace_optval_t dtrace_helper_providers_max = 32; 126 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024); 127 size_t dtrace_strsize_default = 256; 128 dtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */ 129 dtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */ 130 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */ 131 dtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */ 132 dtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */ 133 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */ 134 dtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */ 135 dtrace_optval_t dtrace_nspec_default = 1; 136 dtrace_optval_t dtrace_specsize_default = 32 * 1024; 137 dtrace_optval_t dtrace_stackframes_default = 20; 138 dtrace_optval_t dtrace_ustackframes_default = 20; 139 dtrace_optval_t dtrace_jstackframes_default = 50; 140 dtrace_optval_t dtrace_jstackstrsize_default = 512; 141 int dtrace_msgdsize_max = 128; 142 hrtime_t dtrace_chill_max = MSEC2NSEC(500); /* 500 ms */ 143 hrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */ 144 int dtrace_devdepth_max = 32; 145 int dtrace_err_verbose; 146 hrtime_t dtrace_deadman_interval = NANOSEC; 147 hrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC; 148 hrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC; 149 hrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC; 150 151 /* 152 * DTrace External Variables 153 * 154 * As dtrace(7D) is a kernel module, any DTrace variables are obviously 155 * available to DTrace consumers via the backtick (`) syntax. One of these, 156 * dtrace_zero, is made deliberately so: it is provided as a source of 157 * well-known, zero-filled memory. While this variable is not documented, 158 * it is used by some translators as an implementation detail. 159 */ 160 const char dtrace_zero[256] = { 0 }; /* zero-filled memory */ 161 162 /* 163 * DTrace Internal Variables 164 */ 165 static dev_info_t *dtrace_devi; /* device info */ 166 static vmem_t *dtrace_arena; /* probe ID arena */ 167 static vmem_t *dtrace_minor; /* minor number arena */ 168 static taskq_t *dtrace_taskq; /* task queue */ 169 static dtrace_probe_t **dtrace_probes; /* array of all probes */ 170 static int dtrace_nprobes; /* number of probes */ 171 static dtrace_provider_t *dtrace_provider; /* provider list */ 172 static dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */ 173 static int dtrace_opens; /* number of opens */ 174 static int dtrace_helpers; /* number of helpers */ 175 static int dtrace_getf; /* number of unpriv getf()s */ 176 static void *dtrace_softstate; /* softstate pointer */ 177 static dtrace_hash_t *dtrace_bymod; /* probes hashed by module */ 178 static dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */ 179 static dtrace_hash_t *dtrace_byname; /* probes hashed by name */ 180 static dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */ 181 static int dtrace_toxranges; /* number of toxic ranges */ 182 static int dtrace_toxranges_max; /* size of toxic range array */ 183 static dtrace_anon_t dtrace_anon; /* anonymous enabling */ 184 static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */ 185 static uint64_t dtrace_vtime_references; /* number of vtimestamp refs */ 186 static kthread_t *dtrace_panicked; /* panicking thread */ 187 static dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */ 188 static dtrace_genid_t dtrace_probegen; /* current probe generation */ 189 static dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */ 190 static dtrace_enabling_t *dtrace_retained; /* list of retained enablings */ 191 static dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */ 192 static dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */ 193 static int dtrace_dynvar_failclean; /* dynvars failed to clean */ 194 195 /* 196 * DTrace Locking 197 * DTrace is protected by three (relatively coarse-grained) locks: 198 * 199 * (1) dtrace_lock is required to manipulate essentially any DTrace state, 200 * including enabling state, probes, ECBs, consumer state, helper state, 201 * etc. Importantly, dtrace_lock is _not_ required when in probe context; 202 * probe context is lock-free -- synchronization is handled via the 203 * dtrace_sync() cross call mechanism. 204 * 205 * (2) dtrace_provider_lock is required when manipulating provider state, or 206 * when provider state must be held constant. 207 * 208 * (3) dtrace_meta_lock is required when manipulating meta provider state, or 209 * when meta provider state must be held constant. 210 * 211 * The lock ordering between these three locks is dtrace_meta_lock before 212 * dtrace_provider_lock before dtrace_lock. (In particular, there are 213 * several places where dtrace_provider_lock is held by the framework as it 214 * calls into the providers -- which then call back into the framework, 215 * grabbing dtrace_lock.) 216 * 217 * There are two other locks in the mix: mod_lock and cpu_lock. With respect 218 * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical 219 * role as a coarse-grained lock; it is acquired before both of these locks. 220 * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must 221 * be acquired _between_ dtrace_meta_lock and any other DTrace locks. 222 * mod_lock is similar with respect to dtrace_provider_lock in that it must be 223 * acquired _between_ dtrace_provider_lock and dtrace_lock. 224 */ 225 static kmutex_t dtrace_lock; /* probe state lock */ 226 static kmutex_t dtrace_provider_lock; /* provider state lock */ 227 static kmutex_t dtrace_meta_lock; /* meta-provider state lock */ 228 229 /* 230 * DTrace Provider Variables 231 * 232 * These are the variables relating to DTrace as a provider (that is, the 233 * provider of the BEGIN, END, and ERROR probes). 234 */ 235 static dtrace_pattr_t dtrace_provider_attr = { 236 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, 237 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 238 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 239 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, 240 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, 241 }; 242 243 static void 244 dtrace_nullop(void) 245 {} 246 247 static int 248 dtrace_enable_nullop(void) 249 { 250 return (0); 251 } 252 253 static dtrace_pops_t dtrace_provider_ops = { 254 (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop, 255 (void (*)(void *, struct modctl *))dtrace_nullop, 256 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop, 257 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 258 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 259 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 260 NULL, 261 NULL, 262 NULL, 263 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop 264 }; 265 266 static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */ 267 static dtrace_id_t dtrace_probeid_end; /* special END probe */ 268 dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ 269 270 /* 271 * DTrace Helper Tracing Variables 272 * 273 * These variables should be set dynamically to enable helper tracing. The 274 * only variables that should be set are dtrace_helptrace_enable (which should 275 * be set to a non-zero value to allocate helper tracing buffers on the next 276 * open of /dev/dtrace) and dtrace_helptrace_disable (which should be set to a 277 * non-zero value to deallocate helper tracing buffers on the next close of 278 * /dev/dtrace). When (and only when) helper tracing is disabled, the 279 * buffer size may also be set via dtrace_helptrace_bufsize. 280 */ 281 int dtrace_helptrace_enable = 0; 282 int dtrace_helptrace_disable = 0; 283 int dtrace_helptrace_bufsize = 16 * 1024 * 1024; 284 uint32_t dtrace_helptrace_nlocals; 285 static dtrace_helptrace_t *dtrace_helptrace_buffer; 286 static uint32_t dtrace_helptrace_next = 0; 287 static int dtrace_helptrace_wrapped = 0; 288 289 /* 290 * DTrace Error Hashing 291 * 292 * On DEBUG kernels, DTrace will track the errors that has seen in a hash 293 * table. This is very useful for checking coverage of tests that are 294 * expected to induce DIF or DOF processing errors, and may be useful for 295 * debugging problems in the DIF code generator or in DOF generation . The 296 * error hash may be examined with the ::dtrace_errhash MDB dcmd. 297 */ 298 #ifdef DEBUG 299 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ]; 300 static const char *dtrace_errlast; 301 static kthread_t *dtrace_errthread; 302 static kmutex_t dtrace_errlock; 303 #endif 304 305 /* 306 * DTrace Macros and Constants 307 * 308 * These are various macros that are useful in various spots in the 309 * implementation, along with a few random constants that have no meaning 310 * outside of the implementation. There is no real structure to this cpp 311 * mishmash -- but is there ever? 312 */ 313 #define DTRACE_HASHSTR(hash, probe) \ 314 dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs))) 315 316 #define DTRACE_HASHNEXT(hash, probe) \ 317 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs) 318 319 #define DTRACE_HASHPREV(hash, probe) \ 320 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs) 321 322 #define DTRACE_HASHEQ(hash, lhs, rhs) \ 323 (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \ 324 *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0) 325 326 #define DTRACE_AGGHASHSIZE_SLEW 17 327 328 #define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3) 329 330 /* 331 * The key for a thread-local variable consists of the lower 61 bits of the 332 * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL. 333 * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never 334 * equal to a variable identifier. This is necessary (but not sufficient) to 335 * assure that global associative arrays never collide with thread-local 336 * variables. To guarantee that they cannot collide, we must also define the 337 * order for keying dynamic variables. That order is: 338 * 339 * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ] 340 * 341 * Because the variable-key and the tls-key are in orthogonal spaces, there is 342 * no way for a global variable key signature to match a thread-local key 343 * signature. 344 */ 345 #define DTRACE_TLS_THRKEY(where) { \ 346 uint_t intr = 0; \ 347 uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \ 348 for (; actv; actv >>= 1) \ 349 intr++; \ 350 ASSERT(intr < (1 << 3)); \ 351 (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \ 352 (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ 353 } 354 355 #define DT_BSWAP_8(x) ((x) & 0xff) 356 #define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8)) 357 #define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16)) 358 #define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32)) 359 360 #define DT_MASK_LO 0x00000000FFFFFFFFULL 361 362 #define DTRACE_STORE(type, tomax, offset, what) \ 363 *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what); 364 365 #ifndef __x86 366 #define DTRACE_ALIGNCHECK(addr, size, flags) \ 367 if (addr & (size - 1)) { \ 368 *flags |= CPU_DTRACE_BADALIGN; \ 369 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \ 370 return (0); \ 371 } 372 #else 373 #define DTRACE_ALIGNCHECK(addr, size, flags) 374 #endif 375 376 /* 377 * Test whether a range of memory starting at testaddr of size testsz falls 378 * within the range of memory described by addr, sz. We take care to avoid 379 * problems with overflow and underflow of the unsigned quantities, and 380 * disallow all negative sizes. Ranges of size 0 are allowed. 381 */ 382 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \ 383 ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \ 384 (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \ 385 (testaddr) + (testsz) >= (testaddr)) 386 387 #define DTRACE_RANGE_REMAIN(remp, addr, baseaddr, basesz) \ 388 do { \ 389 if ((remp) != NULL) { \ 390 *(remp) = (uintptr_t)(baseaddr) + (basesz) - (addr); \ 391 } \ 392 _NOTE(CONSTCOND) } while (0) 393 394 395 /* 396 * Test whether alloc_sz bytes will fit in the scratch region. We isolate 397 * alloc_sz on the righthand side of the comparison in order to avoid overflow 398 * or underflow in the comparison with it. This is simpler than the INRANGE 399 * check above, because we know that the dtms_scratch_ptr is valid in the 400 * range. Allocations of size zero are allowed. 401 */ 402 #define DTRACE_INSCRATCH(mstate, alloc_sz) \ 403 ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \ 404 (mstate)->dtms_scratch_ptr >= (alloc_sz)) 405 406 #define DTRACE_LOADFUNC(bits) \ 407 /*CSTYLED*/ \ 408 uint##bits##_t \ 409 dtrace_load##bits(uintptr_t addr) \ 410 { \ 411 size_t size = bits / NBBY; \ 412 /*CSTYLED*/ \ 413 uint##bits##_t rval; \ 414 int i; \ 415 volatile uint16_t *flags = (volatile uint16_t *) \ 416 &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; \ 417 \ 418 DTRACE_ALIGNCHECK(addr, size, flags); \ 419 \ 420 for (i = 0; i < dtrace_toxranges; i++) { \ 421 if (addr >= dtrace_toxrange[i].dtt_limit) \ 422 continue; \ 423 \ 424 if (addr + size <= dtrace_toxrange[i].dtt_base) \ 425 continue; \ 426 \ 427 /* \ 428 * This address falls within a toxic region; return 0. \ 429 */ \ 430 *flags |= CPU_DTRACE_BADADDR; \ 431 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr; \ 432 return (0); \ 433 } \ 434 \ 435 *flags |= CPU_DTRACE_NOFAULT; \ 436 /*CSTYLED*/ \ 437 rval = *((volatile uint##bits##_t *)addr); \ 438 *flags &= ~CPU_DTRACE_NOFAULT; \ 439 \ 440 return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \ 441 } 442 443 #ifdef _LP64 444 #define dtrace_loadptr dtrace_load64 445 #else 446 #define dtrace_loadptr dtrace_load32 447 #endif 448 449 #define DTRACE_DYNHASH_FREE 0 450 #define DTRACE_DYNHASH_SINK 1 451 #define DTRACE_DYNHASH_VALID 2 452 453 #define DTRACE_MATCH_FAIL -1 454 #define DTRACE_MATCH_NEXT 0 455 #define DTRACE_MATCH_DONE 1 456 #define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0') 457 #define DTRACE_STATE_ALIGN 64 458 459 #define DTRACE_FLAGS2FLT(flags) \ 460 (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \ 461 ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \ 462 ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \ 463 ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \ 464 ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \ 465 ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \ 466 ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \ 467 ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \ 468 ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \ 469 DTRACEFLT_UNKNOWN) 470 471 #define DTRACEACT_ISSTRING(act) \ 472 ((act)->dta_kind == DTRACEACT_DIFEXPR && \ 473 (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) 474 475 static size_t dtrace_strlen(const char *, size_t); 476 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id); 477 static void dtrace_enabling_provide(dtrace_provider_t *); 478 static int dtrace_enabling_match(dtrace_enabling_t *, int *); 479 static void dtrace_enabling_matchall(void); 480 static void dtrace_enabling_reap(void); 481 static dtrace_state_t *dtrace_anon_grab(void); 482 static uint64_t dtrace_helper(int, dtrace_mstate_t *, 483 dtrace_state_t *, uint64_t, uint64_t); 484 static dtrace_helpers_t *dtrace_helpers_create(proc_t *); 485 static void dtrace_buffer_drop(dtrace_buffer_t *); 486 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when); 487 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t, 488 dtrace_state_t *, dtrace_mstate_t *); 489 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t, 490 dtrace_optval_t); 491 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *); 492 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *); 493 static int dtrace_priv_proc(dtrace_state_t *, dtrace_mstate_t *); 494 static void dtrace_getf_barrier(void); 495 static int dtrace_canload_remains(uint64_t, size_t, size_t *, 496 dtrace_mstate_t *, dtrace_vstate_t *); 497 static int dtrace_canstore_remains(uint64_t, size_t, size_t *, 498 dtrace_mstate_t *, dtrace_vstate_t *); 499 500 /* 501 * DTrace Probe Context Functions 502 * 503 * These functions are called from probe context. Because probe context is 504 * any context in which C may be called, arbitrarily locks may be held, 505 * interrupts may be disabled, we may be in arbitrary dispatched state, etc. 506 * As a result, functions called from probe context may only call other DTrace 507 * support functions -- they may not interact at all with the system at large. 508 * (Note that the ASSERT macro is made probe-context safe by redefining it in 509 * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary 510 * loads are to be performed from probe context, they _must_ be in terms of 511 * the safe dtrace_load*() variants. 512 * 513 * Some functions in this block are not actually called from probe context; 514 * for these functions, there will be a comment above the function reading 515 * "Note: not called from probe context." 516 */ 517 void 518 dtrace_panic(const char *format, ...) 519 { 520 va_list alist; 521 522 va_start(alist, format); 523 dtrace_vpanic(format, alist); 524 va_end(alist); 525 } 526 527 int 528 dtrace_assfail(const char *a, const char *f, int l) 529 { 530 dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l); 531 532 /* 533 * We just need something here that even the most clever compiler 534 * cannot optimize away. 535 */ 536 return (a[(uintptr_t)f]); 537 } 538 539 /* 540 * Atomically increment a specified error counter from probe context. 541 */ 542 static void 543 dtrace_error(uint32_t *counter) 544 { 545 /* 546 * Most counters stored to in probe context are per-CPU counters. 547 * However, there are some error conditions that are sufficiently 548 * arcane that they don't merit per-CPU storage. If these counters 549 * are incremented concurrently on different CPUs, scalability will be 550 * adversely affected -- but we don't expect them to be white-hot in a 551 * correctly constructed enabling... 552 */ 553 uint32_t oval, nval; 554 555 do { 556 oval = *counter; 557 558 if ((nval = oval + 1) == 0) { 559 /* 560 * If the counter would wrap, set it to 1 -- assuring 561 * that the counter is never zero when we have seen 562 * errors. (The counter must be 32-bits because we 563 * aren't guaranteed a 64-bit compare&swap operation.) 564 * To save this code both the infamy of being fingered 565 * by a priggish news story and the indignity of being 566 * the target of a neo-puritan witch trial, we're 567 * carefully avoiding any colorful description of the 568 * likelihood of this condition -- but suffice it to 569 * say that it is only slightly more likely than the 570 * overflow of predicate cache IDs, as discussed in 571 * dtrace_predicate_create(). 572 */ 573 nval = 1; 574 } 575 } while (dtrace_cas32(counter, oval, nval) != oval); 576 } 577 578 /* 579 * Use the DTRACE_LOADFUNC macro to define functions for each of loading a 580 * uint8_t, a uint16_t, a uint32_t and a uint64_t. 581 */ 582 /* BEGIN CSTYLED */ 583 DTRACE_LOADFUNC(8) 584 DTRACE_LOADFUNC(16) 585 DTRACE_LOADFUNC(32) 586 DTRACE_LOADFUNC(64) 587 /* END CSTYLED */ 588 589 static int 590 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate) 591 { 592 if (dest < mstate->dtms_scratch_base) 593 return (0); 594 595 if (dest + size < dest) 596 return (0); 597 598 if (dest + size > mstate->dtms_scratch_ptr) 599 return (0); 600 601 return (1); 602 } 603 604 static int 605 dtrace_canstore_statvar(uint64_t addr, size_t sz, size_t *remain, 606 dtrace_statvar_t **svars, int nsvars) 607 { 608 int i; 609 size_t maxglobalsize, maxlocalsize; 610 611 if (nsvars == 0) 612 return (0); 613 614 maxglobalsize = dtrace_statvar_maxsize + sizeof (uint64_t); 615 maxlocalsize = maxglobalsize * NCPU; 616 617 for (i = 0; i < nsvars; i++) { 618 dtrace_statvar_t *svar = svars[i]; 619 uint8_t scope; 620 size_t size; 621 622 if (svar == NULL || (size = svar->dtsv_size) == 0) 623 continue; 624 625 scope = svar->dtsv_var.dtdv_scope; 626 627 /* 628 * We verify that our size is valid in the spirit of providing 629 * defense in depth: we want to prevent attackers from using 630 * DTrace to escalate an orthogonal kernel heap corruption bug 631 * into the ability to store to arbitrary locations in memory. 632 */ 633 VERIFY((scope == DIFV_SCOPE_GLOBAL && size <= maxglobalsize) || 634 (scope == DIFV_SCOPE_LOCAL && size <= maxlocalsize)); 635 636 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, 637 svar->dtsv_size)) { 638 DTRACE_RANGE_REMAIN(remain, addr, svar->dtsv_data, 639 svar->dtsv_size); 640 return (1); 641 } 642 } 643 644 return (0); 645 } 646 647 /* 648 * Check to see if the address is within a memory region to which a store may 649 * be issued. This includes the DTrace scratch areas, and any DTrace variable 650 * region. The caller of dtrace_canstore() is responsible for performing any 651 * alignment checks that are needed before stores are actually executed. 652 */ 653 static int 654 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, 655 dtrace_vstate_t *vstate) 656 { 657 return (dtrace_canstore_remains(addr, sz, NULL, mstate, vstate)); 658 } 659 660 /* 661 * Implementation of dtrace_canstore which communicates the upper bound of the 662 * allowed memory region. 663 */ 664 static int 665 dtrace_canstore_remains(uint64_t addr, size_t sz, size_t *remain, 666 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) 667 { 668 /* 669 * First, check to see if the address is in scratch space... 670 */ 671 if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base, 672 mstate->dtms_scratch_size)) { 673 DTRACE_RANGE_REMAIN(remain, addr, mstate->dtms_scratch_base, 674 mstate->dtms_scratch_size); 675 return (1); 676 } 677 678 /* 679 * Now check to see if it's a dynamic variable. This check will pick 680 * up both thread-local variables and any global dynamically-allocated 681 * variables. 682 */ 683 if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base, 684 vstate->dtvs_dynvars.dtds_size)) { 685 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; 686 uintptr_t base = (uintptr_t)dstate->dtds_base + 687 (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t)); 688 uintptr_t chunkoffs; 689 dtrace_dynvar_t *dvar; 690 691 /* 692 * Before we assume that we can store here, we need to make 693 * sure that it isn't in our metadata -- storing to our 694 * dynamic variable metadata would corrupt our state. For 695 * the range to not include any dynamic variable metadata, 696 * it must: 697 * 698 * (1) Start above the hash table that is at the base of 699 * the dynamic variable space 700 * 701 * (2) Have a starting chunk offset that is beyond the 702 * dtrace_dynvar_t that is at the base of every chunk 703 * 704 * (3) Not span a chunk boundary 705 * 706 * (4) Not be in the tuple space of a dynamic variable 707 * 708 */ 709 if (addr < base) 710 return (0); 711 712 chunkoffs = (addr - base) % dstate->dtds_chunksize; 713 714 if (chunkoffs < sizeof (dtrace_dynvar_t)) 715 return (0); 716 717 if (chunkoffs + sz > dstate->dtds_chunksize) 718 return (0); 719 720 dvar = (dtrace_dynvar_t *)((uintptr_t)addr - chunkoffs); 721 722 if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) 723 return (0); 724 725 if (chunkoffs < sizeof (dtrace_dynvar_t) + 726 ((dvar->dtdv_tuple.dtt_nkeys - 1) * sizeof (dtrace_key_t))) 727 return (0); 728 729 DTRACE_RANGE_REMAIN(remain, addr, dvar, dstate->dtds_chunksize); 730 return (1); 731 } 732 733 /* 734 * Finally, check the static local and global variables. These checks 735 * take the longest, so we perform them last. 736 */ 737 if (dtrace_canstore_statvar(addr, sz, remain, 738 vstate->dtvs_locals, vstate->dtvs_nlocals)) 739 return (1); 740 741 if (dtrace_canstore_statvar(addr, sz, remain, 742 vstate->dtvs_globals, vstate->dtvs_nglobals)) 743 return (1); 744 745 return (0); 746 } 747 748 749 /* 750 * Convenience routine to check to see if the address is within a memory 751 * region in which a load may be issued given the user's privilege level; 752 * if not, it sets the appropriate error flags and loads 'addr' into the 753 * illegal value slot. 754 * 755 * DTrace subroutines (DIF_SUBR_*) should use this helper to implement 756 * appropriate memory access protection. 757 */ 758 static int 759 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, 760 dtrace_vstate_t *vstate) 761 { 762 return (dtrace_canload_remains(addr, sz, NULL, mstate, vstate)); 763 } 764 765 /* 766 * Implementation of dtrace_canload which communicates the upper bound of the 767 * allowed memory region. 768 */ 769 static int 770 dtrace_canload_remains(uint64_t addr, size_t sz, size_t *remain, 771 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) 772 { 773 volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval; 774 file_t *fp; 775 776 /* 777 * If we hold the privilege to read from kernel memory, then 778 * everything is readable. 779 */ 780 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) { 781 DTRACE_RANGE_REMAIN(remain, addr, addr, sz); 782 return (1); 783 } 784 785 /* 786 * You can obviously read that which you can store. 787 */ 788 if (dtrace_canstore_remains(addr, sz, remain, mstate, vstate)) 789 return (1); 790 791 /* 792 * We're allowed to read from our own string table. 793 */ 794 if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab, 795 mstate->dtms_difo->dtdo_strlen)) { 796 DTRACE_RANGE_REMAIN(remain, addr, 797 mstate->dtms_difo->dtdo_strtab, 798 mstate->dtms_difo->dtdo_strlen); 799 return (1); 800 } 801 802 if (vstate->dtvs_state != NULL && 803 dtrace_priv_proc(vstate->dtvs_state, mstate)) { 804 proc_t *p; 805 806 /* 807 * When we have privileges to the current process, there are 808 * several context-related kernel structures that are safe to 809 * read, even absent the privilege to read from kernel memory. 810 * These reads are safe because these structures contain only 811 * state that (1) we're permitted to read, (2) is harmless or 812 * (3) contains pointers to additional kernel state that we're 813 * not permitted to read (and as such, do not present an 814 * opportunity for privilege escalation). Finally (and 815 * critically), because of the nature of their relation with 816 * the current thread context, the memory associated with these 817 * structures cannot change over the duration of probe context, 818 * and it is therefore impossible for this memory to be 819 * deallocated and reallocated as something else while it's 820 * being operated upon. 821 */ 822 if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t))) { 823 DTRACE_RANGE_REMAIN(remain, addr, curthread, 824 sizeof (kthread_t)); 825 return (1); 826 } 827 828 if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr, 829 sz, curthread->t_procp, sizeof (proc_t))) { 830 DTRACE_RANGE_REMAIN(remain, addr, curthread->t_procp, 831 sizeof (proc_t)); 832 return (1); 833 } 834 835 if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz, 836 curthread->t_cred, sizeof (cred_t))) { 837 DTRACE_RANGE_REMAIN(remain, addr, curthread->t_cred, 838 sizeof (cred_t)); 839 return (1); 840 } 841 842 if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz, 843 &(p->p_pidp->pid_id), sizeof (pid_t))) { 844 DTRACE_RANGE_REMAIN(remain, addr, &(p->p_pidp->pid_id), 845 sizeof (pid_t)); 846 return (1); 847 } 848 849 if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz, 850 curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) { 851 DTRACE_RANGE_REMAIN(remain, addr, curthread->t_cpu, 852 offsetof(cpu_t, cpu_pause_thread)); 853 return (1); 854 } 855 } 856 857 if ((fp = mstate->dtms_getf) != NULL) { 858 uintptr_t psz = sizeof (void *); 859 vnode_t *vp; 860 vnodeops_t *op; 861 862 /* 863 * When getf() returns a file_t, the enabling is implicitly 864 * granted the (transient) right to read the returned file_t 865 * as well as the v_path and v_op->vnop_name of the underlying 866 * vnode. These accesses are allowed after a successful 867 * getf() because the members that they refer to cannot change 868 * once set -- and the barrier logic in the kernel's closef() 869 * path assures that the file_t and its referenced vode_t 870 * cannot themselves be stale (that is, it impossible for 871 * either dtms_getf itself or its f_vnode member to reference 872 * freed memory). 873 */ 874 if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t))) { 875 DTRACE_RANGE_REMAIN(remain, addr, fp, sizeof (file_t)); 876 return (1); 877 } 878 879 if ((vp = fp->f_vnode) != NULL) { 880 size_t slen; 881 882 if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz)) { 883 DTRACE_RANGE_REMAIN(remain, addr, &vp->v_path, 884 psz); 885 return (1); 886 } 887 888 slen = strlen(vp->v_path) + 1; 889 if (DTRACE_INRANGE(addr, sz, vp->v_path, slen)) { 890 DTRACE_RANGE_REMAIN(remain, addr, vp->v_path, 891 slen); 892 return (1); 893 } 894 895 if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz)) { 896 DTRACE_RANGE_REMAIN(remain, addr, &vp->v_op, 897 psz); 898 return (1); 899 } 900 901 if ((op = vp->v_op) != NULL && 902 DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) { 903 DTRACE_RANGE_REMAIN(remain, addr, 904 &op->vnop_name, psz); 905 return (1); 906 } 907 908 if (op != NULL && op->vnop_name != NULL && 909 DTRACE_INRANGE(addr, sz, op->vnop_name, 910 (slen = strlen(op->vnop_name) + 1))) { 911 DTRACE_RANGE_REMAIN(remain, addr, 912 op->vnop_name, slen); 913 return (1); 914 } 915 } 916 } 917 918 DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV); 919 *illval = addr; 920 return (0); 921 } 922 923 /* 924 * Convenience routine to check to see if a given string is within a memory 925 * region in which a load may be issued given the user's privilege level; 926 * this exists so that we don't need to issue unnecessary dtrace_strlen() 927 * calls in the event that the user has all privileges. 928 */ 929 static int 930 dtrace_strcanload(uint64_t addr, size_t sz, size_t *remain, 931 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) 932 { 933 size_t rsize; 934 935 /* 936 * If we hold the privilege to read from kernel memory, then 937 * everything is readable. 938 */ 939 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) { 940 DTRACE_RANGE_REMAIN(remain, addr, addr, sz); 941 return (1); 942 } 943 944 /* 945 * Even if the caller is uninterested in querying the remaining valid 946 * range, it is required to ensure that the access is allowed. 947 */ 948 if (remain == NULL) { 949 remain = &rsize; 950 } 951 if (dtrace_canload_remains(addr, 0, remain, mstate, vstate)) { 952 size_t strsz; 953 /* 954 * Perform the strlen after determining the length of the 955 * memory region which is accessible. This prevents timing 956 * information from being used to find NULs in memory which is 957 * not accessible to the caller. 958 */ 959 strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, 960 MIN(sz, *remain)); 961 if (strsz <= *remain) { 962 return (1); 963 } 964 } 965 966 return (0); 967 } 968 969 /* 970 * Convenience routine to check to see if a given variable is within a memory 971 * region in which a load may be issued given the user's privilege level. 972 */ 973 static int 974 dtrace_vcanload(void *src, dtrace_diftype_t *type, size_t *remain, 975 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) 976 { 977 size_t sz; 978 ASSERT(type->dtdt_flags & DIF_TF_BYREF); 979 980 /* 981 * Calculate the max size before performing any checks since even 982 * DTRACE_ACCESS_KERNEL-credentialed callers expect that this function 983 * return the max length via 'remain'. 984 */ 985 if (type->dtdt_kind == DIF_TYPE_STRING) { 986 dtrace_state_t *state = vstate->dtvs_state; 987 988 if (state != NULL) { 989 sz = state->dts_options[DTRACEOPT_STRSIZE]; 990 } else { 991 /* 992 * In helper context, we have a NULL state; fall back 993 * to using the system-wide default for the string size 994 * in this case. 995 */ 996 sz = dtrace_strsize_default; 997 } 998 } else { 999 sz = type->dtdt_size; 1000 } 1001 1002 /* 1003 * If we hold the privilege to read from kernel memory, then 1004 * everything is readable. 1005 */ 1006 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) { 1007 DTRACE_RANGE_REMAIN(remain, (uintptr_t)src, src, sz); 1008 return (1); 1009 } 1010 1011 if (type->dtdt_kind == DIF_TYPE_STRING) { 1012 return (dtrace_strcanload((uintptr_t)src, sz, remain, mstate, 1013 vstate)); 1014 } 1015 return (dtrace_canload_remains((uintptr_t)src, sz, remain, mstate, 1016 vstate)); 1017 } 1018 1019 /* 1020 * Convert a string to a signed integer using safe loads. 1021 * 1022 * NOTE: This function uses various macros from strtolctype.h to manipulate 1023 * digit values, etc -- these have all been checked to ensure they make 1024 * no additional function calls. 1025 */ 1026 static int64_t 1027 dtrace_strtoll(char *input, int base, size_t limit) 1028 { 1029 uintptr_t pos = (uintptr_t)input; 1030 int64_t val = 0; 1031 int x; 1032 boolean_t neg = B_FALSE; 1033 char c, cc, ccc; 1034 uintptr_t end = pos + limit; 1035 1036 /* 1037 * Consume any whitespace preceding digits. 1038 */ 1039 while ((c = dtrace_load8(pos)) == ' ' || c == '\t') 1040 pos++; 1041 1042 /* 1043 * Handle an explicit sign if one is present. 1044 */ 1045 if (c == '-' || c == '+') { 1046 if (c == '-') 1047 neg = B_TRUE; 1048 c = dtrace_load8(++pos); 1049 } 1050 1051 /* 1052 * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it 1053 * if present. 1054 */ 1055 if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' || 1056 cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) { 1057 pos += 2; 1058 c = ccc; 1059 } 1060 1061 /* 1062 * Read in contiguous digits until the first non-digit character. 1063 */ 1064 for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base; 1065 c = dtrace_load8(++pos)) 1066 val = val * base + x; 1067 1068 return (neg ? -val : val); 1069 } 1070 1071 /* 1072 * Compare two strings using safe loads. 1073 */ 1074 static int 1075 dtrace_strncmp(char *s1, char *s2, size_t limit) 1076 { 1077 uint8_t c1, c2; 1078 volatile uint16_t *flags; 1079 1080 if (s1 == s2 || limit == 0) 1081 return (0); 1082 1083 flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 1084 1085 do { 1086 if (s1 == NULL) { 1087 c1 = '\0'; 1088 } else { 1089 c1 = dtrace_load8((uintptr_t)s1++); 1090 } 1091 1092 if (s2 == NULL) { 1093 c2 = '\0'; 1094 } else { 1095 c2 = dtrace_load8((uintptr_t)s2++); 1096 } 1097 1098 if (c1 != c2) 1099 return (c1 - c2); 1100 } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT)); 1101 1102 return (0); 1103 } 1104 1105 /* 1106 * Compute strlen(s) for a string using safe memory accesses. The additional 1107 * len parameter is used to specify a maximum length to ensure completion. 1108 */ 1109 static size_t 1110 dtrace_strlen(const char *s, size_t lim) 1111 { 1112 uint_t len; 1113 1114 for (len = 0; len != lim; len++) { 1115 if (dtrace_load8((uintptr_t)s++) == '\0') 1116 break; 1117 } 1118 1119 return (len); 1120 } 1121 1122 /* 1123 * Check if an address falls within a toxic region. 1124 */ 1125 static int 1126 dtrace_istoxic(uintptr_t kaddr, size_t size) 1127 { 1128 uintptr_t taddr, tsize; 1129 int i; 1130 1131 for (i = 0; i < dtrace_toxranges; i++) { 1132 taddr = dtrace_toxrange[i].dtt_base; 1133 tsize = dtrace_toxrange[i].dtt_limit - taddr; 1134 1135 if (kaddr - taddr < tsize) { 1136 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 1137 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr; 1138 return (1); 1139 } 1140 1141 if (taddr - kaddr < size) { 1142 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 1143 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr; 1144 return (1); 1145 } 1146 } 1147 1148 return (0); 1149 } 1150 1151 /* 1152 * Copy src to dst using safe memory accesses. The src is assumed to be unsafe 1153 * memory specified by the DIF program. The dst is assumed to be safe memory 1154 * that we can store to directly because it is managed by DTrace. As with 1155 * standard bcopy, overlapping copies are handled properly. 1156 */ 1157 static void 1158 dtrace_bcopy(const void *src, void *dst, size_t len) 1159 { 1160 if (len != 0) { 1161 uint8_t *s1 = dst; 1162 const uint8_t *s2 = src; 1163 1164 if (s1 <= s2) { 1165 do { 1166 *s1++ = dtrace_load8((uintptr_t)s2++); 1167 } while (--len != 0); 1168 } else { 1169 s2 += len; 1170 s1 += len; 1171 1172 do { 1173 *--s1 = dtrace_load8((uintptr_t)--s2); 1174 } while (--len != 0); 1175 } 1176 } 1177 } 1178 1179 /* 1180 * Copy src to dst using safe memory accesses, up to either the specified 1181 * length, or the point that a nul byte is encountered. The src is assumed to 1182 * be unsafe memory specified by the DIF program. The dst is assumed to be 1183 * safe memory that we can store to directly because it is managed by DTrace. 1184 * Unlike dtrace_bcopy(), overlapping regions are not handled. 1185 */ 1186 static void 1187 dtrace_strcpy(const void *src, void *dst, size_t len) 1188 { 1189 if (len != 0) { 1190 uint8_t *s1 = dst, c; 1191 const uint8_t *s2 = src; 1192 1193 do { 1194 *s1++ = c = dtrace_load8((uintptr_t)s2++); 1195 } while (--len != 0 && c != '\0'); 1196 } 1197 } 1198 1199 /* 1200 * Copy src to dst, deriving the size and type from the specified (BYREF) 1201 * variable type. The src is assumed to be unsafe memory specified by the DIF 1202 * program. The dst is assumed to be DTrace variable memory that is of the 1203 * specified type; we assume that we can store to directly. 1204 */ 1205 static void 1206 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type, size_t limit) 1207 { 1208 ASSERT(type->dtdt_flags & DIF_TF_BYREF); 1209 1210 if (type->dtdt_kind == DIF_TYPE_STRING) { 1211 dtrace_strcpy(src, dst, MIN(type->dtdt_size, limit)); 1212 } else { 1213 dtrace_bcopy(src, dst, MIN(type->dtdt_size, limit)); 1214 } 1215 } 1216 1217 /* 1218 * Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be 1219 * unsafe memory specified by the DIF program. The s2 data is assumed to be 1220 * safe memory that we can access directly because it is managed by DTrace. 1221 */ 1222 static int 1223 dtrace_bcmp(const void *s1, const void *s2, size_t len) 1224 { 1225 volatile uint16_t *flags; 1226 1227 flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 1228 1229 if (s1 == s2) 1230 return (0); 1231 1232 if (s1 == NULL || s2 == NULL) 1233 return (1); 1234 1235 if (s1 != s2 && len != 0) { 1236 const uint8_t *ps1 = s1; 1237 const uint8_t *ps2 = s2; 1238 1239 do { 1240 if (dtrace_load8((uintptr_t)ps1++) != *ps2++) 1241 return (1); 1242 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT)); 1243 } 1244 return (0); 1245 } 1246 1247 /* 1248 * Zero the specified region using a simple byte-by-byte loop. Note that this 1249 * is for safe DTrace-managed memory only. 1250 */ 1251 static void 1252 dtrace_bzero(void *dst, size_t len) 1253 { 1254 uchar_t *cp; 1255 1256 for (cp = dst; len != 0; len--) 1257 *cp++ = 0; 1258 } 1259 1260 static void 1261 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum) 1262 { 1263 uint64_t result[2]; 1264 1265 result[0] = addend1[0] + addend2[0]; 1266 result[1] = addend1[1] + addend2[1] + 1267 (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0); 1268 1269 sum[0] = result[0]; 1270 sum[1] = result[1]; 1271 } 1272 1273 /* 1274 * Shift the 128-bit value in a by b. If b is positive, shift left. 1275 * If b is negative, shift right. 1276 */ 1277 static void 1278 dtrace_shift_128(uint64_t *a, int b) 1279 { 1280 uint64_t mask; 1281 1282 if (b == 0) 1283 return; 1284 1285 if (b < 0) { 1286 b = -b; 1287 if (b >= 64) { 1288 a[0] = a[1] >> (b - 64); 1289 a[1] = 0; 1290 } else { 1291 a[0] >>= b; 1292 mask = 1LL << (64 - b); 1293 mask -= 1; 1294 a[0] |= ((a[1] & mask) << (64 - b)); 1295 a[1] >>= b; 1296 } 1297 } else { 1298 if (b >= 64) { 1299 a[1] = a[0] << (b - 64); 1300 a[0] = 0; 1301 } else { 1302 a[1] <<= b; 1303 mask = a[0] >> (64 - b); 1304 a[1] |= mask; 1305 a[0] <<= b; 1306 } 1307 } 1308 } 1309 1310 /* 1311 * The basic idea is to break the 2 64-bit values into 4 32-bit values, 1312 * use native multiplication on those, and then re-combine into the 1313 * resulting 128-bit value. 1314 * 1315 * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) = 1316 * hi1 * hi2 << 64 + 1317 * hi1 * lo2 << 32 + 1318 * hi2 * lo1 << 32 + 1319 * lo1 * lo2 1320 */ 1321 static void 1322 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product) 1323 { 1324 uint64_t hi1, hi2, lo1, lo2; 1325 uint64_t tmp[2]; 1326 1327 hi1 = factor1 >> 32; 1328 hi2 = factor2 >> 32; 1329 1330 lo1 = factor1 & DT_MASK_LO; 1331 lo2 = factor2 & DT_MASK_LO; 1332 1333 product[0] = lo1 * lo2; 1334 product[1] = hi1 * hi2; 1335 1336 tmp[0] = hi1 * lo2; 1337 tmp[1] = 0; 1338 dtrace_shift_128(tmp, 32); 1339 dtrace_add_128(product, tmp, product); 1340 1341 tmp[0] = hi2 * lo1; 1342 tmp[1] = 0; 1343 dtrace_shift_128(tmp, 32); 1344 dtrace_add_128(product, tmp, product); 1345 } 1346 1347 /* 1348 * This privilege check should be used by actions and subroutines to 1349 * verify that the user credentials of the process that enabled the 1350 * invoking ECB match the target credentials 1351 */ 1352 static int 1353 dtrace_priv_proc_common_user(dtrace_state_t *state) 1354 { 1355 cred_t *cr, *s_cr = state->dts_cred.dcr_cred; 1356 1357 /* 1358 * We should always have a non-NULL state cred here, since if cred 1359 * is null (anonymous tracing), we fast-path bypass this routine. 1360 */ 1361 ASSERT(s_cr != NULL); 1362 1363 if ((cr = CRED()) != NULL && 1364 s_cr->cr_uid == cr->cr_uid && 1365 s_cr->cr_uid == cr->cr_ruid && 1366 s_cr->cr_uid == cr->cr_suid && 1367 s_cr->cr_gid == cr->cr_gid && 1368 s_cr->cr_gid == cr->cr_rgid && 1369 s_cr->cr_gid == cr->cr_sgid) 1370 return (1); 1371 1372 return (0); 1373 } 1374 1375 /* 1376 * This privilege check should be used by actions and subroutines to 1377 * verify that the zone of the process that enabled the invoking ECB 1378 * matches the target credentials 1379 */ 1380 static int 1381 dtrace_priv_proc_common_zone(dtrace_state_t *state) 1382 { 1383 cred_t *cr, *s_cr = state->dts_cred.dcr_cred; 1384 1385 /* 1386 * We should always have a non-NULL state cred here, since if cred 1387 * is null (anonymous tracing), we fast-path bypass this routine. 1388 */ 1389 ASSERT(s_cr != NULL); 1390 1391 if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone) 1392 return (1); 1393 1394 return (0); 1395 } 1396 1397 /* 1398 * This privilege check should be used by actions and subroutines to 1399 * verify that the process has not setuid or changed credentials. 1400 */ 1401 static int 1402 dtrace_priv_proc_common_nocd() 1403 { 1404 proc_t *proc; 1405 1406 if ((proc = ttoproc(curthread)) != NULL && 1407 !(proc->p_flag & SNOCD)) 1408 return (1); 1409 1410 return (0); 1411 } 1412 1413 static int 1414 dtrace_priv_proc_destructive(dtrace_state_t *state, dtrace_mstate_t *mstate) 1415 { 1416 int action = state->dts_cred.dcr_action; 1417 1418 if (!(mstate->dtms_access & DTRACE_ACCESS_PROC)) 1419 goto bad; 1420 1421 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) && 1422 dtrace_priv_proc_common_zone(state) == 0) 1423 goto bad; 1424 1425 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) && 1426 dtrace_priv_proc_common_user(state) == 0) 1427 goto bad; 1428 1429 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) && 1430 dtrace_priv_proc_common_nocd() == 0) 1431 goto bad; 1432 1433 return (1); 1434 1435 bad: 1436 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; 1437 1438 return (0); 1439 } 1440 1441 static int 1442 dtrace_priv_proc_control(dtrace_state_t *state, dtrace_mstate_t *mstate) 1443 { 1444 if (mstate->dtms_access & DTRACE_ACCESS_PROC) { 1445 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL) 1446 return (1); 1447 1448 if (dtrace_priv_proc_common_zone(state) && 1449 dtrace_priv_proc_common_user(state) && 1450 dtrace_priv_proc_common_nocd()) 1451 return (1); 1452 } 1453 1454 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; 1455 1456 return (0); 1457 } 1458 1459 static int 1460 dtrace_priv_proc(dtrace_state_t *state, dtrace_mstate_t *mstate) 1461 { 1462 if ((mstate->dtms_access & DTRACE_ACCESS_PROC) && 1463 (state->dts_cred.dcr_action & DTRACE_CRA_PROC)) 1464 return (1); 1465 1466 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; 1467 1468 return (0); 1469 } 1470 1471 static int 1472 dtrace_priv_kernel(dtrace_state_t *state) 1473 { 1474 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL) 1475 return (1); 1476 1477 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV; 1478 1479 return (0); 1480 } 1481 1482 static int 1483 dtrace_priv_kernel_destructive(dtrace_state_t *state) 1484 { 1485 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE) 1486 return (1); 1487 1488 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV; 1489 1490 return (0); 1491 } 1492 1493 /* 1494 * Determine if the dte_cond of the specified ECB allows for processing of 1495 * the current probe to continue. Note that this routine may allow continued 1496 * processing, but with access(es) stripped from the mstate's dtms_access 1497 * field. 1498 */ 1499 static int 1500 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate, 1501 dtrace_ecb_t *ecb) 1502 { 1503 dtrace_probe_t *probe = ecb->dte_probe; 1504 dtrace_provider_t *prov = probe->dtpr_provider; 1505 dtrace_pops_t *pops = &prov->dtpv_pops; 1506 int mode = DTRACE_MODE_NOPRIV_DROP; 1507 1508 ASSERT(ecb->dte_cond); 1509 1510 if (pops->dtps_mode != NULL) { 1511 mode = pops->dtps_mode(prov->dtpv_arg, 1512 probe->dtpr_id, probe->dtpr_arg); 1513 1514 ASSERT(mode & (DTRACE_MODE_USER | DTRACE_MODE_KERNEL)); 1515 ASSERT(mode & (DTRACE_MODE_NOPRIV_RESTRICT | 1516 DTRACE_MODE_NOPRIV_DROP)); 1517 } 1518 1519 /* 1520 * If the dte_cond bits indicate that this consumer is only allowed to 1521 * see user-mode firings of this probe, check that the probe was fired 1522 * while in a user context. If that's not the case, use the policy 1523 * specified by the provider to determine if we drop the probe or 1524 * merely restrict operation. 1525 */ 1526 if (ecb->dte_cond & DTRACE_COND_USERMODE) { 1527 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP); 1528 1529 if (!(mode & DTRACE_MODE_USER)) { 1530 if (mode & DTRACE_MODE_NOPRIV_DROP) 1531 return (0); 1532 1533 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS; 1534 } 1535 } 1536 1537 /* 1538 * This is more subtle than it looks. We have to be absolutely certain 1539 * that CRED() isn't going to change out from under us so it's only 1540 * legit to examine that structure if we're in constrained situations. 1541 * Currently, the only times we'll this check is if a non-super-user 1542 * has enabled the profile or syscall providers -- providers that 1543 * allow visibility of all processes. For the profile case, the check 1544 * above will ensure that we're examining a user context. 1545 */ 1546 if (ecb->dte_cond & DTRACE_COND_OWNER) { 1547 cred_t *cr; 1548 cred_t *s_cr = state->dts_cred.dcr_cred; 1549 proc_t *proc; 1550 1551 ASSERT(s_cr != NULL); 1552 1553 if ((cr = CRED()) == NULL || 1554 s_cr->cr_uid != cr->cr_uid || 1555 s_cr->cr_uid != cr->cr_ruid || 1556 s_cr->cr_uid != cr->cr_suid || 1557 s_cr->cr_gid != cr->cr_gid || 1558 s_cr->cr_gid != cr->cr_rgid || 1559 s_cr->cr_gid != cr->cr_sgid || 1560 (proc = ttoproc(curthread)) == NULL || 1561 (proc->p_flag & SNOCD)) { 1562 if (mode & DTRACE_MODE_NOPRIV_DROP) 1563 return (0); 1564 1565 mstate->dtms_access &= ~DTRACE_ACCESS_PROC; 1566 } 1567 } 1568 1569 /* 1570 * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not 1571 * in our zone, check to see if our mode policy is to restrict rather 1572 * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC 1573 * and DTRACE_ACCESS_ARGS 1574 */ 1575 if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) { 1576 cred_t *cr; 1577 cred_t *s_cr = state->dts_cred.dcr_cred; 1578 1579 ASSERT(s_cr != NULL); 1580 1581 if ((cr = CRED()) == NULL || 1582 s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) { 1583 if (mode & DTRACE_MODE_NOPRIV_DROP) 1584 return (0); 1585 1586 mstate->dtms_access &= 1587 ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS); 1588 } 1589 } 1590 1591 /* 1592 * By merits of being in this code path at all, we have limited 1593 * privileges. If the provider has indicated that limited privileges 1594 * are to denote restricted operation, strip off the ability to access 1595 * arguments. 1596 */ 1597 if (mode & DTRACE_MODE_LIMITEDPRIV_RESTRICT) 1598 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS; 1599 1600 return (1); 1601 } 1602 1603 /* 1604 * Note: not called from probe context. This function is called 1605 * asynchronously (and at a regular interval) from outside of probe context to 1606 * clean the dirty dynamic variable lists on all CPUs. Dynamic variable 1607 * cleaning is explained in detail in <sys/dtrace_impl.h>. 1608 */ 1609 void 1610 dtrace_dynvar_clean(dtrace_dstate_t *dstate) 1611 { 1612 dtrace_dynvar_t *dirty; 1613 dtrace_dstate_percpu_t *dcpu; 1614 dtrace_dynvar_t **rinsep; 1615 int i, j, work = 0; 1616 1617 for (i = 0; i < NCPU; i++) { 1618 dcpu = &dstate->dtds_percpu[i]; 1619 rinsep = &dcpu->dtdsc_rinsing; 1620 1621 /* 1622 * If the dirty list is NULL, there is no dirty work to do. 1623 */ 1624 if (dcpu->dtdsc_dirty == NULL) 1625 continue; 1626 1627 if (dcpu->dtdsc_rinsing != NULL) { 1628 /* 1629 * If the rinsing list is non-NULL, then it is because 1630 * this CPU was selected to accept another CPU's 1631 * dirty list -- and since that time, dirty buffers 1632 * have accumulated. This is a highly unlikely 1633 * condition, but we choose to ignore the dirty 1634 * buffers -- they'll be picked up a future cleanse. 1635 */ 1636 continue; 1637 } 1638 1639 if (dcpu->dtdsc_clean != NULL) { 1640 /* 1641 * If the clean list is non-NULL, then we're in a 1642 * situation where a CPU has done deallocations (we 1643 * have a non-NULL dirty list) but no allocations (we 1644 * also have a non-NULL clean list). We can't simply 1645 * move the dirty list into the clean list on this 1646 * CPU, yet we also don't want to allow this condition 1647 * to persist, lest a short clean list prevent a 1648 * massive dirty list from being cleaned (which in 1649 * turn could lead to otherwise avoidable dynamic 1650 * drops). To deal with this, we look for some CPU 1651 * with a NULL clean list, NULL dirty list, and NULL 1652 * rinsing list -- and then we borrow this CPU to 1653 * rinse our dirty list. 1654 */ 1655 for (j = 0; j < NCPU; j++) { 1656 dtrace_dstate_percpu_t *rinser; 1657 1658 rinser = &dstate->dtds_percpu[j]; 1659 1660 if (rinser->dtdsc_rinsing != NULL) 1661 continue; 1662 1663 if (rinser->dtdsc_dirty != NULL) 1664 continue; 1665 1666 if (rinser->dtdsc_clean != NULL) 1667 continue; 1668 1669 rinsep = &rinser->dtdsc_rinsing; 1670 break; 1671 } 1672 1673 if (j == NCPU) { 1674 /* 1675 * We were unable to find another CPU that 1676 * could accept this dirty list -- we are 1677 * therefore unable to clean it now. 1678 */ 1679 dtrace_dynvar_failclean++; 1680 continue; 1681 } 1682 } 1683 1684 work = 1; 1685 1686 /* 1687 * Atomically move the dirty list aside. 1688 */ 1689 do { 1690 dirty = dcpu->dtdsc_dirty; 1691 1692 /* 1693 * Before we zap the dirty list, set the rinsing list. 1694 * (This allows for a potential assertion in 1695 * dtrace_dynvar(): if a free dynamic variable appears 1696 * on a hash chain, either the dirty list or the 1697 * rinsing list for some CPU must be non-NULL.) 1698 */ 1699 *rinsep = dirty; 1700 dtrace_membar_producer(); 1701 } while (dtrace_casptr(&dcpu->dtdsc_dirty, 1702 dirty, NULL) != dirty); 1703 } 1704 1705 if (!work) { 1706 /* 1707 * We have no work to do; we can simply return. 1708 */ 1709 return; 1710 } 1711 1712 dtrace_sync(); 1713 1714 for (i = 0; i < NCPU; i++) { 1715 dcpu = &dstate->dtds_percpu[i]; 1716 1717 if (dcpu->dtdsc_rinsing == NULL) 1718 continue; 1719 1720 /* 1721 * We are now guaranteed that no hash chain contains a pointer 1722 * into this dirty list; we can make it clean. 1723 */ 1724 ASSERT(dcpu->dtdsc_clean == NULL); 1725 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing; 1726 dcpu->dtdsc_rinsing = NULL; 1727 } 1728 1729 /* 1730 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make 1731 * sure that all CPUs have seen all of the dtdsc_clean pointers. 1732 * This prevents a race whereby a CPU incorrectly decides that 1733 * the state should be something other than DTRACE_DSTATE_CLEAN 1734 * after dtrace_dynvar_clean() has completed. 1735 */ 1736 dtrace_sync(); 1737 1738 dstate->dtds_state = DTRACE_DSTATE_CLEAN; 1739 } 1740 1741 /* 1742 * Depending on the value of the op parameter, this function looks-up, 1743 * allocates or deallocates an arbitrarily-keyed dynamic variable. If an 1744 * allocation is requested, this function will return a pointer to a 1745 * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no 1746 * variable can be allocated. If NULL is returned, the appropriate counter 1747 * will be incremented. 1748 */ 1749 dtrace_dynvar_t * 1750 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys, 1751 dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op, 1752 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) 1753 { 1754 uint64_t hashval = DTRACE_DYNHASH_VALID; 1755 dtrace_dynhash_t *hash = dstate->dtds_hash; 1756 dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL; 1757 processorid_t me = CPU->cpu_id, cpu = me; 1758 dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me]; 1759 size_t bucket, ksize; 1760 size_t chunksize = dstate->dtds_chunksize; 1761 uintptr_t kdata, lock, nstate; 1762 uint_t i; 1763 1764 ASSERT(nkeys != 0); 1765 1766 /* 1767 * Hash the key. As with aggregations, we use Jenkins' "One-at-a-time" 1768 * algorithm. For the by-value portions, we perform the algorithm in 1769 * 16-bit chunks (as opposed to 8-bit chunks). This speeds things up a 1770 * bit, and seems to have only a minute effect on distribution. For 1771 * the by-reference data, we perform "One-at-a-time" iterating (safely) 1772 * over each referenced byte. It's painful to do this, but it's much 1773 * better than pathological hash distribution. The efficacy of the 1774 * hashing algorithm (and a comparison with other algorithms) may be 1775 * found by running the ::dtrace_dynstat MDB dcmd. 1776 */ 1777 for (i = 0; i < nkeys; i++) { 1778 if (key[i].dttk_size == 0) { 1779 uint64_t val = key[i].dttk_value; 1780 1781 hashval += (val >> 48) & 0xffff; 1782 hashval += (hashval << 10); 1783 hashval ^= (hashval >> 6); 1784 1785 hashval += (val >> 32) & 0xffff; 1786 hashval += (hashval << 10); 1787 hashval ^= (hashval >> 6); 1788 1789 hashval += (val >> 16) & 0xffff; 1790 hashval += (hashval << 10); 1791 hashval ^= (hashval >> 6); 1792 1793 hashval += val & 0xffff; 1794 hashval += (hashval << 10); 1795 hashval ^= (hashval >> 6); 1796 } else { 1797 /* 1798 * This is incredibly painful, but it beats the hell 1799 * out of the alternative. 1800 */ 1801 uint64_t j, size = key[i].dttk_size; 1802 uintptr_t base = (uintptr_t)key[i].dttk_value; 1803 1804 if (!dtrace_canload(base, size, mstate, vstate)) 1805 break; 1806 1807 for (j = 0; j < size; j++) { 1808 hashval += dtrace_load8(base + j); 1809 hashval += (hashval << 10); 1810 hashval ^= (hashval >> 6); 1811 } 1812 } 1813 } 1814 1815 if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) 1816 return (NULL); 1817 1818 hashval += (hashval << 3); 1819 hashval ^= (hashval >> 11); 1820 hashval += (hashval << 15); 1821 1822 /* 1823 * There is a remote chance (ideally, 1 in 2^31) that our hashval 1824 * comes out to be one of our two sentinel hash values. If this 1825 * actually happens, we set the hashval to be a value known to be a 1826 * non-sentinel value. 1827 */ 1828 if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK) 1829 hashval = DTRACE_DYNHASH_VALID; 1830 1831 /* 1832 * Yes, it's painful to do a divide here. If the cycle count becomes 1833 * important here, tricks can be pulled to reduce it. (However, it's 1834 * critical that hash collisions be kept to an absolute minimum; 1835 * they're much more painful than a divide.) It's better to have a 1836 * solution that generates few collisions and still keeps things 1837 * relatively simple. 1838 */ 1839 bucket = hashval % dstate->dtds_hashsize; 1840 1841 if (op == DTRACE_DYNVAR_DEALLOC) { 1842 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock; 1843 1844 for (;;) { 1845 while ((lock = *lockp) & 1) 1846 continue; 1847 1848 if (dtrace_casptr((void *)lockp, 1849 (void *)lock, (void *)(lock + 1)) == (void *)lock) 1850 break; 1851 } 1852 1853 dtrace_membar_producer(); 1854 } 1855 1856 top: 1857 prev = NULL; 1858 lock = hash[bucket].dtdh_lock; 1859 1860 dtrace_membar_consumer(); 1861 1862 start = hash[bucket].dtdh_chain; 1863 ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK || 1864 start->dtdv_hashval != DTRACE_DYNHASH_FREE || 1865 op != DTRACE_DYNVAR_DEALLOC)); 1866 1867 for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) { 1868 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple; 1869 dtrace_key_t *dkey = &dtuple->dtt_key[0]; 1870 1871 if (dvar->dtdv_hashval != hashval) { 1872 if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) { 1873 /* 1874 * We've reached the sink, and therefore the 1875 * end of the hash chain; we can kick out of 1876 * the loop knowing that we have seen a valid 1877 * snapshot of state. 1878 */ 1879 ASSERT(dvar->dtdv_next == NULL); 1880 ASSERT(dvar == &dtrace_dynhash_sink); 1881 break; 1882 } 1883 1884 if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) { 1885 /* 1886 * We've gone off the rails: somewhere along 1887 * the line, one of the members of this hash 1888 * chain was deleted. Note that we could also 1889 * detect this by simply letting this loop run 1890 * to completion, as we would eventually hit 1891 * the end of the dirty list. However, we 1892 * want to avoid running the length of the 1893 * dirty list unnecessarily (it might be quite 1894 * long), so we catch this as early as 1895 * possible by detecting the hash marker. In 1896 * this case, we simply set dvar to NULL and 1897 * break; the conditional after the loop will 1898 * send us back to top. 1899 */ 1900 dvar = NULL; 1901 break; 1902 } 1903 1904 goto next; 1905 } 1906 1907 if (dtuple->dtt_nkeys != nkeys) 1908 goto next; 1909 1910 for (i = 0; i < nkeys; i++, dkey++) { 1911 if (dkey->dttk_size != key[i].dttk_size) 1912 goto next; /* size or type mismatch */ 1913 1914 if (dkey->dttk_size != 0) { 1915 if (dtrace_bcmp( 1916 (void *)(uintptr_t)key[i].dttk_value, 1917 (void *)(uintptr_t)dkey->dttk_value, 1918 dkey->dttk_size)) 1919 goto next; 1920 } else { 1921 if (dkey->dttk_value != key[i].dttk_value) 1922 goto next; 1923 } 1924 } 1925 1926 if (op != DTRACE_DYNVAR_DEALLOC) 1927 return (dvar); 1928 1929 ASSERT(dvar->dtdv_next == NULL || 1930 dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE); 1931 1932 if (prev != NULL) { 1933 ASSERT(hash[bucket].dtdh_chain != dvar); 1934 ASSERT(start != dvar); 1935 ASSERT(prev->dtdv_next == dvar); 1936 prev->dtdv_next = dvar->dtdv_next; 1937 } else { 1938 if (dtrace_casptr(&hash[bucket].dtdh_chain, 1939 start, dvar->dtdv_next) != start) { 1940 /* 1941 * We have failed to atomically swing the 1942 * hash table head pointer, presumably because 1943 * of a conflicting allocation on another CPU. 1944 * We need to reread the hash chain and try 1945 * again. 1946 */ 1947 goto top; 1948 } 1949 } 1950 1951 dtrace_membar_producer(); 1952 1953 /* 1954 * Now set the hash value to indicate that it's free. 1955 */ 1956 ASSERT(hash[bucket].dtdh_chain != dvar); 1957 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE; 1958 1959 dtrace_membar_producer(); 1960 1961 /* 1962 * Set the next pointer to point at the dirty list, and 1963 * atomically swing the dirty pointer to the newly freed dvar. 1964 */ 1965 do { 1966 next = dcpu->dtdsc_dirty; 1967 dvar->dtdv_next = next; 1968 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next); 1969 1970 /* 1971 * Finally, unlock this hash bucket. 1972 */ 1973 ASSERT(hash[bucket].dtdh_lock == lock); 1974 ASSERT(lock & 1); 1975 hash[bucket].dtdh_lock++; 1976 1977 return (NULL); 1978 next: 1979 prev = dvar; 1980 continue; 1981 } 1982 1983 if (dvar == NULL) { 1984 /* 1985 * If dvar is NULL, it is because we went off the rails: 1986 * one of the elements that we traversed in the hash chain 1987 * was deleted while we were traversing it. In this case, 1988 * we assert that we aren't doing a dealloc (deallocs lock 1989 * the hash bucket to prevent themselves from racing with 1990 * one another), and retry the hash chain traversal. 1991 */ 1992 ASSERT(op != DTRACE_DYNVAR_DEALLOC); 1993 goto top; 1994 } 1995 1996 if (op != DTRACE_DYNVAR_ALLOC) { 1997 /* 1998 * If we are not to allocate a new variable, we want to 1999 * return NULL now. Before we return, check that the value 2000 * of the lock word hasn't changed. If it has, we may have 2001 * seen an inconsistent snapshot. 2002 */ 2003 if (op == DTRACE_DYNVAR_NOALLOC) { 2004 if (hash[bucket].dtdh_lock != lock) 2005 goto top; 2006 } else { 2007 ASSERT(op == DTRACE_DYNVAR_DEALLOC); 2008 ASSERT(hash[bucket].dtdh_lock == lock); 2009 ASSERT(lock & 1); 2010 hash[bucket].dtdh_lock++; 2011 } 2012 2013 return (NULL); 2014 } 2015 2016 /* 2017 * We need to allocate a new dynamic variable. The size we need is the 2018 * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the 2019 * size of any auxiliary key data (rounded up to 8-byte alignment) plus 2020 * the size of any referred-to data (dsize). We then round the final 2021 * size up to the chunksize for allocation. 2022 */ 2023 for (ksize = 0, i = 0; i < nkeys; i++) 2024 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t)); 2025 2026 /* 2027 * This should be pretty much impossible, but could happen if, say, 2028 * strange DIF specified the tuple. Ideally, this should be an 2029 * assertion and not an error condition -- but that requires that the 2030 * chunksize calculation in dtrace_difo_chunksize() be absolutely 2031 * bullet-proof. (That is, it must not be able to be fooled by 2032 * malicious DIF.) Given the lack of backwards branches in DIF, 2033 * solving this would presumably not amount to solving the Halting 2034 * Problem -- but it still seems awfully hard. 2035 */ 2036 if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) + 2037 ksize + dsize > chunksize) { 2038 dcpu->dtdsc_drops++; 2039 return (NULL); 2040 } 2041 2042 nstate = DTRACE_DSTATE_EMPTY; 2043 2044 do { 2045 retry: 2046 free = dcpu->dtdsc_free; 2047 2048 if (free == NULL) { 2049 dtrace_dynvar_t *clean = dcpu->dtdsc_clean; 2050 void *rval; 2051 2052 if (clean == NULL) { 2053 /* 2054 * We're out of dynamic variable space on 2055 * this CPU. Unless we have tried all CPUs, 2056 * we'll try to allocate from a different 2057 * CPU. 2058 */ 2059 switch (dstate->dtds_state) { 2060 case DTRACE_DSTATE_CLEAN: { 2061 void *sp = &dstate->dtds_state; 2062 2063 if (++cpu >= NCPU) 2064 cpu = 0; 2065 2066 if (dcpu->dtdsc_dirty != NULL && 2067 nstate == DTRACE_DSTATE_EMPTY) 2068 nstate = DTRACE_DSTATE_DIRTY; 2069 2070 if (dcpu->dtdsc_rinsing != NULL) 2071 nstate = DTRACE_DSTATE_RINSING; 2072 2073 dcpu = &dstate->dtds_percpu[cpu]; 2074 2075 if (cpu != me) 2076 goto retry; 2077 2078 (void) dtrace_cas32(sp, 2079 DTRACE_DSTATE_CLEAN, nstate); 2080 2081 /* 2082 * To increment the correct bean 2083 * counter, take another lap. 2084 */ 2085 goto retry; 2086 } 2087 2088 case DTRACE_DSTATE_DIRTY: 2089 dcpu->dtdsc_dirty_drops++; 2090 break; 2091 2092 case DTRACE_DSTATE_RINSING: 2093 dcpu->dtdsc_rinsing_drops++; 2094 break; 2095 2096 case DTRACE_DSTATE_EMPTY: 2097 dcpu->dtdsc_drops++; 2098 break; 2099 } 2100 2101 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP); 2102 return (NULL); 2103 } 2104 2105 /* 2106 * The clean list appears to be non-empty. We want to 2107 * move the clean list to the free list; we start by 2108 * moving the clean pointer aside. 2109 */ 2110 if (dtrace_casptr(&dcpu->dtdsc_clean, 2111 clean, NULL) != clean) { 2112 /* 2113 * We are in one of two situations: 2114 * 2115 * (a) The clean list was switched to the 2116 * free list by another CPU. 2117 * 2118 * (b) The clean list was added to by the 2119 * cleansing cyclic. 2120 * 2121 * In either of these situations, we can 2122 * just reattempt the free list allocation. 2123 */ 2124 goto retry; 2125 } 2126 2127 ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE); 2128 2129 /* 2130 * Now we'll move the clean list to our free list. 2131 * It's impossible for this to fail: the only way 2132 * the free list can be updated is through this 2133 * code path, and only one CPU can own the clean list. 2134 * Thus, it would only be possible for this to fail if 2135 * this code were racing with dtrace_dynvar_clean(). 2136 * (That is, if dtrace_dynvar_clean() updated the clean 2137 * list, and we ended up racing to update the free 2138 * list.) This race is prevented by the dtrace_sync() 2139 * in dtrace_dynvar_clean() -- which flushes the 2140 * owners of the clean lists out before resetting 2141 * the clean lists. 2142 */ 2143 dcpu = &dstate->dtds_percpu[me]; 2144 rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean); 2145 ASSERT(rval == NULL); 2146 goto retry; 2147 } 2148 2149 dvar = free; 2150 new_free = dvar->dtdv_next; 2151 } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free); 2152 2153 /* 2154 * We have now allocated a new chunk. We copy the tuple keys into the 2155 * tuple array and copy any referenced key data into the data space 2156 * following the tuple array. As we do this, we relocate dttk_value 2157 * in the final tuple to point to the key data address in the chunk. 2158 */ 2159 kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys]; 2160 dvar->dtdv_data = (void *)(kdata + ksize); 2161 dvar->dtdv_tuple.dtt_nkeys = nkeys; 2162 2163 for (i = 0; i < nkeys; i++) { 2164 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i]; 2165 size_t kesize = key[i].dttk_size; 2166 2167 if (kesize != 0) { 2168 dtrace_bcopy( 2169 (const void *)(uintptr_t)key[i].dttk_value, 2170 (void *)kdata, kesize); 2171 dkey->dttk_value = kdata; 2172 kdata += P2ROUNDUP(kesize, sizeof (uint64_t)); 2173 } else { 2174 dkey->dttk_value = key[i].dttk_value; 2175 } 2176 2177 dkey->dttk_size = kesize; 2178 } 2179 2180 ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE); 2181 dvar->dtdv_hashval = hashval; 2182 dvar->dtdv_next = start; 2183 2184 if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start) 2185 return (dvar); 2186 2187 /* 2188 * The cas has failed. Either another CPU is adding an element to 2189 * this hash chain, or another CPU is deleting an element from this 2190 * hash chain. The simplest way to deal with both of these cases 2191 * (though not necessarily the most efficient) is to free our 2192 * allocated block and re-attempt it all. Note that the free is 2193 * to the dirty list and _not_ to the free list. This is to prevent 2194 * races with allocators, above. 2195 */ 2196 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE; 2197 2198 dtrace_membar_producer(); 2199 2200 do { 2201 free = dcpu->dtdsc_dirty; 2202 dvar->dtdv_next = free; 2203 } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free); 2204 2205 goto top; 2206 } 2207 2208 /*ARGSUSED*/ 2209 static void 2210 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg) 2211 { 2212 if ((int64_t)nval < (int64_t)*oval) 2213 *oval = nval; 2214 } 2215 2216 /*ARGSUSED*/ 2217 static void 2218 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg) 2219 { 2220 if ((int64_t)nval > (int64_t)*oval) 2221 *oval = nval; 2222 } 2223 2224 static void 2225 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr) 2226 { 2227 int i, zero = DTRACE_QUANTIZE_ZEROBUCKET; 2228 int64_t val = (int64_t)nval; 2229 2230 if (val < 0) { 2231 for (i = 0; i < zero; i++) { 2232 if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) { 2233 quanta[i] += incr; 2234 return; 2235 } 2236 } 2237 } else { 2238 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) { 2239 if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) { 2240 quanta[i - 1] += incr; 2241 return; 2242 } 2243 } 2244 2245 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr; 2246 return; 2247 } 2248 2249 ASSERT(0); 2250 } 2251 2252 static void 2253 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr) 2254 { 2255 uint64_t arg = *lquanta++; 2256 int32_t base = DTRACE_LQUANTIZE_BASE(arg); 2257 uint16_t step = DTRACE_LQUANTIZE_STEP(arg); 2258 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg); 2259 int32_t val = (int32_t)nval, level; 2260 2261 ASSERT(step != 0); 2262 ASSERT(levels != 0); 2263 2264 if (val < base) { 2265 /* 2266 * This is an underflow. 2267 */ 2268 lquanta[0] += incr; 2269 return; 2270 } 2271 2272 level = (val - base) / step; 2273 2274 if (level < levels) { 2275 lquanta[level + 1] += incr; 2276 return; 2277 } 2278 2279 /* 2280 * This is an overflow. 2281 */ 2282 lquanta[levels + 1] += incr; 2283 } 2284 2285 static int 2286 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low, 2287 uint16_t high, uint16_t nsteps, int64_t value) 2288 { 2289 int64_t this = 1, last, next; 2290 int base = 1, order; 2291 2292 ASSERT(factor <= nsteps); 2293 ASSERT(nsteps % factor == 0); 2294 2295 for (order = 0; order < low; order++) 2296 this *= factor; 2297 2298 /* 2299 * If our value is less than our factor taken to the power of the 2300 * low order of magnitude, it goes into the zeroth bucket. 2301 */ 2302 if (value < (last = this)) 2303 return (0); 2304 2305 for (this *= factor; order <= high; order++) { 2306 int nbuckets = this > nsteps ? nsteps : this; 2307 2308 if ((next = this * factor) < this) { 2309 /* 2310 * We should not generally get log/linear quantizations 2311 * with a high magnitude that allows 64-bits to 2312 * overflow, but we nonetheless protect against this 2313 * by explicitly checking for overflow, and clamping 2314 * our value accordingly. 2315 */ 2316 value = this - 1; 2317 } 2318 2319 if (value < this) { 2320 /* 2321 * If our value lies within this order of magnitude, 2322 * determine its position by taking the offset within 2323 * the order of magnitude, dividing by the bucket 2324 * width, and adding to our (accumulated) base. 2325 */ 2326 return (base + (value - last) / (this / nbuckets)); 2327 } 2328 2329 base += nbuckets - (nbuckets / factor); 2330 last = this; 2331 this = next; 2332 } 2333 2334 /* 2335 * Our value is greater than or equal to our factor taken to the 2336 * power of one plus the high magnitude -- return the top bucket. 2337 */ 2338 return (base); 2339 } 2340 2341 static void 2342 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr) 2343 { 2344 uint64_t arg = *llquanta++; 2345 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg); 2346 uint16_t low = DTRACE_LLQUANTIZE_LOW(arg); 2347 uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg); 2348 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg); 2349 2350 llquanta[dtrace_aggregate_llquantize_bucket(factor, 2351 low, high, nsteps, nval)] += incr; 2352 } 2353 2354 /*ARGSUSED*/ 2355 static void 2356 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg) 2357 { 2358 data[0]++; 2359 data[1] += nval; 2360 } 2361 2362 /*ARGSUSED*/ 2363 static void 2364 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg) 2365 { 2366 int64_t snval = (int64_t)nval; 2367 uint64_t tmp[2]; 2368 2369 data[0]++; 2370 data[1] += nval; 2371 2372 /* 2373 * What we want to say here is: 2374 * 2375 * data[2] += nval * nval; 2376 * 2377 * But given that nval is 64-bit, we could easily overflow, so 2378 * we do this as 128-bit arithmetic. 2379 */ 2380 if (snval < 0) 2381 snval = -snval; 2382 2383 dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp); 2384 dtrace_add_128(data + 2, tmp, data + 2); 2385 } 2386 2387 /*ARGSUSED*/ 2388 static void 2389 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg) 2390 { 2391 *oval = *oval + 1; 2392 } 2393 2394 /*ARGSUSED*/ 2395 static void 2396 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg) 2397 { 2398 *oval += nval; 2399 } 2400 2401 /* 2402 * Aggregate given the tuple in the principal data buffer, and the aggregating 2403 * action denoted by the specified dtrace_aggregation_t. The aggregation 2404 * buffer is specified as the buf parameter. This routine does not return 2405 * failure; if there is no space in the aggregation buffer, the data will be 2406 * dropped, and a corresponding counter incremented. 2407 */ 2408 static void 2409 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf, 2410 intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg) 2411 { 2412 dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec; 2413 uint32_t i, ndx, size, fsize; 2414 uint32_t align = sizeof (uint64_t) - 1; 2415 dtrace_aggbuffer_t *agb; 2416 dtrace_aggkey_t *key; 2417 uint32_t hashval = 0, limit, isstr; 2418 caddr_t tomax, data, kdata; 2419 dtrace_actkind_t action; 2420 dtrace_action_t *act; 2421 uintptr_t offs; 2422 2423 if (buf == NULL) 2424 return; 2425 2426 if (!agg->dtag_hasarg) { 2427 /* 2428 * Currently, only quantize() and lquantize() take additional 2429 * arguments, and they have the same semantics: an increment 2430 * value that defaults to 1 when not present. If additional 2431 * aggregating actions take arguments, the setting of the 2432 * default argument value will presumably have to become more 2433 * sophisticated... 2434 */ 2435 arg = 1; 2436 } 2437 2438 action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION; 2439 size = rec->dtrd_offset - agg->dtag_base; 2440 fsize = size + rec->dtrd_size; 2441 2442 ASSERT(dbuf->dtb_tomax != NULL); 2443 data = dbuf->dtb_tomax + offset + agg->dtag_base; 2444 2445 if ((tomax = buf->dtb_tomax) == NULL) { 2446 dtrace_buffer_drop(buf); 2447 return; 2448 } 2449 2450 /* 2451 * The metastructure is always at the bottom of the buffer. 2452 */ 2453 agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size - 2454 sizeof (dtrace_aggbuffer_t)); 2455 2456 if (buf->dtb_offset == 0) { 2457 /* 2458 * We just kludge up approximately 1/8th of the size to be 2459 * buckets. If this guess ends up being routinely 2460 * off-the-mark, we may need to dynamically readjust this 2461 * based on past performance. 2462 */ 2463 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t); 2464 2465 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) < 2466 (uintptr_t)tomax || hashsize == 0) { 2467 /* 2468 * We've been given a ludicrously small buffer; 2469 * increment our drop count and leave. 2470 */ 2471 dtrace_buffer_drop(buf); 2472 return; 2473 } 2474 2475 /* 2476 * And now, a pathetic attempt to try to get a an odd (or 2477 * perchance, a prime) hash size for better hash distribution. 2478 */ 2479 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3)) 2480 hashsize -= DTRACE_AGGHASHSIZE_SLEW; 2481 2482 agb->dtagb_hashsize = hashsize; 2483 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb - 2484 agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *)); 2485 agb->dtagb_free = (uintptr_t)agb->dtagb_hash; 2486 2487 for (i = 0; i < agb->dtagb_hashsize; i++) 2488 agb->dtagb_hash[i] = NULL; 2489 } 2490 2491 ASSERT(agg->dtag_first != NULL); 2492 ASSERT(agg->dtag_first->dta_intuple); 2493 2494 /* 2495 * Calculate the hash value based on the key. Note that we _don't_ 2496 * include the aggid in the hashing (but we will store it as part of 2497 * the key). The hashing algorithm is Bob Jenkins' "One-at-a-time" 2498 * algorithm: a simple, quick algorithm that has no known funnels, and 2499 * gets good distribution in practice. The efficacy of the hashing 2500 * algorithm (and a comparison with other algorithms) may be found by 2501 * running the ::dtrace_aggstat MDB dcmd. 2502 */ 2503 for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { 2504 i = act->dta_rec.dtrd_offset - agg->dtag_base; 2505 limit = i + act->dta_rec.dtrd_size; 2506 ASSERT(limit <= size); 2507 isstr = DTRACEACT_ISSTRING(act); 2508 2509 for (; i < limit; i++) { 2510 hashval += data[i]; 2511 hashval += (hashval << 10); 2512 hashval ^= (hashval >> 6); 2513 2514 if (isstr && data[i] == '\0') 2515 break; 2516 } 2517 } 2518 2519 hashval += (hashval << 3); 2520 hashval ^= (hashval >> 11); 2521 hashval += (hashval << 15); 2522 2523 /* 2524 * Yes, the divide here is expensive -- but it's generally the least 2525 * of the performance issues given the amount of data that we iterate 2526 * over to compute hash values, compare data, etc. 2527 */ 2528 ndx = hashval % agb->dtagb_hashsize; 2529 2530 for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) { 2531 ASSERT((caddr_t)key >= tomax); 2532 ASSERT((caddr_t)key < tomax + buf->dtb_size); 2533 2534 if (hashval != key->dtak_hashval || key->dtak_size != size) 2535 continue; 2536 2537 kdata = key->dtak_data; 2538 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size); 2539 2540 for (act = agg->dtag_first; act->dta_intuple; 2541 act = act->dta_next) { 2542 i = act->dta_rec.dtrd_offset - agg->dtag_base; 2543 limit = i + act->dta_rec.dtrd_size; 2544 ASSERT(limit <= size); 2545 isstr = DTRACEACT_ISSTRING(act); 2546 2547 for (; i < limit; i++) { 2548 if (kdata[i] != data[i]) 2549 goto next; 2550 2551 if (isstr && data[i] == '\0') 2552 break; 2553 } 2554 } 2555 2556 if (action != key->dtak_action) { 2557 /* 2558 * We are aggregating on the same value in the same 2559 * aggregation with two different aggregating actions. 2560 * (This should have been picked up in the compiler, 2561 * so we may be dealing with errant or devious DIF.) 2562 * This is an error condition; we indicate as much, 2563 * and return. 2564 */ 2565 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 2566 return; 2567 } 2568 2569 /* 2570 * This is a hit: we need to apply the aggregator to 2571 * the value at this key. 2572 */ 2573 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg); 2574 return; 2575 next: 2576 continue; 2577 } 2578 2579 /* 2580 * We didn't find it. We need to allocate some zero-filled space, 2581 * link it into the hash table appropriately, and apply the aggregator 2582 * to the (zero-filled) value. 2583 */ 2584 offs = buf->dtb_offset; 2585 while (offs & (align - 1)) 2586 offs += sizeof (uint32_t); 2587 2588 /* 2589 * If we don't have enough room to both allocate a new key _and_ 2590 * its associated data, increment the drop count and return. 2591 */ 2592 if ((uintptr_t)tomax + offs + fsize > 2593 agb->dtagb_free - sizeof (dtrace_aggkey_t)) { 2594 dtrace_buffer_drop(buf); 2595 return; 2596 } 2597 2598 /*CONSTCOND*/ 2599 ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1))); 2600 key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t)); 2601 agb->dtagb_free -= sizeof (dtrace_aggkey_t); 2602 2603 key->dtak_data = kdata = tomax + offs; 2604 buf->dtb_offset = offs + fsize; 2605 2606 /* 2607 * Now copy the data across. 2608 */ 2609 *((dtrace_aggid_t *)kdata) = agg->dtag_id; 2610 2611 for (i = sizeof (dtrace_aggid_t); i < size; i++) 2612 kdata[i] = data[i]; 2613 2614 /* 2615 * Because strings are not zeroed out by default, we need to iterate 2616 * looking for actions that store strings, and we need to explicitly 2617 * pad these strings out with zeroes. 2618 */ 2619 for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { 2620 int nul; 2621 2622 if (!DTRACEACT_ISSTRING(act)) 2623 continue; 2624 2625 i = act->dta_rec.dtrd_offset - agg->dtag_base; 2626 limit = i + act->dta_rec.dtrd_size; 2627 ASSERT(limit <= size); 2628 2629 for (nul = 0; i < limit; i++) { 2630 if (nul) { 2631 kdata[i] = '\0'; 2632 continue; 2633 } 2634 2635 if (data[i] != '\0') 2636 continue; 2637 2638 nul = 1; 2639 } 2640 } 2641 2642 for (i = size; i < fsize; i++) 2643 kdata[i] = 0; 2644 2645 key->dtak_hashval = hashval; 2646 key->dtak_size = size; 2647 key->dtak_action = action; 2648 key->dtak_next = agb->dtagb_hash[ndx]; 2649 agb->dtagb_hash[ndx] = key; 2650 2651 /* 2652 * Finally, apply the aggregator. 2653 */ 2654 *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial; 2655 agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg); 2656 } 2657 2658 /* 2659 * Given consumer state, this routine finds a speculation in the INACTIVE 2660 * state and transitions it into the ACTIVE state. If there is no speculation 2661 * in the INACTIVE state, 0 is returned. In this case, no error counter is 2662 * incremented -- it is up to the caller to take appropriate action. 2663 */ 2664 static int 2665 dtrace_speculation(dtrace_state_t *state) 2666 { 2667 int i = 0; 2668 dtrace_speculation_state_t current; 2669 uint32_t *stat = &state->dts_speculations_unavail, count; 2670 2671 while (i < state->dts_nspeculations) { 2672 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2673 2674 current = spec->dtsp_state; 2675 2676 if (current != DTRACESPEC_INACTIVE) { 2677 if (current == DTRACESPEC_COMMITTINGMANY || 2678 current == DTRACESPEC_COMMITTING || 2679 current == DTRACESPEC_DISCARDING) 2680 stat = &state->dts_speculations_busy; 2681 i++; 2682 continue; 2683 } 2684 2685 if (dtrace_cas32((uint32_t *)&spec->dtsp_state, 2686 current, DTRACESPEC_ACTIVE) == current) 2687 return (i + 1); 2688 } 2689 2690 /* 2691 * We couldn't find a speculation. If we found as much as a single 2692 * busy speculation buffer, we'll attribute this failure as "busy" 2693 * instead of "unavail". 2694 */ 2695 do { 2696 count = *stat; 2697 } while (dtrace_cas32(stat, count, count + 1) != count); 2698 2699 return (0); 2700 } 2701 2702 /* 2703 * This routine commits an active speculation. If the specified speculation 2704 * is not in a valid state to perform a commit(), this routine will silently do 2705 * nothing. The state of the specified speculation is transitioned according 2706 * to the state transition diagram outlined in <sys/dtrace_impl.h> 2707 */ 2708 static void 2709 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu, 2710 dtrace_specid_t which) 2711 { 2712 dtrace_speculation_t *spec; 2713 dtrace_buffer_t *src, *dest; 2714 uintptr_t daddr, saddr, dlimit, slimit; 2715 dtrace_speculation_state_t current, new; 2716 intptr_t offs; 2717 uint64_t timestamp; 2718 2719 if (which == 0) 2720 return; 2721 2722 if (which > state->dts_nspeculations) { 2723 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 2724 return; 2725 } 2726 2727 spec = &state->dts_speculations[which - 1]; 2728 src = &spec->dtsp_buffer[cpu]; 2729 dest = &state->dts_buffer[cpu]; 2730 2731 do { 2732 current = spec->dtsp_state; 2733 2734 if (current == DTRACESPEC_COMMITTINGMANY) 2735 break; 2736 2737 switch (current) { 2738 case DTRACESPEC_INACTIVE: 2739 case DTRACESPEC_DISCARDING: 2740 return; 2741 2742 case DTRACESPEC_COMMITTING: 2743 /* 2744 * This is only possible if we are (a) commit()'ing 2745 * without having done a prior speculate() on this CPU 2746 * and (b) racing with another commit() on a different 2747 * CPU. There's nothing to do -- we just assert that 2748 * our offset is 0. 2749 */ 2750 ASSERT(src->dtb_offset == 0); 2751 return; 2752 2753 case DTRACESPEC_ACTIVE: 2754 new = DTRACESPEC_COMMITTING; 2755 break; 2756 2757 case DTRACESPEC_ACTIVEONE: 2758 /* 2759 * This speculation is active on one CPU. If our 2760 * buffer offset is non-zero, we know that the one CPU 2761 * must be us. Otherwise, we are committing on a 2762 * different CPU from the speculate(), and we must 2763 * rely on being asynchronously cleaned. 2764 */ 2765 if (src->dtb_offset != 0) { 2766 new = DTRACESPEC_COMMITTING; 2767 break; 2768 } 2769 /*FALLTHROUGH*/ 2770 2771 case DTRACESPEC_ACTIVEMANY: 2772 new = DTRACESPEC_COMMITTINGMANY; 2773 break; 2774 2775 default: 2776 ASSERT(0); 2777 } 2778 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, 2779 current, new) != current); 2780 2781 /* 2782 * We have set the state to indicate that we are committing this 2783 * speculation. Now reserve the necessary space in the destination 2784 * buffer. 2785 */ 2786 if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset, 2787 sizeof (uint64_t), state, NULL)) < 0) { 2788 dtrace_buffer_drop(dest); 2789 goto out; 2790 } 2791 2792 /* 2793 * We have sufficient space to copy the speculative buffer into the 2794 * primary buffer. First, modify the speculative buffer, filling 2795 * in the timestamp of all entries with the current time. The data 2796 * must have the commit() time rather than the time it was traced, 2797 * so that all entries in the primary buffer are in timestamp order. 2798 */ 2799 timestamp = dtrace_gethrtime(); 2800 saddr = (uintptr_t)src->dtb_tomax; 2801 slimit = saddr + src->dtb_offset; 2802 while (saddr < slimit) { 2803 size_t size; 2804 dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr; 2805 2806 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) { 2807 saddr += sizeof (dtrace_epid_t); 2808 continue; 2809 } 2810 ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs); 2811 size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size; 2812 2813 ASSERT3U(saddr + size, <=, slimit); 2814 ASSERT3U(size, >=, sizeof (dtrace_rechdr_t)); 2815 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX); 2816 2817 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp); 2818 2819 saddr += size; 2820 } 2821 2822 /* 2823 * Copy the buffer across. (Note that this is a 2824 * highly subobtimal bcopy(); in the unlikely event that this becomes 2825 * a serious performance issue, a high-performance DTrace-specific 2826 * bcopy() should obviously be invented.) 2827 */ 2828 daddr = (uintptr_t)dest->dtb_tomax + offs; 2829 dlimit = daddr + src->dtb_offset; 2830 saddr = (uintptr_t)src->dtb_tomax; 2831 2832 /* 2833 * First, the aligned portion. 2834 */ 2835 while (dlimit - daddr >= sizeof (uint64_t)) { 2836 *((uint64_t *)daddr) = *((uint64_t *)saddr); 2837 2838 daddr += sizeof (uint64_t); 2839 saddr += sizeof (uint64_t); 2840 } 2841 2842 /* 2843 * Now any left-over bit... 2844 */ 2845 while (dlimit - daddr) 2846 *((uint8_t *)daddr++) = *((uint8_t *)saddr++); 2847 2848 /* 2849 * Finally, commit the reserved space in the destination buffer. 2850 */ 2851 dest->dtb_offset = offs + src->dtb_offset; 2852 2853 out: 2854 /* 2855 * If we're lucky enough to be the only active CPU on this speculation 2856 * buffer, we can just set the state back to DTRACESPEC_INACTIVE. 2857 */ 2858 if (current == DTRACESPEC_ACTIVE || 2859 (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) { 2860 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state, 2861 DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE); 2862 2863 ASSERT(rval == DTRACESPEC_COMMITTING); 2864 } 2865 2866 src->dtb_offset = 0; 2867 src->dtb_xamot_drops += src->dtb_drops; 2868 src->dtb_drops = 0; 2869 } 2870 2871 /* 2872 * This routine discards an active speculation. If the specified speculation 2873 * is not in a valid state to perform a discard(), this routine will silently 2874 * do nothing. The state of the specified speculation is transitioned 2875 * according to the state transition diagram outlined in <sys/dtrace_impl.h> 2876 */ 2877 static void 2878 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu, 2879 dtrace_specid_t which) 2880 { 2881 dtrace_speculation_t *spec; 2882 dtrace_speculation_state_t current, new; 2883 dtrace_buffer_t *buf; 2884 2885 if (which == 0) 2886 return; 2887 2888 if (which > state->dts_nspeculations) { 2889 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 2890 return; 2891 } 2892 2893 spec = &state->dts_speculations[which - 1]; 2894 buf = &spec->dtsp_buffer[cpu]; 2895 2896 do { 2897 current = spec->dtsp_state; 2898 2899 switch (current) { 2900 case DTRACESPEC_INACTIVE: 2901 case DTRACESPEC_COMMITTINGMANY: 2902 case DTRACESPEC_COMMITTING: 2903 case DTRACESPEC_DISCARDING: 2904 return; 2905 2906 case DTRACESPEC_ACTIVE: 2907 case DTRACESPEC_ACTIVEMANY: 2908 new = DTRACESPEC_DISCARDING; 2909 break; 2910 2911 case DTRACESPEC_ACTIVEONE: 2912 if (buf->dtb_offset != 0) { 2913 new = DTRACESPEC_INACTIVE; 2914 } else { 2915 new = DTRACESPEC_DISCARDING; 2916 } 2917 break; 2918 2919 default: 2920 ASSERT(0); 2921 } 2922 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, 2923 current, new) != current); 2924 2925 buf->dtb_offset = 0; 2926 buf->dtb_drops = 0; 2927 } 2928 2929 /* 2930 * Note: not called from probe context. This function is called 2931 * asynchronously from cross call context to clean any speculations that are 2932 * in the COMMITTINGMANY or DISCARDING states. These speculations may not be 2933 * transitioned back to the INACTIVE state until all CPUs have cleaned the 2934 * speculation. 2935 */ 2936 static void 2937 dtrace_speculation_clean_here(dtrace_state_t *state) 2938 { 2939 dtrace_icookie_t cookie; 2940 processorid_t cpu = CPU->cpu_id; 2941 dtrace_buffer_t *dest = &state->dts_buffer[cpu]; 2942 dtrace_specid_t i; 2943 2944 cookie = dtrace_interrupt_disable(); 2945 2946 if (dest->dtb_tomax == NULL) { 2947 dtrace_interrupt_enable(cookie); 2948 return; 2949 } 2950 2951 for (i = 0; i < state->dts_nspeculations; i++) { 2952 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2953 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu]; 2954 2955 if (src->dtb_tomax == NULL) 2956 continue; 2957 2958 if (spec->dtsp_state == DTRACESPEC_DISCARDING) { 2959 src->dtb_offset = 0; 2960 continue; 2961 } 2962 2963 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY) 2964 continue; 2965 2966 if (src->dtb_offset == 0) 2967 continue; 2968 2969 dtrace_speculation_commit(state, cpu, i + 1); 2970 } 2971 2972 dtrace_interrupt_enable(cookie); 2973 } 2974 2975 /* 2976 * Note: not called from probe context. This function is called 2977 * asynchronously (and at a regular interval) to clean any speculations that 2978 * are in the COMMITTINGMANY or DISCARDING states. If it discovers that there 2979 * is work to be done, it cross calls all CPUs to perform that work; 2980 * COMMITMANY and DISCARDING speculations may not be transitioned back to the 2981 * INACTIVE state until they have been cleaned by all CPUs. 2982 */ 2983 static void 2984 dtrace_speculation_clean(dtrace_state_t *state) 2985 { 2986 int work = 0, rv; 2987 dtrace_specid_t i; 2988 2989 for (i = 0; i < state->dts_nspeculations; i++) { 2990 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2991 2992 ASSERT(!spec->dtsp_cleaning); 2993 2994 if (spec->dtsp_state != DTRACESPEC_DISCARDING && 2995 spec->dtsp_state != DTRACESPEC_COMMITTINGMANY) 2996 continue; 2997 2998 work++; 2999 spec->dtsp_cleaning = 1; 3000 } 3001 3002 if (!work) 3003 return; 3004 3005 dtrace_xcall(DTRACE_CPUALL, 3006 (dtrace_xcall_t)dtrace_speculation_clean_here, state); 3007 3008 /* 3009 * We now know that all CPUs have committed or discarded their 3010 * speculation buffers, as appropriate. We can now set the state 3011 * to inactive. 3012 */ 3013 for (i = 0; i < state->dts_nspeculations; i++) { 3014 dtrace_speculation_t *spec = &state->dts_speculations[i]; 3015 dtrace_speculation_state_t current, new; 3016 3017 if (!spec->dtsp_cleaning) 3018 continue; 3019 3020 current = spec->dtsp_state; 3021 ASSERT(current == DTRACESPEC_DISCARDING || 3022 current == DTRACESPEC_COMMITTINGMANY); 3023 3024 new = DTRACESPEC_INACTIVE; 3025 3026 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new); 3027 ASSERT(rv == current); 3028 spec->dtsp_cleaning = 0; 3029 } 3030 } 3031 3032 /* 3033 * Called as part of a speculate() to get the speculative buffer associated 3034 * with a given speculation. Returns NULL if the specified speculation is not 3035 * in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and 3036 * the active CPU is not the specified CPU -- the speculation will be 3037 * atomically transitioned into the ACTIVEMANY state. 3038 */ 3039 static dtrace_buffer_t * 3040 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid, 3041 dtrace_specid_t which) 3042 { 3043 dtrace_speculation_t *spec; 3044 dtrace_speculation_state_t current, new; 3045 dtrace_buffer_t *buf; 3046 3047 if (which == 0) 3048 return (NULL); 3049 3050 if (which > state->dts_nspeculations) { 3051 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 3052 return (NULL); 3053 } 3054 3055 spec = &state->dts_speculations[which - 1]; 3056 buf = &spec->dtsp_buffer[cpuid]; 3057 3058 do { 3059 current = spec->dtsp_state; 3060 3061 switch (current) { 3062 case DTRACESPEC_INACTIVE: 3063 case DTRACESPEC_COMMITTINGMANY: 3064 case DTRACESPEC_DISCARDING: 3065 return (NULL); 3066 3067 case DTRACESPEC_COMMITTING: 3068 ASSERT(buf->dtb_offset == 0); 3069 return (NULL); 3070 3071 case DTRACESPEC_ACTIVEONE: 3072 /* 3073 * This speculation is currently active on one CPU. 3074 * Check the offset in the buffer; if it's non-zero, 3075 * that CPU must be us (and we leave the state alone). 3076 * If it's zero, assume that we're starting on a new 3077 * CPU -- and change the state to indicate that the 3078 * speculation is active on more than one CPU. 3079 */ 3080 if (buf->dtb_offset != 0) 3081 return (buf); 3082 3083 new = DTRACESPEC_ACTIVEMANY; 3084 break; 3085 3086 case DTRACESPEC_ACTIVEMANY: 3087 return (buf); 3088 3089 case DTRACESPEC_ACTIVE: 3090 new = DTRACESPEC_ACTIVEONE; 3091 break; 3092 3093 default: 3094 ASSERT(0); 3095 } 3096 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, 3097 current, new) != current); 3098 3099 ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY); 3100 return (buf); 3101 } 3102 3103 /* 3104 * Return a string. In the event that the user lacks the privilege to access 3105 * arbitrary kernel memory, we copy the string out to scratch memory so that we 3106 * don't fail access checking. 3107 * 3108 * dtrace_dif_variable() uses this routine as a helper for various 3109 * builtin values such as 'execname' and 'probefunc.' 3110 */ 3111 uintptr_t 3112 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state, 3113 dtrace_mstate_t *mstate) 3114 { 3115 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 3116 uintptr_t ret; 3117 size_t strsz; 3118 3119 /* 3120 * The easy case: this probe is allowed to read all of memory, so 3121 * we can just return this as a vanilla pointer. 3122 */ 3123 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) 3124 return (addr); 3125 3126 /* 3127 * This is the tougher case: we copy the string in question from 3128 * kernel memory into scratch memory and return it that way: this 3129 * ensures that we won't trip up when access checking tests the 3130 * BYREF return value. 3131 */ 3132 strsz = dtrace_strlen((char *)addr, size) + 1; 3133 3134 if (mstate->dtms_scratch_ptr + strsz > 3135 mstate->dtms_scratch_base + mstate->dtms_scratch_size) { 3136 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 3137 return (NULL); 3138 } 3139 3140 dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr, 3141 strsz); 3142 ret = mstate->dtms_scratch_ptr; 3143 mstate->dtms_scratch_ptr += strsz; 3144 return (ret); 3145 } 3146 3147 /* 3148 * This function implements the DIF emulator's variable lookups. The emulator 3149 * passes a reserved variable identifier and optional built-in array index. 3150 */ 3151 static uint64_t 3152 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, 3153 uint64_t ndx) 3154 { 3155 /* 3156 * If we're accessing one of the uncached arguments, we'll turn this 3157 * into a reference in the args array. 3158 */ 3159 if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) { 3160 ndx = v - DIF_VAR_ARG0; 3161 v = DIF_VAR_ARGS; 3162 } 3163 3164 switch (v) { 3165 case DIF_VAR_ARGS: 3166 if (!(mstate->dtms_access & DTRACE_ACCESS_ARGS)) { 3167 cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= 3168 CPU_DTRACE_KPRIV; 3169 return (0); 3170 } 3171 3172 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS); 3173 if (ndx >= sizeof (mstate->dtms_arg) / 3174 sizeof (mstate->dtms_arg[0])) { 3175 int aframes = mstate->dtms_probe->dtpr_aframes + 2; 3176 dtrace_provider_t *pv; 3177 uint64_t val; 3178 3179 pv = mstate->dtms_probe->dtpr_provider; 3180 if (pv->dtpv_pops.dtps_getargval != NULL) 3181 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg, 3182 mstate->dtms_probe->dtpr_id, 3183 mstate->dtms_probe->dtpr_arg, ndx, aframes); 3184 else 3185 val = dtrace_getarg(ndx, aframes); 3186 3187 /* 3188 * This is regrettably required to keep the compiler 3189 * from tail-optimizing the call to dtrace_getarg(). 3190 * The condition always evaluates to true, but the 3191 * compiler has no way of figuring that out a priori. 3192 * (None of this would be necessary if the compiler 3193 * could be relied upon to _always_ tail-optimize 3194 * the call to dtrace_getarg() -- but it can't.) 3195 */ 3196 if (mstate->dtms_probe != NULL) 3197 return (val); 3198 3199 ASSERT(0); 3200 } 3201 3202 return (mstate->dtms_arg[ndx]); 3203 3204 case DIF_VAR_UREGS: { 3205 klwp_t *lwp; 3206 3207 if (!dtrace_priv_proc(state, mstate)) 3208 return (0); 3209 3210 if ((lwp = curthread->t_lwp) == NULL) { 3211 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 3212 cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL; 3213 return (0); 3214 } 3215 3216 return (dtrace_getreg(lwp->lwp_regs, ndx)); 3217 } 3218 3219 case DIF_VAR_VMREGS: { 3220 uint64_t rval; 3221 3222 if (!dtrace_priv_kernel(state)) 3223 return (0); 3224 3225 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 3226 3227 rval = dtrace_getvmreg(ndx, 3228 &cpu_core[CPU->cpu_id].cpuc_dtrace_flags); 3229 3230 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 3231 3232 return (rval); 3233 } 3234 3235 case DIF_VAR_CURTHREAD: 3236 if (!dtrace_priv_proc(state, mstate)) 3237 return (0); 3238 return ((uint64_t)(uintptr_t)curthread); 3239 3240 case DIF_VAR_TIMESTAMP: 3241 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) { 3242 mstate->dtms_timestamp = dtrace_gethrtime(); 3243 mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP; 3244 } 3245 return (mstate->dtms_timestamp); 3246 3247 case DIF_VAR_VTIMESTAMP: 3248 ASSERT(dtrace_vtime_references != 0); 3249 return (curthread->t_dtrace_vtime); 3250 3251 case DIF_VAR_WALLTIMESTAMP: 3252 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) { 3253 mstate->dtms_walltimestamp = dtrace_gethrestime(); 3254 mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP; 3255 } 3256 return (mstate->dtms_walltimestamp); 3257 3258 case DIF_VAR_IPL: 3259 if (!dtrace_priv_kernel(state)) 3260 return (0); 3261 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) { 3262 mstate->dtms_ipl = dtrace_getipl(); 3263 mstate->dtms_present |= DTRACE_MSTATE_IPL; 3264 } 3265 return (mstate->dtms_ipl); 3266 3267 case DIF_VAR_EPID: 3268 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID); 3269 return (mstate->dtms_epid); 3270 3271 case DIF_VAR_ID: 3272 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3273 return (mstate->dtms_probe->dtpr_id); 3274 3275 case DIF_VAR_STACKDEPTH: 3276 if (!dtrace_priv_kernel(state)) 3277 return (0); 3278 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) { 3279 int aframes = mstate->dtms_probe->dtpr_aframes + 2; 3280 3281 mstate->dtms_stackdepth = dtrace_getstackdepth(aframes); 3282 mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH; 3283 } 3284 return (mstate->dtms_stackdepth); 3285 3286 case DIF_VAR_USTACKDEPTH: 3287 if (!dtrace_priv_proc(state, mstate)) 3288 return (0); 3289 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) { 3290 /* 3291 * See comment in DIF_VAR_PID. 3292 */ 3293 if (DTRACE_ANCHORED(mstate->dtms_probe) && 3294 CPU_ON_INTR(CPU)) { 3295 mstate->dtms_ustackdepth = 0; 3296 } else { 3297 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 3298 mstate->dtms_ustackdepth = 3299 dtrace_getustackdepth(); 3300 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 3301 } 3302 mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH; 3303 } 3304 return (mstate->dtms_ustackdepth); 3305 3306 case DIF_VAR_CALLER: 3307 if (!dtrace_priv_kernel(state)) 3308 return (0); 3309 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) { 3310 int aframes = mstate->dtms_probe->dtpr_aframes + 2; 3311 3312 if (!DTRACE_ANCHORED(mstate->dtms_probe)) { 3313 /* 3314 * If this is an unanchored probe, we are 3315 * required to go through the slow path: 3316 * dtrace_caller() only guarantees correct 3317 * results for anchored probes. 3318 */ 3319 pc_t caller[2]; 3320 3321 dtrace_getpcstack(caller, 2, aframes, 3322 (uint32_t *)(uintptr_t)mstate->dtms_arg[0]); 3323 mstate->dtms_caller = caller[1]; 3324 } else if ((mstate->dtms_caller = 3325 dtrace_caller(aframes)) == -1) { 3326 /* 3327 * We have failed to do this the quick way; 3328 * we must resort to the slower approach of 3329 * calling dtrace_getpcstack(). 3330 */ 3331 pc_t caller; 3332 3333 dtrace_getpcstack(&caller, 1, aframes, NULL); 3334 mstate->dtms_caller = caller; 3335 } 3336 3337 mstate->dtms_present |= DTRACE_MSTATE_CALLER; 3338 } 3339 return (mstate->dtms_caller); 3340 3341 case DIF_VAR_UCALLER: 3342 if (!dtrace_priv_proc(state, mstate)) 3343 return (0); 3344 3345 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) { 3346 uint64_t ustack[3]; 3347 3348 /* 3349 * dtrace_getupcstack() fills in the first uint64_t 3350 * with the current PID. The second uint64_t will 3351 * be the program counter at user-level. The third 3352 * uint64_t will contain the caller, which is what 3353 * we're after. 3354 */ 3355 ustack[2] = NULL; 3356 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 3357 dtrace_getupcstack(ustack, 3); 3358 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 3359 mstate->dtms_ucaller = ustack[2]; 3360 mstate->dtms_present |= DTRACE_MSTATE_UCALLER; 3361 } 3362 3363 return (mstate->dtms_ucaller); 3364 3365 case DIF_VAR_PROBEPROV: 3366 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3367 return (dtrace_dif_varstr( 3368 (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name, 3369 state, mstate)); 3370 3371 case DIF_VAR_PROBEMOD: 3372 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3373 return (dtrace_dif_varstr( 3374 (uintptr_t)mstate->dtms_probe->dtpr_mod, 3375 state, mstate)); 3376 3377 case DIF_VAR_PROBEFUNC: 3378 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3379 return (dtrace_dif_varstr( 3380 (uintptr_t)mstate->dtms_probe->dtpr_func, 3381 state, mstate)); 3382 3383 case DIF_VAR_PROBENAME: 3384 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3385 return (dtrace_dif_varstr( 3386 (uintptr_t)mstate->dtms_probe->dtpr_name, 3387 state, mstate)); 3388 3389 case DIF_VAR_PID: 3390 if (!dtrace_priv_proc(state, mstate)) 3391 return (0); 3392 3393 /* 3394 * Note that we are assuming that an unanchored probe is 3395 * always due to a high-level interrupt. (And we're assuming 3396 * that there is only a single high level interrupt.) 3397 */ 3398 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3399 return (pid0.pid_id); 3400 3401 /* 3402 * It is always safe to dereference one's own t_procp pointer: 3403 * it always points to a valid, allocated proc structure. 3404 * Further, it is always safe to dereference the p_pidp member 3405 * of one's own proc structure. (These are truisms becuase 3406 * threads and processes don't clean up their own state -- 3407 * they leave that task to whomever reaps them.) 3408 */ 3409 return ((uint64_t)curthread->t_procp->p_pidp->pid_id); 3410 3411 case DIF_VAR_PPID: 3412 if (!dtrace_priv_proc(state, mstate)) 3413 return (0); 3414 3415 /* 3416 * See comment in DIF_VAR_PID. 3417 */ 3418 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3419 return (pid0.pid_id); 3420 3421 /* 3422 * It is always safe to dereference one's own t_procp pointer: 3423 * it always points to a valid, allocated proc structure. 3424 * (This is true because threads don't clean up their own 3425 * state -- they leave that task to whomever reaps them.) 3426 */ 3427 return ((uint64_t)curthread->t_procp->p_ppid); 3428 3429 case DIF_VAR_TID: 3430 /* 3431 * See comment in DIF_VAR_PID. 3432 */ 3433 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3434 return (0); 3435 3436 return ((uint64_t)curthread->t_tid); 3437 3438 case DIF_VAR_EXECNAME: 3439 if (!dtrace_priv_proc(state, mstate)) 3440 return (0); 3441 3442 /* 3443 * See comment in DIF_VAR_PID. 3444 */ 3445 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3446 return ((uint64_t)(uintptr_t)p0.p_user.u_comm); 3447 3448 /* 3449 * It is always safe to dereference one's own t_procp pointer: 3450 * it always points to a valid, allocated proc structure. 3451 * (This is true because threads don't clean up their own 3452 * state -- they leave that task to whomever reaps them.) 3453 */ 3454 return (dtrace_dif_varstr( 3455 (uintptr_t)curthread->t_procp->p_user.u_comm, 3456 state, mstate)); 3457 3458 case DIF_VAR_ZONENAME: 3459 if (!dtrace_priv_proc(state, mstate)) 3460 return (0); 3461 3462 /* 3463 * See comment in DIF_VAR_PID. 3464 */ 3465 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3466 return ((uint64_t)(uintptr_t)p0.p_zone->zone_name); 3467 3468 /* 3469 * It is always safe to dereference one's own t_procp pointer: 3470 * it always points to a valid, allocated proc structure. 3471 * (This is true because threads don't clean up their own 3472 * state -- they leave that task to whomever reaps them.) 3473 */ 3474 return (dtrace_dif_varstr( 3475 (uintptr_t)curthread->t_procp->p_zone->zone_name, 3476 state, mstate)); 3477 3478 case DIF_VAR_UID: 3479 if (!dtrace_priv_proc(state, mstate)) 3480 return (0); 3481 3482 /* 3483 * See comment in DIF_VAR_PID. 3484 */ 3485 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3486 return ((uint64_t)p0.p_cred->cr_uid); 3487 3488 /* 3489 * It is always safe to dereference one's own t_procp pointer: 3490 * it always points to a valid, allocated proc structure. 3491 * (This is true because threads don't clean up their own 3492 * state -- they leave that task to whomever reaps them.) 3493 * 3494 * Additionally, it is safe to dereference one's own process 3495 * credential, since this is never NULL after process birth. 3496 */ 3497 return ((uint64_t)curthread->t_procp->p_cred->cr_uid); 3498 3499 case DIF_VAR_GID: 3500 if (!dtrace_priv_proc(state, mstate)) 3501 return (0); 3502 3503 /* 3504 * See comment in DIF_VAR_PID. 3505 */ 3506 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3507 return ((uint64_t)p0.p_cred->cr_gid); 3508 3509 /* 3510 * It is always safe to dereference one's own t_procp pointer: 3511 * it always points to a valid, allocated proc structure. 3512 * (This is true because threads don't clean up their own 3513 * state -- they leave that task to whomever reaps them.) 3514 * 3515 * Additionally, it is safe to dereference one's own process 3516 * credential, since this is never NULL after process birth. 3517 */ 3518 return ((uint64_t)curthread->t_procp->p_cred->cr_gid); 3519 3520 case DIF_VAR_ERRNO: { 3521 klwp_t *lwp; 3522 if (!dtrace_priv_proc(state, mstate)) 3523 return (0); 3524 3525 /* 3526 * See comment in DIF_VAR_PID. 3527 */ 3528 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3529 return (0); 3530 3531 /* 3532 * It is always safe to dereference one's own t_lwp pointer in 3533 * the event that this pointer is non-NULL. (This is true 3534 * because threads and lwps don't clean up their own state -- 3535 * they leave that task to whomever reaps them.) 3536 */ 3537 if ((lwp = curthread->t_lwp) == NULL) 3538 return (0); 3539 3540 return ((uint64_t)lwp->lwp_errno); 3541 } 3542 default: 3543 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 3544 return (0); 3545 } 3546 } 3547 3548 3549 typedef enum dtrace_json_state { 3550 DTRACE_JSON_REST = 1, 3551 DTRACE_JSON_OBJECT, 3552 DTRACE_JSON_STRING, 3553 DTRACE_JSON_STRING_ESCAPE, 3554 DTRACE_JSON_STRING_ESCAPE_UNICODE, 3555 DTRACE_JSON_COLON, 3556 DTRACE_JSON_COMMA, 3557 DTRACE_JSON_VALUE, 3558 DTRACE_JSON_IDENTIFIER, 3559 DTRACE_JSON_NUMBER, 3560 DTRACE_JSON_NUMBER_FRAC, 3561 DTRACE_JSON_NUMBER_EXP, 3562 DTRACE_JSON_COLLECT_OBJECT 3563 } dtrace_json_state_t; 3564 3565 /* 3566 * This function possesses just enough knowledge about JSON to extract a single 3567 * value from a JSON string and store it in the scratch buffer. It is able 3568 * to extract nested object values, and members of arrays by index. 3569 * 3570 * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to 3571 * be looked up as we descend into the object tree. e.g. 3572 * 3573 * foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL 3574 * with nelems = 5. 3575 * 3576 * The run time of this function must be bounded above by strsize to limit the 3577 * amount of work done in probe context. As such, it is implemented as a 3578 * simple state machine, reading one character at a time using safe loads 3579 * until we find the requested element, hit a parsing error or run off the 3580 * end of the object or string. 3581 * 3582 * As there is no way for a subroutine to return an error without interrupting 3583 * clause execution, we simply return NULL in the event of a missing key or any 3584 * other error condition. Each NULL return in this function is commented with 3585 * the error condition it represents -- parsing or otherwise. 3586 * 3587 * The set of states for the state machine closely matches the JSON 3588 * specification (http://json.org/). Briefly: 3589 * 3590 * DTRACE_JSON_REST: 3591 * Skip whitespace until we find either a top-level Object, moving 3592 * to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE. 3593 * 3594 * DTRACE_JSON_OBJECT: 3595 * Locate the next key String in an Object. Sets a flag to denote 3596 * the next String as a key string and moves to DTRACE_JSON_STRING. 3597 * 3598 * DTRACE_JSON_COLON: 3599 * Skip whitespace until we find the colon that separates key Strings 3600 * from their values. Once found, move to DTRACE_JSON_VALUE. 3601 * 3602 * DTRACE_JSON_VALUE: 3603 * Detects the type of the next value (String, Number, Identifier, Object 3604 * or Array) and routes to the states that process that type. Here we also 3605 * deal with the element selector list if we are requested to traverse down 3606 * into the object tree. 3607 * 3608 * DTRACE_JSON_COMMA: 3609 * Skip whitespace until we find the comma that separates key-value pairs 3610 * in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays 3611 * (similarly DTRACE_JSON_VALUE). All following literal value processing 3612 * states return to this state at the end of their value, unless otherwise 3613 * noted. 3614 * 3615 * DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP: 3616 * Processes a Number literal from the JSON, including any exponent 3617 * component that may be present. Numbers are returned as strings, which 3618 * may be passed to strtoll() if an integer is required. 3619 * 3620 * DTRACE_JSON_IDENTIFIER: 3621 * Processes a "true", "false" or "null" literal in the JSON. 3622 * 3623 * DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE, 3624 * DTRACE_JSON_STRING_ESCAPE_UNICODE: 3625 * Processes a String literal from the JSON, whether the String denotes 3626 * a key, a value or part of a larger Object. Handles all escape sequences 3627 * present in the specification, including four-digit unicode characters, 3628 * but merely includes the escape sequence without converting it to the 3629 * actual escaped character. If the String is flagged as a key, we 3630 * move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA. 3631 * 3632 * DTRACE_JSON_COLLECT_OBJECT: 3633 * This state collects an entire Object (or Array), correctly handling 3634 * embedded strings. If the full element selector list matches this nested 3635 * object, we return the Object in full as a string. If not, we use this 3636 * state to skip to the next value at this level and continue processing. 3637 * 3638 * NOTE: This function uses various macros from strtolctype.h to manipulate 3639 * digit values, etc -- these have all been checked to ensure they make 3640 * no additional function calls. 3641 */ 3642 static char * 3643 dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems, 3644 char *dest) 3645 { 3646 dtrace_json_state_t state = DTRACE_JSON_REST; 3647 int64_t array_elem = INT64_MIN; 3648 int64_t array_pos = 0; 3649 uint8_t escape_unicount = 0; 3650 boolean_t string_is_key = B_FALSE; 3651 boolean_t collect_object = B_FALSE; 3652 boolean_t found_key = B_FALSE; 3653 boolean_t in_array = B_FALSE; 3654 uint32_t braces = 0, brackets = 0; 3655 char *elem = elemlist; 3656 char *dd = dest; 3657 uintptr_t cur; 3658 3659 for (cur = json; cur < json + size; cur++) { 3660 char cc = dtrace_load8(cur); 3661 if (cc == '\0') 3662 return (NULL); 3663 3664 switch (state) { 3665 case DTRACE_JSON_REST: 3666 if (isspace(cc)) 3667 break; 3668 3669 if (cc == '{') { 3670 state = DTRACE_JSON_OBJECT; 3671 break; 3672 } 3673 3674 if (cc == '[') { 3675 in_array = B_TRUE; 3676 array_pos = 0; 3677 array_elem = dtrace_strtoll(elem, 10, size); 3678 found_key = array_elem == 0 ? B_TRUE : B_FALSE; 3679 state = DTRACE_JSON_VALUE; 3680 break; 3681 } 3682 3683 /* 3684 * ERROR: expected to find a top-level object or array. 3685 */ 3686 return (NULL); 3687 case DTRACE_JSON_OBJECT: 3688 if (isspace(cc)) 3689 break; 3690 3691 if (cc == '"') { 3692 state = DTRACE_JSON_STRING; 3693 string_is_key = B_TRUE; 3694 break; 3695 } 3696 3697 /* 3698 * ERROR: either the object did not start with a key 3699 * string, or we've run off the end of the object 3700 * without finding the requested key. 3701 */ 3702 return (NULL); 3703 case DTRACE_JSON_STRING: 3704 if (cc == '\\') { 3705 *dd++ = '\\'; 3706 state = DTRACE_JSON_STRING_ESCAPE; 3707 break; 3708 } 3709 3710 if (cc == '"') { 3711 if (collect_object) { 3712 /* 3713 * We don't reset the dest here, as 3714 * the string is part of a larger 3715 * object being collected. 3716 */ 3717 *dd++ = cc; 3718 collect_object = B_FALSE; 3719 state = DTRACE_JSON_COLLECT_OBJECT; 3720 break; 3721 } 3722 *dd = '\0'; 3723 dd = dest; /* reset string buffer */ 3724 if (string_is_key) { 3725 if (dtrace_strncmp(dest, elem, 3726 size) == 0) 3727 found_key = B_TRUE; 3728 } else if (found_key) { 3729 if (nelems > 1) { 3730 /* 3731 * We expected an object, not 3732 * this string. 3733 */ 3734 return (NULL); 3735 } 3736 return (dest); 3737 } 3738 state = string_is_key ? DTRACE_JSON_COLON : 3739 DTRACE_JSON_COMMA; 3740 string_is_key = B_FALSE; 3741 break; 3742 } 3743 3744 *dd++ = cc; 3745 break; 3746 case DTRACE_JSON_STRING_ESCAPE: 3747 *dd++ = cc; 3748 if (cc == 'u') { 3749 escape_unicount = 0; 3750 state = DTRACE_JSON_STRING_ESCAPE_UNICODE; 3751 } else { 3752 state = DTRACE_JSON_STRING; 3753 } 3754 break; 3755 case DTRACE_JSON_STRING_ESCAPE_UNICODE: 3756 if (!isxdigit(cc)) { 3757 /* 3758 * ERROR: invalid unicode escape, expected 3759 * four valid hexidecimal digits. 3760 */ 3761 return (NULL); 3762 } 3763 3764 *dd++ = cc; 3765 if (++escape_unicount == 4) 3766 state = DTRACE_JSON_STRING; 3767 break; 3768 case DTRACE_JSON_COLON: 3769 if (isspace(cc)) 3770 break; 3771 3772 if (cc == ':') { 3773 state = DTRACE_JSON_VALUE; 3774 break; 3775 } 3776 3777 /* 3778 * ERROR: expected a colon. 3779 */ 3780 return (NULL); 3781 case DTRACE_JSON_COMMA: 3782 if (isspace(cc)) 3783 break; 3784 3785 if (cc == ',') { 3786 if (in_array) { 3787 state = DTRACE_JSON_VALUE; 3788 if (++array_pos == array_elem) 3789 found_key = B_TRUE; 3790 } else { 3791 state = DTRACE_JSON_OBJECT; 3792 } 3793 break; 3794 } 3795 3796 /* 3797 * ERROR: either we hit an unexpected character, or 3798 * we reached the end of the object or array without 3799 * finding the requested key. 3800 */ 3801 return (NULL); 3802 case DTRACE_JSON_IDENTIFIER: 3803 if (islower(cc)) { 3804 *dd++ = cc; 3805 break; 3806 } 3807 3808 *dd = '\0'; 3809 dd = dest; /* reset string buffer */ 3810 3811 if (dtrace_strncmp(dest, "true", 5) == 0 || 3812 dtrace_strncmp(dest, "false", 6) == 0 || 3813 dtrace_strncmp(dest, "null", 5) == 0) { 3814 if (found_key) { 3815 if (nelems > 1) { 3816 /* 3817 * ERROR: We expected an object, 3818 * not this identifier. 3819 */ 3820 return (NULL); 3821 } 3822 return (dest); 3823 } else { 3824 cur--; 3825 state = DTRACE_JSON_COMMA; 3826 break; 3827 } 3828 } 3829 3830 /* 3831 * ERROR: we did not recognise the identifier as one 3832 * of those in the JSON specification. 3833 */ 3834 return (NULL); 3835 case DTRACE_JSON_NUMBER: 3836 if (cc == '.') { 3837 *dd++ = cc; 3838 state = DTRACE_JSON_NUMBER_FRAC; 3839 break; 3840 } 3841 3842 if (cc == 'x' || cc == 'X') { 3843 /* 3844 * ERROR: specification explicitly excludes 3845 * hexidecimal or octal numbers. 3846 */ 3847 return (NULL); 3848 } 3849 3850 /* FALLTHRU */ 3851 case DTRACE_JSON_NUMBER_FRAC: 3852 if (cc == 'e' || cc == 'E') { 3853 *dd++ = cc; 3854 state = DTRACE_JSON_NUMBER_EXP; 3855 break; 3856 } 3857 3858 if (cc == '+' || cc == '-') { 3859 /* 3860 * ERROR: expect sign as part of exponent only. 3861 */ 3862 return (NULL); 3863 } 3864 /* FALLTHRU */ 3865 case DTRACE_JSON_NUMBER_EXP: 3866 if (isdigit(cc) || cc == '+' || cc == '-') { 3867 *dd++ = cc; 3868 break; 3869 } 3870 3871 *dd = '\0'; 3872 dd = dest; /* reset string buffer */ 3873 if (found_key) { 3874 if (nelems > 1) { 3875 /* 3876 * ERROR: We expected an object, not 3877 * this number. 3878 */ 3879 return (NULL); 3880 } 3881 return (dest); 3882 } 3883 3884 cur--; 3885 state = DTRACE_JSON_COMMA; 3886 break; 3887 case DTRACE_JSON_VALUE: 3888 if (isspace(cc)) 3889 break; 3890 3891 if (cc == '{' || cc == '[') { 3892 if (nelems > 1 && found_key) { 3893 in_array = cc == '[' ? B_TRUE : B_FALSE; 3894 /* 3895 * If our element selector directs us 3896 * to descend into this nested object, 3897 * then move to the next selector 3898 * element in the list and restart the 3899 * state machine. 3900 */ 3901 while (*elem != '\0') 3902 elem++; 3903 elem++; /* skip the inter-element NUL */ 3904 nelems--; 3905 dd = dest; 3906 if (in_array) { 3907 state = DTRACE_JSON_VALUE; 3908 array_pos = 0; 3909 array_elem = dtrace_strtoll( 3910 elem, 10, size); 3911 found_key = array_elem == 0 ? 3912 B_TRUE : B_FALSE; 3913 } else { 3914 found_key = B_FALSE; 3915 state = DTRACE_JSON_OBJECT; 3916 } 3917 break; 3918 } 3919 3920 /* 3921 * Otherwise, we wish to either skip this 3922 * nested object or return it in full. 3923 */ 3924 if (cc == '[') 3925 brackets = 1; 3926 else 3927 braces = 1; 3928 *dd++ = cc; 3929 state = DTRACE_JSON_COLLECT_OBJECT; 3930 break; 3931 } 3932 3933 if (cc == '"') { 3934 state = DTRACE_JSON_STRING; 3935 break; 3936 } 3937 3938 if (islower(cc)) { 3939 /* 3940 * Here we deal with true, false and null. 3941 */ 3942 *dd++ = cc; 3943 state = DTRACE_JSON_IDENTIFIER; 3944 break; 3945 } 3946 3947 if (cc == '-' || isdigit(cc)) { 3948 *dd++ = cc; 3949 state = DTRACE_JSON_NUMBER; 3950 break; 3951 } 3952 3953 /* 3954 * ERROR: unexpected character at start of value. 3955 */ 3956 return (NULL); 3957 case DTRACE_JSON_COLLECT_OBJECT: 3958 if (cc == '\0') 3959 /* 3960 * ERROR: unexpected end of input. 3961 */ 3962 return (NULL); 3963 3964 *dd++ = cc; 3965 if (cc == '"') { 3966 collect_object = B_TRUE; 3967 state = DTRACE_JSON_STRING; 3968 break; 3969 } 3970 3971 if (cc == ']') { 3972 if (brackets-- == 0) { 3973 /* 3974 * ERROR: unbalanced brackets. 3975 */ 3976 return (NULL); 3977 } 3978 } else if (cc == '}') { 3979 if (braces-- == 0) { 3980 /* 3981 * ERROR: unbalanced braces. 3982 */ 3983 return (NULL); 3984 } 3985 } else if (cc == '{') { 3986 braces++; 3987 } else if (cc == '[') { 3988 brackets++; 3989 } 3990 3991 if (brackets == 0 && braces == 0) { 3992 if (found_key) { 3993 *dd = '\0'; 3994 return (dest); 3995 } 3996 dd = dest; /* reset string buffer */ 3997 state = DTRACE_JSON_COMMA; 3998 } 3999 break; 4000 } 4001 } 4002 return (NULL); 4003 } 4004 4005 /* 4006 * Emulate the execution of DTrace ID subroutines invoked by the call opcode. 4007 * Notice that we don't bother validating the proper number of arguments or 4008 * their types in the tuple stack. This isn't needed because all argument 4009 * interpretation is safe because of our load safety -- the worst that can 4010 * happen is that a bogus program can obtain bogus results. 4011 */ 4012 static void 4013 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, 4014 dtrace_key_t *tupregs, int nargs, 4015 dtrace_mstate_t *mstate, dtrace_state_t *state) 4016 { 4017 volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 4018 volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval; 4019 dtrace_vstate_t *vstate = &state->dts_vstate; 4020 4021 union { 4022 mutex_impl_t mi; 4023 uint64_t mx; 4024 } m; 4025 4026 union { 4027 krwlock_t ri; 4028 uintptr_t rw; 4029 } r; 4030 4031 switch (subr) { 4032 case DIF_SUBR_RAND: 4033 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875; 4034 break; 4035 4036 case DIF_SUBR_MUTEX_OWNED: 4037 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4038 mstate, vstate)) { 4039 regs[rd] = NULL; 4040 break; 4041 } 4042 4043 m.mx = dtrace_load64(tupregs[0].dttk_value); 4044 if (MUTEX_TYPE_ADAPTIVE(&m.mi)) 4045 regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER; 4046 else 4047 regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock); 4048 break; 4049 4050 case DIF_SUBR_MUTEX_OWNER: 4051 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4052 mstate, vstate)) { 4053 regs[rd] = NULL; 4054 break; 4055 } 4056 4057 m.mx = dtrace_load64(tupregs[0].dttk_value); 4058 if (MUTEX_TYPE_ADAPTIVE(&m.mi) && 4059 MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER) 4060 regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi); 4061 else 4062 regs[rd] = 0; 4063 break; 4064 4065 case DIF_SUBR_MUTEX_TYPE_ADAPTIVE: 4066 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4067 mstate, vstate)) { 4068 regs[rd] = NULL; 4069 break; 4070 } 4071 4072 m.mx = dtrace_load64(tupregs[0].dttk_value); 4073 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi); 4074 break; 4075 4076 case DIF_SUBR_MUTEX_TYPE_SPIN: 4077 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4078 mstate, vstate)) { 4079 regs[rd] = NULL; 4080 break; 4081 } 4082 4083 m.mx = dtrace_load64(tupregs[0].dttk_value); 4084 regs[rd] = MUTEX_TYPE_SPIN(&m.mi); 4085 break; 4086 4087 case DIF_SUBR_RW_READ_HELD: { 4088 uintptr_t tmp; 4089 4090 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), 4091 mstate, vstate)) { 4092 regs[rd] = NULL; 4093 break; 4094 } 4095 4096 r.rw = dtrace_loadptr(tupregs[0].dttk_value); 4097 regs[rd] = _RW_READ_HELD(&r.ri, tmp); 4098 break; 4099 } 4100 4101 case DIF_SUBR_RW_WRITE_HELD: 4102 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t), 4103 mstate, vstate)) { 4104 regs[rd] = NULL; 4105 break; 4106 } 4107 4108 r.rw = dtrace_loadptr(tupregs[0].dttk_value); 4109 regs[rd] = _RW_WRITE_HELD(&r.ri); 4110 break; 4111 4112 case DIF_SUBR_RW_ISWRITER: 4113 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t), 4114 mstate, vstate)) { 4115 regs[rd] = NULL; 4116 break; 4117 } 4118 4119 r.rw = dtrace_loadptr(tupregs[0].dttk_value); 4120 regs[rd] = _RW_ISWRITER(&r.ri); 4121 break; 4122 4123 case DIF_SUBR_BCOPY: { 4124 /* 4125 * We need to be sure that the destination is in the scratch 4126 * region -- no other region is allowed. 4127 */ 4128 uintptr_t src = tupregs[0].dttk_value; 4129 uintptr_t dest = tupregs[1].dttk_value; 4130 size_t size = tupregs[2].dttk_value; 4131 4132 if (!dtrace_inscratch(dest, size, mstate)) { 4133 *flags |= CPU_DTRACE_BADADDR; 4134 *illval = regs[rd]; 4135 break; 4136 } 4137 4138 if (!dtrace_canload(src, size, mstate, vstate)) { 4139 regs[rd] = NULL; 4140 break; 4141 } 4142 4143 dtrace_bcopy((void *)src, (void *)dest, size); 4144 break; 4145 } 4146 4147 case DIF_SUBR_ALLOCA: 4148 case DIF_SUBR_COPYIN: { 4149 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8); 4150 uint64_t size = 4151 tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value; 4152 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size; 4153 4154 /* 4155 * This action doesn't require any credential checks since 4156 * probes will not activate in user contexts to which the 4157 * enabling user does not have permissions. 4158 */ 4159 4160 /* 4161 * Rounding up the user allocation size could have overflowed 4162 * a large, bogus allocation (like -1ULL) to 0. 4163 */ 4164 if (scratch_size < size || 4165 !DTRACE_INSCRATCH(mstate, scratch_size)) { 4166 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4167 regs[rd] = NULL; 4168 break; 4169 } 4170 4171 if (subr == DIF_SUBR_COPYIN) { 4172 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4173 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags); 4174 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4175 } 4176 4177 mstate->dtms_scratch_ptr += scratch_size; 4178 regs[rd] = dest; 4179 break; 4180 } 4181 4182 case DIF_SUBR_COPYINTO: { 4183 uint64_t size = tupregs[1].dttk_value; 4184 uintptr_t dest = tupregs[2].dttk_value; 4185 4186 /* 4187 * This action doesn't require any credential checks since 4188 * probes will not activate in user contexts to which the 4189 * enabling user does not have permissions. 4190 */ 4191 if (!dtrace_inscratch(dest, size, mstate)) { 4192 *flags |= CPU_DTRACE_BADADDR; 4193 *illval = regs[rd]; 4194 break; 4195 } 4196 4197 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4198 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags); 4199 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4200 break; 4201 } 4202 4203 case DIF_SUBR_COPYINSTR: { 4204 uintptr_t dest = mstate->dtms_scratch_ptr; 4205 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4206 4207 if (nargs > 1 && tupregs[1].dttk_value < size) 4208 size = tupregs[1].dttk_value + 1; 4209 4210 /* 4211 * This action doesn't require any credential checks since 4212 * probes will not activate in user contexts to which the 4213 * enabling user does not have permissions. 4214 */ 4215 if (!DTRACE_INSCRATCH(mstate, size)) { 4216 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4217 regs[rd] = NULL; 4218 break; 4219 } 4220 4221 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4222 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags); 4223 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4224 4225 ((char *)dest)[size - 1] = '\0'; 4226 mstate->dtms_scratch_ptr += size; 4227 regs[rd] = dest; 4228 break; 4229 } 4230 4231 case DIF_SUBR_MSGSIZE: 4232 case DIF_SUBR_MSGDSIZE: { 4233 uintptr_t baddr = tupregs[0].dttk_value, daddr; 4234 uintptr_t wptr, rptr; 4235 size_t count = 0; 4236 int cont = 0; 4237 4238 while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { 4239 4240 if (!dtrace_canload(baddr, sizeof (mblk_t), mstate, 4241 vstate)) { 4242 regs[rd] = NULL; 4243 break; 4244 } 4245 4246 wptr = dtrace_loadptr(baddr + 4247 offsetof(mblk_t, b_wptr)); 4248 4249 rptr = dtrace_loadptr(baddr + 4250 offsetof(mblk_t, b_rptr)); 4251 4252 if (wptr < rptr) { 4253 *flags |= CPU_DTRACE_BADADDR; 4254 *illval = tupregs[0].dttk_value; 4255 break; 4256 } 4257 4258 daddr = dtrace_loadptr(baddr + 4259 offsetof(mblk_t, b_datap)); 4260 4261 baddr = dtrace_loadptr(baddr + 4262 offsetof(mblk_t, b_cont)); 4263 4264 /* 4265 * We want to prevent against denial-of-service here, 4266 * so we're only going to search the list for 4267 * dtrace_msgdsize_max mblks. 4268 */ 4269 if (cont++ > dtrace_msgdsize_max) { 4270 *flags |= CPU_DTRACE_ILLOP; 4271 break; 4272 } 4273 4274 if (subr == DIF_SUBR_MSGDSIZE) { 4275 if (dtrace_load8(daddr + 4276 offsetof(dblk_t, db_type)) != M_DATA) 4277 continue; 4278 } 4279 4280 count += wptr - rptr; 4281 } 4282 4283 if (!(*flags & CPU_DTRACE_FAULT)) 4284 regs[rd] = count; 4285 4286 break; 4287 } 4288 4289 case DIF_SUBR_PROGENYOF: { 4290 pid_t pid = tupregs[0].dttk_value; 4291 proc_t *p; 4292 int rval = 0; 4293 4294 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4295 4296 for (p = curthread->t_procp; p != NULL; p = p->p_parent) { 4297 if (p->p_pidp->pid_id == pid) { 4298 rval = 1; 4299 break; 4300 } 4301 } 4302 4303 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4304 4305 regs[rd] = rval; 4306 break; 4307 } 4308 4309 case DIF_SUBR_SPECULATION: 4310 regs[rd] = dtrace_speculation(state); 4311 break; 4312 4313 case DIF_SUBR_COPYOUT: { 4314 uintptr_t kaddr = tupregs[0].dttk_value; 4315 uintptr_t uaddr = tupregs[1].dttk_value; 4316 uint64_t size = tupregs[2].dttk_value; 4317 4318 if (!dtrace_destructive_disallow && 4319 dtrace_priv_proc_control(state, mstate) && 4320 !dtrace_istoxic(kaddr, size) && 4321 dtrace_canload(kaddr, size, mstate, vstate)) { 4322 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4323 dtrace_copyout(kaddr, uaddr, size, flags); 4324 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4325 } 4326 break; 4327 } 4328 4329 case DIF_SUBR_COPYOUTSTR: { 4330 uintptr_t kaddr = tupregs[0].dttk_value; 4331 uintptr_t uaddr = tupregs[1].dttk_value; 4332 uint64_t size = tupregs[2].dttk_value; 4333 size_t lim; 4334 4335 if (!dtrace_destructive_disallow && 4336 dtrace_priv_proc_control(state, mstate) && 4337 !dtrace_istoxic(kaddr, size) && 4338 dtrace_strcanload(kaddr, size, &lim, mstate, vstate)) { 4339 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4340 dtrace_copyoutstr(kaddr, uaddr, lim, flags); 4341 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4342 } 4343 break; 4344 } 4345 4346 case DIF_SUBR_STRLEN: { 4347 size_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4348 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value; 4349 size_t lim; 4350 4351 if (!dtrace_strcanload(addr, size, &lim, mstate, vstate)) { 4352 regs[rd] = NULL; 4353 break; 4354 } 4355 regs[rd] = dtrace_strlen((char *)addr, lim); 4356 4357 break; 4358 } 4359 4360 case DIF_SUBR_STRCHR: 4361 case DIF_SUBR_STRRCHR: { 4362 /* 4363 * We're going to iterate over the string looking for the 4364 * specified character. We will iterate until we have reached 4365 * the string length or we have found the character. If this 4366 * is DIF_SUBR_STRRCHR, we will look for the last occurrence 4367 * of the specified character instead of the first. 4368 */ 4369 uintptr_t addr = tupregs[0].dttk_value; 4370 uintptr_t addr_limit; 4371 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4372 size_t lim; 4373 char c, target = (char)tupregs[1].dttk_value; 4374 4375 if (!dtrace_strcanload(addr, size, &lim, mstate, vstate)) { 4376 regs[rd] = NULL; 4377 break; 4378 } 4379 addr_limit = addr + lim; 4380 4381 for (regs[rd] = NULL; addr < addr_limit; addr++) { 4382 if ((c = dtrace_load8(addr)) == target) { 4383 regs[rd] = addr; 4384 4385 if (subr == DIF_SUBR_STRCHR) 4386 break; 4387 } 4388 if (c == '\0') 4389 break; 4390 } 4391 4392 break; 4393 } 4394 4395 case DIF_SUBR_STRSTR: 4396 case DIF_SUBR_INDEX: 4397 case DIF_SUBR_RINDEX: { 4398 /* 4399 * We're going to iterate over the string looking for the 4400 * specified string. We will iterate until we have reached 4401 * the string length or we have found the string. (Yes, this 4402 * is done in the most naive way possible -- but considering 4403 * that the string we're searching for is likely to be 4404 * relatively short, the complexity of Rabin-Karp or similar 4405 * hardly seems merited.) 4406 */ 4407 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value; 4408 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value; 4409 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4410 size_t len = dtrace_strlen(addr, size); 4411 size_t sublen = dtrace_strlen(substr, size); 4412 char *limit = addr + len, *orig = addr; 4413 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1; 4414 int inc = 1; 4415 4416 regs[rd] = notfound; 4417 4418 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) { 4419 regs[rd] = NULL; 4420 break; 4421 } 4422 4423 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate, 4424 vstate)) { 4425 regs[rd] = NULL; 4426 break; 4427 } 4428 4429 /* 4430 * strstr() and index()/rindex() have similar semantics if 4431 * both strings are the empty string: strstr() returns a 4432 * pointer to the (empty) string, and index() and rindex() 4433 * both return index 0 (regardless of any position argument). 4434 */ 4435 if (sublen == 0 && len == 0) { 4436 if (subr == DIF_SUBR_STRSTR) 4437 regs[rd] = (uintptr_t)addr; 4438 else 4439 regs[rd] = 0; 4440 break; 4441 } 4442 4443 if (subr != DIF_SUBR_STRSTR) { 4444 if (subr == DIF_SUBR_RINDEX) { 4445 limit = orig - 1; 4446 addr += len; 4447 inc = -1; 4448 } 4449 4450 /* 4451 * Both index() and rindex() take an optional position 4452 * argument that denotes the starting position. 4453 */ 4454 if (nargs == 3) { 4455 int64_t pos = (int64_t)tupregs[2].dttk_value; 4456 4457 /* 4458 * If the position argument to index() is 4459 * negative, Perl implicitly clamps it at 4460 * zero. This semantic is a little surprising 4461 * given the special meaning of negative 4462 * positions to similar Perl functions like 4463 * substr(), but it appears to reflect a 4464 * notion that index() can start from a 4465 * negative index and increment its way up to 4466 * the string. Given this notion, Perl's 4467 * rindex() is at least self-consistent in 4468 * that it implicitly clamps positions greater 4469 * than the string length to be the string 4470 * length. Where Perl completely loses 4471 * coherence, however, is when the specified 4472 * substring is the empty string (""). In 4473 * this case, even if the position is 4474 * negative, rindex() returns 0 -- and even if 4475 * the position is greater than the length, 4476 * index() returns the string length. These 4477 * semantics violate the notion that index() 4478 * should never return a value less than the 4479 * specified position and that rindex() should 4480 * never return a value greater than the 4481 * specified position. (One assumes that 4482 * these semantics are artifacts of Perl's 4483 * implementation and not the results of 4484 * deliberate design -- it beggars belief that 4485 * even Larry Wall could desire such oddness.) 4486 * While in the abstract one would wish for 4487 * consistent position semantics across 4488 * substr(), index() and rindex() -- or at the 4489 * very least self-consistent position 4490 * semantics for index() and rindex() -- we 4491 * instead opt to keep with the extant Perl 4492 * semantics, in all their broken glory. (Do 4493 * we have more desire to maintain Perl's 4494 * semantics than Perl does? Probably.) 4495 */ 4496 if (subr == DIF_SUBR_RINDEX) { 4497 if (pos < 0) { 4498 if (sublen == 0) 4499 regs[rd] = 0; 4500 break; 4501 } 4502 4503 if (pos > len) 4504 pos = len; 4505 } else { 4506 if (pos < 0) 4507 pos = 0; 4508 4509 if (pos >= len) { 4510 if (sublen == 0) 4511 regs[rd] = len; 4512 break; 4513 } 4514 } 4515 4516 addr = orig + pos; 4517 } 4518 } 4519 4520 for (regs[rd] = notfound; addr != limit; addr += inc) { 4521 if (dtrace_strncmp(addr, substr, sublen) == 0) { 4522 if (subr != DIF_SUBR_STRSTR) { 4523 /* 4524 * As D index() and rindex() are 4525 * modeled on Perl (and not on awk), 4526 * we return a zero-based (and not a 4527 * one-based) index. (For you Perl 4528 * weenies: no, we're not going to add 4529 * $[ -- and shouldn't you be at a con 4530 * or something?) 4531 */ 4532 regs[rd] = (uintptr_t)(addr - orig); 4533 break; 4534 } 4535 4536 ASSERT(subr == DIF_SUBR_STRSTR); 4537 regs[rd] = (uintptr_t)addr; 4538 break; 4539 } 4540 } 4541 4542 break; 4543 } 4544 4545 case DIF_SUBR_STRTOK: { 4546 uintptr_t addr = tupregs[0].dttk_value; 4547 uintptr_t tokaddr = tupregs[1].dttk_value; 4548 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4549 uintptr_t limit, toklimit; 4550 size_t clim; 4551 uint8_t c, tokmap[32]; /* 256 / 8 */ 4552 char *dest = (char *)mstate->dtms_scratch_ptr; 4553 int i; 4554 4555 /* 4556 * Check both the token buffer and (later) the input buffer, 4557 * since both could be non-scratch addresses. 4558 */ 4559 if (!dtrace_strcanload(tokaddr, size, &clim, mstate, vstate)) { 4560 regs[rd] = NULL; 4561 break; 4562 } 4563 toklimit = tokaddr + clim; 4564 4565 if (!DTRACE_INSCRATCH(mstate, size)) { 4566 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4567 regs[rd] = NULL; 4568 break; 4569 } 4570 4571 if (addr == NULL) { 4572 /* 4573 * If the address specified is NULL, we use our saved 4574 * strtok pointer from the mstate. Note that this 4575 * means that the saved strtok pointer is _only_ 4576 * valid within multiple enablings of the same probe -- 4577 * it behaves like an implicit clause-local variable. 4578 */ 4579 addr = mstate->dtms_strtok; 4580 limit = mstate->dtms_strtok_limit; 4581 } else { 4582 /* 4583 * If the user-specified address is non-NULL we must 4584 * access check it. This is the only time we have 4585 * a chance to do so, since this address may reside 4586 * in the string table of this clause-- future calls 4587 * (when we fetch addr from mstate->dtms_strtok) 4588 * would fail this access check. 4589 */ 4590 if (!dtrace_strcanload(addr, size, &clim, mstate, 4591 vstate)) { 4592 regs[rd] = NULL; 4593 break; 4594 } 4595 limit = addr + clim; 4596 } 4597 4598 /* 4599 * First, zero the token map, and then process the token 4600 * string -- setting a bit in the map for every character 4601 * found in the token string. 4602 */ 4603 for (i = 0; i < sizeof (tokmap); i++) 4604 tokmap[i] = 0; 4605 4606 for (; tokaddr < toklimit; tokaddr++) { 4607 if ((c = dtrace_load8(tokaddr)) == '\0') 4608 break; 4609 4610 ASSERT((c >> 3) < sizeof (tokmap)); 4611 tokmap[c >> 3] |= (1 << (c & 0x7)); 4612 } 4613 4614 for (; addr < limit; addr++) { 4615 /* 4616 * We're looking for a character that is _not_ 4617 * contained in the token string. 4618 */ 4619 if ((c = dtrace_load8(addr)) == '\0') 4620 break; 4621 4622 if (!(tokmap[c >> 3] & (1 << (c & 0x7)))) 4623 break; 4624 } 4625 4626 if (c == '\0') { 4627 /* 4628 * We reached the end of the string without finding 4629 * any character that was not in the token string. 4630 * We return NULL in this case, and we set the saved 4631 * address to NULL as well. 4632 */ 4633 regs[rd] = NULL; 4634 mstate->dtms_strtok = NULL; 4635 mstate->dtms_strtok_limit = NULL; 4636 break; 4637 } 4638 4639 /* 4640 * From here on, we're copying into the destination string. 4641 */ 4642 for (i = 0; addr < limit && i < size - 1; addr++) { 4643 if ((c = dtrace_load8(addr)) == '\0') 4644 break; 4645 4646 if (tokmap[c >> 3] & (1 << (c & 0x7))) 4647 break; 4648 4649 ASSERT(i < size); 4650 dest[i++] = c; 4651 } 4652 4653 ASSERT(i < size); 4654 dest[i] = '\0'; 4655 regs[rd] = (uintptr_t)dest; 4656 mstate->dtms_scratch_ptr += size; 4657 mstate->dtms_strtok = addr; 4658 mstate->dtms_strtok_limit = limit; 4659 break; 4660 } 4661 4662 case DIF_SUBR_SUBSTR: { 4663 uintptr_t s = tupregs[0].dttk_value; 4664 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4665 char *d = (char *)mstate->dtms_scratch_ptr; 4666 int64_t index = (int64_t)tupregs[1].dttk_value; 4667 int64_t remaining = (int64_t)tupregs[2].dttk_value; 4668 size_t len = dtrace_strlen((char *)s, size); 4669 int64_t i; 4670 4671 if (!dtrace_canload(s, len + 1, mstate, vstate)) { 4672 regs[rd] = NULL; 4673 break; 4674 } 4675 4676 if (!DTRACE_INSCRATCH(mstate, size)) { 4677 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4678 regs[rd] = NULL; 4679 break; 4680 } 4681 4682 if (nargs <= 2) 4683 remaining = (int64_t)size; 4684 4685 if (index < 0) { 4686 index += len; 4687 4688 if (index < 0 && index + remaining > 0) { 4689 remaining += index; 4690 index = 0; 4691 } 4692 } 4693 4694 if (index >= len || index < 0) { 4695 remaining = 0; 4696 } else if (remaining < 0) { 4697 remaining += len - index; 4698 } else if (index + remaining > size) { 4699 remaining = size - index; 4700 } 4701 4702 for (i = 0; i < remaining; i++) { 4703 if ((d[i] = dtrace_load8(s + index + i)) == '\0') 4704 break; 4705 } 4706 4707 d[i] = '\0'; 4708 4709 mstate->dtms_scratch_ptr += size; 4710 regs[rd] = (uintptr_t)d; 4711 break; 4712 } 4713 4714 case DIF_SUBR_JSON: { 4715 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4716 uintptr_t json = tupregs[0].dttk_value; 4717 size_t jsonlen = dtrace_strlen((char *)json, size); 4718 uintptr_t elem = tupregs[1].dttk_value; 4719 size_t elemlen = dtrace_strlen((char *)elem, size); 4720 4721 char *dest = (char *)mstate->dtms_scratch_ptr; 4722 char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1; 4723 char *ee = elemlist; 4724 int nelems = 1; 4725 uintptr_t cur; 4726 4727 if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) || 4728 !dtrace_canload(elem, elemlen + 1, mstate, vstate)) { 4729 regs[rd] = NULL; 4730 break; 4731 } 4732 4733 if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) { 4734 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4735 regs[rd] = NULL; 4736 break; 4737 } 4738 4739 /* 4740 * Read the element selector and split it up into a packed list 4741 * of strings. 4742 */ 4743 for (cur = elem; cur < elem + elemlen; cur++) { 4744 char cc = dtrace_load8(cur); 4745 4746 if (cur == elem && cc == '[') { 4747 /* 4748 * If the first element selector key is 4749 * actually an array index then ignore the 4750 * bracket. 4751 */ 4752 continue; 4753 } 4754 4755 if (cc == ']') 4756 continue; 4757 4758 if (cc == '.' || cc == '[') { 4759 nelems++; 4760 cc = '\0'; 4761 } 4762 4763 *ee++ = cc; 4764 } 4765 *ee++ = '\0'; 4766 4767 if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist, 4768 nelems, dest)) != NULL) 4769 mstate->dtms_scratch_ptr += jsonlen + 1; 4770 break; 4771 } 4772 4773 case DIF_SUBR_TOUPPER: 4774 case DIF_SUBR_TOLOWER: { 4775 uintptr_t s = tupregs[0].dttk_value; 4776 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4777 char *dest = (char *)mstate->dtms_scratch_ptr, c; 4778 size_t len = dtrace_strlen((char *)s, size); 4779 char lower, upper, convert; 4780 int64_t i; 4781 4782 if (subr == DIF_SUBR_TOUPPER) { 4783 lower = 'a'; 4784 upper = 'z'; 4785 convert = 'A'; 4786 } else { 4787 lower = 'A'; 4788 upper = 'Z'; 4789 convert = 'a'; 4790 } 4791 4792 if (!dtrace_canload(s, len + 1, mstate, vstate)) { 4793 regs[rd] = NULL; 4794 break; 4795 } 4796 4797 if (!DTRACE_INSCRATCH(mstate, size)) { 4798 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4799 regs[rd] = NULL; 4800 break; 4801 } 4802 4803 for (i = 0; i < size - 1; i++) { 4804 if ((c = dtrace_load8(s + i)) == '\0') 4805 break; 4806 4807 if (c >= lower && c <= upper) 4808 c = convert + (c - lower); 4809 4810 dest[i] = c; 4811 } 4812 4813 ASSERT(i < size); 4814 dest[i] = '\0'; 4815 regs[rd] = (uintptr_t)dest; 4816 mstate->dtms_scratch_ptr += size; 4817 break; 4818 } 4819 4820 case DIF_SUBR_GETMAJOR: 4821 #ifdef _LP64 4822 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64; 4823 #else 4824 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ; 4825 #endif 4826 break; 4827 4828 case DIF_SUBR_GETMINOR: 4829 #ifdef _LP64 4830 regs[rd] = tupregs[0].dttk_value & MAXMIN64; 4831 #else 4832 regs[rd] = tupregs[0].dttk_value & MAXMIN; 4833 #endif 4834 break; 4835 4836 case DIF_SUBR_DDI_PATHNAME: { 4837 /* 4838 * This one is a galactic mess. We are going to roughly 4839 * emulate ddi_pathname(), but it's made more complicated 4840 * by the fact that we (a) want to include the minor name and 4841 * (b) must proceed iteratively instead of recursively. 4842 */ 4843 uintptr_t dest = mstate->dtms_scratch_ptr; 4844 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4845 char *start = (char *)dest, *end = start + size - 1; 4846 uintptr_t daddr = tupregs[0].dttk_value; 4847 int64_t minor = (int64_t)tupregs[1].dttk_value; 4848 char *s; 4849 int i, len, depth = 0; 4850 4851 /* 4852 * Due to all the pointer jumping we do and context we must 4853 * rely upon, we just mandate that the user must have kernel 4854 * read privileges to use this routine. 4855 */ 4856 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) { 4857 *flags |= CPU_DTRACE_KPRIV; 4858 *illval = daddr; 4859 regs[rd] = NULL; 4860 } 4861 4862 if (!DTRACE_INSCRATCH(mstate, size)) { 4863 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4864 regs[rd] = NULL; 4865 break; 4866 } 4867 4868 *end = '\0'; 4869 4870 /* 4871 * We want to have a name for the minor. In order to do this, 4872 * we need to walk the minor list from the devinfo. We want 4873 * to be sure that we don't infinitely walk a circular list, 4874 * so we check for circularity by sending a scout pointer 4875 * ahead two elements for every element that we iterate over; 4876 * if the list is circular, these will ultimately point to the 4877 * same element. You may recognize this little trick as the 4878 * answer to a stupid interview question -- one that always 4879 * seems to be asked by those who had to have it laboriously 4880 * explained to them, and who can't even concisely describe 4881 * the conditions under which one would be forced to resort to 4882 * this technique. Needless to say, those conditions are 4883 * found here -- and probably only here. Is this the only use 4884 * of this infamous trick in shipping, production code? If it 4885 * isn't, it probably should be... 4886 */ 4887 if (minor != -1) { 4888 uintptr_t maddr = dtrace_loadptr(daddr + 4889 offsetof(struct dev_info, devi_minor)); 4890 4891 uintptr_t next = offsetof(struct ddi_minor_data, next); 4892 uintptr_t name = offsetof(struct ddi_minor_data, 4893 d_minor) + offsetof(struct ddi_minor, name); 4894 uintptr_t dev = offsetof(struct ddi_minor_data, 4895 d_minor) + offsetof(struct ddi_minor, dev); 4896 uintptr_t scout; 4897 4898 if (maddr != NULL) 4899 scout = dtrace_loadptr(maddr + next); 4900 4901 while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { 4902 uint64_t m; 4903 #ifdef _LP64 4904 m = dtrace_load64(maddr + dev) & MAXMIN64; 4905 #else 4906 m = dtrace_load32(maddr + dev) & MAXMIN; 4907 #endif 4908 if (m != minor) { 4909 maddr = dtrace_loadptr(maddr + next); 4910 4911 if (scout == NULL) 4912 continue; 4913 4914 scout = dtrace_loadptr(scout + next); 4915 4916 if (scout == NULL) 4917 continue; 4918 4919 scout = dtrace_loadptr(scout + next); 4920 4921 if (scout == NULL) 4922 continue; 4923 4924 if (scout == maddr) { 4925 *flags |= CPU_DTRACE_ILLOP; 4926 break; 4927 } 4928 4929 continue; 4930 } 4931 4932 /* 4933 * We have the minor data. Now we need to 4934 * copy the minor's name into the end of the 4935 * pathname. 4936 */ 4937 s = (char *)dtrace_loadptr(maddr + name); 4938 len = dtrace_strlen(s, size); 4939 4940 if (*flags & CPU_DTRACE_FAULT) 4941 break; 4942 4943 if (len != 0) { 4944 if ((end -= (len + 1)) < start) 4945 break; 4946 4947 *end = ':'; 4948 } 4949 4950 for (i = 1; i <= len; i++) 4951 end[i] = dtrace_load8((uintptr_t)s++); 4952 break; 4953 } 4954 } 4955 4956 while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { 4957 ddi_node_state_t devi_state; 4958 4959 devi_state = dtrace_load32(daddr + 4960 offsetof(struct dev_info, devi_node_state)); 4961 4962 if (*flags & CPU_DTRACE_FAULT) 4963 break; 4964 4965 if (devi_state >= DS_INITIALIZED) { 4966 s = (char *)dtrace_loadptr(daddr + 4967 offsetof(struct dev_info, devi_addr)); 4968 len = dtrace_strlen(s, size); 4969 4970 if (*flags & CPU_DTRACE_FAULT) 4971 break; 4972 4973 if (len != 0) { 4974 if ((end -= (len + 1)) < start) 4975 break; 4976 4977 *end = '@'; 4978 } 4979 4980 for (i = 1; i <= len; i++) 4981 end[i] = dtrace_load8((uintptr_t)s++); 4982 } 4983 4984 /* 4985 * Now for the node name... 4986 */ 4987 s = (char *)dtrace_loadptr(daddr + 4988 offsetof(struct dev_info, devi_node_name)); 4989 4990 daddr = dtrace_loadptr(daddr + 4991 offsetof(struct dev_info, devi_parent)); 4992 4993 /* 4994 * If our parent is NULL (that is, if we're the root 4995 * node), we're going to use the special path 4996 * "devices". 4997 */ 4998 if (daddr == NULL) 4999 s = "devices"; 5000 5001 len = dtrace_strlen(s, size); 5002 if (*flags & CPU_DTRACE_FAULT) 5003 break; 5004 5005 if ((end -= (len + 1)) < start) 5006 break; 5007 5008 for (i = 1; i <= len; i++) 5009 end[i] = dtrace_load8((uintptr_t)s++); 5010 *end = '/'; 5011 5012 if (depth++ > dtrace_devdepth_max) { 5013 *flags |= CPU_DTRACE_ILLOP; 5014 break; 5015 } 5016 } 5017 5018 if (end < start) 5019 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5020 5021 if (daddr == NULL) { 5022 regs[rd] = (uintptr_t)end; 5023 mstate->dtms_scratch_ptr += size; 5024 } 5025 5026 break; 5027 } 5028 5029 case DIF_SUBR_STRJOIN: { 5030 char *d = (char *)mstate->dtms_scratch_ptr; 5031 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5032 uintptr_t s1 = tupregs[0].dttk_value; 5033 uintptr_t s2 = tupregs[1].dttk_value; 5034 int i = 0, j = 0; 5035 size_t lim1, lim2; 5036 char c; 5037 5038 if (!dtrace_strcanload(s1, size, &lim1, mstate, vstate) || 5039 !dtrace_strcanload(s2, size, &lim2, mstate, vstate)) { 5040 regs[rd] = NULL; 5041 break; 5042 } 5043 5044 if (!DTRACE_INSCRATCH(mstate, size)) { 5045 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5046 regs[rd] = NULL; 5047 break; 5048 } 5049 5050 for (;;) { 5051 if (i >= size) { 5052 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5053 regs[rd] = NULL; 5054 break; 5055 } 5056 c = (i >= lim1) ? '\0' : dtrace_load8(s1++); 5057 if ((d[i++] = c) == '\0') { 5058 i--; 5059 break; 5060 } 5061 } 5062 5063 for (;;) { 5064 if (i >= size) { 5065 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5066 regs[rd] = NULL; 5067 break; 5068 } 5069 5070 c = (j++ >= lim2) ? '\0' : dtrace_load8(s2++); 5071 if ((d[i++] = c) == '\0') 5072 break; 5073 } 5074 5075 if (i < size) { 5076 mstate->dtms_scratch_ptr += i; 5077 regs[rd] = (uintptr_t)d; 5078 } 5079 5080 break; 5081 } 5082 5083 case DIF_SUBR_STRTOLL: { 5084 uintptr_t s = tupregs[0].dttk_value; 5085 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5086 size_t lim; 5087 int base = 10; 5088 5089 if (nargs > 1) { 5090 if ((base = tupregs[1].dttk_value) <= 1 || 5091 base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { 5092 *flags |= CPU_DTRACE_ILLOP; 5093 break; 5094 } 5095 } 5096 5097 if (!dtrace_strcanload(s, size, &lim, mstate, vstate)) { 5098 regs[rd] = INT64_MIN; 5099 break; 5100 } 5101 5102 regs[rd] = dtrace_strtoll((char *)s, base, lim); 5103 break; 5104 } 5105 5106 case DIF_SUBR_LLTOSTR: { 5107 int64_t i = (int64_t)tupregs[0].dttk_value; 5108 uint64_t val, digit; 5109 uint64_t size = 65; /* enough room for 2^64 in binary */ 5110 char *end = (char *)mstate->dtms_scratch_ptr + size - 1; 5111 int base = 10; 5112 5113 if (nargs > 1) { 5114 if ((base = tupregs[1].dttk_value) <= 1 || 5115 base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { 5116 *flags |= CPU_DTRACE_ILLOP; 5117 break; 5118 } 5119 } 5120 5121 val = (base == 10 && i < 0) ? i * -1 : i; 5122 5123 if (!DTRACE_INSCRATCH(mstate, size)) { 5124 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5125 regs[rd] = NULL; 5126 break; 5127 } 5128 5129 for (*end-- = '\0'; val; val /= base) { 5130 if ((digit = val % base) <= '9' - '0') { 5131 *end-- = '0' + digit; 5132 } else { 5133 *end-- = 'a' + (digit - ('9' - '0') - 1); 5134 } 5135 } 5136 5137 if (i == 0 && base == 16) 5138 *end-- = '0'; 5139 5140 if (base == 16) 5141 *end-- = 'x'; 5142 5143 if (i == 0 || base == 8 || base == 16) 5144 *end-- = '0'; 5145 5146 if (i < 0 && base == 10) 5147 *end-- = '-'; 5148 5149 regs[rd] = (uintptr_t)end + 1; 5150 mstate->dtms_scratch_ptr += size; 5151 break; 5152 } 5153 5154 case DIF_SUBR_HTONS: 5155 case DIF_SUBR_NTOHS: 5156 #ifdef _BIG_ENDIAN 5157 regs[rd] = (uint16_t)tupregs[0].dttk_value; 5158 #else 5159 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value); 5160 #endif 5161 break; 5162 5163 5164 case DIF_SUBR_HTONL: 5165 case DIF_SUBR_NTOHL: 5166 #ifdef _BIG_ENDIAN 5167 regs[rd] = (uint32_t)tupregs[0].dttk_value; 5168 #else 5169 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value); 5170 #endif 5171 break; 5172 5173 5174 case DIF_SUBR_HTONLL: 5175 case DIF_SUBR_NTOHLL: 5176 #ifdef _BIG_ENDIAN 5177 regs[rd] = (uint64_t)tupregs[0].dttk_value; 5178 #else 5179 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value); 5180 #endif 5181 break; 5182 5183 5184 case DIF_SUBR_DIRNAME: 5185 case DIF_SUBR_BASENAME: { 5186 char *dest = (char *)mstate->dtms_scratch_ptr; 5187 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5188 uintptr_t src = tupregs[0].dttk_value; 5189 int i, j, len = dtrace_strlen((char *)src, size); 5190 int lastbase = -1, firstbase = -1, lastdir = -1; 5191 int start, end; 5192 5193 if (!dtrace_canload(src, len + 1, mstate, vstate)) { 5194 regs[rd] = NULL; 5195 break; 5196 } 5197 5198 if (!DTRACE_INSCRATCH(mstate, size)) { 5199 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5200 regs[rd] = NULL; 5201 break; 5202 } 5203 5204 /* 5205 * The basename and dirname for a zero-length string is 5206 * defined to be "." 5207 */ 5208 if (len == 0) { 5209 len = 1; 5210 src = (uintptr_t)"."; 5211 } 5212 5213 /* 5214 * Start from the back of the string, moving back toward the 5215 * front until we see a character that isn't a slash. That 5216 * character is the last character in the basename. 5217 */ 5218 for (i = len - 1; i >= 0; i--) { 5219 if (dtrace_load8(src + i) != '/') 5220 break; 5221 } 5222 5223 if (i >= 0) 5224 lastbase = i; 5225 5226 /* 5227 * Starting from the last character in the basename, move 5228 * towards the front until we find a slash. The character 5229 * that we processed immediately before that is the first 5230 * character in the basename. 5231 */ 5232 for (; i >= 0; i--) { 5233 if (dtrace_load8(src + i) == '/') 5234 break; 5235 } 5236 5237 if (i >= 0) 5238 firstbase = i + 1; 5239 5240 /* 5241 * Now keep going until we find a non-slash character. That 5242 * character is the last character in the dirname. 5243 */ 5244 for (; i >= 0; i--) { 5245 if (dtrace_load8(src + i) != '/') 5246 break; 5247 } 5248 5249 if (i >= 0) 5250 lastdir = i; 5251 5252 ASSERT(!(lastbase == -1 && firstbase != -1)); 5253 ASSERT(!(firstbase == -1 && lastdir != -1)); 5254 5255 if (lastbase == -1) { 5256 /* 5257 * We didn't find a non-slash character. We know that 5258 * the length is non-zero, so the whole string must be 5259 * slashes. In either the dirname or the basename 5260 * case, we return '/'. 5261 */ 5262 ASSERT(firstbase == -1); 5263 firstbase = lastbase = lastdir = 0; 5264 } 5265 5266 if (firstbase == -1) { 5267 /* 5268 * The entire string consists only of a basename 5269 * component. If we're looking for dirname, we need 5270 * to change our string to be just "."; if we're 5271 * looking for a basename, we'll just set the first 5272 * character of the basename to be 0. 5273 */ 5274 if (subr == DIF_SUBR_DIRNAME) { 5275 ASSERT(lastdir == -1); 5276 src = (uintptr_t)"."; 5277 lastdir = 0; 5278 } else { 5279 firstbase = 0; 5280 } 5281 } 5282 5283 if (subr == DIF_SUBR_DIRNAME) { 5284 if (lastdir == -1) { 5285 /* 5286 * We know that we have a slash in the name -- 5287 * or lastdir would be set to 0, above. And 5288 * because lastdir is -1, we know that this 5289 * slash must be the first character. (That 5290 * is, the full string must be of the form 5291 * "/basename".) In this case, the last 5292 * character of the directory name is 0. 5293 */ 5294 lastdir = 0; 5295 } 5296 5297 start = 0; 5298 end = lastdir; 5299 } else { 5300 ASSERT(subr == DIF_SUBR_BASENAME); 5301 ASSERT(firstbase != -1 && lastbase != -1); 5302 start = firstbase; 5303 end = lastbase; 5304 } 5305 5306 for (i = start, j = 0; i <= end && j < size - 1; i++, j++) 5307 dest[j] = dtrace_load8(src + i); 5308 5309 dest[j] = '\0'; 5310 regs[rd] = (uintptr_t)dest; 5311 mstate->dtms_scratch_ptr += size; 5312 break; 5313 } 5314 5315 case DIF_SUBR_GETF: { 5316 uintptr_t fd = tupregs[0].dttk_value; 5317 uf_info_t *finfo = &curthread->t_procp->p_user.u_finfo; 5318 file_t *fp; 5319 5320 if (!dtrace_priv_proc(state, mstate)) { 5321 regs[rd] = NULL; 5322 break; 5323 } 5324 5325 /* 5326 * This is safe because fi_nfiles only increases, and the 5327 * fi_list array is not freed when the array size doubles. 5328 * (See the comment in flist_grow() for details on the 5329 * management of the u_finfo structure.) 5330 */ 5331 fp = fd < finfo->fi_nfiles ? finfo->fi_list[fd].uf_file : NULL; 5332 5333 mstate->dtms_getf = fp; 5334 regs[rd] = (uintptr_t)fp; 5335 break; 5336 } 5337 5338 case DIF_SUBR_CLEANPATH: { 5339 char *dest = (char *)mstate->dtms_scratch_ptr, c; 5340 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5341 uintptr_t src = tupregs[0].dttk_value; 5342 size_t lim; 5343 int i = 0, j = 0; 5344 zone_t *z; 5345 5346 if (!dtrace_strcanload(src, size, &lim, mstate, vstate)) { 5347 regs[rd] = NULL; 5348 break; 5349 } 5350 5351 if (!DTRACE_INSCRATCH(mstate, size)) { 5352 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5353 regs[rd] = NULL; 5354 break; 5355 } 5356 5357 /* 5358 * Move forward, loading each character. 5359 */ 5360 do { 5361 c = (i >= lim) ? '\0' : dtrace_load8(src + i++); 5362 next: 5363 if (j + 5 >= size) /* 5 = strlen("/..c\0") */ 5364 break; 5365 5366 if (c != '/') { 5367 dest[j++] = c; 5368 continue; 5369 } 5370 5371 c = (i >= lim) ? '\0' : dtrace_load8(src + i++); 5372 5373 if (c == '/') { 5374 /* 5375 * We have two slashes -- we can just advance 5376 * to the next character. 5377 */ 5378 goto next; 5379 } 5380 5381 if (c != '.') { 5382 /* 5383 * This is not "." and it's not ".." -- we can 5384 * just store the "/" and this character and 5385 * drive on. 5386 */ 5387 dest[j++] = '/'; 5388 dest[j++] = c; 5389 continue; 5390 } 5391 5392 c = (i >= lim) ? '\0' : dtrace_load8(src + i++); 5393 5394 if (c == '/') { 5395 /* 5396 * This is a "/./" component. We're not going 5397 * to store anything in the destination buffer; 5398 * we're just going to go to the next component. 5399 */ 5400 goto next; 5401 } 5402 5403 if (c != '.') { 5404 /* 5405 * This is not ".." -- we can just store the 5406 * "/." and this character and continue 5407 * processing. 5408 */ 5409 dest[j++] = '/'; 5410 dest[j++] = '.'; 5411 dest[j++] = c; 5412 continue; 5413 } 5414 5415 c = (i >= lim) ? '\0' : dtrace_load8(src + i++); 5416 5417 if (c != '/' && c != '\0') { 5418 /* 5419 * This is not ".." -- it's "..[mumble]". 5420 * We'll store the "/.." and this character 5421 * and continue processing. 5422 */ 5423 dest[j++] = '/'; 5424 dest[j++] = '.'; 5425 dest[j++] = '.'; 5426 dest[j++] = c; 5427 continue; 5428 } 5429 5430 /* 5431 * This is "/../" or "/..\0". We need to back up 5432 * our destination pointer until we find a "/". 5433 */ 5434 i--; 5435 while (j != 0 && dest[--j] != '/') 5436 continue; 5437 5438 if (c == '\0') 5439 dest[++j] = '/'; 5440 } while (c != '\0'); 5441 5442 dest[j] = '\0'; 5443 5444 if (mstate->dtms_getf != NULL && 5445 !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) && 5446 (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) { 5447 /* 5448 * If we've done a getf() as a part of this ECB and we 5449 * don't have kernel access (and we're not in the global 5450 * zone), check if the path we cleaned up begins with 5451 * the zone's root path, and trim it off if so. Note 5452 * that this is an output cleanliness issue, not a 5453 * security issue: knowing one's zone root path does 5454 * not enable privilege escalation. 5455 */ 5456 if (strstr(dest, z->zone_rootpath) == dest) 5457 dest += strlen(z->zone_rootpath) - 1; 5458 } 5459 5460 regs[rd] = (uintptr_t)dest; 5461 mstate->dtms_scratch_ptr += size; 5462 break; 5463 } 5464 5465 case DIF_SUBR_INET_NTOA: 5466 case DIF_SUBR_INET_NTOA6: 5467 case DIF_SUBR_INET_NTOP: { 5468 size_t size; 5469 int af, argi, i; 5470 char *base, *end; 5471 5472 if (subr == DIF_SUBR_INET_NTOP) { 5473 af = (int)tupregs[0].dttk_value; 5474 argi = 1; 5475 } else { 5476 af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6; 5477 argi = 0; 5478 } 5479 5480 if (af == AF_INET) { 5481 ipaddr_t ip4; 5482 uint8_t *ptr8, val; 5483 5484 if (!dtrace_canload(tupregs[argi].dttk_value, 5485 sizeof (ipaddr_t), mstate, vstate)) { 5486 regs[rd] = NULL; 5487 break; 5488 } 5489 5490 /* 5491 * Safely load the IPv4 address. 5492 */ 5493 ip4 = dtrace_load32(tupregs[argi].dttk_value); 5494 5495 /* 5496 * Check an IPv4 string will fit in scratch. 5497 */ 5498 size = INET_ADDRSTRLEN; 5499 if (!DTRACE_INSCRATCH(mstate, size)) { 5500 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5501 regs[rd] = NULL; 5502 break; 5503 } 5504 base = (char *)mstate->dtms_scratch_ptr; 5505 end = (char *)mstate->dtms_scratch_ptr + size - 1; 5506 5507 /* 5508 * Stringify as a dotted decimal quad. 5509 */ 5510 *end-- = '\0'; 5511 ptr8 = (uint8_t *)&ip4; 5512 for (i = 3; i >= 0; i--) { 5513 val = ptr8[i]; 5514 5515 if (val == 0) { 5516 *end-- = '0'; 5517 } else { 5518 for (; val; val /= 10) { 5519 *end-- = '0' + (val % 10); 5520 } 5521 } 5522 5523 if (i > 0) 5524 *end-- = '.'; 5525 } 5526 ASSERT(end + 1 >= base); 5527 5528 } else if (af == AF_INET6) { 5529 struct in6_addr ip6; 5530 int firstzero, tryzero, numzero, v6end; 5531 uint16_t val; 5532 const char digits[] = "0123456789abcdef"; 5533 5534 /* 5535 * Stringify using RFC 1884 convention 2 - 16 bit 5536 * hexadecimal values with a zero-run compression. 5537 * Lower case hexadecimal digits are used. 5538 * eg, fe80::214:4fff:fe0b:76c8. 5539 * The IPv4 embedded form is returned for inet_ntop, 5540 * just the IPv4 string is returned for inet_ntoa6. 5541 */ 5542 5543 if (!dtrace_canload(tupregs[argi].dttk_value, 5544 sizeof (struct in6_addr), mstate, vstate)) { 5545 regs[rd] = NULL; 5546 break; 5547 } 5548 5549 /* 5550 * Safely load the IPv6 address. 5551 */ 5552 dtrace_bcopy( 5553 (void *)(uintptr_t)tupregs[argi].dttk_value, 5554 (void *)(uintptr_t)&ip6, sizeof (struct in6_addr)); 5555 5556 /* 5557 * Check an IPv6 string will fit in scratch. 5558 */ 5559 size = INET6_ADDRSTRLEN; 5560 if (!DTRACE_INSCRATCH(mstate, size)) { 5561 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5562 regs[rd] = NULL; 5563 break; 5564 } 5565 base = (char *)mstate->dtms_scratch_ptr; 5566 end = (char *)mstate->dtms_scratch_ptr + size - 1; 5567 *end-- = '\0'; 5568 5569 /* 5570 * Find the longest run of 16 bit zero values 5571 * for the single allowed zero compression - "::". 5572 */ 5573 firstzero = -1; 5574 tryzero = -1; 5575 numzero = 1; 5576 for (i = 0; i < sizeof (struct in6_addr); i++) { 5577 if (ip6._S6_un._S6_u8[i] == 0 && 5578 tryzero == -1 && i % 2 == 0) { 5579 tryzero = i; 5580 continue; 5581 } 5582 5583 if (tryzero != -1 && 5584 (ip6._S6_un._S6_u8[i] != 0 || 5585 i == sizeof (struct in6_addr) - 1)) { 5586 5587 if (i - tryzero <= numzero) { 5588 tryzero = -1; 5589 continue; 5590 } 5591 5592 firstzero = tryzero; 5593 numzero = i - i % 2 - tryzero; 5594 tryzero = -1; 5595 5596 if (ip6._S6_un._S6_u8[i] == 0 && 5597 i == sizeof (struct in6_addr) - 1) 5598 numzero += 2; 5599 } 5600 } 5601 ASSERT(firstzero + numzero <= sizeof (struct in6_addr)); 5602 5603 /* 5604 * Check for an IPv4 embedded address. 5605 */ 5606 v6end = sizeof (struct in6_addr) - 2; 5607 if (IN6_IS_ADDR_V4MAPPED(&ip6) || 5608 IN6_IS_ADDR_V4COMPAT(&ip6)) { 5609 for (i = sizeof (struct in6_addr) - 1; 5610 i >= DTRACE_V4MAPPED_OFFSET; i--) { 5611 ASSERT(end >= base); 5612 5613 val = ip6._S6_un._S6_u8[i]; 5614 5615 if (val == 0) { 5616 *end-- = '0'; 5617 } else { 5618 for (; val; val /= 10) { 5619 *end-- = '0' + val % 10; 5620 } 5621 } 5622 5623 if (i > DTRACE_V4MAPPED_OFFSET) 5624 *end-- = '.'; 5625 } 5626 5627 if (subr == DIF_SUBR_INET_NTOA6) 5628 goto inetout; 5629 5630 /* 5631 * Set v6end to skip the IPv4 address that 5632 * we have already stringified. 5633 */ 5634 v6end = 10; 5635 } 5636 5637 /* 5638 * Build the IPv6 string by working through the 5639 * address in reverse. 5640 */ 5641 for (i = v6end; i >= 0; i -= 2) { 5642 ASSERT(end >= base); 5643 5644 if (i == firstzero + numzero - 2) { 5645 *end-- = ':'; 5646 *end-- = ':'; 5647 i -= numzero - 2; 5648 continue; 5649 } 5650 5651 if (i < 14 && i != firstzero - 2) 5652 *end-- = ':'; 5653 5654 val = (ip6._S6_un._S6_u8[i] << 8) + 5655 ip6._S6_un._S6_u8[i + 1]; 5656 5657 if (val == 0) { 5658 *end-- = '0'; 5659 } else { 5660 for (; val; val /= 16) { 5661 *end-- = digits[val % 16]; 5662 } 5663 } 5664 } 5665 ASSERT(end + 1 >= base); 5666 5667 } else { 5668 /* 5669 * The user didn't use AH_INET or AH_INET6. 5670 */ 5671 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 5672 regs[rd] = NULL; 5673 break; 5674 } 5675 5676 inetout: regs[rd] = (uintptr_t)end + 1; 5677 mstate->dtms_scratch_ptr += size; 5678 break; 5679 } 5680 5681 } 5682 } 5683 5684 /* 5685 * Emulate the execution of DTrace IR instructions specified by the given 5686 * DIF object. This function is deliberately void of assertions as all of 5687 * the necessary checks are handled by a call to dtrace_difo_validate(). 5688 */ 5689 static uint64_t 5690 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate, 5691 dtrace_vstate_t *vstate, dtrace_state_t *state) 5692 { 5693 const dif_instr_t *text = difo->dtdo_buf; 5694 const uint_t textlen = difo->dtdo_len; 5695 const char *strtab = difo->dtdo_strtab; 5696 const uint64_t *inttab = difo->dtdo_inttab; 5697 5698 uint64_t rval = 0; 5699 dtrace_statvar_t *svar; 5700 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; 5701 dtrace_difv_t *v; 5702 volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 5703 volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval; 5704 5705 dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ 5706 uint64_t regs[DIF_DIR_NREGS]; 5707 uint64_t *tmp; 5708 5709 uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0; 5710 int64_t cc_r; 5711 uint_t pc = 0, id, opc; 5712 uint8_t ttop = 0; 5713 dif_instr_t instr; 5714 uint_t r1, r2, rd; 5715 5716 /* 5717 * We stash the current DIF object into the machine state: we need it 5718 * for subsequent access checking. 5719 */ 5720 mstate->dtms_difo = difo; 5721 5722 regs[DIF_REG_R0] = 0; /* %r0 is fixed at zero */ 5723 5724 while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) { 5725 opc = pc; 5726 5727 instr = text[pc++]; 5728 r1 = DIF_INSTR_R1(instr); 5729 r2 = DIF_INSTR_R2(instr); 5730 rd = DIF_INSTR_RD(instr); 5731 5732 switch (DIF_INSTR_OP(instr)) { 5733 case DIF_OP_OR: 5734 regs[rd] = regs[r1] | regs[r2]; 5735 break; 5736 case DIF_OP_XOR: 5737 regs[rd] = regs[r1] ^ regs[r2]; 5738 break; 5739 case DIF_OP_AND: 5740 regs[rd] = regs[r1] & regs[r2]; 5741 break; 5742 case DIF_OP_SLL: 5743 regs[rd] = regs[r1] << regs[r2]; 5744 break; 5745 case DIF_OP_SRL: 5746 regs[rd] = regs[r1] >> regs[r2]; 5747 break; 5748 case DIF_OP_SUB: 5749 regs[rd] = regs[r1] - regs[r2]; 5750 break; 5751 case DIF_OP_ADD: 5752 regs[rd] = regs[r1] + regs[r2]; 5753 break; 5754 case DIF_OP_MUL: 5755 regs[rd] = regs[r1] * regs[r2]; 5756 break; 5757 case DIF_OP_SDIV: 5758 if (regs[r2] == 0) { 5759 regs[rd] = 0; 5760 *flags |= CPU_DTRACE_DIVZERO; 5761 } else { 5762 regs[rd] = (int64_t)regs[r1] / 5763 (int64_t)regs[r2]; 5764 } 5765 break; 5766 5767 case DIF_OP_UDIV: 5768 if (regs[r2] == 0) { 5769 regs[rd] = 0; 5770 *flags |= CPU_DTRACE_DIVZERO; 5771 } else { 5772 regs[rd] = regs[r1] / regs[r2]; 5773 } 5774 break; 5775 5776 case DIF_OP_SREM: 5777 if (regs[r2] == 0) { 5778 regs[rd] = 0; 5779 *flags |= CPU_DTRACE_DIVZERO; 5780 } else { 5781 regs[rd] = (int64_t)regs[r1] % 5782 (int64_t)regs[r2]; 5783 } 5784 break; 5785 5786 case DIF_OP_UREM: 5787 if (regs[r2] == 0) { 5788 regs[rd] = 0; 5789 *flags |= CPU_DTRACE_DIVZERO; 5790 } else { 5791 regs[rd] = regs[r1] % regs[r2]; 5792 } 5793 break; 5794 5795 case DIF_OP_NOT: 5796 regs[rd] = ~regs[r1]; 5797 break; 5798 case DIF_OP_MOV: 5799 regs[rd] = regs[r1]; 5800 break; 5801 case DIF_OP_CMP: 5802 cc_r = regs[r1] - regs[r2]; 5803 cc_n = cc_r < 0; 5804 cc_z = cc_r == 0; 5805 cc_v = 0; 5806 cc_c = regs[r1] < regs[r2]; 5807 break; 5808 case DIF_OP_TST: 5809 cc_n = cc_v = cc_c = 0; 5810 cc_z = regs[r1] == 0; 5811 break; 5812 case DIF_OP_BA: 5813 pc = DIF_INSTR_LABEL(instr); 5814 break; 5815 case DIF_OP_BE: 5816 if (cc_z) 5817 pc = DIF_INSTR_LABEL(instr); 5818 break; 5819 case DIF_OP_BNE: 5820 if (cc_z == 0) 5821 pc = DIF_INSTR_LABEL(instr); 5822 break; 5823 case DIF_OP_BG: 5824 if ((cc_z | (cc_n ^ cc_v)) == 0) 5825 pc = DIF_INSTR_LABEL(instr); 5826 break; 5827 case DIF_OP_BGU: 5828 if ((cc_c | cc_z) == 0) 5829 pc = DIF_INSTR_LABEL(instr); 5830 break; 5831 case DIF_OP_BGE: 5832 if ((cc_n ^ cc_v) == 0) 5833 pc = DIF_INSTR_LABEL(instr); 5834 break; 5835 case DIF_OP_BGEU: 5836 if (cc_c == 0) 5837 pc = DIF_INSTR_LABEL(instr); 5838 break; 5839 case DIF_OP_BL: 5840 if (cc_n ^ cc_v) 5841 pc = DIF_INSTR_LABEL(instr); 5842 break; 5843 case DIF_OP_BLU: 5844 if (cc_c) 5845 pc = DIF_INSTR_LABEL(instr); 5846 break; 5847 case DIF_OP_BLE: 5848 if (cc_z | (cc_n ^ cc_v)) 5849 pc = DIF_INSTR_LABEL(instr); 5850 break; 5851 case DIF_OP_BLEU: 5852 if (cc_c | cc_z) 5853 pc = DIF_INSTR_LABEL(instr); 5854 break; 5855 case DIF_OP_RLDSB: 5856 if (!dtrace_canload(regs[r1], 1, mstate, vstate)) 5857 break; 5858 /*FALLTHROUGH*/ 5859 case DIF_OP_LDSB: 5860 regs[rd] = (int8_t)dtrace_load8(regs[r1]); 5861 break; 5862 case DIF_OP_RLDSH: 5863 if (!dtrace_canload(regs[r1], 2, mstate, vstate)) 5864 break; 5865 /*FALLTHROUGH*/ 5866 case DIF_OP_LDSH: 5867 regs[rd] = (int16_t)dtrace_load16(regs[r1]); 5868 break; 5869 case DIF_OP_RLDSW: 5870 if (!dtrace_canload(regs[r1], 4, mstate, vstate)) 5871 break; 5872 /*FALLTHROUGH*/ 5873 case DIF_OP_LDSW: 5874 regs[rd] = (int32_t)dtrace_load32(regs[r1]); 5875 break; 5876 case DIF_OP_RLDUB: 5877 if (!dtrace_canload(regs[r1], 1, mstate, vstate)) 5878 break; 5879 /*FALLTHROUGH*/ 5880 case DIF_OP_LDUB: 5881 regs[rd] = dtrace_load8(regs[r1]); 5882 break; 5883 case DIF_OP_RLDUH: 5884 if (!dtrace_canload(regs[r1], 2, mstate, vstate)) 5885 break; 5886 /*FALLTHROUGH*/ 5887 case DIF_OP_LDUH: 5888 regs[rd] = dtrace_load16(regs[r1]); 5889 break; 5890 case DIF_OP_RLDUW: 5891 if (!dtrace_canload(regs[r1], 4, mstate, vstate)) 5892 break; 5893 /*FALLTHROUGH*/ 5894 case DIF_OP_LDUW: 5895 regs[rd] = dtrace_load32(regs[r1]); 5896 break; 5897 case DIF_OP_RLDX: 5898 if (!dtrace_canload(regs[r1], 8, mstate, vstate)) 5899 break; 5900 /*FALLTHROUGH*/ 5901 case DIF_OP_LDX: 5902 regs[rd] = dtrace_load64(regs[r1]); 5903 break; 5904 case DIF_OP_ULDSB: 5905 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5906 regs[rd] = (int8_t) 5907 dtrace_fuword8((void *)(uintptr_t)regs[r1]); 5908 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5909 break; 5910 case DIF_OP_ULDSH: 5911 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5912 regs[rd] = (int16_t) 5913 dtrace_fuword16((void *)(uintptr_t)regs[r1]); 5914 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5915 break; 5916 case DIF_OP_ULDSW: 5917 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5918 regs[rd] = (int32_t) 5919 dtrace_fuword32((void *)(uintptr_t)regs[r1]); 5920 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5921 break; 5922 case DIF_OP_ULDUB: 5923 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5924 regs[rd] = 5925 dtrace_fuword8((void *)(uintptr_t)regs[r1]); 5926 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5927 break; 5928 case DIF_OP_ULDUH: 5929 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5930 regs[rd] = 5931 dtrace_fuword16((void *)(uintptr_t)regs[r1]); 5932 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5933 break; 5934 case DIF_OP_ULDUW: 5935 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5936 regs[rd] = 5937 dtrace_fuword32((void *)(uintptr_t)regs[r1]); 5938 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5939 break; 5940 case DIF_OP_ULDX: 5941 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 5942 regs[rd] = 5943 dtrace_fuword64((void *)(uintptr_t)regs[r1]); 5944 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 5945 break; 5946 case DIF_OP_RET: 5947 rval = regs[rd]; 5948 pc = textlen; 5949 break; 5950 case DIF_OP_NOP: 5951 break; 5952 case DIF_OP_SETX: 5953 regs[rd] = inttab[DIF_INSTR_INTEGER(instr)]; 5954 break; 5955 case DIF_OP_SETS: 5956 regs[rd] = (uint64_t)(uintptr_t) 5957 (strtab + DIF_INSTR_STRING(instr)); 5958 break; 5959 case DIF_OP_SCMP: { 5960 size_t sz = state->dts_options[DTRACEOPT_STRSIZE]; 5961 uintptr_t s1 = regs[r1]; 5962 uintptr_t s2 = regs[r2]; 5963 size_t lim1, lim2; 5964 5965 if (s1 != NULL && 5966 !dtrace_strcanload(s1, sz, &lim1, mstate, vstate)) 5967 break; 5968 if (s2 != NULL && 5969 !dtrace_strcanload(s2, sz, &lim2, mstate, vstate)) 5970 break; 5971 5972 cc_r = dtrace_strncmp((char *)s1, (char *)s2, 5973 MIN(lim1, lim2)); 5974 5975 cc_n = cc_r < 0; 5976 cc_z = cc_r == 0; 5977 cc_v = cc_c = 0; 5978 break; 5979 } 5980 case DIF_OP_LDGA: 5981 regs[rd] = dtrace_dif_variable(mstate, state, 5982 r1, regs[r2]); 5983 break; 5984 case DIF_OP_LDGS: 5985 id = DIF_INSTR_VAR(instr); 5986 5987 if (id >= DIF_VAR_OTHER_UBASE) { 5988 uintptr_t a; 5989 5990 id -= DIF_VAR_OTHER_UBASE; 5991 svar = vstate->dtvs_globals[id]; 5992 ASSERT(svar != NULL); 5993 v = &svar->dtsv_var; 5994 5995 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) { 5996 regs[rd] = svar->dtsv_data; 5997 break; 5998 } 5999 6000 a = (uintptr_t)svar->dtsv_data; 6001 6002 if (*(uint8_t *)a == UINT8_MAX) { 6003 /* 6004 * If the 0th byte is set to UINT8_MAX 6005 * then this is to be treated as a 6006 * reference to a NULL variable. 6007 */ 6008 regs[rd] = NULL; 6009 } else { 6010 regs[rd] = a + sizeof (uint64_t); 6011 } 6012 6013 break; 6014 } 6015 6016 regs[rd] = dtrace_dif_variable(mstate, state, id, 0); 6017 break; 6018 6019 case DIF_OP_STGS: 6020 id = DIF_INSTR_VAR(instr); 6021 6022 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6023 id -= DIF_VAR_OTHER_UBASE; 6024 6025 VERIFY(id < vstate->dtvs_nglobals); 6026 svar = vstate->dtvs_globals[id]; 6027 ASSERT(svar != NULL); 6028 v = &svar->dtsv_var; 6029 6030 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6031 uintptr_t a = (uintptr_t)svar->dtsv_data; 6032 size_t lim; 6033 6034 ASSERT(a != NULL); 6035 ASSERT(svar->dtsv_size != 0); 6036 6037 if (regs[rd] == NULL) { 6038 *(uint8_t *)a = UINT8_MAX; 6039 break; 6040 } else { 6041 *(uint8_t *)a = 0; 6042 a += sizeof (uint64_t); 6043 } 6044 if (!dtrace_vcanload( 6045 (void *)(uintptr_t)regs[rd], &v->dtdv_type, 6046 &lim, mstate, vstate)) 6047 break; 6048 6049 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6050 (void *)a, &v->dtdv_type, lim); 6051 break; 6052 } 6053 6054 svar->dtsv_data = regs[rd]; 6055 break; 6056 6057 case DIF_OP_LDTA: 6058 /* 6059 * There are no DTrace built-in thread-local arrays at 6060 * present. This opcode is saved for future work. 6061 */ 6062 *flags |= CPU_DTRACE_ILLOP; 6063 regs[rd] = 0; 6064 break; 6065 6066 case DIF_OP_LDLS: 6067 id = DIF_INSTR_VAR(instr); 6068 6069 if (id < DIF_VAR_OTHER_UBASE) { 6070 /* 6071 * For now, this has no meaning. 6072 */ 6073 regs[rd] = 0; 6074 break; 6075 } 6076 6077 id -= DIF_VAR_OTHER_UBASE; 6078 6079 ASSERT(id < vstate->dtvs_nlocals); 6080 ASSERT(vstate->dtvs_locals != NULL); 6081 6082 svar = vstate->dtvs_locals[id]; 6083 ASSERT(svar != NULL); 6084 v = &svar->dtsv_var; 6085 6086 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6087 uintptr_t a = (uintptr_t)svar->dtsv_data; 6088 size_t sz = v->dtdv_type.dtdt_size; 6089 6090 sz += sizeof (uint64_t); 6091 ASSERT(svar->dtsv_size == NCPU * sz); 6092 a += CPU->cpu_id * sz; 6093 6094 if (*(uint8_t *)a == UINT8_MAX) { 6095 /* 6096 * If the 0th byte is set to UINT8_MAX 6097 * then this is to be treated as a 6098 * reference to a NULL variable. 6099 */ 6100 regs[rd] = NULL; 6101 } else { 6102 regs[rd] = a + sizeof (uint64_t); 6103 } 6104 6105 break; 6106 } 6107 6108 ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t)); 6109 tmp = (uint64_t *)(uintptr_t)svar->dtsv_data; 6110 regs[rd] = tmp[CPU->cpu_id]; 6111 break; 6112 6113 case DIF_OP_STLS: 6114 id = DIF_INSTR_VAR(instr); 6115 6116 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6117 id -= DIF_VAR_OTHER_UBASE; 6118 VERIFY(id < vstate->dtvs_nlocals); 6119 6120 ASSERT(vstate->dtvs_locals != NULL); 6121 svar = vstate->dtvs_locals[id]; 6122 ASSERT(svar != NULL); 6123 v = &svar->dtsv_var; 6124 6125 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6126 uintptr_t a = (uintptr_t)svar->dtsv_data; 6127 size_t sz = v->dtdv_type.dtdt_size; 6128 size_t lim; 6129 6130 sz += sizeof (uint64_t); 6131 ASSERT(svar->dtsv_size == NCPU * sz); 6132 a += CPU->cpu_id * sz; 6133 6134 if (regs[rd] == NULL) { 6135 *(uint8_t *)a = UINT8_MAX; 6136 break; 6137 } else { 6138 *(uint8_t *)a = 0; 6139 a += sizeof (uint64_t); 6140 } 6141 6142 if (!dtrace_vcanload( 6143 (void *)(uintptr_t)regs[rd], &v->dtdv_type, 6144 &lim, mstate, vstate)) 6145 break; 6146 6147 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6148 (void *)a, &v->dtdv_type, lim); 6149 break; 6150 } 6151 6152 ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t)); 6153 tmp = (uint64_t *)(uintptr_t)svar->dtsv_data; 6154 tmp[CPU->cpu_id] = regs[rd]; 6155 break; 6156 6157 case DIF_OP_LDTS: { 6158 dtrace_dynvar_t *dvar; 6159 dtrace_key_t *key; 6160 6161 id = DIF_INSTR_VAR(instr); 6162 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6163 id -= DIF_VAR_OTHER_UBASE; 6164 v = &vstate->dtvs_tlocals[id]; 6165 6166 key = &tupregs[DIF_DTR_NREGS]; 6167 key[0].dttk_value = (uint64_t)id; 6168 key[0].dttk_size = 0; 6169 DTRACE_TLS_THRKEY(key[1].dttk_value); 6170 key[1].dttk_size = 0; 6171 6172 dvar = dtrace_dynvar(dstate, 2, key, 6173 sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC, 6174 mstate, vstate); 6175 6176 if (dvar == NULL) { 6177 regs[rd] = 0; 6178 break; 6179 } 6180 6181 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6182 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data; 6183 } else { 6184 regs[rd] = *((uint64_t *)dvar->dtdv_data); 6185 } 6186 6187 break; 6188 } 6189 6190 case DIF_OP_STTS: { 6191 dtrace_dynvar_t *dvar; 6192 dtrace_key_t *key; 6193 6194 id = DIF_INSTR_VAR(instr); 6195 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6196 id -= DIF_VAR_OTHER_UBASE; 6197 VERIFY(id < vstate->dtvs_ntlocals); 6198 6199 key = &tupregs[DIF_DTR_NREGS]; 6200 key[0].dttk_value = (uint64_t)id; 6201 key[0].dttk_size = 0; 6202 DTRACE_TLS_THRKEY(key[1].dttk_value); 6203 key[1].dttk_size = 0; 6204 v = &vstate->dtvs_tlocals[id]; 6205 6206 dvar = dtrace_dynvar(dstate, 2, key, 6207 v->dtdv_type.dtdt_size > sizeof (uint64_t) ? 6208 v->dtdv_type.dtdt_size : sizeof (uint64_t), 6209 regs[rd] ? DTRACE_DYNVAR_ALLOC : 6210 DTRACE_DYNVAR_DEALLOC, mstate, vstate); 6211 6212 /* 6213 * Given that we're storing to thread-local data, 6214 * we need to flush our predicate cache. 6215 */ 6216 curthread->t_predcache = NULL; 6217 6218 if (dvar == NULL) 6219 break; 6220 6221 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6222 size_t lim; 6223 6224 if (!dtrace_vcanload( 6225 (void *)(uintptr_t)regs[rd], 6226 &v->dtdv_type, &lim, mstate, vstate)) 6227 break; 6228 6229 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6230 dvar->dtdv_data, &v->dtdv_type, lim); 6231 } else { 6232 *((uint64_t *)dvar->dtdv_data) = regs[rd]; 6233 } 6234 6235 break; 6236 } 6237 6238 case DIF_OP_SRA: 6239 regs[rd] = (int64_t)regs[r1] >> regs[r2]; 6240 break; 6241 6242 case DIF_OP_CALL: 6243 dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd, 6244 regs, tupregs, ttop, mstate, state); 6245 break; 6246 6247 case DIF_OP_PUSHTR: 6248 if (ttop == DIF_DTR_NREGS) { 6249 *flags |= CPU_DTRACE_TUPOFLOW; 6250 break; 6251 } 6252 6253 if (r1 == DIF_TYPE_STRING) { 6254 /* 6255 * If this is a string type and the size is 0, 6256 * we'll use the system-wide default string 6257 * size. Note that we are _not_ looking at 6258 * the value of the DTRACEOPT_STRSIZE option; 6259 * had this been set, we would expect to have 6260 * a non-zero size value in the "pushtr". 6261 */ 6262 tupregs[ttop].dttk_size = 6263 dtrace_strlen((char *)(uintptr_t)regs[rd], 6264 regs[r2] ? regs[r2] : 6265 dtrace_strsize_default) + 1; 6266 } else { 6267 if (regs[r2] > LONG_MAX) { 6268 *flags |= CPU_DTRACE_ILLOP; 6269 break; 6270 } 6271 6272 tupregs[ttop].dttk_size = regs[r2]; 6273 } 6274 6275 tupregs[ttop++].dttk_value = regs[rd]; 6276 break; 6277 6278 case DIF_OP_PUSHTV: 6279 if (ttop == DIF_DTR_NREGS) { 6280 *flags |= CPU_DTRACE_TUPOFLOW; 6281 break; 6282 } 6283 6284 tupregs[ttop].dttk_value = regs[rd]; 6285 tupregs[ttop++].dttk_size = 0; 6286 break; 6287 6288 case DIF_OP_POPTS: 6289 if (ttop != 0) 6290 ttop--; 6291 break; 6292 6293 case DIF_OP_FLUSHTS: 6294 ttop = 0; 6295 break; 6296 6297 case DIF_OP_LDGAA: 6298 case DIF_OP_LDTAA: { 6299 dtrace_dynvar_t *dvar; 6300 dtrace_key_t *key = tupregs; 6301 uint_t nkeys = ttop; 6302 6303 id = DIF_INSTR_VAR(instr); 6304 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6305 id -= DIF_VAR_OTHER_UBASE; 6306 6307 key[nkeys].dttk_value = (uint64_t)id; 6308 key[nkeys++].dttk_size = 0; 6309 6310 if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) { 6311 DTRACE_TLS_THRKEY(key[nkeys].dttk_value); 6312 key[nkeys++].dttk_size = 0; 6313 VERIFY(id < vstate->dtvs_ntlocals); 6314 v = &vstate->dtvs_tlocals[id]; 6315 } else { 6316 VERIFY(id < vstate->dtvs_nglobals); 6317 v = &vstate->dtvs_globals[id]->dtsv_var; 6318 } 6319 6320 dvar = dtrace_dynvar(dstate, nkeys, key, 6321 v->dtdv_type.dtdt_size > sizeof (uint64_t) ? 6322 v->dtdv_type.dtdt_size : sizeof (uint64_t), 6323 DTRACE_DYNVAR_NOALLOC, mstate, vstate); 6324 6325 if (dvar == NULL) { 6326 regs[rd] = 0; 6327 break; 6328 } 6329 6330 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6331 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data; 6332 } else { 6333 regs[rd] = *((uint64_t *)dvar->dtdv_data); 6334 } 6335 6336 break; 6337 } 6338 6339 case DIF_OP_STGAA: 6340 case DIF_OP_STTAA: { 6341 dtrace_dynvar_t *dvar; 6342 dtrace_key_t *key = tupregs; 6343 uint_t nkeys = ttop; 6344 6345 id = DIF_INSTR_VAR(instr); 6346 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6347 id -= DIF_VAR_OTHER_UBASE; 6348 6349 key[nkeys].dttk_value = (uint64_t)id; 6350 key[nkeys++].dttk_size = 0; 6351 6352 if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) { 6353 DTRACE_TLS_THRKEY(key[nkeys].dttk_value); 6354 key[nkeys++].dttk_size = 0; 6355 VERIFY(id < vstate->dtvs_ntlocals); 6356 v = &vstate->dtvs_tlocals[id]; 6357 } else { 6358 VERIFY(id < vstate->dtvs_nglobals); 6359 v = &vstate->dtvs_globals[id]->dtsv_var; 6360 } 6361 6362 dvar = dtrace_dynvar(dstate, nkeys, key, 6363 v->dtdv_type.dtdt_size > sizeof (uint64_t) ? 6364 v->dtdv_type.dtdt_size : sizeof (uint64_t), 6365 regs[rd] ? DTRACE_DYNVAR_ALLOC : 6366 DTRACE_DYNVAR_DEALLOC, mstate, vstate); 6367 6368 if (dvar == NULL) 6369 break; 6370 6371 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6372 size_t lim; 6373 6374 if (!dtrace_vcanload( 6375 (void *)(uintptr_t)regs[rd], &v->dtdv_type, 6376 &lim, mstate, vstate)) 6377 break; 6378 6379 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6380 dvar->dtdv_data, &v->dtdv_type, lim); 6381 } else { 6382 *((uint64_t *)dvar->dtdv_data) = regs[rd]; 6383 } 6384 6385 break; 6386 } 6387 6388 case DIF_OP_ALLOCS: { 6389 uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8); 6390 size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1]; 6391 6392 /* 6393 * Rounding up the user allocation size could have 6394 * overflowed large, bogus allocations (like -1ULL) to 6395 * 0. 6396 */ 6397 if (size < regs[r1] || 6398 !DTRACE_INSCRATCH(mstate, size)) { 6399 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 6400 regs[rd] = NULL; 6401 break; 6402 } 6403 6404 dtrace_bzero((void *) mstate->dtms_scratch_ptr, size); 6405 mstate->dtms_scratch_ptr += size; 6406 regs[rd] = ptr; 6407 break; 6408 } 6409 6410 case DIF_OP_COPYS: 6411 if (!dtrace_canstore(regs[rd], regs[r2], 6412 mstate, vstate)) { 6413 *flags |= CPU_DTRACE_BADADDR; 6414 *illval = regs[rd]; 6415 break; 6416 } 6417 6418 if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate)) 6419 break; 6420 6421 dtrace_bcopy((void *)(uintptr_t)regs[r1], 6422 (void *)(uintptr_t)regs[rd], (size_t)regs[r2]); 6423 break; 6424 6425 case DIF_OP_STB: 6426 if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) { 6427 *flags |= CPU_DTRACE_BADADDR; 6428 *illval = regs[rd]; 6429 break; 6430 } 6431 *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1]; 6432 break; 6433 6434 case DIF_OP_STH: 6435 if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) { 6436 *flags |= CPU_DTRACE_BADADDR; 6437 *illval = regs[rd]; 6438 break; 6439 } 6440 if (regs[rd] & 1) { 6441 *flags |= CPU_DTRACE_BADALIGN; 6442 *illval = regs[rd]; 6443 break; 6444 } 6445 *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1]; 6446 break; 6447 6448 case DIF_OP_STW: 6449 if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) { 6450 *flags |= CPU_DTRACE_BADADDR; 6451 *illval = regs[rd]; 6452 break; 6453 } 6454 if (regs[rd] & 3) { 6455 *flags |= CPU_DTRACE_BADALIGN; 6456 *illval = regs[rd]; 6457 break; 6458 } 6459 *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1]; 6460 break; 6461 6462 case DIF_OP_STX: 6463 if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) { 6464 *flags |= CPU_DTRACE_BADADDR; 6465 *illval = regs[rd]; 6466 break; 6467 } 6468 if (regs[rd] & 7) { 6469 *flags |= CPU_DTRACE_BADALIGN; 6470 *illval = regs[rd]; 6471 break; 6472 } 6473 *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1]; 6474 break; 6475 } 6476 } 6477 6478 if (!(*flags & CPU_DTRACE_FAULT)) 6479 return (rval); 6480 6481 mstate->dtms_fltoffs = opc * sizeof (dif_instr_t); 6482 mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS; 6483 6484 return (0); 6485 } 6486 6487 static void 6488 dtrace_action_breakpoint(dtrace_ecb_t *ecb) 6489 { 6490 dtrace_probe_t *probe = ecb->dte_probe; 6491 dtrace_provider_t *prov = probe->dtpr_provider; 6492 char c[DTRACE_FULLNAMELEN + 80], *str; 6493 char *msg = "dtrace: breakpoint action at probe "; 6494 char *ecbmsg = " (ecb "; 6495 uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4)); 6496 uintptr_t val = (uintptr_t)ecb; 6497 int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0; 6498 6499 if (dtrace_destructive_disallow) 6500 return; 6501 6502 /* 6503 * It's impossible to be taking action on the NULL probe. 6504 */ 6505 ASSERT(probe != NULL); 6506 6507 /* 6508 * This is a poor man's (destitute man's?) sprintf(): we want to 6509 * print the provider name, module name, function name and name of 6510 * the probe, along with the hex address of the ECB with the breakpoint 6511 * action -- all of which we must place in the character buffer by 6512 * hand. 6513 */ 6514 while (*msg != '\0') 6515 c[i++] = *msg++; 6516 6517 for (str = prov->dtpv_name; *str != '\0'; str++) 6518 c[i++] = *str; 6519 c[i++] = ':'; 6520 6521 for (str = probe->dtpr_mod; *str != '\0'; str++) 6522 c[i++] = *str; 6523 c[i++] = ':'; 6524 6525 for (str = probe->dtpr_func; *str != '\0'; str++) 6526 c[i++] = *str; 6527 c[i++] = ':'; 6528 6529 for (str = probe->dtpr_name; *str != '\0'; str++) 6530 c[i++] = *str; 6531 6532 while (*ecbmsg != '\0') 6533 c[i++] = *ecbmsg++; 6534 6535 while (shift >= 0) { 6536 mask = (uintptr_t)0xf << shift; 6537 6538 if (val >= ((uintptr_t)1 << shift)) 6539 c[i++] = "0123456789abcdef"[(val & mask) >> shift]; 6540 shift -= 4; 6541 } 6542 6543 c[i++] = ')'; 6544 c[i] = '\0'; 6545 6546 debug_enter(c); 6547 } 6548 6549 static void 6550 dtrace_action_panic(dtrace_ecb_t *ecb) 6551 { 6552 dtrace_probe_t *probe = ecb->dte_probe; 6553 6554 /* 6555 * It's impossible to be taking action on the NULL probe. 6556 */ 6557 ASSERT(probe != NULL); 6558 6559 if (dtrace_destructive_disallow) 6560 return; 6561 6562 if (dtrace_panicked != NULL) 6563 return; 6564 6565 if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL) 6566 return; 6567 6568 /* 6569 * We won the right to panic. (We want to be sure that only one 6570 * thread calls panic() from dtrace_probe(), and that panic() is 6571 * called exactly once.) 6572 */ 6573 dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)", 6574 probe->dtpr_provider->dtpv_name, probe->dtpr_mod, 6575 probe->dtpr_func, probe->dtpr_name, (void *)ecb); 6576 } 6577 6578 static void 6579 dtrace_action_raise(uint64_t sig) 6580 { 6581 if (dtrace_destructive_disallow) 6582 return; 6583 6584 if (sig >= NSIG) { 6585 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 6586 return; 6587 } 6588 6589 /* 6590 * raise() has a queue depth of 1 -- we ignore all subsequent 6591 * invocations of the raise() action. 6592 */ 6593 if (curthread->t_dtrace_sig == 0) 6594 curthread->t_dtrace_sig = (uint8_t)sig; 6595 6596 curthread->t_sig_check = 1; 6597 aston(curthread); 6598 } 6599 6600 static void 6601 dtrace_action_stop(void) 6602 { 6603 if (dtrace_destructive_disallow) 6604 return; 6605 6606 if (!curthread->t_dtrace_stop) { 6607 curthread->t_dtrace_stop = 1; 6608 curthread->t_sig_check = 1; 6609 aston(curthread); 6610 } 6611 } 6612 6613 static void 6614 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val) 6615 { 6616 hrtime_t now; 6617 volatile uint16_t *flags; 6618 cpu_t *cpu = CPU; 6619 6620 if (dtrace_destructive_disallow) 6621 return; 6622 6623 flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags; 6624 6625 now = dtrace_gethrtime(); 6626 6627 if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) { 6628 /* 6629 * We need to advance the mark to the current time. 6630 */ 6631 cpu->cpu_dtrace_chillmark = now; 6632 cpu->cpu_dtrace_chilled = 0; 6633 } 6634 6635 /* 6636 * Now check to see if the requested chill time would take us over 6637 * the maximum amount of time allowed in the chill interval. (Or 6638 * worse, if the calculation itself induces overflow.) 6639 */ 6640 if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max || 6641 cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) { 6642 *flags |= CPU_DTRACE_ILLOP; 6643 return; 6644 } 6645 6646 while (dtrace_gethrtime() - now < val) 6647 continue; 6648 6649 /* 6650 * Normally, we assure that the value of the variable "timestamp" does 6651 * not change within an ECB. The presence of chill() represents an 6652 * exception to this rule, however. 6653 */ 6654 mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP; 6655 cpu->cpu_dtrace_chilled += val; 6656 } 6657 6658 static void 6659 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state, 6660 uint64_t *buf, uint64_t arg) 6661 { 6662 int nframes = DTRACE_USTACK_NFRAMES(arg); 6663 int strsize = DTRACE_USTACK_STRSIZE(arg); 6664 uint64_t *pcs = &buf[1], *fps; 6665 char *str = (char *)&pcs[nframes]; 6666 int size, offs = 0, i, j; 6667 size_t rem; 6668 uintptr_t old = mstate->dtms_scratch_ptr, saved; 6669 uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 6670 char *sym; 6671 6672 /* 6673 * Should be taking a faster path if string space has not been 6674 * allocated. 6675 */ 6676 ASSERT(strsize != 0); 6677 6678 /* 6679 * We will first allocate some temporary space for the frame pointers. 6680 */ 6681 fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8); 6682 size = (uintptr_t)fps - mstate->dtms_scratch_ptr + 6683 (nframes * sizeof (uint64_t)); 6684 6685 if (!DTRACE_INSCRATCH(mstate, size)) { 6686 /* 6687 * Not enough room for our frame pointers -- need to indicate 6688 * that we ran out of scratch space. 6689 */ 6690 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 6691 return; 6692 } 6693 6694 mstate->dtms_scratch_ptr += size; 6695 saved = mstate->dtms_scratch_ptr; 6696 6697 /* 6698 * Now get a stack with both program counters and frame pointers. 6699 */ 6700 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6701 dtrace_getufpstack(buf, fps, nframes + 1); 6702 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6703 6704 /* 6705 * If that faulted, we're cooked. 6706 */ 6707 if (*flags & CPU_DTRACE_FAULT) 6708 goto out; 6709 6710 /* 6711 * Now we want to walk up the stack, calling the USTACK helper. For 6712 * each iteration, we restore the scratch pointer. 6713 */ 6714 for (i = 0; i < nframes; i++) { 6715 mstate->dtms_scratch_ptr = saved; 6716 6717 if (offs >= strsize) 6718 break; 6719 6720 sym = (char *)(uintptr_t)dtrace_helper( 6721 DTRACE_HELPER_ACTION_USTACK, 6722 mstate, state, pcs[i], fps[i]); 6723 6724 /* 6725 * If we faulted while running the helper, we're going to 6726 * clear the fault and null out the corresponding string. 6727 */ 6728 if (*flags & CPU_DTRACE_FAULT) { 6729 *flags &= ~CPU_DTRACE_FAULT; 6730 str[offs++] = '\0'; 6731 continue; 6732 } 6733 6734 if (sym == NULL) { 6735 str[offs++] = '\0'; 6736 continue; 6737 } 6738 6739 if (!dtrace_strcanload((uintptr_t)sym, strsize, &rem, mstate, 6740 &(state->dts_vstate))) { 6741 str[offs++] = '\0'; 6742 continue; 6743 } 6744 6745 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6746 6747 /* 6748 * Now copy in the string that the helper returned to us. 6749 */ 6750 for (j = 0; offs + j < strsize && j < rem; j++) { 6751 if ((str[offs + j] = sym[j]) == '\0') 6752 break; 6753 } 6754 6755 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6756 6757 offs += j + 1; 6758 } 6759 6760 if (offs >= strsize) { 6761 /* 6762 * If we didn't have room for all of the strings, we don't 6763 * abort processing -- this needn't be a fatal error -- but we 6764 * still want to increment a counter (dts_stkstroverflows) to 6765 * allow this condition to be warned about. (If this is from 6766 * a jstack() action, it is easily tuned via jstackstrsize.) 6767 */ 6768 dtrace_error(&state->dts_stkstroverflows); 6769 } 6770 6771 while (offs < strsize) 6772 str[offs++] = '\0'; 6773 6774 out: 6775 mstate->dtms_scratch_ptr = old; 6776 } 6777 6778 static void 6779 dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size, 6780 size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind) 6781 { 6782 volatile uint16_t *flags; 6783 uint64_t val = *valp; 6784 size_t valoffs = *valoffsp; 6785 6786 flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 6787 ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF); 6788 6789 /* 6790 * If this is a string, we're going to only load until we find the zero 6791 * byte -- after which we'll store zero bytes. 6792 */ 6793 if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) { 6794 char c = '\0' + 1; 6795 size_t s; 6796 6797 for (s = 0; s < size; s++) { 6798 if (c != '\0' && dtkind == DIF_TF_BYREF) { 6799 c = dtrace_load8(val++); 6800 } else if (c != '\0' && dtkind == DIF_TF_BYUREF) { 6801 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6802 c = dtrace_fuword8((void *)(uintptr_t)val++); 6803 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6804 if (*flags & CPU_DTRACE_FAULT) 6805 break; 6806 } 6807 6808 DTRACE_STORE(uint8_t, tomax, valoffs++, c); 6809 6810 if (c == '\0' && intuple) 6811 break; 6812 } 6813 } else { 6814 uint8_t c; 6815 while (valoffs < end) { 6816 if (dtkind == DIF_TF_BYREF) { 6817 c = dtrace_load8(val++); 6818 } else if (dtkind == DIF_TF_BYUREF) { 6819 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6820 c = dtrace_fuword8((void *)(uintptr_t)val++); 6821 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6822 if (*flags & CPU_DTRACE_FAULT) 6823 break; 6824 } 6825 6826 DTRACE_STORE(uint8_t, tomax, 6827 valoffs++, c); 6828 } 6829 } 6830 6831 *valp = val; 6832 *valoffsp = valoffs; 6833 } 6834 6835 /* 6836 * If you're looking for the epicenter of DTrace, you just found it. This 6837 * is the function called by the provider to fire a probe -- from which all 6838 * subsequent probe-context DTrace activity emanates. 6839 */ 6840 void 6841 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, 6842 uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) 6843 { 6844 processorid_t cpuid; 6845 dtrace_icookie_t cookie; 6846 dtrace_probe_t *probe; 6847 dtrace_mstate_t mstate; 6848 dtrace_ecb_t *ecb; 6849 dtrace_action_t *act; 6850 intptr_t offs; 6851 size_t size; 6852 int vtime, onintr; 6853 volatile uint16_t *flags; 6854 hrtime_t now, end; 6855 6856 /* 6857 * Kick out immediately if this CPU is still being born (in which case 6858 * curthread will be set to -1) or the current thread can't allow 6859 * probes in its current context. 6860 */ 6861 if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE)) 6862 return; 6863 6864 cookie = dtrace_interrupt_disable(); 6865 probe = dtrace_probes[id - 1]; 6866 cpuid = CPU->cpu_id; 6867 onintr = CPU_ON_INTR(CPU); 6868 6869 CPU->cpu_dtrace_probes++; 6870 6871 if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE && 6872 probe->dtpr_predcache == curthread->t_predcache) { 6873 /* 6874 * We have hit in the predicate cache; we know that 6875 * this predicate would evaluate to be false. 6876 */ 6877 dtrace_interrupt_enable(cookie); 6878 return; 6879 } 6880 6881 if (panic_quiesce) { 6882 /* 6883 * We don't trace anything if we're panicking. 6884 */ 6885 dtrace_interrupt_enable(cookie); 6886 return; 6887 } 6888 6889 now = mstate.dtms_timestamp = dtrace_gethrtime(); 6890 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP; 6891 vtime = dtrace_vtime_references != 0; 6892 6893 if (vtime && curthread->t_dtrace_start) 6894 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start; 6895 6896 mstate.dtms_difo = NULL; 6897 mstate.dtms_probe = probe; 6898 mstate.dtms_strtok = NULL; 6899 mstate.dtms_arg[0] = arg0; 6900 mstate.dtms_arg[1] = arg1; 6901 mstate.dtms_arg[2] = arg2; 6902 mstate.dtms_arg[3] = arg3; 6903 mstate.dtms_arg[4] = arg4; 6904 6905 flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags; 6906 6907 for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { 6908 dtrace_predicate_t *pred = ecb->dte_predicate; 6909 dtrace_state_t *state = ecb->dte_state; 6910 dtrace_buffer_t *buf = &state->dts_buffer[cpuid]; 6911 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid]; 6912 dtrace_vstate_t *vstate = &state->dts_vstate; 6913 dtrace_provider_t *prov = probe->dtpr_provider; 6914 uint64_t tracememsize = 0; 6915 int committed = 0; 6916 caddr_t tomax; 6917 6918 /* 6919 * A little subtlety with the following (seemingly innocuous) 6920 * declaration of the automatic 'val': by looking at the 6921 * code, you might think that it could be declared in the 6922 * action processing loop, below. (That is, it's only used in 6923 * the action processing loop.) However, it must be declared 6924 * out of that scope because in the case of DIF expression 6925 * arguments to aggregating actions, one iteration of the 6926 * action loop will use the last iteration's value. 6927 */ 6928 #ifdef lint 6929 uint64_t val = 0; 6930 #else 6931 uint64_t val; 6932 #endif 6933 6934 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE; 6935 mstate.dtms_access = DTRACE_ACCESS_ARGS | DTRACE_ACCESS_PROC; 6936 mstate.dtms_getf = NULL; 6937 6938 *flags &= ~CPU_DTRACE_ERROR; 6939 6940 if (prov == dtrace_provider) { 6941 /* 6942 * If dtrace itself is the provider of this probe, 6943 * we're only going to continue processing the ECB if 6944 * arg0 (the dtrace_state_t) is equal to the ECB's 6945 * creating state. (This prevents disjoint consumers 6946 * from seeing one another's metaprobes.) 6947 */ 6948 if (arg0 != (uint64_t)(uintptr_t)state) 6949 continue; 6950 } 6951 6952 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) { 6953 /* 6954 * We're not currently active. If our provider isn't 6955 * the dtrace pseudo provider, we're not interested. 6956 */ 6957 if (prov != dtrace_provider) 6958 continue; 6959 6960 /* 6961 * Now we must further check if we are in the BEGIN 6962 * probe. If we are, we will only continue processing 6963 * if we're still in WARMUP -- if one BEGIN enabling 6964 * has invoked the exit() action, we don't want to 6965 * evaluate subsequent BEGIN enablings. 6966 */ 6967 if (probe->dtpr_id == dtrace_probeid_begin && 6968 state->dts_activity != DTRACE_ACTIVITY_WARMUP) { 6969 ASSERT(state->dts_activity == 6970 DTRACE_ACTIVITY_DRAINING); 6971 continue; 6972 } 6973 } 6974 6975 if (ecb->dte_cond && !dtrace_priv_probe(state, &mstate, ecb)) 6976 continue; 6977 6978 if (now - state->dts_alive > dtrace_deadman_timeout) { 6979 /* 6980 * We seem to be dead. Unless we (a) have kernel 6981 * destructive permissions (b) have explicitly enabled 6982 * destructive actions and (c) destructive actions have 6983 * not been disabled, we're going to transition into 6984 * the KILLED state, from which no further processing 6985 * on this state will be performed. 6986 */ 6987 if (!dtrace_priv_kernel_destructive(state) || 6988 !state->dts_cred.dcr_destructive || 6989 dtrace_destructive_disallow) { 6990 void *activity = &state->dts_activity; 6991 dtrace_activity_t current; 6992 6993 do { 6994 current = state->dts_activity; 6995 } while (dtrace_cas32(activity, current, 6996 DTRACE_ACTIVITY_KILLED) != current); 6997 6998 continue; 6999 } 7000 } 7001 7002 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed, 7003 ecb->dte_alignment, state, &mstate)) < 0) 7004 continue; 7005 7006 tomax = buf->dtb_tomax; 7007 ASSERT(tomax != NULL); 7008 7009 if (ecb->dte_size != 0) { 7010 dtrace_rechdr_t dtrh; 7011 if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) { 7012 mstate.dtms_timestamp = dtrace_gethrtime(); 7013 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP; 7014 } 7015 ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t)); 7016 dtrh.dtrh_epid = ecb->dte_epid; 7017 DTRACE_RECORD_STORE_TIMESTAMP(&dtrh, 7018 mstate.dtms_timestamp); 7019 *((dtrace_rechdr_t *)(tomax + offs)) = dtrh; 7020 } 7021 7022 mstate.dtms_epid = ecb->dte_epid; 7023 mstate.dtms_present |= DTRACE_MSTATE_EPID; 7024 7025 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) 7026 mstate.dtms_access |= DTRACE_ACCESS_KERNEL; 7027 7028 if (pred != NULL) { 7029 dtrace_difo_t *dp = pred->dtp_difo; 7030 int rval; 7031 7032 rval = dtrace_dif_emulate(dp, &mstate, vstate, state); 7033 7034 if (!(*flags & CPU_DTRACE_ERROR) && !rval) { 7035 dtrace_cacheid_t cid = probe->dtpr_predcache; 7036 7037 if (cid != DTRACE_CACHEIDNONE && !onintr) { 7038 /* 7039 * Update the predicate cache... 7040 */ 7041 ASSERT(cid == pred->dtp_cacheid); 7042 curthread->t_predcache = cid; 7043 } 7044 7045 continue; 7046 } 7047 } 7048 7049 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) && 7050 act != NULL; act = act->dta_next) { 7051 size_t valoffs; 7052 dtrace_difo_t *dp; 7053 dtrace_recdesc_t *rec = &act->dta_rec; 7054 7055 size = rec->dtrd_size; 7056 valoffs = offs + rec->dtrd_offset; 7057 7058 if (DTRACEACT_ISAGG(act->dta_kind)) { 7059 uint64_t v = 0xbad; 7060 dtrace_aggregation_t *agg; 7061 7062 agg = (dtrace_aggregation_t *)act; 7063 7064 if ((dp = act->dta_difo) != NULL) 7065 v = dtrace_dif_emulate(dp, 7066 &mstate, vstate, state); 7067 7068 if (*flags & CPU_DTRACE_ERROR) 7069 continue; 7070 7071 /* 7072 * Note that we always pass the expression 7073 * value from the previous iteration of the 7074 * action loop. This value will only be used 7075 * if there is an expression argument to the 7076 * aggregating action, denoted by the 7077 * dtag_hasarg field. 7078 */ 7079 dtrace_aggregate(agg, buf, 7080 offs, aggbuf, v, val); 7081 continue; 7082 } 7083 7084 switch (act->dta_kind) { 7085 case DTRACEACT_STOP: 7086 if (dtrace_priv_proc_destructive(state, 7087 &mstate)) 7088 dtrace_action_stop(); 7089 continue; 7090 7091 case DTRACEACT_BREAKPOINT: 7092 if (dtrace_priv_kernel_destructive(state)) 7093 dtrace_action_breakpoint(ecb); 7094 continue; 7095 7096 case DTRACEACT_PANIC: 7097 if (dtrace_priv_kernel_destructive(state)) 7098 dtrace_action_panic(ecb); 7099 continue; 7100 7101 case DTRACEACT_STACK: 7102 if (!dtrace_priv_kernel(state)) 7103 continue; 7104 7105 dtrace_getpcstack((pc_t *)(tomax + valoffs), 7106 size / sizeof (pc_t), probe->dtpr_aframes, 7107 DTRACE_ANCHORED(probe) ? NULL : 7108 (uint32_t *)arg0); 7109 7110 continue; 7111 7112 case DTRACEACT_JSTACK: 7113 case DTRACEACT_USTACK: 7114 if (!dtrace_priv_proc(state, &mstate)) 7115 continue; 7116 7117 /* 7118 * See comment in DIF_VAR_PID. 7119 */ 7120 if (DTRACE_ANCHORED(mstate.dtms_probe) && 7121 CPU_ON_INTR(CPU)) { 7122 int depth = DTRACE_USTACK_NFRAMES( 7123 rec->dtrd_arg) + 1; 7124 7125 dtrace_bzero((void *)(tomax + valoffs), 7126 DTRACE_USTACK_STRSIZE(rec->dtrd_arg) 7127 + depth * sizeof (uint64_t)); 7128 7129 continue; 7130 } 7131 7132 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 && 7133 curproc->p_dtrace_helpers != NULL) { 7134 /* 7135 * This is the slow path -- we have 7136 * allocated string space, and we're 7137 * getting the stack of a process that 7138 * has helpers. Call into a separate 7139 * routine to perform this processing. 7140 */ 7141 dtrace_action_ustack(&mstate, state, 7142 (uint64_t *)(tomax + valoffs), 7143 rec->dtrd_arg); 7144 continue; 7145 } 7146 7147 /* 7148 * Clear the string space, since there's no 7149 * helper to do it for us. 7150 */ 7151 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0) { 7152 int depth = DTRACE_USTACK_NFRAMES( 7153 rec->dtrd_arg); 7154 size_t strsize = DTRACE_USTACK_STRSIZE( 7155 rec->dtrd_arg); 7156 uint64_t *buf = (uint64_t *)(tomax + 7157 valoffs); 7158 void *strspace = &buf[depth + 1]; 7159 7160 dtrace_bzero(strspace, 7161 MIN(depth, strsize)); 7162 } 7163 7164 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 7165 dtrace_getupcstack((uint64_t *) 7166 (tomax + valoffs), 7167 DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1); 7168 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 7169 continue; 7170 7171 default: 7172 break; 7173 } 7174 7175 dp = act->dta_difo; 7176 ASSERT(dp != NULL); 7177 7178 val = dtrace_dif_emulate(dp, &mstate, vstate, state); 7179 7180 if (*flags & CPU_DTRACE_ERROR) 7181 continue; 7182 7183 switch (act->dta_kind) { 7184 case DTRACEACT_SPECULATE: { 7185 dtrace_rechdr_t *dtrh; 7186 7187 ASSERT(buf == &state->dts_buffer[cpuid]); 7188 buf = dtrace_speculation_buffer(state, 7189 cpuid, val); 7190 7191 if (buf == NULL) { 7192 *flags |= CPU_DTRACE_DROP; 7193 continue; 7194 } 7195 7196 offs = dtrace_buffer_reserve(buf, 7197 ecb->dte_needed, ecb->dte_alignment, 7198 state, NULL); 7199 7200 if (offs < 0) { 7201 *flags |= CPU_DTRACE_DROP; 7202 continue; 7203 } 7204 7205 tomax = buf->dtb_tomax; 7206 ASSERT(tomax != NULL); 7207 7208 if (ecb->dte_size == 0) 7209 continue; 7210 7211 ASSERT3U(ecb->dte_size, >=, 7212 sizeof (dtrace_rechdr_t)); 7213 dtrh = ((void *)(tomax + offs)); 7214 dtrh->dtrh_epid = ecb->dte_epid; 7215 /* 7216 * When the speculation is committed, all of 7217 * the records in the speculative buffer will 7218 * have their timestamps set to the commit 7219 * time. Until then, it is set to a sentinel 7220 * value, for debugability. 7221 */ 7222 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX); 7223 continue; 7224 } 7225 7226 case DTRACEACT_CHILL: 7227 if (dtrace_priv_kernel_destructive(state)) 7228 dtrace_action_chill(&mstate, val); 7229 continue; 7230 7231 case DTRACEACT_RAISE: 7232 if (dtrace_priv_proc_destructive(state, 7233 &mstate)) 7234 dtrace_action_raise(val); 7235 continue; 7236 7237 case DTRACEACT_COMMIT: 7238 ASSERT(!committed); 7239 7240 /* 7241 * We need to commit our buffer state. 7242 */ 7243 if (ecb->dte_size) 7244 buf->dtb_offset = offs + ecb->dte_size; 7245 buf = &state->dts_buffer[cpuid]; 7246 dtrace_speculation_commit(state, cpuid, val); 7247 committed = 1; 7248 continue; 7249 7250 case DTRACEACT_DISCARD: 7251 dtrace_speculation_discard(state, cpuid, val); 7252 continue; 7253 7254 case DTRACEACT_DIFEXPR: 7255 case DTRACEACT_LIBACT: 7256 case DTRACEACT_PRINTF: 7257 case DTRACEACT_PRINTA: 7258 case DTRACEACT_SYSTEM: 7259 case DTRACEACT_FREOPEN: 7260 case DTRACEACT_TRACEMEM: 7261 break; 7262 7263 case DTRACEACT_TRACEMEM_DYNSIZE: 7264 tracememsize = val; 7265 break; 7266 7267 case DTRACEACT_SYM: 7268 case DTRACEACT_MOD: 7269 if (!dtrace_priv_kernel(state)) 7270 continue; 7271 break; 7272 7273 case DTRACEACT_USYM: 7274 case DTRACEACT_UMOD: 7275 case DTRACEACT_UADDR: { 7276 struct pid *pid = curthread->t_procp->p_pidp; 7277 7278 if (!dtrace_priv_proc(state, &mstate)) 7279 continue; 7280 7281 DTRACE_STORE(uint64_t, tomax, 7282 valoffs, (uint64_t)pid->pid_id); 7283 DTRACE_STORE(uint64_t, tomax, 7284 valoffs + sizeof (uint64_t), val); 7285 7286 continue; 7287 } 7288 7289 case DTRACEACT_EXIT: { 7290 /* 7291 * For the exit action, we are going to attempt 7292 * to atomically set our activity to be 7293 * draining. If this fails (either because 7294 * another CPU has beat us to the exit action, 7295 * or because our current activity is something 7296 * other than ACTIVE or WARMUP), we will 7297 * continue. This assures that the exit action 7298 * can be successfully recorded at most once 7299 * when we're in the ACTIVE state. If we're 7300 * encountering the exit() action while in 7301 * COOLDOWN, however, we want to honor the new 7302 * status code. (We know that we're the only 7303 * thread in COOLDOWN, so there is no race.) 7304 */ 7305 void *activity = &state->dts_activity; 7306 dtrace_activity_t current = state->dts_activity; 7307 7308 if (current == DTRACE_ACTIVITY_COOLDOWN) 7309 break; 7310 7311 if (current != DTRACE_ACTIVITY_WARMUP) 7312 current = DTRACE_ACTIVITY_ACTIVE; 7313 7314 if (dtrace_cas32(activity, current, 7315 DTRACE_ACTIVITY_DRAINING) != current) { 7316 *flags |= CPU_DTRACE_DROP; 7317 continue; 7318 } 7319 7320 break; 7321 } 7322 7323 default: 7324 ASSERT(0); 7325 } 7326 7327 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF || 7328 dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) { 7329 uintptr_t end = valoffs + size; 7330 7331 if (tracememsize != 0 && 7332 valoffs + tracememsize < end) { 7333 end = valoffs + tracememsize; 7334 tracememsize = 0; 7335 } 7336 7337 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF && 7338 !dtrace_vcanload((void *)(uintptr_t)val, 7339 &dp->dtdo_rtype, NULL, &mstate, vstate)) 7340 continue; 7341 7342 dtrace_store_by_ref(dp, tomax, size, &valoffs, 7343 &val, end, act->dta_intuple, 7344 dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ? 7345 DIF_TF_BYREF: DIF_TF_BYUREF); 7346 continue; 7347 } 7348 7349 switch (size) { 7350 case 0: 7351 break; 7352 7353 case sizeof (uint8_t): 7354 DTRACE_STORE(uint8_t, tomax, valoffs, val); 7355 break; 7356 case sizeof (uint16_t): 7357 DTRACE_STORE(uint16_t, tomax, valoffs, val); 7358 break; 7359 case sizeof (uint32_t): 7360 DTRACE_STORE(uint32_t, tomax, valoffs, val); 7361 break; 7362 case sizeof (uint64_t): 7363 DTRACE_STORE(uint64_t, tomax, valoffs, val); 7364 break; 7365 default: 7366 /* 7367 * Any other size should have been returned by 7368 * reference, not by value. 7369 */ 7370 ASSERT(0); 7371 break; 7372 } 7373 } 7374 7375 if (*flags & CPU_DTRACE_DROP) 7376 continue; 7377 7378 if (*flags & CPU_DTRACE_FAULT) { 7379 int ndx; 7380 dtrace_action_t *err; 7381 7382 buf->dtb_errors++; 7383 7384 if (probe->dtpr_id == dtrace_probeid_error) { 7385 /* 7386 * There's nothing we can do -- we had an 7387 * error on the error probe. We bump an 7388 * error counter to at least indicate that 7389 * this condition happened. 7390 */ 7391 dtrace_error(&state->dts_dblerrors); 7392 continue; 7393 } 7394 7395 if (vtime) { 7396 /* 7397 * Before recursing on dtrace_probe(), we 7398 * need to explicitly clear out our start 7399 * time to prevent it from being accumulated 7400 * into t_dtrace_vtime. 7401 */ 7402 curthread->t_dtrace_start = 0; 7403 } 7404 7405 /* 7406 * Iterate over the actions to figure out which action 7407 * we were processing when we experienced the error. 7408 * Note that act points _past_ the faulting action; if 7409 * act is ecb->dte_action, the fault was in the 7410 * predicate, if it's ecb->dte_action->dta_next it's 7411 * in action #1, and so on. 7412 */ 7413 for (err = ecb->dte_action, ndx = 0; 7414 err != act; err = err->dta_next, ndx++) 7415 continue; 7416 7417 dtrace_probe_error(state, ecb->dte_epid, ndx, 7418 (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ? 7419 mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags), 7420 cpu_core[cpuid].cpuc_dtrace_illval); 7421 7422 continue; 7423 } 7424 7425 if (!committed) 7426 buf->dtb_offset = offs + ecb->dte_size; 7427 } 7428 7429 end = dtrace_gethrtime(); 7430 if (vtime) 7431 curthread->t_dtrace_start = end; 7432 7433 CPU->cpu_dtrace_nsec += end - now; 7434 7435 dtrace_interrupt_enable(cookie); 7436 } 7437 7438 /* 7439 * DTrace Probe Hashing Functions 7440 * 7441 * The functions in this section (and indeed, the functions in remaining 7442 * sections) are not _called_ from probe context. (Any exceptions to this are 7443 * marked with a "Note:".) Rather, they are called from elsewhere in the 7444 * DTrace framework to look-up probes in, add probes to and remove probes from 7445 * the DTrace probe hashes. (Each probe is hashed by each element of the 7446 * probe tuple -- allowing for fast lookups, regardless of what was 7447 * specified.) 7448 */ 7449 static uint_t 7450 dtrace_hash_str(char *p) 7451 { 7452 unsigned int g; 7453 uint_t hval = 0; 7454 7455 while (*p) { 7456 hval = (hval << 4) + *p++; 7457 if ((g = (hval & 0xf0000000)) != 0) 7458 hval ^= g >> 24; 7459 hval &= ~g; 7460 } 7461 return (hval); 7462 } 7463 7464 static dtrace_hash_t * 7465 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs) 7466 { 7467 dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP); 7468 7469 hash->dth_stroffs = stroffs; 7470 hash->dth_nextoffs = nextoffs; 7471 hash->dth_prevoffs = prevoffs; 7472 7473 hash->dth_size = 1; 7474 hash->dth_mask = hash->dth_size - 1; 7475 7476 hash->dth_tab = kmem_zalloc(hash->dth_size * 7477 sizeof (dtrace_hashbucket_t *), KM_SLEEP); 7478 7479 return (hash); 7480 } 7481 7482 static void 7483 dtrace_hash_destroy(dtrace_hash_t *hash) 7484 { 7485 #ifdef DEBUG 7486 int i; 7487 7488 for (i = 0; i < hash->dth_size; i++) 7489 ASSERT(hash->dth_tab[i] == NULL); 7490 #endif 7491 7492 kmem_free(hash->dth_tab, 7493 hash->dth_size * sizeof (dtrace_hashbucket_t *)); 7494 kmem_free(hash, sizeof (dtrace_hash_t)); 7495 } 7496 7497 static void 7498 dtrace_hash_resize(dtrace_hash_t *hash) 7499 { 7500 int size = hash->dth_size, i, ndx; 7501 int new_size = hash->dth_size << 1; 7502 int new_mask = new_size - 1; 7503 dtrace_hashbucket_t **new_tab, *bucket, *next; 7504 7505 ASSERT((new_size & new_mask) == 0); 7506 7507 new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP); 7508 7509 for (i = 0; i < size; i++) { 7510 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) { 7511 dtrace_probe_t *probe = bucket->dthb_chain; 7512 7513 ASSERT(probe != NULL); 7514 ndx = DTRACE_HASHSTR(hash, probe) & new_mask; 7515 7516 next = bucket->dthb_next; 7517 bucket->dthb_next = new_tab[ndx]; 7518 new_tab[ndx] = bucket; 7519 } 7520 } 7521 7522 kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *)); 7523 hash->dth_tab = new_tab; 7524 hash->dth_size = new_size; 7525 hash->dth_mask = new_mask; 7526 } 7527 7528 static void 7529 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new) 7530 { 7531 int hashval = DTRACE_HASHSTR(hash, new); 7532 int ndx = hashval & hash->dth_mask; 7533 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7534 dtrace_probe_t **nextp, **prevp; 7535 7536 for (; bucket != NULL; bucket = bucket->dthb_next) { 7537 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new)) 7538 goto add; 7539 } 7540 7541 if ((hash->dth_nbuckets >> 1) > hash->dth_size) { 7542 dtrace_hash_resize(hash); 7543 dtrace_hash_add(hash, new); 7544 return; 7545 } 7546 7547 bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP); 7548 bucket->dthb_next = hash->dth_tab[ndx]; 7549 hash->dth_tab[ndx] = bucket; 7550 hash->dth_nbuckets++; 7551 7552 add: 7553 nextp = DTRACE_HASHNEXT(hash, new); 7554 ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL); 7555 *nextp = bucket->dthb_chain; 7556 7557 if (bucket->dthb_chain != NULL) { 7558 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain); 7559 ASSERT(*prevp == NULL); 7560 *prevp = new; 7561 } 7562 7563 bucket->dthb_chain = new; 7564 bucket->dthb_len++; 7565 } 7566 7567 static dtrace_probe_t * 7568 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template) 7569 { 7570 int hashval = DTRACE_HASHSTR(hash, template); 7571 int ndx = hashval & hash->dth_mask; 7572 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7573 7574 for (; bucket != NULL; bucket = bucket->dthb_next) { 7575 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template)) 7576 return (bucket->dthb_chain); 7577 } 7578 7579 return (NULL); 7580 } 7581 7582 static int 7583 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template) 7584 { 7585 int hashval = DTRACE_HASHSTR(hash, template); 7586 int ndx = hashval & hash->dth_mask; 7587 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7588 7589 for (; bucket != NULL; bucket = bucket->dthb_next) { 7590 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template)) 7591 return (bucket->dthb_len); 7592 } 7593 7594 return (NULL); 7595 } 7596 7597 static void 7598 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe) 7599 { 7600 int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask; 7601 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7602 7603 dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe); 7604 dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe); 7605 7606 /* 7607 * Find the bucket that we're removing this probe from. 7608 */ 7609 for (; bucket != NULL; bucket = bucket->dthb_next) { 7610 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe)) 7611 break; 7612 } 7613 7614 ASSERT(bucket != NULL); 7615 7616 if (*prevp == NULL) { 7617 if (*nextp == NULL) { 7618 /* 7619 * The removed probe was the only probe on this 7620 * bucket; we need to remove the bucket. 7621 */ 7622 dtrace_hashbucket_t *b = hash->dth_tab[ndx]; 7623 7624 ASSERT(bucket->dthb_chain == probe); 7625 ASSERT(b != NULL); 7626 7627 if (b == bucket) { 7628 hash->dth_tab[ndx] = bucket->dthb_next; 7629 } else { 7630 while (b->dthb_next != bucket) 7631 b = b->dthb_next; 7632 b->dthb_next = bucket->dthb_next; 7633 } 7634 7635 ASSERT(hash->dth_nbuckets > 0); 7636 hash->dth_nbuckets--; 7637 kmem_free(bucket, sizeof (dtrace_hashbucket_t)); 7638 return; 7639 } 7640 7641 bucket->dthb_chain = *nextp; 7642 } else { 7643 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp; 7644 } 7645 7646 if (*nextp != NULL) 7647 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp; 7648 } 7649 7650 /* 7651 * DTrace Utility Functions 7652 * 7653 * These are random utility functions that are _not_ called from probe context. 7654 */ 7655 static int 7656 dtrace_badattr(const dtrace_attribute_t *a) 7657 { 7658 return (a->dtat_name > DTRACE_STABILITY_MAX || 7659 a->dtat_data > DTRACE_STABILITY_MAX || 7660 a->dtat_class > DTRACE_CLASS_MAX); 7661 } 7662 7663 /* 7664 * Return a duplicate copy of a string. If the specified string is NULL, 7665 * this function returns a zero-length string. 7666 */ 7667 static char * 7668 dtrace_strdup(const char *str) 7669 { 7670 char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP); 7671 7672 if (str != NULL) 7673 (void) strcpy(new, str); 7674 7675 return (new); 7676 } 7677 7678 #define DTRACE_ISALPHA(c) \ 7679 (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) 7680 7681 static int 7682 dtrace_badname(const char *s) 7683 { 7684 char c; 7685 7686 if (s == NULL || (c = *s++) == '\0') 7687 return (0); 7688 7689 if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.') 7690 return (1); 7691 7692 while ((c = *s++) != '\0') { 7693 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') && 7694 c != '-' && c != '_' && c != '.' && c != '`') 7695 return (1); 7696 } 7697 7698 return (0); 7699 } 7700 7701 static void 7702 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp) 7703 { 7704 uint32_t priv; 7705 7706 if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { 7707 /* 7708 * For DTRACE_PRIV_ALL, the uid and zoneid don't matter. 7709 */ 7710 priv = DTRACE_PRIV_ALL; 7711 } else { 7712 *uidp = crgetuid(cr); 7713 *zoneidp = crgetzoneid(cr); 7714 7715 priv = 0; 7716 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) 7717 priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER; 7718 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) 7719 priv |= DTRACE_PRIV_USER; 7720 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) 7721 priv |= DTRACE_PRIV_PROC; 7722 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) 7723 priv |= DTRACE_PRIV_OWNER; 7724 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) 7725 priv |= DTRACE_PRIV_ZONEOWNER; 7726 } 7727 7728 *privp = priv; 7729 } 7730 7731 #ifdef DTRACE_ERRDEBUG 7732 static void 7733 dtrace_errdebug(const char *str) 7734 { 7735 int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ; 7736 int occupied = 0; 7737 7738 mutex_enter(&dtrace_errlock); 7739 dtrace_errlast = str; 7740 dtrace_errthread = curthread; 7741 7742 while (occupied++ < DTRACE_ERRHASHSZ) { 7743 if (dtrace_errhash[hval].dter_msg == str) { 7744 dtrace_errhash[hval].dter_count++; 7745 goto out; 7746 } 7747 7748 if (dtrace_errhash[hval].dter_msg != NULL) { 7749 hval = (hval + 1) % DTRACE_ERRHASHSZ; 7750 continue; 7751 } 7752 7753 dtrace_errhash[hval].dter_msg = str; 7754 dtrace_errhash[hval].dter_count = 1; 7755 goto out; 7756 } 7757 7758 panic("dtrace: undersized error hash"); 7759 out: 7760 mutex_exit(&dtrace_errlock); 7761 } 7762 #endif 7763 7764 /* 7765 * DTrace Matching Functions 7766 * 7767 * These functions are used to match groups of probes, given some elements of 7768 * a probe tuple, or some globbed expressions for elements of a probe tuple. 7769 */ 7770 static int 7771 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid, 7772 zoneid_t zoneid) 7773 { 7774 if (priv != DTRACE_PRIV_ALL) { 7775 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags; 7776 uint32_t match = priv & ppriv; 7777 7778 /* 7779 * No PRIV_DTRACE_* privileges... 7780 */ 7781 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER | 7782 DTRACE_PRIV_KERNEL)) == 0) 7783 return (0); 7784 7785 /* 7786 * No matching bits, but there were bits to match... 7787 */ 7788 if (match == 0 && ppriv != 0) 7789 return (0); 7790 7791 /* 7792 * Need to have permissions to the process, but don't... 7793 */ 7794 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 && 7795 uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) { 7796 return (0); 7797 } 7798 7799 /* 7800 * Need to be in the same zone unless we possess the 7801 * privilege to examine all zones. 7802 */ 7803 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 && 7804 zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) { 7805 return (0); 7806 } 7807 } 7808 7809 return (1); 7810 } 7811 7812 /* 7813 * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which 7814 * consists of input pattern strings and an ops-vector to evaluate them. 7815 * This function returns >0 for match, 0 for no match, and <0 for error. 7816 */ 7817 static int 7818 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp, 7819 uint32_t priv, uid_t uid, zoneid_t zoneid) 7820 { 7821 dtrace_provider_t *pvp = prp->dtpr_provider; 7822 int rv; 7823 7824 if (pvp->dtpv_defunct) 7825 return (0); 7826 7827 if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0) 7828 return (rv); 7829 7830 if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0) 7831 return (rv); 7832 7833 if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0) 7834 return (rv); 7835 7836 if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0) 7837 return (rv); 7838 7839 if (dtrace_match_priv(prp, priv, uid, zoneid) == 0) 7840 return (0); 7841 7842 return (rv); 7843 } 7844 7845 /* 7846 * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN) 7847 * interface for matching a glob pattern 'p' to an input string 's'. Unlike 7848 * libc's version, the kernel version only applies to 8-bit ASCII strings. 7849 * In addition, all of the recursion cases except for '*' matching have been 7850 * unwound. For '*', we still implement recursive evaluation, but a depth 7851 * counter is maintained and matching is aborted if we recurse too deep. 7852 * The function returns 0 if no match, >0 if match, and <0 if recursion error. 7853 */ 7854 static int 7855 dtrace_match_glob(const char *s, const char *p, int depth) 7856 { 7857 const char *olds; 7858 char s1, c; 7859 int gs; 7860 7861 if (depth > DTRACE_PROBEKEY_MAXDEPTH) 7862 return (-1); 7863 7864 if (s == NULL) 7865 s = ""; /* treat NULL as empty string */ 7866 7867 top: 7868 olds = s; 7869 s1 = *s++; 7870 7871 if (p == NULL) 7872 return (0); 7873 7874 if ((c = *p++) == '\0') 7875 return (s1 == '\0'); 7876 7877 switch (c) { 7878 case '[': { 7879 int ok = 0, notflag = 0; 7880 char lc = '\0'; 7881 7882 if (s1 == '\0') 7883 return (0); 7884 7885 if (*p == '!') { 7886 notflag = 1; 7887 p++; 7888 } 7889 7890 if ((c = *p++) == '\0') 7891 return (0); 7892 7893 do { 7894 if (c == '-' && lc != '\0' && *p != ']') { 7895 if ((c = *p++) == '\0') 7896 return (0); 7897 if (c == '\\' && (c = *p++) == '\0') 7898 return (0); 7899 7900 if (notflag) { 7901 if (s1 < lc || s1 > c) 7902 ok++; 7903 else 7904 return (0); 7905 } else if (lc <= s1 && s1 <= c) 7906 ok++; 7907 7908 } else if (c == '\\' && (c = *p++) == '\0') 7909 return (0); 7910 7911 lc = c; /* save left-hand 'c' for next iteration */ 7912 7913 if (notflag) { 7914 if (s1 != c) 7915 ok++; 7916 else 7917 return (0); 7918 } else if (s1 == c) 7919 ok++; 7920 7921 if ((c = *p++) == '\0') 7922 return (0); 7923 7924 } while (c != ']'); 7925 7926 if (ok) 7927 goto top; 7928 7929 return (0); 7930 } 7931 7932 case '\\': 7933 if ((c = *p++) == '\0') 7934 return (0); 7935 /*FALLTHRU*/ 7936 7937 default: 7938 if (c != s1) 7939 return (0); 7940 /*FALLTHRU*/ 7941 7942 case '?': 7943 if (s1 != '\0') 7944 goto top; 7945 return (0); 7946 7947 case '*': 7948 while (*p == '*') 7949 p++; /* consecutive *'s are identical to a single one */ 7950 7951 if (*p == '\0') 7952 return (1); 7953 7954 for (s = olds; *s != '\0'; s++) { 7955 if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0) 7956 return (gs); 7957 } 7958 7959 return (0); 7960 } 7961 } 7962 7963 /*ARGSUSED*/ 7964 static int 7965 dtrace_match_string(const char *s, const char *p, int depth) 7966 { 7967 return (s != NULL && strcmp(s, p) == 0); 7968 } 7969 7970 /*ARGSUSED*/ 7971 static int 7972 dtrace_match_nul(const char *s, const char *p, int depth) 7973 { 7974 return (1); /* always match the empty pattern */ 7975 } 7976 7977 /*ARGSUSED*/ 7978 static int 7979 dtrace_match_nonzero(const char *s, const char *p, int depth) 7980 { 7981 return (s != NULL && s[0] != '\0'); 7982 } 7983 7984 static int 7985 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid, 7986 zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg) 7987 { 7988 dtrace_probe_t template, *probe; 7989 dtrace_hash_t *hash = NULL; 7990 int len, rc, best = INT_MAX, nmatched = 0; 7991 dtrace_id_t i; 7992 7993 ASSERT(MUTEX_HELD(&dtrace_lock)); 7994 7995 /* 7996 * If the probe ID is specified in the key, just lookup by ID and 7997 * invoke the match callback once if a matching probe is found. 7998 */ 7999 if (pkp->dtpk_id != DTRACE_IDNONE) { 8000 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL && 8001 dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) { 8002 if ((*matched)(probe, arg) == DTRACE_MATCH_FAIL) 8003 return (DTRACE_MATCH_FAIL); 8004 nmatched++; 8005 } 8006 return (nmatched); 8007 } 8008 8009 template.dtpr_mod = (char *)pkp->dtpk_mod; 8010 template.dtpr_func = (char *)pkp->dtpk_func; 8011 template.dtpr_name = (char *)pkp->dtpk_name; 8012 8013 /* 8014 * We want to find the most distinct of the module name, function 8015 * name, and name. So for each one that is not a glob pattern or 8016 * empty string, we perform a lookup in the corresponding hash and 8017 * use the hash table with the fewest collisions to do our search. 8018 */ 8019 if (pkp->dtpk_mmatch == &dtrace_match_string && 8020 (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) { 8021 best = len; 8022 hash = dtrace_bymod; 8023 } 8024 8025 if (pkp->dtpk_fmatch == &dtrace_match_string && 8026 (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) { 8027 best = len; 8028 hash = dtrace_byfunc; 8029 } 8030 8031 if (pkp->dtpk_nmatch == &dtrace_match_string && 8032 (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) { 8033 best = len; 8034 hash = dtrace_byname; 8035 } 8036 8037 /* 8038 * If we did not select a hash table, iterate over every probe and 8039 * invoke our callback for each one that matches our input probe key. 8040 */ 8041 if (hash == NULL) { 8042 for (i = 0; i < dtrace_nprobes; i++) { 8043 if ((probe = dtrace_probes[i]) == NULL || 8044 dtrace_match_probe(probe, pkp, priv, uid, 8045 zoneid) <= 0) 8046 continue; 8047 8048 nmatched++; 8049 8050 if ((rc = (*matched)(probe, arg)) != 8051 DTRACE_MATCH_NEXT) { 8052 if (rc == DTRACE_MATCH_FAIL) 8053 return (DTRACE_MATCH_FAIL); 8054 break; 8055 } 8056 } 8057 8058 return (nmatched); 8059 } 8060 8061 /* 8062 * If we selected a hash table, iterate over each probe of the same key 8063 * name and invoke the callback for every probe that matches the other 8064 * attributes of our input probe key. 8065 */ 8066 for (probe = dtrace_hash_lookup(hash, &template); probe != NULL; 8067 probe = *(DTRACE_HASHNEXT(hash, probe))) { 8068 8069 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0) 8070 continue; 8071 8072 nmatched++; 8073 8074 if ((rc = (*matched)(probe, arg)) != DTRACE_MATCH_NEXT) { 8075 if (rc == DTRACE_MATCH_FAIL) 8076 return (DTRACE_MATCH_FAIL); 8077 break; 8078 } 8079 } 8080 8081 return (nmatched); 8082 } 8083 8084 /* 8085 * Return the function pointer dtrace_probecmp() should use to compare the 8086 * specified pattern with a string. For NULL or empty patterns, we select 8087 * dtrace_match_nul(). For glob pattern strings, we use dtrace_match_glob(). 8088 * For non-empty non-glob strings, we use dtrace_match_string(). 8089 */ 8090 static dtrace_probekey_f * 8091 dtrace_probekey_func(const char *p) 8092 { 8093 char c; 8094 8095 if (p == NULL || *p == '\0') 8096 return (&dtrace_match_nul); 8097 8098 while ((c = *p++) != '\0') { 8099 if (c == '[' || c == '?' || c == '*' || c == '\\') 8100 return (&dtrace_match_glob); 8101 } 8102 8103 return (&dtrace_match_string); 8104 } 8105 8106 /* 8107 * Build a probe comparison key for use with dtrace_match_probe() from the 8108 * given probe description. By convention, a null key only matches anchored 8109 * probes: if each field is the empty string, reset dtpk_fmatch to 8110 * dtrace_match_nonzero(). 8111 */ 8112 static void 8113 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp) 8114 { 8115 pkp->dtpk_prov = pdp->dtpd_provider; 8116 pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider); 8117 8118 pkp->dtpk_mod = pdp->dtpd_mod; 8119 pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod); 8120 8121 pkp->dtpk_func = pdp->dtpd_func; 8122 pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func); 8123 8124 pkp->dtpk_name = pdp->dtpd_name; 8125 pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name); 8126 8127 pkp->dtpk_id = pdp->dtpd_id; 8128 8129 if (pkp->dtpk_id == DTRACE_IDNONE && 8130 pkp->dtpk_pmatch == &dtrace_match_nul && 8131 pkp->dtpk_mmatch == &dtrace_match_nul && 8132 pkp->dtpk_fmatch == &dtrace_match_nul && 8133 pkp->dtpk_nmatch == &dtrace_match_nul) 8134 pkp->dtpk_fmatch = &dtrace_match_nonzero; 8135 } 8136 8137 /* 8138 * DTrace Provider-to-Framework API Functions 8139 * 8140 * These functions implement much of the Provider-to-Framework API, as 8141 * described in <sys/dtrace.h>. The parts of the API not in this section are 8142 * the functions in the API for probe management (found below), and 8143 * dtrace_probe() itself (found above). 8144 */ 8145 8146 /* 8147 * Register the calling provider with the DTrace framework. This should 8148 * generally be called by DTrace providers in their attach(9E) entry point. 8149 */ 8150 int 8151 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, 8152 cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp) 8153 { 8154 dtrace_provider_t *provider; 8155 8156 if (name == NULL || pap == NULL || pops == NULL || idp == NULL) { 8157 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8158 "arguments", name ? name : "<NULL>"); 8159 return (EINVAL); 8160 } 8161 8162 if (name[0] == '\0' || dtrace_badname(name)) { 8163 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8164 "provider name", name); 8165 return (EINVAL); 8166 } 8167 8168 if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) || 8169 pops->dtps_enable == NULL || pops->dtps_disable == NULL || 8170 pops->dtps_destroy == NULL || 8171 ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) { 8172 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8173 "provider ops", name); 8174 return (EINVAL); 8175 } 8176 8177 if (dtrace_badattr(&pap->dtpa_provider) || 8178 dtrace_badattr(&pap->dtpa_mod) || 8179 dtrace_badattr(&pap->dtpa_func) || 8180 dtrace_badattr(&pap->dtpa_name) || 8181 dtrace_badattr(&pap->dtpa_args)) { 8182 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8183 "provider attributes", name); 8184 return (EINVAL); 8185 } 8186 8187 if (priv & ~DTRACE_PRIV_ALL) { 8188 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8189 "privilege attributes", name); 8190 return (EINVAL); 8191 } 8192 8193 if ((priv & DTRACE_PRIV_KERNEL) && 8194 (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) && 8195 pops->dtps_mode == NULL) { 8196 cmn_err(CE_WARN, "failed to register provider '%s': need " 8197 "dtps_mode() op for given privilege attributes", name); 8198 return (EINVAL); 8199 } 8200 8201 provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP); 8202 provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP); 8203 (void) strcpy(provider->dtpv_name, name); 8204 8205 provider->dtpv_attr = *pap; 8206 provider->dtpv_priv.dtpp_flags = priv; 8207 if (cr != NULL) { 8208 provider->dtpv_priv.dtpp_uid = crgetuid(cr); 8209 provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr); 8210 } 8211 provider->dtpv_pops = *pops; 8212 8213 if (pops->dtps_provide == NULL) { 8214 ASSERT(pops->dtps_provide_module != NULL); 8215 provider->dtpv_pops.dtps_provide = 8216 (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop; 8217 } 8218 8219 if (pops->dtps_provide_module == NULL) { 8220 ASSERT(pops->dtps_provide != NULL); 8221 provider->dtpv_pops.dtps_provide_module = 8222 (void (*)(void *, struct modctl *))dtrace_nullop; 8223 } 8224 8225 if (pops->dtps_suspend == NULL) { 8226 ASSERT(pops->dtps_resume == NULL); 8227 provider->dtpv_pops.dtps_suspend = 8228 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; 8229 provider->dtpv_pops.dtps_resume = 8230 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; 8231 } 8232 8233 provider->dtpv_arg = arg; 8234 *idp = (dtrace_provider_id_t)provider; 8235 8236 if (pops == &dtrace_provider_ops) { 8237 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 8238 ASSERT(MUTEX_HELD(&dtrace_lock)); 8239 ASSERT(dtrace_anon.dta_enabling == NULL); 8240 8241 /* 8242 * We make sure that the DTrace provider is at the head of 8243 * the provider chain. 8244 */ 8245 provider->dtpv_next = dtrace_provider; 8246 dtrace_provider = provider; 8247 return (0); 8248 } 8249 8250 mutex_enter(&dtrace_provider_lock); 8251 mutex_enter(&dtrace_lock); 8252 8253 /* 8254 * If there is at least one provider registered, we'll add this 8255 * provider after the first provider. 8256 */ 8257 if (dtrace_provider != NULL) { 8258 provider->dtpv_next = dtrace_provider->dtpv_next; 8259 dtrace_provider->dtpv_next = provider; 8260 } else { 8261 dtrace_provider = provider; 8262 } 8263 8264 if (dtrace_retained != NULL) { 8265 dtrace_enabling_provide(provider); 8266 8267 /* 8268 * Now we need to call dtrace_enabling_matchall() -- which 8269 * will acquire cpu_lock and dtrace_lock. We therefore need 8270 * to drop all of our locks before calling into it... 8271 */ 8272 mutex_exit(&dtrace_lock); 8273 mutex_exit(&dtrace_provider_lock); 8274 dtrace_enabling_matchall(); 8275 8276 return (0); 8277 } 8278 8279 mutex_exit(&dtrace_lock); 8280 mutex_exit(&dtrace_provider_lock); 8281 8282 return (0); 8283 } 8284 8285 /* 8286 * Unregister the specified provider from the DTrace framework. This should 8287 * generally be called by DTrace providers in their detach(9E) entry point. 8288 */ 8289 int 8290 dtrace_unregister(dtrace_provider_id_t id) 8291 { 8292 dtrace_provider_t *old = (dtrace_provider_t *)id; 8293 dtrace_provider_t *prev = NULL; 8294 int i, self = 0, noreap = 0; 8295 dtrace_probe_t *probe, *first = NULL; 8296 8297 if (old->dtpv_pops.dtps_enable == 8298 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop) { 8299 /* 8300 * If DTrace itself is the provider, we're called with locks 8301 * already held. 8302 */ 8303 ASSERT(old == dtrace_provider); 8304 ASSERT(dtrace_devi != NULL); 8305 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 8306 ASSERT(MUTEX_HELD(&dtrace_lock)); 8307 self = 1; 8308 8309 if (dtrace_provider->dtpv_next != NULL) { 8310 /* 8311 * There's another provider here; return failure. 8312 */ 8313 return (EBUSY); 8314 } 8315 } else { 8316 mutex_enter(&dtrace_provider_lock); 8317 mutex_enter(&mod_lock); 8318 mutex_enter(&dtrace_lock); 8319 } 8320 8321 /* 8322 * If anyone has /dev/dtrace open, or if there are anonymous enabled 8323 * probes, we refuse to let providers slither away, unless this 8324 * provider has already been explicitly invalidated. 8325 */ 8326 if (!old->dtpv_defunct && 8327 (dtrace_opens || (dtrace_anon.dta_state != NULL && 8328 dtrace_anon.dta_state->dts_necbs > 0))) { 8329 if (!self) { 8330 mutex_exit(&dtrace_lock); 8331 mutex_exit(&mod_lock); 8332 mutex_exit(&dtrace_provider_lock); 8333 } 8334 return (EBUSY); 8335 } 8336 8337 /* 8338 * Attempt to destroy the probes associated with this provider. 8339 */ 8340 for (i = 0; i < dtrace_nprobes; i++) { 8341 if ((probe = dtrace_probes[i]) == NULL) 8342 continue; 8343 8344 if (probe->dtpr_provider != old) 8345 continue; 8346 8347 if (probe->dtpr_ecb == NULL) 8348 continue; 8349 8350 /* 8351 * If we are trying to unregister a defunct provider, and the 8352 * provider was made defunct within the interval dictated by 8353 * dtrace_unregister_defunct_reap, we'll (asynchronously) 8354 * attempt to reap our enablings. To denote that the provider 8355 * should reattempt to unregister itself at some point in the 8356 * future, we will return a differentiable error code (EAGAIN 8357 * instead of EBUSY) in this case. 8358 */ 8359 if (dtrace_gethrtime() - old->dtpv_defunct > 8360 dtrace_unregister_defunct_reap) 8361 noreap = 1; 8362 8363 if (!self) { 8364 mutex_exit(&dtrace_lock); 8365 mutex_exit(&mod_lock); 8366 mutex_exit(&dtrace_provider_lock); 8367 } 8368 8369 if (noreap) 8370 return (EBUSY); 8371 8372 (void) taskq_dispatch(dtrace_taskq, 8373 (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP); 8374 8375 return (EAGAIN); 8376 } 8377 8378 /* 8379 * All of the probes for this provider are disabled; we can safely 8380 * remove all of them from their hash chains and from the probe array. 8381 */ 8382 for (i = 0; i < dtrace_nprobes; i++) { 8383 if ((probe = dtrace_probes[i]) == NULL) 8384 continue; 8385 8386 if (probe->dtpr_provider != old) 8387 continue; 8388 8389 dtrace_probes[i] = NULL; 8390 8391 dtrace_hash_remove(dtrace_bymod, probe); 8392 dtrace_hash_remove(dtrace_byfunc, probe); 8393 dtrace_hash_remove(dtrace_byname, probe); 8394 8395 if (first == NULL) { 8396 first = probe; 8397 probe->dtpr_nextmod = NULL; 8398 } else { 8399 probe->dtpr_nextmod = first; 8400 first = probe; 8401 } 8402 } 8403 8404 /* 8405 * The provider's probes have been removed from the hash chains and 8406 * from the probe array. Now issue a dtrace_sync() to be sure that 8407 * everyone has cleared out from any probe array processing. 8408 */ 8409 dtrace_sync(); 8410 8411 for (probe = first; probe != NULL; probe = first) { 8412 first = probe->dtpr_nextmod; 8413 8414 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id, 8415 probe->dtpr_arg); 8416 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); 8417 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); 8418 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); 8419 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1); 8420 kmem_free(probe, sizeof (dtrace_probe_t)); 8421 } 8422 8423 if ((prev = dtrace_provider) == old) { 8424 ASSERT(self || dtrace_devi == NULL); 8425 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL); 8426 dtrace_provider = old->dtpv_next; 8427 } else { 8428 while (prev != NULL && prev->dtpv_next != old) 8429 prev = prev->dtpv_next; 8430 8431 if (prev == NULL) { 8432 panic("attempt to unregister non-existent " 8433 "dtrace provider %p\n", (void *)id); 8434 } 8435 8436 prev->dtpv_next = old->dtpv_next; 8437 } 8438 8439 if (!self) { 8440 mutex_exit(&dtrace_lock); 8441 mutex_exit(&mod_lock); 8442 mutex_exit(&dtrace_provider_lock); 8443 } 8444 8445 kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1); 8446 kmem_free(old, sizeof (dtrace_provider_t)); 8447 8448 return (0); 8449 } 8450 8451 /* 8452 * Invalidate the specified provider. All subsequent probe lookups for the 8453 * specified provider will fail, but its probes will not be removed. 8454 */ 8455 void 8456 dtrace_invalidate(dtrace_provider_id_t id) 8457 { 8458 dtrace_provider_t *pvp = (dtrace_provider_t *)id; 8459 8460 ASSERT(pvp->dtpv_pops.dtps_enable != 8461 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop); 8462 8463 mutex_enter(&dtrace_provider_lock); 8464 mutex_enter(&dtrace_lock); 8465 8466 pvp->dtpv_defunct = dtrace_gethrtime(); 8467 8468 mutex_exit(&dtrace_lock); 8469 mutex_exit(&dtrace_provider_lock); 8470 } 8471 8472 /* 8473 * Indicate whether or not DTrace has attached. 8474 */ 8475 int 8476 dtrace_attached(void) 8477 { 8478 /* 8479 * dtrace_provider will be non-NULL iff the DTrace driver has 8480 * attached. (It's non-NULL because DTrace is always itself a 8481 * provider.) 8482 */ 8483 return (dtrace_provider != NULL); 8484 } 8485 8486 /* 8487 * Remove all the unenabled probes for the given provider. This function is 8488 * not unlike dtrace_unregister(), except that it doesn't remove the provider 8489 * -- just as many of its associated probes as it can. 8490 */ 8491 int 8492 dtrace_condense(dtrace_provider_id_t id) 8493 { 8494 dtrace_provider_t *prov = (dtrace_provider_t *)id; 8495 int i; 8496 dtrace_probe_t *probe; 8497 8498 /* 8499 * Make sure this isn't the dtrace provider itself. 8500 */ 8501 ASSERT(prov->dtpv_pops.dtps_enable != 8502 (int (*)(void *, dtrace_id_t, void *))dtrace_enable_nullop); 8503 8504 mutex_enter(&dtrace_provider_lock); 8505 mutex_enter(&dtrace_lock); 8506 8507 /* 8508 * Attempt to destroy the probes associated with this provider. 8509 */ 8510 for (i = 0; i < dtrace_nprobes; i++) { 8511 if ((probe = dtrace_probes[i]) == NULL) 8512 continue; 8513 8514 if (probe->dtpr_provider != prov) 8515 continue; 8516 8517 if (probe->dtpr_ecb != NULL) 8518 continue; 8519 8520 dtrace_probes[i] = NULL; 8521 8522 dtrace_hash_remove(dtrace_bymod, probe); 8523 dtrace_hash_remove(dtrace_byfunc, probe); 8524 dtrace_hash_remove(dtrace_byname, probe); 8525 8526 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1, 8527 probe->dtpr_arg); 8528 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); 8529 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); 8530 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); 8531 kmem_free(probe, sizeof (dtrace_probe_t)); 8532 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1); 8533 } 8534 8535 mutex_exit(&dtrace_lock); 8536 mutex_exit(&dtrace_provider_lock); 8537 8538 return (0); 8539 } 8540 8541 /* 8542 * DTrace Probe Management Functions 8543 * 8544 * The functions in this section perform the DTrace probe management, 8545 * including functions to create probes, look-up probes, and call into the 8546 * providers to request that probes be provided. Some of these functions are 8547 * in the Provider-to-Framework API; these functions can be identified by the 8548 * fact that they are not declared "static". 8549 */ 8550 8551 /* 8552 * Create a probe with the specified module name, function name, and name. 8553 */ 8554 dtrace_id_t 8555 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, 8556 const char *func, const char *name, int aframes, void *arg) 8557 { 8558 dtrace_probe_t *probe, **probes; 8559 dtrace_provider_t *provider = (dtrace_provider_t *)prov; 8560 dtrace_id_t id; 8561 8562 if (provider == dtrace_provider) { 8563 ASSERT(MUTEX_HELD(&dtrace_lock)); 8564 } else { 8565 mutex_enter(&dtrace_lock); 8566 } 8567 8568 id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1, 8569 VM_BESTFIT | VM_SLEEP); 8570 probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP); 8571 8572 probe->dtpr_id = id; 8573 probe->dtpr_gen = dtrace_probegen++; 8574 probe->dtpr_mod = dtrace_strdup(mod); 8575 probe->dtpr_func = dtrace_strdup(func); 8576 probe->dtpr_name = dtrace_strdup(name); 8577 probe->dtpr_arg = arg; 8578 probe->dtpr_aframes = aframes; 8579 probe->dtpr_provider = provider; 8580 8581 dtrace_hash_add(dtrace_bymod, probe); 8582 dtrace_hash_add(dtrace_byfunc, probe); 8583 dtrace_hash_add(dtrace_byname, probe); 8584 8585 if (id - 1 >= dtrace_nprobes) { 8586 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *); 8587 size_t nsize = osize << 1; 8588 8589 if (nsize == 0) { 8590 ASSERT(osize == 0); 8591 ASSERT(dtrace_probes == NULL); 8592 nsize = sizeof (dtrace_probe_t *); 8593 } 8594 8595 probes = kmem_zalloc(nsize, KM_SLEEP); 8596 8597 if (dtrace_probes == NULL) { 8598 ASSERT(osize == 0); 8599 dtrace_probes = probes; 8600 dtrace_nprobes = 1; 8601 } else { 8602 dtrace_probe_t **oprobes = dtrace_probes; 8603 8604 bcopy(oprobes, probes, osize); 8605 dtrace_membar_producer(); 8606 dtrace_probes = probes; 8607 8608 dtrace_sync(); 8609 8610 /* 8611 * All CPUs are now seeing the new probes array; we can 8612 * safely free the old array. 8613 */ 8614 kmem_free(oprobes, osize); 8615 dtrace_nprobes <<= 1; 8616 } 8617 8618 ASSERT(id - 1 < dtrace_nprobes); 8619 } 8620 8621 ASSERT(dtrace_probes[id - 1] == NULL); 8622 dtrace_probes[id - 1] = probe; 8623 8624 if (provider != dtrace_provider) 8625 mutex_exit(&dtrace_lock); 8626 8627 return (id); 8628 } 8629 8630 static dtrace_probe_t * 8631 dtrace_probe_lookup_id(dtrace_id_t id) 8632 { 8633 ASSERT(MUTEX_HELD(&dtrace_lock)); 8634 8635 if (id == 0 || id > dtrace_nprobes) 8636 return (NULL); 8637 8638 return (dtrace_probes[id - 1]); 8639 } 8640 8641 static int 8642 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg) 8643 { 8644 *((dtrace_id_t *)arg) = probe->dtpr_id; 8645 8646 return (DTRACE_MATCH_DONE); 8647 } 8648 8649 /* 8650 * Look up a probe based on provider and one or more of module name, function 8651 * name and probe name. 8652 */ 8653 dtrace_id_t 8654 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod, 8655 const char *func, const char *name) 8656 { 8657 dtrace_probekey_t pkey; 8658 dtrace_id_t id; 8659 int match; 8660 8661 pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name; 8662 pkey.dtpk_pmatch = &dtrace_match_string; 8663 pkey.dtpk_mod = mod; 8664 pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul; 8665 pkey.dtpk_func = func; 8666 pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul; 8667 pkey.dtpk_name = name; 8668 pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul; 8669 pkey.dtpk_id = DTRACE_IDNONE; 8670 8671 mutex_enter(&dtrace_lock); 8672 match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0, 8673 dtrace_probe_lookup_match, &id); 8674 mutex_exit(&dtrace_lock); 8675 8676 ASSERT(match == 1 || match == 0); 8677 return (match ? id : 0); 8678 } 8679 8680 /* 8681 * Returns the probe argument associated with the specified probe. 8682 */ 8683 void * 8684 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid) 8685 { 8686 dtrace_probe_t *probe; 8687 void *rval = NULL; 8688 8689 mutex_enter(&dtrace_lock); 8690 8691 if ((probe = dtrace_probe_lookup_id(pid)) != NULL && 8692 probe->dtpr_provider == (dtrace_provider_t *)id) 8693 rval = probe->dtpr_arg; 8694 8695 mutex_exit(&dtrace_lock); 8696 8697 return (rval); 8698 } 8699 8700 /* 8701 * Copy a probe into a probe description. 8702 */ 8703 static void 8704 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp) 8705 { 8706 bzero(pdp, sizeof (dtrace_probedesc_t)); 8707 pdp->dtpd_id = prp->dtpr_id; 8708 8709 (void) strncpy(pdp->dtpd_provider, 8710 prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1); 8711 8712 (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1); 8713 (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1); 8714 (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1); 8715 } 8716 8717 /* 8718 * Called to indicate that a probe -- or probes -- should be provided by a 8719 * specfied provider. If the specified description is NULL, the provider will 8720 * be told to provide all of its probes. (This is done whenever a new 8721 * consumer comes along, or whenever a retained enabling is to be matched.) If 8722 * the specified description is non-NULL, the provider is given the 8723 * opportunity to dynamically provide the specified probe, allowing providers 8724 * to support the creation of probes on-the-fly. (So-called _autocreated_ 8725 * probes.) If the provider is NULL, the operations will be applied to all 8726 * providers; if the provider is non-NULL the operations will only be applied 8727 * to the specified provider. The dtrace_provider_lock must be held, and the 8728 * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation 8729 * will need to grab the dtrace_lock when it reenters the framework through 8730 * dtrace_probe_lookup(), dtrace_probe_create(), etc. 8731 */ 8732 static void 8733 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) 8734 { 8735 struct modctl *ctl; 8736 int all = 0; 8737 8738 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 8739 8740 if (prv == NULL) { 8741 all = 1; 8742 prv = dtrace_provider; 8743 } 8744 8745 do { 8746 /* 8747 * First, call the blanket provide operation. 8748 */ 8749 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc); 8750 8751 /* 8752 * Now call the per-module provide operation. We will grab 8753 * mod_lock to prevent the list from being modified. Note 8754 * that this also prevents the mod_busy bits from changing. 8755 * (mod_busy can only be changed with mod_lock held.) 8756 */ 8757 mutex_enter(&mod_lock); 8758 8759 ctl = &modules; 8760 do { 8761 if (ctl->mod_busy || ctl->mod_mp == NULL) 8762 continue; 8763 8764 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); 8765 8766 } while ((ctl = ctl->mod_next) != &modules); 8767 8768 mutex_exit(&mod_lock); 8769 } while (all && (prv = prv->dtpv_next) != NULL); 8770 } 8771 8772 /* 8773 * Iterate over each probe, and call the Framework-to-Provider API function 8774 * denoted by offs. 8775 */ 8776 static void 8777 dtrace_probe_foreach(uintptr_t offs) 8778 { 8779 dtrace_provider_t *prov; 8780 void (*func)(void *, dtrace_id_t, void *); 8781 dtrace_probe_t *probe; 8782 dtrace_icookie_t cookie; 8783 int i; 8784 8785 /* 8786 * We disable interrupts to walk through the probe array. This is 8787 * safe -- the dtrace_sync() in dtrace_unregister() assures that we 8788 * won't see stale data. 8789 */ 8790 cookie = dtrace_interrupt_disable(); 8791 8792 for (i = 0; i < dtrace_nprobes; i++) { 8793 if ((probe = dtrace_probes[i]) == NULL) 8794 continue; 8795 8796 if (probe->dtpr_ecb == NULL) { 8797 /* 8798 * This probe isn't enabled -- don't call the function. 8799 */ 8800 continue; 8801 } 8802 8803 prov = probe->dtpr_provider; 8804 func = *((void(**)(void *, dtrace_id_t, void *)) 8805 ((uintptr_t)&prov->dtpv_pops + offs)); 8806 8807 func(prov->dtpv_arg, i + 1, probe->dtpr_arg); 8808 } 8809 8810 dtrace_interrupt_enable(cookie); 8811 } 8812 8813 static int 8814 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab) 8815 { 8816 dtrace_probekey_t pkey; 8817 uint32_t priv; 8818 uid_t uid; 8819 zoneid_t zoneid; 8820 8821 ASSERT(MUTEX_HELD(&dtrace_lock)); 8822 dtrace_ecb_create_cache = NULL; 8823 8824 if (desc == NULL) { 8825 /* 8826 * If we're passed a NULL description, we're being asked to 8827 * create an ECB with a NULL probe. 8828 */ 8829 (void) dtrace_ecb_create_enable(NULL, enab); 8830 return (0); 8831 } 8832 8833 dtrace_probekey(desc, &pkey); 8834 dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred, 8835 &priv, &uid, &zoneid); 8836 8837 return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable, 8838 enab)); 8839 } 8840 8841 /* 8842 * DTrace Helper Provider Functions 8843 */ 8844 static void 8845 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr) 8846 { 8847 attr->dtat_name = DOF_ATTR_NAME(dofattr); 8848 attr->dtat_data = DOF_ATTR_DATA(dofattr); 8849 attr->dtat_class = DOF_ATTR_CLASS(dofattr); 8850 } 8851 8852 static void 8853 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov, 8854 const dof_provider_t *dofprov, char *strtab) 8855 { 8856 hprov->dthpv_provname = strtab + dofprov->dofpv_name; 8857 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider, 8858 dofprov->dofpv_provattr); 8859 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod, 8860 dofprov->dofpv_modattr); 8861 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func, 8862 dofprov->dofpv_funcattr); 8863 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name, 8864 dofprov->dofpv_nameattr); 8865 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args, 8866 dofprov->dofpv_argsattr); 8867 } 8868 8869 static void 8870 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid) 8871 { 8872 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 8873 dof_hdr_t *dof = (dof_hdr_t *)daddr; 8874 dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; 8875 dof_provider_t *provider; 8876 dof_probe_t *probe; 8877 uint32_t *off, *enoff; 8878 uint8_t *arg; 8879 char *strtab; 8880 uint_t i, nprobes; 8881 dtrace_helper_provdesc_t dhpv; 8882 dtrace_helper_probedesc_t dhpb; 8883 dtrace_meta_t *meta = dtrace_meta_pid; 8884 dtrace_mops_t *mops = &meta->dtm_mops; 8885 void *parg; 8886 8887 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); 8888 str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 8889 provider->dofpv_strtab * dof->dofh_secsize); 8890 prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 8891 provider->dofpv_probes * dof->dofh_secsize); 8892 arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 8893 provider->dofpv_prargs * dof->dofh_secsize); 8894 off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 8895 provider->dofpv_proffs * dof->dofh_secsize); 8896 8897 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); 8898 off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset); 8899 arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); 8900 enoff = NULL; 8901 8902 /* 8903 * See dtrace_helper_provider_validate(). 8904 */ 8905 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && 8906 provider->dofpv_prenoffs != DOF_SECT_NONE) { 8907 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 8908 provider->dofpv_prenoffs * dof->dofh_secsize); 8909 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset); 8910 } 8911 8912 nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; 8913 8914 /* 8915 * Create the provider. 8916 */ 8917 dtrace_dofprov2hprov(&dhpv, provider, strtab); 8918 8919 if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL) 8920 return; 8921 8922 meta->dtm_count++; 8923 8924 /* 8925 * Create the probes. 8926 */ 8927 for (i = 0; i < nprobes; i++) { 8928 probe = (dof_probe_t *)(uintptr_t)(daddr + 8929 prb_sec->dofs_offset + i * prb_sec->dofs_entsize); 8930 8931 dhpb.dthpb_mod = dhp->dofhp_mod; 8932 dhpb.dthpb_func = strtab + probe->dofpr_func; 8933 dhpb.dthpb_name = strtab + probe->dofpr_name; 8934 dhpb.dthpb_base = probe->dofpr_addr; 8935 dhpb.dthpb_offs = off + probe->dofpr_offidx; 8936 dhpb.dthpb_noffs = probe->dofpr_noffs; 8937 if (enoff != NULL) { 8938 dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx; 8939 dhpb.dthpb_nenoffs = probe->dofpr_nenoffs; 8940 } else { 8941 dhpb.dthpb_enoffs = NULL; 8942 dhpb.dthpb_nenoffs = 0; 8943 } 8944 dhpb.dthpb_args = arg + probe->dofpr_argidx; 8945 dhpb.dthpb_nargc = probe->dofpr_nargc; 8946 dhpb.dthpb_xargc = probe->dofpr_xargc; 8947 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv; 8948 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv; 8949 8950 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb); 8951 } 8952 } 8953 8954 static void 8955 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid) 8956 { 8957 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 8958 dof_hdr_t *dof = (dof_hdr_t *)daddr; 8959 int i; 8960 8961 ASSERT(MUTEX_HELD(&dtrace_meta_lock)); 8962 8963 for (i = 0; i < dof->dofh_secnum; i++) { 8964 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + 8965 dof->dofh_secoff + i * dof->dofh_secsize); 8966 8967 if (sec->dofs_type != DOF_SECT_PROVIDER) 8968 continue; 8969 8970 dtrace_helper_provide_one(dhp, sec, pid); 8971 } 8972 8973 /* 8974 * We may have just created probes, so we must now rematch against 8975 * any retained enablings. Note that this call will acquire both 8976 * cpu_lock and dtrace_lock; the fact that we are holding 8977 * dtrace_meta_lock now is what defines the ordering with respect to 8978 * these three locks. 8979 */ 8980 dtrace_enabling_matchall(); 8981 } 8982 8983 static void 8984 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid) 8985 { 8986 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 8987 dof_hdr_t *dof = (dof_hdr_t *)daddr; 8988 dof_sec_t *str_sec; 8989 dof_provider_t *provider; 8990 char *strtab; 8991 dtrace_helper_provdesc_t dhpv; 8992 dtrace_meta_t *meta = dtrace_meta_pid; 8993 dtrace_mops_t *mops = &meta->dtm_mops; 8994 8995 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); 8996 str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 8997 provider->dofpv_strtab * dof->dofh_secsize); 8998 8999 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); 9000 9001 /* 9002 * Create the provider. 9003 */ 9004 dtrace_dofprov2hprov(&dhpv, provider, strtab); 9005 9006 mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid); 9007 9008 meta->dtm_count--; 9009 } 9010 9011 static void 9012 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid) 9013 { 9014 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 9015 dof_hdr_t *dof = (dof_hdr_t *)daddr; 9016 int i; 9017 9018 ASSERT(MUTEX_HELD(&dtrace_meta_lock)); 9019 9020 for (i = 0; i < dof->dofh_secnum; i++) { 9021 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + 9022 dof->dofh_secoff + i * dof->dofh_secsize); 9023 9024 if (sec->dofs_type != DOF_SECT_PROVIDER) 9025 continue; 9026 9027 dtrace_helper_provider_remove_one(dhp, sec, pid); 9028 } 9029 } 9030 9031 /* 9032 * DTrace Meta Provider-to-Framework API Functions 9033 * 9034 * These functions implement the Meta Provider-to-Framework API, as described 9035 * in <sys/dtrace.h>. 9036 */ 9037 int 9038 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg, 9039 dtrace_meta_provider_id_t *idp) 9040 { 9041 dtrace_meta_t *meta; 9042 dtrace_helpers_t *help, *next; 9043 int i; 9044 9045 *idp = DTRACE_METAPROVNONE; 9046 9047 /* 9048 * We strictly don't need the name, but we hold onto it for 9049 * debuggability. All hail error queues! 9050 */ 9051 if (name == NULL) { 9052 cmn_err(CE_WARN, "failed to register meta-provider: " 9053 "invalid name"); 9054 return (EINVAL); 9055 } 9056 9057 if (mops == NULL || 9058 mops->dtms_create_probe == NULL || 9059 mops->dtms_provide_pid == NULL || 9060 mops->dtms_remove_pid == NULL) { 9061 cmn_err(CE_WARN, "failed to register meta-register %s: " 9062 "invalid ops", name); 9063 return (EINVAL); 9064 } 9065 9066 meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP); 9067 meta->dtm_mops = *mops; 9068 meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP); 9069 (void) strcpy(meta->dtm_name, name); 9070 meta->dtm_arg = arg; 9071 9072 mutex_enter(&dtrace_meta_lock); 9073 mutex_enter(&dtrace_lock); 9074 9075 if (dtrace_meta_pid != NULL) { 9076 mutex_exit(&dtrace_lock); 9077 mutex_exit(&dtrace_meta_lock); 9078 cmn_err(CE_WARN, "failed to register meta-register %s: " 9079 "user-land meta-provider exists", name); 9080 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1); 9081 kmem_free(meta, sizeof (dtrace_meta_t)); 9082 return (EINVAL); 9083 } 9084 9085 dtrace_meta_pid = meta; 9086 *idp = (dtrace_meta_provider_id_t)meta; 9087 9088 /* 9089 * If there are providers and probes ready to go, pass them 9090 * off to the new meta provider now. 9091 */ 9092 9093 help = dtrace_deferred_pid; 9094 dtrace_deferred_pid = NULL; 9095 9096 mutex_exit(&dtrace_lock); 9097 9098 while (help != NULL) { 9099 for (i = 0; i < help->dthps_nprovs; i++) { 9100 dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, 9101 help->dthps_pid); 9102 } 9103 9104 next = help->dthps_next; 9105 help->dthps_next = NULL; 9106 help->dthps_prev = NULL; 9107 help->dthps_deferred = 0; 9108 help = next; 9109 } 9110 9111 mutex_exit(&dtrace_meta_lock); 9112 9113 return (0); 9114 } 9115 9116 int 9117 dtrace_meta_unregister(dtrace_meta_provider_id_t id) 9118 { 9119 dtrace_meta_t **pp, *old = (dtrace_meta_t *)id; 9120 9121 mutex_enter(&dtrace_meta_lock); 9122 mutex_enter(&dtrace_lock); 9123 9124 if (old == dtrace_meta_pid) { 9125 pp = &dtrace_meta_pid; 9126 } else { 9127 panic("attempt to unregister non-existent " 9128 "dtrace meta-provider %p\n", (void *)old); 9129 } 9130 9131 if (old->dtm_count != 0) { 9132 mutex_exit(&dtrace_lock); 9133 mutex_exit(&dtrace_meta_lock); 9134 return (EBUSY); 9135 } 9136 9137 *pp = NULL; 9138 9139 mutex_exit(&dtrace_lock); 9140 mutex_exit(&dtrace_meta_lock); 9141 9142 kmem_free(old->dtm_name, strlen(old->dtm_name) + 1); 9143 kmem_free(old, sizeof (dtrace_meta_t)); 9144 9145 return (0); 9146 } 9147 9148 9149 /* 9150 * DTrace DIF Object Functions 9151 */ 9152 static int 9153 dtrace_difo_err(uint_t pc, const char *format, ...) 9154 { 9155 if (dtrace_err_verbose) { 9156 va_list alist; 9157 9158 (void) uprintf("dtrace DIF object error: [%u]: ", pc); 9159 va_start(alist, format); 9160 (void) vuprintf(format, alist); 9161 va_end(alist); 9162 } 9163 9164 #ifdef DTRACE_ERRDEBUG 9165 dtrace_errdebug(format); 9166 #endif 9167 return (1); 9168 } 9169 9170 /* 9171 * Validate a DTrace DIF object by checking the IR instructions. The following 9172 * rules are currently enforced by dtrace_difo_validate(): 9173 * 9174 * 1. Each instruction must have a valid opcode 9175 * 2. Each register, string, variable, or subroutine reference must be valid 9176 * 3. No instruction can modify register %r0 (must be zero) 9177 * 4. All instruction reserved bits must be set to zero 9178 * 5. The last instruction must be a "ret" instruction 9179 * 6. All branch targets must reference a valid instruction _after_ the branch 9180 */ 9181 static int 9182 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs, 9183 cred_t *cr) 9184 { 9185 int err = 0, i; 9186 int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; 9187 int kcheckload; 9188 uint_t pc; 9189 int maxglobal = -1, maxlocal = -1, maxtlocal = -1; 9190 9191 kcheckload = cr == NULL || 9192 (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0; 9193 9194 dp->dtdo_destructive = 0; 9195 9196 for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) { 9197 dif_instr_t instr = dp->dtdo_buf[pc]; 9198 9199 uint_t r1 = DIF_INSTR_R1(instr); 9200 uint_t r2 = DIF_INSTR_R2(instr); 9201 uint_t rd = DIF_INSTR_RD(instr); 9202 uint_t rs = DIF_INSTR_RS(instr); 9203 uint_t label = DIF_INSTR_LABEL(instr); 9204 uint_t v = DIF_INSTR_VAR(instr); 9205 uint_t subr = DIF_INSTR_SUBR(instr); 9206 uint_t type = DIF_INSTR_TYPE(instr); 9207 uint_t op = DIF_INSTR_OP(instr); 9208 9209 switch (op) { 9210 case DIF_OP_OR: 9211 case DIF_OP_XOR: 9212 case DIF_OP_AND: 9213 case DIF_OP_SLL: 9214 case DIF_OP_SRL: 9215 case DIF_OP_SRA: 9216 case DIF_OP_SUB: 9217 case DIF_OP_ADD: 9218 case DIF_OP_MUL: 9219 case DIF_OP_SDIV: 9220 case DIF_OP_UDIV: 9221 case DIF_OP_SREM: 9222 case DIF_OP_UREM: 9223 case DIF_OP_COPYS: 9224 if (r1 >= nregs) 9225 err += efunc(pc, "invalid register %u\n", r1); 9226 if (r2 >= nregs) 9227 err += efunc(pc, "invalid register %u\n", r2); 9228 if (rd >= nregs) 9229 err += efunc(pc, "invalid register %u\n", rd); 9230 if (rd == 0) 9231 err += efunc(pc, "cannot write to %r0\n"); 9232 break; 9233 case DIF_OP_NOT: 9234 case DIF_OP_MOV: 9235 case DIF_OP_ALLOCS: 9236 if (r1 >= nregs) 9237 err += efunc(pc, "invalid register %u\n", r1); 9238 if (r2 != 0) 9239 err += efunc(pc, "non-zero reserved bits\n"); 9240 if (rd >= nregs) 9241 err += efunc(pc, "invalid register %u\n", rd); 9242 if (rd == 0) 9243 err += efunc(pc, "cannot write to %r0\n"); 9244 break; 9245 case DIF_OP_LDSB: 9246 case DIF_OP_LDSH: 9247 case DIF_OP_LDSW: 9248 case DIF_OP_LDUB: 9249 case DIF_OP_LDUH: 9250 case DIF_OP_LDUW: 9251 case DIF_OP_LDX: 9252 if (r1 >= nregs) 9253 err += efunc(pc, "invalid register %u\n", r1); 9254 if (r2 != 0) 9255 err += efunc(pc, "non-zero reserved bits\n"); 9256 if (rd >= nregs) 9257 err += efunc(pc, "invalid register %u\n", rd); 9258 if (rd == 0) 9259 err += efunc(pc, "cannot write to %r0\n"); 9260 if (kcheckload) 9261 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op + 9262 DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd); 9263 break; 9264 case DIF_OP_RLDSB: 9265 case DIF_OP_RLDSH: 9266 case DIF_OP_RLDSW: 9267 case DIF_OP_RLDUB: 9268 case DIF_OP_RLDUH: 9269 case DIF_OP_RLDUW: 9270 case DIF_OP_RLDX: 9271 if (r1 >= nregs) 9272 err += efunc(pc, "invalid register %u\n", r1); 9273 if (r2 != 0) 9274 err += efunc(pc, "non-zero reserved bits\n"); 9275 if (rd >= nregs) 9276 err += efunc(pc, "invalid register %u\n", rd); 9277 if (rd == 0) 9278 err += efunc(pc, "cannot write to %r0\n"); 9279 break; 9280 case DIF_OP_ULDSB: 9281 case DIF_OP_ULDSH: 9282 case DIF_OP_ULDSW: 9283 case DIF_OP_ULDUB: 9284 case DIF_OP_ULDUH: 9285 case DIF_OP_ULDUW: 9286 case DIF_OP_ULDX: 9287 if (r1 >= nregs) 9288 err += efunc(pc, "invalid register %u\n", r1); 9289 if (r2 != 0) 9290 err += efunc(pc, "non-zero reserved bits\n"); 9291 if (rd >= nregs) 9292 err += efunc(pc, "invalid register %u\n", rd); 9293 if (rd == 0) 9294 err += efunc(pc, "cannot write to %r0\n"); 9295 break; 9296 case DIF_OP_STB: 9297 case DIF_OP_STH: 9298 case DIF_OP_STW: 9299 case DIF_OP_STX: 9300 if (r1 >= nregs) 9301 err += efunc(pc, "invalid register %u\n", r1); 9302 if (r2 != 0) 9303 err += efunc(pc, "non-zero reserved bits\n"); 9304 if (rd >= nregs) 9305 err += efunc(pc, "invalid register %u\n", rd); 9306 if (rd == 0) 9307 err += efunc(pc, "cannot write to 0 address\n"); 9308 break; 9309 case DIF_OP_CMP: 9310 case DIF_OP_SCMP: 9311 if (r1 >= nregs) 9312 err += efunc(pc, "invalid register %u\n", r1); 9313 if (r2 >= nregs) 9314 err += efunc(pc, "invalid register %u\n", r2); 9315 if (rd != 0) 9316 err += efunc(pc, "non-zero reserved bits\n"); 9317 break; 9318 case DIF_OP_TST: 9319 if (r1 >= nregs) 9320 err += efunc(pc, "invalid register %u\n", r1); 9321 if (r2 != 0 || rd != 0) 9322 err += efunc(pc, "non-zero reserved bits\n"); 9323 break; 9324 case DIF_OP_BA: 9325 case DIF_OP_BE: 9326 case DIF_OP_BNE: 9327 case DIF_OP_BG: 9328 case DIF_OP_BGU: 9329 case DIF_OP_BGE: 9330 case DIF_OP_BGEU: 9331 case DIF_OP_BL: 9332 case DIF_OP_BLU: 9333 case DIF_OP_BLE: 9334 case DIF_OP_BLEU: 9335 if (label >= dp->dtdo_len) { 9336 err += efunc(pc, "invalid branch target %u\n", 9337 label); 9338 } 9339 if (label <= pc) { 9340 err += efunc(pc, "backward branch to %u\n", 9341 label); 9342 } 9343 break; 9344 case DIF_OP_RET: 9345 if (r1 != 0 || r2 != 0) 9346 err += efunc(pc, "non-zero reserved bits\n"); 9347 if (rd >= nregs) 9348 err += efunc(pc, "invalid register %u\n", rd); 9349 break; 9350 case DIF_OP_NOP: 9351 case DIF_OP_POPTS: 9352 case DIF_OP_FLUSHTS: 9353 if (r1 != 0 || r2 != 0 || rd != 0) 9354 err += efunc(pc, "non-zero reserved bits\n"); 9355 break; 9356 case DIF_OP_SETX: 9357 if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) { 9358 err += efunc(pc, "invalid integer ref %u\n", 9359 DIF_INSTR_INTEGER(instr)); 9360 } 9361 if (rd >= nregs) 9362 err += efunc(pc, "invalid register %u\n", rd); 9363 if (rd == 0) 9364 err += efunc(pc, "cannot write to %r0\n"); 9365 break; 9366 case DIF_OP_SETS: 9367 if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) { 9368 err += efunc(pc, "invalid string ref %u\n", 9369 DIF_INSTR_STRING(instr)); 9370 } 9371 if (rd >= nregs) 9372 err += efunc(pc, "invalid register %u\n", rd); 9373 if (rd == 0) 9374 err += efunc(pc, "cannot write to %r0\n"); 9375 break; 9376 case DIF_OP_LDGA: 9377 case DIF_OP_LDTA: 9378 if (r1 > DIF_VAR_ARRAY_MAX) 9379 err += efunc(pc, "invalid array %u\n", r1); 9380 if (r2 >= nregs) 9381 err += efunc(pc, "invalid register %u\n", r2); 9382 if (rd >= nregs) 9383 err += efunc(pc, "invalid register %u\n", rd); 9384 if (rd == 0) 9385 err += efunc(pc, "cannot write to %r0\n"); 9386 break; 9387 case DIF_OP_LDGS: 9388 case DIF_OP_LDTS: 9389 case DIF_OP_LDLS: 9390 case DIF_OP_LDGAA: 9391 case DIF_OP_LDTAA: 9392 if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX) 9393 err += efunc(pc, "invalid variable %u\n", v); 9394 if (rd >= nregs) 9395 err += efunc(pc, "invalid register %u\n", rd); 9396 if (rd == 0) 9397 err += efunc(pc, "cannot write to %r0\n"); 9398 break; 9399 case DIF_OP_STGS: 9400 case DIF_OP_STTS: 9401 case DIF_OP_STLS: 9402 case DIF_OP_STGAA: 9403 case DIF_OP_STTAA: 9404 if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX) 9405 err += efunc(pc, "invalid variable %u\n", v); 9406 if (rs >= nregs) 9407 err += efunc(pc, "invalid register %u\n", rd); 9408 break; 9409 case DIF_OP_CALL: 9410 if (subr > DIF_SUBR_MAX) 9411 err += efunc(pc, "invalid subr %u\n", subr); 9412 if (rd >= nregs) 9413 err += efunc(pc, "invalid register %u\n", rd); 9414 if (rd == 0) 9415 err += efunc(pc, "cannot write to %r0\n"); 9416 9417 if (subr == DIF_SUBR_COPYOUT || 9418 subr == DIF_SUBR_COPYOUTSTR) { 9419 dp->dtdo_destructive = 1; 9420 } 9421 9422 if (subr == DIF_SUBR_GETF) { 9423 /* 9424 * If we have a getf() we need to record that 9425 * in our state. Note that our state can be 9426 * NULL if this is a helper -- but in that 9427 * case, the call to getf() is itself illegal, 9428 * and will be caught (slightly later) when 9429 * the helper is validated. 9430 */ 9431 if (vstate->dtvs_state != NULL) 9432 vstate->dtvs_state->dts_getf++; 9433 } 9434 9435 break; 9436 case DIF_OP_PUSHTR: 9437 if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF) 9438 err += efunc(pc, "invalid ref type %u\n", type); 9439 if (r2 >= nregs) 9440 err += efunc(pc, "invalid register %u\n", r2); 9441 if (rs >= nregs) 9442 err += efunc(pc, "invalid register %u\n", rs); 9443 break; 9444 case DIF_OP_PUSHTV: 9445 if (type != DIF_TYPE_CTF) 9446 err += efunc(pc, "invalid val type %u\n", type); 9447 if (r2 >= nregs) 9448 err += efunc(pc, "invalid register %u\n", r2); 9449 if (rs >= nregs) 9450 err += efunc(pc, "invalid register %u\n", rs); 9451 break; 9452 default: 9453 err += efunc(pc, "invalid opcode %u\n", 9454 DIF_INSTR_OP(instr)); 9455 } 9456 } 9457 9458 if (dp->dtdo_len != 0 && 9459 DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) { 9460 err += efunc(dp->dtdo_len - 1, 9461 "expected 'ret' as last DIF instruction\n"); 9462 } 9463 9464 if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) { 9465 /* 9466 * If we're not returning by reference, the size must be either 9467 * 0 or the size of one of the base types. 9468 */ 9469 switch (dp->dtdo_rtype.dtdt_size) { 9470 case 0: 9471 case sizeof (uint8_t): 9472 case sizeof (uint16_t): 9473 case sizeof (uint32_t): 9474 case sizeof (uint64_t): 9475 break; 9476 9477 default: 9478 err += efunc(dp->dtdo_len - 1, "bad return size\n"); 9479 } 9480 } 9481 9482 for (i = 0; i < dp->dtdo_varlen && err == 0; i++) { 9483 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL; 9484 dtrace_diftype_t *vt, *et; 9485 uint_t id, ndx; 9486 9487 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL && 9488 v->dtdv_scope != DIFV_SCOPE_THREAD && 9489 v->dtdv_scope != DIFV_SCOPE_LOCAL) { 9490 err += efunc(i, "unrecognized variable scope %d\n", 9491 v->dtdv_scope); 9492 break; 9493 } 9494 9495 if (v->dtdv_kind != DIFV_KIND_ARRAY && 9496 v->dtdv_kind != DIFV_KIND_SCALAR) { 9497 err += efunc(i, "unrecognized variable type %d\n", 9498 v->dtdv_kind); 9499 break; 9500 } 9501 9502 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) { 9503 err += efunc(i, "%d exceeds variable id limit\n", id); 9504 break; 9505 } 9506 9507 if (id < DIF_VAR_OTHER_UBASE) 9508 continue; 9509 9510 /* 9511 * For user-defined variables, we need to check that this 9512 * definition is identical to any previous definition that we 9513 * encountered. 9514 */ 9515 ndx = id - DIF_VAR_OTHER_UBASE; 9516 9517 switch (v->dtdv_scope) { 9518 case DIFV_SCOPE_GLOBAL: 9519 if (maxglobal == -1 || ndx > maxglobal) 9520 maxglobal = ndx; 9521 9522 if (ndx < vstate->dtvs_nglobals) { 9523 dtrace_statvar_t *svar; 9524 9525 if ((svar = vstate->dtvs_globals[ndx]) != NULL) 9526 existing = &svar->dtsv_var; 9527 } 9528 9529 break; 9530 9531 case DIFV_SCOPE_THREAD: 9532 if (maxtlocal == -1 || ndx > maxtlocal) 9533 maxtlocal = ndx; 9534 9535 if (ndx < vstate->dtvs_ntlocals) 9536 existing = &vstate->dtvs_tlocals[ndx]; 9537 break; 9538 9539 case DIFV_SCOPE_LOCAL: 9540 if (maxlocal == -1 || ndx > maxlocal) 9541 maxlocal = ndx; 9542 9543 if (ndx < vstate->dtvs_nlocals) { 9544 dtrace_statvar_t *svar; 9545 9546 if ((svar = vstate->dtvs_locals[ndx]) != NULL) 9547 existing = &svar->dtsv_var; 9548 } 9549 9550 break; 9551 } 9552 9553 vt = &v->dtdv_type; 9554 9555 if (vt->dtdt_flags & DIF_TF_BYREF) { 9556 if (vt->dtdt_size == 0) { 9557 err += efunc(i, "zero-sized variable\n"); 9558 break; 9559 } 9560 9561 if ((v->dtdv_scope == DIFV_SCOPE_GLOBAL || 9562 v->dtdv_scope == DIFV_SCOPE_LOCAL) && 9563 vt->dtdt_size > dtrace_statvar_maxsize) { 9564 err += efunc(i, "oversized by-ref static\n"); 9565 break; 9566 } 9567 } 9568 9569 if (existing == NULL || existing->dtdv_id == 0) 9570 continue; 9571 9572 ASSERT(existing->dtdv_id == v->dtdv_id); 9573 ASSERT(existing->dtdv_scope == v->dtdv_scope); 9574 9575 if (existing->dtdv_kind != v->dtdv_kind) 9576 err += efunc(i, "%d changed variable kind\n", id); 9577 9578 et = &existing->dtdv_type; 9579 9580 if (vt->dtdt_flags != et->dtdt_flags) { 9581 err += efunc(i, "%d changed variable type flags\n", id); 9582 break; 9583 } 9584 9585 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) { 9586 err += efunc(i, "%d changed variable type size\n", id); 9587 break; 9588 } 9589 } 9590 9591 for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) { 9592 dif_instr_t instr = dp->dtdo_buf[pc]; 9593 9594 uint_t v = DIF_INSTR_VAR(instr); 9595 uint_t op = DIF_INSTR_OP(instr); 9596 9597 switch (op) { 9598 case DIF_OP_LDGS: 9599 case DIF_OP_LDGAA: 9600 case DIF_OP_STGS: 9601 case DIF_OP_STGAA: 9602 if (v > DIF_VAR_OTHER_UBASE + maxglobal) 9603 err += efunc(pc, "invalid variable %u\n", v); 9604 break; 9605 case DIF_OP_LDTS: 9606 case DIF_OP_LDTAA: 9607 case DIF_OP_STTS: 9608 case DIF_OP_STTAA: 9609 if (v > DIF_VAR_OTHER_UBASE + maxtlocal) 9610 err += efunc(pc, "invalid variable %u\n", v); 9611 break; 9612 case DIF_OP_LDLS: 9613 case DIF_OP_STLS: 9614 if (v > DIF_VAR_OTHER_UBASE + maxlocal) 9615 err += efunc(pc, "invalid variable %u\n", v); 9616 break; 9617 default: 9618 break; 9619 } 9620 } 9621 9622 return (err); 9623 } 9624 9625 /* 9626 * Validate a DTrace DIF object that it is to be used as a helper. Helpers 9627 * are much more constrained than normal DIFOs. Specifically, they may 9628 * not: 9629 * 9630 * 1. Make calls to subroutines other than copyin(), copyinstr() or 9631 * miscellaneous string routines 9632 * 2. Access DTrace variables other than the args[] array, and the 9633 * curthread, pid, ppid, tid, execname, zonename, uid and gid variables. 9634 * 3. Have thread-local variables. 9635 * 4. Have dynamic variables. 9636 */ 9637 static int 9638 dtrace_difo_validate_helper(dtrace_difo_t *dp) 9639 { 9640 int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; 9641 int err = 0; 9642 uint_t pc; 9643 9644 for (pc = 0; pc < dp->dtdo_len; pc++) { 9645 dif_instr_t instr = dp->dtdo_buf[pc]; 9646 9647 uint_t v = DIF_INSTR_VAR(instr); 9648 uint_t subr = DIF_INSTR_SUBR(instr); 9649 uint_t op = DIF_INSTR_OP(instr); 9650 9651 switch (op) { 9652 case DIF_OP_OR: 9653 case DIF_OP_XOR: 9654 case DIF_OP_AND: 9655 case DIF_OP_SLL: 9656 case DIF_OP_SRL: 9657 case DIF_OP_SRA: 9658 case DIF_OP_SUB: 9659 case DIF_OP_ADD: 9660 case DIF_OP_MUL: 9661 case DIF_OP_SDIV: 9662 case DIF_OP_UDIV: 9663 case DIF_OP_SREM: 9664 case DIF_OP_UREM: 9665 case DIF_OP_COPYS: 9666 case DIF_OP_NOT: 9667 case DIF_OP_MOV: 9668 case DIF_OP_RLDSB: 9669 case DIF_OP_RLDSH: 9670 case DIF_OP_RLDSW: 9671 case DIF_OP_RLDUB: 9672 case DIF_OP_RLDUH: 9673 case DIF_OP_RLDUW: 9674 case DIF_OP_RLDX: 9675 case DIF_OP_ULDSB: 9676 case DIF_OP_ULDSH: 9677 case DIF_OP_ULDSW: 9678 case DIF_OP_ULDUB: 9679 case DIF_OP_ULDUH: 9680 case DIF_OP_ULDUW: 9681 case DIF_OP_ULDX: 9682 case DIF_OP_STB: 9683 case DIF_OP_STH: 9684 case DIF_OP_STW: 9685 case DIF_OP_STX: 9686 case DIF_OP_ALLOCS: 9687 case DIF_OP_CMP: 9688 case DIF_OP_SCMP: 9689 case DIF_OP_TST: 9690 case DIF_OP_BA: 9691 case DIF_OP_BE: 9692 case DIF_OP_BNE: 9693 case DIF_OP_BG: 9694 case DIF_OP_BGU: 9695 case DIF_OP_BGE: 9696 case DIF_OP_BGEU: 9697 case DIF_OP_BL: 9698 case DIF_OP_BLU: 9699 case DIF_OP_BLE: 9700 case DIF_OP_BLEU: 9701 case DIF_OP_RET: 9702 case DIF_OP_NOP: 9703 case DIF_OP_POPTS: 9704 case DIF_OP_FLUSHTS: 9705 case DIF_OP_SETX: 9706 case DIF_OP_SETS: 9707 case DIF_OP_LDGA: 9708 case DIF_OP_LDLS: 9709 case DIF_OP_STGS: 9710 case DIF_OP_STLS: 9711 case DIF_OP_PUSHTR: 9712 case DIF_OP_PUSHTV: 9713 break; 9714 9715 case DIF_OP_LDGS: 9716 if (v >= DIF_VAR_OTHER_UBASE) 9717 break; 9718 9719 if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) 9720 break; 9721 9722 if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID || 9723 v == DIF_VAR_PPID || v == DIF_VAR_TID || 9724 v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME || 9725 v == DIF_VAR_UID || v == DIF_VAR_GID) 9726 break; 9727 9728 err += efunc(pc, "illegal variable %u\n", v); 9729 break; 9730 9731 case DIF_OP_LDTA: 9732 case DIF_OP_LDTS: 9733 case DIF_OP_LDGAA: 9734 case DIF_OP_LDTAA: 9735 err += efunc(pc, "illegal dynamic variable load\n"); 9736 break; 9737 9738 case DIF_OP_STTS: 9739 case DIF_OP_STGAA: 9740 case DIF_OP_STTAA: 9741 err += efunc(pc, "illegal dynamic variable store\n"); 9742 break; 9743 9744 case DIF_OP_CALL: 9745 if (subr == DIF_SUBR_ALLOCA || 9746 subr == DIF_SUBR_BCOPY || 9747 subr == DIF_SUBR_COPYIN || 9748 subr == DIF_SUBR_COPYINTO || 9749 subr == DIF_SUBR_COPYINSTR || 9750 subr == DIF_SUBR_INDEX || 9751 subr == DIF_SUBR_INET_NTOA || 9752 subr == DIF_SUBR_INET_NTOA6 || 9753 subr == DIF_SUBR_INET_NTOP || 9754 subr == DIF_SUBR_JSON || 9755 subr == DIF_SUBR_LLTOSTR || 9756 subr == DIF_SUBR_STRTOLL || 9757 subr == DIF_SUBR_RINDEX || 9758 subr == DIF_SUBR_STRCHR || 9759 subr == DIF_SUBR_STRJOIN || 9760 subr == DIF_SUBR_STRRCHR || 9761 subr == DIF_SUBR_STRSTR || 9762 subr == DIF_SUBR_HTONS || 9763 subr == DIF_SUBR_HTONL || 9764 subr == DIF_SUBR_HTONLL || 9765 subr == DIF_SUBR_NTOHS || 9766 subr == DIF_SUBR_NTOHL || 9767 subr == DIF_SUBR_NTOHLL) 9768 break; 9769 9770 err += efunc(pc, "invalid subr %u\n", subr); 9771 break; 9772 9773 default: 9774 err += efunc(pc, "invalid opcode %u\n", 9775 DIF_INSTR_OP(instr)); 9776 } 9777 } 9778 9779 return (err); 9780 } 9781 9782 /* 9783 * Returns 1 if the expression in the DIF object can be cached on a per-thread 9784 * basis; 0 if not. 9785 */ 9786 static int 9787 dtrace_difo_cacheable(dtrace_difo_t *dp) 9788 { 9789 int i; 9790 9791 if (dp == NULL) 9792 return (0); 9793 9794 for (i = 0; i < dp->dtdo_varlen; i++) { 9795 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 9796 9797 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL) 9798 continue; 9799 9800 switch (v->dtdv_id) { 9801 case DIF_VAR_CURTHREAD: 9802 case DIF_VAR_PID: 9803 case DIF_VAR_TID: 9804 case DIF_VAR_EXECNAME: 9805 case DIF_VAR_ZONENAME: 9806 break; 9807 9808 default: 9809 return (0); 9810 } 9811 } 9812 9813 /* 9814 * This DIF object may be cacheable. Now we need to look for any 9815 * array loading instructions, any memory loading instructions, or 9816 * any stores to thread-local variables. 9817 */ 9818 for (i = 0; i < dp->dtdo_len; i++) { 9819 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]); 9820 9821 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) || 9822 (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) || 9823 (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) || 9824 op == DIF_OP_LDGA || op == DIF_OP_STTS) 9825 return (0); 9826 } 9827 9828 return (1); 9829 } 9830 9831 static void 9832 dtrace_difo_hold(dtrace_difo_t *dp) 9833 { 9834 int i; 9835 9836 ASSERT(MUTEX_HELD(&dtrace_lock)); 9837 9838 dp->dtdo_refcnt++; 9839 ASSERT(dp->dtdo_refcnt != 0); 9840 9841 /* 9842 * We need to check this DIF object for references to the variable 9843 * DIF_VAR_VTIMESTAMP. 9844 */ 9845 for (i = 0; i < dp->dtdo_varlen; i++) { 9846 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 9847 9848 if (v->dtdv_id != DIF_VAR_VTIMESTAMP) 9849 continue; 9850 9851 if (dtrace_vtime_references++ == 0) 9852 dtrace_vtime_enable(); 9853 } 9854 } 9855 9856 /* 9857 * This routine calculates the dynamic variable chunksize for a given DIF 9858 * object. The calculation is not fool-proof, and can probably be tricked by 9859 * malicious DIF -- but it works for all compiler-generated DIF. Because this 9860 * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail 9861 * if a dynamic variable size exceeds the chunksize. 9862 */ 9863 static void 9864 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 9865 { 9866 uint64_t sval; 9867 dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ 9868 const dif_instr_t *text = dp->dtdo_buf; 9869 uint_t pc, srd = 0; 9870 uint_t ttop = 0; 9871 size_t size, ksize; 9872 uint_t id, i; 9873 9874 for (pc = 0; pc < dp->dtdo_len; pc++) { 9875 dif_instr_t instr = text[pc]; 9876 uint_t op = DIF_INSTR_OP(instr); 9877 uint_t rd = DIF_INSTR_RD(instr); 9878 uint_t r1 = DIF_INSTR_R1(instr); 9879 uint_t nkeys = 0; 9880 uchar_t scope; 9881 9882 dtrace_key_t *key = tupregs; 9883 9884 switch (op) { 9885 case DIF_OP_SETX: 9886 sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)]; 9887 srd = rd; 9888 continue; 9889 9890 case DIF_OP_STTS: 9891 key = &tupregs[DIF_DTR_NREGS]; 9892 key[0].dttk_size = 0; 9893 key[1].dttk_size = 0; 9894 nkeys = 2; 9895 scope = DIFV_SCOPE_THREAD; 9896 break; 9897 9898 case DIF_OP_STGAA: 9899 case DIF_OP_STTAA: 9900 nkeys = ttop; 9901 9902 if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) 9903 key[nkeys++].dttk_size = 0; 9904 9905 key[nkeys++].dttk_size = 0; 9906 9907 if (op == DIF_OP_STTAA) { 9908 scope = DIFV_SCOPE_THREAD; 9909 } else { 9910 scope = DIFV_SCOPE_GLOBAL; 9911 } 9912 9913 break; 9914 9915 case DIF_OP_PUSHTR: 9916 if (ttop == DIF_DTR_NREGS) 9917 return; 9918 9919 if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) { 9920 /* 9921 * If the register for the size of the "pushtr" 9922 * is %r0 (or the value is 0) and the type is 9923 * a string, we'll use the system-wide default 9924 * string size. 9925 */ 9926 tupregs[ttop++].dttk_size = 9927 dtrace_strsize_default; 9928 } else { 9929 if (srd == 0) 9930 return; 9931 9932 if (sval > LONG_MAX) 9933 return; 9934 9935 tupregs[ttop++].dttk_size = sval; 9936 } 9937 9938 break; 9939 9940 case DIF_OP_PUSHTV: 9941 if (ttop == DIF_DTR_NREGS) 9942 return; 9943 9944 tupregs[ttop++].dttk_size = 0; 9945 break; 9946 9947 case DIF_OP_FLUSHTS: 9948 ttop = 0; 9949 break; 9950 9951 case DIF_OP_POPTS: 9952 if (ttop != 0) 9953 ttop--; 9954 break; 9955 } 9956 9957 sval = 0; 9958 srd = 0; 9959 9960 if (nkeys == 0) 9961 continue; 9962 9963 /* 9964 * We have a dynamic variable allocation; calculate its size. 9965 */ 9966 for (ksize = 0, i = 0; i < nkeys; i++) 9967 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t)); 9968 9969 size = sizeof (dtrace_dynvar_t); 9970 size += sizeof (dtrace_key_t) * (nkeys - 1); 9971 size += ksize; 9972 9973 /* 9974 * Now we need to determine the size of the stored data. 9975 */ 9976 id = DIF_INSTR_VAR(instr); 9977 9978 for (i = 0; i < dp->dtdo_varlen; i++) { 9979 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 9980 9981 if (v->dtdv_id == id && v->dtdv_scope == scope) { 9982 size += v->dtdv_type.dtdt_size; 9983 break; 9984 } 9985 } 9986 9987 if (i == dp->dtdo_varlen) 9988 return; 9989 9990 /* 9991 * We have the size. If this is larger than the chunk size 9992 * for our dynamic variable state, reset the chunk size. 9993 */ 9994 size = P2ROUNDUP(size, sizeof (uint64_t)); 9995 9996 /* 9997 * Before setting the chunk size, check that we're not going 9998 * to set it to a negative value... 9999 */ 10000 if (size > LONG_MAX) 10001 return; 10002 10003 /* 10004 * ...and make certain that we didn't badly overflow. 10005 */ 10006 if (size < ksize || size < sizeof (dtrace_dynvar_t)) 10007 return; 10008 10009 if (size > vstate->dtvs_dynvars.dtds_chunksize) 10010 vstate->dtvs_dynvars.dtds_chunksize = size; 10011 } 10012 } 10013 10014 static void 10015 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10016 { 10017 int i, oldsvars, osz, nsz, otlocals, ntlocals; 10018 uint_t id; 10019 10020 ASSERT(MUTEX_HELD(&dtrace_lock)); 10021 ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0); 10022 10023 for (i = 0; i < dp->dtdo_varlen; i++) { 10024 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10025 dtrace_statvar_t *svar, ***svarp; 10026 size_t dsize = 0; 10027 uint8_t scope = v->dtdv_scope; 10028 int *np; 10029 10030 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE) 10031 continue; 10032 10033 id -= DIF_VAR_OTHER_UBASE; 10034 10035 switch (scope) { 10036 case DIFV_SCOPE_THREAD: 10037 while (id >= (otlocals = vstate->dtvs_ntlocals)) { 10038 dtrace_difv_t *tlocals; 10039 10040 if ((ntlocals = (otlocals << 1)) == 0) 10041 ntlocals = 1; 10042 10043 osz = otlocals * sizeof (dtrace_difv_t); 10044 nsz = ntlocals * sizeof (dtrace_difv_t); 10045 10046 tlocals = kmem_zalloc(nsz, KM_SLEEP); 10047 10048 if (osz != 0) { 10049 bcopy(vstate->dtvs_tlocals, 10050 tlocals, osz); 10051 kmem_free(vstate->dtvs_tlocals, osz); 10052 } 10053 10054 vstate->dtvs_tlocals = tlocals; 10055 vstate->dtvs_ntlocals = ntlocals; 10056 } 10057 10058 vstate->dtvs_tlocals[id] = *v; 10059 continue; 10060 10061 case DIFV_SCOPE_LOCAL: 10062 np = &vstate->dtvs_nlocals; 10063 svarp = &vstate->dtvs_locals; 10064 10065 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) 10066 dsize = NCPU * (v->dtdv_type.dtdt_size + 10067 sizeof (uint64_t)); 10068 else 10069 dsize = NCPU * sizeof (uint64_t); 10070 10071 break; 10072 10073 case DIFV_SCOPE_GLOBAL: 10074 np = &vstate->dtvs_nglobals; 10075 svarp = &vstate->dtvs_globals; 10076 10077 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) 10078 dsize = v->dtdv_type.dtdt_size + 10079 sizeof (uint64_t); 10080 10081 break; 10082 10083 default: 10084 ASSERT(0); 10085 } 10086 10087 while (id >= (oldsvars = *np)) { 10088 dtrace_statvar_t **statics; 10089 int newsvars, oldsize, newsize; 10090 10091 if ((newsvars = (oldsvars << 1)) == 0) 10092 newsvars = 1; 10093 10094 oldsize = oldsvars * sizeof (dtrace_statvar_t *); 10095 newsize = newsvars * sizeof (dtrace_statvar_t *); 10096 10097 statics = kmem_zalloc(newsize, KM_SLEEP); 10098 10099 if (oldsize != 0) { 10100 bcopy(*svarp, statics, oldsize); 10101 kmem_free(*svarp, oldsize); 10102 } 10103 10104 *svarp = statics; 10105 *np = newsvars; 10106 } 10107 10108 if ((svar = (*svarp)[id]) == NULL) { 10109 svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP); 10110 svar->dtsv_var = *v; 10111 10112 if ((svar->dtsv_size = dsize) != 0) { 10113 svar->dtsv_data = (uint64_t)(uintptr_t) 10114 kmem_zalloc(dsize, KM_SLEEP); 10115 } 10116 10117 (*svarp)[id] = svar; 10118 } 10119 10120 svar->dtsv_refcnt++; 10121 } 10122 10123 dtrace_difo_chunksize(dp, vstate); 10124 dtrace_difo_hold(dp); 10125 } 10126 10127 static dtrace_difo_t * 10128 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10129 { 10130 dtrace_difo_t *new; 10131 size_t sz; 10132 10133 ASSERT(dp->dtdo_buf != NULL); 10134 ASSERT(dp->dtdo_refcnt != 0); 10135 10136 new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP); 10137 10138 ASSERT(dp->dtdo_buf != NULL); 10139 sz = dp->dtdo_len * sizeof (dif_instr_t); 10140 new->dtdo_buf = kmem_alloc(sz, KM_SLEEP); 10141 bcopy(dp->dtdo_buf, new->dtdo_buf, sz); 10142 new->dtdo_len = dp->dtdo_len; 10143 10144 if (dp->dtdo_strtab != NULL) { 10145 ASSERT(dp->dtdo_strlen != 0); 10146 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP); 10147 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen); 10148 new->dtdo_strlen = dp->dtdo_strlen; 10149 } 10150 10151 if (dp->dtdo_inttab != NULL) { 10152 ASSERT(dp->dtdo_intlen != 0); 10153 sz = dp->dtdo_intlen * sizeof (uint64_t); 10154 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP); 10155 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz); 10156 new->dtdo_intlen = dp->dtdo_intlen; 10157 } 10158 10159 if (dp->dtdo_vartab != NULL) { 10160 ASSERT(dp->dtdo_varlen != 0); 10161 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t); 10162 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP); 10163 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz); 10164 new->dtdo_varlen = dp->dtdo_varlen; 10165 } 10166 10167 dtrace_difo_init(new, vstate); 10168 return (new); 10169 } 10170 10171 static void 10172 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10173 { 10174 int i; 10175 10176 ASSERT(dp->dtdo_refcnt == 0); 10177 10178 for (i = 0; i < dp->dtdo_varlen; i++) { 10179 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10180 dtrace_statvar_t *svar, **svarp; 10181 uint_t id; 10182 uint8_t scope = v->dtdv_scope; 10183 int *np; 10184 10185 switch (scope) { 10186 case DIFV_SCOPE_THREAD: 10187 continue; 10188 10189 case DIFV_SCOPE_LOCAL: 10190 np = &vstate->dtvs_nlocals; 10191 svarp = vstate->dtvs_locals; 10192 break; 10193 10194 case DIFV_SCOPE_GLOBAL: 10195 np = &vstate->dtvs_nglobals; 10196 svarp = vstate->dtvs_globals; 10197 break; 10198 10199 default: 10200 ASSERT(0); 10201 } 10202 10203 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE) 10204 continue; 10205 10206 id -= DIF_VAR_OTHER_UBASE; 10207 ASSERT(id < *np); 10208 10209 svar = svarp[id]; 10210 ASSERT(svar != NULL); 10211 ASSERT(svar->dtsv_refcnt > 0); 10212 10213 if (--svar->dtsv_refcnt > 0) 10214 continue; 10215 10216 if (svar->dtsv_size != 0) { 10217 ASSERT(svar->dtsv_data != NULL); 10218 kmem_free((void *)(uintptr_t)svar->dtsv_data, 10219 svar->dtsv_size); 10220 } 10221 10222 kmem_free(svar, sizeof (dtrace_statvar_t)); 10223 svarp[id] = NULL; 10224 } 10225 10226 kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t)); 10227 kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t)); 10228 kmem_free(dp->dtdo_strtab, dp->dtdo_strlen); 10229 kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t)); 10230 10231 kmem_free(dp, sizeof (dtrace_difo_t)); 10232 } 10233 10234 static void 10235 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10236 { 10237 int i; 10238 10239 ASSERT(MUTEX_HELD(&dtrace_lock)); 10240 ASSERT(dp->dtdo_refcnt != 0); 10241 10242 for (i = 0; i < dp->dtdo_varlen; i++) { 10243 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10244 10245 if (v->dtdv_id != DIF_VAR_VTIMESTAMP) 10246 continue; 10247 10248 ASSERT(dtrace_vtime_references > 0); 10249 if (--dtrace_vtime_references == 0) 10250 dtrace_vtime_disable(); 10251 } 10252 10253 if (--dp->dtdo_refcnt == 0) 10254 dtrace_difo_destroy(dp, vstate); 10255 } 10256 10257 /* 10258 * DTrace Format Functions 10259 */ 10260 static uint16_t 10261 dtrace_format_add(dtrace_state_t *state, char *str) 10262 { 10263 char *fmt, **new; 10264 uint16_t ndx, len = strlen(str) + 1; 10265 10266 fmt = kmem_zalloc(len, KM_SLEEP); 10267 bcopy(str, fmt, len); 10268 10269 for (ndx = 0; ndx < state->dts_nformats; ndx++) { 10270 if (state->dts_formats[ndx] == NULL) { 10271 state->dts_formats[ndx] = fmt; 10272 return (ndx + 1); 10273 } 10274 } 10275 10276 if (state->dts_nformats == USHRT_MAX) { 10277 /* 10278 * This is only likely if a denial-of-service attack is being 10279 * attempted. As such, it's okay to fail silently here. 10280 */ 10281 kmem_free(fmt, len); 10282 return (0); 10283 } 10284 10285 /* 10286 * For simplicity, we always resize the formats array to be exactly the 10287 * number of formats. 10288 */ 10289 ndx = state->dts_nformats++; 10290 new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP); 10291 10292 if (state->dts_formats != NULL) { 10293 ASSERT(ndx != 0); 10294 bcopy(state->dts_formats, new, ndx * sizeof (char *)); 10295 kmem_free(state->dts_formats, ndx * sizeof (char *)); 10296 } 10297 10298 state->dts_formats = new; 10299 state->dts_formats[ndx] = fmt; 10300 10301 return (ndx + 1); 10302 } 10303 10304 static void 10305 dtrace_format_remove(dtrace_state_t *state, uint16_t format) 10306 { 10307 char *fmt; 10308 10309 ASSERT(state->dts_formats != NULL); 10310 ASSERT(format <= state->dts_nformats); 10311 ASSERT(state->dts_formats[format - 1] != NULL); 10312 10313 fmt = state->dts_formats[format - 1]; 10314 kmem_free(fmt, strlen(fmt) + 1); 10315 state->dts_formats[format - 1] = NULL; 10316 } 10317 10318 static void 10319 dtrace_format_destroy(dtrace_state_t *state) 10320 { 10321 int i; 10322 10323 if (state->dts_nformats == 0) { 10324 ASSERT(state->dts_formats == NULL); 10325 return; 10326 } 10327 10328 ASSERT(state->dts_formats != NULL); 10329 10330 for (i = 0; i < state->dts_nformats; i++) { 10331 char *fmt = state->dts_formats[i]; 10332 10333 if (fmt == NULL) 10334 continue; 10335 10336 kmem_free(fmt, strlen(fmt) + 1); 10337 } 10338 10339 kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *)); 10340 state->dts_nformats = 0; 10341 state->dts_formats = NULL; 10342 } 10343 10344 /* 10345 * DTrace Predicate Functions 10346 */ 10347 static dtrace_predicate_t * 10348 dtrace_predicate_create(dtrace_difo_t *dp) 10349 { 10350 dtrace_predicate_t *pred; 10351 10352 ASSERT(MUTEX_HELD(&dtrace_lock)); 10353 ASSERT(dp->dtdo_refcnt != 0); 10354 10355 pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP); 10356 pred->dtp_difo = dp; 10357 pred->dtp_refcnt = 1; 10358 10359 if (!dtrace_difo_cacheable(dp)) 10360 return (pred); 10361 10362 if (dtrace_predcache_id == DTRACE_CACHEIDNONE) { 10363 /* 10364 * This is only theoretically possible -- we have had 2^32 10365 * cacheable predicates on this machine. We cannot allow any 10366 * more predicates to become cacheable: as unlikely as it is, 10367 * there may be a thread caching a (now stale) predicate cache 10368 * ID. (N.B.: the temptation is being successfully resisted to 10369 * have this cmn_err() "Holy shit -- we executed this code!") 10370 */ 10371 return (pred); 10372 } 10373 10374 pred->dtp_cacheid = dtrace_predcache_id++; 10375 10376 return (pred); 10377 } 10378 10379 static void 10380 dtrace_predicate_hold(dtrace_predicate_t *pred) 10381 { 10382 ASSERT(MUTEX_HELD(&dtrace_lock)); 10383 ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0); 10384 ASSERT(pred->dtp_refcnt > 0); 10385 10386 pred->dtp_refcnt++; 10387 } 10388 10389 static void 10390 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate) 10391 { 10392 dtrace_difo_t *dp = pred->dtp_difo; 10393 10394 ASSERT(MUTEX_HELD(&dtrace_lock)); 10395 ASSERT(dp != NULL && dp->dtdo_refcnt != 0); 10396 ASSERT(pred->dtp_refcnt > 0); 10397 10398 if (--pred->dtp_refcnt == 0) { 10399 dtrace_difo_release(pred->dtp_difo, vstate); 10400 kmem_free(pred, sizeof (dtrace_predicate_t)); 10401 } 10402 } 10403 10404 /* 10405 * DTrace Action Description Functions 10406 */ 10407 static dtrace_actdesc_t * 10408 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple, 10409 uint64_t uarg, uint64_t arg) 10410 { 10411 dtrace_actdesc_t *act; 10412 10413 ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL && 10414 arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA)); 10415 10416 act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP); 10417 act->dtad_kind = kind; 10418 act->dtad_ntuple = ntuple; 10419 act->dtad_uarg = uarg; 10420 act->dtad_arg = arg; 10421 act->dtad_refcnt = 1; 10422 10423 return (act); 10424 } 10425 10426 static void 10427 dtrace_actdesc_hold(dtrace_actdesc_t *act) 10428 { 10429 ASSERT(act->dtad_refcnt >= 1); 10430 act->dtad_refcnt++; 10431 } 10432 10433 static void 10434 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate) 10435 { 10436 dtrace_actkind_t kind = act->dtad_kind; 10437 dtrace_difo_t *dp; 10438 10439 ASSERT(act->dtad_refcnt >= 1); 10440 10441 if (--act->dtad_refcnt != 0) 10442 return; 10443 10444 if ((dp = act->dtad_difo) != NULL) 10445 dtrace_difo_release(dp, vstate); 10446 10447 if (DTRACEACT_ISPRINTFLIKE(kind)) { 10448 char *str = (char *)(uintptr_t)act->dtad_arg; 10449 10450 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) || 10451 (str == NULL && act->dtad_kind == DTRACEACT_PRINTA)); 10452 10453 if (str != NULL) 10454 kmem_free(str, strlen(str) + 1); 10455 } 10456 10457 kmem_free(act, sizeof (dtrace_actdesc_t)); 10458 } 10459 10460 /* 10461 * DTrace ECB Functions 10462 */ 10463 static dtrace_ecb_t * 10464 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe) 10465 { 10466 dtrace_ecb_t *ecb; 10467 dtrace_epid_t epid; 10468 10469 ASSERT(MUTEX_HELD(&dtrace_lock)); 10470 10471 ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP); 10472 ecb->dte_predicate = NULL; 10473 ecb->dte_probe = probe; 10474 10475 /* 10476 * The default size is the size of the default action: recording 10477 * the header. 10478 */ 10479 ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t); 10480 ecb->dte_alignment = sizeof (dtrace_epid_t); 10481 10482 epid = state->dts_epid++; 10483 10484 if (epid - 1 >= state->dts_necbs) { 10485 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs; 10486 int necbs = state->dts_necbs << 1; 10487 10488 ASSERT(epid == state->dts_necbs + 1); 10489 10490 if (necbs == 0) { 10491 ASSERT(oecbs == NULL); 10492 necbs = 1; 10493 } 10494 10495 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP); 10496 10497 if (oecbs != NULL) 10498 bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs)); 10499 10500 dtrace_membar_producer(); 10501 state->dts_ecbs = ecbs; 10502 10503 if (oecbs != NULL) { 10504 /* 10505 * If this state is active, we must dtrace_sync() 10506 * before we can free the old dts_ecbs array: we're 10507 * coming in hot, and there may be active ring 10508 * buffer processing (which indexes into the dts_ecbs 10509 * array) on another CPU. 10510 */ 10511 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) 10512 dtrace_sync(); 10513 10514 kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs)); 10515 } 10516 10517 dtrace_membar_producer(); 10518 state->dts_necbs = necbs; 10519 } 10520 10521 ecb->dte_state = state; 10522 10523 ASSERT(state->dts_ecbs[epid - 1] == NULL); 10524 dtrace_membar_producer(); 10525 state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb; 10526 10527 return (ecb); 10528 } 10529 10530 static int 10531 dtrace_ecb_enable(dtrace_ecb_t *ecb) 10532 { 10533 dtrace_probe_t *probe = ecb->dte_probe; 10534 10535 ASSERT(MUTEX_HELD(&cpu_lock)); 10536 ASSERT(MUTEX_HELD(&dtrace_lock)); 10537 ASSERT(ecb->dte_next == NULL); 10538 10539 if (probe == NULL) { 10540 /* 10541 * This is the NULL probe -- there's nothing to do. 10542 */ 10543 return (0); 10544 } 10545 10546 if (probe->dtpr_ecb == NULL) { 10547 dtrace_provider_t *prov = probe->dtpr_provider; 10548 10549 /* 10550 * We're the first ECB on this probe. 10551 */ 10552 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb; 10553 10554 if (ecb->dte_predicate != NULL) 10555 probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid; 10556 10557 return (prov->dtpv_pops.dtps_enable(prov->dtpv_arg, 10558 probe->dtpr_id, probe->dtpr_arg)); 10559 } else { 10560 /* 10561 * This probe is already active. Swing the last pointer to 10562 * point to the new ECB, and issue a dtrace_sync() to assure 10563 * that all CPUs have seen the change. 10564 */ 10565 ASSERT(probe->dtpr_ecb_last != NULL); 10566 probe->dtpr_ecb_last->dte_next = ecb; 10567 probe->dtpr_ecb_last = ecb; 10568 probe->dtpr_predcache = 0; 10569 10570 dtrace_sync(); 10571 return (0); 10572 } 10573 } 10574 10575 static int 10576 dtrace_ecb_resize(dtrace_ecb_t *ecb) 10577 { 10578 dtrace_action_t *act; 10579 uint32_t curneeded = UINT32_MAX; 10580 uint32_t aggbase = UINT32_MAX; 10581 10582 /* 10583 * If we record anything, we always record the dtrace_rechdr_t. (And 10584 * we always record it first.) 10585 */ 10586 ecb->dte_size = sizeof (dtrace_rechdr_t); 10587 ecb->dte_alignment = sizeof (dtrace_epid_t); 10588 10589 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 10590 dtrace_recdesc_t *rec = &act->dta_rec; 10591 ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1); 10592 10593 ecb->dte_alignment = MAX(ecb->dte_alignment, 10594 rec->dtrd_alignment); 10595 10596 if (DTRACEACT_ISAGG(act->dta_kind)) { 10597 dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act; 10598 10599 ASSERT(rec->dtrd_size != 0); 10600 ASSERT(agg->dtag_first != NULL); 10601 ASSERT(act->dta_prev->dta_intuple); 10602 ASSERT(aggbase != UINT32_MAX); 10603 ASSERT(curneeded != UINT32_MAX); 10604 10605 agg->dtag_base = aggbase; 10606 10607 curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment); 10608 rec->dtrd_offset = curneeded; 10609 if (curneeded + rec->dtrd_size < curneeded) 10610 return (EINVAL); 10611 curneeded += rec->dtrd_size; 10612 ecb->dte_needed = MAX(ecb->dte_needed, curneeded); 10613 10614 aggbase = UINT32_MAX; 10615 curneeded = UINT32_MAX; 10616 } else if (act->dta_intuple) { 10617 if (curneeded == UINT32_MAX) { 10618 /* 10619 * This is the first record in a tuple. Align 10620 * curneeded to be at offset 4 in an 8-byte 10621 * aligned block. 10622 */ 10623 ASSERT(act->dta_prev == NULL || 10624 !act->dta_prev->dta_intuple); 10625 ASSERT3U(aggbase, ==, UINT32_MAX); 10626 curneeded = P2PHASEUP(ecb->dte_size, 10627 sizeof (uint64_t), sizeof (dtrace_aggid_t)); 10628 10629 aggbase = curneeded - sizeof (dtrace_aggid_t); 10630 ASSERT(IS_P2ALIGNED(aggbase, 10631 sizeof (uint64_t))); 10632 } 10633 curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment); 10634 rec->dtrd_offset = curneeded; 10635 if (curneeded + rec->dtrd_size < curneeded) 10636 return (EINVAL); 10637 curneeded += rec->dtrd_size; 10638 } else { 10639 /* tuples must be followed by an aggregation */ 10640 ASSERT(act->dta_prev == NULL || 10641 !act->dta_prev->dta_intuple); 10642 10643 ecb->dte_size = P2ROUNDUP(ecb->dte_size, 10644 rec->dtrd_alignment); 10645 rec->dtrd_offset = ecb->dte_size; 10646 if (ecb->dte_size + rec->dtrd_size < ecb->dte_size) 10647 return (EINVAL); 10648 ecb->dte_size += rec->dtrd_size; 10649 ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size); 10650 } 10651 } 10652 10653 if ((act = ecb->dte_action) != NULL && 10654 !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) && 10655 ecb->dte_size == sizeof (dtrace_rechdr_t)) { 10656 /* 10657 * If the size is still sizeof (dtrace_rechdr_t), then all 10658 * actions store no data; set the size to 0. 10659 */ 10660 ecb->dte_size = 0; 10661 } 10662 10663 ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t)); 10664 ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t))); 10665 ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed, 10666 ecb->dte_needed); 10667 return (0); 10668 } 10669 10670 static dtrace_action_t * 10671 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) 10672 { 10673 dtrace_aggregation_t *agg; 10674 size_t size = sizeof (uint64_t); 10675 int ntuple = desc->dtad_ntuple; 10676 dtrace_action_t *act; 10677 dtrace_recdesc_t *frec; 10678 dtrace_aggid_t aggid; 10679 dtrace_state_t *state = ecb->dte_state; 10680 10681 agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP); 10682 agg->dtag_ecb = ecb; 10683 10684 ASSERT(DTRACEACT_ISAGG(desc->dtad_kind)); 10685 10686 switch (desc->dtad_kind) { 10687 case DTRACEAGG_MIN: 10688 agg->dtag_initial = INT64_MAX; 10689 agg->dtag_aggregate = dtrace_aggregate_min; 10690 break; 10691 10692 case DTRACEAGG_MAX: 10693 agg->dtag_initial = INT64_MIN; 10694 agg->dtag_aggregate = dtrace_aggregate_max; 10695 break; 10696 10697 case DTRACEAGG_COUNT: 10698 agg->dtag_aggregate = dtrace_aggregate_count; 10699 break; 10700 10701 case DTRACEAGG_QUANTIZE: 10702 agg->dtag_aggregate = dtrace_aggregate_quantize; 10703 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) * 10704 sizeof (uint64_t); 10705 break; 10706 10707 case DTRACEAGG_LQUANTIZE: { 10708 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg); 10709 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg); 10710 10711 agg->dtag_initial = desc->dtad_arg; 10712 agg->dtag_aggregate = dtrace_aggregate_lquantize; 10713 10714 if (step == 0 || levels == 0) 10715 goto err; 10716 10717 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t); 10718 break; 10719 } 10720 10721 case DTRACEAGG_LLQUANTIZE: { 10722 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg); 10723 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg); 10724 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg); 10725 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg); 10726 int64_t v; 10727 10728 agg->dtag_initial = desc->dtad_arg; 10729 agg->dtag_aggregate = dtrace_aggregate_llquantize; 10730 10731 if (factor < 2 || low >= high || nsteps < factor) 10732 goto err; 10733 10734 /* 10735 * Now check that the number of steps evenly divides a power 10736 * of the factor. (This assures both integer bucket size and 10737 * linearity within each magnitude.) 10738 */ 10739 for (v = factor; v < nsteps; v *= factor) 10740 continue; 10741 10742 if ((v % nsteps) || (nsteps % factor)) 10743 goto err; 10744 10745 size = (dtrace_aggregate_llquantize_bucket(factor, 10746 low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t); 10747 break; 10748 } 10749 10750 case DTRACEAGG_AVG: 10751 agg->dtag_aggregate = dtrace_aggregate_avg; 10752 size = sizeof (uint64_t) * 2; 10753 break; 10754 10755 case DTRACEAGG_STDDEV: 10756 agg->dtag_aggregate = dtrace_aggregate_stddev; 10757 size = sizeof (uint64_t) * 4; 10758 break; 10759 10760 case DTRACEAGG_SUM: 10761 agg->dtag_aggregate = dtrace_aggregate_sum; 10762 break; 10763 10764 default: 10765 goto err; 10766 } 10767 10768 agg->dtag_action.dta_rec.dtrd_size = size; 10769 10770 if (ntuple == 0) 10771 goto err; 10772 10773 /* 10774 * We must make sure that we have enough actions for the n-tuple. 10775 */ 10776 for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) { 10777 if (DTRACEACT_ISAGG(act->dta_kind)) 10778 break; 10779 10780 if (--ntuple == 0) { 10781 /* 10782 * This is the action with which our n-tuple begins. 10783 */ 10784 agg->dtag_first = act; 10785 goto success; 10786 } 10787 } 10788 10789 /* 10790 * This n-tuple is short by ntuple elements. Return failure. 10791 */ 10792 ASSERT(ntuple != 0); 10793 err: 10794 kmem_free(agg, sizeof (dtrace_aggregation_t)); 10795 return (NULL); 10796 10797 success: 10798 /* 10799 * If the last action in the tuple has a size of zero, it's actually 10800 * an expression argument for the aggregating action. 10801 */ 10802 ASSERT(ecb->dte_action_last != NULL); 10803 act = ecb->dte_action_last; 10804 10805 if (act->dta_kind == DTRACEACT_DIFEXPR) { 10806 ASSERT(act->dta_difo != NULL); 10807 10808 if (act->dta_difo->dtdo_rtype.dtdt_size == 0) 10809 agg->dtag_hasarg = 1; 10810 } 10811 10812 /* 10813 * We need to allocate an id for this aggregation. 10814 */ 10815 aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1, 10816 VM_BESTFIT | VM_SLEEP); 10817 10818 if (aggid - 1 >= state->dts_naggregations) { 10819 dtrace_aggregation_t **oaggs = state->dts_aggregations; 10820 dtrace_aggregation_t **aggs; 10821 int naggs = state->dts_naggregations << 1; 10822 int onaggs = state->dts_naggregations; 10823 10824 ASSERT(aggid == state->dts_naggregations + 1); 10825 10826 if (naggs == 0) { 10827 ASSERT(oaggs == NULL); 10828 naggs = 1; 10829 } 10830 10831 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP); 10832 10833 if (oaggs != NULL) { 10834 bcopy(oaggs, aggs, onaggs * sizeof (*aggs)); 10835 kmem_free(oaggs, onaggs * sizeof (*aggs)); 10836 } 10837 10838 state->dts_aggregations = aggs; 10839 state->dts_naggregations = naggs; 10840 } 10841 10842 ASSERT(state->dts_aggregations[aggid - 1] == NULL); 10843 state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg; 10844 10845 frec = &agg->dtag_first->dta_rec; 10846 if (frec->dtrd_alignment < sizeof (dtrace_aggid_t)) 10847 frec->dtrd_alignment = sizeof (dtrace_aggid_t); 10848 10849 for (act = agg->dtag_first; act != NULL; act = act->dta_next) { 10850 ASSERT(!act->dta_intuple); 10851 act->dta_intuple = 1; 10852 } 10853 10854 return (&agg->dtag_action); 10855 } 10856 10857 static void 10858 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act) 10859 { 10860 dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act; 10861 dtrace_state_t *state = ecb->dte_state; 10862 dtrace_aggid_t aggid = agg->dtag_id; 10863 10864 ASSERT(DTRACEACT_ISAGG(act->dta_kind)); 10865 vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1); 10866 10867 ASSERT(state->dts_aggregations[aggid - 1] == agg); 10868 state->dts_aggregations[aggid - 1] = NULL; 10869 10870 kmem_free(agg, sizeof (dtrace_aggregation_t)); 10871 } 10872 10873 static int 10874 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) 10875 { 10876 dtrace_action_t *action, *last; 10877 dtrace_difo_t *dp = desc->dtad_difo; 10878 uint32_t size = 0, align = sizeof (uint8_t), mask; 10879 uint16_t format = 0; 10880 dtrace_recdesc_t *rec; 10881 dtrace_state_t *state = ecb->dte_state; 10882 dtrace_optval_t *opt = state->dts_options, nframes, strsize; 10883 uint64_t arg = desc->dtad_arg; 10884 10885 ASSERT(MUTEX_HELD(&dtrace_lock)); 10886 ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1); 10887 10888 if (DTRACEACT_ISAGG(desc->dtad_kind)) { 10889 /* 10890 * If this is an aggregating action, there must be neither 10891 * a speculate nor a commit on the action chain. 10892 */ 10893 dtrace_action_t *act; 10894 10895 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 10896 if (act->dta_kind == DTRACEACT_COMMIT) 10897 return (EINVAL); 10898 10899 if (act->dta_kind == DTRACEACT_SPECULATE) 10900 return (EINVAL); 10901 } 10902 10903 action = dtrace_ecb_aggregation_create(ecb, desc); 10904 10905 if (action == NULL) 10906 return (EINVAL); 10907 } else { 10908 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) || 10909 (desc->dtad_kind == DTRACEACT_DIFEXPR && 10910 dp != NULL && dp->dtdo_destructive)) { 10911 state->dts_destructive = 1; 10912 } 10913 10914 switch (desc->dtad_kind) { 10915 case DTRACEACT_PRINTF: 10916 case DTRACEACT_PRINTA: 10917 case DTRACEACT_SYSTEM: 10918 case DTRACEACT_FREOPEN: 10919 case DTRACEACT_DIFEXPR: 10920 /* 10921 * We know that our arg is a string -- turn it into a 10922 * format. 10923 */ 10924 if (arg == NULL) { 10925 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA || 10926 desc->dtad_kind == DTRACEACT_DIFEXPR); 10927 format = 0; 10928 } else { 10929 ASSERT(arg != NULL); 10930 ASSERT(arg > KERNELBASE); 10931 format = dtrace_format_add(state, 10932 (char *)(uintptr_t)arg); 10933 } 10934 10935 /*FALLTHROUGH*/ 10936 case DTRACEACT_LIBACT: 10937 case DTRACEACT_TRACEMEM: 10938 case DTRACEACT_TRACEMEM_DYNSIZE: 10939 if (dp == NULL) 10940 return (EINVAL); 10941 10942 if ((size = dp->dtdo_rtype.dtdt_size) != 0) 10943 break; 10944 10945 if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) { 10946 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 10947 return (EINVAL); 10948 10949 size = opt[DTRACEOPT_STRSIZE]; 10950 } 10951 10952 break; 10953 10954 case DTRACEACT_STACK: 10955 if ((nframes = arg) == 0) { 10956 nframes = opt[DTRACEOPT_STACKFRAMES]; 10957 ASSERT(nframes > 0); 10958 arg = nframes; 10959 } 10960 10961 size = nframes * sizeof (pc_t); 10962 break; 10963 10964 case DTRACEACT_JSTACK: 10965 if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0) 10966 strsize = opt[DTRACEOPT_JSTACKSTRSIZE]; 10967 10968 if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) 10969 nframes = opt[DTRACEOPT_JSTACKFRAMES]; 10970 10971 arg = DTRACE_USTACK_ARG(nframes, strsize); 10972 10973 /*FALLTHROUGH*/ 10974 case DTRACEACT_USTACK: 10975 if (desc->dtad_kind != DTRACEACT_JSTACK && 10976 (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) { 10977 strsize = DTRACE_USTACK_STRSIZE(arg); 10978 nframes = opt[DTRACEOPT_USTACKFRAMES]; 10979 ASSERT(nframes > 0); 10980 arg = DTRACE_USTACK_ARG(nframes, strsize); 10981 } 10982 10983 /* 10984 * Save a slot for the pid. 10985 */ 10986 size = (nframes + 1) * sizeof (uint64_t); 10987 size += DTRACE_USTACK_STRSIZE(arg); 10988 size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t))); 10989 10990 break; 10991 10992 case DTRACEACT_SYM: 10993 case DTRACEACT_MOD: 10994 if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) != 10995 sizeof (uint64_t)) || 10996 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 10997 return (EINVAL); 10998 break; 10999 11000 case DTRACEACT_USYM: 11001 case DTRACEACT_UMOD: 11002 case DTRACEACT_UADDR: 11003 if (dp == NULL || 11004 (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) || 11005 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 11006 return (EINVAL); 11007 11008 /* 11009 * We have a slot for the pid, plus a slot for the 11010 * argument. To keep things simple (aligned with 11011 * bitness-neutral sizing), we store each as a 64-bit 11012 * quantity. 11013 */ 11014 size = 2 * sizeof (uint64_t); 11015 break; 11016 11017 case DTRACEACT_STOP: 11018 case DTRACEACT_BREAKPOINT: 11019 case DTRACEACT_PANIC: 11020 break; 11021 11022 case DTRACEACT_CHILL: 11023 case DTRACEACT_DISCARD: 11024 case DTRACEACT_RAISE: 11025 if (dp == NULL) 11026 return (EINVAL); 11027 break; 11028 11029 case DTRACEACT_EXIT: 11030 if (dp == NULL || 11031 (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) || 11032 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 11033 return (EINVAL); 11034 break; 11035 11036 case DTRACEACT_SPECULATE: 11037 if (ecb->dte_size > sizeof (dtrace_rechdr_t)) 11038 return (EINVAL); 11039 11040 if (dp == NULL) 11041 return (EINVAL); 11042 11043 state->dts_speculates = 1; 11044 break; 11045 11046 case DTRACEACT_COMMIT: { 11047 dtrace_action_t *act = ecb->dte_action; 11048 11049 for (; act != NULL; act = act->dta_next) { 11050 if (act->dta_kind == DTRACEACT_COMMIT) 11051 return (EINVAL); 11052 } 11053 11054 if (dp == NULL) 11055 return (EINVAL); 11056 break; 11057 } 11058 11059 default: 11060 return (EINVAL); 11061 } 11062 11063 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) { 11064 /* 11065 * If this is a data-storing action or a speculate, 11066 * we must be sure that there isn't a commit on the 11067 * action chain. 11068 */ 11069 dtrace_action_t *act = ecb->dte_action; 11070 11071 for (; act != NULL; act = act->dta_next) { 11072 if (act->dta_kind == DTRACEACT_COMMIT) 11073 return (EINVAL); 11074 } 11075 } 11076 11077 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP); 11078 action->dta_rec.dtrd_size = size; 11079 } 11080 11081 action->dta_refcnt = 1; 11082 rec = &action->dta_rec; 11083 size = rec->dtrd_size; 11084 11085 for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) { 11086 if (!(size & mask)) { 11087 align = mask + 1; 11088 break; 11089 } 11090 } 11091 11092 action->dta_kind = desc->dtad_kind; 11093 11094 if ((action->dta_difo = dp) != NULL) 11095 dtrace_difo_hold(dp); 11096 11097 rec->dtrd_action = action->dta_kind; 11098 rec->dtrd_arg = arg; 11099 rec->dtrd_uarg = desc->dtad_uarg; 11100 rec->dtrd_alignment = (uint16_t)align; 11101 rec->dtrd_format = format; 11102 11103 if ((last = ecb->dte_action_last) != NULL) { 11104 ASSERT(ecb->dte_action != NULL); 11105 action->dta_prev = last; 11106 last->dta_next = action; 11107 } else { 11108 ASSERT(ecb->dte_action == NULL); 11109 ecb->dte_action = action; 11110 } 11111 11112 ecb->dte_action_last = action; 11113 11114 return (0); 11115 } 11116 11117 static void 11118 dtrace_ecb_action_remove(dtrace_ecb_t *ecb) 11119 { 11120 dtrace_action_t *act = ecb->dte_action, *next; 11121 dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate; 11122 dtrace_difo_t *dp; 11123 uint16_t format; 11124 11125 if (act != NULL && act->dta_refcnt > 1) { 11126 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1); 11127 act->dta_refcnt--; 11128 } else { 11129 for (; act != NULL; act = next) { 11130 next = act->dta_next; 11131 ASSERT(next != NULL || act == ecb->dte_action_last); 11132 ASSERT(act->dta_refcnt == 1); 11133 11134 if ((format = act->dta_rec.dtrd_format) != 0) 11135 dtrace_format_remove(ecb->dte_state, format); 11136 11137 if ((dp = act->dta_difo) != NULL) 11138 dtrace_difo_release(dp, vstate); 11139 11140 if (DTRACEACT_ISAGG(act->dta_kind)) { 11141 dtrace_ecb_aggregation_destroy(ecb, act); 11142 } else { 11143 kmem_free(act, sizeof (dtrace_action_t)); 11144 } 11145 } 11146 } 11147 11148 ecb->dte_action = NULL; 11149 ecb->dte_action_last = NULL; 11150 ecb->dte_size = 0; 11151 } 11152 11153 static void 11154 dtrace_ecb_disable(dtrace_ecb_t *ecb) 11155 { 11156 /* 11157 * We disable the ECB by removing it from its probe. 11158 */ 11159 dtrace_ecb_t *pecb, *prev = NULL; 11160 dtrace_probe_t *probe = ecb->dte_probe; 11161 11162 ASSERT(MUTEX_HELD(&dtrace_lock)); 11163 11164 if (probe == NULL) { 11165 /* 11166 * This is the NULL probe; there is nothing to disable. 11167 */ 11168 return; 11169 } 11170 11171 for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) { 11172 if (pecb == ecb) 11173 break; 11174 prev = pecb; 11175 } 11176 11177 ASSERT(pecb != NULL); 11178 11179 if (prev == NULL) { 11180 probe->dtpr_ecb = ecb->dte_next; 11181 } else { 11182 prev->dte_next = ecb->dte_next; 11183 } 11184 11185 if (ecb == probe->dtpr_ecb_last) { 11186 ASSERT(ecb->dte_next == NULL); 11187 probe->dtpr_ecb_last = prev; 11188 } 11189 11190 /* 11191 * The ECB has been disconnected from the probe; now sync to assure 11192 * that all CPUs have seen the change before returning. 11193 */ 11194 dtrace_sync(); 11195 11196 if (probe->dtpr_ecb == NULL) { 11197 /* 11198 * That was the last ECB on the probe; clear the predicate 11199 * cache ID for the probe, disable it and sync one more time 11200 * to assure that we'll never hit it again. 11201 */ 11202 dtrace_provider_t *prov = probe->dtpr_provider; 11203 11204 ASSERT(ecb->dte_next == NULL); 11205 ASSERT(probe->dtpr_ecb_last == NULL); 11206 probe->dtpr_predcache = DTRACE_CACHEIDNONE; 11207 prov->dtpv_pops.dtps_disable(prov->dtpv_arg, 11208 probe->dtpr_id, probe->dtpr_arg); 11209 dtrace_sync(); 11210 } else { 11211 /* 11212 * There is at least one ECB remaining on the probe. If there 11213 * is _exactly_ one, set the probe's predicate cache ID to be 11214 * the predicate cache ID of the remaining ECB. 11215 */ 11216 ASSERT(probe->dtpr_ecb_last != NULL); 11217 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE); 11218 11219 if (probe->dtpr_ecb == probe->dtpr_ecb_last) { 11220 dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate; 11221 11222 ASSERT(probe->dtpr_ecb->dte_next == NULL); 11223 11224 if (p != NULL) 11225 probe->dtpr_predcache = p->dtp_cacheid; 11226 } 11227 11228 ecb->dte_next = NULL; 11229 } 11230 } 11231 11232 static void 11233 dtrace_ecb_destroy(dtrace_ecb_t *ecb) 11234 { 11235 dtrace_state_t *state = ecb->dte_state; 11236 dtrace_vstate_t *vstate = &state->dts_vstate; 11237 dtrace_predicate_t *pred; 11238 dtrace_epid_t epid = ecb->dte_epid; 11239 11240 ASSERT(MUTEX_HELD(&dtrace_lock)); 11241 ASSERT(ecb->dte_next == NULL); 11242 ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb); 11243 11244 if ((pred = ecb->dte_predicate) != NULL) 11245 dtrace_predicate_release(pred, vstate); 11246 11247 dtrace_ecb_action_remove(ecb); 11248 11249 ASSERT(state->dts_ecbs[epid - 1] == ecb); 11250 state->dts_ecbs[epid - 1] = NULL; 11251 11252 kmem_free(ecb, sizeof (dtrace_ecb_t)); 11253 } 11254 11255 static dtrace_ecb_t * 11256 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe, 11257 dtrace_enabling_t *enab) 11258 { 11259 dtrace_ecb_t *ecb; 11260 dtrace_predicate_t *pred; 11261 dtrace_actdesc_t *act; 11262 dtrace_provider_t *prov; 11263 dtrace_ecbdesc_t *desc = enab->dten_current; 11264 11265 ASSERT(MUTEX_HELD(&dtrace_lock)); 11266 ASSERT(state != NULL); 11267 11268 ecb = dtrace_ecb_add(state, probe); 11269 ecb->dte_uarg = desc->dted_uarg; 11270 11271 if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) { 11272 dtrace_predicate_hold(pred); 11273 ecb->dte_predicate = pred; 11274 } 11275 11276 if (probe != NULL) { 11277 /* 11278 * If the provider shows more leg than the consumer is old 11279 * enough to see, we need to enable the appropriate implicit 11280 * predicate bits to prevent the ecb from activating at 11281 * revealing times. 11282 * 11283 * Providers specifying DTRACE_PRIV_USER at register time 11284 * are stating that they need the /proc-style privilege 11285 * model to be enforced, and this is what DTRACE_COND_OWNER 11286 * and DTRACE_COND_ZONEOWNER will then do at probe time. 11287 */ 11288 prov = probe->dtpr_provider; 11289 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) && 11290 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER)) 11291 ecb->dte_cond |= DTRACE_COND_OWNER; 11292 11293 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) && 11294 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER)) 11295 ecb->dte_cond |= DTRACE_COND_ZONEOWNER; 11296 11297 /* 11298 * If the provider shows us kernel innards and the user 11299 * is lacking sufficient privilege, enable the 11300 * DTRACE_COND_USERMODE implicit predicate. 11301 */ 11302 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) && 11303 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL)) 11304 ecb->dte_cond |= DTRACE_COND_USERMODE; 11305 } 11306 11307 if (dtrace_ecb_create_cache != NULL) { 11308 /* 11309 * If we have a cached ecb, we'll use its action list instead 11310 * of creating our own (saving both time and space). 11311 */ 11312 dtrace_ecb_t *cached = dtrace_ecb_create_cache; 11313 dtrace_action_t *act = cached->dte_action; 11314 11315 if (act != NULL) { 11316 ASSERT(act->dta_refcnt > 0); 11317 act->dta_refcnt++; 11318 ecb->dte_action = act; 11319 ecb->dte_action_last = cached->dte_action_last; 11320 ecb->dte_needed = cached->dte_needed; 11321 ecb->dte_size = cached->dte_size; 11322 ecb->dte_alignment = cached->dte_alignment; 11323 } 11324 11325 return (ecb); 11326 } 11327 11328 for (act = desc->dted_action; act != NULL; act = act->dtad_next) { 11329 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) { 11330 dtrace_ecb_destroy(ecb); 11331 return (NULL); 11332 } 11333 } 11334 11335 if ((enab->dten_error = dtrace_ecb_resize(ecb)) != 0) { 11336 dtrace_ecb_destroy(ecb); 11337 return (NULL); 11338 } 11339 11340 return (dtrace_ecb_create_cache = ecb); 11341 } 11342 11343 static int 11344 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg) 11345 { 11346 dtrace_ecb_t *ecb; 11347 dtrace_enabling_t *enab = arg; 11348 dtrace_state_t *state = enab->dten_vstate->dtvs_state; 11349 11350 ASSERT(state != NULL); 11351 11352 if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) { 11353 /* 11354 * This probe was created in a generation for which this 11355 * enabling has previously created ECBs; we don't want to 11356 * enable it again, so just kick out. 11357 */ 11358 return (DTRACE_MATCH_NEXT); 11359 } 11360 11361 if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL) 11362 return (DTRACE_MATCH_DONE); 11363 11364 if (dtrace_ecb_enable(ecb) < 0) 11365 return (DTRACE_MATCH_FAIL); 11366 11367 return (DTRACE_MATCH_NEXT); 11368 } 11369 11370 static dtrace_ecb_t * 11371 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id) 11372 { 11373 dtrace_ecb_t *ecb; 11374 11375 ASSERT(MUTEX_HELD(&dtrace_lock)); 11376 11377 if (id == 0 || id > state->dts_necbs) 11378 return (NULL); 11379 11380 ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL); 11381 ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id); 11382 11383 return (state->dts_ecbs[id - 1]); 11384 } 11385 11386 static dtrace_aggregation_t * 11387 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id) 11388 { 11389 dtrace_aggregation_t *agg; 11390 11391 ASSERT(MUTEX_HELD(&dtrace_lock)); 11392 11393 if (id == 0 || id > state->dts_naggregations) 11394 return (NULL); 11395 11396 ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL); 11397 ASSERT((agg = state->dts_aggregations[id - 1]) == NULL || 11398 agg->dtag_id == id); 11399 11400 return (state->dts_aggregations[id - 1]); 11401 } 11402 11403 /* 11404 * DTrace Buffer Functions 11405 * 11406 * The following functions manipulate DTrace buffers. Most of these functions 11407 * are called in the context of establishing or processing consumer state; 11408 * exceptions are explicitly noted. 11409 */ 11410 11411 /* 11412 * Note: called from cross call context. This function switches the two 11413 * buffers on a given CPU. The atomicity of this operation is assured by 11414 * disabling interrupts while the actual switch takes place; the disabling of 11415 * interrupts serializes the execution with any execution of dtrace_probe() on 11416 * the same CPU. 11417 */ 11418 static void 11419 dtrace_buffer_switch(dtrace_buffer_t *buf) 11420 { 11421 caddr_t tomax = buf->dtb_tomax; 11422 caddr_t xamot = buf->dtb_xamot; 11423 dtrace_icookie_t cookie; 11424 hrtime_t now; 11425 11426 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); 11427 ASSERT(!(buf->dtb_flags & DTRACEBUF_RING)); 11428 11429 cookie = dtrace_interrupt_disable(); 11430 now = dtrace_gethrtime(); 11431 buf->dtb_tomax = xamot; 11432 buf->dtb_xamot = tomax; 11433 buf->dtb_xamot_drops = buf->dtb_drops; 11434 buf->dtb_xamot_offset = buf->dtb_offset; 11435 buf->dtb_xamot_errors = buf->dtb_errors; 11436 buf->dtb_xamot_flags = buf->dtb_flags; 11437 buf->dtb_offset = 0; 11438 buf->dtb_drops = 0; 11439 buf->dtb_errors = 0; 11440 buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED); 11441 buf->dtb_interval = now - buf->dtb_switched; 11442 buf->dtb_switched = now; 11443 dtrace_interrupt_enable(cookie); 11444 } 11445 11446 /* 11447 * Note: called from cross call context. This function activates a buffer 11448 * on a CPU. As with dtrace_buffer_switch(), the atomicity of the operation 11449 * is guaranteed by the disabling of interrupts. 11450 */ 11451 static void 11452 dtrace_buffer_activate(dtrace_state_t *state) 11453 { 11454 dtrace_buffer_t *buf; 11455 dtrace_icookie_t cookie = dtrace_interrupt_disable(); 11456 11457 buf = &state->dts_buffer[CPU->cpu_id]; 11458 11459 if (buf->dtb_tomax != NULL) { 11460 /* 11461 * We might like to assert that the buffer is marked inactive, 11462 * but this isn't necessarily true: the buffer for the CPU 11463 * that processes the BEGIN probe has its buffer activated 11464 * manually. In this case, we take the (harmless) action 11465 * re-clearing the bit INACTIVE bit. 11466 */ 11467 buf->dtb_flags &= ~DTRACEBUF_INACTIVE; 11468 } 11469 11470 dtrace_interrupt_enable(cookie); 11471 } 11472 11473 static int 11474 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags, 11475 processorid_t cpu, int *factor) 11476 { 11477 cpu_t *cp; 11478 dtrace_buffer_t *buf; 11479 int allocated = 0, desired = 0; 11480 11481 ASSERT(MUTEX_HELD(&cpu_lock)); 11482 ASSERT(MUTEX_HELD(&dtrace_lock)); 11483 11484 *factor = 1; 11485 11486 if (size > dtrace_nonroot_maxsize && 11487 !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE)) 11488 return (EFBIG); 11489 11490 cp = cpu_list; 11491 11492 do { 11493 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id) 11494 continue; 11495 11496 buf = &bufs[cp->cpu_id]; 11497 11498 /* 11499 * If there is already a buffer allocated for this CPU, it 11500 * is only possible that this is a DR event. In this case, 11501 * the buffer size must match our specified size. 11502 */ 11503 if (buf->dtb_tomax != NULL) { 11504 ASSERT(buf->dtb_size == size); 11505 continue; 11506 } 11507 11508 ASSERT(buf->dtb_xamot == NULL); 11509 11510 if ((buf->dtb_tomax = kmem_zalloc(size, 11511 KM_NOSLEEP | KM_NORMALPRI)) == NULL) 11512 goto err; 11513 11514 buf->dtb_size = size; 11515 buf->dtb_flags = flags; 11516 buf->dtb_offset = 0; 11517 buf->dtb_drops = 0; 11518 11519 if (flags & DTRACEBUF_NOSWITCH) 11520 continue; 11521 11522 if ((buf->dtb_xamot = kmem_zalloc(size, 11523 KM_NOSLEEP | KM_NORMALPRI)) == NULL) 11524 goto err; 11525 } while ((cp = cp->cpu_next) != cpu_list); 11526 11527 return (0); 11528 11529 err: 11530 cp = cpu_list; 11531 11532 do { 11533 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id) 11534 continue; 11535 11536 buf = &bufs[cp->cpu_id]; 11537 desired += 2; 11538 11539 if (buf->dtb_xamot != NULL) { 11540 ASSERT(buf->dtb_tomax != NULL); 11541 ASSERT(buf->dtb_size == size); 11542 kmem_free(buf->dtb_xamot, size); 11543 allocated++; 11544 } 11545 11546 if (buf->dtb_tomax != NULL) { 11547 ASSERT(buf->dtb_size == size); 11548 kmem_free(buf->dtb_tomax, size); 11549 allocated++; 11550 } 11551 11552 buf->dtb_tomax = NULL; 11553 buf->dtb_xamot = NULL; 11554 buf->dtb_size = 0; 11555 } while ((cp = cp->cpu_next) != cpu_list); 11556 11557 *factor = desired / (allocated > 0 ? allocated : 1); 11558 11559 return (ENOMEM); 11560 } 11561 11562 /* 11563 * Note: called from probe context. This function just increments the drop 11564 * count on a buffer. It has been made a function to allow for the 11565 * possibility of understanding the source of mysterious drop counts. (A 11566 * problem for which one may be particularly disappointed that DTrace cannot 11567 * be used to understand DTrace.) 11568 */ 11569 static void 11570 dtrace_buffer_drop(dtrace_buffer_t *buf) 11571 { 11572 buf->dtb_drops++; 11573 } 11574 11575 /* 11576 * Note: called from probe context. This function is called to reserve space 11577 * in a buffer. If mstate is non-NULL, sets the scratch base and size in the 11578 * mstate. Returns the new offset in the buffer, or a negative value if an 11579 * error has occurred. 11580 */ 11581 static intptr_t 11582 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align, 11583 dtrace_state_t *state, dtrace_mstate_t *mstate) 11584 { 11585 intptr_t offs = buf->dtb_offset, soffs; 11586 intptr_t woffs; 11587 caddr_t tomax; 11588 size_t total; 11589 11590 if (buf->dtb_flags & DTRACEBUF_INACTIVE) 11591 return (-1); 11592 11593 if ((tomax = buf->dtb_tomax) == NULL) { 11594 dtrace_buffer_drop(buf); 11595 return (-1); 11596 } 11597 11598 if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) { 11599 while (offs & (align - 1)) { 11600 /* 11601 * Assert that our alignment is off by a number which 11602 * is itself sizeof (uint32_t) aligned. 11603 */ 11604 ASSERT(!((align - (offs & (align - 1))) & 11605 (sizeof (uint32_t) - 1))); 11606 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE); 11607 offs += sizeof (uint32_t); 11608 } 11609 11610 if ((soffs = offs + needed) > buf->dtb_size) { 11611 dtrace_buffer_drop(buf); 11612 return (-1); 11613 } 11614 11615 if (mstate == NULL) 11616 return (offs); 11617 11618 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs; 11619 mstate->dtms_scratch_size = buf->dtb_size - soffs; 11620 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base; 11621 11622 return (offs); 11623 } 11624 11625 if (buf->dtb_flags & DTRACEBUF_FILL) { 11626 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN && 11627 (buf->dtb_flags & DTRACEBUF_FULL)) 11628 return (-1); 11629 goto out; 11630 } 11631 11632 total = needed + (offs & (align - 1)); 11633 11634 /* 11635 * For a ring buffer, life is quite a bit more complicated. Before 11636 * we can store any padding, we need to adjust our wrapping offset. 11637 * (If we've never before wrapped or we're not about to, no adjustment 11638 * is required.) 11639 */ 11640 if ((buf->dtb_flags & DTRACEBUF_WRAPPED) || 11641 offs + total > buf->dtb_size) { 11642 woffs = buf->dtb_xamot_offset; 11643 11644 if (offs + total > buf->dtb_size) { 11645 /* 11646 * We can't fit in the end of the buffer. First, a 11647 * sanity check that we can fit in the buffer at all. 11648 */ 11649 if (total > buf->dtb_size) { 11650 dtrace_buffer_drop(buf); 11651 return (-1); 11652 } 11653 11654 /* 11655 * We're going to be storing at the top of the buffer, 11656 * so now we need to deal with the wrapped offset. We 11657 * only reset our wrapped offset to 0 if it is 11658 * currently greater than the current offset. If it 11659 * is less than the current offset, it is because a 11660 * previous allocation induced a wrap -- but the 11661 * allocation didn't subsequently take the space due 11662 * to an error or false predicate evaluation. In this 11663 * case, we'll just leave the wrapped offset alone: if 11664 * the wrapped offset hasn't been advanced far enough 11665 * for this allocation, it will be adjusted in the 11666 * lower loop. 11667 */ 11668 if (buf->dtb_flags & DTRACEBUF_WRAPPED) { 11669 if (woffs >= offs) 11670 woffs = 0; 11671 } else { 11672 woffs = 0; 11673 } 11674 11675 /* 11676 * Now we know that we're going to be storing to the 11677 * top of the buffer and that there is room for us 11678 * there. We need to clear the buffer from the current 11679 * offset to the end (there may be old gunk there). 11680 */ 11681 while (offs < buf->dtb_size) 11682 tomax[offs++] = 0; 11683 11684 /* 11685 * We need to set our offset to zero. And because we 11686 * are wrapping, we need to set the bit indicating as 11687 * much. We can also adjust our needed space back 11688 * down to the space required by the ECB -- we know 11689 * that the top of the buffer is aligned. 11690 */ 11691 offs = 0; 11692 total = needed; 11693 buf->dtb_flags |= DTRACEBUF_WRAPPED; 11694 } else { 11695 /* 11696 * There is room for us in the buffer, so we simply 11697 * need to check the wrapped offset. 11698 */ 11699 if (woffs < offs) { 11700 /* 11701 * The wrapped offset is less than the offset. 11702 * This can happen if we allocated buffer space 11703 * that induced a wrap, but then we didn't 11704 * subsequently take the space due to an error 11705 * or false predicate evaluation. This is 11706 * okay; we know that _this_ allocation isn't 11707 * going to induce a wrap. We still can't 11708 * reset the wrapped offset to be zero, 11709 * however: the space may have been trashed in 11710 * the previous failed probe attempt. But at 11711 * least the wrapped offset doesn't need to 11712 * be adjusted at all... 11713 */ 11714 goto out; 11715 } 11716 } 11717 11718 while (offs + total > woffs) { 11719 dtrace_epid_t epid = *(uint32_t *)(tomax + woffs); 11720 size_t size; 11721 11722 if (epid == DTRACE_EPIDNONE) { 11723 size = sizeof (uint32_t); 11724 } else { 11725 ASSERT3U(epid, <=, state->dts_necbs); 11726 ASSERT(state->dts_ecbs[epid - 1] != NULL); 11727 11728 size = state->dts_ecbs[epid - 1]->dte_size; 11729 } 11730 11731 ASSERT(woffs + size <= buf->dtb_size); 11732 ASSERT(size != 0); 11733 11734 if (woffs + size == buf->dtb_size) { 11735 /* 11736 * We've reached the end of the buffer; we want 11737 * to set the wrapped offset to 0 and break 11738 * out. However, if the offs is 0, then we're 11739 * in a strange edge-condition: the amount of 11740 * space that we want to reserve plus the size 11741 * of the record that we're overwriting is 11742 * greater than the size of the buffer. This 11743 * is problematic because if we reserve the 11744 * space but subsequently don't consume it (due 11745 * to a failed predicate or error) the wrapped 11746 * offset will be 0 -- yet the EPID at offset 0 11747 * will not be committed. This situation is 11748 * relatively easy to deal with: if we're in 11749 * this case, the buffer is indistinguishable 11750 * from one that hasn't wrapped; we need only 11751 * finish the job by clearing the wrapped bit, 11752 * explicitly setting the offset to be 0, and 11753 * zero'ing out the old data in the buffer. 11754 */ 11755 if (offs == 0) { 11756 buf->dtb_flags &= ~DTRACEBUF_WRAPPED; 11757 buf->dtb_offset = 0; 11758 woffs = total; 11759 11760 while (woffs < buf->dtb_size) 11761 tomax[woffs++] = 0; 11762 } 11763 11764 woffs = 0; 11765 break; 11766 } 11767 11768 woffs += size; 11769 } 11770 11771 /* 11772 * We have a wrapped offset. It may be that the wrapped offset 11773 * has become zero -- that's okay. 11774 */ 11775 buf->dtb_xamot_offset = woffs; 11776 } 11777 11778 out: 11779 /* 11780 * Now we can plow the buffer with any necessary padding. 11781 */ 11782 while (offs & (align - 1)) { 11783 /* 11784 * Assert that our alignment is off by a number which 11785 * is itself sizeof (uint32_t) aligned. 11786 */ 11787 ASSERT(!((align - (offs & (align - 1))) & 11788 (sizeof (uint32_t) - 1))); 11789 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE); 11790 offs += sizeof (uint32_t); 11791 } 11792 11793 if (buf->dtb_flags & DTRACEBUF_FILL) { 11794 if (offs + needed > buf->dtb_size - state->dts_reserve) { 11795 buf->dtb_flags |= DTRACEBUF_FULL; 11796 return (-1); 11797 } 11798 } 11799 11800 if (mstate == NULL) 11801 return (offs); 11802 11803 /* 11804 * For ring buffers and fill buffers, the scratch space is always 11805 * the inactive buffer. 11806 */ 11807 mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot; 11808 mstate->dtms_scratch_size = buf->dtb_size; 11809 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base; 11810 11811 return (offs); 11812 } 11813 11814 static void 11815 dtrace_buffer_polish(dtrace_buffer_t *buf) 11816 { 11817 ASSERT(buf->dtb_flags & DTRACEBUF_RING); 11818 ASSERT(MUTEX_HELD(&dtrace_lock)); 11819 11820 if (!(buf->dtb_flags & DTRACEBUF_WRAPPED)) 11821 return; 11822 11823 /* 11824 * We need to polish the ring buffer. There are three cases: 11825 * 11826 * - The first (and presumably most common) is that there is no gap 11827 * between the buffer offset and the wrapped offset. In this case, 11828 * there is nothing in the buffer that isn't valid data; we can 11829 * mark the buffer as polished and return. 11830 * 11831 * - The second (less common than the first but still more common 11832 * than the third) is that there is a gap between the buffer offset 11833 * and the wrapped offset, and the wrapped offset is larger than the 11834 * buffer offset. This can happen because of an alignment issue, or 11835 * can happen because of a call to dtrace_buffer_reserve() that 11836 * didn't subsequently consume the buffer space. In this case, 11837 * we need to zero the data from the buffer offset to the wrapped 11838 * offset. 11839 * 11840 * - The third (and least common) is that there is a gap between the 11841 * buffer offset and the wrapped offset, but the wrapped offset is 11842 * _less_ than the buffer offset. This can only happen because a 11843 * call to dtrace_buffer_reserve() induced a wrap, but the space 11844 * was not subsequently consumed. In this case, we need to zero the 11845 * space from the offset to the end of the buffer _and_ from the 11846 * top of the buffer to the wrapped offset. 11847 */ 11848 if (buf->dtb_offset < buf->dtb_xamot_offset) { 11849 bzero(buf->dtb_tomax + buf->dtb_offset, 11850 buf->dtb_xamot_offset - buf->dtb_offset); 11851 } 11852 11853 if (buf->dtb_offset > buf->dtb_xamot_offset) { 11854 bzero(buf->dtb_tomax + buf->dtb_offset, 11855 buf->dtb_size - buf->dtb_offset); 11856 bzero(buf->dtb_tomax, buf->dtb_xamot_offset); 11857 } 11858 } 11859 11860 /* 11861 * This routine determines if data generated at the specified time has likely 11862 * been entirely consumed at user-level. This routine is called to determine 11863 * if an ECB on a defunct probe (but for an active enabling) can be safely 11864 * disabled and destroyed. 11865 */ 11866 static int 11867 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when) 11868 { 11869 int i; 11870 11871 for (i = 0; i < NCPU; i++) { 11872 dtrace_buffer_t *buf = &bufs[i]; 11873 11874 if (buf->dtb_size == 0) 11875 continue; 11876 11877 if (buf->dtb_flags & DTRACEBUF_RING) 11878 return (0); 11879 11880 if (!buf->dtb_switched && buf->dtb_offset != 0) 11881 return (0); 11882 11883 if (buf->dtb_switched - buf->dtb_interval < when) 11884 return (0); 11885 } 11886 11887 return (1); 11888 } 11889 11890 static void 11891 dtrace_buffer_free(dtrace_buffer_t *bufs) 11892 { 11893 int i; 11894 11895 for (i = 0; i < NCPU; i++) { 11896 dtrace_buffer_t *buf = &bufs[i]; 11897 11898 if (buf->dtb_tomax == NULL) { 11899 ASSERT(buf->dtb_xamot == NULL); 11900 ASSERT(buf->dtb_size == 0); 11901 continue; 11902 } 11903 11904 if (buf->dtb_xamot != NULL) { 11905 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); 11906 kmem_free(buf->dtb_xamot, buf->dtb_size); 11907 } 11908 11909 kmem_free(buf->dtb_tomax, buf->dtb_size); 11910 buf->dtb_size = 0; 11911 buf->dtb_tomax = NULL; 11912 buf->dtb_xamot = NULL; 11913 } 11914 } 11915 11916 /* 11917 * DTrace Enabling Functions 11918 */ 11919 static dtrace_enabling_t * 11920 dtrace_enabling_create(dtrace_vstate_t *vstate) 11921 { 11922 dtrace_enabling_t *enab; 11923 11924 enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP); 11925 enab->dten_vstate = vstate; 11926 11927 return (enab); 11928 } 11929 11930 static void 11931 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb) 11932 { 11933 dtrace_ecbdesc_t **ndesc; 11934 size_t osize, nsize; 11935 11936 /* 11937 * We can't add to enablings after we've enabled them, or after we've 11938 * retained them. 11939 */ 11940 ASSERT(enab->dten_probegen == 0); 11941 ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); 11942 11943 if (enab->dten_ndesc < enab->dten_maxdesc) { 11944 enab->dten_desc[enab->dten_ndesc++] = ecb; 11945 return; 11946 } 11947 11948 osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *); 11949 11950 if (enab->dten_maxdesc == 0) { 11951 enab->dten_maxdesc = 1; 11952 } else { 11953 enab->dten_maxdesc <<= 1; 11954 } 11955 11956 ASSERT(enab->dten_ndesc < enab->dten_maxdesc); 11957 11958 nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *); 11959 ndesc = kmem_zalloc(nsize, KM_SLEEP); 11960 bcopy(enab->dten_desc, ndesc, osize); 11961 kmem_free(enab->dten_desc, osize); 11962 11963 enab->dten_desc = ndesc; 11964 enab->dten_desc[enab->dten_ndesc++] = ecb; 11965 } 11966 11967 static void 11968 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb, 11969 dtrace_probedesc_t *pd) 11970 { 11971 dtrace_ecbdesc_t *new; 11972 dtrace_predicate_t *pred; 11973 dtrace_actdesc_t *act; 11974 11975 /* 11976 * We're going to create a new ECB description that matches the 11977 * specified ECB in every way, but has the specified probe description. 11978 */ 11979 new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP); 11980 11981 if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL) 11982 dtrace_predicate_hold(pred); 11983 11984 for (act = ecb->dted_action; act != NULL; act = act->dtad_next) 11985 dtrace_actdesc_hold(act); 11986 11987 new->dted_action = ecb->dted_action; 11988 new->dted_pred = ecb->dted_pred; 11989 new->dted_probe = *pd; 11990 new->dted_uarg = ecb->dted_uarg; 11991 11992 dtrace_enabling_add(enab, new); 11993 } 11994 11995 static void 11996 dtrace_enabling_dump(dtrace_enabling_t *enab) 11997 { 11998 int i; 11999 12000 for (i = 0; i < enab->dten_ndesc; i++) { 12001 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe; 12002 12003 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i, 12004 desc->dtpd_provider, desc->dtpd_mod, 12005 desc->dtpd_func, desc->dtpd_name); 12006 } 12007 } 12008 12009 static void 12010 dtrace_enabling_destroy(dtrace_enabling_t *enab) 12011 { 12012 int i; 12013 dtrace_ecbdesc_t *ep; 12014 dtrace_vstate_t *vstate = enab->dten_vstate; 12015 12016 ASSERT(MUTEX_HELD(&dtrace_lock)); 12017 12018 for (i = 0; i < enab->dten_ndesc; i++) { 12019 dtrace_actdesc_t *act, *next; 12020 dtrace_predicate_t *pred; 12021 12022 ep = enab->dten_desc[i]; 12023 12024 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) 12025 dtrace_predicate_release(pred, vstate); 12026 12027 for (act = ep->dted_action; act != NULL; act = next) { 12028 next = act->dtad_next; 12029 dtrace_actdesc_release(act, vstate); 12030 } 12031 12032 kmem_free(ep, sizeof (dtrace_ecbdesc_t)); 12033 } 12034 12035 kmem_free(enab->dten_desc, 12036 enab->dten_maxdesc * sizeof (dtrace_enabling_t *)); 12037 12038 /* 12039 * If this was a retained enabling, decrement the dts_nretained count 12040 * and take it off of the dtrace_retained list. 12041 */ 12042 if (enab->dten_prev != NULL || enab->dten_next != NULL || 12043 dtrace_retained == enab) { 12044 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12045 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0); 12046 enab->dten_vstate->dtvs_state->dts_nretained--; 12047 dtrace_retained_gen++; 12048 } 12049 12050 if (enab->dten_prev == NULL) { 12051 if (dtrace_retained == enab) { 12052 dtrace_retained = enab->dten_next; 12053 12054 if (dtrace_retained != NULL) 12055 dtrace_retained->dten_prev = NULL; 12056 } 12057 } else { 12058 ASSERT(enab != dtrace_retained); 12059 ASSERT(dtrace_retained != NULL); 12060 enab->dten_prev->dten_next = enab->dten_next; 12061 } 12062 12063 if (enab->dten_next != NULL) { 12064 ASSERT(dtrace_retained != NULL); 12065 enab->dten_next->dten_prev = enab->dten_prev; 12066 } 12067 12068 kmem_free(enab, sizeof (dtrace_enabling_t)); 12069 } 12070 12071 static int 12072 dtrace_enabling_retain(dtrace_enabling_t *enab) 12073 { 12074 dtrace_state_t *state; 12075 12076 ASSERT(MUTEX_HELD(&dtrace_lock)); 12077 ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); 12078 ASSERT(enab->dten_vstate != NULL); 12079 12080 state = enab->dten_vstate->dtvs_state; 12081 ASSERT(state != NULL); 12082 12083 /* 12084 * We only allow each state to retain dtrace_retain_max enablings. 12085 */ 12086 if (state->dts_nretained >= dtrace_retain_max) 12087 return (ENOSPC); 12088 12089 state->dts_nretained++; 12090 dtrace_retained_gen++; 12091 12092 if (dtrace_retained == NULL) { 12093 dtrace_retained = enab; 12094 return (0); 12095 } 12096 12097 enab->dten_next = dtrace_retained; 12098 dtrace_retained->dten_prev = enab; 12099 dtrace_retained = enab; 12100 12101 return (0); 12102 } 12103 12104 static int 12105 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match, 12106 dtrace_probedesc_t *create) 12107 { 12108 dtrace_enabling_t *new, *enab; 12109 int found = 0, err = ENOENT; 12110 12111 ASSERT(MUTEX_HELD(&dtrace_lock)); 12112 ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN); 12113 ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN); 12114 ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN); 12115 ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN); 12116 12117 new = dtrace_enabling_create(&state->dts_vstate); 12118 12119 /* 12120 * Iterate over all retained enablings, looking for enablings that 12121 * match the specified state. 12122 */ 12123 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { 12124 int i; 12125 12126 /* 12127 * dtvs_state can only be NULL for helper enablings -- and 12128 * helper enablings can't be retained. 12129 */ 12130 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12131 12132 if (enab->dten_vstate->dtvs_state != state) 12133 continue; 12134 12135 /* 12136 * Now iterate over each probe description; we're looking for 12137 * an exact match to the specified probe description. 12138 */ 12139 for (i = 0; i < enab->dten_ndesc; i++) { 12140 dtrace_ecbdesc_t *ep = enab->dten_desc[i]; 12141 dtrace_probedesc_t *pd = &ep->dted_probe; 12142 12143 if (strcmp(pd->dtpd_provider, match->dtpd_provider)) 12144 continue; 12145 12146 if (strcmp(pd->dtpd_mod, match->dtpd_mod)) 12147 continue; 12148 12149 if (strcmp(pd->dtpd_func, match->dtpd_func)) 12150 continue; 12151 12152 if (strcmp(pd->dtpd_name, match->dtpd_name)) 12153 continue; 12154 12155 /* 12156 * We have a winning probe! Add it to our growing 12157 * enabling. 12158 */ 12159 found = 1; 12160 dtrace_enabling_addlike(new, ep, create); 12161 } 12162 } 12163 12164 if (!found || (err = dtrace_enabling_retain(new)) != 0) { 12165 dtrace_enabling_destroy(new); 12166 return (err); 12167 } 12168 12169 return (0); 12170 } 12171 12172 static void 12173 dtrace_enabling_retract(dtrace_state_t *state) 12174 { 12175 dtrace_enabling_t *enab, *next; 12176 12177 ASSERT(MUTEX_HELD(&dtrace_lock)); 12178 12179 /* 12180 * Iterate over all retained enablings, destroy the enablings retained 12181 * for the specified state. 12182 */ 12183 for (enab = dtrace_retained; enab != NULL; enab = next) { 12184 next = enab->dten_next; 12185 12186 /* 12187 * dtvs_state can only be NULL for helper enablings -- and 12188 * helper enablings can't be retained. 12189 */ 12190 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12191 12192 if (enab->dten_vstate->dtvs_state == state) { 12193 ASSERT(state->dts_nretained > 0); 12194 dtrace_enabling_destroy(enab); 12195 } 12196 } 12197 12198 ASSERT(state->dts_nretained == 0); 12199 } 12200 12201 static int 12202 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched) 12203 { 12204 int i = 0; 12205 int total_matched = 0, matched = 0; 12206 12207 ASSERT(MUTEX_HELD(&cpu_lock)); 12208 ASSERT(MUTEX_HELD(&dtrace_lock)); 12209 12210 for (i = 0; i < enab->dten_ndesc; i++) { 12211 dtrace_ecbdesc_t *ep = enab->dten_desc[i]; 12212 12213 enab->dten_current = ep; 12214 enab->dten_error = 0; 12215 12216 /* 12217 * If a provider failed to enable a probe then get out and 12218 * let the consumer know we failed. 12219 */ 12220 if ((matched = dtrace_probe_enable(&ep->dted_probe, enab)) < 0) 12221 return (EBUSY); 12222 12223 total_matched += matched; 12224 12225 if (enab->dten_error != 0) { 12226 /* 12227 * If we get an error half-way through enabling the 12228 * probes, we kick out -- perhaps with some number of 12229 * them enabled. Leaving enabled probes enabled may 12230 * be slightly confusing for user-level, but we expect 12231 * that no one will attempt to actually drive on in 12232 * the face of such errors. If this is an anonymous 12233 * enabling (indicated with a NULL nmatched pointer), 12234 * we cmn_err() a message. We aren't expecting to 12235 * get such an error -- such as it can exist at all, 12236 * it would be a result of corrupted DOF in the driver 12237 * properties. 12238 */ 12239 if (nmatched == NULL) { 12240 cmn_err(CE_WARN, "dtrace_enabling_match() " 12241 "error on %p: %d", (void *)ep, 12242 enab->dten_error); 12243 } 12244 12245 return (enab->dten_error); 12246 } 12247 } 12248 12249 enab->dten_probegen = dtrace_probegen; 12250 if (nmatched != NULL) 12251 *nmatched = total_matched; 12252 12253 return (0); 12254 } 12255 12256 static void 12257 dtrace_enabling_matchall(void) 12258 { 12259 dtrace_enabling_t *enab; 12260 12261 mutex_enter(&cpu_lock); 12262 mutex_enter(&dtrace_lock); 12263 12264 /* 12265 * Iterate over all retained enablings to see if any probes match 12266 * against them. We only perform this operation on enablings for which 12267 * we have sufficient permissions by virtue of being in the global zone 12268 * or in the same zone as the DTrace client. Because we can be called 12269 * after dtrace_detach() has been called, we cannot assert that there 12270 * are retained enablings. We can safely load from dtrace_retained, 12271 * however: the taskq_destroy() at the end of dtrace_detach() will 12272 * block pending our completion. 12273 */ 12274 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { 12275 dtrace_cred_t *dcr = &enab->dten_vstate->dtvs_state->dts_cred; 12276 cred_t *cr = dcr->dcr_cred; 12277 zoneid_t zone = cr != NULL ? crgetzoneid(cr) : 0; 12278 12279 if ((dcr->dcr_visible & DTRACE_CRV_ALLZONE) || (cr != NULL && 12280 (zone == GLOBAL_ZONEID || getzoneid() == zone))) 12281 (void) dtrace_enabling_match(enab, NULL); 12282 } 12283 12284 mutex_exit(&dtrace_lock); 12285 mutex_exit(&cpu_lock); 12286 } 12287 12288 /* 12289 * If an enabling is to be enabled without having matched probes (that is, if 12290 * dtrace_state_go() is to be called on the underlying dtrace_state_t), the 12291 * enabling must be _primed_ by creating an ECB for every ECB description. 12292 * This must be done to assure that we know the number of speculations, the 12293 * number of aggregations, the minimum buffer size needed, etc. before we 12294 * transition out of DTRACE_ACTIVITY_INACTIVE. To do this without actually 12295 * enabling any probes, we create ECBs for every ECB decription, but with a 12296 * NULL probe -- which is exactly what this function does. 12297 */ 12298 static void 12299 dtrace_enabling_prime(dtrace_state_t *state) 12300 { 12301 dtrace_enabling_t *enab; 12302 int i; 12303 12304 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { 12305 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12306 12307 if (enab->dten_vstate->dtvs_state != state) 12308 continue; 12309 12310 /* 12311 * We don't want to prime an enabling more than once, lest 12312 * we allow a malicious user to induce resource exhaustion. 12313 * (The ECBs that result from priming an enabling aren't 12314 * leaked -- but they also aren't deallocated until the 12315 * consumer state is destroyed.) 12316 */ 12317 if (enab->dten_primed) 12318 continue; 12319 12320 for (i = 0; i < enab->dten_ndesc; i++) { 12321 enab->dten_current = enab->dten_desc[i]; 12322 (void) dtrace_probe_enable(NULL, enab); 12323 } 12324 12325 enab->dten_primed = 1; 12326 } 12327 } 12328 12329 /* 12330 * Called to indicate that probes should be provided due to retained 12331 * enablings. This is implemented in terms of dtrace_probe_provide(), but it 12332 * must take an initial lap through the enabling calling the dtps_provide() 12333 * entry point explicitly to allow for autocreated probes. 12334 */ 12335 static void 12336 dtrace_enabling_provide(dtrace_provider_t *prv) 12337 { 12338 int i, all = 0; 12339 dtrace_probedesc_t desc; 12340 dtrace_genid_t gen; 12341 12342 ASSERT(MUTEX_HELD(&dtrace_lock)); 12343 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 12344 12345 if (prv == NULL) { 12346 all = 1; 12347 prv = dtrace_provider; 12348 } 12349 12350 do { 12351 dtrace_enabling_t *enab; 12352 void *parg = prv->dtpv_arg; 12353 12354 retry: 12355 gen = dtrace_retained_gen; 12356 for (enab = dtrace_retained; enab != NULL; 12357 enab = enab->dten_next) { 12358 for (i = 0; i < enab->dten_ndesc; i++) { 12359 desc = enab->dten_desc[i]->dted_probe; 12360 mutex_exit(&dtrace_lock); 12361 prv->dtpv_pops.dtps_provide(parg, &desc); 12362 mutex_enter(&dtrace_lock); 12363 /* 12364 * Process the retained enablings again if 12365 * they have changed while we weren't holding 12366 * dtrace_lock. 12367 */ 12368 if (gen != dtrace_retained_gen) 12369 goto retry; 12370 } 12371 } 12372 } while (all && (prv = prv->dtpv_next) != NULL); 12373 12374 mutex_exit(&dtrace_lock); 12375 dtrace_probe_provide(NULL, all ? NULL : prv); 12376 mutex_enter(&dtrace_lock); 12377 } 12378 12379 /* 12380 * Called to reap ECBs that are attached to probes from defunct providers. 12381 */ 12382 static void 12383 dtrace_enabling_reap(void) 12384 { 12385 dtrace_provider_t *prov; 12386 dtrace_probe_t *probe; 12387 dtrace_ecb_t *ecb; 12388 hrtime_t when; 12389 int i; 12390 12391 mutex_enter(&cpu_lock); 12392 mutex_enter(&dtrace_lock); 12393 12394 for (i = 0; i < dtrace_nprobes; i++) { 12395 if ((probe = dtrace_probes[i]) == NULL) 12396 continue; 12397 12398 if (probe->dtpr_ecb == NULL) 12399 continue; 12400 12401 prov = probe->dtpr_provider; 12402 12403 if ((when = prov->dtpv_defunct) == 0) 12404 continue; 12405 12406 /* 12407 * We have ECBs on a defunct provider: we want to reap these 12408 * ECBs to allow the provider to unregister. The destruction 12409 * of these ECBs must be done carefully: if we destroy the ECB 12410 * and the consumer later wishes to consume an EPID that 12411 * corresponds to the destroyed ECB (and if the EPID metadata 12412 * has not been previously consumed), the consumer will abort 12413 * processing on the unknown EPID. To reduce (but not, sadly, 12414 * eliminate) the possibility of this, we will only destroy an 12415 * ECB for a defunct provider if, for the state that 12416 * corresponds to the ECB: 12417 * 12418 * (a) There is no speculative tracing (which can effectively 12419 * cache an EPID for an arbitrary amount of time). 12420 * 12421 * (b) The principal buffers have been switched twice since the 12422 * provider became defunct. 12423 * 12424 * (c) The aggregation buffers are of zero size or have been 12425 * switched twice since the provider became defunct. 12426 * 12427 * We use dts_speculates to determine (a) and call a function 12428 * (dtrace_buffer_consumed()) to determine (b) and (c). Note 12429 * that as soon as we've been unable to destroy one of the ECBs 12430 * associated with the probe, we quit trying -- reaping is only 12431 * fruitful in as much as we can destroy all ECBs associated 12432 * with the defunct provider's probes. 12433 */ 12434 while ((ecb = probe->dtpr_ecb) != NULL) { 12435 dtrace_state_t *state = ecb->dte_state; 12436 dtrace_buffer_t *buf = state->dts_buffer; 12437 dtrace_buffer_t *aggbuf = state->dts_aggbuffer; 12438 12439 if (state->dts_speculates) 12440 break; 12441 12442 if (!dtrace_buffer_consumed(buf, when)) 12443 break; 12444 12445 if (!dtrace_buffer_consumed(aggbuf, when)) 12446 break; 12447 12448 dtrace_ecb_disable(ecb); 12449 ASSERT(probe->dtpr_ecb != ecb); 12450 dtrace_ecb_destroy(ecb); 12451 } 12452 } 12453 12454 mutex_exit(&dtrace_lock); 12455 mutex_exit(&cpu_lock); 12456 } 12457 12458 /* 12459 * DTrace DOF Functions 12460 */ 12461 /*ARGSUSED*/ 12462 static void 12463 dtrace_dof_error(dof_hdr_t *dof, const char *str) 12464 { 12465 if (dtrace_err_verbose) 12466 cmn_err(CE_WARN, "failed to process DOF: %s", str); 12467 12468 #ifdef DTRACE_ERRDEBUG 12469 dtrace_errdebug(str); 12470 #endif 12471 } 12472 12473 /* 12474 * Create DOF out of a currently enabled state. Right now, we only create 12475 * DOF containing the run-time options -- but this could be expanded to create 12476 * complete DOF representing the enabled state. 12477 */ 12478 static dof_hdr_t * 12479 dtrace_dof_create(dtrace_state_t *state) 12480 { 12481 dof_hdr_t *dof; 12482 dof_sec_t *sec; 12483 dof_optdesc_t *opt; 12484 int i, len = sizeof (dof_hdr_t) + 12485 roundup(sizeof (dof_sec_t), sizeof (uint64_t)) + 12486 sizeof (dof_optdesc_t) * DTRACEOPT_MAX; 12487 12488 ASSERT(MUTEX_HELD(&dtrace_lock)); 12489 12490 dof = kmem_zalloc(len, KM_SLEEP); 12491 dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0; 12492 dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1; 12493 dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2; 12494 dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3; 12495 12496 dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE; 12497 dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE; 12498 dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION; 12499 dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION; 12500 dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS; 12501 dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS; 12502 12503 dof->dofh_flags = 0; 12504 dof->dofh_hdrsize = sizeof (dof_hdr_t); 12505 dof->dofh_secsize = sizeof (dof_sec_t); 12506 dof->dofh_secnum = 1; /* only DOF_SECT_OPTDESC */ 12507 dof->dofh_secoff = sizeof (dof_hdr_t); 12508 dof->dofh_loadsz = len; 12509 dof->dofh_filesz = len; 12510 dof->dofh_pad = 0; 12511 12512 /* 12513 * Fill in the option section header... 12514 */ 12515 sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t)); 12516 sec->dofs_type = DOF_SECT_OPTDESC; 12517 sec->dofs_align = sizeof (uint64_t); 12518 sec->dofs_flags = DOF_SECF_LOAD; 12519 sec->dofs_entsize = sizeof (dof_optdesc_t); 12520 12521 opt = (dof_optdesc_t *)((uintptr_t)sec + 12522 roundup(sizeof (dof_sec_t), sizeof (uint64_t))); 12523 12524 sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof; 12525 sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX; 12526 12527 for (i = 0; i < DTRACEOPT_MAX; i++) { 12528 opt[i].dofo_option = i; 12529 opt[i].dofo_strtab = DOF_SECIDX_NONE; 12530 opt[i].dofo_value = state->dts_options[i]; 12531 } 12532 12533 return (dof); 12534 } 12535 12536 static dof_hdr_t * 12537 dtrace_dof_copyin(uintptr_t uarg, int *errp) 12538 { 12539 dof_hdr_t hdr, *dof; 12540 12541 ASSERT(!MUTEX_HELD(&dtrace_lock)); 12542 12543 /* 12544 * First, we're going to copyin() the sizeof (dof_hdr_t). 12545 */ 12546 if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) { 12547 dtrace_dof_error(NULL, "failed to copyin DOF header"); 12548 *errp = EFAULT; 12549 return (NULL); 12550 } 12551 12552 /* 12553 * Now we'll allocate the entire DOF and copy it in -- provided 12554 * that the length isn't outrageous. 12555 */ 12556 if (hdr.dofh_loadsz >= dtrace_dof_maxsize) { 12557 dtrace_dof_error(&hdr, "load size exceeds maximum"); 12558 *errp = E2BIG; 12559 return (NULL); 12560 } 12561 12562 if (hdr.dofh_loadsz < sizeof (hdr)) { 12563 dtrace_dof_error(&hdr, "invalid load size"); 12564 *errp = EINVAL; 12565 return (NULL); 12566 } 12567 12568 dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP); 12569 12570 if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 || 12571 dof->dofh_loadsz != hdr.dofh_loadsz) { 12572 kmem_free(dof, hdr.dofh_loadsz); 12573 *errp = EFAULT; 12574 return (NULL); 12575 } 12576 12577 return (dof); 12578 } 12579 12580 static dof_hdr_t * 12581 dtrace_dof_property(const char *name) 12582 { 12583 uchar_t *buf; 12584 uint64_t loadsz; 12585 unsigned int len, i; 12586 dof_hdr_t *dof; 12587 12588 /* 12589 * Unfortunately, array of values in .conf files are always (and 12590 * only) interpreted to be integer arrays. We must read our DOF 12591 * as an integer array, and then squeeze it into a byte array. 12592 */ 12593 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0, 12594 (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS) 12595 return (NULL); 12596 12597 for (i = 0; i < len; i++) 12598 buf[i] = (uchar_t)(((int *)buf)[i]); 12599 12600 if (len < sizeof (dof_hdr_t)) { 12601 ddi_prop_free(buf); 12602 dtrace_dof_error(NULL, "truncated header"); 12603 return (NULL); 12604 } 12605 12606 if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) { 12607 ddi_prop_free(buf); 12608 dtrace_dof_error(NULL, "truncated DOF"); 12609 return (NULL); 12610 } 12611 12612 if (loadsz >= dtrace_dof_maxsize) { 12613 ddi_prop_free(buf); 12614 dtrace_dof_error(NULL, "oversized DOF"); 12615 return (NULL); 12616 } 12617 12618 dof = kmem_alloc(loadsz, KM_SLEEP); 12619 bcopy(buf, dof, loadsz); 12620 ddi_prop_free(buf); 12621 12622 return (dof); 12623 } 12624 12625 static void 12626 dtrace_dof_destroy(dof_hdr_t *dof) 12627 { 12628 kmem_free(dof, dof->dofh_loadsz); 12629 } 12630 12631 /* 12632 * Return the dof_sec_t pointer corresponding to a given section index. If the 12633 * index is not valid, dtrace_dof_error() is called and NULL is returned. If 12634 * a type other than DOF_SECT_NONE is specified, the header is checked against 12635 * this type and NULL is returned if the types do not match. 12636 */ 12637 static dof_sec_t * 12638 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i) 12639 { 12640 dof_sec_t *sec = (dof_sec_t *)(uintptr_t) 12641 ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize); 12642 12643 if (i >= dof->dofh_secnum) { 12644 dtrace_dof_error(dof, "referenced section index is invalid"); 12645 return (NULL); 12646 } 12647 12648 if (!(sec->dofs_flags & DOF_SECF_LOAD)) { 12649 dtrace_dof_error(dof, "referenced section is not loadable"); 12650 return (NULL); 12651 } 12652 12653 if (type != DOF_SECT_NONE && type != sec->dofs_type) { 12654 dtrace_dof_error(dof, "referenced section is the wrong type"); 12655 return (NULL); 12656 } 12657 12658 return (sec); 12659 } 12660 12661 static dtrace_probedesc_t * 12662 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc) 12663 { 12664 dof_probedesc_t *probe; 12665 dof_sec_t *strtab; 12666 uintptr_t daddr = (uintptr_t)dof; 12667 uintptr_t str; 12668 size_t size; 12669 12670 if (sec->dofs_type != DOF_SECT_PROBEDESC) { 12671 dtrace_dof_error(dof, "invalid probe section"); 12672 return (NULL); 12673 } 12674 12675 if (sec->dofs_align != sizeof (dof_secidx_t)) { 12676 dtrace_dof_error(dof, "bad alignment in probe description"); 12677 return (NULL); 12678 } 12679 12680 if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) { 12681 dtrace_dof_error(dof, "truncated probe description"); 12682 return (NULL); 12683 } 12684 12685 probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset); 12686 strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab); 12687 12688 if (strtab == NULL) 12689 return (NULL); 12690 12691 str = daddr + strtab->dofs_offset; 12692 size = strtab->dofs_size; 12693 12694 if (probe->dofp_provider >= strtab->dofs_size) { 12695 dtrace_dof_error(dof, "corrupt probe provider"); 12696 return (NULL); 12697 } 12698 12699 (void) strncpy(desc->dtpd_provider, 12700 (char *)(str + probe->dofp_provider), 12701 MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider)); 12702 12703 if (probe->dofp_mod >= strtab->dofs_size) { 12704 dtrace_dof_error(dof, "corrupt probe module"); 12705 return (NULL); 12706 } 12707 12708 (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod), 12709 MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod)); 12710 12711 if (probe->dofp_func >= strtab->dofs_size) { 12712 dtrace_dof_error(dof, "corrupt probe function"); 12713 return (NULL); 12714 } 12715 12716 (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func), 12717 MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func)); 12718 12719 if (probe->dofp_name >= strtab->dofs_size) { 12720 dtrace_dof_error(dof, "corrupt probe name"); 12721 return (NULL); 12722 } 12723 12724 (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name), 12725 MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name)); 12726 12727 return (desc); 12728 } 12729 12730 static dtrace_difo_t * 12731 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 12732 cred_t *cr) 12733 { 12734 dtrace_difo_t *dp; 12735 size_t ttl = 0; 12736 dof_difohdr_t *dofd; 12737 uintptr_t daddr = (uintptr_t)dof; 12738 size_t max = dtrace_difo_maxsize; 12739 int i, l, n; 12740 12741 static const struct { 12742 int section; 12743 int bufoffs; 12744 int lenoffs; 12745 int entsize; 12746 int align; 12747 const char *msg; 12748 } difo[] = { 12749 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf), 12750 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t), 12751 sizeof (dif_instr_t), "multiple DIF sections" }, 12752 12753 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab), 12754 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t), 12755 sizeof (uint64_t), "multiple integer tables" }, 12756 12757 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab), 12758 offsetof(dtrace_difo_t, dtdo_strlen), 0, 12759 sizeof (char), "multiple string tables" }, 12760 12761 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab), 12762 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t), 12763 sizeof (uint_t), "multiple variable tables" }, 12764 12765 { DOF_SECT_NONE, 0, 0, 0, NULL } 12766 }; 12767 12768 if (sec->dofs_type != DOF_SECT_DIFOHDR) { 12769 dtrace_dof_error(dof, "invalid DIFO header section"); 12770 return (NULL); 12771 } 12772 12773 if (sec->dofs_align != sizeof (dof_secidx_t)) { 12774 dtrace_dof_error(dof, "bad alignment in DIFO header"); 12775 return (NULL); 12776 } 12777 12778 if (sec->dofs_size < sizeof (dof_difohdr_t) || 12779 sec->dofs_size % sizeof (dof_secidx_t)) { 12780 dtrace_dof_error(dof, "bad size in DIFO header"); 12781 return (NULL); 12782 } 12783 12784 dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset); 12785 n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1; 12786 12787 dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP); 12788 dp->dtdo_rtype = dofd->dofd_rtype; 12789 12790 for (l = 0; l < n; l++) { 12791 dof_sec_t *subsec; 12792 void **bufp; 12793 uint32_t *lenp; 12794 12795 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE, 12796 dofd->dofd_links[l])) == NULL) 12797 goto err; /* invalid section link */ 12798 12799 if (ttl + subsec->dofs_size > max) { 12800 dtrace_dof_error(dof, "exceeds maximum size"); 12801 goto err; 12802 } 12803 12804 ttl += subsec->dofs_size; 12805 12806 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) { 12807 if (subsec->dofs_type != difo[i].section) 12808 continue; 12809 12810 if (!(subsec->dofs_flags & DOF_SECF_LOAD)) { 12811 dtrace_dof_error(dof, "section not loaded"); 12812 goto err; 12813 } 12814 12815 if (subsec->dofs_align != difo[i].align) { 12816 dtrace_dof_error(dof, "bad alignment"); 12817 goto err; 12818 } 12819 12820 bufp = (void **)((uintptr_t)dp + difo[i].bufoffs); 12821 lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs); 12822 12823 if (*bufp != NULL) { 12824 dtrace_dof_error(dof, difo[i].msg); 12825 goto err; 12826 } 12827 12828 if (difo[i].entsize != subsec->dofs_entsize) { 12829 dtrace_dof_error(dof, "entry size mismatch"); 12830 goto err; 12831 } 12832 12833 if (subsec->dofs_entsize != 0 && 12834 (subsec->dofs_size % subsec->dofs_entsize) != 0) { 12835 dtrace_dof_error(dof, "corrupt entry size"); 12836 goto err; 12837 } 12838 12839 *lenp = subsec->dofs_size; 12840 *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP); 12841 bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset), 12842 *bufp, subsec->dofs_size); 12843 12844 if (subsec->dofs_entsize != 0) 12845 *lenp /= subsec->dofs_entsize; 12846 12847 break; 12848 } 12849 12850 /* 12851 * If we encounter a loadable DIFO sub-section that is not 12852 * known to us, assume this is a broken program and fail. 12853 */ 12854 if (difo[i].section == DOF_SECT_NONE && 12855 (subsec->dofs_flags & DOF_SECF_LOAD)) { 12856 dtrace_dof_error(dof, "unrecognized DIFO subsection"); 12857 goto err; 12858 } 12859 } 12860 12861 if (dp->dtdo_buf == NULL) { 12862 /* 12863 * We can't have a DIF object without DIF text. 12864 */ 12865 dtrace_dof_error(dof, "missing DIF text"); 12866 goto err; 12867 } 12868 12869 /* 12870 * Before we validate the DIF object, run through the variable table 12871 * looking for the strings -- if any of their size are under, we'll set 12872 * their size to be the system-wide default string size. Note that 12873 * this should _not_ happen if the "strsize" option has been set -- 12874 * in this case, the compiler should have set the size to reflect the 12875 * setting of the option. 12876 */ 12877 for (i = 0; i < dp->dtdo_varlen; i++) { 12878 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 12879 dtrace_diftype_t *t = &v->dtdv_type; 12880 12881 if (v->dtdv_id < DIF_VAR_OTHER_UBASE) 12882 continue; 12883 12884 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0) 12885 t->dtdt_size = dtrace_strsize_default; 12886 } 12887 12888 if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0) 12889 goto err; 12890 12891 dtrace_difo_init(dp, vstate); 12892 return (dp); 12893 12894 err: 12895 kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t)); 12896 kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t)); 12897 kmem_free(dp->dtdo_strtab, dp->dtdo_strlen); 12898 kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t)); 12899 12900 kmem_free(dp, sizeof (dtrace_difo_t)); 12901 return (NULL); 12902 } 12903 12904 static dtrace_predicate_t * 12905 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 12906 cred_t *cr) 12907 { 12908 dtrace_difo_t *dp; 12909 12910 if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL) 12911 return (NULL); 12912 12913 return (dtrace_predicate_create(dp)); 12914 } 12915 12916 static dtrace_actdesc_t * 12917 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 12918 cred_t *cr) 12919 { 12920 dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next; 12921 dof_actdesc_t *desc; 12922 dof_sec_t *difosec; 12923 size_t offs; 12924 uintptr_t daddr = (uintptr_t)dof; 12925 uint64_t arg; 12926 dtrace_actkind_t kind; 12927 12928 if (sec->dofs_type != DOF_SECT_ACTDESC) { 12929 dtrace_dof_error(dof, "invalid action section"); 12930 return (NULL); 12931 } 12932 12933 if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) { 12934 dtrace_dof_error(dof, "truncated action description"); 12935 return (NULL); 12936 } 12937 12938 if (sec->dofs_align != sizeof (uint64_t)) { 12939 dtrace_dof_error(dof, "bad alignment in action description"); 12940 return (NULL); 12941 } 12942 12943 if (sec->dofs_size < sec->dofs_entsize) { 12944 dtrace_dof_error(dof, "section entry size exceeds total size"); 12945 return (NULL); 12946 } 12947 12948 if (sec->dofs_entsize != sizeof (dof_actdesc_t)) { 12949 dtrace_dof_error(dof, "bad entry size in action description"); 12950 return (NULL); 12951 } 12952 12953 if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) { 12954 dtrace_dof_error(dof, "actions exceed dtrace_actions_max"); 12955 return (NULL); 12956 } 12957 12958 for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) { 12959 desc = (dof_actdesc_t *)(daddr + 12960 (uintptr_t)sec->dofs_offset + offs); 12961 kind = (dtrace_actkind_t)desc->dofa_kind; 12962 12963 if ((DTRACEACT_ISPRINTFLIKE(kind) && 12964 (kind != DTRACEACT_PRINTA || 12965 desc->dofa_strtab != DOF_SECIDX_NONE)) || 12966 (kind == DTRACEACT_DIFEXPR && 12967 desc->dofa_strtab != DOF_SECIDX_NONE)) { 12968 dof_sec_t *strtab; 12969 char *str, *fmt; 12970 uint64_t i; 12971 12972 /* 12973 * The argument to these actions is an index into the 12974 * DOF string table. For printf()-like actions, this 12975 * is the format string. For print(), this is the 12976 * CTF type of the expression result. 12977 */ 12978 if ((strtab = dtrace_dof_sect(dof, 12979 DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL) 12980 goto err; 12981 12982 str = (char *)((uintptr_t)dof + 12983 (uintptr_t)strtab->dofs_offset); 12984 12985 for (i = desc->dofa_arg; i < strtab->dofs_size; i++) { 12986 if (str[i] == '\0') 12987 break; 12988 } 12989 12990 if (i >= strtab->dofs_size) { 12991 dtrace_dof_error(dof, "bogus format string"); 12992 goto err; 12993 } 12994 12995 if (i == desc->dofa_arg) { 12996 dtrace_dof_error(dof, "empty format string"); 12997 goto err; 12998 } 12999 13000 i -= desc->dofa_arg; 13001 fmt = kmem_alloc(i + 1, KM_SLEEP); 13002 bcopy(&str[desc->dofa_arg], fmt, i + 1); 13003 arg = (uint64_t)(uintptr_t)fmt; 13004 } else { 13005 if (kind == DTRACEACT_PRINTA) { 13006 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE); 13007 arg = 0; 13008 } else { 13009 arg = desc->dofa_arg; 13010 } 13011 } 13012 13013 act = dtrace_actdesc_create(kind, desc->dofa_ntuple, 13014 desc->dofa_uarg, arg); 13015 13016 if (last != NULL) { 13017 last->dtad_next = act; 13018 } else { 13019 first = act; 13020 } 13021 13022 last = act; 13023 13024 if (desc->dofa_difo == DOF_SECIDX_NONE) 13025 continue; 13026 13027 if ((difosec = dtrace_dof_sect(dof, 13028 DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL) 13029 goto err; 13030 13031 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr); 13032 13033 if (act->dtad_difo == NULL) 13034 goto err; 13035 } 13036 13037 ASSERT(first != NULL); 13038 return (first); 13039 13040 err: 13041 for (act = first; act != NULL; act = next) { 13042 next = act->dtad_next; 13043 dtrace_actdesc_release(act, vstate); 13044 } 13045 13046 return (NULL); 13047 } 13048 13049 static dtrace_ecbdesc_t * 13050 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 13051 cred_t *cr) 13052 { 13053 dtrace_ecbdesc_t *ep; 13054 dof_ecbdesc_t *ecb; 13055 dtrace_probedesc_t *desc; 13056 dtrace_predicate_t *pred = NULL; 13057 13058 if (sec->dofs_size < sizeof (dof_ecbdesc_t)) { 13059 dtrace_dof_error(dof, "truncated ECB description"); 13060 return (NULL); 13061 } 13062 13063 if (sec->dofs_align != sizeof (uint64_t)) { 13064 dtrace_dof_error(dof, "bad alignment in ECB description"); 13065 return (NULL); 13066 } 13067 13068 ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset); 13069 sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes); 13070 13071 if (sec == NULL) 13072 return (NULL); 13073 13074 ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP); 13075 ep->dted_uarg = ecb->dofe_uarg; 13076 desc = &ep->dted_probe; 13077 13078 if (dtrace_dof_probedesc(dof, sec, desc) == NULL) 13079 goto err; 13080 13081 if (ecb->dofe_pred != DOF_SECIDX_NONE) { 13082 if ((sec = dtrace_dof_sect(dof, 13083 DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL) 13084 goto err; 13085 13086 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL) 13087 goto err; 13088 13089 ep->dted_pred.dtpdd_predicate = pred; 13090 } 13091 13092 if (ecb->dofe_actions != DOF_SECIDX_NONE) { 13093 if ((sec = dtrace_dof_sect(dof, 13094 DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL) 13095 goto err; 13096 13097 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr); 13098 13099 if (ep->dted_action == NULL) 13100 goto err; 13101 } 13102 13103 return (ep); 13104 13105 err: 13106 if (pred != NULL) 13107 dtrace_predicate_release(pred, vstate); 13108 kmem_free(ep, sizeof (dtrace_ecbdesc_t)); 13109 return (NULL); 13110 } 13111 13112 /* 13113 * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the 13114 * specified DOF. At present, this amounts to simply adding 'ubase' to the 13115 * site of any user SETX relocations to account for load object base address. 13116 * In the future, if we need other relocations, this function can be extended. 13117 */ 13118 static int 13119 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase) 13120 { 13121 uintptr_t daddr = (uintptr_t)dof; 13122 dof_relohdr_t *dofr = 13123 (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset); 13124 dof_sec_t *ss, *rs, *ts; 13125 dof_relodesc_t *r; 13126 uint_t i, n; 13127 13128 if (sec->dofs_size < sizeof (dof_relohdr_t) || 13129 sec->dofs_align != sizeof (dof_secidx_t)) { 13130 dtrace_dof_error(dof, "invalid relocation header"); 13131 return (-1); 13132 } 13133 13134 ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab); 13135 rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec); 13136 ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec); 13137 13138 if (ss == NULL || rs == NULL || ts == NULL) 13139 return (-1); /* dtrace_dof_error() has been called already */ 13140 13141 if (rs->dofs_entsize < sizeof (dof_relodesc_t) || 13142 rs->dofs_align != sizeof (uint64_t)) { 13143 dtrace_dof_error(dof, "invalid relocation section"); 13144 return (-1); 13145 } 13146 13147 r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset); 13148 n = rs->dofs_size / rs->dofs_entsize; 13149 13150 for (i = 0; i < n; i++) { 13151 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset; 13152 13153 switch (r->dofr_type) { 13154 case DOF_RELO_NONE: 13155 break; 13156 case DOF_RELO_SETX: 13157 if (r->dofr_offset >= ts->dofs_size || r->dofr_offset + 13158 sizeof (uint64_t) > ts->dofs_size) { 13159 dtrace_dof_error(dof, "bad relocation offset"); 13160 return (-1); 13161 } 13162 13163 if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) { 13164 dtrace_dof_error(dof, "misaligned setx relo"); 13165 return (-1); 13166 } 13167 13168 *(uint64_t *)taddr += ubase; 13169 break; 13170 default: 13171 dtrace_dof_error(dof, "invalid relocation type"); 13172 return (-1); 13173 } 13174 13175 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize); 13176 } 13177 13178 return (0); 13179 } 13180 13181 /* 13182 * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated 13183 * header: it should be at the front of a memory region that is at least 13184 * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in 13185 * size. It need not be validated in any other way. 13186 */ 13187 static int 13188 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr, 13189 dtrace_enabling_t **enabp, uint64_t ubase, int noprobes) 13190 { 13191 uint64_t len = dof->dofh_loadsz, seclen; 13192 uintptr_t daddr = (uintptr_t)dof; 13193 dtrace_ecbdesc_t *ep; 13194 dtrace_enabling_t *enab; 13195 uint_t i; 13196 13197 ASSERT(MUTEX_HELD(&dtrace_lock)); 13198 ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t)); 13199 13200 /* 13201 * Check the DOF header identification bytes. In addition to checking 13202 * valid settings, we also verify that unused bits/bytes are zeroed so 13203 * we can use them later without fear of regressing existing binaries. 13204 */ 13205 if (bcmp(&dof->dofh_ident[DOF_ID_MAG0], 13206 DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) { 13207 dtrace_dof_error(dof, "DOF magic string mismatch"); 13208 return (-1); 13209 } 13210 13211 if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 && 13212 dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) { 13213 dtrace_dof_error(dof, "DOF has invalid data model"); 13214 return (-1); 13215 } 13216 13217 if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) { 13218 dtrace_dof_error(dof, "DOF encoding mismatch"); 13219 return (-1); 13220 } 13221 13222 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && 13223 dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) { 13224 dtrace_dof_error(dof, "DOF version mismatch"); 13225 return (-1); 13226 } 13227 13228 if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) { 13229 dtrace_dof_error(dof, "DOF uses unsupported instruction set"); 13230 return (-1); 13231 } 13232 13233 if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) { 13234 dtrace_dof_error(dof, "DOF uses too many integer registers"); 13235 return (-1); 13236 } 13237 13238 if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) { 13239 dtrace_dof_error(dof, "DOF uses too many tuple registers"); 13240 return (-1); 13241 } 13242 13243 for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) { 13244 if (dof->dofh_ident[i] != 0) { 13245 dtrace_dof_error(dof, "DOF has invalid ident byte set"); 13246 return (-1); 13247 } 13248 } 13249 13250 if (dof->dofh_flags & ~DOF_FL_VALID) { 13251 dtrace_dof_error(dof, "DOF has invalid flag bits set"); 13252 return (-1); 13253 } 13254 13255 if (dof->dofh_secsize == 0) { 13256 dtrace_dof_error(dof, "zero section header size"); 13257 return (-1); 13258 } 13259 13260 /* 13261 * Check that the section headers don't exceed the amount of DOF 13262 * data. Note that we cast the section size and number of sections 13263 * to uint64_t's to prevent possible overflow in the multiplication. 13264 */ 13265 seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize; 13266 13267 if (dof->dofh_secoff > len || seclen > len || 13268 dof->dofh_secoff + seclen > len) { 13269 dtrace_dof_error(dof, "truncated section headers"); 13270 return (-1); 13271 } 13272 13273 if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) { 13274 dtrace_dof_error(dof, "misaligned section headers"); 13275 return (-1); 13276 } 13277 13278 if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) { 13279 dtrace_dof_error(dof, "misaligned section size"); 13280 return (-1); 13281 } 13282 13283 /* 13284 * Take an initial pass through the section headers to be sure that 13285 * the headers don't have stray offsets. If the 'noprobes' flag is 13286 * set, do not permit sections relating to providers, probes, or args. 13287 */ 13288 for (i = 0; i < dof->dofh_secnum; i++) { 13289 dof_sec_t *sec = (dof_sec_t *)(daddr + 13290 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13291 13292 if (noprobes) { 13293 switch (sec->dofs_type) { 13294 case DOF_SECT_PROVIDER: 13295 case DOF_SECT_PROBES: 13296 case DOF_SECT_PRARGS: 13297 case DOF_SECT_PROFFS: 13298 dtrace_dof_error(dof, "illegal sections " 13299 "for enabling"); 13300 return (-1); 13301 } 13302 } 13303 13304 if (DOF_SEC_ISLOADABLE(sec->dofs_type) && 13305 !(sec->dofs_flags & DOF_SECF_LOAD)) { 13306 dtrace_dof_error(dof, "loadable section with load " 13307 "flag unset"); 13308 return (-1); 13309 } 13310 13311 if (!(sec->dofs_flags & DOF_SECF_LOAD)) 13312 continue; /* just ignore non-loadable sections */ 13313 13314 if (!ISP2(sec->dofs_align)) { 13315 dtrace_dof_error(dof, "bad section alignment"); 13316 return (-1); 13317 } 13318 13319 if (sec->dofs_offset & (sec->dofs_align - 1)) { 13320 dtrace_dof_error(dof, "misaligned section"); 13321 return (-1); 13322 } 13323 13324 if (sec->dofs_offset > len || sec->dofs_size > len || 13325 sec->dofs_offset + sec->dofs_size > len) { 13326 dtrace_dof_error(dof, "corrupt section header"); 13327 return (-1); 13328 } 13329 13330 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr + 13331 sec->dofs_offset + sec->dofs_size - 1) != '\0') { 13332 dtrace_dof_error(dof, "non-terminating string table"); 13333 return (-1); 13334 } 13335 } 13336 13337 /* 13338 * Take a second pass through the sections and locate and perform any 13339 * relocations that are present. We do this after the first pass to 13340 * be sure that all sections have had their headers validated. 13341 */ 13342 for (i = 0; i < dof->dofh_secnum; i++) { 13343 dof_sec_t *sec = (dof_sec_t *)(daddr + 13344 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13345 13346 if (!(sec->dofs_flags & DOF_SECF_LOAD)) 13347 continue; /* skip sections that are not loadable */ 13348 13349 switch (sec->dofs_type) { 13350 case DOF_SECT_URELHDR: 13351 if (dtrace_dof_relocate(dof, sec, ubase) != 0) 13352 return (-1); 13353 break; 13354 } 13355 } 13356 13357 if ((enab = *enabp) == NULL) 13358 enab = *enabp = dtrace_enabling_create(vstate); 13359 13360 for (i = 0; i < dof->dofh_secnum; i++) { 13361 dof_sec_t *sec = (dof_sec_t *)(daddr + 13362 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13363 13364 if (sec->dofs_type != DOF_SECT_ECBDESC) 13365 continue; 13366 13367 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) { 13368 dtrace_enabling_destroy(enab); 13369 *enabp = NULL; 13370 return (-1); 13371 } 13372 13373 dtrace_enabling_add(enab, ep); 13374 } 13375 13376 return (0); 13377 } 13378 13379 /* 13380 * Process DOF for any options. This routine assumes that the DOF has been 13381 * at least processed by dtrace_dof_slurp(). 13382 */ 13383 static int 13384 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state) 13385 { 13386 int i, rval; 13387 uint32_t entsize; 13388 size_t offs; 13389 dof_optdesc_t *desc; 13390 13391 for (i = 0; i < dof->dofh_secnum; i++) { 13392 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof + 13393 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13394 13395 if (sec->dofs_type != DOF_SECT_OPTDESC) 13396 continue; 13397 13398 if (sec->dofs_align != sizeof (uint64_t)) { 13399 dtrace_dof_error(dof, "bad alignment in " 13400 "option description"); 13401 return (EINVAL); 13402 } 13403 13404 if ((entsize = sec->dofs_entsize) == 0) { 13405 dtrace_dof_error(dof, "zeroed option entry size"); 13406 return (EINVAL); 13407 } 13408 13409 if (entsize < sizeof (dof_optdesc_t)) { 13410 dtrace_dof_error(dof, "bad option entry size"); 13411 return (EINVAL); 13412 } 13413 13414 for (offs = 0; offs < sec->dofs_size; offs += entsize) { 13415 desc = (dof_optdesc_t *)((uintptr_t)dof + 13416 (uintptr_t)sec->dofs_offset + offs); 13417 13418 if (desc->dofo_strtab != DOF_SECIDX_NONE) { 13419 dtrace_dof_error(dof, "non-zero option string"); 13420 return (EINVAL); 13421 } 13422 13423 if (desc->dofo_value == DTRACEOPT_UNSET) { 13424 dtrace_dof_error(dof, "unset option"); 13425 return (EINVAL); 13426 } 13427 13428 if ((rval = dtrace_state_option(state, 13429 desc->dofo_option, desc->dofo_value)) != 0) { 13430 dtrace_dof_error(dof, "rejected option"); 13431 return (rval); 13432 } 13433 } 13434 } 13435 13436 return (0); 13437 } 13438 13439 /* 13440 * DTrace Consumer State Functions 13441 */ 13442 int 13443 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) 13444 { 13445 size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize; 13446 void *base; 13447 uintptr_t limit; 13448 dtrace_dynvar_t *dvar, *next, *start; 13449 int i; 13450 13451 ASSERT(MUTEX_HELD(&dtrace_lock)); 13452 ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL); 13453 13454 bzero(dstate, sizeof (dtrace_dstate_t)); 13455 13456 if ((dstate->dtds_chunksize = chunksize) == 0) 13457 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE; 13458 13459 VERIFY(dstate->dtds_chunksize < LONG_MAX); 13460 13461 if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t))) 13462 size = min; 13463 13464 if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) 13465 return (ENOMEM); 13466 13467 dstate->dtds_size = size; 13468 dstate->dtds_base = base; 13469 dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP); 13470 bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t)); 13471 13472 hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)); 13473 13474 if (hashsize != 1 && (hashsize & 1)) 13475 hashsize--; 13476 13477 dstate->dtds_hashsize = hashsize; 13478 dstate->dtds_hash = dstate->dtds_base; 13479 13480 /* 13481 * Set all of our hash buckets to point to the single sink, and (if 13482 * it hasn't already been set), set the sink's hash value to be the 13483 * sink sentinel value. The sink is needed for dynamic variable 13484 * lookups to know that they have iterated over an entire, valid hash 13485 * chain. 13486 */ 13487 for (i = 0; i < hashsize; i++) 13488 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink; 13489 13490 if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK) 13491 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK; 13492 13493 /* 13494 * Determine number of active CPUs. Divide free list evenly among 13495 * active CPUs. 13496 */ 13497 start = (dtrace_dynvar_t *) 13498 ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t)); 13499 limit = (uintptr_t)base + size; 13500 13501 VERIFY((uintptr_t)start < limit); 13502 VERIFY((uintptr_t)start >= (uintptr_t)base); 13503 13504 maxper = (limit - (uintptr_t)start) / NCPU; 13505 maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize; 13506 13507 for (i = 0; i < NCPU; i++) { 13508 dstate->dtds_percpu[i].dtdsc_free = dvar = start; 13509 13510 /* 13511 * If we don't even have enough chunks to make it once through 13512 * NCPUs, we're just going to allocate everything to the first 13513 * CPU. And if we're on the last CPU, we're going to allocate 13514 * whatever is left over. In either case, we set the limit to 13515 * be the limit of the dynamic variable space. 13516 */ 13517 if (maxper == 0 || i == NCPU - 1) { 13518 limit = (uintptr_t)base + size; 13519 start = NULL; 13520 } else { 13521 limit = (uintptr_t)start + maxper; 13522 start = (dtrace_dynvar_t *)limit; 13523 } 13524 13525 VERIFY(limit <= (uintptr_t)base + size); 13526 13527 for (;;) { 13528 next = (dtrace_dynvar_t *)((uintptr_t)dvar + 13529 dstate->dtds_chunksize); 13530 13531 if ((uintptr_t)next + dstate->dtds_chunksize >= limit) 13532 break; 13533 13534 VERIFY((uintptr_t)dvar >= (uintptr_t)base && 13535 (uintptr_t)dvar <= (uintptr_t)base + size); 13536 dvar->dtdv_next = next; 13537 dvar = next; 13538 } 13539 13540 if (maxper == 0) 13541 break; 13542 } 13543 13544 return (0); 13545 } 13546 13547 void 13548 dtrace_dstate_fini(dtrace_dstate_t *dstate) 13549 { 13550 ASSERT(MUTEX_HELD(&cpu_lock)); 13551 13552 if (dstate->dtds_base == NULL) 13553 return; 13554 13555 kmem_free(dstate->dtds_base, dstate->dtds_size); 13556 kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu); 13557 } 13558 13559 static void 13560 dtrace_vstate_fini(dtrace_vstate_t *vstate) 13561 { 13562 /* 13563 * Logical XOR, where are you? 13564 */ 13565 ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL)); 13566 13567 if (vstate->dtvs_nglobals > 0) { 13568 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals * 13569 sizeof (dtrace_statvar_t *)); 13570 } 13571 13572 if (vstate->dtvs_ntlocals > 0) { 13573 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals * 13574 sizeof (dtrace_difv_t)); 13575 } 13576 13577 ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL)); 13578 13579 if (vstate->dtvs_nlocals > 0) { 13580 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals * 13581 sizeof (dtrace_statvar_t *)); 13582 } 13583 } 13584 13585 static void 13586 dtrace_state_clean(dtrace_state_t *state) 13587 { 13588 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) 13589 return; 13590 13591 dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars); 13592 dtrace_speculation_clean(state); 13593 } 13594 13595 static void 13596 dtrace_state_deadman(dtrace_state_t *state) 13597 { 13598 hrtime_t now; 13599 13600 dtrace_sync(); 13601 13602 now = dtrace_gethrtime(); 13603 13604 if (state != dtrace_anon.dta_state && 13605 now - state->dts_laststatus >= dtrace_deadman_user) 13606 return; 13607 13608 /* 13609 * We must be sure that dts_alive never appears to be less than the 13610 * value upon entry to dtrace_state_deadman(), and because we lack a 13611 * dtrace_cas64(), we cannot store to it atomically. We thus instead 13612 * store INT64_MAX to it, followed by a memory barrier, followed by 13613 * the new value. This assures that dts_alive never appears to be 13614 * less than its true value, regardless of the order in which the 13615 * stores to the underlying storage are issued. 13616 */ 13617 state->dts_alive = INT64_MAX; 13618 dtrace_membar_producer(); 13619 state->dts_alive = now; 13620 } 13621 13622 dtrace_state_t * 13623 dtrace_state_create(dev_t *devp, cred_t *cr) 13624 { 13625 minor_t minor; 13626 major_t major; 13627 char c[30]; 13628 dtrace_state_t *state; 13629 dtrace_optval_t *opt; 13630 int bufsize = NCPU * sizeof (dtrace_buffer_t), i; 13631 13632 ASSERT(MUTEX_HELD(&dtrace_lock)); 13633 ASSERT(MUTEX_HELD(&cpu_lock)); 13634 13635 minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1, 13636 VM_BESTFIT | VM_SLEEP); 13637 13638 if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) { 13639 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1); 13640 return (NULL); 13641 } 13642 13643 state = ddi_get_soft_state(dtrace_softstate, minor); 13644 state->dts_epid = DTRACE_EPIDNONE + 1; 13645 13646 (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor); 13647 state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1, 13648 NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 13649 13650 if (devp != NULL) { 13651 major = getemajor(*devp); 13652 } else { 13653 major = ddi_driver_major(dtrace_devi); 13654 } 13655 13656 state->dts_dev = makedevice(major, minor); 13657 13658 if (devp != NULL) 13659 *devp = state->dts_dev; 13660 13661 /* 13662 * We allocate NCPU buffers. On the one hand, this can be quite 13663 * a bit of memory per instance (nearly 36K on a Starcat). On the 13664 * other hand, it saves an additional memory reference in the probe 13665 * path. 13666 */ 13667 state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP); 13668 state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP); 13669 state->dts_cleaner = CYCLIC_NONE; 13670 state->dts_deadman = CYCLIC_NONE; 13671 state->dts_vstate.dtvs_state = state; 13672 13673 for (i = 0; i < DTRACEOPT_MAX; i++) 13674 state->dts_options[i] = DTRACEOPT_UNSET; 13675 13676 /* 13677 * Set the default options. 13678 */ 13679 opt = state->dts_options; 13680 opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH; 13681 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO; 13682 opt[DTRACEOPT_NSPEC] = dtrace_nspec_default; 13683 opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default; 13684 opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL; 13685 opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default; 13686 opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default; 13687 opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default; 13688 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default; 13689 opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default; 13690 opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default; 13691 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default; 13692 opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default; 13693 opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default; 13694 13695 state->dts_activity = DTRACE_ACTIVITY_INACTIVE; 13696 13697 /* 13698 * Depending on the user credentials, we set flag bits which alter probe 13699 * visibility or the amount of destructiveness allowed. In the case of 13700 * actual anonymous tracing, or the possession of all privileges, all of 13701 * the normal checks are bypassed. 13702 */ 13703 if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { 13704 state->dts_cred.dcr_visible = DTRACE_CRV_ALL; 13705 state->dts_cred.dcr_action = DTRACE_CRA_ALL; 13706 } else { 13707 /* 13708 * Set up the credentials for this instantiation. We take a 13709 * hold on the credential to prevent it from disappearing on 13710 * us; this in turn prevents the zone_t referenced by this 13711 * credential from disappearing. This means that we can 13712 * examine the credential and the zone from probe context. 13713 */ 13714 crhold(cr); 13715 state->dts_cred.dcr_cred = cr; 13716 13717 /* 13718 * CRA_PROC means "we have *some* privilege for dtrace" and 13719 * unlocks the use of variables like pid, zonename, etc. 13720 */ 13721 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) || 13722 PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) { 13723 state->dts_cred.dcr_action |= DTRACE_CRA_PROC; 13724 } 13725 13726 /* 13727 * dtrace_user allows use of syscall and profile providers. 13728 * If the user also has proc_owner and/or proc_zone, we 13729 * extend the scope to include additional visibility and 13730 * destructive power. 13731 */ 13732 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) { 13733 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) { 13734 state->dts_cred.dcr_visible |= 13735 DTRACE_CRV_ALLPROC; 13736 13737 state->dts_cred.dcr_action |= 13738 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; 13739 } 13740 13741 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) { 13742 state->dts_cred.dcr_visible |= 13743 DTRACE_CRV_ALLZONE; 13744 13745 state->dts_cred.dcr_action |= 13746 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; 13747 } 13748 13749 /* 13750 * If we have all privs in whatever zone this is, 13751 * we can do destructive things to processes which 13752 * have altered credentials. 13753 */ 13754 if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE), 13755 cr->cr_zone->zone_privset)) { 13756 state->dts_cred.dcr_action |= 13757 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG; 13758 } 13759 } 13760 13761 /* 13762 * Holding the dtrace_kernel privilege also implies that 13763 * the user has the dtrace_user privilege from a visibility 13764 * perspective. But without further privileges, some 13765 * destructive actions are not available. 13766 */ 13767 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) { 13768 /* 13769 * Make all probes in all zones visible. However, 13770 * this doesn't mean that all actions become available 13771 * to all zones. 13772 */ 13773 state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL | 13774 DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE; 13775 13776 state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL | 13777 DTRACE_CRA_PROC; 13778 /* 13779 * Holding proc_owner means that destructive actions 13780 * for *this* zone are allowed. 13781 */ 13782 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) 13783 state->dts_cred.dcr_action |= 13784 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; 13785 13786 /* 13787 * Holding proc_zone means that destructive actions 13788 * for this user/group ID in all zones is allowed. 13789 */ 13790 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) 13791 state->dts_cred.dcr_action |= 13792 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; 13793 13794 /* 13795 * If we have all privs in whatever zone this is, 13796 * we can do destructive things to processes which 13797 * have altered credentials. 13798 */ 13799 if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE), 13800 cr->cr_zone->zone_privset)) { 13801 state->dts_cred.dcr_action |= 13802 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG; 13803 } 13804 } 13805 13806 /* 13807 * Holding the dtrace_proc privilege gives control over fasttrap 13808 * and pid providers. We need to grant wider destructive 13809 * privileges in the event that the user has proc_owner and/or 13810 * proc_zone. 13811 */ 13812 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) { 13813 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) 13814 state->dts_cred.dcr_action |= 13815 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; 13816 13817 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) 13818 state->dts_cred.dcr_action |= 13819 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; 13820 } 13821 } 13822 13823 return (state); 13824 } 13825 13826 static int 13827 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which) 13828 { 13829 dtrace_optval_t *opt = state->dts_options, size; 13830 processorid_t cpu; 13831 int flags = 0, rval, factor, divisor = 1; 13832 13833 ASSERT(MUTEX_HELD(&dtrace_lock)); 13834 ASSERT(MUTEX_HELD(&cpu_lock)); 13835 ASSERT(which < DTRACEOPT_MAX); 13836 ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE || 13837 (state == dtrace_anon.dta_state && 13838 state->dts_activity == DTRACE_ACTIVITY_ACTIVE)); 13839 13840 if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0) 13841 return (0); 13842 13843 if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET) 13844 cpu = opt[DTRACEOPT_CPU]; 13845 13846 if (which == DTRACEOPT_SPECSIZE) 13847 flags |= DTRACEBUF_NOSWITCH; 13848 13849 if (which == DTRACEOPT_BUFSIZE) { 13850 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING) 13851 flags |= DTRACEBUF_RING; 13852 13853 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL) 13854 flags |= DTRACEBUF_FILL; 13855 13856 if (state != dtrace_anon.dta_state || 13857 state->dts_activity != DTRACE_ACTIVITY_ACTIVE) 13858 flags |= DTRACEBUF_INACTIVE; 13859 } 13860 13861 for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) { 13862 /* 13863 * The size must be 8-byte aligned. If the size is not 8-byte 13864 * aligned, drop it down by the difference. 13865 */ 13866 if (size & (sizeof (uint64_t) - 1)) 13867 size -= size & (sizeof (uint64_t) - 1); 13868 13869 if (size < state->dts_reserve) { 13870 /* 13871 * Buffers always must be large enough to accommodate 13872 * their prereserved space. We return E2BIG instead 13873 * of ENOMEM in this case to allow for user-level 13874 * software to differentiate the cases. 13875 */ 13876 return (E2BIG); 13877 } 13878 13879 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor); 13880 13881 if (rval != ENOMEM) { 13882 opt[which] = size; 13883 return (rval); 13884 } 13885 13886 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL) 13887 return (rval); 13888 13889 for (divisor = 2; divisor < factor; divisor <<= 1) 13890 continue; 13891 } 13892 13893 return (ENOMEM); 13894 } 13895 13896 static int 13897 dtrace_state_buffers(dtrace_state_t *state) 13898 { 13899 dtrace_speculation_t *spec = state->dts_speculations; 13900 int rval, i; 13901 13902 if ((rval = dtrace_state_buffer(state, state->dts_buffer, 13903 DTRACEOPT_BUFSIZE)) != 0) 13904 return (rval); 13905 13906 if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer, 13907 DTRACEOPT_AGGSIZE)) != 0) 13908 return (rval); 13909 13910 for (i = 0; i < state->dts_nspeculations; i++) { 13911 if ((rval = dtrace_state_buffer(state, 13912 spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0) 13913 return (rval); 13914 } 13915 13916 return (0); 13917 } 13918 13919 static void 13920 dtrace_state_prereserve(dtrace_state_t *state) 13921 { 13922 dtrace_ecb_t *ecb; 13923 dtrace_probe_t *probe; 13924 13925 state->dts_reserve = 0; 13926 13927 if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL) 13928 return; 13929 13930 /* 13931 * If our buffer policy is a "fill" buffer policy, we need to set the 13932 * prereserved space to be the space required by the END probes. 13933 */ 13934 probe = dtrace_probes[dtrace_probeid_end - 1]; 13935 ASSERT(probe != NULL); 13936 13937 for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { 13938 if (ecb->dte_state != state) 13939 continue; 13940 13941 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment; 13942 } 13943 } 13944 13945 static int 13946 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu) 13947 { 13948 dtrace_optval_t *opt = state->dts_options, sz, nspec; 13949 dtrace_speculation_t *spec; 13950 dtrace_buffer_t *buf; 13951 cyc_handler_t hdlr; 13952 cyc_time_t when; 13953 int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t); 13954 dtrace_icookie_t cookie; 13955 13956 mutex_enter(&cpu_lock); 13957 mutex_enter(&dtrace_lock); 13958 13959 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { 13960 rval = EBUSY; 13961 goto out; 13962 } 13963 13964 /* 13965 * Before we can perform any checks, we must prime all of the 13966 * retained enablings that correspond to this state. 13967 */ 13968 dtrace_enabling_prime(state); 13969 13970 if (state->dts_destructive && !state->dts_cred.dcr_destructive) { 13971 rval = EACCES; 13972 goto out; 13973 } 13974 13975 dtrace_state_prereserve(state); 13976 13977 /* 13978 * Now we want to do is try to allocate our speculations. 13979 * We do not automatically resize the number of speculations; if 13980 * this fails, we will fail the operation. 13981 */ 13982 nspec = opt[DTRACEOPT_NSPEC]; 13983 ASSERT(nspec != DTRACEOPT_UNSET); 13984 13985 if (nspec > INT_MAX) { 13986 rval = ENOMEM; 13987 goto out; 13988 } 13989 13990 spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t), 13991 KM_NOSLEEP | KM_NORMALPRI); 13992 13993 if (spec == NULL) { 13994 rval = ENOMEM; 13995 goto out; 13996 } 13997 13998 state->dts_speculations = spec; 13999 state->dts_nspeculations = (int)nspec; 14000 14001 for (i = 0; i < nspec; i++) { 14002 if ((buf = kmem_zalloc(bufsize, 14003 KM_NOSLEEP | KM_NORMALPRI)) == NULL) { 14004 rval = ENOMEM; 14005 goto err; 14006 } 14007 14008 spec[i].dtsp_buffer = buf; 14009 } 14010 14011 if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) { 14012 if (dtrace_anon.dta_state == NULL) { 14013 rval = ENOENT; 14014 goto out; 14015 } 14016 14017 if (state->dts_necbs != 0) { 14018 rval = EALREADY; 14019 goto out; 14020 } 14021 14022 state->dts_anon = dtrace_anon_grab(); 14023 ASSERT(state->dts_anon != NULL); 14024 state = state->dts_anon; 14025 14026 /* 14027 * We want "grabanon" to be set in the grabbed state, so we'll 14028 * copy that option value from the grabbing state into the 14029 * grabbed state. 14030 */ 14031 state->dts_options[DTRACEOPT_GRABANON] = 14032 opt[DTRACEOPT_GRABANON]; 14033 14034 *cpu = dtrace_anon.dta_beganon; 14035 14036 /* 14037 * If the anonymous state is active (as it almost certainly 14038 * is if the anonymous enabling ultimately matched anything), 14039 * we don't allow any further option processing -- but we 14040 * don't return failure. 14041 */ 14042 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) 14043 goto out; 14044 } 14045 14046 if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET && 14047 opt[DTRACEOPT_AGGSIZE] != 0) { 14048 if (state->dts_aggregations == NULL) { 14049 /* 14050 * We're not going to create an aggregation buffer 14051 * because we don't have any ECBs that contain 14052 * aggregations -- set this option to 0. 14053 */ 14054 opt[DTRACEOPT_AGGSIZE] = 0; 14055 } else { 14056 /* 14057 * If we have an aggregation buffer, we must also have 14058 * a buffer to use as scratch. 14059 */ 14060 if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET || 14061 opt[DTRACEOPT_BUFSIZE] < state->dts_needed) { 14062 opt[DTRACEOPT_BUFSIZE] = state->dts_needed; 14063 } 14064 } 14065 } 14066 14067 if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET && 14068 opt[DTRACEOPT_SPECSIZE] != 0) { 14069 if (!state->dts_speculates) { 14070 /* 14071 * We're not going to create speculation buffers 14072 * because we don't have any ECBs that actually 14073 * speculate -- set the speculation size to 0. 14074 */ 14075 opt[DTRACEOPT_SPECSIZE] = 0; 14076 } 14077 } 14078 14079 /* 14080 * The bare minimum size for any buffer that we're actually going to 14081 * do anything to is sizeof (uint64_t). 14082 */ 14083 sz = sizeof (uint64_t); 14084 14085 if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) || 14086 (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) || 14087 (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) { 14088 /* 14089 * A buffer size has been explicitly set to 0 (or to a size 14090 * that will be adjusted to 0) and we need the space -- we 14091 * need to return failure. We return ENOSPC to differentiate 14092 * it from failing to allocate a buffer due to failure to meet 14093 * the reserve (for which we return E2BIG). 14094 */ 14095 rval = ENOSPC; 14096 goto out; 14097 } 14098 14099 if ((rval = dtrace_state_buffers(state)) != 0) 14100 goto err; 14101 14102 if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET) 14103 sz = dtrace_dstate_defsize; 14104 14105 do { 14106 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz); 14107 14108 if (rval == 0) 14109 break; 14110 14111 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL) 14112 goto err; 14113 } while (sz >>= 1); 14114 14115 opt[DTRACEOPT_DYNVARSIZE] = sz; 14116 14117 if (rval != 0) 14118 goto err; 14119 14120 if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max) 14121 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max; 14122 14123 if (opt[DTRACEOPT_CLEANRATE] == 0) 14124 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max; 14125 14126 if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min) 14127 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min; 14128 14129 if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max) 14130 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max; 14131 14132 hdlr.cyh_func = (cyc_func_t)dtrace_state_clean; 14133 hdlr.cyh_arg = state; 14134 hdlr.cyh_level = CY_LOW_LEVEL; 14135 14136 when.cyt_when = 0; 14137 when.cyt_interval = opt[DTRACEOPT_CLEANRATE]; 14138 14139 state->dts_cleaner = cyclic_add(&hdlr, &when); 14140 14141 hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman; 14142 hdlr.cyh_arg = state; 14143 hdlr.cyh_level = CY_LOW_LEVEL; 14144 14145 when.cyt_when = 0; 14146 when.cyt_interval = dtrace_deadman_interval; 14147 14148 state->dts_alive = state->dts_laststatus = dtrace_gethrtime(); 14149 state->dts_deadman = cyclic_add(&hdlr, &when); 14150 14151 state->dts_activity = DTRACE_ACTIVITY_WARMUP; 14152 14153 if (state->dts_getf != 0 && 14154 !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) { 14155 /* 14156 * We don't have kernel privs but we have at least one call 14157 * to getf(); we need to bump our zone's count, and (if 14158 * this is the first enabling to have an unprivileged call 14159 * to getf()) we need to hook into closef(). 14160 */ 14161 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++; 14162 14163 if (dtrace_getf++ == 0) { 14164 ASSERT(dtrace_closef == NULL); 14165 dtrace_closef = dtrace_getf_barrier; 14166 } 14167 } 14168 14169 /* 14170 * Now it's time to actually fire the BEGIN probe. We need to disable 14171 * interrupts here both to record the CPU on which we fired the BEGIN 14172 * probe (the data from this CPU will be processed first at user 14173 * level) and to manually activate the buffer for this CPU. 14174 */ 14175 cookie = dtrace_interrupt_disable(); 14176 *cpu = CPU->cpu_id; 14177 ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE); 14178 state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE; 14179 14180 dtrace_probe(dtrace_probeid_begin, 14181 (uint64_t)(uintptr_t)state, 0, 0, 0, 0); 14182 dtrace_interrupt_enable(cookie); 14183 /* 14184 * We may have had an exit action from a BEGIN probe; only change our 14185 * state to ACTIVE if we're still in WARMUP. 14186 */ 14187 ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP || 14188 state->dts_activity == DTRACE_ACTIVITY_DRAINING); 14189 14190 if (state->dts_activity == DTRACE_ACTIVITY_WARMUP) 14191 state->dts_activity = DTRACE_ACTIVITY_ACTIVE; 14192 14193 /* 14194 * Regardless of whether or not now we're in ACTIVE or DRAINING, we 14195 * want each CPU to transition its principal buffer out of the 14196 * INACTIVE state. Doing this assures that no CPU will suddenly begin 14197 * processing an ECB halfway down a probe's ECB chain; all CPUs will 14198 * atomically transition from processing none of a state's ECBs to 14199 * processing all of them. 14200 */ 14201 dtrace_xcall(DTRACE_CPUALL, 14202 (dtrace_xcall_t)dtrace_buffer_activate, state); 14203 goto out; 14204 14205 err: 14206 dtrace_buffer_free(state->dts_buffer); 14207 dtrace_buffer_free(state->dts_aggbuffer); 14208 14209 if ((nspec = state->dts_nspeculations) == 0) { 14210 ASSERT(state->dts_speculations == NULL); 14211 goto out; 14212 } 14213 14214 spec = state->dts_speculations; 14215 ASSERT(spec != NULL); 14216 14217 for (i = 0; i < state->dts_nspeculations; i++) { 14218 if ((buf = spec[i].dtsp_buffer) == NULL) 14219 break; 14220 14221 dtrace_buffer_free(buf); 14222 kmem_free(buf, bufsize); 14223 } 14224 14225 kmem_free(spec, nspec * sizeof (dtrace_speculation_t)); 14226 state->dts_nspeculations = 0; 14227 state->dts_speculations = NULL; 14228 14229 out: 14230 mutex_exit(&dtrace_lock); 14231 mutex_exit(&cpu_lock); 14232 14233 return (rval); 14234 } 14235 14236 static int 14237 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu) 14238 { 14239 dtrace_icookie_t cookie; 14240 14241 ASSERT(MUTEX_HELD(&dtrace_lock)); 14242 14243 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE && 14244 state->dts_activity != DTRACE_ACTIVITY_DRAINING) 14245 return (EINVAL); 14246 14247 /* 14248 * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync 14249 * to be sure that every CPU has seen it. See below for the details 14250 * on why this is done. 14251 */ 14252 state->dts_activity = DTRACE_ACTIVITY_DRAINING; 14253 dtrace_sync(); 14254 14255 /* 14256 * By this point, it is impossible for any CPU to be still processing 14257 * with DTRACE_ACTIVITY_ACTIVE. We can thus set our activity to 14258 * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any 14259 * other CPU in dtrace_buffer_reserve(). This allows dtrace_probe() 14260 * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN 14261 * iff we're in the END probe. 14262 */ 14263 state->dts_activity = DTRACE_ACTIVITY_COOLDOWN; 14264 dtrace_sync(); 14265 ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN); 14266 14267 /* 14268 * Finally, we can release the reserve and call the END probe. We 14269 * disable interrupts across calling the END probe to allow us to 14270 * return the CPU on which we actually called the END probe. This 14271 * allows user-land to be sure that this CPU's principal buffer is 14272 * processed last. 14273 */ 14274 state->dts_reserve = 0; 14275 14276 cookie = dtrace_interrupt_disable(); 14277 *cpu = CPU->cpu_id; 14278 dtrace_probe(dtrace_probeid_end, 14279 (uint64_t)(uintptr_t)state, 0, 0, 0, 0); 14280 dtrace_interrupt_enable(cookie); 14281 14282 state->dts_activity = DTRACE_ACTIVITY_STOPPED; 14283 dtrace_sync(); 14284 14285 if (state->dts_getf != 0 && 14286 !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) { 14287 /* 14288 * We don't have kernel privs but we have at least one call 14289 * to getf(); we need to lower our zone's count, and (if 14290 * this is the last enabling to have an unprivileged call 14291 * to getf()) we need to clear the closef() hook. 14292 */ 14293 ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0); 14294 ASSERT(dtrace_closef == dtrace_getf_barrier); 14295 ASSERT(dtrace_getf > 0); 14296 14297 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--; 14298 14299 if (--dtrace_getf == 0) 14300 dtrace_closef = NULL; 14301 } 14302 14303 return (0); 14304 } 14305 14306 static int 14307 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option, 14308 dtrace_optval_t val) 14309 { 14310 ASSERT(MUTEX_HELD(&dtrace_lock)); 14311 14312 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) 14313 return (EBUSY); 14314 14315 if (option >= DTRACEOPT_MAX) 14316 return (EINVAL); 14317 14318 if (option != DTRACEOPT_CPU && val < 0) 14319 return (EINVAL); 14320 14321 switch (option) { 14322 case DTRACEOPT_DESTRUCTIVE: 14323 if (dtrace_destructive_disallow) 14324 return (EACCES); 14325 14326 state->dts_cred.dcr_destructive = 1; 14327 break; 14328 14329 case DTRACEOPT_BUFSIZE: 14330 case DTRACEOPT_DYNVARSIZE: 14331 case DTRACEOPT_AGGSIZE: 14332 case DTRACEOPT_SPECSIZE: 14333 case DTRACEOPT_STRSIZE: 14334 if (val < 0) 14335 return (EINVAL); 14336 14337 if (val >= LONG_MAX) { 14338 /* 14339 * If this is an otherwise negative value, set it to 14340 * the highest multiple of 128m less than LONG_MAX. 14341 * Technically, we're adjusting the size without 14342 * regard to the buffer resizing policy, but in fact, 14343 * this has no effect -- if we set the buffer size to 14344 * ~LONG_MAX and the buffer policy is ultimately set to 14345 * be "manual", the buffer allocation is guaranteed to 14346 * fail, if only because the allocation requires two 14347 * buffers. (We set the the size to the highest 14348 * multiple of 128m because it ensures that the size 14349 * will remain a multiple of a megabyte when 14350 * repeatedly halved -- all the way down to 15m.) 14351 */ 14352 val = LONG_MAX - (1 << 27) + 1; 14353 } 14354 } 14355 14356 state->dts_options[option] = val; 14357 14358 return (0); 14359 } 14360 14361 static void 14362 dtrace_state_destroy(dtrace_state_t *state) 14363 { 14364 dtrace_ecb_t *ecb; 14365 dtrace_vstate_t *vstate = &state->dts_vstate; 14366 minor_t minor = getminor(state->dts_dev); 14367 int i, bufsize = NCPU * sizeof (dtrace_buffer_t); 14368 dtrace_speculation_t *spec = state->dts_speculations; 14369 int nspec = state->dts_nspeculations; 14370 uint32_t match; 14371 14372 ASSERT(MUTEX_HELD(&dtrace_lock)); 14373 ASSERT(MUTEX_HELD(&cpu_lock)); 14374 14375 /* 14376 * First, retract any retained enablings for this state. 14377 */ 14378 dtrace_enabling_retract(state); 14379 ASSERT(state->dts_nretained == 0); 14380 14381 if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE || 14382 state->dts_activity == DTRACE_ACTIVITY_DRAINING) { 14383 /* 14384 * We have managed to come into dtrace_state_destroy() on a 14385 * hot enabling -- almost certainly because of a disorderly 14386 * shutdown of a consumer. (That is, a consumer that is 14387 * exiting without having called dtrace_stop().) In this case, 14388 * we're going to set our activity to be KILLED, and then 14389 * issue a sync to be sure that everyone is out of probe 14390 * context before we start blowing away ECBs. 14391 */ 14392 state->dts_activity = DTRACE_ACTIVITY_KILLED; 14393 dtrace_sync(); 14394 } 14395 14396 /* 14397 * Release the credential hold we took in dtrace_state_create(). 14398 */ 14399 if (state->dts_cred.dcr_cred != NULL) 14400 crfree(state->dts_cred.dcr_cred); 14401 14402 /* 14403 * Now we can safely disable and destroy any enabled probes. Because 14404 * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress 14405 * (especially if they're all enabled), we take two passes through the 14406 * ECBs: in the first, we disable just DTRACE_PRIV_KERNEL probes, and 14407 * in the second we disable whatever is left over. 14408 */ 14409 for (match = DTRACE_PRIV_KERNEL; ; match = 0) { 14410 for (i = 0; i < state->dts_necbs; i++) { 14411 if ((ecb = state->dts_ecbs[i]) == NULL) 14412 continue; 14413 14414 if (match && ecb->dte_probe != NULL) { 14415 dtrace_probe_t *probe = ecb->dte_probe; 14416 dtrace_provider_t *prov = probe->dtpr_provider; 14417 14418 if (!(prov->dtpv_priv.dtpp_flags & match)) 14419 continue; 14420 } 14421 14422 dtrace_ecb_disable(ecb); 14423 dtrace_ecb_destroy(ecb); 14424 } 14425 14426 if (!match) 14427 break; 14428 } 14429 14430 /* 14431 * Before we free the buffers, perform one more sync to assure that 14432 * every CPU is out of probe context. 14433 */ 14434 dtrace_sync(); 14435 14436 dtrace_buffer_free(state->dts_buffer); 14437 dtrace_buffer_free(state->dts_aggbuffer); 14438 14439 for (i = 0; i < nspec; i++) 14440 dtrace_buffer_free(spec[i].dtsp_buffer); 14441 14442 if (state->dts_cleaner != CYCLIC_NONE) 14443 cyclic_remove(state->dts_cleaner); 14444 14445 if (state->dts_deadman != CYCLIC_NONE) 14446 cyclic_remove(state->dts_deadman); 14447 14448 dtrace_dstate_fini(&vstate->dtvs_dynvars); 14449 dtrace_vstate_fini(vstate); 14450 kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *)); 14451 14452 if (state->dts_aggregations != NULL) { 14453 #ifdef DEBUG 14454 for (i = 0; i < state->dts_naggregations; i++) 14455 ASSERT(state->dts_aggregations[i] == NULL); 14456 #endif 14457 ASSERT(state->dts_naggregations > 0); 14458 kmem_free(state->dts_aggregations, 14459 state->dts_naggregations * sizeof (dtrace_aggregation_t *)); 14460 } 14461 14462 kmem_free(state->dts_buffer, bufsize); 14463 kmem_free(state->dts_aggbuffer, bufsize); 14464 14465 for (i = 0; i < nspec; i++) 14466 kmem_free(spec[i].dtsp_buffer, bufsize); 14467 14468 kmem_free(spec, nspec * sizeof (dtrace_speculation_t)); 14469 14470 dtrace_format_destroy(state); 14471 14472 vmem_destroy(state->dts_aggid_arena); 14473 ddi_soft_state_free(dtrace_softstate, minor); 14474 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1); 14475 } 14476 14477 /* 14478 * DTrace Anonymous Enabling Functions 14479 */ 14480 static dtrace_state_t * 14481 dtrace_anon_grab(void) 14482 { 14483 dtrace_state_t *state; 14484 14485 ASSERT(MUTEX_HELD(&dtrace_lock)); 14486 14487 if ((state = dtrace_anon.dta_state) == NULL) { 14488 ASSERT(dtrace_anon.dta_enabling == NULL); 14489 return (NULL); 14490 } 14491 14492 ASSERT(dtrace_anon.dta_enabling != NULL); 14493 ASSERT(dtrace_retained != NULL); 14494 14495 dtrace_enabling_destroy(dtrace_anon.dta_enabling); 14496 dtrace_anon.dta_enabling = NULL; 14497 dtrace_anon.dta_state = NULL; 14498 14499 return (state); 14500 } 14501 14502 static void 14503 dtrace_anon_property(void) 14504 { 14505 int i, rv; 14506 dtrace_state_t *state; 14507 dof_hdr_t *dof; 14508 char c[32]; /* enough for "dof-data-" + digits */ 14509 14510 ASSERT(MUTEX_HELD(&dtrace_lock)); 14511 ASSERT(MUTEX_HELD(&cpu_lock)); 14512 14513 for (i = 0; ; i++) { 14514 (void) snprintf(c, sizeof (c), "dof-data-%d", i); 14515 14516 dtrace_err_verbose = 1; 14517 14518 if ((dof = dtrace_dof_property(c)) == NULL) { 14519 dtrace_err_verbose = 0; 14520 break; 14521 } 14522 14523 /* 14524 * We want to create anonymous state, so we need to transition 14525 * the kernel debugger to indicate that DTrace is active. If 14526 * this fails (e.g. because the debugger has modified text in 14527 * some way), we won't continue with the processing. 14528 */ 14529 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { 14530 cmn_err(CE_NOTE, "kernel debugger active; anonymous " 14531 "enabling ignored."); 14532 dtrace_dof_destroy(dof); 14533 break; 14534 } 14535 14536 /* 14537 * If we haven't allocated an anonymous state, we'll do so now. 14538 */ 14539 if ((state = dtrace_anon.dta_state) == NULL) { 14540 state = dtrace_state_create(NULL, NULL); 14541 dtrace_anon.dta_state = state; 14542 14543 if (state == NULL) { 14544 /* 14545 * This basically shouldn't happen: the only 14546 * failure mode from dtrace_state_create() is a 14547 * failure of ddi_soft_state_zalloc() that 14548 * itself should never happen. Still, the 14549 * interface allows for a failure mode, and 14550 * we want to fail as gracefully as possible: 14551 * we'll emit an error message and cease 14552 * processing anonymous state in this case. 14553 */ 14554 cmn_err(CE_WARN, "failed to create " 14555 "anonymous state"); 14556 dtrace_dof_destroy(dof); 14557 break; 14558 } 14559 } 14560 14561 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(), 14562 &dtrace_anon.dta_enabling, 0, B_TRUE); 14563 14564 if (rv == 0) 14565 rv = dtrace_dof_options(dof, state); 14566 14567 dtrace_err_verbose = 0; 14568 dtrace_dof_destroy(dof); 14569 14570 if (rv != 0) { 14571 /* 14572 * This is malformed DOF; chuck any anonymous state 14573 * that we created. 14574 */ 14575 ASSERT(dtrace_anon.dta_enabling == NULL); 14576 dtrace_state_destroy(state); 14577 dtrace_anon.dta_state = NULL; 14578 break; 14579 } 14580 14581 ASSERT(dtrace_anon.dta_enabling != NULL); 14582 } 14583 14584 if (dtrace_anon.dta_enabling != NULL) { 14585 int rval; 14586 14587 /* 14588 * dtrace_enabling_retain() can only fail because we are 14589 * trying to retain more enablings than are allowed -- but 14590 * we only have one anonymous enabling, and we are guaranteed 14591 * to be allowed at least one retained enabling; we assert 14592 * that dtrace_enabling_retain() returns success. 14593 */ 14594 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling); 14595 ASSERT(rval == 0); 14596 14597 dtrace_enabling_dump(dtrace_anon.dta_enabling); 14598 } 14599 } 14600 14601 /* 14602 * DTrace Helper Functions 14603 */ 14604 static void 14605 dtrace_helper_trace(dtrace_helper_action_t *helper, 14606 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where) 14607 { 14608 uint32_t size, next, nnext, i; 14609 dtrace_helptrace_t *ent, *buffer; 14610 uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 14611 14612 if ((buffer = dtrace_helptrace_buffer) == NULL) 14613 return; 14614 14615 ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals); 14616 14617 /* 14618 * What would a tracing framework be without its own tracing 14619 * framework? (Well, a hell of a lot simpler, for starters...) 14620 */ 14621 size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals * 14622 sizeof (uint64_t) - sizeof (uint64_t); 14623 14624 /* 14625 * Iterate until we can allocate a slot in the trace buffer. 14626 */ 14627 do { 14628 next = dtrace_helptrace_next; 14629 14630 if (next + size < dtrace_helptrace_bufsize) { 14631 nnext = next + size; 14632 } else { 14633 nnext = size; 14634 } 14635 } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next); 14636 14637 /* 14638 * We have our slot; fill it in. 14639 */ 14640 if (nnext == size) { 14641 dtrace_helptrace_wrapped++; 14642 next = 0; 14643 } 14644 14645 ent = (dtrace_helptrace_t *)((uintptr_t)buffer + next); 14646 ent->dtht_helper = helper; 14647 ent->dtht_where = where; 14648 ent->dtht_nlocals = vstate->dtvs_nlocals; 14649 14650 ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ? 14651 mstate->dtms_fltoffs : -1; 14652 ent->dtht_fault = DTRACE_FLAGS2FLT(flags); 14653 ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval; 14654 14655 for (i = 0; i < vstate->dtvs_nlocals; i++) { 14656 dtrace_statvar_t *svar; 14657 14658 if ((svar = vstate->dtvs_locals[i]) == NULL) 14659 continue; 14660 14661 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t)); 14662 ent->dtht_locals[i] = 14663 ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id]; 14664 } 14665 } 14666 14667 static uint64_t 14668 dtrace_helper(int which, dtrace_mstate_t *mstate, 14669 dtrace_state_t *state, uint64_t arg0, uint64_t arg1) 14670 { 14671 uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags; 14672 uint64_t sarg0 = mstate->dtms_arg[0]; 14673 uint64_t sarg1 = mstate->dtms_arg[1]; 14674 uint64_t rval; 14675 dtrace_helpers_t *helpers = curproc->p_dtrace_helpers; 14676 dtrace_helper_action_t *helper; 14677 dtrace_vstate_t *vstate; 14678 dtrace_difo_t *pred; 14679 int i, trace = dtrace_helptrace_buffer != NULL; 14680 14681 ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS); 14682 14683 if (helpers == NULL) 14684 return (0); 14685 14686 if ((helper = helpers->dthps_actions[which]) == NULL) 14687 return (0); 14688 14689 vstate = &helpers->dthps_vstate; 14690 mstate->dtms_arg[0] = arg0; 14691 mstate->dtms_arg[1] = arg1; 14692 14693 /* 14694 * Now iterate over each helper. If its predicate evaluates to 'true', 14695 * we'll call the corresponding actions. Note that the below calls 14696 * to dtrace_dif_emulate() may set faults in machine state. This is 14697 * okay: our caller (the outer dtrace_dif_emulate()) will simply plow 14698 * the stored DIF offset with its own (which is the desired behavior). 14699 * Also, note the calls to dtrace_dif_emulate() may allocate scratch 14700 * from machine state; this is okay, too. 14701 */ 14702 for (; helper != NULL; helper = helper->dtha_next) { 14703 if ((pred = helper->dtha_predicate) != NULL) { 14704 if (trace) 14705 dtrace_helper_trace(helper, mstate, vstate, 0); 14706 14707 if (!dtrace_dif_emulate(pred, mstate, vstate, state)) 14708 goto next; 14709 14710 if (*flags & CPU_DTRACE_FAULT) 14711 goto err; 14712 } 14713 14714 for (i = 0; i < helper->dtha_nactions; i++) { 14715 if (trace) 14716 dtrace_helper_trace(helper, 14717 mstate, vstate, i + 1); 14718 14719 rval = dtrace_dif_emulate(helper->dtha_actions[i], 14720 mstate, vstate, state); 14721 14722 if (*flags & CPU_DTRACE_FAULT) 14723 goto err; 14724 } 14725 14726 next: 14727 if (trace) 14728 dtrace_helper_trace(helper, mstate, vstate, 14729 DTRACE_HELPTRACE_NEXT); 14730 } 14731 14732 if (trace) 14733 dtrace_helper_trace(helper, mstate, vstate, 14734 DTRACE_HELPTRACE_DONE); 14735 14736 /* 14737 * Restore the arg0 that we saved upon entry. 14738 */ 14739 mstate->dtms_arg[0] = sarg0; 14740 mstate->dtms_arg[1] = sarg1; 14741 14742 return (rval); 14743 14744 err: 14745 if (trace) 14746 dtrace_helper_trace(helper, mstate, vstate, 14747 DTRACE_HELPTRACE_ERR); 14748 14749 /* 14750 * Restore the arg0 that we saved upon entry. 14751 */ 14752 mstate->dtms_arg[0] = sarg0; 14753 mstate->dtms_arg[1] = sarg1; 14754 14755 return (NULL); 14756 } 14757 14758 static void 14759 dtrace_helper_action_destroy(dtrace_helper_action_t *helper, 14760 dtrace_vstate_t *vstate) 14761 { 14762 int i; 14763 14764 if (helper->dtha_predicate != NULL) 14765 dtrace_difo_release(helper->dtha_predicate, vstate); 14766 14767 for (i = 0; i < helper->dtha_nactions; i++) { 14768 ASSERT(helper->dtha_actions[i] != NULL); 14769 dtrace_difo_release(helper->dtha_actions[i], vstate); 14770 } 14771 14772 kmem_free(helper->dtha_actions, 14773 helper->dtha_nactions * sizeof (dtrace_difo_t *)); 14774 kmem_free(helper, sizeof (dtrace_helper_action_t)); 14775 } 14776 14777 static int 14778 dtrace_helper_destroygen(int gen) 14779 { 14780 proc_t *p = curproc; 14781 dtrace_helpers_t *help = p->p_dtrace_helpers; 14782 dtrace_vstate_t *vstate; 14783 int i; 14784 14785 ASSERT(MUTEX_HELD(&dtrace_lock)); 14786 14787 if (help == NULL || gen > help->dthps_generation) 14788 return (EINVAL); 14789 14790 vstate = &help->dthps_vstate; 14791 14792 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { 14793 dtrace_helper_action_t *last = NULL, *h, *next; 14794 14795 for (h = help->dthps_actions[i]; h != NULL; h = next) { 14796 next = h->dtha_next; 14797 14798 if (h->dtha_generation == gen) { 14799 if (last != NULL) { 14800 last->dtha_next = next; 14801 } else { 14802 help->dthps_actions[i] = next; 14803 } 14804 14805 dtrace_helper_action_destroy(h, vstate); 14806 } else { 14807 last = h; 14808 } 14809 } 14810 } 14811 14812 /* 14813 * Interate until we've cleared out all helper providers with the 14814 * given generation number. 14815 */ 14816 for (;;) { 14817 dtrace_helper_provider_t *prov; 14818 14819 /* 14820 * Look for a helper provider with the right generation. We 14821 * have to start back at the beginning of the list each time 14822 * because we drop dtrace_lock. It's unlikely that we'll make 14823 * more than two passes. 14824 */ 14825 for (i = 0; i < help->dthps_nprovs; i++) { 14826 prov = help->dthps_provs[i]; 14827 14828 if (prov->dthp_generation == gen) 14829 break; 14830 } 14831 14832 /* 14833 * If there were no matches, we're done. 14834 */ 14835 if (i == help->dthps_nprovs) 14836 break; 14837 14838 /* 14839 * Move the last helper provider into this slot. 14840 */ 14841 help->dthps_nprovs--; 14842 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs]; 14843 help->dthps_provs[help->dthps_nprovs] = NULL; 14844 14845 mutex_exit(&dtrace_lock); 14846 14847 /* 14848 * If we have a meta provider, remove this helper provider. 14849 */ 14850 mutex_enter(&dtrace_meta_lock); 14851 if (dtrace_meta_pid != NULL) { 14852 ASSERT(dtrace_deferred_pid == NULL); 14853 dtrace_helper_provider_remove(&prov->dthp_prov, 14854 p->p_pid); 14855 } 14856 mutex_exit(&dtrace_meta_lock); 14857 14858 dtrace_helper_provider_destroy(prov); 14859 14860 mutex_enter(&dtrace_lock); 14861 } 14862 14863 return (0); 14864 } 14865 14866 static int 14867 dtrace_helper_validate(dtrace_helper_action_t *helper) 14868 { 14869 int err = 0, i; 14870 dtrace_difo_t *dp; 14871 14872 if ((dp = helper->dtha_predicate) != NULL) 14873 err += dtrace_difo_validate_helper(dp); 14874 14875 for (i = 0; i < helper->dtha_nactions; i++) 14876 err += dtrace_difo_validate_helper(helper->dtha_actions[i]); 14877 14878 return (err == 0); 14879 } 14880 14881 static int 14882 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep) 14883 { 14884 dtrace_helpers_t *help; 14885 dtrace_helper_action_t *helper, *last; 14886 dtrace_actdesc_t *act; 14887 dtrace_vstate_t *vstate; 14888 dtrace_predicate_t *pred; 14889 int count = 0, nactions = 0, i; 14890 14891 if (which < 0 || which >= DTRACE_NHELPER_ACTIONS) 14892 return (EINVAL); 14893 14894 help = curproc->p_dtrace_helpers; 14895 last = help->dthps_actions[which]; 14896 vstate = &help->dthps_vstate; 14897 14898 for (count = 0; last != NULL; last = last->dtha_next) { 14899 count++; 14900 if (last->dtha_next == NULL) 14901 break; 14902 } 14903 14904 /* 14905 * If we already have dtrace_helper_actions_max helper actions for this 14906 * helper action type, we'll refuse to add a new one. 14907 */ 14908 if (count >= dtrace_helper_actions_max) 14909 return (ENOSPC); 14910 14911 helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP); 14912 helper->dtha_generation = help->dthps_generation; 14913 14914 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) { 14915 ASSERT(pred->dtp_difo != NULL); 14916 dtrace_difo_hold(pred->dtp_difo); 14917 helper->dtha_predicate = pred->dtp_difo; 14918 } 14919 14920 for (act = ep->dted_action; act != NULL; act = act->dtad_next) { 14921 if (act->dtad_kind != DTRACEACT_DIFEXPR) 14922 goto err; 14923 14924 if (act->dtad_difo == NULL) 14925 goto err; 14926 14927 nactions++; 14928 } 14929 14930 helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) * 14931 (helper->dtha_nactions = nactions), KM_SLEEP); 14932 14933 for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) { 14934 dtrace_difo_hold(act->dtad_difo); 14935 helper->dtha_actions[i++] = act->dtad_difo; 14936 } 14937 14938 if (!dtrace_helper_validate(helper)) 14939 goto err; 14940 14941 if (last == NULL) { 14942 help->dthps_actions[which] = helper; 14943 } else { 14944 last->dtha_next = helper; 14945 } 14946 14947 if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) { 14948 dtrace_helptrace_nlocals = vstate->dtvs_nlocals; 14949 dtrace_helptrace_next = 0; 14950 } 14951 14952 return (0); 14953 err: 14954 dtrace_helper_action_destroy(helper, vstate); 14955 return (EINVAL); 14956 } 14957 14958 static void 14959 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help, 14960 dof_helper_t *dofhp) 14961 { 14962 ASSERT(MUTEX_NOT_HELD(&dtrace_lock)); 14963 14964 mutex_enter(&dtrace_meta_lock); 14965 mutex_enter(&dtrace_lock); 14966 14967 if (!dtrace_attached() || dtrace_meta_pid == NULL) { 14968 /* 14969 * If the dtrace module is loaded but not attached, or if 14970 * there aren't isn't a meta provider registered to deal with 14971 * these provider descriptions, we need to postpone creating 14972 * the actual providers until later. 14973 */ 14974 14975 if (help->dthps_next == NULL && help->dthps_prev == NULL && 14976 dtrace_deferred_pid != help) { 14977 help->dthps_deferred = 1; 14978 help->dthps_pid = p->p_pid; 14979 help->dthps_next = dtrace_deferred_pid; 14980 help->dthps_prev = NULL; 14981 if (dtrace_deferred_pid != NULL) 14982 dtrace_deferred_pid->dthps_prev = help; 14983 dtrace_deferred_pid = help; 14984 } 14985 14986 mutex_exit(&dtrace_lock); 14987 14988 } else if (dofhp != NULL) { 14989 /* 14990 * If the dtrace module is loaded and we have a particular 14991 * helper provider description, pass that off to the 14992 * meta provider. 14993 */ 14994 14995 mutex_exit(&dtrace_lock); 14996 14997 dtrace_helper_provide(dofhp, p->p_pid); 14998 14999 } else { 15000 /* 15001 * Otherwise, just pass all the helper provider descriptions 15002 * off to the meta provider. 15003 */ 15004 15005 int i; 15006 mutex_exit(&dtrace_lock); 15007 15008 for (i = 0; i < help->dthps_nprovs; i++) { 15009 dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, 15010 p->p_pid); 15011 } 15012 } 15013 15014 mutex_exit(&dtrace_meta_lock); 15015 } 15016 15017 static int 15018 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen) 15019 { 15020 dtrace_helpers_t *help; 15021 dtrace_helper_provider_t *hprov, **tmp_provs; 15022 uint_t tmp_maxprovs, i; 15023 15024 ASSERT(MUTEX_HELD(&dtrace_lock)); 15025 15026 help = curproc->p_dtrace_helpers; 15027 ASSERT(help != NULL); 15028 15029 /* 15030 * If we already have dtrace_helper_providers_max helper providers, 15031 * we're refuse to add a new one. 15032 */ 15033 if (help->dthps_nprovs >= dtrace_helper_providers_max) 15034 return (ENOSPC); 15035 15036 /* 15037 * Check to make sure this isn't a duplicate. 15038 */ 15039 for (i = 0; i < help->dthps_nprovs; i++) { 15040 if (dofhp->dofhp_addr == 15041 help->dthps_provs[i]->dthp_prov.dofhp_addr) 15042 return (EALREADY); 15043 } 15044 15045 hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP); 15046 hprov->dthp_prov = *dofhp; 15047 hprov->dthp_ref = 1; 15048 hprov->dthp_generation = gen; 15049 15050 /* 15051 * Allocate a bigger table for helper providers if it's already full. 15052 */ 15053 if (help->dthps_maxprovs == help->dthps_nprovs) { 15054 tmp_maxprovs = help->dthps_maxprovs; 15055 tmp_provs = help->dthps_provs; 15056 15057 if (help->dthps_maxprovs == 0) 15058 help->dthps_maxprovs = 2; 15059 else 15060 help->dthps_maxprovs *= 2; 15061 if (help->dthps_maxprovs > dtrace_helper_providers_max) 15062 help->dthps_maxprovs = dtrace_helper_providers_max; 15063 15064 ASSERT(tmp_maxprovs < help->dthps_maxprovs); 15065 15066 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs * 15067 sizeof (dtrace_helper_provider_t *), KM_SLEEP); 15068 15069 if (tmp_provs != NULL) { 15070 bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs * 15071 sizeof (dtrace_helper_provider_t *)); 15072 kmem_free(tmp_provs, tmp_maxprovs * 15073 sizeof (dtrace_helper_provider_t *)); 15074 } 15075 } 15076 15077 help->dthps_provs[help->dthps_nprovs] = hprov; 15078 help->dthps_nprovs++; 15079 15080 return (0); 15081 } 15082 15083 static void 15084 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov) 15085 { 15086 mutex_enter(&dtrace_lock); 15087 15088 if (--hprov->dthp_ref == 0) { 15089 dof_hdr_t *dof; 15090 mutex_exit(&dtrace_lock); 15091 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof; 15092 dtrace_dof_destroy(dof); 15093 kmem_free(hprov, sizeof (dtrace_helper_provider_t)); 15094 } else { 15095 mutex_exit(&dtrace_lock); 15096 } 15097 } 15098 15099 static int 15100 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec) 15101 { 15102 uintptr_t daddr = (uintptr_t)dof; 15103 dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; 15104 dof_provider_t *provider; 15105 dof_probe_t *probe; 15106 uint8_t *arg; 15107 char *strtab, *typestr; 15108 dof_stridx_t typeidx; 15109 size_t typesz; 15110 uint_t nprobes, j, k; 15111 15112 ASSERT(sec->dofs_type == DOF_SECT_PROVIDER); 15113 15114 if (sec->dofs_offset & (sizeof (uint_t) - 1)) { 15115 dtrace_dof_error(dof, "misaligned section offset"); 15116 return (-1); 15117 } 15118 15119 /* 15120 * The section needs to be large enough to contain the DOF provider 15121 * structure appropriate for the given version. 15122 */ 15123 if (sec->dofs_size < 15124 ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ? 15125 offsetof(dof_provider_t, dofpv_prenoffs) : 15126 sizeof (dof_provider_t))) { 15127 dtrace_dof_error(dof, "provider section too small"); 15128 return (-1); 15129 } 15130 15131 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); 15132 str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab); 15133 prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes); 15134 arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs); 15135 off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs); 15136 15137 if (str_sec == NULL || prb_sec == NULL || 15138 arg_sec == NULL || off_sec == NULL) 15139 return (-1); 15140 15141 enoff_sec = NULL; 15142 15143 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && 15144 provider->dofpv_prenoffs != DOF_SECT_NONE && 15145 (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS, 15146 provider->dofpv_prenoffs)) == NULL) 15147 return (-1); 15148 15149 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); 15150 15151 if (provider->dofpv_name >= str_sec->dofs_size || 15152 strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) { 15153 dtrace_dof_error(dof, "invalid provider name"); 15154 return (-1); 15155 } 15156 15157 if (prb_sec->dofs_entsize == 0 || 15158 prb_sec->dofs_entsize > prb_sec->dofs_size) { 15159 dtrace_dof_error(dof, "invalid entry size"); 15160 return (-1); 15161 } 15162 15163 if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) { 15164 dtrace_dof_error(dof, "misaligned entry size"); 15165 return (-1); 15166 } 15167 15168 if (off_sec->dofs_entsize != sizeof (uint32_t)) { 15169 dtrace_dof_error(dof, "invalid entry size"); 15170 return (-1); 15171 } 15172 15173 if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) { 15174 dtrace_dof_error(dof, "misaligned section offset"); 15175 return (-1); 15176 } 15177 15178 if (arg_sec->dofs_entsize != sizeof (uint8_t)) { 15179 dtrace_dof_error(dof, "invalid entry size"); 15180 return (-1); 15181 } 15182 15183 arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); 15184 15185 nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; 15186 15187 /* 15188 * Take a pass through the probes to check for errors. 15189 */ 15190 for (j = 0; j < nprobes; j++) { 15191 probe = (dof_probe_t *)(uintptr_t)(daddr + 15192 prb_sec->dofs_offset + j * prb_sec->dofs_entsize); 15193 15194 if (probe->dofpr_func >= str_sec->dofs_size) { 15195 dtrace_dof_error(dof, "invalid function name"); 15196 return (-1); 15197 } 15198 15199 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) { 15200 dtrace_dof_error(dof, "function name too long"); 15201 return (-1); 15202 } 15203 15204 if (probe->dofpr_name >= str_sec->dofs_size || 15205 strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) { 15206 dtrace_dof_error(dof, "invalid probe name"); 15207 return (-1); 15208 } 15209 15210 /* 15211 * The offset count must not wrap the index, and the offsets 15212 * must also not overflow the section's data. 15213 */ 15214 if (probe->dofpr_offidx + probe->dofpr_noffs < 15215 probe->dofpr_offidx || 15216 (probe->dofpr_offidx + probe->dofpr_noffs) * 15217 off_sec->dofs_entsize > off_sec->dofs_size) { 15218 dtrace_dof_error(dof, "invalid probe offset"); 15219 return (-1); 15220 } 15221 15222 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) { 15223 /* 15224 * If there's no is-enabled offset section, make sure 15225 * there aren't any is-enabled offsets. Otherwise 15226 * perform the same checks as for probe offsets 15227 * (immediately above). 15228 */ 15229 if (enoff_sec == NULL) { 15230 if (probe->dofpr_enoffidx != 0 || 15231 probe->dofpr_nenoffs != 0) { 15232 dtrace_dof_error(dof, "is-enabled " 15233 "offsets with null section"); 15234 return (-1); 15235 } 15236 } else if (probe->dofpr_enoffidx + 15237 probe->dofpr_nenoffs < probe->dofpr_enoffidx || 15238 (probe->dofpr_enoffidx + probe->dofpr_nenoffs) * 15239 enoff_sec->dofs_entsize > enoff_sec->dofs_size) { 15240 dtrace_dof_error(dof, "invalid is-enabled " 15241 "offset"); 15242 return (-1); 15243 } 15244 15245 if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) { 15246 dtrace_dof_error(dof, "zero probe and " 15247 "is-enabled offsets"); 15248 return (-1); 15249 } 15250 } else if (probe->dofpr_noffs == 0) { 15251 dtrace_dof_error(dof, "zero probe offsets"); 15252 return (-1); 15253 } 15254 15255 if (probe->dofpr_argidx + probe->dofpr_xargc < 15256 probe->dofpr_argidx || 15257 (probe->dofpr_argidx + probe->dofpr_xargc) * 15258 arg_sec->dofs_entsize > arg_sec->dofs_size) { 15259 dtrace_dof_error(dof, "invalid args"); 15260 return (-1); 15261 } 15262 15263 typeidx = probe->dofpr_nargv; 15264 typestr = strtab + probe->dofpr_nargv; 15265 for (k = 0; k < probe->dofpr_nargc; k++) { 15266 if (typeidx >= str_sec->dofs_size) { 15267 dtrace_dof_error(dof, "bad " 15268 "native argument type"); 15269 return (-1); 15270 } 15271 15272 typesz = strlen(typestr) + 1; 15273 if (typesz > DTRACE_ARGTYPELEN) { 15274 dtrace_dof_error(dof, "native " 15275 "argument type too long"); 15276 return (-1); 15277 } 15278 typeidx += typesz; 15279 typestr += typesz; 15280 } 15281 15282 typeidx = probe->dofpr_xargv; 15283 typestr = strtab + probe->dofpr_xargv; 15284 for (k = 0; k < probe->dofpr_xargc; k++) { 15285 if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) { 15286 dtrace_dof_error(dof, "bad " 15287 "native argument index"); 15288 return (-1); 15289 } 15290 15291 if (typeidx >= str_sec->dofs_size) { 15292 dtrace_dof_error(dof, "bad " 15293 "translated argument type"); 15294 return (-1); 15295 } 15296 15297 typesz = strlen(typestr) + 1; 15298 if (typesz > DTRACE_ARGTYPELEN) { 15299 dtrace_dof_error(dof, "translated argument " 15300 "type too long"); 15301 return (-1); 15302 } 15303 15304 typeidx += typesz; 15305 typestr += typesz; 15306 } 15307 } 15308 15309 return (0); 15310 } 15311 15312 static int 15313 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp) 15314 { 15315 dtrace_helpers_t *help; 15316 dtrace_vstate_t *vstate; 15317 dtrace_enabling_t *enab = NULL; 15318 int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1; 15319 uintptr_t daddr = (uintptr_t)dof; 15320 15321 ASSERT(MUTEX_HELD(&dtrace_lock)); 15322 15323 if ((help = curproc->p_dtrace_helpers) == NULL) 15324 help = dtrace_helpers_create(curproc); 15325 15326 vstate = &help->dthps_vstate; 15327 15328 if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab, 15329 dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) { 15330 dtrace_dof_destroy(dof); 15331 return (rv); 15332 } 15333 15334 /* 15335 * Look for helper providers and validate their descriptions. 15336 */ 15337 if (dhp != NULL) { 15338 for (i = 0; i < dof->dofh_secnum; i++) { 15339 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + 15340 dof->dofh_secoff + i * dof->dofh_secsize); 15341 15342 if (sec->dofs_type != DOF_SECT_PROVIDER) 15343 continue; 15344 15345 if (dtrace_helper_provider_validate(dof, sec) != 0) { 15346 dtrace_enabling_destroy(enab); 15347 dtrace_dof_destroy(dof); 15348 return (-1); 15349 } 15350 15351 nprovs++; 15352 } 15353 } 15354 15355 /* 15356 * Now we need to walk through the ECB descriptions in the enabling. 15357 */ 15358 for (i = 0; i < enab->dten_ndesc; i++) { 15359 dtrace_ecbdesc_t *ep = enab->dten_desc[i]; 15360 dtrace_probedesc_t *desc = &ep->dted_probe; 15361 15362 if (strcmp(desc->dtpd_provider, "dtrace") != 0) 15363 continue; 15364 15365 if (strcmp(desc->dtpd_mod, "helper") != 0) 15366 continue; 15367 15368 if (strcmp(desc->dtpd_func, "ustack") != 0) 15369 continue; 15370 15371 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK, 15372 ep)) != 0) { 15373 /* 15374 * Adding this helper action failed -- we are now going 15375 * to rip out the entire generation and return failure. 15376 */ 15377 (void) dtrace_helper_destroygen(help->dthps_generation); 15378 dtrace_enabling_destroy(enab); 15379 dtrace_dof_destroy(dof); 15380 return (-1); 15381 } 15382 15383 nhelpers++; 15384 } 15385 15386 if (nhelpers < enab->dten_ndesc) 15387 dtrace_dof_error(dof, "unmatched helpers"); 15388 15389 gen = help->dthps_generation++; 15390 dtrace_enabling_destroy(enab); 15391 15392 if (dhp != NULL && nprovs > 0) { 15393 /* 15394 * Now that this is in-kernel, we change the sense of the 15395 * members: dofhp_dof denotes the in-kernel copy of the DOF 15396 * and dofhp_addr denotes the address at user-level. 15397 */ 15398 dhp->dofhp_addr = dhp->dofhp_dof; 15399 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof; 15400 15401 if (dtrace_helper_provider_add(dhp, gen) == 0) { 15402 mutex_exit(&dtrace_lock); 15403 dtrace_helper_provider_register(curproc, help, dhp); 15404 mutex_enter(&dtrace_lock); 15405 15406 destroy = 0; 15407 } 15408 } 15409 15410 if (destroy) 15411 dtrace_dof_destroy(dof); 15412 15413 return (gen); 15414 } 15415 15416 static dtrace_helpers_t * 15417 dtrace_helpers_create(proc_t *p) 15418 { 15419 dtrace_helpers_t *help; 15420 15421 ASSERT(MUTEX_HELD(&dtrace_lock)); 15422 ASSERT(p->p_dtrace_helpers == NULL); 15423 15424 help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP); 15425 help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) * 15426 DTRACE_NHELPER_ACTIONS, KM_SLEEP); 15427 15428 p->p_dtrace_helpers = help; 15429 dtrace_helpers++; 15430 15431 return (help); 15432 } 15433 15434 static void 15435 dtrace_helpers_destroy(void) 15436 { 15437 dtrace_helpers_t *help; 15438 dtrace_vstate_t *vstate; 15439 proc_t *p = curproc; 15440 int i; 15441 15442 mutex_enter(&dtrace_lock); 15443 15444 ASSERT(p->p_dtrace_helpers != NULL); 15445 ASSERT(dtrace_helpers > 0); 15446 15447 help = p->p_dtrace_helpers; 15448 vstate = &help->dthps_vstate; 15449 15450 /* 15451 * We're now going to lose the help from this process. 15452 */ 15453 p->p_dtrace_helpers = NULL; 15454 dtrace_sync(); 15455 15456 /* 15457 * Destory the helper actions. 15458 */ 15459 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { 15460 dtrace_helper_action_t *h, *next; 15461 15462 for (h = help->dthps_actions[i]; h != NULL; h = next) { 15463 next = h->dtha_next; 15464 dtrace_helper_action_destroy(h, vstate); 15465 h = next; 15466 } 15467 } 15468 15469 mutex_exit(&dtrace_lock); 15470 15471 /* 15472 * Destroy the helper providers. 15473 */ 15474 if (help->dthps_maxprovs > 0) { 15475 mutex_enter(&dtrace_meta_lock); 15476 if (dtrace_meta_pid != NULL) { 15477 ASSERT(dtrace_deferred_pid == NULL); 15478 15479 for (i = 0; i < help->dthps_nprovs; i++) { 15480 dtrace_helper_provider_remove( 15481 &help->dthps_provs[i]->dthp_prov, p->p_pid); 15482 } 15483 } else { 15484 mutex_enter(&dtrace_lock); 15485 ASSERT(help->dthps_deferred == 0 || 15486 help->dthps_next != NULL || 15487 help->dthps_prev != NULL || 15488 help == dtrace_deferred_pid); 15489 15490 /* 15491 * Remove the helper from the deferred list. 15492 */ 15493 if (help->dthps_next != NULL) 15494 help->dthps_next->dthps_prev = help->dthps_prev; 15495 if (help->dthps_prev != NULL) 15496 help->dthps_prev->dthps_next = help->dthps_next; 15497 if (dtrace_deferred_pid == help) { 15498 dtrace_deferred_pid = help->dthps_next; 15499 ASSERT(help->dthps_prev == NULL); 15500 } 15501 15502 mutex_exit(&dtrace_lock); 15503 } 15504 15505 mutex_exit(&dtrace_meta_lock); 15506 15507 for (i = 0; i < help->dthps_nprovs; i++) { 15508 dtrace_helper_provider_destroy(help->dthps_provs[i]); 15509 } 15510 15511 kmem_free(help->dthps_provs, help->dthps_maxprovs * 15512 sizeof (dtrace_helper_provider_t *)); 15513 } 15514 15515 mutex_enter(&dtrace_lock); 15516 15517 dtrace_vstate_fini(&help->dthps_vstate); 15518 kmem_free(help->dthps_actions, 15519 sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS); 15520 kmem_free(help, sizeof (dtrace_helpers_t)); 15521 15522 --dtrace_helpers; 15523 mutex_exit(&dtrace_lock); 15524 } 15525 15526 static void 15527 dtrace_helpers_duplicate(proc_t *from, proc_t *to) 15528 { 15529 dtrace_helpers_t *help, *newhelp; 15530 dtrace_helper_action_t *helper, *new, *last; 15531 dtrace_difo_t *dp; 15532 dtrace_vstate_t *vstate; 15533 int i, j, sz, hasprovs = 0; 15534 15535 mutex_enter(&dtrace_lock); 15536 ASSERT(from->p_dtrace_helpers != NULL); 15537 ASSERT(dtrace_helpers > 0); 15538 15539 help = from->p_dtrace_helpers; 15540 newhelp = dtrace_helpers_create(to); 15541 ASSERT(to->p_dtrace_helpers != NULL); 15542 15543 newhelp->dthps_generation = help->dthps_generation; 15544 vstate = &newhelp->dthps_vstate; 15545 15546 /* 15547 * Duplicate the helper actions. 15548 */ 15549 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { 15550 if ((helper = help->dthps_actions[i]) == NULL) 15551 continue; 15552 15553 for (last = NULL; helper != NULL; helper = helper->dtha_next) { 15554 new = kmem_zalloc(sizeof (dtrace_helper_action_t), 15555 KM_SLEEP); 15556 new->dtha_generation = helper->dtha_generation; 15557 15558 if ((dp = helper->dtha_predicate) != NULL) { 15559 dp = dtrace_difo_duplicate(dp, vstate); 15560 new->dtha_predicate = dp; 15561 } 15562 15563 new->dtha_nactions = helper->dtha_nactions; 15564 sz = sizeof (dtrace_difo_t *) * new->dtha_nactions; 15565 new->dtha_actions = kmem_alloc(sz, KM_SLEEP); 15566 15567 for (j = 0; j < new->dtha_nactions; j++) { 15568 dtrace_difo_t *dp = helper->dtha_actions[j]; 15569 15570 ASSERT(dp != NULL); 15571 dp = dtrace_difo_duplicate(dp, vstate); 15572 new->dtha_actions[j] = dp; 15573 } 15574 15575 if (last != NULL) { 15576 last->dtha_next = new; 15577 } else { 15578 newhelp->dthps_actions[i] = new; 15579 } 15580 15581 last = new; 15582 } 15583 } 15584 15585 /* 15586 * Duplicate the helper providers and register them with the 15587 * DTrace framework. 15588 */ 15589 if (help->dthps_nprovs > 0) { 15590 newhelp->dthps_nprovs = help->dthps_nprovs; 15591 newhelp->dthps_maxprovs = help->dthps_nprovs; 15592 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs * 15593 sizeof (dtrace_helper_provider_t *), KM_SLEEP); 15594 for (i = 0; i < newhelp->dthps_nprovs; i++) { 15595 newhelp->dthps_provs[i] = help->dthps_provs[i]; 15596 newhelp->dthps_provs[i]->dthp_ref++; 15597 } 15598 15599 hasprovs = 1; 15600 } 15601 15602 mutex_exit(&dtrace_lock); 15603 15604 if (hasprovs) 15605 dtrace_helper_provider_register(to, newhelp, NULL); 15606 } 15607 15608 /* 15609 * DTrace Hook Functions 15610 */ 15611 static void 15612 dtrace_module_loaded(struct modctl *ctl) 15613 { 15614 dtrace_provider_t *prv; 15615 15616 mutex_enter(&dtrace_provider_lock); 15617 mutex_enter(&mod_lock); 15618 15619 ASSERT(ctl->mod_busy); 15620 15621 /* 15622 * We're going to call each providers per-module provide operation 15623 * specifying only this module. 15624 */ 15625 for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next) 15626 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); 15627 15628 mutex_exit(&mod_lock); 15629 mutex_exit(&dtrace_provider_lock); 15630 15631 /* 15632 * If we have any retained enablings, we need to match against them. 15633 * Enabling probes requires that cpu_lock be held, and we cannot hold 15634 * cpu_lock here -- it is legal for cpu_lock to be held when loading a 15635 * module. (In particular, this happens when loading scheduling 15636 * classes.) So if we have any retained enablings, we need to dispatch 15637 * our task queue to do the match for us. 15638 */ 15639 mutex_enter(&dtrace_lock); 15640 15641 if (dtrace_retained == NULL) { 15642 mutex_exit(&dtrace_lock); 15643 return; 15644 } 15645 15646 (void) taskq_dispatch(dtrace_taskq, 15647 (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP); 15648 15649 mutex_exit(&dtrace_lock); 15650 15651 /* 15652 * And now, for a little heuristic sleaze: in general, we want to 15653 * match modules as soon as they load. However, we cannot guarantee 15654 * this, because it would lead us to the lock ordering violation 15655 * outlined above. The common case, of course, is that cpu_lock is 15656 * _not_ held -- so we delay here for a clock tick, hoping that that's 15657 * long enough for the task queue to do its work. If it's not, it's 15658 * not a serious problem -- it just means that the module that we 15659 * just loaded may not be immediately instrumentable. 15660 */ 15661 delay(1); 15662 } 15663 15664 static void 15665 dtrace_module_unloaded(struct modctl *ctl) 15666 { 15667 dtrace_probe_t template, *probe, *first, *next; 15668 dtrace_provider_t *prov; 15669 15670 template.dtpr_mod = ctl->mod_modname; 15671 15672 mutex_enter(&dtrace_provider_lock); 15673 mutex_enter(&mod_lock); 15674 mutex_enter(&dtrace_lock); 15675 15676 if (dtrace_bymod == NULL) { 15677 /* 15678 * The DTrace module is loaded (obviously) but not attached; 15679 * we don't have any work to do. 15680 */ 15681 mutex_exit(&dtrace_provider_lock); 15682 mutex_exit(&mod_lock); 15683 mutex_exit(&dtrace_lock); 15684 return; 15685 } 15686 15687 for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template); 15688 probe != NULL; probe = probe->dtpr_nextmod) { 15689 if (probe->dtpr_ecb != NULL) { 15690 mutex_exit(&dtrace_provider_lock); 15691 mutex_exit(&mod_lock); 15692 mutex_exit(&dtrace_lock); 15693 15694 /* 15695 * This shouldn't _actually_ be possible -- we're 15696 * unloading a module that has an enabled probe in it. 15697 * (It's normally up to the provider to make sure that 15698 * this can't happen.) However, because dtps_enable() 15699 * doesn't have a failure mode, there can be an 15700 * enable/unload race. Upshot: we don't want to 15701 * assert, but we're not going to disable the 15702 * probe, either. 15703 */ 15704 if (dtrace_err_verbose) { 15705 cmn_err(CE_WARN, "unloaded module '%s' had " 15706 "enabled probes", ctl->mod_modname); 15707 } 15708 15709 return; 15710 } 15711 } 15712 15713 probe = first; 15714 15715 for (first = NULL; probe != NULL; probe = next) { 15716 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe); 15717 15718 dtrace_probes[probe->dtpr_id - 1] = NULL; 15719 15720 next = probe->dtpr_nextmod; 15721 dtrace_hash_remove(dtrace_bymod, probe); 15722 dtrace_hash_remove(dtrace_byfunc, probe); 15723 dtrace_hash_remove(dtrace_byname, probe); 15724 15725 if (first == NULL) { 15726 first = probe; 15727 probe->dtpr_nextmod = NULL; 15728 } else { 15729 probe->dtpr_nextmod = first; 15730 first = probe; 15731 } 15732 } 15733 15734 /* 15735 * We've removed all of the module's probes from the hash chains and 15736 * from the probe array. Now issue a dtrace_sync() to be sure that 15737 * everyone has cleared out from any probe array processing. 15738 */ 15739 dtrace_sync(); 15740 15741 for (probe = first; probe != NULL; probe = first) { 15742 first = probe->dtpr_nextmod; 15743 prov = probe->dtpr_provider; 15744 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id, 15745 probe->dtpr_arg); 15746 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); 15747 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); 15748 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); 15749 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1); 15750 kmem_free(probe, sizeof (dtrace_probe_t)); 15751 } 15752 15753 mutex_exit(&dtrace_lock); 15754 mutex_exit(&mod_lock); 15755 mutex_exit(&dtrace_provider_lock); 15756 } 15757 15758 void 15759 dtrace_suspend(void) 15760 { 15761 dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend)); 15762 } 15763 15764 void 15765 dtrace_resume(void) 15766 { 15767 dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume)); 15768 } 15769 15770 static int 15771 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu) 15772 { 15773 ASSERT(MUTEX_HELD(&cpu_lock)); 15774 mutex_enter(&dtrace_lock); 15775 15776 switch (what) { 15777 case CPU_CONFIG: { 15778 dtrace_state_t *state; 15779 dtrace_optval_t *opt, rs, c; 15780 15781 /* 15782 * For now, we only allocate a new buffer for anonymous state. 15783 */ 15784 if ((state = dtrace_anon.dta_state) == NULL) 15785 break; 15786 15787 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) 15788 break; 15789 15790 opt = state->dts_options; 15791 c = opt[DTRACEOPT_CPU]; 15792 15793 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu) 15794 break; 15795 15796 /* 15797 * Regardless of what the actual policy is, we're going to 15798 * temporarily set our resize policy to be manual. We're 15799 * also going to temporarily set our CPU option to denote 15800 * the newly configured CPU. 15801 */ 15802 rs = opt[DTRACEOPT_BUFRESIZE]; 15803 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL; 15804 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu; 15805 15806 (void) dtrace_state_buffers(state); 15807 15808 opt[DTRACEOPT_BUFRESIZE] = rs; 15809 opt[DTRACEOPT_CPU] = c; 15810 15811 break; 15812 } 15813 15814 case CPU_UNCONFIG: 15815 /* 15816 * We don't free the buffer in the CPU_UNCONFIG case. (The 15817 * buffer will be freed when the consumer exits.) 15818 */ 15819 break; 15820 15821 default: 15822 break; 15823 } 15824 15825 mutex_exit(&dtrace_lock); 15826 return (0); 15827 } 15828 15829 static void 15830 dtrace_cpu_setup_initial(processorid_t cpu) 15831 { 15832 (void) dtrace_cpu_setup(CPU_CONFIG, cpu); 15833 } 15834 15835 static void 15836 dtrace_toxrange_add(uintptr_t base, uintptr_t limit) 15837 { 15838 if (dtrace_toxranges >= dtrace_toxranges_max) { 15839 int osize, nsize; 15840 dtrace_toxrange_t *range; 15841 15842 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t); 15843 15844 if (osize == 0) { 15845 ASSERT(dtrace_toxrange == NULL); 15846 ASSERT(dtrace_toxranges_max == 0); 15847 dtrace_toxranges_max = 1; 15848 } else { 15849 dtrace_toxranges_max <<= 1; 15850 } 15851 15852 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t); 15853 range = kmem_zalloc(nsize, KM_SLEEP); 15854 15855 if (dtrace_toxrange != NULL) { 15856 ASSERT(osize != 0); 15857 bcopy(dtrace_toxrange, range, osize); 15858 kmem_free(dtrace_toxrange, osize); 15859 } 15860 15861 dtrace_toxrange = range; 15862 } 15863 15864 ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL); 15865 ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL); 15866 15867 dtrace_toxrange[dtrace_toxranges].dtt_base = base; 15868 dtrace_toxrange[dtrace_toxranges].dtt_limit = limit; 15869 dtrace_toxranges++; 15870 } 15871 15872 static void 15873 dtrace_getf_barrier() 15874 { 15875 /* 15876 * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings 15877 * that contain calls to getf(), this routine will be called on every 15878 * closef() before either the underlying vnode is released or the 15879 * file_t itself is freed. By the time we are here, it is essential 15880 * that the file_t can no longer be accessed from a call to getf() 15881 * in probe context -- that assures that a dtrace_sync() can be used 15882 * to clear out any enablings referring to the old structures. 15883 */ 15884 if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 || 15885 kcred->cr_zone->zone_dtrace_getf != 0) 15886 dtrace_sync(); 15887 } 15888 15889 /* 15890 * DTrace Driver Cookbook Functions 15891 */ 15892 /*ARGSUSED*/ 15893 static int 15894 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 15895 { 15896 dtrace_provider_id_t id; 15897 dtrace_state_t *state = NULL; 15898 dtrace_enabling_t *enab; 15899 15900 mutex_enter(&cpu_lock); 15901 mutex_enter(&dtrace_provider_lock); 15902 mutex_enter(&dtrace_lock); 15903 15904 if (ddi_soft_state_init(&dtrace_softstate, 15905 sizeof (dtrace_state_t), 0) != 0) { 15906 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state"); 15907 mutex_exit(&cpu_lock); 15908 mutex_exit(&dtrace_provider_lock); 15909 mutex_exit(&dtrace_lock); 15910 return (DDI_FAILURE); 15911 } 15912 15913 if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR, 15914 DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE || 15915 ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR, 15916 DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) { 15917 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes"); 15918 ddi_remove_minor_node(devi, NULL); 15919 ddi_soft_state_fini(&dtrace_softstate); 15920 mutex_exit(&cpu_lock); 15921 mutex_exit(&dtrace_provider_lock); 15922 mutex_exit(&dtrace_lock); 15923 return (DDI_FAILURE); 15924 } 15925 15926 ddi_report_dev(devi); 15927 dtrace_devi = devi; 15928 15929 dtrace_modload = dtrace_module_loaded; 15930 dtrace_modunload = dtrace_module_unloaded; 15931 dtrace_cpu_init = dtrace_cpu_setup_initial; 15932 dtrace_helpers_cleanup = dtrace_helpers_destroy; 15933 dtrace_helpers_fork = dtrace_helpers_duplicate; 15934 dtrace_cpustart_init = dtrace_suspend; 15935 dtrace_cpustart_fini = dtrace_resume; 15936 dtrace_debugger_init = dtrace_suspend; 15937 dtrace_debugger_fini = dtrace_resume; 15938 15939 register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); 15940 15941 ASSERT(MUTEX_HELD(&cpu_lock)); 15942 15943 dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1, 15944 NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 15945 dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE, 15946 UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0, 15947 VM_SLEEP | VMC_IDENTIFIER); 15948 dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri, 15949 1, INT_MAX, 0); 15950 15951 dtrace_state_cache = kmem_cache_create("dtrace_state_cache", 15952 sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN, 15953 NULL, NULL, NULL, NULL, NULL, 0); 15954 15955 ASSERT(MUTEX_HELD(&cpu_lock)); 15956 dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod), 15957 offsetof(dtrace_probe_t, dtpr_nextmod), 15958 offsetof(dtrace_probe_t, dtpr_prevmod)); 15959 15960 dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func), 15961 offsetof(dtrace_probe_t, dtpr_nextfunc), 15962 offsetof(dtrace_probe_t, dtpr_prevfunc)); 15963 15964 dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name), 15965 offsetof(dtrace_probe_t, dtpr_nextname), 15966 offsetof(dtrace_probe_t, dtpr_prevname)); 15967 15968 if (dtrace_retain_max < 1) { 15969 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; " 15970 "setting to 1", dtrace_retain_max); 15971 dtrace_retain_max = 1; 15972 } 15973 15974 /* 15975 * Now discover our toxic ranges. 15976 */ 15977 dtrace_toxic_ranges(dtrace_toxrange_add); 15978 15979 /* 15980 * Before we register ourselves as a provider to our own framework, 15981 * we would like to assert that dtrace_provider is NULL -- but that's 15982 * not true if we were loaded as a dependency of a DTrace provider. 15983 * Once we've registered, we can assert that dtrace_provider is our 15984 * pseudo provider. 15985 */ 15986 (void) dtrace_register("dtrace", &dtrace_provider_attr, 15987 DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id); 15988 15989 ASSERT(dtrace_provider != NULL); 15990 ASSERT((dtrace_provider_id_t)dtrace_provider == id); 15991 15992 dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t) 15993 dtrace_provider, NULL, NULL, "BEGIN", 0, NULL); 15994 dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t) 15995 dtrace_provider, NULL, NULL, "END", 0, NULL); 15996 dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) 15997 dtrace_provider, NULL, NULL, "ERROR", 1, NULL); 15998 15999 dtrace_anon_property(); 16000 mutex_exit(&cpu_lock); 16001 16002 /* 16003 * If there are already providers, we must ask them to provide their 16004 * probes, and then match any anonymous enabling against them. Note 16005 * that there should be no other retained enablings at this time: 16006 * the only retained enablings at this time should be the anonymous 16007 * enabling. 16008 */ 16009 if (dtrace_anon.dta_enabling != NULL) { 16010 ASSERT(dtrace_retained == dtrace_anon.dta_enabling); 16011 16012 dtrace_enabling_provide(NULL); 16013 state = dtrace_anon.dta_state; 16014 16015 /* 16016 * We couldn't hold cpu_lock across the above call to 16017 * dtrace_enabling_provide(), but we must hold it to actually 16018 * enable the probes. We have to drop all of our locks, pick 16019 * up cpu_lock, and regain our locks before matching the 16020 * retained anonymous enabling. 16021 */ 16022 mutex_exit(&dtrace_lock); 16023 mutex_exit(&dtrace_provider_lock); 16024 16025 mutex_enter(&cpu_lock); 16026 mutex_enter(&dtrace_provider_lock); 16027 mutex_enter(&dtrace_lock); 16028 16029 if ((enab = dtrace_anon.dta_enabling) != NULL) 16030 (void) dtrace_enabling_match(enab, NULL); 16031 16032 mutex_exit(&cpu_lock); 16033 } 16034 16035 mutex_exit(&dtrace_lock); 16036 mutex_exit(&dtrace_provider_lock); 16037 16038 if (state != NULL) { 16039 /* 16040 * If we created any anonymous state, set it going now. 16041 */ 16042 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon); 16043 } 16044 16045 return (DDI_SUCCESS); 16046 } 16047 16048 /*ARGSUSED*/ 16049 static int 16050 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 16051 { 16052 dtrace_state_t *state; 16053 uint32_t priv; 16054 uid_t uid; 16055 zoneid_t zoneid; 16056 16057 if (getminor(*devp) == DTRACEMNRN_HELPER) 16058 return (0); 16059 16060 /* 16061 * If this wasn't an open with the "helper" minor, then it must be 16062 * the "dtrace" minor. 16063 */ 16064 if (getminor(*devp) != DTRACEMNRN_DTRACE) 16065 return (ENXIO); 16066 16067 /* 16068 * If no DTRACE_PRIV_* bits are set in the credential, then the 16069 * caller lacks sufficient permission to do anything with DTrace. 16070 */ 16071 dtrace_cred2priv(cred_p, &priv, &uid, &zoneid); 16072 if (priv == DTRACE_PRIV_NONE) 16073 return (EACCES); 16074 16075 /* 16076 * Ask all providers to provide all their probes. 16077 */ 16078 mutex_enter(&dtrace_provider_lock); 16079 dtrace_probe_provide(NULL, NULL); 16080 mutex_exit(&dtrace_provider_lock); 16081 16082 mutex_enter(&cpu_lock); 16083 mutex_enter(&dtrace_lock); 16084 dtrace_opens++; 16085 dtrace_membar_producer(); 16086 16087 /* 16088 * If the kernel debugger is active (that is, if the kernel debugger 16089 * modified text in some way), we won't allow the open. 16090 */ 16091 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { 16092 dtrace_opens--; 16093 mutex_exit(&cpu_lock); 16094 mutex_exit(&dtrace_lock); 16095 return (EBUSY); 16096 } 16097 16098 if (dtrace_helptrace_enable && dtrace_helptrace_buffer == NULL) { 16099 /* 16100 * If DTrace helper tracing is enabled, we need to allocate the 16101 * trace buffer and initialize the values. 16102 */ 16103 dtrace_helptrace_buffer = 16104 kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP); 16105 dtrace_helptrace_next = 0; 16106 dtrace_helptrace_wrapped = 0; 16107 dtrace_helptrace_enable = 0; 16108 } 16109 16110 state = dtrace_state_create(devp, cred_p); 16111 mutex_exit(&cpu_lock); 16112 16113 if (state == NULL) { 16114 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) 16115 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); 16116 mutex_exit(&dtrace_lock); 16117 return (EAGAIN); 16118 } 16119 16120 mutex_exit(&dtrace_lock); 16121 16122 return (0); 16123 } 16124 16125 /*ARGSUSED*/ 16126 static int 16127 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 16128 { 16129 minor_t minor = getminor(dev); 16130 dtrace_state_t *state; 16131 dtrace_helptrace_t *buf = NULL; 16132 16133 if (minor == DTRACEMNRN_HELPER) 16134 return (0); 16135 16136 state = ddi_get_soft_state(dtrace_softstate, minor); 16137 16138 mutex_enter(&cpu_lock); 16139 mutex_enter(&dtrace_lock); 16140 16141 if (state->dts_anon) { 16142 /* 16143 * There is anonymous state. Destroy that first. 16144 */ 16145 ASSERT(dtrace_anon.dta_state == NULL); 16146 dtrace_state_destroy(state->dts_anon); 16147 } 16148 16149 if (dtrace_helptrace_disable) { 16150 /* 16151 * If we have been told to disable helper tracing, set the 16152 * buffer to NULL before calling into dtrace_state_destroy(); 16153 * we take advantage of its dtrace_sync() to know that no 16154 * CPU is in probe context with enabled helper tracing 16155 * after it returns. 16156 */ 16157 buf = dtrace_helptrace_buffer; 16158 dtrace_helptrace_buffer = NULL; 16159 } 16160 16161 dtrace_state_destroy(state); 16162 ASSERT(dtrace_opens > 0); 16163 16164 /* 16165 * Only relinquish control of the kernel debugger interface when there 16166 * are no consumers and no anonymous enablings. 16167 */ 16168 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) 16169 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); 16170 16171 if (buf != NULL) { 16172 kmem_free(buf, dtrace_helptrace_bufsize); 16173 dtrace_helptrace_disable = 0; 16174 } 16175 16176 mutex_exit(&dtrace_lock); 16177 mutex_exit(&cpu_lock); 16178 16179 return (0); 16180 } 16181 16182 /*ARGSUSED*/ 16183 static int 16184 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv) 16185 { 16186 int rval; 16187 dof_helper_t help, *dhp = NULL; 16188 16189 switch (cmd) { 16190 case DTRACEHIOC_ADDDOF: 16191 if (copyin((void *)arg, &help, sizeof (help)) != 0) { 16192 dtrace_dof_error(NULL, "failed to copyin DOF helper"); 16193 return (EFAULT); 16194 } 16195 16196 dhp = &help; 16197 arg = (intptr_t)help.dofhp_dof; 16198 /*FALLTHROUGH*/ 16199 16200 case DTRACEHIOC_ADD: { 16201 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval); 16202 16203 if (dof == NULL) 16204 return (rval); 16205 16206 mutex_enter(&dtrace_lock); 16207 16208 /* 16209 * dtrace_helper_slurp() takes responsibility for the dof -- 16210 * it may free it now or it may save it and free it later. 16211 */ 16212 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) { 16213 *rv = rval; 16214 rval = 0; 16215 } else { 16216 rval = EINVAL; 16217 } 16218 16219 mutex_exit(&dtrace_lock); 16220 return (rval); 16221 } 16222 16223 case DTRACEHIOC_REMOVE: { 16224 mutex_enter(&dtrace_lock); 16225 rval = dtrace_helper_destroygen(arg); 16226 mutex_exit(&dtrace_lock); 16227 16228 return (rval); 16229 } 16230 16231 default: 16232 break; 16233 } 16234 16235 return (ENOTTY); 16236 } 16237 16238 /*ARGSUSED*/ 16239 static int 16240 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 16241 { 16242 minor_t minor = getminor(dev); 16243 dtrace_state_t *state; 16244 int rval; 16245 16246 if (minor == DTRACEMNRN_HELPER) 16247 return (dtrace_ioctl_helper(cmd, arg, rv)); 16248 16249 state = ddi_get_soft_state(dtrace_softstate, minor); 16250 16251 if (state->dts_anon) { 16252 ASSERT(dtrace_anon.dta_state == NULL); 16253 state = state->dts_anon; 16254 } 16255 16256 switch (cmd) { 16257 case DTRACEIOC_PROVIDER: { 16258 dtrace_providerdesc_t pvd; 16259 dtrace_provider_t *pvp; 16260 16261 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0) 16262 return (EFAULT); 16263 16264 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0'; 16265 mutex_enter(&dtrace_provider_lock); 16266 16267 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) { 16268 if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0) 16269 break; 16270 } 16271 16272 mutex_exit(&dtrace_provider_lock); 16273 16274 if (pvp == NULL) 16275 return (ESRCH); 16276 16277 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t)); 16278 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t)); 16279 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0) 16280 return (EFAULT); 16281 16282 return (0); 16283 } 16284 16285 case DTRACEIOC_EPROBE: { 16286 dtrace_eprobedesc_t epdesc; 16287 dtrace_ecb_t *ecb; 16288 dtrace_action_t *act; 16289 void *buf; 16290 size_t size; 16291 uintptr_t dest; 16292 int nrecs; 16293 16294 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0) 16295 return (EFAULT); 16296 16297 mutex_enter(&dtrace_lock); 16298 16299 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) { 16300 mutex_exit(&dtrace_lock); 16301 return (EINVAL); 16302 } 16303 16304 if (ecb->dte_probe == NULL) { 16305 mutex_exit(&dtrace_lock); 16306 return (EINVAL); 16307 } 16308 16309 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id; 16310 epdesc.dtepd_uarg = ecb->dte_uarg; 16311 epdesc.dtepd_size = ecb->dte_size; 16312 16313 nrecs = epdesc.dtepd_nrecs; 16314 epdesc.dtepd_nrecs = 0; 16315 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 16316 if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) 16317 continue; 16318 16319 epdesc.dtepd_nrecs++; 16320 } 16321 16322 /* 16323 * Now that we have the size, we need to allocate a temporary 16324 * buffer in which to store the complete description. We need 16325 * the temporary buffer to be able to drop dtrace_lock() 16326 * across the copyout(), below. 16327 */ 16328 size = sizeof (dtrace_eprobedesc_t) + 16329 (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t)); 16330 16331 buf = kmem_alloc(size, KM_SLEEP); 16332 dest = (uintptr_t)buf; 16333 16334 bcopy(&epdesc, (void *)dest, sizeof (epdesc)); 16335 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]); 16336 16337 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 16338 if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) 16339 continue; 16340 16341 if (nrecs-- == 0) 16342 break; 16343 16344 bcopy(&act->dta_rec, (void *)dest, 16345 sizeof (dtrace_recdesc_t)); 16346 dest += sizeof (dtrace_recdesc_t); 16347 } 16348 16349 mutex_exit(&dtrace_lock); 16350 16351 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) { 16352 kmem_free(buf, size); 16353 return (EFAULT); 16354 } 16355 16356 kmem_free(buf, size); 16357 return (0); 16358 } 16359 16360 case DTRACEIOC_AGGDESC: { 16361 dtrace_aggdesc_t aggdesc; 16362 dtrace_action_t *act; 16363 dtrace_aggregation_t *agg; 16364 int nrecs; 16365 uint32_t offs; 16366 dtrace_recdesc_t *lrec; 16367 void *buf; 16368 size_t size; 16369 uintptr_t dest; 16370 16371 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0) 16372 return (EFAULT); 16373 16374 mutex_enter(&dtrace_lock); 16375 16376 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) { 16377 mutex_exit(&dtrace_lock); 16378 return (EINVAL); 16379 } 16380 16381 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid; 16382 16383 nrecs = aggdesc.dtagd_nrecs; 16384 aggdesc.dtagd_nrecs = 0; 16385 16386 offs = agg->dtag_base; 16387 lrec = &agg->dtag_action.dta_rec; 16388 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs; 16389 16390 for (act = agg->dtag_first; ; act = act->dta_next) { 16391 ASSERT(act->dta_intuple || 16392 DTRACEACT_ISAGG(act->dta_kind)); 16393 16394 /* 16395 * If this action has a record size of zero, it 16396 * denotes an argument to the aggregating action. 16397 * Because the presence of this record doesn't (or 16398 * shouldn't) affect the way the data is interpreted, 16399 * we don't copy it out to save user-level the 16400 * confusion of dealing with a zero-length record. 16401 */ 16402 if (act->dta_rec.dtrd_size == 0) { 16403 ASSERT(agg->dtag_hasarg); 16404 continue; 16405 } 16406 16407 aggdesc.dtagd_nrecs++; 16408 16409 if (act == &agg->dtag_action) 16410 break; 16411 } 16412 16413 /* 16414 * Now that we have the size, we need to allocate a temporary 16415 * buffer in which to store the complete description. We need 16416 * the temporary buffer to be able to drop dtrace_lock() 16417 * across the copyout(), below. 16418 */ 16419 size = sizeof (dtrace_aggdesc_t) + 16420 (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t)); 16421 16422 buf = kmem_alloc(size, KM_SLEEP); 16423 dest = (uintptr_t)buf; 16424 16425 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc)); 16426 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]); 16427 16428 for (act = agg->dtag_first; ; act = act->dta_next) { 16429 dtrace_recdesc_t rec = act->dta_rec; 16430 16431 /* 16432 * See the comment in the above loop for why we pass 16433 * over zero-length records. 16434 */ 16435 if (rec.dtrd_size == 0) { 16436 ASSERT(agg->dtag_hasarg); 16437 continue; 16438 } 16439 16440 if (nrecs-- == 0) 16441 break; 16442 16443 rec.dtrd_offset -= offs; 16444 bcopy(&rec, (void *)dest, sizeof (rec)); 16445 dest += sizeof (dtrace_recdesc_t); 16446 16447 if (act == &agg->dtag_action) 16448 break; 16449 } 16450 16451 mutex_exit(&dtrace_lock); 16452 16453 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) { 16454 kmem_free(buf, size); 16455 return (EFAULT); 16456 } 16457 16458 kmem_free(buf, size); 16459 return (0); 16460 } 16461 16462 case DTRACEIOC_ENABLE: { 16463 dof_hdr_t *dof; 16464 dtrace_enabling_t *enab = NULL; 16465 dtrace_vstate_t *vstate; 16466 int err = 0; 16467 16468 *rv = 0; 16469 16470 /* 16471 * If a NULL argument has been passed, we take this as our 16472 * cue to reevaluate our enablings. 16473 */ 16474 if (arg == NULL) { 16475 dtrace_enabling_matchall(); 16476 16477 return (0); 16478 } 16479 16480 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL) 16481 return (rval); 16482 16483 mutex_enter(&cpu_lock); 16484 mutex_enter(&dtrace_lock); 16485 vstate = &state->dts_vstate; 16486 16487 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { 16488 mutex_exit(&dtrace_lock); 16489 mutex_exit(&cpu_lock); 16490 dtrace_dof_destroy(dof); 16491 return (EBUSY); 16492 } 16493 16494 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) { 16495 mutex_exit(&dtrace_lock); 16496 mutex_exit(&cpu_lock); 16497 dtrace_dof_destroy(dof); 16498 return (EINVAL); 16499 } 16500 16501 if ((rval = dtrace_dof_options(dof, state)) != 0) { 16502 dtrace_enabling_destroy(enab); 16503 mutex_exit(&dtrace_lock); 16504 mutex_exit(&cpu_lock); 16505 dtrace_dof_destroy(dof); 16506 return (rval); 16507 } 16508 16509 if ((err = dtrace_enabling_match(enab, rv)) == 0) { 16510 err = dtrace_enabling_retain(enab); 16511 } else { 16512 dtrace_enabling_destroy(enab); 16513 } 16514 16515 mutex_exit(&cpu_lock); 16516 mutex_exit(&dtrace_lock); 16517 dtrace_dof_destroy(dof); 16518 16519 return (err); 16520 } 16521 16522 case DTRACEIOC_REPLICATE: { 16523 dtrace_repldesc_t desc; 16524 dtrace_probedesc_t *match = &desc.dtrpd_match; 16525 dtrace_probedesc_t *create = &desc.dtrpd_create; 16526 int err; 16527 16528 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 16529 return (EFAULT); 16530 16531 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; 16532 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; 16533 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; 16534 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; 16535 16536 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; 16537 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; 16538 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; 16539 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; 16540 16541 mutex_enter(&dtrace_lock); 16542 err = dtrace_enabling_replicate(state, match, create); 16543 mutex_exit(&dtrace_lock); 16544 16545 return (err); 16546 } 16547 16548 case DTRACEIOC_PROBEMATCH: 16549 case DTRACEIOC_PROBES: { 16550 dtrace_probe_t *probe = NULL; 16551 dtrace_probedesc_t desc; 16552 dtrace_probekey_t pkey; 16553 dtrace_id_t i; 16554 int m = 0; 16555 uint32_t priv; 16556 uid_t uid; 16557 zoneid_t zoneid; 16558 16559 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 16560 return (EFAULT); 16561 16562 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; 16563 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; 16564 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; 16565 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0'; 16566 16567 /* 16568 * Before we attempt to match this probe, we want to give 16569 * all providers the opportunity to provide it. 16570 */ 16571 if (desc.dtpd_id == DTRACE_IDNONE) { 16572 mutex_enter(&dtrace_provider_lock); 16573 dtrace_probe_provide(&desc, NULL); 16574 mutex_exit(&dtrace_provider_lock); 16575 desc.dtpd_id++; 16576 } 16577 16578 if (cmd == DTRACEIOC_PROBEMATCH) { 16579 dtrace_probekey(&desc, &pkey); 16580 pkey.dtpk_id = DTRACE_IDNONE; 16581 } 16582 16583 dtrace_cred2priv(cr, &priv, &uid, &zoneid); 16584 16585 mutex_enter(&dtrace_lock); 16586 16587 if (cmd == DTRACEIOC_PROBEMATCH) { 16588 for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) { 16589 if ((probe = dtrace_probes[i - 1]) != NULL && 16590 (m = dtrace_match_probe(probe, &pkey, 16591 priv, uid, zoneid)) != 0) 16592 break; 16593 } 16594 16595 if (m < 0) { 16596 mutex_exit(&dtrace_lock); 16597 return (EINVAL); 16598 } 16599 16600 } else { 16601 for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) { 16602 if ((probe = dtrace_probes[i - 1]) != NULL && 16603 dtrace_match_priv(probe, priv, uid, zoneid)) 16604 break; 16605 } 16606 } 16607 16608 if (probe == NULL) { 16609 mutex_exit(&dtrace_lock); 16610 return (ESRCH); 16611 } 16612 16613 dtrace_probe_description(probe, &desc); 16614 mutex_exit(&dtrace_lock); 16615 16616 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 16617 return (EFAULT); 16618 16619 return (0); 16620 } 16621 16622 case DTRACEIOC_PROBEARG: { 16623 dtrace_argdesc_t desc; 16624 dtrace_probe_t *probe; 16625 dtrace_provider_t *prov; 16626 16627 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 16628 return (EFAULT); 16629 16630 if (desc.dtargd_id == DTRACE_IDNONE) 16631 return (EINVAL); 16632 16633 if (desc.dtargd_ndx == DTRACE_ARGNONE) 16634 return (EINVAL); 16635 16636 mutex_enter(&dtrace_provider_lock); 16637 mutex_enter(&mod_lock); 16638 mutex_enter(&dtrace_lock); 16639 16640 if (desc.dtargd_id > dtrace_nprobes) { 16641 mutex_exit(&dtrace_lock); 16642 mutex_exit(&mod_lock); 16643 mutex_exit(&dtrace_provider_lock); 16644 return (EINVAL); 16645 } 16646 16647 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) { 16648 mutex_exit(&dtrace_lock); 16649 mutex_exit(&mod_lock); 16650 mutex_exit(&dtrace_provider_lock); 16651 return (EINVAL); 16652 } 16653 16654 mutex_exit(&dtrace_lock); 16655 16656 prov = probe->dtpr_provider; 16657 16658 if (prov->dtpv_pops.dtps_getargdesc == NULL) { 16659 /* 16660 * There isn't any typed information for this probe. 16661 * Set the argument number to DTRACE_ARGNONE. 16662 */ 16663 desc.dtargd_ndx = DTRACE_ARGNONE; 16664 } else { 16665 desc.dtargd_native[0] = '\0'; 16666 desc.dtargd_xlate[0] = '\0'; 16667 desc.dtargd_mapping = desc.dtargd_ndx; 16668 16669 prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg, 16670 probe->dtpr_id, probe->dtpr_arg, &desc); 16671 } 16672 16673 mutex_exit(&mod_lock); 16674 mutex_exit(&dtrace_provider_lock); 16675 16676 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 16677 return (EFAULT); 16678 16679 return (0); 16680 } 16681 16682 case DTRACEIOC_GO: { 16683 processorid_t cpuid; 16684 rval = dtrace_state_go(state, &cpuid); 16685 16686 if (rval != 0) 16687 return (rval); 16688 16689 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) 16690 return (EFAULT); 16691 16692 return (0); 16693 } 16694 16695 case DTRACEIOC_STOP: { 16696 processorid_t cpuid; 16697 16698 mutex_enter(&dtrace_lock); 16699 rval = dtrace_state_stop(state, &cpuid); 16700 mutex_exit(&dtrace_lock); 16701 16702 if (rval != 0) 16703 return (rval); 16704 16705 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) 16706 return (EFAULT); 16707 16708 return (0); 16709 } 16710 16711 case DTRACEIOC_DOFGET: { 16712 dof_hdr_t hdr, *dof; 16713 uint64_t len; 16714 16715 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0) 16716 return (EFAULT); 16717 16718 mutex_enter(&dtrace_lock); 16719 dof = dtrace_dof_create(state); 16720 mutex_exit(&dtrace_lock); 16721 16722 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz); 16723 rval = copyout(dof, (void *)arg, len); 16724 dtrace_dof_destroy(dof); 16725 16726 return (rval == 0 ? 0 : EFAULT); 16727 } 16728 16729 case DTRACEIOC_AGGSNAP: 16730 case DTRACEIOC_BUFSNAP: { 16731 dtrace_bufdesc_t desc; 16732 caddr_t cached; 16733 dtrace_buffer_t *buf; 16734 16735 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 16736 return (EFAULT); 16737 16738 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU) 16739 return (EINVAL); 16740 16741 mutex_enter(&dtrace_lock); 16742 16743 if (cmd == DTRACEIOC_BUFSNAP) { 16744 buf = &state->dts_buffer[desc.dtbd_cpu]; 16745 } else { 16746 buf = &state->dts_aggbuffer[desc.dtbd_cpu]; 16747 } 16748 16749 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) { 16750 size_t sz = buf->dtb_offset; 16751 16752 if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) { 16753 mutex_exit(&dtrace_lock); 16754 return (EBUSY); 16755 } 16756 16757 /* 16758 * If this buffer has already been consumed, we're 16759 * going to indicate that there's nothing left here 16760 * to consume. 16761 */ 16762 if (buf->dtb_flags & DTRACEBUF_CONSUMED) { 16763 mutex_exit(&dtrace_lock); 16764 16765 desc.dtbd_size = 0; 16766 desc.dtbd_drops = 0; 16767 desc.dtbd_errors = 0; 16768 desc.dtbd_oldest = 0; 16769 sz = sizeof (desc); 16770 16771 if (copyout(&desc, (void *)arg, sz) != 0) 16772 return (EFAULT); 16773 16774 return (0); 16775 } 16776 16777 /* 16778 * If this is a ring buffer that has wrapped, we want 16779 * to copy the whole thing out. 16780 */ 16781 if (buf->dtb_flags & DTRACEBUF_WRAPPED) { 16782 dtrace_buffer_polish(buf); 16783 sz = buf->dtb_size; 16784 } 16785 16786 if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) { 16787 mutex_exit(&dtrace_lock); 16788 return (EFAULT); 16789 } 16790 16791 desc.dtbd_size = sz; 16792 desc.dtbd_drops = buf->dtb_drops; 16793 desc.dtbd_errors = buf->dtb_errors; 16794 desc.dtbd_oldest = buf->dtb_xamot_offset; 16795 desc.dtbd_timestamp = dtrace_gethrtime(); 16796 16797 mutex_exit(&dtrace_lock); 16798 16799 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 16800 return (EFAULT); 16801 16802 buf->dtb_flags |= DTRACEBUF_CONSUMED; 16803 16804 return (0); 16805 } 16806 16807 if (buf->dtb_tomax == NULL) { 16808 ASSERT(buf->dtb_xamot == NULL); 16809 mutex_exit(&dtrace_lock); 16810 return (ENOENT); 16811 } 16812 16813 cached = buf->dtb_tomax; 16814 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); 16815 16816 dtrace_xcall(desc.dtbd_cpu, 16817 (dtrace_xcall_t)dtrace_buffer_switch, buf); 16818 16819 state->dts_errors += buf->dtb_xamot_errors; 16820 16821 /* 16822 * If the buffers did not actually switch, then the cross call 16823 * did not take place -- presumably because the given CPU is 16824 * not in the ready set. If this is the case, we'll return 16825 * ENOENT. 16826 */ 16827 if (buf->dtb_tomax == cached) { 16828 ASSERT(buf->dtb_xamot != cached); 16829 mutex_exit(&dtrace_lock); 16830 return (ENOENT); 16831 } 16832 16833 ASSERT(cached == buf->dtb_xamot); 16834 16835 /* 16836 * We have our snapshot; now copy it out. 16837 */ 16838 if (copyout(buf->dtb_xamot, desc.dtbd_data, 16839 buf->dtb_xamot_offset) != 0) { 16840 mutex_exit(&dtrace_lock); 16841 return (EFAULT); 16842 } 16843 16844 desc.dtbd_size = buf->dtb_xamot_offset; 16845 desc.dtbd_drops = buf->dtb_xamot_drops; 16846 desc.dtbd_errors = buf->dtb_xamot_errors; 16847 desc.dtbd_oldest = 0; 16848 desc.dtbd_timestamp = buf->dtb_switched; 16849 16850 mutex_exit(&dtrace_lock); 16851 16852 /* 16853 * Finally, copy out the buffer description. 16854 */ 16855 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 16856 return (EFAULT); 16857 16858 return (0); 16859 } 16860 16861 case DTRACEIOC_CONF: { 16862 dtrace_conf_t conf; 16863 16864 bzero(&conf, sizeof (conf)); 16865 conf.dtc_difversion = DIF_VERSION; 16866 conf.dtc_difintregs = DIF_DIR_NREGS; 16867 conf.dtc_diftupregs = DIF_DTR_NREGS; 16868 conf.dtc_ctfmodel = CTF_MODEL_NATIVE; 16869 16870 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0) 16871 return (EFAULT); 16872 16873 return (0); 16874 } 16875 16876 case DTRACEIOC_STATUS: { 16877 dtrace_status_t stat; 16878 dtrace_dstate_t *dstate; 16879 int i, j; 16880 uint64_t nerrs; 16881 16882 /* 16883 * See the comment in dtrace_state_deadman() for the reason 16884 * for setting dts_laststatus to INT64_MAX before setting 16885 * it to the correct value. 16886 */ 16887 state->dts_laststatus = INT64_MAX; 16888 dtrace_membar_producer(); 16889 state->dts_laststatus = dtrace_gethrtime(); 16890 16891 bzero(&stat, sizeof (stat)); 16892 16893 mutex_enter(&dtrace_lock); 16894 16895 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) { 16896 mutex_exit(&dtrace_lock); 16897 return (ENOENT); 16898 } 16899 16900 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING) 16901 stat.dtst_exiting = 1; 16902 16903 nerrs = state->dts_errors; 16904 dstate = &state->dts_vstate.dtvs_dynvars; 16905 16906 for (i = 0; i < NCPU; i++) { 16907 dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i]; 16908 16909 stat.dtst_dyndrops += dcpu->dtdsc_drops; 16910 stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops; 16911 stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops; 16912 16913 if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL) 16914 stat.dtst_filled++; 16915 16916 nerrs += state->dts_buffer[i].dtb_errors; 16917 16918 for (j = 0; j < state->dts_nspeculations; j++) { 16919 dtrace_speculation_t *spec; 16920 dtrace_buffer_t *buf; 16921 16922 spec = &state->dts_speculations[j]; 16923 buf = &spec->dtsp_buffer[i]; 16924 stat.dtst_specdrops += buf->dtb_xamot_drops; 16925 } 16926 } 16927 16928 stat.dtst_specdrops_busy = state->dts_speculations_busy; 16929 stat.dtst_specdrops_unavail = state->dts_speculations_unavail; 16930 stat.dtst_stkstroverflows = state->dts_stkstroverflows; 16931 stat.dtst_dblerrors = state->dts_dblerrors; 16932 stat.dtst_killed = 16933 (state->dts_activity == DTRACE_ACTIVITY_KILLED); 16934 stat.dtst_errors = nerrs; 16935 16936 mutex_exit(&dtrace_lock); 16937 16938 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0) 16939 return (EFAULT); 16940 16941 return (0); 16942 } 16943 16944 case DTRACEIOC_FORMAT: { 16945 dtrace_fmtdesc_t fmt; 16946 char *str; 16947 int len; 16948 16949 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0) 16950 return (EFAULT); 16951 16952 mutex_enter(&dtrace_lock); 16953 16954 if (fmt.dtfd_format == 0 || 16955 fmt.dtfd_format > state->dts_nformats) { 16956 mutex_exit(&dtrace_lock); 16957 return (EINVAL); 16958 } 16959 16960 /* 16961 * Format strings are allocated contiguously and they are 16962 * never freed; if a format index is less than the number 16963 * of formats, we can assert that the format map is non-NULL 16964 * and that the format for the specified index is non-NULL. 16965 */ 16966 ASSERT(state->dts_formats != NULL); 16967 str = state->dts_formats[fmt.dtfd_format - 1]; 16968 ASSERT(str != NULL); 16969 16970 len = strlen(str) + 1; 16971 16972 if (len > fmt.dtfd_length) { 16973 fmt.dtfd_length = len; 16974 16975 if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) { 16976 mutex_exit(&dtrace_lock); 16977 return (EINVAL); 16978 } 16979 } else { 16980 if (copyout(str, fmt.dtfd_string, len) != 0) { 16981 mutex_exit(&dtrace_lock); 16982 return (EINVAL); 16983 } 16984 } 16985 16986 mutex_exit(&dtrace_lock); 16987 return (0); 16988 } 16989 16990 default: 16991 break; 16992 } 16993 16994 return (ENOTTY); 16995 } 16996 16997 /*ARGSUSED*/ 16998 static int 16999 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 17000 { 17001 dtrace_state_t *state; 17002 17003 switch (cmd) { 17004 case DDI_DETACH: 17005 break; 17006 17007 case DDI_SUSPEND: 17008 return (DDI_SUCCESS); 17009 17010 default: 17011 return (DDI_FAILURE); 17012 } 17013 17014 mutex_enter(&cpu_lock); 17015 mutex_enter(&dtrace_provider_lock); 17016 mutex_enter(&dtrace_lock); 17017 17018 ASSERT(dtrace_opens == 0); 17019 17020 if (dtrace_helpers > 0) { 17021 mutex_exit(&dtrace_provider_lock); 17022 mutex_exit(&dtrace_lock); 17023 mutex_exit(&cpu_lock); 17024 return (DDI_FAILURE); 17025 } 17026 17027 if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) { 17028 mutex_exit(&dtrace_provider_lock); 17029 mutex_exit(&dtrace_lock); 17030 mutex_exit(&cpu_lock); 17031 return (DDI_FAILURE); 17032 } 17033 17034 dtrace_provider = NULL; 17035 17036 if ((state = dtrace_anon_grab()) != NULL) { 17037 /* 17038 * If there were ECBs on this state, the provider should 17039 * have not been allowed to detach; assert that there is 17040 * none. 17041 */ 17042 ASSERT(state->dts_necbs == 0); 17043 dtrace_state_destroy(state); 17044 17045 /* 17046 * If we're being detached with anonymous state, we need to 17047 * indicate to the kernel debugger that DTrace is now inactive. 17048 */ 17049 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); 17050 } 17051 17052 bzero(&dtrace_anon, sizeof (dtrace_anon_t)); 17053 unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); 17054 dtrace_cpu_init = NULL; 17055 dtrace_helpers_cleanup = NULL; 17056 dtrace_helpers_fork = NULL; 17057 dtrace_cpustart_init = NULL; 17058 dtrace_cpustart_fini = NULL; 17059 dtrace_debugger_init = NULL; 17060 dtrace_debugger_fini = NULL; 17061 dtrace_modload = NULL; 17062 dtrace_modunload = NULL; 17063 17064 ASSERT(dtrace_getf == 0); 17065 ASSERT(dtrace_closef == NULL); 17066 17067 mutex_exit(&cpu_lock); 17068 17069 kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *)); 17070 dtrace_probes = NULL; 17071 dtrace_nprobes = 0; 17072 17073 dtrace_hash_destroy(dtrace_bymod); 17074 dtrace_hash_destroy(dtrace_byfunc); 17075 dtrace_hash_destroy(dtrace_byname); 17076 dtrace_bymod = NULL; 17077 dtrace_byfunc = NULL; 17078 dtrace_byname = NULL; 17079 17080 kmem_cache_destroy(dtrace_state_cache); 17081 vmem_destroy(dtrace_minor); 17082 vmem_destroy(dtrace_arena); 17083 17084 if (dtrace_toxrange != NULL) { 17085 kmem_free(dtrace_toxrange, 17086 dtrace_toxranges_max * sizeof (dtrace_toxrange_t)); 17087 dtrace_toxrange = NULL; 17088 dtrace_toxranges = 0; 17089 dtrace_toxranges_max = 0; 17090 } 17091 17092 ddi_remove_minor_node(dtrace_devi, NULL); 17093 dtrace_devi = NULL; 17094 17095 ddi_soft_state_fini(&dtrace_softstate); 17096 17097 ASSERT(dtrace_vtime_references == 0); 17098 ASSERT(dtrace_opens == 0); 17099 ASSERT(dtrace_retained == NULL); 17100 17101 mutex_exit(&dtrace_lock); 17102 mutex_exit(&dtrace_provider_lock); 17103 17104 /* 17105 * We don't destroy the task queue until after we have dropped our 17106 * locks (taskq_destroy() may block on running tasks). To prevent 17107 * attempting to do work after we have effectively detached but before 17108 * the task queue has been destroyed, all tasks dispatched via the 17109 * task queue must check that DTrace is still attached before 17110 * performing any operation. 17111 */ 17112 taskq_destroy(dtrace_taskq); 17113 dtrace_taskq = NULL; 17114 17115 return (DDI_SUCCESS); 17116 } 17117 17118 /*ARGSUSED*/ 17119 static int 17120 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 17121 { 17122 int error; 17123 17124 switch (infocmd) { 17125 case DDI_INFO_DEVT2DEVINFO: 17126 *result = (void *)dtrace_devi; 17127 error = DDI_SUCCESS; 17128 break; 17129 case DDI_INFO_DEVT2INSTANCE: 17130 *result = (void *)0; 17131 error = DDI_SUCCESS; 17132 break; 17133 default: 17134 error = DDI_FAILURE; 17135 } 17136 return (error); 17137 } 17138 17139 static struct cb_ops dtrace_cb_ops = { 17140 dtrace_open, /* open */ 17141 dtrace_close, /* close */ 17142 nulldev, /* strategy */ 17143 nulldev, /* print */ 17144 nodev, /* dump */ 17145 nodev, /* read */ 17146 nodev, /* write */ 17147 dtrace_ioctl, /* ioctl */ 17148 nodev, /* devmap */ 17149 nodev, /* mmap */ 17150 nodev, /* segmap */ 17151 nochpoll, /* poll */ 17152 ddi_prop_op, /* cb_prop_op */ 17153 0, /* streamtab */ 17154 D_NEW | D_MP /* Driver compatibility flag */ 17155 }; 17156 17157 static struct dev_ops dtrace_ops = { 17158 DEVO_REV, /* devo_rev */ 17159 0, /* refcnt */ 17160 dtrace_info, /* get_dev_info */ 17161 nulldev, /* identify */ 17162 nulldev, /* probe */ 17163 dtrace_attach, /* attach */ 17164 dtrace_detach, /* detach */ 17165 nodev, /* reset */ 17166 &dtrace_cb_ops, /* driver operations */ 17167 NULL, /* bus operations */ 17168 nodev, /* dev power */ 17169 ddi_quiesce_not_needed, /* quiesce */ 17170 }; 17171 17172 static struct modldrv modldrv = { 17173 &mod_driverops, /* module type (this is a pseudo driver) */ 17174 "Dynamic Tracing", /* name of module */ 17175 &dtrace_ops, /* driver ops */ 17176 }; 17177 17178 static struct modlinkage modlinkage = { 17179 MODREV_1, 17180 (void *)&modldrv, 17181 NULL 17182 }; 17183 17184 int 17185 _init(void) 17186 { 17187 return (mod_install(&modlinkage)); 17188 } 17189 17190 int 17191 _info(struct modinfo *modinfop) 17192 { 17193 return (mod_info(&modlinkage, modinfop)); 17194 } 17195 17196 int 17197 _fini(void) 17198 { 17199 return (mod_remove(&modlinkage)); 17200 } 17201