1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * $FreeBSD$ 22 */ 23 24 /* 25 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27 * Copyright (c) 2012, 2014 by Delphix. All rights reserved. 28 */ 29 30 /* 31 * DTrace - Dynamic Tracing for Solaris 32 * 33 * This is the implementation of the Solaris Dynamic Tracing framework 34 * (DTrace). The user-visible interface to DTrace is described at length in 35 * the "Solaris Dynamic Tracing Guide". The interfaces between the libdtrace 36 * library, the in-kernel DTrace framework, and the DTrace providers are 37 * described in the block comments in the <sys/dtrace.h> header file. The 38 * internal architecture of DTrace is described in the block comments in the 39 * <sys/dtrace_impl.h> header file. The comments contained within the DTrace 40 * implementation very much assume mastery of all of these sources; if one has 41 * an unanswered question about the implementation, one should consult them 42 * first. 43 * 44 * The functions here are ordered roughly as follows: 45 * 46 * - Probe context functions 47 * - Probe hashing functions 48 * - Non-probe context utility functions 49 * - Matching functions 50 * - Provider-to-Framework API functions 51 * - Probe management functions 52 * - DIF object functions 53 * - Format functions 54 * - Predicate functions 55 * - ECB functions 56 * - Buffer functions 57 * - Enabling functions 58 * - DOF functions 59 * - Anonymous enabling functions 60 * - Consumer state functions 61 * - Helper functions 62 * - Hook functions 63 * - Driver cookbook functions 64 * 65 * Each group of functions begins with a block comment labelled the "DTrace 66 * [Group] Functions", allowing one to find each block by searching forward 67 * on capital-f functions. 68 */ 69 #include <sys/errno.h> 70 #ifndef illumos 71 #include <sys/time.h> 72 #endif 73 #include <sys/stat.h> 74 #include <sys/modctl.h> 75 #include <sys/conf.h> 76 #include <sys/systm.h> 77 #ifdef illumos 78 #include <sys/ddi.h> 79 #include <sys/sunddi.h> 80 #endif 81 #include <sys/cpuvar.h> 82 #include <sys/kmem.h> 83 #ifdef illumos 84 #include <sys/strsubr.h> 85 #endif 86 #include <sys/sysmacros.h> 87 #include <sys/dtrace_impl.h> 88 #include <sys/atomic.h> 89 #include <sys/cmn_err.h> 90 #ifdef illumos 91 #include <sys/mutex_impl.h> 92 #include <sys/rwlock_impl.h> 93 #endif 94 #include <sys/ctf_api.h> 95 #ifdef illumos 96 #include <sys/panic.h> 97 #include <sys/priv_impl.h> 98 #endif 99 #include <sys/policy.h> 100 #ifdef illumos 101 #include <sys/cred_impl.h> 102 #include <sys/procfs_isa.h> 103 #endif 104 #include <sys/taskq.h> 105 #ifdef illumos 106 #include <sys/mkdev.h> 107 #include <sys/kdi.h> 108 #endif 109 #include <sys/zone.h> 110 #include <sys/socket.h> 111 #include <netinet/in.h> 112 #include "strtolctype.h" 113 114 /* FreeBSD includes: */ 115 #ifndef illumos 116 #include <sys/callout.h> 117 #include <sys/ctype.h> 118 #include <sys/eventhandler.h> 119 #include <sys/limits.h> 120 #include <sys/kdb.h> 121 #include <sys/kernel.h> 122 #include <sys/malloc.h> 123 #include <sys/sysctl.h> 124 #include <sys/lock.h> 125 #include <sys/mutex.h> 126 #include <sys/rwlock.h> 127 #include <sys/sx.h> 128 #include <sys/dtrace_bsd.h> 129 #include <netinet/in.h> 130 #include "dtrace_cddl.h" 131 #include "dtrace_debug.c" 132 #endif 133 134 /* 135 * DTrace Tunable Variables 136 * 137 * The following variables may be tuned by adding a line to /etc/system that 138 * includes both the name of the DTrace module ("dtrace") and the name of the 139 * variable. For example: 140 * 141 * set dtrace:dtrace_destructive_disallow = 1 142 * 143 * In general, the only variables that one should be tuning this way are those 144 * that affect system-wide DTrace behavior, and for which the default behavior 145 * is undesirable. Most of these variables are tunable on a per-consumer 146 * basis using DTrace options, and need not be tuned on a system-wide basis. 147 * When tuning these variables, avoid pathological values; while some attempt 148 * is made to verify the integrity of these variables, they are not considered 149 * part of the supported interface to DTrace, and they are therefore not 150 * checked comprehensively. Further, these variables should not be tuned 151 * dynamically via "mdb -kw" or other means; they should only be tuned via 152 * /etc/system. 153 */ 154 int dtrace_destructive_disallow = 0; 155 dtrace_optval_t dtrace_nonroot_maxsize = (16 * 1024 * 1024); 156 size_t dtrace_difo_maxsize = (256 * 1024); 157 dtrace_optval_t dtrace_dof_maxsize = (8 * 1024 * 1024); 158 size_t dtrace_global_maxsize = (16 * 1024); 159 size_t dtrace_actions_max = (16 * 1024); 160 size_t dtrace_retain_max = 1024; 161 dtrace_optval_t dtrace_helper_actions_max = 128; 162 dtrace_optval_t dtrace_helper_providers_max = 32; 163 dtrace_optval_t dtrace_dstate_defsize = (1 * 1024 * 1024); 164 size_t dtrace_strsize_default = 256; 165 dtrace_optval_t dtrace_cleanrate_default = 9900990; /* 101 hz */ 166 dtrace_optval_t dtrace_cleanrate_min = 200000; /* 5000 hz */ 167 dtrace_optval_t dtrace_cleanrate_max = (uint64_t)60 * NANOSEC; /* 1/minute */ 168 dtrace_optval_t dtrace_aggrate_default = NANOSEC; /* 1 hz */ 169 dtrace_optval_t dtrace_statusrate_default = NANOSEC; /* 1 hz */ 170 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC; /* 6/minute */ 171 dtrace_optval_t dtrace_switchrate_default = NANOSEC; /* 1 hz */ 172 dtrace_optval_t dtrace_nspec_default = 1; 173 dtrace_optval_t dtrace_specsize_default = 32 * 1024; 174 dtrace_optval_t dtrace_stackframes_default = 20; 175 dtrace_optval_t dtrace_ustackframes_default = 20; 176 dtrace_optval_t dtrace_jstackframes_default = 50; 177 dtrace_optval_t dtrace_jstackstrsize_default = 512; 178 int dtrace_msgdsize_max = 128; 179 hrtime_t dtrace_chill_max = MSEC2NSEC(500); /* 500 ms */ 180 hrtime_t dtrace_chill_interval = NANOSEC; /* 1000 ms */ 181 int dtrace_devdepth_max = 32; 182 int dtrace_err_verbose; 183 hrtime_t dtrace_deadman_interval = NANOSEC; 184 hrtime_t dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC; 185 hrtime_t dtrace_deadman_user = (hrtime_t)30 * NANOSEC; 186 hrtime_t dtrace_unregister_defunct_reap = (hrtime_t)60 * NANOSEC; 187 #ifndef illumos 188 int dtrace_memstr_max = 4096; 189 #endif 190 191 /* 192 * DTrace External Variables 193 * 194 * As dtrace(7D) is a kernel module, any DTrace variables are obviously 195 * available to DTrace consumers via the backtick (`) syntax. One of these, 196 * dtrace_zero, is made deliberately so: it is provided as a source of 197 * well-known, zero-filled memory. While this variable is not documented, 198 * it is used by some translators as an implementation detail. 199 */ 200 const char dtrace_zero[256] = { 0 }; /* zero-filled memory */ 201 202 /* 203 * DTrace Internal Variables 204 */ 205 #ifdef illumos 206 static dev_info_t *dtrace_devi; /* device info */ 207 #endif 208 #ifdef illumos 209 static vmem_t *dtrace_arena; /* probe ID arena */ 210 static vmem_t *dtrace_minor; /* minor number arena */ 211 #else 212 static taskq_t *dtrace_taskq; /* task queue */ 213 static struct unrhdr *dtrace_arena; /* Probe ID number. */ 214 #endif 215 static dtrace_probe_t **dtrace_probes; /* array of all probes */ 216 static int dtrace_nprobes; /* number of probes */ 217 static dtrace_provider_t *dtrace_provider; /* provider list */ 218 static dtrace_meta_t *dtrace_meta_pid; /* user-land meta provider */ 219 static int dtrace_opens; /* number of opens */ 220 static int dtrace_helpers; /* number of helpers */ 221 static int dtrace_getf; /* number of unpriv getf()s */ 222 #ifdef illumos 223 static void *dtrace_softstate; /* softstate pointer */ 224 #endif 225 static dtrace_hash_t *dtrace_bymod; /* probes hashed by module */ 226 static dtrace_hash_t *dtrace_byfunc; /* probes hashed by function */ 227 static dtrace_hash_t *dtrace_byname; /* probes hashed by name */ 228 static dtrace_toxrange_t *dtrace_toxrange; /* toxic range array */ 229 static int dtrace_toxranges; /* number of toxic ranges */ 230 static int dtrace_toxranges_max; /* size of toxic range array */ 231 static dtrace_anon_t dtrace_anon; /* anonymous enabling */ 232 static kmem_cache_t *dtrace_state_cache; /* cache for dynamic state */ 233 static uint64_t dtrace_vtime_references; /* number of vtimestamp refs */ 234 static kthread_t *dtrace_panicked; /* panicking thread */ 235 static dtrace_ecb_t *dtrace_ecb_create_cache; /* cached created ECB */ 236 static dtrace_genid_t dtrace_probegen; /* current probe generation */ 237 static dtrace_helpers_t *dtrace_deferred_pid; /* deferred helper list */ 238 static dtrace_enabling_t *dtrace_retained; /* list of retained enablings */ 239 static dtrace_genid_t dtrace_retained_gen; /* current retained enab gen */ 240 static dtrace_dynvar_t dtrace_dynhash_sink; /* end of dynamic hash chains */ 241 static int dtrace_dynvar_failclean; /* dynvars failed to clean */ 242 #ifndef illumos 243 static struct mtx dtrace_unr_mtx; 244 MTX_SYSINIT(dtrace_unr_mtx, &dtrace_unr_mtx, "Unique resource identifier", MTX_DEF); 245 int dtrace_in_probe; /* non-zero if executing a probe */ 246 #if defined(__i386__) || defined(__amd64__) || defined(__mips__) || defined(__powerpc__) 247 uintptr_t dtrace_in_probe_addr; /* Address of invop when already in probe */ 248 #endif 249 static eventhandler_tag dtrace_kld_load_tag; 250 static eventhandler_tag dtrace_kld_unload_try_tag; 251 #endif 252 253 /* 254 * DTrace Locking 255 * DTrace is protected by three (relatively coarse-grained) locks: 256 * 257 * (1) dtrace_lock is required to manipulate essentially any DTrace state, 258 * including enabling state, probes, ECBs, consumer state, helper state, 259 * etc. Importantly, dtrace_lock is _not_ required when in probe context; 260 * probe context is lock-free -- synchronization is handled via the 261 * dtrace_sync() cross call mechanism. 262 * 263 * (2) dtrace_provider_lock is required when manipulating provider state, or 264 * when provider state must be held constant. 265 * 266 * (3) dtrace_meta_lock is required when manipulating meta provider state, or 267 * when meta provider state must be held constant. 268 * 269 * The lock ordering between these three locks is dtrace_meta_lock before 270 * dtrace_provider_lock before dtrace_lock. (In particular, there are 271 * several places where dtrace_provider_lock is held by the framework as it 272 * calls into the providers -- which then call back into the framework, 273 * grabbing dtrace_lock.) 274 * 275 * There are two other locks in the mix: mod_lock and cpu_lock. With respect 276 * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical 277 * role as a coarse-grained lock; it is acquired before both of these locks. 278 * With respect to dtrace_meta_lock, its behavior is stranger: cpu_lock must 279 * be acquired _between_ dtrace_meta_lock and any other DTrace locks. 280 * mod_lock is similar with respect to dtrace_provider_lock in that it must be 281 * acquired _between_ dtrace_provider_lock and dtrace_lock. 282 */ 283 static kmutex_t dtrace_lock; /* probe state lock */ 284 static kmutex_t dtrace_provider_lock; /* provider state lock */ 285 static kmutex_t dtrace_meta_lock; /* meta-provider state lock */ 286 287 #ifndef illumos 288 /* XXX FreeBSD hacks. */ 289 #define cr_suid cr_svuid 290 #define cr_sgid cr_svgid 291 #define ipaddr_t in_addr_t 292 #define mod_modname pathname 293 #define vuprintf vprintf 294 #define ttoproc(_a) ((_a)->td_proc) 295 #define crgetzoneid(_a) 0 296 #define NCPU MAXCPU 297 #define SNOCD 0 298 #define CPU_ON_INTR(_a) 0 299 300 #define PRIV_EFFECTIVE (1 << 0) 301 #define PRIV_DTRACE_KERNEL (1 << 1) 302 #define PRIV_DTRACE_PROC (1 << 2) 303 #define PRIV_DTRACE_USER (1 << 3) 304 #define PRIV_PROC_OWNER (1 << 4) 305 #define PRIV_PROC_ZONE (1 << 5) 306 #define PRIV_ALL ~0 307 308 SYSCTL_DECL(_debug_dtrace); 309 SYSCTL_DECL(_kern_dtrace); 310 #endif 311 312 #ifdef illumos 313 #define curcpu CPU->cpu_id 314 #endif 315 316 317 /* 318 * DTrace Provider Variables 319 * 320 * These are the variables relating to DTrace as a provider (that is, the 321 * provider of the BEGIN, END, and ERROR probes). 322 */ 323 static dtrace_pattr_t dtrace_provider_attr = { 324 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, 325 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 326 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 327 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, 328 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON }, 329 }; 330 331 static void 332 dtrace_nullop(void) 333 {} 334 335 static dtrace_pops_t dtrace_provider_ops = { 336 (void (*)(void *, dtrace_probedesc_t *))dtrace_nullop, 337 (void (*)(void *, modctl_t *))dtrace_nullop, 338 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 339 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 340 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 341 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop, 342 NULL, 343 NULL, 344 NULL, 345 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop 346 }; 347 348 static dtrace_id_t dtrace_probeid_begin; /* special BEGIN probe */ 349 static dtrace_id_t dtrace_probeid_end; /* special END probe */ 350 dtrace_id_t dtrace_probeid_error; /* special ERROR probe */ 351 352 /* 353 * DTrace Helper Tracing Variables 354 * 355 * These variables should be set dynamically to enable helper tracing. The 356 * only variables that should be set are dtrace_helptrace_enable (which should 357 * be set to a non-zero value to allocate helper tracing buffers on the next 358 * open of /dev/dtrace) and dtrace_helptrace_disable (which should be set to a 359 * non-zero value to deallocate helper tracing buffers on the next close of 360 * /dev/dtrace). When (and only when) helper tracing is disabled, the 361 * buffer size may also be set via dtrace_helptrace_bufsize. 362 */ 363 int dtrace_helptrace_enable = 0; 364 int dtrace_helptrace_disable = 0; 365 int dtrace_helptrace_bufsize = 16 * 1024 * 1024; 366 uint32_t dtrace_helptrace_nlocals; 367 static dtrace_helptrace_t *dtrace_helptrace_buffer; 368 static uint32_t dtrace_helptrace_next = 0; 369 static int dtrace_helptrace_wrapped = 0; 370 371 /* 372 * DTrace Error Hashing 373 * 374 * On DEBUG kernels, DTrace will track the errors that has seen in a hash 375 * table. This is very useful for checking coverage of tests that are 376 * expected to induce DIF or DOF processing errors, and may be useful for 377 * debugging problems in the DIF code generator or in DOF generation . The 378 * error hash may be examined with the ::dtrace_errhash MDB dcmd. 379 */ 380 #ifdef DEBUG 381 static dtrace_errhash_t dtrace_errhash[DTRACE_ERRHASHSZ]; 382 static const char *dtrace_errlast; 383 static kthread_t *dtrace_errthread; 384 static kmutex_t dtrace_errlock; 385 #endif 386 387 /* 388 * DTrace Macros and Constants 389 * 390 * These are various macros that are useful in various spots in the 391 * implementation, along with a few random constants that have no meaning 392 * outside of the implementation. There is no real structure to this cpp 393 * mishmash -- but is there ever? 394 */ 395 #define DTRACE_HASHSTR(hash, probe) \ 396 dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs))) 397 398 #define DTRACE_HASHNEXT(hash, probe) \ 399 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs) 400 401 #define DTRACE_HASHPREV(hash, probe) \ 402 (dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs) 403 404 #define DTRACE_HASHEQ(hash, lhs, rhs) \ 405 (strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \ 406 *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0) 407 408 #define DTRACE_AGGHASHSIZE_SLEW 17 409 410 #define DTRACE_V4MAPPED_OFFSET (sizeof (uint32_t) * 3) 411 412 /* 413 * The key for a thread-local variable consists of the lower 61 bits of the 414 * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL. 415 * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never 416 * equal to a variable identifier. This is necessary (but not sufficient) to 417 * assure that global associative arrays never collide with thread-local 418 * variables. To guarantee that they cannot collide, we must also define the 419 * order for keying dynamic variables. That order is: 420 * 421 * [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ] 422 * 423 * Because the variable-key and the tls-key are in orthogonal spaces, there is 424 * no way for a global variable key signature to match a thread-local key 425 * signature. 426 */ 427 #ifdef illumos 428 #define DTRACE_TLS_THRKEY(where) { \ 429 uint_t intr = 0; \ 430 uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \ 431 for (; actv; actv >>= 1) \ 432 intr++; \ 433 ASSERT(intr < (1 << 3)); \ 434 (where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \ 435 (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ 436 } 437 #else 438 #define DTRACE_TLS_THRKEY(where) { \ 439 solaris_cpu_t *_c = &solaris_cpu[curcpu]; \ 440 uint_t intr = 0; \ 441 uint_t actv = _c->cpu_intr_actv; \ 442 for (; actv; actv >>= 1) \ 443 intr++; \ 444 ASSERT(intr < (1 << 3)); \ 445 (where) = ((curthread->td_tid + DIF_VARIABLE_MAX) & \ 446 (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \ 447 } 448 #endif 449 450 #define DT_BSWAP_8(x) ((x) & 0xff) 451 #define DT_BSWAP_16(x) ((DT_BSWAP_8(x) << 8) | DT_BSWAP_8((x) >> 8)) 452 #define DT_BSWAP_32(x) ((DT_BSWAP_16(x) << 16) | DT_BSWAP_16((x) >> 16)) 453 #define DT_BSWAP_64(x) ((DT_BSWAP_32(x) << 32) | DT_BSWAP_32((x) >> 32)) 454 455 #define DT_MASK_LO 0x00000000FFFFFFFFULL 456 457 #define DTRACE_STORE(type, tomax, offset, what) \ 458 *((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what); 459 460 #ifndef __x86 461 #define DTRACE_ALIGNCHECK(addr, size, flags) \ 462 if (addr & (size - 1)) { \ 463 *flags |= CPU_DTRACE_BADALIGN; \ 464 cpu_core[curcpu].cpuc_dtrace_illval = addr; \ 465 return (0); \ 466 } 467 #else 468 #define DTRACE_ALIGNCHECK(addr, size, flags) 469 #endif 470 471 /* 472 * Test whether a range of memory starting at testaddr of size testsz falls 473 * within the range of memory described by addr, sz. We take care to avoid 474 * problems with overflow and underflow of the unsigned quantities, and 475 * disallow all negative sizes. Ranges of size 0 are allowed. 476 */ 477 #define DTRACE_INRANGE(testaddr, testsz, baseaddr, basesz) \ 478 ((testaddr) - (uintptr_t)(baseaddr) < (basesz) && \ 479 (testaddr) + (testsz) - (uintptr_t)(baseaddr) <= (basesz) && \ 480 (testaddr) + (testsz) >= (testaddr)) 481 482 /* 483 * Test whether alloc_sz bytes will fit in the scratch region. We isolate 484 * alloc_sz on the righthand side of the comparison in order to avoid overflow 485 * or underflow in the comparison with it. This is simpler than the INRANGE 486 * check above, because we know that the dtms_scratch_ptr is valid in the 487 * range. Allocations of size zero are allowed. 488 */ 489 #define DTRACE_INSCRATCH(mstate, alloc_sz) \ 490 ((mstate)->dtms_scratch_base + (mstate)->dtms_scratch_size - \ 491 (mstate)->dtms_scratch_ptr >= (alloc_sz)) 492 493 #define DTRACE_LOADFUNC(bits) \ 494 /*CSTYLED*/ \ 495 uint##bits##_t \ 496 dtrace_load##bits(uintptr_t addr) \ 497 { \ 498 size_t size = bits / NBBY; \ 499 /*CSTYLED*/ \ 500 uint##bits##_t rval; \ 501 int i; \ 502 volatile uint16_t *flags = (volatile uint16_t *) \ 503 &cpu_core[curcpu].cpuc_dtrace_flags; \ 504 \ 505 DTRACE_ALIGNCHECK(addr, size, flags); \ 506 \ 507 for (i = 0; i < dtrace_toxranges; i++) { \ 508 if (addr >= dtrace_toxrange[i].dtt_limit) \ 509 continue; \ 510 \ 511 if (addr + size <= dtrace_toxrange[i].dtt_base) \ 512 continue; \ 513 \ 514 /* \ 515 * This address falls within a toxic region; return 0. \ 516 */ \ 517 *flags |= CPU_DTRACE_BADADDR; \ 518 cpu_core[curcpu].cpuc_dtrace_illval = addr; \ 519 return (0); \ 520 } \ 521 \ 522 *flags |= CPU_DTRACE_NOFAULT; \ 523 /*CSTYLED*/ \ 524 rval = *((volatile uint##bits##_t *)addr); \ 525 *flags &= ~CPU_DTRACE_NOFAULT; \ 526 \ 527 return (!(*flags & CPU_DTRACE_FAULT) ? rval : 0); \ 528 } 529 530 #ifdef _LP64 531 #define dtrace_loadptr dtrace_load64 532 #else 533 #define dtrace_loadptr dtrace_load32 534 #endif 535 536 #define DTRACE_DYNHASH_FREE 0 537 #define DTRACE_DYNHASH_SINK 1 538 #define DTRACE_DYNHASH_VALID 2 539 540 #define DTRACE_MATCH_NEXT 0 541 #define DTRACE_MATCH_DONE 1 542 #define DTRACE_ANCHORED(probe) ((probe)->dtpr_func[0] != '\0') 543 #define DTRACE_STATE_ALIGN 64 544 545 #define DTRACE_FLAGS2FLT(flags) \ 546 (((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR : \ 547 ((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP : \ 548 ((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO : \ 549 ((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV : \ 550 ((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV : \ 551 ((flags) & CPU_DTRACE_TUPOFLOW) ? DTRACEFLT_TUPOFLOW : \ 552 ((flags) & CPU_DTRACE_BADALIGN) ? DTRACEFLT_BADALIGN : \ 553 ((flags) & CPU_DTRACE_NOSCRATCH) ? DTRACEFLT_NOSCRATCH : \ 554 ((flags) & CPU_DTRACE_BADSTACK) ? DTRACEFLT_BADSTACK : \ 555 DTRACEFLT_UNKNOWN) 556 557 #define DTRACEACT_ISSTRING(act) \ 558 ((act)->dta_kind == DTRACEACT_DIFEXPR && \ 559 (act)->dta_difo->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) 560 561 /* Function prototype definitions: */ 562 static size_t dtrace_strlen(const char *, size_t); 563 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id); 564 static void dtrace_enabling_provide(dtrace_provider_t *); 565 static int dtrace_enabling_match(dtrace_enabling_t *, int *); 566 static void dtrace_enabling_matchall(void); 567 static void dtrace_enabling_reap(void); 568 static dtrace_state_t *dtrace_anon_grab(void); 569 static uint64_t dtrace_helper(int, dtrace_mstate_t *, 570 dtrace_state_t *, uint64_t, uint64_t); 571 static dtrace_helpers_t *dtrace_helpers_create(proc_t *); 572 static void dtrace_buffer_drop(dtrace_buffer_t *); 573 static int dtrace_buffer_consumed(dtrace_buffer_t *, hrtime_t when); 574 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t, 575 dtrace_state_t *, dtrace_mstate_t *); 576 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t, 577 dtrace_optval_t); 578 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *); 579 static void dtrace_helper_provider_destroy(dtrace_helper_provider_t *); 580 uint16_t dtrace_load16(uintptr_t); 581 uint32_t dtrace_load32(uintptr_t); 582 uint64_t dtrace_load64(uintptr_t); 583 uint8_t dtrace_load8(uintptr_t); 584 void dtrace_dynvar_clean(dtrace_dstate_t *); 585 dtrace_dynvar_t *dtrace_dynvar(dtrace_dstate_t *, uint_t, dtrace_key_t *, 586 size_t, dtrace_dynvar_op_t, dtrace_mstate_t *, dtrace_vstate_t *); 587 uintptr_t dtrace_dif_varstr(uintptr_t, dtrace_state_t *, dtrace_mstate_t *); 588 static int dtrace_priv_proc(dtrace_state_t *); 589 static void dtrace_getf_barrier(void); 590 591 /* 592 * DTrace Probe Context Functions 593 * 594 * These functions are called from probe context. Because probe context is 595 * any context in which C may be called, arbitrarily locks may be held, 596 * interrupts may be disabled, we may be in arbitrary dispatched state, etc. 597 * As a result, functions called from probe context may only call other DTrace 598 * support functions -- they may not interact at all with the system at large. 599 * (Note that the ASSERT macro is made probe-context safe by redefining it in 600 * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary 601 * loads are to be performed from probe context, they _must_ be in terms of 602 * the safe dtrace_load*() variants. 603 * 604 * Some functions in this block are not actually called from probe context; 605 * for these functions, there will be a comment above the function reading 606 * "Note: not called from probe context." 607 */ 608 void 609 dtrace_panic(const char *format, ...) 610 { 611 va_list alist; 612 613 va_start(alist, format); 614 dtrace_vpanic(format, alist); 615 va_end(alist); 616 } 617 618 int 619 dtrace_assfail(const char *a, const char *f, int l) 620 { 621 dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l); 622 623 /* 624 * We just need something here that even the most clever compiler 625 * cannot optimize away. 626 */ 627 return (a[(uintptr_t)f]); 628 } 629 630 /* 631 * Atomically increment a specified error counter from probe context. 632 */ 633 static void 634 dtrace_error(uint32_t *counter) 635 { 636 /* 637 * Most counters stored to in probe context are per-CPU counters. 638 * However, there are some error conditions that are sufficiently 639 * arcane that they don't merit per-CPU storage. If these counters 640 * are incremented concurrently on different CPUs, scalability will be 641 * adversely affected -- but we don't expect them to be white-hot in a 642 * correctly constructed enabling... 643 */ 644 uint32_t oval, nval; 645 646 do { 647 oval = *counter; 648 649 if ((nval = oval + 1) == 0) { 650 /* 651 * If the counter would wrap, set it to 1 -- assuring 652 * that the counter is never zero when we have seen 653 * errors. (The counter must be 32-bits because we 654 * aren't guaranteed a 64-bit compare&swap operation.) 655 * To save this code both the infamy of being fingered 656 * by a priggish news story and the indignity of being 657 * the target of a neo-puritan witch trial, we're 658 * carefully avoiding any colorful description of the 659 * likelihood of this condition -- but suffice it to 660 * say that it is only slightly more likely than the 661 * overflow of predicate cache IDs, as discussed in 662 * dtrace_predicate_create(). 663 */ 664 nval = 1; 665 } 666 } while (dtrace_cas32(counter, oval, nval) != oval); 667 } 668 669 /* 670 * Use the DTRACE_LOADFUNC macro to define functions for each of loading a 671 * uint8_t, a uint16_t, a uint32_t and a uint64_t. 672 */ 673 DTRACE_LOADFUNC(8) 674 DTRACE_LOADFUNC(16) 675 DTRACE_LOADFUNC(32) 676 DTRACE_LOADFUNC(64) 677 678 static int 679 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate) 680 { 681 if (dest < mstate->dtms_scratch_base) 682 return (0); 683 684 if (dest + size < dest) 685 return (0); 686 687 if (dest + size > mstate->dtms_scratch_ptr) 688 return (0); 689 690 return (1); 691 } 692 693 static int 694 dtrace_canstore_statvar(uint64_t addr, size_t sz, 695 dtrace_statvar_t **svars, int nsvars) 696 { 697 int i; 698 699 for (i = 0; i < nsvars; i++) { 700 dtrace_statvar_t *svar = svars[i]; 701 702 if (svar == NULL || svar->dtsv_size == 0) 703 continue; 704 705 if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size)) 706 return (1); 707 } 708 709 return (0); 710 } 711 712 /* 713 * Check to see if the address is within a memory region to which a store may 714 * be issued. This includes the DTrace scratch areas, and any DTrace variable 715 * region. The caller of dtrace_canstore() is responsible for performing any 716 * alignment checks that are needed before stores are actually executed. 717 */ 718 static int 719 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, 720 dtrace_vstate_t *vstate) 721 { 722 /* 723 * First, check to see if the address is in scratch space... 724 */ 725 if (DTRACE_INRANGE(addr, sz, mstate->dtms_scratch_base, 726 mstate->dtms_scratch_size)) 727 return (1); 728 729 /* 730 * Now check to see if it's a dynamic variable. This check will pick 731 * up both thread-local variables and any global dynamically-allocated 732 * variables. 733 */ 734 if (DTRACE_INRANGE(addr, sz, vstate->dtvs_dynvars.dtds_base, 735 vstate->dtvs_dynvars.dtds_size)) { 736 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; 737 uintptr_t base = (uintptr_t)dstate->dtds_base + 738 (dstate->dtds_hashsize * sizeof (dtrace_dynhash_t)); 739 uintptr_t chunkoffs; 740 741 /* 742 * Before we assume that we can store here, we need to make 743 * sure that it isn't in our metadata -- storing to our 744 * dynamic variable metadata would corrupt our state. For 745 * the range to not include any dynamic variable metadata, 746 * it must: 747 * 748 * (1) Start above the hash table that is at the base of 749 * the dynamic variable space 750 * 751 * (2) Have a starting chunk offset that is beyond the 752 * dtrace_dynvar_t that is at the base of every chunk 753 * 754 * (3) Not span a chunk boundary 755 * 756 */ 757 if (addr < base) 758 return (0); 759 760 chunkoffs = (addr - base) % dstate->dtds_chunksize; 761 762 if (chunkoffs < sizeof (dtrace_dynvar_t)) 763 return (0); 764 765 if (chunkoffs + sz > dstate->dtds_chunksize) 766 return (0); 767 768 return (1); 769 } 770 771 /* 772 * Finally, check the static local and global variables. These checks 773 * take the longest, so we perform them last. 774 */ 775 if (dtrace_canstore_statvar(addr, sz, 776 vstate->dtvs_locals, vstate->dtvs_nlocals)) 777 return (1); 778 779 if (dtrace_canstore_statvar(addr, sz, 780 vstate->dtvs_globals, vstate->dtvs_nglobals)) 781 return (1); 782 783 return (0); 784 } 785 786 787 /* 788 * Convenience routine to check to see if the address is within a memory 789 * region in which a load may be issued given the user's privilege level; 790 * if not, it sets the appropriate error flags and loads 'addr' into the 791 * illegal value slot. 792 * 793 * DTrace subroutines (DIF_SUBR_*) should use this helper to implement 794 * appropriate memory access protection. 795 */ 796 static int 797 dtrace_canload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, 798 dtrace_vstate_t *vstate) 799 { 800 volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval; 801 file_t *fp; 802 803 /* 804 * If we hold the privilege to read from kernel memory, then 805 * everything is readable. 806 */ 807 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) 808 return (1); 809 810 /* 811 * You can obviously read that which you can store. 812 */ 813 if (dtrace_canstore(addr, sz, mstate, vstate)) 814 return (1); 815 816 /* 817 * We're allowed to read from our own string table. 818 */ 819 if (DTRACE_INRANGE(addr, sz, mstate->dtms_difo->dtdo_strtab, 820 mstate->dtms_difo->dtdo_strlen)) 821 return (1); 822 823 if (vstate->dtvs_state != NULL && 824 dtrace_priv_proc(vstate->dtvs_state)) { 825 proc_t *p; 826 827 /* 828 * When we have privileges to the current process, there are 829 * several context-related kernel structures that are safe to 830 * read, even absent the privilege to read from kernel memory. 831 * These reads are safe because these structures contain only 832 * state that (1) we're permitted to read, (2) is harmless or 833 * (3) contains pointers to additional kernel state that we're 834 * not permitted to read (and as such, do not present an 835 * opportunity for privilege escalation). Finally (and 836 * critically), because of the nature of their relation with 837 * the current thread context, the memory associated with these 838 * structures cannot change over the duration of probe context, 839 * and it is therefore impossible for this memory to be 840 * deallocated and reallocated as something else while it's 841 * being operated upon. 842 */ 843 if (DTRACE_INRANGE(addr, sz, curthread, sizeof (kthread_t))) 844 return (1); 845 846 if ((p = curthread->t_procp) != NULL && DTRACE_INRANGE(addr, 847 sz, curthread->t_procp, sizeof (proc_t))) { 848 return (1); 849 } 850 851 if (curthread->t_cred != NULL && DTRACE_INRANGE(addr, sz, 852 curthread->t_cred, sizeof (cred_t))) { 853 return (1); 854 } 855 856 #ifdef illumos 857 if (p != NULL && p->p_pidp != NULL && DTRACE_INRANGE(addr, sz, 858 &(p->p_pidp->pid_id), sizeof (pid_t))) { 859 return (1); 860 } 861 862 if (curthread->t_cpu != NULL && DTRACE_INRANGE(addr, sz, 863 curthread->t_cpu, offsetof(cpu_t, cpu_pause_thread))) { 864 return (1); 865 } 866 #endif 867 } 868 869 if ((fp = mstate->dtms_getf) != NULL) { 870 uintptr_t psz = sizeof (void *); 871 vnode_t *vp; 872 vnodeops_t *op; 873 874 /* 875 * When getf() returns a file_t, the enabling is implicitly 876 * granted the (transient) right to read the returned file_t 877 * as well as the v_path and v_op->vnop_name of the underlying 878 * vnode. These accesses are allowed after a successful 879 * getf() because the members that they refer to cannot change 880 * once set -- and the barrier logic in the kernel's closef() 881 * path assures that the file_t and its referenced vode_t 882 * cannot themselves be stale (that is, it impossible for 883 * either dtms_getf itself or its f_vnode member to reference 884 * freed memory). 885 */ 886 if (DTRACE_INRANGE(addr, sz, fp, sizeof (file_t))) 887 return (1); 888 889 if ((vp = fp->f_vnode) != NULL) { 890 #ifdef illumos 891 if (DTRACE_INRANGE(addr, sz, &vp->v_path, psz)) 892 return (1); 893 if (vp->v_path != NULL && DTRACE_INRANGE(addr, sz, 894 vp->v_path, strlen(vp->v_path) + 1)) { 895 return (1); 896 } 897 #endif 898 899 if (DTRACE_INRANGE(addr, sz, &vp->v_op, psz)) 900 return (1); 901 902 #ifdef illumos 903 if ((op = vp->v_op) != NULL && 904 DTRACE_INRANGE(addr, sz, &op->vnop_name, psz)) { 905 return (1); 906 } 907 908 if (op != NULL && op->vnop_name != NULL && 909 DTRACE_INRANGE(addr, sz, op->vnop_name, 910 strlen(op->vnop_name) + 1)) { 911 return (1); 912 } 913 #endif 914 } 915 } 916 917 DTRACE_CPUFLAG_SET(CPU_DTRACE_KPRIV); 918 *illval = addr; 919 return (0); 920 } 921 922 /* 923 * Convenience routine to check to see if a given string is within a memory 924 * region in which a load may be issued given the user's privilege level; 925 * this exists so that we don't need to issue unnecessary dtrace_strlen() 926 * calls in the event that the user has all privileges. 927 */ 928 static int 929 dtrace_strcanload(uint64_t addr, size_t sz, dtrace_mstate_t *mstate, 930 dtrace_vstate_t *vstate) 931 { 932 size_t strsz; 933 934 /* 935 * If we hold the privilege to read from kernel memory, then 936 * everything is readable. 937 */ 938 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) 939 return (1); 940 941 strsz = 1 + dtrace_strlen((char *)(uintptr_t)addr, sz); 942 if (dtrace_canload(addr, strsz, mstate, vstate)) 943 return (1); 944 945 return (0); 946 } 947 948 /* 949 * Convenience routine to check to see if a given variable is within a memory 950 * region in which a load may be issued given the user's privilege level. 951 */ 952 static int 953 dtrace_vcanload(void *src, dtrace_diftype_t *type, dtrace_mstate_t *mstate, 954 dtrace_vstate_t *vstate) 955 { 956 size_t sz; 957 ASSERT(type->dtdt_flags & DIF_TF_BYREF); 958 959 /* 960 * If we hold the privilege to read from kernel memory, then 961 * everything is readable. 962 */ 963 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) 964 return (1); 965 966 if (type->dtdt_kind == DIF_TYPE_STRING) 967 sz = dtrace_strlen(src, 968 vstate->dtvs_state->dts_options[DTRACEOPT_STRSIZE]) + 1; 969 else 970 sz = type->dtdt_size; 971 972 return (dtrace_canload((uintptr_t)src, sz, mstate, vstate)); 973 } 974 975 /* 976 * Convert a string to a signed integer using safe loads. 977 * 978 * NOTE: This function uses various macros from strtolctype.h to manipulate 979 * digit values, etc -- these have all been checked to ensure they make 980 * no additional function calls. 981 */ 982 static int64_t 983 dtrace_strtoll(char *input, int base, size_t limit) 984 { 985 uintptr_t pos = (uintptr_t)input; 986 int64_t val = 0; 987 int x; 988 boolean_t neg = B_FALSE; 989 char c, cc, ccc; 990 uintptr_t end = pos + limit; 991 992 /* 993 * Consume any whitespace preceding digits. 994 */ 995 while ((c = dtrace_load8(pos)) == ' ' || c == '\t') 996 pos++; 997 998 /* 999 * Handle an explicit sign if one is present. 1000 */ 1001 if (c == '-' || c == '+') { 1002 if (c == '-') 1003 neg = B_TRUE; 1004 c = dtrace_load8(++pos); 1005 } 1006 1007 /* 1008 * Check for an explicit hexadecimal prefix ("0x" or "0X") and skip it 1009 * if present. 1010 */ 1011 if (base == 16 && c == '0' && ((cc = dtrace_load8(pos + 1)) == 'x' || 1012 cc == 'X') && isxdigit(ccc = dtrace_load8(pos + 2))) { 1013 pos += 2; 1014 c = ccc; 1015 } 1016 1017 /* 1018 * Read in contiguous digits until the first non-digit character. 1019 */ 1020 for (; pos < end && c != '\0' && lisalnum(c) && (x = DIGIT(c)) < base; 1021 c = dtrace_load8(++pos)) 1022 val = val * base + x; 1023 1024 return (neg ? -val : val); 1025 } 1026 1027 /* 1028 * Compare two strings using safe loads. 1029 */ 1030 static int 1031 dtrace_strncmp(char *s1, char *s2, size_t limit) 1032 { 1033 uint8_t c1, c2; 1034 volatile uint16_t *flags; 1035 1036 if (s1 == s2 || limit == 0) 1037 return (0); 1038 1039 flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; 1040 1041 do { 1042 if (s1 == NULL) { 1043 c1 = '\0'; 1044 } else { 1045 c1 = dtrace_load8((uintptr_t)s1++); 1046 } 1047 1048 if (s2 == NULL) { 1049 c2 = '\0'; 1050 } else { 1051 c2 = dtrace_load8((uintptr_t)s2++); 1052 } 1053 1054 if (c1 != c2) 1055 return (c1 - c2); 1056 } while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT)); 1057 1058 return (0); 1059 } 1060 1061 /* 1062 * Compute strlen(s) for a string using safe memory accesses. The additional 1063 * len parameter is used to specify a maximum length to ensure completion. 1064 */ 1065 static size_t 1066 dtrace_strlen(const char *s, size_t lim) 1067 { 1068 uint_t len; 1069 1070 for (len = 0; len != lim; len++) { 1071 if (dtrace_load8((uintptr_t)s++) == '\0') 1072 break; 1073 } 1074 1075 return (len); 1076 } 1077 1078 /* 1079 * Check if an address falls within a toxic region. 1080 */ 1081 static int 1082 dtrace_istoxic(uintptr_t kaddr, size_t size) 1083 { 1084 uintptr_t taddr, tsize; 1085 int i; 1086 1087 for (i = 0; i < dtrace_toxranges; i++) { 1088 taddr = dtrace_toxrange[i].dtt_base; 1089 tsize = dtrace_toxrange[i].dtt_limit - taddr; 1090 1091 if (kaddr - taddr < tsize) { 1092 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 1093 cpu_core[curcpu].cpuc_dtrace_illval = kaddr; 1094 return (1); 1095 } 1096 1097 if (taddr - kaddr < size) { 1098 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 1099 cpu_core[curcpu].cpuc_dtrace_illval = taddr; 1100 return (1); 1101 } 1102 } 1103 1104 return (0); 1105 } 1106 1107 /* 1108 * Copy src to dst using safe memory accesses. The src is assumed to be unsafe 1109 * memory specified by the DIF program. The dst is assumed to be safe memory 1110 * that we can store to directly because it is managed by DTrace. As with 1111 * standard bcopy, overlapping copies are handled properly. 1112 */ 1113 static void 1114 dtrace_bcopy(const void *src, void *dst, size_t len) 1115 { 1116 if (len != 0) { 1117 uint8_t *s1 = dst; 1118 const uint8_t *s2 = src; 1119 1120 if (s1 <= s2) { 1121 do { 1122 *s1++ = dtrace_load8((uintptr_t)s2++); 1123 } while (--len != 0); 1124 } else { 1125 s2 += len; 1126 s1 += len; 1127 1128 do { 1129 *--s1 = dtrace_load8((uintptr_t)--s2); 1130 } while (--len != 0); 1131 } 1132 } 1133 } 1134 1135 /* 1136 * Copy src to dst using safe memory accesses, up to either the specified 1137 * length, or the point that a nul byte is encountered. The src is assumed to 1138 * be unsafe memory specified by the DIF program. The dst is assumed to be 1139 * safe memory that we can store to directly because it is managed by DTrace. 1140 * Unlike dtrace_bcopy(), overlapping regions are not handled. 1141 */ 1142 static void 1143 dtrace_strcpy(const void *src, void *dst, size_t len) 1144 { 1145 if (len != 0) { 1146 uint8_t *s1 = dst, c; 1147 const uint8_t *s2 = src; 1148 1149 do { 1150 *s1++ = c = dtrace_load8((uintptr_t)s2++); 1151 } while (--len != 0 && c != '\0'); 1152 } 1153 } 1154 1155 /* 1156 * Copy src to dst, deriving the size and type from the specified (BYREF) 1157 * variable type. The src is assumed to be unsafe memory specified by the DIF 1158 * program. The dst is assumed to be DTrace variable memory that is of the 1159 * specified type; we assume that we can store to directly. 1160 */ 1161 static void 1162 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type) 1163 { 1164 ASSERT(type->dtdt_flags & DIF_TF_BYREF); 1165 1166 if (type->dtdt_kind == DIF_TYPE_STRING) { 1167 dtrace_strcpy(src, dst, type->dtdt_size); 1168 } else { 1169 dtrace_bcopy(src, dst, type->dtdt_size); 1170 } 1171 } 1172 1173 /* 1174 * Compare s1 to s2 using safe memory accesses. The s1 data is assumed to be 1175 * unsafe memory specified by the DIF program. The s2 data is assumed to be 1176 * safe memory that we can access directly because it is managed by DTrace. 1177 */ 1178 static int 1179 dtrace_bcmp(const void *s1, const void *s2, size_t len) 1180 { 1181 volatile uint16_t *flags; 1182 1183 flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; 1184 1185 if (s1 == s2) 1186 return (0); 1187 1188 if (s1 == NULL || s2 == NULL) 1189 return (1); 1190 1191 if (s1 != s2 && len != 0) { 1192 const uint8_t *ps1 = s1; 1193 const uint8_t *ps2 = s2; 1194 1195 do { 1196 if (dtrace_load8((uintptr_t)ps1++) != *ps2++) 1197 return (1); 1198 } while (--len != 0 && !(*flags & CPU_DTRACE_FAULT)); 1199 } 1200 return (0); 1201 } 1202 1203 /* 1204 * Zero the specified region using a simple byte-by-byte loop. Note that this 1205 * is for safe DTrace-managed memory only. 1206 */ 1207 static void 1208 dtrace_bzero(void *dst, size_t len) 1209 { 1210 uchar_t *cp; 1211 1212 for (cp = dst; len != 0; len--) 1213 *cp++ = 0; 1214 } 1215 1216 static void 1217 dtrace_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum) 1218 { 1219 uint64_t result[2]; 1220 1221 result[0] = addend1[0] + addend2[0]; 1222 result[1] = addend1[1] + addend2[1] + 1223 (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0); 1224 1225 sum[0] = result[0]; 1226 sum[1] = result[1]; 1227 } 1228 1229 /* 1230 * Shift the 128-bit value in a by b. If b is positive, shift left. 1231 * If b is negative, shift right. 1232 */ 1233 static void 1234 dtrace_shift_128(uint64_t *a, int b) 1235 { 1236 uint64_t mask; 1237 1238 if (b == 0) 1239 return; 1240 1241 if (b < 0) { 1242 b = -b; 1243 if (b >= 64) { 1244 a[0] = a[1] >> (b - 64); 1245 a[1] = 0; 1246 } else { 1247 a[0] >>= b; 1248 mask = 1LL << (64 - b); 1249 mask -= 1; 1250 a[0] |= ((a[1] & mask) << (64 - b)); 1251 a[1] >>= b; 1252 } 1253 } else { 1254 if (b >= 64) { 1255 a[1] = a[0] << (b - 64); 1256 a[0] = 0; 1257 } else { 1258 a[1] <<= b; 1259 mask = a[0] >> (64 - b); 1260 a[1] |= mask; 1261 a[0] <<= b; 1262 } 1263 } 1264 } 1265 1266 /* 1267 * The basic idea is to break the 2 64-bit values into 4 32-bit values, 1268 * use native multiplication on those, and then re-combine into the 1269 * resulting 128-bit value. 1270 * 1271 * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) = 1272 * hi1 * hi2 << 64 + 1273 * hi1 * lo2 << 32 + 1274 * hi2 * lo1 << 32 + 1275 * lo1 * lo2 1276 */ 1277 static void 1278 dtrace_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product) 1279 { 1280 uint64_t hi1, hi2, lo1, lo2; 1281 uint64_t tmp[2]; 1282 1283 hi1 = factor1 >> 32; 1284 hi2 = factor2 >> 32; 1285 1286 lo1 = factor1 & DT_MASK_LO; 1287 lo2 = factor2 & DT_MASK_LO; 1288 1289 product[0] = lo1 * lo2; 1290 product[1] = hi1 * hi2; 1291 1292 tmp[0] = hi1 * lo2; 1293 tmp[1] = 0; 1294 dtrace_shift_128(tmp, 32); 1295 dtrace_add_128(product, tmp, product); 1296 1297 tmp[0] = hi2 * lo1; 1298 tmp[1] = 0; 1299 dtrace_shift_128(tmp, 32); 1300 dtrace_add_128(product, tmp, product); 1301 } 1302 1303 /* 1304 * This privilege check should be used by actions and subroutines to 1305 * verify that the user credentials of the process that enabled the 1306 * invoking ECB match the target credentials 1307 */ 1308 static int 1309 dtrace_priv_proc_common_user(dtrace_state_t *state) 1310 { 1311 cred_t *cr, *s_cr = state->dts_cred.dcr_cred; 1312 1313 /* 1314 * We should always have a non-NULL state cred here, since if cred 1315 * is null (anonymous tracing), we fast-path bypass this routine. 1316 */ 1317 ASSERT(s_cr != NULL); 1318 1319 if ((cr = CRED()) != NULL && 1320 s_cr->cr_uid == cr->cr_uid && 1321 s_cr->cr_uid == cr->cr_ruid && 1322 s_cr->cr_uid == cr->cr_suid && 1323 s_cr->cr_gid == cr->cr_gid && 1324 s_cr->cr_gid == cr->cr_rgid && 1325 s_cr->cr_gid == cr->cr_sgid) 1326 return (1); 1327 1328 return (0); 1329 } 1330 1331 /* 1332 * This privilege check should be used by actions and subroutines to 1333 * verify that the zone of the process that enabled the invoking ECB 1334 * matches the target credentials 1335 */ 1336 static int 1337 dtrace_priv_proc_common_zone(dtrace_state_t *state) 1338 { 1339 #ifdef illumos 1340 cred_t *cr, *s_cr = state->dts_cred.dcr_cred; 1341 1342 /* 1343 * We should always have a non-NULL state cred here, since if cred 1344 * is null (anonymous tracing), we fast-path bypass this routine. 1345 */ 1346 ASSERT(s_cr != NULL); 1347 1348 if ((cr = CRED()) != NULL && s_cr->cr_zone == cr->cr_zone) 1349 return (1); 1350 1351 return (0); 1352 #else 1353 return (1); 1354 #endif 1355 } 1356 1357 /* 1358 * This privilege check should be used by actions and subroutines to 1359 * verify that the process has not setuid or changed credentials. 1360 */ 1361 static int 1362 dtrace_priv_proc_common_nocd(void) 1363 { 1364 proc_t *proc; 1365 1366 if ((proc = ttoproc(curthread)) != NULL && 1367 !(proc->p_flag & SNOCD)) 1368 return (1); 1369 1370 return (0); 1371 } 1372 1373 static int 1374 dtrace_priv_proc_destructive(dtrace_state_t *state) 1375 { 1376 int action = state->dts_cred.dcr_action; 1377 1378 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE) == 0) && 1379 dtrace_priv_proc_common_zone(state) == 0) 1380 goto bad; 1381 1382 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER) == 0) && 1383 dtrace_priv_proc_common_user(state) == 0) 1384 goto bad; 1385 1386 if (((action & DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG) == 0) && 1387 dtrace_priv_proc_common_nocd() == 0) 1388 goto bad; 1389 1390 return (1); 1391 1392 bad: 1393 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; 1394 1395 return (0); 1396 } 1397 1398 static int 1399 dtrace_priv_proc_control(dtrace_state_t *state) 1400 { 1401 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL) 1402 return (1); 1403 1404 if (dtrace_priv_proc_common_zone(state) && 1405 dtrace_priv_proc_common_user(state) && 1406 dtrace_priv_proc_common_nocd()) 1407 return (1); 1408 1409 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; 1410 1411 return (0); 1412 } 1413 1414 static int 1415 dtrace_priv_proc(dtrace_state_t *state) 1416 { 1417 if (state->dts_cred.dcr_action & DTRACE_CRA_PROC) 1418 return (1); 1419 1420 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV; 1421 1422 return (0); 1423 } 1424 1425 static int 1426 dtrace_priv_kernel(dtrace_state_t *state) 1427 { 1428 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL) 1429 return (1); 1430 1431 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV; 1432 1433 return (0); 1434 } 1435 1436 static int 1437 dtrace_priv_kernel_destructive(dtrace_state_t *state) 1438 { 1439 if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE) 1440 return (1); 1441 1442 cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV; 1443 1444 return (0); 1445 } 1446 1447 /* 1448 * Determine if the dte_cond of the specified ECB allows for processing of 1449 * the current probe to continue. Note that this routine may allow continued 1450 * processing, but with access(es) stripped from the mstate's dtms_access 1451 * field. 1452 */ 1453 static int 1454 dtrace_priv_probe(dtrace_state_t *state, dtrace_mstate_t *mstate, 1455 dtrace_ecb_t *ecb) 1456 { 1457 dtrace_probe_t *probe = ecb->dte_probe; 1458 dtrace_provider_t *prov = probe->dtpr_provider; 1459 dtrace_pops_t *pops = &prov->dtpv_pops; 1460 int mode = DTRACE_MODE_NOPRIV_DROP; 1461 1462 ASSERT(ecb->dte_cond); 1463 1464 #ifdef illumos 1465 if (pops->dtps_mode != NULL) { 1466 mode = pops->dtps_mode(prov->dtpv_arg, 1467 probe->dtpr_id, probe->dtpr_arg); 1468 1469 ASSERT((mode & DTRACE_MODE_USER) || 1470 (mode & DTRACE_MODE_KERNEL)); 1471 ASSERT((mode & DTRACE_MODE_NOPRIV_RESTRICT) || 1472 (mode & DTRACE_MODE_NOPRIV_DROP)); 1473 } 1474 1475 /* 1476 * If the dte_cond bits indicate that this consumer is only allowed to 1477 * see user-mode firings of this probe, call the provider's dtps_mode() 1478 * entry point to check that the probe was fired while in a user 1479 * context. If that's not the case, use the policy specified by the 1480 * provider to determine if we drop the probe or merely restrict 1481 * operation. 1482 */ 1483 if (ecb->dte_cond & DTRACE_COND_USERMODE) { 1484 ASSERT(mode != DTRACE_MODE_NOPRIV_DROP); 1485 1486 if (!(mode & DTRACE_MODE_USER)) { 1487 if (mode & DTRACE_MODE_NOPRIV_DROP) 1488 return (0); 1489 1490 mstate->dtms_access &= ~DTRACE_ACCESS_ARGS; 1491 } 1492 } 1493 #endif 1494 1495 /* 1496 * This is more subtle than it looks. We have to be absolutely certain 1497 * that CRED() isn't going to change out from under us so it's only 1498 * legit to examine that structure if we're in constrained situations. 1499 * Currently, the only times we'll this check is if a non-super-user 1500 * has enabled the profile or syscall providers -- providers that 1501 * allow visibility of all processes. For the profile case, the check 1502 * above will ensure that we're examining a user context. 1503 */ 1504 if (ecb->dte_cond & DTRACE_COND_OWNER) { 1505 cred_t *cr; 1506 cred_t *s_cr = state->dts_cred.dcr_cred; 1507 proc_t *proc; 1508 1509 ASSERT(s_cr != NULL); 1510 1511 if ((cr = CRED()) == NULL || 1512 s_cr->cr_uid != cr->cr_uid || 1513 s_cr->cr_uid != cr->cr_ruid || 1514 s_cr->cr_uid != cr->cr_suid || 1515 s_cr->cr_gid != cr->cr_gid || 1516 s_cr->cr_gid != cr->cr_rgid || 1517 s_cr->cr_gid != cr->cr_sgid || 1518 (proc = ttoproc(curthread)) == NULL || 1519 (proc->p_flag & SNOCD)) { 1520 if (mode & DTRACE_MODE_NOPRIV_DROP) 1521 return (0); 1522 1523 #ifdef illumos 1524 mstate->dtms_access &= ~DTRACE_ACCESS_PROC; 1525 #endif 1526 } 1527 } 1528 1529 #ifdef illumos 1530 /* 1531 * If our dte_cond is set to DTRACE_COND_ZONEOWNER and we are not 1532 * in our zone, check to see if our mode policy is to restrict rather 1533 * than to drop; if to restrict, strip away both DTRACE_ACCESS_PROC 1534 * and DTRACE_ACCESS_ARGS 1535 */ 1536 if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) { 1537 cred_t *cr; 1538 cred_t *s_cr = state->dts_cred.dcr_cred; 1539 1540 ASSERT(s_cr != NULL); 1541 1542 if ((cr = CRED()) == NULL || 1543 s_cr->cr_zone->zone_id != cr->cr_zone->zone_id) { 1544 if (mode & DTRACE_MODE_NOPRIV_DROP) 1545 return (0); 1546 1547 mstate->dtms_access &= 1548 ~(DTRACE_ACCESS_PROC | DTRACE_ACCESS_ARGS); 1549 } 1550 } 1551 #endif 1552 1553 return (1); 1554 } 1555 1556 /* 1557 * Note: not called from probe context. This function is called 1558 * asynchronously (and at a regular interval) from outside of probe context to 1559 * clean the dirty dynamic variable lists on all CPUs. Dynamic variable 1560 * cleaning is explained in detail in <sys/dtrace_impl.h>. 1561 */ 1562 void 1563 dtrace_dynvar_clean(dtrace_dstate_t *dstate) 1564 { 1565 dtrace_dynvar_t *dirty; 1566 dtrace_dstate_percpu_t *dcpu; 1567 dtrace_dynvar_t **rinsep; 1568 int i, j, work = 0; 1569 1570 for (i = 0; i < NCPU; i++) { 1571 dcpu = &dstate->dtds_percpu[i]; 1572 rinsep = &dcpu->dtdsc_rinsing; 1573 1574 /* 1575 * If the dirty list is NULL, there is no dirty work to do. 1576 */ 1577 if (dcpu->dtdsc_dirty == NULL) 1578 continue; 1579 1580 if (dcpu->dtdsc_rinsing != NULL) { 1581 /* 1582 * If the rinsing list is non-NULL, then it is because 1583 * this CPU was selected to accept another CPU's 1584 * dirty list -- and since that time, dirty buffers 1585 * have accumulated. This is a highly unlikely 1586 * condition, but we choose to ignore the dirty 1587 * buffers -- they'll be picked up a future cleanse. 1588 */ 1589 continue; 1590 } 1591 1592 if (dcpu->dtdsc_clean != NULL) { 1593 /* 1594 * If the clean list is non-NULL, then we're in a 1595 * situation where a CPU has done deallocations (we 1596 * have a non-NULL dirty list) but no allocations (we 1597 * also have a non-NULL clean list). We can't simply 1598 * move the dirty list into the clean list on this 1599 * CPU, yet we also don't want to allow this condition 1600 * to persist, lest a short clean list prevent a 1601 * massive dirty list from being cleaned (which in 1602 * turn could lead to otherwise avoidable dynamic 1603 * drops). To deal with this, we look for some CPU 1604 * with a NULL clean list, NULL dirty list, and NULL 1605 * rinsing list -- and then we borrow this CPU to 1606 * rinse our dirty list. 1607 */ 1608 for (j = 0; j < NCPU; j++) { 1609 dtrace_dstate_percpu_t *rinser; 1610 1611 rinser = &dstate->dtds_percpu[j]; 1612 1613 if (rinser->dtdsc_rinsing != NULL) 1614 continue; 1615 1616 if (rinser->dtdsc_dirty != NULL) 1617 continue; 1618 1619 if (rinser->dtdsc_clean != NULL) 1620 continue; 1621 1622 rinsep = &rinser->dtdsc_rinsing; 1623 break; 1624 } 1625 1626 if (j == NCPU) { 1627 /* 1628 * We were unable to find another CPU that 1629 * could accept this dirty list -- we are 1630 * therefore unable to clean it now. 1631 */ 1632 dtrace_dynvar_failclean++; 1633 continue; 1634 } 1635 } 1636 1637 work = 1; 1638 1639 /* 1640 * Atomically move the dirty list aside. 1641 */ 1642 do { 1643 dirty = dcpu->dtdsc_dirty; 1644 1645 /* 1646 * Before we zap the dirty list, set the rinsing list. 1647 * (This allows for a potential assertion in 1648 * dtrace_dynvar(): if a free dynamic variable appears 1649 * on a hash chain, either the dirty list or the 1650 * rinsing list for some CPU must be non-NULL.) 1651 */ 1652 *rinsep = dirty; 1653 dtrace_membar_producer(); 1654 } while (dtrace_casptr(&dcpu->dtdsc_dirty, 1655 dirty, NULL) != dirty); 1656 } 1657 1658 if (!work) { 1659 /* 1660 * We have no work to do; we can simply return. 1661 */ 1662 return; 1663 } 1664 1665 dtrace_sync(); 1666 1667 for (i = 0; i < NCPU; i++) { 1668 dcpu = &dstate->dtds_percpu[i]; 1669 1670 if (dcpu->dtdsc_rinsing == NULL) 1671 continue; 1672 1673 /* 1674 * We are now guaranteed that no hash chain contains a pointer 1675 * into this dirty list; we can make it clean. 1676 */ 1677 ASSERT(dcpu->dtdsc_clean == NULL); 1678 dcpu->dtdsc_clean = dcpu->dtdsc_rinsing; 1679 dcpu->dtdsc_rinsing = NULL; 1680 } 1681 1682 /* 1683 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make 1684 * sure that all CPUs have seen all of the dtdsc_clean pointers. 1685 * This prevents a race whereby a CPU incorrectly decides that 1686 * the state should be something other than DTRACE_DSTATE_CLEAN 1687 * after dtrace_dynvar_clean() has completed. 1688 */ 1689 dtrace_sync(); 1690 1691 dstate->dtds_state = DTRACE_DSTATE_CLEAN; 1692 } 1693 1694 /* 1695 * Depending on the value of the op parameter, this function looks-up, 1696 * allocates or deallocates an arbitrarily-keyed dynamic variable. If an 1697 * allocation is requested, this function will return a pointer to a 1698 * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no 1699 * variable can be allocated. If NULL is returned, the appropriate counter 1700 * will be incremented. 1701 */ 1702 dtrace_dynvar_t * 1703 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys, 1704 dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op, 1705 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate) 1706 { 1707 uint64_t hashval = DTRACE_DYNHASH_VALID; 1708 dtrace_dynhash_t *hash = dstate->dtds_hash; 1709 dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL; 1710 processorid_t me = curcpu, cpu = me; 1711 dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me]; 1712 size_t bucket, ksize; 1713 size_t chunksize = dstate->dtds_chunksize; 1714 uintptr_t kdata, lock, nstate; 1715 uint_t i; 1716 1717 ASSERT(nkeys != 0); 1718 1719 /* 1720 * Hash the key. As with aggregations, we use Jenkins' "One-at-a-time" 1721 * algorithm. For the by-value portions, we perform the algorithm in 1722 * 16-bit chunks (as opposed to 8-bit chunks). This speeds things up a 1723 * bit, and seems to have only a minute effect on distribution. For 1724 * the by-reference data, we perform "One-at-a-time" iterating (safely) 1725 * over each referenced byte. It's painful to do this, but it's much 1726 * better than pathological hash distribution. The efficacy of the 1727 * hashing algorithm (and a comparison with other algorithms) may be 1728 * found by running the ::dtrace_dynstat MDB dcmd. 1729 */ 1730 for (i = 0; i < nkeys; i++) { 1731 if (key[i].dttk_size == 0) { 1732 uint64_t val = key[i].dttk_value; 1733 1734 hashval += (val >> 48) & 0xffff; 1735 hashval += (hashval << 10); 1736 hashval ^= (hashval >> 6); 1737 1738 hashval += (val >> 32) & 0xffff; 1739 hashval += (hashval << 10); 1740 hashval ^= (hashval >> 6); 1741 1742 hashval += (val >> 16) & 0xffff; 1743 hashval += (hashval << 10); 1744 hashval ^= (hashval >> 6); 1745 1746 hashval += val & 0xffff; 1747 hashval += (hashval << 10); 1748 hashval ^= (hashval >> 6); 1749 } else { 1750 /* 1751 * This is incredibly painful, but it beats the hell 1752 * out of the alternative. 1753 */ 1754 uint64_t j, size = key[i].dttk_size; 1755 uintptr_t base = (uintptr_t)key[i].dttk_value; 1756 1757 if (!dtrace_canload(base, size, mstate, vstate)) 1758 break; 1759 1760 for (j = 0; j < size; j++) { 1761 hashval += dtrace_load8(base + j); 1762 hashval += (hashval << 10); 1763 hashval ^= (hashval >> 6); 1764 } 1765 } 1766 } 1767 1768 if (DTRACE_CPUFLAG_ISSET(CPU_DTRACE_FAULT)) 1769 return (NULL); 1770 1771 hashval += (hashval << 3); 1772 hashval ^= (hashval >> 11); 1773 hashval += (hashval << 15); 1774 1775 /* 1776 * There is a remote chance (ideally, 1 in 2^31) that our hashval 1777 * comes out to be one of our two sentinel hash values. If this 1778 * actually happens, we set the hashval to be a value known to be a 1779 * non-sentinel value. 1780 */ 1781 if (hashval == DTRACE_DYNHASH_FREE || hashval == DTRACE_DYNHASH_SINK) 1782 hashval = DTRACE_DYNHASH_VALID; 1783 1784 /* 1785 * Yes, it's painful to do a divide here. If the cycle count becomes 1786 * important here, tricks can be pulled to reduce it. (However, it's 1787 * critical that hash collisions be kept to an absolute minimum; 1788 * they're much more painful than a divide.) It's better to have a 1789 * solution that generates few collisions and still keeps things 1790 * relatively simple. 1791 */ 1792 bucket = hashval % dstate->dtds_hashsize; 1793 1794 if (op == DTRACE_DYNVAR_DEALLOC) { 1795 volatile uintptr_t *lockp = &hash[bucket].dtdh_lock; 1796 1797 for (;;) { 1798 while ((lock = *lockp) & 1) 1799 continue; 1800 1801 if (dtrace_casptr((volatile void *)lockp, 1802 (volatile void *)lock, (volatile void *)(lock + 1)) == (void *)lock) 1803 break; 1804 } 1805 1806 dtrace_membar_producer(); 1807 } 1808 1809 top: 1810 prev = NULL; 1811 lock = hash[bucket].dtdh_lock; 1812 1813 dtrace_membar_consumer(); 1814 1815 start = hash[bucket].dtdh_chain; 1816 ASSERT(start != NULL && (start->dtdv_hashval == DTRACE_DYNHASH_SINK || 1817 start->dtdv_hashval != DTRACE_DYNHASH_FREE || 1818 op != DTRACE_DYNVAR_DEALLOC)); 1819 1820 for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) { 1821 dtrace_tuple_t *dtuple = &dvar->dtdv_tuple; 1822 dtrace_key_t *dkey = &dtuple->dtt_key[0]; 1823 1824 if (dvar->dtdv_hashval != hashval) { 1825 if (dvar->dtdv_hashval == DTRACE_DYNHASH_SINK) { 1826 /* 1827 * We've reached the sink, and therefore the 1828 * end of the hash chain; we can kick out of 1829 * the loop knowing that we have seen a valid 1830 * snapshot of state. 1831 */ 1832 ASSERT(dvar->dtdv_next == NULL); 1833 ASSERT(dvar == &dtrace_dynhash_sink); 1834 break; 1835 } 1836 1837 if (dvar->dtdv_hashval == DTRACE_DYNHASH_FREE) { 1838 /* 1839 * We've gone off the rails: somewhere along 1840 * the line, one of the members of this hash 1841 * chain was deleted. Note that we could also 1842 * detect this by simply letting this loop run 1843 * to completion, as we would eventually hit 1844 * the end of the dirty list. However, we 1845 * want to avoid running the length of the 1846 * dirty list unnecessarily (it might be quite 1847 * long), so we catch this as early as 1848 * possible by detecting the hash marker. In 1849 * this case, we simply set dvar to NULL and 1850 * break; the conditional after the loop will 1851 * send us back to top. 1852 */ 1853 dvar = NULL; 1854 break; 1855 } 1856 1857 goto next; 1858 } 1859 1860 if (dtuple->dtt_nkeys != nkeys) 1861 goto next; 1862 1863 for (i = 0; i < nkeys; i++, dkey++) { 1864 if (dkey->dttk_size != key[i].dttk_size) 1865 goto next; /* size or type mismatch */ 1866 1867 if (dkey->dttk_size != 0) { 1868 if (dtrace_bcmp( 1869 (void *)(uintptr_t)key[i].dttk_value, 1870 (void *)(uintptr_t)dkey->dttk_value, 1871 dkey->dttk_size)) 1872 goto next; 1873 } else { 1874 if (dkey->dttk_value != key[i].dttk_value) 1875 goto next; 1876 } 1877 } 1878 1879 if (op != DTRACE_DYNVAR_DEALLOC) 1880 return (dvar); 1881 1882 ASSERT(dvar->dtdv_next == NULL || 1883 dvar->dtdv_next->dtdv_hashval != DTRACE_DYNHASH_FREE); 1884 1885 if (prev != NULL) { 1886 ASSERT(hash[bucket].dtdh_chain != dvar); 1887 ASSERT(start != dvar); 1888 ASSERT(prev->dtdv_next == dvar); 1889 prev->dtdv_next = dvar->dtdv_next; 1890 } else { 1891 if (dtrace_casptr(&hash[bucket].dtdh_chain, 1892 start, dvar->dtdv_next) != start) { 1893 /* 1894 * We have failed to atomically swing the 1895 * hash table head pointer, presumably because 1896 * of a conflicting allocation on another CPU. 1897 * We need to reread the hash chain and try 1898 * again. 1899 */ 1900 goto top; 1901 } 1902 } 1903 1904 dtrace_membar_producer(); 1905 1906 /* 1907 * Now set the hash value to indicate that it's free. 1908 */ 1909 ASSERT(hash[bucket].dtdh_chain != dvar); 1910 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE; 1911 1912 dtrace_membar_producer(); 1913 1914 /* 1915 * Set the next pointer to point at the dirty list, and 1916 * atomically swing the dirty pointer to the newly freed dvar. 1917 */ 1918 do { 1919 next = dcpu->dtdsc_dirty; 1920 dvar->dtdv_next = next; 1921 } while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next); 1922 1923 /* 1924 * Finally, unlock this hash bucket. 1925 */ 1926 ASSERT(hash[bucket].dtdh_lock == lock); 1927 ASSERT(lock & 1); 1928 hash[bucket].dtdh_lock++; 1929 1930 return (NULL); 1931 next: 1932 prev = dvar; 1933 continue; 1934 } 1935 1936 if (dvar == NULL) { 1937 /* 1938 * If dvar is NULL, it is because we went off the rails: 1939 * one of the elements that we traversed in the hash chain 1940 * was deleted while we were traversing it. In this case, 1941 * we assert that we aren't doing a dealloc (deallocs lock 1942 * the hash bucket to prevent themselves from racing with 1943 * one another), and retry the hash chain traversal. 1944 */ 1945 ASSERT(op != DTRACE_DYNVAR_DEALLOC); 1946 goto top; 1947 } 1948 1949 if (op != DTRACE_DYNVAR_ALLOC) { 1950 /* 1951 * If we are not to allocate a new variable, we want to 1952 * return NULL now. Before we return, check that the value 1953 * of the lock word hasn't changed. If it has, we may have 1954 * seen an inconsistent snapshot. 1955 */ 1956 if (op == DTRACE_DYNVAR_NOALLOC) { 1957 if (hash[bucket].dtdh_lock != lock) 1958 goto top; 1959 } else { 1960 ASSERT(op == DTRACE_DYNVAR_DEALLOC); 1961 ASSERT(hash[bucket].dtdh_lock == lock); 1962 ASSERT(lock & 1); 1963 hash[bucket].dtdh_lock++; 1964 } 1965 1966 return (NULL); 1967 } 1968 1969 /* 1970 * We need to allocate a new dynamic variable. The size we need is the 1971 * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the 1972 * size of any auxiliary key data (rounded up to 8-byte alignment) plus 1973 * the size of any referred-to data (dsize). We then round the final 1974 * size up to the chunksize for allocation. 1975 */ 1976 for (ksize = 0, i = 0; i < nkeys; i++) 1977 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t)); 1978 1979 /* 1980 * This should be pretty much impossible, but could happen if, say, 1981 * strange DIF specified the tuple. Ideally, this should be an 1982 * assertion and not an error condition -- but that requires that the 1983 * chunksize calculation in dtrace_difo_chunksize() be absolutely 1984 * bullet-proof. (That is, it must not be able to be fooled by 1985 * malicious DIF.) Given the lack of backwards branches in DIF, 1986 * solving this would presumably not amount to solving the Halting 1987 * Problem -- but it still seems awfully hard. 1988 */ 1989 if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) + 1990 ksize + dsize > chunksize) { 1991 dcpu->dtdsc_drops++; 1992 return (NULL); 1993 } 1994 1995 nstate = DTRACE_DSTATE_EMPTY; 1996 1997 do { 1998 retry: 1999 free = dcpu->dtdsc_free; 2000 2001 if (free == NULL) { 2002 dtrace_dynvar_t *clean = dcpu->dtdsc_clean; 2003 void *rval; 2004 2005 if (clean == NULL) { 2006 /* 2007 * We're out of dynamic variable space on 2008 * this CPU. Unless we have tried all CPUs, 2009 * we'll try to allocate from a different 2010 * CPU. 2011 */ 2012 switch (dstate->dtds_state) { 2013 case DTRACE_DSTATE_CLEAN: { 2014 void *sp = &dstate->dtds_state; 2015 2016 if (++cpu >= NCPU) 2017 cpu = 0; 2018 2019 if (dcpu->dtdsc_dirty != NULL && 2020 nstate == DTRACE_DSTATE_EMPTY) 2021 nstate = DTRACE_DSTATE_DIRTY; 2022 2023 if (dcpu->dtdsc_rinsing != NULL) 2024 nstate = DTRACE_DSTATE_RINSING; 2025 2026 dcpu = &dstate->dtds_percpu[cpu]; 2027 2028 if (cpu != me) 2029 goto retry; 2030 2031 (void) dtrace_cas32(sp, 2032 DTRACE_DSTATE_CLEAN, nstate); 2033 2034 /* 2035 * To increment the correct bean 2036 * counter, take another lap. 2037 */ 2038 goto retry; 2039 } 2040 2041 case DTRACE_DSTATE_DIRTY: 2042 dcpu->dtdsc_dirty_drops++; 2043 break; 2044 2045 case DTRACE_DSTATE_RINSING: 2046 dcpu->dtdsc_rinsing_drops++; 2047 break; 2048 2049 case DTRACE_DSTATE_EMPTY: 2050 dcpu->dtdsc_drops++; 2051 break; 2052 } 2053 2054 DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP); 2055 return (NULL); 2056 } 2057 2058 /* 2059 * The clean list appears to be non-empty. We want to 2060 * move the clean list to the free list; we start by 2061 * moving the clean pointer aside. 2062 */ 2063 if (dtrace_casptr(&dcpu->dtdsc_clean, 2064 clean, NULL) != clean) { 2065 /* 2066 * We are in one of two situations: 2067 * 2068 * (a) The clean list was switched to the 2069 * free list by another CPU. 2070 * 2071 * (b) The clean list was added to by the 2072 * cleansing cyclic. 2073 * 2074 * In either of these situations, we can 2075 * just reattempt the free list allocation. 2076 */ 2077 goto retry; 2078 } 2079 2080 ASSERT(clean->dtdv_hashval == DTRACE_DYNHASH_FREE); 2081 2082 /* 2083 * Now we'll move the clean list to our free list. 2084 * It's impossible for this to fail: the only way 2085 * the free list can be updated is through this 2086 * code path, and only one CPU can own the clean list. 2087 * Thus, it would only be possible for this to fail if 2088 * this code were racing with dtrace_dynvar_clean(). 2089 * (That is, if dtrace_dynvar_clean() updated the clean 2090 * list, and we ended up racing to update the free 2091 * list.) This race is prevented by the dtrace_sync() 2092 * in dtrace_dynvar_clean() -- which flushes the 2093 * owners of the clean lists out before resetting 2094 * the clean lists. 2095 */ 2096 dcpu = &dstate->dtds_percpu[me]; 2097 rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean); 2098 ASSERT(rval == NULL); 2099 goto retry; 2100 } 2101 2102 dvar = free; 2103 new_free = dvar->dtdv_next; 2104 } while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free); 2105 2106 /* 2107 * We have now allocated a new chunk. We copy the tuple keys into the 2108 * tuple array and copy any referenced key data into the data space 2109 * following the tuple array. As we do this, we relocate dttk_value 2110 * in the final tuple to point to the key data address in the chunk. 2111 */ 2112 kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys]; 2113 dvar->dtdv_data = (void *)(kdata + ksize); 2114 dvar->dtdv_tuple.dtt_nkeys = nkeys; 2115 2116 for (i = 0; i < nkeys; i++) { 2117 dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i]; 2118 size_t kesize = key[i].dttk_size; 2119 2120 if (kesize != 0) { 2121 dtrace_bcopy( 2122 (const void *)(uintptr_t)key[i].dttk_value, 2123 (void *)kdata, kesize); 2124 dkey->dttk_value = kdata; 2125 kdata += P2ROUNDUP(kesize, sizeof (uint64_t)); 2126 } else { 2127 dkey->dttk_value = key[i].dttk_value; 2128 } 2129 2130 dkey->dttk_size = kesize; 2131 } 2132 2133 ASSERT(dvar->dtdv_hashval == DTRACE_DYNHASH_FREE); 2134 dvar->dtdv_hashval = hashval; 2135 dvar->dtdv_next = start; 2136 2137 if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start) 2138 return (dvar); 2139 2140 /* 2141 * The cas has failed. Either another CPU is adding an element to 2142 * this hash chain, or another CPU is deleting an element from this 2143 * hash chain. The simplest way to deal with both of these cases 2144 * (though not necessarily the most efficient) is to free our 2145 * allocated block and tail-call ourselves. Note that the free is 2146 * to the dirty list and _not_ to the free list. This is to prevent 2147 * races with allocators, above. 2148 */ 2149 dvar->dtdv_hashval = DTRACE_DYNHASH_FREE; 2150 2151 dtrace_membar_producer(); 2152 2153 do { 2154 free = dcpu->dtdsc_dirty; 2155 dvar->dtdv_next = free; 2156 } while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free); 2157 2158 return (dtrace_dynvar(dstate, nkeys, key, dsize, op, mstate, vstate)); 2159 } 2160 2161 /*ARGSUSED*/ 2162 static void 2163 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg) 2164 { 2165 if ((int64_t)nval < (int64_t)*oval) 2166 *oval = nval; 2167 } 2168 2169 /*ARGSUSED*/ 2170 static void 2171 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg) 2172 { 2173 if ((int64_t)nval > (int64_t)*oval) 2174 *oval = nval; 2175 } 2176 2177 static void 2178 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr) 2179 { 2180 int i, zero = DTRACE_QUANTIZE_ZEROBUCKET; 2181 int64_t val = (int64_t)nval; 2182 2183 if (val < 0) { 2184 for (i = 0; i < zero; i++) { 2185 if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) { 2186 quanta[i] += incr; 2187 return; 2188 } 2189 } 2190 } else { 2191 for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) { 2192 if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) { 2193 quanta[i - 1] += incr; 2194 return; 2195 } 2196 } 2197 2198 quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr; 2199 return; 2200 } 2201 2202 ASSERT(0); 2203 } 2204 2205 static void 2206 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr) 2207 { 2208 uint64_t arg = *lquanta++; 2209 int32_t base = DTRACE_LQUANTIZE_BASE(arg); 2210 uint16_t step = DTRACE_LQUANTIZE_STEP(arg); 2211 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg); 2212 int32_t val = (int32_t)nval, level; 2213 2214 ASSERT(step != 0); 2215 ASSERT(levels != 0); 2216 2217 if (val < base) { 2218 /* 2219 * This is an underflow. 2220 */ 2221 lquanta[0] += incr; 2222 return; 2223 } 2224 2225 level = (val - base) / step; 2226 2227 if (level < levels) { 2228 lquanta[level + 1] += incr; 2229 return; 2230 } 2231 2232 /* 2233 * This is an overflow. 2234 */ 2235 lquanta[levels + 1] += incr; 2236 } 2237 2238 static int 2239 dtrace_aggregate_llquantize_bucket(uint16_t factor, uint16_t low, 2240 uint16_t high, uint16_t nsteps, int64_t value) 2241 { 2242 int64_t this = 1, last, next; 2243 int base = 1, order; 2244 2245 ASSERT(factor <= nsteps); 2246 ASSERT(nsteps % factor == 0); 2247 2248 for (order = 0; order < low; order++) 2249 this *= factor; 2250 2251 /* 2252 * If our value is less than our factor taken to the power of the 2253 * low order of magnitude, it goes into the zeroth bucket. 2254 */ 2255 if (value < (last = this)) 2256 return (0); 2257 2258 for (this *= factor; order <= high; order++) { 2259 int nbuckets = this > nsteps ? nsteps : this; 2260 2261 if ((next = this * factor) < this) { 2262 /* 2263 * We should not generally get log/linear quantizations 2264 * with a high magnitude that allows 64-bits to 2265 * overflow, but we nonetheless protect against this 2266 * by explicitly checking for overflow, and clamping 2267 * our value accordingly. 2268 */ 2269 value = this - 1; 2270 } 2271 2272 if (value < this) { 2273 /* 2274 * If our value lies within this order of magnitude, 2275 * determine its position by taking the offset within 2276 * the order of magnitude, dividing by the bucket 2277 * width, and adding to our (accumulated) base. 2278 */ 2279 return (base + (value - last) / (this / nbuckets)); 2280 } 2281 2282 base += nbuckets - (nbuckets / factor); 2283 last = this; 2284 this = next; 2285 } 2286 2287 /* 2288 * Our value is greater than or equal to our factor taken to the 2289 * power of one plus the high magnitude -- return the top bucket. 2290 */ 2291 return (base); 2292 } 2293 2294 static void 2295 dtrace_aggregate_llquantize(uint64_t *llquanta, uint64_t nval, uint64_t incr) 2296 { 2297 uint64_t arg = *llquanta++; 2298 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg); 2299 uint16_t low = DTRACE_LLQUANTIZE_LOW(arg); 2300 uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg); 2301 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg); 2302 2303 llquanta[dtrace_aggregate_llquantize_bucket(factor, 2304 low, high, nsteps, nval)] += incr; 2305 } 2306 2307 /*ARGSUSED*/ 2308 static void 2309 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg) 2310 { 2311 data[0]++; 2312 data[1] += nval; 2313 } 2314 2315 /*ARGSUSED*/ 2316 static void 2317 dtrace_aggregate_stddev(uint64_t *data, uint64_t nval, uint64_t arg) 2318 { 2319 int64_t snval = (int64_t)nval; 2320 uint64_t tmp[2]; 2321 2322 data[0]++; 2323 data[1] += nval; 2324 2325 /* 2326 * What we want to say here is: 2327 * 2328 * data[2] += nval * nval; 2329 * 2330 * But given that nval is 64-bit, we could easily overflow, so 2331 * we do this as 128-bit arithmetic. 2332 */ 2333 if (snval < 0) 2334 snval = -snval; 2335 2336 dtrace_multiply_128((uint64_t)snval, (uint64_t)snval, tmp); 2337 dtrace_add_128(data + 2, tmp, data + 2); 2338 } 2339 2340 /*ARGSUSED*/ 2341 static void 2342 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg) 2343 { 2344 *oval = *oval + 1; 2345 } 2346 2347 /*ARGSUSED*/ 2348 static void 2349 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg) 2350 { 2351 *oval += nval; 2352 } 2353 2354 /* 2355 * Aggregate given the tuple in the principal data buffer, and the aggregating 2356 * action denoted by the specified dtrace_aggregation_t. The aggregation 2357 * buffer is specified as the buf parameter. This routine does not return 2358 * failure; if there is no space in the aggregation buffer, the data will be 2359 * dropped, and a corresponding counter incremented. 2360 */ 2361 static void 2362 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf, 2363 intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg) 2364 { 2365 dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec; 2366 uint32_t i, ndx, size, fsize; 2367 uint32_t align = sizeof (uint64_t) - 1; 2368 dtrace_aggbuffer_t *agb; 2369 dtrace_aggkey_t *key; 2370 uint32_t hashval = 0, limit, isstr; 2371 caddr_t tomax, data, kdata; 2372 dtrace_actkind_t action; 2373 dtrace_action_t *act; 2374 uintptr_t offs; 2375 2376 if (buf == NULL) 2377 return; 2378 2379 if (!agg->dtag_hasarg) { 2380 /* 2381 * Currently, only quantize() and lquantize() take additional 2382 * arguments, and they have the same semantics: an increment 2383 * value that defaults to 1 when not present. If additional 2384 * aggregating actions take arguments, the setting of the 2385 * default argument value will presumably have to become more 2386 * sophisticated... 2387 */ 2388 arg = 1; 2389 } 2390 2391 action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION; 2392 size = rec->dtrd_offset - agg->dtag_base; 2393 fsize = size + rec->dtrd_size; 2394 2395 ASSERT(dbuf->dtb_tomax != NULL); 2396 data = dbuf->dtb_tomax + offset + agg->dtag_base; 2397 2398 if ((tomax = buf->dtb_tomax) == NULL) { 2399 dtrace_buffer_drop(buf); 2400 return; 2401 } 2402 2403 /* 2404 * The metastructure is always at the bottom of the buffer. 2405 */ 2406 agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size - 2407 sizeof (dtrace_aggbuffer_t)); 2408 2409 if (buf->dtb_offset == 0) { 2410 /* 2411 * We just kludge up approximately 1/8th of the size to be 2412 * buckets. If this guess ends up being routinely 2413 * off-the-mark, we may need to dynamically readjust this 2414 * based on past performance. 2415 */ 2416 uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t); 2417 2418 if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) < 2419 (uintptr_t)tomax || hashsize == 0) { 2420 /* 2421 * We've been given a ludicrously small buffer; 2422 * increment our drop count and leave. 2423 */ 2424 dtrace_buffer_drop(buf); 2425 return; 2426 } 2427 2428 /* 2429 * And now, a pathetic attempt to try to get a an odd (or 2430 * perchance, a prime) hash size for better hash distribution. 2431 */ 2432 if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3)) 2433 hashsize -= DTRACE_AGGHASHSIZE_SLEW; 2434 2435 agb->dtagb_hashsize = hashsize; 2436 agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb - 2437 agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *)); 2438 agb->dtagb_free = (uintptr_t)agb->dtagb_hash; 2439 2440 for (i = 0; i < agb->dtagb_hashsize; i++) 2441 agb->dtagb_hash[i] = NULL; 2442 } 2443 2444 ASSERT(agg->dtag_first != NULL); 2445 ASSERT(agg->dtag_first->dta_intuple); 2446 2447 /* 2448 * Calculate the hash value based on the key. Note that we _don't_ 2449 * include the aggid in the hashing (but we will store it as part of 2450 * the key). The hashing algorithm is Bob Jenkins' "One-at-a-time" 2451 * algorithm: a simple, quick algorithm that has no known funnels, and 2452 * gets good distribution in practice. The efficacy of the hashing 2453 * algorithm (and a comparison with other algorithms) may be found by 2454 * running the ::dtrace_aggstat MDB dcmd. 2455 */ 2456 for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { 2457 i = act->dta_rec.dtrd_offset - agg->dtag_base; 2458 limit = i + act->dta_rec.dtrd_size; 2459 ASSERT(limit <= size); 2460 isstr = DTRACEACT_ISSTRING(act); 2461 2462 for (; i < limit; i++) { 2463 hashval += data[i]; 2464 hashval += (hashval << 10); 2465 hashval ^= (hashval >> 6); 2466 2467 if (isstr && data[i] == '\0') 2468 break; 2469 } 2470 } 2471 2472 hashval += (hashval << 3); 2473 hashval ^= (hashval >> 11); 2474 hashval += (hashval << 15); 2475 2476 /* 2477 * Yes, the divide here is expensive -- but it's generally the least 2478 * of the performance issues given the amount of data that we iterate 2479 * over to compute hash values, compare data, etc. 2480 */ 2481 ndx = hashval % agb->dtagb_hashsize; 2482 2483 for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) { 2484 ASSERT((caddr_t)key >= tomax); 2485 ASSERT((caddr_t)key < tomax + buf->dtb_size); 2486 2487 if (hashval != key->dtak_hashval || key->dtak_size != size) 2488 continue; 2489 2490 kdata = key->dtak_data; 2491 ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size); 2492 2493 for (act = agg->dtag_first; act->dta_intuple; 2494 act = act->dta_next) { 2495 i = act->dta_rec.dtrd_offset - agg->dtag_base; 2496 limit = i + act->dta_rec.dtrd_size; 2497 ASSERT(limit <= size); 2498 isstr = DTRACEACT_ISSTRING(act); 2499 2500 for (; i < limit; i++) { 2501 if (kdata[i] != data[i]) 2502 goto next; 2503 2504 if (isstr && data[i] == '\0') 2505 break; 2506 } 2507 } 2508 2509 if (action != key->dtak_action) { 2510 /* 2511 * We are aggregating on the same value in the same 2512 * aggregation with two different aggregating actions. 2513 * (This should have been picked up in the compiler, 2514 * so we may be dealing with errant or devious DIF.) 2515 * This is an error condition; we indicate as much, 2516 * and return. 2517 */ 2518 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 2519 return; 2520 } 2521 2522 /* 2523 * This is a hit: we need to apply the aggregator to 2524 * the value at this key. 2525 */ 2526 agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg); 2527 return; 2528 next: 2529 continue; 2530 } 2531 2532 /* 2533 * We didn't find it. We need to allocate some zero-filled space, 2534 * link it into the hash table appropriately, and apply the aggregator 2535 * to the (zero-filled) value. 2536 */ 2537 offs = buf->dtb_offset; 2538 while (offs & (align - 1)) 2539 offs += sizeof (uint32_t); 2540 2541 /* 2542 * If we don't have enough room to both allocate a new key _and_ 2543 * its associated data, increment the drop count and return. 2544 */ 2545 if ((uintptr_t)tomax + offs + fsize > 2546 agb->dtagb_free - sizeof (dtrace_aggkey_t)) { 2547 dtrace_buffer_drop(buf); 2548 return; 2549 } 2550 2551 /*CONSTCOND*/ 2552 ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1))); 2553 key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t)); 2554 agb->dtagb_free -= sizeof (dtrace_aggkey_t); 2555 2556 key->dtak_data = kdata = tomax + offs; 2557 buf->dtb_offset = offs + fsize; 2558 2559 /* 2560 * Now copy the data across. 2561 */ 2562 *((dtrace_aggid_t *)kdata) = agg->dtag_id; 2563 2564 for (i = sizeof (dtrace_aggid_t); i < size; i++) 2565 kdata[i] = data[i]; 2566 2567 /* 2568 * Because strings are not zeroed out by default, we need to iterate 2569 * looking for actions that store strings, and we need to explicitly 2570 * pad these strings out with zeroes. 2571 */ 2572 for (act = agg->dtag_first; act->dta_intuple; act = act->dta_next) { 2573 int nul; 2574 2575 if (!DTRACEACT_ISSTRING(act)) 2576 continue; 2577 2578 i = act->dta_rec.dtrd_offset - agg->dtag_base; 2579 limit = i + act->dta_rec.dtrd_size; 2580 ASSERT(limit <= size); 2581 2582 for (nul = 0; i < limit; i++) { 2583 if (nul) { 2584 kdata[i] = '\0'; 2585 continue; 2586 } 2587 2588 if (data[i] != '\0') 2589 continue; 2590 2591 nul = 1; 2592 } 2593 } 2594 2595 for (i = size; i < fsize; i++) 2596 kdata[i] = 0; 2597 2598 key->dtak_hashval = hashval; 2599 key->dtak_size = size; 2600 key->dtak_action = action; 2601 key->dtak_next = agb->dtagb_hash[ndx]; 2602 agb->dtagb_hash[ndx] = key; 2603 2604 /* 2605 * Finally, apply the aggregator. 2606 */ 2607 *((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial; 2608 agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg); 2609 } 2610 2611 /* 2612 * Given consumer state, this routine finds a speculation in the INACTIVE 2613 * state and transitions it into the ACTIVE state. If there is no speculation 2614 * in the INACTIVE state, 0 is returned. In this case, no error counter is 2615 * incremented -- it is up to the caller to take appropriate action. 2616 */ 2617 static int 2618 dtrace_speculation(dtrace_state_t *state) 2619 { 2620 int i = 0; 2621 dtrace_speculation_state_t current; 2622 uint32_t *stat = &state->dts_speculations_unavail, count; 2623 2624 while (i < state->dts_nspeculations) { 2625 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2626 2627 current = spec->dtsp_state; 2628 2629 if (current != DTRACESPEC_INACTIVE) { 2630 if (current == DTRACESPEC_COMMITTINGMANY || 2631 current == DTRACESPEC_COMMITTING || 2632 current == DTRACESPEC_DISCARDING) 2633 stat = &state->dts_speculations_busy; 2634 i++; 2635 continue; 2636 } 2637 2638 if (dtrace_cas32((uint32_t *)&spec->dtsp_state, 2639 current, DTRACESPEC_ACTIVE) == current) 2640 return (i + 1); 2641 } 2642 2643 /* 2644 * We couldn't find a speculation. If we found as much as a single 2645 * busy speculation buffer, we'll attribute this failure as "busy" 2646 * instead of "unavail". 2647 */ 2648 do { 2649 count = *stat; 2650 } while (dtrace_cas32(stat, count, count + 1) != count); 2651 2652 return (0); 2653 } 2654 2655 /* 2656 * This routine commits an active speculation. If the specified speculation 2657 * is not in a valid state to perform a commit(), this routine will silently do 2658 * nothing. The state of the specified speculation is transitioned according 2659 * to the state transition diagram outlined in <sys/dtrace_impl.h> 2660 */ 2661 static void 2662 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu, 2663 dtrace_specid_t which) 2664 { 2665 dtrace_speculation_t *spec; 2666 dtrace_buffer_t *src, *dest; 2667 uintptr_t daddr, saddr, dlimit, slimit; 2668 dtrace_speculation_state_t current, new = 0; 2669 intptr_t offs; 2670 uint64_t timestamp; 2671 2672 if (which == 0) 2673 return; 2674 2675 if (which > state->dts_nspeculations) { 2676 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 2677 return; 2678 } 2679 2680 spec = &state->dts_speculations[which - 1]; 2681 src = &spec->dtsp_buffer[cpu]; 2682 dest = &state->dts_buffer[cpu]; 2683 2684 do { 2685 current = spec->dtsp_state; 2686 2687 if (current == DTRACESPEC_COMMITTINGMANY) 2688 break; 2689 2690 switch (current) { 2691 case DTRACESPEC_INACTIVE: 2692 case DTRACESPEC_DISCARDING: 2693 return; 2694 2695 case DTRACESPEC_COMMITTING: 2696 /* 2697 * This is only possible if we are (a) commit()'ing 2698 * without having done a prior speculate() on this CPU 2699 * and (b) racing with another commit() on a different 2700 * CPU. There's nothing to do -- we just assert that 2701 * our offset is 0. 2702 */ 2703 ASSERT(src->dtb_offset == 0); 2704 return; 2705 2706 case DTRACESPEC_ACTIVE: 2707 new = DTRACESPEC_COMMITTING; 2708 break; 2709 2710 case DTRACESPEC_ACTIVEONE: 2711 /* 2712 * This speculation is active on one CPU. If our 2713 * buffer offset is non-zero, we know that the one CPU 2714 * must be us. Otherwise, we are committing on a 2715 * different CPU from the speculate(), and we must 2716 * rely on being asynchronously cleaned. 2717 */ 2718 if (src->dtb_offset != 0) { 2719 new = DTRACESPEC_COMMITTING; 2720 break; 2721 } 2722 /*FALLTHROUGH*/ 2723 2724 case DTRACESPEC_ACTIVEMANY: 2725 new = DTRACESPEC_COMMITTINGMANY; 2726 break; 2727 2728 default: 2729 ASSERT(0); 2730 } 2731 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, 2732 current, new) != current); 2733 2734 /* 2735 * We have set the state to indicate that we are committing this 2736 * speculation. Now reserve the necessary space in the destination 2737 * buffer. 2738 */ 2739 if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset, 2740 sizeof (uint64_t), state, NULL)) < 0) { 2741 dtrace_buffer_drop(dest); 2742 goto out; 2743 } 2744 2745 /* 2746 * We have sufficient space to copy the speculative buffer into the 2747 * primary buffer. First, modify the speculative buffer, filling 2748 * in the timestamp of all entries with the current time. The data 2749 * must have the commit() time rather than the time it was traced, 2750 * so that all entries in the primary buffer are in timestamp order. 2751 */ 2752 timestamp = dtrace_gethrtime(); 2753 saddr = (uintptr_t)src->dtb_tomax; 2754 slimit = saddr + src->dtb_offset; 2755 while (saddr < slimit) { 2756 size_t size; 2757 dtrace_rechdr_t *dtrh = (dtrace_rechdr_t *)saddr; 2758 2759 if (dtrh->dtrh_epid == DTRACE_EPIDNONE) { 2760 saddr += sizeof (dtrace_epid_t); 2761 continue; 2762 } 2763 ASSERT3U(dtrh->dtrh_epid, <=, state->dts_necbs); 2764 size = state->dts_ecbs[dtrh->dtrh_epid - 1]->dte_size; 2765 2766 ASSERT3U(saddr + size, <=, slimit); 2767 ASSERT3U(size, >=, sizeof (dtrace_rechdr_t)); 2768 ASSERT3U(DTRACE_RECORD_LOAD_TIMESTAMP(dtrh), ==, UINT64_MAX); 2769 2770 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, timestamp); 2771 2772 saddr += size; 2773 } 2774 2775 /* 2776 * Copy the buffer across. (Note that this is a 2777 * highly subobtimal bcopy(); in the unlikely event that this becomes 2778 * a serious performance issue, a high-performance DTrace-specific 2779 * bcopy() should obviously be invented.) 2780 */ 2781 daddr = (uintptr_t)dest->dtb_tomax + offs; 2782 dlimit = daddr + src->dtb_offset; 2783 saddr = (uintptr_t)src->dtb_tomax; 2784 2785 /* 2786 * First, the aligned portion. 2787 */ 2788 while (dlimit - daddr >= sizeof (uint64_t)) { 2789 *((uint64_t *)daddr) = *((uint64_t *)saddr); 2790 2791 daddr += sizeof (uint64_t); 2792 saddr += sizeof (uint64_t); 2793 } 2794 2795 /* 2796 * Now any left-over bit... 2797 */ 2798 while (dlimit - daddr) 2799 *((uint8_t *)daddr++) = *((uint8_t *)saddr++); 2800 2801 /* 2802 * Finally, commit the reserved space in the destination buffer. 2803 */ 2804 dest->dtb_offset = offs + src->dtb_offset; 2805 2806 out: 2807 /* 2808 * If we're lucky enough to be the only active CPU on this speculation 2809 * buffer, we can just set the state back to DTRACESPEC_INACTIVE. 2810 */ 2811 if (current == DTRACESPEC_ACTIVE || 2812 (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) { 2813 uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state, 2814 DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE); 2815 2816 ASSERT(rval == DTRACESPEC_COMMITTING); 2817 } 2818 2819 src->dtb_offset = 0; 2820 src->dtb_xamot_drops += src->dtb_drops; 2821 src->dtb_drops = 0; 2822 } 2823 2824 /* 2825 * This routine discards an active speculation. If the specified speculation 2826 * is not in a valid state to perform a discard(), this routine will silently 2827 * do nothing. The state of the specified speculation is transitioned 2828 * according to the state transition diagram outlined in <sys/dtrace_impl.h> 2829 */ 2830 static void 2831 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu, 2832 dtrace_specid_t which) 2833 { 2834 dtrace_speculation_t *spec; 2835 dtrace_speculation_state_t current, new = 0; 2836 dtrace_buffer_t *buf; 2837 2838 if (which == 0) 2839 return; 2840 2841 if (which > state->dts_nspeculations) { 2842 cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 2843 return; 2844 } 2845 2846 spec = &state->dts_speculations[which - 1]; 2847 buf = &spec->dtsp_buffer[cpu]; 2848 2849 do { 2850 current = spec->dtsp_state; 2851 2852 switch (current) { 2853 case DTRACESPEC_INACTIVE: 2854 case DTRACESPEC_COMMITTINGMANY: 2855 case DTRACESPEC_COMMITTING: 2856 case DTRACESPEC_DISCARDING: 2857 return; 2858 2859 case DTRACESPEC_ACTIVE: 2860 case DTRACESPEC_ACTIVEMANY: 2861 new = DTRACESPEC_DISCARDING; 2862 break; 2863 2864 case DTRACESPEC_ACTIVEONE: 2865 if (buf->dtb_offset != 0) { 2866 new = DTRACESPEC_INACTIVE; 2867 } else { 2868 new = DTRACESPEC_DISCARDING; 2869 } 2870 break; 2871 2872 default: 2873 ASSERT(0); 2874 } 2875 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, 2876 current, new) != current); 2877 2878 buf->dtb_offset = 0; 2879 buf->dtb_drops = 0; 2880 } 2881 2882 /* 2883 * Note: not called from probe context. This function is called 2884 * asynchronously from cross call context to clean any speculations that are 2885 * in the COMMITTINGMANY or DISCARDING states. These speculations may not be 2886 * transitioned back to the INACTIVE state until all CPUs have cleaned the 2887 * speculation. 2888 */ 2889 static void 2890 dtrace_speculation_clean_here(dtrace_state_t *state) 2891 { 2892 dtrace_icookie_t cookie; 2893 processorid_t cpu = curcpu; 2894 dtrace_buffer_t *dest = &state->dts_buffer[cpu]; 2895 dtrace_specid_t i; 2896 2897 cookie = dtrace_interrupt_disable(); 2898 2899 if (dest->dtb_tomax == NULL) { 2900 dtrace_interrupt_enable(cookie); 2901 return; 2902 } 2903 2904 for (i = 0; i < state->dts_nspeculations; i++) { 2905 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2906 dtrace_buffer_t *src = &spec->dtsp_buffer[cpu]; 2907 2908 if (src->dtb_tomax == NULL) 2909 continue; 2910 2911 if (spec->dtsp_state == DTRACESPEC_DISCARDING) { 2912 src->dtb_offset = 0; 2913 continue; 2914 } 2915 2916 if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY) 2917 continue; 2918 2919 if (src->dtb_offset == 0) 2920 continue; 2921 2922 dtrace_speculation_commit(state, cpu, i + 1); 2923 } 2924 2925 dtrace_interrupt_enable(cookie); 2926 } 2927 2928 /* 2929 * Note: not called from probe context. This function is called 2930 * asynchronously (and at a regular interval) to clean any speculations that 2931 * are in the COMMITTINGMANY or DISCARDING states. If it discovers that there 2932 * is work to be done, it cross calls all CPUs to perform that work; 2933 * COMMITMANY and DISCARDING speculations may not be transitioned back to the 2934 * INACTIVE state until they have been cleaned by all CPUs. 2935 */ 2936 static void 2937 dtrace_speculation_clean(dtrace_state_t *state) 2938 { 2939 int work = 0, rv; 2940 dtrace_specid_t i; 2941 2942 for (i = 0; i < state->dts_nspeculations; i++) { 2943 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2944 2945 ASSERT(!spec->dtsp_cleaning); 2946 2947 if (spec->dtsp_state != DTRACESPEC_DISCARDING && 2948 spec->dtsp_state != DTRACESPEC_COMMITTINGMANY) 2949 continue; 2950 2951 work++; 2952 spec->dtsp_cleaning = 1; 2953 } 2954 2955 if (!work) 2956 return; 2957 2958 dtrace_xcall(DTRACE_CPUALL, 2959 (dtrace_xcall_t)dtrace_speculation_clean_here, state); 2960 2961 /* 2962 * We now know that all CPUs have committed or discarded their 2963 * speculation buffers, as appropriate. We can now set the state 2964 * to inactive. 2965 */ 2966 for (i = 0; i < state->dts_nspeculations; i++) { 2967 dtrace_speculation_t *spec = &state->dts_speculations[i]; 2968 dtrace_speculation_state_t current, new; 2969 2970 if (!spec->dtsp_cleaning) 2971 continue; 2972 2973 current = spec->dtsp_state; 2974 ASSERT(current == DTRACESPEC_DISCARDING || 2975 current == DTRACESPEC_COMMITTINGMANY); 2976 2977 new = DTRACESPEC_INACTIVE; 2978 2979 rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new); 2980 ASSERT(rv == current); 2981 spec->dtsp_cleaning = 0; 2982 } 2983 } 2984 2985 /* 2986 * Called as part of a speculate() to get the speculative buffer associated 2987 * with a given speculation. Returns NULL if the specified speculation is not 2988 * in an ACTIVE state. If the speculation is in the ACTIVEONE state -- and 2989 * the active CPU is not the specified CPU -- the speculation will be 2990 * atomically transitioned into the ACTIVEMANY state. 2991 */ 2992 static dtrace_buffer_t * 2993 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid, 2994 dtrace_specid_t which) 2995 { 2996 dtrace_speculation_t *spec; 2997 dtrace_speculation_state_t current, new = 0; 2998 dtrace_buffer_t *buf; 2999 3000 if (which == 0) 3001 return (NULL); 3002 3003 if (which > state->dts_nspeculations) { 3004 cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP; 3005 return (NULL); 3006 } 3007 3008 spec = &state->dts_speculations[which - 1]; 3009 buf = &spec->dtsp_buffer[cpuid]; 3010 3011 do { 3012 current = spec->dtsp_state; 3013 3014 switch (current) { 3015 case DTRACESPEC_INACTIVE: 3016 case DTRACESPEC_COMMITTINGMANY: 3017 case DTRACESPEC_DISCARDING: 3018 return (NULL); 3019 3020 case DTRACESPEC_COMMITTING: 3021 ASSERT(buf->dtb_offset == 0); 3022 return (NULL); 3023 3024 case DTRACESPEC_ACTIVEONE: 3025 /* 3026 * This speculation is currently active on one CPU. 3027 * Check the offset in the buffer; if it's non-zero, 3028 * that CPU must be us (and we leave the state alone). 3029 * If it's zero, assume that we're starting on a new 3030 * CPU -- and change the state to indicate that the 3031 * speculation is active on more than one CPU. 3032 */ 3033 if (buf->dtb_offset != 0) 3034 return (buf); 3035 3036 new = DTRACESPEC_ACTIVEMANY; 3037 break; 3038 3039 case DTRACESPEC_ACTIVEMANY: 3040 return (buf); 3041 3042 case DTRACESPEC_ACTIVE: 3043 new = DTRACESPEC_ACTIVEONE; 3044 break; 3045 3046 default: 3047 ASSERT(0); 3048 } 3049 } while (dtrace_cas32((uint32_t *)&spec->dtsp_state, 3050 current, new) != current); 3051 3052 ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY); 3053 return (buf); 3054 } 3055 3056 /* 3057 * Return a string. In the event that the user lacks the privilege to access 3058 * arbitrary kernel memory, we copy the string out to scratch memory so that we 3059 * don't fail access checking. 3060 * 3061 * dtrace_dif_variable() uses this routine as a helper for various 3062 * builtin values such as 'execname' and 'probefunc.' 3063 */ 3064 uintptr_t 3065 dtrace_dif_varstr(uintptr_t addr, dtrace_state_t *state, 3066 dtrace_mstate_t *mstate) 3067 { 3068 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 3069 uintptr_t ret; 3070 size_t strsz; 3071 3072 /* 3073 * The easy case: this probe is allowed to read all of memory, so 3074 * we can just return this as a vanilla pointer. 3075 */ 3076 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) != 0) 3077 return (addr); 3078 3079 /* 3080 * This is the tougher case: we copy the string in question from 3081 * kernel memory into scratch memory and return it that way: this 3082 * ensures that we won't trip up when access checking tests the 3083 * BYREF return value. 3084 */ 3085 strsz = dtrace_strlen((char *)addr, size) + 1; 3086 3087 if (mstate->dtms_scratch_ptr + strsz > 3088 mstate->dtms_scratch_base + mstate->dtms_scratch_size) { 3089 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 3090 return (0); 3091 } 3092 3093 dtrace_strcpy((const void *)addr, (void *)mstate->dtms_scratch_ptr, 3094 strsz); 3095 ret = mstate->dtms_scratch_ptr; 3096 mstate->dtms_scratch_ptr += strsz; 3097 return (ret); 3098 } 3099 3100 /* 3101 * Return a string from a memoy address which is known to have one or 3102 * more concatenated, individually zero terminated, sub-strings. 3103 * In the event that the user lacks the privilege to access 3104 * arbitrary kernel memory, we copy the string out to scratch memory so that we 3105 * don't fail access checking. 3106 * 3107 * dtrace_dif_variable() uses this routine as a helper for various 3108 * builtin values such as 'execargs'. 3109 */ 3110 static uintptr_t 3111 dtrace_dif_varstrz(uintptr_t addr, size_t strsz, dtrace_state_t *state, 3112 dtrace_mstate_t *mstate) 3113 { 3114 char *p; 3115 size_t i; 3116 uintptr_t ret; 3117 3118 if (mstate->dtms_scratch_ptr + strsz > 3119 mstate->dtms_scratch_base + mstate->dtms_scratch_size) { 3120 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 3121 return (0); 3122 } 3123 3124 dtrace_bcopy((const void *)addr, (void *)mstate->dtms_scratch_ptr, 3125 strsz); 3126 3127 /* Replace sub-string termination characters with a space. */ 3128 for (p = (char *) mstate->dtms_scratch_ptr, i = 0; i < strsz - 1; 3129 p++, i++) 3130 if (*p == '\0') 3131 *p = ' '; 3132 3133 ret = mstate->dtms_scratch_ptr; 3134 mstate->dtms_scratch_ptr += strsz; 3135 return (ret); 3136 } 3137 3138 /* 3139 * This function implements the DIF emulator's variable lookups. The emulator 3140 * passes a reserved variable identifier and optional built-in array index. 3141 */ 3142 static uint64_t 3143 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v, 3144 uint64_t ndx) 3145 { 3146 /* 3147 * If we're accessing one of the uncached arguments, we'll turn this 3148 * into a reference in the args array. 3149 */ 3150 if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) { 3151 ndx = v - DIF_VAR_ARG0; 3152 v = DIF_VAR_ARGS; 3153 } 3154 3155 switch (v) { 3156 case DIF_VAR_ARGS: 3157 ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS); 3158 if (ndx >= sizeof (mstate->dtms_arg) / 3159 sizeof (mstate->dtms_arg[0])) { 3160 int aframes = mstate->dtms_probe->dtpr_aframes + 2; 3161 dtrace_provider_t *pv; 3162 uint64_t val; 3163 3164 pv = mstate->dtms_probe->dtpr_provider; 3165 if (pv->dtpv_pops.dtps_getargval != NULL) 3166 val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg, 3167 mstate->dtms_probe->dtpr_id, 3168 mstate->dtms_probe->dtpr_arg, ndx, aframes); 3169 else 3170 val = dtrace_getarg(ndx, aframes); 3171 3172 /* 3173 * This is regrettably required to keep the compiler 3174 * from tail-optimizing the call to dtrace_getarg(). 3175 * The condition always evaluates to true, but the 3176 * compiler has no way of figuring that out a priori. 3177 * (None of this would be necessary if the compiler 3178 * could be relied upon to _always_ tail-optimize 3179 * the call to dtrace_getarg() -- but it can't.) 3180 */ 3181 if (mstate->dtms_probe != NULL) 3182 return (val); 3183 3184 ASSERT(0); 3185 } 3186 3187 return (mstate->dtms_arg[ndx]); 3188 3189 #ifdef illumos 3190 case DIF_VAR_UREGS: { 3191 klwp_t *lwp; 3192 3193 if (!dtrace_priv_proc(state)) 3194 return (0); 3195 3196 if ((lwp = curthread->t_lwp) == NULL) { 3197 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 3198 cpu_core[curcpu].cpuc_dtrace_illval = NULL; 3199 return (0); 3200 } 3201 3202 return (dtrace_getreg(lwp->lwp_regs, ndx)); 3203 return (0); 3204 } 3205 #else 3206 case DIF_VAR_UREGS: { 3207 struct trapframe *tframe; 3208 3209 if (!dtrace_priv_proc(state)) 3210 return (0); 3211 3212 if ((tframe = curthread->td_frame) == NULL) { 3213 DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR); 3214 cpu_core[curcpu].cpuc_dtrace_illval = 0; 3215 return (0); 3216 } 3217 3218 return (dtrace_getreg(tframe, ndx)); 3219 } 3220 #endif 3221 3222 case DIF_VAR_CURTHREAD: 3223 if (!dtrace_priv_proc(state)) 3224 return (0); 3225 return ((uint64_t)(uintptr_t)curthread); 3226 3227 case DIF_VAR_TIMESTAMP: 3228 if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) { 3229 mstate->dtms_timestamp = dtrace_gethrtime(); 3230 mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP; 3231 } 3232 return (mstate->dtms_timestamp); 3233 3234 case DIF_VAR_VTIMESTAMP: 3235 ASSERT(dtrace_vtime_references != 0); 3236 return (curthread->t_dtrace_vtime); 3237 3238 case DIF_VAR_WALLTIMESTAMP: 3239 if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) { 3240 mstate->dtms_walltimestamp = dtrace_gethrestime(); 3241 mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP; 3242 } 3243 return (mstate->dtms_walltimestamp); 3244 3245 #ifdef illumos 3246 case DIF_VAR_IPL: 3247 if (!dtrace_priv_kernel(state)) 3248 return (0); 3249 if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) { 3250 mstate->dtms_ipl = dtrace_getipl(); 3251 mstate->dtms_present |= DTRACE_MSTATE_IPL; 3252 } 3253 return (mstate->dtms_ipl); 3254 #endif 3255 3256 case DIF_VAR_EPID: 3257 ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID); 3258 return (mstate->dtms_epid); 3259 3260 case DIF_VAR_ID: 3261 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3262 return (mstate->dtms_probe->dtpr_id); 3263 3264 case DIF_VAR_STACKDEPTH: 3265 if (!dtrace_priv_kernel(state)) 3266 return (0); 3267 if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) { 3268 int aframes = mstate->dtms_probe->dtpr_aframes + 2; 3269 3270 mstate->dtms_stackdepth = dtrace_getstackdepth(aframes); 3271 mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH; 3272 } 3273 return (mstate->dtms_stackdepth); 3274 3275 case DIF_VAR_USTACKDEPTH: 3276 if (!dtrace_priv_proc(state)) 3277 return (0); 3278 if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) { 3279 /* 3280 * See comment in DIF_VAR_PID. 3281 */ 3282 if (DTRACE_ANCHORED(mstate->dtms_probe) && 3283 CPU_ON_INTR(CPU)) { 3284 mstate->dtms_ustackdepth = 0; 3285 } else { 3286 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 3287 mstate->dtms_ustackdepth = 3288 dtrace_getustackdepth(); 3289 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 3290 } 3291 mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH; 3292 } 3293 return (mstate->dtms_ustackdepth); 3294 3295 case DIF_VAR_CALLER: 3296 if (!dtrace_priv_kernel(state)) 3297 return (0); 3298 if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) { 3299 int aframes = mstate->dtms_probe->dtpr_aframes + 2; 3300 3301 if (!DTRACE_ANCHORED(mstate->dtms_probe)) { 3302 /* 3303 * If this is an unanchored probe, we are 3304 * required to go through the slow path: 3305 * dtrace_caller() only guarantees correct 3306 * results for anchored probes. 3307 */ 3308 pc_t caller[2] = {0, 0}; 3309 3310 dtrace_getpcstack(caller, 2, aframes, 3311 (uint32_t *)(uintptr_t)mstate->dtms_arg[0]); 3312 mstate->dtms_caller = caller[1]; 3313 } else if ((mstate->dtms_caller = 3314 dtrace_caller(aframes)) == -1) { 3315 /* 3316 * We have failed to do this the quick way; 3317 * we must resort to the slower approach of 3318 * calling dtrace_getpcstack(). 3319 */ 3320 pc_t caller = 0; 3321 3322 dtrace_getpcstack(&caller, 1, aframes, NULL); 3323 mstate->dtms_caller = caller; 3324 } 3325 3326 mstate->dtms_present |= DTRACE_MSTATE_CALLER; 3327 } 3328 return (mstate->dtms_caller); 3329 3330 case DIF_VAR_UCALLER: 3331 if (!dtrace_priv_proc(state)) 3332 return (0); 3333 3334 if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) { 3335 uint64_t ustack[3]; 3336 3337 /* 3338 * dtrace_getupcstack() fills in the first uint64_t 3339 * with the current PID. The second uint64_t will 3340 * be the program counter at user-level. The third 3341 * uint64_t will contain the caller, which is what 3342 * we're after. 3343 */ 3344 ustack[2] = 0; 3345 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 3346 dtrace_getupcstack(ustack, 3); 3347 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 3348 mstate->dtms_ucaller = ustack[2]; 3349 mstate->dtms_present |= DTRACE_MSTATE_UCALLER; 3350 } 3351 3352 return (mstate->dtms_ucaller); 3353 3354 case DIF_VAR_PROBEPROV: 3355 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3356 return (dtrace_dif_varstr( 3357 (uintptr_t)mstate->dtms_probe->dtpr_provider->dtpv_name, 3358 state, mstate)); 3359 3360 case DIF_VAR_PROBEMOD: 3361 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3362 return (dtrace_dif_varstr( 3363 (uintptr_t)mstate->dtms_probe->dtpr_mod, 3364 state, mstate)); 3365 3366 case DIF_VAR_PROBEFUNC: 3367 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3368 return (dtrace_dif_varstr( 3369 (uintptr_t)mstate->dtms_probe->dtpr_func, 3370 state, mstate)); 3371 3372 case DIF_VAR_PROBENAME: 3373 ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE); 3374 return (dtrace_dif_varstr( 3375 (uintptr_t)mstate->dtms_probe->dtpr_name, 3376 state, mstate)); 3377 3378 case DIF_VAR_PID: 3379 if (!dtrace_priv_proc(state)) 3380 return (0); 3381 3382 #ifdef illumos 3383 /* 3384 * Note that we are assuming that an unanchored probe is 3385 * always due to a high-level interrupt. (And we're assuming 3386 * that there is only a single high level interrupt.) 3387 */ 3388 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3389 return (pid0.pid_id); 3390 3391 /* 3392 * It is always safe to dereference one's own t_procp pointer: 3393 * it always points to a valid, allocated proc structure. 3394 * Further, it is always safe to dereference the p_pidp member 3395 * of one's own proc structure. (These are truisms becuase 3396 * threads and processes don't clean up their own state -- 3397 * they leave that task to whomever reaps them.) 3398 */ 3399 return ((uint64_t)curthread->t_procp->p_pidp->pid_id); 3400 #else 3401 return ((uint64_t)curproc->p_pid); 3402 #endif 3403 3404 case DIF_VAR_PPID: 3405 if (!dtrace_priv_proc(state)) 3406 return (0); 3407 3408 #ifdef illumos 3409 /* 3410 * See comment in DIF_VAR_PID. 3411 */ 3412 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3413 return (pid0.pid_id); 3414 3415 /* 3416 * It is always safe to dereference one's own t_procp pointer: 3417 * it always points to a valid, allocated proc structure. 3418 * (This is true because threads don't clean up their own 3419 * state -- they leave that task to whomever reaps them.) 3420 */ 3421 return ((uint64_t)curthread->t_procp->p_ppid); 3422 #else 3423 if (curproc->p_pid == proc0.p_pid) 3424 return (curproc->p_pid); 3425 else 3426 return (curproc->p_pptr->p_pid); 3427 #endif 3428 3429 case DIF_VAR_TID: 3430 #ifdef illumos 3431 /* 3432 * See comment in DIF_VAR_PID. 3433 */ 3434 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3435 return (0); 3436 #endif 3437 3438 return ((uint64_t)curthread->t_tid); 3439 3440 case DIF_VAR_EXECARGS: { 3441 struct pargs *p_args = curthread->td_proc->p_args; 3442 3443 if (p_args == NULL) 3444 return(0); 3445 3446 return (dtrace_dif_varstrz( 3447 (uintptr_t) p_args->ar_args, p_args->ar_length, state, mstate)); 3448 } 3449 3450 case DIF_VAR_EXECNAME: 3451 #ifdef illumos 3452 if (!dtrace_priv_proc(state)) 3453 return (0); 3454 3455 /* 3456 * See comment in DIF_VAR_PID. 3457 */ 3458 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3459 return ((uint64_t)(uintptr_t)p0.p_user.u_comm); 3460 3461 /* 3462 * It is always safe to dereference one's own t_procp pointer: 3463 * it always points to a valid, allocated proc structure. 3464 * (This is true because threads don't clean up their own 3465 * state -- they leave that task to whomever reaps them.) 3466 */ 3467 return (dtrace_dif_varstr( 3468 (uintptr_t)curthread->t_procp->p_user.u_comm, 3469 state, mstate)); 3470 #else 3471 return (dtrace_dif_varstr( 3472 (uintptr_t) curthread->td_proc->p_comm, state, mstate)); 3473 #endif 3474 3475 case DIF_VAR_ZONENAME: 3476 #ifdef illumos 3477 if (!dtrace_priv_proc(state)) 3478 return (0); 3479 3480 /* 3481 * See comment in DIF_VAR_PID. 3482 */ 3483 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3484 return ((uint64_t)(uintptr_t)p0.p_zone->zone_name); 3485 3486 /* 3487 * It is always safe to dereference one's own t_procp pointer: 3488 * it always points to a valid, allocated proc structure. 3489 * (This is true because threads don't clean up their own 3490 * state -- they leave that task to whomever reaps them.) 3491 */ 3492 return (dtrace_dif_varstr( 3493 (uintptr_t)curthread->t_procp->p_zone->zone_name, 3494 state, mstate)); 3495 #else 3496 return (0); 3497 #endif 3498 3499 case DIF_VAR_UID: 3500 if (!dtrace_priv_proc(state)) 3501 return (0); 3502 3503 #ifdef illumos 3504 /* 3505 * See comment in DIF_VAR_PID. 3506 */ 3507 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3508 return ((uint64_t)p0.p_cred->cr_uid); 3509 #endif 3510 3511 /* 3512 * It is always safe to dereference one's own t_procp pointer: 3513 * it always points to a valid, allocated proc structure. 3514 * (This is true because threads don't clean up their own 3515 * state -- they leave that task to whomever reaps them.) 3516 * 3517 * Additionally, it is safe to dereference one's own process 3518 * credential, since this is never NULL after process birth. 3519 */ 3520 return ((uint64_t)curthread->t_procp->p_cred->cr_uid); 3521 3522 case DIF_VAR_GID: 3523 if (!dtrace_priv_proc(state)) 3524 return (0); 3525 3526 #ifdef illumos 3527 /* 3528 * See comment in DIF_VAR_PID. 3529 */ 3530 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3531 return ((uint64_t)p0.p_cred->cr_gid); 3532 #endif 3533 3534 /* 3535 * It is always safe to dereference one's own t_procp pointer: 3536 * it always points to a valid, allocated proc structure. 3537 * (This is true because threads don't clean up their own 3538 * state -- they leave that task to whomever reaps them.) 3539 * 3540 * Additionally, it is safe to dereference one's own process 3541 * credential, since this is never NULL after process birth. 3542 */ 3543 return ((uint64_t)curthread->t_procp->p_cred->cr_gid); 3544 3545 case DIF_VAR_ERRNO: { 3546 #ifdef illumos 3547 klwp_t *lwp; 3548 if (!dtrace_priv_proc(state)) 3549 return (0); 3550 3551 /* 3552 * See comment in DIF_VAR_PID. 3553 */ 3554 if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU)) 3555 return (0); 3556 3557 /* 3558 * It is always safe to dereference one's own t_lwp pointer in 3559 * the event that this pointer is non-NULL. (This is true 3560 * because threads and lwps don't clean up their own state -- 3561 * they leave that task to whomever reaps them.) 3562 */ 3563 if ((lwp = curthread->t_lwp) == NULL) 3564 return (0); 3565 3566 return ((uint64_t)lwp->lwp_errno); 3567 #else 3568 return (curthread->td_errno); 3569 #endif 3570 } 3571 #ifndef illumos 3572 case DIF_VAR_CPU: { 3573 return curcpu; 3574 } 3575 #endif 3576 default: 3577 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 3578 return (0); 3579 } 3580 } 3581 3582 3583 typedef enum dtrace_json_state { 3584 DTRACE_JSON_REST = 1, 3585 DTRACE_JSON_OBJECT, 3586 DTRACE_JSON_STRING, 3587 DTRACE_JSON_STRING_ESCAPE, 3588 DTRACE_JSON_STRING_ESCAPE_UNICODE, 3589 DTRACE_JSON_COLON, 3590 DTRACE_JSON_COMMA, 3591 DTRACE_JSON_VALUE, 3592 DTRACE_JSON_IDENTIFIER, 3593 DTRACE_JSON_NUMBER, 3594 DTRACE_JSON_NUMBER_FRAC, 3595 DTRACE_JSON_NUMBER_EXP, 3596 DTRACE_JSON_COLLECT_OBJECT 3597 } dtrace_json_state_t; 3598 3599 /* 3600 * This function possesses just enough knowledge about JSON to extract a single 3601 * value from a JSON string and store it in the scratch buffer. It is able 3602 * to extract nested object values, and members of arrays by index. 3603 * 3604 * elemlist is a list of JSON keys, stored as packed NUL-terminated strings, to 3605 * be looked up as we descend into the object tree. e.g. 3606 * 3607 * foo[0].bar.baz[32] --> "foo" NUL "0" NUL "bar" NUL "baz" NUL "32" NUL 3608 * with nelems = 5. 3609 * 3610 * The run time of this function must be bounded above by strsize to limit the 3611 * amount of work done in probe context. As such, it is implemented as a 3612 * simple state machine, reading one character at a time using safe loads 3613 * until we find the requested element, hit a parsing error or run off the 3614 * end of the object or string. 3615 * 3616 * As there is no way for a subroutine to return an error without interrupting 3617 * clause execution, we simply return NULL in the event of a missing key or any 3618 * other error condition. Each NULL return in this function is commented with 3619 * the error condition it represents -- parsing or otherwise. 3620 * 3621 * The set of states for the state machine closely matches the JSON 3622 * specification (http://json.org/). Briefly: 3623 * 3624 * DTRACE_JSON_REST: 3625 * Skip whitespace until we find either a top-level Object, moving 3626 * to DTRACE_JSON_OBJECT; or an Array, moving to DTRACE_JSON_VALUE. 3627 * 3628 * DTRACE_JSON_OBJECT: 3629 * Locate the next key String in an Object. Sets a flag to denote 3630 * the next String as a key string and moves to DTRACE_JSON_STRING. 3631 * 3632 * DTRACE_JSON_COLON: 3633 * Skip whitespace until we find the colon that separates key Strings 3634 * from their values. Once found, move to DTRACE_JSON_VALUE. 3635 * 3636 * DTRACE_JSON_VALUE: 3637 * Detects the type of the next value (String, Number, Identifier, Object 3638 * or Array) and routes to the states that process that type. Here we also 3639 * deal with the element selector list if we are requested to traverse down 3640 * into the object tree. 3641 * 3642 * DTRACE_JSON_COMMA: 3643 * Skip whitespace until we find the comma that separates key-value pairs 3644 * in Objects (returning to DTRACE_JSON_OBJECT) or values in Arrays 3645 * (similarly DTRACE_JSON_VALUE). All following literal value processing 3646 * states return to this state at the end of their value, unless otherwise 3647 * noted. 3648 * 3649 * DTRACE_JSON_NUMBER, DTRACE_JSON_NUMBER_FRAC, DTRACE_JSON_NUMBER_EXP: 3650 * Processes a Number literal from the JSON, including any exponent 3651 * component that may be present. Numbers are returned as strings, which 3652 * may be passed to strtoll() if an integer is required. 3653 * 3654 * DTRACE_JSON_IDENTIFIER: 3655 * Processes a "true", "false" or "null" literal in the JSON. 3656 * 3657 * DTRACE_JSON_STRING, DTRACE_JSON_STRING_ESCAPE, 3658 * DTRACE_JSON_STRING_ESCAPE_UNICODE: 3659 * Processes a String literal from the JSON, whether the String denotes 3660 * a key, a value or part of a larger Object. Handles all escape sequences 3661 * present in the specification, including four-digit unicode characters, 3662 * but merely includes the escape sequence without converting it to the 3663 * actual escaped character. If the String is flagged as a key, we 3664 * move to DTRACE_JSON_COLON rather than DTRACE_JSON_COMMA. 3665 * 3666 * DTRACE_JSON_COLLECT_OBJECT: 3667 * This state collects an entire Object (or Array), correctly handling 3668 * embedded strings. If the full element selector list matches this nested 3669 * object, we return the Object in full as a string. If not, we use this 3670 * state to skip to the next value at this level and continue processing. 3671 * 3672 * NOTE: This function uses various macros from strtolctype.h to manipulate 3673 * digit values, etc -- these have all been checked to ensure they make 3674 * no additional function calls. 3675 */ 3676 static char * 3677 dtrace_json(uint64_t size, uintptr_t json, char *elemlist, int nelems, 3678 char *dest) 3679 { 3680 dtrace_json_state_t state = DTRACE_JSON_REST; 3681 int64_t array_elem = INT64_MIN; 3682 int64_t array_pos = 0; 3683 uint8_t escape_unicount = 0; 3684 boolean_t string_is_key = B_FALSE; 3685 boolean_t collect_object = B_FALSE; 3686 boolean_t found_key = B_FALSE; 3687 boolean_t in_array = B_FALSE; 3688 uint32_t braces = 0, brackets = 0; 3689 char *elem = elemlist; 3690 char *dd = dest; 3691 uintptr_t cur; 3692 3693 for (cur = json; cur < json + size; cur++) { 3694 char cc = dtrace_load8(cur); 3695 if (cc == '\0') 3696 return (NULL); 3697 3698 switch (state) { 3699 case DTRACE_JSON_REST: 3700 if (isspace(cc)) 3701 break; 3702 3703 if (cc == '{') { 3704 state = DTRACE_JSON_OBJECT; 3705 break; 3706 } 3707 3708 if (cc == '[') { 3709 in_array = B_TRUE; 3710 array_pos = 0; 3711 array_elem = dtrace_strtoll(elem, 10, size); 3712 found_key = array_elem == 0 ? B_TRUE : B_FALSE; 3713 state = DTRACE_JSON_VALUE; 3714 break; 3715 } 3716 3717 /* 3718 * ERROR: expected to find a top-level object or array. 3719 */ 3720 return (NULL); 3721 case DTRACE_JSON_OBJECT: 3722 if (isspace(cc)) 3723 break; 3724 3725 if (cc == '"') { 3726 state = DTRACE_JSON_STRING; 3727 string_is_key = B_TRUE; 3728 break; 3729 } 3730 3731 /* 3732 * ERROR: either the object did not start with a key 3733 * string, or we've run off the end of the object 3734 * without finding the requested key. 3735 */ 3736 return (NULL); 3737 case DTRACE_JSON_STRING: 3738 if (cc == '\\') { 3739 *dd++ = '\\'; 3740 state = DTRACE_JSON_STRING_ESCAPE; 3741 break; 3742 } 3743 3744 if (cc == '"') { 3745 if (collect_object) { 3746 /* 3747 * We don't reset the dest here, as 3748 * the string is part of a larger 3749 * object being collected. 3750 */ 3751 *dd++ = cc; 3752 collect_object = B_FALSE; 3753 state = DTRACE_JSON_COLLECT_OBJECT; 3754 break; 3755 } 3756 *dd = '\0'; 3757 dd = dest; /* reset string buffer */ 3758 if (string_is_key) { 3759 if (dtrace_strncmp(dest, elem, 3760 size) == 0) 3761 found_key = B_TRUE; 3762 } else if (found_key) { 3763 if (nelems > 1) { 3764 /* 3765 * We expected an object, not 3766 * this string. 3767 */ 3768 return (NULL); 3769 } 3770 return (dest); 3771 } 3772 state = string_is_key ? DTRACE_JSON_COLON : 3773 DTRACE_JSON_COMMA; 3774 string_is_key = B_FALSE; 3775 break; 3776 } 3777 3778 *dd++ = cc; 3779 break; 3780 case DTRACE_JSON_STRING_ESCAPE: 3781 *dd++ = cc; 3782 if (cc == 'u') { 3783 escape_unicount = 0; 3784 state = DTRACE_JSON_STRING_ESCAPE_UNICODE; 3785 } else { 3786 state = DTRACE_JSON_STRING; 3787 } 3788 break; 3789 case DTRACE_JSON_STRING_ESCAPE_UNICODE: 3790 if (!isxdigit(cc)) { 3791 /* 3792 * ERROR: invalid unicode escape, expected 3793 * four valid hexidecimal digits. 3794 */ 3795 return (NULL); 3796 } 3797 3798 *dd++ = cc; 3799 if (++escape_unicount == 4) 3800 state = DTRACE_JSON_STRING; 3801 break; 3802 case DTRACE_JSON_COLON: 3803 if (isspace(cc)) 3804 break; 3805 3806 if (cc == ':') { 3807 state = DTRACE_JSON_VALUE; 3808 break; 3809 } 3810 3811 /* 3812 * ERROR: expected a colon. 3813 */ 3814 return (NULL); 3815 case DTRACE_JSON_COMMA: 3816 if (isspace(cc)) 3817 break; 3818 3819 if (cc == ',') { 3820 if (in_array) { 3821 state = DTRACE_JSON_VALUE; 3822 if (++array_pos == array_elem) 3823 found_key = B_TRUE; 3824 } else { 3825 state = DTRACE_JSON_OBJECT; 3826 } 3827 break; 3828 } 3829 3830 /* 3831 * ERROR: either we hit an unexpected character, or 3832 * we reached the end of the object or array without 3833 * finding the requested key. 3834 */ 3835 return (NULL); 3836 case DTRACE_JSON_IDENTIFIER: 3837 if (islower(cc)) { 3838 *dd++ = cc; 3839 break; 3840 } 3841 3842 *dd = '\0'; 3843 dd = dest; /* reset string buffer */ 3844 3845 if (dtrace_strncmp(dest, "true", 5) == 0 || 3846 dtrace_strncmp(dest, "false", 6) == 0 || 3847 dtrace_strncmp(dest, "null", 5) == 0) { 3848 if (found_key) { 3849 if (nelems > 1) { 3850 /* 3851 * ERROR: We expected an object, 3852 * not this identifier. 3853 */ 3854 return (NULL); 3855 } 3856 return (dest); 3857 } else { 3858 cur--; 3859 state = DTRACE_JSON_COMMA; 3860 break; 3861 } 3862 } 3863 3864 /* 3865 * ERROR: we did not recognise the identifier as one 3866 * of those in the JSON specification. 3867 */ 3868 return (NULL); 3869 case DTRACE_JSON_NUMBER: 3870 if (cc == '.') { 3871 *dd++ = cc; 3872 state = DTRACE_JSON_NUMBER_FRAC; 3873 break; 3874 } 3875 3876 if (cc == 'x' || cc == 'X') { 3877 /* 3878 * ERROR: specification explicitly excludes 3879 * hexidecimal or octal numbers. 3880 */ 3881 return (NULL); 3882 } 3883 3884 /* FALLTHRU */ 3885 case DTRACE_JSON_NUMBER_FRAC: 3886 if (cc == 'e' || cc == 'E') { 3887 *dd++ = cc; 3888 state = DTRACE_JSON_NUMBER_EXP; 3889 break; 3890 } 3891 3892 if (cc == '+' || cc == '-') { 3893 /* 3894 * ERROR: expect sign as part of exponent only. 3895 */ 3896 return (NULL); 3897 } 3898 /* FALLTHRU */ 3899 case DTRACE_JSON_NUMBER_EXP: 3900 if (isdigit(cc) || cc == '+' || cc == '-') { 3901 *dd++ = cc; 3902 break; 3903 } 3904 3905 *dd = '\0'; 3906 dd = dest; /* reset string buffer */ 3907 if (found_key) { 3908 if (nelems > 1) { 3909 /* 3910 * ERROR: We expected an object, not 3911 * this number. 3912 */ 3913 return (NULL); 3914 } 3915 return (dest); 3916 } 3917 3918 cur--; 3919 state = DTRACE_JSON_COMMA; 3920 break; 3921 case DTRACE_JSON_VALUE: 3922 if (isspace(cc)) 3923 break; 3924 3925 if (cc == '{' || cc == '[') { 3926 if (nelems > 1 && found_key) { 3927 in_array = cc == '[' ? B_TRUE : B_FALSE; 3928 /* 3929 * If our element selector directs us 3930 * to descend into this nested object, 3931 * then move to the next selector 3932 * element in the list and restart the 3933 * state machine. 3934 */ 3935 while (*elem != '\0') 3936 elem++; 3937 elem++; /* skip the inter-element NUL */ 3938 nelems--; 3939 dd = dest; 3940 if (in_array) { 3941 state = DTRACE_JSON_VALUE; 3942 array_pos = 0; 3943 array_elem = dtrace_strtoll( 3944 elem, 10, size); 3945 found_key = array_elem == 0 ? 3946 B_TRUE : B_FALSE; 3947 } else { 3948 found_key = B_FALSE; 3949 state = DTRACE_JSON_OBJECT; 3950 } 3951 break; 3952 } 3953 3954 /* 3955 * Otherwise, we wish to either skip this 3956 * nested object or return it in full. 3957 */ 3958 if (cc == '[') 3959 brackets = 1; 3960 else 3961 braces = 1; 3962 *dd++ = cc; 3963 state = DTRACE_JSON_COLLECT_OBJECT; 3964 break; 3965 } 3966 3967 if (cc == '"') { 3968 state = DTRACE_JSON_STRING; 3969 break; 3970 } 3971 3972 if (islower(cc)) { 3973 /* 3974 * Here we deal with true, false and null. 3975 */ 3976 *dd++ = cc; 3977 state = DTRACE_JSON_IDENTIFIER; 3978 break; 3979 } 3980 3981 if (cc == '-' || isdigit(cc)) { 3982 *dd++ = cc; 3983 state = DTRACE_JSON_NUMBER; 3984 break; 3985 } 3986 3987 /* 3988 * ERROR: unexpected character at start of value. 3989 */ 3990 return (NULL); 3991 case DTRACE_JSON_COLLECT_OBJECT: 3992 if (cc == '\0') 3993 /* 3994 * ERROR: unexpected end of input. 3995 */ 3996 return (NULL); 3997 3998 *dd++ = cc; 3999 if (cc == '"') { 4000 collect_object = B_TRUE; 4001 state = DTRACE_JSON_STRING; 4002 break; 4003 } 4004 4005 if (cc == ']') { 4006 if (brackets-- == 0) { 4007 /* 4008 * ERROR: unbalanced brackets. 4009 */ 4010 return (NULL); 4011 } 4012 } else if (cc == '}') { 4013 if (braces-- == 0) { 4014 /* 4015 * ERROR: unbalanced braces. 4016 */ 4017 return (NULL); 4018 } 4019 } else if (cc == '{') { 4020 braces++; 4021 } else if (cc == '[') { 4022 brackets++; 4023 } 4024 4025 if (brackets == 0 && braces == 0) { 4026 if (found_key) { 4027 *dd = '\0'; 4028 return (dest); 4029 } 4030 dd = dest; /* reset string buffer */ 4031 state = DTRACE_JSON_COMMA; 4032 } 4033 break; 4034 } 4035 } 4036 return (NULL); 4037 } 4038 4039 /* 4040 * Emulate the execution of DTrace ID subroutines invoked by the call opcode. 4041 * Notice that we don't bother validating the proper number of arguments or 4042 * their types in the tuple stack. This isn't needed because all argument 4043 * interpretation is safe because of our load safety -- the worst that can 4044 * happen is that a bogus program can obtain bogus results. 4045 */ 4046 static void 4047 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs, 4048 dtrace_key_t *tupregs, int nargs, 4049 dtrace_mstate_t *mstate, dtrace_state_t *state) 4050 { 4051 volatile uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; 4052 volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval; 4053 dtrace_vstate_t *vstate = &state->dts_vstate; 4054 4055 #ifdef illumos 4056 union { 4057 mutex_impl_t mi; 4058 uint64_t mx; 4059 } m; 4060 4061 union { 4062 krwlock_t ri; 4063 uintptr_t rw; 4064 } r; 4065 #else 4066 struct thread *lowner; 4067 union { 4068 struct lock_object *li; 4069 uintptr_t lx; 4070 } l; 4071 #endif 4072 4073 switch (subr) { 4074 case DIF_SUBR_RAND: 4075 regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875; 4076 break; 4077 4078 #ifdef illumos 4079 case DIF_SUBR_MUTEX_OWNED: 4080 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4081 mstate, vstate)) { 4082 regs[rd] = 0; 4083 break; 4084 } 4085 4086 m.mx = dtrace_load64(tupregs[0].dttk_value); 4087 if (MUTEX_TYPE_ADAPTIVE(&m.mi)) 4088 regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER; 4089 else 4090 regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock); 4091 break; 4092 4093 case DIF_SUBR_MUTEX_OWNER: 4094 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4095 mstate, vstate)) { 4096 regs[rd] = 0; 4097 break; 4098 } 4099 4100 m.mx = dtrace_load64(tupregs[0].dttk_value); 4101 if (MUTEX_TYPE_ADAPTIVE(&m.mi) && 4102 MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER) 4103 regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi); 4104 else 4105 regs[rd] = 0; 4106 break; 4107 4108 case DIF_SUBR_MUTEX_TYPE_ADAPTIVE: 4109 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4110 mstate, vstate)) { 4111 regs[rd] = 0; 4112 break; 4113 } 4114 4115 m.mx = dtrace_load64(tupregs[0].dttk_value); 4116 regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi); 4117 break; 4118 4119 case DIF_SUBR_MUTEX_TYPE_SPIN: 4120 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (kmutex_t), 4121 mstate, vstate)) { 4122 regs[rd] = 0; 4123 break; 4124 } 4125 4126 m.mx = dtrace_load64(tupregs[0].dttk_value); 4127 regs[rd] = MUTEX_TYPE_SPIN(&m.mi); 4128 break; 4129 4130 case DIF_SUBR_RW_READ_HELD: { 4131 uintptr_t tmp; 4132 4133 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), 4134 mstate, vstate)) { 4135 regs[rd] = 0; 4136 break; 4137 } 4138 4139 r.rw = dtrace_loadptr(tupregs[0].dttk_value); 4140 regs[rd] = _RW_READ_HELD(&r.ri, tmp); 4141 break; 4142 } 4143 4144 case DIF_SUBR_RW_WRITE_HELD: 4145 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t), 4146 mstate, vstate)) { 4147 regs[rd] = 0; 4148 break; 4149 } 4150 4151 r.rw = dtrace_loadptr(tupregs[0].dttk_value); 4152 regs[rd] = _RW_WRITE_HELD(&r.ri); 4153 break; 4154 4155 case DIF_SUBR_RW_ISWRITER: 4156 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (krwlock_t), 4157 mstate, vstate)) { 4158 regs[rd] = 0; 4159 break; 4160 } 4161 4162 r.rw = dtrace_loadptr(tupregs[0].dttk_value); 4163 regs[rd] = _RW_ISWRITER(&r.ri); 4164 break; 4165 4166 #else /* !illumos */ 4167 case DIF_SUBR_MUTEX_OWNED: 4168 if (!dtrace_canload(tupregs[0].dttk_value, 4169 sizeof (struct lock_object), mstate, vstate)) { 4170 regs[rd] = 0; 4171 break; 4172 } 4173 l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); 4174 regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); 4175 break; 4176 4177 case DIF_SUBR_MUTEX_OWNER: 4178 if (!dtrace_canload(tupregs[0].dttk_value, 4179 sizeof (struct lock_object), mstate, vstate)) { 4180 regs[rd] = 0; 4181 break; 4182 } 4183 l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); 4184 LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); 4185 regs[rd] = (uintptr_t)lowner; 4186 break; 4187 4188 case DIF_SUBR_MUTEX_TYPE_ADAPTIVE: 4189 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct mtx), 4190 mstate, vstate)) { 4191 regs[rd] = 0; 4192 break; 4193 } 4194 l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); 4195 /* XXX - should be only LC_SLEEPABLE? */ 4196 regs[rd] = (LOCK_CLASS(l.li)->lc_flags & 4197 (LC_SLEEPLOCK | LC_SLEEPABLE)) != 0; 4198 break; 4199 4200 case DIF_SUBR_MUTEX_TYPE_SPIN: 4201 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (struct mtx), 4202 mstate, vstate)) { 4203 regs[rd] = 0; 4204 break; 4205 } 4206 l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); 4207 regs[rd] = (LOCK_CLASS(l.li)->lc_flags & LC_SPINLOCK) != 0; 4208 break; 4209 4210 case DIF_SUBR_RW_READ_HELD: 4211 case DIF_SUBR_SX_SHARED_HELD: 4212 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), 4213 mstate, vstate)) { 4214 regs[rd] = 0; 4215 break; 4216 } 4217 l.lx = dtrace_loadptr((uintptr_t)&tupregs[0].dttk_value); 4218 regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) && 4219 lowner == NULL; 4220 break; 4221 4222 case DIF_SUBR_RW_WRITE_HELD: 4223 case DIF_SUBR_SX_EXCLUSIVE_HELD: 4224 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), 4225 mstate, vstate)) { 4226 regs[rd] = 0; 4227 break; 4228 } 4229 l.lx = dtrace_loadptr(tupregs[0].dttk_value); 4230 LOCK_CLASS(l.li)->lc_owner(l.li, &lowner); 4231 regs[rd] = (lowner == curthread); 4232 break; 4233 4234 case DIF_SUBR_RW_ISWRITER: 4235 case DIF_SUBR_SX_ISEXCLUSIVE: 4236 if (!dtrace_canload(tupregs[0].dttk_value, sizeof (uintptr_t), 4237 mstate, vstate)) { 4238 regs[rd] = 0; 4239 break; 4240 } 4241 l.lx = dtrace_loadptr(tupregs[0].dttk_value); 4242 regs[rd] = LOCK_CLASS(l.li)->lc_owner(l.li, &lowner) && 4243 lowner != NULL; 4244 break; 4245 #endif /* illumos */ 4246 4247 case DIF_SUBR_BCOPY: { 4248 /* 4249 * We need to be sure that the destination is in the scratch 4250 * region -- no other region is allowed. 4251 */ 4252 uintptr_t src = tupregs[0].dttk_value; 4253 uintptr_t dest = tupregs[1].dttk_value; 4254 size_t size = tupregs[2].dttk_value; 4255 4256 if (!dtrace_inscratch(dest, size, mstate)) { 4257 *flags |= CPU_DTRACE_BADADDR; 4258 *illval = regs[rd]; 4259 break; 4260 } 4261 4262 if (!dtrace_canload(src, size, mstate, vstate)) { 4263 regs[rd] = 0; 4264 break; 4265 } 4266 4267 dtrace_bcopy((void *)src, (void *)dest, size); 4268 break; 4269 } 4270 4271 case DIF_SUBR_ALLOCA: 4272 case DIF_SUBR_COPYIN: { 4273 uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8); 4274 uint64_t size = 4275 tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value; 4276 size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size; 4277 4278 /* 4279 * This action doesn't require any credential checks since 4280 * probes will not activate in user contexts to which the 4281 * enabling user does not have permissions. 4282 */ 4283 4284 /* 4285 * Rounding up the user allocation size could have overflowed 4286 * a large, bogus allocation (like -1ULL) to 0. 4287 */ 4288 if (scratch_size < size || 4289 !DTRACE_INSCRATCH(mstate, scratch_size)) { 4290 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4291 regs[rd] = 0; 4292 break; 4293 } 4294 4295 if (subr == DIF_SUBR_COPYIN) { 4296 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4297 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags); 4298 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4299 } 4300 4301 mstate->dtms_scratch_ptr += scratch_size; 4302 regs[rd] = dest; 4303 break; 4304 } 4305 4306 case DIF_SUBR_COPYINTO: { 4307 uint64_t size = tupregs[1].dttk_value; 4308 uintptr_t dest = tupregs[2].dttk_value; 4309 4310 /* 4311 * This action doesn't require any credential checks since 4312 * probes will not activate in user contexts to which the 4313 * enabling user does not have permissions. 4314 */ 4315 if (!dtrace_inscratch(dest, size, mstate)) { 4316 *flags |= CPU_DTRACE_BADADDR; 4317 *illval = regs[rd]; 4318 break; 4319 } 4320 4321 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4322 dtrace_copyin(tupregs[0].dttk_value, dest, size, flags); 4323 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4324 break; 4325 } 4326 4327 case DIF_SUBR_COPYINSTR: { 4328 uintptr_t dest = mstate->dtms_scratch_ptr; 4329 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4330 4331 if (nargs > 1 && tupregs[1].dttk_value < size) 4332 size = tupregs[1].dttk_value + 1; 4333 4334 /* 4335 * This action doesn't require any credential checks since 4336 * probes will not activate in user contexts to which the 4337 * enabling user does not have permissions. 4338 */ 4339 if (!DTRACE_INSCRATCH(mstate, size)) { 4340 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4341 regs[rd] = 0; 4342 break; 4343 } 4344 4345 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4346 dtrace_copyinstr(tupregs[0].dttk_value, dest, size, flags); 4347 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4348 4349 ((char *)dest)[size - 1] = '\0'; 4350 mstate->dtms_scratch_ptr += size; 4351 regs[rd] = dest; 4352 break; 4353 } 4354 4355 #ifdef illumos 4356 case DIF_SUBR_MSGSIZE: 4357 case DIF_SUBR_MSGDSIZE: { 4358 uintptr_t baddr = tupregs[0].dttk_value, daddr; 4359 uintptr_t wptr, rptr; 4360 size_t count = 0; 4361 int cont = 0; 4362 4363 while (baddr != 0 && !(*flags & CPU_DTRACE_FAULT)) { 4364 4365 if (!dtrace_canload(baddr, sizeof (mblk_t), mstate, 4366 vstate)) { 4367 regs[rd] = 0; 4368 break; 4369 } 4370 4371 wptr = dtrace_loadptr(baddr + 4372 offsetof(mblk_t, b_wptr)); 4373 4374 rptr = dtrace_loadptr(baddr + 4375 offsetof(mblk_t, b_rptr)); 4376 4377 if (wptr < rptr) { 4378 *flags |= CPU_DTRACE_BADADDR; 4379 *illval = tupregs[0].dttk_value; 4380 break; 4381 } 4382 4383 daddr = dtrace_loadptr(baddr + 4384 offsetof(mblk_t, b_datap)); 4385 4386 baddr = dtrace_loadptr(baddr + 4387 offsetof(mblk_t, b_cont)); 4388 4389 /* 4390 * We want to prevent against denial-of-service here, 4391 * so we're only going to search the list for 4392 * dtrace_msgdsize_max mblks. 4393 */ 4394 if (cont++ > dtrace_msgdsize_max) { 4395 *flags |= CPU_DTRACE_ILLOP; 4396 break; 4397 } 4398 4399 if (subr == DIF_SUBR_MSGDSIZE) { 4400 if (dtrace_load8(daddr + 4401 offsetof(dblk_t, db_type)) != M_DATA) 4402 continue; 4403 } 4404 4405 count += wptr - rptr; 4406 } 4407 4408 if (!(*flags & CPU_DTRACE_FAULT)) 4409 regs[rd] = count; 4410 4411 break; 4412 } 4413 #endif 4414 4415 case DIF_SUBR_PROGENYOF: { 4416 pid_t pid = tupregs[0].dttk_value; 4417 proc_t *p; 4418 int rval = 0; 4419 4420 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4421 4422 for (p = curthread->t_procp; p != NULL; p = p->p_parent) { 4423 #ifdef illumos 4424 if (p->p_pidp->pid_id == pid) { 4425 #else 4426 if (p->p_pid == pid) { 4427 #endif 4428 rval = 1; 4429 break; 4430 } 4431 } 4432 4433 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4434 4435 regs[rd] = rval; 4436 break; 4437 } 4438 4439 case DIF_SUBR_SPECULATION: 4440 regs[rd] = dtrace_speculation(state); 4441 break; 4442 4443 case DIF_SUBR_COPYOUT: { 4444 uintptr_t kaddr = tupregs[0].dttk_value; 4445 uintptr_t uaddr = tupregs[1].dttk_value; 4446 uint64_t size = tupregs[2].dttk_value; 4447 4448 if (!dtrace_destructive_disallow && 4449 dtrace_priv_proc_control(state) && 4450 !dtrace_istoxic(kaddr, size)) { 4451 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4452 dtrace_copyout(kaddr, uaddr, size, flags); 4453 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4454 } 4455 break; 4456 } 4457 4458 case DIF_SUBR_COPYOUTSTR: { 4459 uintptr_t kaddr = tupregs[0].dttk_value; 4460 uintptr_t uaddr = tupregs[1].dttk_value; 4461 uint64_t size = tupregs[2].dttk_value; 4462 4463 if (!dtrace_destructive_disallow && 4464 dtrace_priv_proc_control(state) && 4465 !dtrace_istoxic(kaddr, size)) { 4466 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 4467 dtrace_copyoutstr(kaddr, uaddr, size, flags); 4468 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 4469 } 4470 break; 4471 } 4472 4473 case DIF_SUBR_STRLEN: { 4474 size_t sz; 4475 uintptr_t addr = (uintptr_t)tupregs[0].dttk_value; 4476 sz = dtrace_strlen((char *)addr, 4477 state->dts_options[DTRACEOPT_STRSIZE]); 4478 4479 if (!dtrace_canload(addr, sz + 1, mstate, vstate)) { 4480 regs[rd] = 0; 4481 break; 4482 } 4483 4484 regs[rd] = sz; 4485 4486 break; 4487 } 4488 4489 case DIF_SUBR_STRCHR: 4490 case DIF_SUBR_STRRCHR: { 4491 /* 4492 * We're going to iterate over the string looking for the 4493 * specified character. We will iterate until we have reached 4494 * the string length or we have found the character. If this 4495 * is DIF_SUBR_STRRCHR, we will look for the last occurrence 4496 * of the specified character instead of the first. 4497 */ 4498 uintptr_t saddr = tupregs[0].dttk_value; 4499 uintptr_t addr = tupregs[0].dttk_value; 4500 uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE]; 4501 char c, target = (char)tupregs[1].dttk_value; 4502 4503 for (regs[rd] = 0; addr < limit; addr++) { 4504 if ((c = dtrace_load8(addr)) == target) { 4505 regs[rd] = addr; 4506 4507 if (subr == DIF_SUBR_STRCHR) 4508 break; 4509 } 4510 4511 if (c == '\0') 4512 break; 4513 } 4514 4515 if (!dtrace_canload(saddr, addr - saddr, mstate, vstate)) { 4516 regs[rd] = 0; 4517 break; 4518 } 4519 4520 break; 4521 } 4522 4523 case DIF_SUBR_STRSTR: 4524 case DIF_SUBR_INDEX: 4525 case DIF_SUBR_RINDEX: { 4526 /* 4527 * We're going to iterate over the string looking for the 4528 * specified string. We will iterate until we have reached 4529 * the string length or we have found the string. (Yes, this 4530 * is done in the most naive way possible -- but considering 4531 * that the string we're searching for is likely to be 4532 * relatively short, the complexity of Rabin-Karp or similar 4533 * hardly seems merited.) 4534 */ 4535 char *addr = (char *)(uintptr_t)tupregs[0].dttk_value; 4536 char *substr = (char *)(uintptr_t)tupregs[1].dttk_value; 4537 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4538 size_t len = dtrace_strlen(addr, size); 4539 size_t sublen = dtrace_strlen(substr, size); 4540 char *limit = addr + len, *orig = addr; 4541 int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1; 4542 int inc = 1; 4543 4544 regs[rd] = notfound; 4545 4546 if (!dtrace_canload((uintptr_t)addr, len + 1, mstate, vstate)) { 4547 regs[rd] = 0; 4548 break; 4549 } 4550 4551 if (!dtrace_canload((uintptr_t)substr, sublen + 1, mstate, 4552 vstate)) { 4553 regs[rd] = 0; 4554 break; 4555 } 4556 4557 /* 4558 * strstr() and index()/rindex() have similar semantics if 4559 * both strings are the empty string: strstr() returns a 4560 * pointer to the (empty) string, and index() and rindex() 4561 * both return index 0 (regardless of any position argument). 4562 */ 4563 if (sublen == 0 && len == 0) { 4564 if (subr == DIF_SUBR_STRSTR) 4565 regs[rd] = (uintptr_t)addr; 4566 else 4567 regs[rd] = 0; 4568 break; 4569 } 4570 4571 if (subr != DIF_SUBR_STRSTR) { 4572 if (subr == DIF_SUBR_RINDEX) { 4573 limit = orig - 1; 4574 addr += len; 4575 inc = -1; 4576 } 4577 4578 /* 4579 * Both index() and rindex() take an optional position 4580 * argument that denotes the starting position. 4581 */ 4582 if (nargs == 3) { 4583 int64_t pos = (int64_t)tupregs[2].dttk_value; 4584 4585 /* 4586 * If the position argument to index() is 4587 * negative, Perl implicitly clamps it at 4588 * zero. This semantic is a little surprising 4589 * given the special meaning of negative 4590 * positions to similar Perl functions like 4591 * substr(), but it appears to reflect a 4592 * notion that index() can start from a 4593 * negative index and increment its way up to 4594 * the string. Given this notion, Perl's 4595 * rindex() is at least self-consistent in 4596 * that it implicitly clamps positions greater 4597 * than the string length to be the string 4598 * length. Where Perl completely loses 4599 * coherence, however, is when the specified 4600 * substring is the empty string (""). In 4601 * this case, even if the position is 4602 * negative, rindex() returns 0 -- and even if 4603 * the position is greater than the length, 4604 * index() returns the string length. These 4605 * semantics violate the notion that index() 4606 * should never return a value less than the 4607 * specified position and that rindex() should 4608 * never return a value greater than the 4609 * specified position. (One assumes that 4610 * these semantics are artifacts of Perl's 4611 * implementation and not the results of 4612 * deliberate design -- it beggars belief that 4613 * even Larry Wall could desire such oddness.) 4614 * While in the abstract one would wish for 4615 * consistent position semantics across 4616 * substr(), index() and rindex() -- or at the 4617 * very least self-consistent position 4618 * semantics for index() and rindex() -- we 4619 * instead opt to keep with the extant Perl 4620 * semantics, in all their broken glory. (Do 4621 * we have more desire to maintain Perl's 4622 * semantics than Perl does? Probably.) 4623 */ 4624 if (subr == DIF_SUBR_RINDEX) { 4625 if (pos < 0) { 4626 if (sublen == 0) 4627 regs[rd] = 0; 4628 break; 4629 } 4630 4631 if (pos > len) 4632 pos = len; 4633 } else { 4634 if (pos < 0) 4635 pos = 0; 4636 4637 if (pos >= len) { 4638 if (sublen == 0) 4639 regs[rd] = len; 4640 break; 4641 } 4642 } 4643 4644 addr = orig + pos; 4645 } 4646 } 4647 4648 for (regs[rd] = notfound; addr != limit; addr += inc) { 4649 if (dtrace_strncmp(addr, substr, sublen) == 0) { 4650 if (subr != DIF_SUBR_STRSTR) { 4651 /* 4652 * As D index() and rindex() are 4653 * modeled on Perl (and not on awk), 4654 * we return a zero-based (and not a 4655 * one-based) index. (For you Perl 4656 * weenies: no, we're not going to add 4657 * $[ -- and shouldn't you be at a con 4658 * or something?) 4659 */ 4660 regs[rd] = (uintptr_t)(addr - orig); 4661 break; 4662 } 4663 4664 ASSERT(subr == DIF_SUBR_STRSTR); 4665 regs[rd] = (uintptr_t)addr; 4666 break; 4667 } 4668 } 4669 4670 break; 4671 } 4672 4673 case DIF_SUBR_STRTOK: { 4674 uintptr_t addr = tupregs[0].dttk_value; 4675 uintptr_t tokaddr = tupregs[1].dttk_value; 4676 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4677 uintptr_t limit, toklimit = tokaddr + size; 4678 uint8_t c = 0, tokmap[32]; /* 256 / 8 */ 4679 char *dest = (char *)mstate->dtms_scratch_ptr; 4680 int i; 4681 4682 /* 4683 * Check both the token buffer and (later) the input buffer, 4684 * since both could be non-scratch addresses. 4685 */ 4686 if (!dtrace_strcanload(tokaddr, size, mstate, vstate)) { 4687 regs[rd] = 0; 4688 break; 4689 } 4690 4691 if (!DTRACE_INSCRATCH(mstate, size)) { 4692 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4693 regs[rd] = 0; 4694 break; 4695 } 4696 4697 if (addr == 0) { 4698 /* 4699 * If the address specified is NULL, we use our saved 4700 * strtok pointer from the mstate. Note that this 4701 * means that the saved strtok pointer is _only_ 4702 * valid within multiple enablings of the same probe -- 4703 * it behaves like an implicit clause-local variable. 4704 */ 4705 addr = mstate->dtms_strtok; 4706 } else { 4707 /* 4708 * If the user-specified address is non-NULL we must 4709 * access check it. This is the only time we have 4710 * a chance to do so, since this address may reside 4711 * in the string table of this clause-- future calls 4712 * (when we fetch addr from mstate->dtms_strtok) 4713 * would fail this access check. 4714 */ 4715 if (!dtrace_strcanload(addr, size, mstate, vstate)) { 4716 regs[rd] = 0; 4717 break; 4718 } 4719 } 4720 4721 /* 4722 * First, zero the token map, and then process the token 4723 * string -- setting a bit in the map for every character 4724 * found in the token string. 4725 */ 4726 for (i = 0; i < sizeof (tokmap); i++) 4727 tokmap[i] = 0; 4728 4729 for (; tokaddr < toklimit; tokaddr++) { 4730 if ((c = dtrace_load8(tokaddr)) == '\0') 4731 break; 4732 4733 ASSERT((c >> 3) < sizeof (tokmap)); 4734 tokmap[c >> 3] |= (1 << (c & 0x7)); 4735 } 4736 4737 for (limit = addr + size; addr < limit; addr++) { 4738 /* 4739 * We're looking for a character that is _not_ contained 4740 * in the token string. 4741 */ 4742 if ((c = dtrace_load8(addr)) == '\0') 4743 break; 4744 4745 if (!(tokmap[c >> 3] & (1 << (c & 0x7)))) 4746 break; 4747 } 4748 4749 if (c == '\0') { 4750 /* 4751 * We reached the end of the string without finding 4752 * any character that was not in the token string. 4753 * We return NULL in this case, and we set the saved 4754 * address to NULL as well. 4755 */ 4756 regs[rd] = 0; 4757 mstate->dtms_strtok = 0; 4758 break; 4759 } 4760 4761 /* 4762 * From here on, we're copying into the destination string. 4763 */ 4764 for (i = 0; addr < limit && i < size - 1; addr++) { 4765 if ((c = dtrace_load8(addr)) == '\0') 4766 break; 4767 4768 if (tokmap[c >> 3] & (1 << (c & 0x7))) 4769 break; 4770 4771 ASSERT(i < size); 4772 dest[i++] = c; 4773 } 4774 4775 ASSERT(i < size); 4776 dest[i] = '\0'; 4777 regs[rd] = (uintptr_t)dest; 4778 mstate->dtms_scratch_ptr += size; 4779 mstate->dtms_strtok = addr; 4780 break; 4781 } 4782 4783 case DIF_SUBR_SUBSTR: { 4784 uintptr_t s = tupregs[0].dttk_value; 4785 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4786 char *d = (char *)mstate->dtms_scratch_ptr; 4787 int64_t index = (int64_t)tupregs[1].dttk_value; 4788 int64_t remaining = (int64_t)tupregs[2].dttk_value; 4789 size_t len = dtrace_strlen((char *)s, size); 4790 int64_t i; 4791 4792 if (!dtrace_canload(s, len + 1, mstate, vstate)) { 4793 regs[rd] = 0; 4794 break; 4795 } 4796 4797 if (!DTRACE_INSCRATCH(mstate, size)) { 4798 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4799 regs[rd] = 0; 4800 break; 4801 } 4802 4803 if (nargs <= 2) 4804 remaining = (int64_t)size; 4805 4806 if (index < 0) { 4807 index += len; 4808 4809 if (index < 0 && index + remaining > 0) { 4810 remaining += index; 4811 index = 0; 4812 } 4813 } 4814 4815 if (index >= len || index < 0) { 4816 remaining = 0; 4817 } else if (remaining < 0) { 4818 remaining += len - index; 4819 } else if (index + remaining > size) { 4820 remaining = size - index; 4821 } 4822 4823 for (i = 0; i < remaining; i++) { 4824 if ((d[i] = dtrace_load8(s + index + i)) == '\0') 4825 break; 4826 } 4827 4828 d[i] = '\0'; 4829 4830 mstate->dtms_scratch_ptr += size; 4831 regs[rd] = (uintptr_t)d; 4832 break; 4833 } 4834 4835 case DIF_SUBR_JSON: { 4836 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4837 uintptr_t json = tupregs[0].dttk_value; 4838 size_t jsonlen = dtrace_strlen((char *)json, size); 4839 uintptr_t elem = tupregs[1].dttk_value; 4840 size_t elemlen = dtrace_strlen((char *)elem, size); 4841 4842 char *dest = (char *)mstate->dtms_scratch_ptr; 4843 char *elemlist = (char *)mstate->dtms_scratch_ptr + jsonlen + 1; 4844 char *ee = elemlist; 4845 int nelems = 1; 4846 uintptr_t cur; 4847 4848 if (!dtrace_canload(json, jsonlen + 1, mstate, vstate) || 4849 !dtrace_canload(elem, elemlen + 1, mstate, vstate)) { 4850 regs[rd] = 0; 4851 break; 4852 } 4853 4854 if (!DTRACE_INSCRATCH(mstate, jsonlen + 1 + elemlen + 1)) { 4855 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4856 regs[rd] = 0; 4857 break; 4858 } 4859 4860 /* 4861 * Read the element selector and split it up into a packed list 4862 * of strings. 4863 */ 4864 for (cur = elem; cur < elem + elemlen; cur++) { 4865 char cc = dtrace_load8(cur); 4866 4867 if (cur == elem && cc == '[') { 4868 /* 4869 * If the first element selector key is 4870 * actually an array index then ignore the 4871 * bracket. 4872 */ 4873 continue; 4874 } 4875 4876 if (cc == ']') 4877 continue; 4878 4879 if (cc == '.' || cc == '[') { 4880 nelems++; 4881 cc = '\0'; 4882 } 4883 4884 *ee++ = cc; 4885 } 4886 *ee++ = '\0'; 4887 4888 if ((regs[rd] = (uintptr_t)dtrace_json(size, json, elemlist, 4889 nelems, dest)) != 0) 4890 mstate->dtms_scratch_ptr += jsonlen + 1; 4891 break; 4892 } 4893 4894 case DIF_SUBR_TOUPPER: 4895 case DIF_SUBR_TOLOWER: { 4896 uintptr_t s = tupregs[0].dttk_value; 4897 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4898 char *dest = (char *)mstate->dtms_scratch_ptr, c; 4899 size_t len = dtrace_strlen((char *)s, size); 4900 char lower, upper, convert; 4901 int64_t i; 4902 4903 if (subr == DIF_SUBR_TOUPPER) { 4904 lower = 'a'; 4905 upper = 'z'; 4906 convert = 'A'; 4907 } else { 4908 lower = 'A'; 4909 upper = 'Z'; 4910 convert = 'a'; 4911 } 4912 4913 if (!dtrace_canload(s, len + 1, mstate, vstate)) { 4914 regs[rd] = 0; 4915 break; 4916 } 4917 4918 if (!DTRACE_INSCRATCH(mstate, size)) { 4919 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4920 regs[rd] = 0; 4921 break; 4922 } 4923 4924 for (i = 0; i < size - 1; i++) { 4925 if ((c = dtrace_load8(s + i)) == '\0') 4926 break; 4927 4928 if (c >= lower && c <= upper) 4929 c = convert + (c - lower); 4930 4931 dest[i] = c; 4932 } 4933 4934 ASSERT(i < size); 4935 dest[i] = '\0'; 4936 regs[rd] = (uintptr_t)dest; 4937 mstate->dtms_scratch_ptr += size; 4938 break; 4939 } 4940 4941 #ifdef illumos 4942 case DIF_SUBR_GETMAJOR: 4943 #ifdef _LP64 4944 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64; 4945 #else 4946 regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ; 4947 #endif 4948 break; 4949 4950 case DIF_SUBR_GETMINOR: 4951 #ifdef _LP64 4952 regs[rd] = tupregs[0].dttk_value & MAXMIN64; 4953 #else 4954 regs[rd] = tupregs[0].dttk_value & MAXMIN; 4955 #endif 4956 break; 4957 4958 case DIF_SUBR_DDI_PATHNAME: { 4959 /* 4960 * This one is a galactic mess. We are going to roughly 4961 * emulate ddi_pathname(), but it's made more complicated 4962 * by the fact that we (a) want to include the minor name and 4963 * (b) must proceed iteratively instead of recursively. 4964 */ 4965 uintptr_t dest = mstate->dtms_scratch_ptr; 4966 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 4967 char *start = (char *)dest, *end = start + size - 1; 4968 uintptr_t daddr = tupregs[0].dttk_value; 4969 int64_t minor = (int64_t)tupregs[1].dttk_value; 4970 char *s; 4971 int i, len, depth = 0; 4972 4973 /* 4974 * Due to all the pointer jumping we do and context we must 4975 * rely upon, we just mandate that the user must have kernel 4976 * read privileges to use this routine. 4977 */ 4978 if ((mstate->dtms_access & DTRACE_ACCESS_KERNEL) == 0) { 4979 *flags |= CPU_DTRACE_KPRIV; 4980 *illval = daddr; 4981 regs[rd] = 0; 4982 } 4983 4984 if (!DTRACE_INSCRATCH(mstate, size)) { 4985 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 4986 regs[rd] = 0; 4987 break; 4988 } 4989 4990 *end = '\0'; 4991 4992 /* 4993 * We want to have a name for the minor. In order to do this, 4994 * we need to walk the minor list from the devinfo. We want 4995 * to be sure that we don't infinitely walk a circular list, 4996 * so we check for circularity by sending a scout pointer 4997 * ahead two elements for every element that we iterate over; 4998 * if the list is circular, these will ultimately point to the 4999 * same element. You may recognize this little trick as the 5000 * answer to a stupid interview question -- one that always 5001 * seems to be asked by those who had to have it laboriously 5002 * explained to them, and who can't even concisely describe 5003 * the conditions under which one would be forced to resort to 5004 * this technique. Needless to say, those conditions are 5005 * found here -- and probably only here. Is this the only use 5006 * of this infamous trick in shipping, production code? If it 5007 * isn't, it probably should be... 5008 */ 5009 if (minor != -1) { 5010 uintptr_t maddr = dtrace_loadptr(daddr + 5011 offsetof(struct dev_info, devi_minor)); 5012 5013 uintptr_t next = offsetof(struct ddi_minor_data, next); 5014 uintptr_t name = offsetof(struct ddi_minor_data, 5015 d_minor) + offsetof(struct ddi_minor, name); 5016 uintptr_t dev = offsetof(struct ddi_minor_data, 5017 d_minor) + offsetof(struct ddi_minor, dev); 5018 uintptr_t scout; 5019 5020 if (maddr != NULL) 5021 scout = dtrace_loadptr(maddr + next); 5022 5023 while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { 5024 uint64_t m; 5025 #ifdef _LP64 5026 m = dtrace_load64(maddr + dev) & MAXMIN64; 5027 #else 5028 m = dtrace_load32(maddr + dev) & MAXMIN; 5029 #endif 5030 if (m != minor) { 5031 maddr = dtrace_loadptr(maddr + next); 5032 5033 if (scout == NULL) 5034 continue; 5035 5036 scout = dtrace_loadptr(scout + next); 5037 5038 if (scout == NULL) 5039 continue; 5040 5041 scout = dtrace_loadptr(scout + next); 5042 5043 if (scout == NULL) 5044 continue; 5045 5046 if (scout == maddr) { 5047 *flags |= CPU_DTRACE_ILLOP; 5048 break; 5049 } 5050 5051 continue; 5052 } 5053 5054 /* 5055 * We have the minor data. Now we need to 5056 * copy the minor's name into the end of the 5057 * pathname. 5058 */ 5059 s = (char *)dtrace_loadptr(maddr + name); 5060 len = dtrace_strlen(s, size); 5061 5062 if (*flags & CPU_DTRACE_FAULT) 5063 break; 5064 5065 if (len != 0) { 5066 if ((end -= (len + 1)) < start) 5067 break; 5068 5069 *end = ':'; 5070 } 5071 5072 for (i = 1; i <= len; i++) 5073 end[i] = dtrace_load8((uintptr_t)s++); 5074 break; 5075 } 5076 } 5077 5078 while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) { 5079 ddi_node_state_t devi_state; 5080 5081 devi_state = dtrace_load32(daddr + 5082 offsetof(struct dev_info, devi_node_state)); 5083 5084 if (*flags & CPU_DTRACE_FAULT) 5085 break; 5086 5087 if (devi_state >= DS_INITIALIZED) { 5088 s = (char *)dtrace_loadptr(daddr + 5089 offsetof(struct dev_info, devi_addr)); 5090 len = dtrace_strlen(s, size); 5091 5092 if (*flags & CPU_DTRACE_FAULT) 5093 break; 5094 5095 if (len != 0) { 5096 if ((end -= (len + 1)) < start) 5097 break; 5098 5099 *end = '@'; 5100 } 5101 5102 for (i = 1; i <= len; i++) 5103 end[i] = dtrace_load8((uintptr_t)s++); 5104 } 5105 5106 /* 5107 * Now for the node name... 5108 */ 5109 s = (char *)dtrace_loadptr(daddr + 5110 offsetof(struct dev_info, devi_node_name)); 5111 5112 daddr = dtrace_loadptr(daddr + 5113 offsetof(struct dev_info, devi_parent)); 5114 5115 /* 5116 * If our parent is NULL (that is, if we're the root 5117 * node), we're going to use the special path 5118 * "devices". 5119 */ 5120 if (daddr == 0) 5121 s = "devices"; 5122 5123 len = dtrace_strlen(s, size); 5124 if (*flags & CPU_DTRACE_FAULT) 5125 break; 5126 5127 if ((end -= (len + 1)) < start) 5128 break; 5129 5130 for (i = 1; i <= len; i++) 5131 end[i] = dtrace_load8((uintptr_t)s++); 5132 *end = '/'; 5133 5134 if (depth++ > dtrace_devdepth_max) { 5135 *flags |= CPU_DTRACE_ILLOP; 5136 break; 5137 } 5138 } 5139 5140 if (end < start) 5141 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5142 5143 if (daddr == 0) { 5144 regs[rd] = (uintptr_t)end; 5145 mstate->dtms_scratch_ptr += size; 5146 } 5147 5148 break; 5149 } 5150 #endif 5151 5152 case DIF_SUBR_STRJOIN: { 5153 char *d = (char *)mstate->dtms_scratch_ptr; 5154 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5155 uintptr_t s1 = tupregs[0].dttk_value; 5156 uintptr_t s2 = tupregs[1].dttk_value; 5157 int i = 0; 5158 5159 if (!dtrace_strcanload(s1, size, mstate, vstate) || 5160 !dtrace_strcanload(s2, size, mstate, vstate)) { 5161 regs[rd] = 0; 5162 break; 5163 } 5164 5165 if (!DTRACE_INSCRATCH(mstate, size)) { 5166 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5167 regs[rd] = 0; 5168 break; 5169 } 5170 5171 for (;;) { 5172 if (i >= size) { 5173 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5174 regs[rd] = 0; 5175 break; 5176 } 5177 5178 if ((d[i++] = dtrace_load8(s1++)) == '\0') { 5179 i--; 5180 break; 5181 } 5182 } 5183 5184 for (;;) { 5185 if (i >= size) { 5186 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5187 regs[rd] = 0; 5188 break; 5189 } 5190 5191 if ((d[i++] = dtrace_load8(s2++)) == '\0') 5192 break; 5193 } 5194 5195 if (i < size) { 5196 mstate->dtms_scratch_ptr += i; 5197 regs[rd] = (uintptr_t)d; 5198 } 5199 5200 break; 5201 } 5202 5203 case DIF_SUBR_STRTOLL: { 5204 uintptr_t s = tupregs[0].dttk_value; 5205 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5206 int base = 10; 5207 5208 if (nargs > 1) { 5209 if ((base = tupregs[1].dttk_value) <= 1 || 5210 base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { 5211 *flags |= CPU_DTRACE_ILLOP; 5212 break; 5213 } 5214 } 5215 5216 if (!dtrace_strcanload(s, size, mstate, vstate)) { 5217 regs[rd] = INT64_MIN; 5218 break; 5219 } 5220 5221 regs[rd] = dtrace_strtoll((char *)s, base, size); 5222 break; 5223 } 5224 5225 case DIF_SUBR_LLTOSTR: { 5226 int64_t i = (int64_t)tupregs[0].dttk_value; 5227 uint64_t val, digit; 5228 uint64_t size = 65; /* enough room for 2^64 in binary */ 5229 char *end = (char *)mstate->dtms_scratch_ptr + size - 1; 5230 int base = 10; 5231 5232 if (nargs > 1) { 5233 if ((base = tupregs[1].dttk_value) <= 1 || 5234 base > ('z' - 'a' + 1) + ('9' - '0' + 1)) { 5235 *flags |= CPU_DTRACE_ILLOP; 5236 break; 5237 } 5238 } 5239 5240 val = (base == 10 && i < 0) ? i * -1 : i; 5241 5242 if (!DTRACE_INSCRATCH(mstate, size)) { 5243 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5244 regs[rd] = 0; 5245 break; 5246 } 5247 5248 for (*end-- = '\0'; val; val /= base) { 5249 if ((digit = val % base) <= '9' - '0') { 5250 *end-- = '0' + digit; 5251 } else { 5252 *end-- = 'a' + (digit - ('9' - '0') - 1); 5253 } 5254 } 5255 5256 if (i == 0 && base == 16) 5257 *end-- = '0'; 5258 5259 if (base == 16) 5260 *end-- = 'x'; 5261 5262 if (i == 0 || base == 8 || base == 16) 5263 *end-- = '0'; 5264 5265 if (i < 0 && base == 10) 5266 *end-- = '-'; 5267 5268 regs[rd] = (uintptr_t)end + 1; 5269 mstate->dtms_scratch_ptr += size; 5270 break; 5271 } 5272 5273 case DIF_SUBR_HTONS: 5274 case DIF_SUBR_NTOHS: 5275 #if BYTE_ORDER == BIG_ENDIAN 5276 regs[rd] = (uint16_t)tupregs[0].dttk_value; 5277 #else 5278 regs[rd] = DT_BSWAP_16((uint16_t)tupregs[0].dttk_value); 5279 #endif 5280 break; 5281 5282 5283 case DIF_SUBR_HTONL: 5284 case DIF_SUBR_NTOHL: 5285 #if BYTE_ORDER == BIG_ENDIAN 5286 regs[rd] = (uint32_t)tupregs[0].dttk_value; 5287 #else 5288 regs[rd] = DT_BSWAP_32((uint32_t)tupregs[0].dttk_value); 5289 #endif 5290 break; 5291 5292 5293 case DIF_SUBR_HTONLL: 5294 case DIF_SUBR_NTOHLL: 5295 #if BYTE_ORDER == BIG_ENDIAN 5296 regs[rd] = (uint64_t)tupregs[0].dttk_value; 5297 #else 5298 regs[rd] = DT_BSWAP_64((uint64_t)tupregs[0].dttk_value); 5299 #endif 5300 break; 5301 5302 5303 case DIF_SUBR_DIRNAME: 5304 case DIF_SUBR_BASENAME: { 5305 char *dest = (char *)mstate->dtms_scratch_ptr; 5306 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5307 uintptr_t src = tupregs[0].dttk_value; 5308 int i, j, len = dtrace_strlen((char *)src, size); 5309 int lastbase = -1, firstbase = -1, lastdir = -1; 5310 int start, end; 5311 5312 if (!dtrace_canload(src, len + 1, mstate, vstate)) { 5313 regs[rd] = 0; 5314 break; 5315 } 5316 5317 if (!DTRACE_INSCRATCH(mstate, size)) { 5318 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5319 regs[rd] = 0; 5320 break; 5321 } 5322 5323 /* 5324 * The basename and dirname for a zero-length string is 5325 * defined to be "." 5326 */ 5327 if (len == 0) { 5328 len = 1; 5329 src = (uintptr_t)"."; 5330 } 5331 5332 /* 5333 * Start from the back of the string, moving back toward the 5334 * front until we see a character that isn't a slash. That 5335 * character is the last character in the basename. 5336 */ 5337 for (i = len - 1; i >= 0; i--) { 5338 if (dtrace_load8(src + i) != '/') 5339 break; 5340 } 5341 5342 if (i >= 0) 5343 lastbase = i; 5344 5345 /* 5346 * Starting from the last character in the basename, move 5347 * towards the front until we find a slash. The character 5348 * that we processed immediately before that is the first 5349 * character in the basename. 5350 */ 5351 for (; i >= 0; i--) { 5352 if (dtrace_load8(src + i) == '/') 5353 break; 5354 } 5355 5356 if (i >= 0) 5357 firstbase = i + 1; 5358 5359 /* 5360 * Now keep going until we find a non-slash character. That 5361 * character is the last character in the dirname. 5362 */ 5363 for (; i >= 0; i--) { 5364 if (dtrace_load8(src + i) != '/') 5365 break; 5366 } 5367 5368 if (i >= 0) 5369 lastdir = i; 5370 5371 ASSERT(!(lastbase == -1 && firstbase != -1)); 5372 ASSERT(!(firstbase == -1 && lastdir != -1)); 5373 5374 if (lastbase == -1) { 5375 /* 5376 * We didn't find a non-slash character. We know that 5377 * the length is non-zero, so the whole string must be 5378 * slashes. In either the dirname or the basename 5379 * case, we return '/'. 5380 */ 5381 ASSERT(firstbase == -1); 5382 firstbase = lastbase = lastdir = 0; 5383 } 5384 5385 if (firstbase == -1) { 5386 /* 5387 * The entire string consists only of a basename 5388 * component. If we're looking for dirname, we need 5389 * to change our string to be just "."; if we're 5390 * looking for a basename, we'll just set the first 5391 * character of the basename to be 0. 5392 */ 5393 if (subr == DIF_SUBR_DIRNAME) { 5394 ASSERT(lastdir == -1); 5395 src = (uintptr_t)"."; 5396 lastdir = 0; 5397 } else { 5398 firstbase = 0; 5399 } 5400 } 5401 5402 if (subr == DIF_SUBR_DIRNAME) { 5403 if (lastdir == -1) { 5404 /* 5405 * We know that we have a slash in the name -- 5406 * or lastdir would be set to 0, above. And 5407 * because lastdir is -1, we know that this 5408 * slash must be the first character. (That 5409 * is, the full string must be of the form 5410 * "/basename".) In this case, the last 5411 * character of the directory name is 0. 5412 */ 5413 lastdir = 0; 5414 } 5415 5416 start = 0; 5417 end = lastdir; 5418 } else { 5419 ASSERT(subr == DIF_SUBR_BASENAME); 5420 ASSERT(firstbase != -1 && lastbase != -1); 5421 start = firstbase; 5422 end = lastbase; 5423 } 5424 5425 for (i = start, j = 0; i <= end && j < size - 1; i++, j++) 5426 dest[j] = dtrace_load8(src + i); 5427 5428 dest[j] = '\0'; 5429 regs[rd] = (uintptr_t)dest; 5430 mstate->dtms_scratch_ptr += size; 5431 break; 5432 } 5433 5434 case DIF_SUBR_GETF: { 5435 uintptr_t fd = tupregs[0].dttk_value; 5436 struct filedesc *fdp; 5437 file_t *fp; 5438 5439 if (!dtrace_priv_proc(state)) { 5440 regs[rd] = 0; 5441 break; 5442 } 5443 fdp = curproc->p_fd; 5444 FILEDESC_SLOCK(fdp); 5445 fp = fget_locked(fdp, fd); 5446 mstate->dtms_getf = fp; 5447 regs[rd] = (uintptr_t)fp; 5448 FILEDESC_SUNLOCK(fdp); 5449 break; 5450 } 5451 5452 case DIF_SUBR_CLEANPATH: { 5453 char *dest = (char *)mstate->dtms_scratch_ptr, c; 5454 uint64_t size = state->dts_options[DTRACEOPT_STRSIZE]; 5455 uintptr_t src = tupregs[0].dttk_value; 5456 int i = 0, j = 0; 5457 #ifdef illumos 5458 zone_t *z; 5459 #endif 5460 5461 if (!dtrace_strcanload(src, size, mstate, vstate)) { 5462 regs[rd] = 0; 5463 break; 5464 } 5465 5466 if (!DTRACE_INSCRATCH(mstate, size)) { 5467 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5468 regs[rd] = 0; 5469 break; 5470 } 5471 5472 /* 5473 * Move forward, loading each character. 5474 */ 5475 do { 5476 c = dtrace_load8(src + i++); 5477 next: 5478 if (j + 5 >= size) /* 5 = strlen("/..c\0") */ 5479 break; 5480 5481 if (c != '/') { 5482 dest[j++] = c; 5483 continue; 5484 } 5485 5486 c = dtrace_load8(src + i++); 5487 5488 if (c == '/') { 5489 /* 5490 * We have two slashes -- we can just advance 5491 * to the next character. 5492 */ 5493 goto next; 5494 } 5495 5496 if (c != '.') { 5497 /* 5498 * This is not "." and it's not ".." -- we can 5499 * just store the "/" and this character and 5500 * drive on. 5501 */ 5502 dest[j++] = '/'; 5503 dest[j++] = c; 5504 continue; 5505 } 5506 5507 c = dtrace_load8(src + i++); 5508 5509 if (c == '/') { 5510 /* 5511 * This is a "/./" component. We're not going 5512 * to store anything in the destination buffer; 5513 * we're just going to go to the next component. 5514 */ 5515 goto next; 5516 } 5517 5518 if (c != '.') { 5519 /* 5520 * This is not ".." -- we can just store the 5521 * "/." and this character and continue 5522 * processing. 5523 */ 5524 dest[j++] = '/'; 5525 dest[j++] = '.'; 5526 dest[j++] = c; 5527 continue; 5528 } 5529 5530 c = dtrace_load8(src + i++); 5531 5532 if (c != '/' && c != '\0') { 5533 /* 5534 * This is not ".." -- it's "..[mumble]". 5535 * We'll store the "/.." and this character 5536 * and continue processing. 5537 */ 5538 dest[j++] = '/'; 5539 dest[j++] = '.'; 5540 dest[j++] = '.'; 5541 dest[j++] = c; 5542 continue; 5543 } 5544 5545 /* 5546 * This is "/../" or "/..\0". We need to back up 5547 * our destination pointer until we find a "/". 5548 */ 5549 i--; 5550 while (j != 0 && dest[--j] != '/') 5551 continue; 5552 5553 if (c == '\0') 5554 dest[++j] = '/'; 5555 } while (c != '\0'); 5556 5557 dest[j] = '\0'; 5558 5559 #ifdef illumos 5560 if (mstate->dtms_getf != NULL && 5561 !(mstate->dtms_access & DTRACE_ACCESS_KERNEL) && 5562 (z = state->dts_cred.dcr_cred->cr_zone) != kcred->cr_zone) { 5563 /* 5564 * If we've done a getf() as a part of this ECB and we 5565 * don't have kernel access (and we're not in the global 5566 * zone), check if the path we cleaned up begins with 5567 * the zone's root path, and trim it off if so. Note 5568 * that this is an output cleanliness issue, not a 5569 * security issue: knowing one's zone root path does 5570 * not enable privilege escalation. 5571 */ 5572 if (strstr(dest, z->zone_rootpath) == dest) 5573 dest += strlen(z->zone_rootpath) - 1; 5574 } 5575 #endif 5576 5577 regs[rd] = (uintptr_t)dest; 5578 mstate->dtms_scratch_ptr += size; 5579 break; 5580 } 5581 5582 case DIF_SUBR_INET_NTOA: 5583 case DIF_SUBR_INET_NTOA6: 5584 case DIF_SUBR_INET_NTOP: { 5585 size_t size; 5586 int af, argi, i; 5587 char *base, *end; 5588 5589 if (subr == DIF_SUBR_INET_NTOP) { 5590 af = (int)tupregs[0].dttk_value; 5591 argi = 1; 5592 } else { 5593 af = subr == DIF_SUBR_INET_NTOA ? AF_INET: AF_INET6; 5594 argi = 0; 5595 } 5596 5597 if (af == AF_INET) { 5598 ipaddr_t ip4; 5599 uint8_t *ptr8, val; 5600 5601 /* 5602 * Safely load the IPv4 address. 5603 */ 5604 ip4 = dtrace_load32(tupregs[argi].dttk_value); 5605 5606 /* 5607 * Check an IPv4 string will fit in scratch. 5608 */ 5609 size = INET_ADDRSTRLEN; 5610 if (!DTRACE_INSCRATCH(mstate, size)) { 5611 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5612 regs[rd] = 0; 5613 break; 5614 } 5615 base = (char *)mstate->dtms_scratch_ptr; 5616 end = (char *)mstate->dtms_scratch_ptr + size - 1; 5617 5618 /* 5619 * Stringify as a dotted decimal quad. 5620 */ 5621 *end-- = '\0'; 5622 ptr8 = (uint8_t *)&ip4; 5623 for (i = 3; i >= 0; i--) { 5624 val = ptr8[i]; 5625 5626 if (val == 0) { 5627 *end-- = '0'; 5628 } else { 5629 for (; val; val /= 10) { 5630 *end-- = '0' + (val % 10); 5631 } 5632 } 5633 5634 if (i > 0) 5635 *end-- = '.'; 5636 } 5637 ASSERT(end + 1 >= base); 5638 5639 } else if (af == AF_INET6) { 5640 struct in6_addr ip6; 5641 int firstzero, tryzero, numzero, v6end; 5642 uint16_t val; 5643 const char digits[] = "0123456789abcdef"; 5644 5645 /* 5646 * Stringify using RFC 1884 convention 2 - 16 bit 5647 * hexadecimal values with a zero-run compression. 5648 * Lower case hexadecimal digits are used. 5649 * eg, fe80::214:4fff:fe0b:76c8. 5650 * The IPv4 embedded form is returned for inet_ntop, 5651 * just the IPv4 string is returned for inet_ntoa6. 5652 */ 5653 5654 /* 5655 * Safely load the IPv6 address. 5656 */ 5657 dtrace_bcopy( 5658 (void *)(uintptr_t)tupregs[argi].dttk_value, 5659 (void *)(uintptr_t)&ip6, sizeof (struct in6_addr)); 5660 5661 /* 5662 * Check an IPv6 string will fit in scratch. 5663 */ 5664 size = INET6_ADDRSTRLEN; 5665 if (!DTRACE_INSCRATCH(mstate, size)) { 5666 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5667 regs[rd] = 0; 5668 break; 5669 } 5670 base = (char *)mstate->dtms_scratch_ptr; 5671 end = (char *)mstate->dtms_scratch_ptr + size - 1; 5672 *end-- = '\0'; 5673 5674 /* 5675 * Find the longest run of 16 bit zero values 5676 * for the single allowed zero compression - "::". 5677 */ 5678 firstzero = -1; 5679 tryzero = -1; 5680 numzero = 1; 5681 for (i = 0; i < sizeof (struct in6_addr); i++) { 5682 #ifdef illumos 5683 if (ip6._S6_un._S6_u8[i] == 0 && 5684 #else 5685 if (ip6.__u6_addr.__u6_addr8[i] == 0 && 5686 #endif 5687 tryzero == -1 && i % 2 == 0) { 5688 tryzero = i; 5689 continue; 5690 } 5691 5692 if (tryzero != -1 && 5693 #ifdef illumos 5694 (ip6._S6_un._S6_u8[i] != 0 || 5695 #else 5696 (ip6.__u6_addr.__u6_addr8[i] != 0 || 5697 #endif 5698 i == sizeof (struct in6_addr) - 1)) { 5699 5700 if (i - tryzero <= numzero) { 5701 tryzero = -1; 5702 continue; 5703 } 5704 5705 firstzero = tryzero; 5706 numzero = i - i % 2 - tryzero; 5707 tryzero = -1; 5708 5709 #ifdef illumos 5710 if (ip6._S6_un._S6_u8[i] == 0 && 5711 #else 5712 if (ip6.__u6_addr.__u6_addr8[i] == 0 && 5713 #endif 5714 i == sizeof (struct in6_addr) - 1) 5715 numzero += 2; 5716 } 5717 } 5718 ASSERT(firstzero + numzero <= sizeof (struct in6_addr)); 5719 5720 /* 5721 * Check for an IPv4 embedded address. 5722 */ 5723 v6end = sizeof (struct in6_addr) - 2; 5724 if (IN6_IS_ADDR_V4MAPPED(&ip6) || 5725 IN6_IS_ADDR_V4COMPAT(&ip6)) { 5726 for (i = sizeof (struct in6_addr) - 1; 5727 i >= DTRACE_V4MAPPED_OFFSET; i--) { 5728 ASSERT(end >= base); 5729 5730 #ifdef illumos 5731 val = ip6._S6_un._S6_u8[i]; 5732 #else 5733 val = ip6.__u6_addr.__u6_addr8[i]; 5734 #endif 5735 5736 if (val == 0) { 5737 *end-- = '0'; 5738 } else { 5739 for (; val; val /= 10) { 5740 *end-- = '0' + val % 10; 5741 } 5742 } 5743 5744 if (i > DTRACE_V4MAPPED_OFFSET) 5745 *end-- = '.'; 5746 } 5747 5748 if (subr == DIF_SUBR_INET_NTOA6) 5749 goto inetout; 5750 5751 /* 5752 * Set v6end to skip the IPv4 address that 5753 * we have already stringified. 5754 */ 5755 v6end = 10; 5756 } 5757 5758 /* 5759 * Build the IPv6 string by working through the 5760 * address in reverse. 5761 */ 5762 for (i = v6end; i >= 0; i -= 2) { 5763 ASSERT(end >= base); 5764 5765 if (i == firstzero + numzero - 2) { 5766 *end-- = ':'; 5767 *end-- = ':'; 5768 i -= numzero - 2; 5769 continue; 5770 } 5771 5772 if (i < 14 && i != firstzero - 2) 5773 *end-- = ':'; 5774 5775 #ifdef illumos 5776 val = (ip6._S6_un._S6_u8[i] << 8) + 5777 ip6._S6_un._S6_u8[i + 1]; 5778 #else 5779 val = (ip6.__u6_addr.__u6_addr8[i] << 8) + 5780 ip6.__u6_addr.__u6_addr8[i + 1]; 5781 #endif 5782 5783 if (val == 0) { 5784 *end-- = '0'; 5785 } else { 5786 for (; val; val /= 16) { 5787 *end-- = digits[val % 16]; 5788 } 5789 } 5790 } 5791 ASSERT(end + 1 >= base); 5792 5793 } else { 5794 /* 5795 * The user didn't use AH_INET or AH_INET6. 5796 */ 5797 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 5798 regs[rd] = 0; 5799 break; 5800 } 5801 5802 inetout: regs[rd] = (uintptr_t)end + 1; 5803 mstate->dtms_scratch_ptr += size; 5804 break; 5805 } 5806 5807 case DIF_SUBR_MEMREF: { 5808 uintptr_t size = 2 * sizeof(uintptr_t); 5809 uintptr_t *memref = (uintptr_t *) P2ROUNDUP(mstate->dtms_scratch_ptr, sizeof(uintptr_t)); 5810 size_t scratch_size = ((uintptr_t) memref - mstate->dtms_scratch_ptr) + size; 5811 5812 /* address and length */ 5813 memref[0] = tupregs[0].dttk_value; 5814 memref[1] = tupregs[1].dttk_value; 5815 5816 regs[rd] = (uintptr_t) memref; 5817 mstate->dtms_scratch_ptr += scratch_size; 5818 break; 5819 } 5820 5821 #ifndef illumos 5822 case DIF_SUBR_MEMSTR: { 5823 char *str = (char *)mstate->dtms_scratch_ptr; 5824 uintptr_t mem = tupregs[0].dttk_value; 5825 char c = tupregs[1].dttk_value; 5826 size_t size = tupregs[2].dttk_value; 5827 uint8_t n; 5828 int i; 5829 5830 regs[rd] = 0; 5831 5832 if (size == 0) 5833 break; 5834 5835 if (!dtrace_canload(mem, size - 1, mstate, vstate)) 5836 break; 5837 5838 if (!DTRACE_INSCRATCH(mstate, size)) { 5839 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 5840 break; 5841 } 5842 5843 if (dtrace_memstr_max != 0 && size > dtrace_memstr_max) { 5844 *flags |= CPU_DTRACE_ILLOP; 5845 break; 5846 } 5847 5848 for (i = 0; i < size - 1; i++) { 5849 n = dtrace_load8(mem++); 5850 str[i] = (n == 0) ? c : n; 5851 } 5852 str[size - 1] = 0; 5853 5854 regs[rd] = (uintptr_t)str; 5855 mstate->dtms_scratch_ptr += size; 5856 break; 5857 } 5858 #endif 5859 5860 case DIF_SUBR_TYPEREF: { 5861 uintptr_t size = 4 * sizeof(uintptr_t); 5862 uintptr_t *typeref = (uintptr_t *) P2ROUNDUP(mstate->dtms_scratch_ptr, sizeof(uintptr_t)); 5863 size_t scratch_size = ((uintptr_t) typeref - mstate->dtms_scratch_ptr) + size; 5864 5865 /* address, num_elements, type_str, type_len */ 5866 typeref[0] = tupregs[0].dttk_value; 5867 typeref[1] = tupregs[1].dttk_value; 5868 typeref[2] = tupregs[2].dttk_value; 5869 typeref[3] = tupregs[3].dttk_value; 5870 5871 regs[rd] = (uintptr_t) typeref; 5872 mstate->dtms_scratch_ptr += scratch_size; 5873 break; 5874 } 5875 } 5876 } 5877 5878 /* 5879 * Emulate the execution of DTrace IR instructions specified by the given 5880 * DIF object. This function is deliberately void of assertions as all of 5881 * the necessary checks are handled by a call to dtrace_difo_validate(). 5882 */ 5883 static uint64_t 5884 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate, 5885 dtrace_vstate_t *vstate, dtrace_state_t *state) 5886 { 5887 const dif_instr_t *text = difo->dtdo_buf; 5888 const uint_t textlen = difo->dtdo_len; 5889 const char *strtab = difo->dtdo_strtab; 5890 const uint64_t *inttab = difo->dtdo_inttab; 5891 5892 uint64_t rval = 0; 5893 dtrace_statvar_t *svar; 5894 dtrace_dstate_t *dstate = &vstate->dtvs_dynvars; 5895 dtrace_difv_t *v; 5896 volatile uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; 5897 volatile uintptr_t *illval = &cpu_core[curcpu].cpuc_dtrace_illval; 5898 5899 dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ 5900 uint64_t regs[DIF_DIR_NREGS]; 5901 uint64_t *tmp; 5902 5903 uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0; 5904 int64_t cc_r; 5905 uint_t pc = 0, id, opc = 0; 5906 uint8_t ttop = 0; 5907 dif_instr_t instr; 5908 uint_t r1, r2, rd; 5909 5910 /* 5911 * We stash the current DIF object into the machine state: we need it 5912 * for subsequent access checking. 5913 */ 5914 mstate->dtms_difo = difo; 5915 5916 regs[DIF_REG_R0] = 0; /* %r0 is fixed at zero */ 5917 5918 while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) { 5919 opc = pc; 5920 5921 instr = text[pc++]; 5922 r1 = DIF_INSTR_R1(instr); 5923 r2 = DIF_INSTR_R2(instr); 5924 rd = DIF_INSTR_RD(instr); 5925 5926 switch (DIF_INSTR_OP(instr)) { 5927 case DIF_OP_OR: 5928 regs[rd] = regs[r1] | regs[r2]; 5929 break; 5930 case DIF_OP_XOR: 5931 regs[rd] = regs[r1] ^ regs[r2]; 5932 break; 5933 case DIF_OP_AND: 5934 regs[rd] = regs[r1] & regs[r2]; 5935 break; 5936 case DIF_OP_SLL: 5937 regs[rd] = regs[r1] << regs[r2]; 5938 break; 5939 case DIF_OP_SRL: 5940 regs[rd] = regs[r1] >> regs[r2]; 5941 break; 5942 case DIF_OP_SUB: 5943 regs[rd] = regs[r1] - regs[r2]; 5944 break; 5945 case DIF_OP_ADD: 5946 regs[rd] = regs[r1] + regs[r2]; 5947 break; 5948 case DIF_OP_MUL: 5949 regs[rd] = regs[r1] * regs[r2]; 5950 break; 5951 case DIF_OP_SDIV: 5952 if (regs[r2] == 0) { 5953 regs[rd] = 0; 5954 *flags |= CPU_DTRACE_DIVZERO; 5955 } else { 5956 regs[rd] = (int64_t)regs[r1] / 5957 (int64_t)regs[r2]; 5958 } 5959 break; 5960 5961 case DIF_OP_UDIV: 5962 if (regs[r2] == 0) { 5963 regs[rd] = 0; 5964 *flags |= CPU_DTRACE_DIVZERO; 5965 } else { 5966 regs[rd] = regs[r1] / regs[r2]; 5967 } 5968 break; 5969 5970 case DIF_OP_SREM: 5971 if (regs[r2] == 0) { 5972 regs[rd] = 0; 5973 *flags |= CPU_DTRACE_DIVZERO; 5974 } else { 5975 regs[rd] = (int64_t)regs[r1] % 5976 (int64_t)regs[r2]; 5977 } 5978 break; 5979 5980 case DIF_OP_UREM: 5981 if (regs[r2] == 0) { 5982 regs[rd] = 0; 5983 *flags |= CPU_DTRACE_DIVZERO; 5984 } else { 5985 regs[rd] = regs[r1] % regs[r2]; 5986 } 5987 break; 5988 5989 case DIF_OP_NOT: 5990 regs[rd] = ~regs[r1]; 5991 break; 5992 case DIF_OP_MOV: 5993 regs[rd] = regs[r1]; 5994 break; 5995 case DIF_OP_CMP: 5996 cc_r = regs[r1] - regs[r2]; 5997 cc_n = cc_r < 0; 5998 cc_z = cc_r == 0; 5999 cc_v = 0; 6000 cc_c = regs[r1] < regs[r2]; 6001 break; 6002 case DIF_OP_TST: 6003 cc_n = cc_v = cc_c = 0; 6004 cc_z = regs[r1] == 0; 6005 break; 6006 case DIF_OP_BA: 6007 pc = DIF_INSTR_LABEL(instr); 6008 break; 6009 case DIF_OP_BE: 6010 if (cc_z) 6011 pc = DIF_INSTR_LABEL(instr); 6012 break; 6013 case DIF_OP_BNE: 6014 if (cc_z == 0) 6015 pc = DIF_INSTR_LABEL(instr); 6016 break; 6017 case DIF_OP_BG: 6018 if ((cc_z | (cc_n ^ cc_v)) == 0) 6019 pc = DIF_INSTR_LABEL(instr); 6020 break; 6021 case DIF_OP_BGU: 6022 if ((cc_c | cc_z) == 0) 6023 pc = DIF_INSTR_LABEL(instr); 6024 break; 6025 case DIF_OP_BGE: 6026 if ((cc_n ^ cc_v) == 0) 6027 pc = DIF_INSTR_LABEL(instr); 6028 break; 6029 case DIF_OP_BGEU: 6030 if (cc_c == 0) 6031 pc = DIF_INSTR_LABEL(instr); 6032 break; 6033 case DIF_OP_BL: 6034 if (cc_n ^ cc_v) 6035 pc = DIF_INSTR_LABEL(instr); 6036 break; 6037 case DIF_OP_BLU: 6038 if (cc_c) 6039 pc = DIF_INSTR_LABEL(instr); 6040 break; 6041 case DIF_OP_BLE: 6042 if (cc_z | (cc_n ^ cc_v)) 6043 pc = DIF_INSTR_LABEL(instr); 6044 break; 6045 case DIF_OP_BLEU: 6046 if (cc_c | cc_z) 6047 pc = DIF_INSTR_LABEL(instr); 6048 break; 6049 case DIF_OP_RLDSB: 6050 if (!dtrace_canload(regs[r1], 1, mstate, vstate)) 6051 break; 6052 /*FALLTHROUGH*/ 6053 case DIF_OP_LDSB: 6054 regs[rd] = (int8_t)dtrace_load8(regs[r1]); 6055 break; 6056 case DIF_OP_RLDSH: 6057 if (!dtrace_canload(regs[r1], 2, mstate, vstate)) 6058 break; 6059 /*FALLTHROUGH*/ 6060 case DIF_OP_LDSH: 6061 regs[rd] = (int16_t)dtrace_load16(regs[r1]); 6062 break; 6063 case DIF_OP_RLDSW: 6064 if (!dtrace_canload(regs[r1], 4, mstate, vstate)) 6065 break; 6066 /*FALLTHROUGH*/ 6067 case DIF_OP_LDSW: 6068 regs[rd] = (int32_t)dtrace_load32(regs[r1]); 6069 break; 6070 case DIF_OP_RLDUB: 6071 if (!dtrace_canload(regs[r1], 1, mstate, vstate)) 6072 break; 6073 /*FALLTHROUGH*/ 6074 case DIF_OP_LDUB: 6075 regs[rd] = dtrace_load8(regs[r1]); 6076 break; 6077 case DIF_OP_RLDUH: 6078 if (!dtrace_canload(regs[r1], 2, mstate, vstate)) 6079 break; 6080 /*FALLTHROUGH*/ 6081 case DIF_OP_LDUH: 6082 regs[rd] = dtrace_load16(regs[r1]); 6083 break; 6084 case DIF_OP_RLDUW: 6085 if (!dtrace_canload(regs[r1], 4, mstate, vstate)) 6086 break; 6087 /*FALLTHROUGH*/ 6088 case DIF_OP_LDUW: 6089 regs[rd] = dtrace_load32(regs[r1]); 6090 break; 6091 case DIF_OP_RLDX: 6092 if (!dtrace_canload(regs[r1], 8, mstate, vstate)) 6093 break; 6094 /*FALLTHROUGH*/ 6095 case DIF_OP_LDX: 6096 regs[rd] = dtrace_load64(regs[r1]); 6097 break; 6098 case DIF_OP_ULDSB: 6099 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6100 regs[rd] = (int8_t) 6101 dtrace_fuword8((void *)(uintptr_t)regs[r1]); 6102 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6103 break; 6104 case DIF_OP_ULDSH: 6105 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6106 regs[rd] = (int16_t) 6107 dtrace_fuword16((void *)(uintptr_t)regs[r1]); 6108 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6109 break; 6110 case DIF_OP_ULDSW: 6111 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6112 regs[rd] = (int32_t) 6113 dtrace_fuword32((void *)(uintptr_t)regs[r1]); 6114 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6115 break; 6116 case DIF_OP_ULDUB: 6117 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6118 regs[rd] = 6119 dtrace_fuword8((void *)(uintptr_t)regs[r1]); 6120 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6121 break; 6122 case DIF_OP_ULDUH: 6123 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6124 regs[rd] = 6125 dtrace_fuword16((void *)(uintptr_t)regs[r1]); 6126 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6127 break; 6128 case DIF_OP_ULDUW: 6129 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6130 regs[rd] = 6131 dtrace_fuword32((void *)(uintptr_t)regs[r1]); 6132 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6133 break; 6134 case DIF_OP_ULDX: 6135 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6136 regs[rd] = 6137 dtrace_fuword64((void *)(uintptr_t)regs[r1]); 6138 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6139 break; 6140 case DIF_OP_RET: 6141 rval = regs[rd]; 6142 pc = textlen; 6143 break; 6144 case DIF_OP_NOP: 6145 break; 6146 case DIF_OP_SETX: 6147 regs[rd] = inttab[DIF_INSTR_INTEGER(instr)]; 6148 break; 6149 case DIF_OP_SETS: 6150 regs[rd] = (uint64_t)(uintptr_t) 6151 (strtab + DIF_INSTR_STRING(instr)); 6152 break; 6153 case DIF_OP_SCMP: { 6154 size_t sz = state->dts_options[DTRACEOPT_STRSIZE]; 6155 uintptr_t s1 = regs[r1]; 6156 uintptr_t s2 = regs[r2]; 6157 6158 if (s1 != 0 && 6159 !dtrace_strcanload(s1, sz, mstate, vstate)) 6160 break; 6161 if (s2 != 0 && 6162 !dtrace_strcanload(s2, sz, mstate, vstate)) 6163 break; 6164 6165 cc_r = dtrace_strncmp((char *)s1, (char *)s2, sz); 6166 6167 cc_n = cc_r < 0; 6168 cc_z = cc_r == 0; 6169 cc_v = cc_c = 0; 6170 break; 6171 } 6172 case DIF_OP_LDGA: 6173 regs[rd] = dtrace_dif_variable(mstate, state, 6174 r1, regs[r2]); 6175 break; 6176 case DIF_OP_LDGS: 6177 id = DIF_INSTR_VAR(instr); 6178 6179 if (id >= DIF_VAR_OTHER_UBASE) { 6180 uintptr_t a; 6181 6182 id -= DIF_VAR_OTHER_UBASE; 6183 svar = vstate->dtvs_globals[id]; 6184 ASSERT(svar != NULL); 6185 v = &svar->dtsv_var; 6186 6187 if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) { 6188 regs[rd] = svar->dtsv_data; 6189 break; 6190 } 6191 6192 a = (uintptr_t)svar->dtsv_data; 6193 6194 if (*(uint8_t *)a == UINT8_MAX) { 6195 /* 6196 * If the 0th byte is set to UINT8_MAX 6197 * then this is to be treated as a 6198 * reference to a NULL variable. 6199 */ 6200 regs[rd] = 0; 6201 } else { 6202 regs[rd] = a + sizeof (uint64_t); 6203 } 6204 6205 break; 6206 } 6207 6208 regs[rd] = dtrace_dif_variable(mstate, state, id, 0); 6209 break; 6210 6211 case DIF_OP_STGS: 6212 id = DIF_INSTR_VAR(instr); 6213 6214 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6215 id -= DIF_VAR_OTHER_UBASE; 6216 6217 svar = vstate->dtvs_globals[id]; 6218 ASSERT(svar != NULL); 6219 v = &svar->dtsv_var; 6220 6221 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6222 uintptr_t a = (uintptr_t)svar->dtsv_data; 6223 6224 ASSERT(a != 0); 6225 ASSERT(svar->dtsv_size != 0); 6226 6227 if (regs[rd] == 0) { 6228 *(uint8_t *)a = UINT8_MAX; 6229 break; 6230 } else { 6231 *(uint8_t *)a = 0; 6232 a += sizeof (uint64_t); 6233 } 6234 if (!dtrace_vcanload( 6235 (void *)(uintptr_t)regs[rd], &v->dtdv_type, 6236 mstate, vstate)) 6237 break; 6238 6239 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6240 (void *)a, &v->dtdv_type); 6241 break; 6242 } 6243 6244 svar->dtsv_data = regs[rd]; 6245 break; 6246 6247 case DIF_OP_LDTA: 6248 /* 6249 * There are no DTrace built-in thread-local arrays at 6250 * present. This opcode is saved for future work. 6251 */ 6252 *flags |= CPU_DTRACE_ILLOP; 6253 regs[rd] = 0; 6254 break; 6255 6256 case DIF_OP_LDLS: 6257 id = DIF_INSTR_VAR(instr); 6258 6259 if (id < DIF_VAR_OTHER_UBASE) { 6260 /* 6261 * For now, this has no meaning. 6262 */ 6263 regs[rd] = 0; 6264 break; 6265 } 6266 6267 id -= DIF_VAR_OTHER_UBASE; 6268 6269 ASSERT(id < vstate->dtvs_nlocals); 6270 ASSERT(vstate->dtvs_locals != NULL); 6271 6272 svar = vstate->dtvs_locals[id]; 6273 ASSERT(svar != NULL); 6274 v = &svar->dtsv_var; 6275 6276 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6277 uintptr_t a = (uintptr_t)svar->dtsv_data; 6278 size_t sz = v->dtdv_type.dtdt_size; 6279 6280 sz += sizeof (uint64_t); 6281 ASSERT(svar->dtsv_size == NCPU * sz); 6282 a += curcpu * sz; 6283 6284 if (*(uint8_t *)a == UINT8_MAX) { 6285 /* 6286 * If the 0th byte is set to UINT8_MAX 6287 * then this is to be treated as a 6288 * reference to a NULL variable. 6289 */ 6290 regs[rd] = 0; 6291 } else { 6292 regs[rd] = a + sizeof (uint64_t); 6293 } 6294 6295 break; 6296 } 6297 6298 ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t)); 6299 tmp = (uint64_t *)(uintptr_t)svar->dtsv_data; 6300 regs[rd] = tmp[curcpu]; 6301 break; 6302 6303 case DIF_OP_STLS: 6304 id = DIF_INSTR_VAR(instr); 6305 6306 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6307 id -= DIF_VAR_OTHER_UBASE; 6308 ASSERT(id < vstate->dtvs_nlocals); 6309 6310 ASSERT(vstate->dtvs_locals != NULL); 6311 svar = vstate->dtvs_locals[id]; 6312 ASSERT(svar != NULL); 6313 v = &svar->dtsv_var; 6314 6315 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6316 uintptr_t a = (uintptr_t)svar->dtsv_data; 6317 size_t sz = v->dtdv_type.dtdt_size; 6318 6319 sz += sizeof (uint64_t); 6320 ASSERT(svar->dtsv_size == NCPU * sz); 6321 a += curcpu * sz; 6322 6323 if (regs[rd] == 0) { 6324 *(uint8_t *)a = UINT8_MAX; 6325 break; 6326 } else { 6327 *(uint8_t *)a = 0; 6328 a += sizeof (uint64_t); 6329 } 6330 6331 if (!dtrace_vcanload( 6332 (void *)(uintptr_t)regs[rd], &v->dtdv_type, 6333 mstate, vstate)) 6334 break; 6335 6336 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6337 (void *)a, &v->dtdv_type); 6338 break; 6339 } 6340 6341 ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t)); 6342 tmp = (uint64_t *)(uintptr_t)svar->dtsv_data; 6343 tmp[curcpu] = regs[rd]; 6344 break; 6345 6346 case DIF_OP_LDTS: { 6347 dtrace_dynvar_t *dvar; 6348 dtrace_key_t *key; 6349 6350 id = DIF_INSTR_VAR(instr); 6351 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6352 id -= DIF_VAR_OTHER_UBASE; 6353 v = &vstate->dtvs_tlocals[id]; 6354 6355 key = &tupregs[DIF_DTR_NREGS]; 6356 key[0].dttk_value = (uint64_t)id; 6357 key[0].dttk_size = 0; 6358 DTRACE_TLS_THRKEY(key[1].dttk_value); 6359 key[1].dttk_size = 0; 6360 6361 dvar = dtrace_dynvar(dstate, 2, key, 6362 sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC, 6363 mstate, vstate); 6364 6365 if (dvar == NULL) { 6366 regs[rd] = 0; 6367 break; 6368 } 6369 6370 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6371 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data; 6372 } else { 6373 regs[rd] = *((uint64_t *)dvar->dtdv_data); 6374 } 6375 6376 break; 6377 } 6378 6379 case DIF_OP_STTS: { 6380 dtrace_dynvar_t *dvar; 6381 dtrace_key_t *key; 6382 6383 id = DIF_INSTR_VAR(instr); 6384 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6385 id -= DIF_VAR_OTHER_UBASE; 6386 6387 key = &tupregs[DIF_DTR_NREGS]; 6388 key[0].dttk_value = (uint64_t)id; 6389 key[0].dttk_size = 0; 6390 DTRACE_TLS_THRKEY(key[1].dttk_value); 6391 key[1].dttk_size = 0; 6392 v = &vstate->dtvs_tlocals[id]; 6393 6394 dvar = dtrace_dynvar(dstate, 2, key, 6395 v->dtdv_type.dtdt_size > sizeof (uint64_t) ? 6396 v->dtdv_type.dtdt_size : sizeof (uint64_t), 6397 regs[rd] ? DTRACE_DYNVAR_ALLOC : 6398 DTRACE_DYNVAR_DEALLOC, mstate, vstate); 6399 6400 /* 6401 * Given that we're storing to thread-local data, 6402 * we need to flush our predicate cache. 6403 */ 6404 curthread->t_predcache = 0; 6405 6406 if (dvar == NULL) 6407 break; 6408 6409 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6410 if (!dtrace_vcanload( 6411 (void *)(uintptr_t)regs[rd], 6412 &v->dtdv_type, mstate, vstate)) 6413 break; 6414 6415 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6416 dvar->dtdv_data, &v->dtdv_type); 6417 } else { 6418 *((uint64_t *)dvar->dtdv_data) = regs[rd]; 6419 } 6420 6421 break; 6422 } 6423 6424 case DIF_OP_SRA: 6425 regs[rd] = (int64_t)regs[r1] >> regs[r2]; 6426 break; 6427 6428 case DIF_OP_CALL: 6429 dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd, 6430 regs, tupregs, ttop, mstate, state); 6431 break; 6432 6433 case DIF_OP_PUSHTR: 6434 if (ttop == DIF_DTR_NREGS) { 6435 *flags |= CPU_DTRACE_TUPOFLOW; 6436 break; 6437 } 6438 6439 if (r1 == DIF_TYPE_STRING) { 6440 /* 6441 * If this is a string type and the size is 0, 6442 * we'll use the system-wide default string 6443 * size. Note that we are _not_ looking at 6444 * the value of the DTRACEOPT_STRSIZE option; 6445 * had this been set, we would expect to have 6446 * a non-zero size value in the "pushtr". 6447 */ 6448 tupregs[ttop].dttk_size = 6449 dtrace_strlen((char *)(uintptr_t)regs[rd], 6450 regs[r2] ? regs[r2] : 6451 dtrace_strsize_default) + 1; 6452 } else { 6453 tupregs[ttop].dttk_size = regs[r2]; 6454 } 6455 6456 tupregs[ttop++].dttk_value = regs[rd]; 6457 break; 6458 6459 case DIF_OP_PUSHTV: 6460 if (ttop == DIF_DTR_NREGS) { 6461 *flags |= CPU_DTRACE_TUPOFLOW; 6462 break; 6463 } 6464 6465 tupregs[ttop].dttk_value = regs[rd]; 6466 tupregs[ttop++].dttk_size = 0; 6467 break; 6468 6469 case DIF_OP_POPTS: 6470 if (ttop != 0) 6471 ttop--; 6472 break; 6473 6474 case DIF_OP_FLUSHTS: 6475 ttop = 0; 6476 break; 6477 6478 case DIF_OP_LDGAA: 6479 case DIF_OP_LDTAA: { 6480 dtrace_dynvar_t *dvar; 6481 dtrace_key_t *key = tupregs; 6482 uint_t nkeys = ttop; 6483 6484 id = DIF_INSTR_VAR(instr); 6485 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6486 id -= DIF_VAR_OTHER_UBASE; 6487 6488 key[nkeys].dttk_value = (uint64_t)id; 6489 key[nkeys++].dttk_size = 0; 6490 6491 if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) { 6492 DTRACE_TLS_THRKEY(key[nkeys].dttk_value); 6493 key[nkeys++].dttk_size = 0; 6494 v = &vstate->dtvs_tlocals[id]; 6495 } else { 6496 v = &vstate->dtvs_globals[id]->dtsv_var; 6497 } 6498 6499 dvar = dtrace_dynvar(dstate, nkeys, key, 6500 v->dtdv_type.dtdt_size > sizeof (uint64_t) ? 6501 v->dtdv_type.dtdt_size : sizeof (uint64_t), 6502 DTRACE_DYNVAR_NOALLOC, mstate, vstate); 6503 6504 if (dvar == NULL) { 6505 regs[rd] = 0; 6506 break; 6507 } 6508 6509 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6510 regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data; 6511 } else { 6512 regs[rd] = *((uint64_t *)dvar->dtdv_data); 6513 } 6514 6515 break; 6516 } 6517 6518 case DIF_OP_STGAA: 6519 case DIF_OP_STTAA: { 6520 dtrace_dynvar_t *dvar; 6521 dtrace_key_t *key = tupregs; 6522 uint_t nkeys = ttop; 6523 6524 id = DIF_INSTR_VAR(instr); 6525 ASSERT(id >= DIF_VAR_OTHER_UBASE); 6526 id -= DIF_VAR_OTHER_UBASE; 6527 6528 key[nkeys].dttk_value = (uint64_t)id; 6529 key[nkeys++].dttk_size = 0; 6530 6531 if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) { 6532 DTRACE_TLS_THRKEY(key[nkeys].dttk_value); 6533 key[nkeys++].dttk_size = 0; 6534 v = &vstate->dtvs_tlocals[id]; 6535 } else { 6536 v = &vstate->dtvs_globals[id]->dtsv_var; 6537 } 6538 6539 dvar = dtrace_dynvar(dstate, nkeys, key, 6540 v->dtdv_type.dtdt_size > sizeof (uint64_t) ? 6541 v->dtdv_type.dtdt_size : sizeof (uint64_t), 6542 regs[rd] ? DTRACE_DYNVAR_ALLOC : 6543 DTRACE_DYNVAR_DEALLOC, mstate, vstate); 6544 6545 if (dvar == NULL) 6546 break; 6547 6548 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) { 6549 if (!dtrace_vcanload( 6550 (void *)(uintptr_t)regs[rd], &v->dtdv_type, 6551 mstate, vstate)) 6552 break; 6553 6554 dtrace_vcopy((void *)(uintptr_t)regs[rd], 6555 dvar->dtdv_data, &v->dtdv_type); 6556 } else { 6557 *((uint64_t *)dvar->dtdv_data) = regs[rd]; 6558 } 6559 6560 break; 6561 } 6562 6563 case DIF_OP_ALLOCS: { 6564 uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8); 6565 size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1]; 6566 6567 /* 6568 * Rounding up the user allocation size could have 6569 * overflowed large, bogus allocations (like -1ULL) to 6570 * 0. 6571 */ 6572 if (size < regs[r1] || 6573 !DTRACE_INSCRATCH(mstate, size)) { 6574 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 6575 regs[rd] = 0; 6576 break; 6577 } 6578 6579 dtrace_bzero((void *) mstate->dtms_scratch_ptr, size); 6580 mstate->dtms_scratch_ptr += size; 6581 regs[rd] = ptr; 6582 break; 6583 } 6584 6585 case DIF_OP_COPYS: 6586 if (!dtrace_canstore(regs[rd], regs[r2], 6587 mstate, vstate)) { 6588 *flags |= CPU_DTRACE_BADADDR; 6589 *illval = regs[rd]; 6590 break; 6591 } 6592 6593 if (!dtrace_canload(regs[r1], regs[r2], mstate, vstate)) 6594 break; 6595 6596 dtrace_bcopy((void *)(uintptr_t)regs[r1], 6597 (void *)(uintptr_t)regs[rd], (size_t)regs[r2]); 6598 break; 6599 6600 case DIF_OP_STB: 6601 if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) { 6602 *flags |= CPU_DTRACE_BADADDR; 6603 *illval = regs[rd]; 6604 break; 6605 } 6606 *((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1]; 6607 break; 6608 6609 case DIF_OP_STH: 6610 if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) { 6611 *flags |= CPU_DTRACE_BADADDR; 6612 *illval = regs[rd]; 6613 break; 6614 } 6615 if (regs[rd] & 1) { 6616 *flags |= CPU_DTRACE_BADALIGN; 6617 *illval = regs[rd]; 6618 break; 6619 } 6620 *((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1]; 6621 break; 6622 6623 case DIF_OP_STW: 6624 if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) { 6625 *flags |= CPU_DTRACE_BADADDR; 6626 *illval = regs[rd]; 6627 break; 6628 } 6629 if (regs[rd] & 3) { 6630 *flags |= CPU_DTRACE_BADALIGN; 6631 *illval = regs[rd]; 6632 break; 6633 } 6634 *((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1]; 6635 break; 6636 6637 case DIF_OP_STX: 6638 if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) { 6639 *flags |= CPU_DTRACE_BADADDR; 6640 *illval = regs[rd]; 6641 break; 6642 } 6643 if (regs[rd] & 7) { 6644 *flags |= CPU_DTRACE_BADALIGN; 6645 *illval = regs[rd]; 6646 break; 6647 } 6648 *((uint64_t *)(uintptr_t)regs[rd]) = regs[r1]; 6649 break; 6650 } 6651 } 6652 6653 if (!(*flags & CPU_DTRACE_FAULT)) 6654 return (rval); 6655 6656 mstate->dtms_fltoffs = opc * sizeof (dif_instr_t); 6657 mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS; 6658 6659 return (0); 6660 } 6661 6662 static void 6663 dtrace_action_breakpoint(dtrace_ecb_t *ecb) 6664 { 6665 dtrace_probe_t *probe = ecb->dte_probe; 6666 dtrace_provider_t *prov = probe->dtpr_provider; 6667 char c[DTRACE_FULLNAMELEN + 80], *str; 6668 char *msg = "dtrace: breakpoint action at probe "; 6669 char *ecbmsg = " (ecb "; 6670 uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4)); 6671 uintptr_t val = (uintptr_t)ecb; 6672 int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0; 6673 6674 if (dtrace_destructive_disallow) 6675 return; 6676 6677 /* 6678 * It's impossible to be taking action on the NULL probe. 6679 */ 6680 ASSERT(probe != NULL); 6681 6682 /* 6683 * This is a poor man's (destitute man's?) sprintf(): we want to 6684 * print the provider name, module name, function name and name of 6685 * the probe, along with the hex address of the ECB with the breakpoint 6686 * action -- all of which we must place in the character buffer by 6687 * hand. 6688 */ 6689 while (*msg != '\0') 6690 c[i++] = *msg++; 6691 6692 for (str = prov->dtpv_name; *str != '\0'; str++) 6693 c[i++] = *str; 6694 c[i++] = ':'; 6695 6696 for (str = probe->dtpr_mod; *str != '\0'; str++) 6697 c[i++] = *str; 6698 c[i++] = ':'; 6699 6700 for (str = probe->dtpr_func; *str != '\0'; str++) 6701 c[i++] = *str; 6702 c[i++] = ':'; 6703 6704 for (str = probe->dtpr_name; *str != '\0'; str++) 6705 c[i++] = *str; 6706 6707 while (*ecbmsg != '\0') 6708 c[i++] = *ecbmsg++; 6709 6710 while (shift >= 0) { 6711 mask = (uintptr_t)0xf << shift; 6712 6713 if (val >= ((uintptr_t)1 << shift)) 6714 c[i++] = "0123456789abcdef"[(val & mask) >> shift]; 6715 shift -= 4; 6716 } 6717 6718 c[i++] = ')'; 6719 c[i] = '\0'; 6720 6721 #ifdef illumos 6722 debug_enter(c); 6723 #else 6724 kdb_enter(KDB_WHY_DTRACE, "breakpoint action"); 6725 #endif 6726 } 6727 6728 static void 6729 dtrace_action_panic(dtrace_ecb_t *ecb) 6730 { 6731 dtrace_probe_t *probe = ecb->dte_probe; 6732 6733 /* 6734 * It's impossible to be taking action on the NULL probe. 6735 */ 6736 ASSERT(probe != NULL); 6737 6738 if (dtrace_destructive_disallow) 6739 return; 6740 6741 if (dtrace_panicked != NULL) 6742 return; 6743 6744 if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL) 6745 return; 6746 6747 /* 6748 * We won the right to panic. (We want to be sure that only one 6749 * thread calls panic() from dtrace_probe(), and that panic() is 6750 * called exactly once.) 6751 */ 6752 dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)", 6753 probe->dtpr_provider->dtpv_name, probe->dtpr_mod, 6754 probe->dtpr_func, probe->dtpr_name, (void *)ecb); 6755 } 6756 6757 static void 6758 dtrace_action_raise(uint64_t sig) 6759 { 6760 if (dtrace_destructive_disallow) 6761 return; 6762 6763 if (sig >= NSIG) { 6764 DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP); 6765 return; 6766 } 6767 6768 #ifdef illumos 6769 /* 6770 * raise() has a queue depth of 1 -- we ignore all subsequent 6771 * invocations of the raise() action. 6772 */ 6773 if (curthread->t_dtrace_sig == 0) 6774 curthread->t_dtrace_sig = (uint8_t)sig; 6775 6776 curthread->t_sig_check = 1; 6777 aston(curthread); 6778 #else 6779 struct proc *p = curproc; 6780 PROC_LOCK(p); 6781 kern_psignal(p, sig); 6782 PROC_UNLOCK(p); 6783 #endif 6784 } 6785 6786 static void 6787 dtrace_action_stop(void) 6788 { 6789 if (dtrace_destructive_disallow) 6790 return; 6791 6792 #ifdef illumos 6793 if (!curthread->t_dtrace_stop) { 6794 curthread->t_dtrace_stop = 1; 6795 curthread->t_sig_check = 1; 6796 aston(curthread); 6797 } 6798 #else 6799 struct proc *p = curproc; 6800 PROC_LOCK(p); 6801 kern_psignal(p, SIGSTOP); 6802 PROC_UNLOCK(p); 6803 #endif 6804 } 6805 6806 static void 6807 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val) 6808 { 6809 hrtime_t now; 6810 volatile uint16_t *flags; 6811 #ifdef illumos 6812 cpu_t *cpu = CPU; 6813 #else 6814 cpu_t *cpu = &solaris_cpu[curcpu]; 6815 #endif 6816 6817 if (dtrace_destructive_disallow) 6818 return; 6819 6820 flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; 6821 6822 now = dtrace_gethrtime(); 6823 6824 if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) { 6825 /* 6826 * We need to advance the mark to the current time. 6827 */ 6828 cpu->cpu_dtrace_chillmark = now; 6829 cpu->cpu_dtrace_chilled = 0; 6830 } 6831 6832 /* 6833 * Now check to see if the requested chill time would take us over 6834 * the maximum amount of time allowed in the chill interval. (Or 6835 * worse, if the calculation itself induces overflow.) 6836 */ 6837 if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max || 6838 cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) { 6839 *flags |= CPU_DTRACE_ILLOP; 6840 return; 6841 } 6842 6843 while (dtrace_gethrtime() - now < val) 6844 continue; 6845 6846 /* 6847 * Normally, we assure that the value of the variable "timestamp" does 6848 * not change within an ECB. The presence of chill() represents an 6849 * exception to this rule, however. 6850 */ 6851 mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP; 6852 cpu->cpu_dtrace_chilled += val; 6853 } 6854 6855 static void 6856 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state, 6857 uint64_t *buf, uint64_t arg) 6858 { 6859 int nframes = DTRACE_USTACK_NFRAMES(arg); 6860 int strsize = DTRACE_USTACK_STRSIZE(arg); 6861 uint64_t *pcs = &buf[1], *fps; 6862 char *str = (char *)&pcs[nframes]; 6863 int size, offs = 0, i, j; 6864 uintptr_t old = mstate->dtms_scratch_ptr, saved; 6865 uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; 6866 char *sym; 6867 6868 /* 6869 * Should be taking a faster path if string space has not been 6870 * allocated. 6871 */ 6872 ASSERT(strsize != 0); 6873 6874 /* 6875 * We will first allocate some temporary space for the frame pointers. 6876 */ 6877 fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8); 6878 size = (uintptr_t)fps - mstate->dtms_scratch_ptr + 6879 (nframes * sizeof (uint64_t)); 6880 6881 if (!DTRACE_INSCRATCH(mstate, size)) { 6882 /* 6883 * Not enough room for our frame pointers -- need to indicate 6884 * that we ran out of scratch space. 6885 */ 6886 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH); 6887 return; 6888 } 6889 6890 mstate->dtms_scratch_ptr += size; 6891 saved = mstate->dtms_scratch_ptr; 6892 6893 /* 6894 * Now get a stack with both program counters and frame pointers. 6895 */ 6896 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6897 dtrace_getufpstack(buf, fps, nframes + 1); 6898 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6899 6900 /* 6901 * If that faulted, we're cooked. 6902 */ 6903 if (*flags & CPU_DTRACE_FAULT) 6904 goto out; 6905 6906 /* 6907 * Now we want to walk up the stack, calling the USTACK helper. For 6908 * each iteration, we restore the scratch pointer. 6909 */ 6910 for (i = 0; i < nframes; i++) { 6911 mstate->dtms_scratch_ptr = saved; 6912 6913 if (offs >= strsize) 6914 break; 6915 6916 sym = (char *)(uintptr_t)dtrace_helper( 6917 DTRACE_HELPER_ACTION_USTACK, 6918 mstate, state, pcs[i], fps[i]); 6919 6920 /* 6921 * If we faulted while running the helper, we're going to 6922 * clear the fault and null out the corresponding string. 6923 */ 6924 if (*flags & CPU_DTRACE_FAULT) { 6925 *flags &= ~CPU_DTRACE_FAULT; 6926 str[offs++] = '\0'; 6927 continue; 6928 } 6929 6930 if (sym == NULL) { 6931 str[offs++] = '\0'; 6932 continue; 6933 } 6934 6935 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6936 6937 /* 6938 * Now copy in the string that the helper returned to us. 6939 */ 6940 for (j = 0; offs + j < strsize; j++) { 6941 if ((str[offs + j] = sym[j]) == '\0') 6942 break; 6943 } 6944 6945 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6946 6947 offs += j + 1; 6948 } 6949 6950 if (offs >= strsize) { 6951 /* 6952 * If we didn't have room for all of the strings, we don't 6953 * abort processing -- this needn't be a fatal error -- but we 6954 * still want to increment a counter (dts_stkstroverflows) to 6955 * allow this condition to be warned about. (If this is from 6956 * a jstack() action, it is easily tuned via jstackstrsize.) 6957 */ 6958 dtrace_error(&state->dts_stkstroverflows); 6959 } 6960 6961 while (offs < strsize) 6962 str[offs++] = '\0'; 6963 6964 out: 6965 mstate->dtms_scratch_ptr = old; 6966 } 6967 6968 static void 6969 dtrace_store_by_ref(dtrace_difo_t *dp, caddr_t tomax, size_t size, 6970 size_t *valoffsp, uint64_t *valp, uint64_t end, int intuple, int dtkind) 6971 { 6972 volatile uint16_t *flags; 6973 uint64_t val = *valp; 6974 size_t valoffs = *valoffsp; 6975 6976 flags = (volatile uint16_t *)&cpu_core[curcpu].cpuc_dtrace_flags; 6977 ASSERT(dtkind == DIF_TF_BYREF || dtkind == DIF_TF_BYUREF); 6978 6979 /* 6980 * If this is a string, we're going to only load until we find the zero 6981 * byte -- after which we'll store zero bytes. 6982 */ 6983 if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) { 6984 char c = '\0' + 1; 6985 size_t s; 6986 6987 for (s = 0; s < size; s++) { 6988 if (c != '\0' && dtkind == DIF_TF_BYREF) { 6989 c = dtrace_load8(val++); 6990 } else if (c != '\0' && dtkind == DIF_TF_BYUREF) { 6991 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 6992 c = dtrace_fuword8((void *)(uintptr_t)val++); 6993 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 6994 if (*flags & CPU_DTRACE_FAULT) 6995 break; 6996 } 6997 6998 DTRACE_STORE(uint8_t, tomax, valoffs++, c); 6999 7000 if (c == '\0' && intuple) 7001 break; 7002 } 7003 } else { 7004 uint8_t c; 7005 while (valoffs < end) { 7006 if (dtkind == DIF_TF_BYREF) { 7007 c = dtrace_load8(val++); 7008 } else if (dtkind == DIF_TF_BYUREF) { 7009 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 7010 c = dtrace_fuword8((void *)(uintptr_t)val++); 7011 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 7012 if (*flags & CPU_DTRACE_FAULT) 7013 break; 7014 } 7015 7016 DTRACE_STORE(uint8_t, tomax, 7017 valoffs++, c); 7018 } 7019 } 7020 7021 *valp = val; 7022 *valoffsp = valoffs; 7023 } 7024 7025 /* 7026 * If you're looking for the epicenter of DTrace, you just found it. This 7027 * is the function called by the provider to fire a probe -- from which all 7028 * subsequent probe-context DTrace activity emanates. 7029 */ 7030 void 7031 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1, 7032 uintptr_t arg2, uintptr_t arg3, uintptr_t arg4) 7033 { 7034 processorid_t cpuid; 7035 dtrace_icookie_t cookie; 7036 dtrace_probe_t *probe; 7037 dtrace_mstate_t mstate; 7038 dtrace_ecb_t *ecb; 7039 dtrace_action_t *act; 7040 intptr_t offs; 7041 size_t size; 7042 int vtime, onintr; 7043 volatile uint16_t *flags; 7044 hrtime_t now; 7045 7046 if (panicstr != NULL) 7047 return; 7048 7049 #ifdef illumos 7050 /* 7051 * Kick out immediately if this CPU is still being born (in which case 7052 * curthread will be set to -1) or the current thread can't allow 7053 * probes in its current context. 7054 */ 7055 if (((uintptr_t)curthread & 1) || (curthread->t_flag & T_DONTDTRACE)) 7056 return; 7057 #endif 7058 7059 cookie = dtrace_interrupt_disable(); 7060 probe = dtrace_probes[id - 1]; 7061 cpuid = curcpu; 7062 onintr = CPU_ON_INTR(CPU); 7063 7064 if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE && 7065 probe->dtpr_predcache == curthread->t_predcache) { 7066 /* 7067 * We have hit in the predicate cache; we know that 7068 * this predicate would evaluate to be false. 7069 */ 7070 dtrace_interrupt_enable(cookie); 7071 return; 7072 } 7073 7074 #ifdef illumos 7075 if (panic_quiesce) { 7076 #else 7077 if (panicstr != NULL) { 7078 #endif 7079 /* 7080 * We don't trace anything if we're panicking. 7081 */ 7082 dtrace_interrupt_enable(cookie); 7083 return; 7084 } 7085 7086 now = mstate.dtms_timestamp = dtrace_gethrtime(); 7087 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP; 7088 vtime = dtrace_vtime_references != 0; 7089 7090 if (vtime && curthread->t_dtrace_start) 7091 curthread->t_dtrace_vtime += now - curthread->t_dtrace_start; 7092 7093 mstate.dtms_difo = NULL; 7094 mstate.dtms_probe = probe; 7095 mstate.dtms_strtok = 0; 7096 mstate.dtms_arg[0] = arg0; 7097 mstate.dtms_arg[1] = arg1; 7098 mstate.dtms_arg[2] = arg2; 7099 mstate.dtms_arg[3] = arg3; 7100 mstate.dtms_arg[4] = arg4; 7101 7102 flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags; 7103 7104 for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { 7105 dtrace_predicate_t *pred = ecb->dte_predicate; 7106 dtrace_state_t *state = ecb->dte_state; 7107 dtrace_buffer_t *buf = &state->dts_buffer[cpuid]; 7108 dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid]; 7109 dtrace_vstate_t *vstate = &state->dts_vstate; 7110 dtrace_provider_t *prov = probe->dtpr_provider; 7111 uint64_t tracememsize = 0; 7112 int committed = 0; 7113 caddr_t tomax; 7114 7115 /* 7116 * A little subtlety with the following (seemingly innocuous) 7117 * declaration of the automatic 'val': by looking at the 7118 * code, you might think that it could be declared in the 7119 * action processing loop, below. (That is, it's only used in 7120 * the action processing loop.) However, it must be declared 7121 * out of that scope because in the case of DIF expression 7122 * arguments to aggregating actions, one iteration of the 7123 * action loop will use the last iteration's value. 7124 */ 7125 uint64_t val = 0; 7126 7127 mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE; 7128 mstate.dtms_getf = NULL; 7129 7130 *flags &= ~CPU_DTRACE_ERROR; 7131 7132 if (prov == dtrace_provider) { 7133 /* 7134 * If dtrace itself is the provider of this probe, 7135 * we're only going to continue processing the ECB if 7136 * arg0 (the dtrace_state_t) is equal to the ECB's 7137 * creating state. (This prevents disjoint consumers 7138 * from seeing one another's metaprobes.) 7139 */ 7140 if (arg0 != (uint64_t)(uintptr_t)state) 7141 continue; 7142 } 7143 7144 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) { 7145 /* 7146 * We're not currently active. If our provider isn't 7147 * the dtrace pseudo provider, we're not interested. 7148 */ 7149 if (prov != dtrace_provider) 7150 continue; 7151 7152 /* 7153 * Now we must further check if we are in the BEGIN 7154 * probe. If we are, we will only continue processing 7155 * if we're still in WARMUP -- if one BEGIN enabling 7156 * has invoked the exit() action, we don't want to 7157 * evaluate subsequent BEGIN enablings. 7158 */ 7159 if (probe->dtpr_id == dtrace_probeid_begin && 7160 state->dts_activity != DTRACE_ACTIVITY_WARMUP) { 7161 ASSERT(state->dts_activity == 7162 DTRACE_ACTIVITY_DRAINING); 7163 continue; 7164 } 7165 } 7166 7167 if (ecb->dte_cond) { 7168 /* 7169 * If the dte_cond bits indicate that this 7170 * consumer is only allowed to see user-mode firings 7171 * of this probe, call the provider's dtps_usermode() 7172 * entry point to check that the probe was fired 7173 * while in a user context. Skip this ECB if that's 7174 * not the case. 7175 */ 7176 if ((ecb->dte_cond & DTRACE_COND_USERMODE) && 7177 prov->dtpv_pops.dtps_usermode(prov->dtpv_arg, 7178 probe->dtpr_id, probe->dtpr_arg) == 0) 7179 continue; 7180 7181 #ifdef illumos 7182 /* 7183 * This is more subtle than it looks. We have to be 7184 * absolutely certain that CRED() isn't going to 7185 * change out from under us so it's only legit to 7186 * examine that structure if we're in constrained 7187 * situations. Currently, the only times we'll this 7188 * check is if a non-super-user has enabled the 7189 * profile or syscall providers -- providers that 7190 * allow visibility of all processes. For the 7191 * profile case, the check above will ensure that 7192 * we're examining a user context. 7193 */ 7194 if (ecb->dte_cond & DTRACE_COND_OWNER) { 7195 cred_t *cr; 7196 cred_t *s_cr = 7197 ecb->dte_state->dts_cred.dcr_cred; 7198 proc_t *proc; 7199 7200 ASSERT(s_cr != NULL); 7201 7202 if ((cr = CRED()) == NULL || 7203 s_cr->cr_uid != cr->cr_uid || 7204 s_cr->cr_uid != cr->cr_ruid || 7205 s_cr->cr_uid != cr->cr_suid || 7206 s_cr->cr_gid != cr->cr_gid || 7207 s_cr->cr_gid != cr->cr_rgid || 7208 s_cr->cr_gid != cr->cr_sgid || 7209 (proc = ttoproc(curthread)) == NULL || 7210 (proc->p_flag & SNOCD)) 7211 continue; 7212 } 7213 7214 if (ecb->dte_cond & DTRACE_COND_ZONEOWNER) { 7215 cred_t *cr; 7216 cred_t *s_cr = 7217 ecb->dte_state->dts_cred.dcr_cred; 7218 7219 ASSERT(s_cr != NULL); 7220 7221 if ((cr = CRED()) == NULL || 7222 s_cr->cr_zone->zone_id != 7223 cr->cr_zone->zone_id) 7224 continue; 7225 } 7226 #endif 7227 } 7228 7229 if (now - state->dts_alive > dtrace_deadman_timeout) { 7230 /* 7231 * We seem to be dead. Unless we (a) have kernel 7232 * destructive permissions (b) have explicitly enabled 7233 * destructive actions and (c) destructive actions have 7234 * not been disabled, we're going to transition into 7235 * the KILLED state, from which no further processing 7236 * on this state will be performed. 7237 */ 7238 if (!dtrace_priv_kernel_destructive(state) || 7239 !state->dts_cred.dcr_destructive || 7240 dtrace_destructive_disallow) { 7241 void *activity = &state->dts_activity; 7242 dtrace_activity_t current; 7243 7244 do { 7245 current = state->dts_activity; 7246 } while (dtrace_cas32(activity, current, 7247 DTRACE_ACTIVITY_KILLED) != current); 7248 7249 continue; 7250 } 7251 } 7252 7253 if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed, 7254 ecb->dte_alignment, state, &mstate)) < 0) 7255 continue; 7256 7257 tomax = buf->dtb_tomax; 7258 ASSERT(tomax != NULL); 7259 7260 if (ecb->dte_size != 0) { 7261 dtrace_rechdr_t dtrh; 7262 if (!(mstate.dtms_present & DTRACE_MSTATE_TIMESTAMP)) { 7263 mstate.dtms_timestamp = dtrace_gethrtime(); 7264 mstate.dtms_present |= DTRACE_MSTATE_TIMESTAMP; 7265 } 7266 ASSERT3U(ecb->dte_size, >=, sizeof (dtrace_rechdr_t)); 7267 dtrh.dtrh_epid = ecb->dte_epid; 7268 DTRACE_RECORD_STORE_TIMESTAMP(&dtrh, 7269 mstate.dtms_timestamp); 7270 *((dtrace_rechdr_t *)(tomax + offs)) = dtrh; 7271 } 7272 7273 mstate.dtms_epid = ecb->dte_epid; 7274 mstate.dtms_present |= DTRACE_MSTATE_EPID; 7275 7276 if (state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) 7277 mstate.dtms_access = DTRACE_ACCESS_KERNEL; 7278 else 7279 mstate.dtms_access = 0; 7280 7281 if (pred != NULL) { 7282 dtrace_difo_t *dp = pred->dtp_difo; 7283 int rval; 7284 7285 rval = dtrace_dif_emulate(dp, &mstate, vstate, state); 7286 7287 if (!(*flags & CPU_DTRACE_ERROR) && !rval) { 7288 dtrace_cacheid_t cid = probe->dtpr_predcache; 7289 7290 if (cid != DTRACE_CACHEIDNONE && !onintr) { 7291 /* 7292 * Update the predicate cache... 7293 */ 7294 ASSERT(cid == pred->dtp_cacheid); 7295 curthread->t_predcache = cid; 7296 } 7297 7298 continue; 7299 } 7300 } 7301 7302 for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) && 7303 act != NULL; act = act->dta_next) { 7304 size_t valoffs; 7305 dtrace_difo_t *dp; 7306 dtrace_recdesc_t *rec = &act->dta_rec; 7307 7308 size = rec->dtrd_size; 7309 valoffs = offs + rec->dtrd_offset; 7310 7311 if (DTRACEACT_ISAGG(act->dta_kind)) { 7312 uint64_t v = 0xbad; 7313 dtrace_aggregation_t *agg; 7314 7315 agg = (dtrace_aggregation_t *)act; 7316 7317 if ((dp = act->dta_difo) != NULL) 7318 v = dtrace_dif_emulate(dp, 7319 &mstate, vstate, state); 7320 7321 if (*flags & CPU_DTRACE_ERROR) 7322 continue; 7323 7324 /* 7325 * Note that we always pass the expression 7326 * value from the previous iteration of the 7327 * action loop. This value will only be used 7328 * if there is an expression argument to the 7329 * aggregating action, denoted by the 7330 * dtag_hasarg field. 7331 */ 7332 dtrace_aggregate(agg, buf, 7333 offs, aggbuf, v, val); 7334 continue; 7335 } 7336 7337 switch (act->dta_kind) { 7338 case DTRACEACT_STOP: 7339 if (dtrace_priv_proc_destructive(state)) 7340 dtrace_action_stop(); 7341 continue; 7342 7343 case DTRACEACT_BREAKPOINT: 7344 if (dtrace_priv_kernel_destructive(state)) 7345 dtrace_action_breakpoint(ecb); 7346 continue; 7347 7348 case DTRACEACT_PANIC: 7349 if (dtrace_priv_kernel_destructive(state)) 7350 dtrace_action_panic(ecb); 7351 continue; 7352 7353 case DTRACEACT_STACK: 7354 if (!dtrace_priv_kernel(state)) 7355 continue; 7356 7357 dtrace_getpcstack((pc_t *)(tomax + valoffs), 7358 size / sizeof (pc_t), probe->dtpr_aframes, 7359 DTRACE_ANCHORED(probe) ? NULL : 7360 (uint32_t *)arg0); 7361 continue; 7362 7363 case DTRACEACT_JSTACK: 7364 case DTRACEACT_USTACK: 7365 if (!dtrace_priv_proc(state)) 7366 continue; 7367 7368 /* 7369 * See comment in DIF_VAR_PID. 7370 */ 7371 if (DTRACE_ANCHORED(mstate.dtms_probe) && 7372 CPU_ON_INTR(CPU)) { 7373 int depth = DTRACE_USTACK_NFRAMES( 7374 rec->dtrd_arg) + 1; 7375 7376 dtrace_bzero((void *)(tomax + valoffs), 7377 DTRACE_USTACK_STRSIZE(rec->dtrd_arg) 7378 + depth * sizeof (uint64_t)); 7379 7380 continue; 7381 } 7382 7383 if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 && 7384 curproc->p_dtrace_helpers != NULL) { 7385 /* 7386 * This is the slow path -- we have 7387 * allocated string space, and we're 7388 * getting the stack of a process that 7389 * has helpers. Call into a separate 7390 * routine to perform this processing. 7391 */ 7392 dtrace_action_ustack(&mstate, state, 7393 (uint64_t *)(tomax + valoffs), 7394 rec->dtrd_arg); 7395 continue; 7396 } 7397 7398 DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); 7399 dtrace_getupcstack((uint64_t *) 7400 (tomax + valoffs), 7401 DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1); 7402 DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT); 7403 continue; 7404 7405 default: 7406 break; 7407 } 7408 7409 dp = act->dta_difo; 7410 ASSERT(dp != NULL); 7411 7412 val = dtrace_dif_emulate(dp, &mstate, vstate, state); 7413 7414 if (*flags & CPU_DTRACE_ERROR) 7415 continue; 7416 7417 switch (act->dta_kind) { 7418 case DTRACEACT_SPECULATE: { 7419 dtrace_rechdr_t *dtrh; 7420 7421 ASSERT(buf == &state->dts_buffer[cpuid]); 7422 buf = dtrace_speculation_buffer(state, 7423 cpuid, val); 7424 7425 if (buf == NULL) { 7426 *flags |= CPU_DTRACE_DROP; 7427 continue; 7428 } 7429 7430 offs = dtrace_buffer_reserve(buf, 7431 ecb->dte_needed, ecb->dte_alignment, 7432 state, NULL); 7433 7434 if (offs < 0) { 7435 *flags |= CPU_DTRACE_DROP; 7436 continue; 7437 } 7438 7439 tomax = buf->dtb_tomax; 7440 ASSERT(tomax != NULL); 7441 7442 if (ecb->dte_size == 0) 7443 continue; 7444 7445 ASSERT3U(ecb->dte_size, >=, 7446 sizeof (dtrace_rechdr_t)); 7447 dtrh = ((void *)(tomax + offs)); 7448 dtrh->dtrh_epid = ecb->dte_epid; 7449 /* 7450 * When the speculation is committed, all of 7451 * the records in the speculative buffer will 7452 * have their timestamps set to the commit 7453 * time. Until then, it is set to a sentinel 7454 * value, for debugability. 7455 */ 7456 DTRACE_RECORD_STORE_TIMESTAMP(dtrh, UINT64_MAX); 7457 continue; 7458 } 7459 7460 case DTRACEACT_PRINTM: { 7461 /* The DIF returns a 'memref'. */ 7462 uintptr_t *memref = (uintptr_t *)(uintptr_t) val; 7463 7464 /* Get the size from the memref. */ 7465 size = memref[1]; 7466 7467 /* 7468 * Check if the size exceeds the allocated 7469 * buffer size. 7470 */ 7471 if (size + sizeof(uintptr_t) > dp->dtdo_rtype.dtdt_size) { 7472 /* Flag a drop! */ 7473 *flags |= CPU_DTRACE_DROP; 7474 continue; 7475 } 7476 7477 /* Store the size in the buffer first. */ 7478 DTRACE_STORE(uintptr_t, tomax, 7479 valoffs, size); 7480 7481 /* 7482 * Offset the buffer address to the start 7483 * of the data. 7484 */ 7485 valoffs += sizeof(uintptr_t); 7486 7487 /* 7488 * Reset to the memory address rather than 7489 * the memref array, then let the BYREF 7490 * code below do the work to store the 7491 * memory data in the buffer. 7492 */ 7493 val = memref[0]; 7494 break; 7495 } 7496 7497 case DTRACEACT_PRINTT: { 7498 /* The DIF returns a 'typeref'. */ 7499 uintptr_t *typeref = (uintptr_t *)(uintptr_t) val; 7500 char c = '\0' + 1; 7501 size_t s; 7502 7503 /* 7504 * Get the type string length and round it 7505 * up so that the data that follows is 7506 * aligned for easy access. 7507 */ 7508 size_t typs = strlen((char *) typeref[2]) + 1; 7509 typs = roundup(typs, sizeof(uintptr_t)); 7510 7511 /* 7512 *Get the size from the typeref using the 7513 * number of elements and the type size. 7514 */ 7515 size = typeref[1] * typeref[3]; 7516 7517 /* 7518 * Check if the size exceeds the allocated 7519 * buffer size. 7520 */ 7521 if (size + typs + 2 * sizeof(uintptr_t) > dp->dtdo_rtype.dtdt_size) { 7522 /* Flag a drop! */ 7523 *flags |= CPU_DTRACE_DROP; 7524 7525 } 7526 7527 /* Store the size in the buffer first. */ 7528 DTRACE_STORE(uintptr_t, tomax, 7529 valoffs, size); 7530 valoffs += sizeof(uintptr_t); 7531 7532 /* Store the type size in the buffer. */ 7533 DTRACE_STORE(uintptr_t, tomax, 7534 valoffs, typeref[3]); 7535 valoffs += sizeof(uintptr_t); 7536 7537 val = typeref[2]; 7538 7539 for (s = 0; s < typs; s++) { 7540 if (c != '\0') 7541 c = dtrace_load8(val++); 7542 7543 DTRACE_STORE(uint8_t, tomax, 7544 valoffs++, c); 7545 } 7546 7547 /* 7548 * Reset to the memory address rather than 7549 * the typeref array, then let the BYREF 7550 * code below do the work to store the 7551 * memory data in the buffer. 7552 */ 7553 val = typeref[0]; 7554 break; 7555 } 7556 7557 case DTRACEACT_CHILL: 7558 if (dtrace_priv_kernel_destructive(state)) 7559 dtrace_action_chill(&mstate, val); 7560 continue; 7561 7562 case DTRACEACT_RAISE: 7563 if (dtrace_priv_proc_destructive(state)) 7564 dtrace_action_raise(val); 7565 continue; 7566 7567 case DTRACEACT_COMMIT: 7568 ASSERT(!committed); 7569 7570 /* 7571 * We need to commit our buffer state. 7572 */ 7573 if (ecb->dte_size) 7574 buf->dtb_offset = offs + ecb->dte_size; 7575 buf = &state->dts_buffer[cpuid]; 7576 dtrace_speculation_commit(state, cpuid, val); 7577 committed = 1; 7578 continue; 7579 7580 case DTRACEACT_DISCARD: 7581 dtrace_speculation_discard(state, cpuid, val); 7582 continue; 7583 7584 case DTRACEACT_DIFEXPR: 7585 case DTRACEACT_LIBACT: 7586 case DTRACEACT_PRINTF: 7587 case DTRACEACT_PRINTA: 7588 case DTRACEACT_SYSTEM: 7589 case DTRACEACT_FREOPEN: 7590 case DTRACEACT_TRACEMEM: 7591 break; 7592 7593 case DTRACEACT_TRACEMEM_DYNSIZE: 7594 tracememsize = val; 7595 break; 7596 7597 case DTRACEACT_SYM: 7598 case DTRACEACT_MOD: 7599 if (!dtrace_priv_kernel(state)) 7600 continue; 7601 break; 7602 7603 case DTRACEACT_USYM: 7604 case DTRACEACT_UMOD: 7605 case DTRACEACT_UADDR: { 7606 #ifdef illumos 7607 struct pid *pid = curthread->t_procp->p_pidp; 7608 #endif 7609 7610 if (!dtrace_priv_proc(state)) 7611 continue; 7612 7613 DTRACE_STORE(uint64_t, tomax, 7614 #ifdef illumos 7615 valoffs, (uint64_t)pid->pid_id); 7616 #else 7617 valoffs, (uint64_t) curproc->p_pid); 7618 #endif 7619 DTRACE_STORE(uint64_t, tomax, 7620 valoffs + sizeof (uint64_t), val); 7621 7622 continue; 7623 } 7624 7625 case DTRACEACT_EXIT: { 7626 /* 7627 * For the exit action, we are going to attempt 7628 * to atomically set our activity to be 7629 * draining. If this fails (either because 7630 * another CPU has beat us to the exit action, 7631 * or because our current activity is something 7632 * other than ACTIVE or WARMUP), we will 7633 * continue. This assures that the exit action 7634 * can be successfully recorded at most once 7635 * when we're in the ACTIVE state. If we're 7636 * encountering the exit() action while in 7637 * COOLDOWN, however, we want to honor the new 7638 * status code. (We know that we're the only 7639 * thread in COOLDOWN, so there is no race.) 7640 */ 7641 void *activity = &state->dts_activity; 7642 dtrace_activity_t current = state->dts_activity; 7643 7644 if (current == DTRACE_ACTIVITY_COOLDOWN) 7645 break; 7646 7647 if (current != DTRACE_ACTIVITY_WARMUP) 7648 current = DTRACE_ACTIVITY_ACTIVE; 7649 7650 if (dtrace_cas32(activity, current, 7651 DTRACE_ACTIVITY_DRAINING) != current) { 7652 *flags |= CPU_DTRACE_DROP; 7653 continue; 7654 } 7655 7656 break; 7657 } 7658 7659 default: 7660 ASSERT(0); 7661 } 7662 7663 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF || 7664 dp->dtdo_rtype.dtdt_flags & DIF_TF_BYUREF) { 7665 uintptr_t end = valoffs + size; 7666 7667 if (tracememsize != 0 && 7668 valoffs + tracememsize < end) { 7669 end = valoffs + tracememsize; 7670 tracememsize = 0; 7671 } 7672 7673 if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF && 7674 !dtrace_vcanload((void *)(uintptr_t)val, 7675 &dp->dtdo_rtype, &mstate, vstate)) 7676 continue; 7677 7678 dtrace_store_by_ref(dp, tomax, size, &valoffs, 7679 &val, end, act->dta_intuple, 7680 dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF ? 7681 DIF_TF_BYREF: DIF_TF_BYUREF); 7682 continue; 7683 } 7684 7685 switch (size) { 7686 case 0: 7687 break; 7688 7689 case sizeof (uint8_t): 7690 DTRACE_STORE(uint8_t, tomax, valoffs, val); 7691 break; 7692 case sizeof (uint16_t): 7693 DTRACE_STORE(uint16_t, tomax, valoffs, val); 7694 break; 7695 case sizeof (uint32_t): 7696 DTRACE_STORE(uint32_t, tomax, valoffs, val); 7697 break; 7698 case sizeof (uint64_t): 7699 DTRACE_STORE(uint64_t, tomax, valoffs, val); 7700 break; 7701 default: 7702 /* 7703 * Any other size should have been returned by 7704 * reference, not by value. 7705 */ 7706 ASSERT(0); 7707 break; 7708 } 7709 } 7710 7711 if (*flags & CPU_DTRACE_DROP) 7712 continue; 7713 7714 if (*flags & CPU_DTRACE_FAULT) { 7715 int ndx; 7716 dtrace_action_t *err; 7717 7718 buf->dtb_errors++; 7719 7720 if (probe->dtpr_id == dtrace_probeid_error) { 7721 /* 7722 * There's nothing we can do -- we had an 7723 * error on the error probe. We bump an 7724 * error counter to at least indicate that 7725 * this condition happened. 7726 */ 7727 dtrace_error(&state->dts_dblerrors); 7728 continue; 7729 } 7730 7731 if (vtime) { 7732 /* 7733 * Before recursing on dtrace_probe(), we 7734 * need to explicitly clear out our start 7735 * time to prevent it from being accumulated 7736 * into t_dtrace_vtime. 7737 */ 7738 curthread->t_dtrace_start = 0; 7739 } 7740 7741 /* 7742 * Iterate over the actions to figure out which action 7743 * we were processing when we experienced the error. 7744 * Note that act points _past_ the faulting action; if 7745 * act is ecb->dte_action, the fault was in the 7746 * predicate, if it's ecb->dte_action->dta_next it's 7747 * in action #1, and so on. 7748 */ 7749 for (err = ecb->dte_action, ndx = 0; 7750 err != act; err = err->dta_next, ndx++) 7751 continue; 7752 7753 dtrace_probe_error(state, ecb->dte_epid, ndx, 7754 (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ? 7755 mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags), 7756 cpu_core[cpuid].cpuc_dtrace_illval); 7757 7758 continue; 7759 } 7760 7761 if (!committed) 7762 buf->dtb_offset = offs + ecb->dte_size; 7763 } 7764 7765 if (vtime) 7766 curthread->t_dtrace_start = dtrace_gethrtime(); 7767 7768 dtrace_interrupt_enable(cookie); 7769 } 7770 7771 /* 7772 * DTrace Probe Hashing Functions 7773 * 7774 * The functions in this section (and indeed, the functions in remaining 7775 * sections) are not _called_ from probe context. (Any exceptions to this are 7776 * marked with a "Note:".) Rather, they are called from elsewhere in the 7777 * DTrace framework to look-up probes in, add probes to and remove probes from 7778 * the DTrace probe hashes. (Each probe is hashed by each element of the 7779 * probe tuple -- allowing for fast lookups, regardless of what was 7780 * specified.) 7781 */ 7782 static uint_t 7783 dtrace_hash_str(const char *p) 7784 { 7785 unsigned int g; 7786 uint_t hval = 0; 7787 7788 while (*p) { 7789 hval = (hval << 4) + *p++; 7790 if ((g = (hval & 0xf0000000)) != 0) 7791 hval ^= g >> 24; 7792 hval &= ~g; 7793 } 7794 return (hval); 7795 } 7796 7797 static dtrace_hash_t * 7798 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs) 7799 { 7800 dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP); 7801 7802 hash->dth_stroffs = stroffs; 7803 hash->dth_nextoffs = nextoffs; 7804 hash->dth_prevoffs = prevoffs; 7805 7806 hash->dth_size = 1; 7807 hash->dth_mask = hash->dth_size - 1; 7808 7809 hash->dth_tab = kmem_zalloc(hash->dth_size * 7810 sizeof (dtrace_hashbucket_t *), KM_SLEEP); 7811 7812 return (hash); 7813 } 7814 7815 static void 7816 dtrace_hash_destroy(dtrace_hash_t *hash) 7817 { 7818 #ifdef DEBUG 7819 int i; 7820 7821 for (i = 0; i < hash->dth_size; i++) 7822 ASSERT(hash->dth_tab[i] == NULL); 7823 #endif 7824 7825 kmem_free(hash->dth_tab, 7826 hash->dth_size * sizeof (dtrace_hashbucket_t *)); 7827 kmem_free(hash, sizeof (dtrace_hash_t)); 7828 } 7829 7830 static void 7831 dtrace_hash_resize(dtrace_hash_t *hash) 7832 { 7833 int size = hash->dth_size, i, ndx; 7834 int new_size = hash->dth_size << 1; 7835 int new_mask = new_size - 1; 7836 dtrace_hashbucket_t **new_tab, *bucket, *next; 7837 7838 ASSERT((new_size & new_mask) == 0); 7839 7840 new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP); 7841 7842 for (i = 0; i < size; i++) { 7843 for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) { 7844 dtrace_probe_t *probe = bucket->dthb_chain; 7845 7846 ASSERT(probe != NULL); 7847 ndx = DTRACE_HASHSTR(hash, probe) & new_mask; 7848 7849 next = bucket->dthb_next; 7850 bucket->dthb_next = new_tab[ndx]; 7851 new_tab[ndx] = bucket; 7852 } 7853 } 7854 7855 kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *)); 7856 hash->dth_tab = new_tab; 7857 hash->dth_size = new_size; 7858 hash->dth_mask = new_mask; 7859 } 7860 7861 static void 7862 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new) 7863 { 7864 int hashval = DTRACE_HASHSTR(hash, new); 7865 int ndx = hashval & hash->dth_mask; 7866 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7867 dtrace_probe_t **nextp, **prevp; 7868 7869 for (; bucket != NULL; bucket = bucket->dthb_next) { 7870 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new)) 7871 goto add; 7872 } 7873 7874 if ((hash->dth_nbuckets >> 1) > hash->dth_size) { 7875 dtrace_hash_resize(hash); 7876 dtrace_hash_add(hash, new); 7877 return; 7878 } 7879 7880 bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP); 7881 bucket->dthb_next = hash->dth_tab[ndx]; 7882 hash->dth_tab[ndx] = bucket; 7883 hash->dth_nbuckets++; 7884 7885 add: 7886 nextp = DTRACE_HASHNEXT(hash, new); 7887 ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL); 7888 *nextp = bucket->dthb_chain; 7889 7890 if (bucket->dthb_chain != NULL) { 7891 prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain); 7892 ASSERT(*prevp == NULL); 7893 *prevp = new; 7894 } 7895 7896 bucket->dthb_chain = new; 7897 bucket->dthb_len++; 7898 } 7899 7900 static dtrace_probe_t * 7901 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template) 7902 { 7903 int hashval = DTRACE_HASHSTR(hash, template); 7904 int ndx = hashval & hash->dth_mask; 7905 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7906 7907 for (; bucket != NULL; bucket = bucket->dthb_next) { 7908 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template)) 7909 return (bucket->dthb_chain); 7910 } 7911 7912 return (NULL); 7913 } 7914 7915 static int 7916 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template) 7917 { 7918 int hashval = DTRACE_HASHSTR(hash, template); 7919 int ndx = hashval & hash->dth_mask; 7920 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7921 7922 for (; bucket != NULL; bucket = bucket->dthb_next) { 7923 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template)) 7924 return (bucket->dthb_len); 7925 } 7926 7927 return (0); 7928 } 7929 7930 static void 7931 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe) 7932 { 7933 int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask; 7934 dtrace_hashbucket_t *bucket = hash->dth_tab[ndx]; 7935 7936 dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe); 7937 dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe); 7938 7939 /* 7940 * Find the bucket that we're removing this probe from. 7941 */ 7942 for (; bucket != NULL; bucket = bucket->dthb_next) { 7943 if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe)) 7944 break; 7945 } 7946 7947 ASSERT(bucket != NULL); 7948 7949 if (*prevp == NULL) { 7950 if (*nextp == NULL) { 7951 /* 7952 * The removed probe was the only probe on this 7953 * bucket; we need to remove the bucket. 7954 */ 7955 dtrace_hashbucket_t *b = hash->dth_tab[ndx]; 7956 7957 ASSERT(bucket->dthb_chain == probe); 7958 ASSERT(b != NULL); 7959 7960 if (b == bucket) { 7961 hash->dth_tab[ndx] = bucket->dthb_next; 7962 } else { 7963 while (b->dthb_next != bucket) 7964 b = b->dthb_next; 7965 b->dthb_next = bucket->dthb_next; 7966 } 7967 7968 ASSERT(hash->dth_nbuckets > 0); 7969 hash->dth_nbuckets--; 7970 kmem_free(bucket, sizeof (dtrace_hashbucket_t)); 7971 return; 7972 } 7973 7974 bucket->dthb_chain = *nextp; 7975 } else { 7976 *(DTRACE_HASHNEXT(hash, *prevp)) = *nextp; 7977 } 7978 7979 if (*nextp != NULL) 7980 *(DTRACE_HASHPREV(hash, *nextp)) = *prevp; 7981 } 7982 7983 /* 7984 * DTrace Utility Functions 7985 * 7986 * These are random utility functions that are _not_ called from probe context. 7987 */ 7988 static int 7989 dtrace_badattr(const dtrace_attribute_t *a) 7990 { 7991 return (a->dtat_name > DTRACE_STABILITY_MAX || 7992 a->dtat_data > DTRACE_STABILITY_MAX || 7993 a->dtat_class > DTRACE_CLASS_MAX); 7994 } 7995 7996 /* 7997 * Return a duplicate copy of a string. If the specified string is NULL, 7998 * this function returns a zero-length string. 7999 */ 8000 static char * 8001 dtrace_strdup(const char *str) 8002 { 8003 char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP); 8004 8005 if (str != NULL) 8006 (void) strcpy(new, str); 8007 8008 return (new); 8009 } 8010 8011 #define DTRACE_ISALPHA(c) \ 8012 (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z')) 8013 8014 static int 8015 dtrace_badname(const char *s) 8016 { 8017 char c; 8018 8019 if (s == NULL || (c = *s++) == '\0') 8020 return (0); 8021 8022 if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.') 8023 return (1); 8024 8025 while ((c = *s++) != '\0') { 8026 if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') && 8027 c != '-' && c != '_' && c != '.' && c != '`') 8028 return (1); 8029 } 8030 8031 return (0); 8032 } 8033 8034 static void 8035 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp, zoneid_t *zoneidp) 8036 { 8037 uint32_t priv; 8038 8039 #ifdef illumos 8040 if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { 8041 /* 8042 * For DTRACE_PRIV_ALL, the uid and zoneid don't matter. 8043 */ 8044 priv = DTRACE_PRIV_ALL; 8045 } else { 8046 *uidp = crgetuid(cr); 8047 *zoneidp = crgetzoneid(cr); 8048 8049 priv = 0; 8050 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) 8051 priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER; 8052 else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) 8053 priv |= DTRACE_PRIV_USER; 8054 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) 8055 priv |= DTRACE_PRIV_PROC; 8056 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) 8057 priv |= DTRACE_PRIV_OWNER; 8058 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) 8059 priv |= DTRACE_PRIV_ZONEOWNER; 8060 } 8061 #else 8062 priv = DTRACE_PRIV_ALL; 8063 #endif 8064 8065 *privp = priv; 8066 } 8067 8068 #ifdef DTRACE_ERRDEBUG 8069 static void 8070 dtrace_errdebug(const char *str) 8071 { 8072 int hval = dtrace_hash_str(str) % DTRACE_ERRHASHSZ; 8073 int occupied = 0; 8074 8075 mutex_enter(&dtrace_errlock); 8076 dtrace_errlast = str; 8077 dtrace_errthread = curthread; 8078 8079 while (occupied++ < DTRACE_ERRHASHSZ) { 8080 if (dtrace_errhash[hval].dter_msg == str) { 8081 dtrace_errhash[hval].dter_count++; 8082 goto out; 8083 } 8084 8085 if (dtrace_errhash[hval].dter_msg != NULL) { 8086 hval = (hval + 1) % DTRACE_ERRHASHSZ; 8087 continue; 8088 } 8089 8090 dtrace_errhash[hval].dter_msg = str; 8091 dtrace_errhash[hval].dter_count = 1; 8092 goto out; 8093 } 8094 8095 panic("dtrace: undersized error hash"); 8096 out: 8097 mutex_exit(&dtrace_errlock); 8098 } 8099 #endif 8100 8101 /* 8102 * DTrace Matching Functions 8103 * 8104 * These functions are used to match groups of probes, given some elements of 8105 * a probe tuple, or some globbed expressions for elements of a probe tuple. 8106 */ 8107 static int 8108 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid, 8109 zoneid_t zoneid) 8110 { 8111 if (priv != DTRACE_PRIV_ALL) { 8112 uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags; 8113 uint32_t match = priv & ppriv; 8114 8115 /* 8116 * No PRIV_DTRACE_* privileges... 8117 */ 8118 if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER | 8119 DTRACE_PRIV_KERNEL)) == 0) 8120 return (0); 8121 8122 /* 8123 * No matching bits, but there were bits to match... 8124 */ 8125 if (match == 0 && ppriv != 0) 8126 return (0); 8127 8128 /* 8129 * Need to have permissions to the process, but don't... 8130 */ 8131 if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 && 8132 uid != prp->dtpr_provider->dtpv_priv.dtpp_uid) { 8133 return (0); 8134 } 8135 8136 /* 8137 * Need to be in the same zone unless we possess the 8138 * privilege to examine all zones. 8139 */ 8140 if (((ppriv & ~match) & DTRACE_PRIV_ZONEOWNER) != 0 && 8141 zoneid != prp->dtpr_provider->dtpv_priv.dtpp_zoneid) { 8142 return (0); 8143 } 8144 } 8145 8146 return (1); 8147 } 8148 8149 /* 8150 * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which 8151 * consists of input pattern strings and an ops-vector to evaluate them. 8152 * This function returns >0 for match, 0 for no match, and <0 for error. 8153 */ 8154 static int 8155 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp, 8156 uint32_t priv, uid_t uid, zoneid_t zoneid) 8157 { 8158 dtrace_provider_t *pvp = prp->dtpr_provider; 8159 int rv; 8160 8161 if (pvp->dtpv_defunct) 8162 return (0); 8163 8164 if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0) 8165 return (rv); 8166 8167 if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0) 8168 return (rv); 8169 8170 if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0) 8171 return (rv); 8172 8173 if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0) 8174 return (rv); 8175 8176 if (dtrace_match_priv(prp, priv, uid, zoneid) == 0) 8177 return (0); 8178 8179 return (rv); 8180 } 8181 8182 /* 8183 * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN) 8184 * interface for matching a glob pattern 'p' to an input string 's'. Unlike 8185 * libc's version, the kernel version only applies to 8-bit ASCII strings. 8186 * In addition, all of the recursion cases except for '*' matching have been 8187 * unwound. For '*', we still implement recursive evaluation, but a depth 8188 * counter is maintained and matching is aborted if we recurse too deep. 8189 * The function returns 0 if no match, >0 if match, and <0 if recursion error. 8190 */ 8191 static int 8192 dtrace_match_glob(const char *s, const char *p, int depth) 8193 { 8194 const char *olds; 8195 char s1, c; 8196 int gs; 8197 8198 if (depth > DTRACE_PROBEKEY_MAXDEPTH) 8199 return (-1); 8200 8201 if (s == NULL) 8202 s = ""; /* treat NULL as empty string */ 8203 8204 top: 8205 olds = s; 8206 s1 = *s++; 8207 8208 if (p == NULL) 8209 return (0); 8210 8211 if ((c = *p++) == '\0') 8212 return (s1 == '\0'); 8213 8214 switch (c) { 8215 case '[': { 8216 int ok = 0, notflag = 0; 8217 char lc = '\0'; 8218 8219 if (s1 == '\0') 8220 return (0); 8221 8222 if (*p == '!') { 8223 notflag = 1; 8224 p++; 8225 } 8226 8227 if ((c = *p++) == '\0') 8228 return (0); 8229 8230 do { 8231 if (c == '-' && lc != '\0' && *p != ']') { 8232 if ((c = *p++) == '\0') 8233 return (0); 8234 if (c == '\\' && (c = *p++) == '\0') 8235 return (0); 8236 8237 if (notflag) { 8238 if (s1 < lc || s1 > c) 8239 ok++; 8240 else 8241 return (0); 8242 } else if (lc <= s1 && s1 <= c) 8243 ok++; 8244 8245 } else if (c == '\\' && (c = *p++) == '\0') 8246 return (0); 8247 8248 lc = c; /* save left-hand 'c' for next iteration */ 8249 8250 if (notflag) { 8251 if (s1 != c) 8252 ok++; 8253 else 8254 return (0); 8255 } else if (s1 == c) 8256 ok++; 8257 8258 if ((c = *p++) == '\0') 8259 return (0); 8260 8261 } while (c != ']'); 8262 8263 if (ok) 8264 goto top; 8265 8266 return (0); 8267 } 8268 8269 case '\\': 8270 if ((c = *p++) == '\0') 8271 return (0); 8272 /*FALLTHRU*/ 8273 8274 default: 8275 if (c != s1) 8276 return (0); 8277 /*FALLTHRU*/ 8278 8279 case '?': 8280 if (s1 != '\0') 8281 goto top; 8282 return (0); 8283 8284 case '*': 8285 while (*p == '*') 8286 p++; /* consecutive *'s are identical to a single one */ 8287 8288 if (*p == '\0') 8289 return (1); 8290 8291 for (s = olds; *s != '\0'; s++) { 8292 if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0) 8293 return (gs); 8294 } 8295 8296 return (0); 8297 } 8298 } 8299 8300 /*ARGSUSED*/ 8301 static int 8302 dtrace_match_string(const char *s, const char *p, int depth) 8303 { 8304 return (s != NULL && strcmp(s, p) == 0); 8305 } 8306 8307 /*ARGSUSED*/ 8308 static int 8309 dtrace_match_nul(const char *s, const char *p, int depth) 8310 { 8311 return (1); /* always match the empty pattern */ 8312 } 8313 8314 /*ARGSUSED*/ 8315 static int 8316 dtrace_match_nonzero(const char *s, const char *p, int depth) 8317 { 8318 return (s != NULL && s[0] != '\0'); 8319 } 8320 8321 static int 8322 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid, 8323 zoneid_t zoneid, int (*matched)(dtrace_probe_t *, void *), void *arg) 8324 { 8325 dtrace_probe_t template, *probe; 8326 dtrace_hash_t *hash = NULL; 8327 int len, best = INT_MAX, nmatched = 0; 8328 dtrace_id_t i; 8329 8330 ASSERT(MUTEX_HELD(&dtrace_lock)); 8331 8332 /* 8333 * If the probe ID is specified in the key, just lookup by ID and 8334 * invoke the match callback once if a matching probe is found. 8335 */ 8336 if (pkp->dtpk_id != DTRACE_IDNONE) { 8337 if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL && 8338 dtrace_match_probe(probe, pkp, priv, uid, zoneid) > 0) { 8339 (void) (*matched)(probe, arg); 8340 nmatched++; 8341 } 8342 return (nmatched); 8343 } 8344 8345 template.dtpr_mod = (char *)pkp->dtpk_mod; 8346 template.dtpr_func = (char *)pkp->dtpk_func; 8347 template.dtpr_name = (char *)pkp->dtpk_name; 8348 8349 /* 8350 * We want to find the most distinct of the module name, function 8351 * name, and name. So for each one that is not a glob pattern or 8352 * empty string, we perform a lookup in the corresponding hash and 8353 * use the hash table with the fewest collisions to do our search. 8354 */ 8355 if (pkp->dtpk_mmatch == &dtrace_match_string && 8356 (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) { 8357 best = len; 8358 hash = dtrace_bymod; 8359 } 8360 8361 if (pkp->dtpk_fmatch == &dtrace_match_string && 8362 (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) { 8363 best = len; 8364 hash = dtrace_byfunc; 8365 } 8366 8367 if (pkp->dtpk_nmatch == &dtrace_match_string && 8368 (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) { 8369 best = len; 8370 hash = dtrace_byname; 8371 } 8372 8373 /* 8374 * If we did not select a hash table, iterate over every probe and 8375 * invoke our callback for each one that matches our input probe key. 8376 */ 8377 if (hash == NULL) { 8378 for (i = 0; i < dtrace_nprobes; i++) { 8379 if ((probe = dtrace_probes[i]) == NULL || 8380 dtrace_match_probe(probe, pkp, priv, uid, 8381 zoneid) <= 0) 8382 continue; 8383 8384 nmatched++; 8385 8386 if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT) 8387 break; 8388 } 8389 8390 return (nmatched); 8391 } 8392 8393 /* 8394 * If we selected a hash table, iterate over each probe of the same key 8395 * name and invoke the callback for every probe that matches the other 8396 * attributes of our input probe key. 8397 */ 8398 for (probe = dtrace_hash_lookup(hash, &template); probe != NULL; 8399 probe = *(DTRACE_HASHNEXT(hash, probe))) { 8400 8401 if (dtrace_match_probe(probe, pkp, priv, uid, zoneid) <= 0) 8402 continue; 8403 8404 nmatched++; 8405 8406 if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT) 8407 break; 8408 } 8409 8410 return (nmatched); 8411 } 8412 8413 /* 8414 * Return the function pointer dtrace_probecmp() should use to compare the 8415 * specified pattern with a string. For NULL or empty patterns, we select 8416 * dtrace_match_nul(). For glob pattern strings, we use dtrace_match_glob(). 8417 * For non-empty non-glob strings, we use dtrace_match_string(). 8418 */ 8419 static dtrace_probekey_f * 8420 dtrace_probekey_func(const char *p) 8421 { 8422 char c; 8423 8424 if (p == NULL || *p == '\0') 8425 return (&dtrace_match_nul); 8426 8427 while ((c = *p++) != '\0') { 8428 if (c == '[' || c == '?' || c == '*' || c == '\\') 8429 return (&dtrace_match_glob); 8430 } 8431 8432 return (&dtrace_match_string); 8433 } 8434 8435 /* 8436 * Build a probe comparison key for use with dtrace_match_probe() from the 8437 * given probe description. By convention, a null key only matches anchored 8438 * probes: if each field is the empty string, reset dtpk_fmatch to 8439 * dtrace_match_nonzero(). 8440 */ 8441 static void 8442 dtrace_probekey(dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp) 8443 { 8444 pkp->dtpk_prov = pdp->dtpd_provider; 8445 pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider); 8446 8447 pkp->dtpk_mod = pdp->dtpd_mod; 8448 pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod); 8449 8450 pkp->dtpk_func = pdp->dtpd_func; 8451 pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func); 8452 8453 pkp->dtpk_name = pdp->dtpd_name; 8454 pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name); 8455 8456 pkp->dtpk_id = pdp->dtpd_id; 8457 8458 if (pkp->dtpk_id == DTRACE_IDNONE && 8459 pkp->dtpk_pmatch == &dtrace_match_nul && 8460 pkp->dtpk_mmatch == &dtrace_match_nul && 8461 pkp->dtpk_fmatch == &dtrace_match_nul && 8462 pkp->dtpk_nmatch == &dtrace_match_nul) 8463 pkp->dtpk_fmatch = &dtrace_match_nonzero; 8464 } 8465 8466 /* 8467 * DTrace Provider-to-Framework API Functions 8468 * 8469 * These functions implement much of the Provider-to-Framework API, as 8470 * described in <sys/dtrace.h>. The parts of the API not in this section are 8471 * the functions in the API for probe management (found below), and 8472 * dtrace_probe() itself (found above). 8473 */ 8474 8475 /* 8476 * Register the calling provider with the DTrace framework. This should 8477 * generally be called by DTrace providers in their attach(9E) entry point. 8478 */ 8479 int 8480 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv, 8481 cred_t *cr, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp) 8482 { 8483 dtrace_provider_t *provider; 8484 8485 if (name == NULL || pap == NULL || pops == NULL || idp == NULL) { 8486 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8487 "arguments", name ? name : "<NULL>"); 8488 return (EINVAL); 8489 } 8490 8491 if (name[0] == '\0' || dtrace_badname(name)) { 8492 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8493 "provider name", name); 8494 return (EINVAL); 8495 } 8496 8497 if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) || 8498 pops->dtps_enable == NULL || pops->dtps_disable == NULL || 8499 pops->dtps_destroy == NULL || 8500 ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) { 8501 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8502 "provider ops", name); 8503 return (EINVAL); 8504 } 8505 8506 if (dtrace_badattr(&pap->dtpa_provider) || 8507 dtrace_badattr(&pap->dtpa_mod) || 8508 dtrace_badattr(&pap->dtpa_func) || 8509 dtrace_badattr(&pap->dtpa_name) || 8510 dtrace_badattr(&pap->dtpa_args)) { 8511 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8512 "provider attributes", name); 8513 return (EINVAL); 8514 } 8515 8516 if (priv & ~DTRACE_PRIV_ALL) { 8517 cmn_err(CE_WARN, "failed to register provider '%s': invalid " 8518 "privilege attributes", name); 8519 return (EINVAL); 8520 } 8521 8522 if ((priv & DTRACE_PRIV_KERNEL) && 8523 (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) && 8524 pops->dtps_usermode == NULL) { 8525 cmn_err(CE_WARN, "failed to register provider '%s': need " 8526 "dtps_usermode() op for given privilege attributes", name); 8527 return (EINVAL); 8528 } 8529 8530 provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP); 8531 provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP); 8532 (void) strcpy(provider->dtpv_name, name); 8533 8534 provider->dtpv_attr = *pap; 8535 provider->dtpv_priv.dtpp_flags = priv; 8536 if (cr != NULL) { 8537 provider->dtpv_priv.dtpp_uid = crgetuid(cr); 8538 provider->dtpv_priv.dtpp_zoneid = crgetzoneid(cr); 8539 } 8540 provider->dtpv_pops = *pops; 8541 8542 if (pops->dtps_provide == NULL) { 8543 ASSERT(pops->dtps_provide_module != NULL); 8544 provider->dtpv_pops.dtps_provide = 8545 (void (*)(void *, dtrace_probedesc_t *))dtrace_nullop; 8546 } 8547 8548 if (pops->dtps_provide_module == NULL) { 8549 ASSERT(pops->dtps_provide != NULL); 8550 provider->dtpv_pops.dtps_provide_module = 8551 (void (*)(void *, modctl_t *))dtrace_nullop; 8552 } 8553 8554 if (pops->dtps_suspend == NULL) { 8555 ASSERT(pops->dtps_resume == NULL); 8556 provider->dtpv_pops.dtps_suspend = 8557 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; 8558 provider->dtpv_pops.dtps_resume = 8559 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop; 8560 } 8561 8562 provider->dtpv_arg = arg; 8563 *idp = (dtrace_provider_id_t)provider; 8564 8565 if (pops == &dtrace_provider_ops) { 8566 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 8567 ASSERT(MUTEX_HELD(&dtrace_lock)); 8568 ASSERT(dtrace_anon.dta_enabling == NULL); 8569 8570 /* 8571 * We make sure that the DTrace provider is at the head of 8572 * the provider chain. 8573 */ 8574 provider->dtpv_next = dtrace_provider; 8575 dtrace_provider = provider; 8576 return (0); 8577 } 8578 8579 mutex_enter(&dtrace_provider_lock); 8580 mutex_enter(&dtrace_lock); 8581 8582 /* 8583 * If there is at least one provider registered, we'll add this 8584 * provider after the first provider. 8585 */ 8586 if (dtrace_provider != NULL) { 8587 provider->dtpv_next = dtrace_provider->dtpv_next; 8588 dtrace_provider->dtpv_next = provider; 8589 } else { 8590 dtrace_provider = provider; 8591 } 8592 8593 if (dtrace_retained != NULL) { 8594 dtrace_enabling_provide(provider); 8595 8596 /* 8597 * Now we need to call dtrace_enabling_matchall() -- which 8598 * will acquire cpu_lock and dtrace_lock. We therefore need 8599 * to drop all of our locks before calling into it... 8600 */ 8601 mutex_exit(&dtrace_lock); 8602 mutex_exit(&dtrace_provider_lock); 8603 dtrace_enabling_matchall(); 8604 8605 return (0); 8606 } 8607 8608 mutex_exit(&dtrace_lock); 8609 mutex_exit(&dtrace_provider_lock); 8610 8611 return (0); 8612 } 8613 8614 /* 8615 * Unregister the specified provider from the DTrace framework. This should 8616 * generally be called by DTrace providers in their detach(9E) entry point. 8617 */ 8618 int 8619 dtrace_unregister(dtrace_provider_id_t id) 8620 { 8621 dtrace_provider_t *old = (dtrace_provider_t *)id; 8622 dtrace_provider_t *prev = NULL; 8623 int i, self = 0, noreap = 0; 8624 dtrace_probe_t *probe, *first = NULL; 8625 8626 if (old->dtpv_pops.dtps_enable == 8627 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop) { 8628 /* 8629 * If DTrace itself is the provider, we're called with locks 8630 * already held. 8631 */ 8632 ASSERT(old == dtrace_provider); 8633 #ifdef illumos 8634 ASSERT(dtrace_devi != NULL); 8635 #endif 8636 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 8637 ASSERT(MUTEX_HELD(&dtrace_lock)); 8638 self = 1; 8639 8640 if (dtrace_provider->dtpv_next != NULL) { 8641 /* 8642 * There's another provider here; return failure. 8643 */ 8644 return (EBUSY); 8645 } 8646 } else { 8647 mutex_enter(&dtrace_provider_lock); 8648 #ifdef illumos 8649 mutex_enter(&mod_lock); 8650 #endif 8651 mutex_enter(&dtrace_lock); 8652 } 8653 8654 /* 8655 * If anyone has /dev/dtrace open, or if there are anonymous enabled 8656 * probes, we refuse to let providers slither away, unless this 8657 * provider has already been explicitly invalidated. 8658 */ 8659 if (!old->dtpv_defunct && 8660 (dtrace_opens || (dtrace_anon.dta_state != NULL && 8661 dtrace_anon.dta_state->dts_necbs > 0))) { 8662 if (!self) { 8663 mutex_exit(&dtrace_lock); 8664 #ifdef illumos 8665 mutex_exit(&mod_lock); 8666 #endif 8667 mutex_exit(&dtrace_provider_lock); 8668 } 8669 return (EBUSY); 8670 } 8671 8672 /* 8673 * Attempt to destroy the probes associated with this provider. 8674 */ 8675 for (i = 0; i < dtrace_nprobes; i++) { 8676 if ((probe = dtrace_probes[i]) == NULL) 8677 continue; 8678 8679 if (probe->dtpr_provider != old) 8680 continue; 8681 8682 if (probe->dtpr_ecb == NULL) 8683 continue; 8684 8685 /* 8686 * If we are trying to unregister a defunct provider, and the 8687 * provider was made defunct within the interval dictated by 8688 * dtrace_unregister_defunct_reap, we'll (asynchronously) 8689 * attempt to reap our enablings. To denote that the provider 8690 * should reattempt to unregister itself at some point in the 8691 * future, we will return a differentiable error code (EAGAIN 8692 * instead of EBUSY) in this case. 8693 */ 8694 if (dtrace_gethrtime() - old->dtpv_defunct > 8695 dtrace_unregister_defunct_reap) 8696 noreap = 1; 8697 8698 if (!self) { 8699 mutex_exit(&dtrace_lock); 8700 #ifdef illumos 8701 mutex_exit(&mod_lock); 8702 #endif 8703 mutex_exit(&dtrace_provider_lock); 8704 } 8705 8706 if (noreap) 8707 return (EBUSY); 8708 8709 (void) taskq_dispatch(dtrace_taskq, 8710 (task_func_t *)dtrace_enabling_reap, NULL, TQ_SLEEP); 8711 8712 return (EAGAIN); 8713 } 8714 8715 /* 8716 * All of the probes for this provider are disabled; we can safely 8717 * remove all of them from their hash chains and from the probe array. 8718 */ 8719 for (i = 0; i < dtrace_nprobes; i++) { 8720 if ((probe = dtrace_probes[i]) == NULL) 8721 continue; 8722 8723 if (probe->dtpr_provider != old) 8724 continue; 8725 8726 dtrace_probes[i] = NULL; 8727 8728 dtrace_hash_remove(dtrace_bymod, probe); 8729 dtrace_hash_remove(dtrace_byfunc, probe); 8730 dtrace_hash_remove(dtrace_byname, probe); 8731 8732 if (first == NULL) { 8733 first = probe; 8734 probe->dtpr_nextmod = NULL; 8735 } else { 8736 probe->dtpr_nextmod = first; 8737 first = probe; 8738 } 8739 } 8740 8741 /* 8742 * The provider's probes have been removed from the hash chains and 8743 * from the probe array. Now issue a dtrace_sync() to be sure that 8744 * everyone has cleared out from any probe array processing. 8745 */ 8746 dtrace_sync(); 8747 8748 for (probe = first; probe != NULL; probe = first) { 8749 first = probe->dtpr_nextmod; 8750 8751 old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id, 8752 probe->dtpr_arg); 8753 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); 8754 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); 8755 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); 8756 #ifdef illumos 8757 vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1); 8758 #else 8759 free_unr(dtrace_arena, probe->dtpr_id); 8760 #endif 8761 kmem_free(probe, sizeof (dtrace_probe_t)); 8762 } 8763 8764 if ((prev = dtrace_provider) == old) { 8765 #ifdef illumos 8766 ASSERT(self || dtrace_devi == NULL); 8767 ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL); 8768 #endif 8769 dtrace_provider = old->dtpv_next; 8770 } else { 8771 while (prev != NULL && prev->dtpv_next != old) 8772 prev = prev->dtpv_next; 8773 8774 if (prev == NULL) { 8775 panic("attempt to unregister non-existent " 8776 "dtrace provider %p\n", (void *)id); 8777 } 8778 8779 prev->dtpv_next = old->dtpv_next; 8780 } 8781 8782 if (!self) { 8783 mutex_exit(&dtrace_lock); 8784 #ifdef illumos 8785 mutex_exit(&mod_lock); 8786 #endif 8787 mutex_exit(&dtrace_provider_lock); 8788 } 8789 8790 kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1); 8791 kmem_free(old, sizeof (dtrace_provider_t)); 8792 8793 return (0); 8794 } 8795 8796 /* 8797 * Invalidate the specified provider. All subsequent probe lookups for the 8798 * specified provider will fail, but its probes will not be removed. 8799 */ 8800 void 8801 dtrace_invalidate(dtrace_provider_id_t id) 8802 { 8803 dtrace_provider_t *pvp = (dtrace_provider_t *)id; 8804 8805 ASSERT(pvp->dtpv_pops.dtps_enable != 8806 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop); 8807 8808 mutex_enter(&dtrace_provider_lock); 8809 mutex_enter(&dtrace_lock); 8810 8811 pvp->dtpv_defunct = dtrace_gethrtime(); 8812 8813 mutex_exit(&dtrace_lock); 8814 mutex_exit(&dtrace_provider_lock); 8815 } 8816 8817 /* 8818 * Indicate whether or not DTrace has attached. 8819 */ 8820 int 8821 dtrace_attached(void) 8822 { 8823 /* 8824 * dtrace_provider will be non-NULL iff the DTrace driver has 8825 * attached. (It's non-NULL because DTrace is always itself a 8826 * provider.) 8827 */ 8828 return (dtrace_provider != NULL); 8829 } 8830 8831 /* 8832 * Remove all the unenabled probes for the given provider. This function is 8833 * not unlike dtrace_unregister(), except that it doesn't remove the provider 8834 * -- just as many of its associated probes as it can. 8835 */ 8836 int 8837 dtrace_condense(dtrace_provider_id_t id) 8838 { 8839 dtrace_provider_t *prov = (dtrace_provider_t *)id; 8840 int i; 8841 dtrace_probe_t *probe; 8842 8843 /* 8844 * Make sure this isn't the dtrace provider itself. 8845 */ 8846 ASSERT(prov->dtpv_pops.dtps_enable != 8847 (void (*)(void *, dtrace_id_t, void *))dtrace_nullop); 8848 8849 mutex_enter(&dtrace_provider_lock); 8850 mutex_enter(&dtrace_lock); 8851 8852 /* 8853 * Attempt to destroy the probes associated with this provider. 8854 */ 8855 for (i = 0; i < dtrace_nprobes; i++) { 8856 if ((probe = dtrace_probes[i]) == NULL) 8857 continue; 8858 8859 if (probe->dtpr_provider != prov) 8860 continue; 8861 8862 if (probe->dtpr_ecb != NULL) 8863 continue; 8864 8865 dtrace_probes[i] = NULL; 8866 8867 dtrace_hash_remove(dtrace_bymod, probe); 8868 dtrace_hash_remove(dtrace_byfunc, probe); 8869 dtrace_hash_remove(dtrace_byname, probe); 8870 8871 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1, 8872 probe->dtpr_arg); 8873 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); 8874 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); 8875 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); 8876 kmem_free(probe, sizeof (dtrace_probe_t)); 8877 #ifdef illumos 8878 vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1); 8879 #else 8880 free_unr(dtrace_arena, i + 1); 8881 #endif 8882 } 8883 8884 mutex_exit(&dtrace_lock); 8885 mutex_exit(&dtrace_provider_lock); 8886 8887 return (0); 8888 } 8889 8890 /* 8891 * DTrace Probe Management Functions 8892 * 8893 * The functions in this section perform the DTrace probe management, 8894 * including functions to create probes, look-up probes, and call into the 8895 * providers to request that probes be provided. Some of these functions are 8896 * in the Provider-to-Framework API; these functions can be identified by the 8897 * fact that they are not declared "static". 8898 */ 8899 8900 /* 8901 * Create a probe with the specified module name, function name, and name. 8902 */ 8903 dtrace_id_t 8904 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod, 8905 const char *func, const char *name, int aframes, void *arg) 8906 { 8907 dtrace_probe_t *probe, **probes; 8908 dtrace_provider_t *provider = (dtrace_provider_t *)prov; 8909 dtrace_id_t id; 8910 8911 if (provider == dtrace_provider) { 8912 ASSERT(MUTEX_HELD(&dtrace_lock)); 8913 } else { 8914 mutex_enter(&dtrace_lock); 8915 } 8916 8917 #ifdef illumos 8918 id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1, 8919 VM_BESTFIT | VM_SLEEP); 8920 #else 8921 id = alloc_unr(dtrace_arena); 8922 #endif 8923 probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP); 8924 8925 probe->dtpr_id = id; 8926 probe->dtpr_gen = dtrace_probegen++; 8927 probe->dtpr_mod = dtrace_strdup(mod); 8928 probe->dtpr_func = dtrace_strdup(func); 8929 probe->dtpr_name = dtrace_strdup(name); 8930 probe->dtpr_arg = arg; 8931 probe->dtpr_aframes = aframes; 8932 probe->dtpr_provider = provider; 8933 8934 dtrace_hash_add(dtrace_bymod, probe); 8935 dtrace_hash_add(dtrace_byfunc, probe); 8936 dtrace_hash_add(dtrace_byname, probe); 8937 8938 if (id - 1 >= dtrace_nprobes) { 8939 size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *); 8940 size_t nsize = osize << 1; 8941 8942 if (nsize == 0) { 8943 ASSERT(osize == 0); 8944 ASSERT(dtrace_probes == NULL); 8945 nsize = sizeof (dtrace_probe_t *); 8946 } 8947 8948 probes = kmem_zalloc(nsize, KM_SLEEP); 8949 8950 if (dtrace_probes == NULL) { 8951 ASSERT(osize == 0); 8952 dtrace_probes = probes; 8953 dtrace_nprobes = 1; 8954 } else { 8955 dtrace_probe_t **oprobes = dtrace_probes; 8956 8957 bcopy(oprobes, probes, osize); 8958 dtrace_membar_producer(); 8959 dtrace_probes = probes; 8960 8961 dtrace_sync(); 8962 8963 /* 8964 * All CPUs are now seeing the new probes array; we can 8965 * safely free the old array. 8966 */ 8967 kmem_free(oprobes, osize); 8968 dtrace_nprobes <<= 1; 8969 } 8970 8971 ASSERT(id - 1 < dtrace_nprobes); 8972 } 8973 8974 ASSERT(dtrace_probes[id - 1] == NULL); 8975 dtrace_probes[id - 1] = probe; 8976 8977 if (provider != dtrace_provider) 8978 mutex_exit(&dtrace_lock); 8979 8980 return (id); 8981 } 8982 8983 static dtrace_probe_t * 8984 dtrace_probe_lookup_id(dtrace_id_t id) 8985 { 8986 ASSERT(MUTEX_HELD(&dtrace_lock)); 8987 8988 if (id == 0 || id > dtrace_nprobes) 8989 return (NULL); 8990 8991 return (dtrace_probes[id - 1]); 8992 } 8993 8994 static int 8995 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg) 8996 { 8997 *((dtrace_id_t *)arg) = probe->dtpr_id; 8998 8999 return (DTRACE_MATCH_DONE); 9000 } 9001 9002 /* 9003 * Look up a probe based on provider and one or more of module name, function 9004 * name and probe name. 9005 */ 9006 dtrace_id_t 9007 dtrace_probe_lookup(dtrace_provider_id_t prid, char *mod, 9008 char *func, char *name) 9009 { 9010 dtrace_probekey_t pkey; 9011 dtrace_id_t id; 9012 int match; 9013 9014 pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name; 9015 pkey.dtpk_pmatch = &dtrace_match_string; 9016 pkey.dtpk_mod = mod; 9017 pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul; 9018 pkey.dtpk_func = func; 9019 pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul; 9020 pkey.dtpk_name = name; 9021 pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul; 9022 pkey.dtpk_id = DTRACE_IDNONE; 9023 9024 mutex_enter(&dtrace_lock); 9025 match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0, 0, 9026 dtrace_probe_lookup_match, &id); 9027 mutex_exit(&dtrace_lock); 9028 9029 ASSERT(match == 1 || match == 0); 9030 return (match ? id : 0); 9031 } 9032 9033 /* 9034 * Returns the probe argument associated with the specified probe. 9035 */ 9036 void * 9037 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid) 9038 { 9039 dtrace_probe_t *probe; 9040 void *rval = NULL; 9041 9042 mutex_enter(&dtrace_lock); 9043 9044 if ((probe = dtrace_probe_lookup_id(pid)) != NULL && 9045 probe->dtpr_provider == (dtrace_provider_t *)id) 9046 rval = probe->dtpr_arg; 9047 9048 mutex_exit(&dtrace_lock); 9049 9050 return (rval); 9051 } 9052 9053 /* 9054 * Copy a probe into a probe description. 9055 */ 9056 static void 9057 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp) 9058 { 9059 bzero(pdp, sizeof (dtrace_probedesc_t)); 9060 pdp->dtpd_id = prp->dtpr_id; 9061 9062 (void) strncpy(pdp->dtpd_provider, 9063 prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1); 9064 9065 (void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1); 9066 (void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1); 9067 (void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1); 9068 } 9069 9070 /* 9071 * Called to indicate that a probe -- or probes -- should be provided by a 9072 * specfied provider. If the specified description is NULL, the provider will 9073 * be told to provide all of its probes. (This is done whenever a new 9074 * consumer comes along, or whenever a retained enabling is to be matched.) If 9075 * the specified description is non-NULL, the provider is given the 9076 * opportunity to dynamically provide the specified probe, allowing providers 9077 * to support the creation of probes on-the-fly. (So-called _autocreated_ 9078 * probes.) If the provider is NULL, the operations will be applied to all 9079 * providers; if the provider is non-NULL the operations will only be applied 9080 * to the specified provider. The dtrace_provider_lock must be held, and the 9081 * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation 9082 * will need to grab the dtrace_lock when it reenters the framework through 9083 * dtrace_probe_lookup(), dtrace_probe_create(), etc. 9084 */ 9085 static void 9086 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv) 9087 { 9088 #ifdef illumos 9089 modctl_t *ctl; 9090 #endif 9091 int all = 0; 9092 9093 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 9094 9095 if (prv == NULL) { 9096 all = 1; 9097 prv = dtrace_provider; 9098 } 9099 9100 do { 9101 /* 9102 * First, call the blanket provide operation. 9103 */ 9104 prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc); 9105 9106 #ifdef illumos 9107 /* 9108 * Now call the per-module provide operation. We will grab 9109 * mod_lock to prevent the list from being modified. Note 9110 * that this also prevents the mod_busy bits from changing. 9111 * (mod_busy can only be changed with mod_lock held.) 9112 */ 9113 mutex_enter(&mod_lock); 9114 9115 ctl = &modules; 9116 do { 9117 if (ctl->mod_busy || ctl->mod_mp == NULL) 9118 continue; 9119 9120 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); 9121 9122 } while ((ctl = ctl->mod_next) != &modules); 9123 9124 mutex_exit(&mod_lock); 9125 #endif 9126 } while (all && (prv = prv->dtpv_next) != NULL); 9127 } 9128 9129 #ifdef illumos 9130 /* 9131 * Iterate over each probe, and call the Framework-to-Provider API function 9132 * denoted by offs. 9133 */ 9134 static void 9135 dtrace_probe_foreach(uintptr_t offs) 9136 { 9137 dtrace_provider_t *prov; 9138 void (*func)(void *, dtrace_id_t, void *); 9139 dtrace_probe_t *probe; 9140 dtrace_icookie_t cookie; 9141 int i; 9142 9143 /* 9144 * We disable interrupts to walk through the probe array. This is 9145 * safe -- the dtrace_sync() in dtrace_unregister() assures that we 9146 * won't see stale data. 9147 */ 9148 cookie = dtrace_interrupt_disable(); 9149 9150 for (i = 0; i < dtrace_nprobes; i++) { 9151 if ((probe = dtrace_probes[i]) == NULL) 9152 continue; 9153 9154 if (probe->dtpr_ecb == NULL) { 9155 /* 9156 * This probe isn't enabled -- don't call the function. 9157 */ 9158 continue; 9159 } 9160 9161 prov = probe->dtpr_provider; 9162 func = *((void(**)(void *, dtrace_id_t, void *)) 9163 ((uintptr_t)&prov->dtpv_pops + offs)); 9164 9165 func(prov->dtpv_arg, i + 1, probe->dtpr_arg); 9166 } 9167 9168 dtrace_interrupt_enable(cookie); 9169 } 9170 #endif 9171 9172 static int 9173 dtrace_probe_enable(dtrace_probedesc_t *desc, dtrace_enabling_t *enab) 9174 { 9175 dtrace_probekey_t pkey; 9176 uint32_t priv; 9177 uid_t uid; 9178 zoneid_t zoneid; 9179 9180 ASSERT(MUTEX_HELD(&dtrace_lock)); 9181 dtrace_ecb_create_cache = NULL; 9182 9183 if (desc == NULL) { 9184 /* 9185 * If we're passed a NULL description, we're being asked to 9186 * create an ECB with a NULL probe. 9187 */ 9188 (void) dtrace_ecb_create_enable(NULL, enab); 9189 return (0); 9190 } 9191 9192 dtrace_probekey(desc, &pkey); 9193 dtrace_cred2priv(enab->dten_vstate->dtvs_state->dts_cred.dcr_cred, 9194 &priv, &uid, &zoneid); 9195 9196 return (dtrace_match(&pkey, priv, uid, zoneid, dtrace_ecb_create_enable, 9197 enab)); 9198 } 9199 9200 /* 9201 * DTrace Helper Provider Functions 9202 */ 9203 static void 9204 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr) 9205 { 9206 attr->dtat_name = DOF_ATTR_NAME(dofattr); 9207 attr->dtat_data = DOF_ATTR_DATA(dofattr); 9208 attr->dtat_class = DOF_ATTR_CLASS(dofattr); 9209 } 9210 9211 static void 9212 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov, 9213 const dof_provider_t *dofprov, char *strtab) 9214 { 9215 hprov->dthpv_provname = strtab + dofprov->dofpv_name; 9216 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider, 9217 dofprov->dofpv_provattr); 9218 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod, 9219 dofprov->dofpv_modattr); 9220 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func, 9221 dofprov->dofpv_funcattr); 9222 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name, 9223 dofprov->dofpv_nameattr); 9224 dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args, 9225 dofprov->dofpv_argsattr); 9226 } 9227 9228 static void 9229 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid) 9230 { 9231 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 9232 dof_hdr_t *dof = (dof_hdr_t *)daddr; 9233 dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; 9234 dof_provider_t *provider; 9235 dof_probe_t *probe; 9236 uint32_t *off, *enoff; 9237 uint8_t *arg; 9238 char *strtab; 9239 uint_t i, nprobes; 9240 dtrace_helper_provdesc_t dhpv; 9241 dtrace_helper_probedesc_t dhpb; 9242 dtrace_meta_t *meta = dtrace_meta_pid; 9243 dtrace_mops_t *mops = &meta->dtm_mops; 9244 void *parg; 9245 9246 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); 9247 str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 9248 provider->dofpv_strtab * dof->dofh_secsize); 9249 prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 9250 provider->dofpv_probes * dof->dofh_secsize); 9251 arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 9252 provider->dofpv_prargs * dof->dofh_secsize); 9253 off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 9254 provider->dofpv_proffs * dof->dofh_secsize); 9255 9256 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); 9257 off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset); 9258 arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); 9259 enoff = NULL; 9260 9261 /* 9262 * See dtrace_helper_provider_validate(). 9263 */ 9264 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && 9265 provider->dofpv_prenoffs != DOF_SECT_NONE) { 9266 enoff_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 9267 provider->dofpv_prenoffs * dof->dofh_secsize); 9268 enoff = (uint32_t *)(uintptr_t)(daddr + enoff_sec->dofs_offset); 9269 } 9270 9271 nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; 9272 9273 /* 9274 * Create the provider. 9275 */ 9276 dtrace_dofprov2hprov(&dhpv, provider, strtab); 9277 9278 if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL) 9279 return; 9280 9281 meta->dtm_count++; 9282 9283 /* 9284 * Create the probes. 9285 */ 9286 for (i = 0; i < nprobes; i++) { 9287 probe = (dof_probe_t *)(uintptr_t)(daddr + 9288 prb_sec->dofs_offset + i * prb_sec->dofs_entsize); 9289 9290 dhpb.dthpb_mod = dhp->dofhp_mod; 9291 dhpb.dthpb_func = strtab + probe->dofpr_func; 9292 dhpb.dthpb_name = strtab + probe->dofpr_name; 9293 dhpb.dthpb_base = probe->dofpr_addr; 9294 dhpb.dthpb_offs = off + probe->dofpr_offidx; 9295 dhpb.dthpb_noffs = probe->dofpr_noffs; 9296 if (enoff != NULL) { 9297 dhpb.dthpb_enoffs = enoff + probe->dofpr_enoffidx; 9298 dhpb.dthpb_nenoffs = probe->dofpr_nenoffs; 9299 } else { 9300 dhpb.dthpb_enoffs = NULL; 9301 dhpb.dthpb_nenoffs = 0; 9302 } 9303 dhpb.dthpb_args = arg + probe->dofpr_argidx; 9304 dhpb.dthpb_nargc = probe->dofpr_nargc; 9305 dhpb.dthpb_xargc = probe->dofpr_xargc; 9306 dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv; 9307 dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv; 9308 9309 mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb); 9310 } 9311 } 9312 9313 static void 9314 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid) 9315 { 9316 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 9317 dof_hdr_t *dof = (dof_hdr_t *)daddr; 9318 int i; 9319 9320 ASSERT(MUTEX_HELD(&dtrace_meta_lock)); 9321 9322 for (i = 0; i < dof->dofh_secnum; i++) { 9323 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + 9324 dof->dofh_secoff + i * dof->dofh_secsize); 9325 9326 if (sec->dofs_type != DOF_SECT_PROVIDER) 9327 continue; 9328 9329 dtrace_helper_provide_one(dhp, sec, pid); 9330 } 9331 9332 /* 9333 * We may have just created probes, so we must now rematch against 9334 * any retained enablings. Note that this call will acquire both 9335 * cpu_lock and dtrace_lock; the fact that we are holding 9336 * dtrace_meta_lock now is what defines the ordering with respect to 9337 * these three locks. 9338 */ 9339 dtrace_enabling_matchall(); 9340 } 9341 9342 static void 9343 dtrace_helper_provider_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid) 9344 { 9345 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 9346 dof_hdr_t *dof = (dof_hdr_t *)daddr; 9347 dof_sec_t *str_sec; 9348 dof_provider_t *provider; 9349 char *strtab; 9350 dtrace_helper_provdesc_t dhpv; 9351 dtrace_meta_t *meta = dtrace_meta_pid; 9352 dtrace_mops_t *mops = &meta->dtm_mops; 9353 9354 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); 9355 str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff + 9356 provider->dofpv_strtab * dof->dofh_secsize); 9357 9358 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); 9359 9360 /* 9361 * Create the provider. 9362 */ 9363 dtrace_dofprov2hprov(&dhpv, provider, strtab); 9364 9365 mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid); 9366 9367 meta->dtm_count--; 9368 } 9369 9370 static void 9371 dtrace_helper_provider_remove(dof_helper_t *dhp, pid_t pid) 9372 { 9373 uintptr_t daddr = (uintptr_t)dhp->dofhp_dof; 9374 dof_hdr_t *dof = (dof_hdr_t *)daddr; 9375 int i; 9376 9377 ASSERT(MUTEX_HELD(&dtrace_meta_lock)); 9378 9379 for (i = 0; i < dof->dofh_secnum; i++) { 9380 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + 9381 dof->dofh_secoff + i * dof->dofh_secsize); 9382 9383 if (sec->dofs_type != DOF_SECT_PROVIDER) 9384 continue; 9385 9386 dtrace_helper_provider_remove_one(dhp, sec, pid); 9387 } 9388 } 9389 9390 /* 9391 * DTrace Meta Provider-to-Framework API Functions 9392 * 9393 * These functions implement the Meta Provider-to-Framework API, as described 9394 * in <sys/dtrace.h>. 9395 */ 9396 int 9397 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg, 9398 dtrace_meta_provider_id_t *idp) 9399 { 9400 dtrace_meta_t *meta; 9401 dtrace_helpers_t *help, *next; 9402 int i; 9403 9404 *idp = DTRACE_METAPROVNONE; 9405 9406 /* 9407 * We strictly don't need the name, but we hold onto it for 9408 * debuggability. All hail error queues! 9409 */ 9410 if (name == NULL) { 9411 cmn_err(CE_WARN, "failed to register meta-provider: " 9412 "invalid name"); 9413 return (EINVAL); 9414 } 9415 9416 if (mops == NULL || 9417 mops->dtms_create_probe == NULL || 9418 mops->dtms_provide_pid == NULL || 9419 mops->dtms_remove_pid == NULL) { 9420 cmn_err(CE_WARN, "failed to register meta-register %s: " 9421 "invalid ops", name); 9422 return (EINVAL); 9423 } 9424 9425 meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP); 9426 meta->dtm_mops = *mops; 9427 meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP); 9428 (void) strcpy(meta->dtm_name, name); 9429 meta->dtm_arg = arg; 9430 9431 mutex_enter(&dtrace_meta_lock); 9432 mutex_enter(&dtrace_lock); 9433 9434 if (dtrace_meta_pid != NULL) { 9435 mutex_exit(&dtrace_lock); 9436 mutex_exit(&dtrace_meta_lock); 9437 cmn_err(CE_WARN, "failed to register meta-register %s: " 9438 "user-land meta-provider exists", name); 9439 kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1); 9440 kmem_free(meta, sizeof (dtrace_meta_t)); 9441 return (EINVAL); 9442 } 9443 9444 dtrace_meta_pid = meta; 9445 *idp = (dtrace_meta_provider_id_t)meta; 9446 9447 /* 9448 * If there are providers and probes ready to go, pass them 9449 * off to the new meta provider now. 9450 */ 9451 9452 help = dtrace_deferred_pid; 9453 dtrace_deferred_pid = NULL; 9454 9455 mutex_exit(&dtrace_lock); 9456 9457 while (help != NULL) { 9458 for (i = 0; i < help->dthps_nprovs; i++) { 9459 dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, 9460 help->dthps_pid); 9461 } 9462 9463 next = help->dthps_next; 9464 help->dthps_next = NULL; 9465 help->dthps_prev = NULL; 9466 help->dthps_deferred = 0; 9467 help = next; 9468 } 9469 9470 mutex_exit(&dtrace_meta_lock); 9471 9472 return (0); 9473 } 9474 9475 int 9476 dtrace_meta_unregister(dtrace_meta_provider_id_t id) 9477 { 9478 dtrace_meta_t **pp, *old = (dtrace_meta_t *)id; 9479 9480 mutex_enter(&dtrace_meta_lock); 9481 mutex_enter(&dtrace_lock); 9482 9483 if (old == dtrace_meta_pid) { 9484 pp = &dtrace_meta_pid; 9485 } else { 9486 panic("attempt to unregister non-existent " 9487 "dtrace meta-provider %p\n", (void *)old); 9488 } 9489 9490 if (old->dtm_count != 0) { 9491 mutex_exit(&dtrace_lock); 9492 mutex_exit(&dtrace_meta_lock); 9493 return (EBUSY); 9494 } 9495 9496 *pp = NULL; 9497 9498 mutex_exit(&dtrace_lock); 9499 mutex_exit(&dtrace_meta_lock); 9500 9501 kmem_free(old->dtm_name, strlen(old->dtm_name) + 1); 9502 kmem_free(old, sizeof (dtrace_meta_t)); 9503 9504 return (0); 9505 } 9506 9507 9508 /* 9509 * DTrace DIF Object Functions 9510 */ 9511 static int 9512 dtrace_difo_err(uint_t pc, const char *format, ...) 9513 { 9514 if (dtrace_err_verbose) { 9515 va_list alist; 9516 9517 (void) uprintf("dtrace DIF object error: [%u]: ", pc); 9518 va_start(alist, format); 9519 (void) vuprintf(format, alist); 9520 va_end(alist); 9521 } 9522 9523 #ifdef DTRACE_ERRDEBUG 9524 dtrace_errdebug(format); 9525 #endif 9526 return (1); 9527 } 9528 9529 /* 9530 * Validate a DTrace DIF object by checking the IR instructions. The following 9531 * rules are currently enforced by dtrace_difo_validate(): 9532 * 9533 * 1. Each instruction must have a valid opcode 9534 * 2. Each register, string, variable, or subroutine reference must be valid 9535 * 3. No instruction can modify register %r0 (must be zero) 9536 * 4. All instruction reserved bits must be set to zero 9537 * 5. The last instruction must be a "ret" instruction 9538 * 6. All branch targets must reference a valid instruction _after_ the branch 9539 */ 9540 static int 9541 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs, 9542 cred_t *cr) 9543 { 9544 int err = 0, i; 9545 int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; 9546 int kcheckload; 9547 uint_t pc; 9548 9549 kcheckload = cr == NULL || 9550 (vstate->dtvs_state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) == 0; 9551 9552 dp->dtdo_destructive = 0; 9553 9554 for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) { 9555 dif_instr_t instr = dp->dtdo_buf[pc]; 9556 9557 uint_t r1 = DIF_INSTR_R1(instr); 9558 uint_t r2 = DIF_INSTR_R2(instr); 9559 uint_t rd = DIF_INSTR_RD(instr); 9560 uint_t rs = DIF_INSTR_RS(instr); 9561 uint_t label = DIF_INSTR_LABEL(instr); 9562 uint_t v = DIF_INSTR_VAR(instr); 9563 uint_t subr = DIF_INSTR_SUBR(instr); 9564 uint_t type = DIF_INSTR_TYPE(instr); 9565 uint_t op = DIF_INSTR_OP(instr); 9566 9567 switch (op) { 9568 case DIF_OP_OR: 9569 case DIF_OP_XOR: 9570 case DIF_OP_AND: 9571 case DIF_OP_SLL: 9572 case DIF_OP_SRL: 9573 case DIF_OP_SRA: 9574 case DIF_OP_SUB: 9575 case DIF_OP_ADD: 9576 case DIF_OP_MUL: 9577 case DIF_OP_SDIV: 9578 case DIF_OP_UDIV: 9579 case DIF_OP_SREM: 9580 case DIF_OP_UREM: 9581 case DIF_OP_COPYS: 9582 if (r1 >= nregs) 9583 err += efunc(pc, "invalid register %u\n", r1); 9584 if (r2 >= nregs) 9585 err += efunc(pc, "invalid register %u\n", r2); 9586 if (rd >= nregs) 9587 err += efunc(pc, "invalid register %u\n", rd); 9588 if (rd == 0) 9589 err += efunc(pc, "cannot write to %r0\n"); 9590 break; 9591 case DIF_OP_NOT: 9592 case DIF_OP_MOV: 9593 case DIF_OP_ALLOCS: 9594 if (r1 >= nregs) 9595 err += efunc(pc, "invalid register %u\n", r1); 9596 if (r2 != 0) 9597 err += efunc(pc, "non-zero reserved bits\n"); 9598 if (rd >= nregs) 9599 err += efunc(pc, "invalid register %u\n", rd); 9600 if (rd == 0) 9601 err += efunc(pc, "cannot write to %r0\n"); 9602 break; 9603 case DIF_OP_LDSB: 9604 case DIF_OP_LDSH: 9605 case DIF_OP_LDSW: 9606 case DIF_OP_LDUB: 9607 case DIF_OP_LDUH: 9608 case DIF_OP_LDUW: 9609 case DIF_OP_LDX: 9610 if (r1 >= nregs) 9611 err += efunc(pc, "invalid register %u\n", r1); 9612 if (r2 != 0) 9613 err += efunc(pc, "non-zero reserved bits\n"); 9614 if (rd >= nregs) 9615 err += efunc(pc, "invalid register %u\n", rd); 9616 if (rd == 0) 9617 err += efunc(pc, "cannot write to %r0\n"); 9618 if (kcheckload) 9619 dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op + 9620 DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd); 9621 break; 9622 case DIF_OP_RLDSB: 9623 case DIF_OP_RLDSH: 9624 case DIF_OP_RLDSW: 9625 case DIF_OP_RLDUB: 9626 case DIF_OP_RLDUH: 9627 case DIF_OP_RLDUW: 9628 case DIF_OP_RLDX: 9629 if (r1 >= nregs) 9630 err += efunc(pc, "invalid register %u\n", r1); 9631 if (r2 != 0) 9632 err += efunc(pc, "non-zero reserved bits\n"); 9633 if (rd >= nregs) 9634 err += efunc(pc, "invalid register %u\n", rd); 9635 if (rd == 0) 9636 err += efunc(pc, "cannot write to %r0\n"); 9637 break; 9638 case DIF_OP_ULDSB: 9639 case DIF_OP_ULDSH: 9640 case DIF_OP_ULDSW: 9641 case DIF_OP_ULDUB: 9642 case DIF_OP_ULDUH: 9643 case DIF_OP_ULDUW: 9644 case DIF_OP_ULDX: 9645 if (r1 >= nregs) 9646 err += efunc(pc, "invalid register %u\n", r1); 9647 if (r2 != 0) 9648 err += efunc(pc, "non-zero reserved bits\n"); 9649 if (rd >= nregs) 9650 err += efunc(pc, "invalid register %u\n", rd); 9651 if (rd == 0) 9652 err += efunc(pc, "cannot write to %r0\n"); 9653 break; 9654 case DIF_OP_STB: 9655 case DIF_OP_STH: 9656 case DIF_OP_STW: 9657 case DIF_OP_STX: 9658 if (r1 >= nregs) 9659 err += efunc(pc, "invalid register %u\n", r1); 9660 if (r2 != 0) 9661 err += efunc(pc, "non-zero reserved bits\n"); 9662 if (rd >= nregs) 9663 err += efunc(pc, "invalid register %u\n", rd); 9664 if (rd == 0) 9665 err += efunc(pc, "cannot write to 0 address\n"); 9666 break; 9667 case DIF_OP_CMP: 9668 case DIF_OP_SCMP: 9669 if (r1 >= nregs) 9670 err += efunc(pc, "invalid register %u\n", r1); 9671 if (r2 >= nregs) 9672 err += efunc(pc, "invalid register %u\n", r2); 9673 if (rd != 0) 9674 err += efunc(pc, "non-zero reserved bits\n"); 9675 break; 9676 case DIF_OP_TST: 9677 if (r1 >= nregs) 9678 err += efunc(pc, "invalid register %u\n", r1); 9679 if (r2 != 0 || rd != 0) 9680 err += efunc(pc, "non-zero reserved bits\n"); 9681 break; 9682 case DIF_OP_BA: 9683 case DIF_OP_BE: 9684 case DIF_OP_BNE: 9685 case DIF_OP_BG: 9686 case DIF_OP_BGU: 9687 case DIF_OP_BGE: 9688 case DIF_OP_BGEU: 9689 case DIF_OP_BL: 9690 case DIF_OP_BLU: 9691 case DIF_OP_BLE: 9692 case DIF_OP_BLEU: 9693 if (label >= dp->dtdo_len) { 9694 err += efunc(pc, "invalid branch target %u\n", 9695 label); 9696 } 9697 if (label <= pc) { 9698 err += efunc(pc, "backward branch to %u\n", 9699 label); 9700 } 9701 break; 9702 case DIF_OP_RET: 9703 if (r1 != 0 || r2 != 0) 9704 err += efunc(pc, "non-zero reserved bits\n"); 9705 if (rd >= nregs) 9706 err += efunc(pc, "invalid register %u\n", rd); 9707 break; 9708 case DIF_OP_NOP: 9709 case DIF_OP_POPTS: 9710 case DIF_OP_FLUSHTS: 9711 if (r1 != 0 || r2 != 0 || rd != 0) 9712 err += efunc(pc, "non-zero reserved bits\n"); 9713 break; 9714 case DIF_OP_SETX: 9715 if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) { 9716 err += efunc(pc, "invalid integer ref %u\n", 9717 DIF_INSTR_INTEGER(instr)); 9718 } 9719 if (rd >= nregs) 9720 err += efunc(pc, "invalid register %u\n", rd); 9721 if (rd == 0) 9722 err += efunc(pc, "cannot write to %r0\n"); 9723 break; 9724 case DIF_OP_SETS: 9725 if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) { 9726 err += efunc(pc, "invalid string ref %u\n", 9727 DIF_INSTR_STRING(instr)); 9728 } 9729 if (rd >= nregs) 9730 err += efunc(pc, "invalid register %u\n", rd); 9731 if (rd == 0) 9732 err += efunc(pc, "cannot write to %r0\n"); 9733 break; 9734 case DIF_OP_LDGA: 9735 case DIF_OP_LDTA: 9736 if (r1 > DIF_VAR_ARRAY_MAX) 9737 err += efunc(pc, "invalid array %u\n", r1); 9738 if (r2 >= nregs) 9739 err += efunc(pc, "invalid register %u\n", r2); 9740 if (rd >= nregs) 9741 err += efunc(pc, "invalid register %u\n", rd); 9742 if (rd == 0) 9743 err += efunc(pc, "cannot write to %r0\n"); 9744 break; 9745 case DIF_OP_LDGS: 9746 case DIF_OP_LDTS: 9747 case DIF_OP_LDLS: 9748 case DIF_OP_LDGAA: 9749 case DIF_OP_LDTAA: 9750 if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX) 9751 err += efunc(pc, "invalid variable %u\n", v); 9752 if (rd >= nregs) 9753 err += efunc(pc, "invalid register %u\n", rd); 9754 if (rd == 0) 9755 err += efunc(pc, "cannot write to %r0\n"); 9756 break; 9757 case DIF_OP_STGS: 9758 case DIF_OP_STTS: 9759 case DIF_OP_STLS: 9760 case DIF_OP_STGAA: 9761 case DIF_OP_STTAA: 9762 if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX) 9763 err += efunc(pc, "invalid variable %u\n", v); 9764 if (rs >= nregs) 9765 err += efunc(pc, "invalid register %u\n", rd); 9766 break; 9767 case DIF_OP_CALL: 9768 if (subr > DIF_SUBR_MAX) 9769 err += efunc(pc, "invalid subr %u\n", subr); 9770 if (rd >= nregs) 9771 err += efunc(pc, "invalid register %u\n", rd); 9772 if (rd == 0) 9773 err += efunc(pc, "cannot write to %r0\n"); 9774 9775 if (subr == DIF_SUBR_COPYOUT || 9776 subr == DIF_SUBR_COPYOUTSTR) { 9777 dp->dtdo_destructive = 1; 9778 } 9779 9780 if (subr == DIF_SUBR_GETF) { 9781 /* 9782 * If we have a getf() we need to record that 9783 * in our state. Note that our state can be 9784 * NULL if this is a helper -- but in that 9785 * case, the call to getf() is itself illegal, 9786 * and will be caught (slightly later) when 9787 * the helper is validated. 9788 */ 9789 if (vstate->dtvs_state != NULL) 9790 vstate->dtvs_state->dts_getf++; 9791 } 9792 9793 break; 9794 case DIF_OP_PUSHTR: 9795 if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF) 9796 err += efunc(pc, "invalid ref type %u\n", type); 9797 if (r2 >= nregs) 9798 err += efunc(pc, "invalid register %u\n", r2); 9799 if (rs >= nregs) 9800 err += efunc(pc, "invalid register %u\n", rs); 9801 break; 9802 case DIF_OP_PUSHTV: 9803 if (type != DIF_TYPE_CTF) 9804 err += efunc(pc, "invalid val type %u\n", type); 9805 if (r2 >= nregs) 9806 err += efunc(pc, "invalid register %u\n", r2); 9807 if (rs >= nregs) 9808 err += efunc(pc, "invalid register %u\n", rs); 9809 break; 9810 default: 9811 err += efunc(pc, "invalid opcode %u\n", 9812 DIF_INSTR_OP(instr)); 9813 } 9814 } 9815 9816 if (dp->dtdo_len != 0 && 9817 DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) { 9818 err += efunc(dp->dtdo_len - 1, 9819 "expected 'ret' as last DIF instruction\n"); 9820 } 9821 9822 if (!(dp->dtdo_rtype.dtdt_flags & (DIF_TF_BYREF | DIF_TF_BYUREF))) { 9823 /* 9824 * If we're not returning by reference, the size must be either 9825 * 0 or the size of one of the base types. 9826 */ 9827 switch (dp->dtdo_rtype.dtdt_size) { 9828 case 0: 9829 case sizeof (uint8_t): 9830 case sizeof (uint16_t): 9831 case sizeof (uint32_t): 9832 case sizeof (uint64_t): 9833 break; 9834 9835 default: 9836 err += efunc(dp->dtdo_len - 1, "bad return size\n"); 9837 } 9838 } 9839 9840 for (i = 0; i < dp->dtdo_varlen && err == 0; i++) { 9841 dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL; 9842 dtrace_diftype_t *vt, *et; 9843 uint_t id, ndx; 9844 9845 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL && 9846 v->dtdv_scope != DIFV_SCOPE_THREAD && 9847 v->dtdv_scope != DIFV_SCOPE_LOCAL) { 9848 err += efunc(i, "unrecognized variable scope %d\n", 9849 v->dtdv_scope); 9850 break; 9851 } 9852 9853 if (v->dtdv_kind != DIFV_KIND_ARRAY && 9854 v->dtdv_kind != DIFV_KIND_SCALAR) { 9855 err += efunc(i, "unrecognized variable type %d\n", 9856 v->dtdv_kind); 9857 break; 9858 } 9859 9860 if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) { 9861 err += efunc(i, "%d exceeds variable id limit\n", id); 9862 break; 9863 } 9864 9865 if (id < DIF_VAR_OTHER_UBASE) 9866 continue; 9867 9868 /* 9869 * For user-defined variables, we need to check that this 9870 * definition is identical to any previous definition that we 9871 * encountered. 9872 */ 9873 ndx = id - DIF_VAR_OTHER_UBASE; 9874 9875 switch (v->dtdv_scope) { 9876 case DIFV_SCOPE_GLOBAL: 9877 if (ndx < vstate->dtvs_nglobals) { 9878 dtrace_statvar_t *svar; 9879 9880 if ((svar = vstate->dtvs_globals[ndx]) != NULL) 9881 existing = &svar->dtsv_var; 9882 } 9883 9884 break; 9885 9886 case DIFV_SCOPE_THREAD: 9887 if (ndx < vstate->dtvs_ntlocals) 9888 existing = &vstate->dtvs_tlocals[ndx]; 9889 break; 9890 9891 case DIFV_SCOPE_LOCAL: 9892 if (ndx < vstate->dtvs_nlocals) { 9893 dtrace_statvar_t *svar; 9894 9895 if ((svar = vstate->dtvs_locals[ndx]) != NULL) 9896 existing = &svar->dtsv_var; 9897 } 9898 9899 break; 9900 } 9901 9902 vt = &v->dtdv_type; 9903 9904 if (vt->dtdt_flags & DIF_TF_BYREF) { 9905 if (vt->dtdt_size == 0) { 9906 err += efunc(i, "zero-sized variable\n"); 9907 break; 9908 } 9909 9910 if (v->dtdv_scope == DIFV_SCOPE_GLOBAL && 9911 vt->dtdt_size > dtrace_global_maxsize) { 9912 err += efunc(i, "oversized by-ref global\n"); 9913 break; 9914 } 9915 } 9916 9917 if (existing == NULL || existing->dtdv_id == 0) 9918 continue; 9919 9920 ASSERT(existing->dtdv_id == v->dtdv_id); 9921 ASSERT(existing->dtdv_scope == v->dtdv_scope); 9922 9923 if (existing->dtdv_kind != v->dtdv_kind) 9924 err += efunc(i, "%d changed variable kind\n", id); 9925 9926 et = &existing->dtdv_type; 9927 9928 if (vt->dtdt_flags != et->dtdt_flags) { 9929 err += efunc(i, "%d changed variable type flags\n", id); 9930 break; 9931 } 9932 9933 if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) { 9934 err += efunc(i, "%d changed variable type size\n", id); 9935 break; 9936 } 9937 } 9938 9939 return (err); 9940 } 9941 9942 /* 9943 * Validate a DTrace DIF object that it is to be used as a helper. Helpers 9944 * are much more constrained than normal DIFOs. Specifically, they may 9945 * not: 9946 * 9947 * 1. Make calls to subroutines other than copyin(), copyinstr() or 9948 * miscellaneous string routines 9949 * 2. Access DTrace variables other than the args[] array, and the 9950 * curthread, pid, ppid, tid, execname, zonename, uid and gid variables. 9951 * 3. Have thread-local variables. 9952 * 4. Have dynamic variables. 9953 */ 9954 static int 9955 dtrace_difo_validate_helper(dtrace_difo_t *dp) 9956 { 9957 int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err; 9958 int err = 0; 9959 uint_t pc; 9960 9961 for (pc = 0; pc < dp->dtdo_len; pc++) { 9962 dif_instr_t instr = dp->dtdo_buf[pc]; 9963 9964 uint_t v = DIF_INSTR_VAR(instr); 9965 uint_t subr = DIF_INSTR_SUBR(instr); 9966 uint_t op = DIF_INSTR_OP(instr); 9967 9968 switch (op) { 9969 case DIF_OP_OR: 9970 case DIF_OP_XOR: 9971 case DIF_OP_AND: 9972 case DIF_OP_SLL: 9973 case DIF_OP_SRL: 9974 case DIF_OP_SRA: 9975 case DIF_OP_SUB: 9976 case DIF_OP_ADD: 9977 case DIF_OP_MUL: 9978 case DIF_OP_SDIV: 9979 case DIF_OP_UDIV: 9980 case DIF_OP_SREM: 9981 case DIF_OP_UREM: 9982 case DIF_OP_COPYS: 9983 case DIF_OP_NOT: 9984 case DIF_OP_MOV: 9985 case DIF_OP_RLDSB: 9986 case DIF_OP_RLDSH: 9987 case DIF_OP_RLDSW: 9988 case DIF_OP_RLDUB: 9989 case DIF_OP_RLDUH: 9990 case DIF_OP_RLDUW: 9991 case DIF_OP_RLDX: 9992 case DIF_OP_ULDSB: 9993 case DIF_OP_ULDSH: 9994 case DIF_OP_ULDSW: 9995 case DIF_OP_ULDUB: 9996 case DIF_OP_ULDUH: 9997 case DIF_OP_ULDUW: 9998 case DIF_OP_ULDX: 9999 case DIF_OP_STB: 10000 case DIF_OP_STH: 10001 case DIF_OP_STW: 10002 case DIF_OP_STX: 10003 case DIF_OP_ALLOCS: 10004 case DIF_OP_CMP: 10005 case DIF_OP_SCMP: 10006 case DIF_OP_TST: 10007 case DIF_OP_BA: 10008 case DIF_OP_BE: 10009 case DIF_OP_BNE: 10010 case DIF_OP_BG: 10011 case DIF_OP_BGU: 10012 case DIF_OP_BGE: 10013 case DIF_OP_BGEU: 10014 case DIF_OP_BL: 10015 case DIF_OP_BLU: 10016 case DIF_OP_BLE: 10017 case DIF_OP_BLEU: 10018 case DIF_OP_RET: 10019 case DIF_OP_NOP: 10020 case DIF_OP_POPTS: 10021 case DIF_OP_FLUSHTS: 10022 case DIF_OP_SETX: 10023 case DIF_OP_SETS: 10024 case DIF_OP_LDGA: 10025 case DIF_OP_LDLS: 10026 case DIF_OP_STGS: 10027 case DIF_OP_STLS: 10028 case DIF_OP_PUSHTR: 10029 case DIF_OP_PUSHTV: 10030 break; 10031 10032 case DIF_OP_LDGS: 10033 if (v >= DIF_VAR_OTHER_UBASE) 10034 break; 10035 10036 if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) 10037 break; 10038 10039 if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID || 10040 v == DIF_VAR_PPID || v == DIF_VAR_TID || 10041 v == DIF_VAR_EXECARGS || 10042 v == DIF_VAR_EXECNAME || v == DIF_VAR_ZONENAME || 10043 v == DIF_VAR_UID || v == DIF_VAR_GID) 10044 break; 10045 10046 err += efunc(pc, "illegal variable %u\n", v); 10047 break; 10048 10049 case DIF_OP_LDTA: 10050 case DIF_OP_LDTS: 10051 case DIF_OP_LDGAA: 10052 case DIF_OP_LDTAA: 10053 err += efunc(pc, "illegal dynamic variable load\n"); 10054 break; 10055 10056 case DIF_OP_STTS: 10057 case DIF_OP_STGAA: 10058 case DIF_OP_STTAA: 10059 err += efunc(pc, "illegal dynamic variable store\n"); 10060 break; 10061 10062 case DIF_OP_CALL: 10063 if (subr == DIF_SUBR_ALLOCA || 10064 subr == DIF_SUBR_BCOPY || 10065 subr == DIF_SUBR_COPYIN || 10066 subr == DIF_SUBR_COPYINTO || 10067 subr == DIF_SUBR_COPYINSTR || 10068 subr == DIF_SUBR_INDEX || 10069 subr == DIF_SUBR_INET_NTOA || 10070 subr == DIF_SUBR_INET_NTOA6 || 10071 subr == DIF_SUBR_INET_NTOP || 10072 subr == DIF_SUBR_JSON || 10073 subr == DIF_SUBR_LLTOSTR || 10074 subr == DIF_SUBR_STRTOLL || 10075 subr == DIF_SUBR_RINDEX || 10076 subr == DIF_SUBR_STRCHR || 10077 subr == DIF_SUBR_STRJOIN || 10078 subr == DIF_SUBR_STRRCHR || 10079 subr == DIF_SUBR_STRSTR || 10080 subr == DIF_SUBR_HTONS || 10081 subr == DIF_SUBR_HTONL || 10082 subr == DIF_SUBR_HTONLL || 10083 subr == DIF_SUBR_NTOHS || 10084 subr == DIF_SUBR_NTOHL || 10085 subr == DIF_SUBR_NTOHLL || 10086 subr == DIF_SUBR_MEMREF || 10087 #ifndef illumos 10088 subr == DIF_SUBR_MEMSTR || 10089 #endif 10090 subr == DIF_SUBR_TYPEREF) 10091 break; 10092 10093 err += efunc(pc, "invalid subr %u\n", subr); 10094 break; 10095 10096 default: 10097 err += efunc(pc, "invalid opcode %u\n", 10098 DIF_INSTR_OP(instr)); 10099 } 10100 } 10101 10102 return (err); 10103 } 10104 10105 /* 10106 * Returns 1 if the expression in the DIF object can be cached on a per-thread 10107 * basis; 0 if not. 10108 */ 10109 static int 10110 dtrace_difo_cacheable(dtrace_difo_t *dp) 10111 { 10112 int i; 10113 10114 if (dp == NULL) 10115 return (0); 10116 10117 for (i = 0; i < dp->dtdo_varlen; i++) { 10118 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10119 10120 if (v->dtdv_scope != DIFV_SCOPE_GLOBAL) 10121 continue; 10122 10123 switch (v->dtdv_id) { 10124 case DIF_VAR_CURTHREAD: 10125 case DIF_VAR_PID: 10126 case DIF_VAR_TID: 10127 case DIF_VAR_EXECARGS: 10128 case DIF_VAR_EXECNAME: 10129 case DIF_VAR_ZONENAME: 10130 break; 10131 10132 default: 10133 return (0); 10134 } 10135 } 10136 10137 /* 10138 * This DIF object may be cacheable. Now we need to look for any 10139 * array loading instructions, any memory loading instructions, or 10140 * any stores to thread-local variables. 10141 */ 10142 for (i = 0; i < dp->dtdo_len; i++) { 10143 uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]); 10144 10145 if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) || 10146 (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) || 10147 (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) || 10148 op == DIF_OP_LDGA || op == DIF_OP_STTS) 10149 return (0); 10150 } 10151 10152 return (1); 10153 } 10154 10155 static void 10156 dtrace_difo_hold(dtrace_difo_t *dp) 10157 { 10158 int i; 10159 10160 ASSERT(MUTEX_HELD(&dtrace_lock)); 10161 10162 dp->dtdo_refcnt++; 10163 ASSERT(dp->dtdo_refcnt != 0); 10164 10165 /* 10166 * We need to check this DIF object for references to the variable 10167 * DIF_VAR_VTIMESTAMP. 10168 */ 10169 for (i = 0; i < dp->dtdo_varlen; i++) { 10170 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10171 10172 if (v->dtdv_id != DIF_VAR_VTIMESTAMP) 10173 continue; 10174 10175 if (dtrace_vtime_references++ == 0) 10176 dtrace_vtime_enable(); 10177 } 10178 } 10179 10180 /* 10181 * This routine calculates the dynamic variable chunksize for a given DIF 10182 * object. The calculation is not fool-proof, and can probably be tricked by 10183 * malicious DIF -- but it works for all compiler-generated DIF. Because this 10184 * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail 10185 * if a dynamic variable size exceeds the chunksize. 10186 */ 10187 static void 10188 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10189 { 10190 uint64_t sval = 0; 10191 dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */ 10192 const dif_instr_t *text = dp->dtdo_buf; 10193 uint_t pc, srd = 0; 10194 uint_t ttop = 0; 10195 size_t size, ksize; 10196 uint_t id, i; 10197 10198 for (pc = 0; pc < dp->dtdo_len; pc++) { 10199 dif_instr_t instr = text[pc]; 10200 uint_t op = DIF_INSTR_OP(instr); 10201 uint_t rd = DIF_INSTR_RD(instr); 10202 uint_t r1 = DIF_INSTR_R1(instr); 10203 uint_t nkeys = 0; 10204 uchar_t scope = 0; 10205 10206 dtrace_key_t *key = tupregs; 10207 10208 switch (op) { 10209 case DIF_OP_SETX: 10210 sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)]; 10211 srd = rd; 10212 continue; 10213 10214 case DIF_OP_STTS: 10215 key = &tupregs[DIF_DTR_NREGS]; 10216 key[0].dttk_size = 0; 10217 key[1].dttk_size = 0; 10218 nkeys = 2; 10219 scope = DIFV_SCOPE_THREAD; 10220 break; 10221 10222 case DIF_OP_STGAA: 10223 case DIF_OP_STTAA: 10224 nkeys = ttop; 10225 10226 if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) 10227 key[nkeys++].dttk_size = 0; 10228 10229 key[nkeys++].dttk_size = 0; 10230 10231 if (op == DIF_OP_STTAA) { 10232 scope = DIFV_SCOPE_THREAD; 10233 } else { 10234 scope = DIFV_SCOPE_GLOBAL; 10235 } 10236 10237 break; 10238 10239 case DIF_OP_PUSHTR: 10240 if (ttop == DIF_DTR_NREGS) 10241 return; 10242 10243 if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) { 10244 /* 10245 * If the register for the size of the "pushtr" 10246 * is %r0 (or the value is 0) and the type is 10247 * a string, we'll use the system-wide default 10248 * string size. 10249 */ 10250 tupregs[ttop++].dttk_size = 10251 dtrace_strsize_default; 10252 } else { 10253 if (srd == 0) 10254 return; 10255 10256 tupregs[ttop++].dttk_size = sval; 10257 } 10258 10259 break; 10260 10261 case DIF_OP_PUSHTV: 10262 if (ttop == DIF_DTR_NREGS) 10263 return; 10264 10265 tupregs[ttop++].dttk_size = 0; 10266 break; 10267 10268 case DIF_OP_FLUSHTS: 10269 ttop = 0; 10270 break; 10271 10272 case DIF_OP_POPTS: 10273 if (ttop != 0) 10274 ttop--; 10275 break; 10276 } 10277 10278 sval = 0; 10279 srd = 0; 10280 10281 if (nkeys == 0) 10282 continue; 10283 10284 /* 10285 * We have a dynamic variable allocation; calculate its size. 10286 */ 10287 for (ksize = 0, i = 0; i < nkeys; i++) 10288 ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t)); 10289 10290 size = sizeof (dtrace_dynvar_t); 10291 size += sizeof (dtrace_key_t) * (nkeys - 1); 10292 size += ksize; 10293 10294 /* 10295 * Now we need to determine the size of the stored data. 10296 */ 10297 id = DIF_INSTR_VAR(instr); 10298 10299 for (i = 0; i < dp->dtdo_varlen; i++) { 10300 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10301 10302 if (v->dtdv_id == id && v->dtdv_scope == scope) { 10303 size += v->dtdv_type.dtdt_size; 10304 break; 10305 } 10306 } 10307 10308 if (i == dp->dtdo_varlen) 10309 return; 10310 10311 /* 10312 * We have the size. If this is larger than the chunk size 10313 * for our dynamic variable state, reset the chunk size. 10314 */ 10315 size = P2ROUNDUP(size, sizeof (uint64_t)); 10316 10317 if (size > vstate->dtvs_dynvars.dtds_chunksize) 10318 vstate->dtvs_dynvars.dtds_chunksize = size; 10319 } 10320 } 10321 10322 static void 10323 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10324 { 10325 int i, oldsvars, osz, nsz, otlocals, ntlocals; 10326 uint_t id; 10327 10328 ASSERT(MUTEX_HELD(&dtrace_lock)); 10329 ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0); 10330 10331 for (i = 0; i < dp->dtdo_varlen; i++) { 10332 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10333 dtrace_statvar_t *svar, ***svarp = NULL; 10334 size_t dsize = 0; 10335 uint8_t scope = v->dtdv_scope; 10336 int *np = NULL; 10337 10338 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE) 10339 continue; 10340 10341 id -= DIF_VAR_OTHER_UBASE; 10342 10343 switch (scope) { 10344 case DIFV_SCOPE_THREAD: 10345 while (id >= (otlocals = vstate->dtvs_ntlocals)) { 10346 dtrace_difv_t *tlocals; 10347 10348 if ((ntlocals = (otlocals << 1)) == 0) 10349 ntlocals = 1; 10350 10351 osz = otlocals * sizeof (dtrace_difv_t); 10352 nsz = ntlocals * sizeof (dtrace_difv_t); 10353 10354 tlocals = kmem_zalloc(nsz, KM_SLEEP); 10355 10356 if (osz != 0) { 10357 bcopy(vstate->dtvs_tlocals, 10358 tlocals, osz); 10359 kmem_free(vstate->dtvs_tlocals, osz); 10360 } 10361 10362 vstate->dtvs_tlocals = tlocals; 10363 vstate->dtvs_ntlocals = ntlocals; 10364 } 10365 10366 vstate->dtvs_tlocals[id] = *v; 10367 continue; 10368 10369 case DIFV_SCOPE_LOCAL: 10370 np = &vstate->dtvs_nlocals; 10371 svarp = &vstate->dtvs_locals; 10372 10373 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) 10374 dsize = NCPU * (v->dtdv_type.dtdt_size + 10375 sizeof (uint64_t)); 10376 else 10377 dsize = NCPU * sizeof (uint64_t); 10378 10379 break; 10380 10381 case DIFV_SCOPE_GLOBAL: 10382 np = &vstate->dtvs_nglobals; 10383 svarp = &vstate->dtvs_globals; 10384 10385 if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) 10386 dsize = v->dtdv_type.dtdt_size + 10387 sizeof (uint64_t); 10388 10389 break; 10390 10391 default: 10392 ASSERT(0); 10393 } 10394 10395 while (id >= (oldsvars = *np)) { 10396 dtrace_statvar_t **statics; 10397 int newsvars, oldsize, newsize; 10398 10399 if ((newsvars = (oldsvars << 1)) == 0) 10400 newsvars = 1; 10401 10402 oldsize = oldsvars * sizeof (dtrace_statvar_t *); 10403 newsize = newsvars * sizeof (dtrace_statvar_t *); 10404 10405 statics = kmem_zalloc(newsize, KM_SLEEP); 10406 10407 if (oldsize != 0) { 10408 bcopy(*svarp, statics, oldsize); 10409 kmem_free(*svarp, oldsize); 10410 } 10411 10412 *svarp = statics; 10413 *np = newsvars; 10414 } 10415 10416 if ((svar = (*svarp)[id]) == NULL) { 10417 svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP); 10418 svar->dtsv_var = *v; 10419 10420 if ((svar->dtsv_size = dsize) != 0) { 10421 svar->dtsv_data = (uint64_t)(uintptr_t) 10422 kmem_zalloc(dsize, KM_SLEEP); 10423 } 10424 10425 (*svarp)[id] = svar; 10426 } 10427 10428 svar->dtsv_refcnt++; 10429 } 10430 10431 dtrace_difo_chunksize(dp, vstate); 10432 dtrace_difo_hold(dp); 10433 } 10434 10435 static dtrace_difo_t * 10436 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10437 { 10438 dtrace_difo_t *new; 10439 size_t sz; 10440 10441 ASSERT(dp->dtdo_buf != NULL); 10442 ASSERT(dp->dtdo_refcnt != 0); 10443 10444 new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP); 10445 10446 ASSERT(dp->dtdo_buf != NULL); 10447 sz = dp->dtdo_len * sizeof (dif_instr_t); 10448 new->dtdo_buf = kmem_alloc(sz, KM_SLEEP); 10449 bcopy(dp->dtdo_buf, new->dtdo_buf, sz); 10450 new->dtdo_len = dp->dtdo_len; 10451 10452 if (dp->dtdo_strtab != NULL) { 10453 ASSERT(dp->dtdo_strlen != 0); 10454 new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP); 10455 bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen); 10456 new->dtdo_strlen = dp->dtdo_strlen; 10457 } 10458 10459 if (dp->dtdo_inttab != NULL) { 10460 ASSERT(dp->dtdo_intlen != 0); 10461 sz = dp->dtdo_intlen * sizeof (uint64_t); 10462 new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP); 10463 bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz); 10464 new->dtdo_intlen = dp->dtdo_intlen; 10465 } 10466 10467 if (dp->dtdo_vartab != NULL) { 10468 ASSERT(dp->dtdo_varlen != 0); 10469 sz = dp->dtdo_varlen * sizeof (dtrace_difv_t); 10470 new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP); 10471 bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz); 10472 new->dtdo_varlen = dp->dtdo_varlen; 10473 } 10474 10475 dtrace_difo_init(new, vstate); 10476 return (new); 10477 } 10478 10479 static void 10480 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10481 { 10482 int i; 10483 10484 ASSERT(dp->dtdo_refcnt == 0); 10485 10486 for (i = 0; i < dp->dtdo_varlen; i++) { 10487 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10488 dtrace_statvar_t *svar, **svarp = NULL; 10489 uint_t id; 10490 uint8_t scope = v->dtdv_scope; 10491 int *np = NULL; 10492 10493 switch (scope) { 10494 case DIFV_SCOPE_THREAD: 10495 continue; 10496 10497 case DIFV_SCOPE_LOCAL: 10498 np = &vstate->dtvs_nlocals; 10499 svarp = vstate->dtvs_locals; 10500 break; 10501 10502 case DIFV_SCOPE_GLOBAL: 10503 np = &vstate->dtvs_nglobals; 10504 svarp = vstate->dtvs_globals; 10505 break; 10506 10507 default: 10508 ASSERT(0); 10509 } 10510 10511 if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE) 10512 continue; 10513 10514 id -= DIF_VAR_OTHER_UBASE; 10515 ASSERT(id < *np); 10516 10517 svar = svarp[id]; 10518 ASSERT(svar != NULL); 10519 ASSERT(svar->dtsv_refcnt > 0); 10520 10521 if (--svar->dtsv_refcnt > 0) 10522 continue; 10523 10524 if (svar->dtsv_size != 0) { 10525 ASSERT(svar->dtsv_data != 0); 10526 kmem_free((void *)(uintptr_t)svar->dtsv_data, 10527 svar->dtsv_size); 10528 } 10529 10530 kmem_free(svar, sizeof (dtrace_statvar_t)); 10531 svarp[id] = NULL; 10532 } 10533 10534 if (dp->dtdo_buf != NULL) 10535 kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t)); 10536 if (dp->dtdo_inttab != NULL) 10537 kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t)); 10538 if (dp->dtdo_strtab != NULL) 10539 kmem_free(dp->dtdo_strtab, dp->dtdo_strlen); 10540 if (dp->dtdo_vartab != NULL) 10541 kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t)); 10542 10543 kmem_free(dp, sizeof (dtrace_difo_t)); 10544 } 10545 10546 static void 10547 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate) 10548 { 10549 int i; 10550 10551 ASSERT(MUTEX_HELD(&dtrace_lock)); 10552 ASSERT(dp->dtdo_refcnt != 0); 10553 10554 for (i = 0; i < dp->dtdo_varlen; i++) { 10555 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 10556 10557 if (v->dtdv_id != DIF_VAR_VTIMESTAMP) 10558 continue; 10559 10560 ASSERT(dtrace_vtime_references > 0); 10561 if (--dtrace_vtime_references == 0) 10562 dtrace_vtime_disable(); 10563 } 10564 10565 if (--dp->dtdo_refcnt == 0) 10566 dtrace_difo_destroy(dp, vstate); 10567 } 10568 10569 /* 10570 * DTrace Format Functions 10571 */ 10572 static uint16_t 10573 dtrace_format_add(dtrace_state_t *state, char *str) 10574 { 10575 char *fmt, **new; 10576 uint16_t ndx, len = strlen(str) + 1; 10577 10578 fmt = kmem_zalloc(len, KM_SLEEP); 10579 bcopy(str, fmt, len); 10580 10581 for (ndx = 0; ndx < state->dts_nformats; ndx++) { 10582 if (state->dts_formats[ndx] == NULL) { 10583 state->dts_formats[ndx] = fmt; 10584 return (ndx + 1); 10585 } 10586 } 10587 10588 if (state->dts_nformats == USHRT_MAX) { 10589 /* 10590 * This is only likely if a denial-of-service attack is being 10591 * attempted. As such, it's okay to fail silently here. 10592 */ 10593 kmem_free(fmt, len); 10594 return (0); 10595 } 10596 10597 /* 10598 * For simplicity, we always resize the formats array to be exactly the 10599 * number of formats. 10600 */ 10601 ndx = state->dts_nformats++; 10602 new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP); 10603 10604 if (state->dts_formats != NULL) { 10605 ASSERT(ndx != 0); 10606 bcopy(state->dts_formats, new, ndx * sizeof (char *)); 10607 kmem_free(state->dts_formats, ndx * sizeof (char *)); 10608 } 10609 10610 state->dts_formats = new; 10611 state->dts_formats[ndx] = fmt; 10612 10613 return (ndx + 1); 10614 } 10615 10616 static void 10617 dtrace_format_remove(dtrace_state_t *state, uint16_t format) 10618 { 10619 char *fmt; 10620 10621 ASSERT(state->dts_formats != NULL); 10622 ASSERT(format <= state->dts_nformats); 10623 ASSERT(state->dts_formats[format - 1] != NULL); 10624 10625 fmt = state->dts_formats[format - 1]; 10626 kmem_free(fmt, strlen(fmt) + 1); 10627 state->dts_formats[format - 1] = NULL; 10628 } 10629 10630 static void 10631 dtrace_format_destroy(dtrace_state_t *state) 10632 { 10633 int i; 10634 10635 if (state->dts_nformats == 0) { 10636 ASSERT(state->dts_formats == NULL); 10637 return; 10638 } 10639 10640 ASSERT(state->dts_formats != NULL); 10641 10642 for (i = 0; i < state->dts_nformats; i++) { 10643 char *fmt = state->dts_formats[i]; 10644 10645 if (fmt == NULL) 10646 continue; 10647 10648 kmem_free(fmt, strlen(fmt) + 1); 10649 } 10650 10651 kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *)); 10652 state->dts_nformats = 0; 10653 state->dts_formats = NULL; 10654 } 10655 10656 /* 10657 * DTrace Predicate Functions 10658 */ 10659 static dtrace_predicate_t * 10660 dtrace_predicate_create(dtrace_difo_t *dp) 10661 { 10662 dtrace_predicate_t *pred; 10663 10664 ASSERT(MUTEX_HELD(&dtrace_lock)); 10665 ASSERT(dp->dtdo_refcnt != 0); 10666 10667 pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP); 10668 pred->dtp_difo = dp; 10669 pred->dtp_refcnt = 1; 10670 10671 if (!dtrace_difo_cacheable(dp)) 10672 return (pred); 10673 10674 if (dtrace_predcache_id == DTRACE_CACHEIDNONE) { 10675 /* 10676 * This is only theoretically possible -- we have had 2^32 10677 * cacheable predicates on this machine. We cannot allow any 10678 * more predicates to become cacheable: as unlikely as it is, 10679 * there may be a thread caching a (now stale) predicate cache 10680 * ID. (N.B.: the temptation is being successfully resisted to 10681 * have this cmn_err() "Holy shit -- we executed this code!") 10682 */ 10683 return (pred); 10684 } 10685 10686 pred->dtp_cacheid = dtrace_predcache_id++; 10687 10688 return (pred); 10689 } 10690 10691 static void 10692 dtrace_predicate_hold(dtrace_predicate_t *pred) 10693 { 10694 ASSERT(MUTEX_HELD(&dtrace_lock)); 10695 ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0); 10696 ASSERT(pred->dtp_refcnt > 0); 10697 10698 pred->dtp_refcnt++; 10699 } 10700 10701 static void 10702 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate) 10703 { 10704 dtrace_difo_t *dp = pred->dtp_difo; 10705 10706 ASSERT(MUTEX_HELD(&dtrace_lock)); 10707 ASSERT(dp != NULL && dp->dtdo_refcnt != 0); 10708 ASSERT(pred->dtp_refcnt > 0); 10709 10710 if (--pred->dtp_refcnt == 0) { 10711 dtrace_difo_release(pred->dtp_difo, vstate); 10712 kmem_free(pred, sizeof (dtrace_predicate_t)); 10713 } 10714 } 10715 10716 /* 10717 * DTrace Action Description Functions 10718 */ 10719 static dtrace_actdesc_t * 10720 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple, 10721 uint64_t uarg, uint64_t arg) 10722 { 10723 dtrace_actdesc_t *act; 10724 10725 #ifdef illumos 10726 ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL && 10727 arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA)); 10728 #endif 10729 10730 act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP); 10731 act->dtad_kind = kind; 10732 act->dtad_ntuple = ntuple; 10733 act->dtad_uarg = uarg; 10734 act->dtad_arg = arg; 10735 act->dtad_refcnt = 1; 10736 10737 return (act); 10738 } 10739 10740 static void 10741 dtrace_actdesc_hold(dtrace_actdesc_t *act) 10742 { 10743 ASSERT(act->dtad_refcnt >= 1); 10744 act->dtad_refcnt++; 10745 } 10746 10747 static void 10748 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate) 10749 { 10750 dtrace_actkind_t kind = act->dtad_kind; 10751 dtrace_difo_t *dp; 10752 10753 ASSERT(act->dtad_refcnt >= 1); 10754 10755 if (--act->dtad_refcnt != 0) 10756 return; 10757 10758 if ((dp = act->dtad_difo) != NULL) 10759 dtrace_difo_release(dp, vstate); 10760 10761 if (DTRACEACT_ISPRINTFLIKE(kind)) { 10762 char *str = (char *)(uintptr_t)act->dtad_arg; 10763 10764 #ifdef illumos 10765 ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) || 10766 (str == NULL && act->dtad_kind == DTRACEACT_PRINTA)); 10767 #endif 10768 10769 if (str != NULL) 10770 kmem_free(str, strlen(str) + 1); 10771 } 10772 10773 kmem_free(act, sizeof (dtrace_actdesc_t)); 10774 } 10775 10776 /* 10777 * DTrace ECB Functions 10778 */ 10779 static dtrace_ecb_t * 10780 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe) 10781 { 10782 dtrace_ecb_t *ecb; 10783 dtrace_epid_t epid; 10784 10785 ASSERT(MUTEX_HELD(&dtrace_lock)); 10786 10787 ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP); 10788 ecb->dte_predicate = NULL; 10789 ecb->dte_probe = probe; 10790 10791 /* 10792 * The default size is the size of the default action: recording 10793 * the header. 10794 */ 10795 ecb->dte_size = ecb->dte_needed = sizeof (dtrace_rechdr_t); 10796 ecb->dte_alignment = sizeof (dtrace_epid_t); 10797 10798 epid = state->dts_epid++; 10799 10800 if (epid - 1 >= state->dts_necbs) { 10801 dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs; 10802 int necbs = state->dts_necbs << 1; 10803 10804 ASSERT(epid == state->dts_necbs + 1); 10805 10806 if (necbs == 0) { 10807 ASSERT(oecbs == NULL); 10808 necbs = 1; 10809 } 10810 10811 ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP); 10812 10813 if (oecbs != NULL) 10814 bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs)); 10815 10816 dtrace_membar_producer(); 10817 state->dts_ecbs = ecbs; 10818 10819 if (oecbs != NULL) { 10820 /* 10821 * If this state is active, we must dtrace_sync() 10822 * before we can free the old dts_ecbs array: we're 10823 * coming in hot, and there may be active ring 10824 * buffer processing (which indexes into the dts_ecbs 10825 * array) on another CPU. 10826 */ 10827 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) 10828 dtrace_sync(); 10829 10830 kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs)); 10831 } 10832 10833 dtrace_membar_producer(); 10834 state->dts_necbs = necbs; 10835 } 10836 10837 ecb->dte_state = state; 10838 10839 ASSERT(state->dts_ecbs[epid - 1] == NULL); 10840 dtrace_membar_producer(); 10841 state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb; 10842 10843 return (ecb); 10844 } 10845 10846 static void 10847 dtrace_ecb_enable(dtrace_ecb_t *ecb) 10848 { 10849 dtrace_probe_t *probe = ecb->dte_probe; 10850 10851 ASSERT(MUTEX_HELD(&cpu_lock)); 10852 ASSERT(MUTEX_HELD(&dtrace_lock)); 10853 ASSERT(ecb->dte_next == NULL); 10854 10855 if (probe == NULL) { 10856 /* 10857 * This is the NULL probe -- there's nothing to do. 10858 */ 10859 return; 10860 } 10861 10862 if (probe->dtpr_ecb == NULL) { 10863 dtrace_provider_t *prov = probe->dtpr_provider; 10864 10865 /* 10866 * We're the first ECB on this probe. 10867 */ 10868 probe->dtpr_ecb = probe->dtpr_ecb_last = ecb; 10869 10870 if (ecb->dte_predicate != NULL) 10871 probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid; 10872 10873 prov->dtpv_pops.dtps_enable(prov->dtpv_arg, 10874 probe->dtpr_id, probe->dtpr_arg); 10875 } else { 10876 /* 10877 * This probe is already active. Swing the last pointer to 10878 * point to the new ECB, and issue a dtrace_sync() to assure 10879 * that all CPUs have seen the change. 10880 */ 10881 ASSERT(probe->dtpr_ecb_last != NULL); 10882 probe->dtpr_ecb_last->dte_next = ecb; 10883 probe->dtpr_ecb_last = ecb; 10884 probe->dtpr_predcache = 0; 10885 10886 dtrace_sync(); 10887 } 10888 } 10889 10890 static void 10891 dtrace_ecb_resize(dtrace_ecb_t *ecb) 10892 { 10893 dtrace_action_t *act; 10894 uint32_t curneeded = UINT32_MAX; 10895 uint32_t aggbase = UINT32_MAX; 10896 10897 /* 10898 * If we record anything, we always record the dtrace_rechdr_t. (And 10899 * we always record it first.) 10900 */ 10901 ecb->dte_size = sizeof (dtrace_rechdr_t); 10902 ecb->dte_alignment = sizeof (dtrace_epid_t); 10903 10904 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 10905 dtrace_recdesc_t *rec = &act->dta_rec; 10906 ASSERT(rec->dtrd_size > 0 || rec->dtrd_alignment == 1); 10907 10908 ecb->dte_alignment = MAX(ecb->dte_alignment, 10909 rec->dtrd_alignment); 10910 10911 if (DTRACEACT_ISAGG(act->dta_kind)) { 10912 dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act; 10913 10914 ASSERT(rec->dtrd_size != 0); 10915 ASSERT(agg->dtag_first != NULL); 10916 ASSERT(act->dta_prev->dta_intuple); 10917 ASSERT(aggbase != UINT32_MAX); 10918 ASSERT(curneeded != UINT32_MAX); 10919 10920 agg->dtag_base = aggbase; 10921 10922 curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment); 10923 rec->dtrd_offset = curneeded; 10924 curneeded += rec->dtrd_size; 10925 ecb->dte_needed = MAX(ecb->dte_needed, curneeded); 10926 10927 aggbase = UINT32_MAX; 10928 curneeded = UINT32_MAX; 10929 } else if (act->dta_intuple) { 10930 if (curneeded == UINT32_MAX) { 10931 /* 10932 * This is the first record in a tuple. Align 10933 * curneeded to be at offset 4 in an 8-byte 10934 * aligned block. 10935 */ 10936 ASSERT(act->dta_prev == NULL || 10937 !act->dta_prev->dta_intuple); 10938 ASSERT3U(aggbase, ==, UINT32_MAX); 10939 curneeded = P2PHASEUP(ecb->dte_size, 10940 sizeof (uint64_t), sizeof (dtrace_aggid_t)); 10941 10942 aggbase = curneeded - sizeof (dtrace_aggid_t); 10943 ASSERT(IS_P2ALIGNED(aggbase, 10944 sizeof (uint64_t))); 10945 } 10946 curneeded = P2ROUNDUP(curneeded, rec->dtrd_alignment); 10947 rec->dtrd_offset = curneeded; 10948 curneeded += rec->dtrd_size; 10949 } else { 10950 /* tuples must be followed by an aggregation */ 10951 ASSERT(act->dta_prev == NULL || 10952 !act->dta_prev->dta_intuple); 10953 10954 ecb->dte_size = P2ROUNDUP(ecb->dte_size, 10955 rec->dtrd_alignment); 10956 rec->dtrd_offset = ecb->dte_size; 10957 ecb->dte_size += rec->dtrd_size; 10958 ecb->dte_needed = MAX(ecb->dte_needed, ecb->dte_size); 10959 } 10960 } 10961 10962 if ((act = ecb->dte_action) != NULL && 10963 !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) && 10964 ecb->dte_size == sizeof (dtrace_rechdr_t)) { 10965 /* 10966 * If the size is still sizeof (dtrace_rechdr_t), then all 10967 * actions store no data; set the size to 0. 10968 */ 10969 ecb->dte_size = 0; 10970 } 10971 10972 ecb->dte_size = P2ROUNDUP(ecb->dte_size, sizeof (dtrace_epid_t)); 10973 ecb->dte_needed = P2ROUNDUP(ecb->dte_needed, (sizeof (dtrace_epid_t))); 10974 ecb->dte_state->dts_needed = MAX(ecb->dte_state->dts_needed, 10975 ecb->dte_needed); 10976 } 10977 10978 static dtrace_action_t * 10979 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) 10980 { 10981 dtrace_aggregation_t *agg; 10982 size_t size = sizeof (uint64_t); 10983 int ntuple = desc->dtad_ntuple; 10984 dtrace_action_t *act; 10985 dtrace_recdesc_t *frec; 10986 dtrace_aggid_t aggid; 10987 dtrace_state_t *state = ecb->dte_state; 10988 10989 agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP); 10990 agg->dtag_ecb = ecb; 10991 10992 ASSERT(DTRACEACT_ISAGG(desc->dtad_kind)); 10993 10994 switch (desc->dtad_kind) { 10995 case DTRACEAGG_MIN: 10996 agg->dtag_initial = INT64_MAX; 10997 agg->dtag_aggregate = dtrace_aggregate_min; 10998 break; 10999 11000 case DTRACEAGG_MAX: 11001 agg->dtag_initial = INT64_MIN; 11002 agg->dtag_aggregate = dtrace_aggregate_max; 11003 break; 11004 11005 case DTRACEAGG_COUNT: 11006 agg->dtag_aggregate = dtrace_aggregate_count; 11007 break; 11008 11009 case DTRACEAGG_QUANTIZE: 11010 agg->dtag_aggregate = dtrace_aggregate_quantize; 11011 size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) * 11012 sizeof (uint64_t); 11013 break; 11014 11015 case DTRACEAGG_LQUANTIZE: { 11016 uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg); 11017 uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg); 11018 11019 agg->dtag_initial = desc->dtad_arg; 11020 agg->dtag_aggregate = dtrace_aggregate_lquantize; 11021 11022 if (step == 0 || levels == 0) 11023 goto err; 11024 11025 size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t); 11026 break; 11027 } 11028 11029 case DTRACEAGG_LLQUANTIZE: { 11030 uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(desc->dtad_arg); 11031 uint16_t low = DTRACE_LLQUANTIZE_LOW(desc->dtad_arg); 11032 uint16_t high = DTRACE_LLQUANTIZE_HIGH(desc->dtad_arg); 11033 uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(desc->dtad_arg); 11034 int64_t v; 11035 11036 agg->dtag_initial = desc->dtad_arg; 11037 agg->dtag_aggregate = dtrace_aggregate_llquantize; 11038 11039 if (factor < 2 || low >= high || nsteps < factor) 11040 goto err; 11041 11042 /* 11043 * Now check that the number of steps evenly divides a power 11044 * of the factor. (This assures both integer bucket size and 11045 * linearity within each magnitude.) 11046 */ 11047 for (v = factor; v < nsteps; v *= factor) 11048 continue; 11049 11050 if ((v % nsteps) || (nsteps % factor)) 11051 goto err; 11052 11053 size = (dtrace_aggregate_llquantize_bucket(factor, 11054 low, high, nsteps, INT64_MAX) + 2) * sizeof (uint64_t); 11055 break; 11056 } 11057 11058 case DTRACEAGG_AVG: 11059 agg->dtag_aggregate = dtrace_aggregate_avg; 11060 size = sizeof (uint64_t) * 2; 11061 break; 11062 11063 case DTRACEAGG_STDDEV: 11064 agg->dtag_aggregate = dtrace_aggregate_stddev; 11065 size = sizeof (uint64_t) * 4; 11066 break; 11067 11068 case DTRACEAGG_SUM: 11069 agg->dtag_aggregate = dtrace_aggregate_sum; 11070 break; 11071 11072 default: 11073 goto err; 11074 } 11075 11076 agg->dtag_action.dta_rec.dtrd_size = size; 11077 11078 if (ntuple == 0) 11079 goto err; 11080 11081 /* 11082 * We must make sure that we have enough actions for the n-tuple. 11083 */ 11084 for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) { 11085 if (DTRACEACT_ISAGG(act->dta_kind)) 11086 break; 11087 11088 if (--ntuple == 0) { 11089 /* 11090 * This is the action with which our n-tuple begins. 11091 */ 11092 agg->dtag_first = act; 11093 goto success; 11094 } 11095 } 11096 11097 /* 11098 * This n-tuple is short by ntuple elements. Return failure. 11099 */ 11100 ASSERT(ntuple != 0); 11101 err: 11102 kmem_free(agg, sizeof (dtrace_aggregation_t)); 11103 return (NULL); 11104 11105 success: 11106 /* 11107 * If the last action in the tuple has a size of zero, it's actually 11108 * an expression argument for the aggregating action. 11109 */ 11110 ASSERT(ecb->dte_action_last != NULL); 11111 act = ecb->dte_action_last; 11112 11113 if (act->dta_kind == DTRACEACT_DIFEXPR) { 11114 ASSERT(act->dta_difo != NULL); 11115 11116 if (act->dta_difo->dtdo_rtype.dtdt_size == 0) 11117 agg->dtag_hasarg = 1; 11118 } 11119 11120 /* 11121 * We need to allocate an id for this aggregation. 11122 */ 11123 #ifdef illumos 11124 aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1, 11125 VM_BESTFIT | VM_SLEEP); 11126 #else 11127 aggid = alloc_unr(state->dts_aggid_arena); 11128 #endif 11129 11130 if (aggid - 1 >= state->dts_naggregations) { 11131 dtrace_aggregation_t **oaggs = state->dts_aggregations; 11132 dtrace_aggregation_t **aggs; 11133 int naggs = state->dts_naggregations << 1; 11134 int onaggs = state->dts_naggregations; 11135 11136 ASSERT(aggid == state->dts_naggregations + 1); 11137 11138 if (naggs == 0) { 11139 ASSERT(oaggs == NULL); 11140 naggs = 1; 11141 } 11142 11143 aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP); 11144 11145 if (oaggs != NULL) { 11146 bcopy(oaggs, aggs, onaggs * sizeof (*aggs)); 11147 kmem_free(oaggs, onaggs * sizeof (*aggs)); 11148 } 11149 11150 state->dts_aggregations = aggs; 11151 state->dts_naggregations = naggs; 11152 } 11153 11154 ASSERT(state->dts_aggregations[aggid - 1] == NULL); 11155 state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg; 11156 11157 frec = &agg->dtag_first->dta_rec; 11158 if (frec->dtrd_alignment < sizeof (dtrace_aggid_t)) 11159 frec->dtrd_alignment = sizeof (dtrace_aggid_t); 11160 11161 for (act = agg->dtag_first; act != NULL; act = act->dta_next) { 11162 ASSERT(!act->dta_intuple); 11163 act->dta_intuple = 1; 11164 } 11165 11166 return (&agg->dtag_action); 11167 } 11168 11169 static void 11170 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act) 11171 { 11172 dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act; 11173 dtrace_state_t *state = ecb->dte_state; 11174 dtrace_aggid_t aggid = agg->dtag_id; 11175 11176 ASSERT(DTRACEACT_ISAGG(act->dta_kind)); 11177 #ifdef illumos 11178 vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1); 11179 #else 11180 free_unr(state->dts_aggid_arena, aggid); 11181 #endif 11182 11183 ASSERT(state->dts_aggregations[aggid - 1] == agg); 11184 state->dts_aggregations[aggid - 1] = NULL; 11185 11186 kmem_free(agg, sizeof (dtrace_aggregation_t)); 11187 } 11188 11189 static int 11190 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc) 11191 { 11192 dtrace_action_t *action, *last; 11193 dtrace_difo_t *dp = desc->dtad_difo; 11194 uint32_t size = 0, align = sizeof (uint8_t), mask; 11195 uint16_t format = 0; 11196 dtrace_recdesc_t *rec; 11197 dtrace_state_t *state = ecb->dte_state; 11198 dtrace_optval_t *opt = state->dts_options, nframes = 0, strsize; 11199 uint64_t arg = desc->dtad_arg; 11200 11201 ASSERT(MUTEX_HELD(&dtrace_lock)); 11202 ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1); 11203 11204 if (DTRACEACT_ISAGG(desc->dtad_kind)) { 11205 /* 11206 * If this is an aggregating action, there must be neither 11207 * a speculate nor a commit on the action chain. 11208 */ 11209 dtrace_action_t *act; 11210 11211 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 11212 if (act->dta_kind == DTRACEACT_COMMIT) 11213 return (EINVAL); 11214 11215 if (act->dta_kind == DTRACEACT_SPECULATE) 11216 return (EINVAL); 11217 } 11218 11219 action = dtrace_ecb_aggregation_create(ecb, desc); 11220 11221 if (action == NULL) 11222 return (EINVAL); 11223 } else { 11224 if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) || 11225 (desc->dtad_kind == DTRACEACT_DIFEXPR && 11226 dp != NULL && dp->dtdo_destructive)) { 11227 state->dts_destructive = 1; 11228 } 11229 11230 switch (desc->dtad_kind) { 11231 case DTRACEACT_PRINTF: 11232 case DTRACEACT_PRINTA: 11233 case DTRACEACT_SYSTEM: 11234 case DTRACEACT_FREOPEN: 11235 case DTRACEACT_DIFEXPR: 11236 /* 11237 * We know that our arg is a string -- turn it into a 11238 * format. 11239 */ 11240 if (arg == 0) { 11241 ASSERT(desc->dtad_kind == DTRACEACT_PRINTA || 11242 desc->dtad_kind == DTRACEACT_DIFEXPR); 11243 format = 0; 11244 } else { 11245 ASSERT(arg != 0); 11246 #ifdef illumos 11247 ASSERT(arg > KERNELBASE); 11248 #endif 11249 format = dtrace_format_add(state, 11250 (char *)(uintptr_t)arg); 11251 } 11252 11253 /*FALLTHROUGH*/ 11254 case DTRACEACT_LIBACT: 11255 case DTRACEACT_TRACEMEM: 11256 case DTRACEACT_TRACEMEM_DYNSIZE: 11257 if (dp == NULL) 11258 return (EINVAL); 11259 11260 if ((size = dp->dtdo_rtype.dtdt_size) != 0) 11261 break; 11262 11263 if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) { 11264 if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 11265 return (EINVAL); 11266 11267 size = opt[DTRACEOPT_STRSIZE]; 11268 } 11269 11270 break; 11271 11272 case DTRACEACT_STACK: 11273 if ((nframes = arg) == 0) { 11274 nframes = opt[DTRACEOPT_STACKFRAMES]; 11275 ASSERT(nframes > 0); 11276 arg = nframes; 11277 } 11278 11279 size = nframes * sizeof (pc_t); 11280 break; 11281 11282 case DTRACEACT_JSTACK: 11283 if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0) 11284 strsize = opt[DTRACEOPT_JSTACKSTRSIZE]; 11285 11286 if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) 11287 nframes = opt[DTRACEOPT_JSTACKFRAMES]; 11288 11289 arg = DTRACE_USTACK_ARG(nframes, strsize); 11290 11291 /*FALLTHROUGH*/ 11292 case DTRACEACT_USTACK: 11293 if (desc->dtad_kind != DTRACEACT_JSTACK && 11294 (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) { 11295 strsize = DTRACE_USTACK_STRSIZE(arg); 11296 nframes = opt[DTRACEOPT_USTACKFRAMES]; 11297 ASSERT(nframes > 0); 11298 arg = DTRACE_USTACK_ARG(nframes, strsize); 11299 } 11300 11301 /* 11302 * Save a slot for the pid. 11303 */ 11304 size = (nframes + 1) * sizeof (uint64_t); 11305 size += DTRACE_USTACK_STRSIZE(arg); 11306 size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t))); 11307 11308 break; 11309 11310 case DTRACEACT_SYM: 11311 case DTRACEACT_MOD: 11312 if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) != 11313 sizeof (uint64_t)) || 11314 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 11315 return (EINVAL); 11316 break; 11317 11318 case DTRACEACT_USYM: 11319 case DTRACEACT_UMOD: 11320 case DTRACEACT_UADDR: 11321 if (dp == NULL || 11322 (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) || 11323 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 11324 return (EINVAL); 11325 11326 /* 11327 * We have a slot for the pid, plus a slot for the 11328 * argument. To keep things simple (aligned with 11329 * bitness-neutral sizing), we store each as a 64-bit 11330 * quantity. 11331 */ 11332 size = 2 * sizeof (uint64_t); 11333 break; 11334 11335 case DTRACEACT_STOP: 11336 case DTRACEACT_BREAKPOINT: 11337 case DTRACEACT_PANIC: 11338 break; 11339 11340 case DTRACEACT_CHILL: 11341 case DTRACEACT_DISCARD: 11342 case DTRACEACT_RAISE: 11343 if (dp == NULL) 11344 return (EINVAL); 11345 break; 11346 11347 case DTRACEACT_EXIT: 11348 if (dp == NULL || 11349 (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) || 11350 (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) 11351 return (EINVAL); 11352 break; 11353 11354 case DTRACEACT_SPECULATE: 11355 if (ecb->dte_size > sizeof (dtrace_rechdr_t)) 11356 return (EINVAL); 11357 11358 if (dp == NULL) 11359 return (EINVAL); 11360 11361 state->dts_speculates = 1; 11362 break; 11363 11364 case DTRACEACT_PRINTM: 11365 size = dp->dtdo_rtype.dtdt_size; 11366 break; 11367 11368 case DTRACEACT_PRINTT: 11369 size = dp->dtdo_rtype.dtdt_size; 11370 break; 11371 11372 case DTRACEACT_COMMIT: { 11373 dtrace_action_t *act = ecb->dte_action; 11374 11375 for (; act != NULL; act = act->dta_next) { 11376 if (act->dta_kind == DTRACEACT_COMMIT) 11377 return (EINVAL); 11378 } 11379 11380 if (dp == NULL) 11381 return (EINVAL); 11382 break; 11383 } 11384 11385 default: 11386 return (EINVAL); 11387 } 11388 11389 if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) { 11390 /* 11391 * If this is a data-storing action or a speculate, 11392 * we must be sure that there isn't a commit on the 11393 * action chain. 11394 */ 11395 dtrace_action_t *act = ecb->dte_action; 11396 11397 for (; act != NULL; act = act->dta_next) { 11398 if (act->dta_kind == DTRACEACT_COMMIT) 11399 return (EINVAL); 11400 } 11401 } 11402 11403 action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP); 11404 action->dta_rec.dtrd_size = size; 11405 } 11406 11407 action->dta_refcnt = 1; 11408 rec = &action->dta_rec; 11409 size = rec->dtrd_size; 11410 11411 for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) { 11412 if (!(size & mask)) { 11413 align = mask + 1; 11414 break; 11415 } 11416 } 11417 11418 action->dta_kind = desc->dtad_kind; 11419 11420 if ((action->dta_difo = dp) != NULL) 11421 dtrace_difo_hold(dp); 11422 11423 rec->dtrd_action = action->dta_kind; 11424 rec->dtrd_arg = arg; 11425 rec->dtrd_uarg = desc->dtad_uarg; 11426 rec->dtrd_alignment = (uint16_t)align; 11427 rec->dtrd_format = format; 11428 11429 if ((last = ecb->dte_action_last) != NULL) { 11430 ASSERT(ecb->dte_action != NULL); 11431 action->dta_prev = last; 11432 last->dta_next = action; 11433 } else { 11434 ASSERT(ecb->dte_action == NULL); 11435 ecb->dte_action = action; 11436 } 11437 11438 ecb->dte_action_last = action; 11439 11440 return (0); 11441 } 11442 11443 static void 11444 dtrace_ecb_action_remove(dtrace_ecb_t *ecb) 11445 { 11446 dtrace_action_t *act = ecb->dte_action, *next; 11447 dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate; 11448 dtrace_difo_t *dp; 11449 uint16_t format; 11450 11451 if (act != NULL && act->dta_refcnt > 1) { 11452 ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1); 11453 act->dta_refcnt--; 11454 } else { 11455 for (; act != NULL; act = next) { 11456 next = act->dta_next; 11457 ASSERT(next != NULL || act == ecb->dte_action_last); 11458 ASSERT(act->dta_refcnt == 1); 11459 11460 if ((format = act->dta_rec.dtrd_format) != 0) 11461 dtrace_format_remove(ecb->dte_state, format); 11462 11463 if ((dp = act->dta_difo) != NULL) 11464 dtrace_difo_release(dp, vstate); 11465 11466 if (DTRACEACT_ISAGG(act->dta_kind)) { 11467 dtrace_ecb_aggregation_destroy(ecb, act); 11468 } else { 11469 kmem_free(act, sizeof (dtrace_action_t)); 11470 } 11471 } 11472 } 11473 11474 ecb->dte_action = NULL; 11475 ecb->dte_action_last = NULL; 11476 ecb->dte_size = 0; 11477 } 11478 11479 static void 11480 dtrace_ecb_disable(dtrace_ecb_t *ecb) 11481 { 11482 /* 11483 * We disable the ECB by removing it from its probe. 11484 */ 11485 dtrace_ecb_t *pecb, *prev = NULL; 11486 dtrace_probe_t *probe = ecb->dte_probe; 11487 11488 ASSERT(MUTEX_HELD(&dtrace_lock)); 11489 11490 if (probe == NULL) { 11491 /* 11492 * This is the NULL probe; there is nothing to disable. 11493 */ 11494 return; 11495 } 11496 11497 for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) { 11498 if (pecb == ecb) 11499 break; 11500 prev = pecb; 11501 } 11502 11503 ASSERT(pecb != NULL); 11504 11505 if (prev == NULL) { 11506 probe->dtpr_ecb = ecb->dte_next; 11507 } else { 11508 prev->dte_next = ecb->dte_next; 11509 } 11510 11511 if (ecb == probe->dtpr_ecb_last) { 11512 ASSERT(ecb->dte_next == NULL); 11513 probe->dtpr_ecb_last = prev; 11514 } 11515 11516 /* 11517 * The ECB has been disconnected from the probe; now sync to assure 11518 * that all CPUs have seen the change before returning. 11519 */ 11520 dtrace_sync(); 11521 11522 if (probe->dtpr_ecb == NULL) { 11523 /* 11524 * That was the last ECB on the probe; clear the predicate 11525 * cache ID for the probe, disable it and sync one more time 11526 * to assure that we'll never hit it again. 11527 */ 11528 dtrace_provider_t *prov = probe->dtpr_provider; 11529 11530 ASSERT(ecb->dte_next == NULL); 11531 ASSERT(probe->dtpr_ecb_last == NULL); 11532 probe->dtpr_predcache = DTRACE_CACHEIDNONE; 11533 prov->dtpv_pops.dtps_disable(prov->dtpv_arg, 11534 probe->dtpr_id, probe->dtpr_arg); 11535 dtrace_sync(); 11536 } else { 11537 /* 11538 * There is at least one ECB remaining on the probe. If there 11539 * is _exactly_ one, set the probe's predicate cache ID to be 11540 * the predicate cache ID of the remaining ECB. 11541 */ 11542 ASSERT(probe->dtpr_ecb_last != NULL); 11543 ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE); 11544 11545 if (probe->dtpr_ecb == probe->dtpr_ecb_last) { 11546 dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate; 11547 11548 ASSERT(probe->dtpr_ecb->dte_next == NULL); 11549 11550 if (p != NULL) 11551 probe->dtpr_predcache = p->dtp_cacheid; 11552 } 11553 11554 ecb->dte_next = NULL; 11555 } 11556 } 11557 11558 static void 11559 dtrace_ecb_destroy(dtrace_ecb_t *ecb) 11560 { 11561 dtrace_state_t *state = ecb->dte_state; 11562 dtrace_vstate_t *vstate = &state->dts_vstate; 11563 dtrace_predicate_t *pred; 11564 dtrace_epid_t epid = ecb->dte_epid; 11565 11566 ASSERT(MUTEX_HELD(&dtrace_lock)); 11567 ASSERT(ecb->dte_next == NULL); 11568 ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb); 11569 11570 if ((pred = ecb->dte_predicate) != NULL) 11571 dtrace_predicate_release(pred, vstate); 11572 11573 dtrace_ecb_action_remove(ecb); 11574 11575 ASSERT(state->dts_ecbs[epid - 1] == ecb); 11576 state->dts_ecbs[epid - 1] = NULL; 11577 11578 kmem_free(ecb, sizeof (dtrace_ecb_t)); 11579 } 11580 11581 static dtrace_ecb_t * 11582 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe, 11583 dtrace_enabling_t *enab) 11584 { 11585 dtrace_ecb_t *ecb; 11586 dtrace_predicate_t *pred; 11587 dtrace_actdesc_t *act; 11588 dtrace_provider_t *prov; 11589 dtrace_ecbdesc_t *desc = enab->dten_current; 11590 11591 ASSERT(MUTEX_HELD(&dtrace_lock)); 11592 ASSERT(state != NULL); 11593 11594 ecb = dtrace_ecb_add(state, probe); 11595 ecb->dte_uarg = desc->dted_uarg; 11596 11597 if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) { 11598 dtrace_predicate_hold(pred); 11599 ecb->dte_predicate = pred; 11600 } 11601 11602 if (probe != NULL) { 11603 /* 11604 * If the provider shows more leg than the consumer is old 11605 * enough to see, we need to enable the appropriate implicit 11606 * predicate bits to prevent the ecb from activating at 11607 * revealing times. 11608 * 11609 * Providers specifying DTRACE_PRIV_USER at register time 11610 * are stating that they need the /proc-style privilege 11611 * model to be enforced, and this is what DTRACE_COND_OWNER 11612 * and DTRACE_COND_ZONEOWNER will then do at probe time. 11613 */ 11614 prov = probe->dtpr_provider; 11615 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) && 11616 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER)) 11617 ecb->dte_cond |= DTRACE_COND_OWNER; 11618 11619 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLZONE) && 11620 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER)) 11621 ecb->dte_cond |= DTRACE_COND_ZONEOWNER; 11622 11623 /* 11624 * If the provider shows us kernel innards and the user 11625 * is lacking sufficient privilege, enable the 11626 * DTRACE_COND_USERMODE implicit predicate. 11627 */ 11628 if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) && 11629 (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL)) 11630 ecb->dte_cond |= DTRACE_COND_USERMODE; 11631 } 11632 11633 if (dtrace_ecb_create_cache != NULL) { 11634 /* 11635 * If we have a cached ecb, we'll use its action list instead 11636 * of creating our own (saving both time and space). 11637 */ 11638 dtrace_ecb_t *cached = dtrace_ecb_create_cache; 11639 dtrace_action_t *act = cached->dte_action; 11640 11641 if (act != NULL) { 11642 ASSERT(act->dta_refcnt > 0); 11643 act->dta_refcnt++; 11644 ecb->dte_action = act; 11645 ecb->dte_action_last = cached->dte_action_last; 11646 ecb->dte_needed = cached->dte_needed; 11647 ecb->dte_size = cached->dte_size; 11648 ecb->dte_alignment = cached->dte_alignment; 11649 } 11650 11651 return (ecb); 11652 } 11653 11654 for (act = desc->dted_action; act != NULL; act = act->dtad_next) { 11655 if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) { 11656 dtrace_ecb_destroy(ecb); 11657 return (NULL); 11658 } 11659 } 11660 11661 dtrace_ecb_resize(ecb); 11662 11663 return (dtrace_ecb_create_cache = ecb); 11664 } 11665 11666 static int 11667 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg) 11668 { 11669 dtrace_ecb_t *ecb; 11670 dtrace_enabling_t *enab = arg; 11671 dtrace_state_t *state = enab->dten_vstate->dtvs_state; 11672 11673 ASSERT(state != NULL); 11674 11675 if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) { 11676 /* 11677 * This probe was created in a generation for which this 11678 * enabling has previously created ECBs; we don't want to 11679 * enable it again, so just kick out. 11680 */ 11681 return (DTRACE_MATCH_NEXT); 11682 } 11683 11684 if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL) 11685 return (DTRACE_MATCH_DONE); 11686 11687 dtrace_ecb_enable(ecb); 11688 return (DTRACE_MATCH_NEXT); 11689 } 11690 11691 static dtrace_ecb_t * 11692 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id) 11693 { 11694 dtrace_ecb_t *ecb; 11695 11696 ASSERT(MUTEX_HELD(&dtrace_lock)); 11697 11698 if (id == 0 || id > state->dts_necbs) 11699 return (NULL); 11700 11701 ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL); 11702 ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id); 11703 11704 return (state->dts_ecbs[id - 1]); 11705 } 11706 11707 static dtrace_aggregation_t * 11708 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id) 11709 { 11710 dtrace_aggregation_t *agg; 11711 11712 ASSERT(MUTEX_HELD(&dtrace_lock)); 11713 11714 if (id == 0 || id > state->dts_naggregations) 11715 return (NULL); 11716 11717 ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL); 11718 ASSERT((agg = state->dts_aggregations[id - 1]) == NULL || 11719 agg->dtag_id == id); 11720 11721 return (state->dts_aggregations[id - 1]); 11722 } 11723 11724 /* 11725 * DTrace Buffer Functions 11726 * 11727 * The following functions manipulate DTrace buffers. Most of these functions 11728 * are called in the context of establishing or processing consumer state; 11729 * exceptions are explicitly noted. 11730 */ 11731 11732 /* 11733 * Note: called from cross call context. This function switches the two 11734 * buffers on a given CPU. The atomicity of this operation is assured by 11735 * disabling interrupts while the actual switch takes place; the disabling of 11736 * interrupts serializes the execution with any execution of dtrace_probe() on 11737 * the same CPU. 11738 */ 11739 static void 11740 dtrace_buffer_switch(dtrace_buffer_t *buf) 11741 { 11742 caddr_t tomax = buf->dtb_tomax; 11743 caddr_t xamot = buf->dtb_xamot; 11744 dtrace_icookie_t cookie; 11745 hrtime_t now; 11746 11747 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); 11748 ASSERT(!(buf->dtb_flags & DTRACEBUF_RING)); 11749 11750 cookie = dtrace_interrupt_disable(); 11751 now = dtrace_gethrtime(); 11752 buf->dtb_tomax = xamot; 11753 buf->dtb_xamot = tomax; 11754 buf->dtb_xamot_drops = buf->dtb_drops; 11755 buf->dtb_xamot_offset = buf->dtb_offset; 11756 buf->dtb_xamot_errors = buf->dtb_errors; 11757 buf->dtb_xamot_flags = buf->dtb_flags; 11758 buf->dtb_offset = 0; 11759 buf->dtb_drops = 0; 11760 buf->dtb_errors = 0; 11761 buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED); 11762 buf->dtb_interval = now - buf->dtb_switched; 11763 buf->dtb_switched = now; 11764 dtrace_interrupt_enable(cookie); 11765 } 11766 11767 /* 11768 * Note: called from cross call context. This function activates a buffer 11769 * on a CPU. As with dtrace_buffer_switch(), the atomicity of the operation 11770 * is guaranteed by the disabling of interrupts. 11771 */ 11772 static void 11773 dtrace_buffer_activate(dtrace_state_t *state) 11774 { 11775 dtrace_buffer_t *buf; 11776 dtrace_icookie_t cookie = dtrace_interrupt_disable(); 11777 11778 buf = &state->dts_buffer[curcpu]; 11779 11780 if (buf->dtb_tomax != NULL) { 11781 /* 11782 * We might like to assert that the buffer is marked inactive, 11783 * but this isn't necessarily true: the buffer for the CPU 11784 * that processes the BEGIN probe has its buffer activated 11785 * manually. In this case, we take the (harmless) action 11786 * re-clearing the bit INACTIVE bit. 11787 */ 11788 buf->dtb_flags &= ~DTRACEBUF_INACTIVE; 11789 } 11790 11791 dtrace_interrupt_enable(cookie); 11792 } 11793 11794 static int 11795 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags, 11796 processorid_t cpu, int *factor) 11797 { 11798 #ifdef illumos 11799 cpu_t *cp; 11800 #endif 11801 dtrace_buffer_t *buf; 11802 int allocated = 0, desired = 0; 11803 11804 #ifdef illumos 11805 ASSERT(MUTEX_HELD(&cpu_lock)); 11806 ASSERT(MUTEX_HELD(&dtrace_lock)); 11807 11808 *factor = 1; 11809 11810 if (size > dtrace_nonroot_maxsize && 11811 !PRIV_POLICY_CHOICE(CRED(), PRIV_ALL, B_FALSE)) 11812 return (EFBIG); 11813 11814 cp = cpu_list; 11815 11816 do { 11817 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id) 11818 continue; 11819 11820 buf = &bufs[cp->cpu_id]; 11821 11822 /* 11823 * If there is already a buffer allocated for this CPU, it 11824 * is only possible that this is a DR event. In this case, 11825 */ 11826 if (buf->dtb_tomax != NULL) { 11827 ASSERT(buf->dtb_size == size); 11828 continue; 11829 } 11830 11831 ASSERT(buf->dtb_xamot == NULL); 11832 11833 if ((buf->dtb_tomax = kmem_zalloc(size, 11834 KM_NOSLEEP | KM_NORMALPRI)) == NULL) 11835 goto err; 11836 11837 buf->dtb_size = size; 11838 buf->dtb_flags = flags; 11839 buf->dtb_offset = 0; 11840 buf->dtb_drops = 0; 11841 11842 if (flags & DTRACEBUF_NOSWITCH) 11843 continue; 11844 11845 if ((buf->dtb_xamot = kmem_zalloc(size, 11846 KM_NOSLEEP | KM_NORMALPRI)) == NULL) 11847 goto err; 11848 } while ((cp = cp->cpu_next) != cpu_list); 11849 11850 return (0); 11851 11852 err: 11853 cp = cpu_list; 11854 11855 do { 11856 if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id) 11857 continue; 11858 11859 buf = &bufs[cp->cpu_id]; 11860 desired += 2; 11861 11862 if (buf->dtb_xamot != NULL) { 11863 ASSERT(buf->dtb_tomax != NULL); 11864 ASSERT(buf->dtb_size == size); 11865 kmem_free(buf->dtb_xamot, size); 11866 allocated++; 11867 } 11868 11869 if (buf->dtb_tomax != NULL) { 11870 ASSERT(buf->dtb_size == size); 11871 kmem_free(buf->dtb_tomax, size); 11872 allocated++; 11873 } 11874 11875 buf->dtb_tomax = NULL; 11876 buf->dtb_xamot = NULL; 11877 buf->dtb_size = 0; 11878 } while ((cp = cp->cpu_next) != cpu_list); 11879 #else 11880 int i; 11881 11882 *factor = 1; 11883 #if defined(__amd64__) || defined(__mips__) || defined(__powerpc__) 11884 /* 11885 * FreeBSD isn't good at limiting the amount of memory we 11886 * ask to malloc, so let's place a limit here before trying 11887 * to do something that might well end in tears at bedtime. 11888 */ 11889 if (size > physmem * PAGE_SIZE / (128 * (mp_maxid + 1))) 11890 return (ENOMEM); 11891 #endif 11892 11893 ASSERT(MUTEX_HELD(&dtrace_lock)); 11894 CPU_FOREACH(i) { 11895 if (cpu != DTRACE_CPUALL && cpu != i) 11896 continue; 11897 11898 buf = &bufs[i]; 11899 11900 /* 11901 * If there is already a buffer allocated for this CPU, it 11902 * is only possible that this is a DR event. In this case, 11903 * the buffer size must match our specified size. 11904 */ 11905 if (buf->dtb_tomax != NULL) { 11906 ASSERT(buf->dtb_size == size); 11907 continue; 11908 } 11909 11910 ASSERT(buf->dtb_xamot == NULL); 11911 11912 if ((buf->dtb_tomax = kmem_zalloc(size, 11913 KM_NOSLEEP | KM_NORMALPRI)) == NULL) 11914 goto err; 11915 11916 buf->dtb_size = size; 11917 buf->dtb_flags = flags; 11918 buf->dtb_offset = 0; 11919 buf->dtb_drops = 0; 11920 11921 if (flags & DTRACEBUF_NOSWITCH) 11922 continue; 11923 11924 if ((buf->dtb_xamot = kmem_zalloc(size, 11925 KM_NOSLEEP | KM_NORMALPRI)) == NULL) 11926 goto err; 11927 } 11928 11929 return (0); 11930 11931 err: 11932 /* 11933 * Error allocating memory, so free the buffers that were 11934 * allocated before the failed allocation. 11935 */ 11936 CPU_FOREACH(i) { 11937 if (cpu != DTRACE_CPUALL && cpu != i) 11938 continue; 11939 11940 buf = &bufs[i]; 11941 desired += 2; 11942 11943 if (buf->dtb_xamot != NULL) { 11944 ASSERT(buf->dtb_tomax != NULL); 11945 ASSERT(buf->dtb_size == size); 11946 kmem_free(buf->dtb_xamot, size); 11947 allocated++; 11948 } 11949 11950 if (buf->dtb_tomax != NULL) { 11951 ASSERT(buf->dtb_size == size); 11952 kmem_free(buf->dtb_tomax, size); 11953 allocated++; 11954 } 11955 11956 buf->dtb_tomax = NULL; 11957 buf->dtb_xamot = NULL; 11958 buf->dtb_size = 0; 11959 11960 } 11961 #endif 11962 *factor = desired / (allocated > 0 ? allocated : 1); 11963 11964 return (ENOMEM); 11965 } 11966 11967 /* 11968 * Note: called from probe context. This function just increments the drop 11969 * count on a buffer. It has been made a function to allow for the 11970 * possibility of understanding the source of mysterious drop counts. (A 11971 * problem for which one may be particularly disappointed that DTrace cannot 11972 * be used to understand DTrace.) 11973 */ 11974 static void 11975 dtrace_buffer_drop(dtrace_buffer_t *buf) 11976 { 11977 buf->dtb_drops++; 11978 } 11979 11980 /* 11981 * Note: called from probe context. This function is called to reserve space 11982 * in a buffer. If mstate is non-NULL, sets the scratch base and size in the 11983 * mstate. Returns the new offset in the buffer, or a negative value if an 11984 * error has occurred. 11985 */ 11986 static intptr_t 11987 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align, 11988 dtrace_state_t *state, dtrace_mstate_t *mstate) 11989 { 11990 intptr_t offs = buf->dtb_offset, soffs; 11991 intptr_t woffs; 11992 caddr_t tomax; 11993 size_t total; 11994 11995 if (buf->dtb_flags & DTRACEBUF_INACTIVE) 11996 return (-1); 11997 11998 if ((tomax = buf->dtb_tomax) == NULL) { 11999 dtrace_buffer_drop(buf); 12000 return (-1); 12001 } 12002 12003 if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) { 12004 while (offs & (align - 1)) { 12005 /* 12006 * Assert that our alignment is off by a number which 12007 * is itself sizeof (uint32_t) aligned. 12008 */ 12009 ASSERT(!((align - (offs & (align - 1))) & 12010 (sizeof (uint32_t) - 1))); 12011 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE); 12012 offs += sizeof (uint32_t); 12013 } 12014 12015 if ((soffs = offs + needed) > buf->dtb_size) { 12016 dtrace_buffer_drop(buf); 12017 return (-1); 12018 } 12019 12020 if (mstate == NULL) 12021 return (offs); 12022 12023 mstate->dtms_scratch_base = (uintptr_t)tomax + soffs; 12024 mstate->dtms_scratch_size = buf->dtb_size - soffs; 12025 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base; 12026 12027 return (offs); 12028 } 12029 12030 if (buf->dtb_flags & DTRACEBUF_FILL) { 12031 if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN && 12032 (buf->dtb_flags & DTRACEBUF_FULL)) 12033 return (-1); 12034 goto out; 12035 } 12036 12037 total = needed + (offs & (align - 1)); 12038 12039 /* 12040 * For a ring buffer, life is quite a bit more complicated. Before 12041 * we can store any padding, we need to adjust our wrapping offset. 12042 * (If we've never before wrapped or we're not about to, no adjustment 12043 * is required.) 12044 */ 12045 if ((buf->dtb_flags & DTRACEBUF_WRAPPED) || 12046 offs + total > buf->dtb_size) { 12047 woffs = buf->dtb_xamot_offset; 12048 12049 if (offs + total > buf->dtb_size) { 12050 /* 12051 * We can't fit in the end of the buffer. First, a 12052 * sanity check that we can fit in the buffer at all. 12053 */ 12054 if (total > buf->dtb_size) { 12055 dtrace_buffer_drop(buf); 12056 return (-1); 12057 } 12058 12059 /* 12060 * We're going to be storing at the top of the buffer, 12061 * so now we need to deal with the wrapped offset. We 12062 * only reset our wrapped offset to 0 if it is 12063 * currently greater than the current offset. If it 12064 * is less than the current offset, it is because a 12065 * previous allocation induced a wrap -- but the 12066 * allocation didn't subsequently take the space due 12067 * to an error or false predicate evaluation. In this 12068 * case, we'll just leave the wrapped offset alone: if 12069 * the wrapped offset hasn't been advanced far enough 12070 * for this allocation, it will be adjusted in the 12071 * lower loop. 12072 */ 12073 if (buf->dtb_flags & DTRACEBUF_WRAPPED) { 12074 if (woffs >= offs) 12075 woffs = 0; 12076 } else { 12077 woffs = 0; 12078 } 12079 12080 /* 12081 * Now we know that we're going to be storing to the 12082 * top of the buffer and that there is room for us 12083 * there. We need to clear the buffer from the current 12084 * offset to the end (there may be old gunk there). 12085 */ 12086 while (offs < buf->dtb_size) 12087 tomax[offs++] = 0; 12088 12089 /* 12090 * We need to set our offset to zero. And because we 12091 * are wrapping, we need to set the bit indicating as 12092 * much. We can also adjust our needed space back 12093 * down to the space required by the ECB -- we know 12094 * that the top of the buffer is aligned. 12095 */ 12096 offs = 0; 12097 total = needed; 12098 buf->dtb_flags |= DTRACEBUF_WRAPPED; 12099 } else { 12100 /* 12101 * There is room for us in the buffer, so we simply 12102 * need to check the wrapped offset. 12103 */ 12104 if (woffs < offs) { 12105 /* 12106 * The wrapped offset is less than the offset. 12107 * This can happen if we allocated buffer space 12108 * that induced a wrap, but then we didn't 12109 * subsequently take the space due to an error 12110 * or false predicate evaluation. This is 12111 * okay; we know that _this_ allocation isn't 12112 * going to induce a wrap. We still can't 12113 * reset the wrapped offset to be zero, 12114 * however: the space may have been trashed in 12115 * the previous failed probe attempt. But at 12116 * least the wrapped offset doesn't need to 12117 * be adjusted at all... 12118 */ 12119 goto out; 12120 } 12121 } 12122 12123 while (offs + total > woffs) { 12124 dtrace_epid_t epid = *(uint32_t *)(tomax + woffs); 12125 size_t size; 12126 12127 if (epid == DTRACE_EPIDNONE) { 12128 size = sizeof (uint32_t); 12129 } else { 12130 ASSERT3U(epid, <=, state->dts_necbs); 12131 ASSERT(state->dts_ecbs[epid - 1] != NULL); 12132 12133 size = state->dts_ecbs[epid - 1]->dte_size; 12134 } 12135 12136 ASSERT(woffs + size <= buf->dtb_size); 12137 ASSERT(size != 0); 12138 12139 if (woffs + size == buf->dtb_size) { 12140 /* 12141 * We've reached the end of the buffer; we want 12142 * to set the wrapped offset to 0 and break 12143 * out. However, if the offs is 0, then we're 12144 * in a strange edge-condition: the amount of 12145 * space that we want to reserve plus the size 12146 * of the record that we're overwriting is 12147 * greater than the size of the buffer. This 12148 * is problematic because if we reserve the 12149 * space but subsequently don't consume it (due 12150 * to a failed predicate or error) the wrapped 12151 * offset will be 0 -- yet the EPID at offset 0 12152 * will not be committed. This situation is 12153 * relatively easy to deal with: if we're in 12154 * this case, the buffer is indistinguishable 12155 * from one that hasn't wrapped; we need only 12156 * finish the job by clearing the wrapped bit, 12157 * explicitly setting the offset to be 0, and 12158 * zero'ing out the old data in the buffer. 12159 */ 12160 if (offs == 0) { 12161 buf->dtb_flags &= ~DTRACEBUF_WRAPPED; 12162 buf->dtb_offset = 0; 12163 woffs = total; 12164 12165 while (woffs < buf->dtb_size) 12166 tomax[woffs++] = 0; 12167 } 12168 12169 woffs = 0; 12170 break; 12171 } 12172 12173 woffs += size; 12174 } 12175 12176 /* 12177 * We have a wrapped offset. It may be that the wrapped offset 12178 * has become zero -- that's okay. 12179 */ 12180 buf->dtb_xamot_offset = woffs; 12181 } 12182 12183 out: 12184 /* 12185 * Now we can plow the buffer with any necessary padding. 12186 */ 12187 while (offs & (align - 1)) { 12188 /* 12189 * Assert that our alignment is off by a number which 12190 * is itself sizeof (uint32_t) aligned. 12191 */ 12192 ASSERT(!((align - (offs & (align - 1))) & 12193 (sizeof (uint32_t) - 1))); 12194 DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE); 12195 offs += sizeof (uint32_t); 12196 } 12197 12198 if (buf->dtb_flags & DTRACEBUF_FILL) { 12199 if (offs + needed > buf->dtb_size - state->dts_reserve) { 12200 buf->dtb_flags |= DTRACEBUF_FULL; 12201 return (-1); 12202 } 12203 } 12204 12205 if (mstate == NULL) 12206 return (offs); 12207 12208 /* 12209 * For ring buffers and fill buffers, the scratch space is always 12210 * the inactive buffer. 12211 */ 12212 mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot; 12213 mstate->dtms_scratch_size = buf->dtb_size; 12214 mstate->dtms_scratch_ptr = mstate->dtms_scratch_base; 12215 12216 return (offs); 12217 } 12218 12219 static void 12220 dtrace_buffer_polish(dtrace_buffer_t *buf) 12221 { 12222 ASSERT(buf->dtb_flags & DTRACEBUF_RING); 12223 ASSERT(MUTEX_HELD(&dtrace_lock)); 12224 12225 if (!(buf->dtb_flags & DTRACEBUF_WRAPPED)) 12226 return; 12227 12228 /* 12229 * We need to polish the ring buffer. There are three cases: 12230 * 12231 * - The first (and presumably most common) is that there is no gap 12232 * between the buffer offset and the wrapped offset. In this case, 12233 * there is nothing in the buffer that isn't valid data; we can 12234 * mark the buffer as polished and return. 12235 * 12236 * - The second (less common than the first but still more common 12237 * than the third) is that there is a gap between the buffer offset 12238 * and the wrapped offset, and the wrapped offset is larger than the 12239 * buffer offset. This can happen because of an alignment issue, or 12240 * can happen because of a call to dtrace_buffer_reserve() that 12241 * didn't subsequently consume the buffer space. In this case, 12242 * we need to zero the data from the buffer offset to the wrapped 12243 * offset. 12244 * 12245 * - The third (and least common) is that there is a gap between the 12246 * buffer offset and the wrapped offset, but the wrapped offset is 12247 * _less_ than the buffer offset. This can only happen because a 12248 * call to dtrace_buffer_reserve() induced a wrap, but the space 12249 * was not subsequently consumed. In this case, we need to zero the 12250 * space from the offset to the end of the buffer _and_ from the 12251 * top of the buffer to the wrapped offset. 12252 */ 12253 if (buf->dtb_offset < buf->dtb_xamot_offset) { 12254 bzero(buf->dtb_tomax + buf->dtb_offset, 12255 buf->dtb_xamot_offset - buf->dtb_offset); 12256 } 12257 12258 if (buf->dtb_offset > buf->dtb_xamot_offset) { 12259 bzero(buf->dtb_tomax + buf->dtb_offset, 12260 buf->dtb_size - buf->dtb_offset); 12261 bzero(buf->dtb_tomax, buf->dtb_xamot_offset); 12262 } 12263 } 12264 12265 /* 12266 * This routine determines if data generated at the specified time has likely 12267 * been entirely consumed at user-level. This routine is called to determine 12268 * if an ECB on a defunct probe (but for an active enabling) can be safely 12269 * disabled and destroyed. 12270 */ 12271 static int 12272 dtrace_buffer_consumed(dtrace_buffer_t *bufs, hrtime_t when) 12273 { 12274 int i; 12275 12276 for (i = 0; i < NCPU; i++) { 12277 dtrace_buffer_t *buf = &bufs[i]; 12278 12279 if (buf->dtb_size == 0) 12280 continue; 12281 12282 if (buf->dtb_flags & DTRACEBUF_RING) 12283 return (0); 12284 12285 if (!buf->dtb_switched && buf->dtb_offset != 0) 12286 return (0); 12287 12288 if (buf->dtb_switched - buf->dtb_interval < when) 12289 return (0); 12290 } 12291 12292 return (1); 12293 } 12294 12295 static void 12296 dtrace_buffer_free(dtrace_buffer_t *bufs) 12297 { 12298 int i; 12299 12300 for (i = 0; i < NCPU; i++) { 12301 dtrace_buffer_t *buf = &bufs[i]; 12302 12303 if (buf->dtb_tomax == NULL) { 12304 ASSERT(buf->dtb_xamot == NULL); 12305 ASSERT(buf->dtb_size == 0); 12306 continue; 12307 } 12308 12309 if (buf->dtb_xamot != NULL) { 12310 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); 12311 kmem_free(buf->dtb_xamot, buf->dtb_size); 12312 } 12313 12314 kmem_free(buf->dtb_tomax, buf->dtb_size); 12315 buf->dtb_size = 0; 12316 buf->dtb_tomax = NULL; 12317 buf->dtb_xamot = NULL; 12318 } 12319 } 12320 12321 /* 12322 * DTrace Enabling Functions 12323 */ 12324 static dtrace_enabling_t * 12325 dtrace_enabling_create(dtrace_vstate_t *vstate) 12326 { 12327 dtrace_enabling_t *enab; 12328 12329 enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP); 12330 enab->dten_vstate = vstate; 12331 12332 return (enab); 12333 } 12334 12335 static void 12336 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb) 12337 { 12338 dtrace_ecbdesc_t **ndesc; 12339 size_t osize, nsize; 12340 12341 /* 12342 * We can't add to enablings after we've enabled them, or after we've 12343 * retained them. 12344 */ 12345 ASSERT(enab->dten_probegen == 0); 12346 ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); 12347 12348 if (enab->dten_ndesc < enab->dten_maxdesc) { 12349 enab->dten_desc[enab->dten_ndesc++] = ecb; 12350 return; 12351 } 12352 12353 osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *); 12354 12355 if (enab->dten_maxdesc == 0) { 12356 enab->dten_maxdesc = 1; 12357 } else { 12358 enab->dten_maxdesc <<= 1; 12359 } 12360 12361 ASSERT(enab->dten_ndesc < enab->dten_maxdesc); 12362 12363 nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *); 12364 ndesc = kmem_zalloc(nsize, KM_SLEEP); 12365 bcopy(enab->dten_desc, ndesc, osize); 12366 if (enab->dten_desc != NULL) 12367 kmem_free(enab->dten_desc, osize); 12368 12369 enab->dten_desc = ndesc; 12370 enab->dten_desc[enab->dten_ndesc++] = ecb; 12371 } 12372 12373 static void 12374 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb, 12375 dtrace_probedesc_t *pd) 12376 { 12377 dtrace_ecbdesc_t *new; 12378 dtrace_predicate_t *pred; 12379 dtrace_actdesc_t *act; 12380 12381 /* 12382 * We're going to create a new ECB description that matches the 12383 * specified ECB in every way, but has the specified probe description. 12384 */ 12385 new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP); 12386 12387 if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL) 12388 dtrace_predicate_hold(pred); 12389 12390 for (act = ecb->dted_action; act != NULL; act = act->dtad_next) 12391 dtrace_actdesc_hold(act); 12392 12393 new->dted_action = ecb->dted_action; 12394 new->dted_pred = ecb->dted_pred; 12395 new->dted_probe = *pd; 12396 new->dted_uarg = ecb->dted_uarg; 12397 12398 dtrace_enabling_add(enab, new); 12399 } 12400 12401 static void 12402 dtrace_enabling_dump(dtrace_enabling_t *enab) 12403 { 12404 int i; 12405 12406 for (i = 0; i < enab->dten_ndesc; i++) { 12407 dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe; 12408 12409 cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i, 12410 desc->dtpd_provider, desc->dtpd_mod, 12411 desc->dtpd_func, desc->dtpd_name); 12412 } 12413 } 12414 12415 static void 12416 dtrace_enabling_destroy(dtrace_enabling_t *enab) 12417 { 12418 int i; 12419 dtrace_ecbdesc_t *ep; 12420 dtrace_vstate_t *vstate = enab->dten_vstate; 12421 12422 ASSERT(MUTEX_HELD(&dtrace_lock)); 12423 12424 for (i = 0; i < enab->dten_ndesc; i++) { 12425 dtrace_actdesc_t *act, *next; 12426 dtrace_predicate_t *pred; 12427 12428 ep = enab->dten_desc[i]; 12429 12430 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) 12431 dtrace_predicate_release(pred, vstate); 12432 12433 for (act = ep->dted_action; act != NULL; act = next) { 12434 next = act->dtad_next; 12435 dtrace_actdesc_release(act, vstate); 12436 } 12437 12438 kmem_free(ep, sizeof (dtrace_ecbdesc_t)); 12439 } 12440 12441 if (enab->dten_desc != NULL) 12442 kmem_free(enab->dten_desc, 12443 enab->dten_maxdesc * sizeof (dtrace_enabling_t *)); 12444 12445 /* 12446 * If this was a retained enabling, decrement the dts_nretained count 12447 * and take it off of the dtrace_retained list. 12448 */ 12449 if (enab->dten_prev != NULL || enab->dten_next != NULL || 12450 dtrace_retained == enab) { 12451 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12452 ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0); 12453 enab->dten_vstate->dtvs_state->dts_nretained--; 12454 dtrace_retained_gen++; 12455 } 12456 12457 if (enab->dten_prev == NULL) { 12458 if (dtrace_retained == enab) { 12459 dtrace_retained = enab->dten_next; 12460 12461 if (dtrace_retained != NULL) 12462 dtrace_retained->dten_prev = NULL; 12463 } 12464 } else { 12465 ASSERT(enab != dtrace_retained); 12466 ASSERT(dtrace_retained != NULL); 12467 enab->dten_prev->dten_next = enab->dten_next; 12468 } 12469 12470 if (enab->dten_next != NULL) { 12471 ASSERT(dtrace_retained != NULL); 12472 enab->dten_next->dten_prev = enab->dten_prev; 12473 } 12474 12475 kmem_free(enab, sizeof (dtrace_enabling_t)); 12476 } 12477 12478 static int 12479 dtrace_enabling_retain(dtrace_enabling_t *enab) 12480 { 12481 dtrace_state_t *state; 12482 12483 ASSERT(MUTEX_HELD(&dtrace_lock)); 12484 ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL); 12485 ASSERT(enab->dten_vstate != NULL); 12486 12487 state = enab->dten_vstate->dtvs_state; 12488 ASSERT(state != NULL); 12489 12490 /* 12491 * We only allow each state to retain dtrace_retain_max enablings. 12492 */ 12493 if (state->dts_nretained >= dtrace_retain_max) 12494 return (ENOSPC); 12495 12496 state->dts_nretained++; 12497 dtrace_retained_gen++; 12498 12499 if (dtrace_retained == NULL) { 12500 dtrace_retained = enab; 12501 return (0); 12502 } 12503 12504 enab->dten_next = dtrace_retained; 12505 dtrace_retained->dten_prev = enab; 12506 dtrace_retained = enab; 12507 12508 return (0); 12509 } 12510 12511 static int 12512 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match, 12513 dtrace_probedesc_t *create) 12514 { 12515 dtrace_enabling_t *new, *enab; 12516 int found = 0, err = ENOENT; 12517 12518 ASSERT(MUTEX_HELD(&dtrace_lock)); 12519 ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN); 12520 ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN); 12521 ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN); 12522 ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN); 12523 12524 new = dtrace_enabling_create(&state->dts_vstate); 12525 12526 /* 12527 * Iterate over all retained enablings, looking for enablings that 12528 * match the specified state. 12529 */ 12530 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { 12531 int i; 12532 12533 /* 12534 * dtvs_state can only be NULL for helper enablings -- and 12535 * helper enablings can't be retained. 12536 */ 12537 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12538 12539 if (enab->dten_vstate->dtvs_state != state) 12540 continue; 12541 12542 /* 12543 * Now iterate over each probe description; we're looking for 12544 * an exact match to the specified probe description. 12545 */ 12546 for (i = 0; i < enab->dten_ndesc; i++) { 12547 dtrace_ecbdesc_t *ep = enab->dten_desc[i]; 12548 dtrace_probedesc_t *pd = &ep->dted_probe; 12549 12550 if (strcmp(pd->dtpd_provider, match->dtpd_provider)) 12551 continue; 12552 12553 if (strcmp(pd->dtpd_mod, match->dtpd_mod)) 12554 continue; 12555 12556 if (strcmp(pd->dtpd_func, match->dtpd_func)) 12557 continue; 12558 12559 if (strcmp(pd->dtpd_name, match->dtpd_name)) 12560 continue; 12561 12562 /* 12563 * We have a winning probe! Add it to our growing 12564 * enabling. 12565 */ 12566 found = 1; 12567 dtrace_enabling_addlike(new, ep, create); 12568 } 12569 } 12570 12571 if (!found || (err = dtrace_enabling_retain(new)) != 0) { 12572 dtrace_enabling_destroy(new); 12573 return (err); 12574 } 12575 12576 return (0); 12577 } 12578 12579 static void 12580 dtrace_enabling_retract(dtrace_state_t *state) 12581 { 12582 dtrace_enabling_t *enab, *next; 12583 12584 ASSERT(MUTEX_HELD(&dtrace_lock)); 12585 12586 /* 12587 * Iterate over all retained enablings, destroy the enablings retained 12588 * for the specified state. 12589 */ 12590 for (enab = dtrace_retained; enab != NULL; enab = next) { 12591 next = enab->dten_next; 12592 12593 /* 12594 * dtvs_state can only be NULL for helper enablings -- and 12595 * helper enablings can't be retained. 12596 */ 12597 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12598 12599 if (enab->dten_vstate->dtvs_state == state) { 12600 ASSERT(state->dts_nretained > 0); 12601 dtrace_enabling_destroy(enab); 12602 } 12603 } 12604 12605 ASSERT(state->dts_nretained == 0); 12606 } 12607 12608 static int 12609 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched) 12610 { 12611 int i = 0; 12612 int matched = 0; 12613 12614 ASSERT(MUTEX_HELD(&cpu_lock)); 12615 ASSERT(MUTEX_HELD(&dtrace_lock)); 12616 12617 for (i = 0; i < enab->dten_ndesc; i++) { 12618 dtrace_ecbdesc_t *ep = enab->dten_desc[i]; 12619 12620 enab->dten_current = ep; 12621 enab->dten_error = 0; 12622 12623 matched += dtrace_probe_enable(&ep->dted_probe, enab); 12624 12625 if (enab->dten_error != 0) { 12626 /* 12627 * If we get an error half-way through enabling the 12628 * probes, we kick out -- perhaps with some number of 12629 * them enabled. Leaving enabled probes enabled may 12630 * be slightly confusing for user-level, but we expect 12631 * that no one will attempt to actually drive on in 12632 * the face of such errors. If this is an anonymous 12633 * enabling (indicated with a NULL nmatched pointer), 12634 * we cmn_err() a message. We aren't expecting to 12635 * get such an error -- such as it can exist at all, 12636 * it would be a result of corrupted DOF in the driver 12637 * properties. 12638 */ 12639 if (nmatched == NULL) { 12640 cmn_err(CE_WARN, "dtrace_enabling_match() " 12641 "error on %p: %d", (void *)ep, 12642 enab->dten_error); 12643 } 12644 12645 return (enab->dten_error); 12646 } 12647 } 12648 12649 enab->dten_probegen = dtrace_probegen; 12650 if (nmatched != NULL) 12651 *nmatched = matched; 12652 12653 return (0); 12654 } 12655 12656 static void 12657 dtrace_enabling_matchall(void) 12658 { 12659 dtrace_enabling_t *enab; 12660 12661 mutex_enter(&cpu_lock); 12662 mutex_enter(&dtrace_lock); 12663 12664 /* 12665 * Iterate over all retained enablings to see if any probes match 12666 * against them. We only perform this operation on enablings for which 12667 * we have sufficient permissions by virtue of being in the global zone 12668 * or in the same zone as the DTrace client. Because we can be called 12669 * after dtrace_detach() has been called, we cannot assert that there 12670 * are retained enablings. We can safely load from dtrace_retained, 12671 * however: the taskq_destroy() at the end of dtrace_detach() will 12672 * block pending our completion. 12673 */ 12674 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { 12675 #ifdef illumos 12676 cred_t *cr = enab->dten_vstate->dtvs_state->dts_cred.dcr_cred; 12677 12678 if (INGLOBALZONE(curproc) || 12679 cr != NULL && getzoneid() == crgetzoneid(cr)) 12680 #endif 12681 (void) dtrace_enabling_match(enab, NULL); 12682 } 12683 12684 mutex_exit(&dtrace_lock); 12685 mutex_exit(&cpu_lock); 12686 } 12687 12688 /* 12689 * If an enabling is to be enabled without having matched probes (that is, if 12690 * dtrace_state_go() is to be called on the underlying dtrace_state_t), the 12691 * enabling must be _primed_ by creating an ECB for every ECB description. 12692 * This must be done to assure that we know the number of speculations, the 12693 * number of aggregations, the minimum buffer size needed, etc. before we 12694 * transition out of DTRACE_ACTIVITY_INACTIVE. To do this without actually 12695 * enabling any probes, we create ECBs for every ECB decription, but with a 12696 * NULL probe -- which is exactly what this function does. 12697 */ 12698 static void 12699 dtrace_enabling_prime(dtrace_state_t *state) 12700 { 12701 dtrace_enabling_t *enab; 12702 int i; 12703 12704 for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) { 12705 ASSERT(enab->dten_vstate->dtvs_state != NULL); 12706 12707 if (enab->dten_vstate->dtvs_state != state) 12708 continue; 12709 12710 /* 12711 * We don't want to prime an enabling more than once, lest 12712 * we allow a malicious user to induce resource exhaustion. 12713 * (The ECBs that result from priming an enabling aren't 12714 * leaked -- but they also aren't deallocated until the 12715 * consumer state is destroyed.) 12716 */ 12717 if (enab->dten_primed) 12718 continue; 12719 12720 for (i = 0; i < enab->dten_ndesc; i++) { 12721 enab->dten_current = enab->dten_desc[i]; 12722 (void) dtrace_probe_enable(NULL, enab); 12723 } 12724 12725 enab->dten_primed = 1; 12726 } 12727 } 12728 12729 /* 12730 * Called to indicate that probes should be provided due to retained 12731 * enablings. This is implemented in terms of dtrace_probe_provide(), but it 12732 * must take an initial lap through the enabling calling the dtps_provide() 12733 * entry point explicitly to allow for autocreated probes. 12734 */ 12735 static void 12736 dtrace_enabling_provide(dtrace_provider_t *prv) 12737 { 12738 int i, all = 0; 12739 dtrace_probedesc_t desc; 12740 dtrace_genid_t gen; 12741 12742 ASSERT(MUTEX_HELD(&dtrace_lock)); 12743 ASSERT(MUTEX_HELD(&dtrace_provider_lock)); 12744 12745 if (prv == NULL) { 12746 all = 1; 12747 prv = dtrace_provider; 12748 } 12749 12750 do { 12751 dtrace_enabling_t *enab; 12752 void *parg = prv->dtpv_arg; 12753 12754 retry: 12755 gen = dtrace_retained_gen; 12756 for (enab = dtrace_retained; enab != NULL; 12757 enab = enab->dten_next) { 12758 for (i = 0; i < enab->dten_ndesc; i++) { 12759 desc = enab->dten_desc[i]->dted_probe; 12760 mutex_exit(&dtrace_lock); 12761 prv->dtpv_pops.dtps_provide(parg, &desc); 12762 mutex_enter(&dtrace_lock); 12763 /* 12764 * Process the retained enablings again if 12765 * they have changed while we weren't holding 12766 * dtrace_lock. 12767 */ 12768 if (gen != dtrace_retained_gen) 12769 goto retry; 12770 } 12771 } 12772 } while (all && (prv = prv->dtpv_next) != NULL); 12773 12774 mutex_exit(&dtrace_lock); 12775 dtrace_probe_provide(NULL, all ? NULL : prv); 12776 mutex_enter(&dtrace_lock); 12777 } 12778 12779 /* 12780 * Called to reap ECBs that are attached to probes from defunct providers. 12781 */ 12782 static void 12783 dtrace_enabling_reap(void) 12784 { 12785 dtrace_provider_t *prov; 12786 dtrace_probe_t *probe; 12787 dtrace_ecb_t *ecb; 12788 hrtime_t when; 12789 int i; 12790 12791 mutex_enter(&cpu_lock); 12792 mutex_enter(&dtrace_lock); 12793 12794 for (i = 0; i < dtrace_nprobes; i++) { 12795 if ((probe = dtrace_probes[i]) == NULL) 12796 continue; 12797 12798 if (probe->dtpr_ecb == NULL) 12799 continue; 12800 12801 prov = probe->dtpr_provider; 12802 12803 if ((when = prov->dtpv_defunct) == 0) 12804 continue; 12805 12806 /* 12807 * We have ECBs on a defunct provider: we want to reap these 12808 * ECBs to allow the provider to unregister. The destruction 12809 * of these ECBs must be done carefully: if we destroy the ECB 12810 * and the consumer later wishes to consume an EPID that 12811 * corresponds to the destroyed ECB (and if the EPID metadata 12812 * has not been previously consumed), the consumer will abort 12813 * processing on the unknown EPID. To reduce (but not, sadly, 12814 * eliminate) the possibility of this, we will only destroy an 12815 * ECB for a defunct provider if, for the state that 12816 * corresponds to the ECB: 12817 * 12818 * (a) There is no speculative tracing (which can effectively 12819 * cache an EPID for an arbitrary amount of time). 12820 * 12821 * (b) The principal buffers have been switched twice since the 12822 * provider became defunct. 12823 * 12824 * (c) The aggregation buffers are of zero size or have been 12825 * switched twice since the provider became defunct. 12826 * 12827 * We use dts_speculates to determine (a) and call a function 12828 * (dtrace_buffer_consumed()) to determine (b) and (c). Note 12829 * that as soon as we've been unable to destroy one of the ECBs 12830 * associated with the probe, we quit trying -- reaping is only 12831 * fruitful in as much as we can destroy all ECBs associated 12832 * with the defunct provider's probes. 12833 */ 12834 while ((ecb = probe->dtpr_ecb) != NULL) { 12835 dtrace_state_t *state = ecb->dte_state; 12836 dtrace_buffer_t *buf = state->dts_buffer; 12837 dtrace_buffer_t *aggbuf = state->dts_aggbuffer; 12838 12839 if (state->dts_speculates) 12840 break; 12841 12842 if (!dtrace_buffer_consumed(buf, when)) 12843 break; 12844 12845 if (!dtrace_buffer_consumed(aggbuf, when)) 12846 break; 12847 12848 dtrace_ecb_disable(ecb); 12849 ASSERT(probe->dtpr_ecb != ecb); 12850 dtrace_ecb_destroy(ecb); 12851 } 12852 } 12853 12854 mutex_exit(&dtrace_lock); 12855 mutex_exit(&cpu_lock); 12856 } 12857 12858 /* 12859 * DTrace DOF Functions 12860 */ 12861 /*ARGSUSED*/ 12862 static void 12863 dtrace_dof_error(dof_hdr_t *dof, const char *str) 12864 { 12865 if (dtrace_err_verbose) 12866 cmn_err(CE_WARN, "failed to process DOF: %s", str); 12867 12868 #ifdef DTRACE_ERRDEBUG 12869 dtrace_errdebug(str); 12870 #endif 12871 } 12872 12873 /* 12874 * Create DOF out of a currently enabled state. Right now, we only create 12875 * DOF containing the run-time options -- but this could be expanded to create 12876 * complete DOF representing the enabled state. 12877 */ 12878 static dof_hdr_t * 12879 dtrace_dof_create(dtrace_state_t *state) 12880 { 12881 dof_hdr_t *dof; 12882 dof_sec_t *sec; 12883 dof_optdesc_t *opt; 12884 int i, len = sizeof (dof_hdr_t) + 12885 roundup(sizeof (dof_sec_t), sizeof (uint64_t)) + 12886 sizeof (dof_optdesc_t) * DTRACEOPT_MAX; 12887 12888 ASSERT(MUTEX_HELD(&dtrace_lock)); 12889 12890 dof = kmem_zalloc(len, KM_SLEEP); 12891 dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0; 12892 dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1; 12893 dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2; 12894 dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3; 12895 12896 dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE; 12897 dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE; 12898 dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION; 12899 dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION; 12900 dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS; 12901 dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS; 12902 12903 dof->dofh_flags = 0; 12904 dof->dofh_hdrsize = sizeof (dof_hdr_t); 12905 dof->dofh_secsize = sizeof (dof_sec_t); 12906 dof->dofh_secnum = 1; /* only DOF_SECT_OPTDESC */ 12907 dof->dofh_secoff = sizeof (dof_hdr_t); 12908 dof->dofh_loadsz = len; 12909 dof->dofh_filesz = len; 12910 dof->dofh_pad = 0; 12911 12912 /* 12913 * Fill in the option section header... 12914 */ 12915 sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t)); 12916 sec->dofs_type = DOF_SECT_OPTDESC; 12917 sec->dofs_align = sizeof (uint64_t); 12918 sec->dofs_flags = DOF_SECF_LOAD; 12919 sec->dofs_entsize = sizeof (dof_optdesc_t); 12920 12921 opt = (dof_optdesc_t *)((uintptr_t)sec + 12922 roundup(sizeof (dof_sec_t), sizeof (uint64_t))); 12923 12924 sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof; 12925 sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX; 12926 12927 for (i = 0; i < DTRACEOPT_MAX; i++) { 12928 opt[i].dofo_option = i; 12929 opt[i].dofo_strtab = DOF_SECIDX_NONE; 12930 opt[i].dofo_value = state->dts_options[i]; 12931 } 12932 12933 return (dof); 12934 } 12935 12936 static dof_hdr_t * 12937 dtrace_dof_copyin(uintptr_t uarg, int *errp) 12938 { 12939 dof_hdr_t hdr, *dof; 12940 12941 ASSERT(!MUTEX_HELD(&dtrace_lock)); 12942 12943 /* 12944 * First, we're going to copyin() the sizeof (dof_hdr_t). 12945 */ 12946 if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) { 12947 dtrace_dof_error(NULL, "failed to copyin DOF header"); 12948 *errp = EFAULT; 12949 return (NULL); 12950 } 12951 12952 /* 12953 * Now we'll allocate the entire DOF and copy it in -- provided 12954 * that the length isn't outrageous. 12955 */ 12956 if (hdr.dofh_loadsz >= dtrace_dof_maxsize) { 12957 dtrace_dof_error(&hdr, "load size exceeds maximum"); 12958 *errp = E2BIG; 12959 return (NULL); 12960 } 12961 12962 if (hdr.dofh_loadsz < sizeof (hdr)) { 12963 dtrace_dof_error(&hdr, "invalid load size"); 12964 *errp = EINVAL; 12965 return (NULL); 12966 } 12967 12968 dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP); 12969 12970 if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0 || 12971 dof->dofh_loadsz != hdr.dofh_loadsz) { 12972 kmem_free(dof, hdr.dofh_loadsz); 12973 *errp = EFAULT; 12974 return (NULL); 12975 } 12976 12977 return (dof); 12978 } 12979 12980 #ifndef illumos 12981 static __inline uchar_t 12982 dtrace_dof_char(char c) { 12983 switch (c) { 12984 case '0': 12985 case '1': 12986 case '2': 12987 case '3': 12988 case '4': 12989 case '5': 12990 case '6': 12991 case '7': 12992 case '8': 12993 case '9': 12994 return (c - '0'); 12995 case 'A': 12996 case 'B': 12997 case 'C': 12998 case 'D': 12999 case 'E': 13000 case 'F': 13001 return (c - 'A' + 10); 13002 case 'a': 13003 case 'b': 13004 case 'c': 13005 case 'd': 13006 case 'e': 13007 case 'f': 13008 return (c - 'a' + 10); 13009 } 13010 /* Should not reach here. */ 13011 return (0); 13012 } 13013 #endif 13014 13015 static dof_hdr_t * 13016 dtrace_dof_property(const char *name) 13017 { 13018 uchar_t *buf; 13019 uint64_t loadsz; 13020 unsigned int len, i; 13021 dof_hdr_t *dof; 13022 13023 #ifdef illumos 13024 /* 13025 * Unfortunately, array of values in .conf files are always (and 13026 * only) interpreted to be integer arrays. We must read our DOF 13027 * as an integer array, and then squeeze it into a byte array. 13028 */ 13029 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0, 13030 (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS) 13031 return (NULL); 13032 13033 for (i = 0; i < len; i++) 13034 buf[i] = (uchar_t)(((int *)buf)[i]); 13035 13036 if (len < sizeof (dof_hdr_t)) { 13037 ddi_prop_free(buf); 13038 dtrace_dof_error(NULL, "truncated header"); 13039 return (NULL); 13040 } 13041 13042 if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) { 13043 ddi_prop_free(buf); 13044 dtrace_dof_error(NULL, "truncated DOF"); 13045 return (NULL); 13046 } 13047 13048 if (loadsz >= dtrace_dof_maxsize) { 13049 ddi_prop_free(buf); 13050 dtrace_dof_error(NULL, "oversized DOF"); 13051 return (NULL); 13052 } 13053 13054 dof = kmem_alloc(loadsz, KM_SLEEP); 13055 bcopy(buf, dof, loadsz); 13056 ddi_prop_free(buf); 13057 #else 13058 char *p; 13059 char *p_env; 13060 13061 if ((p_env = kern_getenv(name)) == NULL) 13062 return (NULL); 13063 13064 len = strlen(p_env) / 2; 13065 13066 buf = kmem_alloc(len, KM_SLEEP); 13067 13068 dof = (dof_hdr_t *) buf; 13069 13070 p = p_env; 13071 13072 for (i = 0; i < len; i++) { 13073 buf[i] = (dtrace_dof_char(p[0]) << 4) | 13074 dtrace_dof_char(p[1]); 13075 p += 2; 13076 } 13077 13078 freeenv(p_env); 13079 13080 if (len < sizeof (dof_hdr_t)) { 13081 kmem_free(buf, 0); 13082 dtrace_dof_error(NULL, "truncated header"); 13083 return (NULL); 13084 } 13085 13086 if (len < (loadsz = dof->dofh_loadsz)) { 13087 kmem_free(buf, 0); 13088 dtrace_dof_error(NULL, "truncated DOF"); 13089 return (NULL); 13090 } 13091 13092 if (loadsz >= dtrace_dof_maxsize) { 13093 kmem_free(buf, 0); 13094 dtrace_dof_error(NULL, "oversized DOF"); 13095 return (NULL); 13096 } 13097 #endif 13098 13099 return (dof); 13100 } 13101 13102 static void 13103 dtrace_dof_destroy(dof_hdr_t *dof) 13104 { 13105 kmem_free(dof, dof->dofh_loadsz); 13106 } 13107 13108 /* 13109 * Return the dof_sec_t pointer corresponding to a given section index. If the 13110 * index is not valid, dtrace_dof_error() is called and NULL is returned. If 13111 * a type other than DOF_SECT_NONE is specified, the header is checked against 13112 * this type and NULL is returned if the types do not match. 13113 */ 13114 static dof_sec_t * 13115 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i) 13116 { 13117 dof_sec_t *sec = (dof_sec_t *)(uintptr_t) 13118 ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize); 13119 13120 if (i >= dof->dofh_secnum) { 13121 dtrace_dof_error(dof, "referenced section index is invalid"); 13122 return (NULL); 13123 } 13124 13125 if (!(sec->dofs_flags & DOF_SECF_LOAD)) { 13126 dtrace_dof_error(dof, "referenced section is not loadable"); 13127 return (NULL); 13128 } 13129 13130 if (type != DOF_SECT_NONE && type != sec->dofs_type) { 13131 dtrace_dof_error(dof, "referenced section is the wrong type"); 13132 return (NULL); 13133 } 13134 13135 return (sec); 13136 } 13137 13138 static dtrace_probedesc_t * 13139 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc) 13140 { 13141 dof_probedesc_t *probe; 13142 dof_sec_t *strtab; 13143 uintptr_t daddr = (uintptr_t)dof; 13144 uintptr_t str; 13145 size_t size; 13146 13147 if (sec->dofs_type != DOF_SECT_PROBEDESC) { 13148 dtrace_dof_error(dof, "invalid probe section"); 13149 return (NULL); 13150 } 13151 13152 if (sec->dofs_align != sizeof (dof_secidx_t)) { 13153 dtrace_dof_error(dof, "bad alignment in probe description"); 13154 return (NULL); 13155 } 13156 13157 if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) { 13158 dtrace_dof_error(dof, "truncated probe description"); 13159 return (NULL); 13160 } 13161 13162 probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset); 13163 strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab); 13164 13165 if (strtab == NULL) 13166 return (NULL); 13167 13168 str = daddr + strtab->dofs_offset; 13169 size = strtab->dofs_size; 13170 13171 if (probe->dofp_provider >= strtab->dofs_size) { 13172 dtrace_dof_error(dof, "corrupt probe provider"); 13173 return (NULL); 13174 } 13175 13176 (void) strncpy(desc->dtpd_provider, 13177 (char *)(str + probe->dofp_provider), 13178 MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider)); 13179 13180 if (probe->dofp_mod >= strtab->dofs_size) { 13181 dtrace_dof_error(dof, "corrupt probe module"); 13182 return (NULL); 13183 } 13184 13185 (void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod), 13186 MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod)); 13187 13188 if (probe->dofp_func >= strtab->dofs_size) { 13189 dtrace_dof_error(dof, "corrupt probe function"); 13190 return (NULL); 13191 } 13192 13193 (void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func), 13194 MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func)); 13195 13196 if (probe->dofp_name >= strtab->dofs_size) { 13197 dtrace_dof_error(dof, "corrupt probe name"); 13198 return (NULL); 13199 } 13200 13201 (void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name), 13202 MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name)); 13203 13204 return (desc); 13205 } 13206 13207 static dtrace_difo_t * 13208 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 13209 cred_t *cr) 13210 { 13211 dtrace_difo_t *dp; 13212 size_t ttl = 0; 13213 dof_difohdr_t *dofd; 13214 uintptr_t daddr = (uintptr_t)dof; 13215 size_t max = dtrace_difo_maxsize; 13216 int i, l, n; 13217 13218 static const struct { 13219 int section; 13220 int bufoffs; 13221 int lenoffs; 13222 int entsize; 13223 int align; 13224 const char *msg; 13225 } difo[] = { 13226 { DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf), 13227 offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t), 13228 sizeof (dif_instr_t), "multiple DIF sections" }, 13229 13230 { DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab), 13231 offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t), 13232 sizeof (uint64_t), "multiple integer tables" }, 13233 13234 { DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab), 13235 offsetof(dtrace_difo_t, dtdo_strlen), 0, 13236 sizeof (char), "multiple string tables" }, 13237 13238 { DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab), 13239 offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t), 13240 sizeof (uint_t), "multiple variable tables" }, 13241 13242 { DOF_SECT_NONE, 0, 0, 0, 0, NULL } 13243 }; 13244 13245 if (sec->dofs_type != DOF_SECT_DIFOHDR) { 13246 dtrace_dof_error(dof, "invalid DIFO header section"); 13247 return (NULL); 13248 } 13249 13250 if (sec->dofs_align != sizeof (dof_secidx_t)) { 13251 dtrace_dof_error(dof, "bad alignment in DIFO header"); 13252 return (NULL); 13253 } 13254 13255 if (sec->dofs_size < sizeof (dof_difohdr_t) || 13256 sec->dofs_size % sizeof (dof_secidx_t)) { 13257 dtrace_dof_error(dof, "bad size in DIFO header"); 13258 return (NULL); 13259 } 13260 13261 dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset); 13262 n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1; 13263 13264 dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP); 13265 dp->dtdo_rtype = dofd->dofd_rtype; 13266 13267 for (l = 0; l < n; l++) { 13268 dof_sec_t *subsec; 13269 void **bufp; 13270 uint32_t *lenp; 13271 13272 if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE, 13273 dofd->dofd_links[l])) == NULL) 13274 goto err; /* invalid section link */ 13275 13276 if (ttl + subsec->dofs_size > max) { 13277 dtrace_dof_error(dof, "exceeds maximum size"); 13278 goto err; 13279 } 13280 13281 ttl += subsec->dofs_size; 13282 13283 for (i = 0; difo[i].section != DOF_SECT_NONE; i++) { 13284 if (subsec->dofs_type != difo[i].section) 13285 continue; 13286 13287 if (!(subsec->dofs_flags & DOF_SECF_LOAD)) { 13288 dtrace_dof_error(dof, "section not loaded"); 13289 goto err; 13290 } 13291 13292 if (subsec->dofs_align != difo[i].align) { 13293 dtrace_dof_error(dof, "bad alignment"); 13294 goto err; 13295 } 13296 13297 bufp = (void **)((uintptr_t)dp + difo[i].bufoffs); 13298 lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs); 13299 13300 if (*bufp != NULL) { 13301 dtrace_dof_error(dof, difo[i].msg); 13302 goto err; 13303 } 13304 13305 if (difo[i].entsize != subsec->dofs_entsize) { 13306 dtrace_dof_error(dof, "entry size mismatch"); 13307 goto err; 13308 } 13309 13310 if (subsec->dofs_entsize != 0 && 13311 (subsec->dofs_size % subsec->dofs_entsize) != 0) { 13312 dtrace_dof_error(dof, "corrupt entry size"); 13313 goto err; 13314 } 13315 13316 *lenp = subsec->dofs_size; 13317 *bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP); 13318 bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset), 13319 *bufp, subsec->dofs_size); 13320 13321 if (subsec->dofs_entsize != 0) 13322 *lenp /= subsec->dofs_entsize; 13323 13324 break; 13325 } 13326 13327 /* 13328 * If we encounter a loadable DIFO sub-section that is not 13329 * known to us, assume this is a broken program and fail. 13330 */ 13331 if (difo[i].section == DOF_SECT_NONE && 13332 (subsec->dofs_flags & DOF_SECF_LOAD)) { 13333 dtrace_dof_error(dof, "unrecognized DIFO subsection"); 13334 goto err; 13335 } 13336 } 13337 13338 if (dp->dtdo_buf == NULL) { 13339 /* 13340 * We can't have a DIF object without DIF text. 13341 */ 13342 dtrace_dof_error(dof, "missing DIF text"); 13343 goto err; 13344 } 13345 13346 /* 13347 * Before we validate the DIF object, run through the variable table 13348 * looking for the strings -- if any of their size are under, we'll set 13349 * their size to be the system-wide default string size. Note that 13350 * this should _not_ happen if the "strsize" option has been set -- 13351 * in this case, the compiler should have set the size to reflect the 13352 * setting of the option. 13353 */ 13354 for (i = 0; i < dp->dtdo_varlen; i++) { 13355 dtrace_difv_t *v = &dp->dtdo_vartab[i]; 13356 dtrace_diftype_t *t = &v->dtdv_type; 13357 13358 if (v->dtdv_id < DIF_VAR_OTHER_UBASE) 13359 continue; 13360 13361 if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0) 13362 t->dtdt_size = dtrace_strsize_default; 13363 } 13364 13365 if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0) 13366 goto err; 13367 13368 dtrace_difo_init(dp, vstate); 13369 return (dp); 13370 13371 err: 13372 kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t)); 13373 kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t)); 13374 kmem_free(dp->dtdo_strtab, dp->dtdo_strlen); 13375 kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t)); 13376 13377 kmem_free(dp, sizeof (dtrace_difo_t)); 13378 return (NULL); 13379 } 13380 13381 static dtrace_predicate_t * 13382 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 13383 cred_t *cr) 13384 { 13385 dtrace_difo_t *dp; 13386 13387 if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL) 13388 return (NULL); 13389 13390 return (dtrace_predicate_create(dp)); 13391 } 13392 13393 static dtrace_actdesc_t * 13394 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 13395 cred_t *cr) 13396 { 13397 dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next; 13398 dof_actdesc_t *desc; 13399 dof_sec_t *difosec; 13400 size_t offs; 13401 uintptr_t daddr = (uintptr_t)dof; 13402 uint64_t arg; 13403 dtrace_actkind_t kind; 13404 13405 if (sec->dofs_type != DOF_SECT_ACTDESC) { 13406 dtrace_dof_error(dof, "invalid action section"); 13407 return (NULL); 13408 } 13409 13410 if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) { 13411 dtrace_dof_error(dof, "truncated action description"); 13412 return (NULL); 13413 } 13414 13415 if (sec->dofs_align != sizeof (uint64_t)) { 13416 dtrace_dof_error(dof, "bad alignment in action description"); 13417 return (NULL); 13418 } 13419 13420 if (sec->dofs_size < sec->dofs_entsize) { 13421 dtrace_dof_error(dof, "section entry size exceeds total size"); 13422 return (NULL); 13423 } 13424 13425 if (sec->dofs_entsize != sizeof (dof_actdesc_t)) { 13426 dtrace_dof_error(dof, "bad entry size in action description"); 13427 return (NULL); 13428 } 13429 13430 if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) { 13431 dtrace_dof_error(dof, "actions exceed dtrace_actions_max"); 13432 return (NULL); 13433 } 13434 13435 for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) { 13436 desc = (dof_actdesc_t *)(daddr + 13437 (uintptr_t)sec->dofs_offset + offs); 13438 kind = (dtrace_actkind_t)desc->dofa_kind; 13439 13440 if ((DTRACEACT_ISPRINTFLIKE(kind) && 13441 (kind != DTRACEACT_PRINTA || 13442 desc->dofa_strtab != DOF_SECIDX_NONE)) || 13443 (kind == DTRACEACT_DIFEXPR && 13444 desc->dofa_strtab != DOF_SECIDX_NONE)) { 13445 dof_sec_t *strtab; 13446 char *str, *fmt; 13447 uint64_t i; 13448 13449 /* 13450 * The argument to these actions is an index into the 13451 * DOF string table. For printf()-like actions, this 13452 * is the format string. For print(), this is the 13453 * CTF type of the expression result. 13454 */ 13455 if ((strtab = dtrace_dof_sect(dof, 13456 DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL) 13457 goto err; 13458 13459 str = (char *)((uintptr_t)dof + 13460 (uintptr_t)strtab->dofs_offset); 13461 13462 for (i = desc->dofa_arg; i < strtab->dofs_size; i++) { 13463 if (str[i] == '\0') 13464 break; 13465 } 13466 13467 if (i >= strtab->dofs_size) { 13468 dtrace_dof_error(dof, "bogus format string"); 13469 goto err; 13470 } 13471 13472 if (i == desc->dofa_arg) { 13473 dtrace_dof_error(dof, "empty format string"); 13474 goto err; 13475 } 13476 13477 i -= desc->dofa_arg; 13478 fmt = kmem_alloc(i + 1, KM_SLEEP); 13479 bcopy(&str[desc->dofa_arg], fmt, i + 1); 13480 arg = (uint64_t)(uintptr_t)fmt; 13481 } else { 13482 if (kind == DTRACEACT_PRINTA) { 13483 ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE); 13484 arg = 0; 13485 } else { 13486 arg = desc->dofa_arg; 13487 } 13488 } 13489 13490 act = dtrace_actdesc_create(kind, desc->dofa_ntuple, 13491 desc->dofa_uarg, arg); 13492 13493 if (last != NULL) { 13494 last->dtad_next = act; 13495 } else { 13496 first = act; 13497 } 13498 13499 last = act; 13500 13501 if (desc->dofa_difo == DOF_SECIDX_NONE) 13502 continue; 13503 13504 if ((difosec = dtrace_dof_sect(dof, 13505 DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL) 13506 goto err; 13507 13508 act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr); 13509 13510 if (act->dtad_difo == NULL) 13511 goto err; 13512 } 13513 13514 ASSERT(first != NULL); 13515 return (first); 13516 13517 err: 13518 for (act = first; act != NULL; act = next) { 13519 next = act->dtad_next; 13520 dtrace_actdesc_release(act, vstate); 13521 } 13522 13523 return (NULL); 13524 } 13525 13526 static dtrace_ecbdesc_t * 13527 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate, 13528 cred_t *cr) 13529 { 13530 dtrace_ecbdesc_t *ep; 13531 dof_ecbdesc_t *ecb; 13532 dtrace_probedesc_t *desc; 13533 dtrace_predicate_t *pred = NULL; 13534 13535 if (sec->dofs_size < sizeof (dof_ecbdesc_t)) { 13536 dtrace_dof_error(dof, "truncated ECB description"); 13537 return (NULL); 13538 } 13539 13540 if (sec->dofs_align != sizeof (uint64_t)) { 13541 dtrace_dof_error(dof, "bad alignment in ECB description"); 13542 return (NULL); 13543 } 13544 13545 ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset); 13546 sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes); 13547 13548 if (sec == NULL) 13549 return (NULL); 13550 13551 ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP); 13552 ep->dted_uarg = ecb->dofe_uarg; 13553 desc = &ep->dted_probe; 13554 13555 if (dtrace_dof_probedesc(dof, sec, desc) == NULL) 13556 goto err; 13557 13558 if (ecb->dofe_pred != DOF_SECIDX_NONE) { 13559 if ((sec = dtrace_dof_sect(dof, 13560 DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL) 13561 goto err; 13562 13563 if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL) 13564 goto err; 13565 13566 ep->dted_pred.dtpdd_predicate = pred; 13567 } 13568 13569 if (ecb->dofe_actions != DOF_SECIDX_NONE) { 13570 if ((sec = dtrace_dof_sect(dof, 13571 DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL) 13572 goto err; 13573 13574 ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr); 13575 13576 if (ep->dted_action == NULL) 13577 goto err; 13578 } 13579 13580 return (ep); 13581 13582 err: 13583 if (pred != NULL) 13584 dtrace_predicate_release(pred, vstate); 13585 kmem_free(ep, sizeof (dtrace_ecbdesc_t)); 13586 return (NULL); 13587 } 13588 13589 /* 13590 * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the 13591 * specified DOF. At present, this amounts to simply adding 'ubase' to the 13592 * site of any user SETX relocations to account for load object base address. 13593 * In the future, if we need other relocations, this function can be extended. 13594 */ 13595 static int 13596 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase) 13597 { 13598 uintptr_t daddr = (uintptr_t)dof; 13599 dof_relohdr_t *dofr = 13600 (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset); 13601 dof_sec_t *ss, *rs, *ts; 13602 dof_relodesc_t *r; 13603 uint_t i, n; 13604 13605 if (sec->dofs_size < sizeof (dof_relohdr_t) || 13606 sec->dofs_align != sizeof (dof_secidx_t)) { 13607 dtrace_dof_error(dof, "invalid relocation header"); 13608 return (-1); 13609 } 13610 13611 ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab); 13612 rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec); 13613 ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec); 13614 13615 if (ss == NULL || rs == NULL || ts == NULL) 13616 return (-1); /* dtrace_dof_error() has been called already */ 13617 13618 if (rs->dofs_entsize < sizeof (dof_relodesc_t) || 13619 rs->dofs_align != sizeof (uint64_t)) { 13620 dtrace_dof_error(dof, "invalid relocation section"); 13621 return (-1); 13622 } 13623 13624 r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset); 13625 n = rs->dofs_size / rs->dofs_entsize; 13626 13627 for (i = 0; i < n; i++) { 13628 uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset; 13629 13630 switch (r->dofr_type) { 13631 case DOF_RELO_NONE: 13632 break; 13633 case DOF_RELO_SETX: 13634 if (r->dofr_offset >= ts->dofs_size || r->dofr_offset + 13635 sizeof (uint64_t) > ts->dofs_size) { 13636 dtrace_dof_error(dof, "bad relocation offset"); 13637 return (-1); 13638 } 13639 13640 if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) { 13641 dtrace_dof_error(dof, "misaligned setx relo"); 13642 return (-1); 13643 } 13644 13645 *(uint64_t *)taddr += ubase; 13646 break; 13647 default: 13648 dtrace_dof_error(dof, "invalid relocation type"); 13649 return (-1); 13650 } 13651 13652 r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize); 13653 } 13654 13655 return (0); 13656 } 13657 13658 /* 13659 * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated 13660 * header: it should be at the front of a memory region that is at least 13661 * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in 13662 * size. It need not be validated in any other way. 13663 */ 13664 static int 13665 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr, 13666 dtrace_enabling_t **enabp, uint64_t ubase, int noprobes) 13667 { 13668 uint64_t len = dof->dofh_loadsz, seclen; 13669 uintptr_t daddr = (uintptr_t)dof; 13670 dtrace_ecbdesc_t *ep; 13671 dtrace_enabling_t *enab; 13672 uint_t i; 13673 13674 ASSERT(MUTEX_HELD(&dtrace_lock)); 13675 ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t)); 13676 13677 /* 13678 * Check the DOF header identification bytes. In addition to checking 13679 * valid settings, we also verify that unused bits/bytes are zeroed so 13680 * we can use them later without fear of regressing existing binaries. 13681 */ 13682 if (bcmp(&dof->dofh_ident[DOF_ID_MAG0], 13683 DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) { 13684 dtrace_dof_error(dof, "DOF magic string mismatch"); 13685 return (-1); 13686 } 13687 13688 if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 && 13689 dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) { 13690 dtrace_dof_error(dof, "DOF has invalid data model"); 13691 return (-1); 13692 } 13693 13694 if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) { 13695 dtrace_dof_error(dof, "DOF encoding mismatch"); 13696 return (-1); 13697 } 13698 13699 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && 13700 dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_2) { 13701 dtrace_dof_error(dof, "DOF version mismatch"); 13702 return (-1); 13703 } 13704 13705 if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) { 13706 dtrace_dof_error(dof, "DOF uses unsupported instruction set"); 13707 return (-1); 13708 } 13709 13710 if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) { 13711 dtrace_dof_error(dof, "DOF uses too many integer registers"); 13712 return (-1); 13713 } 13714 13715 if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) { 13716 dtrace_dof_error(dof, "DOF uses too many tuple registers"); 13717 return (-1); 13718 } 13719 13720 for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) { 13721 if (dof->dofh_ident[i] != 0) { 13722 dtrace_dof_error(dof, "DOF has invalid ident byte set"); 13723 return (-1); 13724 } 13725 } 13726 13727 if (dof->dofh_flags & ~DOF_FL_VALID) { 13728 dtrace_dof_error(dof, "DOF has invalid flag bits set"); 13729 return (-1); 13730 } 13731 13732 if (dof->dofh_secsize == 0) { 13733 dtrace_dof_error(dof, "zero section header size"); 13734 return (-1); 13735 } 13736 13737 /* 13738 * Check that the section headers don't exceed the amount of DOF 13739 * data. Note that we cast the section size and number of sections 13740 * to uint64_t's to prevent possible overflow in the multiplication. 13741 */ 13742 seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize; 13743 13744 if (dof->dofh_secoff > len || seclen > len || 13745 dof->dofh_secoff + seclen > len) { 13746 dtrace_dof_error(dof, "truncated section headers"); 13747 return (-1); 13748 } 13749 13750 if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) { 13751 dtrace_dof_error(dof, "misaligned section headers"); 13752 return (-1); 13753 } 13754 13755 if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) { 13756 dtrace_dof_error(dof, "misaligned section size"); 13757 return (-1); 13758 } 13759 13760 /* 13761 * Take an initial pass through the section headers to be sure that 13762 * the headers don't have stray offsets. If the 'noprobes' flag is 13763 * set, do not permit sections relating to providers, probes, or args. 13764 */ 13765 for (i = 0; i < dof->dofh_secnum; i++) { 13766 dof_sec_t *sec = (dof_sec_t *)(daddr + 13767 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13768 13769 if (noprobes) { 13770 switch (sec->dofs_type) { 13771 case DOF_SECT_PROVIDER: 13772 case DOF_SECT_PROBES: 13773 case DOF_SECT_PRARGS: 13774 case DOF_SECT_PROFFS: 13775 dtrace_dof_error(dof, "illegal sections " 13776 "for enabling"); 13777 return (-1); 13778 } 13779 } 13780 13781 if (DOF_SEC_ISLOADABLE(sec->dofs_type) && 13782 !(sec->dofs_flags & DOF_SECF_LOAD)) { 13783 dtrace_dof_error(dof, "loadable section with load " 13784 "flag unset"); 13785 return (-1); 13786 } 13787 13788 if (!(sec->dofs_flags & DOF_SECF_LOAD)) 13789 continue; /* just ignore non-loadable sections */ 13790 13791 if (!ISP2(sec->dofs_align)) { 13792 dtrace_dof_error(dof, "bad section alignment"); 13793 return (-1); 13794 } 13795 13796 if (sec->dofs_offset & (sec->dofs_align - 1)) { 13797 dtrace_dof_error(dof, "misaligned section"); 13798 return (-1); 13799 } 13800 13801 if (sec->dofs_offset > len || sec->dofs_size > len || 13802 sec->dofs_offset + sec->dofs_size > len) { 13803 dtrace_dof_error(dof, "corrupt section header"); 13804 return (-1); 13805 } 13806 13807 if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr + 13808 sec->dofs_offset + sec->dofs_size - 1) != '\0') { 13809 dtrace_dof_error(dof, "non-terminating string table"); 13810 return (-1); 13811 } 13812 } 13813 13814 /* 13815 * Take a second pass through the sections and locate and perform any 13816 * relocations that are present. We do this after the first pass to 13817 * be sure that all sections have had their headers validated. 13818 */ 13819 for (i = 0; i < dof->dofh_secnum; i++) { 13820 dof_sec_t *sec = (dof_sec_t *)(daddr + 13821 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13822 13823 if (!(sec->dofs_flags & DOF_SECF_LOAD)) 13824 continue; /* skip sections that are not loadable */ 13825 13826 switch (sec->dofs_type) { 13827 case DOF_SECT_URELHDR: 13828 if (dtrace_dof_relocate(dof, sec, ubase) != 0) 13829 return (-1); 13830 break; 13831 } 13832 } 13833 13834 if ((enab = *enabp) == NULL) 13835 enab = *enabp = dtrace_enabling_create(vstate); 13836 13837 for (i = 0; i < dof->dofh_secnum; i++) { 13838 dof_sec_t *sec = (dof_sec_t *)(daddr + 13839 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13840 13841 if (sec->dofs_type != DOF_SECT_ECBDESC) 13842 continue; 13843 13844 if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) { 13845 dtrace_enabling_destroy(enab); 13846 *enabp = NULL; 13847 return (-1); 13848 } 13849 13850 dtrace_enabling_add(enab, ep); 13851 } 13852 13853 return (0); 13854 } 13855 13856 /* 13857 * Process DOF for any options. This routine assumes that the DOF has been 13858 * at least processed by dtrace_dof_slurp(). 13859 */ 13860 static int 13861 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state) 13862 { 13863 int i, rval; 13864 uint32_t entsize; 13865 size_t offs; 13866 dof_optdesc_t *desc; 13867 13868 for (i = 0; i < dof->dofh_secnum; i++) { 13869 dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof + 13870 (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize); 13871 13872 if (sec->dofs_type != DOF_SECT_OPTDESC) 13873 continue; 13874 13875 if (sec->dofs_align != sizeof (uint64_t)) { 13876 dtrace_dof_error(dof, "bad alignment in " 13877 "option description"); 13878 return (EINVAL); 13879 } 13880 13881 if ((entsize = sec->dofs_entsize) == 0) { 13882 dtrace_dof_error(dof, "zeroed option entry size"); 13883 return (EINVAL); 13884 } 13885 13886 if (entsize < sizeof (dof_optdesc_t)) { 13887 dtrace_dof_error(dof, "bad option entry size"); 13888 return (EINVAL); 13889 } 13890 13891 for (offs = 0; offs < sec->dofs_size; offs += entsize) { 13892 desc = (dof_optdesc_t *)((uintptr_t)dof + 13893 (uintptr_t)sec->dofs_offset + offs); 13894 13895 if (desc->dofo_strtab != DOF_SECIDX_NONE) { 13896 dtrace_dof_error(dof, "non-zero option string"); 13897 return (EINVAL); 13898 } 13899 13900 if (desc->dofo_value == DTRACEOPT_UNSET) { 13901 dtrace_dof_error(dof, "unset option"); 13902 return (EINVAL); 13903 } 13904 13905 if ((rval = dtrace_state_option(state, 13906 desc->dofo_option, desc->dofo_value)) != 0) { 13907 dtrace_dof_error(dof, "rejected option"); 13908 return (rval); 13909 } 13910 } 13911 } 13912 13913 return (0); 13914 } 13915 13916 /* 13917 * DTrace Consumer State Functions 13918 */ 13919 static int 13920 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size) 13921 { 13922 size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize; 13923 void *base; 13924 uintptr_t limit; 13925 dtrace_dynvar_t *dvar, *next, *start; 13926 int i; 13927 13928 ASSERT(MUTEX_HELD(&dtrace_lock)); 13929 ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL); 13930 13931 bzero(dstate, sizeof (dtrace_dstate_t)); 13932 13933 if ((dstate->dtds_chunksize = chunksize) == 0) 13934 dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE; 13935 13936 if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t))) 13937 size = min; 13938 13939 if ((base = kmem_zalloc(size, KM_NOSLEEP | KM_NORMALPRI)) == NULL) 13940 return (ENOMEM); 13941 13942 dstate->dtds_size = size; 13943 dstate->dtds_base = base; 13944 dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP); 13945 bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t)); 13946 13947 hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)); 13948 13949 if (hashsize != 1 && (hashsize & 1)) 13950 hashsize--; 13951 13952 dstate->dtds_hashsize = hashsize; 13953 dstate->dtds_hash = dstate->dtds_base; 13954 13955 /* 13956 * Set all of our hash buckets to point to the single sink, and (if 13957 * it hasn't already been set), set the sink's hash value to be the 13958 * sink sentinel value. The sink is needed for dynamic variable 13959 * lookups to know that they have iterated over an entire, valid hash 13960 * chain. 13961 */ 13962 for (i = 0; i < hashsize; i++) 13963 dstate->dtds_hash[i].dtdh_chain = &dtrace_dynhash_sink; 13964 13965 if (dtrace_dynhash_sink.dtdv_hashval != DTRACE_DYNHASH_SINK) 13966 dtrace_dynhash_sink.dtdv_hashval = DTRACE_DYNHASH_SINK; 13967 13968 /* 13969 * Determine number of active CPUs. Divide free list evenly among 13970 * active CPUs. 13971 */ 13972 start = (dtrace_dynvar_t *) 13973 ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t)); 13974 limit = (uintptr_t)base + size; 13975 13976 maxper = (limit - (uintptr_t)start) / NCPU; 13977 maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize; 13978 13979 #ifndef illumos 13980 CPU_FOREACH(i) { 13981 #else 13982 for (i = 0; i < NCPU; i++) { 13983 #endif 13984 dstate->dtds_percpu[i].dtdsc_free = dvar = start; 13985 13986 /* 13987 * If we don't even have enough chunks to make it once through 13988 * NCPUs, we're just going to allocate everything to the first 13989 * CPU. And if we're on the last CPU, we're going to allocate 13990 * whatever is left over. In either case, we set the limit to 13991 * be the limit of the dynamic variable space. 13992 */ 13993 if (maxper == 0 || i == NCPU - 1) { 13994 limit = (uintptr_t)base + size; 13995 start = NULL; 13996 } else { 13997 limit = (uintptr_t)start + maxper; 13998 start = (dtrace_dynvar_t *)limit; 13999 } 14000 14001 ASSERT(limit <= (uintptr_t)base + size); 14002 14003 for (;;) { 14004 next = (dtrace_dynvar_t *)((uintptr_t)dvar + 14005 dstate->dtds_chunksize); 14006 14007 if ((uintptr_t)next + dstate->dtds_chunksize >= limit) 14008 break; 14009 14010 dvar->dtdv_next = next; 14011 dvar = next; 14012 } 14013 14014 if (maxper == 0) 14015 break; 14016 } 14017 14018 return (0); 14019 } 14020 14021 static void 14022 dtrace_dstate_fini(dtrace_dstate_t *dstate) 14023 { 14024 ASSERT(MUTEX_HELD(&cpu_lock)); 14025 14026 if (dstate->dtds_base == NULL) 14027 return; 14028 14029 kmem_free(dstate->dtds_base, dstate->dtds_size); 14030 kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu); 14031 } 14032 14033 static void 14034 dtrace_vstate_fini(dtrace_vstate_t *vstate) 14035 { 14036 /* 14037 * Logical XOR, where are you? 14038 */ 14039 ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL)); 14040 14041 if (vstate->dtvs_nglobals > 0) { 14042 kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals * 14043 sizeof (dtrace_statvar_t *)); 14044 } 14045 14046 if (vstate->dtvs_ntlocals > 0) { 14047 kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals * 14048 sizeof (dtrace_difv_t)); 14049 } 14050 14051 ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL)); 14052 14053 if (vstate->dtvs_nlocals > 0) { 14054 kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals * 14055 sizeof (dtrace_statvar_t *)); 14056 } 14057 } 14058 14059 #ifdef illumos 14060 static void 14061 dtrace_state_clean(dtrace_state_t *state) 14062 { 14063 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) 14064 return; 14065 14066 dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars); 14067 dtrace_speculation_clean(state); 14068 } 14069 14070 static void 14071 dtrace_state_deadman(dtrace_state_t *state) 14072 { 14073 hrtime_t now; 14074 14075 dtrace_sync(); 14076 14077 now = dtrace_gethrtime(); 14078 14079 if (state != dtrace_anon.dta_state && 14080 now - state->dts_laststatus >= dtrace_deadman_user) 14081 return; 14082 14083 /* 14084 * We must be sure that dts_alive never appears to be less than the 14085 * value upon entry to dtrace_state_deadman(), and because we lack a 14086 * dtrace_cas64(), we cannot store to it atomically. We thus instead 14087 * store INT64_MAX to it, followed by a memory barrier, followed by 14088 * the new value. This assures that dts_alive never appears to be 14089 * less than its true value, regardless of the order in which the 14090 * stores to the underlying storage are issued. 14091 */ 14092 state->dts_alive = INT64_MAX; 14093 dtrace_membar_producer(); 14094 state->dts_alive = now; 14095 } 14096 #else /* !illumos */ 14097 static void 14098 dtrace_state_clean(void *arg) 14099 { 14100 dtrace_state_t *state = arg; 14101 dtrace_optval_t *opt = state->dts_options; 14102 14103 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) 14104 return; 14105 14106 dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars); 14107 dtrace_speculation_clean(state); 14108 14109 callout_reset(&state->dts_cleaner, hz * opt[DTRACEOPT_CLEANRATE] / NANOSEC, 14110 dtrace_state_clean, state); 14111 } 14112 14113 static void 14114 dtrace_state_deadman(void *arg) 14115 { 14116 dtrace_state_t *state = arg; 14117 hrtime_t now; 14118 14119 dtrace_sync(); 14120 14121 dtrace_debug_output(); 14122 14123 now = dtrace_gethrtime(); 14124 14125 if (state != dtrace_anon.dta_state && 14126 now - state->dts_laststatus >= dtrace_deadman_user) 14127 return; 14128 14129 /* 14130 * We must be sure that dts_alive never appears to be less than the 14131 * value upon entry to dtrace_state_deadman(), and because we lack a 14132 * dtrace_cas64(), we cannot store to it atomically. We thus instead 14133 * store INT64_MAX to it, followed by a memory barrier, followed by 14134 * the new value. This assures that dts_alive never appears to be 14135 * less than its true value, regardless of the order in which the 14136 * stores to the underlying storage are issued. 14137 */ 14138 state->dts_alive = INT64_MAX; 14139 dtrace_membar_producer(); 14140 state->dts_alive = now; 14141 14142 callout_reset(&state->dts_deadman, hz * dtrace_deadman_interval / NANOSEC, 14143 dtrace_state_deadman, state); 14144 } 14145 #endif /* illumos */ 14146 14147 static dtrace_state_t * 14148 #ifdef illumos 14149 dtrace_state_create(dev_t *devp, cred_t *cr) 14150 #else 14151 dtrace_state_create(struct cdev *dev) 14152 #endif 14153 { 14154 #ifdef illumos 14155 minor_t minor; 14156 major_t major; 14157 #else 14158 cred_t *cr = NULL; 14159 int m = 0; 14160 #endif 14161 char c[30]; 14162 dtrace_state_t *state; 14163 dtrace_optval_t *opt; 14164 int bufsize = NCPU * sizeof (dtrace_buffer_t), i; 14165 14166 ASSERT(MUTEX_HELD(&dtrace_lock)); 14167 ASSERT(MUTEX_HELD(&cpu_lock)); 14168 14169 #ifdef illumos 14170 minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1, 14171 VM_BESTFIT | VM_SLEEP); 14172 14173 if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) { 14174 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1); 14175 return (NULL); 14176 } 14177 14178 state = ddi_get_soft_state(dtrace_softstate, minor); 14179 #else 14180 if (dev != NULL) { 14181 cr = dev->si_cred; 14182 m = dev2unit(dev); 14183 } 14184 14185 /* Allocate memory for the state. */ 14186 state = kmem_zalloc(sizeof(dtrace_state_t), KM_SLEEP); 14187 #endif 14188 14189 state->dts_epid = DTRACE_EPIDNONE + 1; 14190 14191 (void) snprintf(c, sizeof (c), "dtrace_aggid_%d", m); 14192 #ifdef illumos 14193 state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1, 14194 NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 14195 14196 if (devp != NULL) { 14197 major = getemajor(*devp); 14198 } else { 14199 major = ddi_driver_major(dtrace_devi); 14200 } 14201 14202 state->dts_dev = makedevice(major, minor); 14203 14204 if (devp != NULL) 14205 *devp = state->dts_dev; 14206 #else 14207 state->dts_aggid_arena = new_unrhdr(1, INT_MAX, &dtrace_unr_mtx); 14208 state->dts_dev = dev; 14209 #endif 14210 14211 /* 14212 * We allocate NCPU buffers. On the one hand, this can be quite 14213 * a bit of memory per instance (nearly 36K on a Starcat). On the 14214 * other hand, it saves an additional memory reference in the probe 14215 * path. 14216 */ 14217 state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP); 14218 state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP); 14219 14220 #ifdef illumos 14221 state->dts_cleaner = CYCLIC_NONE; 14222 state->dts_deadman = CYCLIC_NONE; 14223 #else 14224 callout_init(&state->dts_cleaner, CALLOUT_MPSAFE); 14225 callout_init(&state->dts_deadman, CALLOUT_MPSAFE); 14226 #endif 14227 state->dts_vstate.dtvs_state = state; 14228 14229 for (i = 0; i < DTRACEOPT_MAX; i++) 14230 state->dts_options[i] = DTRACEOPT_UNSET; 14231 14232 /* 14233 * Set the default options. 14234 */ 14235 opt = state->dts_options; 14236 opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH; 14237 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO; 14238 opt[DTRACEOPT_NSPEC] = dtrace_nspec_default; 14239 opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default; 14240 opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL; 14241 opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default; 14242 opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default; 14243 opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default; 14244 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default; 14245 opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default; 14246 opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default; 14247 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default; 14248 opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default; 14249 opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default; 14250 14251 state->dts_activity = DTRACE_ACTIVITY_INACTIVE; 14252 14253 /* 14254 * Depending on the user credentials, we set flag bits which alter probe 14255 * visibility or the amount of destructiveness allowed. In the case of 14256 * actual anonymous tracing, or the possession of all privileges, all of 14257 * the normal checks are bypassed. 14258 */ 14259 if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) { 14260 state->dts_cred.dcr_visible = DTRACE_CRV_ALL; 14261 state->dts_cred.dcr_action = DTRACE_CRA_ALL; 14262 } else { 14263 /* 14264 * Set up the credentials for this instantiation. We take a 14265 * hold on the credential to prevent it from disappearing on 14266 * us; this in turn prevents the zone_t referenced by this 14267 * credential from disappearing. This means that we can 14268 * examine the credential and the zone from probe context. 14269 */ 14270 crhold(cr); 14271 state->dts_cred.dcr_cred = cr; 14272 14273 /* 14274 * CRA_PROC means "we have *some* privilege for dtrace" and 14275 * unlocks the use of variables like pid, zonename, etc. 14276 */ 14277 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) || 14278 PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) { 14279 state->dts_cred.dcr_action |= DTRACE_CRA_PROC; 14280 } 14281 14282 /* 14283 * dtrace_user allows use of syscall and profile providers. 14284 * If the user also has proc_owner and/or proc_zone, we 14285 * extend the scope to include additional visibility and 14286 * destructive power. 14287 */ 14288 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE)) { 14289 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) { 14290 state->dts_cred.dcr_visible |= 14291 DTRACE_CRV_ALLPROC; 14292 14293 state->dts_cred.dcr_action |= 14294 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; 14295 } 14296 14297 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) { 14298 state->dts_cred.dcr_visible |= 14299 DTRACE_CRV_ALLZONE; 14300 14301 state->dts_cred.dcr_action |= 14302 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; 14303 } 14304 14305 /* 14306 * If we have all privs in whatever zone this is, 14307 * we can do destructive things to processes which 14308 * have altered credentials. 14309 */ 14310 #ifdef illumos 14311 if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE), 14312 cr->cr_zone->zone_privset)) { 14313 state->dts_cred.dcr_action |= 14314 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG; 14315 } 14316 #endif 14317 } 14318 14319 /* 14320 * Holding the dtrace_kernel privilege also implies that 14321 * the user has the dtrace_user privilege from a visibility 14322 * perspective. But without further privileges, some 14323 * destructive actions are not available. 14324 */ 14325 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) { 14326 /* 14327 * Make all probes in all zones visible. However, 14328 * this doesn't mean that all actions become available 14329 * to all zones. 14330 */ 14331 state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL | 14332 DTRACE_CRV_ALLPROC | DTRACE_CRV_ALLZONE; 14333 14334 state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL | 14335 DTRACE_CRA_PROC; 14336 /* 14337 * Holding proc_owner means that destructive actions 14338 * for *this* zone are allowed. 14339 */ 14340 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) 14341 state->dts_cred.dcr_action |= 14342 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; 14343 14344 /* 14345 * Holding proc_zone means that destructive actions 14346 * for this user/group ID in all zones is allowed. 14347 */ 14348 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) 14349 state->dts_cred.dcr_action |= 14350 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; 14351 14352 #ifdef illumos 14353 /* 14354 * If we have all privs in whatever zone this is, 14355 * we can do destructive things to processes which 14356 * have altered credentials. 14357 */ 14358 if (priv_isequalset(priv_getset(cr, PRIV_EFFECTIVE), 14359 cr->cr_zone->zone_privset)) { 14360 state->dts_cred.dcr_action |= 14361 DTRACE_CRA_PROC_DESTRUCTIVE_CREDCHG; 14362 } 14363 #endif 14364 } 14365 14366 /* 14367 * Holding the dtrace_proc privilege gives control over fasttrap 14368 * and pid providers. We need to grant wider destructive 14369 * privileges in the event that the user has proc_owner and/or 14370 * proc_zone. 14371 */ 14372 if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) { 14373 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) 14374 state->dts_cred.dcr_action |= 14375 DTRACE_CRA_PROC_DESTRUCTIVE_ALLUSER; 14376 14377 if (PRIV_POLICY_ONLY(cr, PRIV_PROC_ZONE, B_FALSE)) 14378 state->dts_cred.dcr_action |= 14379 DTRACE_CRA_PROC_DESTRUCTIVE_ALLZONE; 14380 } 14381 } 14382 14383 return (state); 14384 } 14385 14386 static int 14387 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which) 14388 { 14389 dtrace_optval_t *opt = state->dts_options, size; 14390 processorid_t cpu = 0;; 14391 int flags = 0, rval, factor, divisor = 1; 14392 14393 ASSERT(MUTEX_HELD(&dtrace_lock)); 14394 ASSERT(MUTEX_HELD(&cpu_lock)); 14395 ASSERT(which < DTRACEOPT_MAX); 14396 ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE || 14397 (state == dtrace_anon.dta_state && 14398 state->dts_activity == DTRACE_ACTIVITY_ACTIVE)); 14399 14400 if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0) 14401 return (0); 14402 14403 if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET) 14404 cpu = opt[DTRACEOPT_CPU]; 14405 14406 if (which == DTRACEOPT_SPECSIZE) 14407 flags |= DTRACEBUF_NOSWITCH; 14408 14409 if (which == DTRACEOPT_BUFSIZE) { 14410 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING) 14411 flags |= DTRACEBUF_RING; 14412 14413 if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL) 14414 flags |= DTRACEBUF_FILL; 14415 14416 if (state != dtrace_anon.dta_state || 14417 state->dts_activity != DTRACE_ACTIVITY_ACTIVE) 14418 flags |= DTRACEBUF_INACTIVE; 14419 } 14420 14421 for (size = opt[which]; size >= sizeof (uint64_t); size /= divisor) { 14422 /* 14423 * The size must be 8-byte aligned. If the size is not 8-byte 14424 * aligned, drop it down by the difference. 14425 */ 14426 if (size & (sizeof (uint64_t) - 1)) 14427 size -= size & (sizeof (uint64_t) - 1); 14428 14429 if (size < state->dts_reserve) { 14430 /* 14431 * Buffers always must be large enough to accommodate 14432 * their prereserved space. We return E2BIG instead 14433 * of ENOMEM in this case to allow for user-level 14434 * software to differentiate the cases. 14435 */ 14436 return (E2BIG); 14437 } 14438 14439 rval = dtrace_buffer_alloc(buf, size, flags, cpu, &factor); 14440 14441 if (rval != ENOMEM) { 14442 opt[which] = size; 14443 return (rval); 14444 } 14445 14446 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL) 14447 return (rval); 14448 14449 for (divisor = 2; divisor < factor; divisor <<= 1) 14450 continue; 14451 } 14452 14453 return (ENOMEM); 14454 } 14455 14456 static int 14457 dtrace_state_buffers(dtrace_state_t *state) 14458 { 14459 dtrace_speculation_t *spec = state->dts_speculations; 14460 int rval, i; 14461 14462 if ((rval = dtrace_state_buffer(state, state->dts_buffer, 14463 DTRACEOPT_BUFSIZE)) != 0) 14464 return (rval); 14465 14466 if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer, 14467 DTRACEOPT_AGGSIZE)) != 0) 14468 return (rval); 14469 14470 for (i = 0; i < state->dts_nspeculations; i++) { 14471 if ((rval = dtrace_state_buffer(state, 14472 spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0) 14473 return (rval); 14474 } 14475 14476 return (0); 14477 } 14478 14479 static void 14480 dtrace_state_prereserve(dtrace_state_t *state) 14481 { 14482 dtrace_ecb_t *ecb; 14483 dtrace_probe_t *probe; 14484 14485 state->dts_reserve = 0; 14486 14487 if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL) 14488 return; 14489 14490 /* 14491 * If our buffer policy is a "fill" buffer policy, we need to set the 14492 * prereserved space to be the space required by the END probes. 14493 */ 14494 probe = dtrace_probes[dtrace_probeid_end - 1]; 14495 ASSERT(probe != NULL); 14496 14497 for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) { 14498 if (ecb->dte_state != state) 14499 continue; 14500 14501 state->dts_reserve += ecb->dte_needed + ecb->dte_alignment; 14502 } 14503 } 14504 14505 static int 14506 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu) 14507 { 14508 dtrace_optval_t *opt = state->dts_options, sz, nspec; 14509 dtrace_speculation_t *spec; 14510 dtrace_buffer_t *buf; 14511 #ifdef illumos 14512 cyc_handler_t hdlr; 14513 cyc_time_t when; 14514 #endif 14515 int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t); 14516 dtrace_icookie_t cookie; 14517 14518 mutex_enter(&cpu_lock); 14519 mutex_enter(&dtrace_lock); 14520 14521 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { 14522 rval = EBUSY; 14523 goto out; 14524 } 14525 14526 /* 14527 * Before we can perform any checks, we must prime all of the 14528 * retained enablings that correspond to this state. 14529 */ 14530 dtrace_enabling_prime(state); 14531 14532 if (state->dts_destructive && !state->dts_cred.dcr_destructive) { 14533 rval = EACCES; 14534 goto out; 14535 } 14536 14537 dtrace_state_prereserve(state); 14538 14539 /* 14540 * Now we want to do is try to allocate our speculations. 14541 * We do not automatically resize the number of speculations; if 14542 * this fails, we will fail the operation. 14543 */ 14544 nspec = opt[DTRACEOPT_NSPEC]; 14545 ASSERT(nspec != DTRACEOPT_UNSET); 14546 14547 if (nspec > INT_MAX) { 14548 rval = ENOMEM; 14549 goto out; 14550 } 14551 14552 spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t), 14553 KM_NOSLEEP | KM_NORMALPRI); 14554 14555 if (spec == NULL) { 14556 rval = ENOMEM; 14557 goto out; 14558 } 14559 14560 state->dts_speculations = spec; 14561 state->dts_nspeculations = (int)nspec; 14562 14563 for (i = 0; i < nspec; i++) { 14564 if ((buf = kmem_zalloc(bufsize, 14565 KM_NOSLEEP | KM_NORMALPRI)) == NULL) { 14566 rval = ENOMEM; 14567 goto err; 14568 } 14569 14570 spec[i].dtsp_buffer = buf; 14571 } 14572 14573 if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) { 14574 if (dtrace_anon.dta_state == NULL) { 14575 rval = ENOENT; 14576 goto out; 14577 } 14578 14579 if (state->dts_necbs != 0) { 14580 rval = EALREADY; 14581 goto out; 14582 } 14583 14584 state->dts_anon = dtrace_anon_grab(); 14585 ASSERT(state->dts_anon != NULL); 14586 state = state->dts_anon; 14587 14588 /* 14589 * We want "grabanon" to be set in the grabbed state, so we'll 14590 * copy that option value from the grabbing state into the 14591 * grabbed state. 14592 */ 14593 state->dts_options[DTRACEOPT_GRABANON] = 14594 opt[DTRACEOPT_GRABANON]; 14595 14596 *cpu = dtrace_anon.dta_beganon; 14597 14598 /* 14599 * If the anonymous state is active (as it almost certainly 14600 * is if the anonymous enabling ultimately matched anything), 14601 * we don't allow any further option processing -- but we 14602 * don't return failure. 14603 */ 14604 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) 14605 goto out; 14606 } 14607 14608 if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET && 14609 opt[DTRACEOPT_AGGSIZE] != 0) { 14610 if (state->dts_aggregations == NULL) { 14611 /* 14612 * We're not going to create an aggregation buffer 14613 * because we don't have any ECBs that contain 14614 * aggregations -- set this option to 0. 14615 */ 14616 opt[DTRACEOPT_AGGSIZE] = 0; 14617 } else { 14618 /* 14619 * If we have an aggregation buffer, we must also have 14620 * a buffer to use as scratch. 14621 */ 14622 if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET || 14623 opt[DTRACEOPT_BUFSIZE] < state->dts_needed) { 14624 opt[DTRACEOPT_BUFSIZE] = state->dts_needed; 14625 } 14626 } 14627 } 14628 14629 if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET && 14630 opt[DTRACEOPT_SPECSIZE] != 0) { 14631 if (!state->dts_speculates) { 14632 /* 14633 * We're not going to create speculation buffers 14634 * because we don't have any ECBs that actually 14635 * speculate -- set the speculation size to 0. 14636 */ 14637 opt[DTRACEOPT_SPECSIZE] = 0; 14638 } 14639 } 14640 14641 /* 14642 * The bare minimum size for any buffer that we're actually going to 14643 * do anything to is sizeof (uint64_t). 14644 */ 14645 sz = sizeof (uint64_t); 14646 14647 if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) || 14648 (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) || 14649 (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) { 14650 /* 14651 * A buffer size has been explicitly set to 0 (or to a size 14652 * that will be adjusted to 0) and we need the space -- we 14653 * need to return failure. We return ENOSPC to differentiate 14654 * it from failing to allocate a buffer due to failure to meet 14655 * the reserve (for which we return E2BIG). 14656 */ 14657 rval = ENOSPC; 14658 goto out; 14659 } 14660 14661 if ((rval = dtrace_state_buffers(state)) != 0) 14662 goto err; 14663 14664 if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET) 14665 sz = dtrace_dstate_defsize; 14666 14667 do { 14668 rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz); 14669 14670 if (rval == 0) 14671 break; 14672 14673 if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL) 14674 goto err; 14675 } while (sz >>= 1); 14676 14677 opt[DTRACEOPT_DYNVARSIZE] = sz; 14678 14679 if (rval != 0) 14680 goto err; 14681 14682 if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max) 14683 opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max; 14684 14685 if (opt[DTRACEOPT_CLEANRATE] == 0) 14686 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max; 14687 14688 if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min) 14689 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min; 14690 14691 if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max) 14692 opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max; 14693 14694 state->dts_alive = state->dts_laststatus = dtrace_gethrtime(); 14695 #ifdef illumos 14696 hdlr.cyh_func = (cyc_func_t)dtrace_state_clean; 14697 hdlr.cyh_arg = state; 14698 hdlr.cyh_level = CY_LOW_LEVEL; 14699 14700 when.cyt_when = 0; 14701 when.cyt_interval = opt[DTRACEOPT_CLEANRATE]; 14702 14703 state->dts_cleaner = cyclic_add(&hdlr, &when); 14704 14705 hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman; 14706 hdlr.cyh_arg = state; 14707 hdlr.cyh_level = CY_LOW_LEVEL; 14708 14709 when.cyt_when = 0; 14710 when.cyt_interval = dtrace_deadman_interval; 14711 14712 state->dts_deadman = cyclic_add(&hdlr, &when); 14713 #else 14714 callout_reset(&state->dts_cleaner, hz * opt[DTRACEOPT_CLEANRATE] / NANOSEC, 14715 dtrace_state_clean, state); 14716 callout_reset(&state->dts_deadman, hz * dtrace_deadman_interval / NANOSEC, 14717 dtrace_state_deadman, state); 14718 #endif 14719 14720 state->dts_activity = DTRACE_ACTIVITY_WARMUP; 14721 14722 #ifdef illumos 14723 if (state->dts_getf != 0 && 14724 !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) { 14725 /* 14726 * We don't have kernel privs but we have at least one call 14727 * to getf(); we need to bump our zone's count, and (if 14728 * this is the first enabling to have an unprivileged call 14729 * to getf()) we need to hook into closef(). 14730 */ 14731 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf++; 14732 14733 if (dtrace_getf++ == 0) { 14734 ASSERT(dtrace_closef == NULL); 14735 dtrace_closef = dtrace_getf_barrier; 14736 } 14737 } 14738 #endif 14739 14740 /* 14741 * Now it's time to actually fire the BEGIN probe. We need to disable 14742 * interrupts here both to record the CPU on which we fired the BEGIN 14743 * probe (the data from this CPU will be processed first at user 14744 * level) and to manually activate the buffer for this CPU. 14745 */ 14746 cookie = dtrace_interrupt_disable(); 14747 *cpu = curcpu; 14748 ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE); 14749 state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE; 14750 14751 dtrace_probe(dtrace_probeid_begin, 14752 (uint64_t)(uintptr_t)state, 0, 0, 0, 0); 14753 dtrace_interrupt_enable(cookie); 14754 /* 14755 * We may have had an exit action from a BEGIN probe; only change our 14756 * state to ACTIVE if we're still in WARMUP. 14757 */ 14758 ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP || 14759 state->dts_activity == DTRACE_ACTIVITY_DRAINING); 14760 14761 if (state->dts_activity == DTRACE_ACTIVITY_WARMUP) 14762 state->dts_activity = DTRACE_ACTIVITY_ACTIVE; 14763 14764 /* 14765 * Regardless of whether or not now we're in ACTIVE or DRAINING, we 14766 * want each CPU to transition its principal buffer out of the 14767 * INACTIVE state. Doing this assures that no CPU will suddenly begin 14768 * processing an ECB halfway down a probe's ECB chain; all CPUs will 14769 * atomically transition from processing none of a state's ECBs to 14770 * processing all of them. 14771 */ 14772 dtrace_xcall(DTRACE_CPUALL, 14773 (dtrace_xcall_t)dtrace_buffer_activate, state); 14774 goto out; 14775 14776 err: 14777 dtrace_buffer_free(state->dts_buffer); 14778 dtrace_buffer_free(state->dts_aggbuffer); 14779 14780 if ((nspec = state->dts_nspeculations) == 0) { 14781 ASSERT(state->dts_speculations == NULL); 14782 goto out; 14783 } 14784 14785 spec = state->dts_speculations; 14786 ASSERT(spec != NULL); 14787 14788 for (i = 0; i < state->dts_nspeculations; i++) { 14789 if ((buf = spec[i].dtsp_buffer) == NULL) 14790 break; 14791 14792 dtrace_buffer_free(buf); 14793 kmem_free(buf, bufsize); 14794 } 14795 14796 kmem_free(spec, nspec * sizeof (dtrace_speculation_t)); 14797 state->dts_nspeculations = 0; 14798 state->dts_speculations = NULL; 14799 14800 out: 14801 mutex_exit(&dtrace_lock); 14802 mutex_exit(&cpu_lock); 14803 14804 return (rval); 14805 } 14806 14807 static int 14808 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu) 14809 { 14810 dtrace_icookie_t cookie; 14811 14812 ASSERT(MUTEX_HELD(&dtrace_lock)); 14813 14814 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE && 14815 state->dts_activity != DTRACE_ACTIVITY_DRAINING) 14816 return (EINVAL); 14817 14818 /* 14819 * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync 14820 * to be sure that every CPU has seen it. See below for the details 14821 * on why this is done. 14822 */ 14823 state->dts_activity = DTRACE_ACTIVITY_DRAINING; 14824 dtrace_sync(); 14825 14826 /* 14827 * By this point, it is impossible for any CPU to be still processing 14828 * with DTRACE_ACTIVITY_ACTIVE. We can thus set our activity to 14829 * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any 14830 * other CPU in dtrace_buffer_reserve(). This allows dtrace_probe() 14831 * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN 14832 * iff we're in the END probe. 14833 */ 14834 state->dts_activity = DTRACE_ACTIVITY_COOLDOWN; 14835 dtrace_sync(); 14836 ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN); 14837 14838 /* 14839 * Finally, we can release the reserve and call the END probe. We 14840 * disable interrupts across calling the END probe to allow us to 14841 * return the CPU on which we actually called the END probe. This 14842 * allows user-land to be sure that this CPU's principal buffer is 14843 * processed last. 14844 */ 14845 state->dts_reserve = 0; 14846 14847 cookie = dtrace_interrupt_disable(); 14848 *cpu = curcpu; 14849 dtrace_probe(dtrace_probeid_end, 14850 (uint64_t)(uintptr_t)state, 0, 0, 0, 0); 14851 dtrace_interrupt_enable(cookie); 14852 14853 state->dts_activity = DTRACE_ACTIVITY_STOPPED; 14854 dtrace_sync(); 14855 14856 #ifdef illumos 14857 if (state->dts_getf != 0 && 14858 !(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL)) { 14859 /* 14860 * We don't have kernel privs but we have at least one call 14861 * to getf(); we need to lower our zone's count, and (if 14862 * this is the last enabling to have an unprivileged call 14863 * to getf()) we need to clear the closef() hook. 14864 */ 14865 ASSERT(state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf > 0); 14866 ASSERT(dtrace_closef == dtrace_getf_barrier); 14867 ASSERT(dtrace_getf > 0); 14868 14869 state->dts_cred.dcr_cred->cr_zone->zone_dtrace_getf--; 14870 14871 if (--dtrace_getf == 0) 14872 dtrace_closef = NULL; 14873 } 14874 #endif 14875 14876 return (0); 14877 } 14878 14879 static int 14880 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option, 14881 dtrace_optval_t val) 14882 { 14883 ASSERT(MUTEX_HELD(&dtrace_lock)); 14884 14885 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) 14886 return (EBUSY); 14887 14888 if (option >= DTRACEOPT_MAX) 14889 return (EINVAL); 14890 14891 if (option != DTRACEOPT_CPU && val < 0) 14892 return (EINVAL); 14893 14894 switch (option) { 14895 case DTRACEOPT_DESTRUCTIVE: 14896 if (dtrace_destructive_disallow) 14897 return (EACCES); 14898 14899 state->dts_cred.dcr_destructive = 1; 14900 break; 14901 14902 case DTRACEOPT_BUFSIZE: 14903 case DTRACEOPT_DYNVARSIZE: 14904 case DTRACEOPT_AGGSIZE: 14905 case DTRACEOPT_SPECSIZE: 14906 case DTRACEOPT_STRSIZE: 14907 if (val < 0) 14908 return (EINVAL); 14909 14910 if (val >= LONG_MAX) { 14911 /* 14912 * If this is an otherwise negative value, set it to 14913 * the highest multiple of 128m less than LONG_MAX. 14914 * Technically, we're adjusting the size without 14915 * regard to the buffer resizing policy, but in fact, 14916 * this has no effect -- if we set the buffer size to 14917 * ~LONG_MAX and the buffer policy is ultimately set to 14918 * be "manual", the buffer allocation is guaranteed to 14919 * fail, if only because the allocation requires two 14920 * buffers. (We set the the size to the highest 14921 * multiple of 128m because it ensures that the size 14922 * will remain a multiple of a megabyte when 14923 * repeatedly halved -- all the way down to 15m.) 14924 */ 14925 val = LONG_MAX - (1 << 27) + 1; 14926 } 14927 } 14928 14929 state->dts_options[option] = val; 14930 14931 return (0); 14932 } 14933 14934 static void 14935 dtrace_state_destroy(dtrace_state_t *state) 14936 { 14937 dtrace_ecb_t *ecb; 14938 dtrace_vstate_t *vstate = &state->dts_vstate; 14939 #ifdef illumos 14940 minor_t minor = getminor(state->dts_dev); 14941 #endif 14942 int i, bufsize = NCPU * sizeof (dtrace_buffer_t); 14943 dtrace_speculation_t *spec = state->dts_speculations; 14944 int nspec = state->dts_nspeculations; 14945 uint32_t match; 14946 14947 ASSERT(MUTEX_HELD(&dtrace_lock)); 14948 ASSERT(MUTEX_HELD(&cpu_lock)); 14949 14950 /* 14951 * First, retract any retained enablings for this state. 14952 */ 14953 dtrace_enabling_retract(state); 14954 ASSERT(state->dts_nretained == 0); 14955 14956 if (state->dts_activity == DTRACE_ACTIVITY_ACTIVE || 14957 state->dts_activity == DTRACE_ACTIVITY_DRAINING) { 14958 /* 14959 * We have managed to come into dtrace_state_destroy() on a 14960 * hot enabling -- almost certainly because of a disorderly 14961 * shutdown of a consumer. (That is, a consumer that is 14962 * exiting without having called dtrace_stop().) In this case, 14963 * we're going to set our activity to be KILLED, and then 14964 * issue a sync to be sure that everyone is out of probe 14965 * context before we start blowing away ECBs. 14966 */ 14967 state->dts_activity = DTRACE_ACTIVITY_KILLED; 14968 dtrace_sync(); 14969 } 14970 14971 /* 14972 * Release the credential hold we took in dtrace_state_create(). 14973 */ 14974 if (state->dts_cred.dcr_cred != NULL) 14975 crfree(state->dts_cred.dcr_cred); 14976 14977 /* 14978 * Now we can safely disable and destroy any enabled probes. Because 14979 * any DTRACE_PRIV_KERNEL probes may actually be slowing our progress 14980 * (especially if they're all enabled), we take two passes through the 14981 * ECBs: in the first, we disable just DTRACE_PRIV_KERNEL probes, and 14982 * in the second we disable whatever is left over. 14983 */ 14984 for (match = DTRACE_PRIV_KERNEL; ; match = 0) { 14985 for (i = 0; i < state->dts_necbs; i++) { 14986 if ((ecb = state->dts_ecbs[i]) == NULL) 14987 continue; 14988 14989 if (match && ecb->dte_probe != NULL) { 14990 dtrace_probe_t *probe = ecb->dte_probe; 14991 dtrace_provider_t *prov = probe->dtpr_provider; 14992 14993 if (!(prov->dtpv_priv.dtpp_flags & match)) 14994 continue; 14995 } 14996 14997 dtrace_ecb_disable(ecb); 14998 dtrace_ecb_destroy(ecb); 14999 } 15000 15001 if (!match) 15002 break; 15003 } 15004 15005 /* 15006 * Before we free the buffers, perform one more sync to assure that 15007 * every CPU is out of probe context. 15008 */ 15009 dtrace_sync(); 15010 15011 dtrace_buffer_free(state->dts_buffer); 15012 dtrace_buffer_free(state->dts_aggbuffer); 15013 15014 for (i = 0; i < nspec; i++) 15015 dtrace_buffer_free(spec[i].dtsp_buffer); 15016 15017 #ifdef illumos 15018 if (state->dts_cleaner != CYCLIC_NONE) 15019 cyclic_remove(state->dts_cleaner); 15020 15021 if (state->dts_deadman != CYCLIC_NONE) 15022 cyclic_remove(state->dts_deadman); 15023 #else 15024 callout_stop(&state->dts_cleaner); 15025 callout_drain(&state->dts_cleaner); 15026 callout_stop(&state->dts_deadman); 15027 callout_drain(&state->dts_deadman); 15028 #endif 15029 15030 dtrace_dstate_fini(&vstate->dtvs_dynvars); 15031 dtrace_vstate_fini(vstate); 15032 if (state->dts_ecbs != NULL) 15033 kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *)); 15034 15035 if (state->dts_aggregations != NULL) { 15036 #ifdef DEBUG 15037 for (i = 0; i < state->dts_naggregations; i++) 15038 ASSERT(state->dts_aggregations[i] == NULL); 15039 #endif 15040 ASSERT(state->dts_naggregations > 0); 15041 kmem_free(state->dts_aggregations, 15042 state->dts_naggregations * sizeof (dtrace_aggregation_t *)); 15043 } 15044 15045 kmem_free(state->dts_buffer, bufsize); 15046 kmem_free(state->dts_aggbuffer, bufsize); 15047 15048 for (i = 0; i < nspec; i++) 15049 kmem_free(spec[i].dtsp_buffer, bufsize); 15050 15051 if (spec != NULL) 15052 kmem_free(spec, nspec * sizeof (dtrace_speculation_t)); 15053 15054 dtrace_format_destroy(state); 15055 15056 if (state->dts_aggid_arena != NULL) { 15057 #ifdef illumos 15058 vmem_destroy(state->dts_aggid_arena); 15059 #else 15060 delete_unrhdr(state->dts_aggid_arena); 15061 #endif 15062 state->dts_aggid_arena = NULL; 15063 } 15064 #ifdef illumos 15065 ddi_soft_state_free(dtrace_softstate, minor); 15066 vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1); 15067 #endif 15068 } 15069 15070 /* 15071 * DTrace Anonymous Enabling Functions 15072 */ 15073 static dtrace_state_t * 15074 dtrace_anon_grab(void) 15075 { 15076 dtrace_state_t *state; 15077 15078 ASSERT(MUTEX_HELD(&dtrace_lock)); 15079 15080 if ((state = dtrace_anon.dta_state) == NULL) { 15081 ASSERT(dtrace_anon.dta_enabling == NULL); 15082 return (NULL); 15083 } 15084 15085 ASSERT(dtrace_anon.dta_enabling != NULL); 15086 ASSERT(dtrace_retained != NULL); 15087 15088 dtrace_enabling_destroy(dtrace_anon.dta_enabling); 15089 dtrace_anon.dta_enabling = NULL; 15090 dtrace_anon.dta_state = NULL; 15091 15092 return (state); 15093 } 15094 15095 static void 15096 dtrace_anon_property(void) 15097 { 15098 int i, rv; 15099 dtrace_state_t *state; 15100 dof_hdr_t *dof; 15101 char c[32]; /* enough for "dof-data-" + digits */ 15102 15103 ASSERT(MUTEX_HELD(&dtrace_lock)); 15104 ASSERT(MUTEX_HELD(&cpu_lock)); 15105 15106 for (i = 0; ; i++) { 15107 (void) snprintf(c, sizeof (c), "dof-data-%d", i); 15108 15109 dtrace_err_verbose = 1; 15110 15111 if ((dof = dtrace_dof_property(c)) == NULL) { 15112 dtrace_err_verbose = 0; 15113 break; 15114 } 15115 15116 #ifdef illumos 15117 /* 15118 * We want to create anonymous state, so we need to transition 15119 * the kernel debugger to indicate that DTrace is active. If 15120 * this fails (e.g. because the debugger has modified text in 15121 * some way), we won't continue with the processing. 15122 */ 15123 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { 15124 cmn_err(CE_NOTE, "kernel debugger active; anonymous " 15125 "enabling ignored."); 15126 dtrace_dof_destroy(dof); 15127 break; 15128 } 15129 #endif 15130 15131 /* 15132 * If we haven't allocated an anonymous state, we'll do so now. 15133 */ 15134 if ((state = dtrace_anon.dta_state) == NULL) { 15135 #ifdef illumos 15136 state = dtrace_state_create(NULL, NULL); 15137 #else 15138 state = dtrace_state_create(NULL); 15139 #endif 15140 dtrace_anon.dta_state = state; 15141 15142 if (state == NULL) { 15143 /* 15144 * This basically shouldn't happen: the only 15145 * failure mode from dtrace_state_create() is a 15146 * failure of ddi_soft_state_zalloc() that 15147 * itself should never happen. Still, the 15148 * interface allows for a failure mode, and 15149 * we want to fail as gracefully as possible: 15150 * we'll emit an error message and cease 15151 * processing anonymous state in this case. 15152 */ 15153 cmn_err(CE_WARN, "failed to create " 15154 "anonymous state"); 15155 dtrace_dof_destroy(dof); 15156 break; 15157 } 15158 } 15159 15160 rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(), 15161 &dtrace_anon.dta_enabling, 0, B_TRUE); 15162 15163 if (rv == 0) 15164 rv = dtrace_dof_options(dof, state); 15165 15166 dtrace_err_verbose = 0; 15167 dtrace_dof_destroy(dof); 15168 15169 if (rv != 0) { 15170 /* 15171 * This is malformed DOF; chuck any anonymous state 15172 * that we created. 15173 */ 15174 ASSERT(dtrace_anon.dta_enabling == NULL); 15175 dtrace_state_destroy(state); 15176 dtrace_anon.dta_state = NULL; 15177 break; 15178 } 15179 15180 ASSERT(dtrace_anon.dta_enabling != NULL); 15181 } 15182 15183 if (dtrace_anon.dta_enabling != NULL) { 15184 int rval; 15185 15186 /* 15187 * dtrace_enabling_retain() can only fail because we are 15188 * trying to retain more enablings than are allowed -- but 15189 * we only have one anonymous enabling, and we are guaranteed 15190 * to be allowed at least one retained enabling; we assert 15191 * that dtrace_enabling_retain() returns success. 15192 */ 15193 rval = dtrace_enabling_retain(dtrace_anon.dta_enabling); 15194 ASSERT(rval == 0); 15195 15196 dtrace_enabling_dump(dtrace_anon.dta_enabling); 15197 } 15198 } 15199 15200 /* 15201 * DTrace Helper Functions 15202 */ 15203 static void 15204 dtrace_helper_trace(dtrace_helper_action_t *helper, 15205 dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where) 15206 { 15207 uint32_t size, next, nnext, i; 15208 dtrace_helptrace_t *ent, *buffer; 15209 uint16_t flags = cpu_core[curcpu].cpuc_dtrace_flags; 15210 15211 if ((buffer = dtrace_helptrace_buffer) == NULL) 15212 return; 15213 15214 ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals); 15215 15216 /* 15217 * What would a tracing framework be without its own tracing 15218 * framework? (Well, a hell of a lot simpler, for starters...) 15219 */ 15220 size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals * 15221 sizeof (uint64_t) - sizeof (uint64_t); 15222 15223 /* 15224 * Iterate until we can allocate a slot in the trace buffer. 15225 */ 15226 do { 15227 next = dtrace_helptrace_next; 15228 15229 if (next + size < dtrace_helptrace_bufsize) { 15230 nnext = next + size; 15231 } else { 15232 nnext = size; 15233 } 15234 } while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next); 15235 15236 /* 15237 * We have our slot; fill it in. 15238 */ 15239 if (nnext == size) { 15240 dtrace_helptrace_wrapped++; 15241 next = 0; 15242 } 15243 15244 ent = (dtrace_helptrace_t *)((uintptr_t)buffer + next); 15245 ent->dtht_helper = helper; 15246 ent->dtht_where = where; 15247 ent->dtht_nlocals = vstate->dtvs_nlocals; 15248 15249 ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ? 15250 mstate->dtms_fltoffs : -1; 15251 ent->dtht_fault = DTRACE_FLAGS2FLT(flags); 15252 ent->dtht_illval = cpu_core[curcpu].cpuc_dtrace_illval; 15253 15254 for (i = 0; i < vstate->dtvs_nlocals; i++) { 15255 dtrace_statvar_t *svar; 15256 15257 if ((svar = vstate->dtvs_locals[i]) == NULL) 15258 continue; 15259 15260 ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t)); 15261 ent->dtht_locals[i] = 15262 ((uint64_t *)(uintptr_t)svar->dtsv_data)[curcpu]; 15263 } 15264 } 15265 15266 static uint64_t 15267 dtrace_helper(int which, dtrace_mstate_t *mstate, 15268 dtrace_state_t *state, uint64_t arg0, uint64_t arg1) 15269 { 15270 uint16_t *flags = &cpu_core[curcpu].cpuc_dtrace_flags; 15271 uint64_t sarg0 = mstate->dtms_arg[0]; 15272 uint64_t sarg1 = mstate->dtms_arg[1]; 15273 uint64_t rval = 0; 15274 dtrace_helpers_t *helpers = curproc->p_dtrace_helpers; 15275 dtrace_helper_action_t *helper; 15276 dtrace_vstate_t *vstate; 15277 dtrace_difo_t *pred; 15278 int i, trace = dtrace_helptrace_buffer != NULL; 15279 15280 ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS); 15281 15282 if (helpers == NULL) 15283 return (0); 15284 15285 if ((helper = helpers->dthps_actions[which]) == NULL) 15286 return (0); 15287 15288 vstate = &helpers->dthps_vstate; 15289 mstate->dtms_arg[0] = arg0; 15290 mstate->dtms_arg[1] = arg1; 15291 15292 /* 15293 * Now iterate over each helper. If its predicate evaluates to 'true', 15294 * we'll call the corresponding actions. Note that the below calls 15295 * to dtrace_dif_emulate() may set faults in machine state. This is 15296 * okay: our caller (the outer dtrace_dif_emulate()) will simply plow 15297 * the stored DIF offset with its own (which is the desired behavior). 15298 * Also, note the calls to dtrace_dif_emulate() may allocate scratch 15299 * from machine state; this is okay, too. 15300 */ 15301 for (; helper != NULL; helper = helper->dtha_next) { 15302 if ((pred = helper->dtha_predicate) != NULL) { 15303 if (trace) 15304 dtrace_helper_trace(helper, mstate, vstate, 0); 15305 15306 if (!dtrace_dif_emulate(pred, mstate, vstate, state)) 15307 goto next; 15308 15309 if (*flags & CPU_DTRACE_FAULT) 15310 goto err; 15311 } 15312 15313 for (i = 0; i < helper->dtha_nactions; i++) { 15314 if (trace) 15315 dtrace_helper_trace(helper, 15316 mstate, vstate, i + 1); 15317 15318 rval = dtrace_dif_emulate(helper->dtha_actions[i], 15319 mstate, vstate, state); 15320 15321 if (*flags & CPU_DTRACE_FAULT) 15322 goto err; 15323 } 15324 15325 next: 15326 if (trace) 15327 dtrace_helper_trace(helper, mstate, vstate, 15328 DTRACE_HELPTRACE_NEXT); 15329 } 15330 15331 if (trace) 15332 dtrace_helper_trace(helper, mstate, vstate, 15333 DTRACE_HELPTRACE_DONE); 15334 15335 /* 15336 * Restore the arg0 that we saved upon entry. 15337 */ 15338 mstate->dtms_arg[0] = sarg0; 15339 mstate->dtms_arg[1] = sarg1; 15340 15341 return (rval); 15342 15343 err: 15344 if (trace) 15345 dtrace_helper_trace(helper, mstate, vstate, 15346 DTRACE_HELPTRACE_ERR); 15347 15348 /* 15349 * Restore the arg0 that we saved upon entry. 15350 */ 15351 mstate->dtms_arg[0] = sarg0; 15352 mstate->dtms_arg[1] = sarg1; 15353 15354 return (0); 15355 } 15356 15357 static void 15358 dtrace_helper_action_destroy(dtrace_helper_action_t *helper, 15359 dtrace_vstate_t *vstate) 15360 { 15361 int i; 15362 15363 if (helper->dtha_predicate != NULL) 15364 dtrace_difo_release(helper->dtha_predicate, vstate); 15365 15366 for (i = 0; i < helper->dtha_nactions; i++) { 15367 ASSERT(helper->dtha_actions[i] != NULL); 15368 dtrace_difo_release(helper->dtha_actions[i], vstate); 15369 } 15370 15371 kmem_free(helper->dtha_actions, 15372 helper->dtha_nactions * sizeof (dtrace_difo_t *)); 15373 kmem_free(helper, sizeof (dtrace_helper_action_t)); 15374 } 15375 15376 static int 15377 dtrace_helper_destroygen(int gen) 15378 { 15379 proc_t *p = curproc; 15380 dtrace_helpers_t *help = p->p_dtrace_helpers; 15381 dtrace_vstate_t *vstate; 15382 int i; 15383 15384 ASSERT(MUTEX_HELD(&dtrace_lock)); 15385 15386 if (help == NULL || gen > help->dthps_generation) 15387 return (EINVAL); 15388 15389 vstate = &help->dthps_vstate; 15390 15391 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { 15392 dtrace_helper_action_t *last = NULL, *h, *next; 15393 15394 for (h = help->dthps_actions[i]; h != NULL; h = next) { 15395 next = h->dtha_next; 15396 15397 if (h->dtha_generation == gen) { 15398 if (last != NULL) { 15399 last->dtha_next = next; 15400 } else { 15401 help->dthps_actions[i] = next; 15402 } 15403 15404 dtrace_helper_action_destroy(h, vstate); 15405 } else { 15406 last = h; 15407 } 15408 } 15409 } 15410 15411 /* 15412 * Interate until we've cleared out all helper providers with the 15413 * given generation number. 15414 */ 15415 for (;;) { 15416 dtrace_helper_provider_t *prov; 15417 15418 /* 15419 * Look for a helper provider with the right generation. We 15420 * have to start back at the beginning of the list each time 15421 * because we drop dtrace_lock. It's unlikely that we'll make 15422 * more than two passes. 15423 */ 15424 for (i = 0; i < help->dthps_nprovs; i++) { 15425 prov = help->dthps_provs[i]; 15426 15427 if (prov->dthp_generation == gen) 15428 break; 15429 } 15430 15431 /* 15432 * If there were no matches, we're done. 15433 */ 15434 if (i == help->dthps_nprovs) 15435 break; 15436 15437 /* 15438 * Move the last helper provider into this slot. 15439 */ 15440 help->dthps_nprovs--; 15441 help->dthps_provs[i] = help->dthps_provs[help->dthps_nprovs]; 15442 help->dthps_provs[help->dthps_nprovs] = NULL; 15443 15444 mutex_exit(&dtrace_lock); 15445 15446 /* 15447 * If we have a meta provider, remove this helper provider. 15448 */ 15449 mutex_enter(&dtrace_meta_lock); 15450 if (dtrace_meta_pid != NULL) { 15451 ASSERT(dtrace_deferred_pid == NULL); 15452 dtrace_helper_provider_remove(&prov->dthp_prov, 15453 p->p_pid); 15454 } 15455 mutex_exit(&dtrace_meta_lock); 15456 15457 dtrace_helper_provider_destroy(prov); 15458 15459 mutex_enter(&dtrace_lock); 15460 } 15461 15462 return (0); 15463 } 15464 15465 static int 15466 dtrace_helper_validate(dtrace_helper_action_t *helper) 15467 { 15468 int err = 0, i; 15469 dtrace_difo_t *dp; 15470 15471 if ((dp = helper->dtha_predicate) != NULL) 15472 err += dtrace_difo_validate_helper(dp); 15473 15474 for (i = 0; i < helper->dtha_nactions; i++) 15475 err += dtrace_difo_validate_helper(helper->dtha_actions[i]); 15476 15477 return (err == 0); 15478 } 15479 15480 static int 15481 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep) 15482 { 15483 dtrace_helpers_t *help; 15484 dtrace_helper_action_t *helper, *last; 15485 dtrace_actdesc_t *act; 15486 dtrace_vstate_t *vstate; 15487 dtrace_predicate_t *pred; 15488 int count = 0, nactions = 0, i; 15489 15490 if (which < 0 || which >= DTRACE_NHELPER_ACTIONS) 15491 return (EINVAL); 15492 15493 help = curproc->p_dtrace_helpers; 15494 last = help->dthps_actions[which]; 15495 vstate = &help->dthps_vstate; 15496 15497 for (count = 0; last != NULL; last = last->dtha_next) { 15498 count++; 15499 if (last->dtha_next == NULL) 15500 break; 15501 } 15502 15503 /* 15504 * If we already have dtrace_helper_actions_max helper actions for this 15505 * helper action type, we'll refuse to add a new one. 15506 */ 15507 if (count >= dtrace_helper_actions_max) 15508 return (ENOSPC); 15509 15510 helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP); 15511 helper->dtha_generation = help->dthps_generation; 15512 15513 if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) { 15514 ASSERT(pred->dtp_difo != NULL); 15515 dtrace_difo_hold(pred->dtp_difo); 15516 helper->dtha_predicate = pred->dtp_difo; 15517 } 15518 15519 for (act = ep->dted_action; act != NULL; act = act->dtad_next) { 15520 if (act->dtad_kind != DTRACEACT_DIFEXPR) 15521 goto err; 15522 15523 if (act->dtad_difo == NULL) 15524 goto err; 15525 15526 nactions++; 15527 } 15528 15529 helper->dtha_actions = kmem_zalloc(sizeof (dtrace_difo_t *) * 15530 (helper->dtha_nactions = nactions), KM_SLEEP); 15531 15532 for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) { 15533 dtrace_difo_hold(act->dtad_difo); 15534 helper->dtha_actions[i++] = act->dtad_difo; 15535 } 15536 15537 if (!dtrace_helper_validate(helper)) 15538 goto err; 15539 15540 if (last == NULL) { 15541 help->dthps_actions[which] = helper; 15542 } else { 15543 last->dtha_next = helper; 15544 } 15545 15546 if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) { 15547 dtrace_helptrace_nlocals = vstate->dtvs_nlocals; 15548 dtrace_helptrace_next = 0; 15549 } 15550 15551 return (0); 15552 err: 15553 dtrace_helper_action_destroy(helper, vstate); 15554 return (EINVAL); 15555 } 15556 15557 static void 15558 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help, 15559 dof_helper_t *dofhp) 15560 { 15561 ASSERT(MUTEX_NOT_HELD(&dtrace_lock)); 15562 15563 mutex_enter(&dtrace_meta_lock); 15564 mutex_enter(&dtrace_lock); 15565 15566 if (!dtrace_attached() || dtrace_meta_pid == NULL) { 15567 /* 15568 * If the dtrace module is loaded but not attached, or if 15569 * there aren't isn't a meta provider registered to deal with 15570 * these provider descriptions, we need to postpone creating 15571 * the actual providers until later. 15572 */ 15573 15574 if (help->dthps_next == NULL && help->dthps_prev == NULL && 15575 dtrace_deferred_pid != help) { 15576 help->dthps_deferred = 1; 15577 help->dthps_pid = p->p_pid; 15578 help->dthps_next = dtrace_deferred_pid; 15579 help->dthps_prev = NULL; 15580 if (dtrace_deferred_pid != NULL) 15581 dtrace_deferred_pid->dthps_prev = help; 15582 dtrace_deferred_pid = help; 15583 } 15584 15585 mutex_exit(&dtrace_lock); 15586 15587 } else if (dofhp != NULL) { 15588 /* 15589 * If the dtrace module is loaded and we have a particular 15590 * helper provider description, pass that off to the 15591 * meta provider. 15592 */ 15593 15594 mutex_exit(&dtrace_lock); 15595 15596 dtrace_helper_provide(dofhp, p->p_pid); 15597 15598 } else { 15599 /* 15600 * Otherwise, just pass all the helper provider descriptions 15601 * off to the meta provider. 15602 */ 15603 15604 int i; 15605 mutex_exit(&dtrace_lock); 15606 15607 for (i = 0; i < help->dthps_nprovs; i++) { 15608 dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov, 15609 p->p_pid); 15610 } 15611 } 15612 15613 mutex_exit(&dtrace_meta_lock); 15614 } 15615 15616 static int 15617 dtrace_helper_provider_add(dof_helper_t *dofhp, int gen) 15618 { 15619 dtrace_helpers_t *help; 15620 dtrace_helper_provider_t *hprov, **tmp_provs; 15621 uint_t tmp_maxprovs, i; 15622 15623 ASSERT(MUTEX_HELD(&dtrace_lock)); 15624 15625 help = curproc->p_dtrace_helpers; 15626 ASSERT(help != NULL); 15627 15628 /* 15629 * If we already have dtrace_helper_providers_max helper providers, 15630 * we're refuse to add a new one. 15631 */ 15632 if (help->dthps_nprovs >= dtrace_helper_providers_max) 15633 return (ENOSPC); 15634 15635 /* 15636 * Check to make sure this isn't a duplicate. 15637 */ 15638 for (i = 0; i < help->dthps_nprovs; i++) { 15639 if (dofhp->dofhp_dof == 15640 help->dthps_provs[i]->dthp_prov.dofhp_dof) 15641 return (EALREADY); 15642 } 15643 15644 hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP); 15645 hprov->dthp_prov = *dofhp; 15646 hprov->dthp_ref = 1; 15647 hprov->dthp_generation = gen; 15648 15649 /* 15650 * Allocate a bigger table for helper providers if it's already full. 15651 */ 15652 if (help->dthps_maxprovs == help->dthps_nprovs) { 15653 tmp_maxprovs = help->dthps_maxprovs; 15654 tmp_provs = help->dthps_provs; 15655 15656 if (help->dthps_maxprovs == 0) 15657 help->dthps_maxprovs = 2; 15658 else 15659 help->dthps_maxprovs *= 2; 15660 if (help->dthps_maxprovs > dtrace_helper_providers_max) 15661 help->dthps_maxprovs = dtrace_helper_providers_max; 15662 15663 ASSERT(tmp_maxprovs < help->dthps_maxprovs); 15664 15665 help->dthps_provs = kmem_zalloc(help->dthps_maxprovs * 15666 sizeof (dtrace_helper_provider_t *), KM_SLEEP); 15667 15668 if (tmp_provs != NULL) { 15669 bcopy(tmp_provs, help->dthps_provs, tmp_maxprovs * 15670 sizeof (dtrace_helper_provider_t *)); 15671 kmem_free(tmp_provs, tmp_maxprovs * 15672 sizeof (dtrace_helper_provider_t *)); 15673 } 15674 } 15675 15676 help->dthps_provs[help->dthps_nprovs] = hprov; 15677 help->dthps_nprovs++; 15678 15679 return (0); 15680 } 15681 15682 static void 15683 dtrace_helper_provider_destroy(dtrace_helper_provider_t *hprov) 15684 { 15685 mutex_enter(&dtrace_lock); 15686 15687 if (--hprov->dthp_ref == 0) { 15688 dof_hdr_t *dof; 15689 mutex_exit(&dtrace_lock); 15690 dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof; 15691 dtrace_dof_destroy(dof); 15692 kmem_free(hprov, sizeof (dtrace_helper_provider_t)); 15693 } else { 15694 mutex_exit(&dtrace_lock); 15695 } 15696 } 15697 15698 static int 15699 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec) 15700 { 15701 uintptr_t daddr = (uintptr_t)dof; 15702 dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec, *enoff_sec; 15703 dof_provider_t *provider; 15704 dof_probe_t *probe; 15705 uint8_t *arg; 15706 char *strtab, *typestr; 15707 dof_stridx_t typeidx; 15708 size_t typesz; 15709 uint_t nprobes, j, k; 15710 15711 ASSERT(sec->dofs_type == DOF_SECT_PROVIDER); 15712 15713 if (sec->dofs_offset & (sizeof (uint_t) - 1)) { 15714 dtrace_dof_error(dof, "misaligned section offset"); 15715 return (-1); 15716 } 15717 15718 /* 15719 * The section needs to be large enough to contain the DOF provider 15720 * structure appropriate for the given version. 15721 */ 15722 if (sec->dofs_size < 15723 ((dof->dofh_ident[DOF_ID_VERSION] == DOF_VERSION_1) ? 15724 offsetof(dof_provider_t, dofpv_prenoffs) : 15725 sizeof (dof_provider_t))) { 15726 dtrace_dof_error(dof, "provider section too small"); 15727 return (-1); 15728 } 15729 15730 provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset); 15731 str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab); 15732 prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes); 15733 arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs); 15734 off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs); 15735 15736 if (str_sec == NULL || prb_sec == NULL || 15737 arg_sec == NULL || off_sec == NULL) 15738 return (-1); 15739 15740 enoff_sec = NULL; 15741 15742 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1 && 15743 provider->dofpv_prenoffs != DOF_SECT_NONE && 15744 (enoff_sec = dtrace_dof_sect(dof, DOF_SECT_PRENOFFS, 15745 provider->dofpv_prenoffs)) == NULL) 15746 return (-1); 15747 15748 strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset); 15749 15750 if (provider->dofpv_name >= str_sec->dofs_size || 15751 strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) { 15752 dtrace_dof_error(dof, "invalid provider name"); 15753 return (-1); 15754 } 15755 15756 if (prb_sec->dofs_entsize == 0 || 15757 prb_sec->dofs_entsize > prb_sec->dofs_size) { 15758 dtrace_dof_error(dof, "invalid entry size"); 15759 return (-1); 15760 } 15761 15762 if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) { 15763 dtrace_dof_error(dof, "misaligned entry size"); 15764 return (-1); 15765 } 15766 15767 if (off_sec->dofs_entsize != sizeof (uint32_t)) { 15768 dtrace_dof_error(dof, "invalid entry size"); 15769 return (-1); 15770 } 15771 15772 if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) { 15773 dtrace_dof_error(dof, "misaligned section offset"); 15774 return (-1); 15775 } 15776 15777 if (arg_sec->dofs_entsize != sizeof (uint8_t)) { 15778 dtrace_dof_error(dof, "invalid entry size"); 15779 return (-1); 15780 } 15781 15782 arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset); 15783 15784 nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize; 15785 15786 /* 15787 * Take a pass through the probes to check for errors. 15788 */ 15789 for (j = 0; j < nprobes; j++) { 15790 probe = (dof_probe_t *)(uintptr_t)(daddr + 15791 prb_sec->dofs_offset + j * prb_sec->dofs_entsize); 15792 15793 if (probe->dofpr_func >= str_sec->dofs_size) { 15794 dtrace_dof_error(dof, "invalid function name"); 15795 return (-1); 15796 } 15797 15798 if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) { 15799 dtrace_dof_error(dof, "function name too long"); 15800 return (-1); 15801 } 15802 15803 if (probe->dofpr_name >= str_sec->dofs_size || 15804 strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) { 15805 dtrace_dof_error(dof, "invalid probe name"); 15806 return (-1); 15807 } 15808 15809 /* 15810 * The offset count must not wrap the index, and the offsets 15811 * must also not overflow the section's data. 15812 */ 15813 if (probe->dofpr_offidx + probe->dofpr_noffs < 15814 probe->dofpr_offidx || 15815 (probe->dofpr_offidx + probe->dofpr_noffs) * 15816 off_sec->dofs_entsize > off_sec->dofs_size) { 15817 dtrace_dof_error(dof, "invalid probe offset"); 15818 return (-1); 15819 } 15820 15821 if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) { 15822 /* 15823 * If there's no is-enabled offset section, make sure 15824 * there aren't any is-enabled offsets. Otherwise 15825 * perform the same checks as for probe offsets 15826 * (immediately above). 15827 */ 15828 if (enoff_sec == NULL) { 15829 if (probe->dofpr_enoffidx != 0 || 15830 probe->dofpr_nenoffs != 0) { 15831 dtrace_dof_error(dof, "is-enabled " 15832 "offsets with null section"); 15833 return (-1); 15834 } 15835 } else if (probe->dofpr_enoffidx + 15836 probe->dofpr_nenoffs < probe->dofpr_enoffidx || 15837 (probe->dofpr_enoffidx + probe->dofpr_nenoffs) * 15838 enoff_sec->dofs_entsize > enoff_sec->dofs_size) { 15839 dtrace_dof_error(dof, "invalid is-enabled " 15840 "offset"); 15841 return (-1); 15842 } 15843 15844 if (probe->dofpr_noffs + probe->dofpr_nenoffs == 0) { 15845 dtrace_dof_error(dof, "zero probe and " 15846 "is-enabled offsets"); 15847 return (-1); 15848 } 15849 } else if (probe->dofpr_noffs == 0) { 15850 dtrace_dof_error(dof, "zero probe offsets"); 15851 return (-1); 15852 } 15853 15854 if (probe->dofpr_argidx + probe->dofpr_xargc < 15855 probe->dofpr_argidx || 15856 (probe->dofpr_argidx + probe->dofpr_xargc) * 15857 arg_sec->dofs_entsize > arg_sec->dofs_size) { 15858 dtrace_dof_error(dof, "invalid args"); 15859 return (-1); 15860 } 15861 15862 typeidx = probe->dofpr_nargv; 15863 typestr = strtab + probe->dofpr_nargv; 15864 for (k = 0; k < probe->dofpr_nargc; k++) { 15865 if (typeidx >= str_sec->dofs_size) { 15866 dtrace_dof_error(dof, "bad " 15867 "native argument type"); 15868 return (-1); 15869 } 15870 15871 typesz = strlen(typestr) + 1; 15872 if (typesz > DTRACE_ARGTYPELEN) { 15873 dtrace_dof_error(dof, "native " 15874 "argument type too long"); 15875 return (-1); 15876 } 15877 typeidx += typesz; 15878 typestr += typesz; 15879 } 15880 15881 typeidx = probe->dofpr_xargv; 15882 typestr = strtab + probe->dofpr_xargv; 15883 for (k = 0; k < probe->dofpr_xargc; k++) { 15884 if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) { 15885 dtrace_dof_error(dof, "bad " 15886 "native argument index"); 15887 return (-1); 15888 } 15889 15890 if (typeidx >= str_sec->dofs_size) { 15891 dtrace_dof_error(dof, "bad " 15892 "translated argument type"); 15893 return (-1); 15894 } 15895 15896 typesz = strlen(typestr) + 1; 15897 if (typesz > DTRACE_ARGTYPELEN) { 15898 dtrace_dof_error(dof, "translated argument " 15899 "type too long"); 15900 return (-1); 15901 } 15902 15903 typeidx += typesz; 15904 typestr += typesz; 15905 } 15906 } 15907 15908 return (0); 15909 } 15910 15911 static int 15912 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp) 15913 { 15914 dtrace_helpers_t *help; 15915 dtrace_vstate_t *vstate; 15916 dtrace_enabling_t *enab = NULL; 15917 int i, gen, rv, nhelpers = 0, nprovs = 0, destroy = 1; 15918 uintptr_t daddr = (uintptr_t)dof; 15919 15920 ASSERT(MUTEX_HELD(&dtrace_lock)); 15921 15922 if ((help = curproc->p_dtrace_helpers) == NULL) 15923 help = dtrace_helpers_create(curproc); 15924 15925 vstate = &help->dthps_vstate; 15926 15927 if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab, 15928 dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) { 15929 dtrace_dof_destroy(dof); 15930 return (rv); 15931 } 15932 15933 /* 15934 * Look for helper providers and validate their descriptions. 15935 */ 15936 if (dhp != NULL) { 15937 for (i = 0; i < dof->dofh_secnum; i++) { 15938 dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr + 15939 dof->dofh_secoff + i * dof->dofh_secsize); 15940 15941 if (sec->dofs_type != DOF_SECT_PROVIDER) 15942 continue; 15943 15944 if (dtrace_helper_provider_validate(dof, sec) != 0) { 15945 dtrace_enabling_destroy(enab); 15946 dtrace_dof_destroy(dof); 15947 return (-1); 15948 } 15949 15950 nprovs++; 15951 } 15952 } 15953 15954 /* 15955 * Now we need to walk through the ECB descriptions in the enabling. 15956 */ 15957 for (i = 0; i < enab->dten_ndesc; i++) { 15958 dtrace_ecbdesc_t *ep = enab->dten_desc[i]; 15959 dtrace_probedesc_t *desc = &ep->dted_probe; 15960 15961 if (strcmp(desc->dtpd_provider, "dtrace") != 0) 15962 continue; 15963 15964 if (strcmp(desc->dtpd_mod, "helper") != 0) 15965 continue; 15966 15967 if (strcmp(desc->dtpd_func, "ustack") != 0) 15968 continue; 15969 15970 if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK, 15971 ep)) != 0) { 15972 /* 15973 * Adding this helper action failed -- we are now going 15974 * to rip out the entire generation and return failure. 15975 */ 15976 (void) dtrace_helper_destroygen(help->dthps_generation); 15977 dtrace_enabling_destroy(enab); 15978 dtrace_dof_destroy(dof); 15979 return (-1); 15980 } 15981 15982 nhelpers++; 15983 } 15984 15985 if (nhelpers < enab->dten_ndesc) 15986 dtrace_dof_error(dof, "unmatched helpers"); 15987 15988 gen = help->dthps_generation++; 15989 dtrace_enabling_destroy(enab); 15990 15991 if (dhp != NULL && nprovs > 0) { 15992 dhp->dofhp_dof = (uint64_t)(uintptr_t)dof; 15993 if (dtrace_helper_provider_add(dhp, gen) == 0) { 15994 mutex_exit(&dtrace_lock); 15995 dtrace_helper_provider_register(curproc, help, dhp); 15996 mutex_enter(&dtrace_lock); 15997 15998 destroy = 0; 15999 } 16000 } 16001 16002 if (destroy) 16003 dtrace_dof_destroy(dof); 16004 16005 return (gen); 16006 } 16007 16008 static dtrace_helpers_t * 16009 dtrace_helpers_create(proc_t *p) 16010 { 16011 dtrace_helpers_t *help; 16012 16013 ASSERT(MUTEX_HELD(&dtrace_lock)); 16014 ASSERT(p->p_dtrace_helpers == NULL); 16015 16016 help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP); 16017 help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) * 16018 DTRACE_NHELPER_ACTIONS, KM_SLEEP); 16019 16020 p->p_dtrace_helpers = help; 16021 dtrace_helpers++; 16022 16023 return (help); 16024 } 16025 16026 #ifdef illumos 16027 static 16028 #endif 16029 void 16030 dtrace_helpers_destroy(proc_t *p) 16031 { 16032 dtrace_helpers_t *help; 16033 dtrace_vstate_t *vstate; 16034 #ifdef illumos 16035 proc_t *p = curproc; 16036 #endif 16037 int i; 16038 16039 mutex_enter(&dtrace_lock); 16040 16041 ASSERT(p->p_dtrace_helpers != NULL); 16042 ASSERT(dtrace_helpers > 0); 16043 16044 help = p->p_dtrace_helpers; 16045 vstate = &help->dthps_vstate; 16046 16047 /* 16048 * We're now going to lose the help from this process. 16049 */ 16050 p->p_dtrace_helpers = NULL; 16051 dtrace_sync(); 16052 16053 /* 16054 * Destory the helper actions. 16055 */ 16056 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { 16057 dtrace_helper_action_t *h, *next; 16058 16059 for (h = help->dthps_actions[i]; h != NULL; h = next) { 16060 next = h->dtha_next; 16061 dtrace_helper_action_destroy(h, vstate); 16062 h = next; 16063 } 16064 } 16065 16066 mutex_exit(&dtrace_lock); 16067 16068 /* 16069 * Destroy the helper providers. 16070 */ 16071 if (help->dthps_maxprovs > 0) { 16072 mutex_enter(&dtrace_meta_lock); 16073 if (dtrace_meta_pid != NULL) { 16074 ASSERT(dtrace_deferred_pid == NULL); 16075 16076 for (i = 0; i < help->dthps_nprovs; i++) { 16077 dtrace_helper_provider_remove( 16078 &help->dthps_provs[i]->dthp_prov, p->p_pid); 16079 } 16080 } else { 16081 mutex_enter(&dtrace_lock); 16082 ASSERT(help->dthps_deferred == 0 || 16083 help->dthps_next != NULL || 16084 help->dthps_prev != NULL || 16085 help == dtrace_deferred_pid); 16086 16087 /* 16088 * Remove the helper from the deferred list. 16089 */ 16090 if (help->dthps_next != NULL) 16091 help->dthps_next->dthps_prev = help->dthps_prev; 16092 if (help->dthps_prev != NULL) 16093 help->dthps_prev->dthps_next = help->dthps_next; 16094 if (dtrace_deferred_pid == help) { 16095 dtrace_deferred_pid = help->dthps_next; 16096 ASSERT(help->dthps_prev == NULL); 16097 } 16098 16099 mutex_exit(&dtrace_lock); 16100 } 16101 16102 mutex_exit(&dtrace_meta_lock); 16103 16104 for (i = 0; i < help->dthps_nprovs; i++) { 16105 dtrace_helper_provider_destroy(help->dthps_provs[i]); 16106 } 16107 16108 kmem_free(help->dthps_provs, help->dthps_maxprovs * 16109 sizeof (dtrace_helper_provider_t *)); 16110 } 16111 16112 mutex_enter(&dtrace_lock); 16113 16114 dtrace_vstate_fini(&help->dthps_vstate); 16115 kmem_free(help->dthps_actions, 16116 sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS); 16117 kmem_free(help, sizeof (dtrace_helpers_t)); 16118 16119 --dtrace_helpers; 16120 mutex_exit(&dtrace_lock); 16121 } 16122 16123 #ifdef illumos 16124 static 16125 #endif 16126 void 16127 dtrace_helpers_duplicate(proc_t *from, proc_t *to) 16128 { 16129 dtrace_helpers_t *help, *newhelp; 16130 dtrace_helper_action_t *helper, *new, *last; 16131 dtrace_difo_t *dp; 16132 dtrace_vstate_t *vstate; 16133 int i, j, sz, hasprovs = 0; 16134 16135 mutex_enter(&dtrace_lock); 16136 ASSERT(from->p_dtrace_helpers != NULL); 16137 ASSERT(dtrace_helpers > 0); 16138 16139 help = from->p_dtrace_helpers; 16140 newhelp = dtrace_helpers_create(to); 16141 ASSERT(to->p_dtrace_helpers != NULL); 16142 16143 newhelp->dthps_generation = help->dthps_generation; 16144 vstate = &newhelp->dthps_vstate; 16145 16146 /* 16147 * Duplicate the helper actions. 16148 */ 16149 for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) { 16150 if ((helper = help->dthps_actions[i]) == NULL) 16151 continue; 16152 16153 for (last = NULL; helper != NULL; helper = helper->dtha_next) { 16154 new = kmem_zalloc(sizeof (dtrace_helper_action_t), 16155 KM_SLEEP); 16156 new->dtha_generation = helper->dtha_generation; 16157 16158 if ((dp = helper->dtha_predicate) != NULL) { 16159 dp = dtrace_difo_duplicate(dp, vstate); 16160 new->dtha_predicate = dp; 16161 } 16162 16163 new->dtha_nactions = helper->dtha_nactions; 16164 sz = sizeof (dtrace_difo_t *) * new->dtha_nactions; 16165 new->dtha_actions = kmem_alloc(sz, KM_SLEEP); 16166 16167 for (j = 0; j < new->dtha_nactions; j++) { 16168 dtrace_difo_t *dp = helper->dtha_actions[j]; 16169 16170 ASSERT(dp != NULL); 16171 dp = dtrace_difo_duplicate(dp, vstate); 16172 new->dtha_actions[j] = dp; 16173 } 16174 16175 if (last != NULL) { 16176 last->dtha_next = new; 16177 } else { 16178 newhelp->dthps_actions[i] = new; 16179 } 16180 16181 last = new; 16182 } 16183 } 16184 16185 /* 16186 * Duplicate the helper providers and register them with the 16187 * DTrace framework. 16188 */ 16189 if (help->dthps_nprovs > 0) { 16190 newhelp->dthps_nprovs = help->dthps_nprovs; 16191 newhelp->dthps_maxprovs = help->dthps_nprovs; 16192 newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs * 16193 sizeof (dtrace_helper_provider_t *), KM_SLEEP); 16194 for (i = 0; i < newhelp->dthps_nprovs; i++) { 16195 newhelp->dthps_provs[i] = help->dthps_provs[i]; 16196 newhelp->dthps_provs[i]->dthp_ref++; 16197 } 16198 16199 hasprovs = 1; 16200 } 16201 16202 mutex_exit(&dtrace_lock); 16203 16204 if (hasprovs) 16205 dtrace_helper_provider_register(to, newhelp, NULL); 16206 } 16207 16208 /* 16209 * DTrace Hook Functions 16210 */ 16211 static void 16212 dtrace_module_loaded(modctl_t *ctl) 16213 { 16214 dtrace_provider_t *prv; 16215 16216 mutex_enter(&dtrace_provider_lock); 16217 #ifdef illumos 16218 mutex_enter(&mod_lock); 16219 #endif 16220 16221 #ifdef illumos 16222 ASSERT(ctl->mod_busy); 16223 #endif 16224 16225 /* 16226 * We're going to call each providers per-module provide operation 16227 * specifying only this module. 16228 */ 16229 for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next) 16230 prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl); 16231 16232 #ifdef illumos 16233 mutex_exit(&mod_lock); 16234 #endif 16235 mutex_exit(&dtrace_provider_lock); 16236 16237 /* 16238 * If we have any retained enablings, we need to match against them. 16239 * Enabling probes requires that cpu_lock be held, and we cannot hold 16240 * cpu_lock here -- it is legal for cpu_lock to be held when loading a 16241 * module. (In particular, this happens when loading scheduling 16242 * classes.) So if we have any retained enablings, we need to dispatch 16243 * our task queue to do the match for us. 16244 */ 16245 mutex_enter(&dtrace_lock); 16246 16247 if (dtrace_retained == NULL) { 16248 mutex_exit(&dtrace_lock); 16249 return; 16250 } 16251 16252 (void) taskq_dispatch(dtrace_taskq, 16253 (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP); 16254 16255 mutex_exit(&dtrace_lock); 16256 16257 /* 16258 * And now, for a little heuristic sleaze: in general, we want to 16259 * match modules as soon as they load. However, we cannot guarantee 16260 * this, because it would lead us to the lock ordering violation 16261 * outlined above. The common case, of course, is that cpu_lock is 16262 * _not_ held -- so we delay here for a clock tick, hoping that that's 16263 * long enough for the task queue to do its work. If it's not, it's 16264 * not a serious problem -- it just means that the module that we 16265 * just loaded may not be immediately instrumentable. 16266 */ 16267 delay(1); 16268 } 16269 16270 static void 16271 #ifdef illumos 16272 dtrace_module_unloaded(modctl_t *ctl) 16273 #else 16274 dtrace_module_unloaded(modctl_t *ctl, int *error) 16275 #endif 16276 { 16277 dtrace_probe_t template, *probe, *first, *next; 16278 dtrace_provider_t *prov; 16279 #ifndef illumos 16280 char modname[DTRACE_MODNAMELEN]; 16281 size_t len; 16282 #endif 16283 16284 #ifdef illumos 16285 template.dtpr_mod = ctl->mod_modname; 16286 #else 16287 /* Handle the fact that ctl->filename may end in ".ko". */ 16288 strlcpy(modname, ctl->filename, sizeof(modname)); 16289 len = strlen(ctl->filename); 16290 if (len > 3 && strcmp(modname + len - 3, ".ko") == 0) 16291 modname[len - 3] = '\0'; 16292 template.dtpr_mod = modname; 16293 #endif 16294 16295 mutex_enter(&dtrace_provider_lock); 16296 #ifdef illumos 16297 mutex_enter(&mod_lock); 16298 #endif 16299 mutex_enter(&dtrace_lock); 16300 16301 #ifndef illumos 16302 if (ctl->nenabled > 0) { 16303 /* Don't allow unloads if a probe is enabled. */ 16304 mutex_exit(&dtrace_provider_lock); 16305 mutex_exit(&dtrace_lock); 16306 *error = -1; 16307 printf( 16308 "kldunload: attempt to unload module that has DTrace probes enabled\n"); 16309 return; 16310 } 16311 #endif 16312 16313 if (dtrace_bymod == NULL) { 16314 /* 16315 * The DTrace module is loaded (obviously) but not attached; 16316 * we don't have any work to do. 16317 */ 16318 mutex_exit(&dtrace_provider_lock); 16319 #ifdef illumos 16320 mutex_exit(&mod_lock); 16321 #endif 16322 mutex_exit(&dtrace_lock); 16323 return; 16324 } 16325 16326 for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template); 16327 probe != NULL; probe = probe->dtpr_nextmod) { 16328 if (probe->dtpr_ecb != NULL) { 16329 mutex_exit(&dtrace_provider_lock); 16330 #ifdef illumos 16331 mutex_exit(&mod_lock); 16332 #endif 16333 mutex_exit(&dtrace_lock); 16334 16335 /* 16336 * This shouldn't _actually_ be possible -- we're 16337 * unloading a module that has an enabled probe in it. 16338 * (It's normally up to the provider to make sure that 16339 * this can't happen.) However, because dtps_enable() 16340 * doesn't have a failure mode, there can be an 16341 * enable/unload race. Upshot: we don't want to 16342 * assert, but we're not going to disable the 16343 * probe, either. 16344 */ 16345 if (dtrace_err_verbose) { 16346 #ifdef illumos 16347 cmn_err(CE_WARN, "unloaded module '%s' had " 16348 "enabled probes", ctl->mod_modname); 16349 #else 16350 cmn_err(CE_WARN, "unloaded module '%s' had " 16351 "enabled probes", modname); 16352 #endif 16353 } 16354 16355 return; 16356 } 16357 } 16358 16359 probe = first; 16360 16361 for (first = NULL; probe != NULL; probe = next) { 16362 ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe); 16363 16364 dtrace_probes[probe->dtpr_id - 1] = NULL; 16365 16366 next = probe->dtpr_nextmod; 16367 dtrace_hash_remove(dtrace_bymod, probe); 16368 dtrace_hash_remove(dtrace_byfunc, probe); 16369 dtrace_hash_remove(dtrace_byname, probe); 16370 16371 if (first == NULL) { 16372 first = probe; 16373 probe->dtpr_nextmod = NULL; 16374 } else { 16375 probe->dtpr_nextmod = first; 16376 first = probe; 16377 } 16378 } 16379 16380 /* 16381 * We've removed all of the module's probes from the hash chains and 16382 * from the probe array. Now issue a dtrace_sync() to be sure that 16383 * everyone has cleared out from any probe array processing. 16384 */ 16385 dtrace_sync(); 16386 16387 for (probe = first; probe != NULL; probe = first) { 16388 first = probe->dtpr_nextmod; 16389 prov = probe->dtpr_provider; 16390 prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id, 16391 probe->dtpr_arg); 16392 kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1); 16393 kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1); 16394 kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1); 16395 #ifdef illumos 16396 vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1); 16397 #else 16398 free_unr(dtrace_arena, probe->dtpr_id); 16399 #endif 16400 kmem_free(probe, sizeof (dtrace_probe_t)); 16401 } 16402 16403 mutex_exit(&dtrace_lock); 16404 #ifdef illumos 16405 mutex_exit(&mod_lock); 16406 #endif 16407 mutex_exit(&dtrace_provider_lock); 16408 } 16409 16410 #ifndef illumos 16411 static void 16412 dtrace_kld_load(void *arg __unused, linker_file_t lf) 16413 { 16414 16415 dtrace_module_loaded(lf); 16416 } 16417 16418 static void 16419 dtrace_kld_unload_try(void *arg __unused, linker_file_t lf, int *error) 16420 { 16421 16422 if (*error != 0) 16423 /* We already have an error, so don't do anything. */ 16424 return; 16425 dtrace_module_unloaded(lf, error); 16426 } 16427 #endif 16428 16429 #ifdef illumos 16430 static void 16431 dtrace_suspend(void) 16432 { 16433 dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend)); 16434 } 16435 16436 static void 16437 dtrace_resume(void) 16438 { 16439 dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume)); 16440 } 16441 #endif 16442 16443 static int 16444 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu) 16445 { 16446 ASSERT(MUTEX_HELD(&cpu_lock)); 16447 mutex_enter(&dtrace_lock); 16448 16449 switch (what) { 16450 case CPU_CONFIG: { 16451 dtrace_state_t *state; 16452 dtrace_optval_t *opt, rs, c; 16453 16454 /* 16455 * For now, we only allocate a new buffer for anonymous state. 16456 */ 16457 if ((state = dtrace_anon.dta_state) == NULL) 16458 break; 16459 16460 if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) 16461 break; 16462 16463 opt = state->dts_options; 16464 c = opt[DTRACEOPT_CPU]; 16465 16466 if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu) 16467 break; 16468 16469 /* 16470 * Regardless of what the actual policy is, we're going to 16471 * temporarily set our resize policy to be manual. We're 16472 * also going to temporarily set our CPU option to denote 16473 * the newly configured CPU. 16474 */ 16475 rs = opt[DTRACEOPT_BUFRESIZE]; 16476 opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL; 16477 opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu; 16478 16479 (void) dtrace_state_buffers(state); 16480 16481 opt[DTRACEOPT_BUFRESIZE] = rs; 16482 opt[DTRACEOPT_CPU] = c; 16483 16484 break; 16485 } 16486 16487 case CPU_UNCONFIG: 16488 /* 16489 * We don't free the buffer in the CPU_UNCONFIG case. (The 16490 * buffer will be freed when the consumer exits.) 16491 */ 16492 break; 16493 16494 default: 16495 break; 16496 } 16497 16498 mutex_exit(&dtrace_lock); 16499 return (0); 16500 } 16501 16502 #ifdef illumos 16503 static void 16504 dtrace_cpu_setup_initial(processorid_t cpu) 16505 { 16506 (void) dtrace_cpu_setup(CPU_CONFIG, cpu); 16507 } 16508 #endif 16509 16510 static void 16511 dtrace_toxrange_add(uintptr_t base, uintptr_t limit) 16512 { 16513 if (dtrace_toxranges >= dtrace_toxranges_max) { 16514 int osize, nsize; 16515 dtrace_toxrange_t *range; 16516 16517 osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t); 16518 16519 if (osize == 0) { 16520 ASSERT(dtrace_toxrange == NULL); 16521 ASSERT(dtrace_toxranges_max == 0); 16522 dtrace_toxranges_max = 1; 16523 } else { 16524 dtrace_toxranges_max <<= 1; 16525 } 16526 16527 nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t); 16528 range = kmem_zalloc(nsize, KM_SLEEP); 16529 16530 if (dtrace_toxrange != NULL) { 16531 ASSERT(osize != 0); 16532 bcopy(dtrace_toxrange, range, osize); 16533 kmem_free(dtrace_toxrange, osize); 16534 } 16535 16536 dtrace_toxrange = range; 16537 } 16538 16539 ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == 0); 16540 ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == 0); 16541 16542 dtrace_toxrange[dtrace_toxranges].dtt_base = base; 16543 dtrace_toxrange[dtrace_toxranges].dtt_limit = limit; 16544 dtrace_toxranges++; 16545 } 16546 16547 static void 16548 dtrace_getf_barrier() 16549 { 16550 #ifdef illumos 16551 /* 16552 * When we have unprivileged (that is, non-DTRACE_CRV_KERNEL) enablings 16553 * that contain calls to getf(), this routine will be called on every 16554 * closef() before either the underlying vnode is released or the 16555 * file_t itself is freed. By the time we are here, it is essential 16556 * that the file_t can no longer be accessed from a call to getf() 16557 * in probe context -- that assures that a dtrace_sync() can be used 16558 * to clear out any enablings referring to the old structures. 16559 */ 16560 if (curthread->t_procp->p_zone->zone_dtrace_getf != 0 || 16561 kcred->cr_zone->zone_dtrace_getf != 0) 16562 dtrace_sync(); 16563 #endif 16564 } 16565 16566 /* 16567 * DTrace Driver Cookbook Functions 16568 */ 16569 #ifdef illumos 16570 /*ARGSUSED*/ 16571 static int 16572 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 16573 { 16574 dtrace_provider_id_t id; 16575 dtrace_state_t *state = NULL; 16576 dtrace_enabling_t *enab; 16577 16578 mutex_enter(&cpu_lock); 16579 mutex_enter(&dtrace_provider_lock); 16580 mutex_enter(&dtrace_lock); 16581 16582 if (ddi_soft_state_init(&dtrace_softstate, 16583 sizeof (dtrace_state_t), 0) != 0) { 16584 cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state"); 16585 mutex_exit(&cpu_lock); 16586 mutex_exit(&dtrace_provider_lock); 16587 mutex_exit(&dtrace_lock); 16588 return (DDI_FAILURE); 16589 } 16590 16591 if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR, 16592 DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE || 16593 ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR, 16594 DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) { 16595 cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes"); 16596 ddi_remove_minor_node(devi, NULL); 16597 ddi_soft_state_fini(&dtrace_softstate); 16598 mutex_exit(&cpu_lock); 16599 mutex_exit(&dtrace_provider_lock); 16600 mutex_exit(&dtrace_lock); 16601 return (DDI_FAILURE); 16602 } 16603 16604 ddi_report_dev(devi); 16605 dtrace_devi = devi; 16606 16607 dtrace_modload = dtrace_module_loaded; 16608 dtrace_modunload = dtrace_module_unloaded; 16609 dtrace_cpu_init = dtrace_cpu_setup_initial; 16610 dtrace_helpers_cleanup = dtrace_helpers_destroy; 16611 dtrace_helpers_fork = dtrace_helpers_duplicate; 16612 dtrace_cpustart_init = dtrace_suspend; 16613 dtrace_cpustart_fini = dtrace_resume; 16614 dtrace_debugger_init = dtrace_suspend; 16615 dtrace_debugger_fini = dtrace_resume; 16616 16617 register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); 16618 16619 ASSERT(MUTEX_HELD(&cpu_lock)); 16620 16621 dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1, 16622 NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER); 16623 dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE, 16624 UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0, 16625 VM_SLEEP | VMC_IDENTIFIER); 16626 dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri, 16627 1, INT_MAX, 0); 16628 16629 dtrace_state_cache = kmem_cache_create("dtrace_state_cache", 16630 sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN, 16631 NULL, NULL, NULL, NULL, NULL, 0); 16632 16633 ASSERT(MUTEX_HELD(&cpu_lock)); 16634 dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod), 16635 offsetof(dtrace_probe_t, dtpr_nextmod), 16636 offsetof(dtrace_probe_t, dtpr_prevmod)); 16637 16638 dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func), 16639 offsetof(dtrace_probe_t, dtpr_nextfunc), 16640 offsetof(dtrace_probe_t, dtpr_prevfunc)); 16641 16642 dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name), 16643 offsetof(dtrace_probe_t, dtpr_nextname), 16644 offsetof(dtrace_probe_t, dtpr_prevname)); 16645 16646 if (dtrace_retain_max < 1) { 16647 cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; " 16648 "setting to 1", dtrace_retain_max); 16649 dtrace_retain_max = 1; 16650 } 16651 16652 /* 16653 * Now discover our toxic ranges. 16654 */ 16655 dtrace_toxic_ranges(dtrace_toxrange_add); 16656 16657 /* 16658 * Before we register ourselves as a provider to our own framework, 16659 * we would like to assert that dtrace_provider is NULL -- but that's 16660 * not true if we were loaded as a dependency of a DTrace provider. 16661 * Once we've registered, we can assert that dtrace_provider is our 16662 * pseudo provider. 16663 */ 16664 (void) dtrace_register("dtrace", &dtrace_provider_attr, 16665 DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id); 16666 16667 ASSERT(dtrace_provider != NULL); 16668 ASSERT((dtrace_provider_id_t)dtrace_provider == id); 16669 16670 dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t) 16671 dtrace_provider, NULL, NULL, "BEGIN", 0, NULL); 16672 dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t) 16673 dtrace_provider, NULL, NULL, "END", 0, NULL); 16674 dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t) 16675 dtrace_provider, NULL, NULL, "ERROR", 1, NULL); 16676 16677 dtrace_anon_property(); 16678 mutex_exit(&cpu_lock); 16679 16680 /* 16681 * If there are already providers, we must ask them to provide their 16682 * probes, and then match any anonymous enabling against them. Note 16683 * that there should be no other retained enablings at this time: 16684 * the only retained enablings at this time should be the anonymous 16685 * enabling. 16686 */ 16687 if (dtrace_anon.dta_enabling != NULL) { 16688 ASSERT(dtrace_retained == dtrace_anon.dta_enabling); 16689 16690 dtrace_enabling_provide(NULL); 16691 state = dtrace_anon.dta_state; 16692 16693 /* 16694 * We couldn't hold cpu_lock across the above call to 16695 * dtrace_enabling_provide(), but we must hold it to actually 16696 * enable the probes. We have to drop all of our locks, pick 16697 * up cpu_lock, and regain our locks before matching the 16698 * retained anonymous enabling. 16699 */ 16700 mutex_exit(&dtrace_lock); 16701 mutex_exit(&dtrace_provider_lock); 16702 16703 mutex_enter(&cpu_lock); 16704 mutex_enter(&dtrace_provider_lock); 16705 mutex_enter(&dtrace_lock); 16706 16707 if ((enab = dtrace_anon.dta_enabling) != NULL) 16708 (void) dtrace_enabling_match(enab, NULL); 16709 16710 mutex_exit(&cpu_lock); 16711 } 16712 16713 mutex_exit(&dtrace_lock); 16714 mutex_exit(&dtrace_provider_lock); 16715 16716 if (state != NULL) { 16717 /* 16718 * If we created any anonymous state, set it going now. 16719 */ 16720 (void) dtrace_state_go(state, &dtrace_anon.dta_beganon); 16721 } 16722 16723 return (DDI_SUCCESS); 16724 } 16725 #endif /* illumos */ 16726 16727 #ifndef illumos 16728 static void dtrace_dtr(void *); 16729 #endif 16730 16731 /*ARGSUSED*/ 16732 static int 16733 #ifdef illumos 16734 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 16735 #else 16736 dtrace_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 16737 #endif 16738 { 16739 dtrace_state_t *state; 16740 uint32_t priv; 16741 uid_t uid; 16742 zoneid_t zoneid; 16743 16744 #ifdef illumos 16745 if (getminor(*devp) == DTRACEMNRN_HELPER) 16746 return (0); 16747 16748 /* 16749 * If this wasn't an open with the "helper" minor, then it must be 16750 * the "dtrace" minor. 16751 */ 16752 if (getminor(*devp) == DTRACEMNRN_DTRACE) 16753 return (ENXIO); 16754 #else 16755 cred_t *cred_p = NULL; 16756 cred_p = dev->si_cred; 16757 16758 /* 16759 * If no DTRACE_PRIV_* bits are set in the credential, then the 16760 * caller lacks sufficient permission to do anything with DTrace. 16761 */ 16762 dtrace_cred2priv(cred_p, &priv, &uid, &zoneid); 16763 if (priv == DTRACE_PRIV_NONE) { 16764 #endif 16765 16766 return (EACCES); 16767 } 16768 16769 /* 16770 * Ask all providers to provide all their probes. 16771 */ 16772 mutex_enter(&dtrace_provider_lock); 16773 dtrace_probe_provide(NULL, NULL); 16774 mutex_exit(&dtrace_provider_lock); 16775 16776 mutex_enter(&cpu_lock); 16777 mutex_enter(&dtrace_lock); 16778 dtrace_opens++; 16779 dtrace_membar_producer(); 16780 16781 #ifdef illumos 16782 /* 16783 * If the kernel debugger is active (that is, if the kernel debugger 16784 * modified text in some way), we won't allow the open. 16785 */ 16786 if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) { 16787 dtrace_opens--; 16788 mutex_exit(&cpu_lock); 16789 mutex_exit(&dtrace_lock); 16790 return (EBUSY); 16791 } 16792 16793 if (dtrace_helptrace_enable && dtrace_helptrace_buffer == NULL) { 16794 /* 16795 * If DTrace helper tracing is enabled, we need to allocate the 16796 * trace buffer and initialize the values. 16797 */ 16798 dtrace_helptrace_buffer = 16799 kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP); 16800 dtrace_helptrace_next = 0; 16801 dtrace_helptrace_wrapped = 0; 16802 dtrace_helptrace_enable = 0; 16803 } 16804 16805 state = dtrace_state_create(devp, cred_p); 16806 #else 16807 state = dtrace_state_create(dev); 16808 devfs_set_cdevpriv(state, dtrace_dtr); 16809 #endif 16810 16811 mutex_exit(&cpu_lock); 16812 16813 if (state == NULL) { 16814 #ifdef illumos 16815 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) 16816 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); 16817 #else 16818 --dtrace_opens; 16819 #endif 16820 mutex_exit(&dtrace_lock); 16821 return (EAGAIN); 16822 } 16823 16824 mutex_exit(&dtrace_lock); 16825 16826 return (0); 16827 } 16828 16829 /*ARGSUSED*/ 16830 #ifdef illumos 16831 static int 16832 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 16833 #else 16834 static void 16835 dtrace_dtr(void *data) 16836 #endif 16837 { 16838 #ifdef illumos 16839 minor_t minor = getminor(dev); 16840 dtrace_state_t *state; 16841 #endif 16842 dtrace_helptrace_t *buf = NULL; 16843 16844 #ifdef illumos 16845 if (minor == DTRACEMNRN_HELPER) 16846 return (0); 16847 16848 state = ddi_get_soft_state(dtrace_softstate, minor); 16849 #else 16850 dtrace_state_t *state = data; 16851 #endif 16852 16853 mutex_enter(&cpu_lock); 16854 mutex_enter(&dtrace_lock); 16855 16856 #ifdef illumos 16857 if (state->dts_anon) 16858 #else 16859 if (state != NULL && state->dts_anon) 16860 #endif 16861 { 16862 /* 16863 * There is anonymous state. Destroy that first. 16864 */ 16865 ASSERT(dtrace_anon.dta_state == NULL); 16866 dtrace_state_destroy(state->dts_anon); 16867 } 16868 16869 if (dtrace_helptrace_disable) { 16870 /* 16871 * If we have been told to disable helper tracing, set the 16872 * buffer to NULL before calling into dtrace_state_destroy(); 16873 * we take advantage of its dtrace_sync() to know that no 16874 * CPU is in probe context with enabled helper tracing 16875 * after it returns. 16876 */ 16877 buf = dtrace_helptrace_buffer; 16878 dtrace_helptrace_buffer = NULL; 16879 } 16880 16881 #ifdef illumos 16882 dtrace_state_destroy(state); 16883 #else 16884 if (state != NULL) { 16885 dtrace_state_destroy(state); 16886 kmem_free(state, 0); 16887 } 16888 #endif 16889 ASSERT(dtrace_opens > 0); 16890 16891 #ifdef illumos 16892 /* 16893 * Only relinquish control of the kernel debugger interface when there 16894 * are no consumers and no anonymous enablings. 16895 */ 16896 if (--dtrace_opens == 0 && dtrace_anon.dta_enabling == NULL) 16897 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); 16898 #else 16899 --dtrace_opens; 16900 #endif 16901 16902 if (buf != NULL) { 16903 kmem_free(buf, dtrace_helptrace_bufsize); 16904 dtrace_helptrace_disable = 0; 16905 } 16906 16907 mutex_exit(&dtrace_lock); 16908 mutex_exit(&cpu_lock); 16909 16910 #ifdef illumos 16911 return (0); 16912 #endif 16913 } 16914 16915 #ifdef illumos 16916 /*ARGSUSED*/ 16917 static int 16918 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv) 16919 { 16920 int rval; 16921 dof_helper_t help, *dhp = NULL; 16922 16923 switch (cmd) { 16924 case DTRACEHIOC_ADDDOF: 16925 if (copyin((void *)arg, &help, sizeof (help)) != 0) { 16926 dtrace_dof_error(NULL, "failed to copyin DOF helper"); 16927 return (EFAULT); 16928 } 16929 16930 dhp = &help; 16931 arg = (intptr_t)help.dofhp_dof; 16932 /*FALLTHROUGH*/ 16933 16934 case DTRACEHIOC_ADD: { 16935 dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval); 16936 16937 if (dof == NULL) 16938 return (rval); 16939 16940 mutex_enter(&dtrace_lock); 16941 16942 /* 16943 * dtrace_helper_slurp() takes responsibility for the dof -- 16944 * it may free it now or it may save it and free it later. 16945 */ 16946 if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) { 16947 *rv = rval; 16948 rval = 0; 16949 } else { 16950 rval = EINVAL; 16951 } 16952 16953 mutex_exit(&dtrace_lock); 16954 return (rval); 16955 } 16956 16957 case DTRACEHIOC_REMOVE: { 16958 mutex_enter(&dtrace_lock); 16959 rval = dtrace_helper_destroygen(arg); 16960 mutex_exit(&dtrace_lock); 16961 16962 return (rval); 16963 } 16964 16965 default: 16966 break; 16967 } 16968 16969 return (ENOTTY); 16970 } 16971 16972 /*ARGSUSED*/ 16973 static int 16974 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 16975 { 16976 minor_t minor = getminor(dev); 16977 dtrace_state_t *state; 16978 int rval; 16979 16980 if (minor == DTRACEMNRN_HELPER) 16981 return (dtrace_ioctl_helper(cmd, arg, rv)); 16982 16983 state = ddi_get_soft_state(dtrace_softstate, minor); 16984 16985 if (state->dts_anon) { 16986 ASSERT(dtrace_anon.dta_state == NULL); 16987 state = state->dts_anon; 16988 } 16989 16990 switch (cmd) { 16991 case DTRACEIOC_PROVIDER: { 16992 dtrace_providerdesc_t pvd; 16993 dtrace_provider_t *pvp; 16994 16995 if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0) 16996 return (EFAULT); 16997 16998 pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0'; 16999 mutex_enter(&dtrace_provider_lock); 17000 17001 for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) { 17002 if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0) 17003 break; 17004 } 17005 17006 mutex_exit(&dtrace_provider_lock); 17007 17008 if (pvp == NULL) 17009 return (ESRCH); 17010 17011 bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t)); 17012 bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t)); 17013 17014 if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0) 17015 return (EFAULT); 17016 17017 return (0); 17018 } 17019 17020 case DTRACEIOC_EPROBE: { 17021 dtrace_eprobedesc_t epdesc; 17022 dtrace_ecb_t *ecb; 17023 dtrace_action_t *act; 17024 void *buf; 17025 size_t size; 17026 uintptr_t dest; 17027 int nrecs; 17028 17029 if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0) 17030 return (EFAULT); 17031 17032 mutex_enter(&dtrace_lock); 17033 17034 if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) { 17035 mutex_exit(&dtrace_lock); 17036 return (EINVAL); 17037 } 17038 17039 if (ecb->dte_probe == NULL) { 17040 mutex_exit(&dtrace_lock); 17041 return (EINVAL); 17042 } 17043 17044 epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id; 17045 epdesc.dtepd_uarg = ecb->dte_uarg; 17046 epdesc.dtepd_size = ecb->dte_size; 17047 17048 nrecs = epdesc.dtepd_nrecs; 17049 epdesc.dtepd_nrecs = 0; 17050 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 17051 if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) 17052 continue; 17053 17054 epdesc.dtepd_nrecs++; 17055 } 17056 17057 /* 17058 * Now that we have the size, we need to allocate a temporary 17059 * buffer in which to store the complete description. We need 17060 * the temporary buffer to be able to drop dtrace_lock() 17061 * across the copyout(), below. 17062 */ 17063 size = sizeof (dtrace_eprobedesc_t) + 17064 (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t)); 17065 17066 buf = kmem_alloc(size, KM_SLEEP); 17067 dest = (uintptr_t)buf; 17068 17069 bcopy(&epdesc, (void *)dest, sizeof (epdesc)); 17070 dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]); 17071 17072 for (act = ecb->dte_action; act != NULL; act = act->dta_next) { 17073 if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple) 17074 continue; 17075 17076 if (nrecs-- == 0) 17077 break; 17078 17079 bcopy(&act->dta_rec, (void *)dest, 17080 sizeof (dtrace_recdesc_t)); 17081 dest += sizeof (dtrace_recdesc_t); 17082 } 17083 17084 mutex_exit(&dtrace_lock); 17085 17086 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) { 17087 kmem_free(buf, size); 17088 return (EFAULT); 17089 } 17090 17091 kmem_free(buf, size); 17092 return (0); 17093 } 17094 17095 case DTRACEIOC_AGGDESC: { 17096 dtrace_aggdesc_t aggdesc; 17097 dtrace_action_t *act; 17098 dtrace_aggregation_t *agg; 17099 int nrecs; 17100 uint32_t offs; 17101 dtrace_recdesc_t *lrec; 17102 void *buf; 17103 size_t size; 17104 uintptr_t dest; 17105 17106 if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0) 17107 return (EFAULT); 17108 17109 mutex_enter(&dtrace_lock); 17110 17111 if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) { 17112 mutex_exit(&dtrace_lock); 17113 return (EINVAL); 17114 } 17115 17116 aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid; 17117 17118 nrecs = aggdesc.dtagd_nrecs; 17119 aggdesc.dtagd_nrecs = 0; 17120 17121 offs = agg->dtag_base; 17122 lrec = &agg->dtag_action.dta_rec; 17123 aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs; 17124 17125 for (act = agg->dtag_first; ; act = act->dta_next) { 17126 ASSERT(act->dta_intuple || 17127 DTRACEACT_ISAGG(act->dta_kind)); 17128 17129 /* 17130 * If this action has a record size of zero, it 17131 * denotes an argument to the aggregating action. 17132 * Because the presence of this record doesn't (or 17133 * shouldn't) affect the way the data is interpreted, 17134 * we don't copy it out to save user-level the 17135 * confusion of dealing with a zero-length record. 17136 */ 17137 if (act->dta_rec.dtrd_size == 0) { 17138 ASSERT(agg->dtag_hasarg); 17139 continue; 17140 } 17141 17142 aggdesc.dtagd_nrecs++; 17143 17144 if (act == &agg->dtag_action) 17145 break; 17146 } 17147 17148 /* 17149 * Now that we have the size, we need to allocate a temporary 17150 * buffer in which to store the complete description. We need 17151 * the temporary buffer to be able to drop dtrace_lock() 17152 * across the copyout(), below. 17153 */ 17154 size = sizeof (dtrace_aggdesc_t) + 17155 (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t)); 17156 17157 buf = kmem_alloc(size, KM_SLEEP); 17158 dest = (uintptr_t)buf; 17159 17160 bcopy(&aggdesc, (void *)dest, sizeof (aggdesc)); 17161 dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]); 17162 17163 for (act = agg->dtag_first; ; act = act->dta_next) { 17164 dtrace_recdesc_t rec = act->dta_rec; 17165 17166 /* 17167 * See the comment in the above loop for why we pass 17168 * over zero-length records. 17169 */ 17170 if (rec.dtrd_size == 0) { 17171 ASSERT(agg->dtag_hasarg); 17172 continue; 17173 } 17174 17175 if (nrecs-- == 0) 17176 break; 17177 17178 rec.dtrd_offset -= offs; 17179 bcopy(&rec, (void *)dest, sizeof (rec)); 17180 dest += sizeof (dtrace_recdesc_t); 17181 17182 if (act == &agg->dtag_action) 17183 break; 17184 } 17185 17186 mutex_exit(&dtrace_lock); 17187 17188 if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) { 17189 kmem_free(buf, size); 17190 return (EFAULT); 17191 } 17192 17193 kmem_free(buf, size); 17194 return (0); 17195 } 17196 17197 case DTRACEIOC_ENABLE: { 17198 dof_hdr_t *dof; 17199 dtrace_enabling_t *enab = NULL; 17200 dtrace_vstate_t *vstate; 17201 int err = 0; 17202 17203 *rv = 0; 17204 17205 /* 17206 * If a NULL argument has been passed, we take this as our 17207 * cue to reevaluate our enablings. 17208 */ 17209 if (arg == NULL) { 17210 dtrace_enabling_matchall(); 17211 17212 return (0); 17213 } 17214 17215 if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL) 17216 return (rval); 17217 17218 mutex_enter(&cpu_lock); 17219 mutex_enter(&dtrace_lock); 17220 vstate = &state->dts_vstate; 17221 17222 if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) { 17223 mutex_exit(&dtrace_lock); 17224 mutex_exit(&cpu_lock); 17225 dtrace_dof_destroy(dof); 17226 return (EBUSY); 17227 } 17228 17229 if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) { 17230 mutex_exit(&dtrace_lock); 17231 mutex_exit(&cpu_lock); 17232 dtrace_dof_destroy(dof); 17233 return (EINVAL); 17234 } 17235 17236 if ((rval = dtrace_dof_options(dof, state)) != 0) { 17237 dtrace_enabling_destroy(enab); 17238 mutex_exit(&dtrace_lock); 17239 mutex_exit(&cpu_lock); 17240 dtrace_dof_destroy(dof); 17241 return (rval); 17242 } 17243 17244 if ((err = dtrace_enabling_match(enab, rv)) == 0) { 17245 err = dtrace_enabling_retain(enab); 17246 } else { 17247 dtrace_enabling_destroy(enab); 17248 } 17249 17250 mutex_exit(&cpu_lock); 17251 mutex_exit(&dtrace_lock); 17252 dtrace_dof_destroy(dof); 17253 17254 return (err); 17255 } 17256 17257 case DTRACEIOC_REPLICATE: { 17258 dtrace_repldesc_t desc; 17259 dtrace_probedesc_t *match = &desc.dtrpd_match; 17260 dtrace_probedesc_t *create = &desc.dtrpd_create; 17261 int err; 17262 17263 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 17264 return (EFAULT); 17265 17266 match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; 17267 match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; 17268 match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; 17269 match->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; 17270 17271 create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; 17272 create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; 17273 create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; 17274 create->dtpd_name[DTRACE_NAMELEN - 1] = '\0'; 17275 17276 mutex_enter(&dtrace_lock); 17277 err = dtrace_enabling_replicate(state, match, create); 17278 mutex_exit(&dtrace_lock); 17279 17280 return (err); 17281 } 17282 17283 case DTRACEIOC_PROBEMATCH: 17284 case DTRACEIOC_PROBES: { 17285 dtrace_probe_t *probe = NULL; 17286 dtrace_probedesc_t desc; 17287 dtrace_probekey_t pkey; 17288 dtrace_id_t i; 17289 int m = 0; 17290 uint32_t priv; 17291 uid_t uid; 17292 zoneid_t zoneid; 17293 17294 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 17295 return (EFAULT); 17296 17297 desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0'; 17298 desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0'; 17299 desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0'; 17300 desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0'; 17301 17302 /* 17303 * Before we attempt to match this probe, we want to give 17304 * all providers the opportunity to provide it. 17305 */ 17306 if (desc.dtpd_id == DTRACE_IDNONE) { 17307 mutex_enter(&dtrace_provider_lock); 17308 dtrace_probe_provide(&desc, NULL); 17309 mutex_exit(&dtrace_provider_lock); 17310 desc.dtpd_id++; 17311 } 17312 17313 if (cmd == DTRACEIOC_PROBEMATCH) { 17314 dtrace_probekey(&desc, &pkey); 17315 pkey.dtpk_id = DTRACE_IDNONE; 17316 } 17317 17318 dtrace_cred2priv(cr, &priv, &uid, &zoneid); 17319 17320 mutex_enter(&dtrace_lock); 17321 17322 if (cmd == DTRACEIOC_PROBEMATCH) { 17323 for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) { 17324 if ((probe = dtrace_probes[i - 1]) != NULL && 17325 (m = dtrace_match_probe(probe, &pkey, 17326 priv, uid, zoneid)) != 0) 17327 break; 17328 } 17329 17330 if (m < 0) { 17331 mutex_exit(&dtrace_lock); 17332 return (EINVAL); 17333 } 17334 17335 } else { 17336 for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) { 17337 if ((probe = dtrace_probes[i - 1]) != NULL && 17338 dtrace_match_priv(probe, priv, uid, zoneid)) 17339 break; 17340 } 17341 } 17342 17343 if (probe == NULL) { 17344 mutex_exit(&dtrace_lock); 17345 return (ESRCH); 17346 } 17347 17348 dtrace_probe_description(probe, &desc); 17349 mutex_exit(&dtrace_lock); 17350 17351 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 17352 return (EFAULT); 17353 17354 return (0); 17355 } 17356 17357 case DTRACEIOC_PROBEARG: { 17358 dtrace_argdesc_t desc; 17359 dtrace_probe_t *probe; 17360 dtrace_provider_t *prov; 17361 17362 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 17363 return (EFAULT); 17364 17365 if (desc.dtargd_id == DTRACE_IDNONE) 17366 return (EINVAL); 17367 17368 if (desc.dtargd_ndx == DTRACE_ARGNONE) 17369 return (EINVAL); 17370 17371 mutex_enter(&dtrace_provider_lock); 17372 mutex_enter(&mod_lock); 17373 mutex_enter(&dtrace_lock); 17374 17375 if (desc.dtargd_id > dtrace_nprobes) { 17376 mutex_exit(&dtrace_lock); 17377 mutex_exit(&mod_lock); 17378 mutex_exit(&dtrace_provider_lock); 17379 return (EINVAL); 17380 } 17381 17382 if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) { 17383 mutex_exit(&dtrace_lock); 17384 mutex_exit(&mod_lock); 17385 mutex_exit(&dtrace_provider_lock); 17386 return (EINVAL); 17387 } 17388 17389 mutex_exit(&dtrace_lock); 17390 17391 prov = probe->dtpr_provider; 17392 17393 if (prov->dtpv_pops.dtps_getargdesc == NULL) { 17394 /* 17395 * There isn't any typed information for this probe. 17396 * Set the argument number to DTRACE_ARGNONE. 17397 */ 17398 desc.dtargd_ndx = DTRACE_ARGNONE; 17399 } else { 17400 desc.dtargd_native[0] = '\0'; 17401 desc.dtargd_xlate[0] = '\0'; 17402 desc.dtargd_mapping = desc.dtargd_ndx; 17403 17404 prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg, 17405 probe->dtpr_id, probe->dtpr_arg, &desc); 17406 } 17407 17408 mutex_exit(&mod_lock); 17409 mutex_exit(&dtrace_provider_lock); 17410 17411 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 17412 return (EFAULT); 17413 17414 return (0); 17415 } 17416 17417 case DTRACEIOC_GO: { 17418 processorid_t cpuid; 17419 rval = dtrace_state_go(state, &cpuid); 17420 17421 if (rval != 0) 17422 return (rval); 17423 17424 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) 17425 return (EFAULT); 17426 17427 return (0); 17428 } 17429 17430 case DTRACEIOC_STOP: { 17431 processorid_t cpuid; 17432 17433 mutex_enter(&dtrace_lock); 17434 rval = dtrace_state_stop(state, &cpuid); 17435 mutex_exit(&dtrace_lock); 17436 17437 if (rval != 0) 17438 return (rval); 17439 17440 if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0) 17441 return (EFAULT); 17442 17443 return (0); 17444 } 17445 17446 case DTRACEIOC_DOFGET: { 17447 dof_hdr_t hdr, *dof; 17448 uint64_t len; 17449 17450 if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0) 17451 return (EFAULT); 17452 17453 mutex_enter(&dtrace_lock); 17454 dof = dtrace_dof_create(state); 17455 mutex_exit(&dtrace_lock); 17456 17457 len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz); 17458 rval = copyout(dof, (void *)arg, len); 17459 dtrace_dof_destroy(dof); 17460 17461 return (rval == 0 ? 0 : EFAULT); 17462 } 17463 17464 case DTRACEIOC_AGGSNAP: 17465 case DTRACEIOC_BUFSNAP: { 17466 dtrace_bufdesc_t desc; 17467 caddr_t cached; 17468 dtrace_buffer_t *buf; 17469 17470 if (copyin((void *)arg, &desc, sizeof (desc)) != 0) 17471 return (EFAULT); 17472 17473 if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU) 17474 return (EINVAL); 17475 17476 mutex_enter(&dtrace_lock); 17477 17478 if (cmd == DTRACEIOC_BUFSNAP) { 17479 buf = &state->dts_buffer[desc.dtbd_cpu]; 17480 } else { 17481 buf = &state->dts_aggbuffer[desc.dtbd_cpu]; 17482 } 17483 17484 if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) { 17485 size_t sz = buf->dtb_offset; 17486 17487 if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) { 17488 mutex_exit(&dtrace_lock); 17489 return (EBUSY); 17490 } 17491 17492 /* 17493 * If this buffer has already been consumed, we're 17494 * going to indicate that there's nothing left here 17495 * to consume. 17496 */ 17497 if (buf->dtb_flags & DTRACEBUF_CONSUMED) { 17498 mutex_exit(&dtrace_lock); 17499 17500 desc.dtbd_size = 0; 17501 desc.dtbd_drops = 0; 17502 desc.dtbd_errors = 0; 17503 desc.dtbd_oldest = 0; 17504 sz = sizeof (desc); 17505 17506 if (copyout(&desc, (void *)arg, sz) != 0) 17507 return (EFAULT); 17508 17509 return (0); 17510 } 17511 17512 /* 17513 * If this is a ring buffer that has wrapped, we want 17514 * to copy the whole thing out. 17515 */ 17516 if (buf->dtb_flags & DTRACEBUF_WRAPPED) { 17517 dtrace_buffer_polish(buf); 17518 sz = buf->dtb_size; 17519 } 17520 17521 if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) { 17522 mutex_exit(&dtrace_lock); 17523 return (EFAULT); 17524 } 17525 17526 desc.dtbd_size = sz; 17527 desc.dtbd_drops = buf->dtb_drops; 17528 desc.dtbd_errors = buf->dtb_errors; 17529 desc.dtbd_oldest = buf->dtb_xamot_offset; 17530 desc.dtbd_timestamp = dtrace_gethrtime(); 17531 17532 mutex_exit(&dtrace_lock); 17533 17534 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 17535 return (EFAULT); 17536 17537 buf->dtb_flags |= DTRACEBUF_CONSUMED; 17538 17539 return (0); 17540 } 17541 17542 if (buf->dtb_tomax == NULL) { 17543 ASSERT(buf->dtb_xamot == NULL); 17544 mutex_exit(&dtrace_lock); 17545 return (ENOENT); 17546 } 17547 17548 cached = buf->dtb_tomax; 17549 ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH)); 17550 17551 dtrace_xcall(desc.dtbd_cpu, 17552 (dtrace_xcall_t)dtrace_buffer_switch, buf); 17553 17554 state->dts_errors += buf->dtb_xamot_errors; 17555 17556 /* 17557 * If the buffers did not actually switch, then the cross call 17558 * did not take place -- presumably because the given CPU is 17559 * not in the ready set. If this is the case, we'll return 17560 * ENOENT. 17561 */ 17562 if (buf->dtb_tomax == cached) { 17563 ASSERT(buf->dtb_xamot != cached); 17564 mutex_exit(&dtrace_lock); 17565 return (ENOENT); 17566 } 17567 17568 ASSERT(cached == buf->dtb_xamot); 17569 17570 /* 17571 * We have our snapshot; now copy it out. 17572 */ 17573 if (copyout(buf->dtb_xamot, desc.dtbd_data, 17574 buf->dtb_xamot_offset) != 0) { 17575 mutex_exit(&dtrace_lock); 17576 return (EFAULT); 17577 } 17578 17579 desc.dtbd_size = buf->dtb_xamot_offset; 17580 desc.dtbd_drops = buf->dtb_xamot_drops; 17581 desc.dtbd_errors = buf->dtb_xamot_errors; 17582 desc.dtbd_oldest = 0; 17583 desc.dtbd_timestamp = buf->dtb_switched; 17584 17585 mutex_exit(&dtrace_lock); 17586 17587 /* 17588 * Finally, copy out the buffer description. 17589 */ 17590 if (copyout(&desc, (void *)arg, sizeof (desc)) != 0) 17591 return (EFAULT); 17592 17593 return (0); 17594 } 17595 17596 case DTRACEIOC_CONF: { 17597 dtrace_conf_t conf; 17598 17599 bzero(&conf, sizeof (conf)); 17600 conf.dtc_difversion = DIF_VERSION; 17601 conf.dtc_difintregs = DIF_DIR_NREGS; 17602 conf.dtc_diftupregs = DIF_DTR_NREGS; 17603 conf.dtc_ctfmodel = CTF_MODEL_NATIVE; 17604 17605 if (copyout(&conf, (void *)arg, sizeof (conf)) != 0) 17606 return (EFAULT); 17607 17608 return (0); 17609 } 17610 17611 case DTRACEIOC_STATUS: { 17612 dtrace_status_t stat; 17613 dtrace_dstate_t *dstate; 17614 int i, j; 17615 uint64_t nerrs; 17616 17617 /* 17618 * See the comment in dtrace_state_deadman() for the reason 17619 * for setting dts_laststatus to INT64_MAX before setting 17620 * it to the correct value. 17621 */ 17622 state->dts_laststatus = INT64_MAX; 17623 dtrace_membar_producer(); 17624 state->dts_laststatus = dtrace_gethrtime(); 17625 17626 bzero(&stat, sizeof (stat)); 17627 17628 mutex_enter(&dtrace_lock); 17629 17630 if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) { 17631 mutex_exit(&dtrace_lock); 17632 return (ENOENT); 17633 } 17634 17635 if (state->dts_activity == DTRACE_ACTIVITY_DRAINING) 17636 stat.dtst_exiting = 1; 17637 17638 nerrs = state->dts_errors; 17639 dstate = &state->dts_vstate.dtvs_dynvars; 17640 17641 for (i = 0; i < NCPU; i++) { 17642 dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i]; 17643 17644 stat.dtst_dyndrops += dcpu->dtdsc_drops; 17645 stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops; 17646 stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops; 17647 17648 if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL) 17649 stat.dtst_filled++; 17650 17651 nerrs += state->dts_buffer[i].dtb_errors; 17652 17653 for (j = 0; j < state->dts_nspeculations; j++) { 17654 dtrace_speculation_t *spec; 17655 dtrace_buffer_t *buf; 17656 17657 spec = &state->dts_speculations[j]; 17658 buf = &spec->dtsp_buffer[i]; 17659 stat.dtst_specdrops += buf->dtb_xamot_drops; 17660 } 17661 } 17662 17663 stat.dtst_specdrops_busy = state->dts_speculations_busy; 17664 stat.dtst_specdrops_unavail = state->dts_speculations_unavail; 17665 stat.dtst_stkstroverflows = state->dts_stkstroverflows; 17666 stat.dtst_dblerrors = state->dts_dblerrors; 17667 stat.dtst_killed = 17668 (state->dts_activity == DTRACE_ACTIVITY_KILLED); 17669 stat.dtst_errors = nerrs; 17670 17671 mutex_exit(&dtrace_lock); 17672 17673 if (copyout(&stat, (void *)arg, sizeof (stat)) != 0) 17674 return (EFAULT); 17675 17676 return (0); 17677 } 17678 17679 case DTRACEIOC_FORMAT: { 17680 dtrace_fmtdesc_t fmt; 17681 char *str; 17682 int len; 17683 17684 if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0) 17685 return (EFAULT); 17686 17687 mutex_enter(&dtrace_lock); 17688 17689 if (fmt.dtfd_format == 0 || 17690 fmt.dtfd_format > state->dts_nformats) { 17691 mutex_exit(&dtrace_lock); 17692 return (EINVAL); 17693 } 17694 17695 /* 17696 * Format strings are allocated contiguously and they are 17697 * never freed; if a format index is less than the number 17698 * of formats, we can assert that the format map is non-NULL 17699 * and that the format for the specified index is non-NULL. 17700 */ 17701 ASSERT(state->dts_formats != NULL); 17702 str = state->dts_formats[fmt.dtfd_format - 1]; 17703 ASSERT(str != NULL); 17704 17705 len = strlen(str) + 1; 17706 17707 if (len > fmt.dtfd_length) { 17708 fmt.dtfd_length = len; 17709 17710 if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) { 17711 mutex_exit(&dtrace_lock); 17712 return (EINVAL); 17713 } 17714 } else { 17715 if (copyout(str, fmt.dtfd_string, len) != 0) { 17716 mutex_exit(&dtrace_lock); 17717 return (EINVAL); 17718 } 17719 } 17720 17721 mutex_exit(&dtrace_lock); 17722 return (0); 17723 } 17724 17725 default: 17726 break; 17727 } 17728 17729 return (ENOTTY); 17730 } 17731 17732 /*ARGSUSED*/ 17733 static int 17734 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 17735 { 17736 dtrace_state_t *state; 17737 17738 switch (cmd) { 17739 case DDI_DETACH: 17740 break; 17741 17742 case DDI_SUSPEND: 17743 return (DDI_SUCCESS); 17744 17745 default: 17746 return (DDI_FAILURE); 17747 } 17748 17749 mutex_enter(&cpu_lock); 17750 mutex_enter(&dtrace_provider_lock); 17751 mutex_enter(&dtrace_lock); 17752 17753 ASSERT(dtrace_opens == 0); 17754 17755 if (dtrace_helpers > 0) { 17756 mutex_exit(&dtrace_provider_lock); 17757 mutex_exit(&dtrace_lock); 17758 mutex_exit(&cpu_lock); 17759 return (DDI_FAILURE); 17760 } 17761 17762 if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) { 17763 mutex_exit(&dtrace_provider_lock); 17764 mutex_exit(&dtrace_lock); 17765 mutex_exit(&cpu_lock); 17766 return (DDI_FAILURE); 17767 } 17768 17769 dtrace_provider = NULL; 17770 17771 if ((state = dtrace_anon_grab()) != NULL) { 17772 /* 17773 * If there were ECBs on this state, the provider should 17774 * have not been allowed to detach; assert that there is 17775 * none. 17776 */ 17777 ASSERT(state->dts_necbs == 0); 17778 dtrace_state_destroy(state); 17779 17780 /* 17781 * If we're being detached with anonymous state, we need to 17782 * indicate to the kernel debugger that DTrace is now inactive. 17783 */ 17784 (void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE); 17785 } 17786 17787 bzero(&dtrace_anon, sizeof (dtrace_anon_t)); 17788 unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL); 17789 dtrace_cpu_init = NULL; 17790 dtrace_helpers_cleanup = NULL; 17791 dtrace_helpers_fork = NULL; 17792 dtrace_cpustart_init = NULL; 17793 dtrace_cpustart_fini = NULL; 17794 dtrace_debugger_init = NULL; 17795 dtrace_debugger_fini = NULL; 17796 dtrace_modload = NULL; 17797 dtrace_modunload = NULL; 17798 17799 ASSERT(dtrace_getf == 0); 17800 ASSERT(dtrace_closef == NULL); 17801 17802 mutex_exit(&cpu_lock); 17803 17804 kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *)); 17805 dtrace_probes = NULL; 17806 dtrace_nprobes = 0; 17807 17808 dtrace_hash_destroy(dtrace_bymod); 17809 dtrace_hash_destroy(dtrace_byfunc); 17810 dtrace_hash_destroy(dtrace_byname); 17811 dtrace_bymod = NULL; 17812 dtrace_byfunc = NULL; 17813 dtrace_byname = NULL; 17814 17815 kmem_cache_destroy(dtrace_state_cache); 17816 vmem_destroy(dtrace_minor); 17817 vmem_destroy(dtrace_arena); 17818 17819 if (dtrace_toxrange != NULL) { 17820 kmem_free(dtrace_toxrange, 17821 dtrace_toxranges_max * sizeof (dtrace_toxrange_t)); 17822 dtrace_toxrange = NULL; 17823 dtrace_toxranges = 0; 17824 dtrace_toxranges_max = 0; 17825 } 17826 17827 ddi_remove_minor_node(dtrace_devi, NULL); 17828 dtrace_devi = NULL; 17829 17830 ddi_soft_state_fini(&dtrace_softstate); 17831 17832 ASSERT(dtrace_vtime_references == 0); 17833 ASSERT(dtrace_opens == 0); 17834 ASSERT(dtrace_retained == NULL); 17835 17836 mutex_exit(&dtrace_lock); 17837 mutex_exit(&dtrace_provider_lock); 17838 17839 /* 17840 * We don't destroy the task queue until after we have dropped our 17841 * locks (taskq_destroy() may block on running tasks). To prevent 17842 * attempting to do work after we have effectively detached but before 17843 * the task queue has been destroyed, all tasks dispatched via the 17844 * task queue must check that DTrace is still attached before 17845 * performing any operation. 17846 */ 17847 taskq_destroy(dtrace_taskq); 17848 dtrace_taskq = NULL; 17849 17850 return (DDI_SUCCESS); 17851 } 17852 #endif 17853 17854 #ifdef illumos 17855 /*ARGSUSED*/ 17856 static int 17857 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 17858 { 17859 int error; 17860 17861 switch (infocmd) { 17862 case DDI_INFO_DEVT2DEVINFO: 17863 *result = (void *)dtrace_devi; 17864 error = DDI_SUCCESS; 17865 break; 17866 case DDI_INFO_DEVT2INSTANCE: 17867 *result = (void *)0; 17868 error = DDI_SUCCESS; 17869 break; 17870 default: 17871 error = DDI_FAILURE; 17872 } 17873 return (error); 17874 } 17875 #endif 17876 17877 #ifdef illumos 17878 static struct cb_ops dtrace_cb_ops = { 17879 dtrace_open, /* open */ 17880 dtrace_close, /* close */ 17881 nulldev, /* strategy */ 17882 nulldev, /* print */ 17883 nodev, /* dump */ 17884 nodev, /* read */ 17885 nodev, /* write */ 17886 dtrace_ioctl, /* ioctl */ 17887 nodev, /* devmap */ 17888 nodev, /* mmap */ 17889 nodev, /* segmap */ 17890 nochpoll, /* poll */ 17891 ddi_prop_op, /* cb_prop_op */ 17892 0, /* streamtab */ 17893 D_NEW | D_MP /* Driver compatibility flag */ 17894 }; 17895 17896 static struct dev_ops dtrace_ops = { 17897 DEVO_REV, /* devo_rev */ 17898 0, /* refcnt */ 17899 dtrace_info, /* get_dev_info */ 17900 nulldev, /* identify */ 17901 nulldev, /* probe */ 17902 dtrace_attach, /* attach */ 17903 dtrace_detach, /* detach */ 17904 nodev, /* reset */ 17905 &dtrace_cb_ops, /* driver operations */ 17906 NULL, /* bus operations */ 17907 nodev /* dev power */ 17908 }; 17909 17910 static struct modldrv modldrv = { 17911 &mod_driverops, /* module type (this is a pseudo driver) */ 17912 "Dynamic Tracing", /* name of module */ 17913 &dtrace_ops, /* driver ops */ 17914 }; 17915 17916 static struct modlinkage modlinkage = { 17917 MODREV_1, 17918 (void *)&modldrv, 17919 NULL 17920 }; 17921 17922 int 17923 _init(void) 17924 { 17925 return (mod_install(&modlinkage)); 17926 } 17927 17928 int 17929 _info(struct modinfo *modinfop) 17930 { 17931 return (mod_info(&modlinkage, modinfop)); 17932 } 17933 17934 int 17935 _fini(void) 17936 { 17937 return (mod_remove(&modlinkage)); 17938 } 17939 #else 17940 17941 static d_ioctl_t dtrace_ioctl; 17942 static d_ioctl_t dtrace_ioctl_helper; 17943 static void dtrace_load(void *); 17944 static int dtrace_unload(void); 17945 static struct cdev *dtrace_dev; 17946 static struct cdev *helper_dev; 17947 17948 void dtrace_invop_init(void); 17949 void dtrace_invop_uninit(void); 17950 17951 static struct cdevsw dtrace_cdevsw = { 17952 .d_version = D_VERSION, 17953 .d_ioctl = dtrace_ioctl, 17954 .d_open = dtrace_open, 17955 .d_name = "dtrace", 17956 }; 17957 17958 static struct cdevsw helper_cdevsw = { 17959 .d_version = D_VERSION, 17960 .d_ioctl = dtrace_ioctl_helper, 17961 .d_name = "helper", 17962 }; 17963 17964 #include <dtrace_anon.c> 17965 #include <dtrace_ioctl.c> 17966 #include <dtrace_load.c> 17967 #include <dtrace_modevent.c> 17968 #include <dtrace_sysctl.c> 17969 #include <dtrace_unload.c> 17970 #include <dtrace_vtime.c> 17971 #include <dtrace_hacks.c> 17972 #include <dtrace_isa.c> 17973 17974 SYSINIT(dtrace_load, SI_SUB_DTRACE, SI_ORDER_FIRST, dtrace_load, NULL); 17975 SYSUNINIT(dtrace_unload, SI_SUB_DTRACE, SI_ORDER_FIRST, dtrace_unload, NULL); 17976 SYSINIT(dtrace_anon_init, SI_SUB_DTRACE_ANON, SI_ORDER_FIRST, dtrace_anon_init, NULL); 17977 17978 DEV_MODULE(dtrace, dtrace_modevent, NULL); 17979 MODULE_VERSION(dtrace, 1); 17980 MODULE_DEPEND(dtrace, opensolaris, 1, 1, 1); 17981 #endif 17982