17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 2297eda132Sraf 237c478bd9Sstevel@tonic-gate /* 247c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 257c478bd9Sstevel@tonic-gate * Use is subject to license terms. 267c478bd9Sstevel@tonic-gate */ 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 317c478bd9Sstevel@tonic-gate #include <sys/errno.h> 327c478bd9Sstevel@tonic-gate #include <sys/stat.h> 337c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 347c478bd9Sstevel@tonic-gate #include <sys/conf.h> 357c478bd9Sstevel@tonic-gate #include <sys/systm.h> 367c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 377c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 387c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 397c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 407c478bd9Sstevel@tonic-gate #include <sys/strsubr.h> 417c478bd9Sstevel@tonic-gate #include <sys/fasttrap.h> 427c478bd9Sstevel@tonic-gate #include <sys/fasttrap_impl.h> 437c478bd9Sstevel@tonic-gate #include <sys/fasttrap_isa.h> 447c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 457c478bd9Sstevel@tonic-gate #include <sys/dtrace_impl.h> 467c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 477c478bd9Sstevel@tonic-gate #include <sys/frame.h> 487c478bd9Sstevel@tonic-gate #include <sys/stack.h> 497c478bd9Sstevel@tonic-gate #include <sys/proc.h> 507c478bd9Sstevel@tonic-gate #include <sys/priv.h> 517c478bd9Sstevel@tonic-gate #include <sys/policy.h> 527c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 537c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 547c478bd9Sstevel@tonic-gate #include <sys/prsystm.h> 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate 577c478bd9Sstevel@tonic-gate #include <vm/as.h> 587c478bd9Sstevel@tonic-gate #include <vm/seg.h> 597c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h> 607c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 617c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 627c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 637c478bd9Sstevel@tonic-gate 647c478bd9Sstevel@tonic-gate /* 657c478bd9Sstevel@tonic-gate * User-Land Trap-Based Tracing 667c478bd9Sstevel@tonic-gate * ---------------------------- 677c478bd9Sstevel@tonic-gate * 687c478bd9Sstevel@tonic-gate * The fasttrap provider allows DTrace consumers to instrument any user-level 697c478bd9Sstevel@tonic-gate * instruction to gather data; this includes probes with semantic 707c478bd9Sstevel@tonic-gate * signifigance like entry and return as well as simple offsets into the 717c478bd9Sstevel@tonic-gate * function. While the specific techniques used are very ISA specific, the 727c478bd9Sstevel@tonic-gate * methodology is generalizable to any architecture. 737c478bd9Sstevel@tonic-gate * 747c478bd9Sstevel@tonic-gate * 757c478bd9Sstevel@tonic-gate * The General Methodology 767c478bd9Sstevel@tonic-gate * ----------------------- 777c478bd9Sstevel@tonic-gate * 787c478bd9Sstevel@tonic-gate * With the primary goal of tracing every user-land instruction and the 797c478bd9Sstevel@tonic-gate * limitation that we can't trust user space so don't want to rely on much 807c478bd9Sstevel@tonic-gate * information there, we begin by replacing the instructions we want to trace 817c478bd9Sstevel@tonic-gate * with trap instructions. Each instruction we overwrite is saved into a hash 827c478bd9Sstevel@tonic-gate * table keyed by process ID and pc address. When we enter the kernel due to 837c478bd9Sstevel@tonic-gate * this trap instruction, we need the effects of the replaced instruction to 847c478bd9Sstevel@tonic-gate * appear to have occurred before we proceed with the user thread's 857c478bd9Sstevel@tonic-gate * execution. 867c478bd9Sstevel@tonic-gate * 877c478bd9Sstevel@tonic-gate * Each user level thread is represented by a ulwp_t structure which is 887c478bd9Sstevel@tonic-gate * always easily accessible through a register. The most basic way to produce 897c478bd9Sstevel@tonic-gate * the effects of the instruction we replaced is to copy that instruction out 907c478bd9Sstevel@tonic-gate * to a bit of scratch space reserved in the user thread's ulwp_t structure 917c478bd9Sstevel@tonic-gate * (a sort of kernel-private thread local storage), set the PC to that 927c478bd9Sstevel@tonic-gate * scratch space and single step. When we reenter the kernel after single 937c478bd9Sstevel@tonic-gate * stepping the instruction we must then adjust the PC to point to what would 947c478bd9Sstevel@tonic-gate * normally be the next instruction. Of course, special care must be taken 957c478bd9Sstevel@tonic-gate * for branches and jumps, but these represent such a small fraction of any 967c478bd9Sstevel@tonic-gate * instruction set that writing the code to emulate these in the kernel is 977c478bd9Sstevel@tonic-gate * not too difficult. 987c478bd9Sstevel@tonic-gate * 997c478bd9Sstevel@tonic-gate * Return probes may require several tracepoints to trace every return site, 1007c478bd9Sstevel@tonic-gate * and, conversely, each tracepoint may activate several probes (the entry 1017c478bd9Sstevel@tonic-gate * and offset 0 probes, for example). To solve this muliplexing problem, 1027c478bd9Sstevel@tonic-gate * tracepoints contain lists of probes to activate and probes contain lists 1037c478bd9Sstevel@tonic-gate * of tracepoints to enable. If a probe is activated, it adds its ID to 1047c478bd9Sstevel@tonic-gate * existing tracepoints or creates new ones as necessary. 1057c478bd9Sstevel@tonic-gate * 1067c478bd9Sstevel@tonic-gate * Most probes are activated _before_ the instruction is executed, but return 1077c478bd9Sstevel@tonic-gate * probes are activated _after_ the effects of the last instruction of the 1087c478bd9Sstevel@tonic-gate * function are visible. Return probes must be fired _after_ we have 1097c478bd9Sstevel@tonic-gate * single-stepped the instruction whereas all other probes are fired 1107c478bd9Sstevel@tonic-gate * beforehand. 1117c478bd9Sstevel@tonic-gate */ 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate static dev_info_t *fasttrap_devi; 1147c478bd9Sstevel@tonic-gate static dtrace_provider_id_t fasttrap_id; 1157c478bd9Sstevel@tonic-gate static dtrace_meta_provider_id_t fasttrap_meta_id; 1167c478bd9Sstevel@tonic-gate 1177c478bd9Sstevel@tonic-gate static timeout_id_t fasttrap_timeout; 1187c478bd9Sstevel@tonic-gate static kmutex_t fasttrap_cleanup_mtx; 1197c478bd9Sstevel@tonic-gate static uint_t fasttrap_cleanup_work; 1207c478bd9Sstevel@tonic-gate 1217c478bd9Sstevel@tonic-gate /* 1227c478bd9Sstevel@tonic-gate * Generation count on modifications to the global tracepoint lookup table. 1237c478bd9Sstevel@tonic-gate */ 1247c478bd9Sstevel@tonic-gate static volatile uint64_t fasttrap_mod_gen; 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate /* 1277c478bd9Sstevel@tonic-gate * When the fasttrap provider is loaded, fasttrap_max is set to either 1287c478bd9Sstevel@tonic-gate * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 1297c478bd9Sstevel@tonic-gate * fasttrap.conf file. Each time a probe is created, fasttrap_total is 1307c478bd9Sstevel@tonic-gate * incremented by the number of tracepoints that may be associated with that 1317c478bd9Sstevel@tonic-gate * probe; fasttrap_total is capped at fasttrap_max. 1327c478bd9Sstevel@tonic-gate */ 1337c478bd9Sstevel@tonic-gate #define FASTTRAP_MAX_DEFAULT 250000 1347c478bd9Sstevel@tonic-gate static uint32_t fasttrap_max; 1357c478bd9Sstevel@tonic-gate static uint32_t fasttrap_total; 1367c478bd9Sstevel@tonic-gate 1377c478bd9Sstevel@tonic-gate 1387c478bd9Sstevel@tonic-gate #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 1397c478bd9Sstevel@tonic-gate #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 140b096140dSahl #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 1417c478bd9Sstevel@tonic-gate 1427c478bd9Sstevel@tonic-gate #define FASTTRAP_PID_NAME "pid" 1437c478bd9Sstevel@tonic-gate 1447c478bd9Sstevel@tonic-gate fasttrap_hash_t fasttrap_tpoints; 1457c478bd9Sstevel@tonic-gate static fasttrap_hash_t fasttrap_provs; 146b096140dSahl static fasttrap_hash_t fasttrap_procs; 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate dtrace_id_t fasttrap_probe_id; 1497c478bd9Sstevel@tonic-gate static int fasttrap_count; /* ref count */ 1507c478bd9Sstevel@tonic-gate static int fasttrap_pid_count; /* pid ref count */ 1517c478bd9Sstevel@tonic-gate static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 1527c478bd9Sstevel@tonic-gate 1538f5db3e7Sahl #define FASTTRAP_ENABLE_FAIL 1 1548f5db3e7Sahl #define FASTTRAP_ENABLE_PARTIAL 2 1558f5db3e7Sahl 1567c478bd9Sstevel@tonic-gate static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 1577c478bd9Sstevel@tonic-gate static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 1607c478bd9Sstevel@tonic-gate const dtrace_pattr_t *); 1617c478bd9Sstevel@tonic-gate static void fasttrap_provider_free(fasttrap_provider_t *); 1627c478bd9Sstevel@tonic-gate static void fasttrap_provider_retire(fasttrap_provider_t *); 1637c478bd9Sstevel@tonic-gate 164b096140dSahl static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 165b096140dSahl static void fasttrap_proc_release(fasttrap_proc_t *); 166b096140dSahl 1677c478bd9Sstevel@tonic-gate #define FASTTRAP_PROVS_INDEX(pid, name) \ 1687c478bd9Sstevel@tonic-gate ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 1697c478bd9Sstevel@tonic-gate 170b096140dSahl #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 171b096140dSahl 1727c478bd9Sstevel@tonic-gate static int 1737c478bd9Sstevel@tonic-gate fasttrap_highbit(ulong_t i) 1747c478bd9Sstevel@tonic-gate { 1757c478bd9Sstevel@tonic-gate int h = 1; 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate if (i == 0) 1787c478bd9Sstevel@tonic-gate return (0); 1797c478bd9Sstevel@tonic-gate #ifdef _LP64 1807c478bd9Sstevel@tonic-gate if (i & 0xffffffff00000000ul) { 1817c478bd9Sstevel@tonic-gate h += 32; i >>= 32; 1827c478bd9Sstevel@tonic-gate } 1837c478bd9Sstevel@tonic-gate #endif 1847c478bd9Sstevel@tonic-gate if (i & 0xffff0000) { 1857c478bd9Sstevel@tonic-gate h += 16; i >>= 16; 1867c478bd9Sstevel@tonic-gate } 1877c478bd9Sstevel@tonic-gate if (i & 0xff00) { 1887c478bd9Sstevel@tonic-gate h += 8; i >>= 8; 1897c478bd9Sstevel@tonic-gate } 1907c478bd9Sstevel@tonic-gate if (i & 0xf0) { 1917c478bd9Sstevel@tonic-gate h += 4; i >>= 4; 1927c478bd9Sstevel@tonic-gate } 1937c478bd9Sstevel@tonic-gate if (i & 0xc) { 1947c478bd9Sstevel@tonic-gate h += 2; i >>= 2; 1957c478bd9Sstevel@tonic-gate } 1967c478bd9Sstevel@tonic-gate if (i & 0x2) { 1977c478bd9Sstevel@tonic-gate h += 1; 1987c478bd9Sstevel@tonic-gate } 1997c478bd9Sstevel@tonic-gate return (h); 2007c478bd9Sstevel@tonic-gate } 2017c478bd9Sstevel@tonic-gate 2027c478bd9Sstevel@tonic-gate static uint_t 2037c478bd9Sstevel@tonic-gate fasttrap_hash_str(const char *p) 2047c478bd9Sstevel@tonic-gate { 2057c478bd9Sstevel@tonic-gate unsigned int g; 2067c478bd9Sstevel@tonic-gate uint_t hval = 0; 2077c478bd9Sstevel@tonic-gate 2087c478bd9Sstevel@tonic-gate while (*p) { 2097c478bd9Sstevel@tonic-gate hval = (hval << 4) + *p++; 2107c478bd9Sstevel@tonic-gate if ((g = (hval & 0xf0000000)) != 0) 2117c478bd9Sstevel@tonic-gate hval ^= g >> 24; 2127c478bd9Sstevel@tonic-gate hval &= ~g; 2137c478bd9Sstevel@tonic-gate } 2147c478bd9Sstevel@tonic-gate return (hval); 2157c478bd9Sstevel@tonic-gate } 2167c478bd9Sstevel@tonic-gate 2177c478bd9Sstevel@tonic-gate void 2187c478bd9Sstevel@tonic-gate fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 2197c478bd9Sstevel@tonic-gate { 2207c478bd9Sstevel@tonic-gate sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 2217c478bd9Sstevel@tonic-gate 2227c478bd9Sstevel@tonic-gate sqp->sq_info.si_signo = SIGTRAP; 2237c478bd9Sstevel@tonic-gate sqp->sq_info.si_code = TRAP_DTRACE; 2247c478bd9Sstevel@tonic-gate sqp->sq_info.si_addr = (caddr_t)pc; 2257c478bd9Sstevel@tonic-gate 2267c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 2277c478bd9Sstevel@tonic-gate sigaddqa(p, t, sqp); 2287c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 2297c478bd9Sstevel@tonic-gate 2307c478bd9Sstevel@tonic-gate if (t != NULL) 2317c478bd9Sstevel@tonic-gate aston(t); 2327c478bd9Sstevel@tonic-gate } 2337c478bd9Sstevel@tonic-gate 2347c478bd9Sstevel@tonic-gate /* 2357c478bd9Sstevel@tonic-gate * This function ensures that no threads are actively using the memory 2367c478bd9Sstevel@tonic-gate * associated with probes that were formerly live. 2377c478bd9Sstevel@tonic-gate */ 2387c478bd9Sstevel@tonic-gate static void 2397c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(uint64_t gen) 2407c478bd9Sstevel@tonic-gate { 2417c478bd9Sstevel@tonic-gate int i; 2427c478bd9Sstevel@tonic-gate 2437c478bd9Sstevel@tonic-gate if (gen < fasttrap_mod_gen) 2447c478bd9Sstevel@tonic-gate return; 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate fasttrap_mod_gen++; 2477c478bd9Sstevel@tonic-gate 2487c478bd9Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 2497c478bd9Sstevel@tonic-gate mutex_enter(&cpu_core[i].cpuc_pid_lock); 2507c478bd9Sstevel@tonic-gate mutex_exit(&cpu_core[i].cpuc_pid_lock); 2517c478bd9Sstevel@tonic-gate } 2527c478bd9Sstevel@tonic-gate } 2537c478bd9Sstevel@tonic-gate 2547c478bd9Sstevel@tonic-gate /* 2557c478bd9Sstevel@tonic-gate * This is the timeout's callback for cleaning up the providers and their 2567c478bd9Sstevel@tonic-gate * probes. 2577c478bd9Sstevel@tonic-gate */ 2587c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2597c478bd9Sstevel@tonic-gate static void 2607c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup_cb(void *data) 2617c478bd9Sstevel@tonic-gate { 2627c478bd9Sstevel@tonic-gate fasttrap_provider_t **fpp, *fp; 2637c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 2647c478bd9Sstevel@tonic-gate dtrace_provider_id_t provid; 2657c478bd9Sstevel@tonic-gate int i, later; 2667c478bd9Sstevel@tonic-gate 2677c478bd9Sstevel@tonic-gate static volatile int in = 0; 2687c478bd9Sstevel@tonic-gate ASSERT(in == 0); 2697c478bd9Sstevel@tonic-gate in = 1; 2707c478bd9Sstevel@tonic-gate 2717c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 2727c478bd9Sstevel@tonic-gate while (fasttrap_cleanup_work) { 2737c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 0; 2747c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 2757c478bd9Sstevel@tonic-gate 2767c478bd9Sstevel@tonic-gate later = 0; 2777c478bd9Sstevel@tonic-gate 2787c478bd9Sstevel@tonic-gate /* 2797c478bd9Sstevel@tonic-gate * Iterate over all the providers trying to remove the marked 280b096140dSahl * ones. If a provider is marked but not retired, we just 2817c478bd9Sstevel@tonic-gate * have to take a crack at removing it -- it's no big deal if 2827c478bd9Sstevel@tonic-gate * we can't. 2837c478bd9Sstevel@tonic-gate */ 2847c478bd9Sstevel@tonic-gate for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2857c478bd9Sstevel@tonic-gate bucket = &fasttrap_provs.fth_table[i]; 2867c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 2877c478bd9Sstevel@tonic-gate fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate while ((fp = *fpp) != NULL) { 2907c478bd9Sstevel@tonic-gate if (!fp->ftp_marked) { 2917c478bd9Sstevel@tonic-gate fpp = &fp->ftp_next; 2927c478bd9Sstevel@tonic-gate continue; 2937c478bd9Sstevel@tonic-gate } 2947c478bd9Sstevel@tonic-gate 2957c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 2967c478bd9Sstevel@tonic-gate 2977c478bd9Sstevel@tonic-gate /* 2987c478bd9Sstevel@tonic-gate * If this provider is referenced either 2997c478bd9Sstevel@tonic-gate * because it is a USDT provider or is being 3007c478bd9Sstevel@tonic-gate * modified, we can't unregister or even 3017c478bd9Sstevel@tonic-gate * condense. 3027c478bd9Sstevel@tonic-gate */ 3037c478bd9Sstevel@tonic-gate if (fp->ftp_ccount != 0) { 3047c478bd9Sstevel@tonic-gate mutex_exit(&fp->ftp_mtx); 3057c478bd9Sstevel@tonic-gate fp->ftp_marked = 0; 3067c478bd9Sstevel@tonic-gate continue; 3077c478bd9Sstevel@tonic-gate } 3087c478bd9Sstevel@tonic-gate 309b096140dSahl if (!fp->ftp_retired || fp->ftp_rcount != 0) 3107c478bd9Sstevel@tonic-gate fp->ftp_marked = 0; 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate mutex_exit(&fp->ftp_mtx); 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate /* 3157c478bd9Sstevel@tonic-gate * If we successfully unregister this 3167c478bd9Sstevel@tonic-gate * provider we can remove it from the hash 3177c478bd9Sstevel@tonic-gate * chain and free the memory. If our attempt 318b096140dSahl * to unregister fails and this is a retired 3197c478bd9Sstevel@tonic-gate * provider, increment our flag to try again 3207c478bd9Sstevel@tonic-gate * pretty soon. If we've consumed more than 3217c478bd9Sstevel@tonic-gate * half of our total permitted number of 3227c478bd9Sstevel@tonic-gate * probes call dtrace_condense() to try to 3237c478bd9Sstevel@tonic-gate * clean out the unenabled probes. 3247c478bd9Sstevel@tonic-gate */ 3257c478bd9Sstevel@tonic-gate provid = fp->ftp_provid; 3267c478bd9Sstevel@tonic-gate if (dtrace_unregister(provid) != 0) { 3277c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max / 2) 3287c478bd9Sstevel@tonic-gate (void) dtrace_condense(provid); 3297c478bd9Sstevel@tonic-gate later += fp->ftp_marked; 3307c478bd9Sstevel@tonic-gate fpp = &fp->ftp_next; 3317c478bd9Sstevel@tonic-gate } else { 3327c478bd9Sstevel@tonic-gate *fpp = fp->ftp_next; 3337c478bd9Sstevel@tonic-gate fasttrap_provider_free(fp); 3347c478bd9Sstevel@tonic-gate } 3357c478bd9Sstevel@tonic-gate } 3367c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 3377c478bd9Sstevel@tonic-gate } 3387c478bd9Sstevel@tonic-gate 3397c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 3407c478bd9Sstevel@tonic-gate } 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate ASSERT(fasttrap_timeout != 0); 3437c478bd9Sstevel@tonic-gate 3447c478bd9Sstevel@tonic-gate /* 345b096140dSahl * If we were unable to remove a retired provider, try again after 3467c478bd9Sstevel@tonic-gate * a second. This situation can occur in certain circumstances where 3477c478bd9Sstevel@tonic-gate * providers cannot be unregistered even though they have no probes 3487c478bd9Sstevel@tonic-gate * enabled because of an execution of dtrace -l or something similar. 3497c478bd9Sstevel@tonic-gate * If the timeout has been disabled (set to 1 because we're trying 3507c478bd9Sstevel@tonic-gate * to detach), we set fasttrap_cleanup_work to ensure that we'll 3517c478bd9Sstevel@tonic-gate * get a chance to do that work if and when the timeout is reenabled 3527c478bd9Sstevel@tonic-gate * (if detach fails). 3537c478bd9Sstevel@tonic-gate */ 3547c478bd9Sstevel@tonic-gate if (later > 0 && fasttrap_timeout != (timeout_id_t)1) 3557c478bd9Sstevel@tonic-gate fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 3567c478bd9Sstevel@tonic-gate else if (later > 0) 3577c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 1; 3587c478bd9Sstevel@tonic-gate else 3597c478bd9Sstevel@tonic-gate fasttrap_timeout = 0; 3607c478bd9Sstevel@tonic-gate 3617c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 3627c478bd9Sstevel@tonic-gate in = 0; 3637c478bd9Sstevel@tonic-gate } 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate /* 3667c478bd9Sstevel@tonic-gate * Activates the asynchronous cleanup mechanism. 3677c478bd9Sstevel@tonic-gate */ 3687c478bd9Sstevel@tonic-gate static void 3697c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(void) 3707c478bd9Sstevel@tonic-gate { 3717c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 3727c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 1; 3737c478bd9Sstevel@tonic-gate if (fasttrap_timeout == 0) 3747c478bd9Sstevel@tonic-gate fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 3757c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 3767c478bd9Sstevel@tonic-gate } 3777c478bd9Sstevel@tonic-gate 3787c478bd9Sstevel@tonic-gate /* 3797c478bd9Sstevel@tonic-gate * This is called from cfork() via dtrace_fasttrap_fork(). The child 3807c478bd9Sstevel@tonic-gate * process's address space is a (roughly) a copy of the parent process's so 3817c478bd9Sstevel@tonic-gate * we have to remove all the instrumentation we had previously enabled in the 3827c478bd9Sstevel@tonic-gate * parent. 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate static void 3857c478bd9Sstevel@tonic-gate fasttrap_fork(proc_t *p, proc_t *cp) 3867c478bd9Sstevel@tonic-gate { 3877c478bd9Sstevel@tonic-gate pid_t ppid = p->p_pid; 3887c478bd9Sstevel@tonic-gate int i; 3897c478bd9Sstevel@tonic-gate 3907c478bd9Sstevel@tonic-gate ASSERT(curproc == p); 3917c478bd9Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK); 3927c478bd9Sstevel@tonic-gate ASSERT(p->p_dtrace_count > 0); 3937c478bd9Sstevel@tonic-gate ASSERT(cp->p_dtrace_count == 0); 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate /* 3967c478bd9Sstevel@tonic-gate * This would be simpler and faster if we maintained per-process 3977c478bd9Sstevel@tonic-gate * hash tables of enabled tracepoints. It could, however, potentially 3987c478bd9Sstevel@tonic-gate * slow down execution of a tracepoint since we'd need to go 3997c478bd9Sstevel@tonic-gate * through two levels of indirection. In the future, we should 4007c478bd9Sstevel@tonic-gate * consider either maintaining per-process ancillary lists of 4017c478bd9Sstevel@tonic-gate * enabled tracepoints or hanging a pointer to a per-process hash 4027c478bd9Sstevel@tonic-gate * table of enabled tracepoints off the proc structure. 4037c478bd9Sstevel@tonic-gate */ 4047c478bd9Sstevel@tonic-gate 4057c478bd9Sstevel@tonic-gate /* 4067c478bd9Sstevel@tonic-gate * We don't have to worry about the child process disappearing 4077c478bd9Sstevel@tonic-gate * because we're in fork(). 4087c478bd9Sstevel@tonic-gate */ 4097c478bd9Sstevel@tonic-gate mutex_enter(&cp->p_lock); 4107c478bd9Sstevel@tonic-gate sprlock_proc(cp); 4117c478bd9Sstevel@tonic-gate mutex_exit(&cp->p_lock); 4127c478bd9Sstevel@tonic-gate 4137c478bd9Sstevel@tonic-gate /* 4147c478bd9Sstevel@tonic-gate * Iterate over every tracepoint looking for ones that belong to the 4157c478bd9Sstevel@tonic-gate * parent process, and remove each from the child process. 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 4187c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 4197c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 4207c478bd9Sstevel@tonic-gate 4217c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 4227c478bd9Sstevel@tonic-gate for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 423b096140dSahl if (tp->ftt_pid == ppid && 424b096140dSahl !tp->ftt_proc->ftpc_defunct) { 4257c478bd9Sstevel@tonic-gate int ret = fasttrap_tracepoint_remove(cp, tp); 4267c478bd9Sstevel@tonic-gate ASSERT(ret == 0); 4277c478bd9Sstevel@tonic-gate } 4287c478bd9Sstevel@tonic-gate } 4297c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 4307c478bd9Sstevel@tonic-gate } 4317c478bd9Sstevel@tonic-gate 4327c478bd9Sstevel@tonic-gate mutex_enter(&cp->p_lock); 4337c478bd9Sstevel@tonic-gate sprunlock(cp); 4347c478bd9Sstevel@tonic-gate } 4357c478bd9Sstevel@tonic-gate 4367c478bd9Sstevel@tonic-gate /* 4377c478bd9Sstevel@tonic-gate * This is called from proc_exit() or from exec_common() if p_dtrace_probes 4387c478bd9Sstevel@tonic-gate * is set on the proc structure to indicate that there is a pid provider 4397c478bd9Sstevel@tonic-gate * associated with this process. 4407c478bd9Sstevel@tonic-gate */ 4417c478bd9Sstevel@tonic-gate static void 4427c478bd9Sstevel@tonic-gate fasttrap_exec_exit(proc_t *p) 4437c478bd9Sstevel@tonic-gate { 4447c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider; 4457c478bd9Sstevel@tonic-gate 4467c478bd9Sstevel@tonic-gate ASSERT(p == curproc); 4477c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&p->p_lock)); 4487c478bd9Sstevel@tonic-gate 4497c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 4507c478bd9Sstevel@tonic-gate 4517c478bd9Sstevel@tonic-gate /* 4527c478bd9Sstevel@tonic-gate * We clean up the pid provider for this process here; user-land 4537c478bd9Sstevel@tonic-gate * static probes are handled by the meta-provider remove entry point. 4547c478bd9Sstevel@tonic-gate */ 4557c478bd9Sstevel@tonic-gate if ((provider = fasttrap_provider_lookup(p->p_pid, 4567c478bd9Sstevel@tonic-gate FASTTRAP_PID_NAME, NULL)) != NULL) 4577c478bd9Sstevel@tonic-gate fasttrap_provider_retire(provider); 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 4607c478bd9Sstevel@tonic-gate } 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4647c478bd9Sstevel@tonic-gate static void 4657c478bd9Sstevel@tonic-gate fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc) 4667c478bd9Sstevel@tonic-gate { 4677c478bd9Sstevel@tonic-gate /* 4687c478bd9Sstevel@tonic-gate * There are no "default" pid probes. 4697c478bd9Sstevel@tonic-gate */ 4707c478bd9Sstevel@tonic-gate } 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4737c478bd9Sstevel@tonic-gate static void 4747c478bd9Sstevel@tonic-gate fasttrap_provide(void *arg, const dtrace_probedesc_t *desc) 4757c478bd9Sstevel@tonic-gate { 4767c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(fasttrap_id, NULL, "fasttrap", "fasttrap") == 0) 4777c478bd9Sstevel@tonic-gate fasttrap_probe_id = dtrace_probe_create(fasttrap_id, NULL, 4787c478bd9Sstevel@tonic-gate "fasttrap", "fasttrap", FASTTRAP_AFRAMES, NULL); 4797c478bd9Sstevel@tonic-gate } 4807c478bd9Sstevel@tonic-gate 4817c478bd9Sstevel@tonic-gate static int 4827c478bd9Sstevel@tonic-gate fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 4837c478bd9Sstevel@tonic-gate { 4847c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp, *new_tp = NULL; 4857c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 4867c478bd9Sstevel@tonic-gate fasttrap_id_t *id; 4877c478bd9Sstevel@tonic-gate pid_t pid; 4887c478bd9Sstevel@tonic-gate uintptr_t pc; 4897c478bd9Sstevel@tonic-gate 4907c478bd9Sstevel@tonic-gate ASSERT(index < probe->ftp_ntps); 4917c478bd9Sstevel@tonic-gate 4927c478bd9Sstevel@tonic-gate pid = probe->ftp_pid; 4937c478bd9Sstevel@tonic-gate pc = probe->ftp_tps[index].fit_tp->ftt_pc; 4947c478bd9Sstevel@tonic-gate id = &probe->ftp_tps[index].fit_id; 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 4977c478bd9Sstevel@tonic-gate 4987c478bd9Sstevel@tonic-gate ASSERT(!(p->p_flag & SVFORK)); 4997c478bd9Sstevel@tonic-gate 5007c478bd9Sstevel@tonic-gate /* 5017c478bd9Sstevel@tonic-gate * Before we make any modifications, make sure we've imposed a barrier 5027c478bd9Sstevel@tonic-gate * on the generation in which this probe was last modified. 5037c478bd9Sstevel@tonic-gate */ 5047c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(probe->ftp_gen); 5057c478bd9Sstevel@tonic-gate 5067c478bd9Sstevel@tonic-gate bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 5077c478bd9Sstevel@tonic-gate 5087c478bd9Sstevel@tonic-gate /* 5097c478bd9Sstevel@tonic-gate * If the tracepoint has already been enabled, just add our id to the 5107c478bd9Sstevel@tonic-gate * list of interested probes. This may be our second time through 5117c478bd9Sstevel@tonic-gate * this path in which case we'll have constructed the tracepoint we'd 5127c478bd9Sstevel@tonic-gate * like to install. If we can't find a match, and have an allocated 5137c478bd9Sstevel@tonic-gate * tracepoint ready to go, enable that one now. 5147c478bd9Sstevel@tonic-gate * 515b096140dSahl * A tracepoint whose proc is defunct is also considered defunct. 5167c478bd9Sstevel@tonic-gate */ 5177c478bd9Sstevel@tonic-gate again: 5187c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 5197c478bd9Sstevel@tonic-gate for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 5207c478bd9Sstevel@tonic-gate if (tp->ftt_pid != pid || tp->ftt_pc != pc || 521b096140dSahl tp->ftt_proc->ftpc_defunct) 5227c478bd9Sstevel@tonic-gate continue; 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate /* 5257c478bd9Sstevel@tonic-gate * Now that we've found a matching tracepoint, it would be 5267c478bd9Sstevel@tonic-gate * a decent idea to confirm that the tracepoint is still 5277c478bd9Sstevel@tonic-gate * enabled and the trap instruction hasn't been overwritten. 5287c478bd9Sstevel@tonic-gate * Since this is a little hairy, we'll punt for now. 5297c478bd9Sstevel@tonic-gate */ 5307c478bd9Sstevel@tonic-gate 5317c478bd9Sstevel@tonic-gate /* 5327c478bd9Sstevel@tonic-gate * This can't be the first interested probe. We don't have 5337c478bd9Sstevel@tonic-gate * to worry about another thread being in the midst of 5347c478bd9Sstevel@tonic-gate * deleting this tracepoint (which would be the only valid 5357c478bd9Sstevel@tonic-gate * reason for a tracepoint to have no interested probes) 5367c478bd9Sstevel@tonic-gate * since we're holding P_PR_LOCK for this process. 5377c478bd9Sstevel@tonic-gate */ 5387c478bd9Sstevel@tonic-gate ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 5397c478bd9Sstevel@tonic-gate 5407c478bd9Sstevel@tonic-gate if (probe->ftp_type == DTFTP_RETURN || 5417c478bd9Sstevel@tonic-gate probe->ftp_type == DTFTP_POST_OFFSETS) { 5427c478bd9Sstevel@tonic-gate id->fti_next = tp->ftt_retids; 5437c478bd9Sstevel@tonic-gate membar_producer(); 5447c478bd9Sstevel@tonic-gate tp->ftt_retids = id; 5457c478bd9Sstevel@tonic-gate membar_producer(); 5467c478bd9Sstevel@tonic-gate } else { 5477c478bd9Sstevel@tonic-gate id->fti_next = tp->ftt_ids; 5487c478bd9Sstevel@tonic-gate membar_producer(); 5497c478bd9Sstevel@tonic-gate tp->ftt_ids = id; 5507c478bd9Sstevel@tonic-gate membar_producer(); 5517c478bd9Sstevel@tonic-gate } 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate if (new_tp != NULL) { 5567c478bd9Sstevel@tonic-gate new_tp->ftt_ids = NULL; 5577c478bd9Sstevel@tonic-gate new_tp->ftt_retids = NULL; 5587c478bd9Sstevel@tonic-gate } 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate return (0); 5617c478bd9Sstevel@tonic-gate } 5627c478bd9Sstevel@tonic-gate 5637c478bd9Sstevel@tonic-gate /* 5647c478bd9Sstevel@tonic-gate * If we have a good tracepoint ready to go, install it now while 5657c478bd9Sstevel@tonic-gate * we have the lock held and no one can screw with us. 5667c478bd9Sstevel@tonic-gate */ 5677c478bd9Sstevel@tonic-gate if (new_tp != NULL) { 5688f5db3e7Sahl int rc = 0; 5697c478bd9Sstevel@tonic-gate 5707c478bd9Sstevel@tonic-gate new_tp->ftt_next = bucket->ftb_data; 5717c478bd9Sstevel@tonic-gate membar_producer(); 5727c478bd9Sstevel@tonic-gate bucket->ftb_data = new_tp; 5737c478bd9Sstevel@tonic-gate membar_producer(); 5747c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 5757c478bd9Sstevel@tonic-gate 5767c478bd9Sstevel@tonic-gate /* 5778f5db3e7Sahl * Activate the tracepoint in the ISA-specific manner. 5788f5db3e7Sahl * If this fails, we need to report the failure, but 5798f5db3e7Sahl * indicate that this tracepoint must still be disabled 5808f5db3e7Sahl * by calling fasttrap_tracepoint_disable(). 5817c478bd9Sstevel@tonic-gate */ 5828f5db3e7Sahl if (fasttrap_tracepoint_install(p, new_tp) != 0) 5838f5db3e7Sahl rc = FASTTRAP_ENABLE_PARTIAL; 5848f5db3e7Sahl 5857c478bd9Sstevel@tonic-gate /* 5868f5db3e7Sahl * Increment the count of the number of tracepoints active in 5878f5db3e7Sahl * the victim process. 5887c478bd9Sstevel@tonic-gate */ 5897c478bd9Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK); 5907c478bd9Sstevel@tonic-gate p->p_dtrace_count++; 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate return (rc); 5937c478bd9Sstevel@tonic-gate } 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 5967c478bd9Sstevel@tonic-gate 5977c478bd9Sstevel@tonic-gate /* 5987c478bd9Sstevel@tonic-gate * Initialize the tracepoint that's been preallocated with the probe. 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate new_tp = probe->ftp_tps[index].fit_tp; 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_pid == pid); 6037c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_pc == pc); 604b096140dSahl ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 6057c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_ids == NULL); 6067c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_retids == NULL); 6077c478bd9Sstevel@tonic-gate 6087c478bd9Sstevel@tonic-gate if (probe->ftp_type == DTFTP_RETURN || 6097c478bd9Sstevel@tonic-gate probe->ftp_type == DTFTP_POST_OFFSETS) { 6107c478bd9Sstevel@tonic-gate id->fti_next = NULL; 6117c478bd9Sstevel@tonic-gate new_tp->ftt_retids = id; 6127c478bd9Sstevel@tonic-gate } else { 6137c478bd9Sstevel@tonic-gate id->fti_next = NULL; 6147c478bd9Sstevel@tonic-gate new_tp->ftt_ids = id; 6157c478bd9Sstevel@tonic-gate } 6167c478bd9Sstevel@tonic-gate 6177c478bd9Sstevel@tonic-gate /* 6188f5db3e7Sahl * If the ISA-dependent initialization goes to plan, go back to the 6197c478bd9Sstevel@tonic-gate * beginning and try to install this freshly made tracepoint. 6207c478bd9Sstevel@tonic-gate */ 6217c478bd9Sstevel@tonic-gate if (fasttrap_tracepoint_init(p, probe, new_tp, pc) == 0) 6227c478bd9Sstevel@tonic-gate goto again; 6237c478bd9Sstevel@tonic-gate 6247c478bd9Sstevel@tonic-gate new_tp->ftt_ids = NULL; 6257c478bd9Sstevel@tonic-gate new_tp->ftt_retids = NULL; 6267c478bd9Sstevel@tonic-gate 6278f5db3e7Sahl return (FASTTRAP_ENABLE_FAIL); 6287c478bd9Sstevel@tonic-gate } 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate static void 6317c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 6327c478bd9Sstevel@tonic-gate { 6337c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 6347c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider = probe->ftp_prov; 6357c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t **pp, *tp; 6367c478bd9Sstevel@tonic-gate fasttrap_id_t *id, **idp; 6377c478bd9Sstevel@tonic-gate pid_t pid; 6387c478bd9Sstevel@tonic-gate uintptr_t pc; 6397c478bd9Sstevel@tonic-gate 6407c478bd9Sstevel@tonic-gate ASSERT(index < probe->ftp_ntps); 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate pid = probe->ftp_pid; 6437c478bd9Sstevel@tonic-gate pc = probe->ftp_tps[index].fit_tp->ftt_pc; 6447c478bd9Sstevel@tonic-gate 6457c478bd9Sstevel@tonic-gate ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate /* 6487c478bd9Sstevel@tonic-gate * Find the tracepoint and make sure that our id is one of the 6497c478bd9Sstevel@tonic-gate * ones registered with it. 6507c478bd9Sstevel@tonic-gate */ 6517c478bd9Sstevel@tonic-gate bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 6527c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 6537c478bd9Sstevel@tonic-gate for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 6547c478bd9Sstevel@tonic-gate if (tp->ftt_pid == pid && tp->ftt_pc == pc && 655b096140dSahl tp->ftt_proc == provider->ftp_proc) 6567c478bd9Sstevel@tonic-gate break; 6577c478bd9Sstevel@tonic-gate } 6587c478bd9Sstevel@tonic-gate 6597c478bd9Sstevel@tonic-gate /* 6607c478bd9Sstevel@tonic-gate * If we somehow lost this tracepoint, we're in a world of hurt. 6617c478bd9Sstevel@tonic-gate */ 6627c478bd9Sstevel@tonic-gate ASSERT(tp != NULL); 6637c478bd9Sstevel@tonic-gate 6647c478bd9Sstevel@tonic-gate if (probe->ftp_type == DTFTP_RETURN || 6657c478bd9Sstevel@tonic-gate probe->ftp_type == DTFTP_POST_OFFSETS) { 6667c478bd9Sstevel@tonic-gate ASSERT(tp->ftt_retids != NULL); 6677c478bd9Sstevel@tonic-gate idp = &tp->ftt_retids; 6687c478bd9Sstevel@tonic-gate } else { 6697c478bd9Sstevel@tonic-gate ASSERT(tp->ftt_ids != NULL); 6707c478bd9Sstevel@tonic-gate idp = &tp->ftt_ids; 6717c478bd9Sstevel@tonic-gate } 6727c478bd9Sstevel@tonic-gate 6737c478bd9Sstevel@tonic-gate while ((*idp)->fti_probe != probe) { 6747c478bd9Sstevel@tonic-gate idp = &(*idp)->fti_next; 6757c478bd9Sstevel@tonic-gate ASSERT(*idp != NULL); 6767c478bd9Sstevel@tonic-gate } 6777c478bd9Sstevel@tonic-gate 6787c478bd9Sstevel@tonic-gate id = *idp; 6797c478bd9Sstevel@tonic-gate *idp = id->fti_next; 6807c478bd9Sstevel@tonic-gate membar_producer(); 6817c478bd9Sstevel@tonic-gate 6827c478bd9Sstevel@tonic-gate ASSERT(id->fti_probe == probe); 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate /* 6857c478bd9Sstevel@tonic-gate * If there are other registered enablings of this tracepoint, we're 6867c478bd9Sstevel@tonic-gate * all done, but if this was the last probe assocated with this 6877c478bd9Sstevel@tonic-gate * this tracepoint, we need to remove and free it. 6887c478bd9Sstevel@tonic-gate */ 6897c478bd9Sstevel@tonic-gate if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 6907c478bd9Sstevel@tonic-gate 6917c478bd9Sstevel@tonic-gate /* 6927c478bd9Sstevel@tonic-gate * If the current probe's tracepoint is in use, swap it 6937c478bd9Sstevel@tonic-gate * for an unused tracepoint. 6947c478bd9Sstevel@tonic-gate */ 6957c478bd9Sstevel@tonic-gate if (tp == probe->ftp_tps[index].fit_tp) { 6967c478bd9Sstevel@tonic-gate fasttrap_probe_t *tmp_probe; 6977c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t **tmp_tp; 6987c478bd9Sstevel@tonic-gate uint_t tmp_index; 6997c478bd9Sstevel@tonic-gate 7007c478bd9Sstevel@tonic-gate if (tp->ftt_ids != NULL) { 7017c478bd9Sstevel@tonic-gate tmp_probe = tp->ftt_ids->fti_probe; 7027c478bd9Sstevel@tonic-gate tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 7037c478bd9Sstevel@tonic-gate tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 7047c478bd9Sstevel@tonic-gate } else { 7057c478bd9Sstevel@tonic-gate tmp_probe = tp->ftt_retids->fti_probe; 7067c478bd9Sstevel@tonic-gate tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 7077c478bd9Sstevel@tonic-gate tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 7087c478bd9Sstevel@tonic-gate } 7097c478bd9Sstevel@tonic-gate 7107c478bd9Sstevel@tonic-gate ASSERT(*tmp_tp != NULL); 7117c478bd9Sstevel@tonic-gate ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 7127c478bd9Sstevel@tonic-gate ASSERT((*tmp_tp)->ftt_ids == NULL); 7137c478bd9Sstevel@tonic-gate ASSERT((*tmp_tp)->ftt_retids == NULL); 7147c478bd9Sstevel@tonic-gate 7157c478bd9Sstevel@tonic-gate probe->ftp_tps[index].fit_tp = *tmp_tp; 7167c478bd9Sstevel@tonic-gate *tmp_tp = tp; 7177c478bd9Sstevel@tonic-gate 7187c478bd9Sstevel@tonic-gate } 7197c478bd9Sstevel@tonic-gate 7207c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 7217c478bd9Sstevel@tonic-gate 7227c478bd9Sstevel@tonic-gate /* 7237c478bd9Sstevel@tonic-gate * Tag the modified probe with the generation in which it was 7247c478bd9Sstevel@tonic-gate * changed. 7257c478bd9Sstevel@tonic-gate */ 7267c478bd9Sstevel@tonic-gate probe->ftp_gen = fasttrap_mod_gen; 7277c478bd9Sstevel@tonic-gate return; 7287c478bd9Sstevel@tonic-gate } 7297c478bd9Sstevel@tonic-gate 7307c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 7317c478bd9Sstevel@tonic-gate 7327c478bd9Sstevel@tonic-gate /* 7337c478bd9Sstevel@tonic-gate * We can't safely remove the tracepoint from the set of active 7347c478bd9Sstevel@tonic-gate * tracepoints until we've actually removed the fasttrap instruction 7357c478bd9Sstevel@tonic-gate * from the process's text. We can, however, operate on this 7367c478bd9Sstevel@tonic-gate * tracepoint secure in the knowledge that no other thread is going to 7377c478bd9Sstevel@tonic-gate * be looking at it since we hold P_PR_LOCK on the process if it's 7387c478bd9Sstevel@tonic-gate * live or we hold the provider lock on the process if it's dead and 7397c478bd9Sstevel@tonic-gate * gone. 7407c478bd9Sstevel@tonic-gate */ 7417c478bd9Sstevel@tonic-gate 7427c478bd9Sstevel@tonic-gate /* 7437c478bd9Sstevel@tonic-gate * We only need to remove the actual instruction if we're looking 7447c478bd9Sstevel@tonic-gate * at an existing process 7457c478bd9Sstevel@tonic-gate */ 7467c478bd9Sstevel@tonic-gate if (p != NULL) { 7477c478bd9Sstevel@tonic-gate /* 7487c478bd9Sstevel@tonic-gate * If we fail to restore the instruction we need to kill 7497c478bd9Sstevel@tonic-gate * this process since it's in a completely unrecoverable 7507c478bd9Sstevel@tonic-gate * state. 7517c478bd9Sstevel@tonic-gate */ 7527c478bd9Sstevel@tonic-gate if (fasttrap_tracepoint_remove(p, tp) != 0) 7537c478bd9Sstevel@tonic-gate fasttrap_sigtrap(p, NULL, pc); 7547c478bd9Sstevel@tonic-gate 7557c478bd9Sstevel@tonic-gate /* 7567c478bd9Sstevel@tonic-gate * Decrement the count of the number of tracepoints active 7577c478bd9Sstevel@tonic-gate * in the victim process. 7587c478bd9Sstevel@tonic-gate */ 7597c478bd9Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK); 7607c478bd9Sstevel@tonic-gate p->p_dtrace_count--; 7617c478bd9Sstevel@tonic-gate } 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate /* 7647c478bd9Sstevel@tonic-gate * Remove the probe from the hash table of active tracepoints. 7657c478bd9Sstevel@tonic-gate */ 7667c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 7677c478bd9Sstevel@tonic-gate pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 7687c478bd9Sstevel@tonic-gate ASSERT(*pp != NULL); 7697c478bd9Sstevel@tonic-gate while (*pp != tp) { 7707c478bd9Sstevel@tonic-gate pp = &(*pp)->ftt_next; 7717c478bd9Sstevel@tonic-gate ASSERT(*pp != NULL); 7727c478bd9Sstevel@tonic-gate } 7737c478bd9Sstevel@tonic-gate 7747c478bd9Sstevel@tonic-gate *pp = tp->ftt_next; 7757c478bd9Sstevel@tonic-gate membar_producer(); 7767c478bd9Sstevel@tonic-gate 7777c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 7787c478bd9Sstevel@tonic-gate 7797c478bd9Sstevel@tonic-gate /* 7807c478bd9Sstevel@tonic-gate * Tag the modified probe with the generation in which it was changed. 7817c478bd9Sstevel@tonic-gate */ 7827c478bd9Sstevel@tonic-gate probe->ftp_gen = fasttrap_mod_gen; 7837c478bd9Sstevel@tonic-gate } 7847c478bd9Sstevel@tonic-gate 7857c478bd9Sstevel@tonic-gate typedef int fasttrap_probe_f(struct regs *); 7867c478bd9Sstevel@tonic-gate 7877c478bd9Sstevel@tonic-gate static void 7887c478bd9Sstevel@tonic-gate fasttrap_enable_common(int *count, fasttrap_probe_f **fptr, fasttrap_probe_f *f, 7897c478bd9Sstevel@tonic-gate fasttrap_probe_f **fptr2, fasttrap_probe_f *f2) 7907c478bd9Sstevel@tonic-gate { 7917c478bd9Sstevel@tonic-gate /* 7927c478bd9Sstevel@tonic-gate * We don't have to play the rw lock game here because we're 7937c478bd9Sstevel@tonic-gate * providing something rather than taking something away -- 7947c478bd9Sstevel@tonic-gate * we can be sure that no threads have tried to follow this 7957c478bd9Sstevel@tonic-gate * function pointer yet. 7967c478bd9Sstevel@tonic-gate */ 7977c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_count_mtx); 7987c478bd9Sstevel@tonic-gate if (*count == 0) { 7997c478bd9Sstevel@tonic-gate ASSERT(*fptr == NULL); 8007c478bd9Sstevel@tonic-gate *fptr = f; 8017c478bd9Sstevel@tonic-gate if (fptr2 != NULL) 8027c478bd9Sstevel@tonic-gate *fptr2 = f2; 8037c478bd9Sstevel@tonic-gate } 8047c478bd9Sstevel@tonic-gate ASSERT(*fptr == f); 8057c478bd9Sstevel@tonic-gate ASSERT(fptr2 == NULL || *fptr2 == f2); 8067c478bd9Sstevel@tonic-gate (*count)++; 8077c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_count_mtx); 8087c478bd9Sstevel@tonic-gate } 8097c478bd9Sstevel@tonic-gate 8107c478bd9Sstevel@tonic-gate static void 8117c478bd9Sstevel@tonic-gate fasttrap_disable_common(int *count, fasttrap_probe_f **fptr, 8127c478bd9Sstevel@tonic-gate fasttrap_probe_f **fptr2) 8137c478bd9Sstevel@tonic-gate { 8147c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 8157c478bd9Sstevel@tonic-gate 8167c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_count_mtx); 8177c478bd9Sstevel@tonic-gate (*count)--; 8187c478bd9Sstevel@tonic-gate ASSERT(*count >= 0); 8197c478bd9Sstevel@tonic-gate if (*count == 0) { 8207c478bd9Sstevel@tonic-gate cpu_t *cur, *cpu = CPU; 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate for (cur = cpu->cpu_next_onln; cur != cpu; 8237c478bd9Sstevel@tonic-gate cur = cur->cpu_next_onln) { 8247c478bd9Sstevel@tonic-gate rw_enter(&cur->cpu_ft_lock, RW_WRITER); 8257c478bd9Sstevel@tonic-gate } 8267c478bd9Sstevel@tonic-gate 8277c478bd9Sstevel@tonic-gate *fptr = NULL; 8287c478bd9Sstevel@tonic-gate if (fptr2 != NULL) 8297c478bd9Sstevel@tonic-gate *fptr2 = NULL; 8307c478bd9Sstevel@tonic-gate 8317c478bd9Sstevel@tonic-gate for (cur = cpu->cpu_next_onln; cur != cpu; 8327c478bd9Sstevel@tonic-gate cur = cur->cpu_next_onln) { 8337c478bd9Sstevel@tonic-gate rw_exit(&cur->cpu_ft_lock); 8347c478bd9Sstevel@tonic-gate } 8357c478bd9Sstevel@tonic-gate } 8367c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_count_mtx); 8377c478bd9Sstevel@tonic-gate } 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8407c478bd9Sstevel@tonic-gate static void 8417c478bd9Sstevel@tonic-gate fasttrap_enable(void *arg, dtrace_id_t id, void *parg) 8427c478bd9Sstevel@tonic-gate { 8437c478bd9Sstevel@tonic-gate /* 8447c478bd9Sstevel@tonic-gate * Enable the probe that corresponds to statically placed trace 8457c478bd9Sstevel@tonic-gate * points which have not explicitly been placed in the process's text 8467c478bd9Sstevel@tonic-gate * by the fasttrap provider. 8477c478bd9Sstevel@tonic-gate */ 8487c478bd9Sstevel@tonic-gate ASSERT(arg == NULL); 8497c478bd9Sstevel@tonic-gate ASSERT(id == fasttrap_probe_id); 8507c478bd9Sstevel@tonic-gate 8517c478bd9Sstevel@tonic-gate fasttrap_enable_common(&fasttrap_count, 8527c478bd9Sstevel@tonic-gate &dtrace_fasttrap_probe_ptr, fasttrap_probe, NULL, NULL); 8537c478bd9Sstevel@tonic-gate } 8547c478bd9Sstevel@tonic-gate 8557c478bd9Sstevel@tonic-gate 8567c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8577c478bd9Sstevel@tonic-gate static void 8587c478bd9Sstevel@tonic-gate fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 8597c478bd9Sstevel@tonic-gate { 8607c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 8617c478bd9Sstevel@tonic-gate proc_t *p; 8628f5db3e7Sahl int i, rc; 8637c478bd9Sstevel@tonic-gate 8647c478bd9Sstevel@tonic-gate ASSERT(probe != NULL); 8657c478bd9Sstevel@tonic-gate ASSERT(!probe->ftp_enabled); 8667c478bd9Sstevel@tonic-gate ASSERT(id == probe->ftp_id); 8677c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 8687c478bd9Sstevel@tonic-gate 8697c478bd9Sstevel@tonic-gate /* 8707c478bd9Sstevel@tonic-gate * Increment the count of enabled probes on this probe's provider; 8717c478bd9Sstevel@tonic-gate * the provider can't go away while the probe still exists. We 8727c478bd9Sstevel@tonic-gate * must increment this even if we aren't able to properly enable 8737c478bd9Sstevel@tonic-gate * this probe. 8747c478bd9Sstevel@tonic-gate */ 8757c478bd9Sstevel@tonic-gate mutex_enter(&probe->ftp_prov->ftp_mtx); 8767c478bd9Sstevel@tonic-gate probe->ftp_prov->ftp_rcount++; 8777c478bd9Sstevel@tonic-gate mutex_exit(&probe->ftp_prov->ftp_mtx); 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate /* 8807c478bd9Sstevel@tonic-gate * Bail out if we can't find the process for this probe or its 881b096140dSahl * provider is retired (meaning it was valid in a previously exec'ed 8827c478bd9Sstevel@tonic-gate * incarnation of this address space). The provider can't go away 8837c478bd9Sstevel@tonic-gate * while we're in this code path. 8847c478bd9Sstevel@tonic-gate */ 885b096140dSahl if (probe->ftp_prov->ftp_retired || 8867c478bd9Sstevel@tonic-gate (p = sprlock(probe->ftp_pid)) == NULL) 8877c478bd9Sstevel@tonic-gate return; 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate ASSERT(!(p->p_flag & SVFORK)); 8907c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 8917c478bd9Sstevel@tonic-gate 8927c478bd9Sstevel@tonic-gate /* 8937c478bd9Sstevel@tonic-gate * We have to enable the trap entry before any user threads have 8947c478bd9Sstevel@tonic-gate * the chance to execute the trap instruction we're about to place 8957c478bd9Sstevel@tonic-gate * in their process's text. 8967c478bd9Sstevel@tonic-gate */ 8977c478bd9Sstevel@tonic-gate fasttrap_enable_common(&fasttrap_pid_count, 8987c478bd9Sstevel@tonic-gate &dtrace_pid_probe_ptr, fasttrap_pid_probe, 8997c478bd9Sstevel@tonic-gate &dtrace_return_probe_ptr, fasttrap_return_probe); 9007c478bd9Sstevel@tonic-gate 9017c478bd9Sstevel@tonic-gate /* 9027c478bd9Sstevel@tonic-gate * Enable all the tracepoints and add this probe's id to each 9037c478bd9Sstevel@tonic-gate * tracepoint's list of active probes. 9047c478bd9Sstevel@tonic-gate */ 9057c478bd9Sstevel@tonic-gate for (i = 0; i < probe->ftp_ntps; i++) { 9068f5db3e7Sahl if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 9078f5db3e7Sahl /* 9088f5db3e7Sahl * If enabling the tracepoint failed completely, 9098f5db3e7Sahl * we don't have to disable it; if the failure 9108f5db3e7Sahl * was only partial we must disable it. 9118f5db3e7Sahl */ 9128f5db3e7Sahl if (rc == FASTTRAP_ENABLE_FAIL) 9138f5db3e7Sahl i--; 9148f5db3e7Sahl else 9158f5db3e7Sahl ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 9168f5db3e7Sahl 9177c478bd9Sstevel@tonic-gate /* 9187c478bd9Sstevel@tonic-gate * Back up and pull out all the tracepoints we've 9197c478bd9Sstevel@tonic-gate * created so far for this probe. 9207c478bd9Sstevel@tonic-gate */ 921*8c642573Sahl while (i >= 0) { 9227c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(p, probe, i); 923*8c642573Sahl i--; 9247c478bd9Sstevel@tonic-gate } 9257c478bd9Sstevel@tonic-gate 9267c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 9277c478bd9Sstevel@tonic-gate sprunlock(p); 9287c478bd9Sstevel@tonic-gate 9297c478bd9Sstevel@tonic-gate /* 9307c478bd9Sstevel@tonic-gate * Since we're not actually enabling this probe, 9317c478bd9Sstevel@tonic-gate * drop our reference on the trap table entry. 9327c478bd9Sstevel@tonic-gate */ 9337c478bd9Sstevel@tonic-gate fasttrap_disable_common(&fasttrap_pid_count, 9347c478bd9Sstevel@tonic-gate &dtrace_pid_probe_ptr, &dtrace_return_probe_ptr); 9357c478bd9Sstevel@tonic-gate return; 9367c478bd9Sstevel@tonic-gate } 9377c478bd9Sstevel@tonic-gate } 9387c478bd9Sstevel@tonic-gate 9397c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 9407c478bd9Sstevel@tonic-gate sprunlock(p); 9417c478bd9Sstevel@tonic-gate 9427c478bd9Sstevel@tonic-gate probe->ftp_enabled = 1; 9437c478bd9Sstevel@tonic-gate } 9447c478bd9Sstevel@tonic-gate 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9477c478bd9Sstevel@tonic-gate static void 9487c478bd9Sstevel@tonic-gate fasttrap_disable(void *arg, dtrace_id_t id, void *parg) 9497c478bd9Sstevel@tonic-gate { 9507c478bd9Sstevel@tonic-gate /* 9517c478bd9Sstevel@tonic-gate * Disable the probe the corresponds to statically placed trace 9527c478bd9Sstevel@tonic-gate * points. 9537c478bd9Sstevel@tonic-gate */ 9547c478bd9Sstevel@tonic-gate ASSERT(arg == NULL); 9557c478bd9Sstevel@tonic-gate ASSERT(id == fasttrap_probe_id); 9567c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 9577c478bd9Sstevel@tonic-gate fasttrap_disable_common(&fasttrap_count, &dtrace_fasttrap_probe_ptr, 9587c478bd9Sstevel@tonic-gate NULL); 9597c478bd9Sstevel@tonic-gate } 9607c478bd9Sstevel@tonic-gate 9617c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9627c478bd9Sstevel@tonic-gate static void 9637c478bd9Sstevel@tonic-gate fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 9647c478bd9Sstevel@tonic-gate { 9657c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 9667c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider = probe->ftp_prov; 9677c478bd9Sstevel@tonic-gate proc_t *p; 9687c478bd9Sstevel@tonic-gate int i, whack = 0; 9697c478bd9Sstevel@tonic-gate 9707c478bd9Sstevel@tonic-gate if (!probe->ftp_enabled) { 9717c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 9727c478bd9Sstevel@tonic-gate provider->ftp_rcount--; 9737c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_rcount >= 0); 9747c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 9757c478bd9Sstevel@tonic-gate return; 9767c478bd9Sstevel@tonic-gate } 9777c478bd9Sstevel@tonic-gate 9787c478bd9Sstevel@tonic-gate ASSERT(id == probe->ftp_id); 9797c478bd9Sstevel@tonic-gate 9807c478bd9Sstevel@tonic-gate /* 9817c478bd9Sstevel@tonic-gate * We won't be able to acquire a /proc-esque lock on the process 9827c478bd9Sstevel@tonic-gate * iff the process is dead and gone. In this case, we rely on the 9837c478bd9Sstevel@tonic-gate * provider lock as a point of mutual exclusion to prevent other 9847c478bd9Sstevel@tonic-gate * DTrace consumers from disabling this probe. 9857c478bd9Sstevel@tonic-gate */ 9867c478bd9Sstevel@tonic-gate if ((p = sprlock(probe->ftp_pid)) != NULL) { 9877c478bd9Sstevel@tonic-gate ASSERT(!(p->p_flag & SVFORK)); 9887c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 9897c478bd9Sstevel@tonic-gate } 9907c478bd9Sstevel@tonic-gate 9917c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate /* 9947c478bd9Sstevel@tonic-gate * Disable all the associated tracepoints. 9957c478bd9Sstevel@tonic-gate */ 9967c478bd9Sstevel@tonic-gate for (i = 0; i < probe->ftp_ntps; i++) { 9977c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(p, probe, i); 9987c478bd9Sstevel@tonic-gate } 9997c478bd9Sstevel@tonic-gate 10007c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_rcount > 0); 10017c478bd9Sstevel@tonic-gate provider->ftp_rcount--; 10027c478bd9Sstevel@tonic-gate 10037c478bd9Sstevel@tonic-gate if (p != NULL) { 10047c478bd9Sstevel@tonic-gate /* 10057c478bd9Sstevel@tonic-gate * Even though we may not be able to remove it entirely, we 1006b096140dSahl * mark this retired provider to get a chance to remove some 10077c478bd9Sstevel@tonic-gate * of the associated probes. 10087c478bd9Sstevel@tonic-gate */ 1009b096140dSahl if (provider->ftp_retired && !provider->ftp_marked) 10107c478bd9Sstevel@tonic-gate whack = provider->ftp_marked = 1; 10117c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 10127c478bd9Sstevel@tonic-gate 10137c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 10147c478bd9Sstevel@tonic-gate sprunlock(p); 10157c478bd9Sstevel@tonic-gate } else { 10167c478bd9Sstevel@tonic-gate /* 10177c478bd9Sstevel@tonic-gate * If the process is dead, we're just waiting for the 10187c478bd9Sstevel@tonic-gate * last probe to be disabled to be able to free it. 10197c478bd9Sstevel@tonic-gate */ 10207c478bd9Sstevel@tonic-gate if (provider->ftp_rcount == 0 && !provider->ftp_marked) 10217c478bd9Sstevel@tonic-gate whack = provider->ftp_marked = 1; 10227c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 10237c478bd9Sstevel@tonic-gate } 10247c478bd9Sstevel@tonic-gate 10257c478bd9Sstevel@tonic-gate if (whack) 10267c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate probe->ftp_enabled = 0; 10297c478bd9Sstevel@tonic-gate 10307c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 10317c478bd9Sstevel@tonic-gate fasttrap_disable_common(&fasttrap_pid_count, &dtrace_pid_probe_ptr, 10327c478bd9Sstevel@tonic-gate &dtrace_return_probe_ptr); 10337c478bd9Sstevel@tonic-gate } 10347c478bd9Sstevel@tonic-gate 10357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10367c478bd9Sstevel@tonic-gate static void 10377c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 10387c478bd9Sstevel@tonic-gate dtrace_argdesc_t *desc) 10397c478bd9Sstevel@tonic-gate { 10407c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 10417c478bd9Sstevel@tonic-gate char *str; 10427c478bd9Sstevel@tonic-gate int i; 10437c478bd9Sstevel@tonic-gate 10447c478bd9Sstevel@tonic-gate desc->dtargd_native[0] = '\0'; 10457c478bd9Sstevel@tonic-gate desc->dtargd_xlate[0] = '\0'; 10467c478bd9Sstevel@tonic-gate 1047b096140dSahl if (probe->ftp_prov->ftp_retired != 0 || 10487c478bd9Sstevel@tonic-gate desc->dtargd_ndx >= probe->ftp_nargs) { 10497c478bd9Sstevel@tonic-gate desc->dtargd_ndx = DTRACE_ARGNONE; 10507c478bd9Sstevel@tonic-gate return; 10517c478bd9Sstevel@tonic-gate } 10527c478bd9Sstevel@tonic-gate 10537c478bd9Sstevel@tonic-gate /* 10547c478bd9Sstevel@tonic-gate * We only need to set this member if the argument is remapped. 10557c478bd9Sstevel@tonic-gate */ 10567c478bd9Sstevel@tonic-gate if (probe->ftp_argmap != NULL) 10577c478bd9Sstevel@tonic-gate desc->dtargd_mapping = probe->ftp_argmap[desc->dtargd_ndx]; 10587c478bd9Sstevel@tonic-gate 10597c478bd9Sstevel@tonic-gate str = probe->ftp_ntypes; 10607c478bd9Sstevel@tonic-gate for (i = 0; i < desc->dtargd_mapping; i++) { 10617c478bd9Sstevel@tonic-gate str += strlen(str) + 1; 10627c478bd9Sstevel@tonic-gate } 10637c478bd9Sstevel@tonic-gate 10647c478bd9Sstevel@tonic-gate ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 10657c478bd9Sstevel@tonic-gate (void) strcpy(desc->dtargd_native, str); 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate if (probe->ftp_xtypes == NULL) 10687c478bd9Sstevel@tonic-gate return; 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate str = probe->ftp_xtypes; 10717c478bd9Sstevel@tonic-gate for (i = 0; i < desc->dtargd_ndx; i++) { 10727c478bd9Sstevel@tonic-gate str += strlen(str) + 1; 10737c478bd9Sstevel@tonic-gate } 10747c478bd9Sstevel@tonic-gate 10757c478bd9Sstevel@tonic-gate ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 10767c478bd9Sstevel@tonic-gate (void) strcpy(desc->dtargd_xlate, str); 10777c478bd9Sstevel@tonic-gate } 10787c478bd9Sstevel@tonic-gate 10797c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10807c478bd9Sstevel@tonic-gate static void 10817c478bd9Sstevel@tonic-gate fasttrap_destroy(void *arg, dtrace_id_t id, void *parg) 10827c478bd9Sstevel@tonic-gate { 10837c478bd9Sstevel@tonic-gate ASSERT(arg == NULL); 10847c478bd9Sstevel@tonic-gate ASSERT(id == fasttrap_probe_id); 10857c478bd9Sstevel@tonic-gate } 10867c478bd9Sstevel@tonic-gate 10877c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10887c478bd9Sstevel@tonic-gate static void 10897c478bd9Sstevel@tonic-gate fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 10907c478bd9Sstevel@tonic-gate { 10917c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 10927c478bd9Sstevel@tonic-gate int i; 10937c478bd9Sstevel@tonic-gate size_t size; 10947c478bd9Sstevel@tonic-gate 10957c478bd9Sstevel@tonic-gate ASSERT(probe != NULL); 10967c478bd9Sstevel@tonic-gate ASSERT(!probe->ftp_enabled); 10977c478bd9Sstevel@tonic-gate ASSERT(fasttrap_total >= probe->ftp_ntps); 10987c478bd9Sstevel@tonic-gate 10997c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 11007c478bd9Sstevel@tonic-gate size = sizeof (fasttrap_probe_t) + 11017c478bd9Sstevel@tonic-gate sizeof (probe->ftp_tps[0]) * (probe->ftp_ntps - 1); 11027c478bd9Sstevel@tonic-gate 11037c478bd9Sstevel@tonic-gate if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 11047c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(probe->ftp_gen); 11057c478bd9Sstevel@tonic-gate 11067c478bd9Sstevel@tonic-gate for (i = 0; i < probe->ftp_ntps; i++) { 11077c478bd9Sstevel@tonic-gate kmem_free(probe->ftp_tps[i].fit_tp, 11087c478bd9Sstevel@tonic-gate sizeof (fasttrap_tracepoint_t)); 11097c478bd9Sstevel@tonic-gate } 11107c478bd9Sstevel@tonic-gate 11117c478bd9Sstevel@tonic-gate kmem_free(probe, size); 11127c478bd9Sstevel@tonic-gate } 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate 11157c478bd9Sstevel@tonic-gate static const dtrace_pattr_t fasttrap_attr = { 11167c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11177c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11187c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11197c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11207c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11217c478bd9Sstevel@tonic-gate }; 11227c478bd9Sstevel@tonic-gate 11237c478bd9Sstevel@tonic-gate static dtrace_pops_t fasttrap_pops = { 11247c478bd9Sstevel@tonic-gate fasttrap_provide, 11257c478bd9Sstevel@tonic-gate NULL, 11267c478bd9Sstevel@tonic-gate fasttrap_enable, 11277c478bd9Sstevel@tonic-gate fasttrap_disable, 11287c478bd9Sstevel@tonic-gate NULL, 11297c478bd9Sstevel@tonic-gate NULL, 11307c478bd9Sstevel@tonic-gate NULL, 11317c478bd9Sstevel@tonic-gate fasttrap_getarg, 11327c478bd9Sstevel@tonic-gate NULL, 11337c478bd9Sstevel@tonic-gate fasttrap_destroy 11347c478bd9Sstevel@tonic-gate }; 11357c478bd9Sstevel@tonic-gate 11367c478bd9Sstevel@tonic-gate static const dtrace_pattr_t pid_attr = { 11377c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11387c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11397c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11407c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11417c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11427c478bd9Sstevel@tonic-gate }; 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate static dtrace_pops_t pid_pops = { 11457c478bd9Sstevel@tonic-gate fasttrap_pid_provide, 11467c478bd9Sstevel@tonic-gate NULL, 11477c478bd9Sstevel@tonic-gate fasttrap_pid_enable, 11487c478bd9Sstevel@tonic-gate fasttrap_pid_disable, 11497c478bd9Sstevel@tonic-gate NULL, 11507c478bd9Sstevel@tonic-gate NULL, 11517c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc, 11527c478bd9Sstevel@tonic-gate fasttrap_getarg, 11537c478bd9Sstevel@tonic-gate NULL, 11547c478bd9Sstevel@tonic-gate fasttrap_pid_destroy 11557c478bd9Sstevel@tonic-gate }; 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate static dtrace_pops_t usdt_pops = { 11587c478bd9Sstevel@tonic-gate fasttrap_pid_provide, 11597c478bd9Sstevel@tonic-gate NULL, 11607c478bd9Sstevel@tonic-gate fasttrap_pid_enable, 11617c478bd9Sstevel@tonic-gate fasttrap_pid_disable, 11627c478bd9Sstevel@tonic-gate NULL, 11637c478bd9Sstevel@tonic-gate NULL, 11647c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc, 11657c478bd9Sstevel@tonic-gate fasttrap_usdt_getarg, 11667c478bd9Sstevel@tonic-gate NULL, 11677c478bd9Sstevel@tonic-gate fasttrap_pid_destroy 11687c478bd9Sstevel@tonic-gate }; 11697c478bd9Sstevel@tonic-gate 1170b096140dSahl static fasttrap_proc_t * 1171b096140dSahl fasttrap_proc_lookup(pid_t pid) 1172b096140dSahl { 1173b096140dSahl fasttrap_bucket_t *bucket; 1174b096140dSahl fasttrap_proc_t *fprc, *new_fprc; 1175b096140dSahl 1176b096140dSahl bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1177b096140dSahl mutex_enter(&bucket->ftb_mtx); 1178b096140dSahl 1179b096140dSahl for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1180b096140dSahl if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) { 1181b096140dSahl mutex_enter(&fprc->ftpc_mtx); 1182b096140dSahl mutex_exit(&bucket->ftb_mtx); 1183b096140dSahl fprc->ftpc_count++; 1184b096140dSahl mutex_exit(&fprc->ftpc_mtx); 1185b096140dSahl 1186b096140dSahl return (fprc); 1187b096140dSahl } 1188b096140dSahl } 1189b096140dSahl 1190b096140dSahl /* 1191b096140dSahl * Drop the bucket lock so we don't try to perform a sleeping 1192b096140dSahl * allocation under it. 1193b096140dSahl */ 1194b096140dSahl mutex_exit(&bucket->ftb_mtx); 1195b096140dSahl 1196b096140dSahl new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1197b096140dSahl new_fprc->ftpc_pid = pid; 1198b096140dSahl new_fprc->ftpc_count = 1; 1199b096140dSahl 1200b096140dSahl mutex_enter(&bucket->ftb_mtx); 1201b096140dSahl 1202b096140dSahl /* 1203b096140dSahl * Take another lap through the list to make sure a proc hasn't 1204b096140dSahl * been created for this pid while we weren't under the bucket lock. 1205b096140dSahl */ 1206b096140dSahl for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1207b096140dSahl if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) { 1208b096140dSahl mutex_enter(&fprc->ftpc_mtx); 1209b096140dSahl mutex_exit(&bucket->ftb_mtx); 1210b096140dSahl fprc->ftpc_count++; 1211b096140dSahl mutex_exit(&fprc->ftpc_mtx); 1212b096140dSahl 1213b096140dSahl kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1214b096140dSahl 1215b096140dSahl return (fprc); 1216b096140dSahl } 1217b096140dSahl } 1218b096140dSahl 1219b096140dSahl new_fprc->ftpc_next = bucket->ftb_data; 1220b096140dSahl bucket->ftb_data = new_fprc; 1221b096140dSahl 1222b096140dSahl mutex_exit(&bucket->ftb_mtx); 1223b096140dSahl 1224b096140dSahl return (new_fprc); 1225b096140dSahl } 1226b096140dSahl 1227b096140dSahl static void 1228b096140dSahl fasttrap_proc_release(fasttrap_proc_t *proc) 1229b096140dSahl { 1230b096140dSahl fasttrap_bucket_t *bucket; 1231b096140dSahl fasttrap_proc_t *fprc, **fprcp; 1232b096140dSahl pid_t pid = proc->ftpc_pid; 1233b096140dSahl 1234b096140dSahl mutex_enter(&proc->ftpc_mtx); 1235b096140dSahl 1236b096140dSahl ASSERT(proc->ftpc_count != 0); 1237b096140dSahl 1238b096140dSahl if (--proc->ftpc_count != 0) { 1239b096140dSahl mutex_exit(&proc->ftpc_mtx); 1240b096140dSahl return; 1241b096140dSahl } 1242b096140dSahl 1243b096140dSahl mutex_exit(&proc->ftpc_mtx); 1244b096140dSahl 1245b096140dSahl bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1246b096140dSahl mutex_enter(&bucket->ftb_mtx); 1247b096140dSahl 1248b096140dSahl fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1249b096140dSahl while ((fprc = *fprcp) != NULL) { 1250b096140dSahl if (fprc == proc) 1251b096140dSahl break; 1252b096140dSahl 1253b096140dSahl fprcp = &fprc->ftpc_next; 1254b096140dSahl } 1255b096140dSahl 1256b096140dSahl /* 1257b096140dSahl * Something strange has happened if we can't find the proc. 1258b096140dSahl */ 1259b096140dSahl ASSERT(fprc != NULL); 1260b096140dSahl 1261b096140dSahl *fprcp = fprc->ftpc_next; 1262b096140dSahl 1263b096140dSahl mutex_exit(&bucket->ftb_mtx); 1264b096140dSahl 1265b096140dSahl kmem_free(fprc, sizeof (fasttrap_proc_t)); 1266b096140dSahl } 1267b096140dSahl 12687c478bd9Sstevel@tonic-gate /* 12697c478bd9Sstevel@tonic-gate * Lookup a fasttrap-managed provider based on its name and associated pid. 12707c478bd9Sstevel@tonic-gate * If the pattr argument is non-NULL, this function instantiates the provider 12717c478bd9Sstevel@tonic-gate * if it doesn't exist otherwise it returns NULL. The provider is returned 12727c478bd9Sstevel@tonic-gate * with its lock held. 12737c478bd9Sstevel@tonic-gate */ 12747c478bd9Sstevel@tonic-gate static fasttrap_provider_t * 12757c478bd9Sstevel@tonic-gate fasttrap_provider_lookup(pid_t pid, const char *name, 12767c478bd9Sstevel@tonic-gate const dtrace_pattr_t *pattr) 12777c478bd9Sstevel@tonic-gate { 12787c478bd9Sstevel@tonic-gate fasttrap_provider_t *fp, *new_fp = NULL; 12797c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 12807c478bd9Sstevel@tonic-gate char provname[DTRACE_PROVNAMELEN]; 12817c478bd9Sstevel@tonic-gate proc_t *p; 12827c478bd9Sstevel@tonic-gate uid_t uid = (uid_t)-1; 12837c478bd9Sstevel@tonic-gate 12847c478bd9Sstevel@tonic-gate ASSERT(strlen(name) < sizeof (fp->ftp_name)); 12857c478bd9Sstevel@tonic-gate 12867c478bd9Sstevel@tonic-gate bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 12877c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 12887c478bd9Sstevel@tonic-gate 12897c478bd9Sstevel@tonic-gate /* 12907c478bd9Sstevel@tonic-gate * Take a lap through the list and return the match if we find it. 12917c478bd9Sstevel@tonic-gate */ 12927c478bd9Sstevel@tonic-gate for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 12937c478bd9Sstevel@tonic-gate if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1294b096140dSahl !fp->ftp_retired) { 12957c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 12967c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 12977c478bd9Sstevel@tonic-gate return (fp); 12987c478bd9Sstevel@tonic-gate } 12997c478bd9Sstevel@tonic-gate } 13007c478bd9Sstevel@tonic-gate 13017c478bd9Sstevel@tonic-gate /* 13027c478bd9Sstevel@tonic-gate * Drop the bucket lock so we don't try to perform a sleeping 13037c478bd9Sstevel@tonic-gate * allocation under it. 13047c478bd9Sstevel@tonic-gate */ 13057c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 13067c478bd9Sstevel@tonic-gate 1307b096140dSahl /* 1308b096140dSahl * If we didn't want to create a new provider, just return failure. 1309b096140dSahl */ 13107c478bd9Sstevel@tonic-gate if (pattr == NULL) 13117c478bd9Sstevel@tonic-gate return (NULL); 13127c478bd9Sstevel@tonic-gate 13137c478bd9Sstevel@tonic-gate /* 13147c478bd9Sstevel@tonic-gate * Make sure the process exists, isn't a child created as the result 13157c478bd9Sstevel@tonic-gate * of a vfork(2), and isn't a zombie (but may be in fork). Record the 13167c478bd9Sstevel@tonic-gate * process's uid to pass to dtrace_register(). 13177c478bd9Sstevel@tonic-gate */ 13187c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 131997eda132Sraf if ((p = prfind(pid)) == NULL) { 13207c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 13217c478bd9Sstevel@tonic-gate return (NULL); 13227c478bd9Sstevel@tonic-gate } 13237c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 13247c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 132597eda132Sraf if (p->p_flag & (SVFORK | SEXITING)) { 132697eda132Sraf mutex_exit(&p->p_lock); 132797eda132Sraf return (NULL); 132897eda132Sraf } 13297c478bd9Sstevel@tonic-gate 13307c478bd9Sstevel@tonic-gate /* 13317c478bd9Sstevel@tonic-gate * Increment p_dtrace_probes so that the process knows to inform us 13327c478bd9Sstevel@tonic-gate * when it exits or execs. fasttrap_provider_free() decrements this 13337c478bd9Sstevel@tonic-gate * when we're done with this provider. 13347c478bd9Sstevel@tonic-gate */ 13357c478bd9Sstevel@tonic-gate p->p_dtrace_probes++; 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate mutex_enter(&p->p_crlock); 13387c478bd9Sstevel@tonic-gate uid = crgetruid(p->p_cred); 13397c478bd9Sstevel@tonic-gate mutex_exit(&p->p_crlock); 13407c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 13417c478bd9Sstevel@tonic-gate 13427c478bd9Sstevel@tonic-gate new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1343b096140dSahl new_fp->ftp_pid = pid; 1344b096140dSahl new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1345b096140dSahl 1346b096140dSahl ASSERT(new_fp->ftp_proc != NULL); 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 13497c478bd9Sstevel@tonic-gate 13507c478bd9Sstevel@tonic-gate /* 13517c478bd9Sstevel@tonic-gate * Take another lap through the list to make sure a provider hasn't 13527c478bd9Sstevel@tonic-gate * been created for this pid while we weren't under the bucket lock. 13537c478bd9Sstevel@tonic-gate */ 13547c478bd9Sstevel@tonic-gate for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 13557c478bd9Sstevel@tonic-gate if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1356b096140dSahl !fp->ftp_retired) { 13577c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 13587c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 13597c478bd9Sstevel@tonic-gate fasttrap_provider_free(new_fp); 13607c478bd9Sstevel@tonic-gate return (fp); 13617c478bd9Sstevel@tonic-gate } 13627c478bd9Sstevel@tonic-gate } 13637c478bd9Sstevel@tonic-gate 13647c478bd9Sstevel@tonic-gate (void) strcpy(new_fp->ftp_name, name); 13657c478bd9Sstevel@tonic-gate 13667c478bd9Sstevel@tonic-gate /* 13677c478bd9Sstevel@tonic-gate * Fail and return NULL if either the provider name is too long 13687c478bd9Sstevel@tonic-gate * or we fail to register this new provider with the DTrace 13697c478bd9Sstevel@tonic-gate * framework. Note that this is the only place we ever construct 13707c478bd9Sstevel@tonic-gate * the full provider name -- we keep it in pieces in the provider 13717c478bd9Sstevel@tonic-gate * structure. 13727c478bd9Sstevel@tonic-gate */ 13737c478bd9Sstevel@tonic-gate if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 13747c478bd9Sstevel@tonic-gate sizeof (provname) || 13757c478bd9Sstevel@tonic-gate dtrace_register(provname, pattr, 13767c478bd9Sstevel@tonic-gate DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER, uid, 13777c478bd9Sstevel@tonic-gate pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 13787c478bd9Sstevel@tonic-gate &new_fp->ftp_provid) != 0) { 13797c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 13807c478bd9Sstevel@tonic-gate fasttrap_provider_free(new_fp); 13817c478bd9Sstevel@tonic-gate return (NULL); 13827c478bd9Sstevel@tonic-gate } 13837c478bd9Sstevel@tonic-gate 13847c478bd9Sstevel@tonic-gate new_fp->ftp_next = bucket->ftb_data; 13857c478bd9Sstevel@tonic-gate bucket->ftb_data = new_fp; 13867c478bd9Sstevel@tonic-gate 13877c478bd9Sstevel@tonic-gate mutex_enter(&new_fp->ftp_mtx); 13887c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 13897c478bd9Sstevel@tonic-gate 13907c478bd9Sstevel@tonic-gate return (new_fp); 13917c478bd9Sstevel@tonic-gate } 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate static void 13947c478bd9Sstevel@tonic-gate fasttrap_provider_free(fasttrap_provider_t *provider) 13957c478bd9Sstevel@tonic-gate { 13967c478bd9Sstevel@tonic-gate pid_t pid = provider->ftp_pid; 13977c478bd9Sstevel@tonic-gate proc_t *p; 13987c478bd9Sstevel@tonic-gate 13997c478bd9Sstevel@tonic-gate /* 14007c478bd9Sstevel@tonic-gate * There need to be no consumers using this provider and no 14017c478bd9Sstevel@tonic-gate * associated enabled probes. 14027c478bd9Sstevel@tonic-gate */ 14037c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_ccount == 0); 14047c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_rcount == 0); 14057c478bd9Sstevel@tonic-gate 1406b096140dSahl fasttrap_proc_release(provider->ftp_proc); 1407b096140dSahl 14087c478bd9Sstevel@tonic-gate kmem_free(provider, sizeof (fasttrap_provider_t)); 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate /* 14117c478bd9Sstevel@tonic-gate * Decrement p_dtrace_probes on the process whose provider we're 14127c478bd9Sstevel@tonic-gate * freeing. We don't have to worry about clobbering somone else's 14137c478bd9Sstevel@tonic-gate * modifications to it because we have locked the bucket that 14147c478bd9Sstevel@tonic-gate * corresponds to this process's hash chain in the provider hash 14157c478bd9Sstevel@tonic-gate * table. Don't sweat it if we can't find the process. 14167c478bd9Sstevel@tonic-gate */ 14177c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 14187c478bd9Sstevel@tonic-gate if ((p = prfind(pid)) == NULL) { 14197c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 14207c478bd9Sstevel@tonic-gate return; 14217c478bd9Sstevel@tonic-gate } 14227c478bd9Sstevel@tonic-gate 14237c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 14247c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 14257c478bd9Sstevel@tonic-gate 14267c478bd9Sstevel@tonic-gate p->p_dtrace_probes--; 14277c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 14287c478bd9Sstevel@tonic-gate } 14297c478bd9Sstevel@tonic-gate 14307c478bd9Sstevel@tonic-gate static void 14317c478bd9Sstevel@tonic-gate fasttrap_provider_retire(fasttrap_provider_t *provider) 14327c478bd9Sstevel@tonic-gate { 14337c478bd9Sstevel@tonic-gate dtrace_provider_id_t provid = provider->ftp_provid; 14347c478bd9Sstevel@tonic-gate 14357c478bd9Sstevel@tonic-gate /* 1436b096140dSahl * Mark the provider to be removed in our post-processing step, 1437b096140dSahl * mark it retired, and mark its proc as defunct (though it may 1438b096140dSahl * already be marked defunct by another provider that shares the 1439b096140dSahl * same proc). Marking it indicates that we should try to remove it; 1440b096140dSahl * setting the retired flag indicates that we're done with this 1441b096140dSahl * provider; setting the proc to be defunct indicates that all 1442b096140dSahl * tracepoints associated with the traced process should be ignored. 14437c478bd9Sstevel@tonic-gate */ 1444b096140dSahl provider->ftp_proc->ftpc_defunct = 1; 1445b096140dSahl provider->ftp_retired = 1; 14467c478bd9Sstevel@tonic-gate provider->ftp_marked = 1; 14477c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 14487c478bd9Sstevel@tonic-gate 14497c478bd9Sstevel@tonic-gate /* 14507c478bd9Sstevel@tonic-gate * We don't have to worry about invalidating the same provider twice 14517c478bd9Sstevel@tonic-gate * since fasttrap_provider_lookup() will ignore provider that have 1452b096140dSahl * been marked as retired. 14537c478bd9Sstevel@tonic-gate */ 14547c478bd9Sstevel@tonic-gate dtrace_invalidate(provid); 14557c478bd9Sstevel@tonic-gate 14567c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 14577c478bd9Sstevel@tonic-gate } 14587c478bd9Sstevel@tonic-gate 14597c478bd9Sstevel@tonic-gate static int 14607c478bd9Sstevel@tonic-gate fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 14617c478bd9Sstevel@tonic-gate { 14627c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider; 14637c478bd9Sstevel@tonic-gate fasttrap_probe_t *pp; 14647c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 14657c478bd9Sstevel@tonic-gate char *name; 14667c478bd9Sstevel@tonic-gate size_t size; 14677c478bd9Sstevel@tonic-gate int i, aframes, whack; 14687c478bd9Sstevel@tonic-gate 14697c478bd9Sstevel@tonic-gate switch (pdata->ftps_type) { 14707c478bd9Sstevel@tonic-gate case DTFTP_ENTRY: 14717c478bd9Sstevel@tonic-gate name = "entry"; 14727c478bd9Sstevel@tonic-gate aframes = FASTTRAP_ENTRY_AFRAMES; 14737c478bd9Sstevel@tonic-gate break; 14747c478bd9Sstevel@tonic-gate case DTFTP_RETURN: 14757c478bd9Sstevel@tonic-gate name = "return"; 14767c478bd9Sstevel@tonic-gate aframes = FASTTRAP_RETURN_AFRAMES; 14777c478bd9Sstevel@tonic-gate break; 14787c478bd9Sstevel@tonic-gate case DTFTP_OFFSETS: 14797c478bd9Sstevel@tonic-gate name = NULL; 14807c478bd9Sstevel@tonic-gate break; 14817c478bd9Sstevel@tonic-gate default: 14827c478bd9Sstevel@tonic-gate return (EINVAL); 14837c478bd9Sstevel@tonic-gate } 14847c478bd9Sstevel@tonic-gate 14857c478bd9Sstevel@tonic-gate if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 14867c478bd9Sstevel@tonic-gate FASTTRAP_PID_NAME, &pid_attr)) == NULL) 14877c478bd9Sstevel@tonic-gate return (ESRCH); 14887c478bd9Sstevel@tonic-gate 14897c478bd9Sstevel@tonic-gate /* 14907c478bd9Sstevel@tonic-gate * Increment this reference count to indicate that a consumer is 14917c478bd9Sstevel@tonic-gate * actively adding a new probe associated with this provider. 14927c478bd9Sstevel@tonic-gate */ 14937c478bd9Sstevel@tonic-gate provider->ftp_ccount++; 14947c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate if (name != NULL) { 14977c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(provider->ftp_provid, 14987c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name) != 0) 14997c478bd9Sstevel@tonic-gate goto done; 15007c478bd9Sstevel@tonic-gate 15017c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 15027c478bd9Sstevel@tonic-gate 15037c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max) { 15047c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 15057c478bd9Sstevel@tonic-gate goto no_mem; 15067c478bd9Sstevel@tonic-gate } 15077c478bd9Sstevel@tonic-gate 15087c478bd9Sstevel@tonic-gate ASSERT(pdata->ftps_noffs > 0); 15097c478bd9Sstevel@tonic-gate size = sizeof (fasttrap_probe_t) + 15107c478bd9Sstevel@tonic-gate sizeof (pp->ftp_tps[0]) * (pdata->ftps_noffs - 1); 15117c478bd9Sstevel@tonic-gate 15127c478bd9Sstevel@tonic-gate pp = kmem_zalloc(size, KM_SLEEP); 15137c478bd9Sstevel@tonic-gate 15147c478bd9Sstevel@tonic-gate pp->ftp_prov = provider; 15157c478bd9Sstevel@tonic-gate pp->ftp_faddr = pdata->ftps_pc; 15167c478bd9Sstevel@tonic-gate pp->ftp_fsize = pdata->ftps_size; 15177c478bd9Sstevel@tonic-gate pp->ftp_pid = pdata->ftps_pid; 15187c478bd9Sstevel@tonic-gate pp->ftp_ntps = pdata->ftps_noffs; 15197c478bd9Sstevel@tonic-gate pp->ftp_type = pdata->ftps_type; 15207c478bd9Sstevel@tonic-gate 15217c478bd9Sstevel@tonic-gate for (i = 0; i < pdata->ftps_noffs; i++) { 15227c478bd9Sstevel@tonic-gate tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 15237c478bd9Sstevel@tonic-gate KM_SLEEP); 15247c478bd9Sstevel@tonic-gate 1525b096140dSahl tp->ftt_proc = provider->ftp_proc; 15267c478bd9Sstevel@tonic-gate tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 15277c478bd9Sstevel@tonic-gate tp->ftt_pid = pdata->ftps_pid; 15287c478bd9Sstevel@tonic-gate 15297c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_tp = tp; 15307c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_id.fti_probe = pp; 15317c478bd9Sstevel@tonic-gate } 15327c478bd9Sstevel@tonic-gate 15337c478bd9Sstevel@tonic-gate pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 15347c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 15357c478bd9Sstevel@tonic-gate } else { 15367c478bd9Sstevel@tonic-gate for (i = 0; i < pdata->ftps_noffs; i++) { 15377c478bd9Sstevel@tonic-gate char name_str[17]; 15387c478bd9Sstevel@tonic-gate 15397c478bd9Sstevel@tonic-gate (void) sprintf(name_str, "%llx", 15407c478bd9Sstevel@tonic-gate (unsigned long long)pdata->ftps_offs[i]); 15417c478bd9Sstevel@tonic-gate 15427c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(provider->ftp_provid, 15437c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 15447c478bd9Sstevel@tonic-gate continue; 15457c478bd9Sstevel@tonic-gate 15467c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, 1); 15477c478bd9Sstevel@tonic-gate 15487c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max) { 15497c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -1); 15507c478bd9Sstevel@tonic-gate goto no_mem; 15517c478bd9Sstevel@tonic-gate } 15527c478bd9Sstevel@tonic-gate 15537c478bd9Sstevel@tonic-gate pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 15547c478bd9Sstevel@tonic-gate 15557c478bd9Sstevel@tonic-gate pp->ftp_prov = provider; 15567c478bd9Sstevel@tonic-gate pp->ftp_faddr = pdata->ftps_pc; 15577c478bd9Sstevel@tonic-gate pp->ftp_fsize = pdata->ftps_size; 15587c478bd9Sstevel@tonic-gate pp->ftp_pid = pdata->ftps_pid; 15597c478bd9Sstevel@tonic-gate pp->ftp_ntps = 1; 15607c478bd9Sstevel@tonic-gate pp->ftp_type = pdata->ftps_type; 15617c478bd9Sstevel@tonic-gate 15627c478bd9Sstevel@tonic-gate tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 15637c478bd9Sstevel@tonic-gate KM_SLEEP); 15647c478bd9Sstevel@tonic-gate 1565b096140dSahl tp->ftt_proc = provider->ftp_proc; 15667c478bd9Sstevel@tonic-gate tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 15677c478bd9Sstevel@tonic-gate tp->ftt_pid = pdata->ftps_pid; 15687c478bd9Sstevel@tonic-gate 15697c478bd9Sstevel@tonic-gate pp->ftp_tps[0].fit_tp = tp; 15707c478bd9Sstevel@tonic-gate pp->ftp_tps[0].fit_id.fti_probe = pp; 15717c478bd9Sstevel@tonic-gate 15727c478bd9Sstevel@tonic-gate pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 15737c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name_str, 15747c478bd9Sstevel@tonic-gate FASTTRAP_OFFSET_AFRAMES, pp); 15757c478bd9Sstevel@tonic-gate } 15767c478bd9Sstevel@tonic-gate } 15777c478bd9Sstevel@tonic-gate 15787c478bd9Sstevel@tonic-gate done: 15797c478bd9Sstevel@tonic-gate /* 15807c478bd9Sstevel@tonic-gate * We know that the provider is still valid since we incremented the 15817c478bd9Sstevel@tonic-gate * reference count. If someone tried to free this provider while we 15827c478bd9Sstevel@tonic-gate * were using it (e.g. because the process called exec(2) or exit(2)), 15837c478bd9Sstevel@tonic-gate * take note of that and try to free it now. 15847c478bd9Sstevel@tonic-gate */ 15857c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 15867c478bd9Sstevel@tonic-gate provider->ftp_ccount--; 1587b096140dSahl whack = provider->ftp_retired; 15887c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 15897c478bd9Sstevel@tonic-gate 15907c478bd9Sstevel@tonic-gate if (whack) 15917c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 15927c478bd9Sstevel@tonic-gate 15937c478bd9Sstevel@tonic-gate return (0); 15947c478bd9Sstevel@tonic-gate 15957c478bd9Sstevel@tonic-gate no_mem: 15967c478bd9Sstevel@tonic-gate /* 15977c478bd9Sstevel@tonic-gate * If we've exhausted the allowable resources, we'll try to remove 15987c478bd9Sstevel@tonic-gate * this provider to free some up. This is to cover the case where 15997c478bd9Sstevel@tonic-gate * the user has accidentally created many more probes than was 16007c478bd9Sstevel@tonic-gate * intended (e.g. pid123:::). 16017c478bd9Sstevel@tonic-gate */ 16027c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 16037c478bd9Sstevel@tonic-gate provider->ftp_ccount--; 16047c478bd9Sstevel@tonic-gate provider->ftp_marked = 1; 16057c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 16067c478bd9Sstevel@tonic-gate 16077c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 16087c478bd9Sstevel@tonic-gate 16097c478bd9Sstevel@tonic-gate return (ENOMEM); 16107c478bd9Sstevel@tonic-gate } 16117c478bd9Sstevel@tonic-gate 16127c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16137c478bd9Sstevel@tonic-gate static void * 16147c478bd9Sstevel@tonic-gate fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 16157c478bd9Sstevel@tonic-gate { 16167c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider; 16177c478bd9Sstevel@tonic-gate 16187c478bd9Sstevel@tonic-gate /* 16197c478bd9Sstevel@tonic-gate * A 32-bit unsigned integer (like a pid for example) can be 16207c478bd9Sstevel@tonic-gate * expressed in 10 or fewer decimal digits. Make sure that we'll 16217c478bd9Sstevel@tonic-gate * have enough space for the provider name. 16227c478bd9Sstevel@tonic-gate */ 16237c478bd9Sstevel@tonic-gate if (strlen(dhpv->dthpv_provname) + 10 >= 16247c478bd9Sstevel@tonic-gate sizeof (provider->ftp_name)) { 16257c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "failed to instantiate provider %s: " 16267c478bd9Sstevel@tonic-gate "name too long to accomodate pid", dhpv->dthpv_provname); 16277c478bd9Sstevel@tonic-gate return (NULL); 16287c478bd9Sstevel@tonic-gate } 16297c478bd9Sstevel@tonic-gate 16307c478bd9Sstevel@tonic-gate /* 16317c478bd9Sstevel@tonic-gate * Don't let folks spoof the true pid provider. 16327c478bd9Sstevel@tonic-gate */ 16337c478bd9Sstevel@tonic-gate if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 16347c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "failed to instantiate provider %s: " 16357c478bd9Sstevel@tonic-gate "%s is an invalid name", dhpv->dthpv_provname, 16367c478bd9Sstevel@tonic-gate FASTTRAP_PID_NAME); 16377c478bd9Sstevel@tonic-gate return (NULL); 16387c478bd9Sstevel@tonic-gate } 16397c478bd9Sstevel@tonic-gate 16407c478bd9Sstevel@tonic-gate /* 16417c478bd9Sstevel@tonic-gate * The highest stability class that fasttrap supports is ISA; cap 16427c478bd9Sstevel@tonic-gate * the stability of the new provider accordingly. 16437c478bd9Sstevel@tonic-gate */ 16447c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_provider.dtat_class >= DTRACE_CLASS_COMMON) 16457c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 16467c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_mod.dtat_class >= DTRACE_CLASS_COMMON) 16477c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 16487c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_func.dtat_class >= DTRACE_CLASS_COMMON) 16497c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 16507c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_name.dtat_class >= DTRACE_CLASS_COMMON) 16517c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 16527c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_args.dtat_class >= DTRACE_CLASS_COMMON) 16537c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 16547c478bd9Sstevel@tonic-gate 16557c478bd9Sstevel@tonic-gate if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 16567c478bd9Sstevel@tonic-gate &dhpv->dthpv_pattr)) == NULL) { 16577c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "failed to instantiate provider %s for " 16587c478bd9Sstevel@tonic-gate "process %u", dhpv->dthpv_provname, (uint_t)pid); 16597c478bd9Sstevel@tonic-gate return (NULL); 16607c478bd9Sstevel@tonic-gate } 16617c478bd9Sstevel@tonic-gate 16627c478bd9Sstevel@tonic-gate /* 16637c478bd9Sstevel@tonic-gate * We elevate the consumer count here to ensure that this provider 16647c478bd9Sstevel@tonic-gate * isn't removed until after the meta provider has been told to 16657c478bd9Sstevel@tonic-gate * remove it. 16667c478bd9Sstevel@tonic-gate */ 16677c478bd9Sstevel@tonic-gate provider->ftp_ccount++; 16687c478bd9Sstevel@tonic-gate 16697c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 16707c478bd9Sstevel@tonic-gate 16717c478bd9Sstevel@tonic-gate return (provider); 16727c478bd9Sstevel@tonic-gate } 16737c478bd9Sstevel@tonic-gate 16747c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16757c478bd9Sstevel@tonic-gate static void 16767c478bd9Sstevel@tonic-gate fasttrap_meta_create_probe(void *arg, void *parg, 16777c478bd9Sstevel@tonic-gate dtrace_helper_probedesc_t *dhpb) 16787c478bd9Sstevel@tonic-gate { 16797c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider = parg; 16807c478bd9Sstevel@tonic-gate fasttrap_probe_t *pp; 16817c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 16827c478bd9Sstevel@tonic-gate size_t size; 16837c478bd9Sstevel@tonic-gate int i; 16847c478bd9Sstevel@tonic-gate 16857c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 16867c478bd9Sstevel@tonic-gate 16877c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 16887c478bd9Sstevel@tonic-gate dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 16897c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 16907c478bd9Sstevel@tonic-gate return; 16917c478bd9Sstevel@tonic-gate } 16927c478bd9Sstevel@tonic-gate 16937c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, dhpb->dthpb_noffs); 16947c478bd9Sstevel@tonic-gate 16957c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max) { 16967c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -dhpb->dthpb_noffs); 16977c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 16987c478bd9Sstevel@tonic-gate return; 16997c478bd9Sstevel@tonic-gate } 17007c478bd9Sstevel@tonic-gate 17017c478bd9Sstevel@tonic-gate size = sizeof (fasttrap_probe_t) + 17027c478bd9Sstevel@tonic-gate sizeof (pp->ftp_tps[0]) * (dhpb->dthpb_noffs - 1); 17037c478bd9Sstevel@tonic-gate pp = kmem_zalloc(size, KM_SLEEP); 17047c478bd9Sstevel@tonic-gate 17057c478bd9Sstevel@tonic-gate pp->ftp_prov = provider; 17067c478bd9Sstevel@tonic-gate pp->ftp_pid = provider->ftp_pid; 17077c478bd9Sstevel@tonic-gate pp->ftp_ntps = dhpb->dthpb_noffs; 17087c478bd9Sstevel@tonic-gate #ifdef __sparc 17097c478bd9Sstevel@tonic-gate pp->ftp_type = DTFTP_POST_OFFSETS; 17107c478bd9Sstevel@tonic-gate #else 17117c478bd9Sstevel@tonic-gate pp->ftp_type = DTFTP_OFFSETS; 17127c478bd9Sstevel@tonic-gate #endif 17137c478bd9Sstevel@tonic-gate pp->ftp_nargs = dhpb->dthpb_xargc; 17147c478bd9Sstevel@tonic-gate pp->ftp_xtypes = dhpb->dthpb_xtypes; 17157c478bd9Sstevel@tonic-gate pp->ftp_ntypes = dhpb->dthpb_ntypes; 17167c478bd9Sstevel@tonic-gate 17177c478bd9Sstevel@tonic-gate for (i = 0; i < pp->ftp_ntps; i++) { 17187c478bd9Sstevel@tonic-gate tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 17197c478bd9Sstevel@tonic-gate 1720b096140dSahl tp->ftt_proc = provider->ftp_proc; 17217c478bd9Sstevel@tonic-gate tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 17227c478bd9Sstevel@tonic-gate tp->ftt_pid = provider->ftp_pid; 17237c478bd9Sstevel@tonic-gate 17247c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_tp = tp; 17257c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_id.fti_probe = pp; 17267c478bd9Sstevel@tonic-gate } 17277c478bd9Sstevel@tonic-gate 17287c478bd9Sstevel@tonic-gate /* 17297c478bd9Sstevel@tonic-gate * If the arguments are shuffled around we set the argument remapping 17307c478bd9Sstevel@tonic-gate * table. Later, when the probe fires, we only remap the arguments 17317c478bd9Sstevel@tonic-gate * if the table is non-NULL. 17327c478bd9Sstevel@tonic-gate */ 17337c478bd9Sstevel@tonic-gate for (i = 0; i < dhpb->dthpb_xargc; i++) { 17347c478bd9Sstevel@tonic-gate if (dhpb->dthpb_args[i] != i) { 17357c478bd9Sstevel@tonic-gate pp->ftp_argmap = dhpb->dthpb_args; 17367c478bd9Sstevel@tonic-gate break; 17377c478bd9Sstevel@tonic-gate } 17387c478bd9Sstevel@tonic-gate } 17397c478bd9Sstevel@tonic-gate 17407c478bd9Sstevel@tonic-gate /* 17417c478bd9Sstevel@tonic-gate * The probe is fully constructed -- register it with DTrace. 17427c478bd9Sstevel@tonic-gate */ 17437c478bd9Sstevel@tonic-gate pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 17447c478bd9Sstevel@tonic-gate dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 17457c478bd9Sstevel@tonic-gate 17467c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 17477c478bd9Sstevel@tonic-gate } 17487c478bd9Sstevel@tonic-gate 17497c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17507c478bd9Sstevel@tonic-gate static void 17517c478bd9Sstevel@tonic-gate fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 17527c478bd9Sstevel@tonic-gate { 17537c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider; 17547c478bd9Sstevel@tonic-gate 17557c478bd9Sstevel@tonic-gate if ((provider = fasttrap_provider_lookup(pid, 17567c478bd9Sstevel@tonic-gate dhpv->dthpv_provname, NULL)) != NULL) { 17577c478bd9Sstevel@tonic-gate /* 17587c478bd9Sstevel@tonic-gate * Drop the consumer count now that we're done with this 17597c478bd9Sstevel@tonic-gate * provider. If there are no other consumers retire it now. 17607c478bd9Sstevel@tonic-gate */ 17617c478bd9Sstevel@tonic-gate if (--provider->ftp_ccount == 0) 17627c478bd9Sstevel@tonic-gate fasttrap_provider_retire(provider); 17637c478bd9Sstevel@tonic-gate else 17647c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 17657c478bd9Sstevel@tonic-gate } 17667c478bd9Sstevel@tonic-gate } 17677c478bd9Sstevel@tonic-gate 17687c478bd9Sstevel@tonic-gate static dtrace_mops_t fasttrap_mops = { 17697c478bd9Sstevel@tonic-gate fasttrap_meta_create_probe, 17707c478bd9Sstevel@tonic-gate fasttrap_meta_provide, 17717c478bd9Sstevel@tonic-gate fasttrap_meta_remove 17727c478bd9Sstevel@tonic-gate }; 17737c478bd9Sstevel@tonic-gate 17747c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17757c478bd9Sstevel@tonic-gate static int 17767c478bd9Sstevel@tonic-gate fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 17777c478bd9Sstevel@tonic-gate { 17787c478bd9Sstevel@tonic-gate return (0); 17797c478bd9Sstevel@tonic-gate } 17807c478bd9Sstevel@tonic-gate 17817c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17827c478bd9Sstevel@tonic-gate static int 17837c478bd9Sstevel@tonic-gate fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 17847c478bd9Sstevel@tonic-gate { 17857c478bd9Sstevel@tonic-gate if (!dtrace_attached()) 17867c478bd9Sstevel@tonic-gate return (EAGAIN); 17877c478bd9Sstevel@tonic-gate 17887c478bd9Sstevel@tonic-gate if (cmd == FASTTRAPIOC_MAKEPROBE) { 17897c478bd9Sstevel@tonic-gate fasttrap_probe_spec_t *uprobe = (void *)arg; 17907c478bd9Sstevel@tonic-gate fasttrap_probe_spec_t *probe; 17917c478bd9Sstevel@tonic-gate uint64_t noffs; 17927c478bd9Sstevel@tonic-gate size_t size; 17937c478bd9Sstevel@tonic-gate int ret; 17947c478bd9Sstevel@tonic-gate char *c; 17957c478bd9Sstevel@tonic-gate 17967c478bd9Sstevel@tonic-gate if (copyin(&uprobe->ftps_noffs, &noffs, 17977c478bd9Sstevel@tonic-gate sizeof (uprobe->ftps_noffs))) 17987c478bd9Sstevel@tonic-gate return (EFAULT); 17997c478bd9Sstevel@tonic-gate 18007c478bd9Sstevel@tonic-gate /* 18017c478bd9Sstevel@tonic-gate * Probes must have at least one tracepoint. 18027c478bd9Sstevel@tonic-gate */ 18037c478bd9Sstevel@tonic-gate if (noffs == 0) 18047c478bd9Sstevel@tonic-gate return (EINVAL); 18057c478bd9Sstevel@tonic-gate 18067c478bd9Sstevel@tonic-gate size = sizeof (fasttrap_probe_spec_t) + 18077c478bd9Sstevel@tonic-gate sizeof (probe->ftps_offs[0]) * (noffs - 1); 18087c478bd9Sstevel@tonic-gate 18097c478bd9Sstevel@tonic-gate if (size > 1024 * 1024) 18107c478bd9Sstevel@tonic-gate return (ENOMEM); 18117c478bd9Sstevel@tonic-gate 18127c478bd9Sstevel@tonic-gate probe = kmem_alloc(size, KM_SLEEP); 18137c478bd9Sstevel@tonic-gate 18147c478bd9Sstevel@tonic-gate if (copyin(uprobe, probe, size) != 0) { 18157c478bd9Sstevel@tonic-gate kmem_free(probe, size); 18167c478bd9Sstevel@tonic-gate return (EFAULT); 18177c478bd9Sstevel@tonic-gate } 18187c478bd9Sstevel@tonic-gate 18197c478bd9Sstevel@tonic-gate /* 18207c478bd9Sstevel@tonic-gate * Verify that the function and module strings contain no 18217c478bd9Sstevel@tonic-gate * funny characters. 18227c478bd9Sstevel@tonic-gate */ 18237c478bd9Sstevel@tonic-gate for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 18247c478bd9Sstevel@tonic-gate if (*c < 0x20 || 0x7f <= *c) { 18257c478bd9Sstevel@tonic-gate ret = EINVAL; 18267c478bd9Sstevel@tonic-gate goto err; 18277c478bd9Sstevel@tonic-gate } 18287c478bd9Sstevel@tonic-gate } 18297c478bd9Sstevel@tonic-gate 18307c478bd9Sstevel@tonic-gate for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 18317c478bd9Sstevel@tonic-gate if (*c < 0x20 || 0x7f <= *c) { 18327c478bd9Sstevel@tonic-gate ret = EINVAL; 18337c478bd9Sstevel@tonic-gate goto err; 18347c478bd9Sstevel@tonic-gate } 18357c478bd9Sstevel@tonic-gate } 18367c478bd9Sstevel@tonic-gate 18377c478bd9Sstevel@tonic-gate if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 18387c478bd9Sstevel@tonic-gate proc_t *p; 18397c478bd9Sstevel@tonic-gate pid_t pid = probe->ftps_pid; 18407c478bd9Sstevel@tonic-gate 18417c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 18427c478bd9Sstevel@tonic-gate /* 18437c478bd9Sstevel@tonic-gate * Report an error if the process doesn't exist 18447c478bd9Sstevel@tonic-gate * or is actively being birthed. 18457c478bd9Sstevel@tonic-gate */ 18467c478bd9Sstevel@tonic-gate if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 18477c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 18487c478bd9Sstevel@tonic-gate return (ESRCH); 18497c478bd9Sstevel@tonic-gate } 18507c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 18517c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 18527c478bd9Sstevel@tonic-gate 18537c478bd9Sstevel@tonic-gate if ((ret = priv_proc_cred_perm(cr, p, NULL, 18547c478bd9Sstevel@tonic-gate VREAD | VWRITE)) != 0) { 18557c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 18567c478bd9Sstevel@tonic-gate return (ret); 18577c478bd9Sstevel@tonic-gate } 18587c478bd9Sstevel@tonic-gate 18597c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 18607c478bd9Sstevel@tonic-gate } 18617c478bd9Sstevel@tonic-gate 18627c478bd9Sstevel@tonic-gate ret = fasttrap_add_probe(probe); 18637c478bd9Sstevel@tonic-gate err: 18647c478bd9Sstevel@tonic-gate kmem_free(probe, size); 18657c478bd9Sstevel@tonic-gate 18667c478bd9Sstevel@tonic-gate return (ret); 18677c478bd9Sstevel@tonic-gate 18687c478bd9Sstevel@tonic-gate } else if (cmd == FASTTRAPIOC_GETINSTR) { 18697c478bd9Sstevel@tonic-gate fasttrap_instr_query_t instr; 18707c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 18717c478bd9Sstevel@tonic-gate uint_t index; 18727c478bd9Sstevel@tonic-gate int ret; 18737c478bd9Sstevel@tonic-gate 18747c478bd9Sstevel@tonic-gate if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 18757c478bd9Sstevel@tonic-gate return (EFAULT); 18767c478bd9Sstevel@tonic-gate 18777c478bd9Sstevel@tonic-gate if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 18787c478bd9Sstevel@tonic-gate proc_t *p; 18797c478bd9Sstevel@tonic-gate pid_t pid = instr.ftiq_pid; 18807c478bd9Sstevel@tonic-gate 18817c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 18827c478bd9Sstevel@tonic-gate /* 18837c478bd9Sstevel@tonic-gate * Report an error if the process doesn't exist 18847c478bd9Sstevel@tonic-gate * or is actively being birthed. 18857c478bd9Sstevel@tonic-gate */ 18867c478bd9Sstevel@tonic-gate if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 18877c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 18887c478bd9Sstevel@tonic-gate return (ESRCH); 18897c478bd9Sstevel@tonic-gate } 18907c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 18917c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 18927c478bd9Sstevel@tonic-gate 18937c478bd9Sstevel@tonic-gate if ((ret = priv_proc_cred_perm(cr, p, NULL, 18947c478bd9Sstevel@tonic-gate VREAD)) != 0) { 18957c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 18967c478bd9Sstevel@tonic-gate return (ret); 18977c478bd9Sstevel@tonic-gate } 18987c478bd9Sstevel@tonic-gate 18997c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 19007c478bd9Sstevel@tonic-gate } 19017c478bd9Sstevel@tonic-gate 19027c478bd9Sstevel@tonic-gate index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 19037c478bd9Sstevel@tonic-gate 19047c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 19057c478bd9Sstevel@tonic-gate tp = fasttrap_tpoints.fth_table[index].ftb_data; 19067c478bd9Sstevel@tonic-gate while (tp != NULL) { 19077c478bd9Sstevel@tonic-gate if (instr.ftiq_pid == tp->ftt_pid && 19087c478bd9Sstevel@tonic-gate instr.ftiq_pc == tp->ftt_pc && 1909b096140dSahl !tp->ftt_proc->ftpc_defunct) 19107c478bd9Sstevel@tonic-gate break; 19117c478bd9Sstevel@tonic-gate 19127c478bd9Sstevel@tonic-gate tp = tp->ftt_next; 19137c478bd9Sstevel@tonic-gate } 19147c478bd9Sstevel@tonic-gate 19157c478bd9Sstevel@tonic-gate if (tp == NULL) { 19167c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 19177c478bd9Sstevel@tonic-gate return (ENOENT); 19187c478bd9Sstevel@tonic-gate } 19197c478bd9Sstevel@tonic-gate 19207c478bd9Sstevel@tonic-gate bcopy(&tp->ftt_instr, &instr.ftiq_instr, 19217c478bd9Sstevel@tonic-gate sizeof (instr.ftiq_instr)); 19227c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 19237c478bd9Sstevel@tonic-gate 19247c478bd9Sstevel@tonic-gate if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 19257c478bd9Sstevel@tonic-gate return (EFAULT); 19267c478bd9Sstevel@tonic-gate 19277c478bd9Sstevel@tonic-gate return (0); 19287c478bd9Sstevel@tonic-gate } 19297c478bd9Sstevel@tonic-gate 19307c478bd9Sstevel@tonic-gate return (EINVAL); 19317c478bd9Sstevel@tonic-gate } 19327c478bd9Sstevel@tonic-gate 19337c478bd9Sstevel@tonic-gate static struct cb_ops fasttrap_cb_ops = { 19347c478bd9Sstevel@tonic-gate fasttrap_open, /* open */ 19357c478bd9Sstevel@tonic-gate nodev, /* close */ 19367c478bd9Sstevel@tonic-gate nulldev, /* strategy */ 19377c478bd9Sstevel@tonic-gate nulldev, /* print */ 19387c478bd9Sstevel@tonic-gate nodev, /* dump */ 19397c478bd9Sstevel@tonic-gate nodev, /* read */ 19407c478bd9Sstevel@tonic-gate nodev, /* write */ 19417c478bd9Sstevel@tonic-gate fasttrap_ioctl, /* ioctl */ 19427c478bd9Sstevel@tonic-gate nodev, /* devmap */ 19437c478bd9Sstevel@tonic-gate nodev, /* mmap */ 19447c478bd9Sstevel@tonic-gate nodev, /* segmap */ 19457c478bd9Sstevel@tonic-gate nochpoll, /* poll */ 19467c478bd9Sstevel@tonic-gate ddi_prop_op, /* cb_prop_op */ 19477c478bd9Sstevel@tonic-gate 0, /* streamtab */ 19487c478bd9Sstevel@tonic-gate D_NEW | D_MP /* Driver compatibility flag */ 19497c478bd9Sstevel@tonic-gate }; 19507c478bd9Sstevel@tonic-gate 19517c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 19527c478bd9Sstevel@tonic-gate static int 19537c478bd9Sstevel@tonic-gate fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 19547c478bd9Sstevel@tonic-gate { 19557c478bd9Sstevel@tonic-gate int error; 19567c478bd9Sstevel@tonic-gate 19577c478bd9Sstevel@tonic-gate switch (infocmd) { 19587c478bd9Sstevel@tonic-gate case DDI_INFO_DEVT2DEVINFO: 19597c478bd9Sstevel@tonic-gate *result = (void *)fasttrap_devi; 19607c478bd9Sstevel@tonic-gate error = DDI_SUCCESS; 19617c478bd9Sstevel@tonic-gate break; 19627c478bd9Sstevel@tonic-gate case DDI_INFO_DEVT2INSTANCE: 19637c478bd9Sstevel@tonic-gate *result = (void *)0; 19647c478bd9Sstevel@tonic-gate error = DDI_SUCCESS; 19657c478bd9Sstevel@tonic-gate break; 19667c478bd9Sstevel@tonic-gate default: 19677c478bd9Sstevel@tonic-gate error = DDI_FAILURE; 19687c478bd9Sstevel@tonic-gate } 19697c478bd9Sstevel@tonic-gate return (error); 19707c478bd9Sstevel@tonic-gate } 19717c478bd9Sstevel@tonic-gate 19727c478bd9Sstevel@tonic-gate static int 19737c478bd9Sstevel@tonic-gate fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 19747c478bd9Sstevel@tonic-gate { 19757c478bd9Sstevel@tonic-gate ulong_t nent; 19767c478bd9Sstevel@tonic-gate 19777c478bd9Sstevel@tonic-gate switch (cmd) { 19787c478bd9Sstevel@tonic-gate case DDI_ATTACH: 19797c478bd9Sstevel@tonic-gate break; 19807c478bd9Sstevel@tonic-gate case DDI_RESUME: 19817c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 19827c478bd9Sstevel@tonic-gate default: 19837c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 19847c478bd9Sstevel@tonic-gate } 19857c478bd9Sstevel@tonic-gate 19867c478bd9Sstevel@tonic-gate if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 19877c478bd9Sstevel@tonic-gate DDI_PSEUDO, NULL) == DDI_FAILURE || 19887c478bd9Sstevel@tonic-gate dtrace_register("fasttrap", &fasttrap_attr, DTRACE_PRIV_USER, 0, 19897c478bd9Sstevel@tonic-gate &fasttrap_pops, NULL, &fasttrap_id) != 0) { 19907c478bd9Sstevel@tonic-gate ddi_remove_minor_node(devi, NULL); 19917c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 19927c478bd9Sstevel@tonic-gate } 19937c478bd9Sstevel@tonic-gate 19947c478bd9Sstevel@tonic-gate ddi_report_dev(devi); 19957c478bd9Sstevel@tonic-gate fasttrap_devi = devi; 19967c478bd9Sstevel@tonic-gate 19977c478bd9Sstevel@tonic-gate /* 19987c478bd9Sstevel@tonic-gate * Install our hooks into fork(2), exec(2), and exit(2). 19997c478bd9Sstevel@tonic-gate */ 20007c478bd9Sstevel@tonic-gate dtrace_fasttrap_fork_ptr = &fasttrap_fork; 20017c478bd9Sstevel@tonic-gate dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 20027c478bd9Sstevel@tonic-gate dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 20037c478bd9Sstevel@tonic-gate 20047c478bd9Sstevel@tonic-gate fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 20057c478bd9Sstevel@tonic-gate "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 20067c478bd9Sstevel@tonic-gate fasttrap_total = 0; 20077c478bd9Sstevel@tonic-gate 20087c478bd9Sstevel@tonic-gate /* 20097c478bd9Sstevel@tonic-gate * Conjure up the tracepoints hashtable... 20107c478bd9Sstevel@tonic-gate */ 20117c478bd9Sstevel@tonic-gate nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 20127c478bd9Sstevel@tonic-gate "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 20137c478bd9Sstevel@tonic-gate 20147c478bd9Sstevel@tonic-gate if (nent <= 0 || nent > 0x1000000) 20157c478bd9Sstevel@tonic-gate nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 20167c478bd9Sstevel@tonic-gate 20177c478bd9Sstevel@tonic-gate if ((nent & (nent - 1)) == 0) 20187c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent = nent; 20197c478bd9Sstevel@tonic-gate else 20207c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 20217c478bd9Sstevel@tonic-gate ASSERT(fasttrap_tpoints.fth_nent > 0); 20227c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 20237c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 20247c478bd9Sstevel@tonic-gate sizeof (fasttrap_bucket_t), KM_SLEEP); 20257c478bd9Sstevel@tonic-gate 20267c478bd9Sstevel@tonic-gate /* 2027b096140dSahl * ... and the providers hash table... 20287c478bd9Sstevel@tonic-gate */ 20297c478bd9Sstevel@tonic-gate nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 20307c478bd9Sstevel@tonic-gate if ((nent & (nent - 1)) == 0) 20317c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent = nent; 20327c478bd9Sstevel@tonic-gate else 20337c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 20347c478bd9Sstevel@tonic-gate ASSERT(fasttrap_provs.fth_nent > 0); 20357c478bd9Sstevel@tonic-gate fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 20367c478bd9Sstevel@tonic-gate fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 20377c478bd9Sstevel@tonic-gate sizeof (fasttrap_bucket_t), KM_SLEEP); 20387c478bd9Sstevel@tonic-gate 2039b096140dSahl /* 2040b096140dSahl * ... and the procs hash table. 2041b096140dSahl */ 2042b096140dSahl nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2043b096140dSahl if ((nent & (nent - 1)) == 0) 2044b096140dSahl fasttrap_procs.fth_nent = nent; 2045b096140dSahl else 2046b096140dSahl fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2047b096140dSahl ASSERT(fasttrap_procs.fth_nent > 0); 2048b096140dSahl fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2049b096140dSahl fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2050b096140dSahl sizeof (fasttrap_bucket_t), KM_SLEEP); 2051b096140dSahl 20527c478bd9Sstevel@tonic-gate (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 20537c478bd9Sstevel@tonic-gate &fasttrap_meta_id); 20547c478bd9Sstevel@tonic-gate 20557c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 20567c478bd9Sstevel@tonic-gate } 20577c478bd9Sstevel@tonic-gate 20587c478bd9Sstevel@tonic-gate static int 20597c478bd9Sstevel@tonic-gate fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 20607c478bd9Sstevel@tonic-gate { 20617c478bd9Sstevel@tonic-gate int i, fail = 0; 20627c478bd9Sstevel@tonic-gate timeout_id_t tmp; 20637c478bd9Sstevel@tonic-gate 20647c478bd9Sstevel@tonic-gate switch (cmd) { 20657c478bd9Sstevel@tonic-gate case DDI_DETACH: 20667c478bd9Sstevel@tonic-gate break; 20677c478bd9Sstevel@tonic-gate case DDI_SUSPEND: 20687c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 20697c478bd9Sstevel@tonic-gate default: 20707c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 20717c478bd9Sstevel@tonic-gate } 20727c478bd9Sstevel@tonic-gate 20737c478bd9Sstevel@tonic-gate /* 20747c478bd9Sstevel@tonic-gate * Unregister the meta-provider to make sure no new fasttrap- 20757c478bd9Sstevel@tonic-gate * managed providers come along while we're trying to close up 20767c478bd9Sstevel@tonic-gate * shop. If we fail to detach, we'll need to re-register as a 20777c478bd9Sstevel@tonic-gate * meta-provider. We can fail to unregister as a meta-provider 20787c478bd9Sstevel@tonic-gate * if providers we manage still exist. 20797c478bd9Sstevel@tonic-gate */ 20807c478bd9Sstevel@tonic-gate if (fasttrap_meta_id != DTRACE_METAPROVNONE && 20817c478bd9Sstevel@tonic-gate dtrace_meta_unregister(fasttrap_meta_id) != 0) 20827c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 20837c478bd9Sstevel@tonic-gate 20847c478bd9Sstevel@tonic-gate /* 20857c478bd9Sstevel@tonic-gate * Prevent any new timeouts from running by setting fasttrap_timeout 20867c478bd9Sstevel@tonic-gate * to a non-zero value, and wait for the current timeout to complete. 20877c478bd9Sstevel@tonic-gate */ 20887c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 20897c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 0; 20907c478bd9Sstevel@tonic-gate 20917c478bd9Sstevel@tonic-gate while (fasttrap_timeout != (timeout_id_t)1) { 20927c478bd9Sstevel@tonic-gate tmp = fasttrap_timeout; 20937c478bd9Sstevel@tonic-gate fasttrap_timeout = (timeout_id_t)1; 20947c478bd9Sstevel@tonic-gate 20957c478bd9Sstevel@tonic-gate if (tmp != 0) { 20967c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 20977c478bd9Sstevel@tonic-gate (void) untimeout(tmp); 20987c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 20997c478bd9Sstevel@tonic-gate } 21007c478bd9Sstevel@tonic-gate } 21017c478bd9Sstevel@tonic-gate 21027c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 0; 21037c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 21047c478bd9Sstevel@tonic-gate 21057c478bd9Sstevel@tonic-gate /* 21067c478bd9Sstevel@tonic-gate * Iterate over all of our providers. If there's still a process 21077c478bd9Sstevel@tonic-gate * that corresponds to that pid, fail to detach. 21087c478bd9Sstevel@tonic-gate */ 21097c478bd9Sstevel@tonic-gate for (i = 0; i < fasttrap_provs.fth_nent; i++) { 21107c478bd9Sstevel@tonic-gate fasttrap_provider_t **fpp, *fp; 21117c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 21127c478bd9Sstevel@tonic-gate 21137c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 21147c478bd9Sstevel@tonic-gate fpp = (fasttrap_provider_t **)&bucket->ftb_data; 21157c478bd9Sstevel@tonic-gate while ((fp = *fpp) != NULL) { 21167c478bd9Sstevel@tonic-gate /* 21177c478bd9Sstevel@tonic-gate * Acquire and release the lock as a simple way of 21187c478bd9Sstevel@tonic-gate * waiting for any other consumer to finish with 21197c478bd9Sstevel@tonic-gate * this provider. A thread must first acquire the 21207c478bd9Sstevel@tonic-gate * bucket lock so there's no chance of another thread 21217c478bd9Sstevel@tonic-gate * blocking on the providers lock. 21227c478bd9Sstevel@tonic-gate */ 21237c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 21247c478bd9Sstevel@tonic-gate mutex_exit(&fp->ftp_mtx); 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate if (dtrace_unregister(fp->ftp_provid) != 0) { 21277c478bd9Sstevel@tonic-gate fail = 1; 21287c478bd9Sstevel@tonic-gate fpp = &fp->ftp_next; 21297c478bd9Sstevel@tonic-gate } else { 21307c478bd9Sstevel@tonic-gate *fpp = fp->ftp_next; 21317c478bd9Sstevel@tonic-gate fasttrap_provider_free(fp); 21327c478bd9Sstevel@tonic-gate } 21337c478bd9Sstevel@tonic-gate } 21347c478bd9Sstevel@tonic-gate 21357c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 21367c478bd9Sstevel@tonic-gate } 21377c478bd9Sstevel@tonic-gate 21387c478bd9Sstevel@tonic-gate if (fail || dtrace_unregister(fasttrap_id) != 0) { 21397c478bd9Sstevel@tonic-gate uint_t work; 21407c478bd9Sstevel@tonic-gate /* 21417c478bd9Sstevel@tonic-gate * If we're failing to detach, we need to unblock timeouts 21427c478bd9Sstevel@tonic-gate * and start a new timeout if any work has accumulated while 21437c478bd9Sstevel@tonic-gate * we've been unsuccessfully trying to detach. 21447c478bd9Sstevel@tonic-gate */ 21457c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 21467c478bd9Sstevel@tonic-gate fasttrap_timeout = 0; 21477c478bd9Sstevel@tonic-gate work = fasttrap_cleanup_work; 21487c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 21497c478bd9Sstevel@tonic-gate 21507c478bd9Sstevel@tonic-gate if (work) 21517c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 21527c478bd9Sstevel@tonic-gate 21537c478bd9Sstevel@tonic-gate (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 21547c478bd9Sstevel@tonic-gate &fasttrap_meta_id); 21557c478bd9Sstevel@tonic-gate 21567c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 21577c478bd9Sstevel@tonic-gate } 21587c478bd9Sstevel@tonic-gate 21597c478bd9Sstevel@tonic-gate #ifdef DEBUG 21607c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_count_mtx); 21617c478bd9Sstevel@tonic-gate ASSERT(fasttrap_count == 0); 21627c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_count_mtx); 21637c478bd9Sstevel@tonic-gate #endif 21647c478bd9Sstevel@tonic-gate 21657c478bd9Sstevel@tonic-gate kmem_free(fasttrap_tpoints.fth_table, 21667c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 21677c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent = 0; 21687c478bd9Sstevel@tonic-gate 21697c478bd9Sstevel@tonic-gate kmem_free(fasttrap_provs.fth_table, 21707c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 21717c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent = 0; 21727c478bd9Sstevel@tonic-gate 2173b096140dSahl kmem_free(fasttrap_procs.fth_table, 2174b096140dSahl fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2175b096140dSahl fasttrap_procs.fth_nent = 0; 2176b096140dSahl 21777c478bd9Sstevel@tonic-gate /* 21787c478bd9Sstevel@tonic-gate * We know there are no tracepoints in any process anywhere in 21797c478bd9Sstevel@tonic-gate * the system so there is no process which has its p_dtrace_count 21807c478bd9Sstevel@tonic-gate * greater than zero, therefore we know that no thread can actively 21817c478bd9Sstevel@tonic-gate * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 21827c478bd9Sstevel@tonic-gate * and fasttrap_exec() and fasttrap_exit(). 21837c478bd9Sstevel@tonic-gate */ 21847c478bd9Sstevel@tonic-gate ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 21857c478bd9Sstevel@tonic-gate dtrace_fasttrap_fork_ptr = NULL; 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 21887c478bd9Sstevel@tonic-gate dtrace_fasttrap_exec_ptr = NULL; 21897c478bd9Sstevel@tonic-gate 21907c478bd9Sstevel@tonic-gate ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 21917c478bd9Sstevel@tonic-gate dtrace_fasttrap_exit_ptr = NULL; 21927c478bd9Sstevel@tonic-gate 21937c478bd9Sstevel@tonic-gate ddi_remove_minor_node(devi, NULL); 21947c478bd9Sstevel@tonic-gate 21957c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 21967c478bd9Sstevel@tonic-gate } 21977c478bd9Sstevel@tonic-gate 21987c478bd9Sstevel@tonic-gate static struct dev_ops fasttrap_ops = { 21997c478bd9Sstevel@tonic-gate DEVO_REV, /* devo_rev */ 22007c478bd9Sstevel@tonic-gate 0, /* refcnt */ 22017c478bd9Sstevel@tonic-gate fasttrap_info, /* get_dev_info */ 22027c478bd9Sstevel@tonic-gate nulldev, /* identify */ 22037c478bd9Sstevel@tonic-gate nulldev, /* probe */ 22047c478bd9Sstevel@tonic-gate fasttrap_attach, /* attach */ 22057c478bd9Sstevel@tonic-gate fasttrap_detach, /* detach */ 22067c478bd9Sstevel@tonic-gate nodev, /* reset */ 22077c478bd9Sstevel@tonic-gate &fasttrap_cb_ops, /* driver operations */ 22087c478bd9Sstevel@tonic-gate NULL, /* bus operations */ 22097c478bd9Sstevel@tonic-gate nodev /* dev power */ 22107c478bd9Sstevel@tonic-gate }; 22117c478bd9Sstevel@tonic-gate 22127c478bd9Sstevel@tonic-gate /* 22137c478bd9Sstevel@tonic-gate * Module linkage information for the kernel. 22147c478bd9Sstevel@tonic-gate */ 22157c478bd9Sstevel@tonic-gate static struct modldrv modldrv = { 22167c478bd9Sstevel@tonic-gate &mod_driverops, /* module type (this is a pseudo driver) */ 22177c478bd9Sstevel@tonic-gate "Fasttrap Tracing", /* name of module */ 22187c478bd9Sstevel@tonic-gate &fasttrap_ops, /* driver ops */ 22197c478bd9Sstevel@tonic-gate }; 22207c478bd9Sstevel@tonic-gate 22217c478bd9Sstevel@tonic-gate static struct modlinkage modlinkage = { 22227c478bd9Sstevel@tonic-gate MODREV_1, 22237c478bd9Sstevel@tonic-gate (void *)&modldrv, 22247c478bd9Sstevel@tonic-gate NULL 22257c478bd9Sstevel@tonic-gate }; 22267c478bd9Sstevel@tonic-gate 22277c478bd9Sstevel@tonic-gate int 22287c478bd9Sstevel@tonic-gate _init(void) 22297c478bd9Sstevel@tonic-gate { 22307c478bd9Sstevel@tonic-gate return (mod_install(&modlinkage)); 22317c478bd9Sstevel@tonic-gate } 22327c478bd9Sstevel@tonic-gate 22337c478bd9Sstevel@tonic-gate int 22347c478bd9Sstevel@tonic-gate _info(struct modinfo *modinfop) 22357c478bd9Sstevel@tonic-gate { 22367c478bd9Sstevel@tonic-gate return (mod_info(&modlinkage, modinfop)); 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate 22397c478bd9Sstevel@tonic-gate int 22407c478bd9Sstevel@tonic-gate _fini(void) 22417c478bd9Sstevel@tonic-gate { 22427c478bd9Sstevel@tonic-gate return (mod_remove(&modlinkage)); 22437c478bd9Sstevel@tonic-gate } 2244