17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ad4023c4Sdp * Common Development and Distribution License (the "License"). 6ad4023c4Sdp * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 2197eda132Sraf 227c478bd9Sstevel@tonic-gate /* 23ad4023c4Sdp * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 307c478bd9Sstevel@tonic-gate #include <sys/errno.h> 317c478bd9Sstevel@tonic-gate #include <sys/stat.h> 327c478bd9Sstevel@tonic-gate #include <sys/modctl.h> 337c478bd9Sstevel@tonic-gate #include <sys/conf.h> 347c478bd9Sstevel@tonic-gate #include <sys/systm.h> 357c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 367c478bd9Sstevel@tonic-gate #include <sys/sunddi.h> 377c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 387c478bd9Sstevel@tonic-gate #include <sys/kmem.h> 397c478bd9Sstevel@tonic-gate #include <sys/strsubr.h> 407c478bd9Sstevel@tonic-gate #include <sys/fasttrap.h> 417c478bd9Sstevel@tonic-gate #include <sys/fasttrap_impl.h> 427c478bd9Sstevel@tonic-gate #include <sys/fasttrap_isa.h> 437c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 447c478bd9Sstevel@tonic-gate #include <sys/dtrace_impl.h> 457c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 467c478bd9Sstevel@tonic-gate #include <sys/frame.h> 477c478bd9Sstevel@tonic-gate #include <sys/stack.h> 487c478bd9Sstevel@tonic-gate #include <sys/proc.h> 497c478bd9Sstevel@tonic-gate #include <sys/priv.h> 507c478bd9Sstevel@tonic-gate #include <sys/policy.h> 517c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 527c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 537c478bd9Sstevel@tonic-gate #include <sys/prsystm.h> 547c478bd9Sstevel@tonic-gate 557c478bd9Sstevel@tonic-gate #include <vm/as.h> 567c478bd9Sstevel@tonic-gate #include <vm/seg.h> 577c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h> 587c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h> 597c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h> 607c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h> 617c478bd9Sstevel@tonic-gate 627c478bd9Sstevel@tonic-gate /* 637c478bd9Sstevel@tonic-gate * User-Land Trap-Based Tracing 647c478bd9Sstevel@tonic-gate * ---------------------------- 657c478bd9Sstevel@tonic-gate * 667c478bd9Sstevel@tonic-gate * The fasttrap provider allows DTrace consumers to instrument any user-level 677c478bd9Sstevel@tonic-gate * instruction to gather data; this includes probes with semantic 687c478bd9Sstevel@tonic-gate * signifigance like entry and return as well as simple offsets into the 697c478bd9Sstevel@tonic-gate * function. While the specific techniques used are very ISA specific, the 707c478bd9Sstevel@tonic-gate * methodology is generalizable to any architecture. 717c478bd9Sstevel@tonic-gate * 727c478bd9Sstevel@tonic-gate * 737c478bd9Sstevel@tonic-gate * The General Methodology 747c478bd9Sstevel@tonic-gate * ----------------------- 757c478bd9Sstevel@tonic-gate * 767c478bd9Sstevel@tonic-gate * With the primary goal of tracing every user-land instruction and the 777c478bd9Sstevel@tonic-gate * limitation that we can't trust user space so don't want to rely on much 787c478bd9Sstevel@tonic-gate * information there, we begin by replacing the instructions we want to trace 797c478bd9Sstevel@tonic-gate * with trap instructions. Each instruction we overwrite is saved into a hash 807c478bd9Sstevel@tonic-gate * table keyed by process ID and pc address. When we enter the kernel due to 817c478bd9Sstevel@tonic-gate * this trap instruction, we need the effects of the replaced instruction to 827c478bd9Sstevel@tonic-gate * appear to have occurred before we proceed with the user thread's 837c478bd9Sstevel@tonic-gate * execution. 847c478bd9Sstevel@tonic-gate * 857c478bd9Sstevel@tonic-gate * Each user level thread is represented by a ulwp_t structure which is 867c478bd9Sstevel@tonic-gate * always easily accessible through a register. The most basic way to produce 877c478bd9Sstevel@tonic-gate * the effects of the instruction we replaced is to copy that instruction out 887c478bd9Sstevel@tonic-gate * to a bit of scratch space reserved in the user thread's ulwp_t structure 897c478bd9Sstevel@tonic-gate * (a sort of kernel-private thread local storage), set the PC to that 907c478bd9Sstevel@tonic-gate * scratch space and single step. When we reenter the kernel after single 917c478bd9Sstevel@tonic-gate * stepping the instruction we must then adjust the PC to point to what would 927c478bd9Sstevel@tonic-gate * normally be the next instruction. Of course, special care must be taken 937c478bd9Sstevel@tonic-gate * for branches and jumps, but these represent such a small fraction of any 947c478bd9Sstevel@tonic-gate * instruction set that writing the code to emulate these in the kernel is 957c478bd9Sstevel@tonic-gate * not too difficult. 967c478bd9Sstevel@tonic-gate * 977c478bd9Sstevel@tonic-gate * Return probes may require several tracepoints to trace every return site, 987c478bd9Sstevel@tonic-gate * and, conversely, each tracepoint may activate several probes (the entry 997c478bd9Sstevel@tonic-gate * and offset 0 probes, for example). To solve this muliplexing problem, 1007c478bd9Sstevel@tonic-gate * tracepoints contain lists of probes to activate and probes contain lists 1017c478bd9Sstevel@tonic-gate * of tracepoints to enable. If a probe is activated, it adds its ID to 1027c478bd9Sstevel@tonic-gate * existing tracepoints or creates new ones as necessary. 1037c478bd9Sstevel@tonic-gate * 1047c478bd9Sstevel@tonic-gate * Most probes are activated _before_ the instruction is executed, but return 1057c478bd9Sstevel@tonic-gate * probes are activated _after_ the effects of the last instruction of the 1067c478bd9Sstevel@tonic-gate * function are visible. Return probes must be fired _after_ we have 1077c478bd9Sstevel@tonic-gate * single-stepped the instruction whereas all other probes are fired 1087c478bd9Sstevel@tonic-gate * beforehand. 1097c478bd9Sstevel@tonic-gate */ 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate static dev_info_t *fasttrap_devi; 1127c478bd9Sstevel@tonic-gate static dtrace_provider_id_t fasttrap_id; 1137c478bd9Sstevel@tonic-gate static dtrace_meta_provider_id_t fasttrap_meta_id; 1147c478bd9Sstevel@tonic-gate 1157c478bd9Sstevel@tonic-gate static timeout_id_t fasttrap_timeout; 1167c478bd9Sstevel@tonic-gate static kmutex_t fasttrap_cleanup_mtx; 1177c478bd9Sstevel@tonic-gate static uint_t fasttrap_cleanup_work; 1187c478bd9Sstevel@tonic-gate 1197c478bd9Sstevel@tonic-gate /* 1207c478bd9Sstevel@tonic-gate * Generation count on modifications to the global tracepoint lookup table. 1217c478bd9Sstevel@tonic-gate */ 1227c478bd9Sstevel@tonic-gate static volatile uint64_t fasttrap_mod_gen; 1237c478bd9Sstevel@tonic-gate 1247c478bd9Sstevel@tonic-gate /* 1257c478bd9Sstevel@tonic-gate * When the fasttrap provider is loaded, fasttrap_max is set to either 1267c478bd9Sstevel@tonic-gate * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 1277c478bd9Sstevel@tonic-gate * fasttrap.conf file. Each time a probe is created, fasttrap_total is 1287c478bd9Sstevel@tonic-gate * incremented by the number of tracepoints that may be associated with that 1297c478bd9Sstevel@tonic-gate * probe; fasttrap_total is capped at fasttrap_max. 1307c478bd9Sstevel@tonic-gate */ 1317c478bd9Sstevel@tonic-gate #define FASTTRAP_MAX_DEFAULT 250000 1327c478bd9Sstevel@tonic-gate static uint32_t fasttrap_max; 1337c478bd9Sstevel@tonic-gate static uint32_t fasttrap_total; 1347c478bd9Sstevel@tonic-gate 1357c478bd9Sstevel@tonic-gate 1367c478bd9Sstevel@tonic-gate #define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 1377c478bd9Sstevel@tonic-gate #define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 138b096140dSahl #define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 1397c478bd9Sstevel@tonic-gate 1407c478bd9Sstevel@tonic-gate #define FASTTRAP_PID_NAME "pid" 1417c478bd9Sstevel@tonic-gate 1427c478bd9Sstevel@tonic-gate fasttrap_hash_t fasttrap_tpoints; 1437c478bd9Sstevel@tonic-gate static fasttrap_hash_t fasttrap_provs; 144b096140dSahl static fasttrap_hash_t fasttrap_procs; 1457c478bd9Sstevel@tonic-gate 1467c478bd9Sstevel@tonic-gate dtrace_id_t fasttrap_probe_id; 1477c478bd9Sstevel@tonic-gate static int fasttrap_count; /* ref count */ 1487c478bd9Sstevel@tonic-gate static int fasttrap_pid_count; /* pid ref count */ 1497c478bd9Sstevel@tonic-gate static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 1507c478bd9Sstevel@tonic-gate 1518f5db3e7Sahl #define FASTTRAP_ENABLE_FAIL 1 1528f5db3e7Sahl #define FASTTRAP_ENABLE_PARTIAL 2 1538f5db3e7Sahl 1547c478bd9Sstevel@tonic-gate static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 1557c478bd9Sstevel@tonic-gate static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 1567c478bd9Sstevel@tonic-gate 1577c478bd9Sstevel@tonic-gate static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 1587c478bd9Sstevel@tonic-gate const dtrace_pattr_t *); 159dafb5540Sahl static void fasttrap_provider_retire(pid_t, const char *, int); 1607c478bd9Sstevel@tonic-gate static void fasttrap_provider_free(fasttrap_provider_t *); 1617c478bd9Sstevel@tonic-gate 162b096140dSahl static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 163b096140dSahl static void fasttrap_proc_release(fasttrap_proc_t *); 164b096140dSahl 1657c478bd9Sstevel@tonic-gate #define FASTTRAP_PROVS_INDEX(pid, name) \ 1667c478bd9Sstevel@tonic-gate ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 1677c478bd9Sstevel@tonic-gate 168b096140dSahl #define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 169b096140dSahl 1707c478bd9Sstevel@tonic-gate static int 1717c478bd9Sstevel@tonic-gate fasttrap_highbit(ulong_t i) 1727c478bd9Sstevel@tonic-gate { 1737c478bd9Sstevel@tonic-gate int h = 1; 1747c478bd9Sstevel@tonic-gate 1757c478bd9Sstevel@tonic-gate if (i == 0) 1767c478bd9Sstevel@tonic-gate return (0); 1777c478bd9Sstevel@tonic-gate #ifdef _LP64 1787c478bd9Sstevel@tonic-gate if (i & 0xffffffff00000000ul) { 1797c478bd9Sstevel@tonic-gate h += 32; i >>= 32; 1807c478bd9Sstevel@tonic-gate } 1817c478bd9Sstevel@tonic-gate #endif 1827c478bd9Sstevel@tonic-gate if (i & 0xffff0000) { 1837c478bd9Sstevel@tonic-gate h += 16; i >>= 16; 1847c478bd9Sstevel@tonic-gate } 1857c478bd9Sstevel@tonic-gate if (i & 0xff00) { 1867c478bd9Sstevel@tonic-gate h += 8; i >>= 8; 1877c478bd9Sstevel@tonic-gate } 1887c478bd9Sstevel@tonic-gate if (i & 0xf0) { 1897c478bd9Sstevel@tonic-gate h += 4; i >>= 4; 1907c478bd9Sstevel@tonic-gate } 1917c478bd9Sstevel@tonic-gate if (i & 0xc) { 1927c478bd9Sstevel@tonic-gate h += 2; i >>= 2; 1937c478bd9Sstevel@tonic-gate } 1947c478bd9Sstevel@tonic-gate if (i & 0x2) { 1957c478bd9Sstevel@tonic-gate h += 1; 1967c478bd9Sstevel@tonic-gate } 1977c478bd9Sstevel@tonic-gate return (h); 1987c478bd9Sstevel@tonic-gate } 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate static uint_t 2017c478bd9Sstevel@tonic-gate fasttrap_hash_str(const char *p) 2027c478bd9Sstevel@tonic-gate { 2037c478bd9Sstevel@tonic-gate unsigned int g; 2047c478bd9Sstevel@tonic-gate uint_t hval = 0; 2057c478bd9Sstevel@tonic-gate 2067c478bd9Sstevel@tonic-gate while (*p) { 2077c478bd9Sstevel@tonic-gate hval = (hval << 4) + *p++; 2087c478bd9Sstevel@tonic-gate if ((g = (hval & 0xf0000000)) != 0) 2097c478bd9Sstevel@tonic-gate hval ^= g >> 24; 2107c478bd9Sstevel@tonic-gate hval &= ~g; 2117c478bd9Sstevel@tonic-gate } 2127c478bd9Sstevel@tonic-gate return (hval); 2137c478bd9Sstevel@tonic-gate } 2147c478bd9Sstevel@tonic-gate 2157c478bd9Sstevel@tonic-gate void 2167c478bd9Sstevel@tonic-gate fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 2177c478bd9Sstevel@tonic-gate { 2187c478bd9Sstevel@tonic-gate sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 2197c478bd9Sstevel@tonic-gate 2207c478bd9Sstevel@tonic-gate sqp->sq_info.si_signo = SIGTRAP; 2217c478bd9Sstevel@tonic-gate sqp->sq_info.si_code = TRAP_DTRACE; 2227c478bd9Sstevel@tonic-gate sqp->sq_info.si_addr = (caddr_t)pc; 2237c478bd9Sstevel@tonic-gate 2247c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 2257c478bd9Sstevel@tonic-gate sigaddqa(p, t, sqp); 2267c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 2277c478bd9Sstevel@tonic-gate 2287c478bd9Sstevel@tonic-gate if (t != NULL) 2297c478bd9Sstevel@tonic-gate aston(t); 2307c478bd9Sstevel@tonic-gate } 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate /* 2337c478bd9Sstevel@tonic-gate * This function ensures that no threads are actively using the memory 2347c478bd9Sstevel@tonic-gate * associated with probes that were formerly live. 2357c478bd9Sstevel@tonic-gate */ 2367c478bd9Sstevel@tonic-gate static void 2377c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(uint64_t gen) 2387c478bd9Sstevel@tonic-gate { 2397c478bd9Sstevel@tonic-gate int i; 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate if (gen < fasttrap_mod_gen) 2427c478bd9Sstevel@tonic-gate return; 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate fasttrap_mod_gen++; 2457c478bd9Sstevel@tonic-gate 2467c478bd9Sstevel@tonic-gate for (i = 0; i < NCPU; i++) { 2477c478bd9Sstevel@tonic-gate mutex_enter(&cpu_core[i].cpuc_pid_lock); 2487c478bd9Sstevel@tonic-gate mutex_exit(&cpu_core[i].cpuc_pid_lock); 2497c478bd9Sstevel@tonic-gate } 2507c478bd9Sstevel@tonic-gate } 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate /* 2537c478bd9Sstevel@tonic-gate * This is the timeout's callback for cleaning up the providers and their 2547c478bd9Sstevel@tonic-gate * probes. 2557c478bd9Sstevel@tonic-gate */ 2567c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 2577c478bd9Sstevel@tonic-gate static void 2587c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup_cb(void *data) 2597c478bd9Sstevel@tonic-gate { 2607c478bd9Sstevel@tonic-gate fasttrap_provider_t **fpp, *fp; 2617c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 2627c478bd9Sstevel@tonic-gate dtrace_provider_id_t provid; 2637c478bd9Sstevel@tonic-gate int i, later; 2647c478bd9Sstevel@tonic-gate 2657c478bd9Sstevel@tonic-gate static volatile int in = 0; 2667c478bd9Sstevel@tonic-gate ASSERT(in == 0); 2677c478bd9Sstevel@tonic-gate in = 1; 2687c478bd9Sstevel@tonic-gate 2697c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 2707c478bd9Sstevel@tonic-gate while (fasttrap_cleanup_work) { 2717c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 0; 2727c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 2737c478bd9Sstevel@tonic-gate 2747c478bd9Sstevel@tonic-gate later = 0; 2757c478bd9Sstevel@tonic-gate 2767c478bd9Sstevel@tonic-gate /* 2777c478bd9Sstevel@tonic-gate * Iterate over all the providers trying to remove the marked 278b096140dSahl * ones. If a provider is marked but not retired, we just 2797c478bd9Sstevel@tonic-gate * have to take a crack at removing it -- it's no big deal if 2807c478bd9Sstevel@tonic-gate * we can't. 2817c478bd9Sstevel@tonic-gate */ 2827c478bd9Sstevel@tonic-gate for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2837c478bd9Sstevel@tonic-gate bucket = &fasttrap_provs.fth_table[i]; 2847c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 2857c478bd9Sstevel@tonic-gate fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2867c478bd9Sstevel@tonic-gate 2877c478bd9Sstevel@tonic-gate while ((fp = *fpp) != NULL) { 2887c478bd9Sstevel@tonic-gate if (!fp->ftp_marked) { 2897c478bd9Sstevel@tonic-gate fpp = &fp->ftp_next; 2907c478bd9Sstevel@tonic-gate continue; 2917c478bd9Sstevel@tonic-gate } 2927c478bd9Sstevel@tonic-gate 2937c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 2947c478bd9Sstevel@tonic-gate 2957c478bd9Sstevel@tonic-gate /* 2967c478bd9Sstevel@tonic-gate * If this provider is referenced either 2977c478bd9Sstevel@tonic-gate * because it is a USDT provider or is being 2987c478bd9Sstevel@tonic-gate * modified, we can't unregister or even 2997c478bd9Sstevel@tonic-gate * condense. 3007c478bd9Sstevel@tonic-gate */ 301*ab9a77c7Sahl if (fp->ftp_ccount != 0 || 302*ab9a77c7Sahl fp->ftp_mcount != 0) { 3037c478bd9Sstevel@tonic-gate mutex_exit(&fp->ftp_mtx); 3047c478bd9Sstevel@tonic-gate fp->ftp_marked = 0; 3057c478bd9Sstevel@tonic-gate continue; 3067c478bd9Sstevel@tonic-gate } 3077c478bd9Sstevel@tonic-gate 308b096140dSahl if (!fp->ftp_retired || fp->ftp_rcount != 0) 3097c478bd9Sstevel@tonic-gate fp->ftp_marked = 0; 3107c478bd9Sstevel@tonic-gate 3117c478bd9Sstevel@tonic-gate mutex_exit(&fp->ftp_mtx); 3127c478bd9Sstevel@tonic-gate 3137c478bd9Sstevel@tonic-gate /* 3147c478bd9Sstevel@tonic-gate * If we successfully unregister this 3157c478bd9Sstevel@tonic-gate * provider we can remove it from the hash 3167c478bd9Sstevel@tonic-gate * chain and free the memory. If our attempt 317b096140dSahl * to unregister fails and this is a retired 3187c478bd9Sstevel@tonic-gate * provider, increment our flag to try again 3197c478bd9Sstevel@tonic-gate * pretty soon. If we've consumed more than 3207c478bd9Sstevel@tonic-gate * half of our total permitted number of 3217c478bd9Sstevel@tonic-gate * probes call dtrace_condense() to try to 3227c478bd9Sstevel@tonic-gate * clean out the unenabled probes. 3237c478bd9Sstevel@tonic-gate */ 3247c478bd9Sstevel@tonic-gate provid = fp->ftp_provid; 3257c478bd9Sstevel@tonic-gate if (dtrace_unregister(provid) != 0) { 3267c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max / 2) 3277c478bd9Sstevel@tonic-gate (void) dtrace_condense(provid); 3287c478bd9Sstevel@tonic-gate later += fp->ftp_marked; 3297c478bd9Sstevel@tonic-gate fpp = &fp->ftp_next; 3307c478bd9Sstevel@tonic-gate } else { 3317c478bd9Sstevel@tonic-gate *fpp = fp->ftp_next; 3327c478bd9Sstevel@tonic-gate fasttrap_provider_free(fp); 3337c478bd9Sstevel@tonic-gate } 3347c478bd9Sstevel@tonic-gate } 3357c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 3367c478bd9Sstevel@tonic-gate } 3377c478bd9Sstevel@tonic-gate 3387c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 3397c478bd9Sstevel@tonic-gate } 3407c478bd9Sstevel@tonic-gate 3417c478bd9Sstevel@tonic-gate ASSERT(fasttrap_timeout != 0); 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate /* 344b096140dSahl * If we were unable to remove a retired provider, try again after 3457c478bd9Sstevel@tonic-gate * a second. This situation can occur in certain circumstances where 3467c478bd9Sstevel@tonic-gate * providers cannot be unregistered even though they have no probes 3477c478bd9Sstevel@tonic-gate * enabled because of an execution of dtrace -l or something similar. 3487c478bd9Sstevel@tonic-gate * If the timeout has been disabled (set to 1 because we're trying 3497c478bd9Sstevel@tonic-gate * to detach), we set fasttrap_cleanup_work to ensure that we'll 3507c478bd9Sstevel@tonic-gate * get a chance to do that work if and when the timeout is reenabled 3517c478bd9Sstevel@tonic-gate * (if detach fails). 3527c478bd9Sstevel@tonic-gate */ 3537c478bd9Sstevel@tonic-gate if (later > 0 && fasttrap_timeout != (timeout_id_t)1) 3547c478bd9Sstevel@tonic-gate fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 3557c478bd9Sstevel@tonic-gate else if (later > 0) 3567c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 1; 3577c478bd9Sstevel@tonic-gate else 3587c478bd9Sstevel@tonic-gate fasttrap_timeout = 0; 3597c478bd9Sstevel@tonic-gate 3607c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 3617c478bd9Sstevel@tonic-gate in = 0; 3627c478bd9Sstevel@tonic-gate } 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate /* 3657c478bd9Sstevel@tonic-gate * Activates the asynchronous cleanup mechanism. 3667c478bd9Sstevel@tonic-gate */ 3677c478bd9Sstevel@tonic-gate static void 3687c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(void) 3697c478bd9Sstevel@tonic-gate { 3707c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 3717c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 1; 3727c478bd9Sstevel@tonic-gate if (fasttrap_timeout == 0) 3737c478bd9Sstevel@tonic-gate fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 3747c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 3757c478bd9Sstevel@tonic-gate } 3767c478bd9Sstevel@tonic-gate 3777c478bd9Sstevel@tonic-gate /* 3787c478bd9Sstevel@tonic-gate * This is called from cfork() via dtrace_fasttrap_fork(). The child 3797c478bd9Sstevel@tonic-gate * process's address space is a (roughly) a copy of the parent process's so 3807c478bd9Sstevel@tonic-gate * we have to remove all the instrumentation we had previously enabled in the 3817c478bd9Sstevel@tonic-gate * parent. 3827c478bd9Sstevel@tonic-gate */ 3837c478bd9Sstevel@tonic-gate static void 3847c478bd9Sstevel@tonic-gate fasttrap_fork(proc_t *p, proc_t *cp) 3857c478bd9Sstevel@tonic-gate { 3867c478bd9Sstevel@tonic-gate pid_t ppid = p->p_pid; 3877c478bd9Sstevel@tonic-gate int i; 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate ASSERT(curproc == p); 3907c478bd9Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK); 3917c478bd9Sstevel@tonic-gate ASSERT(p->p_dtrace_count > 0); 3927c478bd9Sstevel@tonic-gate ASSERT(cp->p_dtrace_count == 0); 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate /* 3957c478bd9Sstevel@tonic-gate * This would be simpler and faster if we maintained per-process 3967c478bd9Sstevel@tonic-gate * hash tables of enabled tracepoints. It could, however, potentially 3977c478bd9Sstevel@tonic-gate * slow down execution of a tracepoint since we'd need to go 3987c478bd9Sstevel@tonic-gate * through two levels of indirection. In the future, we should 3997c478bd9Sstevel@tonic-gate * consider either maintaining per-process ancillary lists of 4007c478bd9Sstevel@tonic-gate * enabled tracepoints or hanging a pointer to a per-process hash 4017c478bd9Sstevel@tonic-gate * table of enabled tracepoints off the proc structure. 4027c478bd9Sstevel@tonic-gate */ 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate /* 4057c478bd9Sstevel@tonic-gate * We don't have to worry about the child process disappearing 4067c478bd9Sstevel@tonic-gate * because we're in fork(). 4077c478bd9Sstevel@tonic-gate */ 4087c478bd9Sstevel@tonic-gate mutex_enter(&cp->p_lock); 4097c478bd9Sstevel@tonic-gate sprlock_proc(cp); 4107c478bd9Sstevel@tonic-gate mutex_exit(&cp->p_lock); 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate /* 4137c478bd9Sstevel@tonic-gate * Iterate over every tracepoint looking for ones that belong to the 4147c478bd9Sstevel@tonic-gate * parent process, and remove each from the child process. 4157c478bd9Sstevel@tonic-gate */ 4167c478bd9Sstevel@tonic-gate for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 4177c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 4187c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 4197c478bd9Sstevel@tonic-gate 4207c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 4217c478bd9Sstevel@tonic-gate for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 422b096140dSahl if (tp->ftt_pid == ppid && 423b096140dSahl !tp->ftt_proc->ftpc_defunct) { 4247c478bd9Sstevel@tonic-gate int ret = fasttrap_tracepoint_remove(cp, tp); 4257c478bd9Sstevel@tonic-gate ASSERT(ret == 0); 4267c478bd9Sstevel@tonic-gate } 4277c478bd9Sstevel@tonic-gate } 4287c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 4297c478bd9Sstevel@tonic-gate } 4307c478bd9Sstevel@tonic-gate 4317c478bd9Sstevel@tonic-gate mutex_enter(&cp->p_lock); 4327c478bd9Sstevel@tonic-gate sprunlock(cp); 4337c478bd9Sstevel@tonic-gate } 4347c478bd9Sstevel@tonic-gate 4357c478bd9Sstevel@tonic-gate /* 4367c478bd9Sstevel@tonic-gate * This is called from proc_exit() or from exec_common() if p_dtrace_probes 4377c478bd9Sstevel@tonic-gate * is set on the proc structure to indicate that there is a pid provider 4387c478bd9Sstevel@tonic-gate * associated with this process. 4397c478bd9Sstevel@tonic-gate */ 4407c478bd9Sstevel@tonic-gate static void 4417c478bd9Sstevel@tonic-gate fasttrap_exec_exit(proc_t *p) 4427c478bd9Sstevel@tonic-gate { 4437c478bd9Sstevel@tonic-gate ASSERT(p == curproc); 4447c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&p->p_lock)); 4457c478bd9Sstevel@tonic-gate 4467c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate /* 4497c478bd9Sstevel@tonic-gate * We clean up the pid provider for this process here; user-land 4507c478bd9Sstevel@tonic-gate * static probes are handled by the meta-provider remove entry point. 4517c478bd9Sstevel@tonic-gate */ 452dafb5540Sahl fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 4537c478bd9Sstevel@tonic-gate 4547c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 4557c478bd9Sstevel@tonic-gate } 4567c478bd9Sstevel@tonic-gate 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4597c478bd9Sstevel@tonic-gate static void 4607c478bd9Sstevel@tonic-gate fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc) 4617c478bd9Sstevel@tonic-gate { 4627c478bd9Sstevel@tonic-gate /* 4637c478bd9Sstevel@tonic-gate * There are no "default" pid probes. 4647c478bd9Sstevel@tonic-gate */ 4657c478bd9Sstevel@tonic-gate } 4667c478bd9Sstevel@tonic-gate 4677c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 4687c478bd9Sstevel@tonic-gate static void 4697c478bd9Sstevel@tonic-gate fasttrap_provide(void *arg, const dtrace_probedesc_t *desc) 4707c478bd9Sstevel@tonic-gate { 4717c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(fasttrap_id, NULL, "fasttrap", "fasttrap") == 0) 4727c478bd9Sstevel@tonic-gate fasttrap_probe_id = dtrace_probe_create(fasttrap_id, NULL, 4737c478bd9Sstevel@tonic-gate "fasttrap", "fasttrap", FASTTRAP_AFRAMES, NULL); 4747c478bd9Sstevel@tonic-gate } 4757c478bd9Sstevel@tonic-gate 4767c478bd9Sstevel@tonic-gate static int 4777c478bd9Sstevel@tonic-gate fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 4787c478bd9Sstevel@tonic-gate { 4797c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp, *new_tp = NULL; 4807c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 4817c478bd9Sstevel@tonic-gate fasttrap_id_t *id; 4827c478bd9Sstevel@tonic-gate pid_t pid; 4837c478bd9Sstevel@tonic-gate uintptr_t pc; 4847c478bd9Sstevel@tonic-gate 4857c478bd9Sstevel@tonic-gate ASSERT(index < probe->ftp_ntps); 4867c478bd9Sstevel@tonic-gate 4877c478bd9Sstevel@tonic-gate pid = probe->ftp_pid; 4887c478bd9Sstevel@tonic-gate pc = probe->ftp_tps[index].fit_tp->ftt_pc; 4897c478bd9Sstevel@tonic-gate id = &probe->ftp_tps[index].fit_id; 4907c478bd9Sstevel@tonic-gate 4917c478bd9Sstevel@tonic-gate ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 4927c478bd9Sstevel@tonic-gate 4937c478bd9Sstevel@tonic-gate ASSERT(!(p->p_flag & SVFORK)); 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate /* 4967c478bd9Sstevel@tonic-gate * Before we make any modifications, make sure we've imposed a barrier 4977c478bd9Sstevel@tonic-gate * on the generation in which this probe was last modified. 4987c478bd9Sstevel@tonic-gate */ 4997c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(probe->ftp_gen); 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate /* 5047c478bd9Sstevel@tonic-gate * If the tracepoint has already been enabled, just add our id to the 5057c478bd9Sstevel@tonic-gate * list of interested probes. This may be our second time through 5067c478bd9Sstevel@tonic-gate * this path in which case we'll have constructed the tracepoint we'd 5077c478bd9Sstevel@tonic-gate * like to install. If we can't find a match, and have an allocated 5087c478bd9Sstevel@tonic-gate * tracepoint ready to go, enable that one now. 5097c478bd9Sstevel@tonic-gate * 510ac448965Sahl * A tracepoint whose process is defunct is also considered defunct. 5117c478bd9Sstevel@tonic-gate */ 5127c478bd9Sstevel@tonic-gate again: 5137c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 5147c478bd9Sstevel@tonic-gate for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 5157c478bd9Sstevel@tonic-gate if (tp->ftt_pid != pid || tp->ftt_pc != pc || 516b096140dSahl tp->ftt_proc->ftpc_defunct) 5177c478bd9Sstevel@tonic-gate continue; 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate /* 5207c478bd9Sstevel@tonic-gate * Now that we've found a matching tracepoint, it would be 5217c478bd9Sstevel@tonic-gate * a decent idea to confirm that the tracepoint is still 5227c478bd9Sstevel@tonic-gate * enabled and the trap instruction hasn't been overwritten. 5237c478bd9Sstevel@tonic-gate * Since this is a little hairy, we'll punt for now. 5247c478bd9Sstevel@tonic-gate */ 5257c478bd9Sstevel@tonic-gate 5267c478bd9Sstevel@tonic-gate /* 5277c478bd9Sstevel@tonic-gate * This can't be the first interested probe. We don't have 5287c478bd9Sstevel@tonic-gate * to worry about another thread being in the midst of 5297c478bd9Sstevel@tonic-gate * deleting this tracepoint (which would be the only valid 5307c478bd9Sstevel@tonic-gate * reason for a tracepoint to have no interested probes) 5317c478bd9Sstevel@tonic-gate * since we're holding P_PR_LOCK for this process. 5327c478bd9Sstevel@tonic-gate */ 5337c478bd9Sstevel@tonic-gate ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 5347c478bd9Sstevel@tonic-gate 535ac448965Sahl switch (id->fti_ptype) { 536ac448965Sahl case DTFTP_ENTRY: 537ac448965Sahl case DTFTP_OFFSETS: 538ac448965Sahl case DTFTP_IS_ENABLED: 5397c478bd9Sstevel@tonic-gate id->fti_next = tp->ftt_ids; 5407c478bd9Sstevel@tonic-gate membar_producer(); 5417c478bd9Sstevel@tonic-gate tp->ftt_ids = id; 5427c478bd9Sstevel@tonic-gate membar_producer(); 543ac448965Sahl break; 544ac448965Sahl 545ac448965Sahl case DTFTP_RETURN: 546ac448965Sahl case DTFTP_POST_OFFSETS: 547ac448965Sahl id->fti_next = tp->ftt_retids; 548ac448965Sahl membar_producer(); 549ac448965Sahl tp->ftt_retids = id; 550ac448965Sahl membar_producer(); 551ac448965Sahl break; 552ac448965Sahl 553ac448965Sahl default: 554ac448965Sahl ASSERT(0); 5557c478bd9Sstevel@tonic-gate } 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate if (new_tp != NULL) { 5607c478bd9Sstevel@tonic-gate new_tp->ftt_ids = NULL; 5617c478bd9Sstevel@tonic-gate new_tp->ftt_retids = NULL; 5627c478bd9Sstevel@tonic-gate } 5637c478bd9Sstevel@tonic-gate 5647c478bd9Sstevel@tonic-gate return (0); 5657c478bd9Sstevel@tonic-gate } 5667c478bd9Sstevel@tonic-gate 5677c478bd9Sstevel@tonic-gate /* 5687c478bd9Sstevel@tonic-gate * If we have a good tracepoint ready to go, install it now while 5697c478bd9Sstevel@tonic-gate * we have the lock held and no one can screw with us. 5707c478bd9Sstevel@tonic-gate */ 5717c478bd9Sstevel@tonic-gate if (new_tp != NULL) { 5728f5db3e7Sahl int rc = 0; 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate new_tp->ftt_next = bucket->ftb_data; 5757c478bd9Sstevel@tonic-gate membar_producer(); 5767c478bd9Sstevel@tonic-gate bucket->ftb_data = new_tp; 5777c478bd9Sstevel@tonic-gate membar_producer(); 5787c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate /* 5818f5db3e7Sahl * Activate the tracepoint in the ISA-specific manner. 5828f5db3e7Sahl * If this fails, we need to report the failure, but 5838f5db3e7Sahl * indicate that this tracepoint must still be disabled 5848f5db3e7Sahl * by calling fasttrap_tracepoint_disable(). 5857c478bd9Sstevel@tonic-gate */ 5868f5db3e7Sahl if (fasttrap_tracepoint_install(p, new_tp) != 0) 5878f5db3e7Sahl rc = FASTTRAP_ENABLE_PARTIAL; 5888f5db3e7Sahl 5897c478bd9Sstevel@tonic-gate /* 5908f5db3e7Sahl * Increment the count of the number of tracepoints active in 5918f5db3e7Sahl * the victim process. 5927c478bd9Sstevel@tonic-gate */ 5937c478bd9Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK); 5947c478bd9Sstevel@tonic-gate p->p_dtrace_count++; 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate return (rc); 5977c478bd9Sstevel@tonic-gate } 5987c478bd9Sstevel@tonic-gate 5997c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 6007c478bd9Sstevel@tonic-gate 6017c478bd9Sstevel@tonic-gate /* 6027c478bd9Sstevel@tonic-gate * Initialize the tracepoint that's been preallocated with the probe. 6037c478bd9Sstevel@tonic-gate */ 6047c478bd9Sstevel@tonic-gate new_tp = probe->ftp_tps[index].fit_tp; 6057c478bd9Sstevel@tonic-gate 6067c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_pid == pid); 6077c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_pc == pc); 608b096140dSahl ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 6097c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_ids == NULL); 6107c478bd9Sstevel@tonic-gate ASSERT(new_tp->ftt_retids == NULL); 6117c478bd9Sstevel@tonic-gate 612ac448965Sahl switch (id->fti_ptype) { 613ac448965Sahl case DTFTP_ENTRY: 614ac448965Sahl case DTFTP_OFFSETS: 615ac448965Sahl case DTFTP_IS_ENABLED: 6167c478bd9Sstevel@tonic-gate id->fti_next = NULL; 6177c478bd9Sstevel@tonic-gate new_tp->ftt_ids = id; 618ac448965Sahl break; 619ac448965Sahl 620ac448965Sahl case DTFTP_RETURN: 621ac448965Sahl case DTFTP_POST_OFFSETS: 622ac448965Sahl id->fti_next = NULL; 623ac448965Sahl new_tp->ftt_retids = id; 624ac448965Sahl break; 625ac448965Sahl 626ac448965Sahl default: 627ac448965Sahl ASSERT(0); 6287c478bd9Sstevel@tonic-gate } 6297c478bd9Sstevel@tonic-gate 6307c478bd9Sstevel@tonic-gate /* 6318f5db3e7Sahl * If the ISA-dependent initialization goes to plan, go back to the 6327c478bd9Sstevel@tonic-gate * beginning and try to install this freshly made tracepoint. 6337c478bd9Sstevel@tonic-gate */ 634ac448965Sahl if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 6357c478bd9Sstevel@tonic-gate goto again; 6367c478bd9Sstevel@tonic-gate 6377c478bd9Sstevel@tonic-gate new_tp->ftt_ids = NULL; 6387c478bd9Sstevel@tonic-gate new_tp->ftt_retids = NULL; 6397c478bd9Sstevel@tonic-gate 6408f5db3e7Sahl return (FASTTRAP_ENABLE_FAIL); 6417c478bd9Sstevel@tonic-gate } 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate static void 6447c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 6457c478bd9Sstevel@tonic-gate { 6467c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 6477c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider = probe->ftp_prov; 6487c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t **pp, *tp; 6497c478bd9Sstevel@tonic-gate fasttrap_id_t *id, **idp; 6507c478bd9Sstevel@tonic-gate pid_t pid; 6517c478bd9Sstevel@tonic-gate uintptr_t pc; 6527c478bd9Sstevel@tonic-gate 6537c478bd9Sstevel@tonic-gate ASSERT(index < probe->ftp_ntps); 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate pid = probe->ftp_pid; 6567c478bd9Sstevel@tonic-gate pc = probe->ftp_tps[index].fit_tp->ftt_pc; 657ac448965Sahl id = &probe->ftp_tps[index].fit_id; 6587c478bd9Sstevel@tonic-gate 6597c478bd9Sstevel@tonic-gate ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate /* 6627c478bd9Sstevel@tonic-gate * Find the tracepoint and make sure that our id is one of the 6637c478bd9Sstevel@tonic-gate * ones registered with it. 6647c478bd9Sstevel@tonic-gate */ 6657c478bd9Sstevel@tonic-gate bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 6667c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 6677c478bd9Sstevel@tonic-gate for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 6687c478bd9Sstevel@tonic-gate if (tp->ftt_pid == pid && tp->ftt_pc == pc && 669b096140dSahl tp->ftt_proc == provider->ftp_proc) 6707c478bd9Sstevel@tonic-gate break; 6717c478bd9Sstevel@tonic-gate } 6727c478bd9Sstevel@tonic-gate 6737c478bd9Sstevel@tonic-gate /* 6747c478bd9Sstevel@tonic-gate * If we somehow lost this tracepoint, we're in a world of hurt. 6757c478bd9Sstevel@tonic-gate */ 6767c478bd9Sstevel@tonic-gate ASSERT(tp != NULL); 6777c478bd9Sstevel@tonic-gate 678ac448965Sahl switch (id->fti_ptype) { 679ac448965Sahl case DTFTP_ENTRY: 680ac448965Sahl case DTFTP_OFFSETS: 681ac448965Sahl case DTFTP_IS_ENABLED: 6827c478bd9Sstevel@tonic-gate ASSERT(tp->ftt_ids != NULL); 6837c478bd9Sstevel@tonic-gate idp = &tp->ftt_ids; 684ac448965Sahl break; 685ac448965Sahl 686ac448965Sahl case DTFTP_RETURN: 687ac448965Sahl case DTFTP_POST_OFFSETS: 688ac448965Sahl ASSERT(tp->ftt_retids != NULL); 689ac448965Sahl idp = &tp->ftt_retids; 690ac448965Sahl break; 691ac448965Sahl 692ac448965Sahl default: 693ac448965Sahl ASSERT(0); 6947c478bd9Sstevel@tonic-gate } 6957c478bd9Sstevel@tonic-gate 6967c478bd9Sstevel@tonic-gate while ((*idp)->fti_probe != probe) { 6977c478bd9Sstevel@tonic-gate idp = &(*idp)->fti_next; 6987c478bd9Sstevel@tonic-gate ASSERT(*idp != NULL); 6997c478bd9Sstevel@tonic-gate } 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate id = *idp; 7027c478bd9Sstevel@tonic-gate *idp = id->fti_next; 7037c478bd9Sstevel@tonic-gate membar_producer(); 7047c478bd9Sstevel@tonic-gate 7057c478bd9Sstevel@tonic-gate ASSERT(id->fti_probe == probe); 7067c478bd9Sstevel@tonic-gate 7077c478bd9Sstevel@tonic-gate /* 7087c478bd9Sstevel@tonic-gate * If there are other registered enablings of this tracepoint, we're 7097c478bd9Sstevel@tonic-gate * all done, but if this was the last probe assocated with this 7107c478bd9Sstevel@tonic-gate * this tracepoint, we need to remove and free it. 7117c478bd9Sstevel@tonic-gate */ 7127c478bd9Sstevel@tonic-gate if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 7137c478bd9Sstevel@tonic-gate 7147c478bd9Sstevel@tonic-gate /* 7157c478bd9Sstevel@tonic-gate * If the current probe's tracepoint is in use, swap it 7167c478bd9Sstevel@tonic-gate * for an unused tracepoint. 7177c478bd9Sstevel@tonic-gate */ 7187c478bd9Sstevel@tonic-gate if (tp == probe->ftp_tps[index].fit_tp) { 7197c478bd9Sstevel@tonic-gate fasttrap_probe_t *tmp_probe; 7207c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t **tmp_tp; 7217c478bd9Sstevel@tonic-gate uint_t tmp_index; 7227c478bd9Sstevel@tonic-gate 7237c478bd9Sstevel@tonic-gate if (tp->ftt_ids != NULL) { 7247c478bd9Sstevel@tonic-gate tmp_probe = tp->ftt_ids->fti_probe; 7257c478bd9Sstevel@tonic-gate tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 7267c478bd9Sstevel@tonic-gate tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 7277c478bd9Sstevel@tonic-gate } else { 7287c478bd9Sstevel@tonic-gate tmp_probe = tp->ftt_retids->fti_probe; 7297c478bd9Sstevel@tonic-gate tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 7307c478bd9Sstevel@tonic-gate tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 7317c478bd9Sstevel@tonic-gate } 7327c478bd9Sstevel@tonic-gate 7337c478bd9Sstevel@tonic-gate ASSERT(*tmp_tp != NULL); 7347c478bd9Sstevel@tonic-gate ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 7357c478bd9Sstevel@tonic-gate ASSERT((*tmp_tp)->ftt_ids == NULL); 7367c478bd9Sstevel@tonic-gate ASSERT((*tmp_tp)->ftt_retids == NULL); 7377c478bd9Sstevel@tonic-gate 7387c478bd9Sstevel@tonic-gate probe->ftp_tps[index].fit_tp = *tmp_tp; 7397c478bd9Sstevel@tonic-gate *tmp_tp = tp; 7407c478bd9Sstevel@tonic-gate 7417c478bd9Sstevel@tonic-gate } 7427c478bd9Sstevel@tonic-gate 7437c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 7447c478bd9Sstevel@tonic-gate 7457c478bd9Sstevel@tonic-gate /* 7467c478bd9Sstevel@tonic-gate * Tag the modified probe with the generation in which it was 7477c478bd9Sstevel@tonic-gate * changed. 7487c478bd9Sstevel@tonic-gate */ 7497c478bd9Sstevel@tonic-gate probe->ftp_gen = fasttrap_mod_gen; 7507c478bd9Sstevel@tonic-gate return; 7517c478bd9Sstevel@tonic-gate } 7527c478bd9Sstevel@tonic-gate 7537c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 7547c478bd9Sstevel@tonic-gate 7557c478bd9Sstevel@tonic-gate /* 7567c478bd9Sstevel@tonic-gate * We can't safely remove the tracepoint from the set of active 7577c478bd9Sstevel@tonic-gate * tracepoints until we've actually removed the fasttrap instruction 7587c478bd9Sstevel@tonic-gate * from the process's text. We can, however, operate on this 7597c478bd9Sstevel@tonic-gate * tracepoint secure in the knowledge that no other thread is going to 7607c478bd9Sstevel@tonic-gate * be looking at it since we hold P_PR_LOCK on the process if it's 7617c478bd9Sstevel@tonic-gate * live or we hold the provider lock on the process if it's dead and 7627c478bd9Sstevel@tonic-gate * gone. 7637c478bd9Sstevel@tonic-gate */ 7647c478bd9Sstevel@tonic-gate 7657c478bd9Sstevel@tonic-gate /* 7667c478bd9Sstevel@tonic-gate * We only need to remove the actual instruction if we're looking 7677c478bd9Sstevel@tonic-gate * at an existing process 7687c478bd9Sstevel@tonic-gate */ 7697c478bd9Sstevel@tonic-gate if (p != NULL) { 7707c478bd9Sstevel@tonic-gate /* 7717c478bd9Sstevel@tonic-gate * If we fail to restore the instruction we need to kill 7727c478bd9Sstevel@tonic-gate * this process since it's in a completely unrecoverable 7737c478bd9Sstevel@tonic-gate * state. 7747c478bd9Sstevel@tonic-gate */ 7757c478bd9Sstevel@tonic-gate if (fasttrap_tracepoint_remove(p, tp) != 0) 7767c478bd9Sstevel@tonic-gate fasttrap_sigtrap(p, NULL, pc); 7777c478bd9Sstevel@tonic-gate 7787c478bd9Sstevel@tonic-gate /* 7797c478bd9Sstevel@tonic-gate * Decrement the count of the number of tracepoints active 7807c478bd9Sstevel@tonic-gate * in the victim process. 7817c478bd9Sstevel@tonic-gate */ 7827c478bd9Sstevel@tonic-gate ASSERT(p->p_proc_flag & P_PR_LOCK); 7837c478bd9Sstevel@tonic-gate p->p_dtrace_count--; 7847c478bd9Sstevel@tonic-gate } 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate /* 7877c478bd9Sstevel@tonic-gate * Remove the probe from the hash table of active tracepoints. 7887c478bd9Sstevel@tonic-gate */ 7897c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 7907c478bd9Sstevel@tonic-gate pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 7917c478bd9Sstevel@tonic-gate ASSERT(*pp != NULL); 7927c478bd9Sstevel@tonic-gate while (*pp != tp) { 7937c478bd9Sstevel@tonic-gate pp = &(*pp)->ftt_next; 7947c478bd9Sstevel@tonic-gate ASSERT(*pp != NULL); 7957c478bd9Sstevel@tonic-gate } 7967c478bd9Sstevel@tonic-gate 7977c478bd9Sstevel@tonic-gate *pp = tp->ftt_next; 7987c478bd9Sstevel@tonic-gate membar_producer(); 7997c478bd9Sstevel@tonic-gate 8007c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 8017c478bd9Sstevel@tonic-gate 8027c478bd9Sstevel@tonic-gate /* 8037c478bd9Sstevel@tonic-gate * Tag the modified probe with the generation in which it was changed. 8047c478bd9Sstevel@tonic-gate */ 8057c478bd9Sstevel@tonic-gate probe->ftp_gen = fasttrap_mod_gen; 8067c478bd9Sstevel@tonic-gate } 8077c478bd9Sstevel@tonic-gate 8087c478bd9Sstevel@tonic-gate typedef int fasttrap_probe_f(struct regs *); 8097c478bd9Sstevel@tonic-gate 8107c478bd9Sstevel@tonic-gate static void 8117c478bd9Sstevel@tonic-gate fasttrap_enable_common(int *count, fasttrap_probe_f **fptr, fasttrap_probe_f *f, 8127c478bd9Sstevel@tonic-gate fasttrap_probe_f **fptr2, fasttrap_probe_f *f2) 8137c478bd9Sstevel@tonic-gate { 8147c478bd9Sstevel@tonic-gate /* 8157c478bd9Sstevel@tonic-gate * We don't have to play the rw lock game here because we're 8167c478bd9Sstevel@tonic-gate * providing something rather than taking something away -- 8177c478bd9Sstevel@tonic-gate * we can be sure that no threads have tried to follow this 8187c478bd9Sstevel@tonic-gate * function pointer yet. 8197c478bd9Sstevel@tonic-gate */ 8207c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_count_mtx); 8217c478bd9Sstevel@tonic-gate if (*count == 0) { 8227c478bd9Sstevel@tonic-gate ASSERT(*fptr == NULL); 8237c478bd9Sstevel@tonic-gate *fptr = f; 8247c478bd9Sstevel@tonic-gate if (fptr2 != NULL) 8257c478bd9Sstevel@tonic-gate *fptr2 = f2; 8267c478bd9Sstevel@tonic-gate } 8277c478bd9Sstevel@tonic-gate ASSERT(*fptr == f); 8287c478bd9Sstevel@tonic-gate ASSERT(fptr2 == NULL || *fptr2 == f2); 8297c478bd9Sstevel@tonic-gate (*count)++; 8307c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_count_mtx); 8317c478bd9Sstevel@tonic-gate } 8327c478bd9Sstevel@tonic-gate 8337c478bd9Sstevel@tonic-gate static void 8347c478bd9Sstevel@tonic-gate fasttrap_disable_common(int *count, fasttrap_probe_f **fptr, 8357c478bd9Sstevel@tonic-gate fasttrap_probe_f **fptr2) 8367c478bd9Sstevel@tonic-gate { 8377c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 8387c478bd9Sstevel@tonic-gate 8397c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_count_mtx); 8407c478bd9Sstevel@tonic-gate (*count)--; 8417c478bd9Sstevel@tonic-gate ASSERT(*count >= 0); 8427c478bd9Sstevel@tonic-gate if (*count == 0) { 8437c478bd9Sstevel@tonic-gate cpu_t *cur, *cpu = CPU; 8447c478bd9Sstevel@tonic-gate 8457c478bd9Sstevel@tonic-gate for (cur = cpu->cpu_next_onln; cur != cpu; 8467c478bd9Sstevel@tonic-gate cur = cur->cpu_next_onln) { 8477c478bd9Sstevel@tonic-gate rw_enter(&cur->cpu_ft_lock, RW_WRITER); 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate *fptr = NULL; 8517c478bd9Sstevel@tonic-gate if (fptr2 != NULL) 8527c478bd9Sstevel@tonic-gate *fptr2 = NULL; 8537c478bd9Sstevel@tonic-gate 8547c478bd9Sstevel@tonic-gate for (cur = cpu->cpu_next_onln; cur != cpu; 8557c478bd9Sstevel@tonic-gate cur = cur->cpu_next_onln) { 8567c478bd9Sstevel@tonic-gate rw_exit(&cur->cpu_ft_lock); 8577c478bd9Sstevel@tonic-gate } 8587c478bd9Sstevel@tonic-gate } 8597c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_count_mtx); 8607c478bd9Sstevel@tonic-gate } 8617c478bd9Sstevel@tonic-gate 8627c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8637c478bd9Sstevel@tonic-gate static void 8647c478bd9Sstevel@tonic-gate fasttrap_enable(void *arg, dtrace_id_t id, void *parg) 8657c478bd9Sstevel@tonic-gate { 8667c478bd9Sstevel@tonic-gate /* 8677c478bd9Sstevel@tonic-gate * Enable the probe that corresponds to statically placed trace 8687c478bd9Sstevel@tonic-gate * points which have not explicitly been placed in the process's text 8697c478bd9Sstevel@tonic-gate * by the fasttrap provider. 8707c478bd9Sstevel@tonic-gate */ 8717c478bd9Sstevel@tonic-gate ASSERT(arg == NULL); 8727c478bd9Sstevel@tonic-gate ASSERT(id == fasttrap_probe_id); 8737c478bd9Sstevel@tonic-gate 8747c478bd9Sstevel@tonic-gate fasttrap_enable_common(&fasttrap_count, 8757c478bd9Sstevel@tonic-gate &dtrace_fasttrap_probe_ptr, fasttrap_probe, NULL, NULL); 8767c478bd9Sstevel@tonic-gate } 8777c478bd9Sstevel@tonic-gate 8787c478bd9Sstevel@tonic-gate 8797c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 8807c478bd9Sstevel@tonic-gate static void 8817c478bd9Sstevel@tonic-gate fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 8827c478bd9Sstevel@tonic-gate { 8837c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 8847c478bd9Sstevel@tonic-gate proc_t *p; 8858f5db3e7Sahl int i, rc; 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate ASSERT(probe != NULL); 8887c478bd9Sstevel@tonic-gate ASSERT(!probe->ftp_enabled); 8897c478bd9Sstevel@tonic-gate ASSERT(id == probe->ftp_id); 8907c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 8917c478bd9Sstevel@tonic-gate 8927c478bd9Sstevel@tonic-gate /* 8937c478bd9Sstevel@tonic-gate * Increment the count of enabled probes on this probe's provider; 8947c478bd9Sstevel@tonic-gate * the provider can't go away while the probe still exists. We 8957c478bd9Sstevel@tonic-gate * must increment this even if we aren't able to properly enable 8967c478bd9Sstevel@tonic-gate * this probe. 8977c478bd9Sstevel@tonic-gate */ 8987c478bd9Sstevel@tonic-gate mutex_enter(&probe->ftp_prov->ftp_mtx); 8997c478bd9Sstevel@tonic-gate probe->ftp_prov->ftp_rcount++; 9007c478bd9Sstevel@tonic-gate mutex_exit(&probe->ftp_prov->ftp_mtx); 9017c478bd9Sstevel@tonic-gate 9027c478bd9Sstevel@tonic-gate /* 903*ab9a77c7Sahl * If this probe's provider is retired (meaning it was valid in a 904*ab9a77c7Sahl * previously exec'ed incarnation of this address space), bail out. The 905*ab9a77c7Sahl * provider can't go away while we're in this code path. 9067c478bd9Sstevel@tonic-gate */ 907*ab9a77c7Sahl if (probe->ftp_prov->ftp_retired) 9087c478bd9Sstevel@tonic-gate return; 9097c478bd9Sstevel@tonic-gate 910*ab9a77c7Sahl /* 911*ab9a77c7Sahl * If we can't find the process, it may be that we're in the context of 912*ab9a77c7Sahl * a fork in which the traced process is being born and we're copying 913*ab9a77c7Sahl * USDT probes. Otherwise, the process is gone so bail. 914*ab9a77c7Sahl */ 915*ab9a77c7Sahl if ((p = sprlock(probe->ftp_pid)) == NULL) { 916*ab9a77c7Sahl if ((curproc->p_flag & SFORKING) == 0) 917*ab9a77c7Sahl return; 918*ab9a77c7Sahl 919*ab9a77c7Sahl mutex_enter(&pidlock); 920*ab9a77c7Sahl p = prfind(probe->ftp_pid); 921*ab9a77c7Sahl 922*ab9a77c7Sahl /* 923*ab9a77c7Sahl * Confirm that curproc is indeed forking the process in which 924*ab9a77c7Sahl * we're trying to enable probes. 925*ab9a77c7Sahl */ 926*ab9a77c7Sahl ASSERT(p != NULL); 927*ab9a77c7Sahl ASSERT(p->p_parent == curproc); 928*ab9a77c7Sahl ASSERT(p->p_stat == SIDL); 929*ab9a77c7Sahl 930*ab9a77c7Sahl mutex_enter(&p->p_lock); 931*ab9a77c7Sahl mutex_exit(&pidlock); 932*ab9a77c7Sahl 933*ab9a77c7Sahl sprlock_proc(p); 934*ab9a77c7Sahl } 935*ab9a77c7Sahl 9367c478bd9Sstevel@tonic-gate ASSERT(!(p->p_flag & SVFORK)); 9377c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 9387c478bd9Sstevel@tonic-gate 9397c478bd9Sstevel@tonic-gate /* 9407c478bd9Sstevel@tonic-gate * We have to enable the trap entry before any user threads have 9417c478bd9Sstevel@tonic-gate * the chance to execute the trap instruction we're about to place 9427c478bd9Sstevel@tonic-gate * in their process's text. 9437c478bd9Sstevel@tonic-gate */ 9447c478bd9Sstevel@tonic-gate fasttrap_enable_common(&fasttrap_pid_count, 9457c478bd9Sstevel@tonic-gate &dtrace_pid_probe_ptr, fasttrap_pid_probe, 9467c478bd9Sstevel@tonic-gate &dtrace_return_probe_ptr, fasttrap_return_probe); 9477c478bd9Sstevel@tonic-gate 9487c478bd9Sstevel@tonic-gate /* 9497c478bd9Sstevel@tonic-gate * Enable all the tracepoints and add this probe's id to each 9507c478bd9Sstevel@tonic-gate * tracepoint's list of active probes. 9517c478bd9Sstevel@tonic-gate */ 9527c478bd9Sstevel@tonic-gate for (i = 0; i < probe->ftp_ntps; i++) { 9538f5db3e7Sahl if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 9548f5db3e7Sahl /* 9558f5db3e7Sahl * If enabling the tracepoint failed completely, 9568f5db3e7Sahl * we don't have to disable it; if the failure 9578f5db3e7Sahl * was only partial we must disable it. 9588f5db3e7Sahl */ 9598f5db3e7Sahl if (rc == FASTTRAP_ENABLE_FAIL) 9608f5db3e7Sahl i--; 9618f5db3e7Sahl else 9628f5db3e7Sahl ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 9638f5db3e7Sahl 9647c478bd9Sstevel@tonic-gate /* 9657c478bd9Sstevel@tonic-gate * Back up and pull out all the tracepoints we've 9667c478bd9Sstevel@tonic-gate * created so far for this probe. 9677c478bd9Sstevel@tonic-gate */ 9688c642573Sahl while (i >= 0) { 9697c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(p, probe, i); 9708c642573Sahl i--; 9717c478bd9Sstevel@tonic-gate } 9727c478bd9Sstevel@tonic-gate 9737c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 9747c478bd9Sstevel@tonic-gate sprunlock(p); 9757c478bd9Sstevel@tonic-gate 9767c478bd9Sstevel@tonic-gate /* 9777c478bd9Sstevel@tonic-gate * Since we're not actually enabling this probe, 9787c478bd9Sstevel@tonic-gate * drop our reference on the trap table entry. 9797c478bd9Sstevel@tonic-gate */ 9807c478bd9Sstevel@tonic-gate fasttrap_disable_common(&fasttrap_pid_count, 9817c478bd9Sstevel@tonic-gate &dtrace_pid_probe_ptr, &dtrace_return_probe_ptr); 9827c478bd9Sstevel@tonic-gate return; 9837c478bd9Sstevel@tonic-gate } 9847c478bd9Sstevel@tonic-gate } 9857c478bd9Sstevel@tonic-gate 9867c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 9877c478bd9Sstevel@tonic-gate sprunlock(p); 9887c478bd9Sstevel@tonic-gate 9897c478bd9Sstevel@tonic-gate probe->ftp_enabled = 1; 9907c478bd9Sstevel@tonic-gate } 9917c478bd9Sstevel@tonic-gate 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9947c478bd9Sstevel@tonic-gate static void 9957c478bd9Sstevel@tonic-gate fasttrap_disable(void *arg, dtrace_id_t id, void *parg) 9967c478bd9Sstevel@tonic-gate { 9977c478bd9Sstevel@tonic-gate /* 9987c478bd9Sstevel@tonic-gate * Disable the probe the corresponds to statically placed trace 9997c478bd9Sstevel@tonic-gate * points. 10007c478bd9Sstevel@tonic-gate */ 10017c478bd9Sstevel@tonic-gate ASSERT(arg == NULL); 10027c478bd9Sstevel@tonic-gate ASSERT(id == fasttrap_probe_id); 10037c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 10047c478bd9Sstevel@tonic-gate fasttrap_disable_common(&fasttrap_count, &dtrace_fasttrap_probe_ptr, 10057c478bd9Sstevel@tonic-gate NULL); 10067c478bd9Sstevel@tonic-gate } 10077c478bd9Sstevel@tonic-gate 10087c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10097c478bd9Sstevel@tonic-gate static void 10107c478bd9Sstevel@tonic-gate fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 10117c478bd9Sstevel@tonic-gate { 10127c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 10137c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider = probe->ftp_prov; 10147c478bd9Sstevel@tonic-gate proc_t *p; 10157c478bd9Sstevel@tonic-gate int i, whack = 0; 10167c478bd9Sstevel@tonic-gate 10177c478bd9Sstevel@tonic-gate if (!probe->ftp_enabled) { 10187c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 10197c478bd9Sstevel@tonic-gate provider->ftp_rcount--; 10207c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_rcount >= 0); 10217c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 10227c478bd9Sstevel@tonic-gate return; 10237c478bd9Sstevel@tonic-gate } 10247c478bd9Sstevel@tonic-gate 10257c478bd9Sstevel@tonic-gate ASSERT(id == probe->ftp_id); 10267c478bd9Sstevel@tonic-gate 10277c478bd9Sstevel@tonic-gate /* 10287c478bd9Sstevel@tonic-gate * We won't be able to acquire a /proc-esque lock on the process 10297c478bd9Sstevel@tonic-gate * iff the process is dead and gone. In this case, we rely on the 10307c478bd9Sstevel@tonic-gate * provider lock as a point of mutual exclusion to prevent other 10317c478bd9Sstevel@tonic-gate * DTrace consumers from disabling this probe. 10327c478bd9Sstevel@tonic-gate */ 10337c478bd9Sstevel@tonic-gate if ((p = sprlock(probe->ftp_pid)) != NULL) { 10347c478bd9Sstevel@tonic-gate ASSERT(!(p->p_flag & SVFORK)); 10357c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 10367c478bd9Sstevel@tonic-gate } 10377c478bd9Sstevel@tonic-gate 10387c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 10397c478bd9Sstevel@tonic-gate 10407c478bd9Sstevel@tonic-gate /* 10417c478bd9Sstevel@tonic-gate * Disable all the associated tracepoints. 10427c478bd9Sstevel@tonic-gate */ 10437c478bd9Sstevel@tonic-gate for (i = 0; i < probe->ftp_ntps; i++) { 10447c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(p, probe, i); 10457c478bd9Sstevel@tonic-gate } 10467c478bd9Sstevel@tonic-gate 10477c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_rcount > 0); 10487c478bd9Sstevel@tonic-gate provider->ftp_rcount--; 10497c478bd9Sstevel@tonic-gate 10507c478bd9Sstevel@tonic-gate if (p != NULL) { 10517c478bd9Sstevel@tonic-gate /* 10527c478bd9Sstevel@tonic-gate * Even though we may not be able to remove it entirely, we 1053b096140dSahl * mark this retired provider to get a chance to remove some 10547c478bd9Sstevel@tonic-gate * of the associated probes. 10557c478bd9Sstevel@tonic-gate */ 1056b096140dSahl if (provider->ftp_retired && !provider->ftp_marked) 10577c478bd9Sstevel@tonic-gate whack = provider->ftp_marked = 1; 10587c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 10597c478bd9Sstevel@tonic-gate 10607c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 10617c478bd9Sstevel@tonic-gate sprunlock(p); 10627c478bd9Sstevel@tonic-gate } else { 10637c478bd9Sstevel@tonic-gate /* 10647c478bd9Sstevel@tonic-gate * If the process is dead, we're just waiting for the 10657c478bd9Sstevel@tonic-gate * last probe to be disabled to be able to free it. 10667c478bd9Sstevel@tonic-gate */ 10677c478bd9Sstevel@tonic-gate if (provider->ftp_rcount == 0 && !provider->ftp_marked) 10687c478bd9Sstevel@tonic-gate whack = provider->ftp_marked = 1; 10697c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 10707c478bd9Sstevel@tonic-gate } 10717c478bd9Sstevel@tonic-gate 10727c478bd9Sstevel@tonic-gate if (whack) 10737c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 10747c478bd9Sstevel@tonic-gate 10757c478bd9Sstevel@tonic-gate probe->ftp_enabled = 0; 10767c478bd9Sstevel@tonic-gate 10777c478bd9Sstevel@tonic-gate ASSERT(MUTEX_HELD(&cpu_lock)); 10787c478bd9Sstevel@tonic-gate fasttrap_disable_common(&fasttrap_pid_count, &dtrace_pid_probe_ptr, 10797c478bd9Sstevel@tonic-gate &dtrace_return_probe_ptr); 10807c478bd9Sstevel@tonic-gate } 10817c478bd9Sstevel@tonic-gate 10827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10837c478bd9Sstevel@tonic-gate static void 10847c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 10857c478bd9Sstevel@tonic-gate dtrace_argdesc_t *desc) 10867c478bd9Sstevel@tonic-gate { 10877c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 10887c478bd9Sstevel@tonic-gate char *str; 10897c478bd9Sstevel@tonic-gate int i; 10907c478bd9Sstevel@tonic-gate 10917c478bd9Sstevel@tonic-gate desc->dtargd_native[0] = '\0'; 10927c478bd9Sstevel@tonic-gate desc->dtargd_xlate[0] = '\0'; 10937c478bd9Sstevel@tonic-gate 1094b096140dSahl if (probe->ftp_prov->ftp_retired != 0 || 10957c478bd9Sstevel@tonic-gate desc->dtargd_ndx >= probe->ftp_nargs) { 10967c478bd9Sstevel@tonic-gate desc->dtargd_ndx = DTRACE_ARGNONE; 10977c478bd9Sstevel@tonic-gate return; 10987c478bd9Sstevel@tonic-gate } 10997c478bd9Sstevel@tonic-gate 11007c478bd9Sstevel@tonic-gate /* 11017c478bd9Sstevel@tonic-gate * We only need to set this member if the argument is remapped. 11027c478bd9Sstevel@tonic-gate */ 11037c478bd9Sstevel@tonic-gate if (probe->ftp_argmap != NULL) 11047c478bd9Sstevel@tonic-gate desc->dtargd_mapping = probe->ftp_argmap[desc->dtargd_ndx]; 11057c478bd9Sstevel@tonic-gate 11067c478bd9Sstevel@tonic-gate str = probe->ftp_ntypes; 11077c478bd9Sstevel@tonic-gate for (i = 0; i < desc->dtargd_mapping; i++) { 11087c478bd9Sstevel@tonic-gate str += strlen(str) + 1; 11097c478bd9Sstevel@tonic-gate } 11107c478bd9Sstevel@tonic-gate 11117c478bd9Sstevel@tonic-gate ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 11127c478bd9Sstevel@tonic-gate (void) strcpy(desc->dtargd_native, str); 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate if (probe->ftp_xtypes == NULL) 11157c478bd9Sstevel@tonic-gate return; 11167c478bd9Sstevel@tonic-gate 11177c478bd9Sstevel@tonic-gate str = probe->ftp_xtypes; 11187c478bd9Sstevel@tonic-gate for (i = 0; i < desc->dtargd_ndx; i++) { 11197c478bd9Sstevel@tonic-gate str += strlen(str) + 1; 11207c478bd9Sstevel@tonic-gate } 11217c478bd9Sstevel@tonic-gate 11227c478bd9Sstevel@tonic-gate ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 11237c478bd9Sstevel@tonic-gate (void) strcpy(desc->dtargd_xlate, str); 11247c478bd9Sstevel@tonic-gate } 11257c478bd9Sstevel@tonic-gate 11267c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 11277c478bd9Sstevel@tonic-gate static void 11287c478bd9Sstevel@tonic-gate fasttrap_destroy(void *arg, dtrace_id_t id, void *parg) 11297c478bd9Sstevel@tonic-gate { 11307c478bd9Sstevel@tonic-gate ASSERT(arg == NULL); 11317c478bd9Sstevel@tonic-gate ASSERT(id == fasttrap_probe_id); 11327c478bd9Sstevel@tonic-gate } 11337c478bd9Sstevel@tonic-gate 11347c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 11357c478bd9Sstevel@tonic-gate static void 11367c478bd9Sstevel@tonic-gate fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 11377c478bd9Sstevel@tonic-gate { 11387c478bd9Sstevel@tonic-gate fasttrap_probe_t *probe = parg; 11397c478bd9Sstevel@tonic-gate int i; 11407c478bd9Sstevel@tonic-gate size_t size; 11417c478bd9Sstevel@tonic-gate 11427c478bd9Sstevel@tonic-gate ASSERT(probe != NULL); 11437c478bd9Sstevel@tonic-gate ASSERT(!probe->ftp_enabled); 11447c478bd9Sstevel@tonic-gate ASSERT(fasttrap_total >= probe->ftp_ntps); 11457c478bd9Sstevel@tonic-gate 11467c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1147*ab9a77c7Sahl size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 11507c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(probe->ftp_gen); 11517c478bd9Sstevel@tonic-gate 11527c478bd9Sstevel@tonic-gate for (i = 0; i < probe->ftp_ntps; i++) { 11537c478bd9Sstevel@tonic-gate kmem_free(probe->ftp_tps[i].fit_tp, 11547c478bd9Sstevel@tonic-gate sizeof (fasttrap_tracepoint_t)); 11557c478bd9Sstevel@tonic-gate } 11567c478bd9Sstevel@tonic-gate 11577c478bd9Sstevel@tonic-gate kmem_free(probe, size); 11587c478bd9Sstevel@tonic-gate } 11597c478bd9Sstevel@tonic-gate 11607c478bd9Sstevel@tonic-gate 11617c478bd9Sstevel@tonic-gate static const dtrace_pattr_t fasttrap_attr = { 11627c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11637c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11647c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11657c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11667c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11677c478bd9Sstevel@tonic-gate }; 11687c478bd9Sstevel@tonic-gate 11697c478bd9Sstevel@tonic-gate static dtrace_pops_t fasttrap_pops = { 11707c478bd9Sstevel@tonic-gate fasttrap_provide, 11717c478bd9Sstevel@tonic-gate NULL, 11727c478bd9Sstevel@tonic-gate fasttrap_enable, 11737c478bd9Sstevel@tonic-gate fasttrap_disable, 11747c478bd9Sstevel@tonic-gate NULL, 11757c478bd9Sstevel@tonic-gate NULL, 11767c478bd9Sstevel@tonic-gate NULL, 11777c478bd9Sstevel@tonic-gate fasttrap_getarg, 11787c478bd9Sstevel@tonic-gate NULL, 11797c478bd9Sstevel@tonic-gate fasttrap_destroy 11807c478bd9Sstevel@tonic-gate }; 11817c478bd9Sstevel@tonic-gate 11827c478bd9Sstevel@tonic-gate static const dtrace_pattr_t pid_attr = { 11837c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11847c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11857c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11867c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 11877c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 11887c478bd9Sstevel@tonic-gate }; 11897c478bd9Sstevel@tonic-gate 11907c478bd9Sstevel@tonic-gate static dtrace_pops_t pid_pops = { 11917c478bd9Sstevel@tonic-gate fasttrap_pid_provide, 11927c478bd9Sstevel@tonic-gate NULL, 11937c478bd9Sstevel@tonic-gate fasttrap_pid_enable, 11947c478bd9Sstevel@tonic-gate fasttrap_pid_disable, 11957c478bd9Sstevel@tonic-gate NULL, 11967c478bd9Sstevel@tonic-gate NULL, 11977c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc, 11987c478bd9Sstevel@tonic-gate fasttrap_getarg, 11997c478bd9Sstevel@tonic-gate NULL, 12007c478bd9Sstevel@tonic-gate fasttrap_pid_destroy 12017c478bd9Sstevel@tonic-gate }; 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate static dtrace_pops_t usdt_pops = { 12047c478bd9Sstevel@tonic-gate fasttrap_pid_provide, 12057c478bd9Sstevel@tonic-gate NULL, 12067c478bd9Sstevel@tonic-gate fasttrap_pid_enable, 12077c478bd9Sstevel@tonic-gate fasttrap_pid_disable, 12087c478bd9Sstevel@tonic-gate NULL, 12097c478bd9Sstevel@tonic-gate NULL, 12107c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc, 12117c478bd9Sstevel@tonic-gate fasttrap_usdt_getarg, 12127c478bd9Sstevel@tonic-gate NULL, 12137c478bd9Sstevel@tonic-gate fasttrap_pid_destroy 12147c478bd9Sstevel@tonic-gate }; 12157c478bd9Sstevel@tonic-gate 1216b096140dSahl static fasttrap_proc_t * 1217b096140dSahl fasttrap_proc_lookup(pid_t pid) 1218b096140dSahl { 1219b096140dSahl fasttrap_bucket_t *bucket; 1220b096140dSahl fasttrap_proc_t *fprc, *new_fprc; 1221b096140dSahl 1222b096140dSahl bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1223b096140dSahl mutex_enter(&bucket->ftb_mtx); 1224b096140dSahl 1225b096140dSahl for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1226b096140dSahl if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) { 1227b096140dSahl mutex_enter(&fprc->ftpc_mtx); 1228b096140dSahl mutex_exit(&bucket->ftb_mtx); 1229b096140dSahl fprc->ftpc_count++; 1230b096140dSahl mutex_exit(&fprc->ftpc_mtx); 1231b096140dSahl 1232b096140dSahl return (fprc); 1233b096140dSahl } 1234b096140dSahl } 1235b096140dSahl 1236b096140dSahl /* 1237b096140dSahl * Drop the bucket lock so we don't try to perform a sleeping 1238b096140dSahl * allocation under it. 1239b096140dSahl */ 1240b096140dSahl mutex_exit(&bucket->ftb_mtx); 1241b096140dSahl 1242b096140dSahl new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1243b096140dSahl new_fprc->ftpc_pid = pid; 1244b096140dSahl new_fprc->ftpc_count = 1; 1245b096140dSahl 1246b096140dSahl mutex_enter(&bucket->ftb_mtx); 1247b096140dSahl 1248b096140dSahl /* 1249b096140dSahl * Take another lap through the list to make sure a proc hasn't 1250b096140dSahl * been created for this pid while we weren't under the bucket lock. 1251b096140dSahl */ 1252b096140dSahl for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1253b096140dSahl if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) { 1254b096140dSahl mutex_enter(&fprc->ftpc_mtx); 1255b096140dSahl mutex_exit(&bucket->ftb_mtx); 1256b096140dSahl fprc->ftpc_count++; 1257b096140dSahl mutex_exit(&fprc->ftpc_mtx); 1258b096140dSahl 1259b096140dSahl kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1260b096140dSahl 1261b096140dSahl return (fprc); 1262b096140dSahl } 1263b096140dSahl } 1264b096140dSahl 1265b096140dSahl new_fprc->ftpc_next = bucket->ftb_data; 1266b096140dSahl bucket->ftb_data = new_fprc; 1267b096140dSahl 1268b096140dSahl mutex_exit(&bucket->ftb_mtx); 1269b096140dSahl 1270b096140dSahl return (new_fprc); 1271b096140dSahl } 1272b096140dSahl 1273b096140dSahl static void 1274b096140dSahl fasttrap_proc_release(fasttrap_proc_t *proc) 1275b096140dSahl { 1276b096140dSahl fasttrap_bucket_t *bucket; 1277b096140dSahl fasttrap_proc_t *fprc, **fprcp; 1278b096140dSahl pid_t pid = proc->ftpc_pid; 1279b096140dSahl 1280b096140dSahl mutex_enter(&proc->ftpc_mtx); 1281b096140dSahl 1282b096140dSahl ASSERT(proc->ftpc_count != 0); 1283b096140dSahl 1284b096140dSahl if (--proc->ftpc_count != 0) { 1285b096140dSahl mutex_exit(&proc->ftpc_mtx); 1286b096140dSahl return; 1287b096140dSahl } 1288b096140dSahl 1289b096140dSahl mutex_exit(&proc->ftpc_mtx); 1290b096140dSahl 1291b096140dSahl bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1292b096140dSahl mutex_enter(&bucket->ftb_mtx); 1293b096140dSahl 1294b096140dSahl fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1295b096140dSahl while ((fprc = *fprcp) != NULL) { 1296b096140dSahl if (fprc == proc) 1297b096140dSahl break; 1298b096140dSahl 1299b096140dSahl fprcp = &fprc->ftpc_next; 1300b096140dSahl } 1301b096140dSahl 1302b096140dSahl /* 1303b096140dSahl * Something strange has happened if we can't find the proc. 1304b096140dSahl */ 1305b096140dSahl ASSERT(fprc != NULL); 1306b096140dSahl 1307b096140dSahl *fprcp = fprc->ftpc_next; 1308b096140dSahl 1309b096140dSahl mutex_exit(&bucket->ftb_mtx); 1310b096140dSahl 1311b096140dSahl kmem_free(fprc, sizeof (fasttrap_proc_t)); 1312b096140dSahl } 1313b096140dSahl 13147c478bd9Sstevel@tonic-gate /* 13157c478bd9Sstevel@tonic-gate * Lookup a fasttrap-managed provider based on its name and associated pid. 13167c478bd9Sstevel@tonic-gate * If the pattr argument is non-NULL, this function instantiates the provider 13177c478bd9Sstevel@tonic-gate * if it doesn't exist otherwise it returns NULL. The provider is returned 13187c478bd9Sstevel@tonic-gate * with its lock held. 13197c478bd9Sstevel@tonic-gate */ 13207c478bd9Sstevel@tonic-gate static fasttrap_provider_t * 13217c478bd9Sstevel@tonic-gate fasttrap_provider_lookup(pid_t pid, const char *name, 13227c478bd9Sstevel@tonic-gate const dtrace_pattr_t *pattr) 13237c478bd9Sstevel@tonic-gate { 13247c478bd9Sstevel@tonic-gate fasttrap_provider_t *fp, *new_fp = NULL; 13257c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket; 13267c478bd9Sstevel@tonic-gate char provname[DTRACE_PROVNAMELEN]; 13277c478bd9Sstevel@tonic-gate proc_t *p; 1328ad4023c4Sdp cred_t *cred; 13297c478bd9Sstevel@tonic-gate 13307c478bd9Sstevel@tonic-gate ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1331dafb5540Sahl ASSERT(pattr != NULL); 13327c478bd9Sstevel@tonic-gate 13337c478bd9Sstevel@tonic-gate bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 13347c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 13357c478bd9Sstevel@tonic-gate 13367c478bd9Sstevel@tonic-gate /* 13377c478bd9Sstevel@tonic-gate * Take a lap through the list and return the match if we find it. 13387c478bd9Sstevel@tonic-gate */ 13397c478bd9Sstevel@tonic-gate for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 13407c478bd9Sstevel@tonic-gate if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1341b096140dSahl !fp->ftp_retired) { 13427c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 13437c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 13447c478bd9Sstevel@tonic-gate return (fp); 13457c478bd9Sstevel@tonic-gate } 13467c478bd9Sstevel@tonic-gate } 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate /* 13497c478bd9Sstevel@tonic-gate * Drop the bucket lock so we don't try to perform a sleeping 13507c478bd9Sstevel@tonic-gate * allocation under it. 13517c478bd9Sstevel@tonic-gate */ 13527c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 13537c478bd9Sstevel@tonic-gate 1354b096140dSahl /* 13557c478bd9Sstevel@tonic-gate * Make sure the process exists, isn't a child created as the result 1356ad4023c4Sdp * of a vfork(2), and isn't a zombie (but may be in fork). 13577c478bd9Sstevel@tonic-gate */ 13587c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 135997eda132Sraf if ((p = prfind(pid)) == NULL) { 13607c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 13617c478bd9Sstevel@tonic-gate return (NULL); 13627c478bd9Sstevel@tonic-gate } 13637c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 13647c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 136597eda132Sraf if (p->p_flag & (SVFORK | SEXITING)) { 136697eda132Sraf mutex_exit(&p->p_lock); 136797eda132Sraf return (NULL); 136897eda132Sraf } 13697c478bd9Sstevel@tonic-gate 13707c478bd9Sstevel@tonic-gate /* 13717c478bd9Sstevel@tonic-gate * Increment p_dtrace_probes so that the process knows to inform us 13727c478bd9Sstevel@tonic-gate * when it exits or execs. fasttrap_provider_free() decrements this 13737c478bd9Sstevel@tonic-gate * when we're done with this provider. 13747c478bd9Sstevel@tonic-gate */ 13757c478bd9Sstevel@tonic-gate p->p_dtrace_probes++; 13767c478bd9Sstevel@tonic-gate 1377ad4023c4Sdp /* 1378ad4023c4Sdp * Grab the credentials for this process so we have 1379ad4023c4Sdp * something to pass to dtrace_register(). 1380ad4023c4Sdp */ 13817c478bd9Sstevel@tonic-gate mutex_enter(&p->p_crlock); 1382ad4023c4Sdp crhold(p->p_cred); 1383ad4023c4Sdp cred = p->p_cred; 13847c478bd9Sstevel@tonic-gate mutex_exit(&p->p_crlock); 13857c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 13867c478bd9Sstevel@tonic-gate 13877c478bd9Sstevel@tonic-gate new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1388b096140dSahl new_fp->ftp_pid = pid; 1389b096140dSahl new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1390b096140dSahl 1391b096140dSahl ASSERT(new_fp->ftp_proc != NULL); 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 13947c478bd9Sstevel@tonic-gate 13957c478bd9Sstevel@tonic-gate /* 13967c478bd9Sstevel@tonic-gate * Take another lap through the list to make sure a provider hasn't 13977c478bd9Sstevel@tonic-gate * been created for this pid while we weren't under the bucket lock. 13987c478bd9Sstevel@tonic-gate */ 13997c478bd9Sstevel@tonic-gate for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 14007c478bd9Sstevel@tonic-gate if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1401b096140dSahl !fp->ftp_retired) { 14027c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 14037c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 14047c478bd9Sstevel@tonic-gate fasttrap_provider_free(new_fp); 1405ad4023c4Sdp crfree(cred); 14067c478bd9Sstevel@tonic-gate return (fp); 14077c478bd9Sstevel@tonic-gate } 14087c478bd9Sstevel@tonic-gate } 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate (void) strcpy(new_fp->ftp_name, name); 14117c478bd9Sstevel@tonic-gate 14127c478bd9Sstevel@tonic-gate /* 14137c478bd9Sstevel@tonic-gate * Fail and return NULL if either the provider name is too long 14147c478bd9Sstevel@tonic-gate * or we fail to register this new provider with the DTrace 14157c478bd9Sstevel@tonic-gate * framework. Note that this is the only place we ever construct 14167c478bd9Sstevel@tonic-gate * the full provider name -- we keep it in pieces in the provider 14177c478bd9Sstevel@tonic-gate * structure. 14187c478bd9Sstevel@tonic-gate */ 14197c478bd9Sstevel@tonic-gate if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 14207c478bd9Sstevel@tonic-gate sizeof (provname) || 14217c478bd9Sstevel@tonic-gate dtrace_register(provname, pattr, 1422ad4023c4Sdp DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 14237c478bd9Sstevel@tonic-gate pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 14247c478bd9Sstevel@tonic-gate &new_fp->ftp_provid) != 0) { 14257c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 14267c478bd9Sstevel@tonic-gate fasttrap_provider_free(new_fp); 1427ad4023c4Sdp crfree(cred); 14287c478bd9Sstevel@tonic-gate return (NULL); 14297c478bd9Sstevel@tonic-gate } 14307c478bd9Sstevel@tonic-gate 14317c478bd9Sstevel@tonic-gate new_fp->ftp_next = bucket->ftb_data; 14327c478bd9Sstevel@tonic-gate bucket->ftb_data = new_fp; 14337c478bd9Sstevel@tonic-gate 14347c478bd9Sstevel@tonic-gate mutex_enter(&new_fp->ftp_mtx); 14357c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 14367c478bd9Sstevel@tonic-gate 1437ad4023c4Sdp crfree(cred); 14387c478bd9Sstevel@tonic-gate return (new_fp); 14397c478bd9Sstevel@tonic-gate } 14407c478bd9Sstevel@tonic-gate 14417c478bd9Sstevel@tonic-gate static void 14427c478bd9Sstevel@tonic-gate fasttrap_provider_free(fasttrap_provider_t *provider) 14437c478bd9Sstevel@tonic-gate { 14447c478bd9Sstevel@tonic-gate pid_t pid = provider->ftp_pid; 14457c478bd9Sstevel@tonic-gate proc_t *p; 14467c478bd9Sstevel@tonic-gate 14477c478bd9Sstevel@tonic-gate /* 1448*ab9a77c7Sahl * There need to be no associated enabled probes, no consumers 1449*ab9a77c7Sahl * creating probes, and no meta providers referencing this provider. 14507c478bd9Sstevel@tonic-gate */ 14517c478bd9Sstevel@tonic-gate ASSERT(provider->ftp_rcount == 0); 1452*ab9a77c7Sahl ASSERT(provider->ftp_ccount == 0); 1453*ab9a77c7Sahl ASSERT(provider->ftp_mcount == 0); 14547c478bd9Sstevel@tonic-gate 1455b096140dSahl fasttrap_proc_release(provider->ftp_proc); 1456b096140dSahl 14577c478bd9Sstevel@tonic-gate kmem_free(provider, sizeof (fasttrap_provider_t)); 14587c478bd9Sstevel@tonic-gate 14597c478bd9Sstevel@tonic-gate /* 14607c478bd9Sstevel@tonic-gate * Decrement p_dtrace_probes on the process whose provider we're 14617c478bd9Sstevel@tonic-gate * freeing. We don't have to worry about clobbering somone else's 14627c478bd9Sstevel@tonic-gate * modifications to it because we have locked the bucket that 14637c478bd9Sstevel@tonic-gate * corresponds to this process's hash chain in the provider hash 14647c478bd9Sstevel@tonic-gate * table. Don't sweat it if we can't find the process. 14657c478bd9Sstevel@tonic-gate */ 14667c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 14677c478bd9Sstevel@tonic-gate if ((p = prfind(pid)) == NULL) { 14687c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 14697c478bd9Sstevel@tonic-gate return; 14707c478bd9Sstevel@tonic-gate } 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 14737c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 14747c478bd9Sstevel@tonic-gate 14757c478bd9Sstevel@tonic-gate p->p_dtrace_probes--; 14767c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 14777c478bd9Sstevel@tonic-gate } 14787c478bd9Sstevel@tonic-gate 14797c478bd9Sstevel@tonic-gate static void 1480*ab9a77c7Sahl fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 14817c478bd9Sstevel@tonic-gate { 1482dafb5540Sahl fasttrap_provider_t *fp; 1483dafb5540Sahl fasttrap_bucket_t *bucket; 1484dafb5540Sahl dtrace_provider_id_t provid; 1485dafb5540Sahl 1486dafb5540Sahl ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1487dafb5540Sahl 1488dafb5540Sahl bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1489dafb5540Sahl mutex_enter(&bucket->ftb_mtx); 1490dafb5540Sahl 1491dafb5540Sahl for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1492dafb5540Sahl if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1493dafb5540Sahl !fp->ftp_retired) 1494dafb5540Sahl break; 1495dafb5540Sahl } 1496dafb5540Sahl 1497dafb5540Sahl if (fp == NULL) { 1498dafb5540Sahl mutex_exit(&bucket->ftb_mtx); 1499dafb5540Sahl return; 1500dafb5540Sahl } 15017c478bd9Sstevel@tonic-gate 1502*ab9a77c7Sahl mutex_enter(&fp->ftp_mtx); 1503*ab9a77c7Sahl ASSERT(!mprov || fp->ftp_mcount > 0); 1504*ab9a77c7Sahl if (mprov && --fp->ftp_mcount != 0) { 1505*ab9a77c7Sahl mutex_exit(&fp->ftp_mtx); 1506*ab9a77c7Sahl mutex_exit(&bucket->ftb_mtx); 1507*ab9a77c7Sahl return; 1508*ab9a77c7Sahl } 1509*ab9a77c7Sahl 15107c478bd9Sstevel@tonic-gate /* 1511b096140dSahl * Mark the provider to be removed in our post-processing step, 1512b096140dSahl * mark it retired, and mark its proc as defunct (though it may 1513b096140dSahl * already be marked defunct by another provider that shares the 1514b096140dSahl * same proc). Marking it indicates that we should try to remove it; 1515b096140dSahl * setting the retired flag indicates that we're done with this 1516b096140dSahl * provider; setting the proc to be defunct indicates that all 1517b096140dSahl * tracepoints associated with the traced process should be ignored. 1518dafb5540Sahl * 1519dafb5540Sahl * We obviously need to take the bucket lock before the provider lock 1520dafb5540Sahl * to perform the lookup, but we need to drop the provider lock 1521dafb5540Sahl * before calling into the DTrace framework since we acquire the 1522dafb5540Sahl * provider lock in callbacks invoked from the DTrace framework. The 1523dafb5540Sahl * bucket lock therefore protects the integrity of the provider hash 1524dafb5540Sahl * table. 15257c478bd9Sstevel@tonic-gate */ 1526dafb5540Sahl fp->ftp_proc->ftpc_defunct = 1; 1527dafb5540Sahl fp->ftp_retired = 1; 1528dafb5540Sahl fp->ftp_marked = 1; 1529dafb5540Sahl provid = fp->ftp_provid; 1530dafb5540Sahl mutex_exit(&fp->ftp_mtx); 15317c478bd9Sstevel@tonic-gate 15327c478bd9Sstevel@tonic-gate /* 15337c478bd9Sstevel@tonic-gate * We don't have to worry about invalidating the same provider twice 15347c478bd9Sstevel@tonic-gate * since fasttrap_provider_lookup() will ignore provider that have 1535b096140dSahl * been marked as retired. 15367c478bd9Sstevel@tonic-gate */ 15377c478bd9Sstevel@tonic-gate dtrace_invalidate(provid); 15387c478bd9Sstevel@tonic-gate 1539dafb5540Sahl mutex_exit(&bucket->ftb_mtx); 1540dafb5540Sahl 15417c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 15427c478bd9Sstevel@tonic-gate } 15437c478bd9Sstevel@tonic-gate 15447c478bd9Sstevel@tonic-gate static int 15457c478bd9Sstevel@tonic-gate fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 15467c478bd9Sstevel@tonic-gate { 15477c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider; 15487c478bd9Sstevel@tonic-gate fasttrap_probe_t *pp; 15497c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 15507c478bd9Sstevel@tonic-gate char *name; 15517c478bd9Sstevel@tonic-gate int i, aframes, whack; 15527c478bd9Sstevel@tonic-gate 15537c478bd9Sstevel@tonic-gate switch (pdata->ftps_type) { 15547c478bd9Sstevel@tonic-gate case DTFTP_ENTRY: 15557c478bd9Sstevel@tonic-gate name = "entry"; 15567c478bd9Sstevel@tonic-gate aframes = FASTTRAP_ENTRY_AFRAMES; 15577c478bd9Sstevel@tonic-gate break; 15587c478bd9Sstevel@tonic-gate case DTFTP_RETURN: 15597c478bd9Sstevel@tonic-gate name = "return"; 15607c478bd9Sstevel@tonic-gate aframes = FASTTRAP_RETURN_AFRAMES; 15617c478bd9Sstevel@tonic-gate break; 15627c478bd9Sstevel@tonic-gate case DTFTP_OFFSETS: 15637c478bd9Sstevel@tonic-gate name = NULL; 15647c478bd9Sstevel@tonic-gate break; 15657c478bd9Sstevel@tonic-gate default: 15667c478bd9Sstevel@tonic-gate return (EINVAL); 15677c478bd9Sstevel@tonic-gate } 15687c478bd9Sstevel@tonic-gate 15697c478bd9Sstevel@tonic-gate if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 15707c478bd9Sstevel@tonic-gate FASTTRAP_PID_NAME, &pid_attr)) == NULL) 15717c478bd9Sstevel@tonic-gate return (ESRCH); 15727c478bd9Sstevel@tonic-gate 15737c478bd9Sstevel@tonic-gate /* 15747c478bd9Sstevel@tonic-gate * Increment this reference count to indicate that a consumer is 15757c478bd9Sstevel@tonic-gate * actively adding a new probe associated with this provider. 15767c478bd9Sstevel@tonic-gate */ 15777c478bd9Sstevel@tonic-gate provider->ftp_ccount++; 15787c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 15797c478bd9Sstevel@tonic-gate 15807c478bd9Sstevel@tonic-gate if (name != NULL) { 15817c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(provider->ftp_provid, 15827c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name) != 0) 15837c478bd9Sstevel@tonic-gate goto done; 15847c478bd9Sstevel@tonic-gate 15857c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 15867c478bd9Sstevel@tonic-gate 15877c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max) { 15887c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 15897c478bd9Sstevel@tonic-gate goto no_mem; 15907c478bd9Sstevel@tonic-gate } 15917c478bd9Sstevel@tonic-gate 15927c478bd9Sstevel@tonic-gate ASSERT(pdata->ftps_noffs > 0); 1593*ab9a77c7Sahl pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1594*ab9a77c7Sahl ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 15957c478bd9Sstevel@tonic-gate 15967c478bd9Sstevel@tonic-gate pp->ftp_prov = provider; 15977c478bd9Sstevel@tonic-gate pp->ftp_faddr = pdata->ftps_pc; 15987c478bd9Sstevel@tonic-gate pp->ftp_fsize = pdata->ftps_size; 15997c478bd9Sstevel@tonic-gate pp->ftp_pid = pdata->ftps_pid; 16007c478bd9Sstevel@tonic-gate pp->ftp_ntps = pdata->ftps_noffs; 16017c478bd9Sstevel@tonic-gate 16027c478bd9Sstevel@tonic-gate for (i = 0; i < pdata->ftps_noffs; i++) { 16037c478bd9Sstevel@tonic-gate tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 16047c478bd9Sstevel@tonic-gate KM_SLEEP); 16057c478bd9Sstevel@tonic-gate 1606b096140dSahl tp->ftt_proc = provider->ftp_proc; 16077c478bd9Sstevel@tonic-gate tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 16087c478bd9Sstevel@tonic-gate tp->ftt_pid = pdata->ftps_pid; 16097c478bd9Sstevel@tonic-gate 16107c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_tp = tp; 16117c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_id.fti_probe = pp; 1612ac448965Sahl pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 16137c478bd9Sstevel@tonic-gate } 16147c478bd9Sstevel@tonic-gate 16157c478bd9Sstevel@tonic-gate pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 16167c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 16177c478bd9Sstevel@tonic-gate } else { 16187c478bd9Sstevel@tonic-gate for (i = 0; i < pdata->ftps_noffs; i++) { 16197c478bd9Sstevel@tonic-gate char name_str[17]; 16207c478bd9Sstevel@tonic-gate 16217c478bd9Sstevel@tonic-gate (void) sprintf(name_str, "%llx", 16227c478bd9Sstevel@tonic-gate (unsigned long long)pdata->ftps_offs[i]); 16237c478bd9Sstevel@tonic-gate 16247c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(provider->ftp_provid, 16257c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 16267c478bd9Sstevel@tonic-gate continue; 16277c478bd9Sstevel@tonic-gate 16287c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, 1); 16297c478bd9Sstevel@tonic-gate 16307c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max) { 16317c478bd9Sstevel@tonic-gate atomic_add_32(&fasttrap_total, -1); 16327c478bd9Sstevel@tonic-gate goto no_mem; 16337c478bd9Sstevel@tonic-gate } 16347c478bd9Sstevel@tonic-gate 16357c478bd9Sstevel@tonic-gate pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 16367c478bd9Sstevel@tonic-gate 16377c478bd9Sstevel@tonic-gate pp->ftp_prov = provider; 16387c478bd9Sstevel@tonic-gate pp->ftp_faddr = pdata->ftps_pc; 16397c478bd9Sstevel@tonic-gate pp->ftp_fsize = pdata->ftps_size; 16407c478bd9Sstevel@tonic-gate pp->ftp_pid = pdata->ftps_pid; 16417c478bd9Sstevel@tonic-gate pp->ftp_ntps = 1; 16427c478bd9Sstevel@tonic-gate 16437c478bd9Sstevel@tonic-gate tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 16447c478bd9Sstevel@tonic-gate KM_SLEEP); 16457c478bd9Sstevel@tonic-gate 1646b096140dSahl tp->ftt_proc = provider->ftp_proc; 16477c478bd9Sstevel@tonic-gate tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 16487c478bd9Sstevel@tonic-gate tp->ftt_pid = pdata->ftps_pid; 16497c478bd9Sstevel@tonic-gate 16507c478bd9Sstevel@tonic-gate pp->ftp_tps[0].fit_tp = tp; 16517c478bd9Sstevel@tonic-gate pp->ftp_tps[0].fit_id.fti_probe = pp; 1652ac448965Sahl pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 16537c478bd9Sstevel@tonic-gate 16547c478bd9Sstevel@tonic-gate pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 16557c478bd9Sstevel@tonic-gate pdata->ftps_mod, pdata->ftps_func, name_str, 16567c478bd9Sstevel@tonic-gate FASTTRAP_OFFSET_AFRAMES, pp); 16577c478bd9Sstevel@tonic-gate } 16587c478bd9Sstevel@tonic-gate } 16597c478bd9Sstevel@tonic-gate 16607c478bd9Sstevel@tonic-gate done: 16617c478bd9Sstevel@tonic-gate /* 16627c478bd9Sstevel@tonic-gate * We know that the provider is still valid since we incremented the 16637c478bd9Sstevel@tonic-gate * reference count. If someone tried to free this provider while we 16647c478bd9Sstevel@tonic-gate * were using it (e.g. because the process called exec(2) or exit(2)), 16657c478bd9Sstevel@tonic-gate * take note of that and try to free it now. 16667c478bd9Sstevel@tonic-gate */ 16677c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 16687c478bd9Sstevel@tonic-gate provider->ftp_ccount--; 1669b096140dSahl whack = provider->ftp_retired; 16707c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 16717c478bd9Sstevel@tonic-gate 16727c478bd9Sstevel@tonic-gate if (whack) 16737c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 16747c478bd9Sstevel@tonic-gate 16757c478bd9Sstevel@tonic-gate return (0); 16767c478bd9Sstevel@tonic-gate 16777c478bd9Sstevel@tonic-gate no_mem: 16787c478bd9Sstevel@tonic-gate /* 16797c478bd9Sstevel@tonic-gate * If we've exhausted the allowable resources, we'll try to remove 16807c478bd9Sstevel@tonic-gate * this provider to free some up. This is to cover the case where 16817c478bd9Sstevel@tonic-gate * the user has accidentally created many more probes than was 16827c478bd9Sstevel@tonic-gate * intended (e.g. pid123:::). 16837c478bd9Sstevel@tonic-gate */ 16847c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 16857c478bd9Sstevel@tonic-gate provider->ftp_ccount--; 16867c478bd9Sstevel@tonic-gate provider->ftp_marked = 1; 16877c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 16887c478bd9Sstevel@tonic-gate 16897c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 16907c478bd9Sstevel@tonic-gate 16917c478bd9Sstevel@tonic-gate return (ENOMEM); 16927c478bd9Sstevel@tonic-gate } 16937c478bd9Sstevel@tonic-gate 16947c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16957c478bd9Sstevel@tonic-gate static void * 16967c478bd9Sstevel@tonic-gate fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 16977c478bd9Sstevel@tonic-gate { 16987c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider; 16997c478bd9Sstevel@tonic-gate 17007c478bd9Sstevel@tonic-gate /* 17017c478bd9Sstevel@tonic-gate * A 32-bit unsigned integer (like a pid for example) can be 17027c478bd9Sstevel@tonic-gate * expressed in 10 or fewer decimal digits. Make sure that we'll 17037c478bd9Sstevel@tonic-gate * have enough space for the provider name. 17047c478bd9Sstevel@tonic-gate */ 17057c478bd9Sstevel@tonic-gate if (strlen(dhpv->dthpv_provname) + 10 >= 17067c478bd9Sstevel@tonic-gate sizeof (provider->ftp_name)) { 17077c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "failed to instantiate provider %s: " 17087c478bd9Sstevel@tonic-gate "name too long to accomodate pid", dhpv->dthpv_provname); 17097c478bd9Sstevel@tonic-gate return (NULL); 17107c478bd9Sstevel@tonic-gate } 17117c478bd9Sstevel@tonic-gate 17127c478bd9Sstevel@tonic-gate /* 17137c478bd9Sstevel@tonic-gate * Don't let folks spoof the true pid provider. 17147c478bd9Sstevel@tonic-gate */ 17157c478bd9Sstevel@tonic-gate if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 17167c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "failed to instantiate provider %s: " 17177c478bd9Sstevel@tonic-gate "%s is an invalid name", dhpv->dthpv_provname, 17187c478bd9Sstevel@tonic-gate FASTTRAP_PID_NAME); 17197c478bd9Sstevel@tonic-gate return (NULL); 17207c478bd9Sstevel@tonic-gate } 17217c478bd9Sstevel@tonic-gate 17227c478bd9Sstevel@tonic-gate /* 17237c478bd9Sstevel@tonic-gate * The highest stability class that fasttrap supports is ISA; cap 17247c478bd9Sstevel@tonic-gate * the stability of the new provider accordingly. 17257c478bd9Sstevel@tonic-gate */ 17267c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_provider.dtat_class >= DTRACE_CLASS_COMMON) 17277c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 17287c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_mod.dtat_class >= DTRACE_CLASS_COMMON) 17297c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 17307c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_func.dtat_class >= DTRACE_CLASS_COMMON) 17317c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 17327c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_name.dtat_class >= DTRACE_CLASS_COMMON) 17337c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 17347c478bd9Sstevel@tonic-gate if (dhpv->dthpv_pattr.dtpa_args.dtat_class >= DTRACE_CLASS_COMMON) 17357c478bd9Sstevel@tonic-gate dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 17367c478bd9Sstevel@tonic-gate 17377c478bd9Sstevel@tonic-gate if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 17387c478bd9Sstevel@tonic-gate &dhpv->dthpv_pattr)) == NULL) { 17397c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "failed to instantiate provider %s for " 17407c478bd9Sstevel@tonic-gate "process %u", dhpv->dthpv_provname, (uint_t)pid); 17417c478bd9Sstevel@tonic-gate return (NULL); 17427c478bd9Sstevel@tonic-gate } 17437c478bd9Sstevel@tonic-gate 17447c478bd9Sstevel@tonic-gate /* 1745*ab9a77c7Sahl * Up the meta provider count so this provider isn't removed until 1746*ab9a77c7Sahl * the meta provider has been told to remove it. 17477c478bd9Sstevel@tonic-gate */ 1748*ab9a77c7Sahl provider->ftp_mcount++; 17497c478bd9Sstevel@tonic-gate 17507c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 17517c478bd9Sstevel@tonic-gate 17527c478bd9Sstevel@tonic-gate return (provider); 17537c478bd9Sstevel@tonic-gate } 17547c478bd9Sstevel@tonic-gate 17557c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17567c478bd9Sstevel@tonic-gate static void 17577c478bd9Sstevel@tonic-gate fasttrap_meta_create_probe(void *arg, void *parg, 17587c478bd9Sstevel@tonic-gate dtrace_helper_probedesc_t *dhpb) 17597c478bd9Sstevel@tonic-gate { 17607c478bd9Sstevel@tonic-gate fasttrap_provider_t *provider = parg; 17617c478bd9Sstevel@tonic-gate fasttrap_probe_t *pp; 17627c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 1763ac448965Sahl int i, j; 1764ac448965Sahl uint32_t ntps; 17657c478bd9Sstevel@tonic-gate 17667c478bd9Sstevel@tonic-gate mutex_enter(&provider->ftp_mtx); 17677c478bd9Sstevel@tonic-gate 17687c478bd9Sstevel@tonic-gate if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 17697c478bd9Sstevel@tonic-gate dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 17707c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 17717c478bd9Sstevel@tonic-gate return; 17727c478bd9Sstevel@tonic-gate } 17737c478bd9Sstevel@tonic-gate 1774ac448965Sahl ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1775*ab9a77c7Sahl ASSERT(ntps > 0); 1776ac448965Sahl 1777ac448965Sahl atomic_add_32(&fasttrap_total, ntps); 17787c478bd9Sstevel@tonic-gate 17797c478bd9Sstevel@tonic-gate if (fasttrap_total > fasttrap_max) { 1780ac448965Sahl atomic_add_32(&fasttrap_total, -ntps); 17817c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 17827c478bd9Sstevel@tonic-gate return; 17837c478bd9Sstevel@tonic-gate } 17847c478bd9Sstevel@tonic-gate 1785*ab9a77c7Sahl pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 17867c478bd9Sstevel@tonic-gate 17877c478bd9Sstevel@tonic-gate pp->ftp_prov = provider; 17887c478bd9Sstevel@tonic-gate pp->ftp_pid = provider->ftp_pid; 1789ac448965Sahl pp->ftp_ntps = ntps; 17907c478bd9Sstevel@tonic-gate pp->ftp_nargs = dhpb->dthpb_xargc; 17917c478bd9Sstevel@tonic-gate pp->ftp_xtypes = dhpb->dthpb_xtypes; 17927c478bd9Sstevel@tonic-gate pp->ftp_ntypes = dhpb->dthpb_ntypes; 17937c478bd9Sstevel@tonic-gate 1794ac448965Sahl /* 1795ac448965Sahl * First create a tracepoint for each actual point of interest. 1796ac448965Sahl */ 1797ac448965Sahl for (i = 0; i < dhpb->dthpb_noffs; i++) { 17987c478bd9Sstevel@tonic-gate tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 17997c478bd9Sstevel@tonic-gate 1800b096140dSahl tp->ftt_proc = provider->ftp_proc; 18017c478bd9Sstevel@tonic-gate tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 18027c478bd9Sstevel@tonic-gate tp->ftt_pid = provider->ftp_pid; 18037c478bd9Sstevel@tonic-gate 18047c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_tp = tp; 18057c478bd9Sstevel@tonic-gate pp->ftp_tps[i].fit_id.fti_probe = pp; 1806ac448965Sahl #ifdef __sparc 1807ac448965Sahl pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1808ac448965Sahl #else 1809ac448965Sahl pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1810ac448965Sahl #endif 1811ac448965Sahl } 1812ac448965Sahl 1813ac448965Sahl /* 1814ac448965Sahl * Then create a tracepoint for each is-enabled point. 1815ac448965Sahl */ 1816ac448965Sahl for (j = 0; i < ntps; i++, j++) { 1817ac448965Sahl tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1818ac448965Sahl 1819ac448965Sahl tp->ftt_proc = provider->ftp_proc; 1820ac448965Sahl tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1821ac448965Sahl tp->ftt_pid = provider->ftp_pid; 1822ac448965Sahl 1823ac448965Sahl pp->ftp_tps[i].fit_tp = tp; 1824ac448965Sahl pp->ftp_tps[i].fit_id.fti_probe = pp; 1825ac448965Sahl pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 18267c478bd9Sstevel@tonic-gate } 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate /* 18297c478bd9Sstevel@tonic-gate * If the arguments are shuffled around we set the argument remapping 18307c478bd9Sstevel@tonic-gate * table. Later, when the probe fires, we only remap the arguments 18317c478bd9Sstevel@tonic-gate * if the table is non-NULL. 18327c478bd9Sstevel@tonic-gate */ 18337c478bd9Sstevel@tonic-gate for (i = 0; i < dhpb->dthpb_xargc; i++) { 18347c478bd9Sstevel@tonic-gate if (dhpb->dthpb_args[i] != i) { 18357c478bd9Sstevel@tonic-gate pp->ftp_argmap = dhpb->dthpb_args; 18367c478bd9Sstevel@tonic-gate break; 18377c478bd9Sstevel@tonic-gate } 18387c478bd9Sstevel@tonic-gate } 18397c478bd9Sstevel@tonic-gate 18407c478bd9Sstevel@tonic-gate /* 18417c478bd9Sstevel@tonic-gate * The probe is fully constructed -- register it with DTrace. 18427c478bd9Sstevel@tonic-gate */ 18437c478bd9Sstevel@tonic-gate pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 18447c478bd9Sstevel@tonic-gate dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate mutex_exit(&provider->ftp_mtx); 18477c478bd9Sstevel@tonic-gate } 18487c478bd9Sstevel@tonic-gate 18497c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 18507c478bd9Sstevel@tonic-gate static void 18517c478bd9Sstevel@tonic-gate fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 18527c478bd9Sstevel@tonic-gate { 18537c478bd9Sstevel@tonic-gate /* 1854dafb5540Sahl * Clean up the USDT provider. There may be active consumers of the 1855dafb5540Sahl * provider busy adding probes, no damage will actually befall the 1856dafb5540Sahl * provider until that count has dropped to zero. This just puts 1857dafb5540Sahl * the provider on death row. 18587c478bd9Sstevel@tonic-gate */ 1859dafb5540Sahl fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 18607c478bd9Sstevel@tonic-gate } 18617c478bd9Sstevel@tonic-gate 18627c478bd9Sstevel@tonic-gate static dtrace_mops_t fasttrap_mops = { 18637c478bd9Sstevel@tonic-gate fasttrap_meta_create_probe, 18647c478bd9Sstevel@tonic-gate fasttrap_meta_provide, 18657c478bd9Sstevel@tonic-gate fasttrap_meta_remove 18667c478bd9Sstevel@tonic-gate }; 18677c478bd9Sstevel@tonic-gate 18687c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 18697c478bd9Sstevel@tonic-gate static int 18707c478bd9Sstevel@tonic-gate fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 18717c478bd9Sstevel@tonic-gate { 18727c478bd9Sstevel@tonic-gate return (0); 18737c478bd9Sstevel@tonic-gate } 18747c478bd9Sstevel@tonic-gate 18757c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 18767c478bd9Sstevel@tonic-gate static int 18777c478bd9Sstevel@tonic-gate fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 18787c478bd9Sstevel@tonic-gate { 18797c478bd9Sstevel@tonic-gate if (!dtrace_attached()) 18807c478bd9Sstevel@tonic-gate return (EAGAIN); 18817c478bd9Sstevel@tonic-gate 18827c478bd9Sstevel@tonic-gate if (cmd == FASTTRAPIOC_MAKEPROBE) { 18837c478bd9Sstevel@tonic-gate fasttrap_probe_spec_t *uprobe = (void *)arg; 18847c478bd9Sstevel@tonic-gate fasttrap_probe_spec_t *probe; 18857c478bd9Sstevel@tonic-gate uint64_t noffs; 18867c478bd9Sstevel@tonic-gate size_t size; 18877c478bd9Sstevel@tonic-gate int ret; 18887c478bd9Sstevel@tonic-gate char *c; 18897c478bd9Sstevel@tonic-gate 18907c478bd9Sstevel@tonic-gate if (copyin(&uprobe->ftps_noffs, &noffs, 18917c478bd9Sstevel@tonic-gate sizeof (uprobe->ftps_noffs))) 18927c478bd9Sstevel@tonic-gate return (EFAULT); 18937c478bd9Sstevel@tonic-gate 18947c478bd9Sstevel@tonic-gate /* 18957c478bd9Sstevel@tonic-gate * Probes must have at least one tracepoint. 18967c478bd9Sstevel@tonic-gate */ 18977c478bd9Sstevel@tonic-gate if (noffs == 0) 18987c478bd9Sstevel@tonic-gate return (EINVAL); 18997c478bd9Sstevel@tonic-gate 19007c478bd9Sstevel@tonic-gate size = sizeof (fasttrap_probe_spec_t) + 19017c478bd9Sstevel@tonic-gate sizeof (probe->ftps_offs[0]) * (noffs - 1); 19027c478bd9Sstevel@tonic-gate 19037c478bd9Sstevel@tonic-gate if (size > 1024 * 1024) 19047c478bd9Sstevel@tonic-gate return (ENOMEM); 19057c478bd9Sstevel@tonic-gate 19067c478bd9Sstevel@tonic-gate probe = kmem_alloc(size, KM_SLEEP); 19077c478bd9Sstevel@tonic-gate 19087c478bd9Sstevel@tonic-gate if (copyin(uprobe, probe, size) != 0) { 19097c478bd9Sstevel@tonic-gate kmem_free(probe, size); 19107c478bd9Sstevel@tonic-gate return (EFAULT); 19117c478bd9Sstevel@tonic-gate } 19127c478bd9Sstevel@tonic-gate 19137c478bd9Sstevel@tonic-gate /* 19147c478bd9Sstevel@tonic-gate * Verify that the function and module strings contain no 19157c478bd9Sstevel@tonic-gate * funny characters. 19167c478bd9Sstevel@tonic-gate */ 19177c478bd9Sstevel@tonic-gate for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 19187c478bd9Sstevel@tonic-gate if (*c < 0x20 || 0x7f <= *c) { 19197c478bd9Sstevel@tonic-gate ret = EINVAL; 19207c478bd9Sstevel@tonic-gate goto err; 19217c478bd9Sstevel@tonic-gate } 19227c478bd9Sstevel@tonic-gate } 19237c478bd9Sstevel@tonic-gate 19247c478bd9Sstevel@tonic-gate for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 19257c478bd9Sstevel@tonic-gate if (*c < 0x20 || 0x7f <= *c) { 19267c478bd9Sstevel@tonic-gate ret = EINVAL; 19277c478bd9Sstevel@tonic-gate goto err; 19287c478bd9Sstevel@tonic-gate } 19297c478bd9Sstevel@tonic-gate } 19307c478bd9Sstevel@tonic-gate 19317c478bd9Sstevel@tonic-gate if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 19327c478bd9Sstevel@tonic-gate proc_t *p; 19337c478bd9Sstevel@tonic-gate pid_t pid = probe->ftps_pid; 19347c478bd9Sstevel@tonic-gate 19357c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 19367c478bd9Sstevel@tonic-gate /* 19377c478bd9Sstevel@tonic-gate * Report an error if the process doesn't exist 19387c478bd9Sstevel@tonic-gate * or is actively being birthed. 19397c478bd9Sstevel@tonic-gate */ 19407c478bd9Sstevel@tonic-gate if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 19417c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 19427c478bd9Sstevel@tonic-gate return (ESRCH); 19437c478bd9Sstevel@tonic-gate } 19447c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 19457c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 19467c478bd9Sstevel@tonic-gate 19477c478bd9Sstevel@tonic-gate if ((ret = priv_proc_cred_perm(cr, p, NULL, 19487c478bd9Sstevel@tonic-gate VREAD | VWRITE)) != 0) { 19497c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 19507c478bd9Sstevel@tonic-gate return (ret); 19517c478bd9Sstevel@tonic-gate } 19527c478bd9Sstevel@tonic-gate 19537c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 19547c478bd9Sstevel@tonic-gate } 19557c478bd9Sstevel@tonic-gate 19567c478bd9Sstevel@tonic-gate ret = fasttrap_add_probe(probe); 19577c478bd9Sstevel@tonic-gate err: 19587c478bd9Sstevel@tonic-gate kmem_free(probe, size); 19597c478bd9Sstevel@tonic-gate 19607c478bd9Sstevel@tonic-gate return (ret); 19617c478bd9Sstevel@tonic-gate 19627c478bd9Sstevel@tonic-gate } else if (cmd == FASTTRAPIOC_GETINSTR) { 19637c478bd9Sstevel@tonic-gate fasttrap_instr_query_t instr; 19647c478bd9Sstevel@tonic-gate fasttrap_tracepoint_t *tp; 19657c478bd9Sstevel@tonic-gate uint_t index; 19667c478bd9Sstevel@tonic-gate int ret; 19677c478bd9Sstevel@tonic-gate 19687c478bd9Sstevel@tonic-gate if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 19697c478bd9Sstevel@tonic-gate return (EFAULT); 19707c478bd9Sstevel@tonic-gate 19717c478bd9Sstevel@tonic-gate if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 19727c478bd9Sstevel@tonic-gate proc_t *p; 19737c478bd9Sstevel@tonic-gate pid_t pid = instr.ftiq_pid; 19747c478bd9Sstevel@tonic-gate 19757c478bd9Sstevel@tonic-gate mutex_enter(&pidlock); 19767c478bd9Sstevel@tonic-gate /* 19777c478bd9Sstevel@tonic-gate * Report an error if the process doesn't exist 19787c478bd9Sstevel@tonic-gate * or is actively being birthed. 19797c478bd9Sstevel@tonic-gate */ 19807c478bd9Sstevel@tonic-gate if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 19817c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 19827c478bd9Sstevel@tonic-gate return (ESRCH); 19837c478bd9Sstevel@tonic-gate } 19847c478bd9Sstevel@tonic-gate mutex_enter(&p->p_lock); 19857c478bd9Sstevel@tonic-gate mutex_exit(&pidlock); 19867c478bd9Sstevel@tonic-gate 19877c478bd9Sstevel@tonic-gate if ((ret = priv_proc_cred_perm(cr, p, NULL, 19887c478bd9Sstevel@tonic-gate VREAD)) != 0) { 19897c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 19907c478bd9Sstevel@tonic-gate return (ret); 19917c478bd9Sstevel@tonic-gate } 19927c478bd9Sstevel@tonic-gate 19937c478bd9Sstevel@tonic-gate mutex_exit(&p->p_lock); 19947c478bd9Sstevel@tonic-gate } 19957c478bd9Sstevel@tonic-gate 19967c478bd9Sstevel@tonic-gate index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 19977c478bd9Sstevel@tonic-gate 19987c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 19997c478bd9Sstevel@tonic-gate tp = fasttrap_tpoints.fth_table[index].ftb_data; 20007c478bd9Sstevel@tonic-gate while (tp != NULL) { 20017c478bd9Sstevel@tonic-gate if (instr.ftiq_pid == tp->ftt_pid && 20027c478bd9Sstevel@tonic-gate instr.ftiq_pc == tp->ftt_pc && 2003b096140dSahl !tp->ftt_proc->ftpc_defunct) 20047c478bd9Sstevel@tonic-gate break; 20057c478bd9Sstevel@tonic-gate 20067c478bd9Sstevel@tonic-gate tp = tp->ftt_next; 20077c478bd9Sstevel@tonic-gate } 20087c478bd9Sstevel@tonic-gate 20097c478bd9Sstevel@tonic-gate if (tp == NULL) { 20107c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 20117c478bd9Sstevel@tonic-gate return (ENOENT); 20127c478bd9Sstevel@tonic-gate } 20137c478bd9Sstevel@tonic-gate 20147c478bd9Sstevel@tonic-gate bcopy(&tp->ftt_instr, &instr.ftiq_instr, 20157c478bd9Sstevel@tonic-gate sizeof (instr.ftiq_instr)); 20167c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 20177c478bd9Sstevel@tonic-gate 20187c478bd9Sstevel@tonic-gate if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 20197c478bd9Sstevel@tonic-gate return (EFAULT); 20207c478bd9Sstevel@tonic-gate 20217c478bd9Sstevel@tonic-gate return (0); 20227c478bd9Sstevel@tonic-gate } 20237c478bd9Sstevel@tonic-gate 20247c478bd9Sstevel@tonic-gate return (EINVAL); 20257c478bd9Sstevel@tonic-gate } 20267c478bd9Sstevel@tonic-gate 20277c478bd9Sstevel@tonic-gate static struct cb_ops fasttrap_cb_ops = { 20287c478bd9Sstevel@tonic-gate fasttrap_open, /* open */ 20297c478bd9Sstevel@tonic-gate nodev, /* close */ 20307c478bd9Sstevel@tonic-gate nulldev, /* strategy */ 20317c478bd9Sstevel@tonic-gate nulldev, /* print */ 20327c478bd9Sstevel@tonic-gate nodev, /* dump */ 20337c478bd9Sstevel@tonic-gate nodev, /* read */ 20347c478bd9Sstevel@tonic-gate nodev, /* write */ 20357c478bd9Sstevel@tonic-gate fasttrap_ioctl, /* ioctl */ 20367c478bd9Sstevel@tonic-gate nodev, /* devmap */ 20377c478bd9Sstevel@tonic-gate nodev, /* mmap */ 20387c478bd9Sstevel@tonic-gate nodev, /* segmap */ 20397c478bd9Sstevel@tonic-gate nochpoll, /* poll */ 20407c478bd9Sstevel@tonic-gate ddi_prop_op, /* cb_prop_op */ 20417c478bd9Sstevel@tonic-gate 0, /* streamtab */ 20427c478bd9Sstevel@tonic-gate D_NEW | D_MP /* Driver compatibility flag */ 20437c478bd9Sstevel@tonic-gate }; 20447c478bd9Sstevel@tonic-gate 20457c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20467c478bd9Sstevel@tonic-gate static int 20477c478bd9Sstevel@tonic-gate fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 20487c478bd9Sstevel@tonic-gate { 20497c478bd9Sstevel@tonic-gate int error; 20507c478bd9Sstevel@tonic-gate 20517c478bd9Sstevel@tonic-gate switch (infocmd) { 20527c478bd9Sstevel@tonic-gate case DDI_INFO_DEVT2DEVINFO: 20537c478bd9Sstevel@tonic-gate *result = (void *)fasttrap_devi; 20547c478bd9Sstevel@tonic-gate error = DDI_SUCCESS; 20557c478bd9Sstevel@tonic-gate break; 20567c478bd9Sstevel@tonic-gate case DDI_INFO_DEVT2INSTANCE: 20577c478bd9Sstevel@tonic-gate *result = (void *)0; 20587c478bd9Sstevel@tonic-gate error = DDI_SUCCESS; 20597c478bd9Sstevel@tonic-gate break; 20607c478bd9Sstevel@tonic-gate default: 20617c478bd9Sstevel@tonic-gate error = DDI_FAILURE; 20627c478bd9Sstevel@tonic-gate } 20637c478bd9Sstevel@tonic-gate return (error); 20647c478bd9Sstevel@tonic-gate } 20657c478bd9Sstevel@tonic-gate 20667c478bd9Sstevel@tonic-gate static int 20677c478bd9Sstevel@tonic-gate fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 20687c478bd9Sstevel@tonic-gate { 20697c478bd9Sstevel@tonic-gate ulong_t nent; 20707c478bd9Sstevel@tonic-gate 20717c478bd9Sstevel@tonic-gate switch (cmd) { 20727c478bd9Sstevel@tonic-gate case DDI_ATTACH: 20737c478bd9Sstevel@tonic-gate break; 20747c478bd9Sstevel@tonic-gate case DDI_RESUME: 20757c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 20767c478bd9Sstevel@tonic-gate default: 20777c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 20787c478bd9Sstevel@tonic-gate } 20797c478bd9Sstevel@tonic-gate 20807c478bd9Sstevel@tonic-gate if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 20817c478bd9Sstevel@tonic-gate DDI_PSEUDO, NULL) == DDI_FAILURE || 2082ad4023c4Sdp dtrace_register("fasttrap", &fasttrap_attr, DTRACE_PRIV_USER, NULL, 20837c478bd9Sstevel@tonic-gate &fasttrap_pops, NULL, &fasttrap_id) != 0) { 20847c478bd9Sstevel@tonic-gate ddi_remove_minor_node(devi, NULL); 20857c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 20867c478bd9Sstevel@tonic-gate } 20877c478bd9Sstevel@tonic-gate 20887c478bd9Sstevel@tonic-gate ddi_report_dev(devi); 20897c478bd9Sstevel@tonic-gate fasttrap_devi = devi; 20907c478bd9Sstevel@tonic-gate 20917c478bd9Sstevel@tonic-gate /* 20927c478bd9Sstevel@tonic-gate * Install our hooks into fork(2), exec(2), and exit(2). 20937c478bd9Sstevel@tonic-gate */ 20947c478bd9Sstevel@tonic-gate dtrace_fasttrap_fork_ptr = &fasttrap_fork; 20957c478bd9Sstevel@tonic-gate dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 20967c478bd9Sstevel@tonic-gate dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 20977c478bd9Sstevel@tonic-gate 20987c478bd9Sstevel@tonic-gate fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 20997c478bd9Sstevel@tonic-gate "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 21007c478bd9Sstevel@tonic-gate fasttrap_total = 0; 21017c478bd9Sstevel@tonic-gate 21027c478bd9Sstevel@tonic-gate /* 21037c478bd9Sstevel@tonic-gate * Conjure up the tracepoints hashtable... 21047c478bd9Sstevel@tonic-gate */ 21057c478bd9Sstevel@tonic-gate nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 21067c478bd9Sstevel@tonic-gate "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 21077c478bd9Sstevel@tonic-gate 21087c478bd9Sstevel@tonic-gate if (nent <= 0 || nent > 0x1000000) 21097c478bd9Sstevel@tonic-gate nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 21107c478bd9Sstevel@tonic-gate 21117c478bd9Sstevel@tonic-gate if ((nent & (nent - 1)) == 0) 21127c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent = nent; 21137c478bd9Sstevel@tonic-gate else 21147c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 21157c478bd9Sstevel@tonic-gate ASSERT(fasttrap_tpoints.fth_nent > 0); 21167c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 21177c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 21187c478bd9Sstevel@tonic-gate sizeof (fasttrap_bucket_t), KM_SLEEP); 21197c478bd9Sstevel@tonic-gate 21207c478bd9Sstevel@tonic-gate /* 2121b096140dSahl * ... and the providers hash table... 21227c478bd9Sstevel@tonic-gate */ 21237c478bd9Sstevel@tonic-gate nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 21247c478bd9Sstevel@tonic-gate if ((nent & (nent - 1)) == 0) 21257c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent = nent; 21267c478bd9Sstevel@tonic-gate else 21277c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 21287c478bd9Sstevel@tonic-gate ASSERT(fasttrap_provs.fth_nent > 0); 21297c478bd9Sstevel@tonic-gate fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 21307c478bd9Sstevel@tonic-gate fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 21317c478bd9Sstevel@tonic-gate sizeof (fasttrap_bucket_t), KM_SLEEP); 21327c478bd9Sstevel@tonic-gate 2133b096140dSahl /* 2134b096140dSahl * ... and the procs hash table. 2135b096140dSahl */ 2136b096140dSahl nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2137b096140dSahl if ((nent & (nent - 1)) == 0) 2138b096140dSahl fasttrap_procs.fth_nent = nent; 2139b096140dSahl else 2140b096140dSahl fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2141b096140dSahl ASSERT(fasttrap_procs.fth_nent > 0); 2142b096140dSahl fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2143b096140dSahl fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2144b096140dSahl sizeof (fasttrap_bucket_t), KM_SLEEP); 2145b096140dSahl 21467c478bd9Sstevel@tonic-gate (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 21477c478bd9Sstevel@tonic-gate &fasttrap_meta_id); 21487c478bd9Sstevel@tonic-gate 21497c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 21507c478bd9Sstevel@tonic-gate } 21517c478bd9Sstevel@tonic-gate 21527c478bd9Sstevel@tonic-gate static int 21537c478bd9Sstevel@tonic-gate fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 21547c478bd9Sstevel@tonic-gate { 21557c478bd9Sstevel@tonic-gate int i, fail = 0; 21567c478bd9Sstevel@tonic-gate timeout_id_t tmp; 21577c478bd9Sstevel@tonic-gate 21587c478bd9Sstevel@tonic-gate switch (cmd) { 21597c478bd9Sstevel@tonic-gate case DDI_DETACH: 21607c478bd9Sstevel@tonic-gate break; 21617c478bd9Sstevel@tonic-gate case DDI_SUSPEND: 21627c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 21637c478bd9Sstevel@tonic-gate default: 21647c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 21657c478bd9Sstevel@tonic-gate } 21667c478bd9Sstevel@tonic-gate 21677c478bd9Sstevel@tonic-gate /* 21687c478bd9Sstevel@tonic-gate * Unregister the meta-provider to make sure no new fasttrap- 21697c478bd9Sstevel@tonic-gate * managed providers come along while we're trying to close up 21707c478bd9Sstevel@tonic-gate * shop. If we fail to detach, we'll need to re-register as a 21717c478bd9Sstevel@tonic-gate * meta-provider. We can fail to unregister as a meta-provider 21727c478bd9Sstevel@tonic-gate * if providers we manage still exist. 21737c478bd9Sstevel@tonic-gate */ 21747c478bd9Sstevel@tonic-gate if (fasttrap_meta_id != DTRACE_METAPROVNONE && 21757c478bd9Sstevel@tonic-gate dtrace_meta_unregister(fasttrap_meta_id) != 0) 21767c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 21777c478bd9Sstevel@tonic-gate 21787c478bd9Sstevel@tonic-gate /* 21797c478bd9Sstevel@tonic-gate * Prevent any new timeouts from running by setting fasttrap_timeout 21807c478bd9Sstevel@tonic-gate * to a non-zero value, and wait for the current timeout to complete. 21817c478bd9Sstevel@tonic-gate */ 21827c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 21837c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 0; 21847c478bd9Sstevel@tonic-gate 21857c478bd9Sstevel@tonic-gate while (fasttrap_timeout != (timeout_id_t)1) { 21867c478bd9Sstevel@tonic-gate tmp = fasttrap_timeout; 21877c478bd9Sstevel@tonic-gate fasttrap_timeout = (timeout_id_t)1; 21887c478bd9Sstevel@tonic-gate 21897c478bd9Sstevel@tonic-gate if (tmp != 0) { 21907c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 21917c478bd9Sstevel@tonic-gate (void) untimeout(tmp); 21927c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 21937c478bd9Sstevel@tonic-gate } 21947c478bd9Sstevel@tonic-gate } 21957c478bd9Sstevel@tonic-gate 21967c478bd9Sstevel@tonic-gate fasttrap_cleanup_work = 0; 21977c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 21987c478bd9Sstevel@tonic-gate 21997c478bd9Sstevel@tonic-gate /* 22007c478bd9Sstevel@tonic-gate * Iterate over all of our providers. If there's still a process 22017c478bd9Sstevel@tonic-gate * that corresponds to that pid, fail to detach. 22027c478bd9Sstevel@tonic-gate */ 22037c478bd9Sstevel@tonic-gate for (i = 0; i < fasttrap_provs.fth_nent; i++) { 22047c478bd9Sstevel@tonic-gate fasttrap_provider_t **fpp, *fp; 22057c478bd9Sstevel@tonic-gate fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 22067c478bd9Sstevel@tonic-gate 22077c478bd9Sstevel@tonic-gate mutex_enter(&bucket->ftb_mtx); 22087c478bd9Sstevel@tonic-gate fpp = (fasttrap_provider_t **)&bucket->ftb_data; 22097c478bd9Sstevel@tonic-gate while ((fp = *fpp) != NULL) { 22107c478bd9Sstevel@tonic-gate /* 22117c478bd9Sstevel@tonic-gate * Acquire and release the lock as a simple way of 22127c478bd9Sstevel@tonic-gate * waiting for any other consumer to finish with 22137c478bd9Sstevel@tonic-gate * this provider. A thread must first acquire the 22147c478bd9Sstevel@tonic-gate * bucket lock so there's no chance of another thread 2215dafb5540Sahl * blocking on the provider's lock. 22167c478bd9Sstevel@tonic-gate */ 22177c478bd9Sstevel@tonic-gate mutex_enter(&fp->ftp_mtx); 22187c478bd9Sstevel@tonic-gate mutex_exit(&fp->ftp_mtx); 22197c478bd9Sstevel@tonic-gate 22207c478bd9Sstevel@tonic-gate if (dtrace_unregister(fp->ftp_provid) != 0) { 22217c478bd9Sstevel@tonic-gate fail = 1; 22227c478bd9Sstevel@tonic-gate fpp = &fp->ftp_next; 22237c478bd9Sstevel@tonic-gate } else { 22247c478bd9Sstevel@tonic-gate *fpp = fp->ftp_next; 22257c478bd9Sstevel@tonic-gate fasttrap_provider_free(fp); 22267c478bd9Sstevel@tonic-gate } 22277c478bd9Sstevel@tonic-gate } 22287c478bd9Sstevel@tonic-gate 22297c478bd9Sstevel@tonic-gate mutex_exit(&bucket->ftb_mtx); 22307c478bd9Sstevel@tonic-gate } 22317c478bd9Sstevel@tonic-gate 22327c478bd9Sstevel@tonic-gate if (fail || dtrace_unregister(fasttrap_id) != 0) { 22337c478bd9Sstevel@tonic-gate uint_t work; 22347c478bd9Sstevel@tonic-gate /* 22357c478bd9Sstevel@tonic-gate * If we're failing to detach, we need to unblock timeouts 22367c478bd9Sstevel@tonic-gate * and start a new timeout if any work has accumulated while 22377c478bd9Sstevel@tonic-gate * we've been unsuccessfully trying to detach. 22387c478bd9Sstevel@tonic-gate */ 22397c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_cleanup_mtx); 22407c478bd9Sstevel@tonic-gate fasttrap_timeout = 0; 22417c478bd9Sstevel@tonic-gate work = fasttrap_cleanup_work; 22427c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_cleanup_mtx); 22437c478bd9Sstevel@tonic-gate 22447c478bd9Sstevel@tonic-gate if (work) 22457c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(); 22467c478bd9Sstevel@tonic-gate 22477c478bd9Sstevel@tonic-gate (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 22487c478bd9Sstevel@tonic-gate &fasttrap_meta_id); 22497c478bd9Sstevel@tonic-gate 22507c478bd9Sstevel@tonic-gate return (DDI_FAILURE); 22517c478bd9Sstevel@tonic-gate } 22527c478bd9Sstevel@tonic-gate 22537c478bd9Sstevel@tonic-gate #ifdef DEBUG 22547c478bd9Sstevel@tonic-gate mutex_enter(&fasttrap_count_mtx); 22557c478bd9Sstevel@tonic-gate ASSERT(fasttrap_count == 0); 22567c478bd9Sstevel@tonic-gate mutex_exit(&fasttrap_count_mtx); 22577c478bd9Sstevel@tonic-gate #endif 22587c478bd9Sstevel@tonic-gate 22597c478bd9Sstevel@tonic-gate kmem_free(fasttrap_tpoints.fth_table, 22607c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 22617c478bd9Sstevel@tonic-gate fasttrap_tpoints.fth_nent = 0; 22627c478bd9Sstevel@tonic-gate 22637c478bd9Sstevel@tonic-gate kmem_free(fasttrap_provs.fth_table, 22647c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 22657c478bd9Sstevel@tonic-gate fasttrap_provs.fth_nent = 0; 22667c478bd9Sstevel@tonic-gate 2267b096140dSahl kmem_free(fasttrap_procs.fth_table, 2268b096140dSahl fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2269b096140dSahl fasttrap_procs.fth_nent = 0; 2270b096140dSahl 22717c478bd9Sstevel@tonic-gate /* 22727c478bd9Sstevel@tonic-gate * We know there are no tracepoints in any process anywhere in 22737c478bd9Sstevel@tonic-gate * the system so there is no process which has its p_dtrace_count 22747c478bd9Sstevel@tonic-gate * greater than zero, therefore we know that no thread can actively 22757c478bd9Sstevel@tonic-gate * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 22767c478bd9Sstevel@tonic-gate * and fasttrap_exec() and fasttrap_exit(). 22777c478bd9Sstevel@tonic-gate */ 22787c478bd9Sstevel@tonic-gate ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 22797c478bd9Sstevel@tonic-gate dtrace_fasttrap_fork_ptr = NULL; 22807c478bd9Sstevel@tonic-gate 22817c478bd9Sstevel@tonic-gate ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 22827c478bd9Sstevel@tonic-gate dtrace_fasttrap_exec_ptr = NULL; 22837c478bd9Sstevel@tonic-gate 22847c478bd9Sstevel@tonic-gate ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 22857c478bd9Sstevel@tonic-gate dtrace_fasttrap_exit_ptr = NULL; 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate ddi_remove_minor_node(devi, NULL); 22887c478bd9Sstevel@tonic-gate 22897c478bd9Sstevel@tonic-gate return (DDI_SUCCESS); 22907c478bd9Sstevel@tonic-gate } 22917c478bd9Sstevel@tonic-gate 22927c478bd9Sstevel@tonic-gate static struct dev_ops fasttrap_ops = { 22937c478bd9Sstevel@tonic-gate DEVO_REV, /* devo_rev */ 22947c478bd9Sstevel@tonic-gate 0, /* refcnt */ 22957c478bd9Sstevel@tonic-gate fasttrap_info, /* get_dev_info */ 22967c478bd9Sstevel@tonic-gate nulldev, /* identify */ 22977c478bd9Sstevel@tonic-gate nulldev, /* probe */ 22987c478bd9Sstevel@tonic-gate fasttrap_attach, /* attach */ 22997c478bd9Sstevel@tonic-gate fasttrap_detach, /* detach */ 23007c478bd9Sstevel@tonic-gate nodev, /* reset */ 23017c478bd9Sstevel@tonic-gate &fasttrap_cb_ops, /* driver operations */ 23027c478bd9Sstevel@tonic-gate NULL, /* bus operations */ 23037c478bd9Sstevel@tonic-gate nodev /* dev power */ 23047c478bd9Sstevel@tonic-gate }; 23057c478bd9Sstevel@tonic-gate 23067c478bd9Sstevel@tonic-gate /* 23077c478bd9Sstevel@tonic-gate * Module linkage information for the kernel. 23087c478bd9Sstevel@tonic-gate */ 23097c478bd9Sstevel@tonic-gate static struct modldrv modldrv = { 23107c478bd9Sstevel@tonic-gate &mod_driverops, /* module type (this is a pseudo driver) */ 23117c478bd9Sstevel@tonic-gate "Fasttrap Tracing", /* name of module */ 23127c478bd9Sstevel@tonic-gate &fasttrap_ops, /* driver ops */ 23137c478bd9Sstevel@tonic-gate }; 23147c478bd9Sstevel@tonic-gate 23157c478bd9Sstevel@tonic-gate static struct modlinkage modlinkage = { 23167c478bd9Sstevel@tonic-gate MODREV_1, 23177c478bd9Sstevel@tonic-gate (void *)&modldrv, 23187c478bd9Sstevel@tonic-gate NULL 23197c478bd9Sstevel@tonic-gate }; 23207c478bd9Sstevel@tonic-gate 23217c478bd9Sstevel@tonic-gate int 23227c478bd9Sstevel@tonic-gate _init(void) 23237c478bd9Sstevel@tonic-gate { 23247c478bd9Sstevel@tonic-gate return (mod_install(&modlinkage)); 23257c478bd9Sstevel@tonic-gate } 23267c478bd9Sstevel@tonic-gate 23277c478bd9Sstevel@tonic-gate int 23287c478bd9Sstevel@tonic-gate _info(struct modinfo *modinfop) 23297c478bd9Sstevel@tonic-gate { 23307c478bd9Sstevel@tonic-gate return (mod_info(&modlinkage, modinfop)); 23317c478bd9Sstevel@tonic-gate } 23327c478bd9Sstevel@tonic-gate 23337c478bd9Sstevel@tonic-gate int 23347c478bd9Sstevel@tonic-gate _fini(void) 23357c478bd9Sstevel@tonic-gate { 23367c478bd9Sstevel@tonic-gate return (mod_remove(&modlinkage)); 23377c478bd9Sstevel@tonic-gate } 2338