xref: /titanic_44/usr/src/uts/common/dtrace/fasttrap.c (revision ab9a77c71b58e388d02c6199c8bfbcb998aae845)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ad4023c4Sdp  * Common Development and Distribution License (the "License").
6ad4023c4Sdp  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
2197eda132Sraf 
227c478bd9Sstevel@tonic-gate /*
23ad4023c4Sdp  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
307c478bd9Sstevel@tonic-gate #include <sys/errno.h>
317c478bd9Sstevel@tonic-gate #include <sys/stat.h>
327c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
337c478bd9Sstevel@tonic-gate #include <sys/conf.h>
347c478bd9Sstevel@tonic-gate #include <sys/systm.h>
357c478bd9Sstevel@tonic-gate #include <sys/ddi.h>
367c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
377c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
387c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
397c478bd9Sstevel@tonic-gate #include <sys/strsubr.h>
407c478bd9Sstevel@tonic-gate #include <sys/fasttrap.h>
417c478bd9Sstevel@tonic-gate #include <sys/fasttrap_impl.h>
427c478bd9Sstevel@tonic-gate #include <sys/fasttrap_isa.h>
437c478bd9Sstevel@tonic-gate #include <sys/dtrace.h>
447c478bd9Sstevel@tonic-gate #include <sys/dtrace_impl.h>
457c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
467c478bd9Sstevel@tonic-gate #include <sys/frame.h>
477c478bd9Sstevel@tonic-gate #include <sys/stack.h>
487c478bd9Sstevel@tonic-gate #include <sys/proc.h>
497c478bd9Sstevel@tonic-gate #include <sys/priv.h>
507c478bd9Sstevel@tonic-gate #include <sys/policy.h>
517c478bd9Sstevel@tonic-gate #include <sys/ontrap.h>
527c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h>
537c478bd9Sstevel@tonic-gate #include <sys/prsystm.h>
547c478bd9Sstevel@tonic-gate 
557c478bd9Sstevel@tonic-gate #include <vm/as.h>
567c478bd9Sstevel@tonic-gate #include <vm/seg.h>
577c478bd9Sstevel@tonic-gate #include <vm/seg_dev.h>
587c478bd9Sstevel@tonic-gate #include <vm/seg_vn.h>
597c478bd9Sstevel@tonic-gate #include <vm/seg_spt.h>
607c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
617c478bd9Sstevel@tonic-gate 
627c478bd9Sstevel@tonic-gate /*
637c478bd9Sstevel@tonic-gate  * User-Land Trap-Based Tracing
647c478bd9Sstevel@tonic-gate  * ----------------------------
657c478bd9Sstevel@tonic-gate  *
667c478bd9Sstevel@tonic-gate  * The fasttrap provider allows DTrace consumers to instrument any user-level
677c478bd9Sstevel@tonic-gate  * instruction to gather data; this includes probes with semantic
687c478bd9Sstevel@tonic-gate  * signifigance like entry and return as well as simple offsets into the
697c478bd9Sstevel@tonic-gate  * function. While the specific techniques used are very ISA specific, the
707c478bd9Sstevel@tonic-gate  * methodology is generalizable to any architecture.
717c478bd9Sstevel@tonic-gate  *
727c478bd9Sstevel@tonic-gate  *
737c478bd9Sstevel@tonic-gate  * The General Methodology
747c478bd9Sstevel@tonic-gate  * -----------------------
757c478bd9Sstevel@tonic-gate  *
767c478bd9Sstevel@tonic-gate  * With the primary goal of tracing every user-land instruction and the
777c478bd9Sstevel@tonic-gate  * limitation that we can't trust user space so don't want to rely on much
787c478bd9Sstevel@tonic-gate  * information there, we begin by replacing the instructions we want to trace
797c478bd9Sstevel@tonic-gate  * with trap instructions. Each instruction we overwrite is saved into a hash
807c478bd9Sstevel@tonic-gate  * table keyed by process ID and pc address. When we enter the kernel due to
817c478bd9Sstevel@tonic-gate  * this trap instruction, we need the effects of the replaced instruction to
827c478bd9Sstevel@tonic-gate  * appear to have occurred before we proceed with the user thread's
837c478bd9Sstevel@tonic-gate  * execution.
847c478bd9Sstevel@tonic-gate  *
857c478bd9Sstevel@tonic-gate  * Each user level thread is represented by a ulwp_t structure which is
867c478bd9Sstevel@tonic-gate  * always easily accessible through a register. The most basic way to produce
877c478bd9Sstevel@tonic-gate  * the effects of the instruction we replaced is to copy that instruction out
887c478bd9Sstevel@tonic-gate  * to a bit of scratch space reserved in the user thread's ulwp_t structure
897c478bd9Sstevel@tonic-gate  * (a sort of kernel-private thread local storage), set the PC to that
907c478bd9Sstevel@tonic-gate  * scratch space and single step. When we reenter the kernel after single
917c478bd9Sstevel@tonic-gate  * stepping the instruction we must then adjust the PC to point to what would
927c478bd9Sstevel@tonic-gate  * normally be the next instruction. Of course, special care must be taken
937c478bd9Sstevel@tonic-gate  * for branches and jumps, but these represent such a small fraction of any
947c478bd9Sstevel@tonic-gate  * instruction set that writing the code to emulate these in the kernel is
957c478bd9Sstevel@tonic-gate  * not too difficult.
967c478bd9Sstevel@tonic-gate  *
977c478bd9Sstevel@tonic-gate  * Return probes may require several tracepoints to trace every return site,
987c478bd9Sstevel@tonic-gate  * and, conversely, each tracepoint may activate several probes (the entry
997c478bd9Sstevel@tonic-gate  * and offset 0 probes, for example). To solve this muliplexing problem,
1007c478bd9Sstevel@tonic-gate  * tracepoints contain lists of probes to activate and probes contain lists
1017c478bd9Sstevel@tonic-gate  * of tracepoints to enable. If a probe is activated, it adds its ID to
1027c478bd9Sstevel@tonic-gate  * existing tracepoints or creates new ones as necessary.
1037c478bd9Sstevel@tonic-gate  *
1047c478bd9Sstevel@tonic-gate  * Most probes are activated _before_ the instruction is executed, but return
1057c478bd9Sstevel@tonic-gate  * probes are activated _after_ the effects of the last instruction of the
1067c478bd9Sstevel@tonic-gate  * function are visible. Return probes must be fired _after_ we have
1077c478bd9Sstevel@tonic-gate  * single-stepped the instruction whereas all other probes are fired
1087c478bd9Sstevel@tonic-gate  * beforehand.
1097c478bd9Sstevel@tonic-gate  */
1107c478bd9Sstevel@tonic-gate 
1117c478bd9Sstevel@tonic-gate static dev_info_t *fasttrap_devi;
1127c478bd9Sstevel@tonic-gate static dtrace_provider_id_t fasttrap_id;
1137c478bd9Sstevel@tonic-gate static dtrace_meta_provider_id_t fasttrap_meta_id;
1147c478bd9Sstevel@tonic-gate 
1157c478bd9Sstevel@tonic-gate static timeout_id_t fasttrap_timeout;
1167c478bd9Sstevel@tonic-gate static kmutex_t fasttrap_cleanup_mtx;
1177c478bd9Sstevel@tonic-gate static uint_t fasttrap_cleanup_work;
1187c478bd9Sstevel@tonic-gate 
1197c478bd9Sstevel@tonic-gate /*
1207c478bd9Sstevel@tonic-gate  * Generation count on modifications to the global tracepoint lookup table.
1217c478bd9Sstevel@tonic-gate  */
1227c478bd9Sstevel@tonic-gate static volatile uint64_t fasttrap_mod_gen;
1237c478bd9Sstevel@tonic-gate 
1247c478bd9Sstevel@tonic-gate /*
1257c478bd9Sstevel@tonic-gate  * When the fasttrap provider is loaded, fasttrap_max is set to either
1267c478bd9Sstevel@tonic-gate  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
1277c478bd9Sstevel@tonic-gate  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
1287c478bd9Sstevel@tonic-gate  * incremented by the number of tracepoints that may be associated with that
1297c478bd9Sstevel@tonic-gate  * probe; fasttrap_total is capped at fasttrap_max.
1307c478bd9Sstevel@tonic-gate  */
1317c478bd9Sstevel@tonic-gate #define	FASTTRAP_MAX_DEFAULT		250000
1327c478bd9Sstevel@tonic-gate static uint32_t fasttrap_max;
1337c478bd9Sstevel@tonic-gate static uint32_t fasttrap_total;
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate 
1367c478bd9Sstevel@tonic-gate #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
1377c478bd9Sstevel@tonic-gate #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
138b096140dSahl #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate #define	FASTTRAP_PID_NAME		"pid"
1417c478bd9Sstevel@tonic-gate 
1427c478bd9Sstevel@tonic-gate fasttrap_hash_t			fasttrap_tpoints;
1437c478bd9Sstevel@tonic-gate static fasttrap_hash_t		fasttrap_provs;
144b096140dSahl static fasttrap_hash_t		fasttrap_procs;
1457c478bd9Sstevel@tonic-gate 
1467c478bd9Sstevel@tonic-gate dtrace_id_t			fasttrap_probe_id;
1477c478bd9Sstevel@tonic-gate static int			fasttrap_count;		/* ref count */
1487c478bd9Sstevel@tonic-gate static int			fasttrap_pid_count;	/* pid ref count */
1497c478bd9Sstevel@tonic-gate static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
1507c478bd9Sstevel@tonic-gate 
1518f5db3e7Sahl #define	FASTTRAP_ENABLE_FAIL	1
1528f5db3e7Sahl #define	FASTTRAP_ENABLE_PARTIAL	2
1538f5db3e7Sahl 
1547c478bd9Sstevel@tonic-gate static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
1557c478bd9Sstevel@tonic-gate static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
1567c478bd9Sstevel@tonic-gate 
1577c478bd9Sstevel@tonic-gate static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
1587c478bd9Sstevel@tonic-gate     const dtrace_pattr_t *);
159dafb5540Sahl static void fasttrap_provider_retire(pid_t, const char *, int);
1607c478bd9Sstevel@tonic-gate static void fasttrap_provider_free(fasttrap_provider_t *);
1617c478bd9Sstevel@tonic-gate 
162b096140dSahl static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
163b096140dSahl static void fasttrap_proc_release(fasttrap_proc_t *);
164b096140dSahl 
1657c478bd9Sstevel@tonic-gate #define	FASTTRAP_PROVS_INDEX(pid, name) \
1667c478bd9Sstevel@tonic-gate 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
1677c478bd9Sstevel@tonic-gate 
168b096140dSahl #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
169b096140dSahl 
1707c478bd9Sstevel@tonic-gate static int
1717c478bd9Sstevel@tonic-gate fasttrap_highbit(ulong_t i)
1727c478bd9Sstevel@tonic-gate {
1737c478bd9Sstevel@tonic-gate 	int h = 1;
1747c478bd9Sstevel@tonic-gate 
1757c478bd9Sstevel@tonic-gate 	if (i == 0)
1767c478bd9Sstevel@tonic-gate 		return (0);
1777c478bd9Sstevel@tonic-gate #ifdef _LP64
1787c478bd9Sstevel@tonic-gate 	if (i & 0xffffffff00000000ul) {
1797c478bd9Sstevel@tonic-gate 		h += 32; i >>= 32;
1807c478bd9Sstevel@tonic-gate 	}
1817c478bd9Sstevel@tonic-gate #endif
1827c478bd9Sstevel@tonic-gate 	if (i & 0xffff0000) {
1837c478bd9Sstevel@tonic-gate 		h += 16; i >>= 16;
1847c478bd9Sstevel@tonic-gate 	}
1857c478bd9Sstevel@tonic-gate 	if (i & 0xff00) {
1867c478bd9Sstevel@tonic-gate 		h += 8; i >>= 8;
1877c478bd9Sstevel@tonic-gate 	}
1887c478bd9Sstevel@tonic-gate 	if (i & 0xf0) {
1897c478bd9Sstevel@tonic-gate 		h += 4; i >>= 4;
1907c478bd9Sstevel@tonic-gate 	}
1917c478bd9Sstevel@tonic-gate 	if (i & 0xc) {
1927c478bd9Sstevel@tonic-gate 		h += 2; i >>= 2;
1937c478bd9Sstevel@tonic-gate 	}
1947c478bd9Sstevel@tonic-gate 	if (i & 0x2) {
1957c478bd9Sstevel@tonic-gate 		h += 1;
1967c478bd9Sstevel@tonic-gate 	}
1977c478bd9Sstevel@tonic-gate 	return (h);
1987c478bd9Sstevel@tonic-gate }
1997c478bd9Sstevel@tonic-gate 
2007c478bd9Sstevel@tonic-gate static uint_t
2017c478bd9Sstevel@tonic-gate fasttrap_hash_str(const char *p)
2027c478bd9Sstevel@tonic-gate {
2037c478bd9Sstevel@tonic-gate 	unsigned int g;
2047c478bd9Sstevel@tonic-gate 	uint_t hval = 0;
2057c478bd9Sstevel@tonic-gate 
2067c478bd9Sstevel@tonic-gate 	while (*p) {
2077c478bd9Sstevel@tonic-gate 		hval = (hval << 4) + *p++;
2087c478bd9Sstevel@tonic-gate 		if ((g = (hval & 0xf0000000)) != 0)
2097c478bd9Sstevel@tonic-gate 			hval ^= g >> 24;
2107c478bd9Sstevel@tonic-gate 		hval &= ~g;
2117c478bd9Sstevel@tonic-gate 	}
2127c478bd9Sstevel@tonic-gate 	return (hval);
2137c478bd9Sstevel@tonic-gate }
2147c478bd9Sstevel@tonic-gate 
2157c478bd9Sstevel@tonic-gate void
2167c478bd9Sstevel@tonic-gate fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
2177c478bd9Sstevel@tonic-gate {
2187c478bd9Sstevel@tonic-gate 	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
2197c478bd9Sstevel@tonic-gate 
2207c478bd9Sstevel@tonic-gate 	sqp->sq_info.si_signo = SIGTRAP;
2217c478bd9Sstevel@tonic-gate 	sqp->sq_info.si_code = TRAP_DTRACE;
2227c478bd9Sstevel@tonic-gate 	sqp->sq_info.si_addr = (caddr_t)pc;
2237c478bd9Sstevel@tonic-gate 
2247c478bd9Sstevel@tonic-gate 	mutex_enter(&p->p_lock);
2257c478bd9Sstevel@tonic-gate 	sigaddqa(p, t, sqp);
2267c478bd9Sstevel@tonic-gate 	mutex_exit(&p->p_lock);
2277c478bd9Sstevel@tonic-gate 
2287c478bd9Sstevel@tonic-gate 	if (t != NULL)
2297c478bd9Sstevel@tonic-gate 		aston(t);
2307c478bd9Sstevel@tonic-gate }
2317c478bd9Sstevel@tonic-gate 
2327c478bd9Sstevel@tonic-gate /*
2337c478bd9Sstevel@tonic-gate  * This function ensures that no threads are actively using the memory
2347c478bd9Sstevel@tonic-gate  * associated with probes that were formerly live.
2357c478bd9Sstevel@tonic-gate  */
2367c478bd9Sstevel@tonic-gate static void
2377c478bd9Sstevel@tonic-gate fasttrap_mod_barrier(uint64_t gen)
2387c478bd9Sstevel@tonic-gate {
2397c478bd9Sstevel@tonic-gate 	int i;
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate 	if (gen < fasttrap_mod_gen)
2427c478bd9Sstevel@tonic-gate 		return;
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	fasttrap_mod_gen++;
2457c478bd9Sstevel@tonic-gate 
2467c478bd9Sstevel@tonic-gate 	for (i = 0; i < NCPU; i++) {
2477c478bd9Sstevel@tonic-gate 		mutex_enter(&cpu_core[i].cpuc_pid_lock);
2487c478bd9Sstevel@tonic-gate 		mutex_exit(&cpu_core[i].cpuc_pid_lock);
2497c478bd9Sstevel@tonic-gate 	}
2507c478bd9Sstevel@tonic-gate }
2517c478bd9Sstevel@tonic-gate 
2527c478bd9Sstevel@tonic-gate /*
2537c478bd9Sstevel@tonic-gate  * This is the timeout's callback for cleaning up the providers and their
2547c478bd9Sstevel@tonic-gate  * probes.
2557c478bd9Sstevel@tonic-gate  */
2567c478bd9Sstevel@tonic-gate /*ARGSUSED*/
2577c478bd9Sstevel@tonic-gate static void
2587c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup_cb(void *data)
2597c478bd9Sstevel@tonic-gate {
2607c478bd9Sstevel@tonic-gate 	fasttrap_provider_t **fpp, *fp;
2617c478bd9Sstevel@tonic-gate 	fasttrap_bucket_t *bucket;
2627c478bd9Sstevel@tonic-gate 	dtrace_provider_id_t provid;
2637c478bd9Sstevel@tonic-gate 	int i, later;
2647c478bd9Sstevel@tonic-gate 
2657c478bd9Sstevel@tonic-gate 	static volatile int in = 0;
2667c478bd9Sstevel@tonic-gate 	ASSERT(in == 0);
2677c478bd9Sstevel@tonic-gate 	in = 1;
2687c478bd9Sstevel@tonic-gate 
2697c478bd9Sstevel@tonic-gate 	mutex_enter(&fasttrap_cleanup_mtx);
2707c478bd9Sstevel@tonic-gate 	while (fasttrap_cleanup_work) {
2717c478bd9Sstevel@tonic-gate 		fasttrap_cleanup_work = 0;
2727c478bd9Sstevel@tonic-gate 		mutex_exit(&fasttrap_cleanup_mtx);
2737c478bd9Sstevel@tonic-gate 
2747c478bd9Sstevel@tonic-gate 		later = 0;
2757c478bd9Sstevel@tonic-gate 
2767c478bd9Sstevel@tonic-gate 		/*
2777c478bd9Sstevel@tonic-gate 		 * Iterate over all the providers trying to remove the marked
278b096140dSahl 		 * ones. If a provider is marked but not retired, we just
2797c478bd9Sstevel@tonic-gate 		 * have to take a crack at removing it -- it's no big deal if
2807c478bd9Sstevel@tonic-gate 		 * we can't.
2817c478bd9Sstevel@tonic-gate 		 */
2827c478bd9Sstevel@tonic-gate 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2837c478bd9Sstevel@tonic-gate 			bucket = &fasttrap_provs.fth_table[i];
2847c478bd9Sstevel@tonic-gate 			mutex_enter(&bucket->ftb_mtx);
2857c478bd9Sstevel@tonic-gate 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2867c478bd9Sstevel@tonic-gate 
2877c478bd9Sstevel@tonic-gate 			while ((fp = *fpp) != NULL) {
2887c478bd9Sstevel@tonic-gate 				if (!fp->ftp_marked) {
2897c478bd9Sstevel@tonic-gate 					fpp = &fp->ftp_next;
2907c478bd9Sstevel@tonic-gate 					continue;
2917c478bd9Sstevel@tonic-gate 				}
2927c478bd9Sstevel@tonic-gate 
2937c478bd9Sstevel@tonic-gate 				mutex_enter(&fp->ftp_mtx);
2947c478bd9Sstevel@tonic-gate 
2957c478bd9Sstevel@tonic-gate 				/*
2967c478bd9Sstevel@tonic-gate 				 * If this provider is referenced either
2977c478bd9Sstevel@tonic-gate 				 * because it is a USDT provider or is being
2987c478bd9Sstevel@tonic-gate 				 * modified, we can't unregister or even
2997c478bd9Sstevel@tonic-gate 				 * condense.
3007c478bd9Sstevel@tonic-gate 				 */
301*ab9a77c7Sahl 				if (fp->ftp_ccount != 0 ||
302*ab9a77c7Sahl 				    fp->ftp_mcount != 0) {
3037c478bd9Sstevel@tonic-gate 					mutex_exit(&fp->ftp_mtx);
3047c478bd9Sstevel@tonic-gate 					fp->ftp_marked = 0;
3057c478bd9Sstevel@tonic-gate 					continue;
3067c478bd9Sstevel@tonic-gate 				}
3077c478bd9Sstevel@tonic-gate 
308b096140dSahl 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
3097c478bd9Sstevel@tonic-gate 					fp->ftp_marked = 0;
3107c478bd9Sstevel@tonic-gate 
3117c478bd9Sstevel@tonic-gate 				mutex_exit(&fp->ftp_mtx);
3127c478bd9Sstevel@tonic-gate 
3137c478bd9Sstevel@tonic-gate 				/*
3147c478bd9Sstevel@tonic-gate 				 * If we successfully unregister this
3157c478bd9Sstevel@tonic-gate 				 * provider we can remove it from the hash
3167c478bd9Sstevel@tonic-gate 				 * chain and free the memory. If our attempt
317b096140dSahl 				 * to unregister fails and this is a retired
3187c478bd9Sstevel@tonic-gate 				 * provider, increment our flag to try again
3197c478bd9Sstevel@tonic-gate 				 * pretty soon. If we've consumed more than
3207c478bd9Sstevel@tonic-gate 				 * half of our total permitted number of
3217c478bd9Sstevel@tonic-gate 				 * probes call dtrace_condense() to try to
3227c478bd9Sstevel@tonic-gate 				 * clean out the unenabled probes.
3237c478bd9Sstevel@tonic-gate 				 */
3247c478bd9Sstevel@tonic-gate 				provid = fp->ftp_provid;
3257c478bd9Sstevel@tonic-gate 				if (dtrace_unregister(provid) != 0) {
3267c478bd9Sstevel@tonic-gate 					if (fasttrap_total > fasttrap_max / 2)
3277c478bd9Sstevel@tonic-gate 						(void) dtrace_condense(provid);
3287c478bd9Sstevel@tonic-gate 					later += fp->ftp_marked;
3297c478bd9Sstevel@tonic-gate 					fpp = &fp->ftp_next;
3307c478bd9Sstevel@tonic-gate 				} else {
3317c478bd9Sstevel@tonic-gate 					*fpp = fp->ftp_next;
3327c478bd9Sstevel@tonic-gate 					fasttrap_provider_free(fp);
3337c478bd9Sstevel@tonic-gate 				}
3347c478bd9Sstevel@tonic-gate 			}
3357c478bd9Sstevel@tonic-gate 			mutex_exit(&bucket->ftb_mtx);
3367c478bd9Sstevel@tonic-gate 		}
3377c478bd9Sstevel@tonic-gate 
3387c478bd9Sstevel@tonic-gate 		mutex_enter(&fasttrap_cleanup_mtx);
3397c478bd9Sstevel@tonic-gate 	}
3407c478bd9Sstevel@tonic-gate 
3417c478bd9Sstevel@tonic-gate 	ASSERT(fasttrap_timeout != 0);
3427c478bd9Sstevel@tonic-gate 
3437c478bd9Sstevel@tonic-gate 	/*
344b096140dSahl 	 * If we were unable to remove a retired provider, try again after
3457c478bd9Sstevel@tonic-gate 	 * a second. This situation can occur in certain circumstances where
3467c478bd9Sstevel@tonic-gate 	 * providers cannot be unregistered even though they have no probes
3477c478bd9Sstevel@tonic-gate 	 * enabled because of an execution of dtrace -l or something similar.
3487c478bd9Sstevel@tonic-gate 	 * If the timeout has been disabled (set to 1 because we're trying
3497c478bd9Sstevel@tonic-gate 	 * to detach), we set fasttrap_cleanup_work to ensure that we'll
3507c478bd9Sstevel@tonic-gate 	 * get a chance to do that work if and when the timeout is reenabled
3517c478bd9Sstevel@tonic-gate 	 * (if detach fails).
3527c478bd9Sstevel@tonic-gate 	 */
3537c478bd9Sstevel@tonic-gate 	if (later > 0 && fasttrap_timeout != (timeout_id_t)1)
3547c478bd9Sstevel@tonic-gate 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
3557c478bd9Sstevel@tonic-gate 	else if (later > 0)
3567c478bd9Sstevel@tonic-gate 		fasttrap_cleanup_work = 1;
3577c478bd9Sstevel@tonic-gate 	else
3587c478bd9Sstevel@tonic-gate 		fasttrap_timeout = 0;
3597c478bd9Sstevel@tonic-gate 
3607c478bd9Sstevel@tonic-gate 	mutex_exit(&fasttrap_cleanup_mtx);
3617c478bd9Sstevel@tonic-gate 	in = 0;
3627c478bd9Sstevel@tonic-gate }
3637c478bd9Sstevel@tonic-gate 
3647c478bd9Sstevel@tonic-gate /*
3657c478bd9Sstevel@tonic-gate  * Activates the asynchronous cleanup mechanism.
3667c478bd9Sstevel@tonic-gate  */
3677c478bd9Sstevel@tonic-gate static void
3687c478bd9Sstevel@tonic-gate fasttrap_pid_cleanup(void)
3697c478bd9Sstevel@tonic-gate {
3707c478bd9Sstevel@tonic-gate 	mutex_enter(&fasttrap_cleanup_mtx);
3717c478bd9Sstevel@tonic-gate 	fasttrap_cleanup_work = 1;
3727c478bd9Sstevel@tonic-gate 	if (fasttrap_timeout == 0)
3737c478bd9Sstevel@tonic-gate 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
3747c478bd9Sstevel@tonic-gate 	mutex_exit(&fasttrap_cleanup_mtx);
3757c478bd9Sstevel@tonic-gate }
3767c478bd9Sstevel@tonic-gate 
3777c478bd9Sstevel@tonic-gate /*
3787c478bd9Sstevel@tonic-gate  * This is called from cfork() via dtrace_fasttrap_fork(). The child
3797c478bd9Sstevel@tonic-gate  * process's address space is a (roughly) a copy of the parent process's so
3807c478bd9Sstevel@tonic-gate  * we have to remove all the instrumentation we had previously enabled in the
3817c478bd9Sstevel@tonic-gate  * parent.
3827c478bd9Sstevel@tonic-gate  */
3837c478bd9Sstevel@tonic-gate static void
3847c478bd9Sstevel@tonic-gate fasttrap_fork(proc_t *p, proc_t *cp)
3857c478bd9Sstevel@tonic-gate {
3867c478bd9Sstevel@tonic-gate 	pid_t ppid = p->p_pid;
3877c478bd9Sstevel@tonic-gate 	int i;
3887c478bd9Sstevel@tonic-gate 
3897c478bd9Sstevel@tonic-gate 	ASSERT(curproc == p);
3907c478bd9Sstevel@tonic-gate 	ASSERT(p->p_proc_flag & P_PR_LOCK);
3917c478bd9Sstevel@tonic-gate 	ASSERT(p->p_dtrace_count > 0);
3927c478bd9Sstevel@tonic-gate 	ASSERT(cp->p_dtrace_count == 0);
3937c478bd9Sstevel@tonic-gate 
3947c478bd9Sstevel@tonic-gate 	/*
3957c478bd9Sstevel@tonic-gate 	 * This would be simpler and faster if we maintained per-process
3967c478bd9Sstevel@tonic-gate 	 * hash tables of enabled tracepoints. It could, however, potentially
3977c478bd9Sstevel@tonic-gate 	 * slow down execution of a tracepoint since we'd need to go
3987c478bd9Sstevel@tonic-gate 	 * through two levels of indirection. In the future, we should
3997c478bd9Sstevel@tonic-gate 	 * consider either maintaining per-process ancillary lists of
4007c478bd9Sstevel@tonic-gate 	 * enabled tracepoints or hanging a pointer to a per-process hash
4017c478bd9Sstevel@tonic-gate 	 * table of enabled tracepoints off the proc structure.
4027c478bd9Sstevel@tonic-gate 	 */
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 	/*
4057c478bd9Sstevel@tonic-gate 	 * We don't have to worry about the child process disappearing
4067c478bd9Sstevel@tonic-gate 	 * because we're in fork().
4077c478bd9Sstevel@tonic-gate 	 */
4087c478bd9Sstevel@tonic-gate 	mutex_enter(&cp->p_lock);
4097c478bd9Sstevel@tonic-gate 	sprlock_proc(cp);
4107c478bd9Sstevel@tonic-gate 	mutex_exit(&cp->p_lock);
4117c478bd9Sstevel@tonic-gate 
4127c478bd9Sstevel@tonic-gate 	/*
4137c478bd9Sstevel@tonic-gate 	 * Iterate over every tracepoint looking for ones that belong to the
4147c478bd9Sstevel@tonic-gate 	 * parent process, and remove each from the child process.
4157c478bd9Sstevel@tonic-gate 	 */
4167c478bd9Sstevel@tonic-gate 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
4177c478bd9Sstevel@tonic-gate 		fasttrap_tracepoint_t *tp;
4187c478bd9Sstevel@tonic-gate 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
4197c478bd9Sstevel@tonic-gate 
4207c478bd9Sstevel@tonic-gate 		mutex_enter(&bucket->ftb_mtx);
4217c478bd9Sstevel@tonic-gate 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
422b096140dSahl 			if (tp->ftt_pid == ppid &&
423b096140dSahl 			    !tp->ftt_proc->ftpc_defunct) {
4247c478bd9Sstevel@tonic-gate 				int ret = fasttrap_tracepoint_remove(cp, tp);
4257c478bd9Sstevel@tonic-gate 				ASSERT(ret == 0);
4267c478bd9Sstevel@tonic-gate 			}
4277c478bd9Sstevel@tonic-gate 		}
4287c478bd9Sstevel@tonic-gate 		mutex_exit(&bucket->ftb_mtx);
4297c478bd9Sstevel@tonic-gate 	}
4307c478bd9Sstevel@tonic-gate 
4317c478bd9Sstevel@tonic-gate 	mutex_enter(&cp->p_lock);
4327c478bd9Sstevel@tonic-gate 	sprunlock(cp);
4337c478bd9Sstevel@tonic-gate }
4347c478bd9Sstevel@tonic-gate 
4357c478bd9Sstevel@tonic-gate /*
4367c478bd9Sstevel@tonic-gate  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
4377c478bd9Sstevel@tonic-gate  * is set on the proc structure to indicate that there is a pid provider
4387c478bd9Sstevel@tonic-gate  * associated with this process.
4397c478bd9Sstevel@tonic-gate  */
4407c478bd9Sstevel@tonic-gate static void
4417c478bd9Sstevel@tonic-gate fasttrap_exec_exit(proc_t *p)
4427c478bd9Sstevel@tonic-gate {
4437c478bd9Sstevel@tonic-gate 	ASSERT(p == curproc);
4447c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&p->p_lock));
4457c478bd9Sstevel@tonic-gate 
4467c478bd9Sstevel@tonic-gate 	mutex_exit(&p->p_lock);
4477c478bd9Sstevel@tonic-gate 
4487c478bd9Sstevel@tonic-gate 	/*
4497c478bd9Sstevel@tonic-gate 	 * We clean up the pid provider for this process here; user-land
4507c478bd9Sstevel@tonic-gate 	 * static probes are handled by the meta-provider remove entry point.
4517c478bd9Sstevel@tonic-gate 	 */
452dafb5540Sahl 	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
4537c478bd9Sstevel@tonic-gate 
4547c478bd9Sstevel@tonic-gate 	mutex_enter(&p->p_lock);
4557c478bd9Sstevel@tonic-gate }
4567c478bd9Sstevel@tonic-gate 
4577c478bd9Sstevel@tonic-gate 
4587c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4597c478bd9Sstevel@tonic-gate static void
4607c478bd9Sstevel@tonic-gate fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
4617c478bd9Sstevel@tonic-gate {
4627c478bd9Sstevel@tonic-gate 	/*
4637c478bd9Sstevel@tonic-gate 	 * There are no "default" pid probes.
4647c478bd9Sstevel@tonic-gate 	 */
4657c478bd9Sstevel@tonic-gate }
4667c478bd9Sstevel@tonic-gate 
4677c478bd9Sstevel@tonic-gate /*ARGSUSED*/
4687c478bd9Sstevel@tonic-gate static void
4697c478bd9Sstevel@tonic-gate fasttrap_provide(void *arg, const dtrace_probedesc_t *desc)
4707c478bd9Sstevel@tonic-gate {
4717c478bd9Sstevel@tonic-gate 	if (dtrace_probe_lookup(fasttrap_id, NULL, "fasttrap", "fasttrap") == 0)
4727c478bd9Sstevel@tonic-gate 		fasttrap_probe_id = dtrace_probe_create(fasttrap_id, NULL,
4737c478bd9Sstevel@tonic-gate 		    "fasttrap", "fasttrap", FASTTRAP_AFRAMES, NULL);
4747c478bd9Sstevel@tonic-gate }
4757c478bd9Sstevel@tonic-gate 
4767c478bd9Sstevel@tonic-gate static int
4777c478bd9Sstevel@tonic-gate fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
4787c478bd9Sstevel@tonic-gate {
4797c478bd9Sstevel@tonic-gate 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
4807c478bd9Sstevel@tonic-gate 	fasttrap_bucket_t *bucket;
4817c478bd9Sstevel@tonic-gate 	fasttrap_id_t *id;
4827c478bd9Sstevel@tonic-gate 	pid_t pid;
4837c478bd9Sstevel@tonic-gate 	uintptr_t pc;
4847c478bd9Sstevel@tonic-gate 
4857c478bd9Sstevel@tonic-gate 	ASSERT(index < probe->ftp_ntps);
4867c478bd9Sstevel@tonic-gate 
4877c478bd9Sstevel@tonic-gate 	pid = probe->ftp_pid;
4887c478bd9Sstevel@tonic-gate 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
4897c478bd9Sstevel@tonic-gate 	id = &probe->ftp_tps[index].fit_id;
4907c478bd9Sstevel@tonic-gate 
4917c478bd9Sstevel@tonic-gate 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
4927c478bd9Sstevel@tonic-gate 
4937c478bd9Sstevel@tonic-gate 	ASSERT(!(p->p_flag & SVFORK));
4947c478bd9Sstevel@tonic-gate 
4957c478bd9Sstevel@tonic-gate 	/*
4967c478bd9Sstevel@tonic-gate 	 * Before we make any modifications, make sure we've imposed a barrier
4977c478bd9Sstevel@tonic-gate 	 * on the generation in which this probe was last modified.
4987c478bd9Sstevel@tonic-gate 	 */
4997c478bd9Sstevel@tonic-gate 	fasttrap_mod_barrier(probe->ftp_gen);
5007c478bd9Sstevel@tonic-gate 
5017c478bd9Sstevel@tonic-gate 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 	/*
5047c478bd9Sstevel@tonic-gate 	 * If the tracepoint has already been enabled, just add our id to the
5057c478bd9Sstevel@tonic-gate 	 * list of interested probes. This may be our second time through
5067c478bd9Sstevel@tonic-gate 	 * this path in which case we'll have constructed the tracepoint we'd
5077c478bd9Sstevel@tonic-gate 	 * like to install. If we can't find a match, and have an allocated
5087c478bd9Sstevel@tonic-gate 	 * tracepoint ready to go, enable that one now.
5097c478bd9Sstevel@tonic-gate 	 *
510ac448965Sahl 	 * A tracepoint whose process is defunct is also considered defunct.
5117c478bd9Sstevel@tonic-gate 	 */
5127c478bd9Sstevel@tonic-gate again:
5137c478bd9Sstevel@tonic-gate 	mutex_enter(&bucket->ftb_mtx);
5147c478bd9Sstevel@tonic-gate 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
5157c478bd9Sstevel@tonic-gate 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
516b096140dSahl 		    tp->ftt_proc->ftpc_defunct)
5177c478bd9Sstevel@tonic-gate 			continue;
5187c478bd9Sstevel@tonic-gate 
5197c478bd9Sstevel@tonic-gate 		/*
5207c478bd9Sstevel@tonic-gate 		 * Now that we've found a matching tracepoint, it would be
5217c478bd9Sstevel@tonic-gate 		 * a decent idea to confirm that the tracepoint is still
5227c478bd9Sstevel@tonic-gate 		 * enabled and the trap instruction hasn't been overwritten.
5237c478bd9Sstevel@tonic-gate 		 * Since this is a little hairy, we'll punt for now.
5247c478bd9Sstevel@tonic-gate 		 */
5257c478bd9Sstevel@tonic-gate 
5267c478bd9Sstevel@tonic-gate 		/*
5277c478bd9Sstevel@tonic-gate 		 * This can't be the first interested probe. We don't have
5287c478bd9Sstevel@tonic-gate 		 * to worry about another thread being in the midst of
5297c478bd9Sstevel@tonic-gate 		 * deleting this tracepoint (which would be the only valid
5307c478bd9Sstevel@tonic-gate 		 * reason for a tracepoint to have no interested probes)
5317c478bd9Sstevel@tonic-gate 		 * since we're holding P_PR_LOCK for this process.
5327c478bd9Sstevel@tonic-gate 		 */
5337c478bd9Sstevel@tonic-gate 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
5347c478bd9Sstevel@tonic-gate 
535ac448965Sahl 		switch (id->fti_ptype) {
536ac448965Sahl 		case DTFTP_ENTRY:
537ac448965Sahl 		case DTFTP_OFFSETS:
538ac448965Sahl 		case DTFTP_IS_ENABLED:
5397c478bd9Sstevel@tonic-gate 			id->fti_next = tp->ftt_ids;
5407c478bd9Sstevel@tonic-gate 			membar_producer();
5417c478bd9Sstevel@tonic-gate 			tp->ftt_ids = id;
5427c478bd9Sstevel@tonic-gate 			membar_producer();
543ac448965Sahl 			break;
544ac448965Sahl 
545ac448965Sahl 		case DTFTP_RETURN:
546ac448965Sahl 		case DTFTP_POST_OFFSETS:
547ac448965Sahl 			id->fti_next = tp->ftt_retids;
548ac448965Sahl 			membar_producer();
549ac448965Sahl 			tp->ftt_retids = id;
550ac448965Sahl 			membar_producer();
551ac448965Sahl 			break;
552ac448965Sahl 
553ac448965Sahl 		default:
554ac448965Sahl 			ASSERT(0);
5557c478bd9Sstevel@tonic-gate 		}
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate 		mutex_exit(&bucket->ftb_mtx);
5587c478bd9Sstevel@tonic-gate 
5597c478bd9Sstevel@tonic-gate 		if (new_tp != NULL) {
5607c478bd9Sstevel@tonic-gate 			new_tp->ftt_ids = NULL;
5617c478bd9Sstevel@tonic-gate 			new_tp->ftt_retids = NULL;
5627c478bd9Sstevel@tonic-gate 		}
5637c478bd9Sstevel@tonic-gate 
5647c478bd9Sstevel@tonic-gate 		return (0);
5657c478bd9Sstevel@tonic-gate 	}
5667c478bd9Sstevel@tonic-gate 
5677c478bd9Sstevel@tonic-gate 	/*
5687c478bd9Sstevel@tonic-gate 	 * If we have a good tracepoint ready to go, install it now while
5697c478bd9Sstevel@tonic-gate 	 * we have the lock held and no one can screw with us.
5707c478bd9Sstevel@tonic-gate 	 */
5717c478bd9Sstevel@tonic-gate 	if (new_tp != NULL) {
5728f5db3e7Sahl 		int rc = 0;
5737c478bd9Sstevel@tonic-gate 
5747c478bd9Sstevel@tonic-gate 		new_tp->ftt_next = bucket->ftb_data;
5757c478bd9Sstevel@tonic-gate 		membar_producer();
5767c478bd9Sstevel@tonic-gate 		bucket->ftb_data = new_tp;
5777c478bd9Sstevel@tonic-gate 		membar_producer();
5787c478bd9Sstevel@tonic-gate 		mutex_exit(&bucket->ftb_mtx);
5797c478bd9Sstevel@tonic-gate 
5807c478bd9Sstevel@tonic-gate 		/*
5818f5db3e7Sahl 		 * Activate the tracepoint in the ISA-specific manner.
5828f5db3e7Sahl 		 * If this fails, we need to report the failure, but
5838f5db3e7Sahl 		 * indicate that this tracepoint must still be disabled
5848f5db3e7Sahl 		 * by calling fasttrap_tracepoint_disable().
5857c478bd9Sstevel@tonic-gate 		 */
5868f5db3e7Sahl 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
5878f5db3e7Sahl 			rc = FASTTRAP_ENABLE_PARTIAL;
5888f5db3e7Sahl 
5897c478bd9Sstevel@tonic-gate 		/*
5908f5db3e7Sahl 		 * Increment the count of the number of tracepoints active in
5918f5db3e7Sahl 		 * the victim process.
5927c478bd9Sstevel@tonic-gate 		 */
5937c478bd9Sstevel@tonic-gate 		ASSERT(p->p_proc_flag & P_PR_LOCK);
5947c478bd9Sstevel@tonic-gate 		p->p_dtrace_count++;
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate 		return (rc);
5977c478bd9Sstevel@tonic-gate 	}
5987c478bd9Sstevel@tonic-gate 
5997c478bd9Sstevel@tonic-gate 	mutex_exit(&bucket->ftb_mtx);
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate 	/*
6027c478bd9Sstevel@tonic-gate 	 * Initialize the tracepoint that's been preallocated with the probe.
6037c478bd9Sstevel@tonic-gate 	 */
6047c478bd9Sstevel@tonic-gate 	new_tp = probe->ftp_tps[index].fit_tp;
6057c478bd9Sstevel@tonic-gate 
6067c478bd9Sstevel@tonic-gate 	ASSERT(new_tp->ftt_pid == pid);
6077c478bd9Sstevel@tonic-gate 	ASSERT(new_tp->ftt_pc == pc);
608b096140dSahl 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
6097c478bd9Sstevel@tonic-gate 	ASSERT(new_tp->ftt_ids == NULL);
6107c478bd9Sstevel@tonic-gate 	ASSERT(new_tp->ftt_retids == NULL);
6117c478bd9Sstevel@tonic-gate 
612ac448965Sahl 	switch (id->fti_ptype) {
613ac448965Sahl 	case DTFTP_ENTRY:
614ac448965Sahl 	case DTFTP_OFFSETS:
615ac448965Sahl 	case DTFTP_IS_ENABLED:
6167c478bd9Sstevel@tonic-gate 		id->fti_next = NULL;
6177c478bd9Sstevel@tonic-gate 		new_tp->ftt_ids = id;
618ac448965Sahl 		break;
619ac448965Sahl 
620ac448965Sahl 	case DTFTP_RETURN:
621ac448965Sahl 	case DTFTP_POST_OFFSETS:
622ac448965Sahl 		id->fti_next = NULL;
623ac448965Sahl 		new_tp->ftt_retids = id;
624ac448965Sahl 		break;
625ac448965Sahl 
626ac448965Sahl 	default:
627ac448965Sahl 		ASSERT(0);
6287c478bd9Sstevel@tonic-gate 	}
6297c478bd9Sstevel@tonic-gate 
6307c478bd9Sstevel@tonic-gate 	/*
6318f5db3e7Sahl 	 * If the ISA-dependent initialization goes to plan, go back to the
6327c478bd9Sstevel@tonic-gate 	 * beginning and try to install this freshly made tracepoint.
6337c478bd9Sstevel@tonic-gate 	 */
634ac448965Sahl 	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
6357c478bd9Sstevel@tonic-gate 		goto again;
6367c478bd9Sstevel@tonic-gate 
6377c478bd9Sstevel@tonic-gate 	new_tp->ftt_ids = NULL;
6387c478bd9Sstevel@tonic-gate 	new_tp->ftt_retids = NULL;
6397c478bd9Sstevel@tonic-gate 
6408f5db3e7Sahl 	return (FASTTRAP_ENABLE_FAIL);
6417c478bd9Sstevel@tonic-gate }
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate static void
6447c478bd9Sstevel@tonic-gate fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
6457c478bd9Sstevel@tonic-gate {
6467c478bd9Sstevel@tonic-gate 	fasttrap_bucket_t *bucket;
6477c478bd9Sstevel@tonic-gate 	fasttrap_provider_t *provider = probe->ftp_prov;
6487c478bd9Sstevel@tonic-gate 	fasttrap_tracepoint_t **pp, *tp;
6497c478bd9Sstevel@tonic-gate 	fasttrap_id_t *id, **idp;
6507c478bd9Sstevel@tonic-gate 	pid_t pid;
6517c478bd9Sstevel@tonic-gate 	uintptr_t pc;
6527c478bd9Sstevel@tonic-gate 
6537c478bd9Sstevel@tonic-gate 	ASSERT(index < probe->ftp_ntps);
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 	pid = probe->ftp_pid;
6567c478bd9Sstevel@tonic-gate 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
657ac448965Sahl 	id = &probe->ftp_tps[index].fit_id;
6587c478bd9Sstevel@tonic-gate 
6597c478bd9Sstevel@tonic-gate 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
6607c478bd9Sstevel@tonic-gate 
6617c478bd9Sstevel@tonic-gate 	/*
6627c478bd9Sstevel@tonic-gate 	 * Find the tracepoint and make sure that our id is one of the
6637c478bd9Sstevel@tonic-gate 	 * ones registered with it.
6647c478bd9Sstevel@tonic-gate 	 */
6657c478bd9Sstevel@tonic-gate 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
6667c478bd9Sstevel@tonic-gate 	mutex_enter(&bucket->ftb_mtx);
6677c478bd9Sstevel@tonic-gate 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
6687c478bd9Sstevel@tonic-gate 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
669b096140dSahl 		    tp->ftt_proc == provider->ftp_proc)
6707c478bd9Sstevel@tonic-gate 			break;
6717c478bd9Sstevel@tonic-gate 	}
6727c478bd9Sstevel@tonic-gate 
6737c478bd9Sstevel@tonic-gate 	/*
6747c478bd9Sstevel@tonic-gate 	 * If we somehow lost this tracepoint, we're in a world of hurt.
6757c478bd9Sstevel@tonic-gate 	 */
6767c478bd9Sstevel@tonic-gate 	ASSERT(tp != NULL);
6777c478bd9Sstevel@tonic-gate 
678ac448965Sahl 	switch (id->fti_ptype) {
679ac448965Sahl 	case DTFTP_ENTRY:
680ac448965Sahl 	case DTFTP_OFFSETS:
681ac448965Sahl 	case DTFTP_IS_ENABLED:
6827c478bd9Sstevel@tonic-gate 		ASSERT(tp->ftt_ids != NULL);
6837c478bd9Sstevel@tonic-gate 		idp = &tp->ftt_ids;
684ac448965Sahl 		break;
685ac448965Sahl 
686ac448965Sahl 	case DTFTP_RETURN:
687ac448965Sahl 	case DTFTP_POST_OFFSETS:
688ac448965Sahl 		ASSERT(tp->ftt_retids != NULL);
689ac448965Sahl 		idp = &tp->ftt_retids;
690ac448965Sahl 		break;
691ac448965Sahl 
692ac448965Sahl 	default:
693ac448965Sahl 		ASSERT(0);
6947c478bd9Sstevel@tonic-gate 	}
6957c478bd9Sstevel@tonic-gate 
6967c478bd9Sstevel@tonic-gate 	while ((*idp)->fti_probe != probe) {
6977c478bd9Sstevel@tonic-gate 		idp = &(*idp)->fti_next;
6987c478bd9Sstevel@tonic-gate 		ASSERT(*idp != NULL);
6997c478bd9Sstevel@tonic-gate 	}
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	id = *idp;
7027c478bd9Sstevel@tonic-gate 	*idp = id->fti_next;
7037c478bd9Sstevel@tonic-gate 	membar_producer();
7047c478bd9Sstevel@tonic-gate 
7057c478bd9Sstevel@tonic-gate 	ASSERT(id->fti_probe == probe);
7067c478bd9Sstevel@tonic-gate 
7077c478bd9Sstevel@tonic-gate 	/*
7087c478bd9Sstevel@tonic-gate 	 * If there are other registered enablings of this tracepoint, we're
7097c478bd9Sstevel@tonic-gate 	 * all done, but if this was the last probe assocated with this
7107c478bd9Sstevel@tonic-gate 	 * this tracepoint, we need to remove and free it.
7117c478bd9Sstevel@tonic-gate 	 */
7127c478bd9Sstevel@tonic-gate 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
7137c478bd9Sstevel@tonic-gate 
7147c478bd9Sstevel@tonic-gate 		/*
7157c478bd9Sstevel@tonic-gate 		 * If the current probe's tracepoint is in use, swap it
7167c478bd9Sstevel@tonic-gate 		 * for an unused tracepoint.
7177c478bd9Sstevel@tonic-gate 		 */
7187c478bd9Sstevel@tonic-gate 		if (tp == probe->ftp_tps[index].fit_tp) {
7197c478bd9Sstevel@tonic-gate 			fasttrap_probe_t *tmp_probe;
7207c478bd9Sstevel@tonic-gate 			fasttrap_tracepoint_t **tmp_tp;
7217c478bd9Sstevel@tonic-gate 			uint_t tmp_index;
7227c478bd9Sstevel@tonic-gate 
7237c478bd9Sstevel@tonic-gate 			if (tp->ftt_ids != NULL) {
7247c478bd9Sstevel@tonic-gate 				tmp_probe = tp->ftt_ids->fti_probe;
7257c478bd9Sstevel@tonic-gate 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
7267c478bd9Sstevel@tonic-gate 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
7277c478bd9Sstevel@tonic-gate 			} else {
7287c478bd9Sstevel@tonic-gate 				tmp_probe = tp->ftt_retids->fti_probe;
7297c478bd9Sstevel@tonic-gate 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
7307c478bd9Sstevel@tonic-gate 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
7317c478bd9Sstevel@tonic-gate 			}
7327c478bd9Sstevel@tonic-gate 
7337c478bd9Sstevel@tonic-gate 			ASSERT(*tmp_tp != NULL);
7347c478bd9Sstevel@tonic-gate 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
7357c478bd9Sstevel@tonic-gate 			ASSERT((*tmp_tp)->ftt_ids == NULL);
7367c478bd9Sstevel@tonic-gate 			ASSERT((*tmp_tp)->ftt_retids == NULL);
7377c478bd9Sstevel@tonic-gate 
7387c478bd9Sstevel@tonic-gate 			probe->ftp_tps[index].fit_tp = *tmp_tp;
7397c478bd9Sstevel@tonic-gate 			*tmp_tp = tp;
7407c478bd9Sstevel@tonic-gate 
7417c478bd9Sstevel@tonic-gate 		}
7427c478bd9Sstevel@tonic-gate 
7437c478bd9Sstevel@tonic-gate 		mutex_exit(&bucket->ftb_mtx);
7447c478bd9Sstevel@tonic-gate 
7457c478bd9Sstevel@tonic-gate 		/*
7467c478bd9Sstevel@tonic-gate 		 * Tag the modified probe with the generation in which it was
7477c478bd9Sstevel@tonic-gate 		 * changed.
7487c478bd9Sstevel@tonic-gate 		 */
7497c478bd9Sstevel@tonic-gate 		probe->ftp_gen = fasttrap_mod_gen;
7507c478bd9Sstevel@tonic-gate 		return;
7517c478bd9Sstevel@tonic-gate 	}
7527c478bd9Sstevel@tonic-gate 
7537c478bd9Sstevel@tonic-gate 	mutex_exit(&bucket->ftb_mtx);
7547c478bd9Sstevel@tonic-gate 
7557c478bd9Sstevel@tonic-gate 	/*
7567c478bd9Sstevel@tonic-gate 	 * We can't safely remove the tracepoint from the set of active
7577c478bd9Sstevel@tonic-gate 	 * tracepoints until we've actually removed the fasttrap instruction
7587c478bd9Sstevel@tonic-gate 	 * from the process's text. We can, however, operate on this
7597c478bd9Sstevel@tonic-gate 	 * tracepoint secure in the knowledge that no other thread is going to
7607c478bd9Sstevel@tonic-gate 	 * be looking at it since we hold P_PR_LOCK on the process if it's
7617c478bd9Sstevel@tonic-gate 	 * live or we hold the provider lock on the process if it's dead and
7627c478bd9Sstevel@tonic-gate 	 * gone.
7637c478bd9Sstevel@tonic-gate 	 */
7647c478bd9Sstevel@tonic-gate 
7657c478bd9Sstevel@tonic-gate 	/*
7667c478bd9Sstevel@tonic-gate 	 * We only need to remove the actual instruction if we're looking
7677c478bd9Sstevel@tonic-gate 	 * at an existing process
7687c478bd9Sstevel@tonic-gate 	 */
7697c478bd9Sstevel@tonic-gate 	if (p != NULL) {
7707c478bd9Sstevel@tonic-gate 		/*
7717c478bd9Sstevel@tonic-gate 		 * If we fail to restore the instruction we need to kill
7727c478bd9Sstevel@tonic-gate 		 * this process since it's in a completely unrecoverable
7737c478bd9Sstevel@tonic-gate 		 * state.
7747c478bd9Sstevel@tonic-gate 		 */
7757c478bd9Sstevel@tonic-gate 		if (fasttrap_tracepoint_remove(p, tp) != 0)
7767c478bd9Sstevel@tonic-gate 			fasttrap_sigtrap(p, NULL, pc);
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate 		/*
7797c478bd9Sstevel@tonic-gate 		 * Decrement the count of the number of tracepoints active
7807c478bd9Sstevel@tonic-gate 		 * in the victim process.
7817c478bd9Sstevel@tonic-gate 		 */
7827c478bd9Sstevel@tonic-gate 		ASSERT(p->p_proc_flag & P_PR_LOCK);
7837c478bd9Sstevel@tonic-gate 		p->p_dtrace_count--;
7847c478bd9Sstevel@tonic-gate 	}
7857c478bd9Sstevel@tonic-gate 
7867c478bd9Sstevel@tonic-gate 	/*
7877c478bd9Sstevel@tonic-gate 	 * Remove the probe from the hash table of active tracepoints.
7887c478bd9Sstevel@tonic-gate 	 */
7897c478bd9Sstevel@tonic-gate 	mutex_enter(&bucket->ftb_mtx);
7907c478bd9Sstevel@tonic-gate 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
7917c478bd9Sstevel@tonic-gate 	ASSERT(*pp != NULL);
7927c478bd9Sstevel@tonic-gate 	while (*pp != tp) {
7937c478bd9Sstevel@tonic-gate 		pp = &(*pp)->ftt_next;
7947c478bd9Sstevel@tonic-gate 		ASSERT(*pp != NULL);
7957c478bd9Sstevel@tonic-gate 	}
7967c478bd9Sstevel@tonic-gate 
7977c478bd9Sstevel@tonic-gate 	*pp = tp->ftt_next;
7987c478bd9Sstevel@tonic-gate 	membar_producer();
7997c478bd9Sstevel@tonic-gate 
8007c478bd9Sstevel@tonic-gate 	mutex_exit(&bucket->ftb_mtx);
8017c478bd9Sstevel@tonic-gate 
8027c478bd9Sstevel@tonic-gate 	/*
8037c478bd9Sstevel@tonic-gate 	 * Tag the modified probe with the generation in which it was changed.
8047c478bd9Sstevel@tonic-gate 	 */
8057c478bd9Sstevel@tonic-gate 	probe->ftp_gen = fasttrap_mod_gen;
8067c478bd9Sstevel@tonic-gate }
8077c478bd9Sstevel@tonic-gate 
8087c478bd9Sstevel@tonic-gate typedef int fasttrap_probe_f(struct regs *);
8097c478bd9Sstevel@tonic-gate 
8107c478bd9Sstevel@tonic-gate static void
8117c478bd9Sstevel@tonic-gate fasttrap_enable_common(int *count, fasttrap_probe_f **fptr, fasttrap_probe_f *f,
8127c478bd9Sstevel@tonic-gate     fasttrap_probe_f **fptr2, fasttrap_probe_f *f2)
8137c478bd9Sstevel@tonic-gate {
8147c478bd9Sstevel@tonic-gate 	/*
8157c478bd9Sstevel@tonic-gate 	 * We don't have to play the rw lock game here because we're
8167c478bd9Sstevel@tonic-gate 	 * providing something rather than taking something away --
8177c478bd9Sstevel@tonic-gate 	 * we can be sure that no threads have tried to follow this
8187c478bd9Sstevel@tonic-gate 	 * function pointer yet.
8197c478bd9Sstevel@tonic-gate 	 */
8207c478bd9Sstevel@tonic-gate 	mutex_enter(&fasttrap_count_mtx);
8217c478bd9Sstevel@tonic-gate 	if (*count == 0) {
8227c478bd9Sstevel@tonic-gate 		ASSERT(*fptr == NULL);
8237c478bd9Sstevel@tonic-gate 		*fptr = f;
8247c478bd9Sstevel@tonic-gate 		if (fptr2 != NULL)
8257c478bd9Sstevel@tonic-gate 			*fptr2 = f2;
8267c478bd9Sstevel@tonic-gate 	}
8277c478bd9Sstevel@tonic-gate 	ASSERT(*fptr == f);
8287c478bd9Sstevel@tonic-gate 	ASSERT(fptr2 == NULL || *fptr2 == f2);
8297c478bd9Sstevel@tonic-gate 	(*count)++;
8307c478bd9Sstevel@tonic-gate 	mutex_exit(&fasttrap_count_mtx);
8317c478bd9Sstevel@tonic-gate }
8327c478bd9Sstevel@tonic-gate 
8337c478bd9Sstevel@tonic-gate static void
8347c478bd9Sstevel@tonic-gate fasttrap_disable_common(int *count, fasttrap_probe_f **fptr,
8357c478bd9Sstevel@tonic-gate     fasttrap_probe_f **fptr2)
8367c478bd9Sstevel@tonic-gate {
8377c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
8387c478bd9Sstevel@tonic-gate 
8397c478bd9Sstevel@tonic-gate 	mutex_enter(&fasttrap_count_mtx);
8407c478bd9Sstevel@tonic-gate 	(*count)--;
8417c478bd9Sstevel@tonic-gate 	ASSERT(*count >= 0);
8427c478bd9Sstevel@tonic-gate 	if (*count == 0) {
8437c478bd9Sstevel@tonic-gate 		cpu_t *cur, *cpu = CPU;
8447c478bd9Sstevel@tonic-gate 
8457c478bd9Sstevel@tonic-gate 		for (cur = cpu->cpu_next_onln; cur != cpu;
8467c478bd9Sstevel@tonic-gate 			cur = cur->cpu_next_onln) {
8477c478bd9Sstevel@tonic-gate 			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
8487c478bd9Sstevel@tonic-gate 		}
8497c478bd9Sstevel@tonic-gate 
8507c478bd9Sstevel@tonic-gate 		*fptr = NULL;
8517c478bd9Sstevel@tonic-gate 		if (fptr2 != NULL)
8527c478bd9Sstevel@tonic-gate 			*fptr2 = NULL;
8537c478bd9Sstevel@tonic-gate 
8547c478bd9Sstevel@tonic-gate 		for (cur = cpu->cpu_next_onln; cur != cpu;
8557c478bd9Sstevel@tonic-gate 			cur = cur->cpu_next_onln) {
8567c478bd9Sstevel@tonic-gate 			rw_exit(&cur->cpu_ft_lock);
8577c478bd9Sstevel@tonic-gate 		}
8587c478bd9Sstevel@tonic-gate 	}
8597c478bd9Sstevel@tonic-gate 	mutex_exit(&fasttrap_count_mtx);
8607c478bd9Sstevel@tonic-gate }
8617c478bd9Sstevel@tonic-gate 
8627c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8637c478bd9Sstevel@tonic-gate static void
8647c478bd9Sstevel@tonic-gate fasttrap_enable(void *arg, dtrace_id_t id, void *parg)
8657c478bd9Sstevel@tonic-gate {
8667c478bd9Sstevel@tonic-gate 	/*
8677c478bd9Sstevel@tonic-gate 	 * Enable the probe that corresponds to statically placed trace
8687c478bd9Sstevel@tonic-gate 	 * points which have not explicitly been placed in the process's text
8697c478bd9Sstevel@tonic-gate 	 * by the fasttrap provider.
8707c478bd9Sstevel@tonic-gate 	 */
8717c478bd9Sstevel@tonic-gate 	ASSERT(arg == NULL);
8727c478bd9Sstevel@tonic-gate 	ASSERT(id == fasttrap_probe_id);
8737c478bd9Sstevel@tonic-gate 
8747c478bd9Sstevel@tonic-gate 	fasttrap_enable_common(&fasttrap_count,
8757c478bd9Sstevel@tonic-gate 	    &dtrace_fasttrap_probe_ptr, fasttrap_probe, NULL, NULL);
8767c478bd9Sstevel@tonic-gate }
8777c478bd9Sstevel@tonic-gate 
8787c478bd9Sstevel@tonic-gate 
8797c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8807c478bd9Sstevel@tonic-gate static void
8817c478bd9Sstevel@tonic-gate fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
8827c478bd9Sstevel@tonic-gate {
8837c478bd9Sstevel@tonic-gate 	fasttrap_probe_t *probe = parg;
8847c478bd9Sstevel@tonic-gate 	proc_t *p;
8858f5db3e7Sahl 	int i, rc;
8867c478bd9Sstevel@tonic-gate 
8877c478bd9Sstevel@tonic-gate 	ASSERT(probe != NULL);
8887c478bd9Sstevel@tonic-gate 	ASSERT(!probe->ftp_enabled);
8897c478bd9Sstevel@tonic-gate 	ASSERT(id == probe->ftp_id);
8907c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
8917c478bd9Sstevel@tonic-gate 
8927c478bd9Sstevel@tonic-gate 	/*
8937c478bd9Sstevel@tonic-gate 	 * Increment the count of enabled probes on this probe's provider;
8947c478bd9Sstevel@tonic-gate 	 * the provider can't go away while the probe still exists. We
8957c478bd9Sstevel@tonic-gate 	 * must increment this even if we aren't able to properly enable
8967c478bd9Sstevel@tonic-gate 	 * this probe.
8977c478bd9Sstevel@tonic-gate 	 */
8987c478bd9Sstevel@tonic-gate 	mutex_enter(&probe->ftp_prov->ftp_mtx);
8997c478bd9Sstevel@tonic-gate 	probe->ftp_prov->ftp_rcount++;
9007c478bd9Sstevel@tonic-gate 	mutex_exit(&probe->ftp_prov->ftp_mtx);
9017c478bd9Sstevel@tonic-gate 
9027c478bd9Sstevel@tonic-gate 	/*
903*ab9a77c7Sahl 	 * If this probe's provider is retired (meaning it was valid in a
904*ab9a77c7Sahl 	 * previously exec'ed incarnation of this address space), bail out. The
905*ab9a77c7Sahl 	 * provider can't go away while we're in this code path.
9067c478bd9Sstevel@tonic-gate 	 */
907*ab9a77c7Sahl 	if (probe->ftp_prov->ftp_retired)
9087c478bd9Sstevel@tonic-gate 		return;
9097c478bd9Sstevel@tonic-gate 
910*ab9a77c7Sahl 	/*
911*ab9a77c7Sahl 	 * If we can't find the process, it may be that we're in the context of
912*ab9a77c7Sahl 	 * a fork in which the traced process is being born and we're copying
913*ab9a77c7Sahl 	 * USDT probes. Otherwise, the process is gone so bail.
914*ab9a77c7Sahl 	 */
915*ab9a77c7Sahl 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
916*ab9a77c7Sahl 		if ((curproc->p_flag & SFORKING) == 0)
917*ab9a77c7Sahl 			return;
918*ab9a77c7Sahl 
919*ab9a77c7Sahl 		mutex_enter(&pidlock);
920*ab9a77c7Sahl 		p = prfind(probe->ftp_pid);
921*ab9a77c7Sahl 
922*ab9a77c7Sahl 		/*
923*ab9a77c7Sahl 		 * Confirm that curproc is indeed forking the process in which
924*ab9a77c7Sahl 		 * we're trying to enable probes.
925*ab9a77c7Sahl 		 */
926*ab9a77c7Sahl 		ASSERT(p != NULL);
927*ab9a77c7Sahl 		ASSERT(p->p_parent == curproc);
928*ab9a77c7Sahl 		ASSERT(p->p_stat == SIDL);
929*ab9a77c7Sahl 
930*ab9a77c7Sahl 		mutex_enter(&p->p_lock);
931*ab9a77c7Sahl 		mutex_exit(&pidlock);
932*ab9a77c7Sahl 
933*ab9a77c7Sahl 		sprlock_proc(p);
934*ab9a77c7Sahl 	}
935*ab9a77c7Sahl 
9367c478bd9Sstevel@tonic-gate 	ASSERT(!(p->p_flag & SVFORK));
9377c478bd9Sstevel@tonic-gate 	mutex_exit(&p->p_lock);
9387c478bd9Sstevel@tonic-gate 
9397c478bd9Sstevel@tonic-gate 	/*
9407c478bd9Sstevel@tonic-gate 	 * We have to enable the trap entry before any user threads have
9417c478bd9Sstevel@tonic-gate 	 * the chance to execute the trap instruction we're about to place
9427c478bd9Sstevel@tonic-gate 	 * in their process's text.
9437c478bd9Sstevel@tonic-gate 	 */
9447c478bd9Sstevel@tonic-gate 	fasttrap_enable_common(&fasttrap_pid_count,
9457c478bd9Sstevel@tonic-gate 	    &dtrace_pid_probe_ptr, fasttrap_pid_probe,
9467c478bd9Sstevel@tonic-gate 	    &dtrace_return_probe_ptr, fasttrap_return_probe);
9477c478bd9Sstevel@tonic-gate 
9487c478bd9Sstevel@tonic-gate 	/*
9497c478bd9Sstevel@tonic-gate 	 * Enable all the tracepoints and add this probe's id to each
9507c478bd9Sstevel@tonic-gate 	 * tracepoint's list of active probes.
9517c478bd9Sstevel@tonic-gate 	 */
9527c478bd9Sstevel@tonic-gate 	for (i = 0; i < probe->ftp_ntps; i++) {
9538f5db3e7Sahl 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
9548f5db3e7Sahl 			/*
9558f5db3e7Sahl 			 * If enabling the tracepoint failed completely,
9568f5db3e7Sahl 			 * we don't have to disable it; if the failure
9578f5db3e7Sahl 			 * was only partial we must disable it.
9588f5db3e7Sahl 			 */
9598f5db3e7Sahl 			if (rc == FASTTRAP_ENABLE_FAIL)
9608f5db3e7Sahl 				i--;
9618f5db3e7Sahl 			else
9628f5db3e7Sahl 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
9638f5db3e7Sahl 
9647c478bd9Sstevel@tonic-gate 			/*
9657c478bd9Sstevel@tonic-gate 			 * Back up and pull out all the tracepoints we've
9667c478bd9Sstevel@tonic-gate 			 * created so far for this probe.
9677c478bd9Sstevel@tonic-gate 			 */
9688c642573Sahl 			while (i >= 0) {
9697c478bd9Sstevel@tonic-gate 				fasttrap_tracepoint_disable(p, probe, i);
9708c642573Sahl 				i--;
9717c478bd9Sstevel@tonic-gate 			}
9727c478bd9Sstevel@tonic-gate 
9737c478bd9Sstevel@tonic-gate 			mutex_enter(&p->p_lock);
9747c478bd9Sstevel@tonic-gate 			sprunlock(p);
9757c478bd9Sstevel@tonic-gate 
9767c478bd9Sstevel@tonic-gate 			/*
9777c478bd9Sstevel@tonic-gate 			 * Since we're not actually enabling this probe,
9787c478bd9Sstevel@tonic-gate 			 * drop our reference on the trap table entry.
9797c478bd9Sstevel@tonic-gate 			 */
9807c478bd9Sstevel@tonic-gate 			fasttrap_disable_common(&fasttrap_pid_count,
9817c478bd9Sstevel@tonic-gate 			    &dtrace_pid_probe_ptr, &dtrace_return_probe_ptr);
9827c478bd9Sstevel@tonic-gate 			return;
9837c478bd9Sstevel@tonic-gate 		}
9847c478bd9Sstevel@tonic-gate 	}
9857c478bd9Sstevel@tonic-gate 
9867c478bd9Sstevel@tonic-gate 	mutex_enter(&p->p_lock);
9877c478bd9Sstevel@tonic-gate 	sprunlock(p);
9887c478bd9Sstevel@tonic-gate 
9897c478bd9Sstevel@tonic-gate 	probe->ftp_enabled = 1;
9907c478bd9Sstevel@tonic-gate }
9917c478bd9Sstevel@tonic-gate 
9927c478bd9Sstevel@tonic-gate 
9937c478bd9Sstevel@tonic-gate /*ARGSUSED*/
9947c478bd9Sstevel@tonic-gate static void
9957c478bd9Sstevel@tonic-gate fasttrap_disable(void *arg, dtrace_id_t id, void *parg)
9967c478bd9Sstevel@tonic-gate {
9977c478bd9Sstevel@tonic-gate 	/*
9987c478bd9Sstevel@tonic-gate 	 * Disable the probe the corresponds to statically placed trace
9997c478bd9Sstevel@tonic-gate 	 * points.
10007c478bd9Sstevel@tonic-gate 	 */
10017c478bd9Sstevel@tonic-gate 	ASSERT(arg == NULL);
10027c478bd9Sstevel@tonic-gate 	ASSERT(id == fasttrap_probe_id);
10037c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
10047c478bd9Sstevel@tonic-gate 	fasttrap_disable_common(&fasttrap_count, &dtrace_fasttrap_probe_ptr,
10057c478bd9Sstevel@tonic-gate 	    NULL);
10067c478bd9Sstevel@tonic-gate }
10077c478bd9Sstevel@tonic-gate 
10087c478bd9Sstevel@tonic-gate /*ARGSUSED*/
10097c478bd9Sstevel@tonic-gate static void
10107c478bd9Sstevel@tonic-gate fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
10117c478bd9Sstevel@tonic-gate {
10127c478bd9Sstevel@tonic-gate 	fasttrap_probe_t *probe = parg;
10137c478bd9Sstevel@tonic-gate 	fasttrap_provider_t *provider = probe->ftp_prov;
10147c478bd9Sstevel@tonic-gate 	proc_t *p;
10157c478bd9Sstevel@tonic-gate 	int i, whack = 0;
10167c478bd9Sstevel@tonic-gate 
10177c478bd9Sstevel@tonic-gate 	if (!probe->ftp_enabled) {
10187c478bd9Sstevel@tonic-gate 		mutex_enter(&provider->ftp_mtx);
10197c478bd9Sstevel@tonic-gate 		provider->ftp_rcount--;
10207c478bd9Sstevel@tonic-gate 		ASSERT(provider->ftp_rcount >= 0);
10217c478bd9Sstevel@tonic-gate 		mutex_exit(&provider->ftp_mtx);
10227c478bd9Sstevel@tonic-gate 		return;
10237c478bd9Sstevel@tonic-gate 	}
10247c478bd9Sstevel@tonic-gate 
10257c478bd9Sstevel@tonic-gate 	ASSERT(id == probe->ftp_id);
10267c478bd9Sstevel@tonic-gate 
10277c478bd9Sstevel@tonic-gate 	/*
10287c478bd9Sstevel@tonic-gate 	 * We won't be able to acquire a /proc-esque lock on the process
10297c478bd9Sstevel@tonic-gate 	 * iff the process is dead and gone. In this case, we rely on the
10307c478bd9Sstevel@tonic-gate 	 * provider lock as a point of mutual exclusion to prevent other
10317c478bd9Sstevel@tonic-gate 	 * DTrace consumers from disabling this probe.
10327c478bd9Sstevel@tonic-gate 	 */
10337c478bd9Sstevel@tonic-gate 	if ((p = sprlock(probe->ftp_pid)) != NULL) {
10347c478bd9Sstevel@tonic-gate 		ASSERT(!(p->p_flag & SVFORK));
10357c478bd9Sstevel@tonic-gate 		mutex_exit(&p->p_lock);
10367c478bd9Sstevel@tonic-gate 	}
10377c478bd9Sstevel@tonic-gate 
10387c478bd9Sstevel@tonic-gate 	mutex_enter(&provider->ftp_mtx);
10397c478bd9Sstevel@tonic-gate 
10407c478bd9Sstevel@tonic-gate 	/*
10417c478bd9Sstevel@tonic-gate 	 * Disable all the associated tracepoints.
10427c478bd9Sstevel@tonic-gate 	 */
10437c478bd9Sstevel@tonic-gate 	for (i = 0; i < probe->ftp_ntps; i++) {
10447c478bd9Sstevel@tonic-gate 		fasttrap_tracepoint_disable(p, probe, i);
10457c478bd9Sstevel@tonic-gate 	}
10467c478bd9Sstevel@tonic-gate 
10477c478bd9Sstevel@tonic-gate 	ASSERT(provider->ftp_rcount > 0);
10487c478bd9Sstevel@tonic-gate 	provider->ftp_rcount--;
10497c478bd9Sstevel@tonic-gate 
10507c478bd9Sstevel@tonic-gate 	if (p != NULL) {
10517c478bd9Sstevel@tonic-gate 		/*
10527c478bd9Sstevel@tonic-gate 		 * Even though we may not be able to remove it entirely, we
1053b096140dSahl 		 * mark this retired provider to get a chance to remove some
10547c478bd9Sstevel@tonic-gate 		 * of the associated probes.
10557c478bd9Sstevel@tonic-gate 		 */
1056b096140dSahl 		if (provider->ftp_retired && !provider->ftp_marked)
10577c478bd9Sstevel@tonic-gate 			whack = provider->ftp_marked = 1;
10587c478bd9Sstevel@tonic-gate 		mutex_exit(&provider->ftp_mtx);
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate 		mutex_enter(&p->p_lock);
10617c478bd9Sstevel@tonic-gate 		sprunlock(p);
10627c478bd9Sstevel@tonic-gate 	} else {
10637c478bd9Sstevel@tonic-gate 		/*
10647c478bd9Sstevel@tonic-gate 		 * If the process is dead, we're just waiting for the
10657c478bd9Sstevel@tonic-gate 		 * last probe to be disabled to be able to free it.
10667c478bd9Sstevel@tonic-gate 		 */
10677c478bd9Sstevel@tonic-gate 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
10687c478bd9Sstevel@tonic-gate 			whack = provider->ftp_marked = 1;
10697c478bd9Sstevel@tonic-gate 		mutex_exit(&provider->ftp_mtx);
10707c478bd9Sstevel@tonic-gate 	}
10717c478bd9Sstevel@tonic-gate 
10727c478bd9Sstevel@tonic-gate 	if (whack)
10737c478bd9Sstevel@tonic-gate 		fasttrap_pid_cleanup();
10747c478bd9Sstevel@tonic-gate 
10757c478bd9Sstevel@tonic-gate 	probe->ftp_enabled = 0;
10767c478bd9Sstevel@tonic-gate 
10777c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&cpu_lock));
10787c478bd9Sstevel@tonic-gate 	fasttrap_disable_common(&fasttrap_pid_count, &dtrace_pid_probe_ptr,
10797c478bd9Sstevel@tonic-gate 	    &dtrace_return_probe_ptr);
10807c478bd9Sstevel@tonic-gate }
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate /*ARGSUSED*/
10837c478bd9Sstevel@tonic-gate static void
10847c478bd9Sstevel@tonic-gate fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
10857c478bd9Sstevel@tonic-gate     dtrace_argdesc_t *desc)
10867c478bd9Sstevel@tonic-gate {
10877c478bd9Sstevel@tonic-gate 	fasttrap_probe_t *probe = parg;
10887c478bd9Sstevel@tonic-gate 	char *str;
10897c478bd9Sstevel@tonic-gate 	int i;
10907c478bd9Sstevel@tonic-gate 
10917c478bd9Sstevel@tonic-gate 	desc->dtargd_native[0] = '\0';
10927c478bd9Sstevel@tonic-gate 	desc->dtargd_xlate[0] = '\0';
10937c478bd9Sstevel@tonic-gate 
1094b096140dSahl 	if (probe->ftp_prov->ftp_retired != 0 ||
10957c478bd9Sstevel@tonic-gate 	    desc->dtargd_ndx >= probe->ftp_nargs) {
10967c478bd9Sstevel@tonic-gate 		desc->dtargd_ndx = DTRACE_ARGNONE;
10977c478bd9Sstevel@tonic-gate 		return;
10987c478bd9Sstevel@tonic-gate 	}
10997c478bd9Sstevel@tonic-gate 
11007c478bd9Sstevel@tonic-gate 	/*
11017c478bd9Sstevel@tonic-gate 	 * We only need to set this member if the argument is remapped.
11027c478bd9Sstevel@tonic-gate 	 */
11037c478bd9Sstevel@tonic-gate 	if (probe->ftp_argmap != NULL)
11047c478bd9Sstevel@tonic-gate 		desc->dtargd_mapping = probe->ftp_argmap[desc->dtargd_ndx];
11057c478bd9Sstevel@tonic-gate 
11067c478bd9Sstevel@tonic-gate 	str = probe->ftp_ntypes;
11077c478bd9Sstevel@tonic-gate 	for (i = 0; i < desc->dtargd_mapping; i++) {
11087c478bd9Sstevel@tonic-gate 		str += strlen(str) + 1;
11097c478bd9Sstevel@tonic-gate 	}
11107c478bd9Sstevel@tonic-gate 
11117c478bd9Sstevel@tonic-gate 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
11127c478bd9Sstevel@tonic-gate 	(void) strcpy(desc->dtargd_native, str);
11137c478bd9Sstevel@tonic-gate 
11147c478bd9Sstevel@tonic-gate 	if (probe->ftp_xtypes == NULL)
11157c478bd9Sstevel@tonic-gate 		return;
11167c478bd9Sstevel@tonic-gate 
11177c478bd9Sstevel@tonic-gate 	str = probe->ftp_xtypes;
11187c478bd9Sstevel@tonic-gate 	for (i = 0; i < desc->dtargd_ndx; i++) {
11197c478bd9Sstevel@tonic-gate 		str += strlen(str) + 1;
11207c478bd9Sstevel@tonic-gate 	}
11217c478bd9Sstevel@tonic-gate 
11227c478bd9Sstevel@tonic-gate 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
11237c478bd9Sstevel@tonic-gate 	(void) strcpy(desc->dtargd_xlate, str);
11247c478bd9Sstevel@tonic-gate }
11257c478bd9Sstevel@tonic-gate 
11267c478bd9Sstevel@tonic-gate /*ARGSUSED*/
11277c478bd9Sstevel@tonic-gate static void
11287c478bd9Sstevel@tonic-gate fasttrap_destroy(void *arg, dtrace_id_t id, void *parg)
11297c478bd9Sstevel@tonic-gate {
11307c478bd9Sstevel@tonic-gate 	ASSERT(arg == NULL);
11317c478bd9Sstevel@tonic-gate 	ASSERT(id == fasttrap_probe_id);
11327c478bd9Sstevel@tonic-gate }
11337c478bd9Sstevel@tonic-gate 
11347c478bd9Sstevel@tonic-gate /*ARGSUSED*/
11357c478bd9Sstevel@tonic-gate static void
11367c478bd9Sstevel@tonic-gate fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
11377c478bd9Sstevel@tonic-gate {
11387c478bd9Sstevel@tonic-gate 	fasttrap_probe_t *probe = parg;
11397c478bd9Sstevel@tonic-gate 	int i;
11407c478bd9Sstevel@tonic-gate 	size_t size;
11417c478bd9Sstevel@tonic-gate 
11427c478bd9Sstevel@tonic-gate 	ASSERT(probe != NULL);
11437c478bd9Sstevel@tonic-gate 	ASSERT(!probe->ftp_enabled);
11447c478bd9Sstevel@tonic-gate 	ASSERT(fasttrap_total >= probe->ftp_ntps);
11457c478bd9Sstevel@tonic-gate 
11467c478bd9Sstevel@tonic-gate 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1147*ab9a77c7Sahl 	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
11487c478bd9Sstevel@tonic-gate 
11497c478bd9Sstevel@tonic-gate 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
11507c478bd9Sstevel@tonic-gate 		fasttrap_mod_barrier(probe->ftp_gen);
11517c478bd9Sstevel@tonic-gate 
11527c478bd9Sstevel@tonic-gate 	for (i = 0; i < probe->ftp_ntps; i++) {
11537c478bd9Sstevel@tonic-gate 		kmem_free(probe->ftp_tps[i].fit_tp,
11547c478bd9Sstevel@tonic-gate 		    sizeof (fasttrap_tracepoint_t));
11557c478bd9Sstevel@tonic-gate 	}
11567c478bd9Sstevel@tonic-gate 
11577c478bd9Sstevel@tonic-gate 	kmem_free(probe, size);
11587c478bd9Sstevel@tonic-gate }
11597c478bd9Sstevel@tonic-gate 
11607c478bd9Sstevel@tonic-gate 
11617c478bd9Sstevel@tonic-gate static const dtrace_pattr_t fasttrap_attr = {
11627c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
11637c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
11647c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
11657c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
11667c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
11677c478bd9Sstevel@tonic-gate };
11687c478bd9Sstevel@tonic-gate 
11697c478bd9Sstevel@tonic-gate static dtrace_pops_t fasttrap_pops = {
11707c478bd9Sstevel@tonic-gate 	fasttrap_provide,
11717c478bd9Sstevel@tonic-gate 	NULL,
11727c478bd9Sstevel@tonic-gate 	fasttrap_enable,
11737c478bd9Sstevel@tonic-gate 	fasttrap_disable,
11747c478bd9Sstevel@tonic-gate 	NULL,
11757c478bd9Sstevel@tonic-gate 	NULL,
11767c478bd9Sstevel@tonic-gate 	NULL,
11777c478bd9Sstevel@tonic-gate 	fasttrap_getarg,
11787c478bd9Sstevel@tonic-gate 	NULL,
11797c478bd9Sstevel@tonic-gate 	fasttrap_destroy
11807c478bd9Sstevel@tonic-gate };
11817c478bd9Sstevel@tonic-gate 
11827c478bd9Sstevel@tonic-gate static const dtrace_pattr_t pid_attr = {
11837c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
11847c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
11857c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
11867c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
11877c478bd9Sstevel@tonic-gate { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
11887c478bd9Sstevel@tonic-gate };
11897c478bd9Sstevel@tonic-gate 
11907c478bd9Sstevel@tonic-gate static dtrace_pops_t pid_pops = {
11917c478bd9Sstevel@tonic-gate 	fasttrap_pid_provide,
11927c478bd9Sstevel@tonic-gate 	NULL,
11937c478bd9Sstevel@tonic-gate 	fasttrap_pid_enable,
11947c478bd9Sstevel@tonic-gate 	fasttrap_pid_disable,
11957c478bd9Sstevel@tonic-gate 	NULL,
11967c478bd9Sstevel@tonic-gate 	NULL,
11977c478bd9Sstevel@tonic-gate 	fasttrap_pid_getargdesc,
11987c478bd9Sstevel@tonic-gate 	fasttrap_getarg,
11997c478bd9Sstevel@tonic-gate 	NULL,
12007c478bd9Sstevel@tonic-gate 	fasttrap_pid_destroy
12017c478bd9Sstevel@tonic-gate };
12027c478bd9Sstevel@tonic-gate 
12037c478bd9Sstevel@tonic-gate static dtrace_pops_t usdt_pops = {
12047c478bd9Sstevel@tonic-gate 	fasttrap_pid_provide,
12057c478bd9Sstevel@tonic-gate 	NULL,
12067c478bd9Sstevel@tonic-gate 	fasttrap_pid_enable,
12077c478bd9Sstevel@tonic-gate 	fasttrap_pid_disable,
12087c478bd9Sstevel@tonic-gate 	NULL,
12097c478bd9Sstevel@tonic-gate 	NULL,
12107c478bd9Sstevel@tonic-gate 	fasttrap_pid_getargdesc,
12117c478bd9Sstevel@tonic-gate 	fasttrap_usdt_getarg,
12127c478bd9Sstevel@tonic-gate 	NULL,
12137c478bd9Sstevel@tonic-gate 	fasttrap_pid_destroy
12147c478bd9Sstevel@tonic-gate };
12157c478bd9Sstevel@tonic-gate 
1216b096140dSahl static fasttrap_proc_t *
1217b096140dSahl fasttrap_proc_lookup(pid_t pid)
1218b096140dSahl {
1219b096140dSahl 	fasttrap_bucket_t *bucket;
1220b096140dSahl 	fasttrap_proc_t *fprc, *new_fprc;
1221b096140dSahl 
1222b096140dSahl 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1223b096140dSahl 	mutex_enter(&bucket->ftb_mtx);
1224b096140dSahl 
1225b096140dSahl 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1226b096140dSahl 		if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) {
1227b096140dSahl 			mutex_enter(&fprc->ftpc_mtx);
1228b096140dSahl 			mutex_exit(&bucket->ftb_mtx);
1229b096140dSahl 			fprc->ftpc_count++;
1230b096140dSahl 			mutex_exit(&fprc->ftpc_mtx);
1231b096140dSahl 
1232b096140dSahl 			return (fprc);
1233b096140dSahl 		}
1234b096140dSahl 	}
1235b096140dSahl 
1236b096140dSahl 	/*
1237b096140dSahl 	 * Drop the bucket lock so we don't try to perform a sleeping
1238b096140dSahl 	 * allocation under it.
1239b096140dSahl 	 */
1240b096140dSahl 	mutex_exit(&bucket->ftb_mtx);
1241b096140dSahl 
1242b096140dSahl 	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1243b096140dSahl 	new_fprc->ftpc_pid = pid;
1244b096140dSahl 	new_fprc->ftpc_count = 1;
1245b096140dSahl 
1246b096140dSahl 	mutex_enter(&bucket->ftb_mtx);
1247b096140dSahl 
1248b096140dSahl 	/*
1249b096140dSahl 	 * Take another lap through the list to make sure a proc hasn't
1250b096140dSahl 	 * been created for this pid while we weren't under the bucket lock.
1251b096140dSahl 	 */
1252b096140dSahl 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1253b096140dSahl 		if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) {
1254b096140dSahl 			mutex_enter(&fprc->ftpc_mtx);
1255b096140dSahl 			mutex_exit(&bucket->ftb_mtx);
1256b096140dSahl 			fprc->ftpc_count++;
1257b096140dSahl 			mutex_exit(&fprc->ftpc_mtx);
1258b096140dSahl 
1259b096140dSahl 			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1260b096140dSahl 
1261b096140dSahl 			return (fprc);
1262b096140dSahl 		}
1263b096140dSahl 	}
1264b096140dSahl 
1265b096140dSahl 	new_fprc->ftpc_next = bucket->ftb_data;
1266b096140dSahl 	bucket->ftb_data = new_fprc;
1267b096140dSahl 
1268b096140dSahl 	mutex_exit(&bucket->ftb_mtx);
1269b096140dSahl 
1270b096140dSahl 	return (new_fprc);
1271b096140dSahl }
1272b096140dSahl 
1273b096140dSahl static void
1274b096140dSahl fasttrap_proc_release(fasttrap_proc_t *proc)
1275b096140dSahl {
1276b096140dSahl 	fasttrap_bucket_t *bucket;
1277b096140dSahl 	fasttrap_proc_t *fprc, **fprcp;
1278b096140dSahl 	pid_t pid = proc->ftpc_pid;
1279b096140dSahl 
1280b096140dSahl 	mutex_enter(&proc->ftpc_mtx);
1281b096140dSahl 
1282b096140dSahl 	ASSERT(proc->ftpc_count != 0);
1283b096140dSahl 
1284b096140dSahl 	if (--proc->ftpc_count != 0) {
1285b096140dSahl 		mutex_exit(&proc->ftpc_mtx);
1286b096140dSahl 		return;
1287b096140dSahl 	}
1288b096140dSahl 
1289b096140dSahl 	mutex_exit(&proc->ftpc_mtx);
1290b096140dSahl 
1291b096140dSahl 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1292b096140dSahl 	mutex_enter(&bucket->ftb_mtx);
1293b096140dSahl 
1294b096140dSahl 	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1295b096140dSahl 	while ((fprc = *fprcp) != NULL) {
1296b096140dSahl 		if (fprc == proc)
1297b096140dSahl 			break;
1298b096140dSahl 
1299b096140dSahl 		fprcp = &fprc->ftpc_next;
1300b096140dSahl 	}
1301b096140dSahl 
1302b096140dSahl 	/*
1303b096140dSahl 	 * Something strange has happened if we can't find the proc.
1304b096140dSahl 	 */
1305b096140dSahl 	ASSERT(fprc != NULL);
1306b096140dSahl 
1307b096140dSahl 	*fprcp = fprc->ftpc_next;
1308b096140dSahl 
1309b096140dSahl 	mutex_exit(&bucket->ftb_mtx);
1310b096140dSahl 
1311b096140dSahl 	kmem_free(fprc, sizeof (fasttrap_proc_t));
1312b096140dSahl }
1313b096140dSahl 
13147c478bd9Sstevel@tonic-gate /*
13157c478bd9Sstevel@tonic-gate  * Lookup a fasttrap-managed provider based on its name and associated pid.
13167c478bd9Sstevel@tonic-gate  * If the pattr argument is non-NULL, this function instantiates the provider
13177c478bd9Sstevel@tonic-gate  * if it doesn't exist otherwise it returns NULL. The provider is returned
13187c478bd9Sstevel@tonic-gate  * with its lock held.
13197c478bd9Sstevel@tonic-gate  */
13207c478bd9Sstevel@tonic-gate static fasttrap_provider_t *
13217c478bd9Sstevel@tonic-gate fasttrap_provider_lookup(pid_t pid, const char *name,
13227c478bd9Sstevel@tonic-gate     const dtrace_pattr_t *pattr)
13237c478bd9Sstevel@tonic-gate {
13247c478bd9Sstevel@tonic-gate 	fasttrap_provider_t *fp, *new_fp = NULL;
13257c478bd9Sstevel@tonic-gate 	fasttrap_bucket_t *bucket;
13267c478bd9Sstevel@tonic-gate 	char provname[DTRACE_PROVNAMELEN];
13277c478bd9Sstevel@tonic-gate 	proc_t *p;
1328ad4023c4Sdp 	cred_t *cred;
13297c478bd9Sstevel@tonic-gate 
13307c478bd9Sstevel@tonic-gate 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1331dafb5540Sahl 	ASSERT(pattr != NULL);
13327c478bd9Sstevel@tonic-gate 
13337c478bd9Sstevel@tonic-gate 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
13347c478bd9Sstevel@tonic-gate 	mutex_enter(&bucket->ftb_mtx);
13357c478bd9Sstevel@tonic-gate 
13367c478bd9Sstevel@tonic-gate 	/*
13377c478bd9Sstevel@tonic-gate 	 * Take a lap through the list and return the match if we find it.
13387c478bd9Sstevel@tonic-gate 	 */
13397c478bd9Sstevel@tonic-gate 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
13407c478bd9Sstevel@tonic-gate 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1341b096140dSahl 		    !fp->ftp_retired) {
13427c478bd9Sstevel@tonic-gate 			mutex_enter(&fp->ftp_mtx);
13437c478bd9Sstevel@tonic-gate 			mutex_exit(&bucket->ftb_mtx);
13447c478bd9Sstevel@tonic-gate 			return (fp);
13457c478bd9Sstevel@tonic-gate 		}
13467c478bd9Sstevel@tonic-gate 	}
13477c478bd9Sstevel@tonic-gate 
13487c478bd9Sstevel@tonic-gate 	/*
13497c478bd9Sstevel@tonic-gate 	 * Drop the bucket lock so we don't try to perform a sleeping
13507c478bd9Sstevel@tonic-gate 	 * allocation under it.
13517c478bd9Sstevel@tonic-gate 	 */
13527c478bd9Sstevel@tonic-gate 	mutex_exit(&bucket->ftb_mtx);
13537c478bd9Sstevel@tonic-gate 
1354b096140dSahl 	/*
13557c478bd9Sstevel@tonic-gate 	 * Make sure the process exists, isn't a child created as the result
1356ad4023c4Sdp 	 * of a vfork(2), and isn't a zombie (but may be in fork).
13577c478bd9Sstevel@tonic-gate 	 */
13587c478bd9Sstevel@tonic-gate 	mutex_enter(&pidlock);
135997eda132Sraf 	if ((p = prfind(pid)) == NULL) {
13607c478bd9Sstevel@tonic-gate 		mutex_exit(&pidlock);
13617c478bd9Sstevel@tonic-gate 		return (NULL);
13627c478bd9Sstevel@tonic-gate 	}
13637c478bd9Sstevel@tonic-gate 	mutex_enter(&p->p_lock);
13647c478bd9Sstevel@tonic-gate 	mutex_exit(&pidlock);
136597eda132Sraf 	if (p->p_flag & (SVFORK | SEXITING)) {
136697eda132Sraf 		mutex_exit(&p->p_lock);
136797eda132Sraf 		return (NULL);
136897eda132Sraf 	}
13697c478bd9Sstevel@tonic-gate 
13707c478bd9Sstevel@tonic-gate 	/*
13717c478bd9Sstevel@tonic-gate 	 * Increment p_dtrace_probes so that the process knows to inform us
13727c478bd9Sstevel@tonic-gate 	 * when it exits or execs. fasttrap_provider_free() decrements this
13737c478bd9Sstevel@tonic-gate 	 * when we're done with this provider.
13747c478bd9Sstevel@tonic-gate 	 */
13757c478bd9Sstevel@tonic-gate 	p->p_dtrace_probes++;
13767c478bd9Sstevel@tonic-gate 
1377ad4023c4Sdp 	/*
1378ad4023c4Sdp 	 * Grab the credentials for this process so we have
1379ad4023c4Sdp 	 * something to pass to dtrace_register().
1380ad4023c4Sdp 	 */
13817c478bd9Sstevel@tonic-gate 	mutex_enter(&p->p_crlock);
1382ad4023c4Sdp 	crhold(p->p_cred);
1383ad4023c4Sdp 	cred = p->p_cred;
13847c478bd9Sstevel@tonic-gate 	mutex_exit(&p->p_crlock);
13857c478bd9Sstevel@tonic-gate 	mutex_exit(&p->p_lock);
13867c478bd9Sstevel@tonic-gate 
13877c478bd9Sstevel@tonic-gate 	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1388b096140dSahl 	new_fp->ftp_pid = pid;
1389b096140dSahl 	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1390b096140dSahl 
1391b096140dSahl 	ASSERT(new_fp->ftp_proc != NULL);
13927c478bd9Sstevel@tonic-gate 
13937c478bd9Sstevel@tonic-gate 	mutex_enter(&bucket->ftb_mtx);
13947c478bd9Sstevel@tonic-gate 
13957c478bd9Sstevel@tonic-gate 	/*
13967c478bd9Sstevel@tonic-gate 	 * Take another lap through the list to make sure a provider hasn't
13977c478bd9Sstevel@tonic-gate 	 * been created for this pid while we weren't under the bucket lock.
13987c478bd9Sstevel@tonic-gate 	 */
13997c478bd9Sstevel@tonic-gate 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
14007c478bd9Sstevel@tonic-gate 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1401b096140dSahl 		    !fp->ftp_retired) {
14027c478bd9Sstevel@tonic-gate 			mutex_enter(&fp->ftp_mtx);
14037c478bd9Sstevel@tonic-gate 			mutex_exit(&bucket->ftb_mtx);
14047c478bd9Sstevel@tonic-gate 			fasttrap_provider_free(new_fp);
1405ad4023c4Sdp 			crfree(cred);
14067c478bd9Sstevel@tonic-gate 			return (fp);
14077c478bd9Sstevel@tonic-gate 		}
14087c478bd9Sstevel@tonic-gate 	}
14097c478bd9Sstevel@tonic-gate 
14107c478bd9Sstevel@tonic-gate 	(void) strcpy(new_fp->ftp_name, name);
14117c478bd9Sstevel@tonic-gate 
14127c478bd9Sstevel@tonic-gate 	/*
14137c478bd9Sstevel@tonic-gate 	 * Fail and return NULL if either the provider name is too long
14147c478bd9Sstevel@tonic-gate 	 * or we fail to register this new provider with the DTrace
14157c478bd9Sstevel@tonic-gate 	 * framework. Note that this is the only place we ever construct
14167c478bd9Sstevel@tonic-gate 	 * the full provider name -- we keep it in pieces in the provider
14177c478bd9Sstevel@tonic-gate 	 * structure.
14187c478bd9Sstevel@tonic-gate 	 */
14197c478bd9Sstevel@tonic-gate 	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
14207c478bd9Sstevel@tonic-gate 	    sizeof (provname) ||
14217c478bd9Sstevel@tonic-gate 	    dtrace_register(provname, pattr,
1422ad4023c4Sdp 	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
14237c478bd9Sstevel@tonic-gate 	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
14247c478bd9Sstevel@tonic-gate 	    &new_fp->ftp_provid) != 0) {
14257c478bd9Sstevel@tonic-gate 		mutex_exit(&bucket->ftb_mtx);
14267c478bd9Sstevel@tonic-gate 		fasttrap_provider_free(new_fp);
1427ad4023c4Sdp 		crfree(cred);
14287c478bd9Sstevel@tonic-gate 		return (NULL);
14297c478bd9Sstevel@tonic-gate 	}
14307c478bd9Sstevel@tonic-gate 
14317c478bd9Sstevel@tonic-gate 	new_fp->ftp_next = bucket->ftb_data;
14327c478bd9Sstevel@tonic-gate 	bucket->ftb_data = new_fp;
14337c478bd9Sstevel@tonic-gate 
14347c478bd9Sstevel@tonic-gate 	mutex_enter(&new_fp->ftp_mtx);
14357c478bd9Sstevel@tonic-gate 	mutex_exit(&bucket->ftb_mtx);
14367c478bd9Sstevel@tonic-gate 
1437ad4023c4Sdp 	crfree(cred);
14387c478bd9Sstevel@tonic-gate 	return (new_fp);
14397c478bd9Sstevel@tonic-gate }
14407c478bd9Sstevel@tonic-gate 
14417c478bd9Sstevel@tonic-gate static void
14427c478bd9Sstevel@tonic-gate fasttrap_provider_free(fasttrap_provider_t *provider)
14437c478bd9Sstevel@tonic-gate {
14447c478bd9Sstevel@tonic-gate 	pid_t pid = provider->ftp_pid;
14457c478bd9Sstevel@tonic-gate 	proc_t *p;
14467c478bd9Sstevel@tonic-gate 
14477c478bd9Sstevel@tonic-gate 	/*
1448*ab9a77c7Sahl 	 * There need to be no associated enabled probes, no consumers
1449*ab9a77c7Sahl 	 * creating probes, and no meta providers referencing this provider.
14507c478bd9Sstevel@tonic-gate 	 */
14517c478bd9Sstevel@tonic-gate 	ASSERT(provider->ftp_rcount == 0);
1452*ab9a77c7Sahl 	ASSERT(provider->ftp_ccount == 0);
1453*ab9a77c7Sahl 	ASSERT(provider->ftp_mcount == 0);
14547c478bd9Sstevel@tonic-gate 
1455b096140dSahl 	fasttrap_proc_release(provider->ftp_proc);
1456b096140dSahl 
14577c478bd9Sstevel@tonic-gate 	kmem_free(provider, sizeof (fasttrap_provider_t));
14587c478bd9Sstevel@tonic-gate 
14597c478bd9Sstevel@tonic-gate 	/*
14607c478bd9Sstevel@tonic-gate 	 * Decrement p_dtrace_probes on the process whose provider we're
14617c478bd9Sstevel@tonic-gate 	 * freeing. We don't have to worry about clobbering somone else's
14627c478bd9Sstevel@tonic-gate 	 * modifications to it because we have locked the bucket that
14637c478bd9Sstevel@tonic-gate 	 * corresponds to this process's hash chain in the provider hash
14647c478bd9Sstevel@tonic-gate 	 * table. Don't sweat it if we can't find the process.
14657c478bd9Sstevel@tonic-gate 	 */
14667c478bd9Sstevel@tonic-gate 	mutex_enter(&pidlock);
14677c478bd9Sstevel@tonic-gate 	if ((p = prfind(pid)) == NULL) {
14687c478bd9Sstevel@tonic-gate 		mutex_exit(&pidlock);
14697c478bd9Sstevel@tonic-gate 		return;
14707c478bd9Sstevel@tonic-gate 	}
14717c478bd9Sstevel@tonic-gate 
14727c478bd9Sstevel@tonic-gate 	mutex_enter(&p->p_lock);
14737c478bd9Sstevel@tonic-gate 	mutex_exit(&pidlock);
14747c478bd9Sstevel@tonic-gate 
14757c478bd9Sstevel@tonic-gate 	p->p_dtrace_probes--;
14767c478bd9Sstevel@tonic-gate 	mutex_exit(&p->p_lock);
14777c478bd9Sstevel@tonic-gate }
14787c478bd9Sstevel@tonic-gate 
14797c478bd9Sstevel@tonic-gate static void
1480*ab9a77c7Sahl fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
14817c478bd9Sstevel@tonic-gate {
1482dafb5540Sahl 	fasttrap_provider_t *fp;
1483dafb5540Sahl 	fasttrap_bucket_t *bucket;
1484dafb5540Sahl 	dtrace_provider_id_t provid;
1485dafb5540Sahl 
1486dafb5540Sahl 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1487dafb5540Sahl 
1488dafb5540Sahl 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1489dafb5540Sahl 	mutex_enter(&bucket->ftb_mtx);
1490dafb5540Sahl 
1491dafb5540Sahl 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1492dafb5540Sahl 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1493dafb5540Sahl 		    !fp->ftp_retired)
1494dafb5540Sahl 			break;
1495dafb5540Sahl 	}
1496dafb5540Sahl 
1497dafb5540Sahl 	if (fp == NULL) {
1498dafb5540Sahl 		mutex_exit(&bucket->ftb_mtx);
1499dafb5540Sahl 		return;
1500dafb5540Sahl 	}
15017c478bd9Sstevel@tonic-gate 
1502*ab9a77c7Sahl 	mutex_enter(&fp->ftp_mtx);
1503*ab9a77c7Sahl 	ASSERT(!mprov || fp->ftp_mcount > 0);
1504*ab9a77c7Sahl 	if (mprov && --fp->ftp_mcount != 0)  {
1505*ab9a77c7Sahl 		mutex_exit(&fp->ftp_mtx);
1506*ab9a77c7Sahl 		mutex_exit(&bucket->ftb_mtx);
1507*ab9a77c7Sahl 		return;
1508*ab9a77c7Sahl 	}
1509*ab9a77c7Sahl 
15107c478bd9Sstevel@tonic-gate 	/*
1511b096140dSahl 	 * Mark the provider to be removed in our post-processing step,
1512b096140dSahl 	 * mark it retired, and mark its proc as defunct (though it may
1513b096140dSahl 	 * already be marked defunct by another provider that shares the
1514b096140dSahl 	 * same proc). Marking it indicates that we should try to remove it;
1515b096140dSahl 	 * setting the retired flag indicates that we're done with this
1516b096140dSahl 	 * provider; setting the proc to be defunct indicates that all
1517b096140dSahl 	 * tracepoints associated with the traced process should be ignored.
1518dafb5540Sahl 	 *
1519dafb5540Sahl 	 * We obviously need to take the bucket lock before the provider lock
1520dafb5540Sahl 	 * to perform the lookup, but we need to drop the provider lock
1521dafb5540Sahl 	 * before calling into the DTrace framework since we acquire the
1522dafb5540Sahl 	 * provider lock in callbacks invoked from the DTrace framework. The
1523dafb5540Sahl 	 * bucket lock therefore protects the integrity of the provider hash
1524dafb5540Sahl 	 * table.
15257c478bd9Sstevel@tonic-gate 	 */
1526dafb5540Sahl 	fp->ftp_proc->ftpc_defunct = 1;
1527dafb5540Sahl 	fp->ftp_retired = 1;
1528dafb5540Sahl 	fp->ftp_marked = 1;
1529dafb5540Sahl 	provid = fp->ftp_provid;
1530dafb5540Sahl 	mutex_exit(&fp->ftp_mtx);
15317c478bd9Sstevel@tonic-gate 
15327c478bd9Sstevel@tonic-gate 	/*
15337c478bd9Sstevel@tonic-gate 	 * We don't have to worry about invalidating the same provider twice
15347c478bd9Sstevel@tonic-gate 	 * since fasttrap_provider_lookup() will ignore provider that have
1535b096140dSahl 	 * been marked as retired.
15367c478bd9Sstevel@tonic-gate 	 */
15377c478bd9Sstevel@tonic-gate 	dtrace_invalidate(provid);
15387c478bd9Sstevel@tonic-gate 
1539dafb5540Sahl 	mutex_exit(&bucket->ftb_mtx);
1540dafb5540Sahl 
15417c478bd9Sstevel@tonic-gate 	fasttrap_pid_cleanup();
15427c478bd9Sstevel@tonic-gate }
15437c478bd9Sstevel@tonic-gate 
15447c478bd9Sstevel@tonic-gate static int
15457c478bd9Sstevel@tonic-gate fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
15467c478bd9Sstevel@tonic-gate {
15477c478bd9Sstevel@tonic-gate 	fasttrap_provider_t *provider;
15487c478bd9Sstevel@tonic-gate 	fasttrap_probe_t *pp;
15497c478bd9Sstevel@tonic-gate 	fasttrap_tracepoint_t *tp;
15507c478bd9Sstevel@tonic-gate 	char *name;
15517c478bd9Sstevel@tonic-gate 	int i, aframes, whack;
15527c478bd9Sstevel@tonic-gate 
15537c478bd9Sstevel@tonic-gate 	switch (pdata->ftps_type) {
15547c478bd9Sstevel@tonic-gate 	case DTFTP_ENTRY:
15557c478bd9Sstevel@tonic-gate 		name = "entry";
15567c478bd9Sstevel@tonic-gate 		aframes = FASTTRAP_ENTRY_AFRAMES;
15577c478bd9Sstevel@tonic-gate 		break;
15587c478bd9Sstevel@tonic-gate 	case DTFTP_RETURN:
15597c478bd9Sstevel@tonic-gate 		name = "return";
15607c478bd9Sstevel@tonic-gate 		aframes = FASTTRAP_RETURN_AFRAMES;
15617c478bd9Sstevel@tonic-gate 		break;
15627c478bd9Sstevel@tonic-gate 	case DTFTP_OFFSETS:
15637c478bd9Sstevel@tonic-gate 		name = NULL;
15647c478bd9Sstevel@tonic-gate 		break;
15657c478bd9Sstevel@tonic-gate 	default:
15667c478bd9Sstevel@tonic-gate 		return (EINVAL);
15677c478bd9Sstevel@tonic-gate 	}
15687c478bd9Sstevel@tonic-gate 
15697c478bd9Sstevel@tonic-gate 	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
15707c478bd9Sstevel@tonic-gate 	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
15717c478bd9Sstevel@tonic-gate 		return (ESRCH);
15727c478bd9Sstevel@tonic-gate 
15737c478bd9Sstevel@tonic-gate 	/*
15747c478bd9Sstevel@tonic-gate 	 * Increment this reference count to indicate that a consumer is
15757c478bd9Sstevel@tonic-gate 	 * actively adding a new probe associated with this provider.
15767c478bd9Sstevel@tonic-gate 	 */
15777c478bd9Sstevel@tonic-gate 	provider->ftp_ccount++;
15787c478bd9Sstevel@tonic-gate 	mutex_exit(&provider->ftp_mtx);
15797c478bd9Sstevel@tonic-gate 
15807c478bd9Sstevel@tonic-gate 	if (name != NULL) {
15817c478bd9Sstevel@tonic-gate 		if (dtrace_probe_lookup(provider->ftp_provid,
15827c478bd9Sstevel@tonic-gate 		    pdata->ftps_mod, pdata->ftps_func, name) != 0)
15837c478bd9Sstevel@tonic-gate 			goto done;
15847c478bd9Sstevel@tonic-gate 
15857c478bd9Sstevel@tonic-gate 		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
15867c478bd9Sstevel@tonic-gate 
15877c478bd9Sstevel@tonic-gate 		if (fasttrap_total > fasttrap_max) {
15887c478bd9Sstevel@tonic-gate 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
15897c478bd9Sstevel@tonic-gate 			goto no_mem;
15907c478bd9Sstevel@tonic-gate 		}
15917c478bd9Sstevel@tonic-gate 
15927c478bd9Sstevel@tonic-gate 		ASSERT(pdata->ftps_noffs > 0);
1593*ab9a77c7Sahl 		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1594*ab9a77c7Sahl 		    ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
15957c478bd9Sstevel@tonic-gate 
15967c478bd9Sstevel@tonic-gate 		pp->ftp_prov = provider;
15977c478bd9Sstevel@tonic-gate 		pp->ftp_faddr = pdata->ftps_pc;
15987c478bd9Sstevel@tonic-gate 		pp->ftp_fsize = pdata->ftps_size;
15997c478bd9Sstevel@tonic-gate 		pp->ftp_pid = pdata->ftps_pid;
16007c478bd9Sstevel@tonic-gate 		pp->ftp_ntps = pdata->ftps_noffs;
16017c478bd9Sstevel@tonic-gate 
16027c478bd9Sstevel@tonic-gate 		for (i = 0; i < pdata->ftps_noffs; i++) {
16037c478bd9Sstevel@tonic-gate 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
16047c478bd9Sstevel@tonic-gate 			    KM_SLEEP);
16057c478bd9Sstevel@tonic-gate 
1606b096140dSahl 			tp->ftt_proc = provider->ftp_proc;
16077c478bd9Sstevel@tonic-gate 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
16087c478bd9Sstevel@tonic-gate 			tp->ftt_pid = pdata->ftps_pid;
16097c478bd9Sstevel@tonic-gate 
16107c478bd9Sstevel@tonic-gate 			pp->ftp_tps[i].fit_tp = tp;
16117c478bd9Sstevel@tonic-gate 			pp->ftp_tps[i].fit_id.fti_probe = pp;
1612ac448965Sahl 			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
16137c478bd9Sstevel@tonic-gate 		}
16147c478bd9Sstevel@tonic-gate 
16157c478bd9Sstevel@tonic-gate 		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
16167c478bd9Sstevel@tonic-gate 		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
16177c478bd9Sstevel@tonic-gate 	} else {
16187c478bd9Sstevel@tonic-gate 		for (i = 0; i < pdata->ftps_noffs; i++) {
16197c478bd9Sstevel@tonic-gate 			char name_str[17];
16207c478bd9Sstevel@tonic-gate 
16217c478bd9Sstevel@tonic-gate 			(void) sprintf(name_str, "%llx",
16227c478bd9Sstevel@tonic-gate 			    (unsigned long long)pdata->ftps_offs[i]);
16237c478bd9Sstevel@tonic-gate 
16247c478bd9Sstevel@tonic-gate 			if (dtrace_probe_lookup(provider->ftp_provid,
16257c478bd9Sstevel@tonic-gate 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
16267c478bd9Sstevel@tonic-gate 				continue;
16277c478bd9Sstevel@tonic-gate 
16287c478bd9Sstevel@tonic-gate 			atomic_add_32(&fasttrap_total, 1);
16297c478bd9Sstevel@tonic-gate 
16307c478bd9Sstevel@tonic-gate 			if (fasttrap_total > fasttrap_max) {
16317c478bd9Sstevel@tonic-gate 				atomic_add_32(&fasttrap_total, -1);
16327c478bd9Sstevel@tonic-gate 				goto no_mem;
16337c478bd9Sstevel@tonic-gate 			}
16347c478bd9Sstevel@tonic-gate 
16357c478bd9Sstevel@tonic-gate 			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
16367c478bd9Sstevel@tonic-gate 
16377c478bd9Sstevel@tonic-gate 			pp->ftp_prov = provider;
16387c478bd9Sstevel@tonic-gate 			pp->ftp_faddr = pdata->ftps_pc;
16397c478bd9Sstevel@tonic-gate 			pp->ftp_fsize = pdata->ftps_size;
16407c478bd9Sstevel@tonic-gate 			pp->ftp_pid = pdata->ftps_pid;
16417c478bd9Sstevel@tonic-gate 			pp->ftp_ntps = 1;
16427c478bd9Sstevel@tonic-gate 
16437c478bd9Sstevel@tonic-gate 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
16447c478bd9Sstevel@tonic-gate 			    KM_SLEEP);
16457c478bd9Sstevel@tonic-gate 
1646b096140dSahl 			tp->ftt_proc = provider->ftp_proc;
16477c478bd9Sstevel@tonic-gate 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
16487c478bd9Sstevel@tonic-gate 			tp->ftt_pid = pdata->ftps_pid;
16497c478bd9Sstevel@tonic-gate 
16507c478bd9Sstevel@tonic-gate 			pp->ftp_tps[0].fit_tp = tp;
16517c478bd9Sstevel@tonic-gate 			pp->ftp_tps[0].fit_id.fti_probe = pp;
1652ac448965Sahl 			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
16537c478bd9Sstevel@tonic-gate 
16547c478bd9Sstevel@tonic-gate 			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
16557c478bd9Sstevel@tonic-gate 			    pdata->ftps_mod, pdata->ftps_func, name_str,
16567c478bd9Sstevel@tonic-gate 			    FASTTRAP_OFFSET_AFRAMES, pp);
16577c478bd9Sstevel@tonic-gate 		}
16587c478bd9Sstevel@tonic-gate 	}
16597c478bd9Sstevel@tonic-gate 
16607c478bd9Sstevel@tonic-gate done:
16617c478bd9Sstevel@tonic-gate 	/*
16627c478bd9Sstevel@tonic-gate 	 * We know that the provider is still valid since we incremented the
16637c478bd9Sstevel@tonic-gate 	 * reference count. If someone tried to free this provider while we
16647c478bd9Sstevel@tonic-gate 	 * were using it (e.g. because the process called exec(2) or exit(2)),
16657c478bd9Sstevel@tonic-gate 	 * take note of that and try to free it now.
16667c478bd9Sstevel@tonic-gate 	 */
16677c478bd9Sstevel@tonic-gate 	mutex_enter(&provider->ftp_mtx);
16687c478bd9Sstevel@tonic-gate 	provider->ftp_ccount--;
1669b096140dSahl 	whack = provider->ftp_retired;
16707c478bd9Sstevel@tonic-gate 	mutex_exit(&provider->ftp_mtx);
16717c478bd9Sstevel@tonic-gate 
16727c478bd9Sstevel@tonic-gate 	if (whack)
16737c478bd9Sstevel@tonic-gate 		fasttrap_pid_cleanup();
16747c478bd9Sstevel@tonic-gate 
16757c478bd9Sstevel@tonic-gate 	return (0);
16767c478bd9Sstevel@tonic-gate 
16777c478bd9Sstevel@tonic-gate no_mem:
16787c478bd9Sstevel@tonic-gate 	/*
16797c478bd9Sstevel@tonic-gate 	 * If we've exhausted the allowable resources, we'll try to remove
16807c478bd9Sstevel@tonic-gate 	 * this provider to free some up. This is to cover the case where
16817c478bd9Sstevel@tonic-gate 	 * the user has accidentally created many more probes than was
16827c478bd9Sstevel@tonic-gate 	 * intended (e.g. pid123:::).
16837c478bd9Sstevel@tonic-gate 	 */
16847c478bd9Sstevel@tonic-gate 	mutex_enter(&provider->ftp_mtx);
16857c478bd9Sstevel@tonic-gate 	provider->ftp_ccount--;
16867c478bd9Sstevel@tonic-gate 	provider->ftp_marked = 1;
16877c478bd9Sstevel@tonic-gate 	mutex_exit(&provider->ftp_mtx);
16887c478bd9Sstevel@tonic-gate 
16897c478bd9Sstevel@tonic-gate 	fasttrap_pid_cleanup();
16907c478bd9Sstevel@tonic-gate 
16917c478bd9Sstevel@tonic-gate 	return (ENOMEM);
16927c478bd9Sstevel@tonic-gate }
16937c478bd9Sstevel@tonic-gate 
16947c478bd9Sstevel@tonic-gate /*ARGSUSED*/
16957c478bd9Sstevel@tonic-gate static void *
16967c478bd9Sstevel@tonic-gate fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
16977c478bd9Sstevel@tonic-gate {
16987c478bd9Sstevel@tonic-gate 	fasttrap_provider_t *provider;
16997c478bd9Sstevel@tonic-gate 
17007c478bd9Sstevel@tonic-gate 	/*
17017c478bd9Sstevel@tonic-gate 	 * A 32-bit unsigned integer (like a pid for example) can be
17027c478bd9Sstevel@tonic-gate 	 * expressed in 10 or fewer decimal digits. Make sure that we'll
17037c478bd9Sstevel@tonic-gate 	 * have enough space for the provider name.
17047c478bd9Sstevel@tonic-gate 	 */
17057c478bd9Sstevel@tonic-gate 	if (strlen(dhpv->dthpv_provname) + 10 >=
17067c478bd9Sstevel@tonic-gate 	    sizeof (provider->ftp_name)) {
17077c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
17087c478bd9Sstevel@tonic-gate 		    "name too long to accomodate pid", dhpv->dthpv_provname);
17097c478bd9Sstevel@tonic-gate 		return (NULL);
17107c478bd9Sstevel@tonic-gate 	}
17117c478bd9Sstevel@tonic-gate 
17127c478bd9Sstevel@tonic-gate 	/*
17137c478bd9Sstevel@tonic-gate 	 * Don't let folks spoof the true pid provider.
17147c478bd9Sstevel@tonic-gate 	 */
17157c478bd9Sstevel@tonic-gate 	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
17167c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
17177c478bd9Sstevel@tonic-gate 		    "%s is an invalid name", dhpv->dthpv_provname,
17187c478bd9Sstevel@tonic-gate 		    FASTTRAP_PID_NAME);
17197c478bd9Sstevel@tonic-gate 		return (NULL);
17207c478bd9Sstevel@tonic-gate 	}
17217c478bd9Sstevel@tonic-gate 
17227c478bd9Sstevel@tonic-gate 	/*
17237c478bd9Sstevel@tonic-gate 	 * The highest stability class that fasttrap supports is ISA; cap
17247c478bd9Sstevel@tonic-gate 	 * the stability of the new provider accordingly.
17257c478bd9Sstevel@tonic-gate 	 */
17267c478bd9Sstevel@tonic-gate 	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class >= DTRACE_CLASS_COMMON)
17277c478bd9Sstevel@tonic-gate 		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
17287c478bd9Sstevel@tonic-gate 	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class >= DTRACE_CLASS_COMMON)
17297c478bd9Sstevel@tonic-gate 		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
17307c478bd9Sstevel@tonic-gate 	if (dhpv->dthpv_pattr.dtpa_func.dtat_class >= DTRACE_CLASS_COMMON)
17317c478bd9Sstevel@tonic-gate 		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
17327c478bd9Sstevel@tonic-gate 	if (dhpv->dthpv_pattr.dtpa_name.dtat_class >= DTRACE_CLASS_COMMON)
17337c478bd9Sstevel@tonic-gate 		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
17347c478bd9Sstevel@tonic-gate 	if (dhpv->dthpv_pattr.dtpa_args.dtat_class >= DTRACE_CLASS_COMMON)
17357c478bd9Sstevel@tonic-gate 		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
17367c478bd9Sstevel@tonic-gate 
17377c478bd9Sstevel@tonic-gate 	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
17387c478bd9Sstevel@tonic-gate 	    &dhpv->dthpv_pattr)) == NULL) {
17397c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "failed to instantiate provider %s for "
17407c478bd9Sstevel@tonic-gate 		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
17417c478bd9Sstevel@tonic-gate 		return (NULL);
17427c478bd9Sstevel@tonic-gate 	}
17437c478bd9Sstevel@tonic-gate 
17447c478bd9Sstevel@tonic-gate 	/*
1745*ab9a77c7Sahl 	 * Up the meta provider count so this provider isn't removed until
1746*ab9a77c7Sahl 	 * the meta provider has been told to remove it.
17477c478bd9Sstevel@tonic-gate 	 */
1748*ab9a77c7Sahl 	provider->ftp_mcount++;
17497c478bd9Sstevel@tonic-gate 
17507c478bd9Sstevel@tonic-gate 	mutex_exit(&provider->ftp_mtx);
17517c478bd9Sstevel@tonic-gate 
17527c478bd9Sstevel@tonic-gate 	return (provider);
17537c478bd9Sstevel@tonic-gate }
17547c478bd9Sstevel@tonic-gate 
17557c478bd9Sstevel@tonic-gate /*ARGSUSED*/
17567c478bd9Sstevel@tonic-gate static void
17577c478bd9Sstevel@tonic-gate fasttrap_meta_create_probe(void *arg, void *parg,
17587c478bd9Sstevel@tonic-gate     dtrace_helper_probedesc_t *dhpb)
17597c478bd9Sstevel@tonic-gate {
17607c478bd9Sstevel@tonic-gate 	fasttrap_provider_t *provider = parg;
17617c478bd9Sstevel@tonic-gate 	fasttrap_probe_t *pp;
17627c478bd9Sstevel@tonic-gate 	fasttrap_tracepoint_t *tp;
1763ac448965Sahl 	int i, j;
1764ac448965Sahl 	uint32_t ntps;
17657c478bd9Sstevel@tonic-gate 
17667c478bd9Sstevel@tonic-gate 	mutex_enter(&provider->ftp_mtx);
17677c478bd9Sstevel@tonic-gate 
17687c478bd9Sstevel@tonic-gate 	if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
17697c478bd9Sstevel@tonic-gate 	    dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
17707c478bd9Sstevel@tonic-gate 		mutex_exit(&provider->ftp_mtx);
17717c478bd9Sstevel@tonic-gate 		return;
17727c478bd9Sstevel@tonic-gate 	}
17737c478bd9Sstevel@tonic-gate 
1774ac448965Sahl 	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1775*ab9a77c7Sahl 	ASSERT(ntps > 0);
1776ac448965Sahl 
1777ac448965Sahl 	atomic_add_32(&fasttrap_total, ntps);
17787c478bd9Sstevel@tonic-gate 
17797c478bd9Sstevel@tonic-gate 	if (fasttrap_total > fasttrap_max) {
1780ac448965Sahl 		atomic_add_32(&fasttrap_total, -ntps);
17817c478bd9Sstevel@tonic-gate 		mutex_exit(&provider->ftp_mtx);
17827c478bd9Sstevel@tonic-gate 		return;
17837c478bd9Sstevel@tonic-gate 	}
17847c478bd9Sstevel@tonic-gate 
1785*ab9a77c7Sahl 	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
17867c478bd9Sstevel@tonic-gate 
17877c478bd9Sstevel@tonic-gate 	pp->ftp_prov = provider;
17887c478bd9Sstevel@tonic-gate 	pp->ftp_pid = provider->ftp_pid;
1789ac448965Sahl 	pp->ftp_ntps = ntps;
17907c478bd9Sstevel@tonic-gate 	pp->ftp_nargs = dhpb->dthpb_xargc;
17917c478bd9Sstevel@tonic-gate 	pp->ftp_xtypes = dhpb->dthpb_xtypes;
17927c478bd9Sstevel@tonic-gate 	pp->ftp_ntypes = dhpb->dthpb_ntypes;
17937c478bd9Sstevel@tonic-gate 
1794ac448965Sahl 	/*
1795ac448965Sahl 	 * First create a tracepoint for each actual point of interest.
1796ac448965Sahl 	 */
1797ac448965Sahl 	for (i = 0; i < dhpb->dthpb_noffs; i++) {
17987c478bd9Sstevel@tonic-gate 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
17997c478bd9Sstevel@tonic-gate 
1800b096140dSahl 		tp->ftt_proc = provider->ftp_proc;
18017c478bd9Sstevel@tonic-gate 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
18027c478bd9Sstevel@tonic-gate 		tp->ftt_pid = provider->ftp_pid;
18037c478bd9Sstevel@tonic-gate 
18047c478bd9Sstevel@tonic-gate 		pp->ftp_tps[i].fit_tp = tp;
18057c478bd9Sstevel@tonic-gate 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1806ac448965Sahl #ifdef __sparc
1807ac448965Sahl 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1808ac448965Sahl #else
1809ac448965Sahl 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1810ac448965Sahl #endif
1811ac448965Sahl 	}
1812ac448965Sahl 
1813ac448965Sahl 	/*
1814ac448965Sahl 	 * Then create a tracepoint for each is-enabled point.
1815ac448965Sahl 	 */
1816ac448965Sahl 	for (j = 0; i < ntps; i++, j++) {
1817ac448965Sahl 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1818ac448965Sahl 
1819ac448965Sahl 		tp->ftt_proc = provider->ftp_proc;
1820ac448965Sahl 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1821ac448965Sahl 		tp->ftt_pid = provider->ftp_pid;
1822ac448965Sahl 
1823ac448965Sahl 		pp->ftp_tps[i].fit_tp = tp;
1824ac448965Sahl 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1825ac448965Sahl 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
18267c478bd9Sstevel@tonic-gate 	}
18277c478bd9Sstevel@tonic-gate 
18287c478bd9Sstevel@tonic-gate 	/*
18297c478bd9Sstevel@tonic-gate 	 * If the arguments are shuffled around we set the argument remapping
18307c478bd9Sstevel@tonic-gate 	 * table. Later, when the probe fires, we only remap the arguments
18317c478bd9Sstevel@tonic-gate 	 * if the table is non-NULL.
18327c478bd9Sstevel@tonic-gate 	 */
18337c478bd9Sstevel@tonic-gate 	for (i = 0; i < dhpb->dthpb_xargc; i++) {
18347c478bd9Sstevel@tonic-gate 		if (dhpb->dthpb_args[i] != i) {
18357c478bd9Sstevel@tonic-gate 			pp->ftp_argmap = dhpb->dthpb_args;
18367c478bd9Sstevel@tonic-gate 			break;
18377c478bd9Sstevel@tonic-gate 		}
18387c478bd9Sstevel@tonic-gate 	}
18397c478bd9Sstevel@tonic-gate 
18407c478bd9Sstevel@tonic-gate 	/*
18417c478bd9Sstevel@tonic-gate 	 * The probe is fully constructed -- register it with DTrace.
18427c478bd9Sstevel@tonic-gate 	 */
18437c478bd9Sstevel@tonic-gate 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
18447c478bd9Sstevel@tonic-gate 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
18457c478bd9Sstevel@tonic-gate 
18467c478bd9Sstevel@tonic-gate 	mutex_exit(&provider->ftp_mtx);
18477c478bd9Sstevel@tonic-gate }
18487c478bd9Sstevel@tonic-gate 
18497c478bd9Sstevel@tonic-gate /*ARGSUSED*/
18507c478bd9Sstevel@tonic-gate static void
18517c478bd9Sstevel@tonic-gate fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
18527c478bd9Sstevel@tonic-gate {
18537c478bd9Sstevel@tonic-gate 	/*
1854dafb5540Sahl 	 * Clean up the USDT provider. There may be active consumers of the
1855dafb5540Sahl 	 * provider busy adding probes, no damage will actually befall the
1856dafb5540Sahl 	 * provider until that count has dropped to zero. This just puts
1857dafb5540Sahl 	 * the provider on death row.
18587c478bd9Sstevel@tonic-gate 	 */
1859dafb5540Sahl 	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
18607c478bd9Sstevel@tonic-gate }
18617c478bd9Sstevel@tonic-gate 
18627c478bd9Sstevel@tonic-gate static dtrace_mops_t fasttrap_mops = {
18637c478bd9Sstevel@tonic-gate 	fasttrap_meta_create_probe,
18647c478bd9Sstevel@tonic-gate 	fasttrap_meta_provide,
18657c478bd9Sstevel@tonic-gate 	fasttrap_meta_remove
18667c478bd9Sstevel@tonic-gate };
18677c478bd9Sstevel@tonic-gate 
18687c478bd9Sstevel@tonic-gate /*ARGSUSED*/
18697c478bd9Sstevel@tonic-gate static int
18707c478bd9Sstevel@tonic-gate fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
18717c478bd9Sstevel@tonic-gate {
18727c478bd9Sstevel@tonic-gate 	return (0);
18737c478bd9Sstevel@tonic-gate }
18747c478bd9Sstevel@tonic-gate 
18757c478bd9Sstevel@tonic-gate /*ARGSUSED*/
18767c478bd9Sstevel@tonic-gate static int
18777c478bd9Sstevel@tonic-gate fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
18787c478bd9Sstevel@tonic-gate {
18797c478bd9Sstevel@tonic-gate 	if (!dtrace_attached())
18807c478bd9Sstevel@tonic-gate 		return (EAGAIN);
18817c478bd9Sstevel@tonic-gate 
18827c478bd9Sstevel@tonic-gate 	if (cmd == FASTTRAPIOC_MAKEPROBE) {
18837c478bd9Sstevel@tonic-gate 		fasttrap_probe_spec_t *uprobe = (void *)arg;
18847c478bd9Sstevel@tonic-gate 		fasttrap_probe_spec_t *probe;
18857c478bd9Sstevel@tonic-gate 		uint64_t noffs;
18867c478bd9Sstevel@tonic-gate 		size_t size;
18877c478bd9Sstevel@tonic-gate 		int ret;
18887c478bd9Sstevel@tonic-gate 		char *c;
18897c478bd9Sstevel@tonic-gate 
18907c478bd9Sstevel@tonic-gate 		if (copyin(&uprobe->ftps_noffs, &noffs,
18917c478bd9Sstevel@tonic-gate 		    sizeof (uprobe->ftps_noffs)))
18927c478bd9Sstevel@tonic-gate 			return (EFAULT);
18937c478bd9Sstevel@tonic-gate 
18947c478bd9Sstevel@tonic-gate 		/*
18957c478bd9Sstevel@tonic-gate 		 * Probes must have at least one tracepoint.
18967c478bd9Sstevel@tonic-gate 		 */
18977c478bd9Sstevel@tonic-gate 		if (noffs == 0)
18987c478bd9Sstevel@tonic-gate 			return (EINVAL);
18997c478bd9Sstevel@tonic-gate 
19007c478bd9Sstevel@tonic-gate 		size = sizeof (fasttrap_probe_spec_t) +
19017c478bd9Sstevel@tonic-gate 		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
19027c478bd9Sstevel@tonic-gate 
19037c478bd9Sstevel@tonic-gate 		if (size > 1024 * 1024)
19047c478bd9Sstevel@tonic-gate 			return (ENOMEM);
19057c478bd9Sstevel@tonic-gate 
19067c478bd9Sstevel@tonic-gate 		probe = kmem_alloc(size, KM_SLEEP);
19077c478bd9Sstevel@tonic-gate 
19087c478bd9Sstevel@tonic-gate 		if (copyin(uprobe, probe, size) != 0) {
19097c478bd9Sstevel@tonic-gate 			kmem_free(probe, size);
19107c478bd9Sstevel@tonic-gate 			return (EFAULT);
19117c478bd9Sstevel@tonic-gate 		}
19127c478bd9Sstevel@tonic-gate 
19137c478bd9Sstevel@tonic-gate 		/*
19147c478bd9Sstevel@tonic-gate 		 * Verify that the function and module strings contain no
19157c478bd9Sstevel@tonic-gate 		 * funny characters.
19167c478bd9Sstevel@tonic-gate 		 */
19177c478bd9Sstevel@tonic-gate 		for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
19187c478bd9Sstevel@tonic-gate 			if (*c < 0x20 || 0x7f <= *c) {
19197c478bd9Sstevel@tonic-gate 				ret = EINVAL;
19207c478bd9Sstevel@tonic-gate 				goto err;
19217c478bd9Sstevel@tonic-gate 			}
19227c478bd9Sstevel@tonic-gate 		}
19237c478bd9Sstevel@tonic-gate 
19247c478bd9Sstevel@tonic-gate 		for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
19257c478bd9Sstevel@tonic-gate 			if (*c < 0x20 || 0x7f <= *c) {
19267c478bd9Sstevel@tonic-gate 				ret = EINVAL;
19277c478bd9Sstevel@tonic-gate 				goto err;
19287c478bd9Sstevel@tonic-gate 			}
19297c478bd9Sstevel@tonic-gate 		}
19307c478bd9Sstevel@tonic-gate 
19317c478bd9Sstevel@tonic-gate 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
19327c478bd9Sstevel@tonic-gate 			proc_t *p;
19337c478bd9Sstevel@tonic-gate 			pid_t pid = probe->ftps_pid;
19347c478bd9Sstevel@tonic-gate 
19357c478bd9Sstevel@tonic-gate 			mutex_enter(&pidlock);
19367c478bd9Sstevel@tonic-gate 			/*
19377c478bd9Sstevel@tonic-gate 			 * Report an error if the process doesn't exist
19387c478bd9Sstevel@tonic-gate 			 * or is actively being birthed.
19397c478bd9Sstevel@tonic-gate 			 */
19407c478bd9Sstevel@tonic-gate 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
19417c478bd9Sstevel@tonic-gate 				mutex_exit(&pidlock);
19427c478bd9Sstevel@tonic-gate 				return (ESRCH);
19437c478bd9Sstevel@tonic-gate 			}
19447c478bd9Sstevel@tonic-gate 			mutex_enter(&p->p_lock);
19457c478bd9Sstevel@tonic-gate 			mutex_exit(&pidlock);
19467c478bd9Sstevel@tonic-gate 
19477c478bd9Sstevel@tonic-gate 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
19487c478bd9Sstevel@tonic-gate 			    VREAD | VWRITE)) != 0) {
19497c478bd9Sstevel@tonic-gate 				mutex_exit(&p->p_lock);
19507c478bd9Sstevel@tonic-gate 				return (ret);
19517c478bd9Sstevel@tonic-gate 			}
19527c478bd9Sstevel@tonic-gate 
19537c478bd9Sstevel@tonic-gate 			mutex_exit(&p->p_lock);
19547c478bd9Sstevel@tonic-gate 		}
19557c478bd9Sstevel@tonic-gate 
19567c478bd9Sstevel@tonic-gate 		ret = fasttrap_add_probe(probe);
19577c478bd9Sstevel@tonic-gate err:
19587c478bd9Sstevel@tonic-gate 		kmem_free(probe, size);
19597c478bd9Sstevel@tonic-gate 
19607c478bd9Sstevel@tonic-gate 		return (ret);
19617c478bd9Sstevel@tonic-gate 
19627c478bd9Sstevel@tonic-gate 	} else if (cmd == FASTTRAPIOC_GETINSTR) {
19637c478bd9Sstevel@tonic-gate 		fasttrap_instr_query_t instr;
19647c478bd9Sstevel@tonic-gate 		fasttrap_tracepoint_t *tp;
19657c478bd9Sstevel@tonic-gate 		uint_t index;
19667c478bd9Sstevel@tonic-gate 		int ret;
19677c478bd9Sstevel@tonic-gate 
19687c478bd9Sstevel@tonic-gate 		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
19697c478bd9Sstevel@tonic-gate 			return (EFAULT);
19707c478bd9Sstevel@tonic-gate 
19717c478bd9Sstevel@tonic-gate 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
19727c478bd9Sstevel@tonic-gate 			proc_t *p;
19737c478bd9Sstevel@tonic-gate 			pid_t pid = instr.ftiq_pid;
19747c478bd9Sstevel@tonic-gate 
19757c478bd9Sstevel@tonic-gate 			mutex_enter(&pidlock);
19767c478bd9Sstevel@tonic-gate 			/*
19777c478bd9Sstevel@tonic-gate 			 * Report an error if the process doesn't exist
19787c478bd9Sstevel@tonic-gate 			 * or is actively being birthed.
19797c478bd9Sstevel@tonic-gate 			 */
19807c478bd9Sstevel@tonic-gate 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
19817c478bd9Sstevel@tonic-gate 				mutex_exit(&pidlock);
19827c478bd9Sstevel@tonic-gate 				return (ESRCH);
19837c478bd9Sstevel@tonic-gate 			}
19847c478bd9Sstevel@tonic-gate 			mutex_enter(&p->p_lock);
19857c478bd9Sstevel@tonic-gate 			mutex_exit(&pidlock);
19867c478bd9Sstevel@tonic-gate 
19877c478bd9Sstevel@tonic-gate 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
19887c478bd9Sstevel@tonic-gate 			    VREAD)) != 0) {
19897c478bd9Sstevel@tonic-gate 				mutex_exit(&p->p_lock);
19907c478bd9Sstevel@tonic-gate 				return (ret);
19917c478bd9Sstevel@tonic-gate 			}
19927c478bd9Sstevel@tonic-gate 
19937c478bd9Sstevel@tonic-gate 			mutex_exit(&p->p_lock);
19947c478bd9Sstevel@tonic-gate 		}
19957c478bd9Sstevel@tonic-gate 
19967c478bd9Sstevel@tonic-gate 		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
19977c478bd9Sstevel@tonic-gate 
19987c478bd9Sstevel@tonic-gate 		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
19997c478bd9Sstevel@tonic-gate 		tp = fasttrap_tpoints.fth_table[index].ftb_data;
20007c478bd9Sstevel@tonic-gate 		while (tp != NULL) {
20017c478bd9Sstevel@tonic-gate 			if (instr.ftiq_pid == tp->ftt_pid &&
20027c478bd9Sstevel@tonic-gate 			    instr.ftiq_pc == tp->ftt_pc &&
2003b096140dSahl 			    !tp->ftt_proc->ftpc_defunct)
20047c478bd9Sstevel@tonic-gate 				break;
20057c478bd9Sstevel@tonic-gate 
20067c478bd9Sstevel@tonic-gate 			tp = tp->ftt_next;
20077c478bd9Sstevel@tonic-gate 		}
20087c478bd9Sstevel@tonic-gate 
20097c478bd9Sstevel@tonic-gate 		if (tp == NULL) {
20107c478bd9Sstevel@tonic-gate 			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
20117c478bd9Sstevel@tonic-gate 			return (ENOENT);
20127c478bd9Sstevel@tonic-gate 		}
20137c478bd9Sstevel@tonic-gate 
20147c478bd9Sstevel@tonic-gate 		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
20157c478bd9Sstevel@tonic-gate 		    sizeof (instr.ftiq_instr));
20167c478bd9Sstevel@tonic-gate 		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
20177c478bd9Sstevel@tonic-gate 
20187c478bd9Sstevel@tonic-gate 		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
20197c478bd9Sstevel@tonic-gate 			return (EFAULT);
20207c478bd9Sstevel@tonic-gate 
20217c478bd9Sstevel@tonic-gate 		return (0);
20227c478bd9Sstevel@tonic-gate 	}
20237c478bd9Sstevel@tonic-gate 
20247c478bd9Sstevel@tonic-gate 	return (EINVAL);
20257c478bd9Sstevel@tonic-gate }
20267c478bd9Sstevel@tonic-gate 
20277c478bd9Sstevel@tonic-gate static struct cb_ops fasttrap_cb_ops = {
20287c478bd9Sstevel@tonic-gate 	fasttrap_open,		/* open */
20297c478bd9Sstevel@tonic-gate 	nodev,			/* close */
20307c478bd9Sstevel@tonic-gate 	nulldev,		/* strategy */
20317c478bd9Sstevel@tonic-gate 	nulldev,		/* print */
20327c478bd9Sstevel@tonic-gate 	nodev,			/* dump */
20337c478bd9Sstevel@tonic-gate 	nodev,			/* read */
20347c478bd9Sstevel@tonic-gate 	nodev,			/* write */
20357c478bd9Sstevel@tonic-gate 	fasttrap_ioctl,		/* ioctl */
20367c478bd9Sstevel@tonic-gate 	nodev,			/* devmap */
20377c478bd9Sstevel@tonic-gate 	nodev,			/* mmap */
20387c478bd9Sstevel@tonic-gate 	nodev,			/* segmap */
20397c478bd9Sstevel@tonic-gate 	nochpoll,		/* poll */
20407c478bd9Sstevel@tonic-gate 	ddi_prop_op,		/* cb_prop_op */
20417c478bd9Sstevel@tonic-gate 	0,			/* streamtab  */
20427c478bd9Sstevel@tonic-gate 	D_NEW | D_MP		/* Driver compatibility flag */
20437c478bd9Sstevel@tonic-gate };
20447c478bd9Sstevel@tonic-gate 
20457c478bd9Sstevel@tonic-gate /*ARGSUSED*/
20467c478bd9Sstevel@tonic-gate static int
20477c478bd9Sstevel@tonic-gate fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
20487c478bd9Sstevel@tonic-gate {
20497c478bd9Sstevel@tonic-gate 	int error;
20507c478bd9Sstevel@tonic-gate 
20517c478bd9Sstevel@tonic-gate 	switch (infocmd) {
20527c478bd9Sstevel@tonic-gate 	case DDI_INFO_DEVT2DEVINFO:
20537c478bd9Sstevel@tonic-gate 		*result = (void *)fasttrap_devi;
20547c478bd9Sstevel@tonic-gate 		error = DDI_SUCCESS;
20557c478bd9Sstevel@tonic-gate 		break;
20567c478bd9Sstevel@tonic-gate 	case DDI_INFO_DEVT2INSTANCE:
20577c478bd9Sstevel@tonic-gate 		*result = (void *)0;
20587c478bd9Sstevel@tonic-gate 		error = DDI_SUCCESS;
20597c478bd9Sstevel@tonic-gate 		break;
20607c478bd9Sstevel@tonic-gate 	default:
20617c478bd9Sstevel@tonic-gate 		error = DDI_FAILURE;
20627c478bd9Sstevel@tonic-gate 	}
20637c478bd9Sstevel@tonic-gate 	return (error);
20647c478bd9Sstevel@tonic-gate }
20657c478bd9Sstevel@tonic-gate 
20667c478bd9Sstevel@tonic-gate static int
20677c478bd9Sstevel@tonic-gate fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
20687c478bd9Sstevel@tonic-gate {
20697c478bd9Sstevel@tonic-gate 	ulong_t nent;
20707c478bd9Sstevel@tonic-gate 
20717c478bd9Sstevel@tonic-gate 	switch (cmd) {
20727c478bd9Sstevel@tonic-gate 	case DDI_ATTACH:
20737c478bd9Sstevel@tonic-gate 		break;
20747c478bd9Sstevel@tonic-gate 	case DDI_RESUME:
20757c478bd9Sstevel@tonic-gate 		return (DDI_SUCCESS);
20767c478bd9Sstevel@tonic-gate 	default:
20777c478bd9Sstevel@tonic-gate 		return (DDI_FAILURE);
20787c478bd9Sstevel@tonic-gate 	}
20797c478bd9Sstevel@tonic-gate 
20807c478bd9Sstevel@tonic-gate 	if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
20817c478bd9Sstevel@tonic-gate 	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
2082ad4023c4Sdp 	    dtrace_register("fasttrap", &fasttrap_attr, DTRACE_PRIV_USER, NULL,
20837c478bd9Sstevel@tonic-gate 	    &fasttrap_pops, NULL, &fasttrap_id) != 0) {
20847c478bd9Sstevel@tonic-gate 		ddi_remove_minor_node(devi, NULL);
20857c478bd9Sstevel@tonic-gate 		return (DDI_FAILURE);
20867c478bd9Sstevel@tonic-gate 	}
20877c478bd9Sstevel@tonic-gate 
20887c478bd9Sstevel@tonic-gate 	ddi_report_dev(devi);
20897c478bd9Sstevel@tonic-gate 	fasttrap_devi = devi;
20907c478bd9Sstevel@tonic-gate 
20917c478bd9Sstevel@tonic-gate 	/*
20927c478bd9Sstevel@tonic-gate 	 * Install our hooks into fork(2), exec(2), and exit(2).
20937c478bd9Sstevel@tonic-gate 	 */
20947c478bd9Sstevel@tonic-gate 	dtrace_fasttrap_fork_ptr = &fasttrap_fork;
20957c478bd9Sstevel@tonic-gate 	dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
20967c478bd9Sstevel@tonic-gate 	dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
20977c478bd9Sstevel@tonic-gate 
20987c478bd9Sstevel@tonic-gate 	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
20997c478bd9Sstevel@tonic-gate 	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
21007c478bd9Sstevel@tonic-gate 	fasttrap_total = 0;
21017c478bd9Sstevel@tonic-gate 
21027c478bd9Sstevel@tonic-gate 	/*
21037c478bd9Sstevel@tonic-gate 	 * Conjure up the tracepoints hashtable...
21047c478bd9Sstevel@tonic-gate 	 */
21057c478bd9Sstevel@tonic-gate 	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
21067c478bd9Sstevel@tonic-gate 	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
21077c478bd9Sstevel@tonic-gate 
21087c478bd9Sstevel@tonic-gate 	if (nent <= 0 || nent > 0x1000000)
21097c478bd9Sstevel@tonic-gate 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
21107c478bd9Sstevel@tonic-gate 
21117c478bd9Sstevel@tonic-gate 	if ((nent & (nent - 1)) == 0)
21127c478bd9Sstevel@tonic-gate 		fasttrap_tpoints.fth_nent = nent;
21137c478bd9Sstevel@tonic-gate 	else
21147c478bd9Sstevel@tonic-gate 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
21157c478bd9Sstevel@tonic-gate 	ASSERT(fasttrap_tpoints.fth_nent > 0);
21167c478bd9Sstevel@tonic-gate 	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
21177c478bd9Sstevel@tonic-gate 	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
21187c478bd9Sstevel@tonic-gate 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
21197c478bd9Sstevel@tonic-gate 
21207c478bd9Sstevel@tonic-gate 	/*
2121b096140dSahl 	 * ... and the providers hash table...
21227c478bd9Sstevel@tonic-gate 	 */
21237c478bd9Sstevel@tonic-gate 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
21247c478bd9Sstevel@tonic-gate 	if ((nent & (nent - 1)) == 0)
21257c478bd9Sstevel@tonic-gate 		fasttrap_provs.fth_nent = nent;
21267c478bd9Sstevel@tonic-gate 	else
21277c478bd9Sstevel@tonic-gate 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
21287c478bd9Sstevel@tonic-gate 	ASSERT(fasttrap_provs.fth_nent > 0);
21297c478bd9Sstevel@tonic-gate 	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
21307c478bd9Sstevel@tonic-gate 	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
21317c478bd9Sstevel@tonic-gate 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
21327c478bd9Sstevel@tonic-gate 
2133b096140dSahl 	/*
2134b096140dSahl 	 * ... and the procs hash table.
2135b096140dSahl 	 */
2136b096140dSahl 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2137b096140dSahl 	if ((nent & (nent - 1)) == 0)
2138b096140dSahl 		fasttrap_procs.fth_nent = nent;
2139b096140dSahl 	else
2140b096140dSahl 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2141b096140dSahl 	ASSERT(fasttrap_procs.fth_nent > 0);
2142b096140dSahl 	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2143b096140dSahl 	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2144b096140dSahl 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2145b096140dSahl 
21467c478bd9Sstevel@tonic-gate 	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
21477c478bd9Sstevel@tonic-gate 	    &fasttrap_meta_id);
21487c478bd9Sstevel@tonic-gate 
21497c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
21507c478bd9Sstevel@tonic-gate }
21517c478bd9Sstevel@tonic-gate 
21527c478bd9Sstevel@tonic-gate static int
21537c478bd9Sstevel@tonic-gate fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
21547c478bd9Sstevel@tonic-gate {
21557c478bd9Sstevel@tonic-gate 	int i, fail = 0;
21567c478bd9Sstevel@tonic-gate 	timeout_id_t tmp;
21577c478bd9Sstevel@tonic-gate 
21587c478bd9Sstevel@tonic-gate 	switch (cmd) {
21597c478bd9Sstevel@tonic-gate 	case DDI_DETACH:
21607c478bd9Sstevel@tonic-gate 		break;
21617c478bd9Sstevel@tonic-gate 	case DDI_SUSPEND:
21627c478bd9Sstevel@tonic-gate 		return (DDI_SUCCESS);
21637c478bd9Sstevel@tonic-gate 	default:
21647c478bd9Sstevel@tonic-gate 		return (DDI_FAILURE);
21657c478bd9Sstevel@tonic-gate 	}
21667c478bd9Sstevel@tonic-gate 
21677c478bd9Sstevel@tonic-gate 	/*
21687c478bd9Sstevel@tonic-gate 	 * Unregister the meta-provider to make sure no new fasttrap-
21697c478bd9Sstevel@tonic-gate 	 * managed providers come along while we're trying to close up
21707c478bd9Sstevel@tonic-gate 	 * shop. If we fail to detach, we'll need to re-register as a
21717c478bd9Sstevel@tonic-gate 	 * meta-provider. We can fail to unregister as a meta-provider
21727c478bd9Sstevel@tonic-gate 	 * if providers we manage still exist.
21737c478bd9Sstevel@tonic-gate 	 */
21747c478bd9Sstevel@tonic-gate 	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
21757c478bd9Sstevel@tonic-gate 	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
21767c478bd9Sstevel@tonic-gate 		return (DDI_FAILURE);
21777c478bd9Sstevel@tonic-gate 
21787c478bd9Sstevel@tonic-gate 	/*
21797c478bd9Sstevel@tonic-gate 	 * Prevent any new timeouts from running by setting fasttrap_timeout
21807c478bd9Sstevel@tonic-gate 	 * to a non-zero value, and wait for the current timeout to complete.
21817c478bd9Sstevel@tonic-gate 	 */
21827c478bd9Sstevel@tonic-gate 	mutex_enter(&fasttrap_cleanup_mtx);
21837c478bd9Sstevel@tonic-gate 	fasttrap_cleanup_work = 0;
21847c478bd9Sstevel@tonic-gate 
21857c478bd9Sstevel@tonic-gate 	while (fasttrap_timeout != (timeout_id_t)1) {
21867c478bd9Sstevel@tonic-gate 		tmp = fasttrap_timeout;
21877c478bd9Sstevel@tonic-gate 		fasttrap_timeout = (timeout_id_t)1;
21887c478bd9Sstevel@tonic-gate 
21897c478bd9Sstevel@tonic-gate 		if (tmp != 0) {
21907c478bd9Sstevel@tonic-gate 			mutex_exit(&fasttrap_cleanup_mtx);
21917c478bd9Sstevel@tonic-gate 			(void) untimeout(tmp);
21927c478bd9Sstevel@tonic-gate 			mutex_enter(&fasttrap_cleanup_mtx);
21937c478bd9Sstevel@tonic-gate 		}
21947c478bd9Sstevel@tonic-gate 	}
21957c478bd9Sstevel@tonic-gate 
21967c478bd9Sstevel@tonic-gate 	fasttrap_cleanup_work = 0;
21977c478bd9Sstevel@tonic-gate 	mutex_exit(&fasttrap_cleanup_mtx);
21987c478bd9Sstevel@tonic-gate 
21997c478bd9Sstevel@tonic-gate 	/*
22007c478bd9Sstevel@tonic-gate 	 * Iterate over all of our providers. If there's still a process
22017c478bd9Sstevel@tonic-gate 	 * that corresponds to that pid, fail to detach.
22027c478bd9Sstevel@tonic-gate 	 */
22037c478bd9Sstevel@tonic-gate 	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
22047c478bd9Sstevel@tonic-gate 		fasttrap_provider_t **fpp, *fp;
22057c478bd9Sstevel@tonic-gate 		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
22067c478bd9Sstevel@tonic-gate 
22077c478bd9Sstevel@tonic-gate 		mutex_enter(&bucket->ftb_mtx);
22087c478bd9Sstevel@tonic-gate 		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
22097c478bd9Sstevel@tonic-gate 		while ((fp = *fpp) != NULL) {
22107c478bd9Sstevel@tonic-gate 			/*
22117c478bd9Sstevel@tonic-gate 			 * Acquire and release the lock as a simple way of
22127c478bd9Sstevel@tonic-gate 			 * waiting for any other consumer to finish with
22137c478bd9Sstevel@tonic-gate 			 * this provider. A thread must first acquire the
22147c478bd9Sstevel@tonic-gate 			 * bucket lock so there's no chance of another thread
2215dafb5540Sahl 			 * blocking on the provider's lock.
22167c478bd9Sstevel@tonic-gate 			 */
22177c478bd9Sstevel@tonic-gate 			mutex_enter(&fp->ftp_mtx);
22187c478bd9Sstevel@tonic-gate 			mutex_exit(&fp->ftp_mtx);
22197c478bd9Sstevel@tonic-gate 
22207c478bd9Sstevel@tonic-gate 			if (dtrace_unregister(fp->ftp_provid) != 0) {
22217c478bd9Sstevel@tonic-gate 				fail = 1;
22227c478bd9Sstevel@tonic-gate 				fpp = &fp->ftp_next;
22237c478bd9Sstevel@tonic-gate 			} else {
22247c478bd9Sstevel@tonic-gate 				*fpp = fp->ftp_next;
22257c478bd9Sstevel@tonic-gate 				fasttrap_provider_free(fp);
22267c478bd9Sstevel@tonic-gate 			}
22277c478bd9Sstevel@tonic-gate 		}
22287c478bd9Sstevel@tonic-gate 
22297c478bd9Sstevel@tonic-gate 		mutex_exit(&bucket->ftb_mtx);
22307c478bd9Sstevel@tonic-gate 	}
22317c478bd9Sstevel@tonic-gate 
22327c478bd9Sstevel@tonic-gate 	if (fail || dtrace_unregister(fasttrap_id) != 0) {
22337c478bd9Sstevel@tonic-gate 		uint_t work;
22347c478bd9Sstevel@tonic-gate 		/*
22357c478bd9Sstevel@tonic-gate 		 * If we're failing to detach, we need to unblock timeouts
22367c478bd9Sstevel@tonic-gate 		 * and start a new timeout if any work has accumulated while
22377c478bd9Sstevel@tonic-gate 		 * we've been unsuccessfully trying to detach.
22387c478bd9Sstevel@tonic-gate 		 */
22397c478bd9Sstevel@tonic-gate 		mutex_enter(&fasttrap_cleanup_mtx);
22407c478bd9Sstevel@tonic-gate 		fasttrap_timeout = 0;
22417c478bd9Sstevel@tonic-gate 		work = fasttrap_cleanup_work;
22427c478bd9Sstevel@tonic-gate 		mutex_exit(&fasttrap_cleanup_mtx);
22437c478bd9Sstevel@tonic-gate 
22447c478bd9Sstevel@tonic-gate 		if (work)
22457c478bd9Sstevel@tonic-gate 			fasttrap_pid_cleanup();
22467c478bd9Sstevel@tonic-gate 
22477c478bd9Sstevel@tonic-gate 		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
22487c478bd9Sstevel@tonic-gate 		    &fasttrap_meta_id);
22497c478bd9Sstevel@tonic-gate 
22507c478bd9Sstevel@tonic-gate 		return (DDI_FAILURE);
22517c478bd9Sstevel@tonic-gate 	}
22527c478bd9Sstevel@tonic-gate 
22537c478bd9Sstevel@tonic-gate #ifdef DEBUG
22547c478bd9Sstevel@tonic-gate 	mutex_enter(&fasttrap_count_mtx);
22557c478bd9Sstevel@tonic-gate 	ASSERT(fasttrap_count == 0);
22567c478bd9Sstevel@tonic-gate 	mutex_exit(&fasttrap_count_mtx);
22577c478bd9Sstevel@tonic-gate #endif
22587c478bd9Sstevel@tonic-gate 
22597c478bd9Sstevel@tonic-gate 	kmem_free(fasttrap_tpoints.fth_table,
22607c478bd9Sstevel@tonic-gate 	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
22617c478bd9Sstevel@tonic-gate 	fasttrap_tpoints.fth_nent = 0;
22627c478bd9Sstevel@tonic-gate 
22637c478bd9Sstevel@tonic-gate 	kmem_free(fasttrap_provs.fth_table,
22647c478bd9Sstevel@tonic-gate 	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
22657c478bd9Sstevel@tonic-gate 	fasttrap_provs.fth_nent = 0;
22667c478bd9Sstevel@tonic-gate 
2267b096140dSahl 	kmem_free(fasttrap_procs.fth_table,
2268b096140dSahl 	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2269b096140dSahl 	fasttrap_procs.fth_nent = 0;
2270b096140dSahl 
22717c478bd9Sstevel@tonic-gate 	/*
22727c478bd9Sstevel@tonic-gate 	 * We know there are no tracepoints in any process anywhere in
22737c478bd9Sstevel@tonic-gate 	 * the system so there is no process which has its p_dtrace_count
22747c478bd9Sstevel@tonic-gate 	 * greater than zero, therefore we know that no thread can actively
22757c478bd9Sstevel@tonic-gate 	 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
22767c478bd9Sstevel@tonic-gate 	 * and fasttrap_exec() and fasttrap_exit().
22777c478bd9Sstevel@tonic-gate 	 */
22787c478bd9Sstevel@tonic-gate 	ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
22797c478bd9Sstevel@tonic-gate 	dtrace_fasttrap_fork_ptr = NULL;
22807c478bd9Sstevel@tonic-gate 
22817c478bd9Sstevel@tonic-gate 	ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
22827c478bd9Sstevel@tonic-gate 	dtrace_fasttrap_exec_ptr = NULL;
22837c478bd9Sstevel@tonic-gate 
22847c478bd9Sstevel@tonic-gate 	ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
22857c478bd9Sstevel@tonic-gate 	dtrace_fasttrap_exit_ptr = NULL;
22867c478bd9Sstevel@tonic-gate 
22877c478bd9Sstevel@tonic-gate 	ddi_remove_minor_node(devi, NULL);
22887c478bd9Sstevel@tonic-gate 
22897c478bd9Sstevel@tonic-gate 	return (DDI_SUCCESS);
22907c478bd9Sstevel@tonic-gate }
22917c478bd9Sstevel@tonic-gate 
22927c478bd9Sstevel@tonic-gate static struct dev_ops fasttrap_ops = {
22937c478bd9Sstevel@tonic-gate 	DEVO_REV,		/* devo_rev */
22947c478bd9Sstevel@tonic-gate 	0,			/* refcnt */
22957c478bd9Sstevel@tonic-gate 	fasttrap_info,		/* get_dev_info */
22967c478bd9Sstevel@tonic-gate 	nulldev,		/* identify */
22977c478bd9Sstevel@tonic-gate 	nulldev,		/* probe */
22987c478bd9Sstevel@tonic-gate 	fasttrap_attach,	/* attach */
22997c478bd9Sstevel@tonic-gate 	fasttrap_detach,	/* detach */
23007c478bd9Sstevel@tonic-gate 	nodev,			/* reset */
23017c478bd9Sstevel@tonic-gate 	&fasttrap_cb_ops,	/* driver operations */
23027c478bd9Sstevel@tonic-gate 	NULL,			/* bus operations */
23037c478bd9Sstevel@tonic-gate 	nodev			/* dev power */
23047c478bd9Sstevel@tonic-gate };
23057c478bd9Sstevel@tonic-gate 
23067c478bd9Sstevel@tonic-gate /*
23077c478bd9Sstevel@tonic-gate  * Module linkage information for the kernel.
23087c478bd9Sstevel@tonic-gate  */
23097c478bd9Sstevel@tonic-gate static struct modldrv modldrv = {
23107c478bd9Sstevel@tonic-gate 	&mod_driverops,		/* module type (this is a pseudo driver) */
23117c478bd9Sstevel@tonic-gate 	"Fasttrap Tracing",	/* name of module */
23127c478bd9Sstevel@tonic-gate 	&fasttrap_ops,		/* driver ops */
23137c478bd9Sstevel@tonic-gate };
23147c478bd9Sstevel@tonic-gate 
23157c478bd9Sstevel@tonic-gate static struct modlinkage modlinkage = {
23167c478bd9Sstevel@tonic-gate 	MODREV_1,
23177c478bd9Sstevel@tonic-gate 	(void *)&modldrv,
23187c478bd9Sstevel@tonic-gate 	NULL
23197c478bd9Sstevel@tonic-gate };
23207c478bd9Sstevel@tonic-gate 
23217c478bd9Sstevel@tonic-gate int
23227c478bd9Sstevel@tonic-gate _init(void)
23237c478bd9Sstevel@tonic-gate {
23247c478bd9Sstevel@tonic-gate 	return (mod_install(&modlinkage));
23257c478bd9Sstevel@tonic-gate }
23267c478bd9Sstevel@tonic-gate 
23277c478bd9Sstevel@tonic-gate int
23287c478bd9Sstevel@tonic-gate _info(struct modinfo *modinfop)
23297c478bd9Sstevel@tonic-gate {
23307c478bd9Sstevel@tonic-gate 	return (mod_info(&modlinkage, modinfop));
23317c478bd9Sstevel@tonic-gate }
23327c478bd9Sstevel@tonic-gate 
23337c478bd9Sstevel@tonic-gate int
23347c478bd9Sstevel@tonic-gate _fini(void)
23357c478bd9Sstevel@tonic-gate {
23367c478bd9Sstevel@tonic-gate 	return (mod_remove(&modlinkage));
23377c478bd9Sstevel@tonic-gate }
2338