xref: /freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c (revision 6966ac055c3b7a39266fb982493330df7a097997)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  * Portions Copyright 2010 The FreeBSD Foundation
22  *
23  * $FreeBSD$
24  */
25 
26 /*
27  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
28  * Use is subject to license terms.
29  */
30 
31 /*
32  * Copyright (c) 2015, Joyent, Inc. All rights reserved.
33  */
34 
35 #include <sys/atomic.h>
36 #include <sys/errno.h>
37 #include <sys/stat.h>
38 #include <sys/modctl.h>
39 #include <sys/conf.h>
40 #include <sys/systm.h>
41 #ifdef illumos
42 #include <sys/ddi.h>
43 #endif
44 #include <sys/sunddi.h>
45 #include <sys/cpuvar.h>
46 #include <sys/kmem.h>
47 #ifdef illumos
48 #include <sys/strsubr.h>
49 #endif
50 #include <sys/fasttrap.h>
51 #include <sys/fasttrap_impl.h>
52 #include <sys/fasttrap_isa.h>
53 #include <sys/dtrace.h>
54 #include <sys/dtrace_impl.h>
55 #include <sys/sysmacros.h>
56 #include <sys/proc.h>
57 #include <sys/policy.h>
58 #ifdef illumos
59 #include <util/qsort.h>
60 #endif
61 #include <sys/mutex.h>
62 #include <sys/kernel.h>
63 #ifndef illumos
64 #include <sys/dtrace_bsd.h>
65 #include <sys/eventhandler.h>
66 #include <sys/rmlock.h>
67 #include <sys/sysent.h>
68 #include <sys/sysctl.h>
69 #include <sys/u8_textprep.h>
70 #include <sys/user.h>
71 
72 #include <vm/vm.h>
73 #include <vm/pmap.h>
74 #include <vm/vm_map.h>
75 #include <vm/vm_param.h>
76 
77 #include <cddl/dev/dtrace/dtrace_cddl.h>
78 #endif
79 
80 /*
81  * User-Land Trap-Based Tracing
82  * ----------------------------
83  *
84  * The fasttrap provider allows DTrace consumers to instrument any user-level
85  * instruction to gather data; this includes probes with semantic
86  * signifigance like entry and return as well as simple offsets into the
87  * function. While the specific techniques used are very ISA specific, the
88  * methodology is generalizable to any architecture.
89  *
90  *
91  * The General Methodology
92  * -----------------------
93  *
94  * With the primary goal of tracing every user-land instruction and the
95  * limitation that we can't trust user space so don't want to rely on much
96  * information there, we begin by replacing the instructions we want to trace
97  * with trap instructions. Each instruction we overwrite is saved into a hash
98  * table keyed by process ID and pc address. When we enter the kernel due to
99  * this trap instruction, we need the effects of the replaced instruction to
100  * appear to have occurred before we proceed with the user thread's
101  * execution.
102  *
103  * Each user level thread is represented by a ulwp_t structure which is
104  * always easily accessible through a register. The most basic way to produce
105  * the effects of the instruction we replaced is to copy that instruction out
106  * to a bit of scratch space reserved in the user thread's ulwp_t structure
107  * (a sort of kernel-private thread local storage), set the PC to that
108  * scratch space and single step. When we reenter the kernel after single
109  * stepping the instruction we must then adjust the PC to point to what would
110  * normally be the next instruction. Of course, special care must be taken
111  * for branches and jumps, but these represent such a small fraction of any
112  * instruction set that writing the code to emulate these in the kernel is
113  * not too difficult.
114  *
115  * Return probes may require several tracepoints to trace every return site,
116  * and, conversely, each tracepoint may activate several probes (the entry
117  * and offset 0 probes, for example). To solve this muliplexing problem,
118  * tracepoints contain lists of probes to activate and probes contain lists
119  * of tracepoints to enable. If a probe is activated, it adds its ID to
120  * existing tracepoints or creates new ones as necessary.
121  *
122  * Most probes are activated _before_ the instruction is executed, but return
123  * probes are activated _after_ the effects of the last instruction of the
124  * function are visible. Return probes must be fired _after_ we have
125  * single-stepped the instruction whereas all other probes are fired
126  * beforehand.
127  *
128  *
129  * Lock Ordering
130  * -------------
131  *
132  * The lock ordering below -- both internally and with respect to the DTrace
133  * framework -- is a little tricky and bears some explanation. Each provider
134  * has a lock (ftp_mtx) that protects its members including reference counts
135  * for enabled probes (ftp_rcount), consumers actively creating probes
136  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
137  * from being freed. A provider is looked up by taking the bucket lock for the
138  * provider hash table, and is returned with its lock held. The provider lock
139  * may be taken in functions invoked by the DTrace framework, but may not be
140  * held while calling functions in the DTrace framework.
141  *
142  * To ensure consistency over multiple calls to the DTrace framework, the
143  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
144  * not be taken when holding the provider lock as that would create a cyclic
145  * lock ordering. In situations where one would naturally take the provider
146  * lock and then the creation lock, we instead up a reference count to prevent
147  * the provider from disappearing, drop the provider lock, and acquire the
148  * creation lock.
149  *
150  * Briefly:
151  * 	bucket lock before provider lock
152  *	DTrace before provider lock
153  *	creation lock before DTrace
154  *	never hold the provider lock and creation lock simultaneously
155  */
156 
157 static d_open_t fasttrap_open;
158 static d_ioctl_t fasttrap_ioctl;
159 
160 static struct cdevsw fasttrap_cdevsw = {
161 	.d_version	= D_VERSION,
162 	.d_open		= fasttrap_open,
163 	.d_ioctl	= fasttrap_ioctl,
164 	.d_name		= "fasttrap",
165 };
166 static struct cdev *fasttrap_cdev;
167 static dtrace_meta_provider_id_t fasttrap_meta_id;
168 
169 static struct proc *fasttrap_cleanup_proc;
170 static struct mtx fasttrap_cleanup_mtx;
171 static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
172 
173 /*
174  * Generation count on modifications to the global tracepoint lookup table.
175  */
176 static volatile uint64_t fasttrap_mod_gen;
177 
178 /*
179  * When the fasttrap provider is loaded, fasttrap_max is set to either
180  * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the
181  * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD).
182  * Each time a probe is created, fasttrap_total is incremented by the number
183  * of tracepoints that may be associated with that probe; fasttrap_total is capped
184  * at fasttrap_max.
185  */
186 #define	FASTTRAP_MAX_DEFAULT		250000
187 static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT;
188 static uint32_t fasttrap_total;
189 
190 /*
191  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
192  */
193 
194 #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
195 #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
196 #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
197 
198 #define	FASTTRAP_PID_NAME		"pid"
199 
200 fasttrap_hash_t			fasttrap_tpoints;
201 static fasttrap_hash_t		fasttrap_provs;
202 static fasttrap_hash_t		fasttrap_procs;
203 
204 static uint64_t			fasttrap_pid_count;	/* pid ref count */
205 static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
206 
207 #define	FASTTRAP_ENABLE_FAIL	1
208 #define	FASTTRAP_ENABLE_PARTIAL	2
209 
210 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
211 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
212 
213 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
214     const dtrace_pattr_t *);
215 static void fasttrap_provider_retire(pid_t, const char *, int);
216 static void fasttrap_provider_free(fasttrap_provider_t *);
217 
218 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
219 static void fasttrap_proc_release(fasttrap_proc_t *);
220 
221 #ifndef illumos
222 static void fasttrap_thread_dtor(void *, struct thread *);
223 #endif
224 
225 #define	FASTTRAP_PROVS_INDEX(pid, name) \
226 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
227 
228 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
229 
230 #ifndef illumos
231 struct rmlock fasttrap_tp_lock;
232 static eventhandler_tag fasttrap_thread_dtor_tag;
233 #endif
234 
235 static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE;
236 
237 #ifdef __FreeBSD__
238 SYSCTL_DECL(_kern_dtrace);
239 SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters");
240 SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max,
241     FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes");
242 SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size,
243     FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table");
244 #endif
245 
246 static int
247 fasttrap_highbit(ulong_t i)
248 {
249 	int h = 1;
250 
251 	if (i == 0)
252 		return (0);
253 #ifdef _LP64
254 	if (i & 0xffffffff00000000ul) {
255 		h += 32; i >>= 32;
256 	}
257 #endif
258 	if (i & 0xffff0000) {
259 		h += 16; i >>= 16;
260 	}
261 	if (i & 0xff00) {
262 		h += 8; i >>= 8;
263 	}
264 	if (i & 0xf0) {
265 		h += 4; i >>= 4;
266 	}
267 	if (i & 0xc) {
268 		h += 2; i >>= 2;
269 	}
270 	if (i & 0x2) {
271 		h += 1;
272 	}
273 	return (h);
274 }
275 
276 static uint_t
277 fasttrap_hash_str(const char *p)
278 {
279 	unsigned int g;
280 	uint_t hval = 0;
281 
282 	while (*p) {
283 		hval = (hval << 4) + *p++;
284 		if ((g = (hval & 0xf0000000)) != 0)
285 			hval ^= g >> 24;
286 		hval &= ~g;
287 	}
288 	return (hval);
289 }
290 
291 void
292 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
293 {
294 	ksiginfo_t ksi;
295 
296 	ksiginfo_init(&ksi);
297 	ksi.ksi_signo = SIGTRAP;
298 	ksi.ksi_code = TRAP_DTRACE;
299 	ksi.ksi_addr = (caddr_t)pc;
300 	PROC_LOCK(p);
301 	(void)tdsendsignal(p, t, SIGTRAP, &ksi);
302 	PROC_UNLOCK(p);
303 }
304 
305 #ifndef illumos
306 /*
307  * Obtain a chunk of scratch space in the address space of the target process.
308  */
309 fasttrap_scrspace_t *
310 fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
311 {
312 	fasttrap_scrblock_t *scrblk;
313 	fasttrap_scrspace_t *scrspc;
314 	struct proc *p;
315 	vm_offset_t addr;
316 	int error, i;
317 
318 	scrspc = NULL;
319 	if (td->t_dtrace_sscr != NULL) {
320 		/* If the thread already has scratch space, we're done. */
321 		scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
322 		return (scrspc);
323 	}
324 
325 	p = td->td_proc;
326 
327 	mutex_enter(&fprc->ftpc_mtx);
328 	if (LIST_EMPTY(&fprc->ftpc_fscr)) {
329 		/*
330 		 * No scratch space is available, so we'll map a new scratch
331 		 * space block into the traced process' address space.
332 		 */
333 		addr = 0;
334 		error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr,
335 		    FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL,
336 		    VM_PROT_ALL, 0);
337 		if (error != KERN_SUCCESS)
338 			goto done;
339 
340 		scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK);
341 		scrblk->ftsb_addr = addr;
342 		LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next);
343 
344 		/*
345 		 * Carve the block up into chunks and put them on the free list.
346 		 */
347 		for (i = 0;
348 		    i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) {
349 			scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK);
350 			scrspc->ftss_addr = addr +
351 			    i * FASTTRAP_SCRSPACE_SIZE;
352 			LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,
353 			    ftss_next);
354 		}
355 	}
356 
357 	/*
358 	 * Take the first scratch chunk off the free list, put it on the
359 	 * allocated list, and return its address.
360 	 */
361 	scrspc = LIST_FIRST(&fprc->ftpc_fscr);
362 	LIST_REMOVE(scrspc, ftss_next);
363 	LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next);
364 
365 	/*
366 	 * This scratch space is reserved for use by td until the thread exits.
367 	 */
368 	td->t_dtrace_sscr = scrspc;
369 
370 done:
371 	mutex_exit(&fprc->ftpc_mtx);
372 
373 	return (scrspc);
374 }
375 
376 /*
377  * Return any allocated per-thread scratch space chunks back to the process'
378  * free list.
379  */
380 static void
381 fasttrap_thread_dtor(void *arg __unused, struct thread *td)
382 {
383 	fasttrap_bucket_t *bucket;
384 	fasttrap_proc_t *fprc;
385 	fasttrap_scrspace_t *scrspc;
386 	pid_t pid;
387 
388 	if (td->t_dtrace_sscr == NULL)
389 		return;
390 
391 	pid = td->td_proc->p_pid;
392 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
393 	fprc = NULL;
394 
395 	/* Look up the fasttrap process handle for this process. */
396 	mutex_enter(&bucket->ftb_mtx);
397 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
398 		if (fprc->ftpc_pid == pid) {
399 			mutex_enter(&fprc->ftpc_mtx);
400 			mutex_exit(&bucket->ftb_mtx);
401 			break;
402 		}
403 	}
404 	if (fprc == NULL) {
405 		mutex_exit(&bucket->ftb_mtx);
406 		return;
407 	}
408 
409 	scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
410 	LIST_REMOVE(scrspc, ftss_next);
411 	LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next);
412 
413 	mutex_exit(&fprc->ftpc_mtx);
414 }
415 #endif
416 
417 /*
418  * This function ensures that no threads are actively using the memory
419  * associated with probes that were formerly live.
420  */
421 static void
422 fasttrap_mod_barrier(uint64_t gen)
423 {
424 	int i;
425 
426 	if (gen < fasttrap_mod_gen)
427 		return;
428 
429 	fasttrap_mod_gen++;
430 
431 #ifdef illumos
432 	CPU_FOREACH(i) {
433 		mutex_enter(&fasttrap_cpuc_pid_lock[i]);
434 		mutex_exit(&fasttrap_cpuc_pid_lock[i]);
435 	}
436 #else
437 	rm_wlock(&fasttrap_tp_lock);
438 	rm_wunlock(&fasttrap_tp_lock);
439 #endif
440 }
441 
442 /*
443  * This function performs asynchronous cleanup of fasttrap providers. The
444  * Solaris implementation of this mechanism use a timeout that's activated in
445  * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
446  * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
447  * Thus we use a dedicated process to perform the cleanup when requested.
448  */
449 /*ARGSUSED*/
450 static void
451 fasttrap_pid_cleanup_cb(void *data)
452 {
453 	fasttrap_provider_t **fpp, *fp;
454 	fasttrap_bucket_t *bucket;
455 	dtrace_provider_id_t provid;
456 	int i, later = 0, rval;
457 
458 	mtx_lock(&fasttrap_cleanup_mtx);
459 	while (!fasttrap_cleanup_drain || later > 0) {
460 		fasttrap_cleanup_work = 0;
461 		mtx_unlock(&fasttrap_cleanup_mtx);
462 
463 		later = 0;
464 
465 		/*
466 		 * Iterate over all the providers trying to remove the marked
467 		 * ones. If a provider is marked but not retired, we just
468 		 * have to take a crack at removing it -- it's no big deal if
469 		 * we can't.
470 		 */
471 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
472 			bucket = &fasttrap_provs.fth_table[i];
473 			mutex_enter(&bucket->ftb_mtx);
474 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
475 
476 			while ((fp = *fpp) != NULL) {
477 				if (!fp->ftp_marked) {
478 					fpp = &fp->ftp_next;
479 					continue;
480 				}
481 
482 				mutex_enter(&fp->ftp_mtx);
483 
484 				/*
485 				 * If this provider has consumers actively
486 				 * creating probes (ftp_ccount) or is a USDT
487 				 * provider (ftp_mcount), we can't unregister
488 				 * or even condense.
489 				 */
490 				if (fp->ftp_ccount != 0 ||
491 				    fp->ftp_mcount != 0) {
492 					mutex_exit(&fp->ftp_mtx);
493 					fp->ftp_marked = 0;
494 					continue;
495 				}
496 
497 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
498 					fp->ftp_marked = 0;
499 
500 				mutex_exit(&fp->ftp_mtx);
501 
502 				/*
503 				 * If we successfully unregister this
504 				 * provider we can remove it from the hash
505 				 * chain and free the memory. If our attempt
506 				 * to unregister fails and this is a retired
507 				 * provider, increment our flag to try again
508 				 * pretty soon. If we've consumed more than
509 				 * half of our total permitted number of
510 				 * probes call dtrace_condense() to try to
511 				 * clean out the unenabled probes.
512 				 */
513 				provid = fp->ftp_provid;
514 				if ((rval = dtrace_unregister(provid)) != 0) {
515 					if (fasttrap_total > fasttrap_max / 2)
516 						(void) dtrace_condense(provid);
517 
518 					if (rval == EAGAIN)
519 						fp->ftp_marked = 1;
520 
521 					later += fp->ftp_marked;
522 					fpp = &fp->ftp_next;
523 				} else {
524 					*fpp = fp->ftp_next;
525 					fasttrap_provider_free(fp);
526 				}
527 			}
528 			mutex_exit(&bucket->ftb_mtx);
529 		}
530 		mtx_lock(&fasttrap_cleanup_mtx);
531 
532 		/*
533 		 * If we were unable to retire a provider, try again after a
534 		 * second. This situation can occur in certain circumstances
535 		 * where providers cannot be unregistered even though they have
536 		 * no probes enabled because of an execution of dtrace -l or
537 		 * something similar.
538 		 */
539 		if (later > 0 || fasttrap_cleanup_work ||
540 		    fasttrap_cleanup_drain) {
541 			mtx_unlock(&fasttrap_cleanup_mtx);
542 			pause("ftclean", hz);
543 			mtx_lock(&fasttrap_cleanup_mtx);
544 		} else
545 			mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
546 			    0, "ftcl", 0);
547 	}
548 
549 	/*
550 	 * Wake up the thread in fasttrap_unload() now that we're done.
551 	 */
552 	wakeup(&fasttrap_cleanup_drain);
553 	mtx_unlock(&fasttrap_cleanup_mtx);
554 
555 	kthread_exit();
556 }
557 
558 /*
559  * Activates the asynchronous cleanup mechanism.
560  */
561 static void
562 fasttrap_pid_cleanup(void)
563 {
564 
565 	mtx_lock(&fasttrap_cleanup_mtx);
566 	if (!fasttrap_cleanup_work) {
567 		fasttrap_cleanup_work = 1;
568 		wakeup(&fasttrap_cleanup_cv);
569 	}
570 	mtx_unlock(&fasttrap_cleanup_mtx);
571 }
572 
573 /*
574  * This is called from cfork() via dtrace_fasttrap_fork(). The child
575  * process's address space is (roughly) a copy of the parent process's so
576  * we have to remove all the instrumentation we had previously enabled in the
577  * parent.
578  */
579 static void
580 fasttrap_fork(proc_t *p, proc_t *cp)
581 {
582 #ifndef illumos
583 	fasttrap_scrblock_t *scrblk;
584 	fasttrap_proc_t *fprc = NULL;
585 #endif
586 	pid_t ppid = p->p_pid;
587 	int i;
588 
589 	ASSERT(curproc == p);
590 #ifdef illumos
591 	ASSERT(p->p_proc_flag & P_PR_LOCK);
592 #else
593 	PROC_LOCK_ASSERT(p, MA_OWNED);
594 #endif
595 #ifdef illumos
596 	ASSERT(p->p_dtrace_count > 0);
597 #else
598 	/*
599 	 * This check is purposely here instead of in kern_fork.c because,
600 	 * for legal resons, we cannot include the dtrace_cddl.h header
601 	 * inside kern_fork.c and insert if-clause there.
602 	 */
603 	if (p->p_dtrace_count == 0 && p->p_dtrace_helpers == NULL)
604 		return;
605 #endif
606 
607 	ASSERT(cp->p_dtrace_count == 0);
608 
609 	/*
610 	 * This would be simpler and faster if we maintained per-process
611 	 * hash tables of enabled tracepoints. It could, however, potentially
612 	 * slow down execution of a tracepoint since we'd need to go
613 	 * through two levels of indirection. In the future, we should
614 	 * consider either maintaining per-process ancillary lists of
615 	 * enabled tracepoints or hanging a pointer to a per-process hash
616 	 * table of enabled tracepoints off the proc structure.
617 	 */
618 
619 	/*
620 	 * We don't have to worry about the child process disappearing
621 	 * because we're in fork().
622 	 */
623 #ifdef illumos
624 	mtx_lock_spin(&cp->p_slock);
625 	sprlock_proc(cp);
626 	mtx_unlock_spin(&cp->p_slock);
627 #else
628 	/*
629 	 * fasttrap_tracepoint_remove() expects the child process to be
630 	 * unlocked and the VM then expects curproc to be unlocked.
631 	 */
632 	_PHOLD(cp);
633 	PROC_UNLOCK(cp);
634 	PROC_UNLOCK(p);
635 	if (p->p_dtrace_count == 0)
636 		goto dup_helpers;
637 #endif
638 
639 	/*
640 	 * Iterate over every tracepoint looking for ones that belong to the
641 	 * parent process, and remove each from the child process.
642 	 */
643 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
644 		fasttrap_tracepoint_t *tp;
645 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
646 
647 		mutex_enter(&bucket->ftb_mtx);
648 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
649 			if (tp->ftt_pid == ppid &&
650 			    tp->ftt_proc->ftpc_acount != 0) {
651 				int ret = fasttrap_tracepoint_remove(cp, tp);
652 				ASSERT(ret == 0);
653 
654 				/*
655 				 * The count of active providers can only be
656 				 * decremented (i.e. to zero) during exec,
657 				 * exit, and removal of a meta provider so it
658 				 * should be impossible to drop the count
659 				 * mid-fork.
660 				 */
661 				ASSERT(tp->ftt_proc->ftpc_acount != 0);
662 #ifndef illumos
663 				fprc = tp->ftt_proc;
664 #endif
665 			}
666 		}
667 		mutex_exit(&bucket->ftb_mtx);
668 
669 #ifndef illumos
670 		/*
671 		 * Unmap any scratch space inherited from the parent's address
672 		 * space.
673 		 */
674 		if (fprc != NULL) {
675 			mutex_enter(&fprc->ftpc_mtx);
676 			LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) {
677 				vm_map_remove(&cp->p_vmspace->vm_map,
678 				    scrblk->ftsb_addr,
679 				    scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE);
680 			}
681 			mutex_exit(&fprc->ftpc_mtx);
682 		}
683 #endif
684 	}
685 
686 #ifdef illumos
687 	mutex_enter(&cp->p_lock);
688 	sprunlock(cp);
689 #else
690 dup_helpers:
691 	if (p->p_dtrace_helpers != NULL)
692 		dtrace_helpers_duplicate(p, cp);
693 	PROC_LOCK(p);
694 	PROC_LOCK(cp);
695 	_PRELE(cp);
696 #endif
697 }
698 
699 /*
700  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
701  * is set on the proc structure to indicate that there is a pid provider
702  * associated with this process.
703  */
704 static void
705 fasttrap_exec_exit(proc_t *p)
706 {
707 #ifndef illumos
708 	struct thread *td;
709 #endif
710 
711 #ifdef illumos
712 	ASSERT(p == curproc);
713 #else
714 	PROC_LOCK_ASSERT(p, MA_OWNED);
715 	_PHOLD(p);
716 	/*
717 	 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
718 	 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
719 	 * ourselves when a process exits.
720 	 */
721 	FOREACH_THREAD_IN_PROC(p, td)
722 		td->t_dtrace_sscr = NULL;
723 	PROC_UNLOCK(p);
724 #endif
725 
726 	/*
727 	 * We clean up the pid provider for this process here; user-land
728 	 * static probes are handled by the meta-provider remove entry point.
729 	 */
730 	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
731 #ifndef illumos
732 	if (p->p_dtrace_helpers)
733 		dtrace_helpers_destroy(p);
734 	PROC_LOCK(p);
735 	_PRELE(p);
736 #endif
737 }
738 
739 
740 /*ARGSUSED*/
741 static void
742 fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
743 {
744 	/*
745 	 * There are no "default" pid probes.
746 	 */
747 }
748 
749 static int
750 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
751 {
752 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
753 	fasttrap_bucket_t *bucket;
754 	fasttrap_id_t *id;
755 	pid_t pid;
756 	uintptr_t pc;
757 
758 	ASSERT(index < probe->ftp_ntps);
759 
760 	pid = probe->ftp_pid;
761 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
762 	id = &probe->ftp_tps[index].fit_id;
763 
764 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
765 
766 #ifdef illumos
767 	ASSERT(!(p->p_flag & SVFORK));
768 #endif
769 
770 	/*
771 	 * Before we make any modifications, make sure we've imposed a barrier
772 	 * on the generation in which this probe was last modified.
773 	 */
774 	fasttrap_mod_barrier(probe->ftp_gen);
775 
776 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
777 
778 	/*
779 	 * If the tracepoint has already been enabled, just add our id to the
780 	 * list of interested probes. This may be our second time through
781 	 * this path in which case we'll have constructed the tracepoint we'd
782 	 * like to install. If we can't find a match, and have an allocated
783 	 * tracepoint ready to go, enable that one now.
784 	 *
785 	 * A tracepoint whose process is defunct is also considered defunct.
786 	 */
787 again:
788 	mutex_enter(&bucket->ftb_mtx);
789 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
790 		/*
791 		 * Note that it's safe to access the active count on the
792 		 * associated proc structure because we know that at least one
793 		 * provider (this one) will still be around throughout this
794 		 * operation.
795 		 */
796 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
797 		    tp->ftt_proc->ftpc_acount == 0)
798 			continue;
799 
800 		/*
801 		 * Now that we've found a matching tracepoint, it would be
802 		 * a decent idea to confirm that the tracepoint is still
803 		 * enabled and the trap instruction hasn't been overwritten.
804 		 * Since this is a little hairy, we'll punt for now.
805 		 */
806 
807 		/*
808 		 * This can't be the first interested probe. We don't have
809 		 * to worry about another thread being in the midst of
810 		 * deleting this tracepoint (which would be the only valid
811 		 * reason for a tracepoint to have no interested probes)
812 		 * since we're holding P_PR_LOCK for this process.
813 		 */
814 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
815 
816 		switch (id->fti_ptype) {
817 		case DTFTP_ENTRY:
818 		case DTFTP_OFFSETS:
819 		case DTFTP_IS_ENABLED:
820 			id->fti_next = tp->ftt_ids;
821 			membar_producer();
822 			tp->ftt_ids = id;
823 			membar_producer();
824 			break;
825 
826 		case DTFTP_RETURN:
827 		case DTFTP_POST_OFFSETS:
828 			id->fti_next = tp->ftt_retids;
829 			membar_producer();
830 			tp->ftt_retids = id;
831 			membar_producer();
832 			break;
833 
834 		default:
835 			ASSERT(0);
836 		}
837 
838 		mutex_exit(&bucket->ftb_mtx);
839 
840 		if (new_tp != NULL) {
841 			new_tp->ftt_ids = NULL;
842 			new_tp->ftt_retids = NULL;
843 		}
844 
845 		return (0);
846 	}
847 
848 	/*
849 	 * If we have a good tracepoint ready to go, install it now while
850 	 * we have the lock held and no one can screw with us.
851 	 */
852 	if (new_tp != NULL) {
853 		int rc = 0;
854 
855 		new_tp->ftt_next = bucket->ftb_data;
856 		membar_producer();
857 		bucket->ftb_data = new_tp;
858 		membar_producer();
859 		mutex_exit(&bucket->ftb_mtx);
860 
861 		/*
862 		 * Activate the tracepoint in the ISA-specific manner.
863 		 * If this fails, we need to report the failure, but
864 		 * indicate that this tracepoint must still be disabled
865 		 * by calling fasttrap_tracepoint_disable().
866 		 */
867 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
868 			rc = FASTTRAP_ENABLE_PARTIAL;
869 
870 		/*
871 		 * Increment the count of the number of tracepoints active in
872 		 * the victim process.
873 		 */
874 #ifdef illumos
875 		ASSERT(p->p_proc_flag & P_PR_LOCK);
876 #endif
877 		p->p_dtrace_count++;
878 
879 		return (rc);
880 	}
881 
882 	mutex_exit(&bucket->ftb_mtx);
883 
884 	/*
885 	 * Initialize the tracepoint that's been preallocated with the probe.
886 	 */
887 	new_tp = probe->ftp_tps[index].fit_tp;
888 
889 	ASSERT(new_tp->ftt_pid == pid);
890 	ASSERT(new_tp->ftt_pc == pc);
891 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
892 	ASSERT(new_tp->ftt_ids == NULL);
893 	ASSERT(new_tp->ftt_retids == NULL);
894 
895 	switch (id->fti_ptype) {
896 	case DTFTP_ENTRY:
897 	case DTFTP_OFFSETS:
898 	case DTFTP_IS_ENABLED:
899 		id->fti_next = NULL;
900 		new_tp->ftt_ids = id;
901 		break;
902 
903 	case DTFTP_RETURN:
904 	case DTFTP_POST_OFFSETS:
905 		id->fti_next = NULL;
906 		new_tp->ftt_retids = id;
907 		break;
908 
909 	default:
910 		ASSERT(0);
911 	}
912 
913 #ifdef __FreeBSD__
914 	if (SV_PROC_FLAG(p, SV_LP64))
915 		p->p_model = DATAMODEL_LP64;
916 	else
917 		p->p_model = DATAMODEL_ILP32;
918 #endif
919 
920 	/*
921 	 * If the ISA-dependent initialization goes to plan, go back to the
922 	 * beginning and try to install this freshly made tracepoint.
923 	 */
924 	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
925 		goto again;
926 
927 	new_tp->ftt_ids = NULL;
928 	new_tp->ftt_retids = NULL;
929 
930 	return (FASTTRAP_ENABLE_FAIL);
931 }
932 
933 static void
934 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
935 {
936 	fasttrap_bucket_t *bucket;
937 	fasttrap_provider_t *provider = probe->ftp_prov;
938 	fasttrap_tracepoint_t **pp, *tp;
939 	fasttrap_id_t *id, **idp = NULL;
940 	pid_t pid;
941 	uintptr_t pc;
942 
943 	ASSERT(index < probe->ftp_ntps);
944 
945 	pid = probe->ftp_pid;
946 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
947 	id = &probe->ftp_tps[index].fit_id;
948 
949 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
950 
951 	/*
952 	 * Find the tracepoint and make sure that our id is one of the
953 	 * ones registered with it.
954 	 */
955 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
956 	mutex_enter(&bucket->ftb_mtx);
957 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
958 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
959 		    tp->ftt_proc == provider->ftp_proc)
960 			break;
961 	}
962 
963 	/*
964 	 * If we somehow lost this tracepoint, we're in a world of hurt.
965 	 */
966 	ASSERT(tp != NULL);
967 
968 	switch (id->fti_ptype) {
969 	case DTFTP_ENTRY:
970 	case DTFTP_OFFSETS:
971 	case DTFTP_IS_ENABLED:
972 		ASSERT(tp->ftt_ids != NULL);
973 		idp = &tp->ftt_ids;
974 		break;
975 
976 	case DTFTP_RETURN:
977 	case DTFTP_POST_OFFSETS:
978 		ASSERT(tp->ftt_retids != NULL);
979 		idp = &tp->ftt_retids;
980 		break;
981 
982 	default:
983 		ASSERT(0);
984 	}
985 
986 	while ((*idp)->fti_probe != probe) {
987 		idp = &(*idp)->fti_next;
988 		ASSERT(*idp != NULL);
989 	}
990 
991 	id = *idp;
992 	*idp = id->fti_next;
993 	membar_producer();
994 
995 	ASSERT(id->fti_probe == probe);
996 
997 	/*
998 	 * If there are other registered enablings of this tracepoint, we're
999 	 * all done, but if this was the last probe assocated with this
1000 	 * this tracepoint, we need to remove and free it.
1001 	 */
1002 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
1003 
1004 		/*
1005 		 * If the current probe's tracepoint is in use, swap it
1006 		 * for an unused tracepoint.
1007 		 */
1008 		if (tp == probe->ftp_tps[index].fit_tp) {
1009 			fasttrap_probe_t *tmp_probe;
1010 			fasttrap_tracepoint_t **tmp_tp;
1011 			uint_t tmp_index;
1012 
1013 			if (tp->ftt_ids != NULL) {
1014 				tmp_probe = tp->ftt_ids->fti_probe;
1015 				/* LINTED - alignment */
1016 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
1017 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1018 			} else {
1019 				tmp_probe = tp->ftt_retids->fti_probe;
1020 				/* LINTED - alignment */
1021 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
1022 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1023 			}
1024 
1025 			ASSERT(*tmp_tp != NULL);
1026 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
1027 			ASSERT((*tmp_tp)->ftt_ids == NULL);
1028 			ASSERT((*tmp_tp)->ftt_retids == NULL);
1029 
1030 			probe->ftp_tps[index].fit_tp = *tmp_tp;
1031 			*tmp_tp = tp;
1032 		}
1033 
1034 		mutex_exit(&bucket->ftb_mtx);
1035 
1036 		/*
1037 		 * Tag the modified probe with the generation in which it was
1038 		 * changed.
1039 		 */
1040 		probe->ftp_gen = fasttrap_mod_gen;
1041 		return;
1042 	}
1043 
1044 	mutex_exit(&bucket->ftb_mtx);
1045 
1046 	/*
1047 	 * We can't safely remove the tracepoint from the set of active
1048 	 * tracepoints until we've actually removed the fasttrap instruction
1049 	 * from the process's text. We can, however, operate on this
1050 	 * tracepoint secure in the knowledge that no other thread is going to
1051 	 * be looking at it since we hold P_PR_LOCK on the process if it's
1052 	 * live or we hold the provider lock on the process if it's dead and
1053 	 * gone.
1054 	 */
1055 
1056 	/*
1057 	 * We only need to remove the actual instruction if we're looking
1058 	 * at an existing process
1059 	 */
1060 	if (p != NULL) {
1061 		/*
1062 		 * If we fail to restore the instruction we need to kill
1063 		 * this process since it's in a completely unrecoverable
1064 		 * state.
1065 		 */
1066 		if (fasttrap_tracepoint_remove(p, tp) != 0)
1067 			fasttrap_sigtrap(p, NULL, pc);
1068 
1069 		/*
1070 		 * Decrement the count of the number of tracepoints active
1071 		 * in the victim process.
1072 		 */
1073 #ifdef illumos
1074 		ASSERT(p->p_proc_flag & P_PR_LOCK);
1075 #endif
1076 		p->p_dtrace_count--;
1077 
1078 		atomic_add_rel_64(&p->p_fasttrap_tp_gen, 1);
1079 	}
1080 
1081 	/*
1082 	 * Remove the probe from the hash table of active tracepoints.
1083 	 */
1084 	mutex_enter(&bucket->ftb_mtx);
1085 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
1086 	ASSERT(*pp != NULL);
1087 	while (*pp != tp) {
1088 		pp = &(*pp)->ftt_next;
1089 		ASSERT(*pp != NULL);
1090 	}
1091 
1092 	*pp = tp->ftt_next;
1093 	membar_producer();
1094 
1095 	mutex_exit(&bucket->ftb_mtx);
1096 
1097 	/*
1098 	 * Tag the modified probe with the generation in which it was changed.
1099 	 */
1100 	probe->ftp_gen = fasttrap_mod_gen;
1101 }
1102 
1103 static void
1104 fasttrap_enable_callbacks(void)
1105 {
1106 	/*
1107 	 * We don't have to play the rw lock game here because we're
1108 	 * providing something rather than taking something away --
1109 	 * we can be sure that no threads have tried to follow this
1110 	 * function pointer yet.
1111 	 */
1112 	mutex_enter(&fasttrap_count_mtx);
1113 	if (fasttrap_pid_count == 0) {
1114 		ASSERT(dtrace_pid_probe_ptr == NULL);
1115 		ASSERT(dtrace_return_probe_ptr == NULL);
1116 		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
1117 		dtrace_return_probe_ptr = &fasttrap_return_probe;
1118 	}
1119 	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
1120 	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
1121 	fasttrap_pid_count++;
1122 	mutex_exit(&fasttrap_count_mtx);
1123 }
1124 
1125 static void
1126 fasttrap_disable_callbacks(void)
1127 {
1128 	mutex_enter(&fasttrap_count_mtx);
1129 	ASSERT(fasttrap_pid_count > 0);
1130 	fasttrap_pid_count--;
1131 	if (fasttrap_pid_count == 0) {
1132 		/*
1133 		 * Synchronize with the breakpoint handler, which is careful to
1134 		 * enable interrupts only after loading the hook pointer.
1135 		 */
1136 		dtrace_sync();
1137 		dtrace_pid_probe_ptr = NULL;
1138 		dtrace_return_probe_ptr = NULL;
1139 	}
1140 	mutex_exit(&fasttrap_count_mtx);
1141 }
1142 
1143 /*ARGSUSED*/
1144 static void
1145 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
1146 {
1147 	fasttrap_probe_t *probe = parg;
1148 	proc_t *p = NULL;
1149 	int i, rc;
1150 
1151 	ASSERT(probe != NULL);
1152 	ASSERT(!probe->ftp_enabled);
1153 	ASSERT(id == probe->ftp_id);
1154 #ifdef illumos
1155 	ASSERT(MUTEX_HELD(&cpu_lock));
1156 #endif
1157 
1158 	/*
1159 	 * Increment the count of enabled probes on this probe's provider;
1160 	 * the provider can't go away while the probe still exists. We
1161 	 * must increment this even if we aren't able to properly enable
1162 	 * this probe.
1163 	 */
1164 	mutex_enter(&probe->ftp_prov->ftp_mtx);
1165 	probe->ftp_prov->ftp_rcount++;
1166 	mutex_exit(&probe->ftp_prov->ftp_mtx);
1167 
1168 	/*
1169 	 * If this probe's provider is retired (meaning it was valid in a
1170 	 * previously exec'ed incarnation of this address space), bail out. The
1171 	 * provider can't go away while we're in this code path.
1172 	 */
1173 	if (probe->ftp_prov->ftp_retired)
1174 		return;
1175 
1176 	/*
1177 	 * If we can't find the process, it may be that we're in the context of
1178 	 * a fork in which the traced process is being born and we're copying
1179 	 * USDT probes. Otherwise, the process is gone so bail.
1180 	 */
1181 #ifdef illumos
1182 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
1183 		if ((curproc->p_flag & SFORKING) == 0)
1184 			return;
1185 
1186 		mutex_enter(&pidlock);
1187 		p = prfind(probe->ftp_pid);
1188 
1189 		if (p == NULL) {
1190 			/*
1191 			 * So it's not that the target process is being born,
1192 			 * it's that it isn't there at all (and we simply
1193 			 * happen to be forking).  Anyway, we know that the
1194 			 * target is definitely gone, so bail out.
1195 			 */
1196 			mutex_exit(&pidlock);
1197 			return (0);
1198 		}
1199 
1200 		/*
1201 		 * Confirm that curproc is indeed forking the process in which
1202 		 * we're trying to enable probes.
1203 		 */
1204 		ASSERT(p->p_parent == curproc);
1205 		ASSERT(p->p_stat == SIDL);
1206 
1207 		mutex_enter(&p->p_lock);
1208 		mutex_exit(&pidlock);
1209 
1210 		sprlock_proc(p);
1211 	}
1212 
1213 	ASSERT(!(p->p_flag & SVFORK));
1214 	mutex_exit(&p->p_lock);
1215 #else
1216 	if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0)
1217 		return;
1218 #endif
1219 
1220 	/*
1221 	 * We have to enable the trap entry point before any user threads have
1222 	 * the chance to execute the trap instruction we're about to place
1223 	 * in their process's text.
1224 	 */
1225 	fasttrap_enable_callbacks();
1226 
1227 	/*
1228 	 * Enable all the tracepoints and add this probe's id to each
1229 	 * tracepoint's list of active probes.
1230 	 */
1231 	for (i = 0; i < probe->ftp_ntps; i++) {
1232 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1233 			/*
1234 			 * If enabling the tracepoint failed completely,
1235 			 * we don't have to disable it; if the failure
1236 			 * was only partial we must disable it.
1237 			 */
1238 			if (rc == FASTTRAP_ENABLE_FAIL)
1239 				i--;
1240 			else
1241 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1242 
1243 			/*
1244 			 * Back up and pull out all the tracepoints we've
1245 			 * created so far for this probe.
1246 			 */
1247 			while (i >= 0) {
1248 				fasttrap_tracepoint_disable(p, probe, i);
1249 				i--;
1250 			}
1251 
1252 #ifdef illumos
1253 			mutex_enter(&p->p_lock);
1254 			sprunlock(p);
1255 #else
1256 			PRELE(p);
1257 #endif
1258 
1259 			/*
1260 			 * Since we're not actually enabling this probe,
1261 			 * drop our reference on the trap table entry.
1262 			 */
1263 			fasttrap_disable_callbacks();
1264 			return;
1265 		}
1266 	}
1267 #ifdef illumos
1268 	mutex_enter(&p->p_lock);
1269 	sprunlock(p);
1270 #else
1271 	PRELE(p);
1272 #endif
1273 
1274 	probe->ftp_enabled = 1;
1275 }
1276 
1277 /*ARGSUSED*/
1278 static void
1279 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1280 {
1281 	fasttrap_probe_t *probe = parg;
1282 	fasttrap_provider_t *provider = probe->ftp_prov;
1283 	proc_t *p;
1284 	int i, whack = 0;
1285 
1286 	ASSERT(id == probe->ftp_id);
1287 
1288 	mutex_enter(&provider->ftp_mtx);
1289 
1290 	/*
1291 	 * We won't be able to acquire a /proc-esque lock on the process
1292 	 * iff the process is dead and gone. In this case, we rely on the
1293 	 * provider lock as a point of mutual exclusion to prevent other
1294 	 * DTrace consumers from disabling this probe.
1295 	 */
1296 	if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0)
1297 		p = NULL;
1298 
1299 	/*
1300 	 * Disable all the associated tracepoints (for fully enabled probes).
1301 	 */
1302 	if (probe->ftp_enabled) {
1303 		for (i = 0; i < probe->ftp_ntps; i++) {
1304 			fasttrap_tracepoint_disable(p, probe, i);
1305 		}
1306 	}
1307 
1308 	ASSERT(provider->ftp_rcount > 0);
1309 	provider->ftp_rcount--;
1310 
1311 	if (p != NULL) {
1312 		/*
1313 		 * Even though we may not be able to remove it entirely, we
1314 		 * mark this retired provider to get a chance to remove some
1315 		 * of the associated probes.
1316 		 */
1317 		if (provider->ftp_retired && !provider->ftp_marked)
1318 			whack = provider->ftp_marked = 1;
1319 		mutex_exit(&provider->ftp_mtx);
1320 	} else {
1321 		/*
1322 		 * If the process is dead, we're just waiting for the
1323 		 * last probe to be disabled to be able to free it.
1324 		 */
1325 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1326 			whack = provider->ftp_marked = 1;
1327 		mutex_exit(&provider->ftp_mtx);
1328 	}
1329 
1330 	if (whack)
1331 		fasttrap_pid_cleanup();
1332 
1333 #ifdef __FreeBSD__
1334 	if (p != NULL)
1335 		PRELE(p);
1336 #endif
1337 	if (!probe->ftp_enabled)
1338 		return;
1339 
1340 	probe->ftp_enabled = 0;
1341 
1342 #ifdef illumos
1343 	ASSERT(MUTEX_HELD(&cpu_lock));
1344 #endif
1345 	fasttrap_disable_callbacks();
1346 }
1347 
1348 /*ARGSUSED*/
1349 static void
1350 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1351     dtrace_argdesc_t *desc)
1352 {
1353 	fasttrap_probe_t *probe = parg;
1354 	char *str;
1355 	int i, ndx;
1356 
1357 	desc->dtargd_native[0] = '\0';
1358 	desc->dtargd_xlate[0] = '\0';
1359 
1360 	if (probe->ftp_prov->ftp_retired != 0 ||
1361 	    desc->dtargd_ndx >= probe->ftp_nargs) {
1362 		desc->dtargd_ndx = DTRACE_ARGNONE;
1363 		return;
1364 	}
1365 
1366 	ndx = (probe->ftp_argmap != NULL) ?
1367 	    probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1368 
1369 	str = probe->ftp_ntypes;
1370 	for (i = 0; i < ndx; i++) {
1371 		str += strlen(str) + 1;
1372 	}
1373 
1374 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1375 	(void) strcpy(desc->dtargd_native, str);
1376 
1377 	if (probe->ftp_xtypes == NULL)
1378 		return;
1379 
1380 	str = probe->ftp_xtypes;
1381 	for (i = 0; i < desc->dtargd_ndx; i++) {
1382 		str += strlen(str) + 1;
1383 	}
1384 
1385 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1386 	(void) strcpy(desc->dtargd_xlate, str);
1387 }
1388 
1389 /*ARGSUSED*/
1390 static void
1391 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1392 {
1393 	fasttrap_probe_t *probe = parg;
1394 	int i;
1395 	size_t size;
1396 
1397 	ASSERT(probe != NULL);
1398 	ASSERT(!probe->ftp_enabled);
1399 	ASSERT(fasttrap_total >= probe->ftp_ntps);
1400 
1401 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1402 	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1403 
1404 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1405 		fasttrap_mod_barrier(probe->ftp_gen);
1406 
1407 	for (i = 0; i < probe->ftp_ntps; i++) {
1408 		kmem_free(probe->ftp_tps[i].fit_tp,
1409 		    sizeof (fasttrap_tracepoint_t));
1410 	}
1411 
1412 	kmem_free(probe, size);
1413 }
1414 
1415 
1416 static const dtrace_pattr_t pid_attr = {
1417 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1418 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1419 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1420 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1421 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1422 };
1423 
1424 static dtrace_pops_t pid_pops = {
1425 	.dtps_provide =		fasttrap_pid_provide,
1426 	.dtps_provide_module =	NULL,
1427 	.dtps_enable =		fasttrap_pid_enable,
1428 	.dtps_disable =		fasttrap_pid_disable,
1429 	.dtps_suspend =		NULL,
1430 	.dtps_resume =		NULL,
1431 	.dtps_getargdesc =	fasttrap_pid_getargdesc,
1432 	.dtps_getargval =	fasttrap_pid_getarg,
1433 	.dtps_usermode =	NULL,
1434 	.dtps_destroy =		fasttrap_pid_destroy
1435 };
1436 
1437 static dtrace_pops_t usdt_pops = {
1438 	.dtps_provide =		fasttrap_pid_provide,
1439 	.dtps_provide_module =	NULL,
1440 	.dtps_enable =		fasttrap_pid_enable,
1441 	.dtps_disable =		fasttrap_pid_disable,
1442 	.dtps_suspend =		NULL,
1443 	.dtps_resume =		NULL,
1444 	.dtps_getargdesc =	fasttrap_pid_getargdesc,
1445 	.dtps_getargval =	fasttrap_usdt_getarg,
1446 	.dtps_usermode =	NULL,
1447 	.dtps_destroy =		fasttrap_pid_destroy
1448 };
1449 
1450 static fasttrap_proc_t *
1451 fasttrap_proc_lookup(pid_t pid)
1452 {
1453 	fasttrap_bucket_t *bucket;
1454 	fasttrap_proc_t *fprc, *new_fprc;
1455 
1456 
1457 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1458 	mutex_enter(&bucket->ftb_mtx);
1459 
1460 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1461 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1462 			mutex_enter(&fprc->ftpc_mtx);
1463 			mutex_exit(&bucket->ftb_mtx);
1464 			fprc->ftpc_rcount++;
1465 			atomic_inc_64(&fprc->ftpc_acount);
1466 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1467 			mutex_exit(&fprc->ftpc_mtx);
1468 
1469 			return (fprc);
1470 		}
1471 	}
1472 
1473 	/*
1474 	 * Drop the bucket lock so we don't try to perform a sleeping
1475 	 * allocation under it.
1476 	 */
1477 	mutex_exit(&bucket->ftb_mtx);
1478 
1479 	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1480 	new_fprc->ftpc_pid = pid;
1481 	new_fprc->ftpc_rcount = 1;
1482 	new_fprc->ftpc_acount = 1;
1483 #ifndef illumos
1484 	mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1485 	    NULL);
1486 #endif
1487 
1488 	mutex_enter(&bucket->ftb_mtx);
1489 
1490 	/*
1491 	 * Take another lap through the list to make sure a proc hasn't
1492 	 * been created for this pid while we weren't under the bucket lock.
1493 	 */
1494 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1495 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1496 			mutex_enter(&fprc->ftpc_mtx);
1497 			mutex_exit(&bucket->ftb_mtx);
1498 			fprc->ftpc_rcount++;
1499 			atomic_inc_64(&fprc->ftpc_acount);
1500 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1501 			mutex_exit(&fprc->ftpc_mtx);
1502 
1503 			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1504 
1505 			return (fprc);
1506 		}
1507 	}
1508 
1509 	new_fprc->ftpc_next = bucket->ftb_data;
1510 	bucket->ftb_data = new_fprc;
1511 
1512 	mutex_exit(&bucket->ftb_mtx);
1513 
1514 	return (new_fprc);
1515 }
1516 
1517 static void
1518 fasttrap_proc_release(fasttrap_proc_t *proc)
1519 {
1520 	fasttrap_bucket_t *bucket;
1521 	fasttrap_proc_t *fprc, **fprcp;
1522 	pid_t pid = proc->ftpc_pid;
1523 #ifndef illumos
1524 	fasttrap_scrblock_t *scrblk, *scrblktmp;
1525 	fasttrap_scrspace_t *scrspc, *scrspctmp;
1526 	struct proc *p;
1527 	struct thread *td;
1528 #endif
1529 
1530 	mutex_enter(&proc->ftpc_mtx);
1531 
1532 	ASSERT(proc->ftpc_rcount != 0);
1533 	ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1534 
1535 	if (--proc->ftpc_rcount != 0) {
1536 		mutex_exit(&proc->ftpc_mtx);
1537 		return;
1538 	}
1539 
1540 #ifndef illumos
1541 	/*
1542 	 * Free all structures used to manage per-thread scratch space.
1543 	 */
1544 	LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,
1545 	    scrblktmp) {
1546 		LIST_REMOVE(scrblk, ftsb_next);
1547 		free(scrblk, M_SOLARIS);
1548 	}
1549 	LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) {
1550 		LIST_REMOVE(scrspc, ftss_next);
1551 		free(scrspc, M_SOLARIS);
1552 	}
1553 	LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) {
1554 		LIST_REMOVE(scrspc, ftss_next);
1555 		free(scrspc, M_SOLARIS);
1556 	}
1557 
1558 	if ((p = pfind(pid)) != NULL) {
1559 		FOREACH_THREAD_IN_PROC(p, td)
1560 			td->t_dtrace_sscr = NULL;
1561 		PROC_UNLOCK(p);
1562 	}
1563 #endif
1564 
1565 	mutex_exit(&proc->ftpc_mtx);
1566 
1567 	/*
1568 	 * There should definitely be no live providers associated with this
1569 	 * process at this point.
1570 	 */
1571 	ASSERT(proc->ftpc_acount == 0);
1572 
1573 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1574 	mutex_enter(&bucket->ftb_mtx);
1575 
1576 	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1577 	while ((fprc = *fprcp) != NULL) {
1578 		if (fprc == proc)
1579 			break;
1580 
1581 		fprcp = &fprc->ftpc_next;
1582 	}
1583 
1584 	/*
1585 	 * Something strange has happened if we can't find the proc.
1586 	 */
1587 	ASSERT(fprc != NULL);
1588 
1589 	*fprcp = fprc->ftpc_next;
1590 
1591 	mutex_exit(&bucket->ftb_mtx);
1592 
1593 	kmem_free(fprc, sizeof (fasttrap_proc_t));
1594 }
1595 
1596 /*
1597  * Lookup a fasttrap-managed provider based on its name and associated pid.
1598  * If the pattr argument is non-NULL, this function instantiates the provider
1599  * if it doesn't exist otherwise it returns NULL. The provider is returned
1600  * with its lock held.
1601  */
1602 static fasttrap_provider_t *
1603 fasttrap_provider_lookup(pid_t pid, const char *name,
1604     const dtrace_pattr_t *pattr)
1605 {
1606 	fasttrap_provider_t *fp, *new_fp = NULL;
1607 	fasttrap_bucket_t *bucket;
1608 	char provname[DTRACE_PROVNAMELEN];
1609 	proc_t *p;
1610 	cred_t *cred;
1611 
1612 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1613 	ASSERT(pattr != NULL);
1614 
1615 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1616 	mutex_enter(&bucket->ftb_mtx);
1617 
1618 	/*
1619 	 * Take a lap through the list and return the match if we find it.
1620 	 */
1621 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1622 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1623 		    !fp->ftp_retired) {
1624 			mutex_enter(&fp->ftp_mtx);
1625 			mutex_exit(&bucket->ftb_mtx);
1626 			return (fp);
1627 		}
1628 	}
1629 
1630 	/*
1631 	 * Drop the bucket lock so we don't try to perform a sleeping
1632 	 * allocation under it.
1633 	 */
1634 	mutex_exit(&bucket->ftb_mtx);
1635 
1636 	/*
1637 	 * Make sure the process exists, isn't a child created as the result
1638 	 * of a vfork(2), and isn't a zombie (but may be in fork).
1639 	 */
1640 	if ((p = pfind(pid)) == NULL)
1641 		return (NULL);
1642 
1643 	/*
1644 	 * Increment p_dtrace_probes so that the process knows to inform us
1645 	 * when it exits or execs. fasttrap_provider_free() decrements this
1646 	 * when we're done with this provider.
1647 	 */
1648 	p->p_dtrace_probes++;
1649 
1650 	/*
1651 	 * Grab the credentials for this process so we have
1652 	 * something to pass to dtrace_register().
1653 	 */
1654 	PROC_LOCK_ASSERT(p, MA_OWNED);
1655 	crhold(p->p_ucred);
1656 	cred = p->p_ucred;
1657 	PROC_UNLOCK(p);
1658 
1659 	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1660 	new_fp->ftp_pid = pid;
1661 	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1662 #ifndef illumos
1663 	mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1664 	mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1665 #endif
1666 
1667 	ASSERT(new_fp->ftp_proc != NULL);
1668 
1669 	mutex_enter(&bucket->ftb_mtx);
1670 
1671 	/*
1672 	 * Take another lap through the list to make sure a provider hasn't
1673 	 * been created for this pid while we weren't under the bucket lock.
1674 	 */
1675 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1676 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1677 		    !fp->ftp_retired) {
1678 			mutex_enter(&fp->ftp_mtx);
1679 			mutex_exit(&bucket->ftb_mtx);
1680 			fasttrap_provider_free(new_fp);
1681 			crfree(cred);
1682 			return (fp);
1683 		}
1684 	}
1685 
1686 	(void) strcpy(new_fp->ftp_name, name);
1687 
1688 	/*
1689 	 * Fail and return NULL if either the provider name is too long
1690 	 * or we fail to register this new provider with the DTrace
1691 	 * framework. Note that this is the only place we ever construct
1692 	 * the full provider name -- we keep it in pieces in the provider
1693 	 * structure.
1694 	 */
1695 	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1696 	    sizeof (provname) ||
1697 	    dtrace_register(provname, pattr,
1698 	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1699 	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1700 	    &new_fp->ftp_provid) != 0) {
1701 		mutex_exit(&bucket->ftb_mtx);
1702 		fasttrap_provider_free(new_fp);
1703 		crfree(cred);
1704 		return (NULL);
1705 	}
1706 
1707 	new_fp->ftp_next = bucket->ftb_data;
1708 	bucket->ftb_data = new_fp;
1709 
1710 	mutex_enter(&new_fp->ftp_mtx);
1711 	mutex_exit(&bucket->ftb_mtx);
1712 
1713 	crfree(cred);
1714 	return (new_fp);
1715 }
1716 
1717 static void
1718 fasttrap_provider_free(fasttrap_provider_t *provider)
1719 {
1720 	pid_t pid = provider->ftp_pid;
1721 	proc_t *p;
1722 
1723 	/*
1724 	 * There need to be no associated enabled probes, no consumers
1725 	 * creating probes, and no meta providers referencing this provider.
1726 	 */
1727 	ASSERT(provider->ftp_rcount == 0);
1728 	ASSERT(provider->ftp_ccount == 0);
1729 	ASSERT(provider->ftp_mcount == 0);
1730 
1731 	/*
1732 	 * If this provider hasn't been retired, we need to explicitly drop the
1733 	 * count of active providers on the associated process structure.
1734 	 */
1735 	if (!provider->ftp_retired) {
1736 		atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1737 		ASSERT(provider->ftp_proc->ftpc_acount <
1738 		    provider->ftp_proc->ftpc_rcount);
1739 	}
1740 
1741 	fasttrap_proc_release(provider->ftp_proc);
1742 
1743 #ifndef illumos
1744 	mutex_destroy(&provider->ftp_mtx);
1745 	mutex_destroy(&provider->ftp_cmtx);
1746 #endif
1747 	kmem_free(provider, sizeof (fasttrap_provider_t));
1748 
1749 	/*
1750 	 * Decrement p_dtrace_probes on the process whose provider we're
1751 	 * freeing. We don't have to worry about clobbering somone else's
1752 	 * modifications to it because we have locked the bucket that
1753 	 * corresponds to this process's hash chain in the provider hash
1754 	 * table. Don't sweat it if we can't find the process.
1755 	 */
1756 	if ((p = pfind(pid)) == NULL) {
1757 		return;
1758 	}
1759 
1760 	p->p_dtrace_probes--;
1761 #ifndef illumos
1762 	PROC_UNLOCK(p);
1763 #endif
1764 }
1765 
1766 static void
1767 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1768 {
1769 	fasttrap_provider_t *fp;
1770 	fasttrap_bucket_t *bucket;
1771 	dtrace_provider_id_t provid;
1772 
1773 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1774 
1775 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1776 	mutex_enter(&bucket->ftb_mtx);
1777 
1778 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1779 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1780 		    !fp->ftp_retired)
1781 			break;
1782 	}
1783 
1784 	if (fp == NULL) {
1785 		mutex_exit(&bucket->ftb_mtx);
1786 		return;
1787 	}
1788 
1789 	mutex_enter(&fp->ftp_mtx);
1790 	ASSERT(!mprov || fp->ftp_mcount > 0);
1791 	if (mprov && --fp->ftp_mcount != 0)  {
1792 		mutex_exit(&fp->ftp_mtx);
1793 		mutex_exit(&bucket->ftb_mtx);
1794 		return;
1795 	}
1796 
1797 	/*
1798 	 * Mark the provider to be removed in our post-processing step, mark it
1799 	 * retired, and drop the active count on its proc. Marking it indicates
1800 	 * that we should try to remove it; setting the retired flag indicates
1801 	 * that we're done with this provider; dropping the active the proc
1802 	 * releases our hold, and when this reaches zero (as it will during
1803 	 * exit or exec) the proc and associated providers become defunct.
1804 	 *
1805 	 * We obviously need to take the bucket lock before the provider lock
1806 	 * to perform the lookup, but we need to drop the provider lock
1807 	 * before calling into the DTrace framework since we acquire the
1808 	 * provider lock in callbacks invoked from the DTrace framework. The
1809 	 * bucket lock therefore protects the integrity of the provider hash
1810 	 * table.
1811 	 */
1812 	atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1813 	ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1814 
1815 	fp->ftp_retired = 1;
1816 	fp->ftp_marked = 1;
1817 	provid = fp->ftp_provid;
1818 	mutex_exit(&fp->ftp_mtx);
1819 
1820 	/*
1821 	 * We don't have to worry about invalidating the same provider twice
1822 	 * since fasttrap_provider_lookup() will ignore provider that have
1823 	 * been marked as retired.
1824 	 */
1825 	dtrace_invalidate(provid);
1826 
1827 	mutex_exit(&bucket->ftb_mtx);
1828 
1829 	fasttrap_pid_cleanup();
1830 }
1831 
1832 static int
1833 fasttrap_uint32_cmp(const void *ap, const void *bp)
1834 {
1835 	return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1836 }
1837 
1838 static int
1839 fasttrap_uint64_cmp(const void *ap, const void *bp)
1840 {
1841 	return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1842 }
1843 
1844 static int
1845 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1846 {
1847 	fasttrap_provider_t *provider;
1848 	fasttrap_probe_t *pp;
1849 	fasttrap_tracepoint_t *tp;
1850 	char *name;
1851 	int i, aframes = 0, whack;
1852 
1853 	/*
1854 	 * There needs to be at least one desired trace point.
1855 	 */
1856 	if (pdata->ftps_noffs == 0)
1857 		return (EINVAL);
1858 
1859 	switch (pdata->ftps_type) {
1860 	case DTFTP_ENTRY:
1861 		name = "entry";
1862 		aframes = FASTTRAP_ENTRY_AFRAMES;
1863 		break;
1864 	case DTFTP_RETURN:
1865 		name = "return";
1866 		aframes = FASTTRAP_RETURN_AFRAMES;
1867 		break;
1868 	case DTFTP_OFFSETS:
1869 		name = NULL;
1870 		break;
1871 	default:
1872 		return (EINVAL);
1873 	}
1874 
1875 	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1876 	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1877 		return (ESRCH);
1878 
1879 	/*
1880 	 * Increment this reference count to indicate that a consumer is
1881 	 * actively adding a new probe associated with this provider. This
1882 	 * prevents the provider from being deleted -- we'll need to check
1883 	 * for pending deletions when we drop this reference count.
1884 	 */
1885 	provider->ftp_ccount++;
1886 	mutex_exit(&provider->ftp_mtx);
1887 
1888 	/*
1889 	 * Grab the creation lock to ensure consistency between calls to
1890 	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1891 	 * other threads creating probes. We must drop the provider lock
1892 	 * before taking this lock to avoid a three-way deadlock with the
1893 	 * DTrace framework.
1894 	 */
1895 	mutex_enter(&provider->ftp_cmtx);
1896 
1897 	if (name == NULL) {
1898 		for (i = 0; i < pdata->ftps_noffs; i++) {
1899 			char name_str[17];
1900 
1901 			(void) sprintf(name_str, "%llx",
1902 			    (unsigned long long)pdata->ftps_offs[i]);
1903 
1904 			if (dtrace_probe_lookup(provider->ftp_provid,
1905 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1906 				continue;
1907 
1908 			atomic_inc_32(&fasttrap_total);
1909 
1910 			if (fasttrap_total > fasttrap_max) {
1911 				atomic_dec_32(&fasttrap_total);
1912 				goto no_mem;
1913 			}
1914 
1915 			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1916 
1917 			pp->ftp_prov = provider;
1918 			pp->ftp_faddr = pdata->ftps_pc;
1919 			pp->ftp_fsize = pdata->ftps_size;
1920 			pp->ftp_pid = pdata->ftps_pid;
1921 			pp->ftp_ntps = 1;
1922 
1923 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1924 			    KM_SLEEP);
1925 
1926 			tp->ftt_proc = provider->ftp_proc;
1927 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1928 			tp->ftt_pid = pdata->ftps_pid;
1929 
1930 			pp->ftp_tps[0].fit_tp = tp;
1931 			pp->ftp_tps[0].fit_id.fti_probe = pp;
1932 			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1933 
1934 			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1935 			    pdata->ftps_mod, pdata->ftps_func, name_str,
1936 			    FASTTRAP_OFFSET_AFRAMES, pp);
1937 		}
1938 
1939 	} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1940 	    pdata->ftps_func, name) == 0) {
1941 		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1942 
1943 		if (fasttrap_total > fasttrap_max) {
1944 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1945 			goto no_mem;
1946 		}
1947 
1948 		/*
1949 		 * Make sure all tracepoint program counter values are unique.
1950 		 * We later assume that each probe has exactly one tracepoint
1951 		 * for a given pc.
1952 		 */
1953 		qsort(pdata->ftps_offs, pdata->ftps_noffs,
1954 		    sizeof (uint64_t), fasttrap_uint64_cmp);
1955 		for (i = 1; i < pdata->ftps_noffs; i++) {
1956 			if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1957 				continue;
1958 
1959 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1960 			goto no_mem;
1961 		}
1962 
1963 		ASSERT(pdata->ftps_noffs > 0);
1964 		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1965 		    ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1966 
1967 		pp->ftp_prov = provider;
1968 		pp->ftp_faddr = pdata->ftps_pc;
1969 		pp->ftp_fsize = pdata->ftps_size;
1970 		pp->ftp_pid = pdata->ftps_pid;
1971 		pp->ftp_ntps = pdata->ftps_noffs;
1972 
1973 		for (i = 0; i < pdata->ftps_noffs; i++) {
1974 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1975 			    KM_SLEEP);
1976 
1977 			tp->ftt_proc = provider->ftp_proc;
1978 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1979 			tp->ftt_pid = pdata->ftps_pid;
1980 
1981 			pp->ftp_tps[i].fit_tp = tp;
1982 			pp->ftp_tps[i].fit_id.fti_probe = pp;
1983 			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1984 		}
1985 
1986 		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1987 		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1988 	}
1989 
1990 	mutex_exit(&provider->ftp_cmtx);
1991 
1992 	/*
1993 	 * We know that the provider is still valid since we incremented the
1994 	 * creation reference count. If someone tried to clean up this provider
1995 	 * while we were using it (e.g. because the process called exec(2) or
1996 	 * exit(2)), take note of that and try to clean it up now.
1997 	 */
1998 	mutex_enter(&provider->ftp_mtx);
1999 	provider->ftp_ccount--;
2000 	whack = provider->ftp_retired;
2001 	mutex_exit(&provider->ftp_mtx);
2002 
2003 	if (whack)
2004 		fasttrap_pid_cleanup();
2005 
2006 	return (0);
2007 
2008 no_mem:
2009 	/*
2010 	 * If we've exhausted the allowable resources, we'll try to remove
2011 	 * this provider to free some up. This is to cover the case where
2012 	 * the user has accidentally created many more probes than was
2013 	 * intended (e.g. pid123:::).
2014 	 */
2015 	mutex_exit(&provider->ftp_cmtx);
2016 	mutex_enter(&provider->ftp_mtx);
2017 	provider->ftp_ccount--;
2018 	provider->ftp_marked = 1;
2019 	mutex_exit(&provider->ftp_mtx);
2020 
2021 	fasttrap_pid_cleanup();
2022 
2023 	return (ENOMEM);
2024 }
2025 
2026 /*ARGSUSED*/
2027 static void *
2028 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2029 {
2030 	fasttrap_provider_t *provider;
2031 
2032 	/*
2033 	 * A 32-bit unsigned integer (like a pid for example) can be
2034 	 * expressed in 10 or fewer decimal digits. Make sure that we'll
2035 	 * have enough space for the provider name.
2036 	 */
2037 	if (strlen(dhpv->dthpv_provname) + 10 >=
2038 	    sizeof (provider->ftp_name)) {
2039 		printf("failed to instantiate provider %s: "
2040 		    "name too long to accomodate pid", dhpv->dthpv_provname);
2041 		return (NULL);
2042 	}
2043 
2044 	/*
2045 	 * Don't let folks spoof the true pid provider.
2046 	 */
2047 	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
2048 		printf("failed to instantiate provider %s: "
2049 		    "%s is an invalid name", dhpv->dthpv_provname,
2050 		    FASTTRAP_PID_NAME);
2051 		return (NULL);
2052 	}
2053 
2054 	/*
2055 	 * The highest stability class that fasttrap supports is ISA; cap
2056 	 * the stability of the new provider accordingly.
2057 	 */
2058 	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
2059 		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
2060 	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
2061 		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
2062 	if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
2063 		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
2064 	if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
2065 		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
2066 	if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
2067 		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
2068 
2069 	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
2070 	    &dhpv->dthpv_pattr)) == NULL) {
2071 		printf("failed to instantiate provider %s for "
2072 		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
2073 		return (NULL);
2074 	}
2075 
2076 	/*
2077 	 * Up the meta provider count so this provider isn't removed until
2078 	 * the meta provider has been told to remove it.
2079 	 */
2080 	provider->ftp_mcount++;
2081 
2082 	mutex_exit(&provider->ftp_mtx);
2083 
2084 	return (provider);
2085 }
2086 
2087 /*
2088  * We know a few things about our context here:  we know that the probe being
2089  * created doesn't already exist (DTrace won't load DOF at the same address
2090  * twice, even if explicitly told to do so) and we know that we are
2091  * single-threaded with respect to the meta provider machinery. Knowing that
2092  * this is a new probe and that there is no way for us to race with another
2093  * operation on this provider allows us an important optimization: we need not
2094  * lookup a probe before adding it.  Saving this lookup is important because
2095  * this code is in the fork path for processes with USDT probes, and lookups
2096  * here are potentially very expensive because of long hash conflicts on
2097  * module, function and name (DTrace doesn't hash on provider name).
2098  */
2099 /*ARGSUSED*/
2100 static void
2101 fasttrap_meta_create_probe(void *arg, void *parg,
2102     dtrace_helper_probedesc_t *dhpb)
2103 {
2104 	fasttrap_provider_t *provider = parg;
2105 	fasttrap_probe_t *pp;
2106 	fasttrap_tracepoint_t *tp;
2107 	int i, j;
2108 	uint32_t ntps;
2109 
2110 	/*
2111 	 * Since the meta provider count is non-zero we don't have to worry
2112 	 * about this provider disappearing.
2113 	 */
2114 	ASSERT(provider->ftp_mcount > 0);
2115 
2116 	/*
2117 	 * The offsets must be unique.
2118 	 */
2119 	qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
2120 	    fasttrap_uint32_cmp);
2121 	for (i = 1; i < dhpb->dthpb_noffs; i++) {
2122 		if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
2123 		    dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
2124 			return;
2125 	}
2126 
2127 	qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
2128 	    fasttrap_uint32_cmp);
2129 	for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
2130 		if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
2131 		    dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
2132 			return;
2133 	}
2134 
2135 	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
2136 	ASSERT(ntps > 0);
2137 
2138 	atomic_add_32(&fasttrap_total, ntps);
2139 
2140 	if (fasttrap_total > fasttrap_max) {
2141 		atomic_add_32(&fasttrap_total, -ntps);
2142 		return;
2143 	}
2144 
2145 	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
2146 
2147 	pp->ftp_prov = provider;
2148 	pp->ftp_pid = provider->ftp_pid;
2149 	pp->ftp_ntps = ntps;
2150 	pp->ftp_nargs = dhpb->dthpb_xargc;
2151 	pp->ftp_xtypes = dhpb->dthpb_xtypes;
2152 	pp->ftp_ntypes = dhpb->dthpb_ntypes;
2153 
2154 	/*
2155 	 * First create a tracepoint for each actual point of interest.
2156 	 */
2157 	for (i = 0; i < dhpb->dthpb_noffs; i++) {
2158 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2159 
2160 		tp->ftt_proc = provider->ftp_proc;
2161 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
2162 		tp->ftt_pid = provider->ftp_pid;
2163 
2164 		pp->ftp_tps[i].fit_tp = tp;
2165 		pp->ftp_tps[i].fit_id.fti_probe = pp;
2166 #ifdef __sparc
2167 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
2168 #else
2169 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
2170 #endif
2171 	}
2172 
2173 	/*
2174 	 * Then create a tracepoint for each is-enabled point.
2175 	 */
2176 	for (j = 0; i < ntps; i++, j++) {
2177 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2178 
2179 		tp->ftt_proc = provider->ftp_proc;
2180 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
2181 		tp->ftt_pid = provider->ftp_pid;
2182 
2183 		pp->ftp_tps[i].fit_tp = tp;
2184 		pp->ftp_tps[i].fit_id.fti_probe = pp;
2185 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2186 	}
2187 
2188 	/*
2189 	 * If the arguments are shuffled around we set the argument remapping
2190 	 * table. Later, when the probe fires, we only remap the arguments
2191 	 * if the table is non-NULL.
2192 	 */
2193 	for (i = 0; i < dhpb->dthpb_xargc; i++) {
2194 		if (dhpb->dthpb_args[i] != i) {
2195 			pp->ftp_argmap = dhpb->dthpb_args;
2196 			break;
2197 		}
2198 	}
2199 
2200 	/*
2201 	 * The probe is fully constructed -- register it with DTrace.
2202 	 */
2203 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2204 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2205 }
2206 
2207 /*ARGSUSED*/
2208 static void
2209 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2210 {
2211 	/*
2212 	 * Clean up the USDT provider. There may be active consumers of the
2213 	 * provider busy adding probes, no damage will actually befall the
2214 	 * provider until that count has dropped to zero. This just puts
2215 	 * the provider on death row.
2216 	 */
2217 	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2218 }
2219 
2220 static dtrace_mops_t fasttrap_mops = {
2221 	.dtms_create_probe =	fasttrap_meta_create_probe,
2222 	.dtms_provide_pid =	fasttrap_meta_provide,
2223 	.dtms_remove_pid =	fasttrap_meta_remove
2224 };
2225 
2226 /*ARGSUSED*/
2227 static int
2228 fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2229     int devtype __unused, struct thread *td __unused)
2230 {
2231 	return (0);
2232 }
2233 
2234 /*ARGSUSED*/
2235 static int
2236 fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2237     struct thread *td)
2238 {
2239 	if (!dtrace_attached())
2240 		return (EAGAIN);
2241 
2242 	if (cmd == FASTTRAPIOC_MAKEPROBE) {
2243 		fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2244 		fasttrap_probe_spec_t *probe;
2245 		uint64_t noffs;
2246 		size_t size;
2247 		int ret, err;
2248 
2249 		if (copyin(&uprobe->ftps_noffs, &noffs,
2250 		    sizeof (uprobe->ftps_noffs)))
2251 			return (EFAULT);
2252 
2253 		/*
2254 		 * Probes must have at least one tracepoint.
2255 		 */
2256 		if (noffs == 0)
2257 			return (EINVAL);
2258 
2259 		size = sizeof (fasttrap_probe_spec_t) +
2260 		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
2261 
2262 		if (size > 1024 * 1024)
2263 			return (ENOMEM);
2264 
2265 		probe = kmem_alloc(size, KM_SLEEP);
2266 
2267 		if (copyin(uprobe, probe, size) != 0 ||
2268 		    probe->ftps_noffs != noffs) {
2269 			kmem_free(probe, size);
2270 			return (EFAULT);
2271 		}
2272 
2273 		/*
2274 		 * Verify that the function and module strings contain no
2275 		 * funny characters.
2276 		 */
2277 		if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
2278 		    NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
2279 			ret = EINVAL;
2280 			goto err;
2281 		}
2282 
2283 		if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
2284 		    NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
2285 			ret = EINVAL;
2286 			goto err;
2287 		}
2288 
2289 #ifdef notyet
2290 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2291 			proc_t *p;
2292 			pid_t pid = probe->ftps_pid;
2293 
2294 			mutex_enter(&pidlock);
2295 			/*
2296 			 * Report an error if the process doesn't exist
2297 			 * or is actively being birthed.
2298 			 */
2299 			if ((p = pfind(pid)) == NULL || p->p_stat == SIDL) {
2300 				mutex_exit(&pidlock);
2301 				return (ESRCH);
2302 			}
2303 			mutex_enter(&p->p_lock);
2304 			mutex_exit(&pidlock);
2305 
2306 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2307 			    VREAD | VWRITE)) != 0) {
2308 				mutex_exit(&p->p_lock);
2309 				return (ret);
2310 			}
2311 			mutex_exit(&p->p_lock);
2312 		}
2313 #endif /* notyet */
2314 
2315 		ret = fasttrap_add_probe(probe);
2316 err:
2317 		kmem_free(probe, size);
2318 
2319 		return (ret);
2320 
2321 	} else if (cmd == FASTTRAPIOC_GETINSTR) {
2322 		fasttrap_instr_query_t instr;
2323 		fasttrap_tracepoint_t *tp;
2324 		uint_t index;
2325 #ifdef notyet
2326 		int ret;
2327 #endif
2328 
2329 		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2330 			return (EFAULT);
2331 
2332 #ifdef notyet
2333 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2334 			proc_t *p;
2335 			pid_t pid = instr.ftiq_pid;
2336 
2337 			mutex_enter(&pidlock);
2338 			/*
2339 			 * Report an error if the process doesn't exist
2340 			 * or is actively being birthed.
2341 			 */
2342 			if ((p == pfind(pid)) == NULL || p->p_stat == SIDL) {
2343 				mutex_exit(&pidlock);
2344 				return (ESRCH);
2345 			}
2346 			mutex_enter(&p->p_lock);
2347 			mutex_exit(&pidlock);
2348 
2349 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2350 			    VREAD)) != 0) {
2351 				mutex_exit(&p->p_lock);
2352 				return (ret);
2353 			}
2354 
2355 			mutex_exit(&p->p_lock);
2356 		}
2357 #endif /* notyet */
2358 
2359 		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2360 
2361 		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2362 		tp = fasttrap_tpoints.fth_table[index].ftb_data;
2363 		while (tp != NULL) {
2364 			if (instr.ftiq_pid == tp->ftt_pid &&
2365 			    instr.ftiq_pc == tp->ftt_pc &&
2366 			    tp->ftt_proc->ftpc_acount != 0)
2367 				break;
2368 
2369 			tp = tp->ftt_next;
2370 		}
2371 
2372 		if (tp == NULL) {
2373 			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2374 			return (ENOENT);
2375 		}
2376 
2377 		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2378 		    sizeof (instr.ftiq_instr));
2379 		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2380 
2381 		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2382 			return (EFAULT);
2383 
2384 		return (0);
2385 	}
2386 
2387 	return (EINVAL);
2388 }
2389 
2390 static int
2391 fasttrap_load(void)
2392 {
2393 	ulong_t nent;
2394 	int i, ret;
2395 
2396         /* Create the /dev/dtrace/fasttrap entry. */
2397         fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2398             "dtrace/fasttrap");
2399 
2400 	mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2401 	mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2402 	    NULL);
2403 
2404 #ifdef illumos
2405 	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2406 	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2407 #endif
2408 	fasttrap_total = 0;
2409 
2410 	/*
2411 	 * Conjure up the tracepoints hashtable...
2412 	 */
2413 #ifdef illumos
2414 	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2415 	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2416 #else
2417 	nent = tpoints_hash_size;
2418 #endif
2419 
2420 	if (nent == 0 || nent > 0x1000000)
2421 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2422 
2423 	tpoints_hash_size = nent;
2424 
2425 	if (ISP2(nent))
2426 		fasttrap_tpoints.fth_nent = nent;
2427 	else
2428 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2429 	ASSERT(fasttrap_tpoints.fth_nent > 0);
2430 	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2431 	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2432 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2433 #ifndef illumos
2434 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2435 		mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2436 		    "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2437 #endif
2438 
2439 	/*
2440 	 * ... and the providers hash table...
2441 	 */
2442 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2443 	if (ISP2(nent))
2444 		fasttrap_provs.fth_nent = nent;
2445 	else
2446 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2447 	ASSERT(fasttrap_provs.fth_nent > 0);
2448 	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2449 	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2450 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2451 #ifndef illumos
2452 	for (i = 0; i < fasttrap_provs.fth_nent; i++)
2453 		mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2454 		    "providers bucket mtx", MUTEX_DEFAULT, NULL);
2455 #endif
2456 
2457 	ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2458 	    &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2459 	if (ret != 0) {
2460 		destroy_dev(fasttrap_cdev);
2461 #ifndef illumos
2462 		for (i = 0; i < fasttrap_provs.fth_nent; i++)
2463 			mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2464 		for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2465 			mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2466 #endif
2467 		kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *
2468 		    sizeof (fasttrap_bucket_t));
2469 		mtx_destroy(&fasttrap_cleanup_mtx);
2470 		mutex_destroy(&fasttrap_count_mtx);
2471 		return (ret);
2472 	}
2473 
2474 
2475 	/*
2476 	 * ... and the procs hash table.
2477 	 */
2478 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2479 	if (ISP2(nent))
2480 		fasttrap_procs.fth_nent = nent;
2481 	else
2482 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2483 	ASSERT(fasttrap_procs.fth_nent > 0);
2484 	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2485 	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2486 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2487 #ifndef illumos
2488 	for (i = 0; i < fasttrap_procs.fth_nent; i++)
2489 		mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2490 		    "processes bucket mtx", MUTEX_DEFAULT, NULL);
2491 
2492 	rm_init(&fasttrap_tp_lock, "fasttrap tracepoint");
2493 
2494 	/*
2495 	 * This event handler must run before kdtrace_thread_dtor() since it
2496 	 * accesses the thread's struct kdtrace_thread.
2497 	 */
2498 	fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
2499 	    fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
2500 #endif
2501 
2502 	/*
2503 	 * Install our hooks into fork(2), exec(2), and exit(2).
2504 	 */
2505 	dtrace_fasttrap_fork = &fasttrap_fork;
2506 	dtrace_fasttrap_exit = &fasttrap_exec_exit;
2507 	dtrace_fasttrap_exec = &fasttrap_exec_exit;
2508 
2509 	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2510 	    &fasttrap_meta_id);
2511 
2512 	return (0);
2513 }
2514 
2515 static int
2516 fasttrap_unload(void)
2517 {
2518 	int i, fail = 0;
2519 
2520 	/*
2521 	 * Unregister the meta-provider to make sure no new fasttrap-
2522 	 * managed providers come along while we're trying to close up
2523 	 * shop. If we fail to detach, we'll need to re-register as a
2524 	 * meta-provider. We can fail to unregister as a meta-provider
2525 	 * if providers we manage still exist.
2526 	 */
2527 	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2528 	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
2529 		return (-1);
2530 
2531 	/*
2532 	 * Iterate over all of our providers. If there's still a process
2533 	 * that corresponds to that pid, fail to detach.
2534 	 */
2535 	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2536 		fasttrap_provider_t **fpp, *fp;
2537 		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2538 
2539 		mutex_enter(&bucket->ftb_mtx);
2540 		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2541 		while ((fp = *fpp) != NULL) {
2542 			/*
2543 			 * Acquire and release the lock as a simple way of
2544 			 * waiting for any other consumer to finish with
2545 			 * this provider. A thread must first acquire the
2546 			 * bucket lock so there's no chance of another thread
2547 			 * blocking on the provider's lock.
2548 			 */
2549 			mutex_enter(&fp->ftp_mtx);
2550 			mutex_exit(&fp->ftp_mtx);
2551 
2552 			if (dtrace_unregister(fp->ftp_provid) != 0) {
2553 				fail = 1;
2554 				fpp = &fp->ftp_next;
2555 			} else {
2556 				*fpp = fp->ftp_next;
2557 				fasttrap_provider_free(fp);
2558 			}
2559 		}
2560 
2561 		mutex_exit(&bucket->ftb_mtx);
2562 	}
2563 
2564 	if (fail) {
2565 		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2566 		    &fasttrap_meta_id);
2567 
2568 		return (-1);
2569 	}
2570 
2571 	/*
2572 	 * Stop new processes from entering these hooks now, before the
2573 	 * fasttrap_cleanup thread runs.  That way all processes will hopefully
2574 	 * be out of these hooks before we free fasttrap_provs.fth_table
2575 	 */
2576 	ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2577 	dtrace_fasttrap_fork = NULL;
2578 
2579 	ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2580 	dtrace_fasttrap_exec = NULL;
2581 
2582 	ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2583 	dtrace_fasttrap_exit = NULL;
2584 
2585 	mtx_lock(&fasttrap_cleanup_mtx);
2586 	fasttrap_cleanup_drain = 1;
2587 	/* Wait for the cleanup thread to finish up and signal us. */
2588 	wakeup(&fasttrap_cleanup_cv);
2589 	mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2590 	    0);
2591 	fasttrap_cleanup_proc = NULL;
2592 	mtx_destroy(&fasttrap_cleanup_mtx);
2593 
2594 #ifdef DEBUG
2595 	mutex_enter(&fasttrap_count_mtx);
2596 	ASSERT(fasttrap_pid_count == 0);
2597 	mutex_exit(&fasttrap_count_mtx);
2598 #endif
2599 
2600 #ifndef illumos
2601 	EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag);
2602 
2603 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2604 		mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2605 	for (i = 0; i < fasttrap_provs.fth_nent; i++)
2606 		mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2607 	for (i = 0; i < fasttrap_procs.fth_nent; i++)
2608 		mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2609 #endif
2610 	kmem_free(fasttrap_tpoints.fth_table,
2611 	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2612 	fasttrap_tpoints.fth_nent = 0;
2613 
2614 	kmem_free(fasttrap_provs.fth_table,
2615 	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2616 	fasttrap_provs.fth_nent = 0;
2617 
2618 	kmem_free(fasttrap_procs.fth_table,
2619 	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2620 	fasttrap_procs.fth_nent = 0;
2621 
2622 #ifndef illumos
2623 	destroy_dev(fasttrap_cdev);
2624 	mutex_destroy(&fasttrap_count_mtx);
2625 	rm_destroy(&fasttrap_tp_lock);
2626 #endif
2627 
2628 	return (0);
2629 }
2630 
2631 /* ARGSUSED */
2632 static int
2633 fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2634 {
2635 	int error = 0;
2636 
2637 	switch (type) {
2638 	case MOD_LOAD:
2639 		break;
2640 
2641 	case MOD_UNLOAD:
2642 		break;
2643 
2644 	case MOD_SHUTDOWN:
2645 		break;
2646 
2647 	default:
2648 		error = EOPNOTSUPP;
2649 		break;
2650 	}
2651 	return (error);
2652 }
2653 
2654 SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2655     NULL);
2656 SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2657     fasttrap_unload, NULL);
2658 
2659 DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2660 MODULE_VERSION(fasttrap, 1);
2661 MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2662 MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
2663