xref: /illumos-gate/usr/src/uts/common/dtrace/fasttrap.c (revision 54719d5edb7521bb3a321ad8bd9ebc251d61a1a0)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 
23 /*
24  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
25  * Use is subject to license terms.
26  */
27 
28 #pragma ident	"%Z%%M%	%I%	%E% SMI"
29 
30 #include <sys/atomic.h>
31 #include <sys/errno.h>
32 #include <sys/stat.h>
33 #include <sys/modctl.h>
34 #include <sys/conf.h>
35 #include <sys/systm.h>
36 #include <sys/ddi.h>
37 #include <sys/sunddi.h>
38 #include <sys/cpuvar.h>
39 #include <sys/kmem.h>
40 #include <sys/strsubr.h>
41 #include <sys/fasttrap.h>
42 #include <sys/fasttrap_impl.h>
43 #include <sys/fasttrap_isa.h>
44 #include <sys/dtrace.h>
45 #include <sys/dtrace_impl.h>
46 #include <sys/sysmacros.h>
47 #include <sys/frame.h>
48 #include <sys/stack.h>
49 #include <sys/proc.h>
50 #include <sys/priv.h>
51 #include <sys/policy.h>
52 #include <sys/ontrap.h>
53 #include <sys/vmsystm.h>
54 #include <sys/prsystm.h>
55 
56 
57 #include <vm/as.h>
58 #include <vm/seg.h>
59 #include <vm/seg_dev.h>
60 #include <vm/seg_vn.h>
61 #include <vm/seg_spt.h>
62 #include <vm/seg_kmem.h>
63 
64 /*
65  * User-Land Trap-Based Tracing
66  * ----------------------------
67  *
68  * The fasttrap provider allows DTrace consumers to instrument any user-level
69  * instruction to gather data; this includes probes with semantic
70  * signifigance like entry and return as well as simple offsets into the
71  * function. While the specific techniques used are very ISA specific, the
72  * methodology is generalizable to any architecture.
73  *
74  *
75  * The General Methodology
76  * -----------------------
77  *
78  * With the primary goal of tracing every user-land instruction and the
79  * limitation that we can't trust user space so don't want to rely on much
80  * information there, we begin by replacing the instructions we want to trace
81  * with trap instructions. Each instruction we overwrite is saved into a hash
82  * table keyed by process ID and pc address. When we enter the kernel due to
83  * this trap instruction, we need the effects of the replaced instruction to
84  * appear to have occurred before we proceed with the user thread's
85  * execution.
86  *
87  * Each user level thread is represented by a ulwp_t structure which is
88  * always easily accessible through a register. The most basic way to produce
89  * the effects of the instruction we replaced is to copy that instruction out
90  * to a bit of scratch space reserved in the user thread's ulwp_t structure
91  * (a sort of kernel-private thread local storage), set the PC to that
92  * scratch space and single step. When we reenter the kernel after single
93  * stepping the instruction we must then adjust the PC to point to what would
94  * normally be the next instruction. Of course, special care must be taken
95  * for branches and jumps, but these represent such a small fraction of any
96  * instruction set that writing the code to emulate these in the kernel is
97  * not too difficult.
98  *
99  * Return probes may require several tracepoints to trace every return site,
100  * and, conversely, each tracepoint may activate several probes (the entry
101  * and offset 0 probes, for example). To solve this muliplexing problem,
102  * tracepoints contain lists of probes to activate and probes contain lists
103  * of tracepoints to enable. If a probe is activated, it adds its ID to
104  * existing tracepoints or creates new ones as necessary.
105  *
106  * Most probes are activated _before_ the instruction is executed, but return
107  * probes are activated _after_ the effects of the last instruction of the
108  * function are visible. Return probes must be fired _after_ we have
109  * single-stepped the instruction whereas all other probes are fired
110  * beforehand.
111  */
112 
113 static dev_info_t *fasttrap_devi;
114 static dtrace_provider_id_t fasttrap_id;
115 static dtrace_meta_provider_id_t fasttrap_meta_id;
116 
117 static timeout_id_t fasttrap_timeout;
118 static kmutex_t fasttrap_cleanup_mtx;
119 static uint_t fasttrap_cleanup_work;
120 
121 /*
122  * Generation count on modifications to the global tracepoint lookup table.
123  */
124 static volatile uint64_t fasttrap_mod_gen;
125 
126 /*
127  * When the fasttrap provider is loaded, fasttrap_max is set to either
128  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
129  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
130  * incremented by the number of tracepoints that may be associated with that
131  * probe; fasttrap_total is capped at fasttrap_max.
132  */
133 #define	FASTTRAP_MAX_DEFAULT		250000
134 static uint32_t fasttrap_max;
135 static uint32_t fasttrap_total;
136 
137 
138 #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
139 #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
140 #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
141 
142 #define	FASTTRAP_PID_NAME		"pid"
143 
144 fasttrap_hash_t			fasttrap_tpoints;
145 static fasttrap_hash_t		fasttrap_provs;
146 static fasttrap_hash_t		fasttrap_procs;
147 
148 dtrace_id_t			fasttrap_probe_id;
149 static int			fasttrap_count;		/* ref count */
150 static int			fasttrap_pid_count;	/* pid ref count */
151 static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
152 
153 #define	FASTTRAP_ENABLE_FAIL	1
154 #define	FASTTRAP_ENABLE_PARTIAL	2
155 
156 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
157 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
158 
159 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
160     const dtrace_pattr_t *);
161 static void fasttrap_provider_free(fasttrap_provider_t *);
162 static void fasttrap_provider_retire(fasttrap_provider_t *);
163 
164 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
165 static void fasttrap_proc_release(fasttrap_proc_t *);
166 
167 #define	FASTTRAP_PROVS_INDEX(pid, name) \
168 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
169 
170 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
171 
172 static int
173 fasttrap_highbit(ulong_t i)
174 {
175 	int h = 1;
176 
177 	if (i == 0)
178 		return (0);
179 #ifdef _LP64
180 	if (i & 0xffffffff00000000ul) {
181 		h += 32; i >>= 32;
182 	}
183 #endif
184 	if (i & 0xffff0000) {
185 		h += 16; i >>= 16;
186 	}
187 	if (i & 0xff00) {
188 		h += 8; i >>= 8;
189 	}
190 	if (i & 0xf0) {
191 		h += 4; i >>= 4;
192 	}
193 	if (i & 0xc) {
194 		h += 2; i >>= 2;
195 	}
196 	if (i & 0x2) {
197 		h += 1;
198 	}
199 	return (h);
200 }
201 
202 static uint_t
203 fasttrap_hash_str(const char *p)
204 {
205 	unsigned int g;
206 	uint_t hval = 0;
207 
208 	while (*p) {
209 		hval = (hval << 4) + *p++;
210 		if ((g = (hval & 0xf0000000)) != 0)
211 			hval ^= g >> 24;
212 		hval &= ~g;
213 	}
214 	return (hval);
215 }
216 
217 void
218 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
219 {
220 	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
221 
222 	sqp->sq_info.si_signo = SIGTRAP;
223 	sqp->sq_info.si_code = TRAP_DTRACE;
224 	sqp->sq_info.si_addr = (caddr_t)pc;
225 
226 	mutex_enter(&p->p_lock);
227 	sigaddqa(p, t, sqp);
228 	mutex_exit(&p->p_lock);
229 
230 	if (t != NULL)
231 		aston(t);
232 }
233 
234 /*
235  * This function ensures that no threads are actively using the memory
236  * associated with probes that were formerly live.
237  */
238 static void
239 fasttrap_mod_barrier(uint64_t gen)
240 {
241 	int i;
242 
243 	if (gen < fasttrap_mod_gen)
244 		return;
245 
246 	fasttrap_mod_gen++;
247 
248 	for (i = 0; i < NCPU; i++) {
249 		mutex_enter(&cpu_core[i].cpuc_pid_lock);
250 		mutex_exit(&cpu_core[i].cpuc_pid_lock);
251 	}
252 }
253 
254 /*
255  * This is the timeout's callback for cleaning up the providers and their
256  * probes.
257  */
258 /*ARGSUSED*/
259 static void
260 fasttrap_pid_cleanup_cb(void *data)
261 {
262 	fasttrap_provider_t **fpp, *fp;
263 	fasttrap_bucket_t *bucket;
264 	dtrace_provider_id_t provid;
265 	int i, later;
266 
267 	static volatile int in = 0;
268 	ASSERT(in == 0);
269 	in = 1;
270 
271 	mutex_enter(&fasttrap_cleanup_mtx);
272 	while (fasttrap_cleanup_work) {
273 		fasttrap_cleanup_work = 0;
274 		mutex_exit(&fasttrap_cleanup_mtx);
275 
276 		later = 0;
277 
278 		/*
279 		 * Iterate over all the providers trying to remove the marked
280 		 * ones. If a provider is marked but not retired, we just
281 		 * have to take a crack at removing it -- it's no big deal if
282 		 * we can't.
283 		 */
284 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
285 			bucket = &fasttrap_provs.fth_table[i];
286 			mutex_enter(&bucket->ftb_mtx);
287 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
288 
289 			while ((fp = *fpp) != NULL) {
290 				if (!fp->ftp_marked) {
291 					fpp = &fp->ftp_next;
292 					continue;
293 				}
294 
295 				mutex_enter(&fp->ftp_mtx);
296 
297 				/*
298 				 * If this provider is referenced either
299 				 * because it is a USDT provider or is being
300 				 * modified, we can't unregister or even
301 				 * condense.
302 				 */
303 				if (fp->ftp_ccount != 0) {
304 					mutex_exit(&fp->ftp_mtx);
305 					fp->ftp_marked = 0;
306 					continue;
307 				}
308 
309 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
310 					fp->ftp_marked = 0;
311 
312 				mutex_exit(&fp->ftp_mtx);
313 
314 				/*
315 				 * If we successfully unregister this
316 				 * provider we can remove it from the hash
317 				 * chain and free the memory. If our attempt
318 				 * to unregister fails and this is a retired
319 				 * provider, increment our flag to try again
320 				 * pretty soon. If we've consumed more than
321 				 * half of our total permitted number of
322 				 * probes call dtrace_condense() to try to
323 				 * clean out the unenabled probes.
324 				 */
325 				provid = fp->ftp_provid;
326 				if (dtrace_unregister(provid) != 0) {
327 					if (fasttrap_total > fasttrap_max / 2)
328 						(void) dtrace_condense(provid);
329 					later += fp->ftp_marked;
330 					fpp = &fp->ftp_next;
331 				} else {
332 					*fpp = fp->ftp_next;
333 					fasttrap_provider_free(fp);
334 				}
335 			}
336 			mutex_exit(&bucket->ftb_mtx);
337 		}
338 
339 		mutex_enter(&fasttrap_cleanup_mtx);
340 	}
341 
342 	ASSERT(fasttrap_timeout != 0);
343 
344 	/*
345 	 * If we were unable to remove a retired provider, try again after
346 	 * a second. This situation can occur in certain circumstances where
347 	 * providers cannot be unregistered even though they have no probes
348 	 * enabled because of an execution of dtrace -l or something similar.
349 	 * If the timeout has been disabled (set to 1 because we're trying
350 	 * to detach), we set fasttrap_cleanup_work to ensure that we'll
351 	 * get a chance to do that work if and when the timeout is reenabled
352 	 * (if detach fails).
353 	 */
354 	if (later > 0 && fasttrap_timeout != (timeout_id_t)1)
355 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
356 	else if (later > 0)
357 		fasttrap_cleanup_work = 1;
358 	else
359 		fasttrap_timeout = 0;
360 
361 	mutex_exit(&fasttrap_cleanup_mtx);
362 	in = 0;
363 }
364 
365 /*
366  * Activates the asynchronous cleanup mechanism.
367  */
368 static void
369 fasttrap_pid_cleanup(void)
370 {
371 	mutex_enter(&fasttrap_cleanup_mtx);
372 	fasttrap_cleanup_work = 1;
373 	if (fasttrap_timeout == 0)
374 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
375 	mutex_exit(&fasttrap_cleanup_mtx);
376 }
377 
378 /*
379  * This is called from cfork() via dtrace_fasttrap_fork(). The child
380  * process's address space is a (roughly) a copy of the parent process's so
381  * we have to remove all the instrumentation we had previously enabled in the
382  * parent.
383  */
384 static void
385 fasttrap_fork(proc_t *p, proc_t *cp)
386 {
387 	pid_t ppid = p->p_pid;
388 	int i;
389 
390 	ASSERT(curproc == p);
391 	ASSERT(p->p_proc_flag & P_PR_LOCK);
392 	ASSERT(p->p_dtrace_count > 0);
393 	ASSERT(cp->p_dtrace_count == 0);
394 
395 	/*
396 	 * This would be simpler and faster if we maintained per-process
397 	 * hash tables of enabled tracepoints. It could, however, potentially
398 	 * slow down execution of a tracepoint since we'd need to go
399 	 * through two levels of indirection. In the future, we should
400 	 * consider either maintaining per-process ancillary lists of
401 	 * enabled tracepoints or hanging a pointer to a per-process hash
402 	 * table of enabled tracepoints off the proc structure.
403 	 */
404 
405 	/*
406 	 * We don't have to worry about the child process disappearing
407 	 * because we're in fork().
408 	 */
409 	mutex_enter(&cp->p_lock);
410 	sprlock_proc(cp);
411 	mutex_exit(&cp->p_lock);
412 
413 	/*
414 	 * Iterate over every tracepoint looking for ones that belong to the
415 	 * parent process, and remove each from the child process.
416 	 */
417 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
418 		fasttrap_tracepoint_t *tp;
419 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
420 
421 		mutex_enter(&bucket->ftb_mtx);
422 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
423 			if (tp->ftt_pid == ppid &&
424 			    !tp->ftt_proc->ftpc_defunct) {
425 				int ret = fasttrap_tracepoint_remove(cp, tp);
426 				ASSERT(ret == 0);
427 			}
428 		}
429 		mutex_exit(&bucket->ftb_mtx);
430 	}
431 
432 	mutex_enter(&cp->p_lock);
433 	sprunlock(cp);
434 }
435 
436 /*
437  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
438  * is set on the proc structure to indicate that there is a pid provider
439  * associated with this process.
440  */
441 static void
442 fasttrap_exec_exit(proc_t *p)
443 {
444 	fasttrap_provider_t *provider;
445 
446 	ASSERT(p == curproc);
447 	ASSERT(MUTEX_HELD(&p->p_lock));
448 
449 	mutex_exit(&p->p_lock);
450 
451 	/*
452 	 * We clean up the pid provider for this process here; user-land
453 	 * static probes are handled by the meta-provider remove entry point.
454 	 */
455 	if ((provider = fasttrap_provider_lookup(p->p_pid,
456 	    FASTTRAP_PID_NAME, NULL)) != NULL)
457 		fasttrap_provider_retire(provider);
458 
459 	mutex_enter(&p->p_lock);
460 }
461 
462 
463 /*ARGSUSED*/
464 static void
465 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
466 {
467 	/*
468 	 * There are no "default" pid probes.
469 	 */
470 }
471 
472 /*ARGSUSED*/
473 static void
474 fasttrap_provide(void *arg, const dtrace_probedesc_t *desc)
475 {
476 	if (dtrace_probe_lookup(fasttrap_id, NULL, "fasttrap", "fasttrap") == 0)
477 		fasttrap_probe_id = dtrace_probe_create(fasttrap_id, NULL,
478 		    "fasttrap", "fasttrap", FASTTRAP_AFRAMES, NULL);
479 }
480 
481 static int
482 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
483 {
484 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
485 	fasttrap_bucket_t *bucket;
486 	fasttrap_id_t *id;
487 	pid_t pid;
488 	uintptr_t pc;
489 
490 	ASSERT(index < probe->ftp_ntps);
491 
492 	pid = probe->ftp_pid;
493 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
494 	id = &probe->ftp_tps[index].fit_id;
495 
496 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
497 
498 	ASSERT(!(p->p_flag & SVFORK));
499 
500 	/*
501 	 * Before we make any modifications, make sure we've imposed a barrier
502 	 * on the generation in which this probe was last modified.
503 	 */
504 	fasttrap_mod_barrier(probe->ftp_gen);
505 
506 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
507 
508 	/*
509 	 * If the tracepoint has already been enabled, just add our id to the
510 	 * list of interested probes. This may be our second time through
511 	 * this path in which case we'll have constructed the tracepoint we'd
512 	 * like to install. If we can't find a match, and have an allocated
513 	 * tracepoint ready to go, enable that one now.
514 	 *
515 	 * A tracepoint whose proc is defunct is also considered defunct.
516 	 */
517 again:
518 	mutex_enter(&bucket->ftb_mtx);
519 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
520 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
521 		    tp->ftt_proc->ftpc_defunct)
522 			continue;
523 
524 		/*
525 		 * Now that we've found a matching tracepoint, it would be
526 		 * a decent idea to confirm that the tracepoint is still
527 		 * enabled and the trap instruction hasn't been overwritten.
528 		 * Since this is a little hairy, we'll punt for now.
529 		 */
530 
531 		/*
532 		 * This can't be the first interested probe. We don't have
533 		 * to worry about another thread being in the midst of
534 		 * deleting this tracepoint (which would be the only valid
535 		 * reason for a tracepoint to have no interested probes)
536 		 * since we're holding P_PR_LOCK for this process.
537 		 */
538 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
539 
540 		if (probe->ftp_type == DTFTP_RETURN ||
541 		    probe->ftp_type == DTFTP_POST_OFFSETS) {
542 			id->fti_next = tp->ftt_retids;
543 			membar_producer();
544 			tp->ftt_retids = id;
545 			membar_producer();
546 		} else {
547 			id->fti_next = tp->ftt_ids;
548 			membar_producer();
549 			tp->ftt_ids = id;
550 			membar_producer();
551 		}
552 
553 		mutex_exit(&bucket->ftb_mtx);
554 
555 		if (new_tp != NULL) {
556 			new_tp->ftt_ids = NULL;
557 			new_tp->ftt_retids = NULL;
558 		}
559 
560 		return (0);
561 	}
562 
563 	/*
564 	 * If we have a good tracepoint ready to go, install it now while
565 	 * we have the lock held and no one can screw with us.
566 	 */
567 	if (new_tp != NULL) {
568 		int rc = 0;
569 
570 		new_tp->ftt_next = bucket->ftb_data;
571 		membar_producer();
572 		bucket->ftb_data = new_tp;
573 		membar_producer();
574 		mutex_exit(&bucket->ftb_mtx);
575 
576 		/*
577 		 * Activate the tracepoint in the ISA-specific manner.
578 		 * If this fails, we need to report the failure, but
579 		 * indicate that this tracepoint must still be disabled
580 		 * by calling fasttrap_tracepoint_disable().
581 		 */
582 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
583 			rc = FASTTRAP_ENABLE_PARTIAL;
584 
585 		/*
586 		 * Increment the count of the number of tracepoints active in
587 		 * the victim process.
588 		 */
589 		ASSERT(p->p_proc_flag & P_PR_LOCK);
590 		p->p_dtrace_count++;
591 
592 		return (rc);
593 	}
594 
595 	mutex_exit(&bucket->ftb_mtx);
596 
597 	/*
598 	 * Initialize the tracepoint that's been preallocated with the probe.
599 	 */
600 	new_tp = probe->ftp_tps[index].fit_tp;
601 
602 	ASSERT(new_tp->ftt_pid == pid);
603 	ASSERT(new_tp->ftt_pc == pc);
604 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
605 	ASSERT(new_tp->ftt_ids == NULL);
606 	ASSERT(new_tp->ftt_retids == NULL);
607 
608 	if (probe->ftp_type == DTFTP_RETURN ||
609 	    probe->ftp_type == DTFTP_POST_OFFSETS) {
610 		id->fti_next = NULL;
611 		new_tp->ftt_retids = id;
612 	} else {
613 		id->fti_next = NULL;
614 		new_tp->ftt_ids = id;
615 	}
616 
617 	/*
618 	 * If the ISA-dependent initialization goes to plan, go back to the
619 	 * beginning and try to install this freshly made tracepoint.
620 	 */
621 	if (fasttrap_tracepoint_init(p, probe, new_tp, pc) == 0)
622 		goto again;
623 
624 	new_tp->ftt_ids = NULL;
625 	new_tp->ftt_retids = NULL;
626 
627 	return (FASTTRAP_ENABLE_FAIL);
628 }
629 
630 static void
631 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
632 {
633 	fasttrap_bucket_t *bucket;
634 	fasttrap_provider_t *provider = probe->ftp_prov;
635 	fasttrap_tracepoint_t **pp, *tp;
636 	fasttrap_id_t *id, **idp;
637 	pid_t pid;
638 	uintptr_t pc;
639 
640 	ASSERT(index < probe->ftp_ntps);
641 
642 	pid = probe->ftp_pid;
643 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
644 
645 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
646 
647 	/*
648 	 * Find the tracepoint and make sure that our id is one of the
649 	 * ones registered with it.
650 	 */
651 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
652 	mutex_enter(&bucket->ftb_mtx);
653 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
654 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
655 		    tp->ftt_proc == provider->ftp_proc)
656 			break;
657 	}
658 
659 	/*
660 	 * If we somehow lost this tracepoint, we're in a world of hurt.
661 	 */
662 	ASSERT(tp != NULL);
663 
664 	if (probe->ftp_type == DTFTP_RETURN ||
665 	    probe->ftp_type == DTFTP_POST_OFFSETS) {
666 		ASSERT(tp->ftt_retids != NULL);
667 		idp = &tp->ftt_retids;
668 	} else {
669 		ASSERT(tp->ftt_ids != NULL);
670 		idp = &tp->ftt_ids;
671 	}
672 
673 	while ((*idp)->fti_probe != probe) {
674 		idp = &(*idp)->fti_next;
675 		ASSERT(*idp != NULL);
676 	}
677 
678 	id = *idp;
679 	*idp = id->fti_next;
680 	membar_producer();
681 
682 	ASSERT(id->fti_probe == probe);
683 
684 	/*
685 	 * If there are other registered enablings of this tracepoint, we're
686 	 * all done, but if this was the last probe assocated with this
687 	 * this tracepoint, we need to remove and free it.
688 	 */
689 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
690 
691 		/*
692 		 * If the current probe's tracepoint is in use, swap it
693 		 * for an unused tracepoint.
694 		 */
695 		if (tp == probe->ftp_tps[index].fit_tp) {
696 			fasttrap_probe_t *tmp_probe;
697 			fasttrap_tracepoint_t **tmp_tp;
698 			uint_t tmp_index;
699 
700 			if (tp->ftt_ids != NULL) {
701 				tmp_probe = tp->ftt_ids->fti_probe;
702 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
703 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
704 			} else {
705 				tmp_probe = tp->ftt_retids->fti_probe;
706 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
707 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
708 			}
709 
710 			ASSERT(*tmp_tp != NULL);
711 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
712 			ASSERT((*tmp_tp)->ftt_ids == NULL);
713 			ASSERT((*tmp_tp)->ftt_retids == NULL);
714 
715 			probe->ftp_tps[index].fit_tp = *tmp_tp;
716 			*tmp_tp = tp;
717 
718 		}
719 
720 		mutex_exit(&bucket->ftb_mtx);
721 
722 		/*
723 		 * Tag the modified probe with the generation in which it was
724 		 * changed.
725 		 */
726 		probe->ftp_gen = fasttrap_mod_gen;
727 		return;
728 	}
729 
730 	mutex_exit(&bucket->ftb_mtx);
731 
732 	/*
733 	 * We can't safely remove the tracepoint from the set of active
734 	 * tracepoints until we've actually removed the fasttrap instruction
735 	 * from the process's text. We can, however, operate on this
736 	 * tracepoint secure in the knowledge that no other thread is going to
737 	 * be looking at it since we hold P_PR_LOCK on the process if it's
738 	 * live or we hold the provider lock on the process if it's dead and
739 	 * gone.
740 	 */
741 
742 	/*
743 	 * We only need to remove the actual instruction if we're looking
744 	 * at an existing process
745 	 */
746 	if (p != NULL) {
747 		/*
748 		 * If we fail to restore the instruction we need to kill
749 		 * this process since it's in a completely unrecoverable
750 		 * state.
751 		 */
752 		if (fasttrap_tracepoint_remove(p, tp) != 0)
753 			fasttrap_sigtrap(p, NULL, pc);
754 
755 		/*
756 		 * Decrement the count of the number of tracepoints active
757 		 * in the victim process.
758 		 */
759 		ASSERT(p->p_proc_flag & P_PR_LOCK);
760 		p->p_dtrace_count--;
761 	}
762 
763 	/*
764 	 * Remove the probe from the hash table of active tracepoints.
765 	 */
766 	mutex_enter(&bucket->ftb_mtx);
767 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
768 	ASSERT(*pp != NULL);
769 	while (*pp != tp) {
770 		pp = &(*pp)->ftt_next;
771 		ASSERT(*pp != NULL);
772 	}
773 
774 	*pp = tp->ftt_next;
775 	membar_producer();
776 
777 	mutex_exit(&bucket->ftb_mtx);
778 
779 	/*
780 	 * Tag the modified probe with the generation in which it was changed.
781 	 */
782 	probe->ftp_gen = fasttrap_mod_gen;
783 }
784 
785 typedef int fasttrap_probe_f(struct regs *);
786 
787 static void
788 fasttrap_enable_common(int *count, fasttrap_probe_f **fptr, fasttrap_probe_f *f,
789     fasttrap_probe_f **fptr2, fasttrap_probe_f *f2)
790 {
791 	/*
792 	 * We don't have to play the rw lock game here because we're
793 	 * providing something rather than taking something away --
794 	 * we can be sure that no threads have tried to follow this
795 	 * function pointer yet.
796 	 */
797 	mutex_enter(&fasttrap_count_mtx);
798 	if (*count == 0) {
799 		ASSERT(*fptr == NULL);
800 		*fptr = f;
801 		if (fptr2 != NULL)
802 			*fptr2 = f2;
803 	}
804 	ASSERT(*fptr == f);
805 	ASSERT(fptr2 == NULL || *fptr2 == f2);
806 	(*count)++;
807 	mutex_exit(&fasttrap_count_mtx);
808 }
809 
810 static void
811 fasttrap_disable_common(int *count, fasttrap_probe_f **fptr,
812     fasttrap_probe_f **fptr2)
813 {
814 	ASSERT(MUTEX_HELD(&cpu_lock));
815 
816 	mutex_enter(&fasttrap_count_mtx);
817 	(*count)--;
818 	ASSERT(*count >= 0);
819 	if (*count == 0) {
820 		cpu_t *cur, *cpu = CPU;
821 
822 		for (cur = cpu->cpu_next_onln; cur != cpu;
823 			cur = cur->cpu_next_onln) {
824 			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
825 		}
826 
827 		*fptr = NULL;
828 		if (fptr2 != NULL)
829 			*fptr2 = NULL;
830 
831 		for (cur = cpu->cpu_next_onln; cur != cpu;
832 			cur = cur->cpu_next_onln) {
833 			rw_exit(&cur->cpu_ft_lock);
834 		}
835 	}
836 	mutex_exit(&fasttrap_count_mtx);
837 }
838 
839 /*ARGSUSED*/
840 static void
841 fasttrap_enable(void *arg, dtrace_id_t id, void *parg)
842 {
843 	/*
844 	 * Enable the probe that corresponds to statically placed trace
845 	 * points which have not explicitly been placed in the process's text
846 	 * by the fasttrap provider.
847 	 */
848 	ASSERT(arg == NULL);
849 	ASSERT(id == fasttrap_probe_id);
850 
851 	fasttrap_enable_common(&fasttrap_count,
852 	    &dtrace_fasttrap_probe_ptr, fasttrap_probe, NULL, NULL);
853 }
854 
855 
856 /*ARGSUSED*/
857 static void
858 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
859 {
860 	fasttrap_probe_t *probe = parg;
861 	proc_t *p;
862 	int i, rc;
863 
864 	ASSERT(probe != NULL);
865 	ASSERT(!probe->ftp_enabled);
866 	ASSERT(id == probe->ftp_id);
867 	ASSERT(MUTEX_HELD(&cpu_lock));
868 
869 	/*
870 	 * Increment the count of enabled probes on this probe's provider;
871 	 * the provider can't go away while the probe still exists. We
872 	 * must increment this even if we aren't able to properly enable
873 	 * this probe.
874 	 */
875 	mutex_enter(&probe->ftp_prov->ftp_mtx);
876 	probe->ftp_prov->ftp_rcount++;
877 	mutex_exit(&probe->ftp_prov->ftp_mtx);
878 
879 	/*
880 	 * Bail out if we can't find the process for this probe or its
881 	 * provider is retired (meaning it was valid in a previously exec'ed
882 	 * incarnation of this address space). The provider can't go away
883 	 * while we're in this code path.
884 	 */
885 	if (probe->ftp_prov->ftp_retired ||
886 	    (p = sprlock(probe->ftp_pid)) == NULL)
887 		return;
888 
889 	ASSERT(!(p->p_flag & SVFORK));
890 	mutex_exit(&p->p_lock);
891 
892 	/*
893 	 * We have to enable the trap entry before any user threads have
894 	 * the chance to execute the trap instruction we're about to place
895 	 * in their process's text.
896 	 */
897 	fasttrap_enable_common(&fasttrap_pid_count,
898 	    &dtrace_pid_probe_ptr, fasttrap_pid_probe,
899 	    &dtrace_return_probe_ptr, fasttrap_return_probe);
900 
901 	/*
902 	 * Enable all the tracepoints and add this probe's id to each
903 	 * tracepoint's list of active probes.
904 	 */
905 	for (i = 0; i < probe->ftp_ntps; i++) {
906 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
907 			/*
908 			 * If enabling the tracepoint failed completely,
909 			 * we don't have to disable it; if the failure
910 			 * was only partial we must disable it.
911 			 */
912 			if (rc == FASTTRAP_ENABLE_FAIL)
913 				i--;
914 			else
915 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
916 
917 			/*
918 			 * Back up and pull out all the tracepoints we've
919 			 * created so far for this probe.
920 			 */
921 			while (i >= 0) {
922 				fasttrap_tracepoint_disable(p, probe, i);
923 				i--;
924 			}
925 
926 			mutex_enter(&p->p_lock);
927 			sprunlock(p);
928 
929 			/*
930 			 * Since we're not actually enabling this probe,
931 			 * drop our reference on the trap table entry.
932 			 */
933 			fasttrap_disable_common(&fasttrap_pid_count,
934 			    &dtrace_pid_probe_ptr, &dtrace_return_probe_ptr);
935 			return;
936 		}
937 	}
938 
939 	mutex_enter(&p->p_lock);
940 	sprunlock(p);
941 
942 	probe->ftp_enabled = 1;
943 }
944 
945 
946 /*ARGSUSED*/
947 static void
948 fasttrap_disable(void *arg, dtrace_id_t id, void *parg)
949 {
950 	/*
951 	 * Disable the probe the corresponds to statically placed trace
952 	 * points.
953 	 */
954 	ASSERT(arg == NULL);
955 	ASSERT(id == fasttrap_probe_id);
956 	ASSERT(MUTEX_HELD(&cpu_lock));
957 	fasttrap_disable_common(&fasttrap_count, &dtrace_fasttrap_probe_ptr,
958 	    NULL);
959 }
960 
961 /*ARGSUSED*/
962 static void
963 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
964 {
965 	fasttrap_probe_t *probe = parg;
966 	fasttrap_provider_t *provider = probe->ftp_prov;
967 	proc_t *p;
968 	int i, whack = 0;
969 
970 	if (!probe->ftp_enabled) {
971 		mutex_enter(&provider->ftp_mtx);
972 		provider->ftp_rcount--;
973 		ASSERT(provider->ftp_rcount >= 0);
974 		mutex_exit(&provider->ftp_mtx);
975 		return;
976 	}
977 
978 	ASSERT(id == probe->ftp_id);
979 
980 	/*
981 	 * We won't be able to acquire a /proc-esque lock on the process
982 	 * iff the process is dead and gone. In this case, we rely on the
983 	 * provider lock as a point of mutual exclusion to prevent other
984 	 * DTrace consumers from disabling this probe.
985 	 */
986 	if ((p = sprlock(probe->ftp_pid)) != NULL) {
987 		ASSERT(!(p->p_flag & SVFORK));
988 		mutex_exit(&p->p_lock);
989 	}
990 
991 	mutex_enter(&provider->ftp_mtx);
992 
993 	/*
994 	 * Disable all the associated tracepoints.
995 	 */
996 	for (i = 0; i < probe->ftp_ntps; i++) {
997 		fasttrap_tracepoint_disable(p, probe, i);
998 	}
999 
1000 	ASSERT(provider->ftp_rcount > 0);
1001 	provider->ftp_rcount--;
1002 
1003 	if (p != NULL) {
1004 		/*
1005 		 * Even though we may not be able to remove it entirely, we
1006 		 * mark this retired provider to get a chance to remove some
1007 		 * of the associated probes.
1008 		 */
1009 		if (provider->ftp_retired && !provider->ftp_marked)
1010 			whack = provider->ftp_marked = 1;
1011 		mutex_exit(&provider->ftp_mtx);
1012 
1013 		mutex_enter(&p->p_lock);
1014 		sprunlock(p);
1015 	} else {
1016 		/*
1017 		 * If the process is dead, we're just waiting for the
1018 		 * last probe to be disabled to be able to free it.
1019 		 */
1020 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1021 			whack = provider->ftp_marked = 1;
1022 		mutex_exit(&provider->ftp_mtx);
1023 	}
1024 
1025 	if (whack)
1026 		fasttrap_pid_cleanup();
1027 
1028 	probe->ftp_enabled = 0;
1029 
1030 	ASSERT(MUTEX_HELD(&cpu_lock));
1031 	fasttrap_disable_common(&fasttrap_pid_count, &dtrace_pid_probe_ptr,
1032 	    &dtrace_return_probe_ptr);
1033 }
1034 
1035 /*ARGSUSED*/
1036 static void
1037 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1038     dtrace_argdesc_t *desc)
1039 {
1040 	fasttrap_probe_t *probe = parg;
1041 	char *str;
1042 	int i;
1043 
1044 	desc->dtargd_native[0] = '\0';
1045 	desc->dtargd_xlate[0] = '\0';
1046 
1047 	if (probe->ftp_prov->ftp_retired != 0 ||
1048 	    desc->dtargd_ndx >= probe->ftp_nargs) {
1049 		desc->dtargd_ndx = DTRACE_ARGNONE;
1050 		return;
1051 	}
1052 
1053 	/*
1054 	 * We only need to set this member if the argument is remapped.
1055 	 */
1056 	if (probe->ftp_argmap != NULL)
1057 		desc->dtargd_mapping = probe->ftp_argmap[desc->dtargd_ndx];
1058 
1059 	str = probe->ftp_ntypes;
1060 	for (i = 0; i < desc->dtargd_mapping; i++) {
1061 		str += strlen(str) + 1;
1062 	}
1063 
1064 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1065 	(void) strcpy(desc->dtargd_native, str);
1066 
1067 	if (probe->ftp_xtypes == NULL)
1068 		return;
1069 
1070 	str = probe->ftp_xtypes;
1071 	for (i = 0; i < desc->dtargd_ndx; i++) {
1072 		str += strlen(str) + 1;
1073 	}
1074 
1075 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1076 	(void) strcpy(desc->dtargd_xlate, str);
1077 }
1078 
1079 /*ARGSUSED*/
1080 static void
1081 fasttrap_destroy(void *arg, dtrace_id_t id, void *parg)
1082 {
1083 	ASSERT(arg == NULL);
1084 	ASSERT(id == fasttrap_probe_id);
1085 }
1086 
1087 /*ARGSUSED*/
1088 static void
1089 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1090 {
1091 	fasttrap_probe_t *probe = parg;
1092 	int i;
1093 	size_t size;
1094 
1095 	ASSERT(probe != NULL);
1096 	ASSERT(!probe->ftp_enabled);
1097 	ASSERT(fasttrap_total >= probe->ftp_ntps);
1098 
1099 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1100 	size = sizeof (fasttrap_probe_t) +
1101 	    sizeof (probe->ftp_tps[0]) * (probe->ftp_ntps - 1);
1102 
1103 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1104 		fasttrap_mod_barrier(probe->ftp_gen);
1105 
1106 	for (i = 0; i < probe->ftp_ntps; i++) {
1107 		kmem_free(probe->ftp_tps[i].fit_tp,
1108 		    sizeof (fasttrap_tracepoint_t));
1109 	}
1110 
1111 	kmem_free(probe, size);
1112 }
1113 
1114 
1115 static const dtrace_pattr_t fasttrap_attr = {
1116 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1117 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1118 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1119 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1120 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1121 };
1122 
1123 static dtrace_pops_t fasttrap_pops = {
1124 	fasttrap_provide,
1125 	NULL,
1126 	fasttrap_enable,
1127 	fasttrap_disable,
1128 	NULL,
1129 	NULL,
1130 	NULL,
1131 	fasttrap_getarg,
1132 	NULL,
1133 	fasttrap_destroy
1134 };
1135 
1136 static const dtrace_pattr_t pid_attr = {
1137 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1138 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1139 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1140 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1141 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1142 };
1143 
1144 static dtrace_pops_t pid_pops = {
1145 	fasttrap_pid_provide,
1146 	NULL,
1147 	fasttrap_pid_enable,
1148 	fasttrap_pid_disable,
1149 	NULL,
1150 	NULL,
1151 	fasttrap_pid_getargdesc,
1152 	fasttrap_getarg,
1153 	NULL,
1154 	fasttrap_pid_destroy
1155 };
1156 
1157 static dtrace_pops_t usdt_pops = {
1158 	fasttrap_pid_provide,
1159 	NULL,
1160 	fasttrap_pid_enable,
1161 	fasttrap_pid_disable,
1162 	NULL,
1163 	NULL,
1164 	fasttrap_pid_getargdesc,
1165 	fasttrap_usdt_getarg,
1166 	NULL,
1167 	fasttrap_pid_destroy
1168 };
1169 
1170 static fasttrap_proc_t *
1171 fasttrap_proc_lookup(pid_t pid)
1172 {
1173 	fasttrap_bucket_t *bucket;
1174 	fasttrap_proc_t *fprc, *new_fprc;
1175 
1176 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1177 	mutex_enter(&bucket->ftb_mtx);
1178 
1179 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1180 		if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) {
1181 			mutex_enter(&fprc->ftpc_mtx);
1182 			mutex_exit(&bucket->ftb_mtx);
1183 			fprc->ftpc_count++;
1184 			mutex_exit(&fprc->ftpc_mtx);
1185 
1186 			return (fprc);
1187 		}
1188 	}
1189 
1190 	/*
1191 	 * Drop the bucket lock so we don't try to perform a sleeping
1192 	 * allocation under it.
1193 	 */
1194 	mutex_exit(&bucket->ftb_mtx);
1195 
1196 	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1197 	new_fprc->ftpc_pid = pid;
1198 	new_fprc->ftpc_count = 1;
1199 
1200 	mutex_enter(&bucket->ftb_mtx);
1201 
1202 	/*
1203 	 * Take another lap through the list to make sure a proc hasn't
1204 	 * been created for this pid while we weren't under the bucket lock.
1205 	 */
1206 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1207 		if (fprc->ftpc_pid == pid && !fprc->ftpc_defunct) {
1208 			mutex_enter(&fprc->ftpc_mtx);
1209 			mutex_exit(&bucket->ftb_mtx);
1210 			fprc->ftpc_count++;
1211 			mutex_exit(&fprc->ftpc_mtx);
1212 
1213 			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1214 
1215 			return (fprc);
1216 		}
1217 	}
1218 
1219 	new_fprc->ftpc_next = bucket->ftb_data;
1220 	bucket->ftb_data = new_fprc;
1221 
1222 	mutex_exit(&bucket->ftb_mtx);
1223 
1224 	return (new_fprc);
1225 }
1226 
1227 static void
1228 fasttrap_proc_release(fasttrap_proc_t *proc)
1229 {
1230 	fasttrap_bucket_t *bucket;
1231 	fasttrap_proc_t *fprc, **fprcp;
1232 	pid_t pid = proc->ftpc_pid;
1233 
1234 	mutex_enter(&proc->ftpc_mtx);
1235 
1236 	ASSERT(proc->ftpc_count != 0);
1237 
1238 	if (--proc->ftpc_count != 0) {
1239 		mutex_exit(&proc->ftpc_mtx);
1240 		return;
1241 	}
1242 
1243 	mutex_exit(&proc->ftpc_mtx);
1244 
1245 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1246 	mutex_enter(&bucket->ftb_mtx);
1247 
1248 	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1249 	while ((fprc = *fprcp) != NULL) {
1250 		if (fprc == proc)
1251 			break;
1252 
1253 		fprcp = &fprc->ftpc_next;
1254 	}
1255 
1256 	/*
1257 	 * Something strange has happened if we can't find the proc.
1258 	 */
1259 	ASSERT(fprc != NULL);
1260 
1261 	*fprcp = fprc->ftpc_next;
1262 
1263 	mutex_exit(&bucket->ftb_mtx);
1264 
1265 	kmem_free(fprc, sizeof (fasttrap_proc_t));
1266 }
1267 
1268 /*
1269  * Lookup a fasttrap-managed provider based on its name and associated pid.
1270  * If the pattr argument is non-NULL, this function instantiates the provider
1271  * if it doesn't exist otherwise it returns NULL. The provider is returned
1272  * with its lock held.
1273  */
1274 static fasttrap_provider_t *
1275 fasttrap_provider_lookup(pid_t pid, const char *name,
1276     const dtrace_pattr_t *pattr)
1277 {
1278 	fasttrap_provider_t *fp, *new_fp = NULL;
1279 	fasttrap_bucket_t *bucket;
1280 	char provname[DTRACE_PROVNAMELEN];
1281 	proc_t *p;
1282 	uid_t uid = (uid_t)-1;
1283 
1284 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1285 
1286 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1287 	mutex_enter(&bucket->ftb_mtx);
1288 
1289 	/*
1290 	 * Take a lap through the list and return the match if we find it.
1291 	 */
1292 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1293 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1294 		    !fp->ftp_retired) {
1295 			mutex_enter(&fp->ftp_mtx);
1296 			mutex_exit(&bucket->ftb_mtx);
1297 			return (fp);
1298 		}
1299 	}
1300 
1301 	/*
1302 	 * Drop the bucket lock so we don't try to perform a sleeping
1303 	 * allocation under it.
1304 	 */
1305 	mutex_exit(&bucket->ftb_mtx);
1306 
1307 	/*
1308 	 * If we didn't want to create a new provider, just return failure.
1309 	 */
1310 	if (pattr == NULL)
1311 		return (NULL);
1312 
1313 	/*
1314 	 * Make sure the process exists, isn't a child created as the result
1315 	 * of a vfork(2), and isn't a zombie (but may be in fork). Record the
1316 	 * process's uid to pass to dtrace_register().
1317 	 */
1318 	mutex_enter(&pidlock);
1319 	if ((p = prfind(pid)) == NULL) {
1320 		mutex_exit(&pidlock);
1321 		return (NULL);
1322 	}
1323 	mutex_enter(&p->p_lock);
1324 	mutex_exit(&pidlock);
1325 	if (p->p_flag & (SVFORK | SEXITING)) {
1326 		mutex_exit(&p->p_lock);
1327 		return (NULL);
1328 	}
1329 
1330 	/*
1331 	 * Increment p_dtrace_probes so that the process knows to inform us
1332 	 * when it exits or execs. fasttrap_provider_free() decrements this
1333 	 * when we're done with this provider.
1334 	 */
1335 	p->p_dtrace_probes++;
1336 
1337 	mutex_enter(&p->p_crlock);
1338 	uid = crgetruid(p->p_cred);
1339 	mutex_exit(&p->p_crlock);
1340 	mutex_exit(&p->p_lock);
1341 
1342 	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1343 	new_fp->ftp_pid = pid;
1344 	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1345 
1346 	ASSERT(new_fp->ftp_proc != NULL);
1347 
1348 	mutex_enter(&bucket->ftb_mtx);
1349 
1350 	/*
1351 	 * Take another lap through the list to make sure a provider hasn't
1352 	 * been created for this pid while we weren't under the bucket lock.
1353 	 */
1354 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1355 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1356 		    !fp->ftp_retired) {
1357 			mutex_enter(&fp->ftp_mtx);
1358 			mutex_exit(&bucket->ftb_mtx);
1359 			fasttrap_provider_free(new_fp);
1360 			return (fp);
1361 		}
1362 	}
1363 
1364 	(void) strcpy(new_fp->ftp_name, name);
1365 
1366 	/*
1367 	 * Fail and return NULL if either the provider name is too long
1368 	 * or we fail to register this new provider with the DTrace
1369 	 * framework. Note that this is the only place we ever construct
1370 	 * the full provider name -- we keep it in pieces in the provider
1371 	 * structure.
1372 	 */
1373 	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1374 	    sizeof (provname) ||
1375 	    dtrace_register(provname, pattr,
1376 	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER, uid,
1377 	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1378 	    &new_fp->ftp_provid) != 0) {
1379 		mutex_exit(&bucket->ftb_mtx);
1380 		fasttrap_provider_free(new_fp);
1381 		return (NULL);
1382 	}
1383 
1384 	new_fp->ftp_next = bucket->ftb_data;
1385 	bucket->ftb_data = new_fp;
1386 
1387 	mutex_enter(&new_fp->ftp_mtx);
1388 	mutex_exit(&bucket->ftb_mtx);
1389 
1390 	return (new_fp);
1391 }
1392 
1393 static void
1394 fasttrap_provider_free(fasttrap_provider_t *provider)
1395 {
1396 	pid_t pid = provider->ftp_pid;
1397 	proc_t *p;
1398 
1399 	/*
1400 	 * There need to be no consumers using this provider and no
1401 	 * associated enabled probes.
1402 	 */
1403 	ASSERT(provider->ftp_ccount == 0);
1404 	ASSERT(provider->ftp_rcount == 0);
1405 
1406 	fasttrap_proc_release(provider->ftp_proc);
1407 
1408 	kmem_free(provider, sizeof (fasttrap_provider_t));
1409 
1410 	/*
1411 	 * Decrement p_dtrace_probes on the process whose provider we're
1412 	 * freeing. We don't have to worry about clobbering somone else's
1413 	 * modifications to it because we have locked the bucket that
1414 	 * corresponds to this process's hash chain in the provider hash
1415 	 * table. Don't sweat it if we can't find the process.
1416 	 */
1417 	mutex_enter(&pidlock);
1418 	if ((p = prfind(pid)) == NULL) {
1419 		mutex_exit(&pidlock);
1420 		return;
1421 	}
1422 
1423 	mutex_enter(&p->p_lock);
1424 	mutex_exit(&pidlock);
1425 
1426 	p->p_dtrace_probes--;
1427 	mutex_exit(&p->p_lock);
1428 }
1429 
1430 static void
1431 fasttrap_provider_retire(fasttrap_provider_t *provider)
1432 {
1433 	dtrace_provider_id_t provid = provider->ftp_provid;
1434 
1435 	/*
1436 	 * Mark the provider to be removed in our post-processing step,
1437 	 * mark it retired, and mark its proc as defunct (though it may
1438 	 * already be marked defunct by another provider that shares the
1439 	 * same proc). Marking it indicates that we should try to remove it;
1440 	 * setting the retired flag indicates that we're done with this
1441 	 * provider; setting the proc to be defunct indicates that all
1442 	 * tracepoints associated with the traced process should be ignored.
1443 	 */
1444 	provider->ftp_proc->ftpc_defunct = 1;
1445 	provider->ftp_retired = 1;
1446 	provider->ftp_marked = 1;
1447 	mutex_exit(&provider->ftp_mtx);
1448 
1449 	/*
1450 	 * We don't have to worry about invalidating the same provider twice
1451 	 * since fasttrap_provider_lookup() will ignore provider that have
1452 	 * been marked as retired.
1453 	 */
1454 	dtrace_invalidate(provid);
1455 
1456 	fasttrap_pid_cleanup();
1457 }
1458 
1459 static int
1460 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1461 {
1462 	fasttrap_provider_t *provider;
1463 	fasttrap_probe_t *pp;
1464 	fasttrap_tracepoint_t *tp;
1465 	char *name;
1466 	size_t size;
1467 	int i, aframes, whack;
1468 
1469 	switch (pdata->ftps_type) {
1470 	case DTFTP_ENTRY:
1471 		name = "entry";
1472 		aframes = FASTTRAP_ENTRY_AFRAMES;
1473 		break;
1474 	case DTFTP_RETURN:
1475 		name = "return";
1476 		aframes = FASTTRAP_RETURN_AFRAMES;
1477 		break;
1478 	case DTFTP_OFFSETS:
1479 		name = NULL;
1480 		break;
1481 	default:
1482 		return (EINVAL);
1483 	}
1484 
1485 	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1486 	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1487 		return (ESRCH);
1488 
1489 	/*
1490 	 * Increment this reference count to indicate that a consumer is
1491 	 * actively adding a new probe associated with this provider.
1492 	 */
1493 	provider->ftp_ccount++;
1494 	mutex_exit(&provider->ftp_mtx);
1495 
1496 	if (name != NULL) {
1497 		if (dtrace_probe_lookup(provider->ftp_provid,
1498 		    pdata->ftps_mod, pdata->ftps_func, name) != 0)
1499 			goto done;
1500 
1501 		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1502 
1503 		if (fasttrap_total > fasttrap_max) {
1504 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1505 			goto no_mem;
1506 		}
1507 
1508 		ASSERT(pdata->ftps_noffs > 0);
1509 		size = sizeof (fasttrap_probe_t) +
1510 		    sizeof (pp->ftp_tps[0]) * (pdata->ftps_noffs - 1);
1511 
1512 		pp = kmem_zalloc(size, KM_SLEEP);
1513 
1514 		pp->ftp_prov = provider;
1515 		pp->ftp_faddr = pdata->ftps_pc;
1516 		pp->ftp_fsize = pdata->ftps_size;
1517 		pp->ftp_pid = pdata->ftps_pid;
1518 		pp->ftp_ntps = pdata->ftps_noffs;
1519 		pp->ftp_type = pdata->ftps_type;
1520 
1521 		for (i = 0; i < pdata->ftps_noffs; i++) {
1522 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1523 			    KM_SLEEP);
1524 
1525 			tp->ftt_proc = provider->ftp_proc;
1526 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1527 			tp->ftt_pid = pdata->ftps_pid;
1528 
1529 			pp->ftp_tps[i].fit_tp = tp;
1530 			pp->ftp_tps[i].fit_id.fti_probe = pp;
1531 		}
1532 
1533 		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1534 		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1535 	} else {
1536 		for (i = 0; i < pdata->ftps_noffs; i++) {
1537 			char name_str[17];
1538 
1539 			(void) sprintf(name_str, "%llx",
1540 			    (unsigned long long)pdata->ftps_offs[i]);
1541 
1542 			if (dtrace_probe_lookup(provider->ftp_provid,
1543 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1544 				continue;
1545 
1546 			atomic_add_32(&fasttrap_total, 1);
1547 
1548 			if (fasttrap_total > fasttrap_max) {
1549 				atomic_add_32(&fasttrap_total, -1);
1550 				goto no_mem;
1551 			}
1552 
1553 			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1554 
1555 			pp->ftp_prov = provider;
1556 			pp->ftp_faddr = pdata->ftps_pc;
1557 			pp->ftp_fsize = pdata->ftps_size;
1558 			pp->ftp_pid = pdata->ftps_pid;
1559 			pp->ftp_ntps = 1;
1560 			pp->ftp_type = pdata->ftps_type;
1561 
1562 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1563 			    KM_SLEEP);
1564 
1565 			tp->ftt_proc = provider->ftp_proc;
1566 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1567 			tp->ftt_pid = pdata->ftps_pid;
1568 
1569 			pp->ftp_tps[0].fit_tp = tp;
1570 			pp->ftp_tps[0].fit_id.fti_probe = pp;
1571 
1572 			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1573 			    pdata->ftps_mod, pdata->ftps_func, name_str,
1574 			    FASTTRAP_OFFSET_AFRAMES, pp);
1575 		}
1576 	}
1577 
1578 done:
1579 	/*
1580 	 * We know that the provider is still valid since we incremented the
1581 	 * reference count. If someone tried to free this provider while we
1582 	 * were using it (e.g. because the process called exec(2) or exit(2)),
1583 	 * take note of that and try to free it now.
1584 	 */
1585 	mutex_enter(&provider->ftp_mtx);
1586 	provider->ftp_ccount--;
1587 	whack = provider->ftp_retired;
1588 	mutex_exit(&provider->ftp_mtx);
1589 
1590 	if (whack)
1591 		fasttrap_pid_cleanup();
1592 
1593 	return (0);
1594 
1595 no_mem:
1596 	/*
1597 	 * If we've exhausted the allowable resources, we'll try to remove
1598 	 * this provider to free some up. This is to cover the case where
1599 	 * the user has accidentally created many more probes than was
1600 	 * intended (e.g. pid123:::).
1601 	 */
1602 	mutex_enter(&provider->ftp_mtx);
1603 	provider->ftp_ccount--;
1604 	provider->ftp_marked = 1;
1605 	mutex_exit(&provider->ftp_mtx);
1606 
1607 	fasttrap_pid_cleanup();
1608 
1609 	return (ENOMEM);
1610 }
1611 
1612 /*ARGSUSED*/
1613 static void *
1614 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1615 {
1616 	fasttrap_provider_t *provider;
1617 
1618 	/*
1619 	 * A 32-bit unsigned integer (like a pid for example) can be
1620 	 * expressed in 10 or fewer decimal digits. Make sure that we'll
1621 	 * have enough space for the provider name.
1622 	 */
1623 	if (strlen(dhpv->dthpv_provname) + 10 >=
1624 	    sizeof (provider->ftp_name)) {
1625 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
1626 		    "name too long to accomodate pid", dhpv->dthpv_provname);
1627 		return (NULL);
1628 	}
1629 
1630 	/*
1631 	 * Don't let folks spoof the true pid provider.
1632 	 */
1633 	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1634 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
1635 		    "%s is an invalid name", dhpv->dthpv_provname,
1636 		    FASTTRAP_PID_NAME);
1637 		return (NULL);
1638 	}
1639 
1640 	/*
1641 	 * The highest stability class that fasttrap supports is ISA; cap
1642 	 * the stability of the new provider accordingly.
1643 	 */
1644 	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class >= DTRACE_CLASS_COMMON)
1645 		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1646 	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class >= DTRACE_CLASS_COMMON)
1647 		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1648 	if (dhpv->dthpv_pattr.dtpa_func.dtat_class >= DTRACE_CLASS_COMMON)
1649 		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1650 	if (dhpv->dthpv_pattr.dtpa_name.dtat_class >= DTRACE_CLASS_COMMON)
1651 		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1652 	if (dhpv->dthpv_pattr.dtpa_args.dtat_class >= DTRACE_CLASS_COMMON)
1653 		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1654 
1655 	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1656 	    &dhpv->dthpv_pattr)) == NULL) {
1657 		cmn_err(CE_WARN, "failed to instantiate provider %s for "
1658 		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
1659 		return (NULL);
1660 	}
1661 
1662 	/*
1663 	 * We elevate the consumer count here to ensure that this provider
1664 	 * isn't removed until after the meta provider has been told to
1665 	 * remove it.
1666 	 */
1667 	provider->ftp_ccount++;
1668 
1669 	mutex_exit(&provider->ftp_mtx);
1670 
1671 	return (provider);
1672 }
1673 
1674 /*ARGSUSED*/
1675 static void
1676 fasttrap_meta_create_probe(void *arg, void *parg,
1677     dtrace_helper_probedesc_t *dhpb)
1678 {
1679 	fasttrap_provider_t *provider = parg;
1680 	fasttrap_probe_t *pp;
1681 	fasttrap_tracepoint_t *tp;
1682 	size_t size;
1683 	int i;
1684 
1685 	mutex_enter(&provider->ftp_mtx);
1686 
1687 	if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1688 	    dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1689 		mutex_exit(&provider->ftp_mtx);
1690 		return;
1691 	}
1692 
1693 	atomic_add_32(&fasttrap_total, dhpb->dthpb_noffs);
1694 
1695 	if (fasttrap_total > fasttrap_max) {
1696 		atomic_add_32(&fasttrap_total, -dhpb->dthpb_noffs);
1697 		mutex_exit(&provider->ftp_mtx);
1698 		return;
1699 	}
1700 
1701 	size = sizeof (fasttrap_probe_t) +
1702 	    sizeof (pp->ftp_tps[0]) * (dhpb->dthpb_noffs - 1);
1703 	pp = kmem_zalloc(size, KM_SLEEP);
1704 
1705 	pp->ftp_prov = provider;
1706 	pp->ftp_pid = provider->ftp_pid;
1707 	pp->ftp_ntps = dhpb->dthpb_noffs;
1708 #ifdef __sparc
1709 	pp->ftp_type = DTFTP_POST_OFFSETS;
1710 #else
1711 	pp->ftp_type = DTFTP_OFFSETS;
1712 #endif
1713 	pp->ftp_nargs = dhpb->dthpb_xargc;
1714 	pp->ftp_xtypes = dhpb->dthpb_xtypes;
1715 	pp->ftp_ntypes = dhpb->dthpb_ntypes;
1716 
1717 	for (i = 0; i < pp->ftp_ntps; i++) {
1718 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1719 
1720 		tp->ftt_proc = provider->ftp_proc;
1721 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1722 		tp->ftt_pid = provider->ftp_pid;
1723 
1724 		pp->ftp_tps[i].fit_tp = tp;
1725 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1726 	}
1727 
1728 	/*
1729 	 * If the arguments are shuffled around we set the argument remapping
1730 	 * table. Later, when the probe fires, we only remap the arguments
1731 	 * if the table is non-NULL.
1732 	 */
1733 	for (i = 0; i < dhpb->dthpb_xargc; i++) {
1734 		if (dhpb->dthpb_args[i] != i) {
1735 			pp->ftp_argmap = dhpb->dthpb_args;
1736 			break;
1737 		}
1738 	}
1739 
1740 	/*
1741 	 * The probe is fully constructed -- register it with DTrace.
1742 	 */
1743 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1744 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1745 
1746 	mutex_exit(&provider->ftp_mtx);
1747 }
1748 
1749 /*ARGSUSED*/
1750 static void
1751 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1752 {
1753 	fasttrap_provider_t *provider;
1754 
1755 	if ((provider = fasttrap_provider_lookup(pid,
1756 	    dhpv->dthpv_provname, NULL)) != NULL) {
1757 		/*
1758 		 * Drop the consumer count now that we're done with this
1759 		 * provider. If there are no other consumers retire it now.
1760 		 */
1761 		if (--provider->ftp_ccount == 0)
1762 			fasttrap_provider_retire(provider);
1763 		else
1764 			mutex_exit(&provider->ftp_mtx);
1765 	}
1766 }
1767 
1768 static dtrace_mops_t fasttrap_mops = {
1769 	fasttrap_meta_create_probe,
1770 	fasttrap_meta_provide,
1771 	fasttrap_meta_remove
1772 };
1773 
1774 /*ARGSUSED*/
1775 static int
1776 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1777 {
1778 	return (0);
1779 }
1780 
1781 /*ARGSUSED*/
1782 static int
1783 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1784 {
1785 	if (!dtrace_attached())
1786 		return (EAGAIN);
1787 
1788 	if (cmd == FASTTRAPIOC_MAKEPROBE) {
1789 		fasttrap_probe_spec_t *uprobe = (void *)arg;
1790 		fasttrap_probe_spec_t *probe;
1791 		uint64_t noffs;
1792 		size_t size;
1793 		int ret;
1794 		char *c;
1795 
1796 		if (copyin(&uprobe->ftps_noffs, &noffs,
1797 		    sizeof (uprobe->ftps_noffs)))
1798 			return (EFAULT);
1799 
1800 		/*
1801 		 * Probes must have at least one tracepoint.
1802 		 */
1803 		if (noffs == 0)
1804 			return (EINVAL);
1805 
1806 		size = sizeof (fasttrap_probe_spec_t) +
1807 		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
1808 
1809 		if (size > 1024 * 1024)
1810 			return (ENOMEM);
1811 
1812 		probe = kmem_alloc(size, KM_SLEEP);
1813 
1814 		if (copyin(uprobe, probe, size) != 0) {
1815 			kmem_free(probe, size);
1816 			return (EFAULT);
1817 		}
1818 
1819 		/*
1820 		 * Verify that the function and module strings contain no
1821 		 * funny characters.
1822 		 */
1823 		for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
1824 			if (*c < 0x20 || 0x7f <= *c) {
1825 				ret = EINVAL;
1826 				goto err;
1827 			}
1828 		}
1829 
1830 		for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
1831 			if (*c < 0x20 || 0x7f <= *c) {
1832 				ret = EINVAL;
1833 				goto err;
1834 			}
1835 		}
1836 
1837 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1838 			proc_t *p;
1839 			pid_t pid = probe->ftps_pid;
1840 
1841 			mutex_enter(&pidlock);
1842 			/*
1843 			 * Report an error if the process doesn't exist
1844 			 * or is actively being birthed.
1845 			 */
1846 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1847 				mutex_exit(&pidlock);
1848 				return (ESRCH);
1849 			}
1850 			mutex_enter(&p->p_lock);
1851 			mutex_exit(&pidlock);
1852 
1853 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
1854 			    VREAD | VWRITE)) != 0) {
1855 				mutex_exit(&p->p_lock);
1856 				return (ret);
1857 			}
1858 
1859 			mutex_exit(&p->p_lock);
1860 		}
1861 
1862 		ret = fasttrap_add_probe(probe);
1863 err:
1864 		kmem_free(probe, size);
1865 
1866 		return (ret);
1867 
1868 	} else if (cmd == FASTTRAPIOC_GETINSTR) {
1869 		fasttrap_instr_query_t instr;
1870 		fasttrap_tracepoint_t *tp;
1871 		uint_t index;
1872 		int ret;
1873 
1874 		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
1875 			return (EFAULT);
1876 
1877 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1878 			proc_t *p;
1879 			pid_t pid = instr.ftiq_pid;
1880 
1881 			mutex_enter(&pidlock);
1882 			/*
1883 			 * Report an error if the process doesn't exist
1884 			 * or is actively being birthed.
1885 			 */
1886 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1887 				mutex_exit(&pidlock);
1888 				return (ESRCH);
1889 			}
1890 			mutex_enter(&p->p_lock);
1891 			mutex_exit(&pidlock);
1892 
1893 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
1894 			    VREAD)) != 0) {
1895 				mutex_exit(&p->p_lock);
1896 				return (ret);
1897 			}
1898 
1899 			mutex_exit(&p->p_lock);
1900 		}
1901 
1902 		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
1903 
1904 		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
1905 		tp = fasttrap_tpoints.fth_table[index].ftb_data;
1906 		while (tp != NULL) {
1907 			if (instr.ftiq_pid == tp->ftt_pid &&
1908 			    instr.ftiq_pc == tp->ftt_pc &&
1909 			    !tp->ftt_proc->ftpc_defunct)
1910 				break;
1911 
1912 			tp = tp->ftt_next;
1913 		}
1914 
1915 		if (tp == NULL) {
1916 			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
1917 			return (ENOENT);
1918 		}
1919 
1920 		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
1921 		    sizeof (instr.ftiq_instr));
1922 		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
1923 
1924 		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
1925 			return (EFAULT);
1926 
1927 		return (0);
1928 	}
1929 
1930 	return (EINVAL);
1931 }
1932 
1933 static struct cb_ops fasttrap_cb_ops = {
1934 	fasttrap_open,		/* open */
1935 	nodev,			/* close */
1936 	nulldev,		/* strategy */
1937 	nulldev,		/* print */
1938 	nodev,			/* dump */
1939 	nodev,			/* read */
1940 	nodev,			/* write */
1941 	fasttrap_ioctl,		/* ioctl */
1942 	nodev,			/* devmap */
1943 	nodev,			/* mmap */
1944 	nodev,			/* segmap */
1945 	nochpoll,		/* poll */
1946 	ddi_prop_op,		/* cb_prop_op */
1947 	0,			/* streamtab  */
1948 	D_NEW | D_MP		/* Driver compatibility flag */
1949 };
1950 
1951 /*ARGSUSED*/
1952 static int
1953 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1954 {
1955 	int error;
1956 
1957 	switch (infocmd) {
1958 	case DDI_INFO_DEVT2DEVINFO:
1959 		*result = (void *)fasttrap_devi;
1960 		error = DDI_SUCCESS;
1961 		break;
1962 	case DDI_INFO_DEVT2INSTANCE:
1963 		*result = (void *)0;
1964 		error = DDI_SUCCESS;
1965 		break;
1966 	default:
1967 		error = DDI_FAILURE;
1968 	}
1969 	return (error);
1970 }
1971 
1972 static int
1973 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1974 {
1975 	ulong_t nent;
1976 
1977 	switch (cmd) {
1978 	case DDI_ATTACH:
1979 		break;
1980 	case DDI_RESUME:
1981 		return (DDI_SUCCESS);
1982 	default:
1983 		return (DDI_FAILURE);
1984 	}
1985 
1986 	if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
1987 	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
1988 	    dtrace_register("fasttrap", &fasttrap_attr, DTRACE_PRIV_USER, 0,
1989 	    &fasttrap_pops, NULL, &fasttrap_id) != 0) {
1990 		ddi_remove_minor_node(devi, NULL);
1991 		return (DDI_FAILURE);
1992 	}
1993 
1994 	ddi_report_dev(devi);
1995 	fasttrap_devi = devi;
1996 
1997 	/*
1998 	 * Install our hooks into fork(2), exec(2), and exit(2).
1999 	 */
2000 	dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2001 	dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2002 	dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2003 
2004 	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2005 	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2006 	fasttrap_total = 0;
2007 
2008 	/*
2009 	 * Conjure up the tracepoints hashtable...
2010 	 */
2011 	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2012 	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2013 
2014 	if (nent <= 0 || nent > 0x1000000)
2015 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2016 
2017 	if ((nent & (nent - 1)) == 0)
2018 		fasttrap_tpoints.fth_nent = nent;
2019 	else
2020 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2021 	ASSERT(fasttrap_tpoints.fth_nent > 0);
2022 	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2023 	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2024 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2025 
2026 	/*
2027 	 * ... and the providers hash table...
2028 	 */
2029 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2030 	if ((nent & (nent - 1)) == 0)
2031 		fasttrap_provs.fth_nent = nent;
2032 	else
2033 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2034 	ASSERT(fasttrap_provs.fth_nent > 0);
2035 	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2036 	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2037 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2038 
2039 	/*
2040 	 * ... and the procs hash table.
2041 	 */
2042 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2043 	if ((nent & (nent - 1)) == 0)
2044 		fasttrap_procs.fth_nent = nent;
2045 	else
2046 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2047 	ASSERT(fasttrap_procs.fth_nent > 0);
2048 	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2049 	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2050 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2051 
2052 	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2053 	    &fasttrap_meta_id);
2054 
2055 	return (DDI_SUCCESS);
2056 }
2057 
2058 static int
2059 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2060 {
2061 	int i, fail = 0;
2062 	timeout_id_t tmp;
2063 
2064 	switch (cmd) {
2065 	case DDI_DETACH:
2066 		break;
2067 	case DDI_SUSPEND:
2068 		return (DDI_SUCCESS);
2069 	default:
2070 		return (DDI_FAILURE);
2071 	}
2072 
2073 	/*
2074 	 * Unregister the meta-provider to make sure no new fasttrap-
2075 	 * managed providers come along while we're trying to close up
2076 	 * shop. If we fail to detach, we'll need to re-register as a
2077 	 * meta-provider. We can fail to unregister as a meta-provider
2078 	 * if providers we manage still exist.
2079 	 */
2080 	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2081 	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
2082 		return (DDI_FAILURE);
2083 
2084 	/*
2085 	 * Prevent any new timeouts from running by setting fasttrap_timeout
2086 	 * to a non-zero value, and wait for the current timeout to complete.
2087 	 */
2088 	mutex_enter(&fasttrap_cleanup_mtx);
2089 	fasttrap_cleanup_work = 0;
2090 
2091 	while (fasttrap_timeout != (timeout_id_t)1) {
2092 		tmp = fasttrap_timeout;
2093 		fasttrap_timeout = (timeout_id_t)1;
2094 
2095 		if (tmp != 0) {
2096 			mutex_exit(&fasttrap_cleanup_mtx);
2097 			(void) untimeout(tmp);
2098 			mutex_enter(&fasttrap_cleanup_mtx);
2099 		}
2100 	}
2101 
2102 	fasttrap_cleanup_work = 0;
2103 	mutex_exit(&fasttrap_cleanup_mtx);
2104 
2105 	/*
2106 	 * Iterate over all of our providers. If there's still a process
2107 	 * that corresponds to that pid, fail to detach.
2108 	 */
2109 	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2110 		fasttrap_provider_t **fpp, *fp;
2111 		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2112 
2113 		mutex_enter(&bucket->ftb_mtx);
2114 		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2115 		while ((fp = *fpp) != NULL) {
2116 			/*
2117 			 * Acquire and release the lock as a simple way of
2118 			 * waiting for any other consumer to finish with
2119 			 * this provider. A thread must first acquire the
2120 			 * bucket lock so there's no chance of another thread
2121 			 * blocking on the providers lock.
2122 			 */
2123 			mutex_enter(&fp->ftp_mtx);
2124 			mutex_exit(&fp->ftp_mtx);
2125 
2126 			if (dtrace_unregister(fp->ftp_provid) != 0) {
2127 				fail = 1;
2128 				fpp = &fp->ftp_next;
2129 			} else {
2130 				*fpp = fp->ftp_next;
2131 				fasttrap_provider_free(fp);
2132 			}
2133 		}
2134 
2135 		mutex_exit(&bucket->ftb_mtx);
2136 	}
2137 
2138 	if (fail || dtrace_unregister(fasttrap_id) != 0) {
2139 		uint_t work;
2140 		/*
2141 		 * If we're failing to detach, we need to unblock timeouts
2142 		 * and start a new timeout if any work has accumulated while
2143 		 * we've been unsuccessfully trying to detach.
2144 		 */
2145 		mutex_enter(&fasttrap_cleanup_mtx);
2146 		fasttrap_timeout = 0;
2147 		work = fasttrap_cleanup_work;
2148 		mutex_exit(&fasttrap_cleanup_mtx);
2149 
2150 		if (work)
2151 			fasttrap_pid_cleanup();
2152 
2153 		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2154 		    &fasttrap_meta_id);
2155 
2156 		return (DDI_FAILURE);
2157 	}
2158 
2159 #ifdef DEBUG
2160 	mutex_enter(&fasttrap_count_mtx);
2161 	ASSERT(fasttrap_count == 0);
2162 	mutex_exit(&fasttrap_count_mtx);
2163 #endif
2164 
2165 	kmem_free(fasttrap_tpoints.fth_table,
2166 	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2167 	fasttrap_tpoints.fth_nent = 0;
2168 
2169 	kmem_free(fasttrap_provs.fth_table,
2170 	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2171 	fasttrap_provs.fth_nent = 0;
2172 
2173 	kmem_free(fasttrap_procs.fth_table,
2174 	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2175 	fasttrap_procs.fth_nent = 0;
2176 
2177 	/*
2178 	 * We know there are no tracepoints in any process anywhere in
2179 	 * the system so there is no process which has its p_dtrace_count
2180 	 * greater than zero, therefore we know that no thread can actively
2181 	 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2182 	 * and fasttrap_exec() and fasttrap_exit().
2183 	 */
2184 	ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2185 	dtrace_fasttrap_fork_ptr = NULL;
2186 
2187 	ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2188 	dtrace_fasttrap_exec_ptr = NULL;
2189 
2190 	ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2191 	dtrace_fasttrap_exit_ptr = NULL;
2192 
2193 	ddi_remove_minor_node(devi, NULL);
2194 
2195 	return (DDI_SUCCESS);
2196 }
2197 
2198 static struct dev_ops fasttrap_ops = {
2199 	DEVO_REV,		/* devo_rev */
2200 	0,			/* refcnt */
2201 	fasttrap_info,		/* get_dev_info */
2202 	nulldev,		/* identify */
2203 	nulldev,		/* probe */
2204 	fasttrap_attach,	/* attach */
2205 	fasttrap_detach,	/* detach */
2206 	nodev,			/* reset */
2207 	&fasttrap_cb_ops,	/* driver operations */
2208 	NULL,			/* bus operations */
2209 	nodev			/* dev power */
2210 };
2211 
2212 /*
2213  * Module linkage information for the kernel.
2214  */
2215 static struct modldrv modldrv = {
2216 	&mod_driverops,		/* module type (this is a pseudo driver) */
2217 	"Fasttrap Tracing",	/* name of module */
2218 	&fasttrap_ops,		/* driver ops */
2219 };
2220 
2221 static struct modlinkage modlinkage = {
2222 	MODREV_1,
2223 	(void *)&modldrv,
2224 	NULL
2225 };
2226 
2227 int
2228 _init(void)
2229 {
2230 	return (mod_install(&modlinkage));
2231 }
2232 
2233 int
2234 _info(struct modinfo *modinfop)
2235 {
2236 	return (mod_info(&modlinkage, modinfop));
2237 }
2238 
2239 int
2240 _fini(void)
2241 {
2242 	return (mod_remove(&modlinkage));
2243 }
2244