xref: /titanic_51/usr/src/uts/common/dtrace/fasttrap.c (revision 613b28719c10e84c1202c1045df44d77767de21d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/atomic.h>
30 #include <sys/errno.h>
31 #include <sys/stat.h>
32 #include <sys/modctl.h>
33 #include <sys/conf.h>
34 #include <sys/systm.h>
35 #include <sys/ddi.h>
36 #include <sys/sunddi.h>
37 #include <sys/cpuvar.h>
38 #include <sys/kmem.h>
39 #include <sys/strsubr.h>
40 #include <sys/fasttrap.h>
41 #include <sys/fasttrap_impl.h>
42 #include <sys/fasttrap_isa.h>
43 #include <sys/dtrace.h>
44 #include <sys/dtrace_impl.h>
45 #include <sys/sysmacros.h>
46 #include <sys/proc.h>
47 #include <sys/priv.h>
48 #include <sys/policy.h>
49 #include <util/qsort.h>
50 
51 /*
52  * User-Land Trap-Based Tracing
53  * ----------------------------
54  *
55  * The fasttrap provider allows DTrace consumers to instrument any user-level
56  * instruction to gather data; this includes probes with semantic
57  * signifigance like entry and return as well as simple offsets into the
58  * function. While the specific techniques used are very ISA specific, the
59  * methodology is generalizable to any architecture.
60  *
61  *
62  * The General Methodology
63  * -----------------------
64  *
65  * With the primary goal of tracing every user-land instruction and the
66  * limitation that we can't trust user space so don't want to rely on much
67  * information there, we begin by replacing the instructions we want to trace
68  * with trap instructions. Each instruction we overwrite is saved into a hash
69  * table keyed by process ID and pc address. When we enter the kernel due to
70  * this trap instruction, we need the effects of the replaced instruction to
71  * appear to have occurred before we proceed with the user thread's
72  * execution.
73  *
74  * Each user level thread is represented by a ulwp_t structure which is
75  * always easily accessible through a register. The most basic way to produce
76  * the effects of the instruction we replaced is to copy that instruction out
77  * to a bit of scratch space reserved in the user thread's ulwp_t structure
78  * (a sort of kernel-private thread local storage), set the PC to that
79  * scratch space and single step. When we reenter the kernel after single
80  * stepping the instruction we must then adjust the PC to point to what would
81  * normally be the next instruction. Of course, special care must be taken
82  * for branches and jumps, but these represent such a small fraction of any
83  * instruction set that writing the code to emulate these in the kernel is
84  * not too difficult.
85  *
86  * Return probes may require several tracepoints to trace every return site,
87  * and, conversely, each tracepoint may activate several probes (the entry
88  * and offset 0 probes, for example). To solve this muliplexing problem,
89  * tracepoints contain lists of probes to activate and probes contain lists
90  * of tracepoints to enable. If a probe is activated, it adds its ID to
91  * existing tracepoints or creates new ones as necessary.
92  *
93  * Most probes are activated _before_ the instruction is executed, but return
94  * probes are activated _after_ the effects of the last instruction of the
95  * function are visible. Return probes must be fired _after_ we have
96  * single-stepped the instruction whereas all other probes are fired
97  * beforehand.
98  *
99  *
100  * Lock Ordering
101  * -------------
102  *
103  * The lock ordering below -- both internally and with respect to the DTrace
104  * framework -- is a little tricky and bears some explanation. Each provider
105  * has a lock (ftp_mtx) that protects its members including reference counts
106  * for enabled probes (ftp_rcount), consumers actively creating probes
107  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
108  * from being freed. A provider is looked up by taking the bucket lock for the
109  * provider hash table, and is returned with its lock held. The provider lock
110  * may be taken in functions invoked by the DTrace framework, but may not be
111  * held while calling functions in the DTrace framework.
112  *
113  * To ensure consistency over multiple calls to the DTrace framework, the
114  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
115  * not be taken when holding the provider lock as that would create a cyclic
116  * lock ordering. In situations where one would naturally take the provider
117  * lock and then the creation lock, we instead up a reference count to prevent
118  * the provider from disappearing, drop the provider lock, and acquire the
119  * creation lock.
120  *
121  * Briefly:
122  * 	bucket lock before provider lock
123  *	DTrace before provider lock
124  *	creation lock before DTrace
125  *	never hold the provider lock and creation lock simultaneously
126  */
127 
128 static dev_info_t *fasttrap_devi;
129 static dtrace_meta_provider_id_t fasttrap_meta_id;
130 
131 static timeout_id_t fasttrap_timeout;
132 static kmutex_t fasttrap_cleanup_mtx;
133 static uint_t fasttrap_cleanup_work;
134 
135 /*
136  * Generation count on modifications to the global tracepoint lookup table.
137  */
138 static volatile uint64_t fasttrap_mod_gen;
139 
140 /*
141  * When the fasttrap provider is loaded, fasttrap_max is set to either
142  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
143  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
144  * incremented by the number of tracepoints that may be associated with that
145  * probe; fasttrap_total is capped at fasttrap_max.
146  */
147 #define	FASTTRAP_MAX_DEFAULT		250000
148 static uint32_t fasttrap_max;
149 static uint32_t fasttrap_total;
150 
151 
152 #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
153 #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
154 #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
155 
156 #define	FASTTRAP_PID_NAME		"pid"
157 
158 fasttrap_hash_t			fasttrap_tpoints;
159 static fasttrap_hash_t		fasttrap_provs;
160 static fasttrap_hash_t		fasttrap_procs;
161 
162 static uint64_t			fasttrap_pid_count;	/* pid ref count */
163 static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
164 
165 #define	FASTTRAP_ENABLE_FAIL	1
166 #define	FASTTRAP_ENABLE_PARTIAL	2
167 
168 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
169 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
170 
171 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
172     const dtrace_pattr_t *);
173 static void fasttrap_provider_retire(pid_t, const char *, int);
174 static void fasttrap_provider_free(fasttrap_provider_t *);
175 
176 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
177 static void fasttrap_proc_release(fasttrap_proc_t *);
178 
179 #define	FASTTRAP_PROVS_INDEX(pid, name) \
180 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
181 
182 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
183 
184 static int
185 fasttrap_highbit(ulong_t i)
186 {
187 	int h = 1;
188 
189 	if (i == 0)
190 		return (0);
191 #ifdef _LP64
192 	if (i & 0xffffffff00000000ul) {
193 		h += 32; i >>= 32;
194 	}
195 #endif
196 	if (i & 0xffff0000) {
197 		h += 16; i >>= 16;
198 	}
199 	if (i & 0xff00) {
200 		h += 8; i >>= 8;
201 	}
202 	if (i & 0xf0) {
203 		h += 4; i >>= 4;
204 	}
205 	if (i & 0xc) {
206 		h += 2; i >>= 2;
207 	}
208 	if (i & 0x2) {
209 		h += 1;
210 	}
211 	return (h);
212 }
213 
214 static uint_t
215 fasttrap_hash_str(const char *p)
216 {
217 	unsigned int g;
218 	uint_t hval = 0;
219 
220 	while (*p) {
221 		hval = (hval << 4) + *p++;
222 		if ((g = (hval & 0xf0000000)) != 0)
223 			hval ^= g >> 24;
224 		hval &= ~g;
225 	}
226 	return (hval);
227 }
228 
229 void
230 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
231 {
232 	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
233 
234 	sqp->sq_info.si_signo = SIGTRAP;
235 	sqp->sq_info.si_code = TRAP_DTRACE;
236 	sqp->sq_info.si_addr = (caddr_t)pc;
237 
238 	mutex_enter(&p->p_lock);
239 	sigaddqa(p, t, sqp);
240 	mutex_exit(&p->p_lock);
241 
242 	if (t != NULL)
243 		aston(t);
244 }
245 
246 /*
247  * This function ensures that no threads are actively using the memory
248  * associated with probes that were formerly live.
249  */
250 static void
251 fasttrap_mod_barrier(uint64_t gen)
252 {
253 	int i;
254 
255 	if (gen < fasttrap_mod_gen)
256 		return;
257 
258 	fasttrap_mod_gen++;
259 
260 	for (i = 0; i < NCPU; i++) {
261 		mutex_enter(&cpu_core[i].cpuc_pid_lock);
262 		mutex_exit(&cpu_core[i].cpuc_pid_lock);
263 	}
264 }
265 
266 /*
267  * This is the timeout's callback for cleaning up the providers and their
268  * probes.
269  */
270 /*ARGSUSED*/
271 static void
272 fasttrap_pid_cleanup_cb(void *data)
273 {
274 	fasttrap_provider_t **fpp, *fp;
275 	fasttrap_bucket_t *bucket;
276 	dtrace_provider_id_t provid;
277 	int i, later;
278 
279 	static volatile int in = 0;
280 	ASSERT(in == 0);
281 	in = 1;
282 
283 	mutex_enter(&fasttrap_cleanup_mtx);
284 	while (fasttrap_cleanup_work) {
285 		fasttrap_cleanup_work = 0;
286 		mutex_exit(&fasttrap_cleanup_mtx);
287 
288 		later = 0;
289 
290 		/*
291 		 * Iterate over all the providers trying to remove the marked
292 		 * ones. If a provider is marked but not retired, we just
293 		 * have to take a crack at removing it -- it's no big deal if
294 		 * we can't.
295 		 */
296 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
297 			bucket = &fasttrap_provs.fth_table[i];
298 			mutex_enter(&bucket->ftb_mtx);
299 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
300 
301 			while ((fp = *fpp) != NULL) {
302 				if (!fp->ftp_marked) {
303 					fpp = &fp->ftp_next;
304 					continue;
305 				}
306 
307 				mutex_enter(&fp->ftp_mtx);
308 
309 				/*
310 				 * If this provider has consumers actively
311 				 * creating probes (ftp_ccount) or is a USDT
312 				 * provider (ftp_mcount), we can't unregister
313 				 * or even condense.
314 				 */
315 				if (fp->ftp_ccount != 0 ||
316 				    fp->ftp_mcount != 0) {
317 					mutex_exit(&fp->ftp_mtx);
318 					fp->ftp_marked = 0;
319 					continue;
320 				}
321 
322 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
323 					fp->ftp_marked = 0;
324 
325 				mutex_exit(&fp->ftp_mtx);
326 
327 				/*
328 				 * If we successfully unregister this
329 				 * provider we can remove it from the hash
330 				 * chain and free the memory. If our attempt
331 				 * to unregister fails and this is a retired
332 				 * provider, increment our flag to try again
333 				 * pretty soon. If we've consumed more than
334 				 * half of our total permitted number of
335 				 * probes call dtrace_condense() to try to
336 				 * clean out the unenabled probes.
337 				 */
338 				provid = fp->ftp_provid;
339 				if (dtrace_unregister(provid) != 0) {
340 					if (fasttrap_total > fasttrap_max / 2)
341 						(void) dtrace_condense(provid);
342 					later += fp->ftp_marked;
343 					fpp = &fp->ftp_next;
344 				} else {
345 					*fpp = fp->ftp_next;
346 					fasttrap_provider_free(fp);
347 				}
348 			}
349 			mutex_exit(&bucket->ftb_mtx);
350 		}
351 
352 		mutex_enter(&fasttrap_cleanup_mtx);
353 	}
354 
355 	ASSERT(fasttrap_timeout != 0);
356 
357 	/*
358 	 * If we were unable to remove a retired provider, try again after
359 	 * a second. This situation can occur in certain circumstances where
360 	 * providers cannot be unregistered even though they have no probes
361 	 * enabled because of an execution of dtrace -l or something similar.
362 	 * If the timeout has been disabled (set to 1 because we're trying
363 	 * to detach), we set fasttrap_cleanup_work to ensure that we'll
364 	 * get a chance to do that work if and when the timeout is reenabled
365 	 * (if detach fails).
366 	 */
367 	if (later > 0 && fasttrap_timeout != (timeout_id_t)1)
368 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
369 	else if (later > 0)
370 		fasttrap_cleanup_work = 1;
371 	else
372 		fasttrap_timeout = 0;
373 
374 	mutex_exit(&fasttrap_cleanup_mtx);
375 	in = 0;
376 }
377 
378 /*
379  * Activates the asynchronous cleanup mechanism.
380  */
381 static void
382 fasttrap_pid_cleanup(void)
383 {
384 	mutex_enter(&fasttrap_cleanup_mtx);
385 	fasttrap_cleanup_work = 1;
386 	if (fasttrap_timeout == 0)
387 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
388 	mutex_exit(&fasttrap_cleanup_mtx);
389 }
390 
391 /*
392  * This is called from cfork() via dtrace_fasttrap_fork(). The child
393  * process's address space is (roughly) a copy of the parent process's so
394  * we have to remove all the instrumentation we had previously enabled in the
395  * parent.
396  */
397 static void
398 fasttrap_fork(proc_t *p, proc_t *cp)
399 {
400 	pid_t ppid = p->p_pid;
401 	int i;
402 
403 	ASSERT(curproc == p);
404 	ASSERT(p->p_proc_flag & P_PR_LOCK);
405 	ASSERT(p->p_dtrace_count > 0);
406 	ASSERT(cp->p_dtrace_count == 0);
407 
408 	/*
409 	 * This would be simpler and faster if we maintained per-process
410 	 * hash tables of enabled tracepoints. It could, however, potentially
411 	 * slow down execution of a tracepoint since we'd need to go
412 	 * through two levels of indirection. In the future, we should
413 	 * consider either maintaining per-process ancillary lists of
414 	 * enabled tracepoints or hanging a pointer to a per-process hash
415 	 * table of enabled tracepoints off the proc structure.
416 	 */
417 
418 	/*
419 	 * We don't have to worry about the child process disappearing
420 	 * because we're in fork().
421 	 */
422 	mutex_enter(&cp->p_lock);
423 	sprlock_proc(cp);
424 	mutex_exit(&cp->p_lock);
425 
426 	/*
427 	 * Iterate over every tracepoint looking for ones that belong to the
428 	 * parent process, and remove each from the child process.
429 	 */
430 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
431 		fasttrap_tracepoint_t *tp;
432 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
433 
434 		mutex_enter(&bucket->ftb_mtx);
435 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
436 			if (tp->ftt_pid == ppid &&
437 			    tp->ftt_proc->ftpc_acount != 0) {
438 				int ret = fasttrap_tracepoint_remove(cp, tp);
439 				ASSERT(ret == 0);
440 
441 				/*
442 				 * The count of active providers can only be
443 				 * decremented (i.e. to zero) during exec,
444 				 * exit, and removal of a meta provider so it
445 				 * should be impossible to drop the count
446 				 * mid-fork.
447 				 */
448 				ASSERT(tp->ftt_proc->ftpc_acount != 0);
449 			}
450 		}
451 		mutex_exit(&bucket->ftb_mtx);
452 	}
453 
454 	mutex_enter(&cp->p_lock);
455 	sprunlock(cp);
456 }
457 
458 /*
459  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
460  * is set on the proc structure to indicate that there is a pid provider
461  * associated with this process.
462  */
463 static void
464 fasttrap_exec_exit(proc_t *p)
465 {
466 	ASSERT(p == curproc);
467 	ASSERT(MUTEX_HELD(&p->p_lock));
468 
469 	mutex_exit(&p->p_lock);
470 
471 	/*
472 	 * We clean up the pid provider for this process here; user-land
473 	 * static probes are handled by the meta-provider remove entry point.
474 	 */
475 	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
476 
477 	mutex_enter(&p->p_lock);
478 }
479 
480 
481 /*ARGSUSED*/
482 static void
483 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
484 {
485 	/*
486 	 * There are no "default" pid probes.
487 	 */
488 }
489 
490 static int
491 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
492 {
493 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
494 	fasttrap_bucket_t *bucket;
495 	fasttrap_id_t *id;
496 	pid_t pid;
497 	uintptr_t pc;
498 
499 	ASSERT(index < probe->ftp_ntps);
500 
501 	pid = probe->ftp_pid;
502 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
503 	id = &probe->ftp_tps[index].fit_id;
504 
505 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
506 
507 	ASSERT(!(p->p_flag & SVFORK));
508 
509 	/*
510 	 * Before we make any modifications, make sure we've imposed a barrier
511 	 * on the generation in which this probe was last modified.
512 	 */
513 	fasttrap_mod_barrier(probe->ftp_gen);
514 
515 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
516 
517 	/*
518 	 * If the tracepoint has already been enabled, just add our id to the
519 	 * list of interested probes. This may be our second time through
520 	 * this path in which case we'll have constructed the tracepoint we'd
521 	 * like to install. If we can't find a match, and have an allocated
522 	 * tracepoint ready to go, enable that one now.
523 	 *
524 	 * A tracepoint whose process is defunct is also considered defunct.
525 	 */
526 again:
527 	mutex_enter(&bucket->ftb_mtx);
528 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
529 		/*
530 		 * Note that it's safe to access the active count on the
531 		 * associated proc structure because we know that at least one
532 		 * provider (this one) will still be around throughout this
533 		 * operation.
534 		 */
535 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
536 		    tp->ftt_proc->ftpc_acount == 0)
537 			continue;
538 
539 		/*
540 		 * Now that we've found a matching tracepoint, it would be
541 		 * a decent idea to confirm that the tracepoint is still
542 		 * enabled and the trap instruction hasn't been overwritten.
543 		 * Since this is a little hairy, we'll punt for now.
544 		 */
545 
546 		/*
547 		 * This can't be the first interested probe. We don't have
548 		 * to worry about another thread being in the midst of
549 		 * deleting this tracepoint (which would be the only valid
550 		 * reason for a tracepoint to have no interested probes)
551 		 * since we're holding P_PR_LOCK for this process.
552 		 */
553 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
554 
555 		switch (id->fti_ptype) {
556 		case DTFTP_ENTRY:
557 		case DTFTP_OFFSETS:
558 		case DTFTP_IS_ENABLED:
559 			id->fti_next = tp->ftt_ids;
560 			membar_producer();
561 			tp->ftt_ids = id;
562 			membar_producer();
563 			break;
564 
565 		case DTFTP_RETURN:
566 		case DTFTP_POST_OFFSETS:
567 			id->fti_next = tp->ftt_retids;
568 			membar_producer();
569 			tp->ftt_retids = id;
570 			membar_producer();
571 			break;
572 
573 		default:
574 			ASSERT(0);
575 		}
576 
577 		mutex_exit(&bucket->ftb_mtx);
578 
579 		if (new_tp != NULL) {
580 			new_tp->ftt_ids = NULL;
581 			new_tp->ftt_retids = NULL;
582 		}
583 
584 		return (0);
585 	}
586 
587 	/*
588 	 * If we have a good tracepoint ready to go, install it now while
589 	 * we have the lock held and no one can screw with us.
590 	 */
591 	if (new_tp != NULL) {
592 		int rc = 0;
593 
594 		new_tp->ftt_next = bucket->ftb_data;
595 		membar_producer();
596 		bucket->ftb_data = new_tp;
597 		membar_producer();
598 		mutex_exit(&bucket->ftb_mtx);
599 
600 		/*
601 		 * Activate the tracepoint in the ISA-specific manner.
602 		 * If this fails, we need to report the failure, but
603 		 * indicate that this tracepoint must still be disabled
604 		 * by calling fasttrap_tracepoint_disable().
605 		 */
606 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
607 			rc = FASTTRAP_ENABLE_PARTIAL;
608 
609 		/*
610 		 * Increment the count of the number of tracepoints active in
611 		 * the victim process.
612 		 */
613 		ASSERT(p->p_proc_flag & P_PR_LOCK);
614 		p->p_dtrace_count++;
615 
616 		return (rc);
617 	}
618 
619 	mutex_exit(&bucket->ftb_mtx);
620 
621 	/*
622 	 * Initialize the tracepoint that's been preallocated with the probe.
623 	 */
624 	new_tp = probe->ftp_tps[index].fit_tp;
625 
626 	ASSERT(new_tp->ftt_pid == pid);
627 	ASSERT(new_tp->ftt_pc == pc);
628 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
629 	ASSERT(new_tp->ftt_ids == NULL);
630 	ASSERT(new_tp->ftt_retids == NULL);
631 
632 	switch (id->fti_ptype) {
633 	case DTFTP_ENTRY:
634 	case DTFTP_OFFSETS:
635 	case DTFTP_IS_ENABLED:
636 		id->fti_next = NULL;
637 		new_tp->ftt_ids = id;
638 		break;
639 
640 	case DTFTP_RETURN:
641 	case DTFTP_POST_OFFSETS:
642 		id->fti_next = NULL;
643 		new_tp->ftt_retids = id;
644 		break;
645 
646 	default:
647 		ASSERT(0);
648 	}
649 
650 	/*
651 	 * If the ISA-dependent initialization goes to plan, go back to the
652 	 * beginning and try to install this freshly made tracepoint.
653 	 */
654 	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
655 		goto again;
656 
657 	new_tp->ftt_ids = NULL;
658 	new_tp->ftt_retids = NULL;
659 
660 	return (FASTTRAP_ENABLE_FAIL);
661 }
662 
663 static void
664 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
665 {
666 	fasttrap_bucket_t *bucket;
667 	fasttrap_provider_t *provider = probe->ftp_prov;
668 	fasttrap_tracepoint_t **pp, *tp;
669 	fasttrap_id_t *id, **idp;
670 	pid_t pid;
671 	uintptr_t pc;
672 
673 	ASSERT(index < probe->ftp_ntps);
674 
675 	pid = probe->ftp_pid;
676 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
677 	id = &probe->ftp_tps[index].fit_id;
678 
679 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
680 
681 	/*
682 	 * Find the tracepoint and make sure that our id is one of the
683 	 * ones registered with it.
684 	 */
685 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
686 	mutex_enter(&bucket->ftb_mtx);
687 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
688 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
689 		    tp->ftt_proc == provider->ftp_proc)
690 			break;
691 	}
692 
693 	/*
694 	 * If we somehow lost this tracepoint, we're in a world of hurt.
695 	 */
696 	ASSERT(tp != NULL);
697 
698 	switch (id->fti_ptype) {
699 	case DTFTP_ENTRY:
700 	case DTFTP_OFFSETS:
701 	case DTFTP_IS_ENABLED:
702 		ASSERT(tp->ftt_ids != NULL);
703 		idp = &tp->ftt_ids;
704 		break;
705 
706 	case DTFTP_RETURN:
707 	case DTFTP_POST_OFFSETS:
708 		ASSERT(tp->ftt_retids != NULL);
709 		idp = &tp->ftt_retids;
710 		break;
711 
712 	default:
713 		ASSERT(0);
714 	}
715 
716 	while ((*idp)->fti_probe != probe) {
717 		idp = &(*idp)->fti_next;
718 		ASSERT(*idp != NULL);
719 	}
720 
721 	id = *idp;
722 	*idp = id->fti_next;
723 	membar_producer();
724 
725 	ASSERT(id->fti_probe == probe);
726 
727 	/*
728 	 * If there are other registered enablings of this tracepoint, we're
729 	 * all done, but if this was the last probe assocated with this
730 	 * this tracepoint, we need to remove and free it.
731 	 */
732 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
733 
734 		/*
735 		 * If the current probe's tracepoint is in use, swap it
736 		 * for an unused tracepoint.
737 		 */
738 		if (tp == probe->ftp_tps[index].fit_tp) {
739 			fasttrap_probe_t *tmp_probe;
740 			fasttrap_tracepoint_t **tmp_tp;
741 			uint_t tmp_index;
742 
743 			if (tp->ftt_ids != NULL) {
744 				tmp_probe = tp->ftt_ids->fti_probe;
745 				/* LINTED - alignment */
746 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
747 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
748 			} else {
749 				tmp_probe = tp->ftt_retids->fti_probe;
750 				/* LINTED - alignment */
751 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
752 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
753 			}
754 
755 			ASSERT(*tmp_tp != NULL);
756 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
757 			ASSERT((*tmp_tp)->ftt_ids == NULL);
758 			ASSERT((*tmp_tp)->ftt_retids == NULL);
759 
760 			probe->ftp_tps[index].fit_tp = *tmp_tp;
761 			*tmp_tp = tp;
762 		}
763 
764 		mutex_exit(&bucket->ftb_mtx);
765 
766 		/*
767 		 * Tag the modified probe with the generation in which it was
768 		 * changed.
769 		 */
770 		probe->ftp_gen = fasttrap_mod_gen;
771 		return;
772 	}
773 
774 	mutex_exit(&bucket->ftb_mtx);
775 
776 	/*
777 	 * We can't safely remove the tracepoint from the set of active
778 	 * tracepoints until we've actually removed the fasttrap instruction
779 	 * from the process's text. We can, however, operate on this
780 	 * tracepoint secure in the knowledge that no other thread is going to
781 	 * be looking at it since we hold P_PR_LOCK on the process if it's
782 	 * live or we hold the provider lock on the process if it's dead and
783 	 * gone.
784 	 */
785 
786 	/*
787 	 * We only need to remove the actual instruction if we're looking
788 	 * at an existing process
789 	 */
790 	if (p != NULL) {
791 		/*
792 		 * If we fail to restore the instruction we need to kill
793 		 * this process since it's in a completely unrecoverable
794 		 * state.
795 		 */
796 		if (fasttrap_tracepoint_remove(p, tp) != 0)
797 			fasttrap_sigtrap(p, NULL, pc);
798 
799 		/*
800 		 * Decrement the count of the number of tracepoints active
801 		 * in the victim process.
802 		 */
803 		ASSERT(p->p_proc_flag & P_PR_LOCK);
804 		p->p_dtrace_count--;
805 	}
806 
807 	/*
808 	 * Remove the probe from the hash table of active tracepoints.
809 	 */
810 	mutex_enter(&bucket->ftb_mtx);
811 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
812 	ASSERT(*pp != NULL);
813 	while (*pp != tp) {
814 		pp = &(*pp)->ftt_next;
815 		ASSERT(*pp != NULL);
816 	}
817 
818 	*pp = tp->ftt_next;
819 	membar_producer();
820 
821 	mutex_exit(&bucket->ftb_mtx);
822 
823 	/*
824 	 * Tag the modified probe with the generation in which it was changed.
825 	 */
826 	probe->ftp_gen = fasttrap_mod_gen;
827 }
828 
829 static void
830 fasttrap_enable_callbacks(void)
831 {
832 	/*
833 	 * We don't have to play the rw lock game here because we're
834 	 * providing something rather than taking something away --
835 	 * we can be sure that no threads have tried to follow this
836 	 * function pointer yet.
837 	 */
838 	mutex_enter(&fasttrap_count_mtx);
839 	if (fasttrap_pid_count == 0) {
840 		ASSERT(dtrace_pid_probe_ptr == NULL);
841 		ASSERT(dtrace_return_probe_ptr == NULL);
842 		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
843 		dtrace_return_probe_ptr = &fasttrap_return_probe;
844 	}
845 	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
846 	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
847 	fasttrap_pid_count++;
848 	mutex_exit(&fasttrap_count_mtx);
849 }
850 
851 static void
852 fasttrap_disable_callbacks(void)
853 {
854 	ASSERT(MUTEX_HELD(&cpu_lock));
855 
856 	mutex_enter(&fasttrap_count_mtx);
857 	ASSERT(fasttrap_pid_count > 0);
858 	fasttrap_pid_count--;
859 	if (fasttrap_pid_count == 0) {
860 		cpu_t *cur, *cpu = CPU;
861 
862 		for (cur = cpu->cpu_next_onln; cur != cpu;
863 		    cur = cur->cpu_next_onln) {
864 			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
865 		}
866 
867 		dtrace_pid_probe_ptr = NULL;
868 		dtrace_return_probe_ptr = NULL;
869 
870 		for (cur = cpu->cpu_next_onln; cur != cpu;
871 		    cur = cur->cpu_next_onln) {
872 			rw_exit(&cur->cpu_ft_lock);
873 		}
874 	}
875 	mutex_exit(&fasttrap_count_mtx);
876 }
877 
878 /*ARGSUSED*/
879 static void
880 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
881 {
882 	fasttrap_probe_t *probe = parg;
883 	proc_t *p;
884 	int i, rc;
885 
886 	ASSERT(probe != NULL);
887 	ASSERT(!probe->ftp_enabled);
888 	ASSERT(id == probe->ftp_id);
889 	ASSERT(MUTEX_HELD(&cpu_lock));
890 
891 	/*
892 	 * Increment the count of enabled probes on this probe's provider;
893 	 * the provider can't go away while the probe still exists. We
894 	 * must increment this even if we aren't able to properly enable
895 	 * this probe.
896 	 */
897 	mutex_enter(&probe->ftp_prov->ftp_mtx);
898 	probe->ftp_prov->ftp_rcount++;
899 	mutex_exit(&probe->ftp_prov->ftp_mtx);
900 
901 	/*
902 	 * If this probe's provider is retired (meaning it was valid in a
903 	 * previously exec'ed incarnation of this address space), bail out. The
904 	 * provider can't go away while we're in this code path.
905 	 */
906 	if (probe->ftp_prov->ftp_retired)
907 		return;
908 
909 	/*
910 	 * If we can't find the process, it may be that we're in the context of
911 	 * a fork in which the traced process is being born and we're copying
912 	 * USDT probes. Otherwise, the process is gone so bail.
913 	 */
914 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
915 		if ((curproc->p_flag & SFORKING) == 0)
916 			return;
917 
918 		mutex_enter(&pidlock);
919 		p = prfind(probe->ftp_pid);
920 
921 		/*
922 		 * Confirm that curproc is indeed forking the process in which
923 		 * we're trying to enable probes.
924 		 */
925 		ASSERT(p != NULL);
926 		ASSERT(p->p_parent == curproc);
927 		ASSERT(p->p_stat == SIDL);
928 
929 		mutex_enter(&p->p_lock);
930 		mutex_exit(&pidlock);
931 
932 		sprlock_proc(p);
933 	}
934 
935 	ASSERT(!(p->p_flag & SVFORK));
936 	mutex_exit(&p->p_lock);
937 
938 	/*
939 	 * We have to enable the trap entry point before any user threads have
940 	 * the chance to execute the trap instruction we're about to place
941 	 * in their process's text.
942 	 */
943 	fasttrap_enable_callbacks();
944 
945 	/*
946 	 * Enable all the tracepoints and add this probe's id to each
947 	 * tracepoint's list of active probes.
948 	 */
949 	for (i = 0; i < probe->ftp_ntps; i++) {
950 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
951 			/*
952 			 * If enabling the tracepoint failed completely,
953 			 * we don't have to disable it; if the failure
954 			 * was only partial we must disable it.
955 			 */
956 			if (rc == FASTTRAP_ENABLE_FAIL)
957 				i--;
958 			else
959 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
960 
961 			/*
962 			 * Back up and pull out all the tracepoints we've
963 			 * created so far for this probe.
964 			 */
965 			while (i >= 0) {
966 				fasttrap_tracepoint_disable(p, probe, i);
967 				i--;
968 			}
969 
970 			mutex_enter(&p->p_lock);
971 			sprunlock(p);
972 
973 			/*
974 			 * Since we're not actually enabling this probe,
975 			 * drop our reference on the trap table entry.
976 			 */
977 			fasttrap_disable_callbacks();
978 			return;
979 		}
980 	}
981 
982 	mutex_enter(&p->p_lock);
983 	sprunlock(p);
984 
985 	probe->ftp_enabled = 1;
986 }
987 
988 /*ARGSUSED*/
989 static void
990 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
991 {
992 	fasttrap_probe_t *probe = parg;
993 	fasttrap_provider_t *provider = probe->ftp_prov;
994 	proc_t *p;
995 	int i, whack = 0;
996 
997 	ASSERT(id == probe->ftp_id);
998 
999 	/*
1000 	 * We won't be able to acquire a /proc-esque lock on the process
1001 	 * iff the process is dead and gone. In this case, we rely on the
1002 	 * provider lock as a point of mutual exclusion to prevent other
1003 	 * DTrace consumers from disabling this probe.
1004 	 */
1005 	if ((p = sprlock(probe->ftp_pid)) != NULL) {
1006 		ASSERT(!(p->p_flag & SVFORK));
1007 		mutex_exit(&p->p_lock);
1008 	}
1009 
1010 	mutex_enter(&provider->ftp_mtx);
1011 
1012 	/*
1013 	 * Disable all the associated tracepoints (for fully enabled probes).
1014 	 */
1015 	if (probe->ftp_enabled) {
1016 		for (i = 0; i < probe->ftp_ntps; i++) {
1017 			fasttrap_tracepoint_disable(p, probe, i);
1018 		}
1019 	}
1020 
1021 	ASSERT(provider->ftp_rcount > 0);
1022 	provider->ftp_rcount--;
1023 
1024 	if (p != NULL) {
1025 		/*
1026 		 * Even though we may not be able to remove it entirely, we
1027 		 * mark this retired provider to get a chance to remove some
1028 		 * of the associated probes.
1029 		 */
1030 		if (provider->ftp_retired && !provider->ftp_marked)
1031 			whack = provider->ftp_marked = 1;
1032 		mutex_exit(&provider->ftp_mtx);
1033 
1034 		mutex_enter(&p->p_lock);
1035 		sprunlock(p);
1036 	} else {
1037 		/*
1038 		 * If the process is dead, we're just waiting for the
1039 		 * last probe to be disabled to be able to free it.
1040 		 */
1041 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1042 			whack = provider->ftp_marked = 1;
1043 		mutex_exit(&provider->ftp_mtx);
1044 	}
1045 
1046 	if (whack)
1047 		fasttrap_pid_cleanup();
1048 
1049 	if (!probe->ftp_enabled)
1050 		return;
1051 
1052 	probe->ftp_enabled = 0;
1053 
1054 	ASSERT(MUTEX_HELD(&cpu_lock));
1055 	fasttrap_disable_callbacks();
1056 }
1057 
1058 /*ARGSUSED*/
1059 static void
1060 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1061     dtrace_argdesc_t *desc)
1062 {
1063 	fasttrap_probe_t *probe = parg;
1064 	char *str;
1065 	int i, ndx;
1066 
1067 	desc->dtargd_native[0] = '\0';
1068 	desc->dtargd_xlate[0] = '\0';
1069 
1070 	if (probe->ftp_prov->ftp_retired != 0 ||
1071 	    desc->dtargd_ndx >= probe->ftp_nargs) {
1072 		desc->dtargd_ndx = DTRACE_ARGNONE;
1073 		return;
1074 	}
1075 
1076 	ndx = (probe->ftp_argmap != NULL) ?
1077 	    probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1078 
1079 	str = probe->ftp_ntypes;
1080 	for (i = 0; i < ndx; i++) {
1081 		str += strlen(str) + 1;
1082 	}
1083 
1084 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1085 	(void) strcpy(desc->dtargd_native, str);
1086 
1087 	if (probe->ftp_xtypes == NULL)
1088 		return;
1089 
1090 	str = probe->ftp_xtypes;
1091 	for (i = 0; i < desc->dtargd_ndx; i++) {
1092 		str += strlen(str) + 1;
1093 	}
1094 
1095 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1096 	(void) strcpy(desc->dtargd_xlate, str);
1097 }
1098 
1099 /*ARGSUSED*/
1100 static void
1101 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1102 {
1103 	fasttrap_probe_t *probe = parg;
1104 	int i;
1105 	size_t size;
1106 
1107 	ASSERT(probe != NULL);
1108 	ASSERT(!probe->ftp_enabled);
1109 	ASSERT(fasttrap_total >= probe->ftp_ntps);
1110 
1111 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1112 	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1113 
1114 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1115 		fasttrap_mod_barrier(probe->ftp_gen);
1116 
1117 	for (i = 0; i < probe->ftp_ntps; i++) {
1118 		kmem_free(probe->ftp_tps[i].fit_tp,
1119 		    sizeof (fasttrap_tracepoint_t));
1120 	}
1121 
1122 	kmem_free(probe, size);
1123 }
1124 
1125 
1126 static const dtrace_pattr_t pid_attr = {
1127 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1128 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1129 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1130 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1131 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1132 };
1133 
1134 static dtrace_pops_t pid_pops = {
1135 	fasttrap_pid_provide,
1136 	NULL,
1137 	fasttrap_pid_enable,
1138 	fasttrap_pid_disable,
1139 	NULL,
1140 	NULL,
1141 	fasttrap_pid_getargdesc,
1142 	fasttrap_pid_getarg,
1143 	NULL,
1144 	fasttrap_pid_destroy
1145 };
1146 
1147 static dtrace_pops_t usdt_pops = {
1148 	fasttrap_pid_provide,
1149 	NULL,
1150 	fasttrap_pid_enable,
1151 	fasttrap_pid_disable,
1152 	NULL,
1153 	NULL,
1154 	fasttrap_pid_getargdesc,
1155 	fasttrap_usdt_getarg,
1156 	NULL,
1157 	fasttrap_pid_destroy
1158 };
1159 
1160 static fasttrap_proc_t *
1161 fasttrap_proc_lookup(pid_t pid)
1162 {
1163 	fasttrap_bucket_t *bucket;
1164 	fasttrap_proc_t *fprc, *new_fprc;
1165 
1166 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1167 	mutex_enter(&bucket->ftb_mtx);
1168 
1169 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1170 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1171 			mutex_enter(&fprc->ftpc_mtx);
1172 			mutex_exit(&bucket->ftb_mtx);
1173 			fprc->ftpc_rcount++;
1174 			atomic_add_64(&fprc->ftpc_acount, 1);
1175 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1176 			mutex_exit(&fprc->ftpc_mtx);
1177 
1178 			return (fprc);
1179 		}
1180 	}
1181 
1182 	/*
1183 	 * Drop the bucket lock so we don't try to perform a sleeping
1184 	 * allocation under it.
1185 	 */
1186 	mutex_exit(&bucket->ftb_mtx);
1187 
1188 	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1189 	new_fprc->ftpc_pid = pid;
1190 	new_fprc->ftpc_rcount = 1;
1191 	new_fprc->ftpc_acount = 1;
1192 
1193 	mutex_enter(&bucket->ftb_mtx);
1194 
1195 	/*
1196 	 * Take another lap through the list to make sure a proc hasn't
1197 	 * been created for this pid while we weren't under the bucket lock.
1198 	 */
1199 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1200 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1201 			mutex_enter(&fprc->ftpc_mtx);
1202 			mutex_exit(&bucket->ftb_mtx);
1203 			fprc->ftpc_rcount++;
1204 			atomic_add_64(&fprc->ftpc_acount, 1);
1205 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1206 			mutex_exit(&fprc->ftpc_mtx);
1207 
1208 			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1209 
1210 			return (fprc);
1211 		}
1212 	}
1213 
1214 	new_fprc->ftpc_next = bucket->ftb_data;
1215 	bucket->ftb_data = new_fprc;
1216 
1217 	mutex_exit(&bucket->ftb_mtx);
1218 
1219 	return (new_fprc);
1220 }
1221 
1222 static void
1223 fasttrap_proc_release(fasttrap_proc_t *proc)
1224 {
1225 	fasttrap_bucket_t *bucket;
1226 	fasttrap_proc_t *fprc, **fprcp;
1227 	pid_t pid = proc->ftpc_pid;
1228 
1229 	mutex_enter(&proc->ftpc_mtx);
1230 
1231 	ASSERT(proc->ftpc_rcount != 0);
1232 	ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1233 
1234 	if (--proc->ftpc_rcount != 0) {
1235 		mutex_exit(&proc->ftpc_mtx);
1236 		return;
1237 	}
1238 
1239 	mutex_exit(&proc->ftpc_mtx);
1240 
1241 	/*
1242 	 * There should definitely be no live providers associated with this
1243 	 * process at this point.
1244 	 */
1245 	ASSERT(proc->ftpc_acount == 0);
1246 
1247 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1248 	mutex_enter(&bucket->ftb_mtx);
1249 
1250 	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1251 	while ((fprc = *fprcp) != NULL) {
1252 		if (fprc == proc)
1253 			break;
1254 
1255 		fprcp = &fprc->ftpc_next;
1256 	}
1257 
1258 	/*
1259 	 * Something strange has happened if we can't find the proc.
1260 	 */
1261 	ASSERT(fprc != NULL);
1262 
1263 	*fprcp = fprc->ftpc_next;
1264 
1265 	mutex_exit(&bucket->ftb_mtx);
1266 
1267 	kmem_free(fprc, sizeof (fasttrap_proc_t));
1268 }
1269 
1270 /*
1271  * Lookup a fasttrap-managed provider based on its name and associated pid.
1272  * If the pattr argument is non-NULL, this function instantiates the provider
1273  * if it doesn't exist otherwise it returns NULL. The provider is returned
1274  * with its lock held.
1275  */
1276 static fasttrap_provider_t *
1277 fasttrap_provider_lookup(pid_t pid, const char *name,
1278     const dtrace_pattr_t *pattr)
1279 {
1280 	fasttrap_provider_t *fp, *new_fp = NULL;
1281 	fasttrap_bucket_t *bucket;
1282 	char provname[DTRACE_PROVNAMELEN];
1283 	proc_t *p;
1284 	cred_t *cred;
1285 
1286 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1287 	ASSERT(pattr != NULL);
1288 
1289 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1290 	mutex_enter(&bucket->ftb_mtx);
1291 
1292 	/*
1293 	 * Take a lap through the list and return the match if we find it.
1294 	 */
1295 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1296 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1297 		    !fp->ftp_retired) {
1298 			mutex_enter(&fp->ftp_mtx);
1299 			mutex_exit(&bucket->ftb_mtx);
1300 			return (fp);
1301 		}
1302 	}
1303 
1304 	/*
1305 	 * Drop the bucket lock so we don't try to perform a sleeping
1306 	 * allocation under it.
1307 	 */
1308 	mutex_exit(&bucket->ftb_mtx);
1309 
1310 	/*
1311 	 * Make sure the process exists, isn't a child created as the result
1312 	 * of a vfork(2), and isn't a zombie (but may be in fork).
1313 	 */
1314 	mutex_enter(&pidlock);
1315 	if ((p = prfind(pid)) == NULL) {
1316 		mutex_exit(&pidlock);
1317 		return (NULL);
1318 	}
1319 	mutex_enter(&p->p_lock);
1320 	mutex_exit(&pidlock);
1321 	if (p->p_flag & (SVFORK | SEXITING)) {
1322 		mutex_exit(&p->p_lock);
1323 		return (NULL);
1324 	}
1325 
1326 	/*
1327 	 * Increment p_dtrace_probes so that the process knows to inform us
1328 	 * when it exits or execs. fasttrap_provider_free() decrements this
1329 	 * when we're done with this provider.
1330 	 */
1331 	p->p_dtrace_probes++;
1332 
1333 	/*
1334 	 * Grab the credentials for this process so we have
1335 	 * something to pass to dtrace_register().
1336 	 */
1337 	mutex_enter(&p->p_crlock);
1338 	crhold(p->p_cred);
1339 	cred = p->p_cred;
1340 	mutex_exit(&p->p_crlock);
1341 	mutex_exit(&p->p_lock);
1342 
1343 	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1344 	new_fp->ftp_pid = pid;
1345 	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1346 
1347 	ASSERT(new_fp->ftp_proc != NULL);
1348 
1349 	mutex_enter(&bucket->ftb_mtx);
1350 
1351 	/*
1352 	 * Take another lap through the list to make sure a provider hasn't
1353 	 * been created for this pid while we weren't under the bucket lock.
1354 	 */
1355 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1356 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1357 		    !fp->ftp_retired) {
1358 			mutex_enter(&fp->ftp_mtx);
1359 			mutex_exit(&bucket->ftb_mtx);
1360 			fasttrap_provider_free(new_fp);
1361 			crfree(cred);
1362 			return (fp);
1363 		}
1364 	}
1365 
1366 	(void) strcpy(new_fp->ftp_name, name);
1367 
1368 	/*
1369 	 * Fail and return NULL if either the provider name is too long
1370 	 * or we fail to register this new provider with the DTrace
1371 	 * framework. Note that this is the only place we ever construct
1372 	 * the full provider name -- we keep it in pieces in the provider
1373 	 * structure.
1374 	 */
1375 	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1376 	    sizeof (provname) ||
1377 	    dtrace_register(provname, pattr,
1378 	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1379 	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1380 	    &new_fp->ftp_provid) != 0) {
1381 		mutex_exit(&bucket->ftb_mtx);
1382 		fasttrap_provider_free(new_fp);
1383 		crfree(cred);
1384 		return (NULL);
1385 	}
1386 
1387 	new_fp->ftp_next = bucket->ftb_data;
1388 	bucket->ftb_data = new_fp;
1389 
1390 	mutex_enter(&new_fp->ftp_mtx);
1391 	mutex_exit(&bucket->ftb_mtx);
1392 
1393 	crfree(cred);
1394 	return (new_fp);
1395 }
1396 
1397 static void
1398 fasttrap_provider_free(fasttrap_provider_t *provider)
1399 {
1400 	pid_t pid = provider->ftp_pid;
1401 	proc_t *p;
1402 
1403 	/*
1404 	 * There need to be no associated enabled probes, no consumers
1405 	 * creating probes, and no meta providers referencing this provider.
1406 	 */
1407 	ASSERT(provider->ftp_rcount == 0);
1408 	ASSERT(provider->ftp_ccount == 0);
1409 	ASSERT(provider->ftp_mcount == 0);
1410 
1411 	/*
1412 	 * If this provider hasn't been retired, we need to explicitly drop the
1413 	 * count of active providers on the associated process structure.
1414 	 */
1415 	if (!provider->ftp_retired) {
1416 		atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1417 		ASSERT(provider->ftp_proc->ftpc_acount <
1418 		    provider->ftp_proc->ftpc_rcount);
1419 	}
1420 
1421 	fasttrap_proc_release(provider->ftp_proc);
1422 
1423 	kmem_free(provider, sizeof (fasttrap_provider_t));
1424 
1425 	/*
1426 	 * Decrement p_dtrace_probes on the process whose provider we're
1427 	 * freeing. We don't have to worry about clobbering somone else's
1428 	 * modifications to it because we have locked the bucket that
1429 	 * corresponds to this process's hash chain in the provider hash
1430 	 * table. Don't sweat it if we can't find the process.
1431 	 */
1432 	mutex_enter(&pidlock);
1433 	if ((p = prfind(pid)) == NULL) {
1434 		mutex_exit(&pidlock);
1435 		return;
1436 	}
1437 
1438 	mutex_enter(&p->p_lock);
1439 	mutex_exit(&pidlock);
1440 
1441 	p->p_dtrace_probes--;
1442 	mutex_exit(&p->p_lock);
1443 }
1444 
1445 static void
1446 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1447 {
1448 	fasttrap_provider_t *fp;
1449 	fasttrap_bucket_t *bucket;
1450 	dtrace_provider_id_t provid;
1451 
1452 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1453 
1454 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1455 	mutex_enter(&bucket->ftb_mtx);
1456 
1457 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1458 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1459 		    !fp->ftp_retired)
1460 			break;
1461 	}
1462 
1463 	if (fp == NULL) {
1464 		mutex_exit(&bucket->ftb_mtx);
1465 		return;
1466 	}
1467 
1468 	mutex_enter(&fp->ftp_mtx);
1469 	ASSERT(!mprov || fp->ftp_mcount > 0);
1470 	if (mprov && --fp->ftp_mcount != 0)  {
1471 		mutex_exit(&fp->ftp_mtx);
1472 		mutex_exit(&bucket->ftb_mtx);
1473 		return;
1474 	}
1475 
1476 	/*
1477 	 * Mark the provider to be removed in our post-processing step, mark it
1478 	 * retired, and drop the active count on its proc. Marking it indicates
1479 	 * that we should try to remove it; setting the retired flag indicates
1480 	 * that we're done with this provider; dropping the active the proc
1481 	 * releases our hold, and when this reaches zero (as it will during
1482 	 * exit or exec) the proc and associated providers become defunct.
1483 	 *
1484 	 * We obviously need to take the bucket lock before the provider lock
1485 	 * to perform the lookup, but we need to drop the provider lock
1486 	 * before calling into the DTrace framework since we acquire the
1487 	 * provider lock in callbacks invoked from the DTrace framework. The
1488 	 * bucket lock therefore protects the integrity of the provider hash
1489 	 * table.
1490 	 */
1491 	atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1492 	ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1493 
1494 	fp->ftp_retired = 1;
1495 	fp->ftp_marked = 1;
1496 	provid = fp->ftp_provid;
1497 	mutex_exit(&fp->ftp_mtx);
1498 
1499 	/*
1500 	 * We don't have to worry about invalidating the same provider twice
1501 	 * since fasttrap_provider_lookup() will ignore provider that have
1502 	 * been marked as retired.
1503 	 */
1504 	dtrace_invalidate(provid);
1505 
1506 	mutex_exit(&bucket->ftb_mtx);
1507 
1508 	fasttrap_pid_cleanup();
1509 }
1510 
1511 static int
1512 fasttrap_uint32_cmp(const void *ap, const void *bp)
1513 {
1514 	return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1515 }
1516 
1517 static int
1518 fasttrap_uint64_cmp(const void *ap, const void *bp)
1519 {
1520 	return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1521 }
1522 
1523 static int
1524 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1525 {
1526 	fasttrap_provider_t *provider;
1527 	fasttrap_probe_t *pp;
1528 	fasttrap_tracepoint_t *tp;
1529 	char *name;
1530 	int i, aframes, whack;
1531 
1532 	/*
1533 	 * There needs to be at least one desired trace point.
1534 	 */
1535 	if (pdata->ftps_noffs == 0)
1536 		return (EINVAL);
1537 
1538 	switch (pdata->ftps_type) {
1539 	case DTFTP_ENTRY:
1540 		name = "entry";
1541 		aframes = FASTTRAP_ENTRY_AFRAMES;
1542 		break;
1543 	case DTFTP_RETURN:
1544 		name = "return";
1545 		aframes = FASTTRAP_RETURN_AFRAMES;
1546 		break;
1547 	case DTFTP_OFFSETS:
1548 		name = NULL;
1549 		break;
1550 	default:
1551 		return (EINVAL);
1552 	}
1553 
1554 	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1555 	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1556 		return (ESRCH);
1557 
1558 	/*
1559 	 * Increment this reference count to indicate that a consumer is
1560 	 * actively adding a new probe associated with this provider. This
1561 	 * prevents the provider from being deleted -- we'll need to check
1562 	 * for pending deletions when we drop this reference count.
1563 	 */
1564 	provider->ftp_ccount++;
1565 	mutex_exit(&provider->ftp_mtx);
1566 
1567 	/*
1568 	 * Grab the creation lock to ensure consistency between calls to
1569 	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1570 	 * other threads creating probes. We must drop the provider lock
1571 	 * before taking this lock to avoid a three-way deadlock with the
1572 	 * DTrace framework.
1573 	 */
1574 	mutex_enter(&provider->ftp_cmtx);
1575 
1576 	if (name == NULL) {
1577 		for (i = 0; i < pdata->ftps_noffs; i++) {
1578 			char name_str[17];
1579 
1580 			(void) sprintf(name_str, "%llx",
1581 			    (unsigned long long)pdata->ftps_offs[i]);
1582 
1583 			if (dtrace_probe_lookup(provider->ftp_provid,
1584 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1585 				continue;
1586 
1587 			atomic_add_32(&fasttrap_total, 1);
1588 
1589 			if (fasttrap_total > fasttrap_max) {
1590 				atomic_add_32(&fasttrap_total, -1);
1591 				goto no_mem;
1592 			}
1593 
1594 			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1595 
1596 			pp->ftp_prov = provider;
1597 			pp->ftp_faddr = pdata->ftps_pc;
1598 			pp->ftp_fsize = pdata->ftps_size;
1599 			pp->ftp_pid = pdata->ftps_pid;
1600 			pp->ftp_ntps = 1;
1601 
1602 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1603 			    KM_SLEEP);
1604 
1605 			tp->ftt_proc = provider->ftp_proc;
1606 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1607 			tp->ftt_pid = pdata->ftps_pid;
1608 
1609 			pp->ftp_tps[0].fit_tp = tp;
1610 			pp->ftp_tps[0].fit_id.fti_probe = pp;
1611 			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1612 
1613 			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1614 			    pdata->ftps_mod, pdata->ftps_func, name_str,
1615 			    FASTTRAP_OFFSET_AFRAMES, pp);
1616 		}
1617 
1618 	} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1619 	    pdata->ftps_func, name) == 0) {
1620 		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1621 
1622 		if (fasttrap_total > fasttrap_max) {
1623 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1624 			goto no_mem;
1625 		}
1626 
1627 		/*
1628 		 * Make sure all tracepoint program counter values are unique.
1629 		 * We later assume that each probe has exactly one tracepoint
1630 		 * for a given pc.
1631 		 */
1632 		qsort(pdata->ftps_offs, pdata->ftps_noffs,
1633 		    sizeof (uint64_t), fasttrap_uint64_cmp);
1634 		for (i = 1; i < pdata->ftps_noffs; i++) {
1635 			if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1636 				continue;
1637 
1638 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1639 			goto no_mem;
1640 		}
1641 
1642 		ASSERT(pdata->ftps_noffs > 0);
1643 		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1644 		    ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1645 
1646 		pp->ftp_prov = provider;
1647 		pp->ftp_faddr = pdata->ftps_pc;
1648 		pp->ftp_fsize = pdata->ftps_size;
1649 		pp->ftp_pid = pdata->ftps_pid;
1650 		pp->ftp_ntps = pdata->ftps_noffs;
1651 
1652 		for (i = 0; i < pdata->ftps_noffs; i++) {
1653 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1654 			    KM_SLEEP);
1655 
1656 			tp->ftt_proc = provider->ftp_proc;
1657 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1658 			tp->ftt_pid = pdata->ftps_pid;
1659 
1660 			pp->ftp_tps[i].fit_tp = tp;
1661 			pp->ftp_tps[i].fit_id.fti_probe = pp;
1662 			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1663 		}
1664 
1665 		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1666 		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1667 	}
1668 
1669 	mutex_exit(&provider->ftp_cmtx);
1670 
1671 	/*
1672 	 * We know that the provider is still valid since we incremented the
1673 	 * creation reference count. If someone tried to clean up this provider
1674 	 * while we were using it (e.g. because the process called exec(2) or
1675 	 * exit(2)), take note of that and try to clean it up now.
1676 	 */
1677 	mutex_enter(&provider->ftp_mtx);
1678 	provider->ftp_ccount--;
1679 	whack = provider->ftp_retired;
1680 	mutex_exit(&provider->ftp_mtx);
1681 
1682 	if (whack)
1683 		fasttrap_pid_cleanup();
1684 
1685 	return (0);
1686 
1687 no_mem:
1688 	/*
1689 	 * If we've exhausted the allowable resources, we'll try to remove
1690 	 * this provider to free some up. This is to cover the case where
1691 	 * the user has accidentally created many more probes than was
1692 	 * intended (e.g. pid123:::).
1693 	 */
1694 	mutex_exit(&provider->ftp_cmtx);
1695 	mutex_enter(&provider->ftp_mtx);
1696 	provider->ftp_ccount--;
1697 	provider->ftp_marked = 1;
1698 	mutex_exit(&provider->ftp_mtx);
1699 
1700 	fasttrap_pid_cleanup();
1701 
1702 	return (ENOMEM);
1703 }
1704 
1705 /*ARGSUSED*/
1706 static void *
1707 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1708 {
1709 	fasttrap_provider_t *provider;
1710 
1711 	/*
1712 	 * A 32-bit unsigned integer (like a pid for example) can be
1713 	 * expressed in 10 or fewer decimal digits. Make sure that we'll
1714 	 * have enough space for the provider name.
1715 	 */
1716 	if (strlen(dhpv->dthpv_provname) + 10 >=
1717 	    sizeof (provider->ftp_name)) {
1718 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
1719 		    "name too long to accomodate pid", dhpv->dthpv_provname);
1720 		return (NULL);
1721 	}
1722 
1723 	/*
1724 	 * Don't let folks spoof the true pid provider.
1725 	 */
1726 	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1727 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
1728 		    "%s is an invalid name", dhpv->dthpv_provname,
1729 		    FASTTRAP_PID_NAME);
1730 		return (NULL);
1731 	}
1732 
1733 	/*
1734 	 * The highest stability class that fasttrap supports is ISA; cap
1735 	 * the stability of the new provider accordingly.
1736 	 */
1737 	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1738 		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1739 	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1740 		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1741 	if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1742 		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1743 	if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1744 		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1745 	if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1746 		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1747 
1748 	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1749 	    &dhpv->dthpv_pattr)) == NULL) {
1750 		cmn_err(CE_WARN, "failed to instantiate provider %s for "
1751 		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
1752 		return (NULL);
1753 	}
1754 
1755 	/*
1756 	 * Up the meta provider count so this provider isn't removed until
1757 	 * the meta provider has been told to remove it.
1758 	 */
1759 	provider->ftp_mcount++;
1760 
1761 	mutex_exit(&provider->ftp_mtx);
1762 
1763 	return (provider);
1764 }
1765 
1766 /*ARGSUSED*/
1767 static void
1768 fasttrap_meta_create_probe(void *arg, void *parg,
1769     dtrace_helper_probedesc_t *dhpb)
1770 {
1771 	fasttrap_provider_t *provider = parg;
1772 	fasttrap_probe_t *pp;
1773 	fasttrap_tracepoint_t *tp;
1774 	int i, j;
1775 	uint32_t ntps;
1776 
1777 	/*
1778 	 * Since the meta provider count is non-zero we don't have to worry
1779 	 * about this provider disappearing.
1780 	 */
1781 	ASSERT(provider->ftp_mcount > 0);
1782 
1783 	/*
1784 	 * The offsets must be unique.
1785 	 */
1786 	qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1787 	    fasttrap_uint32_cmp);
1788 	for (i = 1; i < dhpb->dthpb_noffs; i++) {
1789 		if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1790 		    dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1791 			return;
1792 	}
1793 
1794 	qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1795 	    fasttrap_uint32_cmp);
1796 	for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1797 		if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1798 		    dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1799 			return;
1800 	}
1801 
1802 	/*
1803 	 * Grab the creation lock to ensure consistency between calls to
1804 	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1805 	 * other threads creating probes.
1806 	 */
1807 	mutex_enter(&provider->ftp_cmtx);
1808 
1809 	if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1810 	    dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1811 		mutex_exit(&provider->ftp_cmtx);
1812 		return;
1813 	}
1814 
1815 	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1816 	ASSERT(ntps > 0);
1817 
1818 	atomic_add_32(&fasttrap_total, ntps);
1819 
1820 	if (fasttrap_total > fasttrap_max) {
1821 		atomic_add_32(&fasttrap_total, -ntps);
1822 		mutex_exit(&provider->ftp_cmtx);
1823 		return;
1824 	}
1825 
1826 	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1827 
1828 	pp->ftp_prov = provider;
1829 	pp->ftp_pid = provider->ftp_pid;
1830 	pp->ftp_ntps = ntps;
1831 	pp->ftp_nargs = dhpb->dthpb_xargc;
1832 	pp->ftp_xtypes = dhpb->dthpb_xtypes;
1833 	pp->ftp_ntypes = dhpb->dthpb_ntypes;
1834 
1835 	/*
1836 	 * First create a tracepoint for each actual point of interest.
1837 	 */
1838 	for (i = 0; i < dhpb->dthpb_noffs; i++) {
1839 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1840 
1841 		tp->ftt_proc = provider->ftp_proc;
1842 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1843 		tp->ftt_pid = provider->ftp_pid;
1844 
1845 		pp->ftp_tps[i].fit_tp = tp;
1846 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1847 #ifdef __sparc
1848 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1849 #else
1850 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1851 #endif
1852 	}
1853 
1854 	/*
1855 	 * Then create a tracepoint for each is-enabled point.
1856 	 */
1857 	for (j = 0; i < ntps; i++, j++) {
1858 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1859 
1860 		tp->ftt_proc = provider->ftp_proc;
1861 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1862 		tp->ftt_pid = provider->ftp_pid;
1863 
1864 		pp->ftp_tps[i].fit_tp = tp;
1865 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1866 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1867 	}
1868 
1869 	/*
1870 	 * If the arguments are shuffled around we set the argument remapping
1871 	 * table. Later, when the probe fires, we only remap the arguments
1872 	 * if the table is non-NULL.
1873 	 */
1874 	for (i = 0; i < dhpb->dthpb_xargc; i++) {
1875 		if (dhpb->dthpb_args[i] != i) {
1876 			pp->ftp_argmap = dhpb->dthpb_args;
1877 			break;
1878 		}
1879 	}
1880 
1881 	/*
1882 	 * The probe is fully constructed -- register it with DTrace.
1883 	 */
1884 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1885 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1886 
1887 	mutex_exit(&provider->ftp_cmtx);
1888 }
1889 
1890 /*ARGSUSED*/
1891 static void
1892 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1893 {
1894 	/*
1895 	 * Clean up the USDT provider. There may be active consumers of the
1896 	 * provider busy adding probes, no damage will actually befall the
1897 	 * provider until that count has dropped to zero. This just puts
1898 	 * the provider on death row.
1899 	 */
1900 	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1901 }
1902 
1903 static dtrace_mops_t fasttrap_mops = {
1904 	fasttrap_meta_create_probe,
1905 	fasttrap_meta_provide,
1906 	fasttrap_meta_remove
1907 };
1908 
1909 /*ARGSUSED*/
1910 static int
1911 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1912 {
1913 	return (0);
1914 }
1915 
1916 /*ARGSUSED*/
1917 static int
1918 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1919 {
1920 	if (!dtrace_attached())
1921 		return (EAGAIN);
1922 
1923 	if (cmd == FASTTRAPIOC_MAKEPROBE) {
1924 		fasttrap_probe_spec_t *uprobe = (void *)arg;
1925 		fasttrap_probe_spec_t *probe;
1926 		uint64_t noffs;
1927 		size_t size;
1928 		int ret;
1929 		char *c;
1930 
1931 		if (copyin(&uprobe->ftps_noffs, &noffs,
1932 		    sizeof (uprobe->ftps_noffs)))
1933 			return (EFAULT);
1934 
1935 		/*
1936 		 * Probes must have at least one tracepoint.
1937 		 */
1938 		if (noffs == 0)
1939 			return (EINVAL);
1940 
1941 		size = sizeof (fasttrap_probe_spec_t) +
1942 		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
1943 
1944 		if (size > 1024 * 1024)
1945 			return (ENOMEM);
1946 
1947 		probe = kmem_alloc(size, KM_SLEEP);
1948 
1949 		if (copyin(uprobe, probe, size) != 0) {
1950 			kmem_free(probe, size);
1951 			return (EFAULT);
1952 		}
1953 
1954 		/*
1955 		 * Verify that the function and module strings contain no
1956 		 * funny characters.
1957 		 */
1958 		for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
1959 			if (*c < 0x20 || 0x7f <= *c) {
1960 				ret = EINVAL;
1961 				goto err;
1962 			}
1963 		}
1964 
1965 		for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
1966 			if (*c < 0x20 || 0x7f <= *c) {
1967 				ret = EINVAL;
1968 				goto err;
1969 			}
1970 		}
1971 
1972 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1973 			proc_t *p;
1974 			pid_t pid = probe->ftps_pid;
1975 
1976 			mutex_enter(&pidlock);
1977 			/*
1978 			 * Report an error if the process doesn't exist
1979 			 * or is actively being birthed.
1980 			 */
1981 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1982 				mutex_exit(&pidlock);
1983 				return (ESRCH);
1984 			}
1985 			mutex_enter(&p->p_lock);
1986 			mutex_exit(&pidlock);
1987 
1988 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
1989 			    VREAD | VWRITE)) != 0) {
1990 				mutex_exit(&p->p_lock);
1991 				return (ret);
1992 			}
1993 
1994 			mutex_exit(&p->p_lock);
1995 		}
1996 
1997 		ret = fasttrap_add_probe(probe);
1998 err:
1999 		kmem_free(probe, size);
2000 
2001 		return (ret);
2002 
2003 	} else if (cmd == FASTTRAPIOC_GETINSTR) {
2004 		fasttrap_instr_query_t instr;
2005 		fasttrap_tracepoint_t *tp;
2006 		uint_t index;
2007 		int ret;
2008 
2009 		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2010 			return (EFAULT);
2011 
2012 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2013 			proc_t *p;
2014 			pid_t pid = instr.ftiq_pid;
2015 
2016 			mutex_enter(&pidlock);
2017 			/*
2018 			 * Report an error if the process doesn't exist
2019 			 * or is actively being birthed.
2020 			 */
2021 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2022 				mutex_exit(&pidlock);
2023 				return (ESRCH);
2024 			}
2025 			mutex_enter(&p->p_lock);
2026 			mutex_exit(&pidlock);
2027 
2028 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2029 			    VREAD)) != 0) {
2030 				mutex_exit(&p->p_lock);
2031 				return (ret);
2032 			}
2033 
2034 			mutex_exit(&p->p_lock);
2035 		}
2036 
2037 		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2038 
2039 		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2040 		tp = fasttrap_tpoints.fth_table[index].ftb_data;
2041 		while (tp != NULL) {
2042 			if (instr.ftiq_pid == tp->ftt_pid &&
2043 			    instr.ftiq_pc == tp->ftt_pc &&
2044 			    tp->ftt_proc->ftpc_acount != 0)
2045 				break;
2046 
2047 			tp = tp->ftt_next;
2048 		}
2049 
2050 		if (tp == NULL) {
2051 			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2052 			return (ENOENT);
2053 		}
2054 
2055 		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2056 		    sizeof (instr.ftiq_instr));
2057 		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2058 
2059 		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2060 			return (EFAULT);
2061 
2062 		return (0);
2063 	}
2064 
2065 	return (EINVAL);
2066 }
2067 
2068 static struct cb_ops fasttrap_cb_ops = {
2069 	fasttrap_open,		/* open */
2070 	nodev,			/* close */
2071 	nulldev,		/* strategy */
2072 	nulldev,		/* print */
2073 	nodev,			/* dump */
2074 	nodev,			/* read */
2075 	nodev,			/* write */
2076 	fasttrap_ioctl,		/* ioctl */
2077 	nodev,			/* devmap */
2078 	nodev,			/* mmap */
2079 	nodev,			/* segmap */
2080 	nochpoll,		/* poll */
2081 	ddi_prop_op,		/* cb_prop_op */
2082 	0,			/* streamtab  */
2083 	D_NEW | D_MP		/* Driver compatibility flag */
2084 };
2085 
2086 /*ARGSUSED*/
2087 static int
2088 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2089 {
2090 	int error;
2091 
2092 	switch (infocmd) {
2093 	case DDI_INFO_DEVT2DEVINFO:
2094 		*result = (void *)fasttrap_devi;
2095 		error = DDI_SUCCESS;
2096 		break;
2097 	case DDI_INFO_DEVT2INSTANCE:
2098 		*result = (void *)0;
2099 		error = DDI_SUCCESS;
2100 		break;
2101 	default:
2102 		error = DDI_FAILURE;
2103 	}
2104 	return (error);
2105 }
2106 
2107 static int
2108 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2109 {
2110 	ulong_t nent;
2111 
2112 	switch (cmd) {
2113 	case DDI_ATTACH:
2114 		break;
2115 	case DDI_RESUME:
2116 		return (DDI_SUCCESS);
2117 	default:
2118 		return (DDI_FAILURE);
2119 	}
2120 
2121 	if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2122 	    DDI_PSEUDO, NULL) == DDI_FAILURE) {
2123 		ddi_remove_minor_node(devi, NULL);
2124 		return (DDI_FAILURE);
2125 	}
2126 
2127 	ddi_report_dev(devi);
2128 	fasttrap_devi = devi;
2129 
2130 	/*
2131 	 * Install our hooks into fork(2), exec(2), and exit(2).
2132 	 */
2133 	dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2134 	dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2135 	dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2136 
2137 	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2138 	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2139 	fasttrap_total = 0;
2140 
2141 	/*
2142 	 * Conjure up the tracepoints hashtable...
2143 	 */
2144 	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2145 	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2146 
2147 	if (nent == 0 || nent > 0x1000000)
2148 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2149 
2150 	if ((nent & (nent - 1)) == 0)
2151 		fasttrap_tpoints.fth_nent = nent;
2152 	else
2153 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2154 	ASSERT(fasttrap_tpoints.fth_nent > 0);
2155 	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2156 	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2157 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2158 
2159 	/*
2160 	 * ... and the providers hash table...
2161 	 */
2162 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2163 	if ((nent & (nent - 1)) == 0)
2164 		fasttrap_provs.fth_nent = nent;
2165 	else
2166 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2167 	ASSERT(fasttrap_provs.fth_nent > 0);
2168 	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2169 	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2170 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2171 
2172 	/*
2173 	 * ... and the procs hash table.
2174 	 */
2175 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2176 	if ((nent & (nent - 1)) == 0)
2177 		fasttrap_procs.fth_nent = nent;
2178 	else
2179 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2180 	ASSERT(fasttrap_procs.fth_nent > 0);
2181 	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2182 	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2183 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2184 
2185 	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2186 	    &fasttrap_meta_id);
2187 
2188 	return (DDI_SUCCESS);
2189 }
2190 
2191 static int
2192 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2193 {
2194 	int i, fail = 0;
2195 	timeout_id_t tmp;
2196 
2197 	switch (cmd) {
2198 	case DDI_DETACH:
2199 		break;
2200 	case DDI_SUSPEND:
2201 		return (DDI_SUCCESS);
2202 	default:
2203 		return (DDI_FAILURE);
2204 	}
2205 
2206 	/*
2207 	 * Unregister the meta-provider to make sure no new fasttrap-
2208 	 * managed providers come along while we're trying to close up
2209 	 * shop. If we fail to detach, we'll need to re-register as a
2210 	 * meta-provider. We can fail to unregister as a meta-provider
2211 	 * if providers we manage still exist.
2212 	 */
2213 	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2214 	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
2215 		return (DDI_FAILURE);
2216 
2217 	/*
2218 	 * Prevent any new timeouts from running by setting fasttrap_timeout
2219 	 * to a non-zero value, and wait for the current timeout to complete.
2220 	 */
2221 	mutex_enter(&fasttrap_cleanup_mtx);
2222 	fasttrap_cleanup_work = 0;
2223 
2224 	while (fasttrap_timeout != (timeout_id_t)1) {
2225 		tmp = fasttrap_timeout;
2226 		fasttrap_timeout = (timeout_id_t)1;
2227 
2228 		if (tmp != 0) {
2229 			mutex_exit(&fasttrap_cleanup_mtx);
2230 			(void) untimeout(tmp);
2231 			mutex_enter(&fasttrap_cleanup_mtx);
2232 		}
2233 	}
2234 
2235 	fasttrap_cleanup_work = 0;
2236 	mutex_exit(&fasttrap_cleanup_mtx);
2237 
2238 	/*
2239 	 * Iterate over all of our providers. If there's still a process
2240 	 * that corresponds to that pid, fail to detach.
2241 	 */
2242 	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2243 		fasttrap_provider_t **fpp, *fp;
2244 		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2245 
2246 		mutex_enter(&bucket->ftb_mtx);
2247 		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2248 		while ((fp = *fpp) != NULL) {
2249 			/*
2250 			 * Acquire and release the lock as a simple way of
2251 			 * waiting for any other consumer to finish with
2252 			 * this provider. A thread must first acquire the
2253 			 * bucket lock so there's no chance of another thread
2254 			 * blocking on the provider's lock.
2255 			 */
2256 			mutex_enter(&fp->ftp_mtx);
2257 			mutex_exit(&fp->ftp_mtx);
2258 
2259 			if (dtrace_unregister(fp->ftp_provid) != 0) {
2260 				fail = 1;
2261 				fpp = &fp->ftp_next;
2262 			} else {
2263 				*fpp = fp->ftp_next;
2264 				fasttrap_provider_free(fp);
2265 			}
2266 		}
2267 
2268 		mutex_exit(&bucket->ftb_mtx);
2269 	}
2270 
2271 	if (fail) {
2272 		uint_t work;
2273 		/*
2274 		 * If we're failing to detach, we need to unblock timeouts
2275 		 * and start a new timeout if any work has accumulated while
2276 		 * we've been unsuccessfully trying to detach.
2277 		 */
2278 		mutex_enter(&fasttrap_cleanup_mtx);
2279 		fasttrap_timeout = 0;
2280 		work = fasttrap_cleanup_work;
2281 		mutex_exit(&fasttrap_cleanup_mtx);
2282 
2283 		if (work)
2284 			fasttrap_pid_cleanup();
2285 
2286 		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2287 		    &fasttrap_meta_id);
2288 
2289 		return (DDI_FAILURE);
2290 	}
2291 
2292 #ifdef DEBUG
2293 	mutex_enter(&fasttrap_count_mtx);
2294 	ASSERT(fasttrap_pid_count == 0);
2295 	mutex_exit(&fasttrap_count_mtx);
2296 #endif
2297 
2298 	kmem_free(fasttrap_tpoints.fth_table,
2299 	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2300 	fasttrap_tpoints.fth_nent = 0;
2301 
2302 	kmem_free(fasttrap_provs.fth_table,
2303 	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2304 	fasttrap_provs.fth_nent = 0;
2305 
2306 	kmem_free(fasttrap_procs.fth_table,
2307 	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2308 	fasttrap_procs.fth_nent = 0;
2309 
2310 	/*
2311 	 * We know there are no tracepoints in any process anywhere in
2312 	 * the system so there is no process which has its p_dtrace_count
2313 	 * greater than zero, therefore we know that no thread can actively
2314 	 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2315 	 * and fasttrap_exec() and fasttrap_exit().
2316 	 */
2317 	ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2318 	dtrace_fasttrap_fork_ptr = NULL;
2319 
2320 	ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2321 	dtrace_fasttrap_exec_ptr = NULL;
2322 
2323 	ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2324 	dtrace_fasttrap_exit_ptr = NULL;
2325 
2326 	ddi_remove_minor_node(devi, NULL);
2327 
2328 	return (DDI_SUCCESS);
2329 }
2330 
2331 static struct dev_ops fasttrap_ops = {
2332 	DEVO_REV,		/* devo_rev */
2333 	0,			/* refcnt */
2334 	fasttrap_info,		/* get_dev_info */
2335 	nulldev,		/* identify */
2336 	nulldev,		/* probe */
2337 	fasttrap_attach,	/* attach */
2338 	fasttrap_detach,	/* detach */
2339 	nodev,			/* reset */
2340 	&fasttrap_cb_ops,	/* driver operations */
2341 	NULL,			/* bus operations */
2342 	nodev			/* dev power */
2343 };
2344 
2345 /*
2346  * Module linkage information for the kernel.
2347  */
2348 static struct modldrv modldrv = {
2349 	&mod_driverops,		/* module type (this is a pseudo driver) */
2350 	"Fasttrap Tracing",	/* name of module */
2351 	&fasttrap_ops,		/* driver ops */
2352 };
2353 
2354 static struct modlinkage modlinkage = {
2355 	MODREV_1,
2356 	(void *)&modldrv,
2357 	NULL
2358 };
2359 
2360 int
2361 _init(void)
2362 {
2363 	return (mod_install(&modlinkage));
2364 }
2365 
2366 int
2367 _info(struct modinfo *modinfop)
2368 {
2369 	return (mod_info(&modlinkage, modinfop));
2370 }
2371 
2372 int
2373 _fini(void)
2374 {
2375 	return (mod_remove(&modlinkage));
2376 }
2377