xref: /freebsd/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c (revision 4f52dfbb8d6c4d446500c5b097e3806ec219fbd4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  * Portions Copyright 2010 The FreeBSD Foundation
22  *
23  * $FreeBSD$
24  */
25 
26 /*
27  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
28  * Use is subject to license terms.
29  */
30 
31 /*
32  * Copyright (c) 2015, Joyent, Inc. All rights reserved.
33  */
34 
35 #include <sys/atomic.h>
36 #include <sys/errno.h>
37 #include <sys/stat.h>
38 #include <sys/modctl.h>
39 #include <sys/conf.h>
40 #include <sys/systm.h>
41 #ifdef illumos
42 #include <sys/ddi.h>
43 #endif
44 #include <sys/sunddi.h>
45 #include <sys/cpuvar.h>
46 #include <sys/kmem.h>
47 #ifdef illumos
48 #include <sys/strsubr.h>
49 #endif
50 #include <sys/fasttrap.h>
51 #include <sys/fasttrap_impl.h>
52 #include <sys/fasttrap_isa.h>
53 #include <sys/dtrace.h>
54 #include <sys/dtrace_impl.h>
55 #include <sys/sysmacros.h>
56 #include <sys/proc.h>
57 #include <sys/policy.h>
58 #ifdef illumos
59 #include <util/qsort.h>
60 #endif
61 #include <sys/mutex.h>
62 #include <sys/kernel.h>
63 #ifndef illumos
64 #include <sys/dtrace_bsd.h>
65 #include <sys/eventhandler.h>
66 #include <sys/rmlock.h>
67 #include <sys/sysent.h>
68 #include <sys/sysctl.h>
69 #include <sys/u8_textprep.h>
70 #include <sys/user.h>
71 
72 #include <vm/vm.h>
73 #include <vm/pmap.h>
74 #include <vm/vm_map.h>
75 #include <vm/vm_param.h>
76 
77 #include <cddl/dev/dtrace/dtrace_cddl.h>
78 #endif
79 
80 /*
81  * User-Land Trap-Based Tracing
82  * ----------------------------
83  *
84  * The fasttrap provider allows DTrace consumers to instrument any user-level
85  * instruction to gather data; this includes probes with semantic
86  * signifigance like entry and return as well as simple offsets into the
87  * function. While the specific techniques used are very ISA specific, the
88  * methodology is generalizable to any architecture.
89  *
90  *
91  * The General Methodology
92  * -----------------------
93  *
94  * With the primary goal of tracing every user-land instruction and the
95  * limitation that we can't trust user space so don't want to rely on much
96  * information there, we begin by replacing the instructions we want to trace
97  * with trap instructions. Each instruction we overwrite is saved into a hash
98  * table keyed by process ID and pc address. When we enter the kernel due to
99  * this trap instruction, we need the effects of the replaced instruction to
100  * appear to have occurred before we proceed with the user thread's
101  * execution.
102  *
103  * Each user level thread is represented by a ulwp_t structure which is
104  * always easily accessible through a register. The most basic way to produce
105  * the effects of the instruction we replaced is to copy that instruction out
106  * to a bit of scratch space reserved in the user thread's ulwp_t structure
107  * (a sort of kernel-private thread local storage), set the PC to that
108  * scratch space and single step. When we reenter the kernel after single
109  * stepping the instruction we must then adjust the PC to point to what would
110  * normally be the next instruction. Of course, special care must be taken
111  * for branches and jumps, but these represent such a small fraction of any
112  * instruction set that writing the code to emulate these in the kernel is
113  * not too difficult.
114  *
115  * Return probes may require several tracepoints to trace every return site,
116  * and, conversely, each tracepoint may activate several probes (the entry
117  * and offset 0 probes, for example). To solve this muliplexing problem,
118  * tracepoints contain lists of probes to activate and probes contain lists
119  * of tracepoints to enable. If a probe is activated, it adds its ID to
120  * existing tracepoints or creates new ones as necessary.
121  *
122  * Most probes are activated _before_ the instruction is executed, but return
123  * probes are activated _after_ the effects of the last instruction of the
124  * function are visible. Return probes must be fired _after_ we have
125  * single-stepped the instruction whereas all other probes are fired
126  * beforehand.
127  *
128  *
129  * Lock Ordering
130  * -------------
131  *
132  * The lock ordering below -- both internally and with respect to the DTrace
133  * framework -- is a little tricky and bears some explanation. Each provider
134  * has a lock (ftp_mtx) that protects its members including reference counts
135  * for enabled probes (ftp_rcount), consumers actively creating probes
136  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
137  * from being freed. A provider is looked up by taking the bucket lock for the
138  * provider hash table, and is returned with its lock held. The provider lock
139  * may be taken in functions invoked by the DTrace framework, but may not be
140  * held while calling functions in the DTrace framework.
141  *
142  * To ensure consistency over multiple calls to the DTrace framework, the
143  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
144  * not be taken when holding the provider lock as that would create a cyclic
145  * lock ordering. In situations where one would naturally take the provider
146  * lock and then the creation lock, we instead up a reference count to prevent
147  * the provider from disappearing, drop the provider lock, and acquire the
148  * creation lock.
149  *
150  * Briefly:
151  * 	bucket lock before provider lock
152  *	DTrace before provider lock
153  *	creation lock before DTrace
154  *	never hold the provider lock and creation lock simultaneously
155  */
156 
157 static d_open_t fasttrap_open;
158 static d_ioctl_t fasttrap_ioctl;
159 
160 static struct cdevsw fasttrap_cdevsw = {
161 	.d_version	= D_VERSION,
162 	.d_open		= fasttrap_open,
163 	.d_ioctl	= fasttrap_ioctl,
164 	.d_name		= "fasttrap",
165 };
166 static struct cdev *fasttrap_cdev;
167 static dtrace_meta_provider_id_t fasttrap_meta_id;
168 
169 static struct proc *fasttrap_cleanup_proc;
170 static struct mtx fasttrap_cleanup_mtx;
171 static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
172 
173 /*
174  * Generation count on modifications to the global tracepoint lookup table.
175  */
176 static volatile uint64_t fasttrap_mod_gen;
177 
178 /*
179  * When the fasttrap provider is loaded, fasttrap_max is set to either
180  * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the
181  * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD).
182  * Each time a probe is created, fasttrap_total is incremented by the number
183  * of tracepoints that may be associated with that probe; fasttrap_total is capped
184  * at fasttrap_max.
185  */
186 #define	FASTTRAP_MAX_DEFAULT		250000
187 static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT;
188 static uint32_t fasttrap_total;
189 
190 /*
191  * Copyright (c) 2011, Joyent, Inc. All rights reserved.
192  */
193 
194 #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
195 #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
196 #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
197 
198 #define	FASTTRAP_PID_NAME		"pid"
199 
200 fasttrap_hash_t			fasttrap_tpoints;
201 static fasttrap_hash_t		fasttrap_provs;
202 static fasttrap_hash_t		fasttrap_procs;
203 
204 static uint64_t			fasttrap_pid_count;	/* pid ref count */
205 static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
206 
207 #define	FASTTRAP_ENABLE_FAIL	1
208 #define	FASTTRAP_ENABLE_PARTIAL	2
209 
210 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
211 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
212 
213 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
214     const dtrace_pattr_t *);
215 static void fasttrap_provider_retire(pid_t, const char *, int);
216 static void fasttrap_provider_free(fasttrap_provider_t *);
217 
218 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
219 static void fasttrap_proc_release(fasttrap_proc_t *);
220 
221 #ifndef illumos
222 static void fasttrap_thread_dtor(void *, struct thread *);
223 #endif
224 
225 #define	FASTTRAP_PROVS_INDEX(pid, name) \
226 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
227 
228 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
229 
230 #ifndef illumos
231 struct rmlock fasttrap_tp_lock;
232 static eventhandler_tag fasttrap_thread_dtor_tag;
233 #endif
234 
235 static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE;
236 
237 #ifdef __FreeBSD__
238 SYSCTL_DECL(_kern_dtrace);
239 SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters");
240 SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max,
241     FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes");
242 SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size,
243     FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table");
244 #endif
245 
246 static int
247 fasttrap_highbit(ulong_t i)
248 {
249 	int h = 1;
250 
251 	if (i == 0)
252 		return (0);
253 #ifdef _LP64
254 	if (i & 0xffffffff00000000ul) {
255 		h += 32; i >>= 32;
256 	}
257 #endif
258 	if (i & 0xffff0000) {
259 		h += 16; i >>= 16;
260 	}
261 	if (i & 0xff00) {
262 		h += 8; i >>= 8;
263 	}
264 	if (i & 0xf0) {
265 		h += 4; i >>= 4;
266 	}
267 	if (i & 0xc) {
268 		h += 2; i >>= 2;
269 	}
270 	if (i & 0x2) {
271 		h += 1;
272 	}
273 	return (h);
274 }
275 
276 static uint_t
277 fasttrap_hash_str(const char *p)
278 {
279 	unsigned int g;
280 	uint_t hval = 0;
281 
282 	while (*p) {
283 		hval = (hval << 4) + *p++;
284 		if ((g = (hval & 0xf0000000)) != 0)
285 			hval ^= g >> 24;
286 		hval &= ~g;
287 	}
288 	return (hval);
289 }
290 
291 void
292 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
293 {
294 #ifdef illumos
295 	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
296 
297 	sqp->sq_info.si_signo = SIGTRAP;
298 	sqp->sq_info.si_code = TRAP_DTRACE;
299 	sqp->sq_info.si_addr = (caddr_t)pc;
300 
301 	mutex_enter(&p->p_lock);
302 	sigaddqa(p, t, sqp);
303 	mutex_exit(&p->p_lock);
304 
305 	if (t != NULL)
306 		aston(t);
307 #else
308 	ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
309 
310 	ksiginfo_init(ksi);
311 	ksi->ksi_signo = SIGTRAP;
312 	ksi->ksi_code = TRAP_DTRACE;
313 	ksi->ksi_addr = (caddr_t)pc;
314 	PROC_LOCK(p);
315 	(void) tdsendsignal(p, t, SIGTRAP, ksi);
316 	PROC_UNLOCK(p);
317 #endif
318 }
319 
320 #ifndef illumos
321 /*
322  * Obtain a chunk of scratch space in the address space of the target process.
323  */
324 fasttrap_scrspace_t *
325 fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
326 {
327 	fasttrap_scrblock_t *scrblk;
328 	fasttrap_scrspace_t *scrspc;
329 	struct proc *p;
330 	vm_offset_t addr;
331 	int error, i;
332 
333 	scrspc = NULL;
334 	if (td->t_dtrace_sscr != NULL) {
335 		/* If the thread already has scratch space, we're done. */
336 		scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
337 		return (scrspc);
338 	}
339 
340 	p = td->td_proc;
341 
342 	mutex_enter(&fprc->ftpc_mtx);
343 	if (LIST_EMPTY(&fprc->ftpc_fscr)) {
344 		/*
345 		 * No scratch space is available, so we'll map a new scratch
346 		 * space block into the traced process' address space.
347 		 */
348 		addr = 0;
349 		error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr,
350 		    FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL,
351 		    VM_PROT_ALL, 0);
352 		if (error != KERN_SUCCESS)
353 			goto done;
354 
355 		scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK);
356 		scrblk->ftsb_addr = addr;
357 		LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next);
358 
359 		/*
360 		 * Carve the block up into chunks and put them on the free list.
361 		 */
362 		for (i = 0;
363 		    i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) {
364 			scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK);
365 			scrspc->ftss_addr = addr +
366 			    i * FASTTRAP_SCRSPACE_SIZE;
367 			LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,
368 			    ftss_next);
369 		}
370 	}
371 
372 	/*
373 	 * Take the first scratch chunk off the free list, put it on the
374 	 * allocated list, and return its address.
375 	 */
376 	scrspc = LIST_FIRST(&fprc->ftpc_fscr);
377 	LIST_REMOVE(scrspc, ftss_next);
378 	LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next);
379 
380 	/*
381 	 * This scratch space is reserved for use by td until the thread exits.
382 	 */
383 	td->t_dtrace_sscr = scrspc;
384 
385 done:
386 	mutex_exit(&fprc->ftpc_mtx);
387 
388 	return (scrspc);
389 }
390 
391 /*
392  * Return any allocated per-thread scratch space chunks back to the process'
393  * free list.
394  */
395 static void
396 fasttrap_thread_dtor(void *arg __unused, struct thread *td)
397 {
398 	fasttrap_bucket_t *bucket;
399 	fasttrap_proc_t *fprc;
400 	fasttrap_scrspace_t *scrspc;
401 	pid_t pid;
402 
403 	if (td->t_dtrace_sscr == NULL)
404 		return;
405 
406 	pid = td->td_proc->p_pid;
407 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
408 	fprc = NULL;
409 
410 	/* Look up the fasttrap process handle for this process. */
411 	mutex_enter(&bucket->ftb_mtx);
412 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
413 		if (fprc->ftpc_pid == pid) {
414 			mutex_enter(&fprc->ftpc_mtx);
415 			mutex_exit(&bucket->ftb_mtx);
416 			break;
417 		}
418 	}
419 	if (fprc == NULL) {
420 		mutex_exit(&bucket->ftb_mtx);
421 		return;
422 	}
423 
424 	scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
425 	LIST_REMOVE(scrspc, ftss_next);
426 	LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next);
427 
428 	mutex_exit(&fprc->ftpc_mtx);
429 }
430 #endif
431 
432 /*
433  * This function ensures that no threads are actively using the memory
434  * associated with probes that were formerly live.
435  */
436 static void
437 fasttrap_mod_barrier(uint64_t gen)
438 {
439 	int i;
440 
441 	if (gen < fasttrap_mod_gen)
442 		return;
443 
444 	fasttrap_mod_gen++;
445 
446 #ifdef illumos
447 	CPU_FOREACH(i) {
448 		mutex_enter(&fasttrap_cpuc_pid_lock[i]);
449 		mutex_exit(&fasttrap_cpuc_pid_lock[i]);
450 	}
451 #else
452 	rm_wlock(&fasttrap_tp_lock);
453 	rm_wunlock(&fasttrap_tp_lock);
454 #endif
455 }
456 
457 /*
458  * This function performs asynchronous cleanup of fasttrap providers. The
459  * Solaris implementation of this mechanism use a timeout that's activated in
460  * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
461  * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
462  * Thus we use a dedicated process to perform the cleanup when requested.
463  */
464 /*ARGSUSED*/
465 static void
466 fasttrap_pid_cleanup_cb(void *data)
467 {
468 	fasttrap_provider_t **fpp, *fp;
469 	fasttrap_bucket_t *bucket;
470 	dtrace_provider_id_t provid;
471 	int i, later = 0, rval;
472 
473 	mtx_lock(&fasttrap_cleanup_mtx);
474 	while (!fasttrap_cleanup_drain || later > 0) {
475 		fasttrap_cleanup_work = 0;
476 		mtx_unlock(&fasttrap_cleanup_mtx);
477 
478 		later = 0;
479 
480 		/*
481 		 * Iterate over all the providers trying to remove the marked
482 		 * ones. If a provider is marked but not retired, we just
483 		 * have to take a crack at removing it -- it's no big deal if
484 		 * we can't.
485 		 */
486 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
487 			bucket = &fasttrap_provs.fth_table[i];
488 			mutex_enter(&bucket->ftb_mtx);
489 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
490 
491 			while ((fp = *fpp) != NULL) {
492 				if (!fp->ftp_marked) {
493 					fpp = &fp->ftp_next;
494 					continue;
495 				}
496 
497 				mutex_enter(&fp->ftp_mtx);
498 
499 				/*
500 				 * If this provider has consumers actively
501 				 * creating probes (ftp_ccount) or is a USDT
502 				 * provider (ftp_mcount), we can't unregister
503 				 * or even condense.
504 				 */
505 				if (fp->ftp_ccount != 0 ||
506 				    fp->ftp_mcount != 0) {
507 					mutex_exit(&fp->ftp_mtx);
508 					fp->ftp_marked = 0;
509 					continue;
510 				}
511 
512 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
513 					fp->ftp_marked = 0;
514 
515 				mutex_exit(&fp->ftp_mtx);
516 
517 				/*
518 				 * If we successfully unregister this
519 				 * provider we can remove it from the hash
520 				 * chain and free the memory. If our attempt
521 				 * to unregister fails and this is a retired
522 				 * provider, increment our flag to try again
523 				 * pretty soon. If we've consumed more than
524 				 * half of our total permitted number of
525 				 * probes call dtrace_condense() to try to
526 				 * clean out the unenabled probes.
527 				 */
528 				provid = fp->ftp_provid;
529 				if ((rval = dtrace_unregister(provid)) != 0) {
530 					if (fasttrap_total > fasttrap_max / 2)
531 						(void) dtrace_condense(provid);
532 
533 					if (rval == EAGAIN)
534 						fp->ftp_marked = 1;
535 
536 					later += fp->ftp_marked;
537 					fpp = &fp->ftp_next;
538 				} else {
539 					*fpp = fp->ftp_next;
540 					fasttrap_provider_free(fp);
541 				}
542 			}
543 			mutex_exit(&bucket->ftb_mtx);
544 		}
545 		mtx_lock(&fasttrap_cleanup_mtx);
546 
547 		/*
548 		 * If we were unable to retire a provider, try again after a
549 		 * second. This situation can occur in certain circumstances
550 		 * where providers cannot be unregistered even though they have
551 		 * no probes enabled because of an execution of dtrace -l or
552 		 * something similar.
553 		 */
554 		if (later > 0 || fasttrap_cleanup_work ||
555 		    fasttrap_cleanup_drain) {
556 			mtx_unlock(&fasttrap_cleanup_mtx);
557 			pause("ftclean", hz);
558 			mtx_lock(&fasttrap_cleanup_mtx);
559 		} else
560 			mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
561 			    0, "ftcl", 0);
562 	}
563 
564 	/*
565 	 * Wake up the thread in fasttrap_unload() now that we're done.
566 	 */
567 	wakeup(&fasttrap_cleanup_drain);
568 	mtx_unlock(&fasttrap_cleanup_mtx);
569 
570 	kthread_exit();
571 }
572 
573 /*
574  * Activates the asynchronous cleanup mechanism.
575  */
576 static void
577 fasttrap_pid_cleanup(void)
578 {
579 
580 	mtx_lock(&fasttrap_cleanup_mtx);
581 	if (!fasttrap_cleanup_work) {
582 		fasttrap_cleanup_work = 1;
583 		wakeup(&fasttrap_cleanup_cv);
584 	}
585 	mtx_unlock(&fasttrap_cleanup_mtx);
586 }
587 
588 /*
589  * This is called from cfork() via dtrace_fasttrap_fork(). The child
590  * process's address space is (roughly) a copy of the parent process's so
591  * we have to remove all the instrumentation we had previously enabled in the
592  * parent.
593  */
594 static void
595 fasttrap_fork(proc_t *p, proc_t *cp)
596 {
597 #ifndef illumos
598 	fasttrap_scrblock_t *scrblk;
599 	fasttrap_proc_t *fprc = NULL;
600 #endif
601 	pid_t ppid = p->p_pid;
602 	int i;
603 
604 	ASSERT(curproc == p);
605 #ifdef illumos
606 	ASSERT(p->p_proc_flag & P_PR_LOCK);
607 #else
608 	PROC_LOCK_ASSERT(p, MA_OWNED);
609 #endif
610 #ifdef illumos
611 	ASSERT(p->p_dtrace_count > 0);
612 #else
613 	/*
614 	 * This check is purposely here instead of in kern_fork.c because,
615 	 * for legal resons, we cannot include the dtrace_cddl.h header
616 	 * inside kern_fork.c and insert if-clause there.
617 	 */
618 	if (p->p_dtrace_count == 0 && p->p_dtrace_helpers == NULL)
619 		return;
620 #endif
621 
622 	ASSERT(cp->p_dtrace_count == 0);
623 
624 	/*
625 	 * This would be simpler and faster if we maintained per-process
626 	 * hash tables of enabled tracepoints. It could, however, potentially
627 	 * slow down execution of a tracepoint since we'd need to go
628 	 * through two levels of indirection. In the future, we should
629 	 * consider either maintaining per-process ancillary lists of
630 	 * enabled tracepoints or hanging a pointer to a per-process hash
631 	 * table of enabled tracepoints off the proc structure.
632 	 */
633 
634 	/*
635 	 * We don't have to worry about the child process disappearing
636 	 * because we're in fork().
637 	 */
638 #ifdef illumos
639 	mtx_lock_spin(&cp->p_slock);
640 	sprlock_proc(cp);
641 	mtx_unlock_spin(&cp->p_slock);
642 #else
643 	/*
644 	 * fasttrap_tracepoint_remove() expects the child process to be
645 	 * unlocked and the VM then expects curproc to be unlocked.
646 	 */
647 	_PHOLD(cp);
648 	PROC_UNLOCK(cp);
649 	PROC_UNLOCK(p);
650 	if (p->p_dtrace_count == 0)
651 		goto dup_helpers;
652 #endif
653 
654 	/*
655 	 * Iterate over every tracepoint looking for ones that belong to the
656 	 * parent process, and remove each from the child process.
657 	 */
658 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
659 		fasttrap_tracepoint_t *tp;
660 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
661 
662 		mutex_enter(&bucket->ftb_mtx);
663 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
664 			if (tp->ftt_pid == ppid &&
665 			    tp->ftt_proc->ftpc_acount != 0) {
666 				int ret = fasttrap_tracepoint_remove(cp, tp);
667 				ASSERT(ret == 0);
668 
669 				/*
670 				 * The count of active providers can only be
671 				 * decremented (i.e. to zero) during exec,
672 				 * exit, and removal of a meta provider so it
673 				 * should be impossible to drop the count
674 				 * mid-fork.
675 				 */
676 				ASSERT(tp->ftt_proc->ftpc_acount != 0);
677 #ifndef illumos
678 				fprc = tp->ftt_proc;
679 #endif
680 			}
681 		}
682 		mutex_exit(&bucket->ftb_mtx);
683 
684 #ifndef illumos
685 		/*
686 		 * Unmap any scratch space inherited from the parent's address
687 		 * space.
688 		 */
689 		if (fprc != NULL) {
690 			mutex_enter(&fprc->ftpc_mtx);
691 			LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) {
692 				vm_map_remove(&cp->p_vmspace->vm_map,
693 				    scrblk->ftsb_addr,
694 				    scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE);
695 			}
696 			mutex_exit(&fprc->ftpc_mtx);
697 		}
698 #endif
699 	}
700 
701 #ifdef illumos
702 	mutex_enter(&cp->p_lock);
703 	sprunlock(cp);
704 #else
705 dup_helpers:
706 	if (p->p_dtrace_helpers != NULL)
707 		dtrace_helpers_duplicate(p, cp);
708 	PROC_LOCK(p);
709 	PROC_LOCK(cp);
710 	_PRELE(cp);
711 #endif
712 }
713 
714 /*
715  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
716  * is set on the proc structure to indicate that there is a pid provider
717  * associated with this process.
718  */
719 static void
720 fasttrap_exec_exit(proc_t *p)
721 {
722 #ifndef illumos
723 	struct thread *td;
724 #endif
725 
726 #ifdef illumos
727 	ASSERT(p == curproc);
728 #else
729 	PROC_LOCK_ASSERT(p, MA_OWNED);
730 	_PHOLD(p);
731 	/*
732 	 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
733 	 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
734 	 * ourselves when a process exits.
735 	 */
736 	FOREACH_THREAD_IN_PROC(p, td)
737 		td->t_dtrace_sscr = NULL;
738 	PROC_UNLOCK(p);
739 #endif
740 
741 	/*
742 	 * We clean up the pid provider for this process here; user-land
743 	 * static probes are handled by the meta-provider remove entry point.
744 	 */
745 	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
746 #ifndef illumos
747 	if (p->p_dtrace_helpers)
748 		dtrace_helpers_destroy(p);
749 	PROC_LOCK(p);
750 	_PRELE(p);
751 #endif
752 }
753 
754 
755 /*ARGSUSED*/
756 static void
757 fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
758 {
759 	/*
760 	 * There are no "default" pid probes.
761 	 */
762 }
763 
764 static int
765 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
766 {
767 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
768 	fasttrap_bucket_t *bucket;
769 	fasttrap_id_t *id;
770 	pid_t pid;
771 	uintptr_t pc;
772 
773 	ASSERT(index < probe->ftp_ntps);
774 
775 	pid = probe->ftp_pid;
776 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
777 	id = &probe->ftp_tps[index].fit_id;
778 
779 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
780 
781 #ifdef illumos
782 	ASSERT(!(p->p_flag & SVFORK));
783 #endif
784 
785 	/*
786 	 * Before we make any modifications, make sure we've imposed a barrier
787 	 * on the generation in which this probe was last modified.
788 	 */
789 	fasttrap_mod_barrier(probe->ftp_gen);
790 
791 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
792 
793 	/*
794 	 * If the tracepoint has already been enabled, just add our id to the
795 	 * list of interested probes. This may be our second time through
796 	 * this path in which case we'll have constructed the tracepoint we'd
797 	 * like to install. If we can't find a match, and have an allocated
798 	 * tracepoint ready to go, enable that one now.
799 	 *
800 	 * A tracepoint whose process is defunct is also considered defunct.
801 	 */
802 again:
803 	mutex_enter(&bucket->ftb_mtx);
804 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
805 		/*
806 		 * Note that it's safe to access the active count on the
807 		 * associated proc structure because we know that at least one
808 		 * provider (this one) will still be around throughout this
809 		 * operation.
810 		 */
811 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
812 		    tp->ftt_proc->ftpc_acount == 0)
813 			continue;
814 
815 		/*
816 		 * Now that we've found a matching tracepoint, it would be
817 		 * a decent idea to confirm that the tracepoint is still
818 		 * enabled and the trap instruction hasn't been overwritten.
819 		 * Since this is a little hairy, we'll punt for now.
820 		 */
821 
822 		/*
823 		 * This can't be the first interested probe. We don't have
824 		 * to worry about another thread being in the midst of
825 		 * deleting this tracepoint (which would be the only valid
826 		 * reason for a tracepoint to have no interested probes)
827 		 * since we're holding P_PR_LOCK for this process.
828 		 */
829 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
830 
831 		switch (id->fti_ptype) {
832 		case DTFTP_ENTRY:
833 		case DTFTP_OFFSETS:
834 		case DTFTP_IS_ENABLED:
835 			id->fti_next = tp->ftt_ids;
836 			membar_producer();
837 			tp->ftt_ids = id;
838 			membar_producer();
839 			break;
840 
841 		case DTFTP_RETURN:
842 		case DTFTP_POST_OFFSETS:
843 			id->fti_next = tp->ftt_retids;
844 			membar_producer();
845 			tp->ftt_retids = id;
846 			membar_producer();
847 			break;
848 
849 		default:
850 			ASSERT(0);
851 		}
852 
853 		mutex_exit(&bucket->ftb_mtx);
854 
855 		if (new_tp != NULL) {
856 			new_tp->ftt_ids = NULL;
857 			new_tp->ftt_retids = NULL;
858 		}
859 
860 		return (0);
861 	}
862 
863 	/*
864 	 * If we have a good tracepoint ready to go, install it now while
865 	 * we have the lock held and no one can screw with us.
866 	 */
867 	if (new_tp != NULL) {
868 		int rc = 0;
869 
870 		new_tp->ftt_next = bucket->ftb_data;
871 		membar_producer();
872 		bucket->ftb_data = new_tp;
873 		membar_producer();
874 		mutex_exit(&bucket->ftb_mtx);
875 
876 		/*
877 		 * Activate the tracepoint in the ISA-specific manner.
878 		 * If this fails, we need to report the failure, but
879 		 * indicate that this tracepoint must still be disabled
880 		 * by calling fasttrap_tracepoint_disable().
881 		 */
882 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
883 			rc = FASTTRAP_ENABLE_PARTIAL;
884 
885 		/*
886 		 * Increment the count of the number of tracepoints active in
887 		 * the victim process.
888 		 */
889 #ifdef illumos
890 		ASSERT(p->p_proc_flag & P_PR_LOCK);
891 #endif
892 		p->p_dtrace_count++;
893 
894 		return (rc);
895 	}
896 
897 	mutex_exit(&bucket->ftb_mtx);
898 
899 	/*
900 	 * Initialize the tracepoint that's been preallocated with the probe.
901 	 */
902 	new_tp = probe->ftp_tps[index].fit_tp;
903 
904 	ASSERT(new_tp->ftt_pid == pid);
905 	ASSERT(new_tp->ftt_pc == pc);
906 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
907 	ASSERT(new_tp->ftt_ids == NULL);
908 	ASSERT(new_tp->ftt_retids == NULL);
909 
910 	switch (id->fti_ptype) {
911 	case DTFTP_ENTRY:
912 	case DTFTP_OFFSETS:
913 	case DTFTP_IS_ENABLED:
914 		id->fti_next = NULL;
915 		new_tp->ftt_ids = id;
916 		break;
917 
918 	case DTFTP_RETURN:
919 	case DTFTP_POST_OFFSETS:
920 		id->fti_next = NULL;
921 		new_tp->ftt_retids = id;
922 		break;
923 
924 	default:
925 		ASSERT(0);
926 	}
927 
928 #ifdef __FreeBSD__
929 	if (SV_PROC_FLAG(p, SV_LP64))
930 		p->p_model = DATAMODEL_LP64;
931 	else
932 		p->p_model = DATAMODEL_ILP32;
933 #endif
934 
935 	/*
936 	 * If the ISA-dependent initialization goes to plan, go back to the
937 	 * beginning and try to install this freshly made tracepoint.
938 	 */
939 	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
940 		goto again;
941 
942 	new_tp->ftt_ids = NULL;
943 	new_tp->ftt_retids = NULL;
944 
945 	return (FASTTRAP_ENABLE_FAIL);
946 }
947 
948 static void
949 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
950 {
951 	fasttrap_bucket_t *bucket;
952 	fasttrap_provider_t *provider = probe->ftp_prov;
953 	fasttrap_tracepoint_t **pp, *tp;
954 	fasttrap_id_t *id, **idp = NULL;
955 	pid_t pid;
956 	uintptr_t pc;
957 
958 	ASSERT(index < probe->ftp_ntps);
959 
960 	pid = probe->ftp_pid;
961 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
962 	id = &probe->ftp_tps[index].fit_id;
963 
964 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
965 
966 	/*
967 	 * Find the tracepoint and make sure that our id is one of the
968 	 * ones registered with it.
969 	 */
970 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
971 	mutex_enter(&bucket->ftb_mtx);
972 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
973 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
974 		    tp->ftt_proc == provider->ftp_proc)
975 			break;
976 	}
977 
978 	/*
979 	 * If we somehow lost this tracepoint, we're in a world of hurt.
980 	 */
981 	ASSERT(tp != NULL);
982 
983 	switch (id->fti_ptype) {
984 	case DTFTP_ENTRY:
985 	case DTFTP_OFFSETS:
986 	case DTFTP_IS_ENABLED:
987 		ASSERT(tp->ftt_ids != NULL);
988 		idp = &tp->ftt_ids;
989 		break;
990 
991 	case DTFTP_RETURN:
992 	case DTFTP_POST_OFFSETS:
993 		ASSERT(tp->ftt_retids != NULL);
994 		idp = &tp->ftt_retids;
995 		break;
996 
997 	default:
998 		ASSERT(0);
999 	}
1000 
1001 	while ((*idp)->fti_probe != probe) {
1002 		idp = &(*idp)->fti_next;
1003 		ASSERT(*idp != NULL);
1004 	}
1005 
1006 	id = *idp;
1007 	*idp = id->fti_next;
1008 	membar_producer();
1009 
1010 	ASSERT(id->fti_probe == probe);
1011 
1012 	/*
1013 	 * If there are other registered enablings of this tracepoint, we're
1014 	 * all done, but if this was the last probe assocated with this
1015 	 * this tracepoint, we need to remove and free it.
1016 	 */
1017 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
1018 
1019 		/*
1020 		 * If the current probe's tracepoint is in use, swap it
1021 		 * for an unused tracepoint.
1022 		 */
1023 		if (tp == probe->ftp_tps[index].fit_tp) {
1024 			fasttrap_probe_t *tmp_probe;
1025 			fasttrap_tracepoint_t **tmp_tp;
1026 			uint_t tmp_index;
1027 
1028 			if (tp->ftt_ids != NULL) {
1029 				tmp_probe = tp->ftt_ids->fti_probe;
1030 				/* LINTED - alignment */
1031 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
1032 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1033 			} else {
1034 				tmp_probe = tp->ftt_retids->fti_probe;
1035 				/* LINTED - alignment */
1036 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
1037 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1038 			}
1039 
1040 			ASSERT(*tmp_tp != NULL);
1041 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
1042 			ASSERT((*tmp_tp)->ftt_ids == NULL);
1043 			ASSERT((*tmp_tp)->ftt_retids == NULL);
1044 
1045 			probe->ftp_tps[index].fit_tp = *tmp_tp;
1046 			*tmp_tp = tp;
1047 		}
1048 
1049 		mutex_exit(&bucket->ftb_mtx);
1050 
1051 		/*
1052 		 * Tag the modified probe with the generation in which it was
1053 		 * changed.
1054 		 */
1055 		probe->ftp_gen = fasttrap_mod_gen;
1056 		return;
1057 	}
1058 
1059 	mutex_exit(&bucket->ftb_mtx);
1060 
1061 	/*
1062 	 * We can't safely remove the tracepoint from the set of active
1063 	 * tracepoints until we've actually removed the fasttrap instruction
1064 	 * from the process's text. We can, however, operate on this
1065 	 * tracepoint secure in the knowledge that no other thread is going to
1066 	 * be looking at it since we hold P_PR_LOCK on the process if it's
1067 	 * live or we hold the provider lock on the process if it's dead and
1068 	 * gone.
1069 	 */
1070 
1071 	/*
1072 	 * We only need to remove the actual instruction if we're looking
1073 	 * at an existing process
1074 	 */
1075 	if (p != NULL) {
1076 		/*
1077 		 * If we fail to restore the instruction we need to kill
1078 		 * this process since it's in a completely unrecoverable
1079 		 * state.
1080 		 */
1081 		if (fasttrap_tracepoint_remove(p, tp) != 0)
1082 			fasttrap_sigtrap(p, NULL, pc);
1083 
1084 		/*
1085 		 * Decrement the count of the number of tracepoints active
1086 		 * in the victim process.
1087 		 */
1088 #ifdef illumos
1089 		ASSERT(p->p_proc_flag & P_PR_LOCK);
1090 #endif
1091 		p->p_dtrace_count--;
1092 	}
1093 
1094 	/*
1095 	 * Remove the probe from the hash table of active tracepoints.
1096 	 */
1097 	mutex_enter(&bucket->ftb_mtx);
1098 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
1099 	ASSERT(*pp != NULL);
1100 	while (*pp != tp) {
1101 		pp = &(*pp)->ftt_next;
1102 		ASSERT(*pp != NULL);
1103 	}
1104 
1105 	*pp = tp->ftt_next;
1106 	membar_producer();
1107 
1108 	mutex_exit(&bucket->ftb_mtx);
1109 
1110 	/*
1111 	 * Tag the modified probe with the generation in which it was changed.
1112 	 */
1113 	probe->ftp_gen = fasttrap_mod_gen;
1114 }
1115 
1116 static void
1117 fasttrap_enable_callbacks(void)
1118 {
1119 	/*
1120 	 * We don't have to play the rw lock game here because we're
1121 	 * providing something rather than taking something away --
1122 	 * we can be sure that no threads have tried to follow this
1123 	 * function pointer yet.
1124 	 */
1125 	mutex_enter(&fasttrap_count_mtx);
1126 	if (fasttrap_pid_count == 0) {
1127 		ASSERT(dtrace_pid_probe_ptr == NULL);
1128 		ASSERT(dtrace_return_probe_ptr == NULL);
1129 		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
1130 		dtrace_return_probe_ptr = &fasttrap_return_probe;
1131 	}
1132 	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
1133 	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
1134 	fasttrap_pid_count++;
1135 	mutex_exit(&fasttrap_count_mtx);
1136 }
1137 
1138 static void
1139 fasttrap_disable_callbacks(void)
1140 {
1141 #ifdef illumos
1142 	ASSERT(MUTEX_HELD(&cpu_lock));
1143 #endif
1144 
1145 
1146 	mutex_enter(&fasttrap_count_mtx);
1147 	ASSERT(fasttrap_pid_count > 0);
1148 	fasttrap_pid_count--;
1149 	if (fasttrap_pid_count == 0) {
1150 #ifdef illumos
1151 		cpu_t *cur, *cpu = CPU;
1152 
1153 		for (cur = cpu->cpu_next_onln; cur != cpu;
1154 		    cur = cur->cpu_next_onln) {
1155 			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
1156 		}
1157 #endif
1158 		dtrace_pid_probe_ptr = NULL;
1159 		dtrace_return_probe_ptr = NULL;
1160 #ifdef illumos
1161 		for (cur = cpu->cpu_next_onln; cur != cpu;
1162 		    cur = cur->cpu_next_onln) {
1163 			rw_exit(&cur->cpu_ft_lock);
1164 		}
1165 #endif
1166 	}
1167 	mutex_exit(&fasttrap_count_mtx);
1168 }
1169 
1170 /*ARGSUSED*/
1171 static void
1172 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
1173 {
1174 	fasttrap_probe_t *probe = parg;
1175 	proc_t *p = NULL;
1176 	int i, rc;
1177 
1178 	ASSERT(probe != NULL);
1179 	ASSERT(!probe->ftp_enabled);
1180 	ASSERT(id == probe->ftp_id);
1181 #ifdef illumos
1182 	ASSERT(MUTEX_HELD(&cpu_lock));
1183 #endif
1184 
1185 	/*
1186 	 * Increment the count of enabled probes on this probe's provider;
1187 	 * the provider can't go away while the probe still exists. We
1188 	 * must increment this even if we aren't able to properly enable
1189 	 * this probe.
1190 	 */
1191 	mutex_enter(&probe->ftp_prov->ftp_mtx);
1192 	probe->ftp_prov->ftp_rcount++;
1193 	mutex_exit(&probe->ftp_prov->ftp_mtx);
1194 
1195 	/*
1196 	 * If this probe's provider is retired (meaning it was valid in a
1197 	 * previously exec'ed incarnation of this address space), bail out. The
1198 	 * provider can't go away while we're in this code path.
1199 	 */
1200 	if (probe->ftp_prov->ftp_retired)
1201 		return;
1202 
1203 	/*
1204 	 * If we can't find the process, it may be that we're in the context of
1205 	 * a fork in which the traced process is being born and we're copying
1206 	 * USDT probes. Otherwise, the process is gone so bail.
1207 	 */
1208 #ifdef illumos
1209 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
1210 		if ((curproc->p_flag & SFORKING) == 0)
1211 			return;
1212 
1213 		mutex_enter(&pidlock);
1214 		p = prfind(probe->ftp_pid);
1215 
1216 		if (p == NULL) {
1217 			/*
1218 			 * So it's not that the target process is being born,
1219 			 * it's that it isn't there at all (and we simply
1220 			 * happen to be forking).  Anyway, we know that the
1221 			 * target is definitely gone, so bail out.
1222 			 */
1223 			mutex_exit(&pidlock);
1224 			return (0);
1225 		}
1226 
1227 		/*
1228 		 * Confirm that curproc is indeed forking the process in which
1229 		 * we're trying to enable probes.
1230 		 */
1231 		ASSERT(p->p_parent == curproc);
1232 		ASSERT(p->p_stat == SIDL);
1233 
1234 		mutex_enter(&p->p_lock);
1235 		mutex_exit(&pidlock);
1236 
1237 		sprlock_proc(p);
1238 	}
1239 
1240 	ASSERT(!(p->p_flag & SVFORK));
1241 	mutex_exit(&p->p_lock);
1242 #else
1243 	if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0)
1244 		return;
1245 #endif
1246 
1247 	/*
1248 	 * We have to enable the trap entry point before any user threads have
1249 	 * the chance to execute the trap instruction we're about to place
1250 	 * in their process's text.
1251 	 */
1252 	fasttrap_enable_callbacks();
1253 
1254 	/*
1255 	 * Enable all the tracepoints and add this probe's id to each
1256 	 * tracepoint's list of active probes.
1257 	 */
1258 	for (i = 0; i < probe->ftp_ntps; i++) {
1259 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1260 			/*
1261 			 * If enabling the tracepoint failed completely,
1262 			 * we don't have to disable it; if the failure
1263 			 * was only partial we must disable it.
1264 			 */
1265 			if (rc == FASTTRAP_ENABLE_FAIL)
1266 				i--;
1267 			else
1268 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1269 
1270 			/*
1271 			 * Back up and pull out all the tracepoints we've
1272 			 * created so far for this probe.
1273 			 */
1274 			while (i >= 0) {
1275 				fasttrap_tracepoint_disable(p, probe, i);
1276 				i--;
1277 			}
1278 
1279 #ifdef illumos
1280 			mutex_enter(&p->p_lock);
1281 			sprunlock(p);
1282 #else
1283 			PRELE(p);
1284 #endif
1285 
1286 			/*
1287 			 * Since we're not actually enabling this probe,
1288 			 * drop our reference on the trap table entry.
1289 			 */
1290 			fasttrap_disable_callbacks();
1291 			return;
1292 		}
1293 	}
1294 #ifdef illumos
1295 	mutex_enter(&p->p_lock);
1296 	sprunlock(p);
1297 #else
1298 	PRELE(p);
1299 #endif
1300 
1301 	probe->ftp_enabled = 1;
1302 }
1303 
1304 /*ARGSUSED*/
1305 static void
1306 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1307 {
1308 	fasttrap_probe_t *probe = parg;
1309 	fasttrap_provider_t *provider = probe->ftp_prov;
1310 	proc_t *p;
1311 	int i, whack = 0;
1312 
1313 	ASSERT(id == probe->ftp_id);
1314 
1315 	mutex_enter(&provider->ftp_mtx);
1316 
1317 	/*
1318 	 * We won't be able to acquire a /proc-esque lock on the process
1319 	 * iff the process is dead and gone. In this case, we rely on the
1320 	 * provider lock as a point of mutual exclusion to prevent other
1321 	 * DTrace consumers from disabling this probe.
1322 	 */
1323 	if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0)
1324 		p = NULL;
1325 
1326 	/*
1327 	 * Disable all the associated tracepoints (for fully enabled probes).
1328 	 */
1329 	if (probe->ftp_enabled) {
1330 		for (i = 0; i < probe->ftp_ntps; i++) {
1331 			fasttrap_tracepoint_disable(p, probe, i);
1332 		}
1333 	}
1334 
1335 	ASSERT(provider->ftp_rcount > 0);
1336 	provider->ftp_rcount--;
1337 
1338 	if (p != NULL) {
1339 		/*
1340 		 * Even though we may not be able to remove it entirely, we
1341 		 * mark this retired provider to get a chance to remove some
1342 		 * of the associated probes.
1343 		 */
1344 		if (provider->ftp_retired && !provider->ftp_marked)
1345 			whack = provider->ftp_marked = 1;
1346 		mutex_exit(&provider->ftp_mtx);
1347 	} else {
1348 		/*
1349 		 * If the process is dead, we're just waiting for the
1350 		 * last probe to be disabled to be able to free it.
1351 		 */
1352 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1353 			whack = provider->ftp_marked = 1;
1354 		mutex_exit(&provider->ftp_mtx);
1355 	}
1356 
1357 	if (whack)
1358 		fasttrap_pid_cleanup();
1359 
1360 #ifdef __FreeBSD__
1361 	if (p != NULL)
1362 		PRELE(p);
1363 #endif
1364 	if (!probe->ftp_enabled)
1365 		return;
1366 
1367 	probe->ftp_enabled = 0;
1368 
1369 #ifdef illumos
1370 	ASSERT(MUTEX_HELD(&cpu_lock));
1371 #endif
1372 	fasttrap_disable_callbacks();
1373 }
1374 
1375 /*ARGSUSED*/
1376 static void
1377 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1378     dtrace_argdesc_t *desc)
1379 {
1380 	fasttrap_probe_t *probe = parg;
1381 	char *str;
1382 	int i, ndx;
1383 
1384 	desc->dtargd_native[0] = '\0';
1385 	desc->dtargd_xlate[0] = '\0';
1386 
1387 	if (probe->ftp_prov->ftp_retired != 0 ||
1388 	    desc->dtargd_ndx >= probe->ftp_nargs) {
1389 		desc->dtargd_ndx = DTRACE_ARGNONE;
1390 		return;
1391 	}
1392 
1393 	ndx = (probe->ftp_argmap != NULL) ?
1394 	    probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1395 
1396 	str = probe->ftp_ntypes;
1397 	for (i = 0; i < ndx; i++) {
1398 		str += strlen(str) + 1;
1399 	}
1400 
1401 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1402 	(void) strcpy(desc->dtargd_native, str);
1403 
1404 	if (probe->ftp_xtypes == NULL)
1405 		return;
1406 
1407 	str = probe->ftp_xtypes;
1408 	for (i = 0; i < desc->dtargd_ndx; i++) {
1409 		str += strlen(str) + 1;
1410 	}
1411 
1412 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1413 	(void) strcpy(desc->dtargd_xlate, str);
1414 }
1415 
1416 /*ARGSUSED*/
1417 static void
1418 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1419 {
1420 	fasttrap_probe_t *probe = parg;
1421 	int i;
1422 	size_t size;
1423 
1424 	ASSERT(probe != NULL);
1425 	ASSERT(!probe->ftp_enabled);
1426 	ASSERT(fasttrap_total >= probe->ftp_ntps);
1427 
1428 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1429 	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1430 
1431 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1432 		fasttrap_mod_barrier(probe->ftp_gen);
1433 
1434 	for (i = 0; i < probe->ftp_ntps; i++) {
1435 		kmem_free(probe->ftp_tps[i].fit_tp,
1436 		    sizeof (fasttrap_tracepoint_t));
1437 	}
1438 
1439 	kmem_free(probe, size);
1440 }
1441 
1442 
1443 static const dtrace_pattr_t pid_attr = {
1444 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1445 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1446 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1447 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1448 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1449 };
1450 
1451 static dtrace_pops_t pid_pops = {
1452 	.dtps_provide =		fasttrap_pid_provide,
1453 	.dtps_provide_module =	NULL,
1454 	.dtps_enable =		fasttrap_pid_enable,
1455 	.dtps_disable =		fasttrap_pid_disable,
1456 	.dtps_suspend =		NULL,
1457 	.dtps_resume =		NULL,
1458 	.dtps_getargdesc =	fasttrap_pid_getargdesc,
1459 	.dtps_getargval =	fasttrap_pid_getarg,
1460 	.dtps_usermode =	NULL,
1461 	.dtps_destroy =		fasttrap_pid_destroy
1462 };
1463 
1464 static dtrace_pops_t usdt_pops = {
1465 	.dtps_provide =		fasttrap_pid_provide,
1466 	.dtps_provide_module =	NULL,
1467 	.dtps_enable =		fasttrap_pid_enable,
1468 	.dtps_disable =		fasttrap_pid_disable,
1469 	.dtps_suspend =		NULL,
1470 	.dtps_resume =		NULL,
1471 	.dtps_getargdesc =	fasttrap_pid_getargdesc,
1472 	.dtps_getargval =	fasttrap_usdt_getarg,
1473 	.dtps_usermode =	NULL,
1474 	.dtps_destroy =		fasttrap_pid_destroy
1475 };
1476 
1477 static fasttrap_proc_t *
1478 fasttrap_proc_lookup(pid_t pid)
1479 {
1480 	fasttrap_bucket_t *bucket;
1481 	fasttrap_proc_t *fprc, *new_fprc;
1482 
1483 
1484 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1485 	mutex_enter(&bucket->ftb_mtx);
1486 
1487 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1488 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1489 			mutex_enter(&fprc->ftpc_mtx);
1490 			mutex_exit(&bucket->ftb_mtx);
1491 			fprc->ftpc_rcount++;
1492 			atomic_inc_64(&fprc->ftpc_acount);
1493 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1494 			mutex_exit(&fprc->ftpc_mtx);
1495 
1496 			return (fprc);
1497 		}
1498 	}
1499 
1500 	/*
1501 	 * Drop the bucket lock so we don't try to perform a sleeping
1502 	 * allocation under it.
1503 	 */
1504 	mutex_exit(&bucket->ftb_mtx);
1505 
1506 	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1507 	new_fprc->ftpc_pid = pid;
1508 	new_fprc->ftpc_rcount = 1;
1509 	new_fprc->ftpc_acount = 1;
1510 #ifndef illumos
1511 	mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1512 	    NULL);
1513 #endif
1514 
1515 	mutex_enter(&bucket->ftb_mtx);
1516 
1517 	/*
1518 	 * Take another lap through the list to make sure a proc hasn't
1519 	 * been created for this pid while we weren't under the bucket lock.
1520 	 */
1521 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1522 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1523 			mutex_enter(&fprc->ftpc_mtx);
1524 			mutex_exit(&bucket->ftb_mtx);
1525 			fprc->ftpc_rcount++;
1526 			atomic_inc_64(&fprc->ftpc_acount);
1527 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1528 			mutex_exit(&fprc->ftpc_mtx);
1529 
1530 			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1531 
1532 			return (fprc);
1533 		}
1534 	}
1535 
1536 	new_fprc->ftpc_next = bucket->ftb_data;
1537 	bucket->ftb_data = new_fprc;
1538 
1539 	mutex_exit(&bucket->ftb_mtx);
1540 
1541 	return (new_fprc);
1542 }
1543 
1544 static void
1545 fasttrap_proc_release(fasttrap_proc_t *proc)
1546 {
1547 	fasttrap_bucket_t *bucket;
1548 	fasttrap_proc_t *fprc, **fprcp;
1549 	pid_t pid = proc->ftpc_pid;
1550 #ifndef illumos
1551 	fasttrap_scrblock_t *scrblk, *scrblktmp;
1552 	fasttrap_scrspace_t *scrspc, *scrspctmp;
1553 	struct proc *p;
1554 	struct thread *td;
1555 #endif
1556 
1557 	mutex_enter(&proc->ftpc_mtx);
1558 
1559 	ASSERT(proc->ftpc_rcount != 0);
1560 	ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1561 
1562 	if (--proc->ftpc_rcount != 0) {
1563 		mutex_exit(&proc->ftpc_mtx);
1564 		return;
1565 	}
1566 
1567 #ifndef illumos
1568 	/*
1569 	 * Free all structures used to manage per-thread scratch space.
1570 	 */
1571 	LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,
1572 	    scrblktmp) {
1573 		LIST_REMOVE(scrblk, ftsb_next);
1574 		free(scrblk, M_SOLARIS);
1575 	}
1576 	LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) {
1577 		LIST_REMOVE(scrspc, ftss_next);
1578 		free(scrspc, M_SOLARIS);
1579 	}
1580 	LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) {
1581 		LIST_REMOVE(scrspc, ftss_next);
1582 		free(scrspc, M_SOLARIS);
1583 	}
1584 
1585 	if ((p = pfind(pid)) != NULL) {
1586 		FOREACH_THREAD_IN_PROC(p, td)
1587 			td->t_dtrace_sscr = NULL;
1588 		PROC_UNLOCK(p);
1589 	}
1590 #endif
1591 
1592 	mutex_exit(&proc->ftpc_mtx);
1593 
1594 	/*
1595 	 * There should definitely be no live providers associated with this
1596 	 * process at this point.
1597 	 */
1598 	ASSERT(proc->ftpc_acount == 0);
1599 
1600 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1601 	mutex_enter(&bucket->ftb_mtx);
1602 
1603 	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1604 	while ((fprc = *fprcp) != NULL) {
1605 		if (fprc == proc)
1606 			break;
1607 
1608 		fprcp = &fprc->ftpc_next;
1609 	}
1610 
1611 	/*
1612 	 * Something strange has happened if we can't find the proc.
1613 	 */
1614 	ASSERT(fprc != NULL);
1615 
1616 	*fprcp = fprc->ftpc_next;
1617 
1618 	mutex_exit(&bucket->ftb_mtx);
1619 
1620 	kmem_free(fprc, sizeof (fasttrap_proc_t));
1621 }
1622 
1623 /*
1624  * Lookup a fasttrap-managed provider based on its name and associated pid.
1625  * If the pattr argument is non-NULL, this function instantiates the provider
1626  * if it doesn't exist otherwise it returns NULL. The provider is returned
1627  * with its lock held.
1628  */
1629 static fasttrap_provider_t *
1630 fasttrap_provider_lookup(pid_t pid, const char *name,
1631     const dtrace_pattr_t *pattr)
1632 {
1633 	fasttrap_provider_t *fp, *new_fp = NULL;
1634 	fasttrap_bucket_t *bucket;
1635 	char provname[DTRACE_PROVNAMELEN];
1636 	proc_t *p;
1637 	cred_t *cred;
1638 
1639 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1640 	ASSERT(pattr != NULL);
1641 
1642 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1643 	mutex_enter(&bucket->ftb_mtx);
1644 
1645 	/*
1646 	 * Take a lap through the list and return the match if we find it.
1647 	 */
1648 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1649 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1650 		    !fp->ftp_retired) {
1651 			mutex_enter(&fp->ftp_mtx);
1652 			mutex_exit(&bucket->ftb_mtx);
1653 			return (fp);
1654 		}
1655 	}
1656 
1657 	/*
1658 	 * Drop the bucket lock so we don't try to perform a sleeping
1659 	 * allocation under it.
1660 	 */
1661 	mutex_exit(&bucket->ftb_mtx);
1662 
1663 	/*
1664 	 * Make sure the process exists, isn't a child created as the result
1665 	 * of a vfork(2), and isn't a zombie (but may be in fork).
1666 	 */
1667 	if ((p = pfind(pid)) == NULL)
1668 		return (NULL);
1669 
1670 	/*
1671 	 * Increment p_dtrace_probes so that the process knows to inform us
1672 	 * when it exits or execs. fasttrap_provider_free() decrements this
1673 	 * when we're done with this provider.
1674 	 */
1675 	p->p_dtrace_probes++;
1676 
1677 	/*
1678 	 * Grab the credentials for this process so we have
1679 	 * something to pass to dtrace_register().
1680 	 */
1681 	PROC_LOCK_ASSERT(p, MA_OWNED);
1682 	crhold(p->p_ucred);
1683 	cred = p->p_ucred;
1684 	PROC_UNLOCK(p);
1685 
1686 	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1687 	new_fp->ftp_pid = pid;
1688 	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1689 #ifndef illumos
1690 	mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1691 	mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1692 #endif
1693 
1694 	ASSERT(new_fp->ftp_proc != NULL);
1695 
1696 	mutex_enter(&bucket->ftb_mtx);
1697 
1698 	/*
1699 	 * Take another lap through the list to make sure a provider hasn't
1700 	 * been created for this pid while we weren't under the bucket lock.
1701 	 */
1702 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1703 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1704 		    !fp->ftp_retired) {
1705 			mutex_enter(&fp->ftp_mtx);
1706 			mutex_exit(&bucket->ftb_mtx);
1707 			fasttrap_provider_free(new_fp);
1708 			crfree(cred);
1709 			return (fp);
1710 		}
1711 	}
1712 
1713 	(void) strcpy(new_fp->ftp_name, name);
1714 
1715 	/*
1716 	 * Fail and return NULL if either the provider name is too long
1717 	 * or we fail to register this new provider with the DTrace
1718 	 * framework. Note that this is the only place we ever construct
1719 	 * the full provider name -- we keep it in pieces in the provider
1720 	 * structure.
1721 	 */
1722 	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1723 	    sizeof (provname) ||
1724 	    dtrace_register(provname, pattr,
1725 	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1726 	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1727 	    &new_fp->ftp_provid) != 0) {
1728 		mutex_exit(&bucket->ftb_mtx);
1729 		fasttrap_provider_free(new_fp);
1730 		crfree(cred);
1731 		return (NULL);
1732 	}
1733 
1734 	new_fp->ftp_next = bucket->ftb_data;
1735 	bucket->ftb_data = new_fp;
1736 
1737 	mutex_enter(&new_fp->ftp_mtx);
1738 	mutex_exit(&bucket->ftb_mtx);
1739 
1740 	crfree(cred);
1741 	return (new_fp);
1742 }
1743 
1744 static void
1745 fasttrap_provider_free(fasttrap_provider_t *provider)
1746 {
1747 	pid_t pid = provider->ftp_pid;
1748 	proc_t *p;
1749 
1750 	/*
1751 	 * There need to be no associated enabled probes, no consumers
1752 	 * creating probes, and no meta providers referencing this provider.
1753 	 */
1754 	ASSERT(provider->ftp_rcount == 0);
1755 	ASSERT(provider->ftp_ccount == 0);
1756 	ASSERT(provider->ftp_mcount == 0);
1757 
1758 	/*
1759 	 * If this provider hasn't been retired, we need to explicitly drop the
1760 	 * count of active providers on the associated process structure.
1761 	 */
1762 	if (!provider->ftp_retired) {
1763 		atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1764 		ASSERT(provider->ftp_proc->ftpc_acount <
1765 		    provider->ftp_proc->ftpc_rcount);
1766 	}
1767 
1768 	fasttrap_proc_release(provider->ftp_proc);
1769 
1770 #ifndef illumos
1771 	mutex_destroy(&provider->ftp_mtx);
1772 	mutex_destroy(&provider->ftp_cmtx);
1773 #endif
1774 	kmem_free(provider, sizeof (fasttrap_provider_t));
1775 
1776 	/*
1777 	 * Decrement p_dtrace_probes on the process whose provider we're
1778 	 * freeing. We don't have to worry about clobbering somone else's
1779 	 * modifications to it because we have locked the bucket that
1780 	 * corresponds to this process's hash chain in the provider hash
1781 	 * table. Don't sweat it if we can't find the process.
1782 	 */
1783 	if ((p = pfind(pid)) == NULL) {
1784 		return;
1785 	}
1786 
1787 	p->p_dtrace_probes--;
1788 #ifndef illumos
1789 	PROC_UNLOCK(p);
1790 #endif
1791 }
1792 
1793 static void
1794 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1795 {
1796 	fasttrap_provider_t *fp;
1797 	fasttrap_bucket_t *bucket;
1798 	dtrace_provider_id_t provid;
1799 
1800 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1801 
1802 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1803 	mutex_enter(&bucket->ftb_mtx);
1804 
1805 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1806 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1807 		    !fp->ftp_retired)
1808 			break;
1809 	}
1810 
1811 	if (fp == NULL) {
1812 		mutex_exit(&bucket->ftb_mtx);
1813 		return;
1814 	}
1815 
1816 	mutex_enter(&fp->ftp_mtx);
1817 	ASSERT(!mprov || fp->ftp_mcount > 0);
1818 	if (mprov && --fp->ftp_mcount != 0)  {
1819 		mutex_exit(&fp->ftp_mtx);
1820 		mutex_exit(&bucket->ftb_mtx);
1821 		return;
1822 	}
1823 
1824 	/*
1825 	 * Mark the provider to be removed in our post-processing step, mark it
1826 	 * retired, and drop the active count on its proc. Marking it indicates
1827 	 * that we should try to remove it; setting the retired flag indicates
1828 	 * that we're done with this provider; dropping the active the proc
1829 	 * releases our hold, and when this reaches zero (as it will during
1830 	 * exit or exec) the proc and associated providers become defunct.
1831 	 *
1832 	 * We obviously need to take the bucket lock before the provider lock
1833 	 * to perform the lookup, but we need to drop the provider lock
1834 	 * before calling into the DTrace framework since we acquire the
1835 	 * provider lock in callbacks invoked from the DTrace framework. The
1836 	 * bucket lock therefore protects the integrity of the provider hash
1837 	 * table.
1838 	 */
1839 	atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1840 	ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1841 
1842 	fp->ftp_retired = 1;
1843 	fp->ftp_marked = 1;
1844 	provid = fp->ftp_provid;
1845 	mutex_exit(&fp->ftp_mtx);
1846 
1847 	/*
1848 	 * We don't have to worry about invalidating the same provider twice
1849 	 * since fasttrap_provider_lookup() will ignore provider that have
1850 	 * been marked as retired.
1851 	 */
1852 	dtrace_invalidate(provid);
1853 
1854 	mutex_exit(&bucket->ftb_mtx);
1855 
1856 	fasttrap_pid_cleanup();
1857 }
1858 
1859 static int
1860 fasttrap_uint32_cmp(const void *ap, const void *bp)
1861 {
1862 	return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1863 }
1864 
1865 static int
1866 fasttrap_uint64_cmp(const void *ap, const void *bp)
1867 {
1868 	return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1869 }
1870 
1871 static int
1872 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1873 {
1874 	fasttrap_provider_t *provider;
1875 	fasttrap_probe_t *pp;
1876 	fasttrap_tracepoint_t *tp;
1877 	char *name;
1878 	int i, aframes = 0, whack;
1879 
1880 	/*
1881 	 * There needs to be at least one desired trace point.
1882 	 */
1883 	if (pdata->ftps_noffs == 0)
1884 		return (EINVAL);
1885 
1886 	switch (pdata->ftps_type) {
1887 	case DTFTP_ENTRY:
1888 		name = "entry";
1889 		aframes = FASTTRAP_ENTRY_AFRAMES;
1890 		break;
1891 	case DTFTP_RETURN:
1892 		name = "return";
1893 		aframes = FASTTRAP_RETURN_AFRAMES;
1894 		break;
1895 	case DTFTP_OFFSETS:
1896 		name = NULL;
1897 		break;
1898 	default:
1899 		return (EINVAL);
1900 	}
1901 
1902 	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1903 	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1904 		return (ESRCH);
1905 
1906 	/*
1907 	 * Increment this reference count to indicate that a consumer is
1908 	 * actively adding a new probe associated with this provider. This
1909 	 * prevents the provider from being deleted -- we'll need to check
1910 	 * for pending deletions when we drop this reference count.
1911 	 */
1912 	provider->ftp_ccount++;
1913 	mutex_exit(&provider->ftp_mtx);
1914 
1915 	/*
1916 	 * Grab the creation lock to ensure consistency between calls to
1917 	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1918 	 * other threads creating probes. We must drop the provider lock
1919 	 * before taking this lock to avoid a three-way deadlock with the
1920 	 * DTrace framework.
1921 	 */
1922 	mutex_enter(&provider->ftp_cmtx);
1923 
1924 	if (name == NULL) {
1925 		for (i = 0; i < pdata->ftps_noffs; i++) {
1926 			char name_str[17];
1927 
1928 			(void) sprintf(name_str, "%llx",
1929 			    (unsigned long long)pdata->ftps_offs[i]);
1930 
1931 			if (dtrace_probe_lookup(provider->ftp_provid,
1932 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1933 				continue;
1934 
1935 			atomic_inc_32(&fasttrap_total);
1936 
1937 			if (fasttrap_total > fasttrap_max) {
1938 				atomic_dec_32(&fasttrap_total);
1939 				goto no_mem;
1940 			}
1941 
1942 			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1943 
1944 			pp->ftp_prov = provider;
1945 			pp->ftp_faddr = pdata->ftps_pc;
1946 			pp->ftp_fsize = pdata->ftps_size;
1947 			pp->ftp_pid = pdata->ftps_pid;
1948 			pp->ftp_ntps = 1;
1949 
1950 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1951 			    KM_SLEEP);
1952 
1953 			tp->ftt_proc = provider->ftp_proc;
1954 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1955 			tp->ftt_pid = pdata->ftps_pid;
1956 
1957 			pp->ftp_tps[0].fit_tp = tp;
1958 			pp->ftp_tps[0].fit_id.fti_probe = pp;
1959 			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1960 
1961 			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1962 			    pdata->ftps_mod, pdata->ftps_func, name_str,
1963 			    FASTTRAP_OFFSET_AFRAMES, pp);
1964 		}
1965 
1966 	} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1967 	    pdata->ftps_func, name) == 0) {
1968 		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1969 
1970 		if (fasttrap_total > fasttrap_max) {
1971 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1972 			goto no_mem;
1973 		}
1974 
1975 		/*
1976 		 * Make sure all tracepoint program counter values are unique.
1977 		 * We later assume that each probe has exactly one tracepoint
1978 		 * for a given pc.
1979 		 */
1980 		qsort(pdata->ftps_offs, pdata->ftps_noffs,
1981 		    sizeof (uint64_t), fasttrap_uint64_cmp);
1982 		for (i = 1; i < pdata->ftps_noffs; i++) {
1983 			if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1984 				continue;
1985 
1986 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1987 			goto no_mem;
1988 		}
1989 
1990 		ASSERT(pdata->ftps_noffs > 0);
1991 		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1992 		    ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1993 
1994 		pp->ftp_prov = provider;
1995 		pp->ftp_faddr = pdata->ftps_pc;
1996 		pp->ftp_fsize = pdata->ftps_size;
1997 		pp->ftp_pid = pdata->ftps_pid;
1998 		pp->ftp_ntps = pdata->ftps_noffs;
1999 
2000 		for (i = 0; i < pdata->ftps_noffs; i++) {
2001 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
2002 			    KM_SLEEP);
2003 
2004 			tp->ftt_proc = provider->ftp_proc;
2005 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
2006 			tp->ftt_pid = pdata->ftps_pid;
2007 
2008 			pp->ftp_tps[i].fit_tp = tp;
2009 			pp->ftp_tps[i].fit_id.fti_probe = pp;
2010 			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
2011 		}
2012 
2013 		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
2014 		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
2015 	}
2016 
2017 	mutex_exit(&provider->ftp_cmtx);
2018 
2019 	/*
2020 	 * We know that the provider is still valid since we incremented the
2021 	 * creation reference count. If someone tried to clean up this provider
2022 	 * while we were using it (e.g. because the process called exec(2) or
2023 	 * exit(2)), take note of that and try to clean it up now.
2024 	 */
2025 	mutex_enter(&provider->ftp_mtx);
2026 	provider->ftp_ccount--;
2027 	whack = provider->ftp_retired;
2028 	mutex_exit(&provider->ftp_mtx);
2029 
2030 	if (whack)
2031 		fasttrap_pid_cleanup();
2032 
2033 	return (0);
2034 
2035 no_mem:
2036 	/*
2037 	 * If we've exhausted the allowable resources, we'll try to remove
2038 	 * this provider to free some up. This is to cover the case where
2039 	 * the user has accidentally created many more probes than was
2040 	 * intended (e.g. pid123:::).
2041 	 */
2042 	mutex_exit(&provider->ftp_cmtx);
2043 	mutex_enter(&provider->ftp_mtx);
2044 	provider->ftp_ccount--;
2045 	provider->ftp_marked = 1;
2046 	mutex_exit(&provider->ftp_mtx);
2047 
2048 	fasttrap_pid_cleanup();
2049 
2050 	return (ENOMEM);
2051 }
2052 
2053 /*ARGSUSED*/
2054 static void *
2055 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2056 {
2057 	fasttrap_provider_t *provider;
2058 
2059 	/*
2060 	 * A 32-bit unsigned integer (like a pid for example) can be
2061 	 * expressed in 10 or fewer decimal digits. Make sure that we'll
2062 	 * have enough space for the provider name.
2063 	 */
2064 	if (strlen(dhpv->dthpv_provname) + 10 >=
2065 	    sizeof (provider->ftp_name)) {
2066 		printf("failed to instantiate provider %s: "
2067 		    "name too long to accomodate pid", dhpv->dthpv_provname);
2068 		return (NULL);
2069 	}
2070 
2071 	/*
2072 	 * Don't let folks spoof the true pid provider.
2073 	 */
2074 	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
2075 		printf("failed to instantiate provider %s: "
2076 		    "%s is an invalid name", dhpv->dthpv_provname,
2077 		    FASTTRAP_PID_NAME);
2078 		return (NULL);
2079 	}
2080 
2081 	/*
2082 	 * The highest stability class that fasttrap supports is ISA; cap
2083 	 * the stability of the new provider accordingly.
2084 	 */
2085 	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
2086 		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
2087 	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
2088 		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
2089 	if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
2090 		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
2091 	if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
2092 		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
2093 	if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
2094 		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
2095 
2096 	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
2097 	    &dhpv->dthpv_pattr)) == NULL) {
2098 		printf("failed to instantiate provider %s for "
2099 		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
2100 		return (NULL);
2101 	}
2102 
2103 	/*
2104 	 * Up the meta provider count so this provider isn't removed until
2105 	 * the meta provider has been told to remove it.
2106 	 */
2107 	provider->ftp_mcount++;
2108 
2109 	mutex_exit(&provider->ftp_mtx);
2110 
2111 	return (provider);
2112 }
2113 
2114 /*
2115  * We know a few things about our context here:  we know that the probe being
2116  * created doesn't already exist (DTrace won't load DOF at the same address
2117  * twice, even if explicitly told to do so) and we know that we are
2118  * single-threaded with respect to the meta provider machinery. Knowing that
2119  * this is a new probe and that there is no way for us to race with another
2120  * operation on this provider allows us an important optimization: we need not
2121  * lookup a probe before adding it.  Saving this lookup is important because
2122  * this code is in the fork path for processes with USDT probes, and lookups
2123  * here are potentially very expensive because of long hash conflicts on
2124  * module, function and name (DTrace doesn't hash on provider name).
2125  */
2126 /*ARGSUSED*/
2127 static void
2128 fasttrap_meta_create_probe(void *arg, void *parg,
2129     dtrace_helper_probedesc_t *dhpb)
2130 {
2131 	fasttrap_provider_t *provider = parg;
2132 	fasttrap_probe_t *pp;
2133 	fasttrap_tracepoint_t *tp;
2134 	int i, j;
2135 	uint32_t ntps;
2136 
2137 	/*
2138 	 * Since the meta provider count is non-zero we don't have to worry
2139 	 * about this provider disappearing.
2140 	 */
2141 	ASSERT(provider->ftp_mcount > 0);
2142 
2143 	/*
2144 	 * The offsets must be unique.
2145 	 */
2146 	qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
2147 	    fasttrap_uint32_cmp);
2148 	for (i = 1; i < dhpb->dthpb_noffs; i++) {
2149 		if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
2150 		    dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
2151 			return;
2152 	}
2153 
2154 	qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
2155 	    fasttrap_uint32_cmp);
2156 	for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
2157 		if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
2158 		    dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
2159 			return;
2160 	}
2161 
2162 	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
2163 	ASSERT(ntps > 0);
2164 
2165 	atomic_add_32(&fasttrap_total, ntps);
2166 
2167 	if (fasttrap_total > fasttrap_max) {
2168 		atomic_add_32(&fasttrap_total, -ntps);
2169 		return;
2170 	}
2171 
2172 	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
2173 
2174 	pp->ftp_prov = provider;
2175 	pp->ftp_pid = provider->ftp_pid;
2176 	pp->ftp_ntps = ntps;
2177 	pp->ftp_nargs = dhpb->dthpb_xargc;
2178 	pp->ftp_xtypes = dhpb->dthpb_xtypes;
2179 	pp->ftp_ntypes = dhpb->dthpb_ntypes;
2180 
2181 	/*
2182 	 * First create a tracepoint for each actual point of interest.
2183 	 */
2184 	for (i = 0; i < dhpb->dthpb_noffs; i++) {
2185 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2186 
2187 		tp->ftt_proc = provider->ftp_proc;
2188 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
2189 		tp->ftt_pid = provider->ftp_pid;
2190 
2191 		pp->ftp_tps[i].fit_tp = tp;
2192 		pp->ftp_tps[i].fit_id.fti_probe = pp;
2193 #ifdef __sparc
2194 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
2195 #else
2196 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
2197 #endif
2198 	}
2199 
2200 	/*
2201 	 * Then create a tracepoint for each is-enabled point.
2202 	 */
2203 	for (j = 0; i < ntps; i++, j++) {
2204 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2205 
2206 		tp->ftt_proc = provider->ftp_proc;
2207 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
2208 		tp->ftt_pid = provider->ftp_pid;
2209 
2210 		pp->ftp_tps[i].fit_tp = tp;
2211 		pp->ftp_tps[i].fit_id.fti_probe = pp;
2212 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2213 	}
2214 
2215 	/*
2216 	 * If the arguments are shuffled around we set the argument remapping
2217 	 * table. Later, when the probe fires, we only remap the arguments
2218 	 * if the table is non-NULL.
2219 	 */
2220 	for (i = 0; i < dhpb->dthpb_xargc; i++) {
2221 		if (dhpb->dthpb_args[i] != i) {
2222 			pp->ftp_argmap = dhpb->dthpb_args;
2223 			break;
2224 		}
2225 	}
2226 
2227 	/*
2228 	 * The probe is fully constructed -- register it with DTrace.
2229 	 */
2230 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2231 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2232 }
2233 
2234 /*ARGSUSED*/
2235 static void
2236 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2237 {
2238 	/*
2239 	 * Clean up the USDT provider. There may be active consumers of the
2240 	 * provider busy adding probes, no damage will actually befall the
2241 	 * provider until that count has dropped to zero. This just puts
2242 	 * the provider on death row.
2243 	 */
2244 	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2245 }
2246 
2247 static dtrace_mops_t fasttrap_mops = {
2248 	.dtms_create_probe =	fasttrap_meta_create_probe,
2249 	.dtms_provide_pid =	fasttrap_meta_provide,
2250 	.dtms_remove_pid =	fasttrap_meta_remove
2251 };
2252 
2253 /*ARGSUSED*/
2254 static int
2255 fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2256     int devtype __unused, struct thread *td __unused)
2257 {
2258 	return (0);
2259 }
2260 
2261 /*ARGSUSED*/
2262 static int
2263 fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2264     struct thread *td)
2265 {
2266 	if (!dtrace_attached())
2267 		return (EAGAIN);
2268 
2269 	if (cmd == FASTTRAPIOC_MAKEPROBE) {
2270 		fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2271 		fasttrap_probe_spec_t *probe;
2272 		uint64_t noffs;
2273 		size_t size;
2274 		int ret, err;
2275 
2276 		if (copyin(&uprobe->ftps_noffs, &noffs,
2277 		    sizeof (uprobe->ftps_noffs)))
2278 			return (EFAULT);
2279 
2280 		/*
2281 		 * Probes must have at least one tracepoint.
2282 		 */
2283 		if (noffs == 0)
2284 			return (EINVAL);
2285 
2286 		size = sizeof (fasttrap_probe_spec_t) +
2287 		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
2288 
2289 		if (size > 1024 * 1024)
2290 			return (ENOMEM);
2291 
2292 		probe = kmem_alloc(size, KM_SLEEP);
2293 
2294 		if (copyin(uprobe, probe, size) != 0 ||
2295 		    probe->ftps_noffs != noffs) {
2296 			kmem_free(probe, size);
2297 			return (EFAULT);
2298 		}
2299 
2300 		/*
2301 		 * Verify that the function and module strings contain no
2302 		 * funny characters.
2303 		 */
2304 		if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
2305 		    NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
2306 			ret = EINVAL;
2307 			goto err;
2308 		}
2309 
2310 		if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
2311 		    NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
2312 			ret = EINVAL;
2313 			goto err;
2314 		}
2315 
2316 #ifdef notyet
2317 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2318 			proc_t *p;
2319 			pid_t pid = probe->ftps_pid;
2320 
2321 			mutex_enter(&pidlock);
2322 			/*
2323 			 * Report an error if the process doesn't exist
2324 			 * or is actively being birthed.
2325 			 */
2326 			if ((p = pfind(pid)) == NULL || p->p_stat == SIDL) {
2327 				mutex_exit(&pidlock);
2328 				return (ESRCH);
2329 			}
2330 			mutex_enter(&p->p_lock);
2331 			mutex_exit(&pidlock);
2332 
2333 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2334 			    VREAD | VWRITE)) != 0) {
2335 				mutex_exit(&p->p_lock);
2336 				return (ret);
2337 			}
2338 			mutex_exit(&p->p_lock);
2339 		}
2340 #endif /* notyet */
2341 
2342 		ret = fasttrap_add_probe(probe);
2343 err:
2344 		kmem_free(probe, size);
2345 
2346 		return (ret);
2347 
2348 	} else if (cmd == FASTTRAPIOC_GETINSTR) {
2349 		fasttrap_instr_query_t instr;
2350 		fasttrap_tracepoint_t *tp;
2351 		uint_t index;
2352 #ifdef notyet
2353 		int ret;
2354 #endif
2355 
2356 #ifdef illumos
2357 		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2358 			return (EFAULT);
2359 #endif
2360 
2361 #ifdef notyet
2362 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2363 			proc_t *p;
2364 			pid_t pid = instr.ftiq_pid;
2365 
2366 			mutex_enter(&pidlock);
2367 			/*
2368 			 * Report an error if the process doesn't exist
2369 			 * or is actively being birthed.
2370 			 */
2371 			if ((p == pfind(pid)) == NULL || p->p_stat == SIDL) {
2372 				mutex_exit(&pidlock);
2373 				return (ESRCH);
2374 			}
2375 			mutex_enter(&p->p_lock);
2376 			mutex_exit(&pidlock);
2377 
2378 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2379 			    VREAD)) != 0) {
2380 				mutex_exit(&p->p_lock);
2381 				return (ret);
2382 			}
2383 
2384 			mutex_exit(&p->p_lock);
2385 		}
2386 #endif /* notyet */
2387 
2388 		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2389 
2390 		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2391 		tp = fasttrap_tpoints.fth_table[index].ftb_data;
2392 		while (tp != NULL) {
2393 			if (instr.ftiq_pid == tp->ftt_pid &&
2394 			    instr.ftiq_pc == tp->ftt_pc &&
2395 			    tp->ftt_proc->ftpc_acount != 0)
2396 				break;
2397 
2398 			tp = tp->ftt_next;
2399 		}
2400 
2401 		if (tp == NULL) {
2402 			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2403 			return (ENOENT);
2404 		}
2405 
2406 		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2407 		    sizeof (instr.ftiq_instr));
2408 		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2409 
2410 		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2411 			return (EFAULT);
2412 
2413 		return (0);
2414 	}
2415 
2416 	return (EINVAL);
2417 }
2418 
2419 static int
2420 fasttrap_load(void)
2421 {
2422 	ulong_t nent;
2423 	int i, ret;
2424 
2425         /* Create the /dev/dtrace/fasttrap entry. */
2426         fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2427             "dtrace/fasttrap");
2428 
2429 	mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2430 	mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2431 	    NULL);
2432 
2433 #ifdef illumos
2434 	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2435 	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2436 #endif
2437 	fasttrap_total = 0;
2438 
2439 	/*
2440 	 * Conjure up the tracepoints hashtable...
2441 	 */
2442 #ifdef illumos
2443 	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2444 	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2445 #else
2446 	nent = tpoints_hash_size;
2447 #endif
2448 
2449 	if (nent == 0 || nent > 0x1000000)
2450 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2451 
2452 	tpoints_hash_size = nent;
2453 
2454 	if (ISP2(nent))
2455 		fasttrap_tpoints.fth_nent = nent;
2456 	else
2457 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2458 	ASSERT(fasttrap_tpoints.fth_nent > 0);
2459 	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2460 	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2461 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2462 #ifndef illumos
2463 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2464 		mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2465 		    "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2466 #endif
2467 
2468 	/*
2469 	 * ... and the providers hash table...
2470 	 */
2471 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2472 	if (ISP2(nent))
2473 		fasttrap_provs.fth_nent = nent;
2474 	else
2475 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2476 	ASSERT(fasttrap_provs.fth_nent > 0);
2477 	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2478 	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2479 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2480 #ifndef illumos
2481 	for (i = 0; i < fasttrap_provs.fth_nent; i++)
2482 		mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2483 		    "providers bucket mtx", MUTEX_DEFAULT, NULL);
2484 #endif
2485 
2486 	ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2487 	    &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2488 	if (ret != 0) {
2489 		destroy_dev(fasttrap_cdev);
2490 #ifndef illumos
2491 		for (i = 0; i < fasttrap_provs.fth_nent; i++)
2492 			mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2493 		for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2494 			mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2495 #endif
2496 		kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *
2497 		    sizeof (fasttrap_bucket_t));
2498 		mtx_destroy(&fasttrap_cleanup_mtx);
2499 		mutex_destroy(&fasttrap_count_mtx);
2500 		return (ret);
2501 	}
2502 
2503 
2504 	/*
2505 	 * ... and the procs hash table.
2506 	 */
2507 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2508 	if (ISP2(nent))
2509 		fasttrap_procs.fth_nent = nent;
2510 	else
2511 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2512 	ASSERT(fasttrap_procs.fth_nent > 0);
2513 	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2514 	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2515 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2516 #ifndef illumos
2517 	for (i = 0; i < fasttrap_procs.fth_nent; i++)
2518 		mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2519 		    "processes bucket mtx", MUTEX_DEFAULT, NULL);
2520 
2521 	rm_init(&fasttrap_tp_lock, "fasttrap tracepoint");
2522 
2523 	/*
2524 	 * This event handler must run before kdtrace_thread_dtor() since it
2525 	 * accesses the thread's struct kdtrace_thread.
2526 	 */
2527 	fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
2528 	    fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
2529 #endif
2530 
2531 	/*
2532 	 * Install our hooks into fork(2), exec(2), and exit(2).
2533 	 */
2534 	dtrace_fasttrap_fork = &fasttrap_fork;
2535 	dtrace_fasttrap_exit = &fasttrap_exec_exit;
2536 	dtrace_fasttrap_exec = &fasttrap_exec_exit;
2537 
2538 	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2539 	    &fasttrap_meta_id);
2540 
2541 	return (0);
2542 }
2543 
2544 static int
2545 fasttrap_unload(void)
2546 {
2547 	int i, fail = 0;
2548 
2549 	/*
2550 	 * Unregister the meta-provider to make sure no new fasttrap-
2551 	 * managed providers come along while we're trying to close up
2552 	 * shop. If we fail to detach, we'll need to re-register as a
2553 	 * meta-provider. We can fail to unregister as a meta-provider
2554 	 * if providers we manage still exist.
2555 	 */
2556 	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2557 	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
2558 		return (-1);
2559 
2560 	/*
2561 	 * Iterate over all of our providers. If there's still a process
2562 	 * that corresponds to that pid, fail to detach.
2563 	 */
2564 	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2565 		fasttrap_provider_t **fpp, *fp;
2566 		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2567 
2568 		mutex_enter(&bucket->ftb_mtx);
2569 		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2570 		while ((fp = *fpp) != NULL) {
2571 			/*
2572 			 * Acquire and release the lock as a simple way of
2573 			 * waiting for any other consumer to finish with
2574 			 * this provider. A thread must first acquire the
2575 			 * bucket lock so there's no chance of another thread
2576 			 * blocking on the provider's lock.
2577 			 */
2578 			mutex_enter(&fp->ftp_mtx);
2579 			mutex_exit(&fp->ftp_mtx);
2580 
2581 			if (dtrace_unregister(fp->ftp_provid) != 0) {
2582 				fail = 1;
2583 				fpp = &fp->ftp_next;
2584 			} else {
2585 				*fpp = fp->ftp_next;
2586 				fasttrap_provider_free(fp);
2587 			}
2588 		}
2589 
2590 		mutex_exit(&bucket->ftb_mtx);
2591 	}
2592 
2593 	if (fail) {
2594 		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2595 		    &fasttrap_meta_id);
2596 
2597 		return (-1);
2598 	}
2599 
2600 	/*
2601 	 * Stop new processes from entering these hooks now, before the
2602 	 * fasttrap_cleanup thread runs.  That way all processes will hopefully
2603 	 * be out of these hooks before we free fasttrap_provs.fth_table
2604 	 */
2605 	ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2606 	dtrace_fasttrap_fork = NULL;
2607 
2608 	ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2609 	dtrace_fasttrap_exec = NULL;
2610 
2611 	ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2612 	dtrace_fasttrap_exit = NULL;
2613 
2614 	mtx_lock(&fasttrap_cleanup_mtx);
2615 	fasttrap_cleanup_drain = 1;
2616 	/* Wait for the cleanup thread to finish up and signal us. */
2617 	wakeup(&fasttrap_cleanup_cv);
2618 	mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2619 	    0);
2620 	fasttrap_cleanup_proc = NULL;
2621 	mtx_destroy(&fasttrap_cleanup_mtx);
2622 
2623 #ifdef DEBUG
2624 	mutex_enter(&fasttrap_count_mtx);
2625 	ASSERT(fasttrap_pid_count == 0);
2626 	mutex_exit(&fasttrap_count_mtx);
2627 #endif
2628 
2629 #ifndef illumos
2630 	EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag);
2631 
2632 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2633 		mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2634 	for (i = 0; i < fasttrap_provs.fth_nent; i++)
2635 		mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2636 	for (i = 0; i < fasttrap_procs.fth_nent; i++)
2637 		mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2638 #endif
2639 	kmem_free(fasttrap_tpoints.fth_table,
2640 	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2641 	fasttrap_tpoints.fth_nent = 0;
2642 
2643 	kmem_free(fasttrap_provs.fth_table,
2644 	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2645 	fasttrap_provs.fth_nent = 0;
2646 
2647 	kmem_free(fasttrap_procs.fth_table,
2648 	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2649 	fasttrap_procs.fth_nent = 0;
2650 
2651 #ifndef illumos
2652 	destroy_dev(fasttrap_cdev);
2653 	mutex_destroy(&fasttrap_count_mtx);
2654 	rm_destroy(&fasttrap_tp_lock);
2655 #endif
2656 
2657 	return (0);
2658 }
2659 
2660 /* ARGSUSED */
2661 static int
2662 fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2663 {
2664 	int error = 0;
2665 
2666 	switch (type) {
2667 	case MOD_LOAD:
2668 		break;
2669 
2670 	case MOD_UNLOAD:
2671 		break;
2672 
2673 	case MOD_SHUTDOWN:
2674 		break;
2675 
2676 	default:
2677 		error = EOPNOTSUPP;
2678 		break;
2679 	}
2680 	return (error);
2681 }
2682 
2683 SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2684     NULL);
2685 SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2686     fasttrap_unload, NULL);
2687 
2688 DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2689 MODULE_VERSION(fasttrap, 1);
2690 MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2691 MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
2692