xref: /illumos-gate/usr/src/uts/common/dtrace/fasttrap.c (revision ef623aa14b0dc34c6b9edb213cbcda88461ecfc3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright (c) 2015, Joyent, Inc. All rights reserved.
29  */
30 
31 #include <sys/atomic.h>
32 #include <sys/errno.h>
33 #include <sys/stat.h>
34 #include <sys/modctl.h>
35 #include <sys/conf.h>
36 #include <sys/systm.h>
37 #include <sys/ddi.h>
38 #include <sys/sunddi.h>
39 #include <sys/cpuvar.h>
40 #include <sys/kmem.h>
41 #include <sys/strsubr.h>
42 #include <sys/fasttrap.h>
43 #include <sys/fasttrap_impl.h>
44 #include <sys/fasttrap_isa.h>
45 #include <sys/dtrace.h>
46 #include <sys/dtrace_impl.h>
47 #include <sys/sysmacros.h>
48 #include <sys/proc.h>
49 #include <sys/priv.h>
50 #include <sys/policy.h>
51 #include <util/qsort.h>
52 
53 /*
54  * User-Land Trap-Based Tracing
55  * ----------------------------
56  *
57  * The fasttrap provider allows DTrace consumers to instrument any user-level
58  * instruction to gather data; this includes probes with semantic
59  * signifigance like entry and return as well as simple offsets into the
60  * function. While the specific techniques used are very ISA specific, the
61  * methodology is generalizable to any architecture.
62  *
63  *
64  * The General Methodology
65  * -----------------------
66  *
67  * With the primary goal of tracing every user-land instruction and the
68  * limitation that we can't trust user space so don't want to rely on much
69  * information there, we begin by replacing the instructions we want to trace
70  * with trap instructions. Each instruction we overwrite is saved into a hash
71  * table keyed by process ID and pc address. When we enter the kernel due to
72  * this trap instruction, we need the effects of the replaced instruction to
73  * appear to have occurred before we proceed with the user thread's
74  * execution.
75  *
76  * Each user level thread is represented by a ulwp_t structure which is
77  * always easily accessible through a register. The most basic way to produce
78  * the effects of the instruction we replaced is to copy that instruction out
79  * to a bit of scratch space reserved in the user thread's ulwp_t structure
80  * (a sort of kernel-private thread local storage), set the PC to that
81  * scratch space and single step. When we reenter the kernel after single
82  * stepping the instruction we must then adjust the PC to point to what would
83  * normally be the next instruction. Of course, special care must be taken
84  * for branches and jumps, but these represent such a small fraction of any
85  * instruction set that writing the code to emulate these in the kernel is
86  * not too difficult.
87  *
88  * Return probes may require several tracepoints to trace every return site,
89  * and, conversely, each tracepoint may activate several probes (the entry
90  * and offset 0 probes, for example). To solve this muliplexing problem,
91  * tracepoints contain lists of probes to activate and probes contain lists
92  * of tracepoints to enable. If a probe is activated, it adds its ID to
93  * existing tracepoints or creates new ones as necessary.
94  *
95  * Most probes are activated _before_ the instruction is executed, but return
96  * probes are activated _after_ the effects of the last instruction of the
97  * function are visible. Return probes must be fired _after_ we have
98  * single-stepped the instruction whereas all other probes are fired
99  * beforehand.
100  *
101  *
102  * Lock Ordering
103  * -------------
104  *
105  * The lock ordering below -- both internally and with respect to the DTrace
106  * framework -- is a little tricky and bears some explanation. Each provider
107  * has a lock (ftp_mtx) that protects its members including reference counts
108  * for enabled probes (ftp_rcount), consumers actively creating probes
109  * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
110  * from being freed. A provider is looked up by taking the bucket lock for the
111  * provider hash table, and is returned with its lock held. The provider lock
112  * may be taken in functions invoked by the DTrace framework, but may not be
113  * held while calling functions in the DTrace framework.
114  *
115  * To ensure consistency over multiple calls to the DTrace framework, the
116  * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
117  * not be taken when holding the provider lock as that would create a cyclic
118  * lock ordering. In situations where one would naturally take the provider
119  * lock and then the creation lock, we instead up a reference count to prevent
120  * the provider from disappearing, drop the provider lock, and acquire the
121  * creation lock.
122  *
123  * Briefly:
124  *	bucket lock before provider lock
125  *	DTrace before provider lock
126  *	creation lock before DTrace
127  *	never hold the provider lock and creation lock simultaneously
128  */
129 
130 static dev_info_t *fasttrap_devi;
131 static dtrace_meta_provider_id_t fasttrap_meta_id;
132 
133 static timeout_id_t fasttrap_timeout;
134 static kmutex_t fasttrap_cleanup_mtx;
135 static uint_t fasttrap_cleanup_work;
136 
137 /*
138  * Generation count on modifications to the global tracepoint lookup table.
139  */
140 static volatile uint64_t fasttrap_mod_gen;
141 
142 /*
143  * When the fasttrap provider is loaded, fasttrap_max is set to either
144  * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
145  * fasttrap.conf file. Each time a probe is created, fasttrap_total is
146  * incremented by the number of tracepoints that may be associated with that
147  * probe; fasttrap_total is capped at fasttrap_max.
148  */
149 #define	FASTTRAP_MAX_DEFAULT		250000
150 static uint32_t fasttrap_max;
151 static uint32_t fasttrap_total;
152 
153 
154 #define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
155 #define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
156 #define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
157 
158 #define	FASTTRAP_PID_NAME		"pid"
159 
160 fasttrap_hash_t			fasttrap_tpoints;
161 static fasttrap_hash_t		fasttrap_provs;
162 static fasttrap_hash_t		fasttrap_procs;
163 
164 static uint64_t			fasttrap_pid_count;	/* pid ref count */
165 static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
166 
167 #define	FASTTRAP_ENABLE_FAIL	1
168 #define	FASTTRAP_ENABLE_PARTIAL	2
169 
170 static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
171 static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
172 
173 static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
174     const dtrace_pattr_t *);
175 static void fasttrap_provider_retire(pid_t, const char *, int);
176 static void fasttrap_provider_free(fasttrap_provider_t *);
177 
178 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
179 static void fasttrap_proc_release(fasttrap_proc_t *);
180 
181 #define	FASTTRAP_PROVS_INDEX(pid, name) \
182 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
183 
184 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
185 
186 static int
fasttrap_highbit(ulong_t i)187 fasttrap_highbit(ulong_t i)
188 {
189 	int h = 1;
190 
191 	if (i == 0)
192 		return (0);
193 #ifdef _LP64
194 	if (i & 0xffffffff00000000ul) {
195 		h += 32; i >>= 32;
196 	}
197 #endif
198 	if (i & 0xffff0000) {
199 		h += 16; i >>= 16;
200 	}
201 	if (i & 0xff00) {
202 		h += 8; i >>= 8;
203 	}
204 	if (i & 0xf0) {
205 		h += 4; i >>= 4;
206 	}
207 	if (i & 0xc) {
208 		h += 2; i >>= 2;
209 	}
210 	if (i & 0x2) {
211 		h += 1;
212 	}
213 	return (h);
214 }
215 
216 static uint_t
fasttrap_hash_str(const char * p)217 fasttrap_hash_str(const char *p)
218 {
219 	unsigned int g;
220 	uint_t hval = 0;
221 
222 	while (*p) {
223 		hval = (hval << 4) + *p++;
224 		if ((g = (hval & 0xf0000000)) != 0)
225 			hval ^= g >> 24;
226 		hval &= ~g;
227 	}
228 	return (hval);
229 }
230 
231 void
fasttrap_sigtrap(proc_t * p,kthread_t * t,uintptr_t pc)232 fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
233 {
234 	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
235 
236 	sqp->sq_info.si_signo = SIGTRAP;
237 	sqp->sq_info.si_code = TRAP_DTRACE;
238 	sqp->sq_info.si_addr = (caddr_t)pc;
239 
240 	mutex_enter(&p->p_lock);
241 	sigaddqa(p, t, sqp);
242 	mutex_exit(&p->p_lock);
243 
244 	if (t != NULL)
245 		aston(t);
246 }
247 
248 /*
249  * This function ensures that no threads are actively using the memory
250  * associated with probes that were formerly live.
251  */
252 static void
fasttrap_mod_barrier(uint64_t gen)253 fasttrap_mod_barrier(uint64_t gen)
254 {
255 	int i;
256 
257 	if (gen < fasttrap_mod_gen)
258 		return;
259 
260 	fasttrap_mod_gen++;
261 
262 	for (i = 0; i < NCPU; i++) {
263 		mutex_enter(&cpu_core[i].cpuc_pid_lock);
264 		mutex_exit(&cpu_core[i].cpuc_pid_lock);
265 	}
266 }
267 
268 /*
269  * This is the timeout's callback for cleaning up the providers and their
270  * probes.
271  */
272 /*ARGSUSED*/
273 static void
fasttrap_pid_cleanup_cb(void * data)274 fasttrap_pid_cleanup_cb(void *data)
275 {
276 	fasttrap_provider_t **fpp, *fp;
277 	fasttrap_bucket_t *bucket;
278 	dtrace_provider_id_t provid;
279 	int i, later, rval;
280 
281 	static volatile int in = 0;
282 	ASSERT(in == 0);
283 	in = 1;
284 
285 	mutex_enter(&fasttrap_cleanup_mtx);
286 	while (fasttrap_cleanup_work) {
287 		fasttrap_cleanup_work = 0;
288 		mutex_exit(&fasttrap_cleanup_mtx);
289 
290 		later = 0;
291 
292 		/*
293 		 * Iterate over all the providers trying to remove the marked
294 		 * ones. If a provider is marked but not retired, we just
295 		 * have to take a crack at removing it -- it's no big deal if
296 		 * we can't.
297 		 */
298 		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
299 			bucket = &fasttrap_provs.fth_table[i];
300 			mutex_enter(&bucket->ftb_mtx);
301 			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
302 
303 			while ((fp = *fpp) != NULL) {
304 				if (!fp->ftp_marked) {
305 					fpp = &fp->ftp_next;
306 					continue;
307 				}
308 
309 				mutex_enter(&fp->ftp_mtx);
310 
311 				/*
312 				 * If this provider has consumers actively
313 				 * creating probes (ftp_ccount) or is a USDT
314 				 * provider (ftp_mcount), we can't unregister
315 				 * or even condense.
316 				 */
317 				if (fp->ftp_ccount != 0 ||
318 				    fp->ftp_mcount != 0) {
319 					mutex_exit(&fp->ftp_mtx);
320 					fp->ftp_marked = 0;
321 					continue;
322 				}
323 
324 				if (!fp->ftp_retired || fp->ftp_rcount != 0)
325 					fp->ftp_marked = 0;
326 
327 				mutex_exit(&fp->ftp_mtx);
328 
329 				/*
330 				 * If we successfully unregister this
331 				 * provider we can remove it from the hash
332 				 * chain and free the memory. If our attempt
333 				 * to unregister fails and this is a retired
334 				 * provider, increment our flag to try again
335 				 * pretty soon. If we've consumed more than
336 				 * half of our total permitted number of
337 				 * probes call dtrace_condense() to try to
338 				 * clean out the unenabled probes.
339 				 */
340 				provid = fp->ftp_provid;
341 				if ((rval = dtrace_unregister(provid)) != 0) {
342 					if (fasttrap_total > fasttrap_max / 2)
343 						(void) dtrace_condense(provid);
344 
345 					if (rval == EAGAIN)
346 						fp->ftp_marked = 1;
347 
348 					later += fp->ftp_marked;
349 					fpp = &fp->ftp_next;
350 				} else {
351 					*fpp = fp->ftp_next;
352 					fasttrap_provider_free(fp);
353 				}
354 			}
355 			mutex_exit(&bucket->ftb_mtx);
356 		}
357 
358 		mutex_enter(&fasttrap_cleanup_mtx);
359 	}
360 
361 	ASSERT(fasttrap_timeout != 0);
362 
363 	/*
364 	 * If we were unable to remove a retired provider, try again after
365 	 * a second. This situation can occur in certain circumstances where
366 	 * providers cannot be unregistered even though they have no probes
367 	 * enabled because of an execution of dtrace -l or something similar.
368 	 * If the timeout has been disabled (set to 1 because we're trying
369 	 * to detach), we set fasttrap_cleanup_work to ensure that we'll
370 	 * get a chance to do that work if and when the timeout is reenabled
371 	 * (if detach fails).
372 	 */
373 	if (later > 0) {
374 		if (fasttrap_timeout != (timeout_id_t)1) {
375 			fasttrap_timeout =
376 			    timeout(&fasttrap_pid_cleanup_cb, NULL, hz);
377 		}
378 
379 		fasttrap_cleanup_work = 1;
380 	} else {
381 		fasttrap_timeout = 0;
382 	}
383 
384 	mutex_exit(&fasttrap_cleanup_mtx);
385 	in = 0;
386 }
387 
388 /*
389  * Activates the asynchronous cleanup mechanism.
390  */
391 static void
fasttrap_pid_cleanup(void)392 fasttrap_pid_cleanup(void)
393 {
394 	mutex_enter(&fasttrap_cleanup_mtx);
395 	fasttrap_cleanup_work = 1;
396 	if (fasttrap_timeout == 0)
397 		fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1);
398 	mutex_exit(&fasttrap_cleanup_mtx);
399 }
400 
401 /*
402  * This is called from cfork() via dtrace_fasttrap_fork(). The child
403  * process's address space is (roughly) a copy of the parent process's so
404  * we have to remove all the instrumentation we had previously enabled in the
405  * parent.
406  */
407 static void
fasttrap_fork(proc_t * p,proc_t * cp)408 fasttrap_fork(proc_t *p, proc_t *cp)
409 {
410 	pid_t ppid = p->p_pid;
411 	int i;
412 
413 	ASSERT(curproc == p);
414 	ASSERT(p->p_proc_flag & P_PR_LOCK);
415 	ASSERT(p->p_dtrace_count > 0);
416 	ASSERT(cp->p_dtrace_count == 0);
417 
418 	/*
419 	 * This would be simpler and faster if we maintained per-process
420 	 * hash tables of enabled tracepoints. It could, however, potentially
421 	 * slow down execution of a tracepoint since we'd need to go
422 	 * through two levels of indirection. In the future, we should
423 	 * consider either maintaining per-process ancillary lists of
424 	 * enabled tracepoints or hanging a pointer to a per-process hash
425 	 * table of enabled tracepoints off the proc structure.
426 	 */
427 
428 	/*
429 	 * We don't have to worry about the child process disappearing
430 	 * because we're in fork().
431 	 */
432 	mutex_enter(&cp->p_lock);
433 	sprlock_proc(cp);
434 	mutex_exit(&cp->p_lock);
435 
436 	/*
437 	 * Iterate over every tracepoint looking for ones that belong to the
438 	 * parent process, and remove each from the child process.
439 	 */
440 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
441 		fasttrap_tracepoint_t *tp;
442 		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
443 
444 		mutex_enter(&bucket->ftb_mtx);
445 		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
446 			if (tp->ftt_pid == ppid &&
447 			    tp->ftt_proc->ftpc_acount != 0) {
448 				int ret = fasttrap_tracepoint_remove(cp, tp);
449 				ASSERT(ret == 0);
450 
451 				/*
452 				 * The count of active providers can only be
453 				 * decremented (i.e. to zero) during exec,
454 				 * exit, and removal of a meta provider so it
455 				 * should be impossible to drop the count
456 				 * mid-fork.
457 				 */
458 				ASSERT(tp->ftt_proc->ftpc_acount != 0);
459 			}
460 		}
461 		mutex_exit(&bucket->ftb_mtx);
462 	}
463 
464 	mutex_enter(&cp->p_lock);
465 	sprunlock(cp);
466 }
467 
468 /*
469  * This is called from proc_exit() or from exec_common() if p_dtrace_probes
470  * is set on the proc structure to indicate that there is a pid provider
471  * associated with this process.
472  */
473 static void
fasttrap_exec_exit(proc_t * p)474 fasttrap_exec_exit(proc_t *p)
475 {
476 	ASSERT(p == curproc);
477 	ASSERT(MUTEX_HELD(&p->p_lock));
478 
479 	mutex_exit(&p->p_lock);
480 
481 	/*
482 	 * We clean up the pid provider for this process here; user-land
483 	 * static probes are handled by the meta-provider remove entry point.
484 	 */
485 	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
486 
487 	mutex_enter(&p->p_lock);
488 }
489 
490 
491 /*ARGSUSED*/
492 static void
fasttrap_pid_provide(void * arg,const dtrace_probedesc_t * desc)493 fasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc)
494 {
495 	/*
496 	 * There are no "default" pid probes.
497 	 */
498 }
499 
500 static int
fasttrap_tracepoint_enable(proc_t * p,fasttrap_probe_t * probe,uint_t index)501 fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
502 {
503 	fasttrap_tracepoint_t *tp, *new_tp = NULL;
504 	fasttrap_bucket_t *bucket;
505 	fasttrap_id_t *id;
506 	pid_t pid;
507 	uintptr_t pc;
508 
509 	ASSERT(index < probe->ftp_ntps);
510 
511 	pid = probe->ftp_pid;
512 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
513 	id = &probe->ftp_tps[index].fit_id;
514 
515 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
516 
517 	ASSERT(!(p->p_flag & SVFORK));
518 
519 	/*
520 	 * Before we make any modifications, make sure we've imposed a barrier
521 	 * on the generation in which this probe was last modified.
522 	 */
523 	fasttrap_mod_barrier(probe->ftp_gen);
524 
525 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
526 
527 	/*
528 	 * If the tracepoint has already been enabled, just add our id to the
529 	 * list of interested probes. This may be our second time through
530 	 * this path in which case we'll have constructed the tracepoint we'd
531 	 * like to install. If we can't find a match, and have an allocated
532 	 * tracepoint ready to go, enable that one now.
533 	 *
534 	 * A tracepoint whose process is defunct is also considered defunct.
535 	 */
536 again:
537 	mutex_enter(&bucket->ftb_mtx);
538 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
539 		/*
540 		 * Note that it's safe to access the active count on the
541 		 * associated proc structure because we know that at least one
542 		 * provider (this one) will still be around throughout this
543 		 * operation.
544 		 */
545 		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
546 		    tp->ftt_proc->ftpc_acount == 0)
547 			continue;
548 
549 		/*
550 		 * Now that we've found a matching tracepoint, it would be
551 		 * a decent idea to confirm that the tracepoint is still
552 		 * enabled and the trap instruction hasn't been overwritten.
553 		 * Since this is a little hairy, we'll punt for now.
554 		 */
555 
556 		/*
557 		 * This can't be the first interested probe. We don't have
558 		 * to worry about another thread being in the midst of
559 		 * deleting this tracepoint (which would be the only valid
560 		 * reason for a tracepoint to have no interested probes)
561 		 * since we're holding P_PR_LOCK for this process.
562 		 */
563 		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
564 
565 		switch (id->fti_ptype) {
566 		case DTFTP_ENTRY:
567 		case DTFTP_OFFSETS:
568 		case DTFTP_IS_ENABLED:
569 			id->fti_next = tp->ftt_ids;
570 			membar_producer();
571 			tp->ftt_ids = id;
572 			membar_producer();
573 			break;
574 
575 		case DTFTP_RETURN:
576 		case DTFTP_POST_OFFSETS:
577 			id->fti_next = tp->ftt_retids;
578 			membar_producer();
579 			tp->ftt_retids = id;
580 			membar_producer();
581 			break;
582 
583 		default:
584 			ASSERT(0);
585 		}
586 
587 		mutex_exit(&bucket->ftb_mtx);
588 
589 		if (new_tp != NULL) {
590 			new_tp->ftt_ids = NULL;
591 			new_tp->ftt_retids = NULL;
592 		}
593 
594 		return (0);
595 	}
596 
597 	/*
598 	 * If we have a good tracepoint ready to go, install it now while
599 	 * we have the lock held and no one can screw with us.
600 	 */
601 	if (new_tp != NULL) {
602 		int rc = 0;
603 
604 		new_tp->ftt_next = bucket->ftb_data;
605 		membar_producer();
606 		bucket->ftb_data = new_tp;
607 		membar_producer();
608 		mutex_exit(&bucket->ftb_mtx);
609 
610 		/*
611 		 * Activate the tracepoint in the ISA-specific manner.
612 		 * If this fails, we need to report the failure, but
613 		 * indicate that this tracepoint must still be disabled
614 		 * by calling fasttrap_tracepoint_disable().
615 		 */
616 		if (fasttrap_tracepoint_install(p, new_tp) != 0)
617 			rc = FASTTRAP_ENABLE_PARTIAL;
618 
619 		/*
620 		 * Increment the count of the number of tracepoints active in
621 		 * the victim process.
622 		 */
623 		ASSERT(p->p_proc_flag & P_PR_LOCK);
624 		p->p_dtrace_count++;
625 
626 		return (rc);
627 	}
628 
629 	mutex_exit(&bucket->ftb_mtx);
630 
631 	/*
632 	 * Initialize the tracepoint that's been preallocated with the probe.
633 	 */
634 	new_tp = probe->ftp_tps[index].fit_tp;
635 
636 	ASSERT(new_tp->ftt_pid == pid);
637 	ASSERT(new_tp->ftt_pc == pc);
638 	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
639 	ASSERT(new_tp->ftt_ids == NULL);
640 	ASSERT(new_tp->ftt_retids == NULL);
641 
642 	switch (id->fti_ptype) {
643 	case DTFTP_ENTRY:
644 	case DTFTP_OFFSETS:
645 	case DTFTP_IS_ENABLED:
646 		id->fti_next = NULL;
647 		new_tp->ftt_ids = id;
648 		break;
649 
650 	case DTFTP_RETURN:
651 	case DTFTP_POST_OFFSETS:
652 		id->fti_next = NULL;
653 		new_tp->ftt_retids = id;
654 		break;
655 
656 	default:
657 		ASSERT(0);
658 	}
659 
660 	/*
661 	 * If the ISA-dependent initialization goes to plan, go back to the
662 	 * beginning and try to install this freshly made tracepoint.
663 	 */
664 	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
665 		goto again;
666 
667 	new_tp->ftt_ids = NULL;
668 	new_tp->ftt_retids = NULL;
669 
670 	return (FASTTRAP_ENABLE_FAIL);
671 }
672 
673 static void
fasttrap_tracepoint_disable(proc_t * p,fasttrap_probe_t * probe,uint_t index)674 fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
675 {
676 	fasttrap_bucket_t *bucket;
677 	fasttrap_provider_t *provider = probe->ftp_prov;
678 	fasttrap_tracepoint_t **pp, *tp;
679 	fasttrap_id_t *id, **idp;
680 	pid_t pid;
681 	uintptr_t pc;
682 
683 	ASSERT(index < probe->ftp_ntps);
684 
685 	pid = probe->ftp_pid;
686 	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
687 	id = &probe->ftp_tps[index].fit_id;
688 
689 	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
690 
691 	/*
692 	 * Find the tracepoint and make sure that our id is one of the
693 	 * ones registered with it.
694 	 */
695 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
696 	mutex_enter(&bucket->ftb_mtx);
697 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
698 		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
699 		    tp->ftt_proc == provider->ftp_proc)
700 			break;
701 	}
702 
703 	/*
704 	 * If we somehow lost this tracepoint, we're in a world of hurt.
705 	 */
706 	ASSERT(tp != NULL);
707 
708 	switch (id->fti_ptype) {
709 	case DTFTP_ENTRY:
710 	case DTFTP_OFFSETS:
711 	case DTFTP_IS_ENABLED:
712 		ASSERT(tp->ftt_ids != NULL);
713 		idp = &tp->ftt_ids;
714 		break;
715 
716 	case DTFTP_RETURN:
717 	case DTFTP_POST_OFFSETS:
718 		ASSERT(tp->ftt_retids != NULL);
719 		idp = &tp->ftt_retids;
720 		break;
721 
722 	default:
723 		ASSERT(0);
724 	}
725 
726 	while ((*idp)->fti_probe != probe) {
727 		idp = &(*idp)->fti_next;
728 		ASSERT(*idp != NULL);
729 	}
730 
731 	id = *idp;
732 	*idp = id->fti_next;
733 	membar_producer();
734 
735 	ASSERT(id->fti_probe == probe);
736 
737 	/*
738 	 * If there are other registered enablings of this tracepoint, we're
739 	 * all done, but if this was the last probe assocated with this
740 	 * this tracepoint, we need to remove and free it.
741 	 */
742 	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
743 
744 		/*
745 		 * If the current probe's tracepoint is in use, swap it
746 		 * for an unused tracepoint.
747 		 */
748 		if (tp == probe->ftp_tps[index].fit_tp) {
749 			fasttrap_probe_t *tmp_probe;
750 			fasttrap_tracepoint_t **tmp_tp;
751 			uint_t tmp_index;
752 
753 			if (tp->ftt_ids != NULL) {
754 				tmp_probe = tp->ftt_ids->fti_probe;
755 				/* LINTED - alignment */
756 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
757 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
758 			} else {
759 				tmp_probe = tp->ftt_retids->fti_probe;
760 				/* LINTED - alignment */
761 				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
762 				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
763 			}
764 
765 			ASSERT(*tmp_tp != NULL);
766 			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
767 			ASSERT((*tmp_tp)->ftt_ids == NULL);
768 			ASSERT((*tmp_tp)->ftt_retids == NULL);
769 
770 			probe->ftp_tps[index].fit_tp = *tmp_tp;
771 			*tmp_tp = tp;
772 		}
773 
774 		mutex_exit(&bucket->ftb_mtx);
775 
776 		/*
777 		 * Tag the modified probe with the generation in which it was
778 		 * changed.
779 		 */
780 		probe->ftp_gen = fasttrap_mod_gen;
781 		return;
782 	}
783 
784 	mutex_exit(&bucket->ftb_mtx);
785 
786 	/*
787 	 * We can't safely remove the tracepoint from the set of active
788 	 * tracepoints until we've actually removed the fasttrap instruction
789 	 * from the process's text. We can, however, operate on this
790 	 * tracepoint secure in the knowledge that no other thread is going to
791 	 * be looking at it since we hold P_PR_LOCK on the process if it's
792 	 * live or we hold the provider lock on the process if it's dead and
793 	 * gone.
794 	 */
795 
796 	/*
797 	 * We only need to remove the actual instruction if we're looking
798 	 * at an existing process
799 	 */
800 	if (p != NULL) {
801 		/*
802 		 * If we fail to restore the instruction we need to kill
803 		 * this process since it's in a completely unrecoverable
804 		 * state.
805 		 */
806 		if (fasttrap_tracepoint_remove(p, tp) != 0)
807 			fasttrap_sigtrap(p, NULL, pc);
808 
809 		/*
810 		 * Decrement the count of the number of tracepoints active
811 		 * in the victim process.
812 		 */
813 		ASSERT(p->p_proc_flag & P_PR_LOCK);
814 		p->p_dtrace_count--;
815 	}
816 
817 	/*
818 	 * Remove the probe from the hash table of active tracepoints.
819 	 */
820 	mutex_enter(&bucket->ftb_mtx);
821 	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
822 	ASSERT(*pp != NULL);
823 	while (*pp != tp) {
824 		pp = &(*pp)->ftt_next;
825 		ASSERT(*pp != NULL);
826 	}
827 
828 	*pp = tp->ftt_next;
829 	membar_producer();
830 
831 	mutex_exit(&bucket->ftb_mtx);
832 
833 	/*
834 	 * Tag the modified probe with the generation in which it was changed.
835 	 */
836 	probe->ftp_gen = fasttrap_mod_gen;
837 }
838 
839 static void
fasttrap_enable_callbacks(void)840 fasttrap_enable_callbacks(void)
841 {
842 	/*
843 	 * We don't have to play the rw lock game here because we're
844 	 * providing something rather than taking something away --
845 	 * we can be sure that no threads have tried to follow this
846 	 * function pointer yet.
847 	 */
848 	mutex_enter(&fasttrap_count_mtx);
849 	if (fasttrap_pid_count == 0) {
850 		ASSERT(dtrace_pid_probe_ptr == NULL);
851 		ASSERT(dtrace_return_probe_ptr == NULL);
852 		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
853 		dtrace_return_probe_ptr = &fasttrap_return_probe;
854 	}
855 	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
856 	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
857 	fasttrap_pid_count++;
858 	mutex_exit(&fasttrap_count_mtx);
859 }
860 
861 static void
fasttrap_disable_callbacks(void)862 fasttrap_disable_callbacks(void)
863 {
864 	ASSERT(MUTEX_HELD(&cpu_lock));
865 
866 	mutex_enter(&fasttrap_count_mtx);
867 	ASSERT(fasttrap_pid_count > 0);
868 	fasttrap_pid_count--;
869 	if (fasttrap_pid_count == 0) {
870 		cpu_t *cur, *cpu = CPU;
871 
872 		for (cur = cpu->cpu_next_onln; cur != cpu;
873 		    cur = cur->cpu_next_onln) {
874 			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
875 		}
876 
877 		dtrace_pid_probe_ptr = NULL;
878 		dtrace_return_probe_ptr = NULL;
879 
880 		for (cur = cpu->cpu_next_onln; cur != cpu;
881 		    cur = cur->cpu_next_onln) {
882 			rw_exit(&cur->cpu_ft_lock);
883 		}
884 	}
885 	mutex_exit(&fasttrap_count_mtx);
886 }
887 
888 /*ARGSUSED*/
889 static int
fasttrap_pid_enable(void * arg,dtrace_id_t id,void * parg)890 fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
891 {
892 	fasttrap_probe_t *probe = parg;
893 	proc_t *p;
894 	int i, rc;
895 
896 	ASSERT(probe != NULL);
897 	ASSERT(!probe->ftp_enabled);
898 	ASSERT(id == probe->ftp_id);
899 	ASSERT(MUTEX_HELD(&cpu_lock));
900 
901 	/*
902 	 * Increment the count of enabled probes on this probe's provider;
903 	 * the provider can't go away while the probe still exists. We
904 	 * must increment this even if we aren't able to properly enable
905 	 * this probe.
906 	 */
907 	mutex_enter(&probe->ftp_prov->ftp_mtx);
908 	probe->ftp_prov->ftp_rcount++;
909 	mutex_exit(&probe->ftp_prov->ftp_mtx);
910 
911 	/*
912 	 * If this probe's provider is retired (meaning it was valid in a
913 	 * previously exec'ed incarnation of this address space), bail out. The
914 	 * provider can't go away while we're in this code path.
915 	 */
916 	if (probe->ftp_prov->ftp_retired)
917 		return (0);
918 
919 	/*
920 	 * If we can't find the process, it may be that we're in the context of
921 	 * a fork in which the traced process is being born and we're copying
922 	 * USDT probes. Otherwise, the process is gone so bail.
923 	 */
924 	if ((p = sprlock(probe->ftp_pid)) == NULL) {
925 		if ((curproc->p_flag & SFORKING) == 0)
926 			return (0);
927 
928 		mutex_enter(&pidlock);
929 		p = prfind(probe->ftp_pid);
930 
931 		if (p == NULL) {
932 			/*
933 			 * So it's not that the target process is being born,
934 			 * it's that it isn't there at all (and we simply
935 			 * happen to be forking).  Anyway, we know that the
936 			 * target is definitely gone, so bail out.
937 			 */
938 			mutex_exit(&pidlock);
939 			return (0);
940 		}
941 
942 		/*
943 		 * Confirm that curproc is indeed forking the process in which
944 		 * we're trying to enable probes.
945 		 */
946 		ASSERT(p->p_parent == curproc);
947 		ASSERT(p->p_stat == SIDL);
948 
949 		mutex_enter(&p->p_lock);
950 		mutex_exit(&pidlock);
951 
952 		sprlock_proc(p);
953 	}
954 
955 	ASSERT(!(p->p_flag & SVFORK));
956 	mutex_exit(&p->p_lock);
957 
958 	/*
959 	 * We have to enable the trap entry point before any user threads have
960 	 * the chance to execute the trap instruction we're about to place
961 	 * in their process's text.
962 	 */
963 	fasttrap_enable_callbacks();
964 
965 	/*
966 	 * Enable all the tracepoints and add this probe's id to each
967 	 * tracepoint's list of active probes.
968 	 */
969 	for (i = 0; i < probe->ftp_ntps; i++) {
970 		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
971 			/*
972 			 * If enabling the tracepoint failed completely,
973 			 * we don't have to disable it; if the failure
974 			 * was only partial we must disable it.
975 			 */
976 			if (rc == FASTTRAP_ENABLE_FAIL)
977 				i--;
978 			else
979 				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
980 
981 			/*
982 			 * Back up and pull out all the tracepoints we've
983 			 * created so far for this probe.
984 			 */
985 			while (i >= 0) {
986 				fasttrap_tracepoint_disable(p, probe, i);
987 				i--;
988 			}
989 
990 			mutex_enter(&p->p_lock);
991 			sprunlock(p);
992 
993 			/*
994 			 * Since we're not actually enabling this probe,
995 			 * drop our reference on the trap table entry.
996 			 */
997 			fasttrap_disable_callbacks();
998 			return (0);
999 		}
1000 	}
1001 
1002 	mutex_enter(&p->p_lock);
1003 	sprunlock(p);
1004 
1005 	probe->ftp_enabled = 1;
1006 	return (0);
1007 }
1008 
1009 /*ARGSUSED*/
1010 static void
fasttrap_pid_disable(void * arg,dtrace_id_t id,void * parg)1011 fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1012 {
1013 	fasttrap_probe_t *probe = parg;
1014 	fasttrap_provider_t *provider = probe->ftp_prov;
1015 	proc_t *p;
1016 	int i, whack = 0;
1017 
1018 	ASSERT(id == probe->ftp_id);
1019 
1020 	/*
1021 	 * We won't be able to acquire a /proc-esque lock on the process
1022 	 * iff the process is dead and gone. In this case, we rely on the
1023 	 * provider lock as a point of mutual exclusion to prevent other
1024 	 * DTrace consumers from disabling this probe.
1025 	 */
1026 	if ((p = sprlock(probe->ftp_pid)) != NULL) {
1027 		ASSERT(!(p->p_flag & SVFORK));
1028 		mutex_exit(&p->p_lock);
1029 	}
1030 
1031 	mutex_enter(&provider->ftp_mtx);
1032 
1033 	/*
1034 	 * Disable all the associated tracepoints (for fully enabled probes).
1035 	 */
1036 	if (probe->ftp_enabled) {
1037 		for (i = 0; i < probe->ftp_ntps; i++) {
1038 			fasttrap_tracepoint_disable(p, probe, i);
1039 		}
1040 	}
1041 
1042 	ASSERT(provider->ftp_rcount > 0);
1043 	provider->ftp_rcount--;
1044 
1045 	if (p != NULL) {
1046 		/*
1047 		 * Even though we may not be able to remove it entirely, we
1048 		 * mark this retired provider to get a chance to remove some
1049 		 * of the associated probes.
1050 		 */
1051 		if (provider->ftp_retired && !provider->ftp_marked)
1052 			whack = provider->ftp_marked = 1;
1053 		mutex_exit(&provider->ftp_mtx);
1054 
1055 		mutex_enter(&p->p_lock);
1056 		sprunlock(p);
1057 	} else {
1058 		/*
1059 		 * If the process is dead, we're just waiting for the
1060 		 * last probe to be disabled to be able to free it.
1061 		 */
1062 		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1063 			whack = provider->ftp_marked = 1;
1064 		mutex_exit(&provider->ftp_mtx);
1065 	}
1066 
1067 	if (whack)
1068 		fasttrap_pid_cleanup();
1069 
1070 	if (!probe->ftp_enabled)
1071 		return;
1072 
1073 	probe->ftp_enabled = 0;
1074 
1075 	ASSERT(MUTEX_HELD(&cpu_lock));
1076 	fasttrap_disable_callbacks();
1077 }
1078 
1079 /*ARGSUSED*/
1080 static void
fasttrap_pid_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)1081 fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1082     dtrace_argdesc_t *desc)
1083 {
1084 	fasttrap_probe_t *probe = parg;
1085 	char *str;
1086 	int i, ndx;
1087 
1088 	desc->dtargd_native[0] = '\0';
1089 	desc->dtargd_xlate[0] = '\0';
1090 
1091 	if (probe->ftp_prov->ftp_retired != 0 ||
1092 	    desc->dtargd_ndx >= probe->ftp_nargs) {
1093 		desc->dtargd_ndx = DTRACE_ARGNONE;
1094 		return;
1095 	}
1096 
1097 	ndx = (probe->ftp_argmap != NULL) ?
1098 	    probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1099 
1100 	str = probe->ftp_ntypes;
1101 	for (i = 0; i < ndx; i++) {
1102 		str += strlen(str) + 1;
1103 	}
1104 
1105 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1106 	(void) strcpy(desc->dtargd_native, str);
1107 
1108 	if (probe->ftp_xtypes == NULL)
1109 		return;
1110 
1111 	str = probe->ftp_xtypes;
1112 	for (i = 0; i < desc->dtargd_ndx; i++) {
1113 		str += strlen(str) + 1;
1114 	}
1115 
1116 	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1117 	(void) strcpy(desc->dtargd_xlate, str);
1118 }
1119 
1120 /*ARGSUSED*/
1121 static void
fasttrap_pid_destroy(void * arg,dtrace_id_t id,void * parg)1122 fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1123 {
1124 	fasttrap_probe_t *probe = parg;
1125 	int i;
1126 	size_t size;
1127 
1128 	ASSERT(probe != NULL);
1129 	ASSERT(!probe->ftp_enabled);
1130 	ASSERT(fasttrap_total >= probe->ftp_ntps);
1131 
1132 	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1133 	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1134 
1135 	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1136 		fasttrap_mod_barrier(probe->ftp_gen);
1137 
1138 	for (i = 0; i < probe->ftp_ntps; i++) {
1139 		kmem_free(probe->ftp_tps[i].fit_tp,
1140 		    sizeof (fasttrap_tracepoint_t));
1141 	}
1142 
1143 	kmem_free(probe, size);
1144 }
1145 
1146 
1147 static const dtrace_pattr_t pid_attr = {
1148 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1149 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1150 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1151 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1152 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1153 };
1154 
1155 static dtrace_pops_t pid_pops = {
1156 	fasttrap_pid_provide,
1157 	NULL,
1158 	fasttrap_pid_enable,
1159 	fasttrap_pid_disable,
1160 	NULL,
1161 	NULL,
1162 	fasttrap_pid_getargdesc,
1163 	fasttrap_pid_getarg,
1164 	NULL,
1165 	fasttrap_pid_destroy
1166 };
1167 
1168 static dtrace_pops_t usdt_pops = {
1169 	fasttrap_pid_provide,
1170 	NULL,
1171 	fasttrap_pid_enable,
1172 	fasttrap_pid_disable,
1173 	NULL,
1174 	NULL,
1175 	fasttrap_pid_getargdesc,
1176 	fasttrap_usdt_getarg,
1177 	NULL,
1178 	fasttrap_pid_destroy
1179 };
1180 
1181 static fasttrap_proc_t *
fasttrap_proc_lookup(pid_t pid)1182 fasttrap_proc_lookup(pid_t pid)
1183 {
1184 	fasttrap_bucket_t *bucket;
1185 	fasttrap_proc_t *fprc, *new_fprc;
1186 
1187 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1188 	mutex_enter(&bucket->ftb_mtx);
1189 
1190 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1191 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1192 			mutex_enter(&fprc->ftpc_mtx);
1193 			mutex_exit(&bucket->ftb_mtx);
1194 			fprc->ftpc_rcount++;
1195 			atomic_inc_64(&fprc->ftpc_acount);
1196 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1197 			mutex_exit(&fprc->ftpc_mtx);
1198 
1199 			return (fprc);
1200 		}
1201 	}
1202 
1203 	/*
1204 	 * Drop the bucket lock so we don't try to perform a sleeping
1205 	 * allocation under it.
1206 	 */
1207 	mutex_exit(&bucket->ftb_mtx);
1208 
1209 	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1210 	new_fprc->ftpc_pid = pid;
1211 	new_fprc->ftpc_rcount = 1;
1212 	new_fprc->ftpc_acount = 1;
1213 
1214 	mutex_enter(&bucket->ftb_mtx);
1215 
1216 	/*
1217 	 * Take another lap through the list to make sure a proc hasn't
1218 	 * been created for this pid while we weren't under the bucket lock.
1219 	 */
1220 	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1221 		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1222 			mutex_enter(&fprc->ftpc_mtx);
1223 			mutex_exit(&bucket->ftb_mtx);
1224 			fprc->ftpc_rcount++;
1225 			atomic_inc_64(&fprc->ftpc_acount);
1226 			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1227 			mutex_exit(&fprc->ftpc_mtx);
1228 
1229 			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1230 
1231 			return (fprc);
1232 		}
1233 	}
1234 
1235 	new_fprc->ftpc_next = bucket->ftb_data;
1236 	bucket->ftb_data = new_fprc;
1237 
1238 	mutex_exit(&bucket->ftb_mtx);
1239 
1240 	return (new_fprc);
1241 }
1242 
1243 static void
fasttrap_proc_release(fasttrap_proc_t * proc)1244 fasttrap_proc_release(fasttrap_proc_t *proc)
1245 {
1246 	fasttrap_bucket_t *bucket;
1247 	fasttrap_proc_t *fprc, **fprcp;
1248 	pid_t pid = proc->ftpc_pid;
1249 
1250 	mutex_enter(&proc->ftpc_mtx);
1251 
1252 	ASSERT(proc->ftpc_rcount != 0);
1253 	ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1254 
1255 	if (--proc->ftpc_rcount != 0) {
1256 		mutex_exit(&proc->ftpc_mtx);
1257 		return;
1258 	}
1259 
1260 	mutex_exit(&proc->ftpc_mtx);
1261 
1262 	/*
1263 	 * There should definitely be no live providers associated with this
1264 	 * process at this point.
1265 	 */
1266 	ASSERT(proc->ftpc_acount == 0);
1267 
1268 	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1269 	mutex_enter(&bucket->ftb_mtx);
1270 
1271 	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1272 	while ((fprc = *fprcp) != NULL) {
1273 		if (fprc == proc)
1274 			break;
1275 
1276 		fprcp = &fprc->ftpc_next;
1277 	}
1278 
1279 	/*
1280 	 * Something strange has happened if we can't find the proc.
1281 	 */
1282 	ASSERT(fprc != NULL);
1283 
1284 	*fprcp = fprc->ftpc_next;
1285 
1286 	mutex_exit(&bucket->ftb_mtx);
1287 
1288 	kmem_free(fprc, sizeof (fasttrap_proc_t));
1289 }
1290 
1291 /*
1292  * Lookup a fasttrap-managed provider based on its name and associated pid.
1293  * If the pattr argument is non-NULL, this function instantiates the provider
1294  * if it doesn't exist otherwise it returns NULL. The provider is returned
1295  * with its lock held.
1296  */
1297 static fasttrap_provider_t *
fasttrap_provider_lookup(pid_t pid,const char * name,const dtrace_pattr_t * pattr)1298 fasttrap_provider_lookup(pid_t pid, const char *name,
1299     const dtrace_pattr_t *pattr)
1300 {
1301 	fasttrap_provider_t *fp, *new_fp = NULL;
1302 	fasttrap_bucket_t *bucket;
1303 	char provname[DTRACE_PROVNAMELEN];
1304 	proc_t *p;
1305 	cred_t *cred;
1306 
1307 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1308 	ASSERT(pattr != NULL);
1309 
1310 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1311 	mutex_enter(&bucket->ftb_mtx);
1312 
1313 	/*
1314 	 * Take a lap through the list and return the match if we find it.
1315 	 */
1316 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1317 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1318 		    !fp->ftp_retired) {
1319 			mutex_enter(&fp->ftp_mtx);
1320 			mutex_exit(&bucket->ftb_mtx);
1321 			return (fp);
1322 		}
1323 	}
1324 
1325 	/*
1326 	 * Drop the bucket lock so we don't try to perform a sleeping
1327 	 * allocation under it.
1328 	 */
1329 	mutex_exit(&bucket->ftb_mtx);
1330 
1331 	/*
1332 	 * Make sure the process exists, isn't a child created as the result
1333 	 * of a vfork(2), and isn't a zombie (but may be in fork).
1334 	 */
1335 	mutex_enter(&pidlock);
1336 	if ((p = prfind(pid)) == NULL) {
1337 		mutex_exit(&pidlock);
1338 		return (NULL);
1339 	}
1340 	mutex_enter(&p->p_lock);
1341 	mutex_exit(&pidlock);
1342 	if (p->p_flag & (SVFORK | SEXITING)) {
1343 		mutex_exit(&p->p_lock);
1344 		return (NULL);
1345 	}
1346 
1347 	/*
1348 	 * Increment p_dtrace_probes so that the process knows to inform us
1349 	 * when it exits or execs. fasttrap_provider_free() decrements this
1350 	 * when we're done with this provider.
1351 	 */
1352 	p->p_dtrace_probes++;
1353 
1354 	/*
1355 	 * Grab the credentials for this process so we have
1356 	 * something to pass to dtrace_register().
1357 	 */
1358 	mutex_enter(&p->p_crlock);
1359 	crhold(p->p_cred);
1360 	cred = p->p_cred;
1361 	mutex_exit(&p->p_crlock);
1362 	mutex_exit(&p->p_lock);
1363 
1364 	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1365 	new_fp->ftp_pid = pid;
1366 	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1367 
1368 	ASSERT(new_fp->ftp_proc != NULL);
1369 
1370 	mutex_enter(&bucket->ftb_mtx);
1371 
1372 	/*
1373 	 * Take another lap through the list to make sure a provider hasn't
1374 	 * been created for this pid while we weren't under the bucket lock.
1375 	 */
1376 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1377 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1378 		    !fp->ftp_retired) {
1379 			mutex_enter(&fp->ftp_mtx);
1380 			mutex_exit(&bucket->ftb_mtx);
1381 			fasttrap_provider_free(new_fp);
1382 			crfree(cred);
1383 			return (fp);
1384 		}
1385 	}
1386 
1387 	(void) strcpy(new_fp->ftp_name, name);
1388 
1389 	/*
1390 	 * Fail and return NULL if either the provider name is too long
1391 	 * or we fail to register this new provider with the DTrace
1392 	 * framework. Note that this is the only place we ever construct
1393 	 * the full provider name -- we keep it in pieces in the provider
1394 	 * structure.
1395 	 */
1396 	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1397 	    sizeof (provname) ||
1398 	    dtrace_register(provname, pattr,
1399 	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1400 	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1401 	    &new_fp->ftp_provid) != 0) {
1402 		mutex_exit(&bucket->ftb_mtx);
1403 		fasttrap_provider_free(new_fp);
1404 		crfree(cred);
1405 		return (NULL);
1406 	}
1407 
1408 	new_fp->ftp_next = bucket->ftb_data;
1409 	bucket->ftb_data = new_fp;
1410 
1411 	mutex_enter(&new_fp->ftp_mtx);
1412 	mutex_exit(&bucket->ftb_mtx);
1413 
1414 	crfree(cred);
1415 	return (new_fp);
1416 }
1417 
1418 static void
fasttrap_provider_free(fasttrap_provider_t * provider)1419 fasttrap_provider_free(fasttrap_provider_t *provider)
1420 {
1421 	pid_t pid = provider->ftp_pid;
1422 	proc_t *p;
1423 
1424 	/*
1425 	 * There need to be no associated enabled probes, no consumers
1426 	 * creating probes, and no meta providers referencing this provider.
1427 	 */
1428 	ASSERT(provider->ftp_rcount == 0);
1429 	ASSERT(provider->ftp_ccount == 0);
1430 	ASSERT(provider->ftp_mcount == 0);
1431 
1432 	/*
1433 	 * If this provider hasn't been retired, we need to explicitly drop the
1434 	 * count of active providers on the associated process structure.
1435 	 */
1436 	if (!provider->ftp_retired) {
1437 		atomic_dec_64(&provider->ftp_proc->ftpc_acount);
1438 		ASSERT(provider->ftp_proc->ftpc_acount <
1439 		    provider->ftp_proc->ftpc_rcount);
1440 	}
1441 
1442 	fasttrap_proc_release(provider->ftp_proc);
1443 
1444 	kmem_free(provider, sizeof (fasttrap_provider_t));
1445 
1446 	/*
1447 	 * Decrement p_dtrace_probes on the process whose provider we're
1448 	 * freeing. We don't have to worry about clobbering somone else's
1449 	 * modifications to it because we have locked the bucket that
1450 	 * corresponds to this process's hash chain in the provider hash
1451 	 * table. Don't sweat it if we can't find the process.
1452 	 */
1453 	mutex_enter(&pidlock);
1454 	if ((p = prfind(pid)) == NULL) {
1455 		mutex_exit(&pidlock);
1456 		return;
1457 	}
1458 
1459 	mutex_enter(&p->p_lock);
1460 	mutex_exit(&pidlock);
1461 
1462 	p->p_dtrace_probes--;
1463 	mutex_exit(&p->p_lock);
1464 }
1465 
1466 static void
fasttrap_provider_retire(pid_t pid,const char * name,int mprov)1467 fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1468 {
1469 	fasttrap_provider_t *fp;
1470 	fasttrap_bucket_t *bucket;
1471 	dtrace_provider_id_t provid;
1472 
1473 	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1474 
1475 	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1476 	mutex_enter(&bucket->ftb_mtx);
1477 
1478 	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1479 		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1480 		    !fp->ftp_retired)
1481 			break;
1482 	}
1483 
1484 	if (fp == NULL) {
1485 		mutex_exit(&bucket->ftb_mtx);
1486 		return;
1487 	}
1488 
1489 	mutex_enter(&fp->ftp_mtx);
1490 	ASSERT(!mprov || fp->ftp_mcount > 0);
1491 	if (mprov && --fp->ftp_mcount != 0)  {
1492 		mutex_exit(&fp->ftp_mtx);
1493 		mutex_exit(&bucket->ftb_mtx);
1494 		return;
1495 	}
1496 
1497 	/*
1498 	 * Mark the provider to be removed in our post-processing step, mark it
1499 	 * retired, and drop the active count on its proc. Marking it indicates
1500 	 * that we should try to remove it; setting the retired flag indicates
1501 	 * that we're done with this provider; dropping the active the proc
1502 	 * releases our hold, and when this reaches zero (as it will during
1503 	 * exit or exec) the proc and associated providers become defunct.
1504 	 *
1505 	 * We obviously need to take the bucket lock before the provider lock
1506 	 * to perform the lookup, but we need to drop the provider lock
1507 	 * before calling into the DTrace framework since we acquire the
1508 	 * provider lock in callbacks invoked from the DTrace framework. The
1509 	 * bucket lock therefore protects the integrity of the provider hash
1510 	 * table.
1511 	 */
1512 	atomic_dec_64(&fp->ftp_proc->ftpc_acount);
1513 	ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1514 
1515 	fp->ftp_retired = 1;
1516 	fp->ftp_marked = 1;
1517 	provid = fp->ftp_provid;
1518 	mutex_exit(&fp->ftp_mtx);
1519 
1520 	/*
1521 	 * We don't have to worry about invalidating the same provider twice
1522 	 * since fasttrap_provider_lookup() will ignore provider that have
1523 	 * been marked as retired.
1524 	 */
1525 	dtrace_invalidate(provid);
1526 
1527 	mutex_exit(&bucket->ftb_mtx);
1528 
1529 	fasttrap_pid_cleanup();
1530 }
1531 
1532 static int
fasttrap_uint32_cmp(const void * ap,const void * bp)1533 fasttrap_uint32_cmp(const void *ap, const void *bp)
1534 {
1535 	return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1536 }
1537 
1538 static int
fasttrap_uint64_cmp(const void * ap,const void * bp)1539 fasttrap_uint64_cmp(const void *ap, const void *bp)
1540 {
1541 	return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1542 }
1543 
1544 static int
fasttrap_add_probe(fasttrap_probe_spec_t * pdata)1545 fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1546 {
1547 	fasttrap_provider_t *provider;
1548 	fasttrap_probe_t *pp;
1549 	fasttrap_tracepoint_t *tp;
1550 	char *name;
1551 	int i, aframes, whack;
1552 
1553 	/*
1554 	 * There needs to be at least one desired trace point.
1555 	 */
1556 	if (pdata->ftps_noffs == 0)
1557 		return (EINVAL);
1558 
1559 	switch (pdata->ftps_type) {
1560 	case DTFTP_ENTRY:
1561 		name = "entry";
1562 		aframes = FASTTRAP_ENTRY_AFRAMES;
1563 		break;
1564 	case DTFTP_RETURN:
1565 		name = "return";
1566 		aframes = FASTTRAP_RETURN_AFRAMES;
1567 		break;
1568 	case DTFTP_OFFSETS:
1569 		name = NULL;
1570 		break;
1571 	default:
1572 		return (EINVAL);
1573 	}
1574 
1575 	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1576 	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1577 		return (ESRCH);
1578 
1579 	/*
1580 	 * Increment this reference count to indicate that a consumer is
1581 	 * actively adding a new probe associated with this provider. This
1582 	 * prevents the provider from being deleted -- we'll need to check
1583 	 * for pending deletions when we drop this reference count.
1584 	 */
1585 	provider->ftp_ccount++;
1586 	mutex_exit(&provider->ftp_mtx);
1587 
1588 	/*
1589 	 * Grab the creation lock to ensure consistency between calls to
1590 	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1591 	 * other threads creating probes. We must drop the provider lock
1592 	 * before taking this lock to avoid a three-way deadlock with the
1593 	 * DTrace framework.
1594 	 */
1595 	mutex_enter(&provider->ftp_cmtx);
1596 
1597 	if (name == NULL) {
1598 		for (i = 0; i < pdata->ftps_noffs; i++) {
1599 			char name_str[17];
1600 
1601 			(void) sprintf(name_str, "%llx",
1602 			    (unsigned long long)pdata->ftps_offs[i]);
1603 
1604 			if (dtrace_probe_lookup(provider->ftp_provid,
1605 			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1606 				continue;
1607 
1608 			atomic_inc_32(&fasttrap_total);
1609 
1610 			if (fasttrap_total > fasttrap_max) {
1611 				atomic_dec_32(&fasttrap_total);
1612 				goto no_mem;
1613 			}
1614 
1615 			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1616 
1617 			pp->ftp_prov = provider;
1618 			pp->ftp_faddr = pdata->ftps_pc;
1619 			pp->ftp_fsize = pdata->ftps_size;
1620 			pp->ftp_pid = pdata->ftps_pid;
1621 			pp->ftp_ntps = 1;
1622 
1623 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1624 			    KM_SLEEP);
1625 
1626 			tp->ftt_proc = provider->ftp_proc;
1627 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1628 			tp->ftt_pid = pdata->ftps_pid;
1629 
1630 			pp->ftp_tps[0].fit_tp = tp;
1631 			pp->ftp_tps[0].fit_id.fti_probe = pp;
1632 			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1633 
1634 			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1635 			    pdata->ftps_mod, pdata->ftps_func, name_str,
1636 			    FASTTRAP_OFFSET_AFRAMES, pp);
1637 		}
1638 
1639 	} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1640 	    pdata->ftps_func, name) == 0) {
1641 		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1642 
1643 		if (fasttrap_total > fasttrap_max) {
1644 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1645 			goto no_mem;
1646 		}
1647 
1648 		/*
1649 		 * Make sure all tracepoint program counter values are unique.
1650 		 * We later assume that each probe has exactly one tracepoint
1651 		 * for a given pc.
1652 		 */
1653 		qsort(pdata->ftps_offs, pdata->ftps_noffs,
1654 		    sizeof (uint64_t), fasttrap_uint64_cmp);
1655 		for (i = 1; i < pdata->ftps_noffs; i++) {
1656 			if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1657 				continue;
1658 
1659 			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1660 			goto no_mem;
1661 		}
1662 
1663 		ASSERT(pdata->ftps_noffs > 0);
1664 		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1665 		    ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1666 
1667 		pp->ftp_prov = provider;
1668 		pp->ftp_faddr = pdata->ftps_pc;
1669 		pp->ftp_fsize = pdata->ftps_size;
1670 		pp->ftp_pid = pdata->ftps_pid;
1671 		pp->ftp_ntps = pdata->ftps_noffs;
1672 
1673 		for (i = 0; i < pdata->ftps_noffs; i++) {
1674 			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1675 			    KM_SLEEP);
1676 
1677 			tp->ftt_proc = provider->ftp_proc;
1678 			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1679 			tp->ftt_pid = pdata->ftps_pid;
1680 
1681 			pp->ftp_tps[i].fit_tp = tp;
1682 			pp->ftp_tps[i].fit_id.fti_probe = pp;
1683 			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1684 		}
1685 
1686 		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1687 		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1688 	}
1689 
1690 	mutex_exit(&provider->ftp_cmtx);
1691 
1692 	/*
1693 	 * We know that the provider is still valid since we incremented the
1694 	 * creation reference count. If someone tried to clean up this provider
1695 	 * while we were using it (e.g. because the process called exec(2) or
1696 	 * exit(2)), take note of that and try to clean it up now.
1697 	 */
1698 	mutex_enter(&provider->ftp_mtx);
1699 	provider->ftp_ccount--;
1700 	whack = provider->ftp_retired;
1701 	mutex_exit(&provider->ftp_mtx);
1702 
1703 	if (whack)
1704 		fasttrap_pid_cleanup();
1705 
1706 	return (0);
1707 
1708 no_mem:
1709 	/*
1710 	 * If we've exhausted the allowable resources, we'll try to remove
1711 	 * this provider to free some up. This is to cover the case where
1712 	 * the user has accidentally created many more probes than was
1713 	 * intended (e.g. pid123:::).
1714 	 */
1715 	mutex_exit(&provider->ftp_cmtx);
1716 	mutex_enter(&provider->ftp_mtx);
1717 	provider->ftp_ccount--;
1718 	provider->ftp_marked = 1;
1719 	mutex_exit(&provider->ftp_mtx);
1720 
1721 	fasttrap_pid_cleanup();
1722 
1723 	return (ENOMEM);
1724 }
1725 
1726 /*ARGSUSED*/
1727 static void *
fasttrap_meta_provide(void * arg,dtrace_helper_provdesc_t * dhpv,pid_t pid)1728 fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1729 {
1730 	fasttrap_provider_t *provider;
1731 
1732 	/*
1733 	 * A 32-bit unsigned integer (like a pid for example) can be
1734 	 * expressed in 10 or fewer decimal digits. Make sure that we'll
1735 	 * have enough space for the provider name.
1736 	 */
1737 	if (strlen(dhpv->dthpv_provname) + 10 >=
1738 	    sizeof (provider->ftp_name)) {
1739 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
1740 		    "name too long to accomodate pid", dhpv->dthpv_provname);
1741 		return (NULL);
1742 	}
1743 
1744 	/*
1745 	 * Don't let folks spoof the true pid provider.
1746 	 */
1747 	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1748 		cmn_err(CE_WARN, "failed to instantiate provider %s: "
1749 		    "%s is an invalid name", dhpv->dthpv_provname,
1750 		    FASTTRAP_PID_NAME);
1751 		return (NULL);
1752 	}
1753 
1754 	/*
1755 	 * The highest stability class that fasttrap supports is ISA; cap
1756 	 * the stability of the new provider accordingly.
1757 	 */
1758 	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1759 		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1760 	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1761 		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1762 	if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1763 		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1764 	if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1765 		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1766 	if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1767 		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1768 
1769 	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1770 	    &dhpv->dthpv_pattr)) == NULL) {
1771 		cmn_err(CE_WARN, "failed to instantiate provider %s for "
1772 		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
1773 		return (NULL);
1774 	}
1775 
1776 	/*
1777 	 * Up the meta provider count so this provider isn't removed until
1778 	 * the meta provider has been told to remove it.
1779 	 */
1780 	provider->ftp_mcount++;
1781 
1782 	mutex_exit(&provider->ftp_mtx);
1783 
1784 	return (provider);
1785 }
1786 
1787 /*
1788  * We know a few things about our context here:  we know that the probe being
1789  * created doesn't already exist (DTrace won't load DOF at the same address
1790  * twice, even if explicitly told to do so) and we know that we are
1791  * single-threaded with respect to the meta provider machinery. Knowing that
1792  * this is a new probe and that there is no way for us to race with another
1793  * operation on this provider allows us an important optimization: we need not
1794  * lookup a probe before adding it.  Saving this lookup is important because
1795  * this code is in the fork path for processes with USDT probes, and lookups
1796  * here are potentially very expensive because of long hash conflicts on
1797  * module, function and name (DTrace doesn't hash on provider name).
1798  */
1799 /*ARGSUSED*/
1800 static void
fasttrap_meta_create_probe(void * arg,void * parg,dtrace_helper_probedesc_t * dhpb)1801 fasttrap_meta_create_probe(void *arg, void *parg,
1802     dtrace_helper_probedesc_t *dhpb)
1803 {
1804 	fasttrap_provider_t *provider = parg;
1805 	fasttrap_probe_t *pp;
1806 	fasttrap_tracepoint_t *tp;
1807 	int i, j;
1808 	uint32_t ntps;
1809 
1810 	/*
1811 	 * Since the meta provider count is non-zero we don't have to worry
1812 	 * about this provider disappearing.
1813 	 */
1814 	ASSERT(provider->ftp_mcount > 0);
1815 
1816 	/*
1817 	 * The offsets must be unique.
1818 	 */
1819 	qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1820 	    fasttrap_uint32_cmp);
1821 	for (i = 1; i < dhpb->dthpb_noffs; i++) {
1822 		if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1823 		    dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1824 			return;
1825 	}
1826 
1827 	qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1828 	    fasttrap_uint32_cmp);
1829 	for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1830 		if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1831 		    dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1832 			return;
1833 	}
1834 
1835 	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1836 	ASSERT(ntps > 0);
1837 
1838 	atomic_add_32(&fasttrap_total, ntps);
1839 
1840 	if (fasttrap_total > fasttrap_max) {
1841 		atomic_add_32(&fasttrap_total, -ntps);
1842 		return;
1843 	}
1844 
1845 	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1846 
1847 	pp->ftp_prov = provider;
1848 	pp->ftp_pid = provider->ftp_pid;
1849 	pp->ftp_ntps = ntps;
1850 	pp->ftp_nargs = dhpb->dthpb_xargc;
1851 	pp->ftp_xtypes = dhpb->dthpb_xtypes;
1852 	pp->ftp_ntypes = dhpb->dthpb_ntypes;
1853 
1854 	/*
1855 	 * First create a tracepoint for each actual point of interest.
1856 	 */
1857 	for (i = 0; i < dhpb->dthpb_noffs; i++) {
1858 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1859 
1860 		tp->ftt_proc = provider->ftp_proc;
1861 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1862 		tp->ftt_pid = provider->ftp_pid;
1863 
1864 		pp->ftp_tps[i].fit_tp = tp;
1865 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1866 #ifdef __sparc
1867 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1868 #else
1869 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1870 #endif
1871 	}
1872 
1873 	/*
1874 	 * Then create a tracepoint for each is-enabled point.
1875 	 */
1876 	for (j = 0; i < ntps; i++, j++) {
1877 		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1878 
1879 		tp->ftt_proc = provider->ftp_proc;
1880 		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1881 		tp->ftt_pid = provider->ftp_pid;
1882 
1883 		pp->ftp_tps[i].fit_tp = tp;
1884 		pp->ftp_tps[i].fit_id.fti_probe = pp;
1885 		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1886 	}
1887 
1888 	/*
1889 	 * If the arguments are shuffled around we set the argument remapping
1890 	 * table. Later, when the probe fires, we only remap the arguments
1891 	 * if the table is non-NULL.
1892 	 */
1893 	for (i = 0; i < dhpb->dthpb_xargc; i++) {
1894 		if (dhpb->dthpb_args[i] != i) {
1895 			pp->ftp_argmap = dhpb->dthpb_args;
1896 			break;
1897 		}
1898 	}
1899 
1900 	/*
1901 	 * The probe is fully constructed -- register it with DTrace.
1902 	 */
1903 	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
1904 	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
1905 }
1906 
1907 /*ARGSUSED*/
1908 static void
fasttrap_meta_remove(void * arg,dtrace_helper_provdesc_t * dhpv,pid_t pid)1909 fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1910 {
1911 	/*
1912 	 * Clean up the USDT provider. There may be active consumers of the
1913 	 * provider busy adding probes, no damage will actually befall the
1914 	 * provider until that count has dropped to zero. This just puts
1915 	 * the provider on death row.
1916 	 */
1917 	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
1918 }
1919 
1920 static dtrace_mops_t fasttrap_mops = {
1921 	fasttrap_meta_create_probe,
1922 	fasttrap_meta_provide,
1923 	fasttrap_meta_remove
1924 };
1925 
1926 /*ARGSUSED*/
1927 static int
fasttrap_open(dev_t * devp,int flag,int otyp,cred_t * cred_p)1928 fasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1929 {
1930 	return (0);
1931 }
1932 
1933 /*ARGSUSED*/
1934 static int
fasttrap_ioctl(dev_t dev,int cmd,intptr_t arg,int md,cred_t * cr,int * rv)1935 fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
1936 {
1937 	if (!dtrace_attached())
1938 		return (EAGAIN);
1939 
1940 	if (cmd == FASTTRAPIOC_MAKEPROBE) {
1941 		fasttrap_probe_spec_t *uprobe = (void *)arg;
1942 		fasttrap_probe_spec_t *probe;
1943 		uint64_t noffs;
1944 		size_t size;
1945 		int ret, err;
1946 
1947 		if (copyin(&uprobe->ftps_noffs, &noffs,
1948 		    sizeof (uprobe->ftps_noffs)))
1949 			return (EFAULT);
1950 
1951 		/*
1952 		 * Probes must have at least one tracepoint.
1953 		 */
1954 		if (noffs == 0)
1955 			return (EINVAL);
1956 
1957 		size = sizeof (fasttrap_probe_spec_t) +
1958 		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
1959 
1960 		if (size > 1024 * 1024)
1961 			return (ENOMEM);
1962 
1963 		probe = kmem_alloc(size, KM_SLEEP);
1964 
1965 		if (copyin(uprobe, probe, size) != 0 ||
1966 		    probe->ftps_noffs != noffs) {
1967 			kmem_free(probe, size);
1968 			return (EFAULT);
1969 		}
1970 
1971 		/*
1972 		 * Verify that the function and module strings contain no
1973 		 * funny characters.
1974 		 */
1975 		if (u8_validate(probe->ftps_func, strlen(probe->ftps_func),
1976 		    NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1977 			ret = EINVAL;
1978 			goto err;
1979 		}
1980 
1981 		if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod),
1982 		    NULL, U8_VALIDATE_ENTIRE, &err) < 0) {
1983 			ret = EINVAL;
1984 			goto err;
1985 		}
1986 
1987 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
1988 			proc_t *p;
1989 			pid_t pid = probe->ftps_pid;
1990 
1991 			mutex_enter(&pidlock);
1992 			/*
1993 			 * Report an error if the process doesn't exist
1994 			 * or is actively being birthed.
1995 			 */
1996 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
1997 				mutex_exit(&pidlock);
1998 				return (ESRCH);
1999 			}
2000 			mutex_enter(&p->p_lock);
2001 			mutex_exit(&pidlock);
2002 
2003 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2004 			    VREAD | VWRITE)) != 0) {
2005 				mutex_exit(&p->p_lock);
2006 				return (ret);
2007 			}
2008 
2009 			mutex_exit(&p->p_lock);
2010 		}
2011 
2012 		ret = fasttrap_add_probe(probe);
2013 err:
2014 		kmem_free(probe, size);
2015 
2016 		return (ret);
2017 
2018 	} else if (cmd == FASTTRAPIOC_GETINSTR) {
2019 		fasttrap_instr_query_t instr;
2020 		fasttrap_tracepoint_t *tp;
2021 		uint_t index;
2022 		int ret;
2023 
2024 		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2025 			return (EFAULT);
2026 
2027 		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2028 			proc_t *p;
2029 			pid_t pid = instr.ftiq_pid;
2030 
2031 			mutex_enter(&pidlock);
2032 			/*
2033 			 * Report an error if the process doesn't exist
2034 			 * or is actively being birthed.
2035 			 */
2036 			if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) {
2037 				mutex_exit(&pidlock);
2038 				return (ESRCH);
2039 			}
2040 			mutex_enter(&p->p_lock);
2041 			mutex_exit(&pidlock);
2042 
2043 			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2044 			    VREAD)) != 0) {
2045 				mutex_exit(&p->p_lock);
2046 				return (ret);
2047 			}
2048 
2049 			mutex_exit(&p->p_lock);
2050 		}
2051 
2052 		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2053 
2054 		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2055 		tp = fasttrap_tpoints.fth_table[index].ftb_data;
2056 		while (tp != NULL) {
2057 			if (instr.ftiq_pid == tp->ftt_pid &&
2058 			    instr.ftiq_pc == tp->ftt_pc &&
2059 			    tp->ftt_proc->ftpc_acount != 0)
2060 				break;
2061 
2062 			tp = tp->ftt_next;
2063 		}
2064 
2065 		if (tp == NULL) {
2066 			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2067 			return (ENOENT);
2068 		}
2069 
2070 		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2071 		    sizeof (instr.ftiq_instr));
2072 		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2073 
2074 		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2075 			return (EFAULT);
2076 
2077 		return (0);
2078 	}
2079 
2080 	return (EINVAL);
2081 }
2082 
2083 static struct cb_ops fasttrap_cb_ops = {
2084 	fasttrap_open,		/* open */
2085 	nodev,			/* close */
2086 	nulldev,		/* strategy */
2087 	nulldev,		/* print */
2088 	nodev,			/* dump */
2089 	nodev,			/* read */
2090 	nodev,			/* write */
2091 	fasttrap_ioctl,		/* ioctl */
2092 	nodev,			/* devmap */
2093 	nodev,			/* mmap */
2094 	nodev,			/* segmap */
2095 	nochpoll,		/* poll */
2096 	ddi_prop_op,		/* cb_prop_op */
2097 	0,			/* streamtab  */
2098 	D_NEW | D_MP		/* Driver compatibility flag */
2099 };
2100 
2101 /*ARGSUSED*/
2102 static int
fasttrap_info(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)2103 fasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
2104 {
2105 	int error;
2106 
2107 	switch (infocmd) {
2108 	case DDI_INFO_DEVT2DEVINFO:
2109 		*result = (void *)fasttrap_devi;
2110 		error = DDI_SUCCESS;
2111 		break;
2112 	case DDI_INFO_DEVT2INSTANCE:
2113 		*result = (void *)0;
2114 		error = DDI_SUCCESS;
2115 		break;
2116 	default:
2117 		error = DDI_FAILURE;
2118 	}
2119 	return (error);
2120 }
2121 
2122 static int
fasttrap_attach(dev_info_t * devi,ddi_attach_cmd_t cmd)2123 fasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
2124 {
2125 	ulong_t nent;
2126 
2127 	switch (cmd) {
2128 	case DDI_ATTACH:
2129 		break;
2130 	case DDI_RESUME:
2131 		return (DDI_SUCCESS);
2132 	default:
2133 		return (DDI_FAILURE);
2134 	}
2135 
2136 	if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0,
2137 	    DDI_PSEUDO, 0) == DDI_FAILURE) {
2138 		ddi_remove_minor_node(devi, NULL);
2139 		return (DDI_FAILURE);
2140 	}
2141 
2142 	ddi_report_dev(devi);
2143 	fasttrap_devi = devi;
2144 
2145 	/*
2146 	 * Install our hooks into fork(2), exec(2), and exit(2).
2147 	 */
2148 	dtrace_fasttrap_fork_ptr = &fasttrap_fork;
2149 	dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit;
2150 	dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit;
2151 
2152 	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2153 	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2154 	fasttrap_total = 0;
2155 
2156 	/*
2157 	 * Conjure up the tracepoints hashtable...
2158 	 */
2159 	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2160 	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2161 
2162 	if (nent == 0 || nent > 0x1000000)
2163 		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2164 
2165 	if (ISP2(nent))
2166 		fasttrap_tpoints.fth_nent = nent;
2167 	else
2168 		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2169 	ASSERT(fasttrap_tpoints.fth_nent > 0);
2170 	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2171 	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2172 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2173 
2174 	/*
2175 	 * ... and the providers hash table...
2176 	 */
2177 	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2178 	if (ISP2(nent))
2179 		fasttrap_provs.fth_nent = nent;
2180 	else
2181 		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2182 	ASSERT(fasttrap_provs.fth_nent > 0);
2183 	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2184 	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2185 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2186 
2187 	/*
2188 	 * ... and the procs hash table.
2189 	 */
2190 	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2191 	if (ISP2(nent))
2192 		fasttrap_procs.fth_nent = nent;
2193 	else
2194 		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2195 	ASSERT(fasttrap_procs.fth_nent > 0);
2196 	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2197 	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2198 	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2199 
2200 	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2201 	    &fasttrap_meta_id);
2202 
2203 	return (DDI_SUCCESS);
2204 }
2205 
2206 static int
fasttrap_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)2207 fasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
2208 {
2209 	int i, fail = 0;
2210 	timeout_id_t tmp;
2211 
2212 	switch (cmd) {
2213 	case DDI_DETACH:
2214 		break;
2215 	case DDI_SUSPEND:
2216 		return (DDI_SUCCESS);
2217 	default:
2218 		return (DDI_FAILURE);
2219 	}
2220 
2221 	/*
2222 	 * Unregister the meta-provider to make sure no new fasttrap-
2223 	 * managed providers come along while we're trying to close up
2224 	 * shop. If we fail to detach, we'll need to re-register as a
2225 	 * meta-provider. We can fail to unregister as a meta-provider
2226 	 * if providers we manage still exist.
2227 	 */
2228 	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2229 	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
2230 		return (DDI_FAILURE);
2231 
2232 	/*
2233 	 * Prevent any new timeouts from running by setting fasttrap_timeout
2234 	 * to a non-zero value, and wait for the current timeout to complete.
2235 	 */
2236 	mutex_enter(&fasttrap_cleanup_mtx);
2237 	fasttrap_cleanup_work = 0;
2238 
2239 	while (fasttrap_timeout != (timeout_id_t)1) {
2240 		tmp = fasttrap_timeout;
2241 		fasttrap_timeout = (timeout_id_t)1;
2242 
2243 		if (tmp != 0) {
2244 			mutex_exit(&fasttrap_cleanup_mtx);
2245 			(void) untimeout(tmp);
2246 			mutex_enter(&fasttrap_cleanup_mtx);
2247 		}
2248 	}
2249 
2250 	fasttrap_cleanup_work = 0;
2251 	mutex_exit(&fasttrap_cleanup_mtx);
2252 
2253 	/*
2254 	 * Iterate over all of our providers. If there's still a process
2255 	 * that corresponds to that pid, fail to detach.
2256 	 */
2257 	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2258 		fasttrap_provider_t **fpp, *fp;
2259 		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2260 
2261 		mutex_enter(&bucket->ftb_mtx);
2262 		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2263 		while ((fp = *fpp) != NULL) {
2264 			/*
2265 			 * Acquire and release the lock as a simple way of
2266 			 * waiting for any other consumer to finish with
2267 			 * this provider. A thread must first acquire the
2268 			 * bucket lock so there's no chance of another thread
2269 			 * blocking on the provider's lock.
2270 			 */
2271 			mutex_enter(&fp->ftp_mtx);
2272 			mutex_exit(&fp->ftp_mtx);
2273 
2274 			if (dtrace_unregister(fp->ftp_provid) != 0) {
2275 				fail = 1;
2276 				fpp = &fp->ftp_next;
2277 			} else {
2278 				*fpp = fp->ftp_next;
2279 				fasttrap_provider_free(fp);
2280 			}
2281 		}
2282 
2283 		mutex_exit(&bucket->ftb_mtx);
2284 	}
2285 
2286 	if (fail) {
2287 		uint_t work;
2288 		/*
2289 		 * If we're failing to detach, we need to unblock timeouts
2290 		 * and start a new timeout if any work has accumulated while
2291 		 * we've been unsuccessfully trying to detach.
2292 		 */
2293 		mutex_enter(&fasttrap_cleanup_mtx);
2294 		fasttrap_timeout = 0;
2295 		work = fasttrap_cleanup_work;
2296 		mutex_exit(&fasttrap_cleanup_mtx);
2297 
2298 		if (work)
2299 			fasttrap_pid_cleanup();
2300 
2301 		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2302 		    &fasttrap_meta_id);
2303 
2304 		return (DDI_FAILURE);
2305 	}
2306 
2307 #ifdef DEBUG
2308 	mutex_enter(&fasttrap_count_mtx);
2309 	ASSERT(fasttrap_pid_count == 0);
2310 	mutex_exit(&fasttrap_count_mtx);
2311 #endif
2312 
2313 	kmem_free(fasttrap_tpoints.fth_table,
2314 	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2315 	fasttrap_tpoints.fth_nent = 0;
2316 
2317 	kmem_free(fasttrap_provs.fth_table,
2318 	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2319 	fasttrap_provs.fth_nent = 0;
2320 
2321 	kmem_free(fasttrap_procs.fth_table,
2322 	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2323 	fasttrap_procs.fth_nent = 0;
2324 
2325 	/*
2326 	 * We know there are no tracepoints in any process anywhere in
2327 	 * the system so there is no process which has its p_dtrace_count
2328 	 * greater than zero, therefore we know that no thread can actively
2329 	 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes
2330 	 * and fasttrap_exec() and fasttrap_exit().
2331 	 */
2332 	ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork);
2333 	dtrace_fasttrap_fork_ptr = NULL;
2334 
2335 	ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit);
2336 	dtrace_fasttrap_exec_ptr = NULL;
2337 
2338 	ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit);
2339 	dtrace_fasttrap_exit_ptr = NULL;
2340 
2341 	ddi_remove_minor_node(devi, NULL);
2342 
2343 	return (DDI_SUCCESS);
2344 }
2345 
2346 static struct dev_ops fasttrap_ops = {
2347 	DEVO_REV,		/* devo_rev */
2348 	0,			/* refcnt */
2349 	fasttrap_info,		/* get_dev_info */
2350 	nulldev,		/* identify */
2351 	nulldev,		/* probe */
2352 	fasttrap_attach,	/* attach */
2353 	fasttrap_detach,	/* detach */
2354 	nodev,			/* reset */
2355 	&fasttrap_cb_ops,	/* driver operations */
2356 	NULL,			/* bus operations */
2357 	nodev,			/* dev power */
2358 	ddi_quiesce_not_needed,		/* quiesce */
2359 };
2360 
2361 /*
2362  * Module linkage information for the kernel.
2363  */
2364 static struct modldrv modldrv = {
2365 	&mod_driverops,		/* module type (this is a pseudo driver) */
2366 	"Fasttrap Tracing",	/* name of module */
2367 	&fasttrap_ops,		/* driver ops */
2368 };
2369 
2370 static struct modlinkage modlinkage = {
2371 	MODREV_1,
2372 	(void *)&modldrv,
2373 	NULL
2374 };
2375 
2376 int
_init(void)2377 _init(void)
2378 {
2379 	return (mod_install(&modlinkage));
2380 }
2381 
2382 int
_info(struct modinfo * modinfop)2383 _info(struct modinfo *modinfop)
2384 {
2385 	return (mod_info(&modlinkage, modinfop));
2386 }
2387 
2388 int
_fini(void)2389 _fini(void)
2390 {
2391 	return (mod_remove(&modlinkage));
2392 }
2393