libdtrace/common/dt_proc.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * DTrace Process Control
 *
 * This file provides a set of routines that permit libdtrace and its clients
 * to create and grab process handles using libproc, and to share these handles
 * between library mechanisms that need libproc access, such as ustack(), and
 * client mechanisms that need libproc access, such as dtrace(1M) -c and -p.
 * The library provides several mechanisms in the libproc control layer:
 *
 * Reference Counting: The library code and client code can independently grab
 * the same process handles without interfering with one another.  Only when
 * the reference count drops to zero and the handle is not being cached (see
 * below for more information on caching) will Prelease() be called on it.
 *
 * Handle Caching: If a handle is grabbed PGRAB_RDONLY (e.g. by ustack()) and
 * the reference count drops to zero, the handle is not immediately released.
 * Instead, libproc handles are maintained on dph_lrulist in order from most-
 * recently accessed to least-recently accessed.  Idle handles are maintained
 * until a pre-defined LRU cache limit is exceeded, permitting repeated calls
 * to ustack() to avoid the overhead of releasing and re-grabbing processes.
 *
 * Process Control: For processes that are grabbed for control (~PGRAB_RDONLY)
 * or created by dt_proc_create(), a control thread is created to provide
 * callbacks on process exit and symbol table caching on dlopen()s.
 *
 * MT-Safety: Libproc is not MT-Safe, so dt_proc_lock() and dt_proc_unlock()
 * are provided to synchronize access to the libproc handle between libdtrace
 * code and client code and the control thread's use of the ps_prochandle.
 *
 * NOTE: MT-Safety is NOT provided for libdtrace itself, or for use of the
 * dtrace_proc_grab/dtrace_proc_create mechanisms.  Like all exported libdtrace
 * calls, these are assumed to be MT-Unsafe.  MT-Safety is ONLY provided for
 * synchronization between libdtrace control threads and the client thread.
 *
 * The ps_prochandles themselves are maintained along with a dt_proc_t struct
 * in a hash table indexed by PID.  This provides basic locking and reference
 * counting.  The dt_proc_t is also maintained in LRU order on dph_lrulist.
 * The dph_lrucnt and dph_lrulim count the number of cacheable processes and
 * the current limit on the number of actively cached entries.
 *
 * The control thread for a process establishes breakpoints at the rtld_db
 * locations of interest, updates mappings and symbol tables at these points,
 * and handles exec and fork (by always following the parent).  The control
 * thread automatically exits when the process dies or control is lost.
 *
 * A simple notification mechanism is provided for libdtrace clients using
 * dtrace_handle_proc() for notification of PS_UNDEAD or PS_LOST events.  If
 * such an event occurs, the dt_proc_t itself is enqueued on a notification
 * list and the control thread broadcasts to dph_cv.  dtrace_sleep() will wake
 * up using this condition and will then call the client handler as necessary.
 */

#include <sys/wait.h>
#include <sys/lwp.h>
#include <strings.h>
#include <signal.h>
#include <assert.h>
#include <errno.h>

#include <dt_proc.h>
#include <dt_pid.h>
#include <dt_impl.h>

#define	IS_SYS_EXEC(w)	(w == SYS_execve)
#define	IS_SYS_FORK(w)	(w == SYS_vfork || w == SYS_forksys)

static dt_bkpt_t *
dt_proc_bpcreate(dt_proc_t *dpr, uintptr_t addr, dt_bkpt_f *func, void *data)
{
	struct ps_prochandle *P = dpr->dpr_proc;
	dt_bkpt_t *dbp;

	assert(MUTEX_HELD(&dpr->dpr_lock));

	if ((dbp = dt_zalloc(dpr->dpr_hdl, sizeof (dt_bkpt_t))) != NULL) {
		dbp->dbp_func = func;
		dbp->dbp_data = data;
		dbp->dbp_addr = addr;

		if (Psetbkpt(P, dbp->dbp_addr, &dbp->dbp_instr) == 0)
			dbp->dbp_active = B_TRUE;

		dt_list_append(&dpr->dpr_bps, dbp);
	}

	return (dbp);
}

static void
dt_proc_bpdestroy(dt_proc_t *dpr, int delbkpts)
{
	int state = Pstate(dpr->dpr_proc);
	dt_bkpt_t *dbp, *nbp;

	assert(MUTEX_HELD(&dpr->dpr_lock));

	for (dbp = dt_list_next(&dpr->dpr_bps); dbp != NULL; dbp = nbp) {
		if (delbkpts && dbp->dbp_active &&
		    state != PS_LOST && state != PS_UNDEAD) {
			(void) Pdelbkpt(dpr->dpr_proc,
			    dbp->dbp_addr, dbp->dbp_instr);
		}
		nbp = dt_list_next(dbp);
		dt_list_delete(&dpr->dpr_bps, dbp);
		dt_free(dpr->dpr_hdl, dbp);
	}
}

static void
dt_proc_bpmatch(dtrace_hdl_t *dtp, dt_proc_t *dpr)
{
	const lwpstatus_t *psp = &Pstatus(dpr->dpr_proc)->pr_lwp;
	dt_bkpt_t *dbp;

	assert(MUTEX_HELD(&dpr->dpr_lock));

	for (dbp = dt_list_next(&dpr->dpr_bps);
	    dbp != NULL; dbp = dt_list_next(dbp)) {
		if (psp->pr_reg[R_PC] == dbp->dbp_addr)
			break;
	}

	if (dbp == NULL) {
		dt_dprintf("pid %d: spurious breakpoint wakeup for %lx\n",
		    (int)dpr->dpr_pid, (ulong_t)psp->pr_reg[R_PC]);
		return;
	}

	dt_dprintf("pid %d: hit breakpoint at %lx (%lu)\n",
	    (int)dpr->dpr_pid, (ulong_t)dbp->dbp_addr, ++dbp->dbp_hits);

	dbp->dbp_func(dtp, dpr, dbp->dbp_data);
	(void) Pxecbkpt(dpr->dpr_proc, dbp->dbp_instr);
}

static void
dt_proc_bpenable(dt_proc_t *dpr)
{
	dt_bkpt_t *dbp;

	assert(MUTEX_HELD(&dpr->dpr_lock));

	for (dbp = dt_list_next(&dpr->dpr_bps);
	    dbp != NULL; dbp = dt_list_next(dbp)) {
		if (!dbp->dbp_active && Psetbkpt(dpr->dpr_proc,
		    dbp->dbp_addr, &dbp->dbp_instr) == 0)
			dbp->dbp_active = B_TRUE;
	}

	dt_dprintf("breakpoints enabled\n");
}

static void
dt_proc_bpdisable(dt_proc_t *dpr)
{
	dt_bkpt_t *dbp;

	assert(MUTEX_HELD(&dpr->dpr_lock));

	for (dbp = dt_list_next(&dpr->dpr_bps);
	    dbp != NULL; dbp = dt_list_next(dbp)) {
		if (dbp->dbp_active && Pdelbkpt(dpr->dpr_proc,
		    dbp->dbp_addr, dbp->dbp_instr) == 0)
			dbp->dbp_active = B_FALSE;
	}

	dt_dprintf("breakpoints disabled\n");
}

static void
dt_proc_notify(dtrace_hdl_t *dtp, dt_proc_hash_t *dph, dt_proc_t *dpr,
    const char *msg)
{
	dt_proc_notify_t *dprn = dt_alloc(dtp, sizeof (dt_proc_notify_t));

	if (dprn == NULL) {
		dt_dprintf("failed to allocate notification for %d %s\n",
		    (int)dpr->dpr_pid, msg);
	} else {
		dprn->dprn_dpr = dpr;
		if (msg == NULL)
			dprn->dprn_errmsg[0] = '\0';
		else
			(void) strlcpy(dprn->dprn_errmsg, msg,
			    sizeof (dprn->dprn_errmsg));

		(void) pthread_mutex_lock(&dph->dph_lock);

		dprn->dprn_next = dph->dph_notify;
		dph->dph_notify = dprn;

		(void) pthread_cond_broadcast(&dph->dph_cv);
		(void) pthread_mutex_unlock(&dph->dph_lock);
	}
}

/*
 * Check to see if the control thread was requested to stop when the victim
 * process reached a particular event (why) rather than continuing the victim.
 * If 'why' is set in the stop mask, we wait on dpr_cv for dt_proc_continue().
 * If 'why' is not set, this function returns immediately and does nothing.
 */
static void
dt_proc_stop(dt_proc_t *dpr, uint8_t why)
{
	assert(MUTEX_HELD(&dpr->dpr_lock));
	assert(why != DT_PROC_STOP_IDLE);

	if (dpr->dpr_stop & why) {
		dpr->dpr_stop |= DT_PROC_STOP_IDLE;
		dpr->dpr_stop &= ~why;

		(void) pthread_cond_broadcast(&dpr->dpr_cv);

		/*
		 * We disable breakpoints while stopped to preserve the
		 * integrity of the program text for both our own disassembly
		 * and that of the kernel.
		 */
		dt_proc_bpdisable(dpr);

		while (dpr->dpr_stop & DT_PROC_STOP_IDLE)
			(void) pthread_cond_wait(&dpr->dpr_cv, &dpr->dpr_lock);

		dt_proc_bpenable(dpr);
	}
}

/*ARGSUSED*/
static void
dt_proc_bpmain(dtrace_hdl_t *dtp, dt_proc_t *dpr, const char *fname)
{
	dt_dprintf("pid %d: breakpoint at %s()\n", (int)dpr->dpr_pid, fname);
	dt_proc_stop(dpr, DT_PROC_STOP_MAIN);
}

static void
dt_proc_rdevent(dtrace_hdl_t *dtp, dt_proc_t *dpr, const char *evname)
{
	rd_event_msg_t rdm;
	rd_err_e err;

	if ((err = rd_event_getmsg(dpr->dpr_rtld, &rdm)) != RD_OK) {
		dt_dprintf("pid %d: failed to get %s event message: %s\n",
		    (int)dpr->dpr_pid, evname, rd_errstr(err));
		return;
	}

	dt_dprintf("pid %d: rtld event %s type=%d state %d\n",
	    (int)dpr->dpr_pid, evname, rdm.type, rdm.u.state);

	switch (rdm.type) {
	case RD_DLACTIVITY:
		if (rdm.u.state != RD_CONSISTENT)
			break;

		Pupdate_syms(dpr->dpr_proc);
		if (dt_pid_create_probes_module(dtp, dpr) != 0)
			dt_proc_notify(dtp, dtp->dt_procs, dpr,
			    dpr->dpr_errmsg);

		break;
	case RD_PREINIT:
		Pupdate_syms(dpr->dpr_proc);
		dt_proc_stop(dpr, DT_PROC_STOP_PREINIT);
		break;
	case RD_POSTINIT:
		Pupdate_syms(dpr->dpr_proc);
		dt_proc_stop(dpr, DT_PROC_STOP_POSTINIT);
		break;
	}
}

static void
dt_proc_rdwatch(dt_proc_t *dpr, rd_event_e event, const char *evname)
{
	rd_notify_t rdn;
	rd_err_e err;

	if ((err = rd_event_addr(dpr->dpr_rtld, event, &rdn)) != RD_OK) {
		dt_dprintf("pid %d: failed to get event address for %s: %s\n",
		    (int)dpr->dpr_pid, evname, rd_errstr(err));
		return;
	}

	if (rdn.type != RD_NOTIFY_BPT) {
		dt_dprintf("pid %d: event %s has unexpected type %d\n",
		    (int)dpr->dpr_pid, evname, rdn.type);
		return;
	}

	(void) dt_proc_bpcreate(dpr, rdn.u.bptaddr,
	    (dt_bkpt_f *)dt_proc_rdevent, (void *)evname);
}

/*
 * Common code for enabling events associated with the run-time linker after
 * attaching to a process or after a victim process completes an exec(2).
 */
static void
dt_proc_attach(dt_proc_t *dpr, int exec)
{
	const pstatus_t *psp = Pstatus(dpr->dpr_proc);
	rd_err_e err;
	GElf_Sym sym;

	assert(MUTEX_HELD(&dpr->dpr_lock));

	if (exec) {
		if (psp->pr_lwp.pr_errno != 0)
			return; /* exec failed: nothing needs to be done */

		dt_proc_bpdestroy(dpr, B_FALSE);
		Preset_maps(dpr->dpr_proc);
	}

	if ((dpr->dpr_rtld = Prd_agent(dpr->dpr_proc)) != NULL &&
	    (err = rd_event_enable(dpr->dpr_rtld, B_TRUE)) == RD_OK) {
		dt_proc_rdwatch(dpr, RD_PREINIT, "RD_PREINIT");
		dt_proc_rdwatch(dpr, RD_POSTINIT, "RD_POSTINIT");
		dt_proc_rdwatch(dpr, RD_DLACTIVITY, "RD_DLACTIVITY");
	} else {
		dt_dprintf("pid %d: failed to enable rtld events: %s\n",
		    (int)dpr->dpr_pid, dpr->dpr_rtld ? rd_errstr(err) :
		    "rtld_db agent initialization failed");
	}

	Pupdate_maps(dpr->dpr_proc);

	if (Pxlookup_by_name(dpr->dpr_proc, LM_ID_BASE,
	    "a.out", "main", &sym, NULL) == 0) {
		(void) dt_proc_bpcreate(dpr, (uintptr_t)sym.st_value,
		    (dt_bkpt_f *)dt_proc_bpmain, "a.out`main");
	} else {
		dt_dprintf("pid %d: failed to find a.out`main: %s\n",
		    (int)dpr->dpr_pid, strerror(errno));
	}
}

/*
 * Wait for a stopped process to be set running again by some other debugger.
 * This is typically not required by /proc-based debuggers, since the usual
 * model is that one debugger controls one victim.  But DTrace, as usual, has
 * its own needs: the stop() action assumes that prun(1) or some other tool
 * will be applied to resume the victim process.  This could be solved by
 * adding a PCWRUN directive to /proc, but that seems like overkill unless
 * other debuggers end up needing this functionality, so we implement a cheap
 * equivalent to PCWRUN using the set of existing kernel mechanisms.
 *
 * Our intent is really not just to wait for the victim to run, but rather to
 * wait for it to run and then stop again for a reason other than the current
 * PR_REQUESTED stop.  Since PCWSTOP/Pstopstatus() can be applied repeatedly
 * to a stopped process and will return the same result without affecting the
 * victim, we can just perform these operations repeatedly until Pstate()
 * changes, the representative LWP ID changes, or the stop timestamp advances.
 * dt_proc_control() will then rediscover the new state and continue as usual.
 * When the process is still stopped in the same exact state, we sleep for a
 * brief interval before waiting again so as not to spin consuming CPU cycles.
 */
static void
dt_proc_waitrun(dt_proc_t *dpr)
{
	struct ps_prochandle *P = dpr->dpr_proc;
	const lwpstatus_t *psp = &Pstatus(P)->pr_lwp;

	int krflag = psp->pr_flags & (PR_KLC | PR_RLC);
	timestruc_t tstamp = psp->pr_tstamp;
	lwpid_t lwpid = psp->pr_lwpid;

	const long wstop = PCWSTOP;
	int pfd = Pctlfd(P);

	assert(MUTEX_HELD(&dpr->dpr_lock));
	assert(psp->pr_flags & PR_STOPPED);
	assert(Pstate(P) == PS_STOP);

	/*
	 * While we are waiting for the victim to run, clear PR_KLC and PR_RLC
	 * so that if the libdtrace client is killed, the victim stays stopped.
	 * dt_proc_destroy() will also observe this and perform PRELEASE_HANG.
	 */
	(void) Punsetflags(P, krflag);
	Psync(P);

	(void) pthread_mutex_unlock(&dpr->dpr_lock);

	while (!dpr->dpr_quit) {
		if (write(pfd, &wstop, sizeof (wstop)) == -1 && errno == EINTR)
			continue; /* check dpr_quit and continue waiting */

		(void) pthread_mutex_lock(&dpr->dpr_lock);
		(void) Pstopstatus(P, PCNULL, 0);
		psp = &Pstatus(P)->pr_lwp;

		/*
		 * If we've reached a new state, found a new representative, or
		 * the stop timestamp has changed, restore PR_KLC/PR_RLC to its
		 * original setting and then return with dpr_lock held.
		 */
		if (Pstate(P) != PS_STOP || psp->pr_lwpid != lwpid ||
		    bcmp(&psp->pr_tstamp, &tstamp, sizeof (tstamp)) != 0) {
			(void) Psetflags(P, krflag);
			Psync(P);
			return;
		}

		(void) pthread_mutex_unlock(&dpr->dpr_lock);
		(void) poll(NULL, 0, MILLISEC / 2);
	}

	(void) pthread_mutex_lock(&dpr->dpr_lock);
}

typedef struct dt_proc_control_data {
	dtrace_hdl_t *dpcd_hdl;			/* DTrace handle */
	dt_proc_t *dpcd_proc;			/* proccess to control */
} dt_proc_control_data_t;

/*
 * Main loop for all victim process control threads.  We initialize all the
 * appropriate /proc control mechanisms, and then enter a loop waiting for
 * the process to stop on an event or die.  We process any events by calling
 * appropriate subroutines, and exit when the victim dies or we lose control.
 *
 * The control thread synchronizes the use of dpr_proc with other libdtrace
 * threads using dpr_lock.  We hold the lock for all of our operations except
 * waiting while the process is running: this is accomplished by writing a
 * PCWSTOP directive directly to the underlying /proc/<pid>/ctl file.  If the
 * libdtrace client wishes to exit or abort our wait, SIGCANCEL can be used.
 */
static void *
dt_proc_control(void *arg)
{
	dt_proc_control_data_t *datap = arg;
	dtrace_hdl_t *dtp = datap->dpcd_hdl;
	dt_proc_t *dpr = datap->dpcd_proc;
	dt_proc_hash_t *dph = dpr->dpr_hdl->dt_procs;
	struct ps_prochandle *P = dpr->dpr_proc;

	int pfd = Pctlfd(P);
	int pid = dpr->dpr_pid;

	const long wstop = PCWSTOP;
	int notify = B_FALSE;

	/*
	 * We disable the POSIX thread cancellation mechanism so that the
	 * client program using libdtrace can't accidentally cancel our thread.
	 * dt_proc_destroy() uses SIGCANCEL explicitly to simply poke us out
	 * of PCWSTOP with EINTR, at which point we will see dpr_quit and exit.
	 */
	(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);

	/*
	 * Set up the corresponding process for tracing by libdtrace.  We want
	 * to be able to catch breakpoints and efficiently single-step over
	 * them, and we need to enable librtld_db to watch libdl activity.
	 */
	(void) pthread_mutex_lock(&dpr->dpr_lock);

	(void) Punsetflags(P, PR_ASYNC);	/* require synchronous mode */
	(void) Psetflags(P, PR_BPTADJ);		/* always adjust eip on x86 */
	(void) Punsetflags(P, PR_FORK);		/* do not inherit on fork */

	(void) Pfault(P, FLTBPT, B_TRUE);	/* always trace breakpoints */
	(void) Pfault(P, FLTTRACE, B_TRUE);	/* always trace single-step */

	/*
	 * We must trace exit from exec() system calls so that if the exec is
	 * successful, we can reset our breakpoints and re-initialize libproc.
	 */
	(void) Psysexit(P, SYS_execve, B_TRUE);

	/*
	 * We must trace entry and exit for fork() system calls in order to
	 * disable our breakpoints temporarily during the fork.  We do not set
	 * the PR_FORK flag, so if fork succeeds the child begins executing and
	 * does not inherit any other tracing behaviors or a control thread.
	 */
	(void) Psysentry(P, SYS_vfork, B_TRUE);
	(void) Psysexit(P, SYS_vfork, B_TRUE);
	(void) Psysentry(P, SYS_forksys, B_TRUE);
	(void) Psysexit(P, SYS_forksys, B_TRUE);

	Psync(P);				/* enable all /proc changes */
	dt_proc_attach(dpr, B_FALSE);		/* enable rtld breakpoints */

	/*
	 * If PR_KLC is set, we created the process; otherwise we grabbed it.
	 * Check for an appropriate stop request and wait for dt_proc_continue.
	 */
	if (Pstatus(P)->pr_flags & PR_KLC)
		dt_proc_stop(dpr, DT_PROC_STOP_CREATE);
	else
		dt_proc_stop(dpr, DT_PROC_STOP_GRAB);

	if (Psetrun(P, 0, 0) == -1) {
		dt_dprintf("pid %d: failed to set running: %s\n",
		    (int)dpr->dpr_pid, strerror(errno));
	}

	(void) pthread_mutex_unlock(&dpr->dpr_lock);

	/*
	 * Wait for the process corresponding to this control thread to stop,
	 * process the event, and then set it running again.  We want to sleep
	 * with dpr_lock *unheld* so that other parts of libdtrace can use the
	 * ps_prochandle in the meantime (e.g. ustack()).  To do this, we write
	 * a PCWSTOP directive directly to the underlying /proc/<pid>/ctl file.
	 * Once the process stops, we wake up, grab dpr_lock, and then call
	 * Pwait() (which will return immediately) and do our processing.
	 */
	while (!dpr->dpr_quit) {
		const lwpstatus_t *psp;

		if (write(pfd, &wstop, sizeof (wstop)) == -1 && errno == EINTR)
			continue; /* check dpr_quit and continue waiting */

		(void) pthread_mutex_lock(&dpr->dpr_lock);
pwait_locked:
		if (Pstopstatus(P, PCNULL, 0) == -1 && errno == EINTR) {
			(void) pthread_mutex_unlock(&dpr->dpr_lock);
			continue; /* check dpr_quit and continue waiting */
		}

		switch (Pstate(P)) {
		case PS_STOP:
			psp = &Pstatus(P)->pr_lwp;

			dt_dprintf("pid %d: proc stopped showing %d/%d\n",
			    pid, psp->pr_why, psp->pr_what);

			/*
			 * If the process stops showing PR_REQUESTED, then the
			 * DTrace stop() action was applied to it or another
			 * debugging utility (e.g. pstop(1)) asked it to stop.
			 * In either case, the user's intention is for the
			 * process to remain stopped until another external
			 * mechanism (e.g. prun(1)) is applied.  So instead of
			 * setting the process running ourself, we wait for
			 * someone else to do so.  Once that happens, we return
			 * to our normal loop waiting for an event of interest.
			 */
			if (psp->pr_why == PR_REQUESTED) {
				dt_proc_waitrun(dpr);
				(void) pthread_mutex_unlock(&dpr->dpr_lock);
				continue;
			}

			/*
			 * If the process stops showing one of the events that
			 * we are tracing, perform the appropriate response.
			 * Note that we ignore PR_SUSPENDED, PR_CHECKPOINT, and
			 * PR_JOBCONTROL by design: if one of these conditions
			 * occurs, we will fall through to Psetrun() but the
			 * process will remain stopped in the kernel by the
			 * corresponding mechanism (e.g. job control stop).
			 */
			if (psp->pr_why == PR_FAULTED && psp->pr_what == FLTBPT)
				dt_proc_bpmatch(dtp, dpr);
			else if (psp->pr_why == PR_SYSENTRY &&
			    IS_SYS_FORK(psp->pr_what))
				dt_proc_bpdisable(dpr);
			else if (psp->pr_why == PR_SYSEXIT &&
			    IS_SYS_FORK(psp->pr_what))
				dt_proc_bpenable(dpr);
			else if (psp->pr_why == PR_SYSEXIT &&
			    IS_SYS_EXEC(psp->pr_what))
				dt_proc_attach(dpr, B_TRUE);
			break;

		case PS_LOST:
			if (Preopen(P) == 0)
				goto pwait_locked;

			dt_dprintf("pid %d: proc lost: %s\n",
			    pid, strerror(errno));

			dpr->dpr_quit = B_TRUE;
			notify = B_TRUE;
			break;

		case PS_UNDEAD:
			dt_dprintf("pid %d: proc died\n", pid);
			dpr->dpr_quit = B_TRUE;
			notify = B_TRUE;
			break;
		}

		if (Pstate(P) != PS_UNDEAD && Psetrun(P, 0, 0) == -1) {
			dt_dprintf("pid %d: failed to set running: %s\n",
			    (int)dpr->dpr_pid, strerror(errno));
		}

		(void) pthread_mutex_unlock(&dpr->dpr_lock);
	}

	/*
	 * If the control thread detected PS_UNDEAD or PS_LOST, then enqueue
	 * the dt_proc_t structure on the dt_proc_hash_t notification list.
	 */
	if (notify)
		dt_proc_notify(dtp, dph, dpr, NULL);

	/*
	 * Destroy and remove any remaining breakpoints, set dpr_done and clear
	 * dpr_tid to indicate the control thread has exited, and notify any
	 * waiting thread in dt_proc_destroy() that we have succesfully exited.
	 */
	(void) pthread_mutex_lock(&dpr->dpr_lock);

	dt_proc_bpdestroy(dpr, B_TRUE);
	dpr->dpr_done = B_TRUE;
	dpr->dpr_tid = 0;

	(void) pthread_cond_broadcast(&dpr->dpr_cv);
	(void) pthread_mutex_unlock(&dpr->dpr_lock);

	return (NULL);
}

/*PRINTFLIKE3*/
static struct ps_prochandle *
dt_proc_error(dtrace_hdl_t *dtp, dt_proc_t *dpr, const char *format, ...)
{
	va_list ap;

	va_start(ap, format);
	dt_set_errmsg(dtp, NULL, NULL, NULL, 0, format, ap);
	va_end(ap);

	if (dpr->dpr_proc != NULL)
		Prelease(dpr->dpr_proc, 0);

	dt_free(dtp, dpr);
	(void) dt_set_errno(dtp, EDT_COMPILER);
	return (NULL);
}

dt_proc_t *
dt_proc_lookup(dtrace_hdl_t *dtp, struct ps_prochandle *P, int remove)
{
	dt_proc_hash_t *dph = dtp->dt_procs;
	pid_t pid = Pstatus(P)->pr_pid;
	dt_proc_t *dpr, **dpp = &dph->dph_hash[pid & (dph->dph_hashlen - 1)];

	for (dpr = *dpp; dpr != NULL; dpr = dpr->dpr_hash) {
		if (dpr->dpr_pid == pid)
			break;
		else
			dpp = &dpr->dpr_hash;
	}

	assert(dpr != NULL);
	assert(dpr->dpr_proc == P);

	if (remove)
		*dpp = dpr->dpr_hash; /* remove from pid hash chain */

	return (dpr);
}

static void
dt_proc_destroy(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_t *dpr = dt_proc_lookup(dtp, P, B_FALSE);
	dt_proc_hash_t *dph = dtp->dt_procs;
	dt_proc_notify_t *npr, **npp;
	int rflag;

	assert(dpr != NULL);

	/*
	 * If neither PR_KLC nor PR_RLC is set, then the process is stopped by
	 * an external debugger and we were waiting in dt_proc_waitrun().
	 * Leave the process in this condition using PRELEASE_HANG.
	 */
	if (!(Pstatus(dpr->dpr_proc)->pr_flags & (PR_KLC | PR_RLC))) {
		dt_dprintf("abandoning pid %d\n", (int)dpr->dpr_pid);
		rflag = PRELEASE_HANG;
	} else if (Pstatus(dpr->dpr_proc)->pr_flags & PR_KLC) {
		dt_dprintf("killing pid %d\n", (int)dpr->dpr_pid);
		rflag = PRELEASE_KILL; /* apply kill-on-last-close */
	} else {
		dt_dprintf("releasing pid %d\n", (int)dpr->dpr_pid);
		rflag = 0; /* apply run-on-last-close */
	}

	if (dpr->dpr_tid) {
		/*
		 * Set the dpr_quit flag to tell the daemon thread to exit.  We
		 * send it a SIGCANCEL to poke it out of PCWSTOP or any other
		 * long-term /proc system call.  Our daemon threads have POSIX
		 * cancellation disabled, so EINTR will be the only effect.  We
		 * then wait for dpr_done to indicate the thread has exited.
		 *
		 * We can't use pthread_kill() to send SIGCANCEL because the
		 * interface forbids it and we can't use pthread_cancel()
		 * because with cancellation disabled it won't actually
		 * send SIGCANCEL to the target thread, so we use _lwp_kill()
		 * to do the job.  This is all built on evil knowledge of
		 * the details of the cancellation mechanism in libc.
		 */
		(void) pthread_mutex_lock(&dpr->dpr_lock);
		dpr->dpr_quit = B_TRUE;
		(void) _lwp_kill(dpr->dpr_tid, SIGCANCEL);

		/*
		 * If the process is currently idling in dt_proc_stop(), re-
		 * enable breakpoints and poke it into running again.
		 */
		if (dpr->dpr_stop & DT_PROC_STOP_IDLE) {
			dt_proc_bpenable(dpr);
			dpr->dpr_stop &= ~DT_PROC_STOP_IDLE;
			(void) pthread_cond_broadcast(&dpr->dpr_cv);
		}

		while (!dpr->dpr_done)
			(void) pthread_cond_wait(&dpr->dpr_cv, &dpr->dpr_lock);

		(void) pthread_mutex_unlock(&dpr->dpr_lock);
	}

	/*
	 * Before we free the process structure, remove this dt_proc_t from the
	 * lookup hash, and then walk the dt_proc_hash_t's notification list
	 * and remove this dt_proc_t if it is enqueued.
	 */
	(void) pthread_mutex_lock(&dph->dph_lock);
	(void) dt_proc_lookup(dtp, P, B_TRUE);
	npp = &dph->dph_notify;

	while ((npr = *npp) != NULL) {
		if (npr->dprn_dpr == dpr) {
			*npp = npr->dprn_next;
			dt_free(dtp, npr);
		} else {
			npp = &npr->dprn_next;
		}
	}

	(void) pthread_mutex_unlock(&dph->dph_lock);

	/*
	 * Remove the dt_proc_list from the LRU list, release the underlying
	 * libproc handle, and free our dt_proc_t data structure.
	 */
	if (dpr->dpr_cacheable) {
		assert(dph->dph_lrucnt != 0);
		dph->dph_lrucnt--;
	}

	dt_list_delete(&dph->dph_lrulist, dpr);
	Prelease(dpr->dpr_proc, rflag);
	dt_free(dtp, dpr);
}

static int
dt_proc_create_thread(dtrace_hdl_t *dtp, dt_proc_t *dpr, uint_t stop)
{
	dt_proc_control_data_t data;
	sigset_t nset, oset;
	pthread_attr_t a;
	int err;

	(void) pthread_mutex_lock(&dpr->dpr_lock);
	dpr->dpr_stop |= stop; /* set bit for initial rendezvous */

	(void) pthread_attr_init(&a);
	(void) pthread_attr_setdetachstate(&a, PTHREAD_CREATE_DETACHED);

	(void) sigfillset(&nset);
	(void) sigdelset(&nset, SIGABRT);	/* unblocked for assert() */
	(void) sigdelset(&nset, SIGCANCEL);	/* see dt_proc_destroy() */

	data.dpcd_hdl = dtp;
	data.dpcd_proc = dpr;

	(void) pthread_sigmask(SIG_SETMASK, &nset, &oset);
	err = pthread_create(&dpr->dpr_tid, &a, dt_proc_control, &data);
	(void) pthread_sigmask(SIG_SETMASK, &oset, NULL);

	/*
	 * If the control thread was created, then wait on dpr_cv for either
	 * dpr_done to be set (the victim died or the control thread failed)
	 * or DT_PROC_STOP_IDLE to be set, indicating that the victim is now
	 * stopped by /proc and the control thread is at the rendezvous event.
	 * On success, we return with the process and control thread stopped:
	 * the caller can then apply dt_proc_continue() to resume both.
	 */
	if (err == 0) {
		while (!dpr->dpr_done && !(dpr->dpr_stop & DT_PROC_STOP_IDLE))
			(void) pthread_cond_wait(&dpr->dpr_cv, &dpr->dpr_lock);

		/*
		 * If dpr_done is set, the control thread aborted before it
		 * reached the rendezvous event.  This is either due to PS_LOST
		 * or PS_UNDEAD (i.e. the process died).  We try to provide a
		 * small amount of useful information to help figure it out.
		 */
		if (dpr->dpr_done) {
			const psinfo_t *prp = Ppsinfo(dpr->dpr_proc);
			int stat = prp ? prp->pr_wstat : 0;
			int pid = dpr->dpr_pid;

			if (Pstate(dpr->dpr_proc) == PS_LOST) {
				(void) dt_proc_error(dpr->dpr_hdl, dpr,
				    "failed to control pid %d: process exec'd "
				    "set-id or unobservable program\n", pid);
			} else if (WIFSIGNALED(stat)) {
				(void) dt_proc_error(dpr->dpr_hdl, dpr,
				    "failed to control pid %d: process died "
				    "from signal %d\n", pid, WTERMSIG(stat));
			} else {
				(void) dt_proc_error(dpr->dpr_hdl, dpr,
				    "failed to control pid %d: process exited "
				    "with status %d\n", pid, WEXITSTATUS(stat));
			}

			err = ESRCH; /* cause grab() or create() to fail */
		}
	} else {
		(void) dt_proc_error(dpr->dpr_hdl, dpr,
		    "failed to create control thread for process-id %d: %s\n",
		    (int)dpr->dpr_pid, strerror(err));
	}

	(void) pthread_mutex_unlock(&dpr->dpr_lock);
	(void) pthread_attr_destroy(&a);

	return (err);
}

struct ps_prochandle *
dt_proc_create(dtrace_hdl_t *dtp, const char *file, char *const *argv)
{
	dt_proc_hash_t *dph = dtp->dt_procs;
	dt_proc_t *dpr;
	int err;

	if ((dpr = dt_zalloc(dtp, sizeof (dt_proc_t))) == NULL)
		return (NULL); /* errno is set for us */

	(void) pthread_mutex_init(&dpr->dpr_lock, NULL);
	(void) pthread_cond_init(&dpr->dpr_cv, NULL);

	if ((dpr->dpr_proc = Pcreate(file, argv, &err, NULL, 0)) == NULL) {
		return (dt_proc_error(dtp, dpr,
		    "failed to execute %s: %s\n", file, Pcreate_error(err)));
	}

	dpr->dpr_hdl = dtp;
	dpr->dpr_pid = Pstatus(dpr->dpr_proc)->pr_pid;

	(void) Punsetflags(dpr->dpr_proc, PR_RLC);
	(void) Psetflags(dpr->dpr_proc, PR_KLC);

	if (dt_proc_create_thread(dtp, dpr, dtp->dt_prcmode) != 0)
		return (NULL); /* dt_proc_error() has been called for us */

	dpr->dpr_hash = dph->dph_hash[dpr->dpr_pid & (dph->dph_hashlen - 1)];
	dph->dph_hash[dpr->dpr_pid & (dph->dph_hashlen - 1)] = dpr;
	dt_list_prepend(&dph->dph_lrulist, dpr);

	dt_dprintf("created pid %d\n", (int)dpr->dpr_pid);
	dpr->dpr_refs++;

	return (dpr->dpr_proc);
}

struct ps_prochandle *
dt_proc_grab(dtrace_hdl_t *dtp, pid_t pid, int flags, int nomonitor)
{
	dt_proc_hash_t *dph = dtp->dt_procs;
	uint_t h = pid & (dph->dph_hashlen - 1);
	dt_proc_t *dpr, *opr;
	int err;

	/*
	 * Search the hash table for the pid.  If it is already grabbed or
	 * created, move the handle to the front of the lrulist, increment
	 * the reference count, and return the existing ps_prochandle.
	 */
	for (dpr = dph->dph_hash[h]; dpr != NULL; dpr = dpr->dpr_hash) {
		if (dpr->dpr_pid == pid && !dpr->dpr_stale) {
			/*
			 * If the cached handle was opened read-only and
			 * this request is for a writeable handle, mark
			 * the cached handle as stale and open a new handle.
			 * Since it's stale, unmark it as cacheable.
			 */
			if (dpr->dpr_rdonly && !(flags & PGRAB_RDONLY)) {
				dt_dprintf("upgrading pid %d\n", (int)pid);
				dpr->dpr_stale = B_TRUE;
				dpr->dpr_cacheable = B_FALSE;
				dph->dph_lrucnt--;
				break;
			}

			dt_dprintf("grabbed pid %d (cached)\n", (int)pid);
			dt_list_delete(&dph->dph_lrulist, dpr);
			dt_list_prepend(&dph->dph_lrulist, dpr);
			dpr->dpr_refs++;
			return (dpr->dpr_proc);
		}
	}

	if ((dpr = dt_zalloc(dtp, sizeof (dt_proc_t))) == NULL)
		return (NULL); /* errno is set for us */

	(void) pthread_mutex_init(&dpr->dpr_lock, NULL);
	(void) pthread_cond_init(&dpr->dpr_cv, NULL);

	if ((dpr->dpr_proc = Pgrab(pid, flags, &err)) == NULL) {
		return (dt_proc_error(dtp, dpr,
		    "failed to grab pid %d: %s\n", (int)pid, Pgrab_error(err)));
	}

	dpr->dpr_hdl = dtp;
	dpr->dpr_pid = pid;

	(void) Punsetflags(dpr->dpr_proc, PR_KLC);
	(void) Psetflags(dpr->dpr_proc, PR_RLC);

	/*
	 * If we are attempting to grab the process without a monitor
	 * thread, then mark the process cacheable only if it's being
	 * grabbed read-only.  If we're currently caching more process
	 * handles than dph_lrulim permits, attempt to find the
	 * least-recently-used handle that is currently unreferenced and
	 * release it from the cache.  Otherwise we are grabbing the process
	 * for control: create a control thread for this process and store
	 * its ID in dpr->dpr_tid.
	 */
	if (nomonitor || (flags & PGRAB_RDONLY)) {
		if (dph->dph_lrucnt >= dph->dph_lrulim) {
			for (opr = dt_list_prev(&dph->dph_lrulist);
			    opr != NULL; opr = dt_list_prev(opr)) {
				if (opr->dpr_cacheable && opr->dpr_refs == 0) {
					dt_proc_destroy(dtp, opr->dpr_proc);
					break;
				}
			}
		}

		if (flags & PGRAB_RDONLY) {
			dpr->dpr_cacheable = B_TRUE;
			dpr->dpr_rdonly = B_TRUE;
			dph->dph_lrucnt++;
		}

	} else if (dt_proc_create_thread(dtp, dpr, DT_PROC_STOP_GRAB) != 0)
		return (NULL); /* dt_proc_error() has been called for us */

	dpr->dpr_hash = dph->dph_hash[h];
	dph->dph_hash[h] = dpr;
	dt_list_prepend(&dph->dph_lrulist, dpr);

	dt_dprintf("grabbed pid %d\n", (int)pid);
	dpr->dpr_refs++;

	return (dpr->dpr_proc);
}

void
dt_proc_release(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_t *dpr = dt_proc_lookup(dtp, P, B_FALSE);
	dt_proc_hash_t *dph = dtp->dt_procs;

	assert(dpr != NULL);
	assert(dpr->dpr_refs != 0);

	if (--dpr->dpr_refs == 0 &&
	    (!dpr->dpr_cacheable || dph->dph_lrucnt > dph->dph_lrulim))
		dt_proc_destroy(dtp, P);
}

void
dt_proc_continue(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_t *dpr = dt_proc_lookup(dtp, P, B_FALSE);

	(void) pthread_mutex_lock(&dpr->dpr_lock);

	if (dpr->dpr_stop & DT_PROC_STOP_IDLE) {
		dpr->dpr_stop &= ~DT_PROC_STOP_IDLE;
		(void) pthread_cond_broadcast(&dpr->dpr_cv);
	}

	(void) pthread_mutex_unlock(&dpr->dpr_lock);
}

void
dt_proc_lock(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_t *dpr = dt_proc_lookup(dtp, P, B_FALSE);
	int err = pthread_mutex_lock(&dpr->dpr_lock);
	assert(err == 0); /* check for recursion */
}

void
dt_proc_unlock(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_t *dpr = dt_proc_lookup(dtp, P, B_FALSE);
	int err = pthread_mutex_unlock(&dpr->dpr_lock);
	assert(err == 0); /* check for unheld lock */
}

void
dt_proc_hash_create(dtrace_hdl_t *dtp)
{
	if ((dtp->dt_procs = dt_zalloc(dtp, sizeof (dt_proc_hash_t) +
	    sizeof (dt_proc_t *) * _dtrace_pidbuckets - 1)) != NULL) {

		(void) pthread_mutex_init(&dtp->dt_procs->dph_lock, NULL);
		(void) pthread_cond_init(&dtp->dt_procs->dph_cv, NULL);

		dtp->dt_procs->dph_hashlen = _dtrace_pidbuckets;
		dtp->dt_procs->dph_lrulim = _dtrace_pidlrulim;
	}
}

void
dt_proc_hash_destroy(dtrace_hdl_t *dtp)
{
	dt_proc_hash_t *dph = dtp->dt_procs;
	dt_proc_t *dpr;

	while ((dpr = dt_list_next(&dph->dph_lrulist)) != NULL)
		dt_proc_destroy(dtp, dpr->dpr_proc);

	dtp->dt_procs = NULL;
	dt_free(dtp, dph);
}

struct ps_prochandle *
dtrace_proc_create(dtrace_hdl_t *dtp, const char *file, char *const *argv)
{
	dt_ident_t *idp = dt_idhash_lookup(dtp->dt_macros, "target");
	struct ps_prochandle *P = dt_proc_create(dtp, file, argv);

	if (P != NULL && idp != NULL && idp->di_id == 0)
		idp->di_id = Pstatus(P)->pr_pid; /* $target = created pid */

	return (P);
}

struct ps_prochandle *
dtrace_proc_grab(dtrace_hdl_t *dtp, pid_t pid, int flags)
{
	dt_ident_t *idp = dt_idhash_lookup(dtp->dt_macros, "target");
	struct ps_prochandle *P = dt_proc_grab(dtp, pid, flags, 0);

	if (P != NULL && idp != NULL && idp->di_id == 0)
		idp->di_id = pid; /* $target = grabbed pid */

	return (P);
}

void
dtrace_proc_release(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_release(dtp, P);
}

void
dtrace_proc_continue(dtrace_hdl_t *dtp, struct ps_prochandle *P)
{
	dt_proc_continue(dtp, P);
}