fmd/common/fmd_module.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <signal.h>
#include <dirent.h>
#include <limits.h>
#include <alloca.h>
#include <unistd.h>
#include <stdio.h>

#include <fmd_string.h>
#include <fmd_alloc.h>
#include <fmd_module.h>
#include <fmd_error.h>
#include <fmd_conf.h>
#include <fmd_dispq.h>
#include <fmd_eventq.h>
#include <fmd_timerq.h>
#include <fmd_subr.h>
#include <fmd_thread.h>
#include <fmd_ustat.h>
#include <fmd_case.h>
#include <fmd_protocol.h>
#include <fmd_buf.h>
#include <fmd_ckpt.h>
#include <fmd_xprt.h>

#include <fmd.h>

/*
 * Template for per-module statistics installed by fmd on behalf of each active
 * module.  These are used to initialize the per-module mp->mod_stats below.
 * NOTE: FMD_TYPE_STRING statistics should not be used here.  If they are
 * required in the future, the FMD_ADM_MODDSTAT service routine must change.
 */
static const fmd_modstat_t _fmd_modstat_tmpl = {
{
{ "fmd.dispatched", FMD_TYPE_UINT64, "total events dispatched to module" },
{ "fmd.dequeued", FMD_TYPE_UINT64, "total events dequeued by module" },
{ "fmd.prdequeued", FMD_TYPE_UINT64, "protocol events dequeued by module" },
{ "fmd.dropped", FMD_TYPE_UINT64, "total events dropped on queue overflow" },
{ "fmd.wcnt", FMD_TYPE_UINT32, "count of events waiting on queue" },
{ "fmd.wtime", FMD_TYPE_TIME, "total wait time on queue" },
{ "fmd.wlentime", FMD_TYPE_TIME, "total wait length * time product" },
{ "fmd.wlastupdate", FMD_TYPE_TIME, "hrtime of last wait queue update" },
{ "fmd.dtime", FMD_TYPE_TIME, "total processing time after dequeue" },
{ "fmd.dlastupdate", FMD_TYPE_TIME, "hrtime of last event dequeue completion" },
},
{ "fmd.loadtime", FMD_TYPE_TIME, "hrtime at which module was loaded" },
{ "fmd.snaptime", FMD_TYPE_TIME, "hrtime of last statistics snapshot" },
{ "fmd.accepted", FMD_TYPE_UINT64, "total events accepted by module" },
{ "fmd.debugdrop", FMD_TYPE_UINT64, "dropped debug messages" },
{ "fmd.memtotal", FMD_TYPE_SIZE, "total memory allocated by module" },
{ "fmd.memlimit", FMD_TYPE_SIZE, "limit on total memory allocated" },
{ "fmd.buftotal", FMD_TYPE_SIZE, "total buffer space used by module" },
{ "fmd.buflimit", FMD_TYPE_SIZE, "limit on total buffer space" },
{ "fmd.thrtotal", FMD_TYPE_UINT32, "total number of auxiliary threads" },
{ "fmd.thrlimit", FMD_TYPE_UINT32, "limit on number of auxiliary threads" },
{ "fmd.caseopen", FMD_TYPE_UINT64, "cases currently open by module" },
{ "fmd.casesolved", FMD_TYPE_UINT64, "total cases solved by module" },
{ "fmd.caseclosed", FMD_TYPE_UINT64, "total cases closed by module" },
{ "fmd.ckptsave", FMD_TYPE_BOOL, "save checkpoints for module" },
{ "fmd.ckptrestore", FMD_TYPE_BOOL, "restore checkpoints for module" },
{ "fmd.ckptzero", FMD_TYPE_BOOL, "zeroed checkpoint at startup" },
{ "fmd.ckptcnt", FMD_TYPE_UINT64, "number of checkpoints taken" },
{ "fmd.ckpttime", FMD_TYPE_TIME, "total checkpoint time" },
{ "fmd.xprtopen", FMD_TYPE_UINT32, "total number of open transports" },
{ "fmd.xprtlimit", FMD_TYPE_UINT32, "limit on number of open transports" },
{ "fmd.xprtqlimit", FMD_TYPE_UINT32, "limit on transport event queue length" },
};

static void
fmd_module_start(void *arg)
{
	fmd_module_t *mp = arg;
	fmd_event_t *ep;
	fmd_xprt_t *xp;

	(void) pthread_mutex_lock(&mp->mod_lock);

	if (mp->mod_ops->mop_init(mp) != 0 || mp->mod_error != 0) {
		if (mp->mod_error == 0)
			mp->mod_error = errno ? errno : EFMD_MOD_INIT;
		goto out;
	}

	if (fmd.d_mod_event != NULL)
		fmd_eventq_insert_at_head(mp->mod_queue, fmd.d_mod_event);

	ASSERT(MUTEX_HELD(&mp->mod_lock));
	mp->mod_flags |= FMD_MOD_INIT;

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	/*
	 * If the module opened any transports while executing _fmd_init(),
	 * they are suspended. Now that _fmd_init() is done, wake them up.
	 */
	for (xp = fmd_list_next(&mp->mod_transports);
	    xp != NULL; xp = fmd_list_next(xp))
		fmd_xprt_xresume(xp, FMD_XPRT_ISUSPENDED);

	/*
	 * Wait for events to arrive by checking mod_error and then sleeping in
	 * fmd_eventq_delete().  If a NULL event is returned, the eventq has
	 * been aborted and we continue on to call fini and exit the thread.
	 */
	while ((ep = fmd_eventq_delete(mp->mod_queue)) != NULL) {
		/*
		 * If the module has failed, discard the event without ever
		 * passing it to the module and go back to sleep.
		 */
		if (mp->mod_error != 0) {
			fmd_eventq_done(mp->mod_queue);
			fmd_event_rele(ep);
			continue;
		}

		mp->mod_ops->mop_dispatch(mp, ep);
		fmd_eventq_done(mp->mod_queue);

		/*
		 * Once mop_dispatch() is complete, grab the lock and perform
		 * any event-specific post-processing.  Finally, if necessary,
		 * checkpoint the state of the module after this event.
		 */
		fmd_module_lock(mp);

		if (FMD_EVENT_TYPE(ep) == FMD_EVT_CLOSE)
			fmd_case_delete(FMD_EVENT_DATA(ep));

		fmd_ckpt_save(mp);
		fmd_module_unlock(mp);
		fmd_event_rele(ep);
	}

	if (mp->mod_ops->mop_fini(mp) != 0 && mp->mod_error == 0)
		mp->mod_error = errno ? errno : EFMD_MOD_FINI;

	(void) pthread_mutex_lock(&mp->mod_lock);
	mp->mod_flags |= FMD_MOD_FINI;

out:
	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);
}

fmd_module_t *
fmd_module_create(const char *path, const fmd_modops_t *ops)
{
	fmd_module_t *mp = fmd_zalloc(sizeof (fmd_module_t), FMD_SLEEP);

	char buf[PATH_MAX], *p;
	const char *dir;
	uint32_t limit;
	int err;

	(void) strlcpy(buf, fmd_strbasename(path), sizeof (buf));
	if ((p = strrchr(buf, '.')) != NULL && strcmp(p, ".so") == 0)
		*p = '\0'; /* strip trailing .so from any module name */

	(void) pthread_mutex_init(&mp->mod_lock, NULL);
	(void) pthread_cond_init(&mp->mod_cv, NULL);
	(void) pthread_mutex_init(&mp->mod_stats_lock, NULL);

	mp->mod_name = fmd_strdup(buf, FMD_SLEEP);
	mp->mod_path = fmd_strdup(path, FMD_SLEEP);
	mp->mod_ops = ops;
	mp->mod_ustat = fmd_ustat_create();

	(void) fmd_conf_getprop(fmd.d_conf, "ckpt.dir", &dir);
	(void) snprintf(buf, sizeof (buf),
	    "%s/%s/%s", fmd.d_rootdir, dir, mp->mod_name);

	mp->mod_ckpt = fmd_strdup(buf, FMD_SLEEP);

	(void) fmd_conf_getprop(fmd.d_conf, "client.tmrlim", &limit);
	mp->mod_timerids = fmd_idspace_create(mp->mod_name, 1, limit + 1);
	mp->mod_threads = fmd_idspace_create(mp->mod_name, 0, INT_MAX);

	fmd_buf_hash_create(&mp->mod_bufs);
	fmd_serd_hash_create(&mp->mod_serds);

	(void) pthread_mutex_lock(&fmd.d_mod_lock);
	fmd_list_append(&fmd.d_mod_list, mp);
	(void) pthread_mutex_unlock(&fmd.d_mod_lock);

	/*
	 * Initialize the module statistics that are kept on its behalf by fmd.
	 * These are set up using a template defined at the top of this file.
	 */
	if ((mp->mod_stats = (fmd_modstat_t *)fmd_ustat_insert(mp->mod_ustat,
	    FMD_USTAT_ALLOC, sizeof (_fmd_modstat_tmpl) / sizeof (fmd_stat_t),
	    (fmd_stat_t *)&_fmd_modstat_tmpl, NULL)) == NULL) {
		fmd_error(EFMD_MOD_INIT, "failed to initialize per-mod stats");
		fmd_module_destroy(mp);
		return (NULL);
	}

	(void) fmd_conf_getprop(fmd.d_conf, "client.evqlim", &limit);

	mp->mod_queue = fmd_eventq_create(mp,
	    &mp->mod_stats->ms_evqstat, &mp->mod_stats_lock, limit);

	(void) fmd_conf_getprop(fmd.d_conf, "client.memlim",
	    &mp->mod_stats->ms_memlimit.fmds_value.ui64);

	(void) fmd_conf_getprop(fmd.d_conf, "client.buflim",
	    &mp->mod_stats->ms_buflimit.fmds_value.ui64);

	(void) fmd_conf_getprop(fmd.d_conf, "client.thrlim",
	    &mp->mod_stats->ms_thrlimit.fmds_value.ui32);

	(void) fmd_conf_getprop(fmd.d_conf, "client.xprtlim",
	    &mp->mod_stats->ms_xprtlimit.fmds_value.ui32);

	(void) fmd_conf_getprop(fmd.d_conf, "client.xprtqlim",
	    &mp->mod_stats->ms_xprtqlimit.fmds_value.ui32);

	(void) fmd_conf_getprop(fmd.d_conf, "ckpt.save",
	    &mp->mod_stats->ms_ckpt_save.fmds_value.bool);

	(void) fmd_conf_getprop(fmd.d_conf, "ckpt.restore",
	    &mp->mod_stats->ms_ckpt_restore.fmds_value.bool);

	(void) fmd_conf_getprop(fmd.d_conf, "ckpt.zero",
	    &mp->mod_stats->ms_ckpt_zeroed.fmds_value.bool);

	if (mp->mod_stats->ms_ckpt_zeroed.fmds_value.bool)
		fmd_ckpt_delete(mp); /* blow away any pre-existing checkpoint */

	/*
	 * Place a hold on the module and grab the module lock before creating
	 * the module's thread to ensure that it cannot destroy the module and
	 * that it cannot call ops->mop_init() before we're done setting up.
	 * NOTE: from now on, we must use fmd_module_rele() for error paths.
	 */
	fmd_module_hold(mp);
	(void) pthread_mutex_lock(&mp->mod_lock);
	mp->mod_stats->ms_loadtime.fmds_value.ui64 = gethrtime();
	mp->mod_thread = fmd_thread_create(mp, fmd_module_start, mp);

	if (mp->mod_thread == NULL) {
		fmd_error(EFMD_MOD_THR, "failed to create thread for %s", path);
		(void) pthread_mutex_unlock(&mp->mod_lock);
		fmd_module_rele(mp);
		return (NULL);
	}

	/*
	 * At this point our module structure is nearly finished and its thread
	 * is starting execution in fmd_module_start() above, which will begin
	 * by blocking for mod_lock.  We now drop mod_lock and wait for either
	 * FMD_MOD_INIT or mod_error to be set before proceeding.
	 */
	while (!(mp->mod_flags & FMD_MOD_INIT) && mp->mod_error == 0)
		(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);

	/*
	 * If the module has failed to initialize, copy its errno to the errno
	 * of the caller, wait for it to unload, and then destroy it.
	 */
	if (!(mp->mod_flags & FMD_MOD_INIT)) {
		err = mp->mod_error;
		(void) pthread_mutex_unlock(&mp->mod_lock);

		if (err == EFMD_CKPT_INVAL)
			fmd_ckpt_rename(mp); /* move aside bad checkpoint */

		/*
		 * If we're in the background, keep quiet about failure to
		 * load because a handle wasn't registered: this is a module's
		 * way of telling us it didn't want to be loaded for some
		 * reason related to system configuration.  If we're in the
		 * foreground we log this too in order to inform developers.
		 */
		if (fmd.d_fg || err != EFMD_HDL_INIT) {
			fmd_error(EFMD_MOD_INIT, "failed to load %s: %s\n",
			    path, fmd_strerror(err));
		}

		fmd_module_unload(mp);
		fmd_module_rele(mp);

		(void) fmd_set_errno(err);
		return (NULL);
	}

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	fmd_dprintf(FMD_DBG_MOD, "loaded module %s\n", mp->mod_name);
	return (mp);
}

static void
fmd_module_untimeout(fmd_idspace_t *ids, id_t id, fmd_module_t *mp)
{
	void *arg = fmd_timerq_remove(fmd.d_timers, ids, id);

	/*
	 * The root module calls fmd_timerq_install() directly and must take
	 * responsibility for any cleanup of timer arguments that is required.
	 * All other modules use fmd_modtimer_t's as the arg data; free them.
	 */
	if (arg != NULL && mp != fmd.d_rmod)
		fmd_free(arg, sizeof (fmd_modtimer_t));
}

void
fmd_module_unload(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	if (mp->mod_flags & FMD_MOD_QUIT) {
		(void) pthread_mutex_unlock(&mp->mod_lock);
		return; /* module is already unloading */
	}

	ASSERT(mp->mod_thread != NULL);
	mp->mod_flags |= FMD_MOD_QUIT;

	if (mp->mod_queue != NULL)
		fmd_eventq_abort(mp->mod_queue);

	/*
	 * Wait for the module's thread to stop processing events and call
	 * _fmd_fini() and exit.  We do this by waiting for FMD_MOD_FINI to be
	 * set if INIT was set, and then attempting to join with the thread.
	 */
	while ((mp->mod_flags & (FMD_MOD_INIT | FMD_MOD_FINI)) == FMD_MOD_INIT)
		(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	fmd_thread_destroy(mp->mod_thread, FMD_THREAD_JOIN);
	mp->mod_thread = NULL;

	/*
	 * Once the module is no longer active, clean up any data structures
	 * that are only required when the module is loaded.
	 */
	fmd_module_lock(mp);

	if (mp->mod_timerids != NULL) {
		fmd_idspace_apply(mp->mod_timerids,
		    (void (*)())fmd_module_untimeout, mp);

		fmd_idspace_destroy(mp->mod_timerids);
		mp->mod_timerids = NULL;
	}

	if (mp->mod_threads != NULL) {
		fmd_idspace_destroy(mp->mod_threads);
		mp->mod_threads = NULL;
	}

	(void) fmd_buf_hash_destroy(&mp->mod_bufs);
	fmd_serd_hash_destroy(&mp->mod_serds);

	fmd_module_unlock(mp);
	fmd_dprintf(FMD_DBG_MOD, "unloaded module %s\n", mp->mod_name);
}

void
fmd_module_destroy(fmd_module_t *mp)
{
	fmd_conf_formal_t *cfp = mp->mod_argv;
	int i;

	ASSERT(MUTEX_HELD(&mp->mod_lock));

	if (mp->mod_thread != NULL) {
		(void) pthread_mutex_unlock(&mp->mod_lock);
		fmd_module_unload(mp);
		(void) pthread_mutex_lock(&mp->mod_lock);
	}

	ASSERT(mp->mod_thread == NULL);
	ASSERT(mp->mod_refs == 0);

	/*
	 * Once the module's thread is dead, we can safely remove the module
	 * from global visibility and by removing it from d_mod_list.  Any
	 * modhash pointers are already gone by virtue of mod_refs being zero.
	 */
	(void) pthread_mutex_lock(&fmd.d_mod_lock);
	fmd_list_delete(&fmd.d_mod_list, mp);
	(void) pthread_mutex_unlock(&fmd.d_mod_lock);

	/*
	 * Once the module is no longer processing events and no longer visible
	 * through any program data structures, we can free all of its content.
	 */
	if (mp->mod_queue != NULL) {
		fmd_eventq_destroy(mp->mod_queue);
		mp->mod_queue = NULL;
	}

	if (mp->mod_ustat != NULL) {
		(void) pthread_mutex_lock(&mp->mod_stats_lock);
		fmd_ustat_destroy(mp->mod_ustat);
		mp->mod_ustat = NULL;
		mp->mod_stats = NULL;
		(void) pthread_mutex_unlock(&mp->mod_stats_lock);
	}

	for (i = 0; i < mp->mod_dictc; i++)
		fm_dc_closedict(mp->mod_dictv[i]);

	fmd_free(mp->mod_dictv, sizeof (struct fm_dc_handle *) * mp->mod_dictc);

	if (mp->mod_conf != NULL)
		fmd_conf_close(mp->mod_conf);

	for (i = 0; i < mp->mod_argc; i++, cfp++) {
		fmd_strfree((char *)cfp->cf_name);
		fmd_strfree((char *)cfp->cf_default);
	}

	fmd_free(mp->mod_argv, sizeof (fmd_conf_formal_t) * mp->mod_argc);

	fmd_strfree(mp->mod_name);
	fmd_strfree(mp->mod_path);
	fmd_strfree(mp->mod_ckpt);
	nvlist_free(mp->mod_fmri);

	fmd_free(mp, sizeof (fmd_module_t));
}

/*
 * fmd_module_error() is called after the stack is unwound from a call to
 * fmd_module_abort() to indicate that the module has failed.  The mod_error
 * field is used to hold the error code of the first fatal error to the module.
 * An EFMD_MOD_FAIL event is then created and sent to fmd-self-diagnosis.
 */
static void
fmd_module_error(fmd_module_t *mp, int err)
{
	fmd_event_t *e;
	nvlist_t *nvl;
	char *class;

	ASSERT(MUTEX_HELD(&mp->mod_lock));
	ASSERT(err != 0);

	TRACE((FMD_DBG_MOD, "module aborted: err=%d", err));

	if (mp->mod_error == 0)
		mp->mod_error = err;

	if (mp == fmd.d_self)
		return; /* do not post event if fmd.d_self itself fails */

	/*
	 * Send an error indicating the module has now failed to fmd.d_self.
	 * Since the error causing the failure has already been logged by
	 * fmd_api_xerror(), we do not need to bother logging this event.
	 * It only exists for the purpose of notifying fmd.d_self that it can
	 * close the case associated with this module because mod_error is set.
	 */
	nvl = fmd_protocol_moderror(mp, EFMD_MOD_FAIL, fmd_strerror(err));
	(void) nvlist_lookup_string(nvl, FM_CLASS, &class);
	e = fmd_event_create(FMD_EVT_PROTOCOL, FMD_HRT_NOW, nvl, class);
	fmd_dispq_dispatch(fmd.d_disp, e, class);
}

void
fmd_module_dispatch(fmd_module_t *mp, fmd_event_t *e)
{
	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
	fmd_event_impl_t *ep = (fmd_event_impl_t *)e;
	fmd_hdl_t *hdl = (fmd_hdl_t *)mp;
	fmd_modtimer_t *t;
	volatile int err;

	/*
	 * Before calling the appropriate module callback, enter the module as
	 * if by fmd_module_enter() and establish mod_jmpbuf for any aborts.
	 */
	(void) pthread_mutex_lock(&mp->mod_lock);

	ASSERT(!(mp->mod_flags & FMD_MOD_BUSY));
	mp->mod_flags |= FMD_MOD_BUSY;

	if ((err = setjmp(mp->mod_jmpbuf)) != 0) {
		(void) pthread_mutex_lock(&mp->mod_lock);
		fmd_module_error(mp, err);
	}

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	/*
	 * If it's the first time through fmd_module_dispatch(), call the
	 * appropriate module callback based on the event type.  If the call
	 * triggers an fmd_module_abort(), we'll return to setjmp() above with
	 * err set to a non-zero value and then bypass this before exiting.
	 */
	if (err == 0) {
		switch (ep->ev_type) {
		case FMD_EVT_PROTOCOL:
			ops->fmdo_recv(hdl, e, ep->ev_nvl, ep->ev_data);
			break;
		case FMD_EVT_TIMEOUT:
			t = ep->ev_data;
			ASSERT(t->mt_mod == mp);
			ops->fmdo_timeout(hdl, t->mt_id, t->mt_arg);
			break;
		case FMD_EVT_CLOSE:
			ops->fmdo_close(hdl, ep->ev_data);
			break;
		case FMD_EVT_STATS:
			ops->fmdo_stats(hdl);
			fmd_modstat_publish(mp);
			break;
		case FMD_EVT_GC:
			ops->fmdo_gc(hdl);
			break;
		case FMD_EVT_PUBLISH:
			fmd_case_publish(ep->ev_data, FMD_CASE_CURRENT);
			break;
		}
	}

	fmd_module_exit(mp);
}

int
fmd_module_transport(fmd_module_t *mp, fmd_xprt_t *xp, fmd_event_t *e)
{
	fmd_event_impl_t *ep = (fmd_event_impl_t *)e;
	fmd_hdl_t *hdl = (fmd_hdl_t *)mp;

	ASSERT(ep->ev_type == FMD_EVT_PROTOCOL);
	return (mp->mod_info->fmdi_ops->fmdo_send(hdl, xp, e, ep->ev_nvl));
}

void
fmd_module_timeout(fmd_modtimer_t *t, id_t id, hrtime_t hrt)
{
	fmd_event_t *e;

	t->mt_id = id; /* save id in case we need to delete from eventq */
	e = fmd_event_create(FMD_EVT_TIMEOUT, hrt, NULL, t);
	fmd_eventq_insert_at_time(t->mt_mod->mod_queue, e);
}

/*
 * Garbage collection is initiated by a timer callback once per day or at the
 * request of fmadm.  Purge old SERD entries and send the module a GC event.
 */
void
fmd_module_gc(fmd_module_t *mp)
{
	fmd_hdl_info_t *info;
	fmd_event_t *e;

	if (mp->mod_error != 0)
		return; /* do not do anything if the module has failed */

	fmd_module_lock(mp);

	if ((info = mp->mod_info) != NULL) {
		fmd_serd_hash_apply(&mp->mod_serds,
		    (fmd_serd_eng_f *)fmd_serd_eng_gc, NULL);
	}

	fmd_module_unlock(mp);

	if (info != NULL) {
		e = fmd_event_create(FMD_EVT_GC, FMD_HRT_NOW, NULL, NULL);
		fmd_eventq_insert_at_head(mp->mod_queue, e);
	}
}

void
fmd_module_trygc(fmd_module_t *mp)
{
	if (fmd_module_trylock(mp)) {
		fmd_serd_hash_apply(&mp->mod_serds,
		    (fmd_serd_eng_f *)fmd_serd_eng_gc, NULL);
		fmd_module_unlock(mp);
	}
}

int
fmd_module_contains(fmd_module_t *mp, fmd_event_t *ep)
{
	fmd_case_t *cp;
	int rv = 0;

	fmd_module_lock(mp);

	for (cp = fmd_list_next(&mp->mod_cases);
	    cp != NULL; cp = fmd_list_next(cp)) {
		if ((rv = fmd_case_contains(cp, ep)) != 0)
			break;
	}

	if (rv == 0)
		rv = fmd_serd_hash_contains(&mp->mod_serds, ep);

	fmd_module_unlock(mp);
	return (rv);
}

void
fmd_module_setdirty(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);
	mp->mod_flags |= FMD_MOD_MDIRTY;
	(void) pthread_mutex_unlock(&mp->mod_lock);
}

void
fmd_module_setcdirty(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);
	mp->mod_flags |= FMD_MOD_CDIRTY;
	(void) pthread_mutex_unlock(&mp->mod_lock);
}

void
fmd_module_clrdirty(fmd_module_t *mp)
{
	fmd_case_t *cp;

	fmd_module_lock(mp);

	if (mp->mod_flags & FMD_MOD_CDIRTY) {
		for (cp = fmd_list_next(&mp->mod_cases);
		    cp != NULL; cp = fmd_list_next(cp))
			fmd_case_clrdirty(cp);
	}

	if (mp->mod_flags & FMD_MOD_MDIRTY) {
		fmd_serd_hash_apply(&mp->mod_serds,
		    (fmd_serd_eng_f *)fmd_serd_eng_clrdirty, NULL);
		fmd_buf_hash_commit(&mp->mod_bufs);
	}

	(void) pthread_mutex_lock(&mp->mod_lock);
	mp->mod_flags &= ~(FMD_MOD_MDIRTY | FMD_MOD_CDIRTY);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	fmd_module_unlock(mp);
}

void
fmd_module_commit(fmd_module_t *mp)
{
	fmd_case_t *cp;

	ASSERT(fmd_module_locked(mp));

	if (mp->mod_flags & FMD_MOD_CDIRTY) {
		for (cp = fmd_list_next(&mp->mod_cases);
		    cp != NULL; cp = fmd_list_next(cp))
			fmd_case_commit(cp);
	}

	if (mp->mod_flags & FMD_MOD_MDIRTY) {
		fmd_serd_hash_apply(&mp->mod_serds,
		    (fmd_serd_eng_f *)fmd_serd_eng_commit, NULL);
		fmd_buf_hash_commit(&mp->mod_bufs);
	}

	(void) pthread_mutex_lock(&mp->mod_lock);
	mp->mod_flags &= ~(FMD_MOD_MDIRTY | FMD_MOD_CDIRTY);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	mp->mod_gen++;
}

void
fmd_module_lock(fmd_module_t *mp)
{
	pthread_t self = pthread_self();

	(void) pthread_mutex_lock(&mp->mod_lock);

	while (mp->mod_flags & FMD_MOD_LOCK) {
		if (mp->mod_owner != self)
			(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);
		else
			fmd_panic("recursive module lock of %p\n", (void *)mp);
	}

	mp->mod_owner = self;
	mp->mod_flags |= FMD_MOD_LOCK;

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);
}

void
fmd_module_unlock(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	ASSERT(mp->mod_owner == pthread_self());
	ASSERT(mp->mod_flags & FMD_MOD_LOCK);

	mp->mod_owner = 0;
	mp->mod_flags &= ~FMD_MOD_LOCK;

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);
}

int
fmd_module_trylock(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	if (mp->mod_flags & FMD_MOD_LOCK) {
		(void) pthread_mutex_unlock(&mp->mod_lock);
		return (0);
	}

	mp->mod_owner = pthread_self();
	mp->mod_flags |= FMD_MOD_LOCK;

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	return (1);
}

int
fmd_module_locked(fmd_module_t *mp)
{
	return ((mp->mod_flags & FMD_MOD_LOCK) &&
	    mp->mod_owner == pthread_self());
}

int
fmd_module_enter(fmd_module_t *mp, void (*func)(fmd_hdl_t *))
{
	volatile int err;

	(void) pthread_mutex_lock(&mp->mod_lock);

	ASSERT(!(mp->mod_flags & FMD_MOD_BUSY));
	mp->mod_flags |= FMD_MOD_BUSY;

	if ((err = setjmp(mp->mod_jmpbuf)) != 0) {
		(void) pthread_mutex_lock(&mp->mod_lock);
		fmd_module_error(mp, err);
	}

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	/*
	 * If it's the first time through fmd_module_enter(), call the provided
	 * function on the module.  If no fmd_module_abort() results, we will
	 * fall through and return zero.  Otherwise we'll longjmp with an err,
	 * return to the setjmp() above, and return the error to our caller.
	 */
	if (err == 0 && func != NULL)
		(*func)((fmd_hdl_t *)mp);

	return (err);
}

void
fmd_module_exit(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	ASSERT(mp->mod_flags & FMD_MOD_BUSY);
	mp->mod_flags &= ~FMD_MOD_BUSY;

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);
}

/*
 * If the client.error policy has been set by a developer, stop or dump core
 * based on the policy; if we stop and are resumed we'll continue and execute
 * the default behavior to discard events in fmd_module_start().  If the caller
 * is the primary module thread, we reach this state by longjmp'ing back to
 * fmd_module_enter(), above.  If the caller is an auxiliary thread, we cancel
 * ourself and arrange for the primary thread to call fmd_module_abort().
 */
void
fmd_module_abort(fmd_module_t *mp, int err)
{
	uint_t policy = FMD_CERROR_UNLOAD;
	pthread_t tid = pthread_self();

	(void) fmd_conf_getprop(fmd.d_conf, "client.error", &policy);

	if (policy == FMD_CERROR_STOP) {
		fmd_error(err, "stopping after %s in client %s (%p)\n",
		    fmd_errclass(err), mp->mod_name, (void *)mp);
		(void) raise(SIGSTOP);
	} else if (policy == FMD_CERROR_ABORT) {
		fmd_panic("aborting due to %s in client %s (%p)\n",
		    fmd_errclass(err), mp->mod_name, (void *)mp);
	}

	/*
	 * If the caller is an auxiliary thread, cancel the current thread.  We
	 * prefer to cancel because it affords developers the option of using
	 * the pthread_cleanup* APIs.  If cancellations have been disabled,
	 * fall through to forcing the current thread to exit.  In either case
	 * we update mod_error (if zero) to enter the failed state.  Once that
	 * is set, further events received by the module will be discarded.
	 *
	 * We also set the FMD_MOD_FAIL bit, indicating an unrecoverable error.
	 * When an auxiliary thread fails, the module is left in a delicate
	 * state where it is likely not able to continue execution (even to
	 * execute its _fmd_fini() routine) because our caller may hold locks
	 * that are private to the module and can no longer be released.  The
	 * FMD_MOD_FAIL bit forces fmd_api_module_lock() to abort if any other
	 * module threads reach an API call, in an attempt to get them to exit.
	 */
	if (tid != mp->mod_thread->thr_tid) {
		(void) pthread_mutex_lock(&mp->mod_lock);

		if (mp->mod_error == 0)
			mp->mod_error = err;

		mp->mod_flags |= FMD_MOD_FAIL;
		(void) pthread_mutex_unlock(&mp->mod_lock);

		(void) pthread_cancel(tid);
		pthread_exit(NULL);
	}

	ASSERT(mp->mod_flags & FMD_MOD_BUSY);
	longjmp(mp->mod_jmpbuf, err);
}

void
fmd_module_hold(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	TRACE((FMD_DBG_MOD, "hold %p (%s/%u)\n",
	    (void *)mp, mp->mod_name, mp->mod_refs));

	mp->mod_refs++;
	ASSERT(mp->mod_refs != 0);

	(void) pthread_mutex_unlock(&mp->mod_lock);
}

void
fmd_module_rele(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	TRACE((FMD_DBG_MOD, "rele %p (%s/%u)\n",
	    (void *)mp, mp->mod_name, mp->mod_refs));

	ASSERT(mp->mod_refs != 0);

	if (--mp->mod_refs == 0)
		fmd_module_destroy(mp);
	else
		(void) pthread_mutex_unlock(&mp->mod_lock);
}

/*
 * Wrapper around libdiagcode's fm_dc_opendict() to load module dictionaries.
 * If the dictionary open is successful, the new dictionary is added to the
 * mod_dictv[] array and mod_codelen is updated with the new maximum length.
 */
int
fmd_module_dc_opendict(fmd_module_t *mp, const char *dict)
{
	struct fm_dc_handle *dcp, **dcv;
	char *dictdir, *dictnam, *p;
	size_t len;

	ASSERT(fmd_module_locked(mp));

	dictnam = alloca(strlen(dict) + 1);
	(void) strcpy(dictnam, fmd_strbasename(dict));

	if ((p = strrchr(dictnam, '.')) != NULL &&
	    strcmp(p, ".dict") == 0)
		*p = '\0'; /* eliminate any trailing .dict suffix */

	/*
	 * If 'dict' is an absolute path, dictdir = $rootdir/`dirname dict`
	 * If 'dict' is not an absolute path, dictdir = $dictdir/`dirname dict`
	 */
	if (dict[0] == '/') {
		len = strlen(fmd.d_rootdir) + strlen(dict) + 1;
		dictdir = alloca(len);
		(void) snprintf(dictdir, len, "%s%s", fmd.d_rootdir, dict);
		(void) fmd_strdirname(dictdir);
	} else {
		(void) fmd_conf_getprop(fmd.d_conf, "dictdir", &p);
		len = strlen(fmd.d_rootdir) + strlen(p) + strlen(dict) + 3;
		dictdir = alloca(len);
		(void) snprintf(dictdir, len,
		    "%s/%s/%s", fmd.d_rootdir, p, dict);
		(void) fmd_strdirname(dictdir);
	}

	fmd_dprintf(FMD_DBG_MOD, "module %s opening %s -> %s/%s.dict\n",
	    mp->mod_name, dict, dictdir, dictnam);

	if ((dcp = fm_dc_opendict(FM_DC_VERSION, dictdir, dictnam)) == NULL)
		return (-1); /* errno is set for us */

	dcv = fmd_alloc(sizeof (dcp) * (mp->mod_dictc + 1), FMD_SLEEP);
	bcopy(mp->mod_dictv, dcv, sizeof (dcp) * mp->mod_dictc);
	fmd_free(mp->mod_dictv, sizeof (dcp) * mp->mod_dictc);
	mp->mod_dictv = dcv;
	mp->mod_dictv[mp->mod_dictc++] = dcp;

	len = fm_dc_codelen(dcp);
	mp->mod_codelen = MAX(mp->mod_codelen, len);

	return (0);
}

/*
 * Wrapper around libdiagcode's fm_dc_key2code() that examines all the module's
 * dictionaries.  We adhere to the libdiagcode return values and semantics.
 */
int
fmd_module_dc_key2code(fmd_module_t *mp,
    char *const keys[], char *code, size_t codelen)
{
	int i, err;

	for (i = 0; i < mp->mod_dictc; i++) {
		if ((err = fm_dc_key2code(mp->mod_dictv[i], (const char **)keys,
		    code, codelen)) == 0 || errno != ENOMSG)
			return (err);
	}

	return (fmd_set_errno(ENOMSG));
}

fmd_modhash_t *
fmd_modhash_create(void)
{
	fmd_modhash_t *mhp = fmd_alloc(sizeof (fmd_modhash_t), FMD_SLEEP);

	(void) pthread_rwlock_init(&mhp->mh_lock, NULL);
	mhp->mh_hashlen = fmd.d_str_buckets;
	mhp->mh_hash = fmd_zalloc(sizeof (void *) * mhp->mh_hashlen, FMD_SLEEP);
	mhp->mh_nelems = 0;

	return (mhp);
}

void
fmd_modhash_destroy(fmd_modhash_t *mhp)
{
	fmd_module_t *mp, *nmp;
	uint_t i;

	for (i = 0; i < mhp->mh_hashlen; i++) {
		for (mp = mhp->mh_hash[i]; mp != NULL; mp = nmp) {
			nmp = mp->mod_next;
			mp->mod_next = NULL;
			fmd_module_rele(mp);
		}
	}

	fmd_free(mhp->mh_hash, sizeof (void *) * mhp->mh_hashlen);
	(void) pthread_rwlock_destroy(&mhp->mh_lock);
	fmd_free(mhp, sizeof (fmd_modhash_t));
}

static void
fmd_modhash_loaddir(fmd_modhash_t *mhp, const char *dir,
    const fmd_modops_t *ops, const char *suffix)
{
	char path[PATH_MAX];
	struct dirent *dp;
	const char *p;
	DIR *dirp;

	if ((dirp = opendir(dir)) == NULL)
		return; /* failed to open directory; just skip it */

	while ((dp = readdir(dirp)) != NULL) {
		if (dp->d_name[0] == '.')
			continue; /* skip "." and ".." */

		p = strrchr(dp->d_name, '.');

		if (p != NULL && strcmp(p, ".conf") == 0)
			continue; /* skip .conf files */

		if (suffix != NULL && (p == NULL || strcmp(p, suffix) != 0))
			continue; /* skip files with the wrong suffix */

		(void) snprintf(path, sizeof (path), "%s/%s", dir, dp->d_name);
		(void) fmd_modhash_load(mhp, path, ops);
	}

	(void) closedir(dirp);
}

void
fmd_modhash_loadall(fmd_modhash_t *mhp, const fmd_conf_path_t *pap,
    const fmd_modops_t *ops, const char *suffix)
{
	int i;

	for (i = 0; i < pap->cpa_argc; i++)
		fmd_modhash_loaddir(mhp, pap->cpa_argv[i], ops, suffix);
}

void
fmd_modhash_apply(fmd_modhash_t *mhp, void (*func)(fmd_module_t *))
{
	fmd_module_t *mp, *np;
	uint_t i;

	(void) pthread_rwlock_rdlock(&mhp->mh_lock);

	for (i = 0; i < mhp->mh_hashlen; i++) {
		for (mp = mhp->mh_hash[i]; mp != NULL; mp = np) {
			np = mp->mod_next;
			func(mp);
		}
	}

	(void) pthread_rwlock_unlock(&mhp->mh_lock);
}

void
fmd_modhash_tryapply(fmd_modhash_t *mhp, void (*func)(fmd_module_t *))
{
	fmd_module_t *mp, *np;
	uint_t i;

	if (mhp == NULL || pthread_rwlock_tryrdlock(&mhp->mh_lock) != 0)
		return; /* not initialized or couldn't grab lock */

	for (i = 0; i < mhp->mh_hashlen; i++) {
		for (mp = mhp->mh_hash[i]; mp != NULL; mp = np) {
			np = mp->mod_next;
			func(mp);
		}
	}

	(void) pthread_rwlock_unlock(&mhp->mh_lock);
}

void
fmd_modhash_dispatch(fmd_modhash_t *mhp, fmd_event_t *ep)
{
	fmd_module_t *mp;
	uint_t i;

	fmd_event_hold(ep);
	(void) pthread_rwlock_rdlock(&mhp->mh_lock);

	for (i = 0; i < mhp->mh_hashlen; i++) {
		for (mp = mhp->mh_hash[i]; mp != NULL; mp = mp->mod_next) {
			/*
			 * If FMD_MOD_INIT is set but MOD_FINI, MOD_QUIT, and
			 * mod_error are all zero, then the module is active:
			 * enqueue the event in the corresponding event queue.
			 */
			(void) pthread_mutex_lock(&mp->mod_lock);

			if ((mp->mod_flags & (FMD_MOD_INIT | FMD_MOD_FINI |
			    FMD_MOD_QUIT)) == FMD_MOD_INIT && !mp->mod_error)
				fmd_eventq_insert_at_time(mp->mod_queue, ep);

			(void) pthread_mutex_unlock(&mp->mod_lock);
		}
	}

	(void) pthread_rwlock_unlock(&mhp->mh_lock);
	fmd_event_rele(ep);
}

fmd_module_t *
fmd_modhash_lookup(fmd_modhash_t *mhp, const char *name)
{
	fmd_module_t *mp;
	uint_t h;

	(void) pthread_rwlock_rdlock(&mhp->mh_lock);
	h = fmd_strhash(name) % mhp->mh_hashlen;

	for (mp = mhp->mh_hash[h]; mp != NULL; mp = mp->mod_next) {
		if (strcmp(name, mp->mod_name) == 0)
			break;
	}

	if (mp != NULL)
		fmd_module_hold(mp);
	else
		(void) fmd_set_errno(EFMD_MOD_NOMOD);

	(void) pthread_rwlock_unlock(&mhp->mh_lock);
	return (mp);
}

fmd_module_t *
fmd_modhash_load(fmd_modhash_t *mhp, const char *path, const fmd_modops_t *ops)
{
	char name[PATH_MAX], *p;
	fmd_module_t *mp;
	int tries = 0;
	uint_t h;

	(void) strlcpy(name, fmd_strbasename(path), sizeof (name));
	if ((p = strrchr(name, '.')) != NULL && strcmp(p, ".so") == 0)
		*p = '\0'; /* strip trailing .so from any module name */

	(void) pthread_rwlock_wrlock(&mhp->mh_lock);
	h = fmd_strhash(name) % mhp->mh_hashlen;

	/*
	 * First check to see if a module is already present in the hash table
	 * for this name.  If so, the module is already loaded: skip it.
	 */
	for (mp = mhp->mh_hash[h]; mp != NULL; mp = mp->mod_next) {
		if (strcmp(name, mp->mod_name) == 0)
			break;
	}

	if (mp != NULL) {
		(void) pthread_rwlock_unlock(&mhp->mh_lock);
		(void) fmd_set_errno(EFMD_MOD_LOADED);
		return (NULL);
	}

	/*
	 * fmd_module_create() will return a held (as if by fmd_module_hold())
	 * module.  We leave this hold in place to correspond to the hash-in.
	 */
	while ((mp = fmd_module_create(path, ops)) == NULL) {
		if (tries++ != 0 || errno != EFMD_CKPT_INVAL) {
			(void) pthread_rwlock_unlock(&mhp->mh_lock);
			return (NULL); /* errno is set for us */
		}
	}

	mp->mod_hash = mhp;
	mp->mod_next = mhp->mh_hash[h];

	mhp->mh_hash[h] = mp;
	mhp->mh_nelems++;

	(void) pthread_rwlock_unlock(&mhp->mh_lock);
	return (mp);
}

int
fmd_modhash_unload(fmd_modhash_t *mhp, const char *name)
{
	fmd_module_t *mp, **pp;
	uint_t h;

	(void) pthread_rwlock_wrlock(&mhp->mh_lock);
	h = fmd_strhash(name) % mhp->mh_hashlen;
	pp = &mhp->mh_hash[h];

	for (mp = *pp; mp != NULL; mp = mp->mod_next) {
		if (strcmp(name, mp->mod_name) == 0)
			break;
		else
			pp = &mp->mod_next;
	}

	if (mp == NULL) {
		(void) pthread_rwlock_unlock(&mhp->mh_lock);
		return (fmd_set_errno(EFMD_MOD_NOMOD));
	}

	*pp = mp->mod_next;
	mp->mod_next = NULL;

	ASSERT(mhp->mh_nelems != 0);
	mhp->mh_nelems--;

	(void) pthread_rwlock_unlock(&mhp->mh_lock);

	fmd_module_unload(mp);
	fmd_module_rele(mp);

	return (0);
}

void
fmd_modstat_publish(fmd_module_t *mp)
{
	(void) pthread_mutex_lock(&mp->mod_lock);

	ASSERT(mp->mod_flags & FMD_MOD_STSUB);
	mp->mod_flags |= FMD_MOD_STPUB;
	(void) pthread_cond_broadcast(&mp->mod_cv);

	while (mp->mod_flags & FMD_MOD_STPUB)
		(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);

	(void) pthread_mutex_unlock(&mp->mod_lock);
}

int
fmd_modstat_snapshot(fmd_module_t *mp, fmd_ustat_snap_t *uss)
{
	fmd_event_t *e;
	int err;

	/*
	 * Grab the module lock and wait for the STSUB bit to be clear.  Then
	 * set it to indicate we are a subscriber and everyone else must wait.
	 */
	(void) pthread_mutex_lock(&mp->mod_lock);

	while (mp->mod_error == 0 && (mp->mod_flags & FMD_MOD_STSUB))
		(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);

	if (mp->mod_error != 0) {
		(void) pthread_mutex_unlock(&mp->mod_lock);
		return (fmd_set_errno(EFMD_HDL_ABORT));
	}

	mp->mod_flags |= FMD_MOD_STSUB;
	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	/*
	 * Create a stats pseudo-event and dispatch it to the module, forcing
	 * it to next execute its custom snapshot routine (or the empty one).
	 */
	e = fmd_event_create(FMD_EVT_STATS, FMD_HRT_NOW, NULL, NULL);
	fmd_eventq_insert_at_head(mp->mod_queue, e);

	/*
	 * Grab the module lock and then wait on mod_cv for STPUB to be set,
	 * indicating the snapshot routine is completed and the module is idle.
	 */
	(void) pthread_mutex_lock(&mp->mod_lock);

	while (mp->mod_error == 0 && !(mp->mod_flags & FMD_MOD_STPUB))
		(void) pthread_cond_wait(&mp->mod_cv, &mp->mod_lock);

	if (mp->mod_error != 0) {
		(void) pthread_mutex_unlock(&mp->mod_lock);
		return (fmd_set_errno(EFMD_HDL_ABORT));
	}

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	/*
	 * Update ms_snaptime and take the actual snapshot of the various
	 * statistics while the module is quiescent and waiting for us.
	 */
	(void) pthread_mutex_lock(&mp->mod_stats_lock);

	if (mp->mod_stats != NULL) {
		mp->mod_stats->ms_snaptime.fmds_value.ui64 = gethrtime();
		err = fmd_ustat_snapshot(mp->mod_ustat, uss);
	} else
		err = fmd_set_errno(EFMD_HDL_ABORT);

	(void) pthread_mutex_unlock(&mp->mod_stats_lock);

	/*
	 * With the snapshot complete, grab the module lock and clear both
	 * STSUB and STPUB, permitting everyone to wake up and continue.
	 */
	(void) pthread_mutex_lock(&mp->mod_lock);

	ASSERT(mp->mod_flags & FMD_MOD_STSUB);
	ASSERT(mp->mod_flags & FMD_MOD_STPUB);
	mp->mod_flags &= ~(FMD_MOD_STSUB | FMD_MOD_STPUB);

	(void) pthread_cond_broadcast(&mp->mod_cv);
	(void) pthread_mutex_unlock(&mp->mod_lock);

	return (err);
}