daemons/syseventd/syseventd.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 *	syseventd - The system event daemon
 *
 *		This daemon dispatches event buffers received from the
 *		kernel to all interested SLM clients.  SLMs in turn
 *		deliver the buffers to their particular application
 *		clients.
 */
#include <stdio.h>
#include <sys/types.h>
#include <dirent.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdlib.h>
#include <dlfcn.h>
#include <door.h>
#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#include <strings.h>
#include <unistd.h>
#include <synch.h>
#include <syslog.h>
#include <thread.h>
#include <libsysevent.h>
#include <limits.h>
#include <locale.h>
#include <sys/sysevent.h>
#include <sys/sysevent_impl.h>
#include <sys/modctl.h>
#include <sys/stat.h>
#include <sys/systeminfo.h>
#include <sys/wait.h>

#include "sysevent_signal.h"
#include "syseventd.h"
#include "message.h"

extern int insert_client(void *client, int client_type, int retry_limit);
extern void delete_client(int id);
extern void initialize_client_tbl(void);

extern struct sysevent_client *sysevent_client_tbl[];
extern mutex_t client_tbl_lock;

#define	DEBUG_LEVEL_FORK	9	/* will run in background at all */
					/* levels less than DEBUG_LEVEL_FORK */

int debug_level = 0;
char *root_dir = "";			/* Relative root for lock and door */

/* Maximum number of outstanding events dispatched */
#define	SE_EVENT_DISPATCH_CNT	100

static int upcall_door;			/* Kernel event door */
static int door_upcall_retval;		/* Kernel event posting return value */
static int fini_pending = 0;		/* fini pending flag */
static int deliver_buf = 0;		/* Current event buffer from kernel */
static int dispatch_buf = 0;		/* Current event buffer dispatched */
static sysevent_t **eventbuf; 		/* Global array of event buffers */
static struct ev_completion *event_compq;	/* Event completion queue */
static mutex_t ev_comp_lock;		/* Event completion queue lock */
static mutex_t err_mutex;		/* error logging lock */
static mutex_t door_lock;		/* sync door return access */
static rwlock_t mod_unload_lock;		/* sync module unloading */

/* declarations and definitions for avoiding multiple daemons running */
#define	DAEMON_LOCK_FILE "/etc/sysevent/syseventd_lock"
char local_lock_file[PATH_MAX + 1];
static int hold_daemon_lock;
static int daemon_lock_fd;

/*
 * sema_eventbuf - guards against the global buffer eventbuf
 *	being written to before it has been dispatched to clients
 *
 * sema_dispatch - synchronizes between the kernel uploading thread
 *	(producer) and the userland dispatch_message thread (consumer).
 *
 * sema_resource - throttles outstanding event consumption.
 *
 * event_comp_cv - synchronizes threads waiting for the event completion queue
 *			to empty or become active.
 */
static sema_t sema_eventbuf, sema_dispatch, sema_resource;
static cond_t event_comp_cv;

/* Self-tuning concurrency level */
#define	MIN_CONCURRENCY_LEVEL	4
static int concurrency_level = MIN_CONCURRENCY_LEVEL;


/* SLM defines */
#define	MODULE_SUFFIX	".so"
#define	EVENT_FINI	"slm_fini"
#define	EVENT_INIT	"slm_init"

#define	SE_TIMEOUT	60	/* Client dispatch timeout (seconds) */

/* syslog message related */
static int logflag = 0;
static char *prog;

/* function prototypes */
static void door_upcall(void *cookie, char *args, size_t alen, door_desc_t *ddp,
	uint_t ndid);
static void dispatch_message(void);
static int dispatch(void);
static void event_completion_thr(void);
static void usage(void);

static void syseventd_init(void);
static void syseventd_fini(int sig);

static pid_t enter_daemon_lock(void);
static void exit_daemon_lock(void);

static void
usage() {
	(void) fprintf(stderr, "usage: syseventd [-d <debug_level>] "
	    "[-r <root_dir>]\n");
	(void) fprintf(stderr, "higher debug levels get progressively ");
	(void) fprintf(stderr, "more detailed debug information.\n");
	(void) fprintf(stderr, "syseventd will run in background if ");
	(void) fprintf(stderr, "run with a debug_level less than %d.\n",
	    DEBUG_LEVEL_FORK);
	exit(2);
}


/* common exit function which ensures releasing locks */
void
syseventd_exit(int status)
{
	syseventd_print(1, "exit status = %d\n", status);

	if (hold_daemon_lock) {
		exit_daemon_lock();
	}

	exit(status);
}


/*
 * hup_handler - SIGHUP handler.  SIGHUP is used to force a reload of
 *		 all SLMs.  During fini, events are drained from all
 *		 client event queues.  The events that have been consumed
 *		 by all clients are freed from the kernel event queue.
 *
 *		 Events that have not yet been delivered to all clients
 *		 are not freed and will be replayed after all SLMs have
 *		 been (re)loaded.
 *
 *		 After all client event queues have been drained, each
 *		 SLM client is unloaded.  The init phase will (re)load
 *		 each SLM and initiate event replay and delivery from
 *		 the kernel.
 *
 */
/*ARGSUSED*/
static void
hup_handler(int sig)
{
	syseventd_err_print(SIGHUP_CAUGHT);
	(void) fflush(0);
	syseventd_fini(sig);
	syseventd_init();
	syseventd_err_print(DAEMON_RESTARTED);
	(void) fflush(0);
}

/*
 * Fault handler for other signals caught
 */
/*ARGSUSED*/
static void
flt_handler(int sig)
{
	char signame[SIG2STR_MAX];

	if (sig2str(sig, signame) == -1) {
		syseventd_err_print(UNKNOWN_SIGNAL_CAUGHT, sig);
	}

	(void) se_signal_sethandler(sig, SIG_DFL, NULL);

	switch (sig) {
		case SIGINT:
		case SIGSTOP:
		case SIGTERM:
			/* Close kernel door */
			(void) door_revoke(upcall_door);

			/* Gracefully exit current event delivery threads */
			syseventd_fini(sig);

			(void) fflush(0);
			(void) se_signal_unblockall();
			syseventd_exit(1);
			/*NOTREACHED*/
		default:
			syseventd_err_print(FATAL_ERROR);
			(void) fflush(0);

	}
}

static void
sigwait_thr()
{
	int	sig;
	int	err;
	sigset_t signal_set;

	for (;;) {
		syseventd_print(3, "sigwait thread waiting for signal\n");
		(void) sigfillset(&signal_set);
		err = sigwait(&signal_set, &sig);
		if (err) {
			syseventd_exit(2);
		}

		/*
		 * Block all signals until the signal handler completes
		 */
		if (sig == SIGHUP) {
			hup_handler(sig);
		} else {
			flt_handler(sig);
		}
	}
	/* NOTREACHED */
}

static void
set_root_dir(char *dir)
{
	root_dir = malloc(strlen(dir) + 1);
	if (root_dir == NULL) {
		syseventd_err_print(INIT_ROOT_DIR_ERR, strerror(errno));
		syseventd_exit(2);
	}
	(void) strcpy(root_dir, dir);
}

int
main(int argc, char **argv)
{
	int i, c;
	int fd;
	pid_t pid;
	extern char *optarg;

	(void) setlocale(LC_ALL, "");
	(void) textdomain(TEXT_DOMAIN);

	if (getuid() != 0) {
		(void) fprintf(stderr, "Must be root to run syseventd\n");
		syseventd_exit(1);
	}

	if (argc > 5) {
		usage();
	}

	if ((prog = strrchr(argv[0], '/')) == NULL) {
		prog = argv[0];
	} else {
		prog++;
	}

	if ((c = getopt(argc, argv, "d:r:")) != EOF) {
		switch (c) {
		case 'd':
			debug_level = atoi(optarg);
			break;
		case 'r':
			/*
			 * Private flag for suninstall to run
			 * daemon during install.
			 */
			set_root_dir(optarg);
			break;
		case '?':
		default:
			usage();
		}
	}

	/* demonize ourselves */
	if (debug_level < DEBUG_LEVEL_FORK) {

		if (fork()) {
			syseventd_exit(0);
		}

		/* child */

		(void) chdir("/");
		(void) setsid();
		if (debug_level <= 1) {
			closefrom(0);
			fd = open("/dev/null", 0);
			(void) dup2(fd, 1);
			(void) dup2(fd, 2);
			logflag = 1;
		}
	}

	openlog("syseventd", LOG_PID, LOG_DAEMON);

	(void) mutex_init(&err_mutex, USYNC_THREAD, NULL);

	syseventd_print(8,
	    "syseventd started, debug level = %d\n", debug_level);

	/* only one instance of syseventd can run at a time */
	if ((pid = enter_daemon_lock()) != getpid()) {
		syseventd_print(1,
		    "event daemon pid %ld already running\n", pid);
		exit(3);
	}

	/* initialize semaphores and eventbuf */
	(void) sema_init(&sema_eventbuf, SE_EVENT_DISPATCH_CNT,
	    USYNC_THREAD, NULL);
	(void) sema_init(&sema_dispatch, 0, USYNC_THREAD, NULL);
	(void) sema_init(&sema_resource, SE_EVENT_DISPATCH_CNT,
			USYNC_THREAD, NULL);
	(void) cond_init(&event_comp_cv, USYNC_THREAD, NULL);
	eventbuf = (sysevent_t **)calloc(SE_EVENT_DISPATCH_CNT,
	    sizeof (sysevent_t *));
	if (eventbuf == NULL) {
		syseventd_print(1, "Unable to allocate event buffer array\n");
		exit(2);
	}
	for (i = 0; i < SE_EVENT_DISPATCH_CNT; ++i) {
		eventbuf[i] = malloc(LOGEVENT_BUFSIZE);
		if (eventbuf[i] == NULL) {
			syseventd_print(1, "Unable to allocate event "
			    "buffers\n");
			exit(2);
		}
	}

	(void) mutex_init(&client_tbl_lock, USYNC_THREAD, NULL);
	(void) mutex_init(&ev_comp_lock, USYNC_THREAD, NULL);
	(void) mutex_init(&door_lock, USYNC_THREAD, NULL);
	(void) rwlock_init(&mod_unload_lock, USYNC_THREAD, NULL);

	event_compq = NULL;

	syseventd_print(8, "start the message thread running\n");

	/*
	 * Block all signals to all threads include the main thread.
	 * The sigwait_thr thread will process any signals and initiate
	 * a graceful recovery if possible.
	 */
	if (se_signal_blockall() < 0) {
		syseventd_err_print(INIT_SIG_BLOCK_ERR);
		syseventd_exit(2);
	}

	if (thr_create(NULL, NULL, (void *(*)(void *))dispatch_message,
				(void *)0, 0, NULL) < 0) {
		syseventd_err_print(INIT_THR_CREATE_ERR, strerror(errno));
		syseventd_exit(2);
	}
	if (thr_create(NULL, NULL,
		(void *(*)(void *))event_completion_thr, NULL,
			THR_BOUND, NULL) != 0) {
		syseventd_err_print(INIT_THR_CREATE_ERR, strerror(errno));
		syseventd_exit(2);
	}
	/* Create signal catching thread */
	if (thr_create(NULL, NULL, (void *(*)(void *))sigwait_thr,
		NULL, 0, NULL) < 0) {
		syseventd_err_print(INIT_THR_CREATE_ERR, strerror(errno));
		syseventd_exit(2);
	}

	setbuf(stdout, (char *)NULL);

	/* Initialize and load SLM clients */
	initialize_client_tbl();
	syseventd_init();

	syseventd_print(8, "Pausing\n");

	for (;;) {
		(void) pause();
	}
	/* NOTREACHED */
	return (0);
}

/*
 * door_upcall - called from the kernel via kernel sysevent door
 *		to upload event(s).
 *
 *		This routine should never block.  If resources are
 *		not available to immediately accept the event buffer
 *		EAGAIN is returned to the kernel.
 *
 *		Once resources are available, the kernel is notified
 *		via a modctl interface to resume event delivery to
 *		syseventd.
 *
 */
/*ARGSUSED*/
static void
door_upcall(void *cookie, char *args, size_t alen,
    door_desc_t *ddp, uint_t ndid)
{
	sysevent_t *ev;
	int rval;


	(void) mutex_lock(&door_lock);
	if (args == NULL) {
		rval = EINVAL;
	} else if (sema_trywait(&sema_eventbuf)) {
		ev = (sysevent_t *)
		    &((log_event_upcall_arg_t *)(void *)args)->buf;
		syseventd_print(2, "door_upcall: busy event %llx "
			"retry\n", sysevent_get_seq(ev));
		rval = door_upcall_retval = EAGAIN;
	} else {
		/*
		 * Copy received message to local buffer.
		 */
		size_t size;
		ev = (sysevent_t *)
		    &((log_event_upcall_arg_t *)(void *)args)->buf;

		syseventd_print(2, "door_upcall: event %llx in eventbuf %d\n",
		    sysevent_get_seq(ev), deliver_buf);
		size = sysevent_get_size(ev) > LOGEVENT_BUFSIZE ?
		    LOGEVENT_BUFSIZE : sysevent_get_size(ev);
		(void) bcopy(ev, eventbuf[deliver_buf], size);
		deliver_buf = (deliver_buf + 1) % SE_EVENT_DISPATCH_CNT;
		rval = 0;
		(void) sema_post(&sema_dispatch);
	}

	(void) mutex_unlock(&door_lock);

	/*
	 * Filling in return values for door_return
	 */
	(void) door_return((void *)&rval, sizeof (rval), NULL, 0);
	(void) door_return(NULL, 0, NULL, 0);
}

/*
 * dispatch_message - dispatch message thread
 *			This thread spins until an event buffer is delivered
 *			delivered from the kernel.
 *
 *			It will wait to dispatch an event to any clients
 *			until adequate resources are available to process
 *			the event buffer.
 */
static void
dispatch_message(void)
{
	int error;

	for (;;) {
		syseventd_print(3, "dispatch_message: thread started\n");
		/*
		 * Spin till a message comes
		 */
		while (sema_wait(&sema_dispatch) != 0) {
			syseventd_print(1,
			    "dispatch_message: sema_wait failed\n");
			(void) sleep(1);
		}

		syseventd_print(3, "dispatch_message: sema_dispatch\n");

		/*
		 * Wait for available resources
		 */
		while (sema_wait(&sema_resource) != 0) {
			syseventd_print(1, "dispatch_message: sema_wait "
				"failed\n");
			(void) sleep(1);
		}

		syseventd_print(2, "dispatch_message: eventbuf %d\n",
		    dispatch_buf);

		/*
		 * Client dispatch
		 */
		do {
			error = dispatch();
		} while (error == EAGAIN);

		syseventd_print(2, "eventbuf %d dispatched\n", dispatch_buf);
		dispatch_buf = (dispatch_buf + 1) % SE_EVENT_DISPATCH_CNT;

		/*
		 * kernel received a busy signal -
		 * kickstart the kernel delivery thread
		 * door_lock blocks the kernel so we hold it for the
		 * shortest time possible.
		 */
		(void) mutex_lock(&door_lock);
		if (door_upcall_retval == EAGAIN && !fini_pending) {
			syseventd_print(3, "dispatch_message: retrigger "
				"door_upcall_retval = %d\n",
				door_upcall_retval);
			(void) modctl(MODEVENTS, (uintptr_t)MODEVENTS_FLUSH,
				NULL, NULL, NULL, 0);
			door_upcall_retval = 0;
		}
		(void) mutex_unlock(&door_lock);
	}
	/* NOTREACHED */
}

/*
 * drain_eventq - Called to drain all pending events from the client's
 *		event queue.
 */
static void
drain_eventq(struct sysevent_client *scp, int status)
{
	struct event_dispatch_pkg *d_pkg;
	struct event_dispatchq *eventq, *eventq_next;

	syseventd_print(3, "Draining eventq for client %d\n",
			scp->client_num);

	eventq = scp->eventq;
	while (eventq) {
		/*
		 * Mark all dispatched events as completed, but indicate the
		 * error status
		 */
		d_pkg = eventq->d_pkg;

		syseventd_print(4, "drain event 0X%llx for client %d\n",
		    sysevent_get_seq(d_pkg->ev), scp->client_num);

		if (d_pkg->completion_state == SE_NOT_DISPATCHED) {
			d_pkg->completion_status = status;
			d_pkg->completion_state = SE_COMPLETE;
			(void) sema_post(d_pkg->completion_sema);
		}

		eventq_next = eventq->next;
		free(eventq);
		eventq = eventq_next;
		scp->eventq = eventq;
	}
}

/*
 * client_deliver_event_thr - Client delivery thread
 *				This thread will process any events on this
 *				client's eventq.
 */
static void
client_deliver_event_thr(void *arg)
{
	int flag, error, i;
	sysevent_t *ev;
	hrtime_t now;
	module_t *mod;
	struct event_dispatchq *eventq;
	struct sysevent_client *scp;
	struct event_dispatch_pkg *d_pkg;

	scp = (struct sysevent_client *)arg;
	mod = (module_t *)scp->client_data;

	(void) mutex_lock(&scp->client_lock);
	for (;;) {
		while (scp->eventq == NULL) {

			/*
			 * Client has been suspended or unloaded, go no further.
			 */
			if (fini_pending) {
				scp->client_flags &= ~SE_CLIENT_THR_RUNNING;
				syseventd_print(3, "Client %d delivery thread "
					"exiting flags: 0X%x\n",
					scp->client_num, scp->client_flags);
				(void) mutex_unlock(&scp->client_lock);
				return;
			}

			(void) cond_wait(&scp->client_cv, &scp->client_lock);

		}

		/*
		 * Process events from the head of the eventq, eventq is locked
		 * going into the processing.
		 */
		eventq = scp->eventq;
		while (eventq != NULL) {
			d_pkg = eventq->d_pkg;
			d_pkg->completion_state = SE_OUTSTANDING;
			(void) mutex_unlock(&scp->client_lock);


			flag = error = 0;
			ev = d_pkg->ev;

			syseventd_print(3, "Start delivery for client %d "
			    "with retry count %d\n",
			    scp->client_num, d_pkg->retry_count);

			/*
			 * Retry limit has been reached by this client, indicate
			 * that no further retries are allowed
			 */
			for (i = 0; i <= scp->retry_limit; ++i) {
				if (i == scp->retry_limit)
					flag = SE_NO_RETRY;

				/* Start the clock for the event delivery */
				d_pkg->start_time = gethrtime();

				syseventd_print(9, "Deliver to module client "
				    "%s\n", mod->name);

				error = mod->deliver_event(ev, flag);

				/* Can not allow another retry */
				if (i == scp->retry_limit)
					error = 0;

				/* Stop the clock */
				now = gethrtime();

				/*
				 * Suspend event processing and drain the
				 * event q for latent clients
				 */
				if (now - d_pkg->start_time >
				    ((hrtime_t)SE_TIMEOUT * NANOSEC)) {
					syseventd_print(1, "Unresponsive "
					    "client %d: Draining eventq and "
					    "suspending event delivery\n",
					    scp->client_num);
					(void) mutex_lock(&scp->client_lock);
					scp->client_flags &=
					    ~SE_CLIENT_THR_RUNNING;
					scp->client_flags |=
					    SE_CLIENT_SUSPENDED;

					/* Cleanup current event */
					d_pkg->completion_status = EFAULT;
					d_pkg->completion_state = SE_COMPLETE;
					(void) sema_post(
					    d_pkg->completion_sema);

					/*
					 * Drain the remaining events from the
					 * queue.
					 */
					drain_eventq(scp, EINVAL);
					(void) mutex_unlock(&scp->client_lock);
					return;
				}

				/* Event delivery retry requested */
				if (fini_pending || error != EAGAIN) {
					break;
				} else {
					(void) sleep(SE_RETRY_TIME);
				}
			}

			(void) mutex_lock(&scp->client_lock);
			d_pkg->completion_status = error;
			d_pkg->completion_state = SE_COMPLETE;
			(void) sema_post(d_pkg->completion_sema);

			/* Update eventq pointer */
			if (scp->eventq != NULL) {
				scp->eventq = eventq->next;
				free(eventq);
				eventq = scp->eventq;
			} else {
				free(eventq);
				break;
			}

			syseventd_print(3, "Completed delivery with "
			    "error %d\n", error);
		}

		syseventd_print(3, "No more events to process for client %d\n",
			scp->client_num);

		/* Return if this was a synchronous delivery */
		if (!SE_CLIENT_IS_THR_RUNNING(scp)) {
			(void) mutex_unlock(&scp->client_lock);
			return;
		}

	}
}

/*
 * client_deliver_event - Client specific event delivery
 *			This routine will allocate and initialize the
 *			neccessary per-client dispatch data.
 *
 *			If the eventq is not empty, it may be assumed that
 *			a delivery thread exists for this client and the
 *			dispatch data is appended to the eventq.
 *
 *			The dispatch package is freed by the event completion
 *			thread (event_completion_thr) and the eventq entry
 *			is freed by the event delivery thread.
 */
static struct event_dispatch_pkg *
client_deliver_event(struct sysevent_client *scp, sysevent_t *ev,
	sema_t *completion_sema)
{
	size_t ev_sz = sysevent_get_size(ev);
	struct event_dispatchq *newq, *tmp;
	struct event_dispatch_pkg *d_pkg;

	syseventd_print(3, "client_deliver_event: id 0x%llx size %d\n",
		(longlong_t)sysevent_get_seq(ev), ev_sz);
	if (debug_level == 9) {
		se_print(stdout, ev);
	}

	/*
	 * Check for suspended client
	 */
	(void) mutex_lock(&scp->client_lock);
	if (SE_CLIENT_IS_SUSPENDED(scp) || !SE_CLIENT_IS_THR_RUNNING(scp)) {
		(void) mutex_unlock(&scp->client_lock);
		return (NULL);
	}

	/*
	 * Allocate a new dispatch package and eventq entry
	 */
	newq = (struct event_dispatchq *)malloc(
		sizeof (struct event_dispatchq));
	if (newq == NULL) {
		(void) mutex_unlock(&scp->client_lock);
		return (NULL);
	}

	d_pkg = (struct event_dispatch_pkg *)malloc(
		sizeof (struct event_dispatch_pkg));
	if (d_pkg == NULL) {
		free(newq);
		(void) mutex_unlock(&scp->client_lock);
		return (NULL);
	}

	/* Initialize the dispatch package */
	d_pkg->scp = scp;
	d_pkg->retry_count = 0;
	d_pkg->completion_status = 0;
	d_pkg->completion_state = SE_NOT_DISPATCHED;
	d_pkg->completion_sema = completion_sema;
	d_pkg->ev = ev;
	newq->d_pkg = d_pkg;
	newq->next = NULL;

	if (scp->eventq != NULL) {

		/* Add entry to the end of the eventq */
		tmp = scp->eventq;
		while (tmp->next != NULL)
			tmp = tmp->next;
		tmp->next = newq;
	} else {
		/* event queue empty, wakeup delivery thread */
		scp->eventq = newq;
		(void) cond_signal(&scp->client_cv);
	}
	(void) mutex_unlock(&scp->client_lock);

	return (d_pkg);
}

/*
 * event_completion_thr - Event completion thread.  This thread routine
 *			waits for all client delivery thread to complete
 *			delivery of a particular event.
 */
static void
event_completion_thr()
{
	int ret, i, client_count, ok_to_free;
	sysevent_id_t eid;
	struct sysevent_client *scp;
	struct ev_completion *ev_comp;
	struct event_dispatchq *dispatchq;
	struct event_dispatch_pkg *d_pkg;

	(void) mutex_lock(&ev_comp_lock);
	for (;;) {
		while (event_compq == NULL) {
			(void) cond_wait(&event_comp_cv, &ev_comp_lock);
		}

		/*
		 * Process event completions from the head of the
		 * completion queue
		 */
		ev_comp = event_compq;
		while (ev_comp) {
			(void) mutex_unlock(&ev_comp_lock);
			eid.eid_seq = sysevent_get_seq(ev_comp->ev);
			sysevent_get_time(ev_comp->ev, &eid.eid_ts);
			client_count = ev_comp->client_count;
			ok_to_free = 1;

			syseventd_print(3, "Wait for event completion of "
			    "event 0X%llx on %d clients\n",
			    eid.eid_seq, client_count);

			while (client_count) {
				syseventd_print(9, "Waiting for %d clients on "
					"event id 0X%llx\n", client_count,
					eid.eid_seq);

				(void) sema_wait(&ev_comp->client_sema);
				--client_count;
			}

			syseventd_print(3, "Cleaning up clients for event "
			    "0X%llx\n", eid.eid_seq);
			dispatchq = ev_comp->dispatch_list;
			while (dispatchq != NULL) {
				d_pkg = dispatchq->d_pkg;
				scp = d_pkg->scp;

				if (d_pkg->completion_status == EAGAIN)
					ok_to_free = 0;

				syseventd_print(4, "Delivery of 0X%llx "
				    "complete for client %d retry count %d "
				    "status %d\n", eid.eid_seq,
				    scp->client_num,
				    d_pkg->retry_count,
				    d_pkg->completion_status);

				free(d_pkg);
				ev_comp->dispatch_list = dispatchq->next;
				free(dispatchq);
				dispatchq = ev_comp->dispatch_list;
			}

			if (ok_to_free) {
				for (i = 0; i < MAX_MODCTL_RETRY; ++i) {
					if ((ret = modctl(MODEVENTS,
						(uintptr_t)MODEVENTS_FREEDATA,
						(uintptr_t)&eid, NULL,
						NULL, 0)) != 0) {
						syseventd_print(1, "attempting "
						    "to free event 0X%llx\n",
						    eid.eid_seq);

						/*
						 * Kernel may need time to
						 * move this event buffer to
						 * the sysevent sent queue
						 */
						(void) sleep(1);
					} else {
						break;
					}
				}
				if (ret) {
					syseventd_print(1, "Unable to free "
						"event 0X%llx from the "
						"kernel\n", eid.eid_seq);
				}
			} else {
				syseventd_print(1, "Not freeing event 0X%llx\n",
					eid.eid_seq);
			}

			syseventd_print(2, "Event delivery complete for id "
				"0X%llx\n", eid.eid_seq);

			(void) mutex_lock(&ev_comp_lock);
			event_compq = ev_comp->next;
			free(ev_comp->ev);
			free(ev_comp);
			ev_comp = event_compq;
			(void) sema_post(&sema_resource);
		}

		/*
		 * Event completion queue is empty, signal possible unload
		 * operation
		 */
		(void) cond_signal(&event_comp_cv);

		syseventd_print(3, "No more events\n");
	}
}

/*
 * dispatch - Dispatch the current event buffer to all valid SLM clients.
 */
static int
dispatch(void)
{
	int ev_sz, i, client_count = 0;
	sysevent_t *new_ev;
	sysevent_id_t eid;
	struct ev_completion *ev_comp, *tmp;
	struct event_dispatchq *dispatchq, *client_list;
	struct event_dispatch_pkg *d_pkg;

	/* Check for module unload operation */
	if (rw_tryrdlock(&mod_unload_lock) != 0) {
		syseventd_print(2, "unload in progress abort delivery\n");
		(void) sema_post(&sema_eventbuf);
		(void) sema_post(&sema_resource);
		return (0);
	}

	syseventd_print(3, "deliver dispatch buffer %d", dispatch_buf);
	eid.eid_seq = sysevent_get_seq(eventbuf[dispatch_buf]);
	sysevent_get_time(eventbuf[dispatch_buf], &eid.eid_ts);
	syseventd_print(3, "deliver msg id: 0x%llx\n", eid.eid_seq);

	/*
	 * ev_comp is used to hold event completion data.  It is freed
	 * by the event completion thread (event_completion_thr).
	 */
	ev_comp = (struct ev_completion *)
		malloc(sizeof (struct ev_completion));
	if (ev_comp == NULL) {
		(void) rw_unlock(&mod_unload_lock);
		syseventd_print(1, "Can not allocate event completion buffer "
			"for event id 0X%llx\n", eid.eid_seq);
		return (EAGAIN);
	}
	ev_comp->dispatch_list = NULL;
	ev_comp->next = NULL;
	(void) sema_init(&ev_comp->client_sema, 0, USYNC_THREAD, NULL);

	ev_sz = sysevent_get_size(eventbuf[dispatch_buf]);
	new_ev = calloc(1, ev_sz);
	if (new_ev == NULL) {
		free(ev_comp);
		(void) rw_unlock(&mod_unload_lock);
		syseventd_print(1, "Can not allocate new event buffer "
		"for event id 0X%llx\n", eid.eid_seq);
		return (EAGAIN);
	}


	/*
	 * For long messages, copy additional data from kernel
	 */
	if (ev_sz > LOGEVENT_BUFSIZE) {
		int ret = 0;

		/* Ok to release eventbuf for next event buffer from kernel */
		(void) sema_post(&sema_eventbuf);

		for (i = 0; i < MAX_MODCTL_RETRY; ++i) {
			if ((ret = modctl(MODEVENTS,
				(uintptr_t)MODEVENTS_GETDATA,
				(uintptr_t)&eid,
				(uintptr_t)ev_sz,
				(uintptr_t)new_ev, 0))
				== 0)
				break;
			else
				(void) sleep(1);
		}
		if (ret) {
			syseventd_print(1, "GET_DATA failed for 0X%llx:%llx\n",
			    eid.eid_ts, eid.eid_seq);
			free(new_ev);
			free(ev_comp);
			(void) rw_unlock(&mod_unload_lock);
			return (EAGAIN);
		}
	} else {
		(void) bcopy(eventbuf[dispatch_buf], new_ev, ev_sz);
		/* Ok to release eventbuf for next event buffer from kernel */
		(void) sema_post(&sema_eventbuf);
	}


	/*
	 * Deliver a copy of eventbuf to clients so
	 * eventbuf can be used for the next message
	 */
	for (i = 0; i < MAX_SLM; ++i) {

		/* Don't bother for suspended or unloaded clients */
		if (!SE_CLIENT_IS_LOADED(sysevent_client_tbl[i]) ||
		    SE_CLIENT_IS_SUSPENDED(sysevent_client_tbl[i]))
			continue;

		/*
		 * Allocate event dispatch queue entry.  All queue entries
		 * are freed by the event completion thread as client
		 * delivery completes.
		 */
		dispatchq = (struct event_dispatchq *)malloc(
			sizeof (struct event_dispatchq));
		if (dispatchq == NULL) {
			syseventd_print(1, "Can not allocate dispatch q "
			"for event id 0X%llx client %d\n", eid.eid_seq, i);
			continue;
		}
		dispatchq->next = NULL;

		/* Initiate client delivery */
		d_pkg = client_deliver_event(sysevent_client_tbl[i],
			new_ev, &ev_comp->client_sema);
		if (d_pkg == NULL) {
			syseventd_print(1, "Can not allocate dispatch "
				"package for event id 0X%llx client %d\n",
				eid.eid_seq, i);
			free(dispatchq);
			continue;
		}
		dispatchq->d_pkg = d_pkg;
		++client_count;

		if (ev_comp->dispatch_list == NULL) {
			ev_comp->dispatch_list = dispatchq;
			client_list = dispatchq;
		} else {
			client_list->next = dispatchq;
			client_list = client_list->next;
		}
	}

	ev_comp->client_count = client_count;
	ev_comp->ev = new_ev;

	(void) mutex_lock(&ev_comp_lock);

	if (event_compq == NULL) {
		syseventd_print(3, "Wakeup event completion thread for "
		    "id 0X%llx\n", eid.eid_seq);
		event_compq = ev_comp;
		(void) cond_signal(&event_comp_cv);
	} else {

		/* Add entry to the end of the event completion queue */
		tmp = event_compq;
		while (tmp->next != NULL)
			tmp = tmp->next;
		tmp->next = ev_comp;
		syseventd_print(3, "event added to completion queue for "
		    "id 0X%llx\n", eid.eid_seq);
	}
	(void) mutex_unlock(&ev_comp_lock);
	(void) rw_unlock(&mod_unload_lock);

	return (0);
}

#define	MODULE_DIR_HW	"/usr/platform/%s/lib/sysevent/modules/"
#define	MODULE_DIR_GEN	"/usr/lib/sysevent/modules/"
#define	MOD_DIR_NUM	3
static char dirname[MOD_DIR_NUM][MAXPATHLEN];

static char *
dir_num2name(int dirnum)
{
	char infobuf[MAXPATHLEN];

	if (dirnum >= MOD_DIR_NUM)
		return (NULL);

	if (dirname[0][0] == '\0') {
		if (sysinfo(SI_PLATFORM, infobuf, MAXPATHLEN) == -1) {
			syseventd_print(1, "dir_num2name: "
				"sysinfo error %s\n", strerror(errno));
			return (NULL);
		} else if (snprintf(dirname[0], sizeof (dirname[0]),
		    MODULE_DIR_HW, infobuf) >= sizeof (dirname[0])) {
			syseventd_print(1, "dir_num2name: "
				"platform name too long: %s\n",
				infobuf);
			return (NULL);
		}
		if (sysinfo(SI_MACHINE, infobuf, MAXPATHLEN) == -1) {
			syseventd_print(1, "dir_num2name: "
				"sysinfo error %s\n", strerror(errno));
			return (NULL);
		} else if (snprintf(dirname[1], sizeof (dirname[1]),
		    MODULE_DIR_HW, infobuf) >= sizeof (dirname[1])) {
			syseventd_print(1, "dir_num2name: "
				"machine name too long: %s\n",
				infobuf);
			return (NULL);
		}
		(void) strcpy(dirname[2], MODULE_DIR_GEN);
	}

	return (dirname[dirnum]);
}


/*
 * load_modules - Load modules found in the common syseventd module directories
 *		Modules that do not provide valid interfaces are rejected.
 */
static void
load_modules(char *dirname)
{
	int client_id;
	DIR *mod_dir;
	module_t *mod;
	struct dirent *entp;
	struct slm_mod_ops *mod_ops;
	struct sysevent_client *scp;

	if (dirname == NULL)
		return;

	/* Return silently if module directory does not exist */
	if ((mod_dir = opendir(dirname)) == NULL) {
		syseventd_print(1, "Unable to open module directory %s: %s\n",
			dirname, strerror(errno));
		return;
	}

	syseventd_print(3, "loading modules from %s\n", dirname);

	/*
	 * Go through directory, looking for files ending with .so
	 */
	while ((entp = readdir(mod_dir)) != NULL) {
		void *dlh, *f;
		char *tmp, modpath[MAXPATHLEN];

		if (((tmp = strstr(entp->d_name, MODULE_SUFFIX)) == NULL) ||
		    (tmp[strlen(MODULE_SUFFIX)] != '\0')) {
			continue;
		}

		if (snprintf(modpath, sizeof (modpath), "%s%s",
		    dirname, entp->d_name) >= sizeof (modpath)) {
			syseventd_err_print(INIT_PATH_ERR, modpath);
			continue;
		}
		if ((dlh = dlopen(modpath, RTLD_LAZY)) == NULL) {
			syseventd_err_print(LOAD_MOD_DLOPEN_ERR,
				modpath, dlerror());
			continue;
		} else if ((f = dlsym(dlh, EVENT_INIT)) == NULL) {
			syseventd_err_print(LOAD_MOD_NO_INIT,
				modpath, dlerror());
			(void) dlclose(dlh);
			continue;
		}

		mod = malloc(sizeof (*mod));
		if (mod == NULL) {
			syseventd_err_print(LOAD_MOD_ALLOC_ERR, "mod",
				strerror(errno));
			(void) dlclose(dlh);
			continue;
		}

		mod->name = strdup(entp->d_name);
		if (mod->name == NULL) {
			syseventd_err_print(LOAD_MOD_ALLOC_ERR, "mod->name",
				strerror(errno));
			(void) dlclose(dlh);
			free(mod);
			continue;
		}

		mod->dlhandle = dlh;
		mod->event_mod_init = (struct slm_mod_ops *(*)())f;

		/* load in other module functions */
		mod->event_mod_fini = (void (*)())dlsym(dlh, EVENT_FINI);
		if (mod->event_mod_fini == NULL) {
			syseventd_err_print(LOAD_MOD_DLSYM_ERR, mod->name,
				dlerror());
			free(mod->name);
			free(mod);
			(void) dlclose(dlh);
			continue;
		}

		/* Call module init routine */
		if ((mod_ops = mod->event_mod_init()) == NULL) {
			syseventd_err_print(LOAD_MOD_EINVAL, mod->name);
			free(mod->name);
			free(mod);
			(void) dlclose(dlh);
			continue;
		}
		if (mod_ops->major_version != SE_MAJOR_VERSION) {
			syseventd_err_print(LOAD_MOD_VERSION_MISMATCH,
				mod->name, SE_MAJOR_VERSION,
				mod_ops->major_version);
			mod->event_mod_fini();
			free(mod->name);
			free(mod);
			(void) dlclose(dlh);
			continue;
		}

		mod->deliver_event = mod_ops->deliver_event;
		/* Add module entry to client list */
		if ((client_id = insert_client((void *)mod, SLM_CLIENT,
			(mod_ops->retry_limit <= SE_MAX_RETRY_LIMIT ?
				mod_ops->retry_limit : SE_MAX_RETRY_LIMIT)))
				< 0) {;
			syseventd_err_print(LOAD_MOD_ALLOC_ERR, "insert_client",
				strerror(errno));
			mod->event_mod_fini();
			free(mod->name);
			free(mod);
			(void) dlclose(dlh);
			continue;
		}

		scp = sysevent_client_tbl[client_id];
		++concurrency_level;
		(void) thr_setconcurrency(concurrency_level);
		if (thr_create(NULL, 0,
			(void *(*)(void *))client_deliver_event_thr,
			(void *)scp, THR_BOUND, &scp->tid) != 0) {

			syseventd_err_print(LOAD_MOD_ALLOC_ERR, "insert_client",
				strerror(errno));
			mod->event_mod_fini();
			free(mod->name);
			free(mod);
			(void) dlclose(dlh);
			continue;
		}
		scp->client_flags |= SE_CLIENT_THR_RUNNING;

		syseventd_print(3, "loaded module %s\n", entp->d_name);
	}

	(void) closedir(mod_dir);
	syseventd_print(3, "modules loaded\n");
}

/*
 * unload_modules - modules are unloaded prior to graceful shutdown or
 *			before restarting the daemon upon receipt of
 *			SIGHUP.
 */
static void
unload_modules(int sig)
{
	int			i, count, done;
	module_t		*mod;
	struct sysevent_client	*scp;

	/*
	 * unload modules that are ready, skip those that have not
	 * drained their event queues.
	 */
	count = done = 0;
	while (done < MAX_SLM) {
		/* Don't wait indefinitely for unresponsive clients */
		if (sig != SIGHUP && count > SE_TIMEOUT) {
			break;
		}

		done = 0;

		/* Shutdown clients */
		for (i = 0; i < MAX_SLM; ++i) {
			scp = sysevent_client_tbl[i];
			if (mutex_trylock(&scp->client_lock) == 0) {
				if (scp->client_type != SLM_CLIENT ||
					scp->client_data == NULL) {
					(void) mutex_unlock(&scp->client_lock);
					done++;
					continue;
				}
			} else {
				syseventd_print(3, "Skipping unload of "
					"client %d: client locked\n",
						scp->client_num);
				continue;
			}

			/*
			 * Drain the eventq and wait for delivery thread to
			 * cleanly exit
			 */
			drain_eventq(scp, EAGAIN);
			(void) cond_signal(&scp->client_cv);
			(void) mutex_unlock(&scp->client_lock);
			(void) thr_join(scp->tid, NULL, NULL);

			/*
			 * It is now safe to unload the module
			 */
			mod = (module_t *)scp->client_data;
			syseventd_print(2, "Unload %s\n", mod->name);
			mod->event_mod_fini();
			(void) dlclose(mod->dlhandle);
			free(mod->name);
			(void) mutex_lock(&client_tbl_lock);
			delete_client(i);
			(void) mutex_unlock(&client_tbl_lock);
			++done;

		}
		++count;
		(void) sleep(1);
	}

	/*
	 * Wait for event completions
	 */
	syseventd_print(2, "waiting for event completions\n");
	(void) mutex_lock(&ev_comp_lock);
	while (event_compq != NULL) {
		(void) cond_wait(&event_comp_cv, &ev_comp_lock);
	}
	(void) mutex_unlock(&ev_comp_lock);
}

/*
 * syseventd_init - Called at daemon (re)start-up time to load modules
 *			and kickstart the kernel delivery engine.
 */
static void
syseventd_init()
{
	int i, fd;
	char local_door_file[PATH_MAX + 1];

	fini_pending = 0;

	concurrency_level = MIN_CONCURRENCY_LEVEL;
	(void) thr_setconcurrency(concurrency_level);

	/*
	 * Load client modules for event delivering
	 */
	for (i = 0; i < MOD_DIR_NUM; ++i) {
		load_modules(dir_num2name(i));
	}

	/*
	 * Create kernel delivery door service
	 */
	syseventd_print(8, "Create a door for kernel upcalls\n");
	if (snprintf(local_door_file, sizeof (local_door_file), "%s%s",
	    root_dir, LOGEVENT_DOOR_UPCALL) >= sizeof (local_door_file)) {
		syseventd_err_print(INIT_PATH_ERR, local_door_file);
		syseventd_exit(5);
	}

	/*
	 * Remove door file for robustness.
	 */
	if (unlink(local_door_file) != 0)
		syseventd_print(8, "Unlink of %s failed.\n", local_door_file);

	fd = open(local_door_file, O_CREAT|O_RDWR, S_IREAD|S_IWRITE);
	if ((fd == -1) && (errno != EEXIST)) {
		syseventd_err_print(INIT_OPEN_DOOR_ERR, strerror(errno));
		syseventd_exit(5);
	}
	(void) close(fd);

	upcall_door = door_create(door_upcall, NULL,
	    DOOR_REFUSE_DESC | DOOR_NO_CANCEL);
	if (upcall_door == -1) {
		syseventd_err_print(INIT_CREATE_DOOR_ERR, strerror(errno));
		syseventd_exit(5);
	}

	(void) fdetach(local_door_file);
retry:
	if (fattach(upcall_door, local_door_file) != 0) {
		if (errno == EBUSY)
			goto retry;
		syseventd_err_print(INIT_FATTACH_ERR, strerror(errno));
		(void) door_revoke(upcall_door);
		syseventd_exit(5);
	}

	/*
	 * Tell kernel the door name and start delivery
	 */
	syseventd_print(2,
	    "local_door_file = %s\n", local_door_file);
	if (modctl(MODEVENTS,
	    (uintptr_t)MODEVENTS_SET_DOOR_UPCALL_FILENAME,
	    (uintptr_t)local_door_file, NULL, NULL, 0) < 0) {
		syseventd_err_print(INIT_DOOR_NAME_ERR, strerror(errno));
		syseventd_exit(6);
	}

	door_upcall_retval = 0;

	if (modctl(MODEVENTS, (uintptr_t)MODEVENTS_FLUSH, NULL, NULL, NULL, 0)
		< 0) {
		syseventd_err_print(KERNEL_REPLAY_ERR, strerror(errno));
		syseventd_exit(7);
	}
}

/*
 * syseventd_fini - shut down daemon, but do not exit
 */
static void
syseventd_fini(int sig)
{
	/*
	 * Indicate that event queues should be drained and no
	 * additional events be accepted
	 */
	fini_pending = 1;

	/* Close the kernel event door to halt delivery */
	(void) door_revoke(upcall_door);

	syseventd_print(1, "Unloading modules\n");
	(void) rw_wrlock(&mod_unload_lock);
	unload_modules(sig);
	(void) rw_unlock(&mod_unload_lock);

}

/*
 * enter_daemon_lock - lock the daemon file lock
 *
 * Use an advisory lock to ensure that only one daemon process is active
 * in the system at any point in time.	If the lock is held by another
 * process, do not block but return the pid owner of the lock to the
 * caller immediately.	The lock is cleared if the holding daemon process
 * exits for any reason even if the lock file remains, so the daemon can
 * be restarted if necessary.  The lock file is DAEMON_LOCK_FILE.
 */
static pid_t
enter_daemon_lock(void)
{
	struct flock	lock;

	syseventd_print(8, "enter_daemon_lock: lock file = %s\n",
		DAEMON_LOCK_FILE);

	if (snprintf(local_lock_file, sizeof (local_lock_file), "%s%s",
	    root_dir, DAEMON_LOCK_FILE) >= sizeof (local_lock_file)) {
		syseventd_err_print(INIT_PATH_ERR, local_lock_file);
		syseventd_exit(8);
	}
	daemon_lock_fd = open(local_lock_file, O_CREAT|O_RDWR, 0644);
	if (daemon_lock_fd < 0) {
		syseventd_err_print(INIT_LOCK_OPEN_ERR,
			local_lock_file, strerror(errno));
		syseventd_exit(8);
	}

	lock.l_type = F_WRLCK;
	lock.l_whence = SEEK_SET;
	lock.l_start = 0;
	lock.l_len = 0;

	if (fcntl(daemon_lock_fd, F_SETLK, &lock) == -1) {
		if (fcntl(daemon_lock_fd, F_GETLK, &lock) == -1) {
			syseventd_err_print(INIT_LOCK_ERR,
				local_lock_file, strerror(errno));
			exit(2);
		}
		return (lock.l_pid);
	}
	hold_daemon_lock = 1;

	return (getpid());
}

/*
 * exit_daemon_lock - release the daemon file lock
 */
static void
exit_daemon_lock(void)
{
	struct flock lock;

	lock.l_type = F_UNLCK;
	lock.l_whence = SEEK_SET;
	lock.l_start = 0;
	lock.l_len = 0;

	if (fcntl(daemon_lock_fd, F_SETLK, &lock) == -1) {
		syseventd_err_print(INIT_UNLOCK_ERR,
			local_lock_file, strerror(errno));
	}

	if (close(daemon_lock_fd) == -1) {
		syseventd_err_print(INIT_LOCK_CLOSE_ERR,
			local_lock_file, strerror(errno));
		exit(-1);
	}
}

/*
 * syseventd_err_print - print error messages to the terminal if not
 *			yet daemonized or to syslog.
 */
/*PRINTFLIKE1*/
void
syseventd_err_print(char *message, ...)
{
	va_list ap;

	(void) mutex_lock(&err_mutex);
	va_start(ap, message);

	if (logflag) {
		(void) vsyslog(LOG_ERR, message, ap);
	} else {
		(void) fprintf(stderr, "%s: ", prog);
		(void) vfprintf(stderr, message, ap);
	}
	va_end(ap);
	(void) mutex_unlock(&err_mutex);
}

/*
 * syseventd_print -  print messages to the terminal or to syslog
 *			the following levels are implemented:
 *
 * 1 - transient errors that does not affect normal program flow
 * 2 - upcall/dispatch interaction
 * 3 - program flow trace as each message goes through the daemon
 * 8 - all the nit-gritty details of startup and shutdown
 * 9 - very verbose event flow tracing (no daemonization of syseventd)
 *
 */
/*PRINTFLIKE2*/
void
syseventd_print(int level, char *message, ...)
{
	va_list ap;
	static int newline = 1;

	if (level > debug_level) {
		return;
	}

	(void) mutex_lock(&err_mutex);
	va_start(ap, message);
	if (logflag) {
		(void) syslog(LOG_DEBUG, "%s[%ld]: ",
			    prog, getpid());
		(void) vsyslog(LOG_DEBUG, message, ap);
	} else {
		if (newline) {
			(void) fprintf(stdout, "%s[%ld]: ",
				prog, getpid());
			(void) vfprintf(stdout, message, ap);
		} else {
			(void) vfprintf(stdout, message, ap);
		}
	}
	if (message[strlen(message)-1] == '\n') {
		newline = 1;
	} else {
		newline = 0;
	}
	va_end(ap);
	(void) mutex_unlock(&err_mutex);
}