xref: /titanic_52/usr/src/cmd/fm/fmd/common/fmd.c (revision 6e628f2786bf7adece487b606a56068e35e3fcd2)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/types.h>
30 #include <sys/utsname.h>
31 #include <sys/param.h>
32 #include <sys/systeminfo.h>
33 #include <sys/fm/util.h>
34 #include <fm/libtopo.h>
35 
36 #include <smbios.h>
37 #include <limits.h>
38 #include <unistd.h>
39 #include <signal.h>
40 #include <stdlib.h>
41 #include <stdio.h>
42 #include <door.h>
43 
44 #include <fmd_conf.h>
45 #include <fmd_dispq.h>
46 #include <fmd_timerq.h>
47 #include <fmd_subr.h>
48 #include <fmd_error.h>
49 #include <fmd_module.h>
50 #include <fmd_thread.h>
51 #include <fmd_alloc.h>
52 #include <fmd_string.h>
53 #include <fmd_builtin.h>
54 #include <fmd_ustat.h>
55 #include <fmd_protocol.h>
56 #include <fmd_scheme.h>
57 #include <fmd_asru.h>
58 #include <fmd_case.h>
59 #include <fmd_log.h>
60 #include <fmd_idspace.h>
61 #include <fmd_rpc.h>
62 #include <fmd_dr.h>
63 #include <fmd_xprt.h>
64 #include <fmd_ctl.h>
65 #include <sys/openpromio.h>
66 #include <libdevinfo.h>
67 
68 #include <fmd.h>
69 
70 extern const nv_alloc_ops_t fmd_nv_alloc_ops;	/* see fmd_nv.c */
71 
72 const char _fmd_version[] = "1.1";		/* daemon version string */
73 static char _fmd_plat[MAXNAMELEN];		/* native platform string */
74 static char _fmd_isa[MAXNAMELEN];		/* native instruction set */
75 static struct utsname _fmd_uts;			/* native uname(2) info */
76 static char _fmd_csn[MAXNAMELEN];		/* chassis serial number */
77 static char _fmd_prod[MAXNAMELEN];		/* product name string */
78 
79 /*
80  * Note: the configuration file path is ordered from most common to most host-
81  * specific because new conf files are merged/override previous ones.  The
82  * module paths are in the opposite order, from most specific to most common,
83  * because once a module is loaded fmd will not try to load over the same name.
84  */
85 
86 static const char _fmd_conf_path[] =
87 	"%r/usr/lib/fm/fmd:"
88 	"%r/usr/platform/%m/lib/fm/fmd:"
89 	"%r/usr/platform/%i/lib/fm/fmd:"
90 	"%r/etc/fm/fmd";
91 
92 static const char _fmd_agent_path[] =
93 	"%r/usr/platform/%i/lib/fm/fmd/agents:"
94 	"%r/usr/platform/%m/lib/fm/fmd/agents:"
95 	"%r/usr/lib/fm/fmd/agents";
96 
97 static const char _fmd_plugin_path[] =
98 	"%r/usr/platform/%i/lib/fm/fmd/plugins:"
99 	"%r/usr/platform/%m/lib/fm/fmd/plugins:"
100 	"%r/usr/lib/fm/fmd/plugins";
101 
102 static const char _fmd_scheme_path[] =
103 	"usr/lib/fm/fmd/schemes";
104 
105 static const fmd_conf_mode_t _fmd_cerror_modes[] = {
106 	{ "unload", "unload offending client module", FMD_CERROR_UNLOAD },
107 	{ "stop", "stop daemon for debugger attach", FMD_CERROR_STOP },
108 	{ "abort", "abort daemon and force core dump", FMD_CERROR_ABORT },
109 	{ NULL, NULL, 0 }
110 };
111 
112 static const fmd_conf_mode_t _fmd_dbout_modes[] = {
113 	{ "stderr", "send debug messages to stderr", FMD_DBOUT_STDERR },
114 	{ "syslog", "send debug messages to syslog", FMD_DBOUT_SYSLOG },
115 	{ NULL, NULL, 0 }
116 };
117 
118 static const fmd_conf_mode_t _fmd_debug_modes[] = {
119 	{ "help", "display debugging modes and exit", FMD_DBG_HELP },
120 	{ "mod", "debug module load/unload/locking", FMD_DBG_MOD },
121 	{ "disp", "debug dispatch queue processing", FMD_DBG_DISP },
122 	{ "xprt", "debug transport-specific routines", FMD_DBG_XPRT },
123 	{ "evt", "debug event subsystem routines", FMD_DBG_EVT },
124 	{ "log", "debug log subsystem routines", FMD_DBG_LOG },
125 	{ "tmr", "debug timer subsystem routines", FMD_DBG_TMR },
126 	{ "fmri", "debug fmri subsystem routines", FMD_DBG_FMRI },
127 	{ "asru", "debug asru subsystem routines", FMD_DBG_ASRU },
128 	{ "case", "debug case subsystem routines", FMD_DBG_CASE },
129 	{ "ckpt", "debug checkpoint routines", FMD_DBG_CKPT },
130 	{ "rpc", "debug rpc service routines", FMD_DBG_RPC },
131 	{ "trace", "display matching trace calls", FMD_DBG_TRACE },
132 	{ "all", "enable all available debug modes", FMD_DBG_ALL },
133 	{ NULL, NULL, 0 }
134 };
135 
136 static int
137 fmd_cerror_set(fmd_conf_param_t *pp, const char *value)
138 {
139 	return (fmd_conf_mode_set(_fmd_cerror_modes, pp, value));
140 }
141 
142 static int
143 fmd_dbout_set(fmd_conf_param_t *pp, const char *value)
144 {
145 	return (fmd_conf_mode_set(_fmd_dbout_modes, pp, value));
146 }
147 
148 static int
149 fmd_debug_set(fmd_conf_param_t *pp, const char *value)
150 {
151 	int err = fmd_conf_mode_set(_fmd_debug_modes, pp, value);
152 
153 	if (err == 0)
154 		fmd.d_fmd_debug = pp->cp_value.cpv_num;
155 
156 	return (err);
157 }
158 
159 static int
160 fmd_trmode_set(fmd_conf_param_t *pp, const char *value)
161 {
162 	fmd_tracebuf_f *func;
163 
164 	if (strcasecmp(value, "none") == 0)
165 		func = fmd_trace_none;
166 	else if (strcasecmp(value, "lite") == 0)
167 		func = fmd_trace_lite;
168 	else if (strcasecmp(value, "full") == 0)
169 		func = fmd_trace_full;
170 	else
171 		return (fmd_set_errno(EFMD_CONF_INVAL));
172 
173 	fmd.d_thr_trace = (void (*)())func;
174 	pp->cp_value.cpv_ptr = (void *)func;
175 	return (0);
176 }
177 
178 static void
179 fmd_trmode_get(const fmd_conf_param_t *pp, void *ptr)
180 {
181 	*((void **)ptr) = pp->cp_value.cpv_ptr;
182 }
183 
184 static int
185 fmd_clkmode_set(fmd_conf_param_t *pp, const char *value)
186 {
187 	const fmd_timeops_t *ops;
188 
189 	if (strcasecmp(value, "native") == 0)
190 		ops = &fmd_timeops_native;
191 	else if (strcasecmp(value, "simulated") == 0)
192 		ops = &fmd_timeops_simulated;
193 	else
194 		return (fmd_set_errno(EFMD_CONF_INVAL));
195 
196 	fmd.d_clockops = ops;
197 	pp->cp_value.cpv_ptr = (void *)ops;
198 	return (0);
199 }
200 
201 static void
202 fmd_clkmode_get(const fmd_conf_param_t *pp, void *ptr)
203 {
204 	*((void **)ptr) = pp->cp_value.cpv_ptr;
205 }
206 
207 static const fmd_conf_ops_t fmd_cerror_ops = {
208 	fmd_cerror_set, fmd_conf_mode_get, fmd_conf_notsup, fmd_conf_nop
209 };
210 
211 static const fmd_conf_ops_t fmd_dbout_ops = {
212 	fmd_dbout_set, fmd_conf_mode_get, fmd_conf_notsup, fmd_conf_nop
213 };
214 
215 static const fmd_conf_ops_t fmd_debug_ops = {
216 	fmd_debug_set, fmd_conf_mode_get, fmd_conf_notsup, fmd_conf_nop
217 };
218 
219 static const fmd_conf_ops_t fmd_trmode_ops = {
220 	fmd_trmode_set, fmd_trmode_get, fmd_conf_notsup, fmd_conf_nop
221 };
222 
223 static const fmd_conf_ops_t fmd_clkmode_ops = {
224 	fmd_clkmode_set, fmd_clkmode_get, fmd_conf_notsup, fmd_conf_nop
225 };
226 
227 static const fmd_conf_formal_t _fmd_conf[] = {
228 { "agent.path", &fmd_conf_path, _fmd_agent_path }, /* path for agents */
229 { "alloc_msecs", &fmd_conf_uint32, "10" },	/* msecs before alloc retry */
230 { "alloc_tries", &fmd_conf_uint32, "3" },	/* max # of alloc retries */
231 { "chassis", &fmd_conf_string, _fmd_csn },	/* chassis serial number */
232 { "ckpt.dir", &fmd_conf_string, "var/fm/fmd/ckpt" }, /* ckpt directory path */
233 { "ckpt.dirmode", &fmd_conf_int32, "0700" },	/* ckpt directory perm mode */
234 { "ckpt.mode", &fmd_conf_int32, "0400" },	/* ckpt file perm mode */
235 { "ckpt.restore", &fmd_conf_bool, "true" },	/* restore checkpoints? */
236 { "ckpt.save", &fmd_conf_bool, "true" },	/* save checkpoints? */
237 { "ckpt.zero", &fmd_conf_bool, "false" },	/* zero checkpoints on start? */
238 { "client.buflim", &fmd_conf_size, "10m" },	/* client buffer space limit */
239 { "client.dbout", &fmd_dbout_ops, NULL },	/* client debug output sinks */
240 { "client.debug", &fmd_conf_bool, NULL },	/* client debug enable */
241 { "client.error", &fmd_cerror_ops, "unload" },	/* client error policy */
242 { "client.memlim", &fmd_conf_size, "10m" },	/* client allocation limit */
243 { "client.evqlim", &fmd_conf_uint32, "256" },	/* client event queue limit */
244 { "client.thrlim", &fmd_conf_uint32, "8" },	/* client aux thread limit */
245 { "client.thrsig", &fmd_conf_signal, "SIGUSR1" }, /* fmd_thr_signal() value */
246 { "client.tmrlim", &fmd_conf_uint32, "1024" },	/* client pending timer limit */
247 { "client.xprtlim", &fmd_conf_uint32, "256" },	/* client transport limit */
248 { "client.xprtlog", &fmd_conf_bool, NULL },	/* client transport logging? */
249 { "client.xprtqlim", &fmd_conf_uint32, "256" }, /* client transport queue lim */
250 { "clock", &fmd_clkmode_ops, "native" },	/* clock operation mode */
251 { "conf_path", &fmd_conf_path, _fmd_conf_path }, /* root config file path */
252 { "conf_file", &fmd_conf_string, "fmd.conf" },	/* root config file name */
253 { "core", &fmd_conf_bool, "false" },		/* force core dump on quit */
254 { "dbout", &fmd_dbout_ops, NULL },		/* daemon debug output sinks */
255 { "debug", &fmd_debug_ops, NULL },		/* daemon debugging flags */
256 { "dictdir", &fmd_conf_string, "usr/lib/fm/dict" }, /* default diagcode dir */
257 { "domain", &fmd_conf_string, NULL },		/* domain id for de auth */
258 { "fg", &fmd_conf_bool, "false" },		/* run daemon in foreground */
259 { "gc_interval", &fmd_conf_time, "1d" },	/* garbage collection intvl */
260 { "ids.avg", &fmd_conf_uint32, "4" },		/* desired idspace chain len */
261 { "ids.max", &fmd_conf_uint32, "1024" },	/* maximum idspace buckets */
262 { "isaname", &fmd_conf_string, _fmd_isa },	/* instruction set (uname -p) */
263 { "log.creator", &fmd_conf_string, "fmd" },	/* exacct log creator string */
264 { "log.error", &fmd_conf_string, "var/fm/fmd/errlog" }, /* error log path */
265 { "log.fault", &fmd_conf_string, "var/fm/fmd/fltlog" }, /* fault log path */
266 { "log.minfree", &fmd_conf_size, "2m" },	/* min log fsys free space */
267 { "log.rsrc", &fmd_conf_string, "var/fm/fmd/rsrc" }, /* asru log dir path */
268 { "log.tryrotate", &fmd_conf_uint32, "10" },	/* max log rotation attempts */
269 { "log.waitrotate", &fmd_conf_time, "200ms" },	/* log rotation retry delay */
270 { "log.xprt", &fmd_conf_string, "var/fm/fmd/xprt" }, /* transport log dir */
271 { "machine", &fmd_conf_string, _fmd_uts.machine }, /* machine name (uname -m) */
272 { "nodiagcode", &fmd_conf_string, "-" },	/* diagcode to use if error */
273 { "osrelease", &fmd_conf_string, _fmd_uts.release }, /* release (uname -r) */
274 { "osversion", &fmd_conf_string, _fmd_uts.version }, /* version (uname -v) */
275 { "platform", &fmd_conf_string, _fmd_plat },	/* platform string (uname -i) */
276 { "plugin.close", &fmd_conf_bool, "true" },	/* dlclose plugins on fini */
277 { "plugin.path", &fmd_conf_path, _fmd_plugin_path }, /* path for plugin mods */
278 { "product", &fmd_conf_string, _fmd_prod },	/* product name string */
279 { "rootdir", &fmd_conf_string, "" },		/* root directory for paths */
280 { "rpc.adm.path", &fmd_conf_string, NULL },	/* FMD_ADM rendezvous file */
281 { "rpc.adm.prog", &fmd_conf_uint32, "100169" },	/* FMD_ADM rpc program num */
282 { "rpc.api.path", &fmd_conf_string, NULL },	/* FMD_API rendezvous file */
283 { "rpc.api.prog", &fmd_conf_uint32, "100170" },	/* FMD_API rpc program num */
284 { "rpc.rcvsize", &fmd_conf_size, "128k" },	/* rpc receive buffer size */
285 { "rpc.sndsize", &fmd_conf_size, "128k" },	/* rpc send buffer size */
286 { "rsrc.age", &fmd_conf_time, "30d" },		/* max age of old rsrc log */
287 { "rsrc.zero", &fmd_conf_bool, "false" },	/* zero rsrc cache on start? */
288 { "schemedir", &fmd_conf_string, _fmd_scheme_path }, /* path for scheme mods */
289 { "self.name", &fmd_conf_string, "fmd-self-diagnosis" }, /* self-diag module */
290 { "self.dict", &fmd_conf_list, "FMD.dict" },	/* self-diag dictionary list */
291 { "server", &fmd_conf_string, _fmd_uts.nodename }, /* server id for de auth */
292 { "strbuckets", &fmd_conf_uint32, "211" },	/* size of string hashes */
293 #ifdef DEBUG
294 { "trace.mode", &fmd_trmode_ops, "full" },	/* trace mode: none/lite/full */
295 #else
296 { "trace.mode", &fmd_trmode_ops, "lite" },	/* trace mode: none/lite/full */
297 #endif
298 { "trace.recs", &fmd_conf_uint32, "128" },	/* trace records per thread */
299 { "trace.frames", &fmd_conf_uint32, "16" },	/* max trace rec stack frames */
300 { "uuidlen", &fmd_conf_uint32, "36" },		/* UUID ASCII string length */
301 { "xprt.ttl", &fmd_conf_uint8, "1" },		/* default event time-to-live */
302 };
303 
304 /*
305  * Statistics maintained by fmd itself on behalf of various global subsystems.
306  * NOTE: FMD_TYPE_STRING statistics should not be used here.  If they are
307  * required in the future, the FMD_ADM_MODGSTAT service routine must change.
308  */
309 static fmd_statistics_t _fmd_stats = {
310 { "errlog.replayed", FMD_TYPE_UINT64, "total events replayed from errlog" },
311 { "errlog.partials", FMD_TYPE_UINT64, "events partially committed in errlog" },
312 { "errlog.enospc", FMD_TYPE_UINT64, "events not appended to errlog (ENOSPC)" },
313 { "fltlog.enospc", FMD_TYPE_UINT64, "events not appended to fltlog (ENOSPC)" },
314 { "log.enospc", FMD_TYPE_UINT64, "events not appended to other logs (ENOSPC)" },
315 { "dr.gen", FMD_TYPE_UINT64, "dynamic reconfiguration generation" },
316 };
317 
318 void
319 fmd_create(fmd_t *dp, const char *arg0, const char *root, const char *conf)
320 {
321 	fmd_conf_path_t *pap;
322 	char file[PATH_MAX];
323 	const char *name;
324 	fmd_stat_t *sp;
325 	int i;
326 
327 	smbios_hdl_t *shp;
328 	smbios_system_t s1;
329 	smbios_info_t s2;
330 	id_t id;
331 
332 	di_prom_handle_t promh = DI_PROM_HANDLE_NIL;
333 	di_node_t rooth = DI_NODE_NIL;
334 	char *bufp;
335 
336 	(void) sysinfo(SI_PLATFORM, _fmd_plat, sizeof (_fmd_plat));
337 	(void) sysinfo(SI_ARCHITECTURE, _fmd_isa, sizeof (_fmd_isa));
338 	(void) uname(&_fmd_uts);
339 
340 	if ((shp = smbios_open(NULL, SMB_VERSION, 0, NULL)) != NULL) {
341 		if ((id = smbios_info_system(shp, &s1)) != SMB_ERR &&
342 		    smbios_info_common(shp, id, &s2) != SMB_ERR) {
343 			(void) strlcpy(_fmd_prod, s2.smbi_product, MAXNAMELEN);
344 			(void) strlcpy(_fmd_csn, s2.smbi_serial, MAXNAMELEN);
345 		}
346 		smbios_close(shp);
347 	} else if ((rooth = di_init("/", DINFOPROP)) != DI_NODE_NIL &&
348 	    (promh = di_prom_init()) != DI_PROM_HANDLE_NIL) {
349 		if (di_prom_prop_lookup_bytes(promh, rooth, "chassis-sn",
350 		    (unsigned char **)&bufp) != -1) {
351 			(void) strlcpy(_fmd_csn, bufp, MAXNAMELEN);
352 		}
353 	}
354 
355 	if (promh != DI_PROM_HANDLE_NIL)
356 		di_prom_fini(promh);
357 	if (rooth != DI_NODE_NIL)
358 		di_fini(rooth);
359 
360 	bzero(dp, sizeof (fmd_t));
361 
362 	dp->d_version = _fmd_version;
363 	dp->d_pname = fmd_strbasename(arg0);
364 	dp->d_pid = getpid();
365 
366 	if (pthread_key_create(&dp->d_key, NULL) != 0)
367 		fmd_error(EFMD_EXIT, "failed to create pthread key");
368 
369 	(void) pthread_mutex_init(&dp->d_xprt_lock, NULL);
370 	(void) pthread_mutex_init(&dp->d_err_lock, NULL);
371 	(void) pthread_mutex_init(&dp->d_thr_lock, NULL);
372 	(void) pthread_mutex_init(&dp->d_mod_lock, NULL);
373 	(void) pthread_mutex_init(&dp->d_stats_lock, NULL);
374 	(void) pthread_rwlock_init(&dp->d_log_lock, NULL);
375 
376 	/*
377 	 * A small number of properties must be set manually before we open
378 	 * the root configuration file.  These include any settings for our
379 	 * memory allocator and path expansion token values, because these
380 	 * values are needed by the routines in fmd_conf.c itself.  After
381 	 * the root configuration file is processed, we reset these properties
382 	 * based upon the latest values from the configuration file.
383 	 */
384 	dp->d_alloc_msecs = 10;
385 	dp->d_alloc_tries = 3;
386 	dp->d_str_buckets = 211;
387 
388 	dp->d_rootdir = root ? root : "";
389 	dp->d_platform = _fmd_plat;
390 	dp->d_machine = _fmd_uts.machine;
391 	dp->d_isaname = _fmd_isa;
392 
393 	dp->d_conf = fmd_conf_open(conf, sizeof (_fmd_conf) /
394 	    sizeof (_fmd_conf[0]), _fmd_conf, FMD_CONF_DEFER);
395 
396 	if (dp->d_conf == NULL) {
397 		fmd_error(EFMD_EXIT,
398 		    "failed to load required configuration properties\n");
399 	}
400 
401 	(void) fmd_conf_getprop(dp->d_conf, "alloc.msecs", &dp->d_alloc_msecs);
402 	(void) fmd_conf_getprop(dp->d_conf, "alloc.tries", &dp->d_alloc_tries);
403 	(void) fmd_conf_getprop(dp->d_conf, "strbuckets", &dp->d_str_buckets);
404 
405 	(void) fmd_conf_getprop(dp->d_conf, "platform", &dp->d_platform);
406 	(void) fmd_conf_getprop(dp->d_conf, "machine", &dp->d_machine);
407 	(void) fmd_conf_getprop(dp->d_conf, "isaname", &dp->d_isaname);
408 
409 	/*
410 	 * Manually specified rootdirs override config files, so only update
411 	 * d_rootdir based on the config files we parsed if no 'root' was set.
412 	 */
413 	if (root == NULL)
414 		(void) fmd_conf_getprop(dp->d_conf, "rootdir", &dp->d_rootdir);
415 	else
416 		(void) fmd_conf_setprop(dp->d_conf, "rootdir", dp->d_rootdir);
417 
418 	/*
419 	 * Once the base conf file properties are loaded, lookup the values
420 	 * of $conf_path and $conf_file and merge in any other conf files.
421 	 */
422 	(void) fmd_conf_getprop(dp->d_conf, "conf_path", &pap);
423 	(void) fmd_conf_getprop(dp->d_conf, "conf_file", &name);
424 
425 	for (i = 0; i < pap->cpa_argc; i++) {
426 		(void) snprintf(file, sizeof (file),
427 		    "%s/%s", pap->cpa_argv[i], name);
428 		if (access(file, F_OK) == 0)
429 			fmd_conf_merge(dp->d_conf, file);
430 	}
431 
432 	/*
433 	 * Update the value of fmd.d_fg based on "fg".  We cache this property
434 	 * because it must be accessed deep within fmd at fmd_verror() time.
435 	 * Update any other properties that must be cached for performance.
436 	 */
437 	(void) fmd_conf_getprop(fmd.d_conf, "fg", &fmd.d_fg);
438 	(void) fmd_conf_getprop(fmd.d_conf, "xprt.ttl", &fmd.d_xprt_ttl);
439 
440 	/*
441 	 * Initialize our custom libnvpair allocator and create an nvlist for
442 	 * authority elements corresponding to this instance of the daemon.
443 	 */
444 	(void) nv_alloc_init(&dp->d_nva, &fmd_nv_alloc_ops);
445 	dp->d_auth = fmd_protocol_authority();
446 
447 	/*
448 	 * The fmd_module_t for the root module must be created manually.  Most
449 	 * of it remains unused and zero, except for the few things we fill in.
450 	 */
451 	dp->d_rmod = fmd_zalloc(sizeof (fmd_module_t), FMD_SLEEP);
452 	dp->d_rmod->mod_name = fmd_strdup(dp->d_pname, FMD_SLEEP);
453 	dp->d_rmod->mod_fmri = fmd_protocol_fmri_module(dp->d_rmod);
454 
455 	fmd_list_append(&dp->d_mod_list, dp->d_rmod);
456 	fmd_module_hold(dp->d_rmod);
457 
458 	(void) pthread_mutex_init(&dp->d_rmod->mod_lock, NULL);
459 	(void) pthread_cond_init(&dp->d_rmod->mod_cv, NULL);
460 	(void) pthread_mutex_init(&dp->d_rmod->mod_stats_lock, NULL);
461 
462 	dp->d_rmod->mod_thread = fmd_thread_xcreate(dp->d_rmod, pthread_self());
463 	dp->d_rmod->mod_stats = fmd_zalloc(sizeof (fmd_modstat_t), FMD_SLEEP);
464 	dp->d_rmod->mod_ustat = fmd_ustat_create();
465 
466 	if (pthread_setspecific(dp->d_key, dp->d_rmod->mod_thread) != 0)
467 		fmd_error(EFMD_EXIT, "failed to attach main thread key");
468 
469 	if ((dp->d_stats = (fmd_statistics_t *)fmd_ustat_insert(
470 	    dp->d_rmod->mod_ustat, FMD_USTAT_NOALLOC, sizeof (_fmd_stats) /
471 	    sizeof (fmd_stat_t), (fmd_stat_t *)&_fmd_stats, NULL)) == NULL)
472 		fmd_error(EFMD_EXIT, "failed to initialize statistics");
473 
474 	(void) pthread_mutex_lock(&dp->d_rmod->mod_lock);
475 	dp->d_rmod->mod_flags |= FMD_MOD_INIT;
476 	(void) pthread_mutex_unlock(&dp->d_rmod->mod_lock);
477 
478 	/*
479 	 * In addition to inserting the _fmd_stats collection of program-wide
480 	 * statistics, we also insert a statistic named after each of our
481 	 * errors and update these counts in fmd_verror() (see fmd_subr.c).
482 	 */
483 	dp->d_errstats = sp = fmd_zalloc(sizeof (fmd_stat_t) *
484 	    (EFMD_END - EFMD_UNKNOWN), FMD_SLEEP);
485 
486 	for (i = 0; i < EFMD_END - EFMD_UNKNOWN; i++, sp++) {
487 		(void) snprintf(sp->fmds_name, sizeof (sp->fmds_name), "err.%s",
488 		    strrchr(fmd_errclass(EFMD_UNKNOWN + i), '.') + 1);
489 		sp->fmds_type = FMD_TYPE_UINT64;
490 	}
491 
492 	(void) fmd_ustat_insert(dp->d_rmod->mod_ustat, FMD_USTAT_NOALLOC,
493 	    EFMD_END - EFMD_UNKNOWN, dp->d_errstats, NULL);
494 }
495 
496 void
497 fmd_destroy(fmd_t *dp)
498 {
499 	fmd_module_t *mp;
500 	fmd_case_t *cp;
501 	int core;
502 
503 	(void) fmd_conf_getprop(fmd.d_conf, "core", &core);
504 
505 	fmd_rpc_fini();
506 	fmd_dr_fini();
507 
508 	if (dp->d_xprt_ids != NULL)
509 		fmd_xprt_suspend_all();
510 
511 	/*
512 	 * Unload the self-diagnosis module first.  This ensures that it does
513 	 * not get confused as we start unloading other modules, etc.  We must
514 	 * hold the dispq lock as a writer while doing so since it uses d_self.
515 	 */
516 	if (dp->d_self != NULL) {
517 		fmd_module_t *self;
518 
519 		(void) pthread_rwlock_wrlock(&dp->d_disp->dq_lock);
520 		self = dp->d_self;
521 		dp->d_self = NULL;
522 		(void) pthread_rwlock_unlock(&dp->d_disp->dq_lock);
523 
524 		fmd_module_unload(self);
525 		fmd_module_rele(self);
526 	}
527 
528 	/*
529 	 * Unload modules in reverse order *except* for the root module, which
530 	 * is first in the list.  This allows it to keep its thread and trace.
531 	 */
532 	for (mp = fmd_list_prev(&dp->d_mod_list); mp != dp->d_rmod; ) {
533 		fmd_module_unload(mp);
534 		mp = fmd_list_prev(mp);
535 	}
536 
537 	if (dp->d_mod_hash != NULL) {
538 		fmd_modhash_destroy(dp->d_mod_hash);
539 		dp->d_mod_hash = NULL;
540 	}
541 
542 	/*
543 	 * Close both log files now that modules are no longer active.  We must
544 	 * set these pointers to NULL in case any subsequent errors occur.
545 	 */
546 	if (dp->d_errlog != NULL) {
547 		fmd_log_rele(dp->d_errlog);
548 		dp->d_errlog = NULL;
549 	}
550 
551 	if (dp->d_fltlog != NULL) {
552 		fmd_log_rele(dp->d_fltlog);
553 		dp->d_fltlog = NULL;
554 	}
555 
556 	/*
557 	 * Now destroy the resource cache: each ASRU contains a case reference,
558 	 * which may in turn contain a pointer to a referenced owning module.
559 	 */
560 	if (dp->d_asrus != NULL) {
561 		fmd_asru_hash_destroy(dp->d_asrus);
562 		dp->d_asrus = NULL;
563 	}
564 
565 	/*
566 	 * Now that all data structures that refer to modules are torn down,
567 	 * no modules should be remaining on the module list except for d_rmod.
568 	 * If we trip one of these assertions, we're missing a rele somewhere.
569 	 */
570 	ASSERT(fmd_list_prev(&dp->d_mod_list) == dp->d_rmod);
571 	ASSERT(fmd_list_next(&dp->d_mod_list) == dp->d_rmod);
572 
573 	/*
574 	 * Now destroy the root module.  We clear its thread key first so any
575 	 * calls to fmd_trace() inside of the module code will be ignored.
576 	 */
577 	(void) pthread_setspecific(dp->d_key, NULL);
578 	fmd_module_lock(dp->d_rmod);
579 
580 	while ((cp = fmd_list_next(&dp->d_rmod->mod_cases)) != NULL)
581 		fmd_case_discard(cp);
582 
583 	fmd_module_unlock(dp->d_rmod);
584 	fmd_free(dp->d_rmod->mod_stats, sizeof (fmd_modstat_t));
585 	dp->d_rmod->mod_stats = NULL;
586 
587 	(void) pthread_mutex_lock(&dp->d_rmod->mod_lock);
588 	dp->d_rmod->mod_flags |= FMD_MOD_FINI;
589 	(void) pthread_mutex_unlock(&dp->d_rmod->mod_lock);
590 
591 	fmd_module_rele(dp->d_rmod);
592 	ASSERT(fmd_list_next(&dp->d_mod_list) == NULL);
593 
594 	/*
595 	 * Now destroy the remaining global data structures.  If 'core' was
596 	 * set to true, force a core dump so we can check for memory leaks.
597 	 */
598 	if (dp->d_cases != NULL)
599 		fmd_case_hash_destroy(dp->d_cases);
600 	if (dp->d_disp != NULL)
601 		fmd_dispq_destroy(dp->d_disp);
602 	if (dp->d_timers != NULL)
603 		fmd_timerq_destroy(dp->d_timers);
604 	if (dp->d_schemes != NULL)
605 		fmd_scheme_hash_destroy(dp->d_schemes);
606 	if (dp->d_xprt_ids != NULL)
607 		fmd_idspace_destroy(dp->d_xprt_ids);
608 
609 	if (dp->d_errstats != NULL) {
610 		fmd_free(dp->d_errstats,
611 		    sizeof (fmd_stat_t) * (EFMD_END - EFMD_UNKNOWN));
612 	}
613 
614 	if (dp->d_conf != NULL)
615 		fmd_conf_close(dp->d_conf);
616 
617 	if (dp->d_topo != NULL)
618 		topo_close(dp->d_topo);
619 
620 	nvlist_free(dp->d_auth);
621 	(void) nv_alloc_fini(&dp->d_nva);
622 	dp->d_clockops->fto_fini(dp->d_clockptr);
623 
624 	(void) pthread_key_delete(dp->d_key);
625 	bzero(dp, sizeof (fmd_t));
626 
627 	if (core)
628 		fmd_panic("forcing core dump at user request\n");
629 }
630 
631 /*ARGSUSED*/
632 static void
633 fmd_gc(fmd_t *dp, id_t id, hrtime_t hrt)
634 {
635 	hrtime_t delta;
636 
637 	if (id != 0) {
638 		TRACE((FMD_DBG_MOD, "garbage collect start"));
639 		fmd_modhash_apply(dp->d_mod_hash, fmd_module_gc);
640 		TRACE((FMD_DBG_MOD, "garbage collect end"));
641 
642 		(void) pthread_rwlock_rdlock(&dp->d_log_lock);
643 		fmd_log_update(dp->d_errlog);
644 		(void) pthread_rwlock_unlock(&dp->d_log_lock);
645 	}
646 
647 	(void) fmd_conf_getprop(dp->d_conf, "gc_interval", &delta);
648 	(void) fmd_timerq_install(dp->d_timers, dp->d_rmod->mod_timerids,
649 	    (fmd_timer_f *)fmd_gc, dp, NULL, delta);
650 }
651 
652 /*
653  * Events are committed to the errlog after cases are checkpointed.  If fmd
654  * crashes before an event is ever associated with a module, this function will
655  * be called to replay it to all subscribers.  If fmd crashes in between the
656  * subscriber checkpointing and committing the event in the error log, the
657  * module will have seen the event and we don't want to replay it.  So we look
658  * for the event in all modules and transition it to the proper state.  If
659  * it is found, we commit it to the error log and do not replay it.  The in-
660  * memory case search used by fmd_module_contains() et al isn't particularly
661  * efficient, but it is faster than doing read i/o's on every case event to
662  * check their status or write i/o's on every event to replay to update states.
663  * We can improve the efficiency of this lookup algorithm later if necessary.
664  */
665 /*ARGSUSED*/
666 static void
667 fmd_err_replay(fmd_log_t *lp, fmd_event_t *ep, fmd_t *dp)
668 {
669 	fmd_module_t *mp;
670 	fmd_stat_t *sp;
671 
672 	(void) pthread_mutex_lock(&dp->d_mod_lock);
673 
674 	for (mp = fmd_list_next(&dp->d_mod_list);
675 	    mp != NULL; mp = fmd_list_next(mp)) {
676 		if (fmd_module_contains(mp, ep)) {
677 			fmd_module_hold(mp);
678 			break;
679 		}
680 	}
681 
682 	(void) pthread_mutex_unlock(&dp->d_mod_lock);
683 
684 	if (mp != NULL) {
685 		fmd_event_commit(ep);
686 		fmd_module_rele(mp);
687 		sp = &dp->d_stats->ds_log_partials;
688 	} else {
689 		fmd_dispq_dispatch(dp->d_disp, ep, FMD_EVENT_DATA(ep));
690 		sp = &dp->d_stats->ds_log_replayed;
691 	}
692 
693 	(void) pthread_mutex_lock(&dp->d_stats_lock);
694 	sp->fmds_value.ui64++;
695 	(void) pthread_mutex_unlock(&dp->d_stats_lock);
696 }
697 
698 void
699 fmd_door_server(void *dip)
700 {
701 	fmd_dprintf(FMD_DBG_XPRT, "door server starting for %p\n", dip);
702 	(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
703 	(void) door_return(NULL, 0, NULL, 0);
704 }
705 
706 /*
707  * Custom door server create callback.  Any fmd services that use doors will
708  * require those threads to have their fmd-specific TSD initialized, etc.
709  */
710 static void
711 fmd_door(door_info_t *dip)
712 {
713 	if (fmd_thread_create(fmd.d_rmod, fmd_door_server, dip) == NULL)
714 		fmd_panic("failed to create server for door %p", (void *)dip);
715 }
716 
717 /*
718  * This signal handler is installed for the client.thrsig signal to be used to
719  * force an auxiliary thread to wake up from a system call and return EINTR in
720  * response to a module's use of fmd_thr_signal().  We also trace the event.
721  */
722 static void
723 fmd_signal(int sig)
724 {
725 	TRACE((FMD_DBG_MOD, "module thread received sig #%d", sig));
726 }
727 
728 void
729 fmd_run(fmd_t *dp, int pfd)
730 {
731 	char *nodc_key[] = { FMD_FLT_NODC, NULL };
732 	char nodc_str[128];
733 	struct sigaction act;
734 
735 	int status = FMD_EXIT_SUCCESS;
736 	const char *name;
737 	fmd_conf_path_t *pap;
738 	fmd_event_t *e;
739 	int dbout, err;
740 
741 	/*
742 	 * Cache all the current debug property settings in d_fmd_debug,
743 	 * d_fmd_dbout, d_hdl_debug, and d_hdl_dbout.  If a given debug mask
744 	 * is non-zero and the corresponding dbout mask is zero, set dbout
745 	 * to a sensible default value based on whether we have daemonized.
746 	 */
747 	(void) fmd_conf_getprop(dp->d_conf, "dbout", &dbout);
748 
749 	if (dp->d_fmd_debug != 0 && dbout == 0)
750 		dp->d_fmd_dbout = dp->d_fg? FMD_DBOUT_STDERR : FMD_DBOUT_SYSLOG;
751 	else
752 		dp->d_fmd_dbout = dbout;
753 
754 	(void) fmd_conf_getprop(dp->d_conf, "client.debug", &dp->d_hdl_debug);
755 	(void) fmd_conf_getprop(dp->d_conf, "client.dbout", &dbout);
756 
757 	if (dp->d_hdl_debug != 0 && dbout == 0)
758 		dp->d_hdl_dbout = dp->d_fg? FMD_DBOUT_STDERR : FMD_DBOUT_SYSLOG;
759 	else
760 		dp->d_hdl_dbout = dbout;
761 
762 	/*
763 	 * Initialize remaining major program data structures such as the
764 	 * clock, dispatch queues, log files, module hash collections, etc.
765 	 * This work is done here rather than in fmd_create() to permit the -o
766 	 * command-line option to modify properties after fmd_create() is done.
767 	 */
768 	name = dp->d_rootdir != NULL &&
769 	    *dp->d_rootdir != '\0' ? dp->d_rootdir : NULL;
770 
771 	if ((dp->d_topo = topo_open(TOPO_VERSION, name, &err)) == NULL) {
772 		fmd_error(EFMD_EXIT, "failed to initialize "
773 		    "topology library: %s\n", topo_strerror(err));
774 	}
775 
776 	dp->d_clockptr = dp->d_clockops->fto_init();
777 	dp->d_xprt_ids = fmd_idspace_create("xprt_ids", 1, INT_MAX);
778 	fmd_xprt_suspend_all();
779 
780 	(void) door_server_create(fmd_door);
781 	fmd_dr_init();
782 
783 	dp->d_rmod->mod_timerids = fmd_idspace_create(dp->d_pname, 1, 16);
784 	dp->d_timers = fmd_timerq_create();
785 	dp->d_disp = fmd_dispq_create();
786 	dp->d_cases = fmd_case_hash_create();
787 
788 	/*
789 	 * The root module's mod_queue is created with limit zero, making it
790 	 * act like /dev/null; anything inserted here is simply ignored.
791 	 */
792 	dp->d_rmod->mod_queue = fmd_eventq_create(dp->d_rmod,
793 	    &dp->d_rmod->mod_stats->ms_evqstat, &dp->d_rmod->mod_stats_lock, 0);
794 
795 	/*
796 	 * Once our subsystems that use signals have been set up, install the
797 	 * signal handler for the fmd_thr_signal() API.  Verify that the signal
798 	 * being used for this purpose doesn't conflict with something else.
799 	 */
800 	(void) fmd_conf_getprop(dp->d_conf, "client.thrsig", &dp->d_thr_sig);
801 
802 	if (sigaction(dp->d_thr_sig, NULL, &act) != 0) {
803 		fmd_error(EFMD_EXIT, "invalid signal selected for "
804 		    "client.thrsig property: %d\n", dp->d_thr_sig);
805 	}
806 
807 	if (act.sa_handler != SIG_IGN && act.sa_handler != SIG_DFL) {
808 		fmd_error(EFMD_EXIT, "signal selected for client.thrsig "
809 		    "property is already in use: %d\n", dp->d_thr_sig);
810 	}
811 
812 	act.sa_handler = fmd_signal;
813 	act.sa_flags = 0;
814 
815 	(void) sigemptyset(&act.sa_mask);
816 	(void) sigaction(dp->d_thr_sig, &act, NULL);
817 
818 	(void) fmd_conf_getprop(dp->d_conf, "schemedir", &name);
819 	dp->d_schemes = fmd_scheme_hash_create(dp->d_rootdir, name);
820 
821 	(void) fmd_conf_getprop(dp->d_conf, "log.rsrc", &name);
822 	dp->d_asrus = fmd_asru_hash_create(dp->d_rootdir, name);
823 
824 	(void) fmd_conf_getprop(dp->d_conf, "log.error", &name);
825 	dp->d_errlog = fmd_log_open(dp->d_rootdir, name, FMD_LOG_ERROR);
826 
827 	(void) fmd_conf_getprop(dp->d_conf, "log.fault", &name);
828 	dp->d_fltlog = fmd_log_open(dp->d_rootdir, name, FMD_LOG_FAULT);
829 
830 	if (dp->d_asrus == NULL || dp->d_errlog == NULL || dp->d_fltlog == NULL)
831 		fmd_error(EFMD_EXIT, "failed to initialize log files\n");
832 
833 	/*
834 	 * Before loading modules, create an empty control event which will act
835 	 * as a global barrier for module event processing.  Each module we
836 	 * load successfully will insert it at their head of their event queue,
837 	 * and then pause inside of fmd_ctl_rele() after dequeuing the event.
838 	 * This module barrier is required for two reasons:
839 	 *
840 	 * (a) During module loading, the restoration of case checkpoints may
841 	 *    result in a list.* event being recreated for which the intended
842 	 *    subscriber has not yet loaded depending on the load order. Such
843 	 *    events could then result in spurious "no subscriber" errors.
844 	 *
845 	 * (b) During errlog replay, a sequence of errors from a long time ago
846 	 *    may be replayed, and the module may attempt to install relative
847 	 *    timers associated with one or more of these events.  If errlog
848 	 *    replay were "racing" with active module threads, an event E1
849 	 *    that resulted in a relative timer T at time E1 + N nsec could
850 	 *    fire prior to an event E2 being enqueued, even if the relative
851 	 *    time ordering was E1 < E2 < E1 + N, causing mis-diagnosis.
852 	 */
853 	dp->d_mod_event = e = fmd_event_create(FMD_EVT_CTL,
854 	    FMD_HRT_NOW, NULL, fmd_ctl_init(NULL));
855 
856 	fmd_event_hold(e);
857 
858 	/*
859 	 * Once all data structures are initialized, we load all of our modules
860 	 * in order according to class in order to load up any subscriptions.
861 	 * Once built-in modules are loaded, we detach from our waiting parent.
862 	 */
863 	dp->d_mod_hash = fmd_modhash_create();
864 
865 	if (fmd_builtin_loadall(dp->d_mod_hash) != 0 && !dp->d_fg)
866 		fmd_error(EFMD_EXIT, "failed to initialize fault manager\n");
867 
868 	(void) fmd_conf_getprop(dp->d_conf, "self.name", &name);
869 	dp->d_self = fmd_modhash_lookup(dp->d_mod_hash, name);
870 
871 	if (dp->d_self != NULL && fmd_module_dc_key2code(dp->d_self,
872 	    nodc_key, nodc_str, sizeof (nodc_str)) == 0)
873 		(void) fmd_conf_setprop(dp->d_conf, "nodiagcode", nodc_str);
874 
875 	fmd_rpc_init();
876 	dp->d_running = 1; /* we are now officially an active fmd */
877 
878 	/*
879 	 * Now that we're running, if a pipe fd was specified, write an exit
880 	 * status to it to indicate that our parent process can safely detach.
881 	 * Then proceed to loading the remaining non-built-in modules.
882 	 */
883 	if (pfd >= 0)
884 		(void) write(pfd, &status, sizeof (status));
885 
886 	/*
887 	 * Before loading all modules, repopulate the ASRU cache from its
888 	 * persistent repository on disk.  Then during module loading, the
889 	 * restoration of checkpoint files will reparent any active cases.
890 	 */
891 	fmd_asru_hash_refresh(dp->d_asrus);
892 
893 	(void) fmd_conf_getprop(dp->d_conf, "plugin.path", &pap);
894 	fmd_modhash_loadall(dp->d_mod_hash, pap, &fmd_rtld_ops, ".so");
895 
896 	(void) fmd_conf_getprop(dp->d_conf, "agent.path", &pap);
897 	fmd_modhash_loadall(dp->d_mod_hash, pap, &fmd_proc_ops, NULL);
898 
899 	/*
900 	 * With all modules loaded, replay fault events from the ASRU cache for
901 	 * any ASRUs that must be retired, replay error events from the errlog
902 	 * that did not finish processing the last time ran, and then release
903 	 * the global module barrier by executing a final rele on d_mod_event.
904 	 */
905 	fmd_asru_hash_replay(dp->d_asrus);
906 
907 	(void) pthread_rwlock_rdlock(&dp->d_log_lock);
908 	fmd_log_replay(dp->d_errlog, (fmd_log_f *)fmd_err_replay, dp);
909 	fmd_log_update(dp->d_errlog);
910 	(void) pthread_rwlock_unlock(&dp->d_log_lock);
911 
912 	dp->d_mod_event = NULL;
913 	fmd_event_rele(e);
914 
915 	/*
916 	 * Finally, awaken any threads associated with receiving events from
917 	 * open transports and tell them to proceed with fmd_xprt_recv().
918 	 */
919 	fmd_xprt_resume_all();
920 	fmd_gc(dp, 0, 0);
921 
922 	dp->d_booted = 1;
923 }
924 
925 void
926 fmd_help(fmd_t *dp)
927 {
928 	const fmd_conf_mode_t *cmp;
929 
930 	(void) printf("Usage: %s -o debug=mode[,mode]\n", dp->d_pname);
931 
932 	for (cmp = _fmd_debug_modes; cmp->cm_name != NULL; cmp++)
933 		(void) printf("\t%s\t%s\n", cmp->cm_name, cmp->cm_desc);
934 }
935