xref: /freebsd/sys/contrib/openzfs/cmd/zed/agents/fmd_api.c (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23  *
24  * Copyright (c) 2016, Intel Corporation.
25  */
26 
27 /*
28  * This file implements the minimal FMD module API required to support the
29  * fault logic modules in ZED. This support includes module registration,
30  * memory allocation, module property accessors, basic case management,
31  * one-shot timers and SERD engines.
32  *
33  * In the ZED runtime, the modules are called from a single thread so no
34  * locking is required in this emulated FMD environment.
35  */
36 
37 #include <sys/types.h>
38 #include <sys/fm/protocol.h>
39 #include <uuid/uuid.h>
40 #include <signal.h>
41 #include <strings.h>
42 #include <time.h>
43 
44 #include "fmd_api.h"
45 #include "fmd_serd.h"
46 
47 #include "zfs_agents.h"
48 #include "../zed_log.h"
49 
50 typedef struct fmd_modstat {
51 	fmd_stat_t	ms_accepted;	/* total events accepted by module */
52 	fmd_stat_t	ms_caseopen;	/* cases currently open */
53 	fmd_stat_t	ms_casesolved;	/* total cases solved by module */
54 	fmd_stat_t	ms_caseclosed;	/* total cases closed by module */
55 } fmd_modstat_t;
56 
57 typedef struct fmd_module {
58 	const char	*mod_name;	/* basename of module (ro) */
59 	const fmd_hdl_info_t *mod_info;	/* module info registered with handle */
60 	void		*mod_spec;	/* fmd_hdl_get/setspecific data value */
61 	fmd_stat_t	*mod_ustat;	/* module specific custom stats */
62 	uint_t		mod_ustat_cnt;	/* count of ustat stats */
63 	fmd_modstat_t	mod_stats;	/* fmd built-in per-module statistics */
64 	fmd_serd_hash_t	mod_serds;	/* hash of serd engs owned by module */
65 	char		*mod_vers;	/* a copy of module version string */
66 } fmd_module_t;
67 
68 /*
69  * ZED has two FMD hardwired module instances
70  */
71 fmd_module_t	zfs_retire_module;
72 fmd_module_t	zfs_diagnosis_module;
73 
74 /*
75  * Enable a reasonable set of defaults for libumem debugging on DEBUG builds.
76  */
77 
78 #ifdef DEBUG
79 const char *
80 _umem_debug_init(void)
81 {
82 	return ("default,verbose"); /* $UMEM_DEBUG setting */
83 }
84 
85 const char *
86 _umem_logging_init(void)
87 {
88 	return ("fail,contents"); /* $UMEM_LOGGING setting */
89 }
90 #endif
91 
92 /*
93  * Register a module with fmd and finish module initialization.
94  * Returns an integer indicating whether it succeeded (zero) or
95  * failed (non-zero).
96  */
97 int
98 fmd_hdl_register(fmd_hdl_t *hdl, int version, const fmd_hdl_info_t *mip)
99 {
100 	fmd_module_t *mp = (fmd_module_t *)hdl;
101 
102 	mp->mod_info = mip;
103 	mp->mod_name = mip->fmdi_desc + 4;	/* drop 'ZFS ' prefix */
104 	mp->mod_spec = NULL;
105 
106 	/* bare minimum module stats */
107 	(void) strcpy(mp->mod_stats.ms_accepted.fmds_name, "fmd.accepted");
108 	(void) strcpy(mp->mod_stats.ms_caseopen.fmds_name, "fmd.caseopen");
109 	(void) strcpy(mp->mod_stats.ms_casesolved.fmds_name, "fmd.casesolved");
110 	(void) strcpy(mp->mod_stats.ms_caseclosed.fmds_name, "fmd.caseclosed");
111 
112 	fmd_serd_hash_create(&mp->mod_serds);
113 
114 	fmd_hdl_debug(hdl, "register module");
115 
116 	return (0);
117 }
118 
119 void
120 fmd_hdl_unregister(fmd_hdl_t *hdl)
121 {
122 	fmd_module_t *mp = (fmd_module_t *)hdl;
123 	fmd_modstat_t *msp = &mp->mod_stats;
124 	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
125 
126 	/* dump generic module stats */
127 	fmd_hdl_debug(hdl, "%s: %llu", msp->ms_accepted.fmds_name,
128 	    msp->ms_accepted.fmds_value.ui64);
129 	if (ops->fmdo_close != NULL) {
130 		fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseopen.fmds_name,
131 		    msp->ms_caseopen.fmds_value.ui64);
132 		fmd_hdl_debug(hdl, "%s: %llu", msp->ms_casesolved.fmds_name,
133 		    msp->ms_casesolved.fmds_value.ui64);
134 		fmd_hdl_debug(hdl, "%s: %llu", msp->ms_caseclosed.fmds_name,
135 		    msp->ms_caseclosed.fmds_value.ui64);
136 	}
137 
138 	/* dump module specific stats */
139 	if (mp->mod_ustat != NULL) {
140 		int i;
141 
142 		for (i = 0; i < mp->mod_ustat_cnt; i++) {
143 			fmd_hdl_debug(hdl, "%s: %llu",
144 			    mp->mod_ustat[i].fmds_name,
145 			    mp->mod_ustat[i].fmds_value.ui64);
146 		}
147 	}
148 
149 	fmd_serd_hash_destroy(&mp->mod_serds);
150 
151 	fmd_hdl_debug(hdl, "unregister module");
152 }
153 
154 /*
155  * fmd_hdl_setspecific() is used to associate a data pointer with
156  * the specified handle for the duration of the module's lifetime.
157  * This pointer can be retrieved using fmd_hdl_getspecific().
158  */
159 void
160 fmd_hdl_setspecific(fmd_hdl_t *hdl, void *spec)
161 {
162 	fmd_module_t *mp = (fmd_module_t *)hdl;
163 
164 	mp->mod_spec = spec;
165 }
166 
167 /*
168  * Return the module-specific data pointer previously associated
169  * with the handle using fmd_hdl_setspecific().
170  */
171 void *
172 fmd_hdl_getspecific(fmd_hdl_t *hdl)
173 {
174 	fmd_module_t *mp = (fmd_module_t *)hdl;
175 
176 	return (mp->mod_spec);
177 }
178 
179 void *
180 fmd_hdl_alloc(fmd_hdl_t *hdl, size_t size, int flags)
181 {
182 	return (umem_alloc(size, flags));
183 }
184 
185 void *
186 fmd_hdl_zalloc(fmd_hdl_t *hdl, size_t size, int flags)
187 {
188 	return (umem_zalloc(size, flags));
189 }
190 
191 void
192 fmd_hdl_free(fmd_hdl_t *hdl, void *data, size_t size)
193 {
194 	umem_free(data, size);
195 }
196 
197 /*
198  * Record a module debug message using the specified format.
199  */
200 void
201 fmd_hdl_debug(fmd_hdl_t *hdl, const char *format, ...)
202 {
203 	char message[256];
204 	va_list vargs;
205 	fmd_module_t *mp = (fmd_module_t *)hdl;
206 
207 	va_start(vargs, format);
208 	(void) vsnprintf(message, sizeof (message), format, vargs);
209 	va_end(vargs);
210 
211 	/* prefix message with module name */
212 	zed_log_msg(LOG_INFO, "%s: %s", mp->mod_name, message);
213 }
214 
215 /* Property Retrieval */
216 
217 int32_t
218 fmd_prop_get_int32(fmd_hdl_t *hdl, const char *name)
219 {
220 	/*
221 	 * These can be looked up in mp->modinfo->fmdi_props
222 	 * For now we just hard code for phase 2. In the
223 	 * future, there can be a ZED based override.
224 	 */
225 	if (strcmp(name, "spare_on_remove") == 0)
226 		return (1);
227 
228 	if (strcmp(name, "io_N") == 0 || strcmp(name, "checksum_N") == 0)
229 		return (10);	/* N = 10 events */
230 
231 	return (0);
232 }
233 
234 int64_t
235 fmd_prop_get_int64(fmd_hdl_t *hdl, const char *name)
236 {
237 	/*
238 	 * These can be looked up in mp->modinfo->fmdi_props
239 	 * For now we just hard code for phase 2. In the
240 	 * future, there can be a ZED based override.
241 	 */
242 	if (strcmp(name, "remove_timeout") == 0)
243 		return (15ULL * 1000ULL * 1000ULL * 1000ULL);	/* 15 sec */
244 
245 	if (strcmp(name, "io_T") == 0 || strcmp(name, "checksum_T") == 0)
246 		return (1000ULL * 1000ULL * 1000ULL * 600ULL);	/* 10 min */
247 
248 	return (0);
249 }
250 
251 /* FMD Statistics */
252 
253 fmd_stat_t *
254 fmd_stat_create(fmd_hdl_t *hdl, uint_t flags, uint_t nstats, fmd_stat_t *statv)
255 {
256 	fmd_module_t *mp = (fmd_module_t *)hdl;
257 
258 	if (flags == FMD_STAT_NOALLOC) {
259 		mp->mod_ustat = statv;
260 		mp->mod_ustat_cnt = nstats;
261 	}
262 
263 	return (statv);
264 }
265 
266 /* Case Management */
267 
268 fmd_case_t *
269 fmd_case_open(fmd_hdl_t *hdl, void *data)
270 {
271 	fmd_module_t *mp = (fmd_module_t *)hdl;
272 	uuid_t uuid;
273 
274 	fmd_case_t *cp;
275 
276 	cp = fmd_hdl_zalloc(hdl, sizeof (fmd_case_t), FMD_SLEEP);
277 	cp->ci_mod = hdl;
278 	cp->ci_state = FMD_CASE_UNSOLVED;
279 	cp->ci_flags = FMD_CF_DIRTY;
280 	cp->ci_data = data;
281 	cp->ci_bufptr = NULL;
282 	cp->ci_bufsiz = 0;
283 
284 	uuid_generate(uuid);
285 	uuid_unparse(uuid, cp->ci_uuid);
286 
287 	fmd_hdl_debug(hdl, "case opened (%s)", cp->ci_uuid);
288 	mp->mod_stats.ms_caseopen.fmds_value.ui64++;
289 
290 	return (cp);
291 }
292 
293 void
294 fmd_case_solve(fmd_hdl_t *hdl, fmd_case_t *cp)
295 {
296 	fmd_module_t *mp = (fmd_module_t *)hdl;
297 
298 	/*
299 	 * For ZED, the event was already sent from fmd_case_add_suspect()
300 	 */
301 
302 	if (cp->ci_state >= FMD_CASE_SOLVED)
303 		fmd_hdl_debug(hdl, "case is already solved or closed");
304 
305 	cp->ci_state = FMD_CASE_SOLVED;
306 
307 	fmd_hdl_debug(hdl, "case solved (%s)", cp->ci_uuid);
308 	mp->mod_stats.ms_casesolved.fmds_value.ui64++;
309 }
310 
311 void
312 fmd_case_close(fmd_hdl_t *hdl, fmd_case_t *cp)
313 {
314 	fmd_module_t *mp = (fmd_module_t *)hdl;
315 	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
316 
317 	fmd_hdl_debug(hdl, "case closed (%s)", cp->ci_uuid);
318 
319 	if (ops->fmdo_close != NULL)
320 		ops->fmdo_close(hdl, cp);
321 
322 	mp->mod_stats.ms_caseopen.fmds_value.ui64--;
323 	mp->mod_stats.ms_caseclosed.fmds_value.ui64++;
324 
325 	if (cp->ci_bufptr != NULL && cp->ci_bufsiz > 0)
326 		fmd_hdl_free(hdl, cp->ci_bufptr, cp->ci_bufsiz);
327 
328 	fmd_hdl_free(hdl, cp, sizeof (fmd_case_t));
329 }
330 
331 void
332 fmd_case_uuresolved(fmd_hdl_t *hdl, const char *uuid)
333 {
334 	fmd_hdl_debug(hdl, "case resolved by uuid (%s)", uuid);
335 }
336 
337 int
338 fmd_case_solved(fmd_hdl_t *hdl, fmd_case_t *cp)
339 {
340 	return ((cp->ci_state >= FMD_CASE_SOLVED) ? FMD_B_TRUE : FMD_B_FALSE);
341 }
342 
343 void
344 fmd_case_add_ereport(fmd_hdl_t *hdl, fmd_case_t *cp, fmd_event_t *ep)
345 {
346 }
347 
348 static void
349 zed_log_fault(nvlist_t *nvl, const char *uuid, const char *code)
350 {
351 	nvlist_t *rsrc;
352 	char *strval;
353 	uint64_t guid;
354 	uint8_t byte;
355 
356 	zed_log_msg(LOG_INFO, "\nzed_fault_event:");
357 
358 	if (uuid != NULL)
359 		zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_UUID, uuid);
360 	if (nvlist_lookup_string(nvl, FM_CLASS, &strval) == 0)
361 		zed_log_msg(LOG_INFO, "\t%s: %s", FM_CLASS, strval);
362 	if (code != NULL)
363 		zed_log_msg(LOG_INFO, "\t%s: %s", FM_SUSPECT_DIAG_CODE, code);
364 	if (nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &byte) == 0)
365 		zed_log_msg(LOG_INFO, "\t%s: %llu", FM_FAULT_CERTAINTY, byte);
366 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
367 		if (nvlist_lookup_string(rsrc, FM_FMRI_SCHEME, &strval) == 0)
368 			zed_log_msg(LOG_INFO, "\t%s: %s", FM_FMRI_SCHEME,
369 			    strval);
370 		if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_POOL, &guid) == 0)
371 			zed_log_msg(LOG_INFO, "\t%s: %llu", FM_FMRI_ZFS_POOL,
372 			    guid);
373 		if (nvlist_lookup_uint64(rsrc, FM_FMRI_ZFS_VDEV, &guid) == 0)
374 			zed_log_msg(LOG_INFO, "\t%s: %llu \n", FM_FMRI_ZFS_VDEV,
375 			    guid);
376 	}
377 }
378 
379 static const char *
380 fmd_fault_mkcode(nvlist_t *fault)
381 {
382 	char *class, *code = "-";
383 
384 	/*
385 	 * Note: message codes come from: openzfs/usr/src/cmd/fm/dicts/ZFS.po
386 	 */
387 	if (nvlist_lookup_string(fault, FM_CLASS, &class) == 0) {
388 		if (strcmp(class, "fault.fs.zfs.vdev.io") == 0)
389 			code = "ZFS-8000-FD";
390 		else if (strcmp(class, "fault.fs.zfs.vdev.checksum") == 0)
391 			code = "ZFS-8000-GH";
392 		else if (strcmp(class, "fault.fs.zfs.io_failure_wait") == 0)
393 			code = "ZFS-8000-HC";
394 		else if (strcmp(class, "fault.fs.zfs.io_failure_continue") == 0)
395 			code = "ZFS-8000-JQ";
396 		else if (strcmp(class, "fault.fs.zfs.log_replay") == 0)
397 			code = "ZFS-8000-K4";
398 		else if (strcmp(class, "fault.fs.zfs.pool") == 0)
399 			code = "ZFS-8000-CS";
400 		else if (strcmp(class, "fault.fs.zfs.device") == 0)
401 			code = "ZFS-8000-D3";
402 
403 	}
404 	return (code);
405 }
406 
407 void
408 fmd_case_add_suspect(fmd_hdl_t *hdl, fmd_case_t *cp, nvlist_t *fault)
409 {
410 	nvlist_t *nvl;
411 	const char *code = fmd_fault_mkcode(fault);
412 	int64_t tod[2];
413 	int err = 0;
414 
415 	/*
416 	 * payload derived from fmd_protocol_list()
417 	 */
418 
419 	(void) gettimeofday(&cp->ci_tv, NULL);
420 	tod[0] = cp->ci_tv.tv_sec;
421 	tod[1] = cp->ci_tv.tv_usec;
422 
423 	nvl = fmd_nvl_alloc(hdl, FMD_SLEEP);
424 
425 	err |= nvlist_add_uint8(nvl, FM_VERSION, FM_SUSPECT_VERSION);
426 	err |= nvlist_add_string(nvl, FM_CLASS, FM_LIST_SUSPECT_CLASS);
427 	err |= nvlist_add_string(nvl, FM_SUSPECT_UUID, cp->ci_uuid);
428 	err |= nvlist_add_string(nvl, FM_SUSPECT_DIAG_CODE, code);
429 	err |= nvlist_add_int64_array(nvl, FM_SUSPECT_DIAG_TIME, tod, 2);
430 	err |= nvlist_add_uint32(nvl, FM_SUSPECT_FAULT_SZ, 1);
431 	err |= nvlist_add_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
432 	    (const nvlist_t **)&fault, 1);
433 
434 	if (err)
435 		zed_log_die("failed to populate nvlist");
436 
437 	zed_log_fault(fault, cp->ci_uuid, code);
438 	zfs_agent_post_event(FM_LIST_SUSPECT_CLASS, NULL, nvl);
439 
440 	nvlist_free(nvl);
441 	nvlist_free(fault);
442 }
443 
444 void
445 fmd_case_setspecific(fmd_hdl_t *hdl, fmd_case_t *cp, void *data)
446 {
447 	cp->ci_data = data;
448 }
449 
450 void *
451 fmd_case_getspecific(fmd_hdl_t *hdl, fmd_case_t *cp)
452 {
453 	return (cp->ci_data);
454 }
455 
456 void
457 fmd_buf_create(fmd_hdl_t *hdl, fmd_case_t *cp, const char *name, size_t size)
458 {
459 	assert(strcmp(name, "data") == 0);
460 	assert(cp->ci_bufptr == NULL);
461 	assert(size < (1024 * 1024));
462 
463 	cp->ci_bufptr = fmd_hdl_alloc(hdl, size, FMD_SLEEP);
464 	cp->ci_bufsiz = size;
465 }
466 
467 void
468 fmd_buf_read(fmd_hdl_t *hdl, fmd_case_t *cp,
469     const char *name, void *buf, size_t size)
470 {
471 	assert(strcmp(name, "data") == 0);
472 	assert(cp->ci_bufptr != NULL);
473 	assert(size <= cp->ci_bufsiz);
474 
475 	bcopy(cp->ci_bufptr, buf, size);
476 }
477 
478 void
479 fmd_buf_write(fmd_hdl_t *hdl, fmd_case_t *cp,
480     const char *name, const void *buf, size_t size)
481 {
482 	assert(strcmp(name, "data") == 0);
483 	assert(cp->ci_bufptr != NULL);
484 	assert(cp->ci_bufsiz >= size);
485 
486 	bcopy(buf, cp->ci_bufptr, size);
487 }
488 
489 /* SERD Engines */
490 
491 void
492 fmd_serd_create(fmd_hdl_t *hdl, const char *name, uint_t n, hrtime_t t)
493 {
494 	fmd_module_t *mp = (fmd_module_t *)hdl;
495 
496 	if (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL) {
497 		zed_log_msg(LOG_ERR, "failed to create SERD engine '%s': "
498 		    " name already exists", name);
499 		return;
500 	}
501 
502 	(void) fmd_serd_eng_insert(&mp->mod_serds, name, n, t);
503 }
504 
505 void
506 fmd_serd_destroy(fmd_hdl_t *hdl, const char *name)
507 {
508 	fmd_module_t *mp = (fmd_module_t *)hdl;
509 
510 	fmd_serd_eng_delete(&mp->mod_serds, name);
511 
512 	fmd_hdl_debug(hdl, "serd_destroy %s", name);
513 }
514 
515 int
516 fmd_serd_exists(fmd_hdl_t *hdl, const char *name)
517 {
518 	fmd_module_t *mp = (fmd_module_t *)hdl;
519 
520 	return (fmd_serd_eng_lookup(&mp->mod_serds, name) != NULL);
521 }
522 
523 void
524 fmd_serd_reset(fmd_hdl_t *hdl, const char *name)
525 {
526 	fmd_module_t *mp = (fmd_module_t *)hdl;
527 	fmd_serd_eng_t *sgp;
528 
529 	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
530 		zed_log_msg(LOG_ERR, "serd engine '%s' does not exist", name);
531 		return;
532 	}
533 
534 	fmd_serd_eng_reset(sgp);
535 
536 	fmd_hdl_debug(hdl, "serd_reset %s", name);
537 }
538 
539 int
540 fmd_serd_record(fmd_hdl_t *hdl, const char *name, fmd_event_t *ep)
541 {
542 	fmd_module_t *mp = (fmd_module_t *)hdl;
543 	fmd_serd_eng_t *sgp;
544 	int err;
545 
546 	if ((sgp = fmd_serd_eng_lookup(&mp->mod_serds, name)) == NULL) {
547 		zed_log_msg(LOG_ERR, "failed to add record to SERD engine '%s'",
548 		    name);
549 		return (FMD_B_FALSE);
550 	}
551 	err = fmd_serd_eng_record(sgp, ep->ev_hrt);
552 
553 	return (err);
554 }
555 
556 /* FMD Timers */
557 
558 static void
559 _timer_notify(union sigval sv)
560 {
561 	fmd_timer_t *ftp = sv.sival_ptr;
562 	fmd_hdl_t *hdl = ftp->ft_hdl;
563 	fmd_module_t *mp = (fmd_module_t *)hdl;
564 	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
565 	struct itimerspec its;
566 
567 	fmd_hdl_debug(hdl, "timer fired (%p)", ftp->ft_tid);
568 
569 	/* disarm the timer */
570 	bzero(&its, sizeof (struct itimerspec));
571 	timer_settime(ftp->ft_tid, 0, &its, NULL);
572 
573 	/* Note that the fmdo_timeout can remove this timer */
574 	if (ops->fmdo_timeout != NULL)
575 		ops->fmdo_timeout(hdl, ftp, ftp->ft_arg);
576 }
577 
578 /*
579  * Install a new timer which will fire at least delta nanoseconds after the
580  * current time. After the timeout has expired, the module's fmdo_timeout
581  * entry point is called.
582  */
583 fmd_timer_t *
584 fmd_timer_install(fmd_hdl_t *hdl, void *arg, fmd_event_t *ep, hrtime_t delta)
585 {
586 	struct sigevent sev;
587 	struct itimerspec its;
588 	fmd_timer_t *ftp;
589 
590 	ftp = fmd_hdl_alloc(hdl, sizeof (fmd_timer_t), FMD_SLEEP);
591 	ftp->ft_arg = arg;
592 	ftp->ft_hdl = hdl;
593 
594 	its.it_value.tv_sec = delta / 1000000000;
595 	its.it_value.tv_nsec = delta % 1000000000;
596 	its.it_interval.tv_sec = its.it_value.tv_sec;
597 	its.it_interval.tv_nsec = its.it_value.tv_nsec;
598 
599 	sev.sigev_notify = SIGEV_THREAD;
600 	sev.sigev_notify_function = _timer_notify;
601 	sev.sigev_notify_attributes = NULL;
602 	sev.sigev_value.sival_ptr = ftp;
603 
604 	timer_create(CLOCK_REALTIME, &sev, &ftp->ft_tid);
605 	timer_settime(ftp->ft_tid, 0, &its, NULL);
606 
607 	fmd_hdl_debug(hdl, "installing timer for %d secs (%p)",
608 	    (int)its.it_value.tv_sec, ftp->ft_tid);
609 
610 	return (ftp);
611 }
612 
613 void
614 fmd_timer_remove(fmd_hdl_t *hdl, fmd_timer_t *ftp)
615 {
616 	fmd_hdl_debug(hdl, "removing timer (%p)", ftp->ft_tid);
617 
618 	timer_delete(ftp->ft_tid);
619 
620 	fmd_hdl_free(hdl, ftp, sizeof (fmd_timer_t));
621 }
622 
623 /* Name-Value Pair Lists */
624 
625 nvlist_t *
626 fmd_nvl_create_fault(fmd_hdl_t *hdl, const char *class, uint8_t certainty,
627     nvlist_t *asru, nvlist_t *fru, nvlist_t *resource)
628 {
629 	nvlist_t *nvl;
630 	int err = 0;
631 
632 	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
633 		zed_log_die("failed to xalloc fault nvlist");
634 
635 	err |= nvlist_add_uint8(nvl, FM_VERSION, FM_FAULT_VERSION);
636 	err |= nvlist_add_string(nvl, FM_CLASS, class);
637 	err |= nvlist_add_uint8(nvl, FM_FAULT_CERTAINTY, certainty);
638 
639 	if (asru != NULL)
640 		err |= nvlist_add_nvlist(nvl, FM_FAULT_ASRU, asru);
641 	if (fru != NULL)
642 		err |= nvlist_add_nvlist(nvl, FM_FAULT_FRU, fru);
643 	if (resource != NULL)
644 		err |= nvlist_add_nvlist(nvl, FM_FAULT_RESOURCE, resource);
645 
646 	if (err)
647 		zed_log_die("failed to populate nvlist: %s\n", strerror(err));
648 
649 	return (nvl);
650 }
651 
652 /*
653  * sourced from fmd_string.c
654  */
655 static int
656 fmd_strmatch(const char *s, const char *p)
657 {
658 	char c;
659 
660 	if (p == NULL)
661 		return (0);
662 
663 	if (s == NULL)
664 		s = ""; /* treat NULL string as the empty string */
665 
666 	do {
667 		if ((c = *p++) == '\0')
668 			return (*s == '\0');
669 
670 		if (c == '*') {
671 			while (*p == '*')
672 				p++; /* consecutive *'s can be collapsed */
673 
674 			if (*p == '\0')
675 				return (1);
676 
677 			while (*s != '\0') {
678 				if (fmd_strmatch(s++, p) != 0)
679 					return (1);
680 			}
681 
682 			return (0);
683 		}
684 	} while (c == *s++);
685 
686 	return (0);
687 }
688 
689 int
690 fmd_nvl_class_match(fmd_hdl_t *hdl, nvlist_t *nvl, const char *pattern)
691 {
692 	char *class;
693 
694 	return (nvl != NULL &&
695 	    nvlist_lookup_string(nvl, FM_CLASS, &class) == 0 &&
696 	    fmd_strmatch(class, pattern));
697 }
698 
699 nvlist_t *
700 fmd_nvl_alloc(fmd_hdl_t *hdl, int flags)
701 {
702 	nvlist_t *nvl = NULL;
703 
704 	if (nvlist_alloc(&nvl, NV_UNIQUE_NAME, 0) != 0)
705 		return (NULL);
706 
707 	return (nvl);
708 }
709 
710 
711 /*
712  * ZED Agent specific APIs
713  */
714 
715 fmd_hdl_t *
716 fmd_module_hdl(const char *name)
717 {
718 	if (strcmp(name, "zfs-retire") == 0)
719 		return ((fmd_hdl_t *)&zfs_retire_module);
720 	if (strcmp(name, "zfs-diagnosis") == 0)
721 		return ((fmd_hdl_t *)&zfs_diagnosis_module);
722 
723 	return (NULL);
724 }
725 
726 boolean_t
727 fmd_module_initialized(fmd_hdl_t *hdl)
728 {
729 	fmd_module_t *mp = (fmd_module_t *)hdl;
730 
731 	return (mp->mod_info != NULL);
732 }
733 
734 /*
735  * fmd_module_recv is called for each event that is received by
736  * the fault manager that has a class that matches one of the
737  * module's subscriptions.
738  */
739 void
740 fmd_module_recv(fmd_hdl_t *hdl, nvlist_t *nvl, const char *class)
741 {
742 	fmd_module_t *mp = (fmd_module_t *)hdl;
743 	const fmd_hdl_ops_t *ops = mp->mod_info->fmdi_ops;
744 	fmd_event_t faux_event = {0};
745 	int64_t *tv;
746 	uint_t n;
747 
748 	/*
749 	 * Will need to normalized this if we persistently store the case data
750 	 */
751 	if (nvlist_lookup_int64_array(nvl, FM_EREPORT_TIME, &tv, &n) == 0)
752 		faux_event.ev_hrt = tv[0] * NANOSEC + tv[1];
753 	else
754 		faux_event.ev_hrt = 0;
755 
756 	ops->fmdo_recv(hdl, &faux_event, nvl, class);
757 
758 	mp->mod_stats.ms_accepted.fmds_value.ui64++;
759 
760 	/* TBD - should we initiate fm_module_gc() periodically? */
761 }
762