xref: /titanic_50/usr/src/cmd/fm/fmd/common/fmd_asru.c (revision 275c9da86e89f8abf71135cf63d9fc23671b2e60)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 #include <sys/fm/protocol.h>
30 #include <uuid/uuid.h>
31 
32 #include <dirent.h>
33 #include <limits.h>
34 #include <unistd.h>
35 #include <alloca.h>
36 #include <stddef.h>
37 
38 #include <fmd_alloc.h>
39 #include <fmd_string.h>
40 #include <fmd_error.h>
41 #include <fmd_subr.h>
42 #include <fmd_protocol.h>
43 #include <fmd_event.h>
44 #include <fmd_conf.h>
45 #include <fmd_fmri.h>
46 #include <fmd_dispq.h>
47 #include <fmd_case.h>
48 #include <fmd_module.h>
49 #include <fmd_asru.h>
50 
51 #include <fmd.h>
52 
53 static const char *const _fmd_asru_events[] = {
54 	FMD_RSRC_CLASS "asru.ok",		/* UNUSABLE=0 FAULTED=0 */
55 	FMD_RSRC_CLASS "asru.degraded",		/* UNUSABLE=0 FAULTED=1 */
56 	FMD_RSRC_CLASS "asru.unknown",		/* UNUSABLE=1 FAULTED=0 */
57 	FMD_RSRC_CLASS "asru.faulted"		/* UNUSABLE=1 FAULTED=1 */
58 };
59 
60 static const char *const _fmd_asru_snames[] = {
61 	"uf", "uF", "Uf", "UF"			/* same order as above */
62 };
63 
64 volatile uint32_t fmd_asru_fake_not_present = 0;
65 
66 static fmd_asru_t *
67 fmd_asru_create(fmd_asru_hash_t *ahp, const char *uuid,
68     const char *name, nvlist_t *fmri)
69 {
70 	fmd_asru_t *ap = fmd_zalloc(sizeof (fmd_asru_t), FMD_SLEEP);
71 	char *s;
72 
73 	(void) pthread_mutex_init(&ap->asru_lock, NULL);
74 	(void) pthread_cond_init(&ap->asru_cv, NULL);
75 
76 	ap->asru_name = fmd_strdup(name, FMD_SLEEP);
77 	if (fmri)
78 		(void) nvlist_xdup(fmri, &ap->asru_fmri, &fmd.d_nva);
79 	ap->asru_root = fmd_strdup(ahp->ah_dirpath, FMD_SLEEP);
80 	ap->asru_uuid = fmd_strdup(uuid, FMD_SLEEP);
81 	ap->asru_uuidlen = ap->asru_uuid ? strlen(ap->asru_uuid) : 0;
82 	ap->asru_refs = 1;
83 
84 	if (fmri && nvlist_lookup_string(fmri, FM_FMRI_SCHEME, &s) == 0 &&
85 	    strcmp(s, FM_FMRI_SCHEME_FMD) == 0)
86 		ap->asru_flags |= FMD_ASRU_INTERNAL;
87 
88 	return (ap);
89 }
90 
91 static void
92 fmd_asru_destroy(fmd_asru_t *ap)
93 {
94 	ASSERT(MUTEX_HELD(&ap->asru_lock));
95 	ASSERT(ap->asru_refs == 0);
96 
97 	nvlist_free(ap->asru_event);
98 	fmd_strfree(ap->asru_name);
99 	nvlist_free(ap->asru_fmri);
100 	fmd_strfree(ap->asru_root);
101 	fmd_free(ap->asru_uuid, ap->asru_uuidlen + 1);
102 	fmd_free(ap, sizeof (fmd_asru_t));
103 }
104 
105 static void
106 fmd_asru_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_t *ap)
107 {
108 	uint_t h = fmd_strhash(ap->asru_name) % ahp->ah_hashlen;
109 
110 	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
111 	ap->asru_next = ahp->ah_hash[h];
112 	ahp->ah_hash[h] = ap;
113 	ahp->ah_count++;
114 }
115 
116 static fmd_asru_t *
117 fmd_asru_hold(fmd_asru_t *ap)
118 {
119 	(void) pthread_mutex_lock(&ap->asru_lock);
120 	ap->asru_refs++;
121 	ASSERT(ap->asru_refs != 0);
122 	(void) pthread_mutex_unlock(&ap->asru_lock);
123 	return (ap);
124 }
125 
126 /*
127  * Lookup an asru in the hash by name and place a hold on it.  If the asru is
128  * not found, no entry is created and NULL is returned.  This internal function
129  * is for callers who have the ah_lock held and is used by lookup_name below.
130  */
131 fmd_asru_t *
132 fmd_asru_hash_lookup(fmd_asru_hash_t *ahp, const char *name)
133 {
134 	fmd_asru_t *ap;
135 	uint_t h;
136 
137 	ASSERT(RW_LOCK_HELD(&ahp->ah_lock));
138 	h = fmd_strhash(name) % ahp->ah_hashlen;
139 
140 	for (ap = ahp->ah_hash[h]; ap != NULL; ap = ap->asru_next) {
141 		if (strcmp(ap->asru_name, name) == 0)
142 			break;
143 	}
144 
145 	if (ap != NULL)
146 		(void) fmd_asru_hold(ap);
147 	else
148 		(void) fmd_set_errno(EFMD_ASRU_NOENT);
149 
150 	return (ap);
151 }
152 
153 static int
154 fmd_asru_is_present(nvlist_t *event)
155 {
156 	int ps = -1;
157 	nvlist_t *asru, *fru, *rsrc;
158 
159 	/*
160 	 * Check if there is evidence that this object is no longer present.
161 	 * In general fmd_fmri_present() should be supported on resources and/or
162 	 * frus, as those are the things that are physically present or not
163 	 * present - an asru can be spread over a number of frus some of which
164 	 * are present and some not, so fmd_fmri_present() is not generally
165 	 * meaningful. However retain a check for asru first for compatibility.
166 	 * If we have checked all three and we still get -1 then nothing knows
167 	 * whether it's present or not, so err on the safe side and treat it
168 	 * as still present.
169 	 */
170 	if (fmd_asru_fake_not_present)
171 		ps = 0;
172 	if (ps == -1 && nvlist_lookup_nvlist(event, FM_FAULT_ASRU, &asru) == 0)
173 		ps = fmd_fmri_present(asru);
174 	if (ps == -1 && nvlist_lookup_nvlist(event, FM_FAULT_RESOURCE,
175 	    &rsrc) == 0)
176 		ps = fmd_fmri_present(rsrc);
177 	if (ps == -1 && nvlist_lookup_nvlist(event, FM_FAULT_FRU, &fru) == 0)
178 		ps = fmd_fmri_present(fru);
179 	if (ps == -1)
180 		ps = 1;
181 	return (ps);
182 }
183 
184 static void
185 fmd_asru_asru_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
186     char *name)
187 {
188 	uint_t h = fmd_strhash(name) % ahp->ah_hashlen;
189 
190 	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
191 	alp->al_asru_next = ahp->ah_asru_hash[h];
192 	ahp->ah_asru_hash[h] = alp;
193 	ahp->ah_al_count++;
194 }
195 
196 static void
197 fmd_asru_case_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
198     char *name)
199 {
200 	uint_t h = fmd_strhash(name) % ahp->ah_hashlen;
201 
202 	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
203 	alp->al_case_next = ahp->ah_case_hash[h];
204 	ahp->ah_case_hash[h] = alp;
205 }
206 
207 static void
208 fmd_asru_fru_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp, char *name)
209 {
210 	uint_t h = fmd_strhash(name) % ahp->ah_hashlen;
211 
212 	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
213 	alp->al_fru_next = ahp->ah_fru_hash[h];
214 	ahp->ah_fru_hash[h] = alp;
215 }
216 
217 static void
218 fmd_asru_label_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
219     char *name)
220 {
221 	uint_t h = fmd_strhash(name) % ahp->ah_hashlen;
222 
223 	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
224 	alp->al_label_next = ahp->ah_label_hash[h];
225 	ahp->ah_label_hash[h] = alp;
226 }
227 
228 static void
229 fmd_asru_rsrc_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
230     char *name)
231 {
232 	uint_t h = fmd_strhash(name) % ahp->ah_hashlen;
233 
234 	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
235 	alp->al_rsrc_next = ahp->ah_rsrc_hash[h];
236 	ahp->ah_rsrc_hash[h] = alp;
237 }
238 
239 static void
240 fmd_asru_al_destroy(fmd_asru_link_t *alp)
241 {
242 	ASSERT(alp->al_refs == 0);
243 	ASSERT(MUTEX_HELD(&alp->al_asru->asru_lock));
244 
245 	if (alp->al_log != NULL)
246 		fmd_log_rele(alp->al_log);
247 
248 	fmd_free(alp->al_uuid, alp->al_uuidlen + 1);
249 	nvlist_free(alp->al_event);
250 	fmd_strfree(alp->al_rsrc_name);
251 	fmd_strfree(alp->al_case_uuid);
252 	fmd_strfree(alp->al_fru_name);
253 	fmd_strfree(alp->al_asru_name);
254 	fmd_strfree(alp->al_label);
255 	nvlist_free(alp->al_asru_fmri);
256 	fmd_free(alp, sizeof (fmd_asru_link_t));
257 }
258 
259 static fmd_asru_link_t *
260 fmd_asru_al_hold(fmd_asru_link_t *alp)
261 {
262 	fmd_asru_t *ap = alp->al_asru;
263 
264 	(void) pthread_mutex_lock(&ap->asru_lock);
265 	ap->asru_refs++;
266 	alp->al_refs++;
267 	ASSERT(alp->al_refs != 0);
268 	(void) pthread_mutex_unlock(&ap->asru_lock);
269 	return (alp);
270 }
271 
272 static void fmd_asru_destroy(fmd_asru_t *ap);
273 
274 /*ARGSUSED*/
275 static void
276 fmd_asru_al_hash_release(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp)
277 {
278 	fmd_asru_t *ap = alp->al_asru;
279 
280 	(void) pthread_mutex_lock(&ap->asru_lock);
281 	ASSERT(alp->al_refs != 0);
282 	if (--alp->al_refs == 0)
283 		fmd_asru_al_destroy(alp);
284 	ASSERT(ap->asru_refs != 0);
285 	if (--ap->asru_refs == 0)
286 		fmd_asru_destroy(ap);
287 	else
288 		(void) pthread_mutex_unlock(&ap->asru_lock);
289 }
290 
291 static int
292 fmd_asru_get_namestr(nvlist_t *nvl, char **name, ssize_t *namelen)
293 {
294 	if ((*namelen = fmd_fmri_nvl2str(nvl, NULL, 0)) == -1)
295 		return (EFMD_ASRU_FMRI);
296 	*name = fmd_alloc(*namelen + 1, FMD_SLEEP);
297 	if (fmd_fmri_nvl2str(nvl, *name, *namelen + 1) == -1) {
298 		if (*name != NULL)
299 			fmd_free(*name, *namelen + 1);
300 		return (EFMD_ASRU_FMRI);
301 	}
302 	return (0);
303 }
304 
305 static fmd_asru_link_t *
306 fmd_asru_al_create(fmd_asru_hash_t *ahp, nvlist_t *nvl, fmd_case_t *cp,
307     const char *al_uuid)
308 {
309 	nvlist_t *asru = NULL, *fru, *rsrc;
310 	int got_rsrc = 0, got_asru = 0, got_fru = 0;
311 	ssize_t fru_namelen, rsrc_namelen, asru_namelen;
312 	char *asru_name, *rsrc_name, *fru_name, *name, *label;
313 	fmd_asru_link_t *alp;
314 	fmd_asru_t *ap;
315 	boolean_t msg;
316 	fmd_case_impl_t *cip = (fmd_case_impl_t *)cp;
317 
318 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &asru) == 0 &&
319 	    fmd_asru_get_namestr(asru, &asru_name, &asru_namelen) == 0)
320 		got_asru = 1;
321 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &fru) == 0 &&
322 	    fmd_asru_get_namestr(fru, &fru_name, &fru_namelen) == 0)
323 		got_fru = 1;
324 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0 &&
325 	    fmd_asru_get_namestr(rsrc, &rsrc_name, &rsrc_namelen) == 0)
326 		got_rsrc = 1;
327 	if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION, &label) != 0)
328 		label = "";
329 
330 	/*
331 	 * Grab the rwlock as a writer; Then create and insert the asru with
332 	 * ahp->ah_lock held and hash it in. We'll then drop the rwlock and
333 	 * proceed to initializing the asru.
334 	 */
335 	(void) pthread_rwlock_wrlock(&ahp->ah_lock);
336 
337 	/*
338 	 * Create and initialise the per-fault "link" structure.
339 	 */
340 	alp = fmd_zalloc(sizeof (fmd_asru_link_t), FMD_SLEEP);
341 	if (got_asru)
342 		(void) nvlist_xdup(asru, &alp->al_asru_fmri, &fmd.d_nva);
343 	alp->al_uuid = fmd_strdup(al_uuid, FMD_SLEEP);
344 	alp->al_uuidlen = strlen(alp->al_uuid);
345 	alp->al_refs = 1;
346 
347 	/*
348 	 * If this is the first fault for this asru, then create the per-asru
349 	 * structure and link into the hash.
350 	 */
351 	name = got_asru ? asru_name : "";
352 	if ((ap = fmd_asru_hash_lookup(ahp, name)) == NULL) {
353 		ap = fmd_asru_create(ahp, al_uuid, name, got_asru ? asru :
354 		    NULL);
355 		fmd_asru_hash_insert(ahp, ap);
356 	} else
357 		nvlist_free(ap->asru_event);
358 	(void) nvlist_xdup(nvl, &ap->asru_event, &fmd.d_nva);
359 
360 	/*
361 	 * Put the link structure on the list associated with the per-asru
362 	 * structure. Then put the link structure on the various hashes.
363 	 */
364 	fmd_list_append(&ap->asru_list, (fmd_list_t *)alp);
365 	alp->al_asru = ap;
366 	alp->al_asru_name = got_asru ? asru_name : fmd_strdup("", FMD_SLEEP);
367 	fmd_asru_asru_hash_insert(ahp, alp, alp->al_asru_name);
368 	alp->al_fru_name = got_fru ? fru_name : fmd_strdup("", FMD_SLEEP);
369 	fmd_asru_fru_hash_insert(ahp, alp, alp->al_fru_name);
370 	alp->al_rsrc_name = got_rsrc ? rsrc_name : fmd_strdup("", FMD_SLEEP);
371 	fmd_asru_rsrc_hash_insert(ahp, alp, alp->al_rsrc_name);
372 	alp->al_label = fmd_strdup(label, FMD_SLEEP);
373 	fmd_asru_label_hash_insert(ahp, alp, label);
374 	alp->al_case_uuid = fmd_strdup(cip->ci_uuid, FMD_SLEEP);
375 	fmd_asru_case_hash_insert(ahp, alp, cip->ci_uuid);
376 	(void) pthread_mutex_lock(&ap->asru_lock);
377 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
378 
379 	ap->asru_case = alp->al_case = cp;
380 	if (nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE, &msg) == 0 &&
381 	    msg == B_FALSE)
382 		ap->asru_flags |= FMD_ASRU_INVISIBLE;
383 	(void) nvlist_xdup(nvl, &alp->al_event, &fmd.d_nva);
384 	ap->asru_flags |= FMD_ASRU_VALID;
385 	(void) pthread_cond_broadcast(&ap->asru_cv);
386 	(void) pthread_mutex_unlock(&ap->asru_lock);
387 	return (alp);
388 }
389 
390 static void
391 fmd_asru_hash_recreate(fmd_log_t *lp, fmd_event_t *ep, fmd_asru_hash_t *ahp)
392 {
393 	nvlist_t *nvl = FMD_EVENT_NVL(ep);
394 	boolean_t f, u, ps, us;
395 	nvlist_t *flt, *flt_copy, *asru;
396 	char *case_uuid = NULL, *case_code = NULL;
397 	fmd_asru_t *ap;
398 	fmd_asru_link_t *alp;
399 	fmd_case_t *cp;
400 	int64_t *diag_time;
401 	uint_t nelem;
402 
403 	/*
404 	 * Extract the most recent values of 'faulty' from the event log.
405 	 */
406 	if (nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_FAULTY, &f) != 0) {
407 		fmd_error(EFMD_ASRU_EVENT, "failed to reload asru %s: "
408 		    "invalid event log record\n", lp->log_name);
409 		ahp->ah_error = EFMD_ASRU_EVENT;
410 		return;
411 	}
412 	if (nvlist_lookup_nvlist(nvl, FM_RSRC_ASRU_EVENT, &flt) != 0) {
413 		fmd_error(EFMD_ASRU_EVENT, "failed to reload asru %s: "
414 		    "invalid event log record\n", lp->log_name);
415 		ahp->ah_error = EFMD_ASRU_EVENT;
416 		return;
417 	}
418 	(void) nvlist_lookup_string(nvl, FM_RSRC_ASRU_UUID, &case_uuid);
419 	(void) nvlist_lookup_string(nvl, FM_RSRC_ASRU_CODE, &case_code);
420 
421 	/*
422 	 * Attempt to recreate the case in the CLOSED state.
423 	 * If the case is already present, fmd_case_recreate() will return it.
424 	 * If not, we'll create a new orphaned case. Either way,  we use the
425 	 * ASRU event to insert a suspect into the partially-restored case.
426 	 */
427 	fmd_module_lock(fmd.d_rmod);
428 	cp = fmd_case_recreate(fmd.d_rmod, NULL, FMD_CASE_CLOSED, case_uuid,
429 	    case_code);
430 	fmd_case_hold(cp);
431 	fmd_module_unlock(fmd.d_rmod);
432 	if (nvlist_lookup_int64_array(nvl, FM_SUSPECT_DIAG_TIME, &diag_time,
433 	    &nelem) == 0 && nelem >= 2)
434 		fmd_case_settime(cp, diag_time[0], diag_time[1]);
435 	else
436 		fmd_case_settime(cp, lp->log_stat.st_ctime, 0);
437 	(void) nvlist_xdup(flt, &flt_copy, &fmd.d_nva);
438 	fmd_case_recreate_suspect(cp, flt_copy);
439 
440 	/*
441 	 * Now create the resource cache entries.
442 	 */
443 	alp = fmd_asru_al_create(ahp, flt, cp, fmd_strbasename(lp->log_name));
444 	ap = alp->al_asru;
445 
446 	/*
447 	 * Check to see if the resource is still present in the system.  If
448 	 * so, then update the value of the unusable bit based on the current
449 	 * system configuration.  If not, then consider unusable.
450 	 */
451 	ps = fmd_asru_is_present(flt);
452 	if (ps) {
453 		if (nvlist_lookup_nvlist(flt, FM_FAULT_ASRU, &asru) != 0)
454 			u = FMD_B_FALSE;
455 		else if ((us = fmd_fmri_unusable(asru)) == -1) {
456 			fmd_error(EFMD_ASRU_FMRI, "failed to update "
457 			    "status of asru %s", lp->log_name);
458 			u = FMD_B_FALSE;
459 		} else
460 			u = us != 0;
461 
462 	} else
463 		u = FMD_B_TRUE;	/* not present; set unusable */
464 
465 	ap->asru_flags |= FMD_ASRU_RECREATED;
466 	if (ps)
467 		ap->asru_flags |= FMD_ASRU_PRESENT;
468 	if (f) {
469 		alp->al_flags |= FMD_ASRU_FAULTY;
470 		ap->asru_flags |= FMD_ASRU_FAULTY;
471 	}
472 	if (u) {
473 		alp->al_flags |= FMD_ASRU_UNUSABLE;
474 		ap->asru_flags |= FMD_ASRU_UNUSABLE;
475 	}
476 
477 	TRACE((FMD_DBG_ASRU, "asru %s recreated as %p (%s)", alp->al_uuid,
478 	    (void *)ap, _fmd_asru_snames[ap->asru_flags & FMD_ASRU_STATE]));
479 }
480 
481 static void
482 fmd_asru_hash_discard(fmd_asru_hash_t *ahp, const char *uuid, int err)
483 {
484 	char src[PATH_MAX], dst[PATH_MAX];
485 
486 	(void) snprintf(src, PATH_MAX, "%s/%s", ahp->ah_dirpath, uuid);
487 	(void) snprintf(dst, PATH_MAX, "%s/%s-", ahp->ah_dirpath, uuid);
488 
489 	if (err != 0)
490 		err = rename(src, dst);
491 	else
492 		err = unlink(src);
493 
494 	if (err != 0 && errno != ENOENT)
495 		fmd_error(EFMD_ASRU_EVENT, "failed to rename log %s", src);
496 }
497 
498 /*
499  * Open a saved log file and restore it into the ASRU hash.  If we can't even
500  * open the log, rename the log file to <uuid>- to indicate it is corrupt.  If
501  * fmd_log_replay() fails, we either delete the file (if it has reached the
502  * upper limit on cache age) or rename it for debugging if it was corrupted.
503  */
504 static void
505 fmd_asru_hash_logopen(fmd_asru_hash_t *ahp, const char *uuid)
506 {
507 	fmd_log_t *lp = fmd_log_tryopen(ahp->ah_dirpath, uuid, FMD_LOG_ASRU);
508 	uint_t n;
509 
510 	if (lp == NULL) {
511 		fmd_asru_hash_discard(ahp, uuid, errno);
512 		return;
513 	}
514 
515 	ahp->ah_error = 0;
516 	n = ahp->ah_al_count;
517 
518 	fmd_log_replay(lp, (fmd_log_f *)fmd_asru_hash_recreate, ahp);
519 	fmd_log_rele(lp);
520 
521 	if (ahp->ah_al_count == n)
522 		fmd_asru_hash_discard(ahp, uuid, ahp->ah_error);
523 }
524 
525 void
526 fmd_asru_hash_refresh(fmd_asru_hash_t *ahp)
527 {
528 	struct dirent *dp;
529 	DIR *dirp;
530 	int zero;
531 
532 	if ((dirp = opendir(ahp->ah_dirpath)) == NULL) {
533 		fmd_error(EFMD_ASRU_NODIR,
534 		    "failed to open asru cache directory %s", ahp->ah_dirpath);
535 		return;
536 	}
537 
538 	(void) fmd_conf_getprop(fmd.d_conf, "rsrc.zero", &zero);
539 
540 	(void) pthread_rwlock_wrlock(&ahp->ah_lock);
541 
542 	while ((dp = readdir(dirp)) != NULL) {
543 		if (dp->d_name[0] == '.')
544 			continue; /* skip "." and ".." */
545 
546 		if (zero)
547 			fmd_asru_hash_discard(ahp, dp->d_name, 0);
548 		else if (!fmd_strmatch(dp->d_name, "*-"))
549 			fmd_asru_hash_logopen(ahp, dp->d_name);
550 	}
551 
552 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
553 	(void) closedir(dirp);
554 }
555 
556 /*
557  * If the resource is present and faulty but not unusable, replay the fault
558  * event that caused it be marked faulty.  This will cause the agent
559  * subscribing to this fault class to again disable the resource.
560  */
561 /*ARGSUSED*/
562 static void
563 fmd_asru_hash_replay_asru(fmd_asru_t *ap, void *data)
564 {
565 	fmd_event_t *e;
566 	nvlist_t *nvl;
567 	char *class;
568 
569 	if (ap->asru_event != NULL && (ap->asru_flags & (FMD_ASRU_STATE |
570 	    FMD_ASRU_PRESENT)) == (FMD_ASRU_FAULTY | FMD_ASRU_PRESENT)) {
571 
572 		fmd_dprintf(FMD_DBG_ASRU,
573 		    "replaying fault event for %s", ap->asru_name);
574 
575 		(void) nvlist_xdup(ap->asru_event, &nvl, &fmd.d_nva);
576 		(void) nvlist_lookup_string(nvl, FM_CLASS, &class);
577 
578 		(void) nvlist_add_string(nvl, FMD_EVN_UUID,
579 		    ((fmd_case_impl_t *)ap->asru_case)->ci_uuid);
580 
581 		e = fmd_event_create(FMD_EVT_PROTOCOL, FMD_HRT_NOW, nvl, class);
582 		fmd_dispq_dispatch(fmd.d_disp, e, class);
583 	}
584 }
585 
586 void
587 fmd_asru_hash_replay(fmd_asru_hash_t *ahp)
588 {
589 	fmd_asru_hash_apply(ahp, fmd_asru_hash_replay_asru, NULL);
590 }
591 
592 /*
593  * Check if the resource is still present. If not, and if the rsrc.age time
594  * has expired, then do an implicit repair on the resource.
595  */
596 static void
597 fmd_asru_repair_if_aged(fmd_asru_link_t *alp, void *er)
598 {
599 	struct timeval tv;
600 	fmd_log_t *lp;
601 	hrtime_t hrt;
602 
603 	if (fmd_asru_is_present(alp->al_event))
604 		return;
605 	fmd_time_gettimeofday(&tv);
606 	lp = fmd_log_open(alp->al_asru->asru_root, alp->al_uuid, FMD_LOG_ASRU);
607 	hrt = (hrtime_t)(tv.tv_sec - lp->log_stat.st_mtime);
608 	fmd_log_rele(lp);
609 	if (hrt * NANOSEC >= fmd.d_asrus->ah_lifetime)
610 		fmd_asru_repair(alp, er);
611 }
612 
613 void
614 fmd_asru_clear_aged_rsrcs()
615 {
616 	int err;
617 
618 	fmd_asru_al_hash_apply(fmd.d_asrus, fmd_asru_repair_if_aged, &err);
619 }
620 
621 fmd_asru_hash_t *
622 fmd_asru_hash_create(const char *root, const char *dir)
623 {
624 	fmd_asru_hash_t *ahp;
625 	char path[PATH_MAX];
626 
627 	ahp = fmd_alloc(sizeof (fmd_asru_hash_t), FMD_SLEEP);
628 	(void) pthread_rwlock_init(&ahp->ah_lock, NULL);
629 	ahp->ah_hashlen = fmd.d_str_buckets;
630 	ahp->ah_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen, FMD_SLEEP);
631 	ahp->ah_asru_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
632 	    FMD_SLEEP);
633 	ahp->ah_case_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
634 	    FMD_SLEEP);
635 	ahp->ah_fru_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
636 	    FMD_SLEEP);
637 	ahp->ah_label_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
638 	    FMD_SLEEP);
639 	ahp->ah_rsrc_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
640 	    FMD_SLEEP);
641 	(void) snprintf(path, sizeof (path), "%s/%s", root, dir);
642 	ahp->ah_dirpath = fmd_strdup(path, FMD_SLEEP);
643 	(void) fmd_conf_getprop(fmd.d_conf, "rsrc.age", &ahp->ah_lifetime);
644 	(void) fmd_conf_getprop(fmd.d_conf, "fakenotpresent",
645 	    (uint32_t *)&fmd_asru_fake_not_present);
646 	ahp->ah_al_count = 0;
647 	ahp->ah_count = 0;
648 	ahp->ah_error = 0;
649 
650 	return (ahp);
651 }
652 
653 void
654 fmd_asru_hash_destroy(fmd_asru_hash_t *ahp)
655 {
656 	fmd_asru_link_t *alp, *np;
657 	uint_t i;
658 
659 	for (i = 0; i < ahp->ah_hashlen; i++) {
660 		for (alp = ahp->ah_case_hash[i]; alp != NULL; alp = np) {
661 			np = alp->al_case_next;
662 			alp->al_case_next = NULL;
663 			fmd_case_rele(alp->al_case);
664 			alp->al_case = NULL;
665 			fmd_asru_al_hash_release(ahp, alp);
666 		}
667 	}
668 
669 	fmd_strfree(ahp->ah_dirpath);
670 	fmd_free(ahp->ah_hash, sizeof (void *) * ahp->ah_hashlen);
671 	fmd_free(ahp->ah_asru_hash, sizeof (void *) * ahp->ah_hashlen);
672 	fmd_free(ahp->ah_case_hash, sizeof (void *) * ahp->ah_hashlen);
673 	fmd_free(ahp->ah_fru_hash, sizeof (void *) * ahp->ah_hashlen);
674 	fmd_free(ahp->ah_label_hash, sizeof (void *) * ahp->ah_hashlen);
675 	fmd_free(ahp->ah_rsrc_hash, sizeof (void *) * ahp->ah_hashlen);
676 	fmd_free(ahp, sizeof (fmd_asru_hash_t));
677 }
678 
679 /*
680  * Take a snapshot of the ASRU database by placing an additional hold on each
681  * member in an auxiliary array, and then call 'func' for each ASRU.
682  */
683 void
684 fmd_asru_hash_apply(fmd_asru_hash_t *ahp,
685     void (*func)(fmd_asru_t *, void *), void *arg)
686 {
687 	fmd_asru_t *ap, **aps, **app;
688 	uint_t apc, i;
689 
690 	(void) pthread_rwlock_rdlock(&ahp->ah_lock);
691 
692 	aps = app = fmd_alloc(ahp->ah_count * sizeof (fmd_asru_t *), FMD_SLEEP);
693 	apc = ahp->ah_count;
694 
695 	for (i = 0; i < ahp->ah_hashlen; i++) {
696 		for (ap = ahp->ah_hash[i]; ap != NULL; ap = ap->asru_next)
697 			*app++ = fmd_asru_hold(ap);
698 	}
699 
700 	ASSERT(app == aps + apc);
701 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
702 
703 	for (i = 0; i < apc; i++) {
704 		if (aps[i]->asru_fmri != NULL)
705 			func(aps[i], arg);
706 		fmd_asru_hash_release(ahp, aps[i]);
707 	}
708 
709 	fmd_free(aps, apc * sizeof (fmd_asru_t *));
710 }
711 
712 void
713 fmd_asru_al_hash_apply(fmd_asru_hash_t *ahp,
714     void (*func)(fmd_asru_link_t *, void *), void *arg)
715 {
716 	fmd_asru_link_t *alp, **alps, **alpp;
717 	uint_t alpc, i;
718 
719 	(void) pthread_rwlock_rdlock(&ahp->ah_lock);
720 
721 	alps = alpp = fmd_alloc(ahp->ah_al_count * sizeof (fmd_asru_link_t *),
722 	    FMD_SLEEP);
723 	alpc = ahp->ah_al_count;
724 
725 	for (i = 0; i < ahp->ah_hashlen; i++) {
726 		for (alp = ahp->ah_case_hash[i]; alp != NULL;
727 		    alp = alp->al_case_next)
728 			*alpp++ = fmd_asru_al_hold(alp);
729 	}
730 
731 	ASSERT(alpp == alps + alpc);
732 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
733 
734 	for (i = 0; i < alpc; i++) {
735 		func(alps[i], arg);
736 		fmd_asru_al_hash_release(ahp, alps[i]);
737 	}
738 
739 	fmd_free(alps, alpc * sizeof (fmd_asru_link_t *));
740 }
741 
742 static void
743 fmd_asru_do_hash_apply(fmd_asru_hash_t *ahp, char *name,
744     void (*func)(fmd_asru_link_t *, void *), void *arg,
745     fmd_asru_link_t **hash, size_t match_offset, size_t next_offset)
746 {
747 	fmd_asru_link_t *alp, **alps, **alpp;
748 	uint_t alpc = 0, i;
749 	uint_t h;
750 
751 	(void) pthread_rwlock_rdlock(&ahp->ah_lock);
752 
753 	h = fmd_strhash(name) % ahp->ah_hashlen;
754 
755 	for (alp = hash[h]; alp != NULL; alp =
756 	    /* LINTED pointer alignment */
757 	    FMD_ASRU_AL_HASH_NEXT(alp, next_offset))
758 		/* LINTED pointer alignment */
759 		if (strcmp(FMD_ASRU_AL_HASH_NAME(alp, match_offset), name) == 0)
760 			alpc++;
761 
762 	alps = alpp = fmd_alloc(alpc * sizeof (fmd_asru_link_t *), FMD_SLEEP);
763 
764 	for (alp = hash[h]; alp != NULL; alp =
765 	    /* LINTED pointer alignment */
766 	    FMD_ASRU_AL_HASH_NEXT(alp, next_offset))
767 		/* LINTED pointer alignment */
768 		if (strcmp(FMD_ASRU_AL_HASH_NAME(alp, match_offset), name) == 0)
769 			*alpp++ = fmd_asru_al_hold(alp);
770 
771 	ASSERT(alpp == alps + alpc);
772 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
773 
774 	for (i = 0; i < alpc; i++) {
775 		func(alps[i], arg);
776 		fmd_asru_al_hash_release(ahp, alps[i]);
777 	}
778 
779 	fmd_free(alps, alpc * sizeof (fmd_asru_link_t *));
780 }
781 
782 void
783 fmd_asru_hash_apply_by_asru(fmd_asru_hash_t *ahp, char *name,
784     void (*func)(fmd_asru_link_t *, void *), void *arg)
785 {
786 	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_asru_hash,
787 	    offsetof(fmd_asru_link_t, al_asru_name),
788 	    offsetof(fmd_asru_link_t, al_asru_next));
789 }
790 
791 void
792 fmd_asru_hash_apply_by_case(fmd_asru_hash_t *ahp, fmd_case_t *cp,
793 	void (*func)(fmd_asru_link_t *, void *), void *arg)
794 {
795 	fmd_asru_do_hash_apply(ahp, ((fmd_case_impl_t *)cp)->ci_uuid, func, arg,
796 	    ahp->ah_case_hash, offsetof(fmd_asru_link_t, al_case_uuid),
797 	    offsetof(fmd_asru_link_t, al_case_next));
798 }
799 
800 void
801 fmd_asru_hash_apply_by_fru(fmd_asru_hash_t *ahp, char *name,
802     void (*func)(fmd_asru_link_t *, void *), void *arg)
803 {
804 	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_fru_hash,
805 	    offsetof(fmd_asru_link_t, al_fru_name),
806 	    offsetof(fmd_asru_link_t, al_fru_next));
807 }
808 
809 void
810 fmd_asru_hash_apply_by_rsrc(fmd_asru_hash_t *ahp, char *name,
811     void (*func)(fmd_asru_link_t *, void *), void *arg)
812 {
813 	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_rsrc_hash,
814 	    offsetof(fmd_asru_link_t, al_rsrc_name),
815 	    offsetof(fmd_asru_link_t, al_rsrc_next));
816 }
817 
818 void
819 fmd_asru_hash_apply_by_label(fmd_asru_hash_t *ahp, char *name,
820     void (*func)(fmd_asru_link_t *, void *), void *arg)
821 {
822 	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_label_hash,
823 	    offsetof(fmd_asru_link_t, al_label),
824 	    offsetof(fmd_asru_link_t, al_label_next));
825 }
826 
827 /*
828  * Lookup an asru in the hash by name and place a hold on it.  If the asru is
829  * not found, no entry is created and NULL is returned.
830  */
831 fmd_asru_t *
832 fmd_asru_hash_lookup_name(fmd_asru_hash_t *ahp, const char *name)
833 {
834 	fmd_asru_t *ap;
835 
836 	(void) pthread_rwlock_rdlock(&ahp->ah_lock);
837 	ap = fmd_asru_hash_lookup(ahp, name);
838 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
839 
840 	return (ap);
841 }
842 
843 /*
844  * Lookup an asru in the hash and place a hold on it.
845  */
846 fmd_asru_t *
847 fmd_asru_hash_lookup_nvl(fmd_asru_hash_t *ahp, nvlist_t *fmri)
848 {
849 	fmd_asru_t *ap;
850 	char *name = NULL;
851 	ssize_t namelen;
852 
853 	if (fmd_asru_get_namestr(fmri, &name, &namelen) != 0)
854 		return (NULL);
855 	(void) pthread_rwlock_rdlock(&ahp->ah_lock);
856 	ap = fmd_asru_hash_lookup(ahp, name);
857 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
858 	fmd_free(name, namelen + 1);
859 	return (ap);
860 }
861 
862 /*
863  * Create a resource cache entry using the fault event "nvl" for one of the
864  * suspects from the case "cp".
865  *
866  * The fault event can have the following components :  FM_FAULT_ASRU,
867  * FM_FAULT_FRU, FM_FAULT_RESOURCE. These should be set by the Diagnosis Engine
868  * when calling fmd_nvl_create_fault(). In the general case, these are all
869  * optional and an entry will always be added into the cache even if one or all
870  * of these fields is missing.
871  *
872  * However, for hardware faults the recommended practice is that the fault
873  * event should always have the FM_FAULT_RESOURCE field present and that this
874  * should be represented in hc-scheme.
875  *
876  * Currently the DE should also add the FM_FAULT_ASRU and FM_FAULT_FRU fields
877  * where known, though at some future stage fmd might be able to fill these
878  * in automatically from the topology.
879  */
880 fmd_asru_link_t *
881 fmd_asru_hash_create_entry(fmd_asru_hash_t *ahp, fmd_case_t *cp, nvlist_t *nvl)
882 {
883 	char *parsed_uuid;
884 	uuid_t uuid;
885 	int uuidlen;
886 	fmd_asru_link_t *alp;
887 
888 	/*
889 	 * Generate a UUID for the ASRU.  libuuid cleverly gives us no
890 	 * interface for specifying or learning the buffer size.  Sigh.
891 	 * The spec says 36 bytes but we use a tunable just to be safe.
892 	 */
893 	(void) fmd_conf_getprop(fmd.d_conf, "uuidlen", &uuidlen);
894 	parsed_uuid = fmd_zalloc(uuidlen + 1, FMD_SLEEP);
895 	uuid_generate(uuid);
896 	uuid_unparse(uuid, parsed_uuid);
897 
898 	/*
899 	 * Now create the resource cache entries.
900 	 */
901 	fmd_case_hold_locked(cp);
902 	alp = fmd_asru_al_create(ahp, nvl, cp, parsed_uuid);
903 	TRACE((FMD_DBG_ASRU, "asru %s created as %p",
904 	    alp->al_uuid, (void *)alp->al_asru));
905 
906 	fmd_free(parsed_uuid, uuidlen + 1);
907 	return (alp);
908 
909 }
910 
911 /*
912  * Release the reference count on an asru obtained using fmd_asru_hash_lookup.
913  * We take 'ahp' for symmetry and in case we need to use it in future work.
914  */
915 /*ARGSUSED*/
916 void
917 fmd_asru_hash_release(fmd_asru_hash_t *ahp, fmd_asru_t *ap)
918 {
919 	(void) pthread_mutex_lock(&ap->asru_lock);
920 
921 	ASSERT(ap->asru_refs != 0);
922 	if (--ap->asru_refs == 0)
923 		fmd_asru_destroy(ap);
924 	else
925 		(void) pthread_mutex_unlock(&ap->asru_lock);
926 }
927 
928 static void
929 fmd_asru_do_delete_entry(fmd_asru_hash_t *ahp, fmd_case_t *cp,
930     fmd_asru_link_t **hash, size_t next_offset, char *name)
931 {
932 	uint_t h;
933 	fmd_asru_link_t *alp, **pp, *alpnext, **alpnextp;
934 
935 	(void) pthread_rwlock_wrlock(&ahp->ah_lock);
936 	h = fmd_strhash(name) % ahp->ah_hashlen;
937 	pp = &hash[h];
938 	for (alp = *pp; alp != NULL; alp = alpnext) {
939 		/* LINTED pointer alignment */
940 		alpnextp = FMD_ASRU_AL_HASH_NEXTP(alp, next_offset);
941 		alpnext = *alpnextp;
942 		if (alp->al_case == cp) {
943 			*pp = *alpnextp;
944 			*alpnextp = NULL;
945 		} else
946 			pp = alpnextp;
947 	}
948 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
949 }
950 
951 static void
952 fmd_asru_do_hash_delete(fmd_asru_hash_t *ahp, fmd_case_susp_t *cis,
953     fmd_case_t *cp, fmd_asru_link_t **hash, size_t next_offset, char *nvname)
954 {
955 	nvlist_t *nvl;
956 	char *name = NULL;
957 	ssize_t namelen;
958 
959 	if (nvlist_lookup_nvlist(cis->cis_nvl, nvname, &nvl) == 0 &&
960 	    (namelen = fmd_fmri_nvl2str(nvl, NULL, 0)) != -1 &&
961 	    (name = fmd_alloc(namelen + 1, FMD_SLEEP)) != NULL) {
962 		if (fmd_fmri_nvl2str(nvl, name, namelen + 1) != -1)
963 			fmd_asru_do_delete_entry(ahp, cp, hash, next_offset,
964 			    name);
965 		fmd_free(name, namelen + 1);
966 	} else
967 		fmd_asru_do_delete_entry(ahp, cp, hash, next_offset, "");
968 }
969 
970 void
971 fmd_asru_hash_delete_case(fmd_asru_hash_t *ahp, fmd_case_t *cp)
972 {
973 	fmd_case_impl_t *cip = (fmd_case_impl_t *)cp;
974 	fmd_case_susp_t *cis;
975 	fmd_asru_link_t *alp, **plp, *alpnext;
976 	fmd_asru_t *ap;
977 	char path[PATH_MAX];
978 	char *label;
979 	uint_t h;
980 
981 	/*
982 	 * first delete hash entries for each suspect
983 	 */
984 	for (cis = cip->ci_suspects; cis != NULL; cis = cis->cis_next) {
985 		fmd_asru_do_hash_delete(ahp, cis, cp, ahp->ah_fru_hash,
986 		    offsetof(fmd_asru_link_t, al_fru_next), FM_FAULT_FRU);
987 		fmd_asru_do_hash_delete(ahp, cis, cp, ahp->ah_rsrc_hash,
988 		    offsetof(fmd_asru_link_t, al_rsrc_next), FM_FAULT_RESOURCE);
989 		if (nvlist_lookup_string(cis->cis_nvl, FM_FAULT_LOCATION,
990 		    &label) != 0)
991 			label = "";
992 		fmd_asru_do_delete_entry(ahp, cp, ahp->ah_label_hash,
993 		    offsetof(fmd_asru_link_t, al_label_next), label);
994 		fmd_asru_do_hash_delete(ahp, cis, cp, ahp->ah_asru_hash,
995 		    offsetof(fmd_asru_link_t, al_asru_next), FM_FAULT_ASRU);
996 	}
997 
998 	/*
999 	 * then delete associated case hash entries
1000 	 */
1001 	(void) pthread_rwlock_wrlock(&ahp->ah_lock);
1002 	h = fmd_strhash(cip->ci_uuid) % ahp->ah_hashlen;
1003 	plp = &ahp->ah_case_hash[h];
1004 	for (alp = *plp; alp != NULL; alp = alpnext) {
1005 		alpnext = alp->al_case_next;
1006 		if (alp->al_case == cp) {
1007 			*plp = alp->al_case_next;
1008 			alp->al_case_next = NULL;
1009 			ASSERT(ahp->ah_al_count != 0);
1010 			ahp->ah_al_count--;
1011 
1012 			/*
1013 			 * decrement case ref.
1014 			 */
1015 			fmd_case_rele_locked(cp);
1016 			alp->al_case = NULL;
1017 
1018 			/*
1019 			 * If we found a matching ASRU, unlink its log file and
1020 			 * then release the hash entry. Note that it may still
1021 			 * be referenced if another thread is manipulating it;
1022 			 * this is ok because once we unlink, the log file will
1023 			 * not be restored, and the log data will be freed when
1024 			 * all of the referencing threads release their
1025 			 * respective references.
1026 			 */
1027 			(void) snprintf(path, sizeof (path), "%s/%s",
1028 			    ahp->ah_dirpath, alp->al_uuid);
1029 			if (unlink(path) != 0)
1030 				fmd_error(EFMD_ASRU_UNLINK,
1031 				    "failed to unlink asru %s", path);
1032 
1033 			/*
1034 			 * Now unlink from the global per-resource cache
1035 			 * and if this is the last link then remove that from
1036 			 * it's own hash too.
1037 			 */
1038 			ap = alp->al_asru;
1039 			(void) pthread_mutex_lock(&ap->asru_lock);
1040 			fmd_list_delete(&ap->asru_list, alp);
1041 			if (ap->asru_list.l_next == NULL) {
1042 				uint_t h;
1043 				fmd_asru_t *ap2, **pp;
1044 				fmd_asru_t *apnext, **apnextp;
1045 
1046 				ASSERT(ahp->ah_count != 0);
1047 				ahp->ah_count--;
1048 				h = fmd_strhash(ap->asru_name) %
1049 				    ahp->ah_hashlen;
1050 					pp = &ahp->ah_hash[h];
1051 				for (ap2 = *pp; ap2 != NULL; ap2 = apnext) {
1052 					apnextp = &ap2->asru_next;
1053 					apnext = *apnextp;
1054 					if (ap2 == ap) {
1055 						*pp = *apnextp;
1056 						*apnextp = NULL;
1057 					} else
1058 						pp = apnextp;
1059 				}
1060 			}
1061 			(void) pthread_mutex_unlock(&ap->asru_lock);
1062 			fmd_asru_al_hash_release(ahp, alp);
1063 		} else
1064 			plp = &alp->al_case_next;
1065 	}
1066 	(void) pthread_rwlock_unlock(&ahp->ah_lock);
1067 }
1068 
1069 static void
1070 fmd_asru_repair_containee(fmd_asru_link_t *alp, void *er)
1071 {
1072 	if (er && alp->al_asru_fmri && fmd_fmri_contains(er,
1073 	    alp->al_asru_fmri) > 0 && fmd_asru_clrflags(alp, FMD_ASRU_FAULTY))
1074 		fmd_case_update(alp->al_case);
1075 }
1076 
1077 void
1078 fmd_asru_repair(fmd_asru_link_t *alp, void *er)
1079 {
1080 	int flags;
1081 	int rval;
1082 
1083 	/*
1084 	 * repair this asru cache entry
1085 	 */
1086 	rval = fmd_asru_clrflags(alp, FMD_ASRU_FAULTY);
1087 
1088 	/*
1089 	 * now check if all entries associated with this asru are repaired and
1090 	 * if so repair containees
1091 	 */
1092 	(void) pthread_mutex_lock(&alp->al_asru->asru_lock);
1093 	flags = alp->al_asru->asru_flags;
1094 	(void) pthread_mutex_unlock(&alp->al_asru->asru_lock);
1095 	if (!(flags & FMD_ASRU_FAULTY))
1096 		fmd_asru_al_hash_apply(fmd.d_asrus, fmd_asru_repair_containee,
1097 		    alp->al_asru_fmri);
1098 
1099 	/*
1100 	 * if called from fmd_adm_repair() and we really did clear the bit then
1101 	 * we need to do a case update to see if the associated case can be
1102 	 * repaired. No need to do this if called from fmd_case_repair() (ie
1103 	 * when er is NULL) as the case will be explicitly repaired anyway.
1104 	 */
1105 	if (er) {
1106 		*(int *)er = 0;
1107 		if (rval)
1108 			fmd_case_update(alp->al_case);
1109 	}
1110 }
1111 
1112 static void
1113 fmd_asru_logevent(fmd_asru_link_t *alp)
1114 {
1115 	fmd_asru_t *ap = alp->al_asru;
1116 	boolean_t f = (ap->asru_flags & FMD_ASRU_FAULTY) != 0;
1117 	boolean_t u = (ap->asru_flags & FMD_ASRU_UNUSABLE) != 0;
1118 	boolean_t m = (ap->asru_flags & FMD_ASRU_INVISIBLE) == 0;
1119 
1120 	fmd_case_impl_t *cip;
1121 	fmd_event_t *e;
1122 	fmd_log_t *lp;
1123 	nvlist_t *nvl;
1124 	char *class;
1125 
1126 	ASSERT(MUTEX_HELD(&ap->asru_lock));
1127 	cip = (fmd_case_impl_t *)alp->al_case;
1128 	ASSERT(cip != NULL);
1129 
1130 	if ((lp = alp->al_log) == NULL)
1131 		lp = fmd_log_open(ap->asru_root, alp->al_uuid, FMD_LOG_ASRU);
1132 
1133 	if (lp == NULL)
1134 		return; /* can't log events if we can't open the log */
1135 
1136 	nvl = fmd_protocol_rsrc_asru(_fmd_asru_events[f | (u << 1)],
1137 	    alp->al_asru_fmri, cip->ci_uuid, cip->ci_code, f, u, m,
1138 	    alp->al_event, &cip->ci_tv);
1139 
1140 	(void) nvlist_lookup_string(nvl, FM_CLASS, &class);
1141 	e = fmd_event_create(FMD_EVT_PROTOCOL, FMD_HRT_NOW, nvl, class);
1142 
1143 	fmd_event_hold(e);
1144 	fmd_log_append(lp, e, NULL);
1145 	fmd_event_rele(e);
1146 
1147 	/*
1148 	 * For now, we close the log file after every update to conserve file
1149 	 * descriptors and daemon overhead.  If this becomes a performance
1150 	 * issue this code can change to keep a fixed-size LRU cache of logs.
1151 	 */
1152 	fmd_log_rele(lp);
1153 	alp->al_log = NULL;
1154 }
1155 
1156 int
1157 fmd_asru_setflags(fmd_asru_link_t *alp, uint_t sflag)
1158 {
1159 	fmd_asru_t *ap = alp->al_asru;
1160 	uint_t nstate, ostate;
1161 
1162 	ASSERT(!(sflag & ~FMD_ASRU_STATE));
1163 	ASSERT(sflag != FMD_ASRU_STATE);
1164 
1165 	(void) pthread_mutex_lock(&ap->asru_lock);
1166 
1167 	ostate = alp->al_flags & FMD_ASRU_STATE;
1168 	alp->al_flags |= sflag;
1169 	nstate = alp->al_flags & FMD_ASRU_STATE;
1170 
1171 	if (nstate == ostate) {
1172 		(void) pthread_mutex_unlock(&ap->asru_lock);
1173 		return (0);
1174 	}
1175 
1176 	ap->asru_flags |= sflag;
1177 	TRACE((FMD_DBG_ASRU, "asru %s %s->%s", alp->al_uuid,
1178 	    _fmd_asru_snames[ostate], _fmd_asru_snames[nstate]));
1179 
1180 	fmd_asru_logevent(alp);
1181 
1182 	(void) pthread_cond_broadcast(&ap->asru_cv);
1183 	(void) pthread_mutex_unlock(&ap->asru_lock);
1184 	return (1);
1185 }
1186 
1187 int
1188 fmd_asru_clrflags(fmd_asru_link_t *alp, uint_t sflag)
1189 {
1190 	fmd_asru_t *ap = alp->al_asru;
1191 	fmd_asru_link_t *nalp;
1192 	uint_t nstate, ostate, flags = 0;
1193 
1194 	ASSERT(!(sflag & ~FMD_ASRU_STATE));
1195 	ASSERT(sflag != FMD_ASRU_STATE);
1196 
1197 	(void) pthread_mutex_lock(&ap->asru_lock);
1198 
1199 	ostate = alp->al_flags & FMD_ASRU_STATE;
1200 	alp->al_flags &= ~sflag;
1201 	nstate = alp->al_flags & FMD_ASRU_STATE;
1202 
1203 	if (nstate == ostate) {
1204 		(void) pthread_mutex_unlock(&ap->asru_lock);
1205 		return (0);
1206 	}
1207 
1208 	if (sflag == FMD_ASRU_UNUSABLE)
1209 		ap->asru_flags &= ~sflag;
1210 	else if (sflag == FMD_ASRU_FAULTY) {
1211 		/*
1212 		 * only clear the faulty bit if all links are clear
1213 		 */
1214 		for (nalp = fmd_list_next(&ap->asru_list); nalp != NULL;
1215 		    nalp = fmd_list_next(nalp))
1216 			flags |= nalp->al_flags;
1217 		if (!(flags & FMD_ASRU_FAULTY))
1218 			ap->asru_flags &= ~sflag;
1219 	}
1220 
1221 	TRACE((FMD_DBG_ASRU, "asru %s %s->%s", alp->al_uuid,
1222 	    _fmd_asru_snames[ostate], _fmd_asru_snames[nstate]));
1223 
1224 	fmd_asru_logevent(alp);
1225 
1226 	(void) pthread_cond_broadcast(&ap->asru_cv);
1227 	(void) pthread_mutex_unlock(&ap->asru_lock);
1228 
1229 	return (1);
1230 }
1231 
1232 /*
1233  * Report the current known state of the link entry (ie this particular fault
1234  * affecting this particular ASRU).
1235  */
1236 int
1237 fmd_asru_al_getstate(fmd_asru_link_t *alp)
1238 {
1239 	int us, st;
1240 	nvlist_t *asru;
1241 
1242 	if (fmd_asru_is_present(alp->al_event) == 0)
1243 		return ((alp->al_flags & FMD_ASRU_FAULTY) | FMD_ASRU_UNUSABLE);
1244 
1245 	if (nvlist_lookup_nvlist(alp->al_event, FM_FAULT_ASRU, &asru) == 0)
1246 		us = fmd_fmri_unusable(asru);
1247 	else
1248 		us = (alp->al_flags & FMD_ASRU_UNUSABLE);
1249 	st = (alp->al_flags & FMD_ASRU_STATE) | FMD_ASRU_PRESENT;
1250 	if (us > 0)
1251 		st |= FMD_ASRU_UNUSABLE;
1252 	else if (us == 0)
1253 		st &= ~FMD_ASRU_UNUSABLE;
1254 	return (st);
1255 }
1256 
1257 /*
1258  * Report the current known state of the ASRU by refreshing its unusable status
1259  * based upon the routines provided by the scheme module.  If the unusable bit
1260  * is different, we do *not* generate a state change here because that change
1261  * may be unrelated to fmd activities and therefore we have no case or event.
1262  * The absence of the transition is harmless as this function is only provided
1263  * for RPC observability and fmd's clients are only concerned with ASRU_FAULTY.
1264  */
1265 int
1266 fmd_asru_getstate(fmd_asru_t *ap)
1267 {
1268 	int us, st;
1269 
1270 	if (!(ap->asru_flags & FMD_ASRU_INTERNAL) &&
1271 	    (fmd_asru_fake_not_present || fmd_fmri_present(ap->asru_fmri) <= 0))
1272 		return (0); /* do not report non-fmd non-present resources */
1273 
1274 	us = fmd_fmri_unusable(ap->asru_fmri);
1275 	st = ap->asru_flags & FMD_ASRU_STATE;
1276 
1277 	if (us > 0)
1278 		st |= FMD_ASRU_UNUSABLE;
1279 	else if (us == 0)
1280 		st &= ~FMD_ASRU_UNUSABLE;
1281 
1282 	return (st);
1283 }
1284