fmd/common/fmd_asru.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/fm/protocol.h>
#include <uuid/uuid.h>

#include <dirent.h>
#include <limits.h>
#include <unistd.h>
#include <alloca.h>
#include <stddef.h>
#include <fm/libtopo.h>

#include <fmd_alloc.h>
#include <fmd_string.h>
#include <fmd_error.h>
#include <fmd_subr.h>
#include <fmd_protocol.h>
#include <fmd_event.h>
#include <fmd_conf.h>
#include <fmd_fmri.h>
#include <fmd_dispq.h>
#include <fmd_case.h>
#include <fmd_module.h>
#include <fmd_asru.h>

#include <fmd.h>

static const char *const _fmd_asru_events[] = {
	FMD_RSRC_CLASS "asru.ok",		/* UNUSABLE=0 FAULTED=0 */
	FMD_RSRC_CLASS "asru.degraded",		/* UNUSABLE=0 FAULTED=1 */
	FMD_RSRC_CLASS "asru.unknown",		/* UNUSABLE=1 FAULTED=0 */
	FMD_RSRC_CLASS "asru.faulted"		/* UNUSABLE=1 FAULTED=1 */
};

static const char *const _fmd_asru_snames[] = {
	"uf", "uF", "Uf", "UF"			/* same order as above */
};

volatile uint32_t fmd_asru_fake_not_present = 0;

static uint_t
fmd_asru_strhash(fmd_asru_hash_t *ahp, const char *val)
{
	return (topo_fmri_strhash(ahp->ah_topo->ft_hdl, val) % ahp->ah_hashlen);
}

static boolean_t
fmd_asru_strcmp(fmd_asru_hash_t *ahp, const char *a, const char *b)
{
	return (topo_fmri_strcmp(ahp->ah_topo->ft_hdl, a, b));
}

static fmd_asru_t *
fmd_asru_create(fmd_asru_hash_t *ahp, const char *uuid,
    const char *name, nvlist_t *fmri)
{
	fmd_asru_t *ap = fmd_zalloc(sizeof (fmd_asru_t), FMD_SLEEP);
	char *s;

	(void) pthread_mutex_init(&ap->asru_lock, NULL);
	(void) pthread_cond_init(&ap->asru_cv, NULL);

	ap->asru_name = fmd_strdup(name, FMD_SLEEP);
	if (fmri)
		(void) nvlist_xdup(fmri, &ap->asru_fmri, &fmd.d_nva);
	ap->asru_root = fmd_strdup(ahp->ah_dirpath, FMD_SLEEP);
	ap->asru_uuid = fmd_strdup(uuid, FMD_SLEEP);
	ap->asru_uuidlen = ap->asru_uuid ? strlen(ap->asru_uuid) : 0;
	ap->asru_refs = 1;

	if (fmri && nvlist_lookup_string(fmri, FM_FMRI_SCHEME, &s) == 0 &&
	    strcmp(s, FM_FMRI_SCHEME_FMD) == 0)
		ap->asru_flags |= FMD_ASRU_INTERNAL;

	return (ap);
}

static void
fmd_asru_destroy(fmd_asru_t *ap)
{
	ASSERT(MUTEX_HELD(&ap->asru_lock));
	ASSERT(ap->asru_refs == 0);

	nvlist_free(ap->asru_event);
	fmd_strfree(ap->asru_name);
	nvlist_free(ap->asru_fmri);
	fmd_strfree(ap->asru_root);
	fmd_free(ap->asru_uuid, ap->asru_uuidlen + 1);
	fmd_free(ap, sizeof (fmd_asru_t));
}

static void
fmd_asru_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_t *ap)
{
	uint_t h = fmd_asru_strhash(ahp, ap->asru_name);

	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
	ap->asru_next = ahp->ah_hash[h];
	ahp->ah_hash[h] = ap;
	ahp->ah_count++;
}

static fmd_asru_t *
fmd_asru_hold(fmd_asru_t *ap)
{
	(void) pthread_mutex_lock(&ap->asru_lock);
	ap->asru_refs++;
	ASSERT(ap->asru_refs != 0);
	(void) pthread_mutex_unlock(&ap->asru_lock);
	return (ap);
}

/*
 * Lookup an asru in the hash by name and place a hold on it.  If the asru is
 * not found, no entry is created and NULL is returned.  This internal function
 * is for callers who have the ah_lock held and is used by lookup_name below.
 */
fmd_asru_t *
fmd_asru_hash_lookup(fmd_asru_hash_t *ahp, const char *name)
{
	fmd_asru_t *ap;
	uint_t h;

	ASSERT(RW_LOCK_HELD(&ahp->ah_lock));
	h = fmd_asru_strhash(ahp, name);

	for (ap = ahp->ah_hash[h]; ap != NULL; ap = ap->asru_next) {
		if (fmd_asru_strcmp(ahp, ap->asru_name, name))
			break;
	}

	if (ap != NULL)
		(void) fmd_asru_hold(ap);
	else
		(void) fmd_set_errno(EFMD_ASRU_NOENT);

	return (ap);
}

#define	HC_ONLY_FALSE	0
#define	HC_ONLY_TRUE	1

static int
fmd_asru_replacement_state(nvlist_t *event, int hc_only)
{
	int ps = -1;
	nvlist_t *asru, *fru, *rsrc;
	char *s;

	/*
	 * Check if there is evidence that this object is no longer present.
	 * In general fmd_fmri_present() should be supported on resources and/or
	 * frus, as those are the things that are physically present or not
	 * present - an asru can be spread over a number of frus some of which
	 * are present and some not, so fmd_fmri_present() is not generally
	 * meaningful. However retain a check for asru first for compatibility.
	 * If we have checked all three and we still get -1 then nothing knows
	 * whether it's present or not, so err on the safe side and treat it
	 * as still present.
	 *
	 * Note that if hc_only is set, then we only check status using fmris
	 * that are in hc-scheme.
	 */
	if (fmd_asru_fake_not_present)
		return (fmd_asru_fake_not_present);
	if (nvlist_lookup_nvlist(event, FM_FAULT_ASRU, &asru) == 0 &&
	    (hc_only == HC_ONLY_FALSE || (nvlist_lookup_string(asru,
	    FM_FMRI_SCHEME, &s) == 0 && strcmp(s, FM_FMRI_SCHEME_HC) == 0)))
		ps = fmd_fmri_replaced(asru);
	if (ps == -1 || ps == FMD_OBJ_STATE_UNKNOWN) {
		if (nvlist_lookup_nvlist(event, FM_FAULT_RESOURCE,
		    &rsrc) == 0 && (hc_only == HC_ONLY_FALSE ||
		    (nvlist_lookup_string(rsrc, FM_FMRI_SCHEME, &s) == 0 &&
		    strcmp(s, FM_FMRI_SCHEME_HC) == 0))) {
			if (ps == -1) {
				ps = fmd_fmri_replaced(rsrc);
			} else {
				/* see if we can improve on UNKNOWN */
				int ps2 = fmd_fmri_replaced(rsrc);
				if (ps2 == FMD_OBJ_STATE_STILL_PRESENT ||
				    ps2 == FMD_OBJ_STATE_REPLACED)
					ps = ps2;
			}
		}
	}
	if (ps == -1 || ps == FMD_OBJ_STATE_UNKNOWN) {
		if (nvlist_lookup_nvlist(event, FM_FAULT_FRU, &fru) == 0 &&
		    (hc_only == HC_ONLY_FALSE || (nvlist_lookup_string(fru,
		    FM_FMRI_SCHEME, &s) == 0 &&
		    strcmp(s, FM_FMRI_SCHEME_HC) == 0))) {
			if (ps == -1) {
				ps = fmd_fmri_replaced(fru);
			} else {
				/* see if we can improve on UNKNOWN */
				int ps2 = fmd_fmri_replaced(fru);
				if (ps2 == FMD_OBJ_STATE_STILL_PRESENT ||
				    ps2 == FMD_OBJ_STATE_REPLACED)
					ps = ps2;
			}
		}
	}
	if (ps == -1)
		ps = FMD_OBJ_STATE_UNKNOWN;
	return (ps);
}

static void
fmd_asru_asru_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
    char *name)
{
	uint_t h = fmd_asru_strhash(ahp, name);

	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
	alp->al_asru_next = ahp->ah_asru_hash[h];
	ahp->ah_asru_hash[h] = alp;
	ahp->ah_al_count++;
}

static void
fmd_asru_case_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
    char *name)
{
	uint_t h = fmd_asru_strhash(ahp, name);

	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
	alp->al_case_next = ahp->ah_case_hash[h];
	ahp->ah_case_hash[h] = alp;
}

static void
fmd_asru_fru_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp, char *name)
{
	uint_t h = fmd_asru_strhash(ahp, name);

	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
	alp->al_fru_next = ahp->ah_fru_hash[h];
	ahp->ah_fru_hash[h] = alp;
}

static void
fmd_asru_label_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
    char *name)
{
	uint_t h = fmd_asru_strhash(ahp, name);

	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
	alp->al_label_next = ahp->ah_label_hash[h];
	ahp->ah_label_hash[h] = alp;
}

static void
fmd_asru_rsrc_hash_insert(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp,
    char *name)
{
	uint_t h = fmd_asru_strhash(ahp, name);

	ASSERT(RW_WRITE_HELD(&ahp->ah_lock));
	alp->al_rsrc_next = ahp->ah_rsrc_hash[h];
	ahp->ah_rsrc_hash[h] = alp;
}

static void
fmd_asru_al_destroy(fmd_asru_link_t *alp)
{
	ASSERT(alp->al_refs == 0);
	ASSERT(MUTEX_HELD(&alp->al_asru->asru_lock));

	if (alp->al_log != NULL)
		fmd_log_rele(alp->al_log);

	fmd_free(alp->al_uuid, alp->al_uuidlen + 1);
	nvlist_free(alp->al_event);
	fmd_strfree(alp->al_rsrc_name);
	fmd_strfree(alp->al_case_uuid);
	fmd_strfree(alp->al_fru_name);
	fmd_strfree(alp->al_asru_name);
	fmd_strfree(alp->al_label);
	nvlist_free(alp->al_asru_fmri);
	fmd_free(alp, sizeof (fmd_asru_link_t));
}

static fmd_asru_link_t *
fmd_asru_al_hold(fmd_asru_link_t *alp)
{
	fmd_asru_t *ap = alp->al_asru;

	(void) pthread_mutex_lock(&ap->asru_lock);
	ap->asru_refs++;
	alp->al_refs++;
	ASSERT(alp->al_refs != 0);
	(void) pthread_mutex_unlock(&ap->asru_lock);
	return (alp);
}

static void fmd_asru_destroy(fmd_asru_t *ap);

/*ARGSUSED*/
static void
fmd_asru_al_hash_release(fmd_asru_hash_t *ahp, fmd_asru_link_t *alp)
{
	fmd_asru_t *ap = alp->al_asru;

	(void) pthread_mutex_lock(&ap->asru_lock);
	ASSERT(alp->al_refs != 0);
	if (--alp->al_refs == 0)
		fmd_asru_al_destroy(alp);
	ASSERT(ap->asru_refs != 0);
	if (--ap->asru_refs == 0)
		fmd_asru_destroy(ap);
	else
		(void) pthread_mutex_unlock(&ap->asru_lock);
}

static int
fmd_asru_get_namestr(nvlist_t *nvl, char **name, ssize_t *namelen)
{
	if ((*namelen = fmd_fmri_nvl2str(nvl, NULL, 0)) == -1)
		return (EFMD_ASRU_FMRI);
	*name = fmd_alloc(*namelen + 1, FMD_SLEEP);
	if (fmd_fmri_nvl2str(nvl, *name, *namelen + 1) == -1) {
		if (*name != NULL)
			fmd_free(*name, *namelen + 1);
		return (EFMD_ASRU_FMRI);
	}
	return (0);
}

static fmd_asru_link_t *
fmd_asru_al_create(fmd_asru_hash_t *ahp, nvlist_t *nvl, fmd_case_t *cp,
    const char *al_uuid)
{
	nvlist_t *asru = NULL, *fru, *rsrc;
	int got_rsrc = 0, got_asru = 0, got_fru = 0;
	ssize_t fru_namelen, rsrc_namelen, asru_namelen;
	char *asru_name, *rsrc_name, *fru_name, *name, *label;
	fmd_asru_link_t *alp;
	fmd_asru_t *ap;
	boolean_t msg;
	fmd_case_impl_t *cip = (fmd_case_impl_t *)cp;

	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &asru) == 0 &&
	    fmd_asru_get_namestr(asru, &asru_name, &asru_namelen) == 0)
		got_asru = 1;
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &fru) == 0 &&
	    fmd_asru_get_namestr(fru, &fru_name, &fru_namelen) == 0)
		got_fru = 1;
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0 &&
	    fmd_asru_get_namestr(rsrc, &rsrc_name, &rsrc_namelen) == 0)
		got_rsrc = 1;
	if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION, &label) != 0)
		label = "";

	/*
	 * Grab the rwlock as a writer; Then create and insert the asru with
	 * ahp->ah_lock held and hash it in. We'll then drop the rwlock and
	 * proceed to initializing the asru.
	 */
	(void) pthread_rwlock_wrlock(&ahp->ah_lock);

	/*
	 * Create and initialise the per-fault "link" structure.
	 */
	alp = fmd_zalloc(sizeof (fmd_asru_link_t), FMD_SLEEP);
	if (got_asru)
		(void) nvlist_xdup(asru, &alp->al_asru_fmri, &fmd.d_nva);
	alp->al_uuid = fmd_strdup(al_uuid, FMD_SLEEP);
	alp->al_uuidlen = strlen(alp->al_uuid);
	alp->al_refs = 1;

	/*
	 * If this is the first fault for this asru, then create the per-asru
	 * structure and link into the hash.
	 */
	name = got_asru ? asru_name : "";
	if ((ap = fmd_asru_hash_lookup(ahp, name)) == NULL) {
		ap = fmd_asru_create(ahp, al_uuid, name, got_asru ? asru :
		    NULL);
		fmd_asru_hash_insert(ahp, ap);
	} else
		nvlist_free(ap->asru_event);
	(void) nvlist_xdup(nvl, &ap->asru_event, &fmd.d_nva);

	/*
	 * Put the link structure on the list associated with the per-asru
	 * structure. Then put the link structure on the various hashes.
	 */
	fmd_list_append(&ap->asru_list, (fmd_list_t *)alp);
	alp->al_asru = ap;
	alp->al_asru_name = got_asru ? asru_name : fmd_strdup("", FMD_SLEEP);
	fmd_asru_asru_hash_insert(ahp, alp, alp->al_asru_name);
	alp->al_fru_name = got_fru ? fru_name : fmd_strdup("", FMD_SLEEP);
	fmd_asru_fru_hash_insert(ahp, alp, alp->al_fru_name);
	alp->al_rsrc_name = got_rsrc ? rsrc_name : fmd_strdup("", FMD_SLEEP);
	fmd_asru_rsrc_hash_insert(ahp, alp, alp->al_rsrc_name);
	alp->al_label = fmd_strdup(label, FMD_SLEEP);
	fmd_asru_label_hash_insert(ahp, alp, label);
	alp->al_case_uuid = fmd_strdup(cip->ci_uuid, FMD_SLEEP);
	fmd_asru_case_hash_insert(ahp, alp, cip->ci_uuid);
	(void) pthread_mutex_lock(&ap->asru_lock);
	(void) pthread_rwlock_unlock(&ahp->ah_lock);

	ap->asru_case = alp->al_case = cp;
	if (nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE, &msg) == 0 &&
	    msg == B_FALSE)
		ap->asru_flags |= FMD_ASRU_INVISIBLE;
	(void) nvlist_xdup(nvl, &alp->al_event, &fmd.d_nva);
	ap->asru_flags |= FMD_ASRU_VALID;
	(void) pthread_cond_broadcast(&ap->asru_cv);
	(void) pthread_mutex_unlock(&ap->asru_lock);
	return (alp);
}

static void
fmd_asru_hash_recreate(fmd_log_t *lp, fmd_event_t *ep, fmd_asru_hash_t *ahp)
{
	nvlist_t *nvl = FMD_EVENT_NVL(ep);
	boolean_t faulty = FMD_B_FALSE, unusable = FMD_B_FALSE;
	int ps;
	boolean_t repaired = FMD_B_FALSE, replaced = FMD_B_FALSE;
	boolean_t acquitted = FMD_B_FALSE, resolved = FMD_B_FALSE;
	nvlist_t *flt, *flt_copy, *asru;
	char *case_uuid = NULL, *case_code = NULL;
	fmd_asru_t *ap;
	fmd_asru_link_t *alp;
	fmd_case_t *cp;
	int64_t *diag_time;
	nvlist_t *de_fmri, *de_fmri_dup;
	uint_t nelem;
	topo_hdl_t *thp;
	char *class;
	nvlist_t *rsrc;
	int err;
	boolean_t injected;

	/*
	 * Extract the most recent values of 'faulty' from the event log.
	 */
	if (nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_FAULTY,
	    &faulty) != 0) {
		fmd_error(EFMD_ASRU_EVENT, "failed to reload asru %s: "
		    "invalid event log record\n", lp->log_name);
		ahp->ah_error = EFMD_ASRU_EVENT;
		return;
	}
	if (nvlist_lookup_nvlist(nvl, FM_RSRC_ASRU_EVENT, &flt) != 0) {
		fmd_error(EFMD_ASRU_EVENT, "failed to reload asru %s: "
		    "invalid event log record\n", lp->log_name);
		ahp->ah_error = EFMD_ASRU_EVENT;
		return;
	}
	(void) nvlist_lookup_string(nvl, FM_RSRC_ASRU_UUID, &case_uuid);
	(void) nvlist_lookup_string(nvl, FM_RSRC_ASRU_CODE, &case_code);
	(void) nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_UNUSABLE,
	    &unusable);
	(void) nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_REPAIRED,
	    &repaired);
	(void) nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_REPLACED,
	    &replaced);
	(void) nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_ACQUITTED,
	    &acquitted);
	(void) nvlist_lookup_boolean_value(nvl, FM_RSRC_ASRU_RESOLVED,
	    &resolved);

	/*
	 * Attempt to recreate the case in CLOSED, REPAIRED or RESOLVED state
	 * (depending on whether the faulty/resolved bits are set).
	 * If the case is already present, fmd_case_recreate() will return it.
	 * If not, we'll create a new orphaned case. Either way,  we use the
	 * ASRU event to insert a suspect into the partially-restored case.
	 */
	fmd_module_lock(fmd.d_rmod);
	cp = fmd_case_recreate(fmd.d_rmod, NULL, faulty ? FMD_CASE_CLOSED :
	    resolved ? FMD_CASE_RESOLVED : FMD_CASE_REPAIRED, case_uuid,
	    case_code);
	fmd_case_hold(cp);
	fmd_module_unlock(fmd.d_rmod);
	if (nvlist_lookup_boolean_value(nvl, FM_SUSPECT_INJECTED,
	    &injected) == 0 && injected)
		fmd_case_set_injected(cp);
	if (nvlist_lookup_int64_array(nvl, FM_SUSPECT_DIAG_TIME, &diag_time,
	    &nelem) == 0 && nelem >= 2)
		fmd_case_settime(cp, diag_time[0], diag_time[1]);
	else
		fmd_case_settime(cp, lp->log_stat.st_ctime, 0);
	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &de_fmri) == 0) {
		(void) nvlist_xdup(de_fmri, &de_fmri_dup, &fmd.d_nva);
		fmd_case_set_de_fmri(cp, de_fmri_dup);
	}
	(void) nvlist_xdup(flt, &flt_copy, &fmd.d_nva);

	/*
	 * For faults with a resource, re-evaluate the asru from the resource.
	 */
	thp = fmd_fmri_topo_hold(TOPO_VERSION);
	if (nvlist_lookup_string(flt_copy, FM_CLASS, &class) == 0 &&
	    strncmp(class, "fault", 5) == 0 &&
	    nvlist_lookup_nvlist(flt_copy, FM_FAULT_RESOURCE, &rsrc) == 0 &&
	    rsrc != NULL &&
	    (fmd_fmri_replaced(rsrc) != FMD_OBJ_STATE_REPLACED) &&
	    topo_fmri_asru(thp, rsrc, &asru, &err) == 0) {
		(void) nvlist_remove(flt_copy, FM_FAULT_ASRU, DATA_TYPE_NVLIST);
		(void) nvlist_add_nvlist(flt_copy, FM_FAULT_ASRU, asru);
		nvlist_free(asru);
	}
	fmd_fmri_topo_rele(thp);

	(void) nvlist_xdup(flt_copy, &flt, &fmd.d_nva);

	fmd_case_recreate_suspect(cp, flt_copy);

	/*
	 * Now create the resource cache entries.
	 */
	alp = fmd_asru_al_create(ahp, flt, cp, fmd_strbasename(lp->log_name));
	ap = alp->al_asru;

	/*
	 * Check to see if the resource is still present in the system.
	 */
	ps = fmd_asru_replacement_state(flt, HC_ONLY_FALSE);
	if (ps == FMD_OBJ_STATE_REPLACED) {
		replaced = FMD_B_TRUE;
	} else if (ps == FMD_OBJ_STATE_STILL_PRESENT ||
	    ps == FMD_OBJ_STATE_UNKNOWN) {
		ap->asru_flags |= FMD_ASRU_PRESENT;
		if (nvlist_lookup_nvlist(alp->al_event, FM_FAULT_ASRU,
		    &asru) == 0) {
			int us;

			switch (fmd_fmri_service_state(asru)) {
			case FMD_SERVICE_STATE_UNUSABLE:
				unusable = FMD_B_TRUE;
				break;
			case FMD_SERVICE_STATE_OK:
			case FMD_SERVICE_STATE_ISOLATE_PENDING:
			case FMD_SERVICE_STATE_DEGRADED:
				unusable = FMD_B_FALSE;
				break;
			case FMD_SERVICE_STATE_UNKNOWN:
			case -1:
				/* not supported by scheme */
				us = fmd_fmri_unusable(asru);
				if (us > 0)
					unusable = FMD_B_TRUE;
				else if (us == 0)
					unusable = FMD_B_FALSE;
				break;
			}
		}
	}

	nvlist_free(flt);

	ap->asru_flags |= FMD_ASRU_RECREATED;
	if (faulty) {
		alp->al_flags |= FMD_ASRU_FAULTY;
		ap->asru_flags |= FMD_ASRU_FAULTY;
	}
	if (unusable) {
		alp->al_flags |= FMD_ASRU_UNUSABLE;
		ap->asru_flags |= FMD_ASRU_UNUSABLE;
	}
	if (replaced)
		alp->al_reason = FMD_ASRU_REPLACED;
	else if (repaired)
		alp->al_reason = FMD_ASRU_REPAIRED;
	else if (acquitted)
		alp->al_reason = FMD_ASRU_ACQUITTED;
	else
		alp->al_reason = FMD_ASRU_REMOVED;

	TRACE((FMD_DBG_ASRU, "asru %s recreated as %p (%s)", alp->al_uuid,
	    (void *)ap, _fmd_asru_snames[ap->asru_flags & FMD_ASRU_STATE]));
}

static void
fmd_asru_hash_discard(fmd_asru_hash_t *ahp, const char *uuid, int err)
{
	char src[PATH_MAX], dst[PATH_MAX];

	(void) snprintf(src, PATH_MAX, "%s/%s", ahp->ah_dirpath, uuid);
	(void) snprintf(dst, PATH_MAX, "%s/%s-", ahp->ah_dirpath, uuid);

	if (err != 0)
		err = rename(src, dst);
	else
		err = unlink(src);

	if (err != 0 && errno != ENOENT)
		fmd_error(EFMD_ASRU_EVENT, "failed to rename log %s", src);
}

/*
 * Open a saved log file and restore it into the ASRU hash.  If we can't even
 * open the log, rename the log file to <uuid>- to indicate it is corrupt.  If
 * fmd_log_replay() fails, we either delete the file (if it has reached the
 * upper limit on cache age) or rename it for debugging if it was corrupted.
 */
static void
fmd_asru_hash_logopen(fmd_asru_hash_t *ahp, const char *uuid)
{
	fmd_log_t *lp = fmd_log_tryopen(ahp->ah_dirpath, uuid, FMD_LOG_ASRU);
	uint_t n;

	if (lp == NULL) {
		fmd_asru_hash_discard(ahp, uuid, errno);
		return;
	}

	ahp->ah_error = 0;
	n = ahp->ah_al_count;

	fmd_log_replay(lp, (fmd_log_f *)fmd_asru_hash_recreate, ahp);
	fmd_log_rele(lp);

	if (ahp->ah_al_count == n)
		fmd_asru_hash_discard(ahp, uuid, ahp->ah_error);
}

void
fmd_asru_hash_refresh(fmd_asru_hash_t *ahp)
{
	struct dirent *dp;
	DIR *dirp;
	int zero;

	if ((dirp = opendir(ahp->ah_dirpath)) == NULL) {
		fmd_error(EFMD_ASRU_NODIR,
		    "failed to open asru cache directory %s", ahp->ah_dirpath);
		return;
	}

	(void) fmd_conf_getprop(fmd.d_conf, "rsrc.zero", &zero);

	(void) pthread_rwlock_wrlock(&ahp->ah_lock);

	while ((dp = readdir(dirp)) != NULL) {
		if (dp->d_name[0] == '.')
			continue; /* skip "." and ".." */

		if (zero)
			fmd_asru_hash_discard(ahp, dp->d_name, 0);
		else if (!fmd_strmatch(dp->d_name, "*-"))
			fmd_asru_hash_logopen(ahp, dp->d_name);
	}

	(void) pthread_rwlock_unlock(&ahp->ah_lock);
	(void) closedir(dirp);
}

/*
 * If the resource is present and faulty but not unusable, replay the fault
 * event that caused it be marked faulty.  This will cause the agent
 * subscribing to this fault class to again disable the resource.
 */
/*ARGSUSED*/
static void
fmd_asru_hash_replay_asru(fmd_asru_t *ap, void *data)
{
	fmd_event_t *e;
	nvlist_t *nvl;
	char *class;

	if (ap->asru_event != NULL && (ap->asru_flags & (FMD_ASRU_STATE |
	    FMD_ASRU_PRESENT)) == (FMD_ASRU_FAULTY | FMD_ASRU_PRESENT)) {

		fmd_dprintf(FMD_DBG_ASRU,
		    "replaying fault event for %s", ap->asru_name);

		(void) nvlist_xdup(ap->asru_event, &nvl, &fmd.d_nva);
		(void) nvlist_lookup_string(nvl, FM_CLASS, &class);

		(void) nvlist_add_string(nvl, FMD_EVN_UUID,
		    ((fmd_case_impl_t *)ap->asru_case)->ci_uuid);

		e = fmd_event_create(FMD_EVT_PROTOCOL, FMD_HRT_NOW, nvl, class);
		fmd_dispq_dispatch(fmd.d_disp, e, class);
	}
}

void
fmd_asru_hash_replay(fmd_asru_hash_t *ahp)
{
	fmd_asru_hash_apply(ahp, fmd_asru_hash_replay_asru, NULL);
}

/*
 * Check if the resource is still present. If not, and if the rsrc.age time
 * has expired, then do an implicit repair on the resource.
 */
/*ARGSUSED*/
static void
fmd_asru_repair_if_aged(fmd_asru_link_t *alp, void *arg)
{
	struct timeval tv;
	fmd_log_t *lp;
	hrtime_t hrt;
	int ps;
	int err;
	fmd_asru_rep_arg_t fara;

	if (!(alp->al_flags & FMD_ASRU_FAULTY))
		return;

	/*
	 * Checking for aged resources only happens on the diagnosing side
	 * not on a proxy.
	 */
	if (alp->al_flags & FMD_ASRU_PROXY)
		return;

	ps = fmd_asru_replacement_state(alp->al_event, HC_ONLY_FALSE);
	if (ps == FMD_OBJ_STATE_REPLACED) {
		fara.fara_reason = FMD_ASRU_REPLACED;
		fara.fara_bywhat = FARA_ALL;
		fara.fara_rval = &err;
		fmd_asru_repaired(alp, &fara);
	} else if (ps == FMD_OBJ_STATE_NOT_PRESENT) {
		fmd_time_gettimeofday(&tv);
		lp = fmd_log_open(alp->al_asru->asru_root, alp->al_uuid,
		    FMD_LOG_ASRU);
		if (lp == NULL)
			return;
		hrt = (hrtime_t)(tv.tv_sec - lp->log_stat.st_mtime);
		fmd_log_rele(lp);
		if (hrt * NANOSEC >= fmd.d_asrus->ah_lifetime) {
			fara.fara_reason = FMD_ASRU_REMOVED;
			fara.fara_bywhat = FARA_ALL;
			fara.fara_rval = &err;
			fmd_asru_repaired(alp, &fara);
		}
	}
}

/*ARGSUSED*/
void
fmd_asru_check_if_aged(fmd_asru_link_t *alp, void *arg)
{
	struct timeval tv;
	fmd_log_t *lp;
	hrtime_t hrt;

	/*
	 * Case must be in resolved state for this to be called. So modified
	 * time on resource cache entry should be the time the resolve occurred.
	 * Return 0 if not yet hit rsrc.aged.
	 */
	fmd_time_gettimeofday(&tv);
	lp = fmd_log_open(alp->al_asru->asru_root, alp->al_uuid, FMD_LOG_ASRU);
	if (lp == NULL)
		return;
	hrt = (hrtime_t)(tv.tv_sec - lp->log_stat.st_mtime);
	fmd_log_rele(lp);
	if (hrt * NANOSEC < fmd.d_asrus->ah_lifetime)
		*(int *)arg = 0;
}

/*ARGSUSED*/
void
fmd_asru_most_recent(fmd_asru_link_t *alp, void *arg)
{
	fmd_log_t *lp;
	uint64_t hrt;

	/*
	 * Find most recent modified time of a set of resource cache entries.
	 */
	lp = fmd_log_open(alp->al_asru->asru_root, alp->al_uuid, FMD_LOG_ASRU);
	if (lp == NULL)
		return;
	hrt = lp->log_stat.st_mtime;
	fmd_log_rele(lp);
	if (*(uint64_t *)arg < hrt)
		*(uint64_t *)arg = hrt;
}

void
fmd_asru_clear_aged_rsrcs()
{
	int check_if_aged = 1;
	fmd_asru_al_hash_apply(fmd.d_asrus, fmd_asru_repair_if_aged, NULL);
	fmd_case_hash_apply(fmd.d_cases, fmd_case_discard_resolved,
	    &check_if_aged);
}

fmd_asru_hash_t *
fmd_asru_hash_create(const char *root, const char *dir)
{
	fmd_asru_hash_t *ahp;
	char path[PATH_MAX];

	ahp = fmd_alloc(sizeof (fmd_asru_hash_t), FMD_SLEEP);
	(void) pthread_rwlock_init(&ahp->ah_lock, NULL);
	ahp->ah_hashlen = fmd.d_str_buckets;
	ahp->ah_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen, FMD_SLEEP);
	ahp->ah_asru_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
	    FMD_SLEEP);
	ahp->ah_case_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
	    FMD_SLEEP);
	ahp->ah_fru_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
	    FMD_SLEEP);
	ahp->ah_label_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
	    FMD_SLEEP);
	ahp->ah_rsrc_hash = fmd_zalloc(sizeof (void *) * ahp->ah_hashlen,
	    FMD_SLEEP);
	(void) snprintf(path, sizeof (path), "%s/%s", root, dir);
	ahp->ah_dirpath = fmd_strdup(path, FMD_SLEEP);
	(void) fmd_conf_getprop(fmd.d_conf, "rsrc.age", &ahp->ah_lifetime);
	(void) fmd_conf_getprop(fmd.d_conf, "fakenotpresent",
	    (uint32_t *)&fmd_asru_fake_not_present);
	ahp->ah_al_count = 0;
	ahp->ah_count = 0;
	ahp->ah_error = 0;
	ahp->ah_topo = fmd_topo_hold();

	return (ahp);
}

void
fmd_asru_hash_destroy(fmd_asru_hash_t *ahp)
{
	fmd_asru_link_t *alp, *np;
	uint_t i;

	for (i = 0; i < ahp->ah_hashlen; i++) {
		for (alp = ahp->ah_case_hash[i]; alp != NULL; alp = np) {
			np = alp->al_case_next;
			alp->al_case_next = NULL;
			fmd_case_rele(alp->al_case);
			alp->al_case = NULL;
			fmd_asru_al_hash_release(ahp, alp);
		}
	}

	fmd_strfree(ahp->ah_dirpath);
	fmd_free(ahp->ah_hash, sizeof (void *) * ahp->ah_hashlen);
	fmd_free(ahp->ah_asru_hash, sizeof (void *) * ahp->ah_hashlen);
	fmd_free(ahp->ah_case_hash, sizeof (void *) * ahp->ah_hashlen);
	fmd_free(ahp->ah_fru_hash, sizeof (void *) * ahp->ah_hashlen);
	fmd_free(ahp->ah_label_hash, sizeof (void *) * ahp->ah_hashlen);
	fmd_free(ahp->ah_rsrc_hash, sizeof (void *) * ahp->ah_hashlen);
	fmd_topo_rele(ahp->ah_topo);
	fmd_free(ahp, sizeof (fmd_asru_hash_t));
}

/*
 * Take a snapshot of the ASRU database by placing an additional hold on each
 * member in an auxiliary array, and then call 'func' for each ASRU.
 */
void
fmd_asru_hash_apply(fmd_asru_hash_t *ahp,
    void (*func)(fmd_asru_t *, void *), void *arg)
{
	fmd_asru_t *ap, **aps, **app;
	uint_t apc, i;

	(void) pthread_rwlock_rdlock(&ahp->ah_lock);

	aps = app = fmd_alloc(ahp->ah_count * sizeof (fmd_asru_t *), FMD_SLEEP);
	apc = ahp->ah_count;

	for (i = 0; i < ahp->ah_hashlen; i++) {
		for (ap = ahp->ah_hash[i]; ap != NULL; ap = ap->asru_next)
			*app++ = fmd_asru_hold(ap);
	}

	ASSERT(app == aps + apc);
	(void) pthread_rwlock_unlock(&ahp->ah_lock);

	for (i = 0; i < apc; i++) {
		if (aps[i]->asru_fmri != NULL)
			func(aps[i], arg);
		fmd_asru_hash_release(ahp, aps[i]);
	}

	fmd_free(aps, apc * sizeof (fmd_asru_t *));
}

void
fmd_asru_al_hash_apply(fmd_asru_hash_t *ahp,
    void (*func)(fmd_asru_link_t *, void *), void *arg)
{
	fmd_asru_link_t *alp, **alps, **alpp;
	uint_t alpc, i;

	(void) pthread_rwlock_rdlock(&ahp->ah_lock);

	alps = alpp = fmd_alloc(ahp->ah_al_count * sizeof (fmd_asru_link_t *),
	    FMD_SLEEP);
	alpc = ahp->ah_al_count;

	for (i = 0; i < ahp->ah_hashlen; i++) {
		for (alp = ahp->ah_case_hash[i]; alp != NULL;
		    alp = alp->al_case_next)
			*alpp++ = fmd_asru_al_hold(alp);
	}

	ASSERT(alpp == alps + alpc);
	(void) pthread_rwlock_unlock(&ahp->ah_lock);

	for (i = 0; i < alpc; i++) {
		func(alps[i], arg);
		fmd_asru_al_hash_release(ahp, alps[i]);
	}

	fmd_free(alps, alpc * sizeof (fmd_asru_link_t *));
}

static void
fmd_asru_do_hash_apply(fmd_asru_hash_t *ahp, const char *name,
    void (*func)(fmd_asru_link_t *, void *), void *arg,
    fmd_asru_link_t **hash, size_t match_offset, size_t next_offset)
{
	fmd_asru_link_t *alp, **alps, **alpp;
	uint_t alpc = 0, i;
	uint_t h;

	(void) pthread_rwlock_rdlock(&ahp->ah_lock);

	h = fmd_asru_strhash(ahp, name);

	for (alp = hash[h]; alp != NULL; alp =
	    /* LINTED pointer alignment */
	    FMD_ASRU_AL_HASH_NEXT(alp, next_offset))
		if (fmd_asru_strcmp(ahp,
		    /* LINTED pointer alignment */
		    FMD_ASRU_AL_HASH_NAME(alp, match_offset), name))
			alpc++;

	alps = alpp = fmd_alloc(alpc * sizeof (fmd_asru_link_t *), FMD_SLEEP);

	for (alp = hash[h]; alp != NULL; alp =
	    /* LINTED pointer alignment */
	    FMD_ASRU_AL_HASH_NEXT(alp, next_offset))
		if (fmd_asru_strcmp(ahp,
		    /* LINTED pointer alignment */
		    FMD_ASRU_AL_HASH_NAME(alp, match_offset), name))
			*alpp++ = fmd_asru_al_hold(alp);

	ASSERT(alpp == alps + alpc);
	(void) pthread_rwlock_unlock(&ahp->ah_lock);

	for (i = 0; i < alpc; i++) {
		func(alps[i], arg);
		fmd_asru_al_hash_release(ahp, alps[i]);
	}

	fmd_free(alps, alpc * sizeof (fmd_asru_link_t *));
}

void
fmd_asru_hash_apply_by_asru(fmd_asru_hash_t *ahp, const char *name,
    void (*func)(fmd_asru_link_t *, void *), void *arg)
{
	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_asru_hash,
	    offsetof(fmd_asru_link_t, al_asru_name),
	    offsetof(fmd_asru_link_t, al_asru_next));
}

void
fmd_asru_hash_apply_by_case(fmd_asru_hash_t *ahp, fmd_case_t *cp,
	void (*func)(fmd_asru_link_t *, void *), void *arg)
{
	fmd_asru_do_hash_apply(ahp, ((fmd_case_impl_t *)cp)->ci_uuid, func, arg,
	    ahp->ah_case_hash, offsetof(fmd_asru_link_t, al_case_uuid),
	    offsetof(fmd_asru_link_t, al_case_next));
}

void
fmd_asru_hash_apply_by_fru(fmd_asru_hash_t *ahp, const char *name,
    void (*func)(fmd_asru_link_t *, void *), void *arg)
{
	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_fru_hash,
	    offsetof(fmd_asru_link_t, al_fru_name),
	    offsetof(fmd_asru_link_t, al_fru_next));
}

void
fmd_asru_hash_apply_by_rsrc(fmd_asru_hash_t *ahp, const char *name,
    void (*func)(fmd_asru_link_t *, void *), void *arg)
{
	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_rsrc_hash,
	    offsetof(fmd_asru_link_t, al_rsrc_name),
	    offsetof(fmd_asru_link_t, al_rsrc_next));
}

void
fmd_asru_hash_apply_by_label(fmd_asru_hash_t *ahp, const char *name,
    void (*func)(fmd_asru_link_t *, void *), void *arg)
{
	fmd_asru_do_hash_apply(ahp, name, func, arg, ahp->ah_label_hash,
	    offsetof(fmd_asru_link_t, al_label),
	    offsetof(fmd_asru_link_t, al_label_next));
}

/*
 * Lookup an asru in the hash by name and place a hold on it.  If the asru is
 * not found, no entry is created and NULL is returned.
 */
fmd_asru_t *
fmd_asru_hash_lookup_name(fmd_asru_hash_t *ahp, const char *name)
{
	fmd_asru_t *ap;

	(void) pthread_rwlock_rdlock(&ahp->ah_lock);
	ap = fmd_asru_hash_lookup(ahp, name);
	(void) pthread_rwlock_unlock(&ahp->ah_lock);

	return (ap);
}

/*
 * Create a resource cache entry using the fault event "nvl" for one of the
 * suspects from the case "cp".
 *
 * The fault event can have the following components :  FM_FAULT_ASRU,
 * FM_FAULT_FRU, FM_FAULT_RESOURCE. These should be set by the Diagnosis Engine
 * when calling fmd_nvl_create_fault(). In the general case, these are all
 * optional and an entry will always be added into the cache even if one or all
 * of these fields is missing.
 *
 * However, for hardware faults the recommended practice is that the fault
 * event should always have the FM_FAULT_RESOURCE field present and that this
 * should be represented in hc-scheme.
 *
 * Currently the DE should also add the FM_FAULT_ASRU and FM_FAULT_FRU fields
 * where known, though at some future stage fmd might be able to fill these
 * in automatically from the topology.
 */
fmd_asru_link_t *
fmd_asru_hash_create_entry(fmd_asru_hash_t *ahp, fmd_case_t *cp, nvlist_t *nvl)
{
	char *parsed_uuid;
	uuid_t uuid;
	int uuidlen;
	fmd_asru_link_t *alp;

	/*
	 * Generate a UUID for the ASRU.  libuuid cleverly gives us no
	 * interface for specifying or learning the buffer size.  Sigh.
	 * The spec says 36 bytes but we use a tunable just to be safe.
	 */
	(void) fmd_conf_getprop(fmd.d_conf, "uuidlen", &uuidlen);
	parsed_uuid = fmd_zalloc(uuidlen + 1, FMD_SLEEP);
	uuid_generate(uuid);
	uuid_unparse(uuid, parsed_uuid);

	/*
	 * Now create the resource cache entries.
	 */
	fmd_case_hold_locked(cp);
	alp = fmd_asru_al_create(ahp, nvl, cp, parsed_uuid);
	TRACE((FMD_DBG_ASRU, "asru %s created as %p",
	    alp->al_uuid, (void *)alp->al_asru));

	fmd_free(parsed_uuid, uuidlen + 1);
	return (alp);

}

/*
 * Release the reference count on an asru obtained using fmd_asru_hash_lookup.
 * We take 'ahp' for symmetry and in case we need to use it in future work.
 */
/*ARGSUSED*/
void
fmd_asru_hash_release(fmd_asru_hash_t *ahp, fmd_asru_t *ap)
{
	(void) pthread_mutex_lock(&ap->asru_lock);

	ASSERT(ap->asru_refs != 0);
	if (--ap->asru_refs == 0)
		fmd_asru_destroy(ap);
	else
		(void) pthread_mutex_unlock(&ap->asru_lock);
}

static void
fmd_asru_do_delete_entry(fmd_asru_hash_t *ahp, fmd_case_t *cp,
    fmd_asru_link_t **hash, size_t next_offset, char *name)
{
	uint_t h;
	fmd_asru_link_t *alp, **pp, *alpnext, **alpnextp;

	(void) pthread_rwlock_wrlock(&ahp->ah_lock);
	h = fmd_asru_strhash(ahp, name);
	pp = &hash[h];
	for (alp = *pp; alp != NULL; alp = alpnext) {
		/* LINTED pointer alignment */
		alpnextp = FMD_ASRU_AL_HASH_NEXTP(alp, next_offset);
		alpnext = *alpnextp;
		if (alp->al_case == cp) {
			*pp = *alpnextp;
			*alpnextp = NULL;
		} else
			pp = alpnextp;
	}
	(void) pthread_rwlock_unlock(&ahp->ah_lock);
}

static void
fmd_asru_do_hash_delete(fmd_asru_hash_t *ahp, fmd_case_susp_t *cis,
    fmd_case_t *cp, fmd_asru_link_t **hash, size_t next_offset, char *nvname)
{
	nvlist_t *nvl;
	char *name = NULL;
	ssize_t namelen;

	if (nvlist_lookup_nvlist(cis->cis_nvl, nvname, &nvl) == 0 &&
	    (namelen = fmd_fmri_nvl2str(nvl, NULL, 0)) != -1 &&
	    (name = fmd_alloc(namelen + 1, FMD_SLEEP)) != NULL) {
		if (fmd_fmri_nvl2str(nvl, name, namelen + 1) != -1)
			fmd_asru_do_delete_entry(ahp, cp, hash, next_offset,
			    name);
		fmd_free(name, namelen + 1);
	} else
		fmd_asru_do_delete_entry(ahp, cp, hash, next_offset, "");
}

void
fmd_asru_hash_delete_case(fmd_asru_hash_t *ahp, fmd_case_t *cp)
{
	fmd_case_impl_t *cip = (fmd_case_impl_t *)cp;
	fmd_case_susp_t *cis;
	fmd_asru_link_t *alp, **plp, *alpnext;
	fmd_asru_t *ap;
	char path[PATH_MAX];
	char *label;
	uint_t h;

	/*
	 * first delete hash entries for each suspect
	 */
	for (cis = cip->ci_suspects; cis != NULL; cis = cis->cis_next) {
		fmd_asru_do_hash_delete(ahp, cis, cp, ahp->ah_fru_hash,
		    offsetof(fmd_asru_link_t, al_fru_next), FM_FAULT_FRU);
		fmd_asru_do_hash_delete(ahp, cis, cp, ahp->ah_rsrc_hash,
		    offsetof(fmd_asru_link_t, al_rsrc_next), FM_FAULT_RESOURCE);
		if (nvlist_lookup_string(cis->cis_nvl, FM_FAULT_LOCATION,
		    &label) != 0)
			label = "";
		fmd_asru_do_delete_entry(ahp, cp, ahp->ah_label_hash,
		    offsetof(fmd_asru_link_t, al_label_next), label);
		fmd_asru_do_hash_delete(ahp, cis, cp, ahp->ah_asru_hash,
		    offsetof(fmd_asru_link_t, al_asru_next), FM_FAULT_ASRU);
	}

	/*
	 * then delete associated case hash entries
	 */
	(void) pthread_rwlock_wrlock(&ahp->ah_lock);
	h = fmd_asru_strhash(ahp, cip->ci_uuid);
	plp = &ahp->ah_case_hash[h];
	for (alp = *plp; alp != NULL; alp = alpnext) {
		alpnext = alp->al_case_next;
		if (alp->al_case == cp) {
			*plp = alp->al_case_next;
			alp->al_case_next = NULL;
			ASSERT(ahp->ah_al_count != 0);
			ahp->ah_al_count--;

			/*
			 * decrement case ref.
			 */
			fmd_case_rele_locked(cp);
			alp->al_case = NULL;

			/*
			 * If we found a matching ASRU, unlink its log file and
			 * then release the hash entry. Note that it may still
			 * be referenced if another thread is manipulating it;
			 * this is ok because once we unlink, the log file will
			 * not be restored, and the log data will be freed when
			 * all of the referencing threads release their
			 * respective references.
			 */
			(void) snprintf(path, sizeof (path), "%s/%s",
			    ahp->ah_dirpath, alp->al_uuid);
			if (cip->ci_xprt == NULL && unlink(path) != 0)
				fmd_error(EFMD_ASRU_UNLINK,
				    "failed to unlink asru %s", path);

			/*
			 * Now unlink from the global per-resource cache
			 * and if this is the last link then remove that from
			 * it's own hash too.
			 */
			ap = alp->al_asru;
			(void) pthread_mutex_lock(&ap->asru_lock);
			fmd_list_delete(&ap->asru_list, alp);
			if (ap->asru_list.l_next == NULL) {
				uint_t h;
				fmd_asru_t *ap2, **pp;
				fmd_asru_t *apnext, **apnextp;

				ASSERT(ahp->ah_count != 0);
				ahp->ah_count--;
				h = fmd_asru_strhash(ahp, ap->asru_name);
				pp = &ahp->ah_hash[h];
				for (ap2 = *pp; ap2 != NULL; ap2 = apnext) {
					apnextp = &ap2->asru_next;
					apnext = *apnextp;
					if (ap2 == ap) {
						*pp = *apnextp;
						*apnextp = NULL;
					} else
						pp = apnextp;
				}
			}
			(void) pthread_mutex_unlock(&ap->asru_lock);
			fmd_asru_al_hash_release(ahp, alp);
		} else
			plp = &alp->al_case_next;
	}
	(void) pthread_rwlock_unlock(&ahp->ah_lock);
}

typedef struct {
	nvlist_t *farc_parent_fmri;
	uint8_t farc_reason;
} fmd_asru_farc_t;

static void
fmd_asru_repair_containee(fmd_asru_link_t *alp, void *arg)
{
	fmd_asru_farc_t *farcp = (fmd_asru_farc_t *)arg;

	if ((alp->al_asru->asru_flags & FMD_ASRU_INVISIBLE) &&
	    alp->al_asru_fmri &&
	    fmd_fmri_contains(farcp->farc_parent_fmri, alp->al_asru_fmri) > 0) {
		if (fmd_asru_clrflags(alp, FMD_ASRU_FAULTY,
		    farcp->farc_reason)) {
			if (alp->al_flags & FMD_ASRU_PROXY)
				fmd_case_xprt_updated(alp->al_case);
			else
				fmd_case_update(alp->al_case);
		}
	}
}

static void
fmd_asru_do_repair_containees(fmd_asru_link_t *alp, uint8_t reason)
{
	int flags;

	/*
	 * Check if all entries associated with this asru are acquitted and
	 * if so acquit containees. Don't try to repair containees on proxy
	 * side unless we have local asru.
	 */
	if (alp->al_asru_fmri != NULL && (!(alp->al_flags & FMD_ASRU_PROXY) ||
	    (alp->al_flags & FMD_ASRU_PROXY_WITH_ASRU))) {
		(void) pthread_mutex_lock(&alp->al_asru->asru_lock);
		flags = alp->al_asru->asru_flags;
		(void) pthread_mutex_unlock(&alp->al_asru->asru_lock);
		if (!(flags & (FMD_ASRU_FAULTY | FMD_ASRU_INVISIBLE))) {
			fmd_asru_farc_t farc;

			farc.farc_parent_fmri = alp->al_asru_fmri;
			farc.farc_reason = reason;
			fmd_asru_al_hash_apply(fmd.d_asrus,
			    fmd_asru_repair_containee, &farc);
		}
	}
}

void
fmd_asru_repaired(fmd_asru_link_t *alp, void *arg)
{
	int cleared;
	fmd_asru_rep_arg_t *farap = (fmd_asru_rep_arg_t *)arg;

	/*
	 * don't allow remote repair over readonly transport
	 */
	if (alp->al_flags & FMD_ASRU_PROXY_RDONLY)
		return;

	/*
	 * don't allow repair etc by asru on proxy unless asru is local
	 */
	if (farap->fara_bywhat == FARA_BY_ASRU &&
	    (alp->al_flags & FMD_ASRU_PROXY) &&
	    !(alp->al_flags & FMD_ASRU_PROXY_WITH_ASRU))
		return;
	/*
	 * For acquit, need to check both name and uuid if specified
	 */
	if (farap->fara_reason == FMD_ASRU_ACQUITTED &&
	    farap->fara_rval != NULL && strcmp(farap->fara_uuid, "") != 0 &&
	    strcmp(farap->fara_uuid, alp->al_case_uuid) != 0)
		return;

	/*
	 * For replaced, verify it has been replaced if we have serial number.
	 * If not set *farap->fara_rval to FARA_ERR_RSRCNOTR.
	 */
	if (farap->fara_reason == FMD_ASRU_REPLACED &&
	    !(alp->al_flags & FMD_ASRU_PROXY_EXTERNAL) &&
	    fmd_asru_replacement_state(alp->al_event,
	    (alp->al_flags & FMD_ASRU_PROXY) ? HC_ONLY_TRUE : HC_ONLY_FALSE) ==
	    FMD_OBJ_STATE_STILL_PRESENT) {
		if (farap->fara_rval)
			*farap->fara_rval = FARA_ERR_RSRCNOTR;
		return;
	}

	cleared = fmd_asru_clrflags(alp, FMD_ASRU_FAULTY, farap->fara_reason);
	fmd_asru_do_repair_containees(alp, farap->fara_reason);

	/*
	 * if called from fmd_adm_*() and we really did clear the bit then
	 * we need to do a case update to see if the associated case can be
	 * repaired. No need to do this if called from fmd_case_*() (ie
	 * when arg is NULL) as the case will be explicitly repaired anyway.
	 */
	if (farap->fara_rval) {
		/*
		 * *farap->fara_rval defaults to FARA_ERR_RSRCNOTF (not found).
		 * If we find a valid cache entry which we repair then we
		 * set it to FARA_OK. However we don't want to do this if
		 * we have already set it to FARA_ERR_RSRCNOTR (not replaced)
		 * in a previous iteration (see above). So only set it to
		 * FARA_OK if the current value is still FARA_ERR_RSRCNOTF.
		 */
		if (*farap->fara_rval == FARA_ERR_RSRCNOTF)
			*farap->fara_rval = FARA_OK;
		if (cleared) {
			if (alp->al_flags & FMD_ASRU_PROXY)
				fmd_case_xprt_updated(alp->al_case);
			else
				fmd_case_update(alp->al_case);
		}
	}
}

/*
 * Discard the case associated with this alp if it is in resolved state.
 * Called on "fmadm flush".
 */
/*ARGSUSED*/
void
fmd_asru_flush(fmd_asru_link_t *alp, void *arg)
{
	int check_if_aged = 0;
	int *rval = (int *)arg;

	if (alp->al_case)
		fmd_case_discard_resolved(alp->al_case, &check_if_aged);
	*rval = 0;
}

/*
 * This is only called for proxied faults. Set various flags so we can
 * find the nature of the transport from the resource cache code.
 */
/*ARGSUSED*/
void
fmd_asru_set_on_proxy(fmd_asru_link_t *alp, void *arg)
{
	fmd_asru_set_on_proxy_t *entryp = (fmd_asru_set_on_proxy_t *)arg;

	if (*entryp->fasp_countp >= entryp->fasp_maxcount)
		return;

	/*
	 * Note that this is a proxy fault and save whetehr transport is
	 * RDONLY or EXTERNAL.
	 */
	alp->al_flags |= FMD_ASRU_PROXY;
	alp->al_asru->asru_flags |= FMD_ASRU_PROXY;

	if (entryp->fasp_proxy_external) {
		alp->al_flags |= FMD_ASRU_PROXY_EXTERNAL;
		alp->al_asru->asru_flags |= FMD_ASRU_PROXY_EXTERNAL;
	}

	if (entryp->fasp_proxy_rdonly)
		alp->al_flags |= FMD_ASRU_PROXY_RDONLY;

	/*
	 * Save whether asru is accessible in local domain
	 */
	if (entryp->fasp_proxy_asru[*entryp->fasp_countp]) {
		alp->al_flags |= FMD_ASRU_PROXY_WITH_ASRU;
		alp->al_asru->asru_flags |= FMD_ASRU_PROXY_WITH_ASRU;
	}
	(*entryp->fasp_countp)++;
}

/*ARGSUSED*/
void
fmd_asru_update_containees(fmd_asru_link_t *alp, void *arg)
{
	fmd_asru_do_repair_containees(alp, alp->al_reason);
}

/*
 * This function is used for fault proxying. It updates the resource status in
 * the resource cache based on information that has come from the other side of
 * the transport. This can be called on either the proxy side or the
 * diagnosing side.
 */
void
fmd_asru_update_status(fmd_asru_link_t *alp, void *arg)
{
	fmd_asru_update_status_t *entryp = (fmd_asru_update_status_t *)arg;
	uint8_t status;

	if (*entryp->faus_countp >= entryp->faus_maxcount)
		return;

	status = entryp->faus_ba[*entryp->faus_countp];

	/*
	 * For proxy, if there is no asru on the proxy side, but there is on
	 * the diag side, then take the diag side asru status.
	 * For diag, if there is an asru on the proxy side, then take the proxy
	 * side asru status.
	 */
	if (entryp->faus_is_proxy ?
	    (entryp->faus_diag_asru[*entryp->faus_countp] &&
	    !entryp->faus_proxy_asru[*entryp->faus_countp]) :
	    entryp->faus_proxy_asru[*entryp->faus_countp]) {
		if (status & FM_SUSPECT_DEGRADED)
			alp->al_flags |= FMD_ASRU_DEGRADED;
		else
			alp->al_flags &= ~FMD_ASRU_DEGRADED;
		if (status & FM_SUSPECT_UNUSABLE)
			(void) fmd_asru_setflags(alp, FMD_ASRU_UNUSABLE);
		else
			(void) fmd_asru_clrflags(alp, FMD_ASRU_UNUSABLE, 0);
	}

	/*
	 * Update the faulty status too.
	 */
	if (!(status & FM_SUSPECT_FAULTY))
		(void) fmd_asru_clrflags(alp, FMD_ASRU_FAULTY,
		    (status & FM_SUSPECT_REPAIRED) ? FMD_ASRU_REPAIRED :
		    (status & FM_SUSPECT_REPLACED) ? FMD_ASRU_REPLACED :
		    (status & FM_SUSPECT_ACQUITTED) ? FMD_ASRU_ACQUITTED :
		    FMD_ASRU_REMOVED);
	else if (entryp->faus_is_proxy)
		(void) fmd_asru_setflags(alp, FMD_ASRU_FAULTY);

	/*
	 * for proxy only, update the present status too.
	 */
	if (entryp->faus_is_proxy) {
		if (!(status & FM_SUSPECT_NOT_PRESENT)) {
			alp->al_flags |= FMD_ASRU_PRESENT;
			alp->al_asru->asru_flags |= FMD_ASRU_PRESENT;
		} else {
			alp->al_flags &= ~FMD_ASRU_PRESENT;
			alp->al_asru->asru_flags &= ~FMD_ASRU_PRESENT;
		}
	}
	(*entryp->faus_countp)++;
}

/*
 * This function is called on the diagnosing side when fault proxying is
 * in use and the proxy has sent a uuclose. It updates the status of the
 * resource cache entries.
 */
void
fmd_asru_close_status(fmd_asru_link_t *alp, void *arg)
{
	fmd_asru_close_status_t *entryp = (fmd_asru_close_status_t *)arg;

	if (*entryp->facs_countp >= entryp->facs_maxcount)
		return;
	alp->al_flags &= ~FMD_ASRU_DEGRADED;
	(void) fmd_asru_setflags(alp, FMD_ASRU_UNUSABLE);
	(*entryp->facs_countp)++;
}

static void
fmd_asru_logevent(fmd_asru_link_t *alp)
{
	fmd_asru_t *ap = alp->al_asru;
	boolean_t faulty = (alp->al_flags & FMD_ASRU_FAULTY) != 0;
	boolean_t unusable = (alp->al_flags & FMD_ASRU_UNUSABLE) != 0;
	boolean_t message = (ap->asru_flags & FMD_ASRU_INVISIBLE) == 0;
	boolean_t repaired = (alp->al_reason == FMD_ASRU_REPAIRED);
	boolean_t replaced = (alp->al_reason == FMD_ASRU_REPLACED);
	boolean_t acquitted = (alp->al_reason == FMD_ASRU_ACQUITTED);

	fmd_case_impl_t *cip;
	fmd_event_t *e;
	fmd_log_t *lp;
	nvlist_t *nvl;
	char *class;

	ASSERT(MUTEX_HELD(&ap->asru_lock));
	cip = (fmd_case_impl_t *)alp->al_case;
	ASSERT(cip != NULL);

	/*
	 * Don't log to disk on proxy side
	 */
	if (cip->ci_xprt != NULL)
		return;

	if ((lp = alp->al_log) == NULL)
		lp = fmd_log_open(ap->asru_root, alp->al_uuid, FMD_LOG_ASRU);

	if (lp == NULL)
		return; /* can't log events if we can't open the log */

	nvl = fmd_protocol_rsrc_asru(_fmd_asru_events[faulty | (unusable << 1)],
	    alp->al_asru_fmri, cip->ci_uuid, cip->ci_code, faulty, unusable,
	    message, alp->al_event, &cip->ci_tv, repaired, replaced, acquitted,
	    cip->ci_state == FMD_CASE_RESOLVED, cip->ci_diag_de == NULL ?
	    cip->ci_mod->mod_fmri : cip->ci_diag_de, cip->ci_injected == 1);

	(void) nvlist_lookup_string(nvl, FM_CLASS, &class);
	e = fmd_event_create(FMD_EVT_PROTOCOL, FMD_HRT_NOW, nvl, class);

	fmd_event_hold(e);
	fmd_log_append(lp, e, NULL);
	fmd_event_rele(e);

	/*
	 * For now, we close the log file after every update to conserve file
	 * descriptors and daemon overhead.  If this becomes a performance
	 * issue this code can change to keep a fixed-size LRU cache of logs.
	 */
	fmd_log_rele(lp);
	alp->al_log = NULL;
}

int
fmd_asru_setflags(fmd_asru_link_t *alp, uint_t sflag)
{
	fmd_asru_t *ap = alp->al_asru;
	uint_t nstate, ostate;

	ASSERT(!(sflag & ~FMD_ASRU_STATE));
	ASSERT(sflag != FMD_ASRU_STATE);

	(void) pthread_mutex_lock(&ap->asru_lock);

	ostate = alp->al_flags & FMD_ASRU_STATE;
	alp->al_flags |= sflag;
	nstate = alp->al_flags & FMD_ASRU_STATE;

	if (nstate == ostate) {
		(void) pthread_mutex_unlock(&ap->asru_lock);
		return (0);
	}

	ap->asru_flags |= sflag;
	TRACE((FMD_DBG_ASRU, "asru %s %s->%s", alp->al_uuid,
	    _fmd_asru_snames[ostate], _fmd_asru_snames[nstate]));

	fmd_asru_logevent(alp);

	(void) pthread_cond_broadcast(&ap->asru_cv);
	(void) pthread_mutex_unlock(&ap->asru_lock);
	return (1);
}

int
fmd_asru_clrflags(fmd_asru_link_t *alp, uint_t sflag, uint8_t reason)
{
	fmd_asru_t *ap = alp->al_asru;
	fmd_asru_link_t *nalp;
	uint_t nstate, ostate, flags = 0;

	ASSERT(!(sflag & ~FMD_ASRU_STATE));
	ASSERT(sflag != FMD_ASRU_STATE);

	(void) pthread_mutex_lock(&ap->asru_lock);

	ostate = alp->al_flags & FMD_ASRU_STATE;
	alp->al_flags &= ~sflag;
	nstate = alp->al_flags & FMD_ASRU_STATE;

	if (nstate == ostate) {
		if (reason > alp->al_reason &&
		    ((fmd_case_impl_t *)alp->al_case)->ci_state <
		    FMD_CASE_REPAIRED) {
			alp->al_reason = reason;
			fmd_asru_logevent(alp);
			(void) pthread_cond_broadcast(&ap->asru_cv);
		}
		(void) pthread_mutex_unlock(&ap->asru_lock);
		return (0);
	}
	if (reason > alp->al_reason)
		alp->al_reason = reason;

	if (sflag == FMD_ASRU_UNUSABLE)
		ap->asru_flags &= ~sflag;
	else if (sflag == FMD_ASRU_FAULTY) {
		/*
		 * only clear the faulty bit if all links are clear
		 */
		for (nalp = fmd_list_next(&ap->asru_list); nalp != NULL;
		    nalp = fmd_list_next(nalp))
			flags |= nalp->al_flags;
		if (!(flags & FMD_ASRU_FAULTY))
			ap->asru_flags &= ~sflag;
	}

	TRACE((FMD_DBG_ASRU, "asru %s %s->%s", alp->al_uuid,
	    _fmd_asru_snames[ostate], _fmd_asru_snames[nstate]));

	fmd_asru_logevent(alp);

	(void) pthread_cond_broadcast(&ap->asru_cv);
	(void) pthread_mutex_unlock(&ap->asru_lock);

	return (1);
}

/*ARGSUSED*/
void
fmd_asru_log_resolved(fmd_asru_link_t *alp, void *unused)
{
	fmd_asru_t *ap = alp->al_asru;

	(void) pthread_mutex_lock(&ap->asru_lock);
	fmd_asru_logevent(alp);
	(void) pthread_cond_broadcast(&ap->asru_cv);
	(void) pthread_mutex_unlock(&ap->asru_lock);
}

/*
 * Report the current known state of the link entry (ie this particular fault
 * affecting this particular ASRU).
 */
int
fmd_asru_al_getstate(fmd_asru_link_t *alp)
{
	int us, st = (alp->al_flags & (FMD_ASRU_FAULTY | FMD_ASRU_UNUSABLE));
	nvlist_t *asru;
	int ps = FMD_OBJ_STATE_UNKNOWN;

	/*
	 * For fault proxying with an EXTERNAL transport, believe the presence
	 * state as sent by the diagnosing side. Otherwise find the presence
	 * state here. Note that if fault proxying with an INTERNAL transport
	 * we can only trust the presence state where we are using hc-scheme
	 * fmris which should be consistant across domains in the same system -
	 * other schemes can refer to different devices in different domains.
	 */
	if (!(alp->al_flags & FMD_ASRU_PROXY_EXTERNAL)) {
		ps = fmd_asru_replacement_state(alp->al_event, (alp->al_flags &
		    FMD_ASRU_PROXY)? HC_ONLY_TRUE : HC_ONLY_FALSE);
		if (ps == FMD_OBJ_STATE_NOT_PRESENT)
			return (st | FMD_ASRU_UNUSABLE);
		if (ps == FMD_OBJ_STATE_REPLACED) {
			if (alp->al_reason < FMD_ASRU_REPLACED)
				alp->al_reason = FMD_ASRU_REPLACED;
			return (st | FMD_ASRU_UNUSABLE);
		}
	}
	if (ps == FMD_OBJ_STATE_UNKNOWN && (alp->al_flags & FMD_ASRU_PROXY))
		st |= (alp->al_flags & (FMD_ASRU_DEGRADED | FMD_ASRU_PRESENT));
	else
		st |= (alp->al_flags & (FMD_ASRU_DEGRADED)) | FMD_ASRU_PRESENT;

	/*
	 * For fault proxying, unless we have a local ASRU, then believe the
	 * service state sent by the diagnosing side. Otherwise find the service
	 * state here. Try fmd_fmri_service_state() first, but if that's not
	 * supported by the scheme then fall back to fmd_fmri_unusable().
	 */
	if ((!(alp->al_flags & FMD_ASRU_PROXY) ||
	    (alp->al_flags & FMD_ASRU_PROXY_WITH_ASRU)) &&
	    nvlist_lookup_nvlist(alp->al_event, FM_FAULT_ASRU, &asru) == 0) {
		us = fmd_fmri_service_state(asru);
		if (us == -1 || us == FMD_SERVICE_STATE_UNKNOWN) {
			/* not supported by scheme - try fmd_fmri_unusable */
			us = fmd_fmri_unusable(asru);
			if (us > 0)
				st |= FMD_ASRU_UNUSABLE;
			else if (us == 0)
				st &= ~FMD_ASRU_UNUSABLE;
		} else {
			if (us == FMD_SERVICE_STATE_UNUSABLE) {
				st &= ~FMD_ASRU_DEGRADED;
				st |= FMD_ASRU_UNUSABLE;
			} else if (us == FMD_SERVICE_STATE_OK) {
				st &= ~(FMD_ASRU_DEGRADED | FMD_ASRU_UNUSABLE);
			} else if (us == FMD_SERVICE_STATE_ISOLATE_PENDING) {
				st &= ~(FMD_ASRU_DEGRADED | FMD_ASRU_UNUSABLE);
			} else if (us == FMD_SERVICE_STATE_DEGRADED) {
				st &= ~FMD_ASRU_UNUSABLE;
				st |= FMD_ASRU_DEGRADED;
			}
		}
	}
	return (st);
}

/*
 * Report the current known state of the ASRU by refreshing its unusable status
 * based upon the routines provided by the scheme module.  If the unusable bit
 * is different, we do *not* generate a state change here because that change
 * may be unrelated to fmd activities and therefore we have no case or event.
 * The absence of the transition is harmless as this function is only provided
 * for RPC observability and fmd's clients are only concerned with ASRU_FAULTY.
 */
int
fmd_asru_getstate(fmd_asru_t *ap)
{
	int us, st, p = -1;
	char *s;

	/* do not report non-fmd non-present resources */
	if (!(ap->asru_flags & FMD_ASRU_INTERNAL)) {
		/*
		 * As with fmd_asru_al_getstate(), we can only trust the
		 * local presence state on a proxy if the transport is
		 * internal and the scheme is hc. Otherwise we believe the
		 * state as sent by the diagnosing side.
		 */
		if (!(ap->asru_flags & FMD_ASRU_PROXY) ||
		    (!(ap->asru_flags & FMD_ASRU_PROXY_EXTERNAL) &&
		    (nvlist_lookup_string(ap->asru_fmri, FM_FMRI_SCHEME,
		    &s) == 0 && strcmp(s, FM_FMRI_SCHEME_HC) == 0))) {
			if (fmd_asru_fake_not_present >=
			    FMD_OBJ_STATE_REPLACED)
				return (0);
			p = fmd_fmri_present(ap->asru_fmri);
		}
		if (p == 0 || (p < 0 && !(ap->asru_flags & FMD_ASRU_PROXY) ||
		    !(ap->asru_flags & FMD_ASRU_PRESENT)))
			return (0);
	}

	/*
	 * As with fmd_asru_al_getstate(), we can only trust the local unusable
	 * state on a proxy if there is a local ASRU.
	 */
	st = ap->asru_flags & (FMD_ASRU_FAULTY | FMD_ASRU_UNUSABLE);
	if (!(ap->asru_flags & FMD_ASRU_PROXY) ||
	    (ap->asru_flags & FMD_ASRU_PROXY_WITH_ASRU)) {
		us = fmd_fmri_unusable(ap->asru_fmri);
		if (us > 0)
			st |= FMD_ASRU_UNUSABLE;
		else if (us == 0)
			st &= ~FMD_ASRU_UNUSABLE;
	}
	return (st);
}