xref: /illumos-gate/usr/src/lib/fm/topo/modules/common/ses/ses.c (revision a92282e44f968185a6bba094d1e5fece2da819cf)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2012 Milan Jurik. All rights reserved.
 * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
 * Copyright 2020 Joyent, Inc.
 */

#include <alloca.h>
#include <dirent.h>
#include <devid.h>
#include <fm/libdiskstatus.h>
#include <inttypes.h>
#include <pthread.h>
#include <strings.h>
#include <string.h>
#include <unistd.h>
#include <sys/dkio.h>
#include <sys/fm/protocol.h>
#include <sys/libdevid.h>
#include <sys/scsi/scsi_types.h>
#include <sys/byteorder.h>
#include <pthread.h>
#include <signal.h>
#include <fcntl.h>
#include <sys/ctfs.h>
#include <libcontract.h>
#include <poll.h>
#include <sys/contract/device.h>
#include <libsysevent.h>
#include <sys/sysevent/eventdefs.h>
#include <scsi/plugins/ses/vendor/sun.h>

#include "disk.h"
#include "ses.h"

#define	SES_VERSION	1

#define	SES_STARTING_SUBCHASSIS 256	/* valid subchassis IDs are uint8_t */
#define	NO_SUBCHASSIS	((uint64_t)-1)

static int ses_snap_freq = 250;		/* in milliseconds */

#define	SES_STATUS_UNAVAIL(s)	\
	((s) == SES_ESC_UNSUPPORTED || (s) >= SES_ESC_NOT_INSTALLED)

#define	HR_SECOND   1000000000

#define	SES_INST_NOTSET		UINT64_MAX

/*
 * Because multiple SES targets can be part of a single chassis, we construct
 * our own hierarchy that takes this into account.  These SES targets may refer
 * to the same devices (multiple paths) or to different devices (managing
 * different portions of the space).  We arrange things into a
 * ses_enum_enclosure_t, which contains a set of ses targets, and a list of all
 * nodes found so far.
 */
typedef struct ses_alt_node {
	topo_list_t		san_link;
	ses_node_t		*san_node;
} ses_alt_node_t;

typedef struct ses_enum_node {
	topo_list_t		sen_link;
	ses_node_t		*sen_node;
	topo_list_t		sen_alt_nodes;
	uint64_t		sen_type;
	uint64_t		sen_instance;
	ses_enum_target_t	*sen_target;
} ses_enum_node_t;

typedef struct ses_enum_chassis {
	topo_list_t		sec_link;
	topo_list_t		sec_subchassis;
	topo_list_t		sec_nodes;
	topo_list_t		sec_targets;
	const char		*sec_csn;
	ses_node_t		*sec_enclosure;
	ses_enum_target_t	*sec_target;
	topo_instance_t		sec_instance;
	topo_instance_t		sec_scinstance;
	topo_instance_t		sec_maxinstance;
	boolean_t		sec_hasdev;
	boolean_t		sec_internal;
} ses_enum_chassis_t;

typedef struct ses_enum_data {
	topo_list_t		sed_devs;
	topo_list_t		sed_chassis;
	ses_enum_chassis_t	*sed_current;
	ses_enum_target_t	*sed_target;
	int			sed_errno;
	char			*sed_name;
	topo_mod_t		*sed_mod;
	topo_instance_t		sed_instance;
} ses_enum_data_t;

typedef struct sas_connector_phy_data {
	uint64_t    scpd_index;
	uint64_t    scpd_pm;
} sas_connector_phy_data_t;

typedef struct sas_connector_type {
	uint64_t    sct_type;
	char	    *sct_name;
} sas_connector_type_t;

static const sas_connector_type_t sas_connector_type_list[] = {
	{   0x0, "Information unknown"  },
	{   0x1, "External SAS 4x receptacle (see SAS-2 and SFF-8470)"	},
	{   0x2, "Exteranl Mini SAS 4x receptacle (see SAS-2 and SFF-8088)" },
	{   0x3, "QSFP+ receptacle (see SAS-2.1 and SFF-8436)" },
	{   0x4, "Mini SAS 4x active receptacle (see SAS-2.1 and SFF-8088)" },
	{   0x5, "Mini SAS HD 4x receptacle (see SAS-2.1 and SFF-8644)" },
	{   0x6, "Mini SAS HD 8x receptacle (see SAS-2.1 and SFF-8644)" },
	{   0x7, "Mini SAS HD 16x receptacle (see SAS-2.1 and SFF-8644)" },
	{   0xF, "Vendor-specific external connector"	},
	{   0x10, "Internal wide SAS 4i plug (see SAS-2 and SFF-8484)"	},
	{   0x11,
	"Internal wide Mini SAS 4i receptacle (see SAS-2 and SFF-8087)"	},
	{   0x12, "Mini SAS HD 4i receptacle (see SAS-2.1 and SFF-8643)" },
	{   0x20, "Internal SAS Drive receptacle (see SAS-2 and SFF-8482)" },
	{   0x21, "Internal SATA host plug (see SAS-2 and SATA-2)"	},
	{   0x22, "Internal SAS Drive plug (see SAS-2 and SFF-8482)"	},
	{   0x23, "Internal SATA device plug (see SAS-2 and SATA-2)"	},
	{   0x24, "Micro SAS receptacle (see SAS-2.14)"	},
	{   0x25, "Micro SATA device plug (see SAS-2.1 and SATA)"	},
	{   0x26, "Micro SAS plug (see SAS-2.1 and SFF-8486)"	},
	{   0x27, "Micro SAS/SATA plug (see SAS-2.1 and SFF-8486)"	},
	{   0x28,
	"12 Gb/s SAS Drive backplane receptacle (see SAS-34 and SFF-8680)" },
	{   0x29, "12Gb/s SAS Drive Plug (see SAS-3 and SFF-8680)"	},
	{   0x2A, "Multifunction 12 Gb/s 6x Unshielded receptacle connector "
		"receptacle (see SAS-3 and SFF-8639)"	},
	{   0x2B, "Multifunction 12 Gb/s 6x Unshielded receptable connector "
		"plug (see SAS-3 and SFF-8639)"	},
	{   0x2F, "Internal SAS virtual connector"  },
	{   0x3F, "Vendor-specific internal connector"	},
	{   0x70, "Other Vendor-specific connector"	},
	{   0x71, "Other Vendor-specific connector"	},
	{   0x72, "Other Vendor-specific connector"	},
	{   0x73, "Other Vendor-specific connector"	},
	{   0x74, "Other Vendor-specific connector"	},
	{   0x75, "Other Vendor-specific connector"	},
	{   0x76, "Other Vendor-specific connector"	},
	{   0x77, "Other Vendor-specific connector"	},
	{   0x78, "Other Vendor-specific connector"	},
	{   0x79, "Other Vendor-specific connector"	},
	{   0x7A, "Other Vendor-specific connector"	},
	{   0x7B, "Other Vendor-specific connector"	},
	{   0x7C, "Other Vendor-specific connector"	},
	{   0x7D, "Other Vendor-specific connector"	},
	{   0x7E, "Other Vendor-specific connector"	},
	{   0x7F, "Other Vendor-specific connector"	},
	{   0x80, "Not Defined"	}
};

#define	SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED  0x80
#define	SAS_CONNECTOR_TYPE_NOT_DEFINED \
	"Connector type not defined by SES-2 standard"
#define	SAS_CONNECTOR_TYPE_RESERVED \
	"Connector type reserved by SES-2 standard"

typedef struct phys_enum_type {
	uint64_t    pet_type;
	char	    *pet_nodename;
	char	    *pet_defaultlabel;
	boolean_t   pet_dorange;
} phys_enum_type_t;

static const phys_enum_type_t phys_enum_type_list[] = {
	{   SES_ET_ARRAY_DEVICE, BAY, "BAY", B_TRUE  },
	{   SES_ET_COOLING, FAN, "FAN", B_TRUE  },
	{   SES_ET_DEVICE, BAY, "BAY", B_TRUE  },
	{   SES_ET_ESC_ELECTRONICS, CONTROLLER, "CONTROLLER", B_TRUE  },
	{   SES_ET_POWER_SUPPLY, PSU, "PSU", B_TRUE  },
	{   SES_ET_SUNW_FANBOARD, FANBOARD, "FANBOARD", B_TRUE  },
	{   SES_ET_SUNW_FANMODULE, FANMODULE, "FANMODULE", B_TRUE  },
	{   SES_ET_SUNW_POWERBOARD, POWERBOARD, "POWERBOARD", B_TRUE  },
	{   SES_ET_SUNW_POWERMODULE, POWERMODULE, "POWERMODULE", B_TRUE  }
};

#define	N_PHYS_ENUM_TYPES (sizeof (phys_enum_type_list) / \
	sizeof (phys_enum_type_list[0]))

/*
 * Structure for the hierarchical tree for element nodes.
 */
typedef struct ses_phys_tree {
    ses_node_t	*spt_snode;
    ses_enum_node_t	*spt_senumnode;
    boolean_t	spt_isfru;
    uint64_t	spt_eonlyindex;
    uint64_t	spt_cindex;
    uint64_t	spt_pindex;
    uint64_t	spt_maxinst;
    struct ses_phys_tree    *spt_parent;
    struct ses_phys_tree    *spt_child;
    struct ses_phys_tree    *spt_sibling;
    tnode_t	*spt_tnode;
} ses_phys_tree_t;

typedef enum {
	SES_NEW_CHASSIS		= 0x1,
	SES_NEW_SUBCHASSIS	= 0x2,
	SES_DUP_CHASSIS		= 0x4,
	SES_DUP_SUBCHASSIS	= 0x8
} ses_chassis_type_e;

static const topo_pgroup_info_t smp_pgroup = {
	TOPO_PGROUP_SMP,
	TOPO_STABILITY_PRIVATE,
	TOPO_STABILITY_PRIVATE,
	1
};

static const topo_pgroup_info_t ses_pgroup = {
	TOPO_PGROUP_SES,
	TOPO_STABILITY_PRIVATE,
	TOPO_STABILITY_PRIVATE,
	1
};

static int ses_present(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
    nvlist_t **);
static int ses_contains(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
    nvlist_t **);

static const topo_method_t ses_component_methods[] = {
	{ TOPO_METH_PRESENT, TOPO_METH_PRESENT_DESC,
	    TOPO_METH_PRESENT_VERSION0, TOPO_STABILITY_INTERNAL, ses_present },
	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
	{ TOPO_METH_SENSOR_FAILURE, TOPO_METH_SENSOR_FAILURE_DESC,
	    TOPO_METH_SENSOR_FAILURE_VERSION, TOPO_STABILITY_INTERNAL,
	    topo_method_sensor_failure },
	{ NULL }
};

#define	TOPO_METH_SMCI_4U36_LABEL		"smci_4u36_bay_label"
#define	TOPO_METH_SMCI_4U36_LABEL_DESC	\
	"compute bay labels on SMCI 4U36 storage platform variants"
#define	TOPO_METH_SMCI_4U36_LABEL_VERSION	0
static int smci_4u36_bay_label(topo_mod_t *, tnode_t *, topo_version_t,
    nvlist_t *, nvlist_t **);

static const topo_method_t ses_bay_methods[] = {
	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
	    topo_mod_hc_occupied },
	{ TOPO_METH_SMCI_4U36_LABEL, TOPO_METH_SMCI_4U36_LABEL_DESC,
	    TOPO_METH_SMCI_4U36_LABEL_VERSION, TOPO_STABILITY_INTERNAL,
	    smci_4u36_bay_label },
	{ NULL }
};

static const topo_method_t ses_recep_methods[] = {
	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
	    topo_mod_hc_occupied },
	{ NULL }
};

static const topo_method_t ses_enclosure_methods[] = {
	{ TOPO_METH_CONTAINS, TOPO_METH_CONTAINS_DESC,
	    TOPO_METH_CONTAINS_VERSION, TOPO_STABILITY_INTERNAL, ses_contains },
	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
	    TOPO_STABILITY_INTERNAL, ses_enc_enum_facility },
	{ NULL }
};

/*
 * The bay_label_overrides table can be used to map a server product ID to a
 * topo method that will be invoked to override the value of the label property
 * for all bay nodes.  By default the property value is static, derived from
 * the corresponding SES array device element's descriptor string.
 */
typedef struct ses_label_overrides {
	const char *slbl_product;
	const char *slbl_mname;
} ses_label_overrides_t;

/*
 * This table covers three generations of SMCI's 4U 36-bay storage server
 * (and the Joyent-branded versions).  There was also an Ivy Bridge variant
 * which has been omitted due to an inability to find one to test on.
 */
static const ses_label_overrides_t bay_label_overrides[] = {
	/* Sandy Bridge variant */
	{ "SSG-6047R-E1R36L", TOPO_METH_SMCI_4U36_LABEL },
	{ "Joyent-Storage-Platform-5001", TOPO_METH_SMCI_4U36_LABEL },

	/* Broadwell variant */
	{ "SSG-6048R-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
	{ "Joyent-Storage-Platform-7001", TOPO_METH_SMCI_4U36_LABEL },

	/* Skylake variant */
	{ "SSG-6049P-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
	{ "Joyent-S10G5", TOPO_METH_SMCI_4U36_LABEL }
};

#define	N_BAY_LBL_OVERRIDES (sizeof (bay_label_overrides) / \
	sizeof (bay_label_overrides[0]))

/*
 * Functions for tracking ses devices which we were unable to open. We retry
 * these at regular intervals using ses_recheck_dir() and if we find that we
 * can now open any of them then we send a sysevent to indicate that a new topo
 * snapshot should be taken.
 */
typedef struct ses_open_fail_list {
	struct ses_open_fail_list	*sof_next;
	char				*sof_path;
} ses_open_fail_list_t;

static ses_open_fail_list_t *ses_sofh;
static pthread_mutex_t ses_sofmt;
static void ses_ct_print(char *ptr);

static void
ses_recheck_dir()
{
	ses_target_t *target;
	sysevent_id_t eid;
	char buf[80];
	ses_open_fail_list_t *sof;

	/*
	 * check list of "unable to open" devices
	 */
	(void) pthread_mutex_lock(&ses_sofmt);
	for (sof = ses_sofh; sof != NULL; sof = sof->sof_next) {
		/*
		 * see if we can open it now
		 */
		if ((target = ses_open(LIBSES_VERSION,
		    sof->sof_path)) == NULL) {
			(void) snprintf(buf, sizeof (buf),
			    "recheck_dir - still can't open %s", sof->sof_path);
			ses_ct_print(buf);
			continue;
		}

		/*
		 * ok - better force a new snapshot
		 */
		(void) snprintf(buf, sizeof (buf),
		    "recheck_dir - can now open %s", sof->sof_path);
		ses_ct_print(buf);
		(void) sysevent_post_event(EC_PLATFORM, ESC_PLATFORM_SP_RESET,
		    SUNW_VENDOR, "fmd", NULL, &eid);
		ses_close(target);
		break;
	}
	(void) pthread_mutex_unlock(&ses_sofmt);
}

static void
ses_sof_alloc(topo_mod_t *mod, char *path)
{
	ses_open_fail_list_t *sof;

	(void) pthread_mutex_lock(&ses_sofmt);
	sof = topo_mod_zalloc(mod, sizeof (*sof));
	topo_mod_dprintf(mod, "sof_alloc %s", path);
	sof->sof_path = path;
	sof->sof_next = ses_sofh;
	ses_sofh = sof;
	(void) pthread_mutex_unlock(&ses_sofmt);
}

static void
ses_sof_freeall(topo_mod_t *mod)
{
	ses_open_fail_list_t *sof, *next_sof;

	(void) pthread_mutex_lock(&ses_sofmt);
	for (sof = ses_sofh; sof != NULL; sof = next_sof) {
		next_sof = sof->sof_next;
		topo_mod_dprintf(mod, "sof_freeall %s", sof->sof_path);
		topo_mod_strfree(mod, sof->sof_path);
		topo_mod_free(mod, sof, sizeof (*sof));
	}
	ses_sofh = NULL;
	(void) pthread_mutex_unlock(&ses_sofmt);
}

/*
 * functions for verifying that the ses_enum_target_t held in a device
 * contract's cookie field is still valid (it may have been freed by
 * ses_release()).
 */
typedef struct ses_stp_list {
	struct ses_stp_list	*ssl_next;
	ses_enum_target_t	*ssl_tgt;
} ses_stp_list_t;

static ses_stp_list_t *ses_sslh;
static pthread_mutex_t ses_sslmt;

static void
ses_ssl_alloc(topo_mod_t *mod, ses_enum_target_t *stp)
{
	ses_stp_list_t *ssl;

	(void) pthread_mutex_lock(&ses_sslmt);
	ssl = topo_mod_zalloc(mod, sizeof (*ssl));
	topo_mod_dprintf(mod, "ssl_alloc %p", stp);
	ssl->ssl_tgt = stp;
	ssl->ssl_next = ses_sslh;
	ses_sslh = ssl;
	(void) pthread_mutex_unlock(&ses_sslmt);
}

static void
ses_ssl_free(topo_mod_t *mod, ses_enum_target_t *stp)
{
	ses_stp_list_t *ssl, *prev_ssl;

	(void) pthread_mutex_lock(&ses_sslmt);
	prev_ssl = NULL;
	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next) {
		if (ssl->ssl_tgt == stp) {
			topo_mod_dprintf(mod, "ssl_free %p", ssl->ssl_tgt);
			if (prev_ssl == NULL)
				ses_sslh = ssl->ssl_next;
			else
				prev_ssl->ssl_next = ssl->ssl_next;
			topo_mod_free(mod, ssl, sizeof (*ssl));
			break;
		}
		prev_ssl = ssl;
	}
	(void) pthread_mutex_unlock(&ses_sslmt);
}

static int
ses_ssl_valid(ses_enum_target_t *stp)
{
	ses_stp_list_t *ssl;

	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next)
		if (ssl->ssl_tgt == stp)
			return (1);
	return (0);
}

/*
 * Functions for creating and destroying a background thread
 * (ses_contract_thread) used for detecting when ses devices have been
 * retired/unretired.
 */
static struct ses_thread_s {
	pthread_mutex_t mt;
	pthread_t tid;
	int thr_sig;
	int doexit;
	int count;
} sesthread = {
	PTHREAD_MUTEX_INITIALIZER,
	0,
	SIGTERM,
	0,
	0
};

typedef struct ses_mod_list {
	struct ses_mod_list	*smod_next;
	topo_mod_t		*smod_mod;
} ses_mod_list_t;

static ses_mod_list_t *ses_smod;

static void
ses_ct_print(char *ptr)
{
	(void) pthread_mutex_lock(&sesthread.mt);
	if (ses_smod != NULL && ses_smod->smod_mod != NULL)
		topo_mod_dprintf(ses_smod->smod_mod, ptr);
	(void) pthread_mutex_unlock(&sesthread.mt);
}

/*ARGSUSED*/
static void *
ses_contract_thread(void *arg)
{
	int efd, ctlfd, statfd;
	ct_evthdl_t ev;
	ctevid_t evid;
	uint_t event;
	char path[PATH_MAX];
	char buf[80];
	ses_enum_target_t *stp;
	ct_stathdl_t stathdl;
	ctid_t ctid;
	struct pollfd fds;
	int pollret;
	sigset_t sigset;

	ses_ct_print("start contract event thread");
	efd = open64(CTFS_ROOT "/device/pbundle", O_RDONLY);
	fds.fd = efd;
	fds.events = POLLIN;
	fds.revents = 0;
	(void) sigaddset(&sigset, sesthread.thr_sig);
	(void) pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
	for (;;) {
		/* check if we've been asked to exit */
		(void) pthread_mutex_lock(&sesthread.mt);
		if (sesthread.doexit) {
			(void) pthread_mutex_unlock(&sesthread.mt);
			break;
		}
		(void) pthread_mutex_unlock(&sesthread.mt);

		/* poll until an event arrives */
		if ((pollret = poll(&fds, 1, 10000)) <= 0) {
			if (pollret == 0)
				ses_recheck_dir();
			continue;
		}

		/* read the event */
		(void) pthread_mutex_lock(&ses_sslmt);
		ses_ct_print("read contract event");
		if (ct_event_read(efd, &ev) != 0) {
			(void) pthread_mutex_unlock(&ses_sslmt);
			continue;
		}

		/* see if it is an event we are expecting */
		ctid = ct_event_get_ctid(ev);
		(void) snprintf(buf, sizeof (buf),
		    "got contract event ctid=%d", ctid);
		ses_ct_print(buf);
		event = ct_event_get_type(ev);
		if (event != CT_DEV_EV_OFFLINE && event != CT_EV_NEGEND) {
			(void) snprintf(buf, sizeof (buf),
			    "bad contract event %x", event);
			ses_ct_print(buf);
			ct_event_free(ev);
			(void) pthread_mutex_unlock(&ses_sslmt);
			continue;
		}

		/* find target pointer saved in cookie */
		evid = ct_event_get_evid(ev);
		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/status",
		    ctid);
		statfd = open64(path, O_RDONLY);
		(void) ct_status_read(statfd, CTD_COMMON, &stathdl);
		stp = (ses_enum_target_t *)(uintptr_t)
		    ct_status_get_cookie(stathdl);
		ct_status_free(stathdl);
		(void) close(statfd);

		/* check if target pointer is still valid */
		if (ses_ssl_valid(stp) == 0) {
			(void) snprintf(buf, sizeof (buf),
			    "contract already abandoned %x", event);
			ses_ct_print(buf);
			(void) snprintf(path, PATH_MAX,
			    CTFS_ROOT "/device/%ld/ctl", ctid);
			ctlfd = open64(path, O_WRONLY);
			if (event != CT_EV_NEGEND)
				(void) ct_ctl_ack(ctlfd, evid);
			else
				(void) ct_ctl_abandon(ctlfd);
			(void) close(ctlfd);
			ct_event_free(ev);
			(void) pthread_mutex_unlock(&ses_sslmt);
			continue;
		}

		/* find control device for ack/abandon */
		(void) pthread_mutex_lock(&stp->set_lock);
		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/ctl",
		    ctid);
		ctlfd = open64(path, O_WRONLY);
		if (event != CT_EV_NEGEND) {
			/* if this is an offline event, do the offline */
			ses_ct_print("got contract offline event");
			if (stp->set_target) {
				ses_ct_print("contract thread rele");
				ses_snap_rele(stp->set_snap);
				ses_close(stp->set_target);
				stp->set_target = NULL;
			}
			(void) ct_ctl_ack(ctlfd, evid);
		} else {
			/* if this is the negend, then abandon the contract */
			ses_ct_print("got contract negend");
			if (stp->set_ctid) {
				(void) snprintf(buf, sizeof (buf),
				    "abandon old contract %d", stp->set_ctid);
				ses_ct_print(buf);
				stp->set_ctid = 0;
			}
			(void) ct_ctl_abandon(ctlfd);
		}
		(void) close(ctlfd);
		(void) pthread_mutex_unlock(&stp->set_lock);
		ct_event_free(ev);
		(void) pthread_mutex_unlock(&ses_sslmt);
	}
	(void) close(efd);
	return (NULL);
}

int
find_thr_sig(void)
{
	int i;
	sigset_t oset, rset;
	int sig[] = {SIGTERM, SIGUSR1, SIGUSR2};
	int sig_sz = sizeof (sig) / sizeof (int);
	int rc = SIGTERM;

	/* prefered set of signals that are likely used to terminate threads */
	(void) sigemptyset(&oset);
	(void) pthread_sigmask(SIG_SETMASK, NULL, &oset);
	for (i = 0; i < sig_sz; i++) {
		if (sigismember(&oset, sig[i]) == 0) {
			return (sig[i]);
		}
	}

	/* reserved set of signals that are not allowed to terminate thread */
	(void) sigemptyset(&rset);
	(void) sigaddset(&rset, SIGABRT);
	(void) sigaddset(&rset, SIGKILL);
	(void) sigaddset(&rset, SIGSTOP);
	(void) sigaddset(&rset, SIGCANCEL);

	/* Find signal that is not masked and not in the reserved list. */
	for (i = 1; i < MAXSIG; i++) {
		if (sigismember(&rset, i) == 1) {
			continue;
		}
		if (sigismember(&oset, i) == 0) {
			return (i);
		}
	}

	return (rc);
}

/*ARGSUSED*/
static void
ses_handler(int sig)
{
}

static void
ses_thread_init(topo_mod_t *mod)
{
	pthread_attr_t *attr = NULL;
	struct sigaction act;
	ses_mod_list_t *smod;

	(void) pthread_mutex_lock(&sesthread.mt);
	sesthread.count++;
	smod = topo_mod_zalloc(mod, sizeof (*smod));
	smod->smod_mod = mod;
	smod->smod_next = ses_smod;
	ses_smod = smod;
	if (sesthread.tid == 0) {
		/* find a suitable signal to use for killing the thread below */
		sesthread.thr_sig = find_thr_sig();

		/* if don't have a handler for this signal, create one */
		(void) sigaction(sesthread.thr_sig, NULL, &act);
		if (act.sa_handler == SIG_DFL || act.sa_handler == SIG_IGN)
			act.sa_handler = ses_handler;
		(void) sigaction(sesthread.thr_sig, &act, NULL);

		/* create a thread to listen for offline events */
		(void) pthread_create(&sesthread.tid,
		    attr, ses_contract_thread, NULL);
	}
	(void) pthread_mutex_unlock(&sesthread.mt);
}

static void
ses_thread_fini(topo_mod_t *mod)
{
	ses_mod_list_t *smod, *prev_smod;

	(void) pthread_mutex_lock(&sesthread.mt);
	prev_smod = NULL;
	for (smod = ses_smod; smod != NULL; smod = smod->smod_next) {
		if (smod->smod_mod == mod) {
			if (prev_smod == NULL)
				ses_smod = smod->smod_next;
			else
				prev_smod->smod_next = smod->smod_next;
			topo_mod_free(mod, smod, sizeof (*smod));
			break;
		}
		prev_smod = smod;
	}
	if (--sesthread.count > 0) {
		(void) pthread_mutex_unlock(&sesthread.mt);
		return;
	}
	sesthread.doexit = 1;
	(void) pthread_mutex_unlock(&sesthread.mt);
	(void) pthread_kill(sesthread.tid, sesthread.thr_sig);
	(void) pthread_join(sesthread.tid, NULL);
	sesthread.tid = 0;
}

static void
ses_create_contract(topo_mod_t *mod, ses_enum_target_t *stp)
{
	int tfd, len, rval;
	char link_path[PATH_MAX];

	stp->set_ctid = 0;

	/* convert "/dev" path into "/devices" path */
	if ((len = readlink(stp->set_devpath, link_path, PATH_MAX)) < 0) {
		topo_mod_dprintf(mod, "readlink failed");
		return;
	}
	link_path[len] = '\0';

	/* set up template to create new contract */
	tfd = open64(CTFS_ROOT "/device/template", O_RDWR);
	(void) ct_tmpl_set_critical(tfd, CT_DEV_EV_OFFLINE);
	(void) ct_tmpl_set_cookie(tfd, (uint64_t)(uintptr_t)stp);

	/* strip "../../devices" off the front and create the contract */
	if ((rval = ct_dev_tmpl_set_minor(tfd, &link_path[13])) != 0)
		topo_mod_dprintf(mod, "failed to set minor %s rval = %d",
		    &link_path[13], rval);
	else if ((rval = ct_tmpl_create(tfd, &stp->set_ctid)) != 0)
		topo_mod_dprintf(mod, "failed to create ctid rval = %d", rval);
	else
		topo_mod_dprintf(mod, "created ctid=%d", stp->set_ctid);
	(void) close(tfd);
}

static void
ses_target_free(topo_mod_t *mod, ses_enum_target_t *stp)
{
	if (--stp->set_refcount == 0) {
		/* check if already closed due to contract offline request */
		(void) pthread_mutex_lock(&stp->set_lock);
		if (stp->set_target) {
			ses_snap_rele(stp->set_snap);
			ses_close(stp->set_target);
			stp->set_target = NULL;
		}
		if (stp->set_ctid) {
			int ctlfd;
			char path[PATH_MAX];

			topo_mod_dprintf(mod, "abandon old contract %d",
			    stp->set_ctid);
			(void) snprintf(path, PATH_MAX,
			    CTFS_ROOT "/device/%ld/ctl", stp->set_ctid);
			ctlfd = open64(path, O_WRONLY);
			(void) ct_ctl_abandon(ctlfd);
			(void) close(ctlfd);
			stp->set_ctid = 0;
		}
		(void) pthread_mutex_unlock(&stp->set_lock);
		ses_ssl_free(mod, stp);
		topo_mod_strfree(mod, stp->set_devpath);
		topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
	}
}

static void
ses_data_free(ses_enum_data_t *sdp, ses_enum_chassis_t *pcp)
{
	topo_mod_t *mod = sdp->sed_mod;
	ses_enum_chassis_t *cp;
	ses_enum_node_t *np;
	ses_enum_target_t *tp;
	ses_alt_node_t *ap;
	topo_list_t *cpl;


	if (pcp != NULL)
		cpl = &pcp->sec_subchassis;
	else
		cpl = &sdp->sed_chassis;

	while ((cp = topo_list_next(cpl)) != NULL) {
		topo_list_delete(cpl, cp);

		while ((np = topo_list_next(&cp->sec_nodes)) != NULL) {
			while ((ap = topo_list_next(&np->sen_alt_nodes)) !=
			    NULL) {
				topo_list_delete(&np->sen_alt_nodes, ap);
				topo_mod_free(mod, ap, sizeof (ses_alt_node_t));
			}
			topo_list_delete(&cp->sec_nodes, np);
			topo_mod_free(mod, np, sizeof (ses_enum_node_t));
		}

		while ((tp = topo_list_next(&cp->sec_targets)) != NULL) {
			topo_list_delete(&cp->sec_targets, tp);
			ses_target_free(mod, tp);
		}

		topo_mod_free(mod, cp, sizeof (ses_enum_chassis_t));
	}

	if (pcp == NULL) {
		dev_list_free(mod, &sdp->sed_devs);
		topo_mod_free(mod, sdp, sizeof (ses_enum_data_t));
	}
}

/*
 * For enclosure nodes, we have a special contains method.  By default, the hc
 * walker will compare the node name and instance number to determine if an
 * FMRI matches.  For enclosures where the enumeration order is impossible to
 * predict, we instead use the chassis-id as a unique identifier, and ignore
 * the instance number.
 */
static int
fmri_contains(topo_mod_t *mod, nvlist_t *nv1, nvlist_t *nv2)
{
	uint8_t v1, v2;
	nvlist_t **hcp1, **hcp2;
	int err, i;
	uint_t nhcp1, nhcp2;
	nvlist_t *a1, *a2;
	char *c1, *c2;
	int mindepth;

	if (nvlist_lookup_uint8(nv1, FM_VERSION, &v1) != 0 ||
	    nvlist_lookup_uint8(nv2, FM_VERSION, &v2) != 0 ||
	    v1 > FM_HC_SCHEME_VERSION || v2 > FM_HC_SCHEME_VERSION)
		return (topo_mod_seterrno(mod, EMOD_FMRI_VERSION));

	err = nvlist_lookup_nvlist_array(nv1, FM_FMRI_HC_LIST, &hcp1, &nhcp1);
	err |= nvlist_lookup_nvlist_array(nv2, FM_FMRI_HC_LIST, &hcp2, &nhcp2);
	if (err != 0)
		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));

	/*
	 * If the chassis-id doesn't match, then these FMRIs are not
	 * equivalent.  If one of the FMRIs doesn't have a chassis ID, then we
	 * have no choice but to fall back to the instance ID.
	 */
	if (nvlist_lookup_nvlist(nv1, FM_FMRI_AUTHORITY, &a1) == 0 &&
	    nvlist_lookup_nvlist(nv2, FM_FMRI_AUTHORITY, &a2) == 0 &&
	    nvlist_lookup_string(a1, FM_FMRI_AUTH_CHASSIS, &c1) == 0 &&
	    nvlist_lookup_string(a2, FM_FMRI_AUTH_CHASSIS, &c2) == 0) {
		if (strcmp(c1, c2) != 0)
			return (0);

		mindepth = 1;
	} else {
		mindepth = 0;
	}

	if (nhcp2 < nhcp1)
		return (0);

	for (i = 0; i < nhcp1; i++) {
		char *nm1 = NULL;
		char *nm2 = NULL;
		char *id1 = NULL;
		char *id2 = NULL;

		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_NAME, &nm1);
		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_NAME, &nm2);
		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_ID, &id1);
		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_ID, &id2);
		if (nm1 == NULL || nm2 == NULL || id1 == NULL || id2 == NULL)
			return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));

		if (strcmp(nm1, nm2) == 0 &&
		    (i < mindepth || strcmp(id1, id2) == 0))
			continue;

		return (0);
	}

	return (1);
}

/*ARGSUSED*/
static int
ses_contains(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
    nvlist_t *in, nvlist_t **out)
{
	int ret;
	nvlist_t *nv1, *nv2;

	if (version > TOPO_METH_CONTAINS_VERSION)
		return (topo_mod_seterrno(mod, EMOD_VER_NEW));

	if (nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_FMRI, &nv1) != 0 ||
	    nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_SUBFMRI, &nv2) != 0)
		return (topo_mod_seterrno(mod, EMOD_METHOD_INVAL));

	ret = fmri_contains(mod, nv1, nv2);
	if (ret < 0)
		return (-1);

	if (topo_mod_nvalloc(mod, out, NV_UNIQUE_NAME) == 0) {
		if (nvlist_add_uint32(*out, TOPO_METH_CONTAINS_RET,
		    ret) == 0)
			return (0);
		else
			nvlist_free(*out);
	}

	return (-1);

}

/*
 * Return a current instance of the node.  This is somewhat complicated because
 * we need to take a new snapshot in order to get the new data, but we don't
 * want to be constantly taking SES snapshots if the consumer is going to do a
 * series of queries.  So we adopt the strategy of assuming that the SES state
 * is not going to be rapidly changing, and limit our snapshot frequency to
 * some defined bounds.
 */
ses_node_t *
ses_node_lock(topo_mod_t *mod, tnode_t *tn)
{
	ses_enum_target_t *tp = topo_node_getspecific(tn);
	hrtime_t now;
	ses_snap_t *snap;
	int err;
	uint64_t nodeid;
	ses_node_t *np;

	if (tp == NULL) {
		(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
		return (NULL);
	}

	(void) pthread_mutex_lock(&tp->set_lock);

	/*
	 * Determine if we need to take a new snapshot.
	 */
	now = gethrtime();

	if (tp->set_target == NULL) {
		/*
		 * We may have closed the device but not yet abandoned the
		 * contract (ie we've had the offline event but not yet the
		 * negend). If so, just return failure.
		 */
		if (tp->set_ctid != 0) {
			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
			(void) pthread_mutex_unlock(&tp->set_lock);
			return (NULL);
		}

		/*
		 * The device has been closed due to a contract offline
		 * request, then we need to reopen it and create a new contract.
		 */
		if ((tp->set_target =
		    ses_open(LIBSES_VERSION, tp->set_devpath)) == NULL) {
			sysevent_id_t eid;

			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
			(void) pthread_mutex_unlock(&tp->set_lock);
			topo_mod_dprintf(mod, "recheck_dir - "
			    "can no longer open %s", tp->set_devpath);
			(void) sysevent_post_event(EC_PLATFORM,
			    ESC_PLATFORM_SP_RESET, SUNW_VENDOR, "fmd", NULL,
			    &eid);
			return (NULL);
		}
		topo_mod_dprintf(mod, "reopen contract");
		ses_create_contract(mod, tp);
		tp->set_snap = ses_snap_hold(tp->set_target);
		tp->set_snaptime = gethrtime();
	} else if (now - tp->set_snaptime > (ses_snap_freq * 1000 * 1000) &&
	    (snap = ses_snap_new(tp->set_target)) != NULL) {
		if (ses_snap_generation(snap) !=
		    ses_snap_generation(tp->set_snap)) {
			/*
			 * If we find ourselves in this situation, we're in
			 * trouble.  The generation count has changed, which
			 * indicates that our current topology is out of date.
			 * But we need to consult the new topology in order to
			 * determine presence at this moment in time.  We can't
			 * go back and change the topo snapshot in situ, so
			 * we'll just have to fail the call in this unlikely
			 * scenario.
			 */
			ses_snap_rele(snap);
			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
			(void) pthread_mutex_unlock(&tp->set_lock);
			return (NULL);
		} else {
			ses_snap_rele(tp->set_snap);
			tp->set_snap = snap;
		}
		tp->set_snaptime = gethrtime();
	}

	snap = tp->set_snap;

	verify(topo_prop_get_uint64(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_NODE_ID, &nodeid, &err) == 0);
	verify((np = ses_node_lookup(snap, nodeid)) != NULL);

	return (np);
}

/*ARGSUSED*/
void
ses_node_unlock(topo_mod_t *mod, tnode_t *tn)
{
	ses_enum_target_t *tp = topo_node_getspecific(tn);

	verify(tp != NULL);

	(void) pthread_mutex_unlock(&tp->set_lock);
}

/*
 * Determine if the element is present.
 */
/*ARGSUSED*/
static int
ses_present(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
    nvlist_t *in, nvlist_t **out)
{
	boolean_t present;
	ses_node_t *np;
	nvlist_t *props, *nvl;
	uint64_t status;

	if ((np = ses_node_lock(mod, tn)) == NULL)
		return (-1);

	verify((props = ses_node_props(np)) != NULL);
	verify(nvlist_lookup_uint64(props,
	    SES_PROP_STATUS_CODE, &status) == 0);

	ses_node_unlock(mod, tn);

	present = (status != SES_ESC_NOT_INSTALLED);

	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0)
		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));

	if (nvlist_add_uint32(nvl, TOPO_METH_PRESENT_RET,
	    present) != 0) {
		nvlist_free(nvl);
		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
	}

	*out = nvl;

	return (0);
}

/*
 * Sets standard properties for a ses node (enclosure, bay, controller
 * or expander).
 * This includes setting the FRU, as well as setting the
 * authority information.  When  the fru topo node(frutn) is not NULL
 * its resouce should be used as FRU.
 */
static int
ses_set_standard_props(topo_mod_t *mod, tnode_t *frutn, tnode_t *tn,
    nvlist_t *auth, uint64_t nodeid, const char *path)
{
	int err;
	char *product, *chassis;
	nvlist_t *fmri;

	/*
	 * Set the authority explicitly if specified.
	 */
	if (auth) {
		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
		    &product) == 0);
		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
		    &chassis) == 0);
		if (topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
		    FM_FMRI_AUTH_PRODUCT, TOPO_PROP_IMMUTABLE, product,
		    &err) != 0 ||
		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
		    FM_FMRI_AUTH_CHASSIS, TOPO_PROP_IMMUTABLE, chassis,
		    &err) != 0 ||
		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
		    FM_FMRI_AUTH_SERVER, TOPO_PROP_IMMUTABLE, "",
		    &err) != 0) {
			topo_mod_dprintf(mod, "failed to add authority "
			    "properties: %s\n", topo_strerror(err));
			return (topo_mod_seterrno(mod, err));
		}
	}

	/*
	 * Copy the resource and set that as the FRU.
	 */
	if (frutn != NULL) {
		if (topo_node_resource(frutn, &fmri, &err) != 0) {
			topo_mod_dprintf(mod,
			    "topo_node_resource() failed : %s\n",
			    topo_strerror(err));
			return (topo_mod_seterrno(mod, err));
		}
	} else {
		if (topo_node_resource(tn, &fmri, &err) != 0) {
			topo_mod_dprintf(mod,
			    "topo_node_resource() failed : %s\n",
			    topo_strerror(err));
			return (topo_mod_seterrno(mod, err));
		}
	}

	if (topo_node_fru_set(tn, fmri, 0, &err) != 0) {
		topo_mod_dprintf(mod,
		    "topo_node_fru_set() failed : %s\n",
		    topo_strerror(err));
		nvlist_free(fmri);
		return (topo_mod_seterrno(mod, err));
	}

	nvlist_free(fmri);

	/*
	 * Set the SES-specific properties so that consumers can query
	 * additional information about the particular SES element.
	 */
	if (topo_pgroup_create(tn, &ses_pgroup, &err) != 0) {
		topo_mod_dprintf(mod, "failed to create propgroup "
		    "%s: %s\n", TOPO_PGROUP_SES, topo_strerror(err));
		return (-1);
	}

	if (topo_prop_set_uint64(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_NODE_ID, TOPO_PROP_IMMUTABLE,
	    nodeid, &err) != 0) {
		topo_mod_dprintf(mod,
		    "failed to create property %s: %s\n",
		    TOPO_PROP_NODE_ID, topo_strerror(err));
		return (-1);
	}

	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_TARGET_PATH, TOPO_PROP_IMMUTABLE,
	    path, &err) != 0) {
		topo_mod_dprintf(mod,
		    "failed to create property %s: %s\n",
		    TOPO_PROP_TARGET_PATH, topo_strerror(err));
		return (-1);
	}

	return (0);
}

/*
 * Iterate over the SES phy information. If any of the ports indicates that it's
 * a SATA device and we haven't matched any disk devices yet, that means
 * that the HBA was able to create a WWN for the SATA device based on its GUID,
 * which is good. However, SES includes the WWN for the device's STP bridge. In
 * theory, if the driver includes the WWN based on the SATA guid then it should
 * also set the bridge-port property indicating the WWN that should match the
 * SATA device.
 */
static int
ses_create_disk_bridge(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props,
    tnode_t **child)
{
	nvlist_t **phys;
	uint_t i, n_phys;
	topo_mod_t *mod = sdp->sed_mod;

	if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS, &phys,
	    &n_phys) != 0)
		return (1);

	for (i = 0; i < n_phys; i++) {
		uint64_t wwn;
		boolean_t sata;
		char wwnstr[64];

		if (nvlist_lookup_uint64(phys[i], SES_SAS_PROP_ADDR,
		    &wwn) != 0 || wwn == 0) {
			continue;
		}

		if (nvlist_lookup_boolean_value(phys[i],
		    SES_SAS_PROP_SATA_DEVICE, &sata) != 0 || !sata) {
			continue;
		}

		if (scsi_wwn_to_wwnstr(wwn, 0, wwnstr) == NULL)
			continue;

		if (disk_declare_bridge(mod, pnode, &sdp->sed_devs,
		    wwnstr, child) == 0) {
			return (0);
		}

	}

	return (1);
}

/*
 * Callback to add a disk to a given bay.  We first check the status-code to
 * determine if a disk is present, ignoring those that aren't in an appropriate
 * state.  We then scan the parent bay node's SAS address array to determine
 * possible attached SAS addresses.  We create a disk node if the disk is not
 * SAS or the SES target does not support the necessary pages for this; if we
 * find the SAS address, we create a disk node and also correlate it with
 * the corresponding Solaris device node to fill in the rest of the data.
 */
static int
ses_create_disk(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props)
{
	topo_mod_t *mod = sdp->sed_mod;
	uint64_t status;
	uint_t s, nsas;
	char **paths;
	int err, ret;
	tnode_t *child = NULL;

	/*
	 * Skip devices that are not in a present (and possibly damaged) state.
	 * Also, skip devices that this expander is either not fully wired to,
	 * or are hidden due to SAS zoning, as indicated by the
	 * SES_ESC_NO_ACCESS state.
	 */
	if (nvlist_lookup_uint64(props, SES_PROP_STATUS_CODE, &status) != 0)
		return (0);

	if (status != SES_ESC_UNSUPPORTED &&
	    status != SES_ESC_OK &&
	    status != SES_ESC_CRITICAL &&
	    status != SES_ESC_NONCRITICAL &&
	    status != SES_ESC_UNRECOVERABLE &&
	    status != SES_ESC_UNKNOWN)
		return (0);

	topo_mod_dprintf(mod, "found attached disk");

	/*
	 * Create the disk range.
	 */
	if (topo_node_range_create(mod, pnode, DISK, 0, 0) != 0) {
		topo_mod_dprintf(mod,
		    "topo_node_create_range() failed: %s",
		    topo_mod_errmsg(mod));
		return (-1);
	}

	/*
	 * Look through all SAS addresses and attempt to correlate them to a
	 * known Solaris device.  If we don't find a matching node, then we
	 * don't enumerate the disk node.
	 * Note that TOPO_PROP_SAS_ADDR prop includes SAS address from
	 * alternate elements that represent the same device.
	 */
	if (topo_prop_get_string_array(pnode, TOPO_PGROUP_SES,
	    TOPO_PROP_SAS_ADDR, &paths, &nsas, &err) != 0)
		return (0);

	err = 0;

	for (s = 0; s < nsas; s++) {
		ret = disk_declare_addr(mod, pnode, &sdp->sed_devs, paths[s],
		    &child);
		if (ret == 0) {
			break;
		} else if (ret < 0) {
			err = -1;
			break;
		}
	}

	/*
	 * We need to take another pass through the properties for this bay by
	 * iterating over the phys and noting if any of these are SATA. Note,
	 * this information isn't commonly part of the topo tree at this time,
	 * hence why we end up going back and iterating over the properties
	 * ourselves.
	 */
	if (s == nsas) {
		if (ses_create_disk_bridge(sdp, pnode, props, &child) != 0)
			(void) disk_declare_non_enumerated(mod, pnode, &child);
	}

	/* copy sas_addresses (target-ports) from parent (with 'w'added) */
	if (child != NULL) {
		int i;
		char **tports;
		uint64_t wwn;

		tports = topo_mod_zalloc(mod, sizeof (char *) * nsas);
		if (tports != NULL) {
			for (i = 0; i < nsas; i++) {
				if (scsi_wwnstr_to_wwn(paths[i], &wwn) !=
				    DDI_SUCCESS)
					break;
				tports[i] = scsi_wwn_to_wwnstr(wwn, 1, NULL);
				if (tports[i] == NULL)
					break;
			}
			/* if they all worked then create the property */
			if (i == nsas)
				(void) topo_prop_set_string_array(child,
				    TOPO_PGROUP_STORAGE,
				    TOPO_STORAGE_TARGET_PORT_L0IDS,
				    TOPO_PROP_IMMUTABLE, (const char **)tports,
				    nsas, &err);

			for (i = 0; i < nsas; i++)
				if (tports[i] != NULL)
					scsi_free_wwnstr(tports[i]);
			topo_mod_free(mod, tports, sizeof (char *) * nsas);
		}
	}

	for (s = 0; s < nsas; s++)
		topo_mod_free(mod, paths[s], strlen(paths[s]) + 1);
	topo_mod_free(mod, paths, nsas * sizeof (char *));

	return (err);
}

static int
ses_add_bay_props(topo_mod_t *mod, tnode_t *tn, ses_enum_node_t *snp)
{
	ses_alt_node_t *ap;
	ses_node_t *np;
	nvlist_t *props;

	nvlist_t **phys;
	uint_t i, j, n_phys, all_phys = 0;
	char **paths;
	uint64_t addr;
	size_t len;
	int terr, err = -1;

	for (ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
	    ap = topo_list_next(ap)) {
		np = ap->san_node;
		props = ses_node_props(np);

		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
		    &phys, &n_phys) != 0)
			continue;

		all_phys += n_phys;
	}

	if (all_phys == 0)
		return (0);

	if ((paths = topo_mod_zalloc(mod, all_phys * sizeof (char *))) == NULL)
		return (-1);

	for (i = 0, ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
	    ap = topo_list_next(ap)) {
		np = ap->san_node;
		props = ses_node_props(np);

		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
		    &phys, &n_phys) != 0)
			continue;

		for (j = 0; j < n_phys; j++) {
			if (nvlist_lookup_uint64(phys[j], SES_SAS_PROP_ADDR,
			    &addr) != 0)
				continue;

			len = snprintf(NULL, 0, "%016llx", addr) + 1;
			if ((paths[i] = topo_mod_alloc(mod, len)) == NULL)
				goto error;

			(void) snprintf(paths[i], len, "%016llx", addr);

			++i;
		}
	}

	err = topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE,
	    (const char **)paths, i, &terr);
	if (err != 0)
		err = topo_mod_seterrno(mod, terr);

error:
	for (i = 0; i < all_phys && paths[i] != NULL; i++)
		topo_mod_free(mod, paths[i], strlen(paths[i]) + 1);
	topo_mod_free(mod, paths, all_phys * sizeof (char *));

	return (err);
}

static const char *
lookup_bay_override(const char *product_id)
{
	for (uint_t i = 0; i < N_BAY_LBL_OVERRIDES; i++) {
		if (strcmp(product_id,
		    bay_label_overrides[i].slbl_product) == 0) {
			return (bay_label_overrides[i].slbl_mname);
		}
	}
	return (NULL);
}

/*
 * Callback to create a basic node (bay, psu, fan, or controller and expander).
 */
static int
ses_create_generic(ses_enum_data_t *sdp, ses_enum_node_t *snp, tnode_t *pnode,
    tnode_t *frutn, const char *nodename, const char *labelname,
    tnode_t **node)
{
	ses_node_t *np = snp->sen_node;
	ses_node_t *parent;
	uint64_t instance = snp->sen_instance;
	topo_mod_t *mod = sdp->sed_mod;
	nvlist_t *props, *aprops;
	nvlist_t *auth = NULL, *fmri = NULL;
	tnode_t *tn = NULL;
	char *clean_label = NULL, label[128];
	int err;
	char *part = NULL, *serial = NULL, *revision = NULL;
	char *desc;
	boolean_t report;

	props = ses_node_props(np);

	(void) nvlist_lookup_string(props, LIBSES_PROP_PART, &part);
	(void) nvlist_lookup_string(props, LIBSES_PROP_SERIAL, &serial);

	topo_mod_dprintf(mod, "adding %s %llu", nodename, instance);

	/*
	 * Create the node.  The interesting information is all copied from the
	 * parent enclosure node, so there is not much to do.
	 */
	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
		goto error;

	/*
	 * We want to report revision information for the controller nodes, but
	 * we do not get per-element revision information.  However, we do have
	 * revision information for the entire enclosure, and we can use the
	 * 'reported-via' property to know that this controller corresponds to
	 * the given revision information.  This means we cannot get revision
	 * information for targets we are not explicitly connected to, but
	 * there is little we can do about the situation.
	 */
	if (strcmp(nodename, CONTROLLER) == 0 &&
	    nvlist_lookup_boolean_value(props, SES_PROP_REPORT, &report) == 0 &&
	    report) {
		for (parent = ses_node_parent(np); parent != NULL;
		    parent = ses_node_parent(parent)) {
			if (ses_node_type(parent) == SES_NODE_ENCLOSURE) {
				(void) nvlist_lookup_string(
				    ses_node_props(parent),
				    SES_EN_PROP_REV, &revision);
				break;
			}
		}
	}

	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
	    nodename, (topo_instance_t)instance, NULL, auth, part, revision,
	    serial)) == NULL) {
		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	if ((tn = topo_node_bind(mod, pnode, nodename,
	    instance, fmri)) == NULL) {
		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	/*
	 * For the node label, we look for the following in order:
	 *
	 * <ses-description>
	 * <ses-class-description> <instance>
	 * <default-type-label> <instance>
	 */
	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
	    desc[0] == '\0') {
		parent = ses_node_parent(np);
		aprops = ses_node_props(parent);
		if (nvlist_lookup_string(aprops, SES_PROP_CLASS_DESCRIPTION,
		    &desc) != 0 || desc[0] == '\0')
			desc = (char *)labelname;
		(void) snprintf(label, sizeof (label), "%s %llu", desc,
		    instance);
		desc = label;
	}

	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
		goto error;

	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
		goto error;

	if (ses_set_standard_props(mod, frutn, tn, NULL, ses_node_id(np),
	    snp->sen_target->set_devpath) != 0)
		goto error;

	if (strcmp(nodename, BAY) == 0) {
		const char *label_method;
		char *product;
		nvlist_t *args = NULL;

		if (ses_add_bay_props(mod, tn, snp) != 0)
			goto error;

		if (topo_method_register(mod, tn, ses_bay_methods) != 0) {
			topo_mod_dprintf(mod,
			    "topo_method_register() failed: %s",
			    topo_mod_errmsg(mod));
			goto error;
		}

		/*
		 * Ideally we'd perform this sort of override with a platform
		 * specific XML map file, and that would work here if we only
		 * wanted to override the bay node label.  However, we'd also
		 * like the disk node label (if the bay is occupied) to inherit
		 * the overriden bay label.  So we need to ensure the
		 * propmethod is registered before we create the child disk
		 * node.
		 */
		if ((product = topo_mod_product(mod)) == NULL) {
			(void) topo_mod_seterrno(mod, EMOD_NOMEM);
			goto error;
		}
		if ((label_method = lookup_bay_override(product)) != NULL) {
			if (topo_mod_nvalloc(mod, &args, NV_UNIQUE_NAME) != 0 ||
			    topo_prop_method_register(tn, TOPO_PGROUP_PROTOCOL,
			    TOPO_PROP_LABEL, TOPO_TYPE_STRING, label_method,
			    args, &err)) {
				topo_mod_dprintf(mod,
				    "Failed to register method: %s on %s=%"
				    PRIu64, label_method, BAY,
				    topo_node_instance(tn));
				topo_mod_strfree(mod, product);
				nvlist_free(args);
				goto error;
			}
			nvlist_free(args);
		}
		topo_mod_strfree(mod, product);

		if (ses_create_disk(sdp, tn, props) != 0)
			goto error;
	} else if ((strcmp(nodename, FAN) == 0) ||
	    (strcmp(nodename, PSU) == 0) ||
	    (strcmp(nodename, CONTROLLER) == 0)) {
		/*
		 * Only fan, psu, and controller nodes have a 'present' method.
		 * Bay nodes are always present, and disk nodes are present by
		 * virtue of being enumerated and SAS expander nodes and
		 * SAS connector nodes are also always present once
		 * the parent controller is found.
		 */
		if (topo_method_register(mod, tn, ses_component_methods) != 0) {
			topo_mod_dprintf(mod,
			    "topo_method_register() failed: %s",
			    topo_mod_errmsg(mod));
			goto error;
		}

	}

	snp->sen_target->set_refcount++;
	topo_node_setspecific(tn, snp->sen_target);

	nvlist_free(auth);
	nvlist_free(fmri);
	topo_mod_strfree(mod, clean_label);
	if (node != NULL) *node = tn;
	return (0);

error:
	nvlist_free(auth);
	nvlist_free(fmri);
	topo_mod_strfree(mod, clean_label);
	return (-1);
}

/*
 * Create SAS expander specific props.
 */
/*ARGSUSED*/
static int
ses_set_expander_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
    tnode_t *ptnode, tnode_t *tnode, int *phycount, int64_t *connlist)
{
	ses_node_t *np = snp->sen_node;
	topo_mod_t *mod = sdp->sed_mod;
	nvlist_t *auth = NULL, *fmri = NULL;
	nvlist_t *props, **phylist;
	int err, i;
	uint_t pcount;
	uint64_t sasaddr, connidx;
	char sasaddr_str[17];
	boolean_t found = B_FALSE, ses_found = B_FALSE;
	dev_di_node_t *dnode, *sesdnode;

	props = ses_node_props(np);

	/*
	 * the uninstalled expander is not enumerated by checking
	 * the element status code.  No present present' method provided.
	 */
	/*
	 * Get the Expander SAS address.  It should exist.
	 */
	if (nvlist_lookup_uint64(props, SES_EXP_PROP_SAS_ADDR,
	    &sasaddr) != 0) {
		topo_mod_dprintf(mod,
		    "Failed to get prop %s.", SES_EXP_PROP_SAS_ADDR);
		goto error;
	}

	(void) sprintf(sasaddr_str, "%llx", sasaddr);

	/* search matching dev_di_node. */
	for (dnode = topo_list_next(&sdp->sed_devs); dnode != NULL;
	    dnode = topo_list_next(dnode)) {
		for (i = 0; i < dnode->ddn_ppath_count; i++) {
			if ((dnode->ddn_target_port[i] != NULL) &&
			    (strstr(dnode->ddn_target_port[i],
			    sasaddr_str) != NULL)) {
				found = B_TRUE;
				break;
			}
		}
		if (found)
			break;
	}

	if (!found) {
		topo_mod_dprintf(mod,
		    "ses_set_expander_props: Failed to find matching "
		    "devinfo node for Exapnder SAS address %s",
		    SES_EXP_PROP_SAS_ADDR);
		/* continue on to get storage group props. */
	} else {
		/* create/set the devfs-path and devid in the smp group */
		if (topo_pgroup_create(tnode, &smp_pgroup, &err) != 0) {
			topo_mod_dprintf(mod, "ses_set_expander_props: "
			    "failed to create smp property group %s\n",
			    topo_strerror(err));
			goto error;
		} else {
			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
			    TOPO_PROP_SMP_TARGET_PORT, TOPO_PROP_IMMUTABLE,
			    dnode->ddn_target_port[i], &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
				    topo_strerror(err));
			}
			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
			    TOPO_PROP_SMP_DEV_PATH, TOPO_PROP_IMMUTABLE,
			    dnode->ddn_dpath, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set dev error %s\n", topo_strerror(err));
			}
			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
			    TOPO_PROP_SMP_DEVID, TOPO_PROP_IMMUTABLE,
			    dnode->ddn_devid, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set devid error %s\n", topo_strerror(err));
			}
			if (dnode->ddn_ppath_count != 0 &&
			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SMP,
			    TOPO_PROP_SMP_PHYS_PATH, TOPO_PROP_IMMUTABLE,
			    (const char **)dnode->ddn_ppath,
			    dnode->ddn_ppath_count, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set phys-path error %s\n",
				    topo_strerror(err));
			}
		}
	}

	/* update the ses property group with SES target info */
	if ((topo_pgroup_create(tnode, &ses_pgroup, &err) != 0) &&
	    (err != ETOPO_PROP_DEFD)) {
		/* SES prop group doesn't exist but failed to be created. */
		topo_mod_dprintf(mod, "ses_set_expander_props: "
		    "ses pgroup create error %s\n", topo_strerror(err));
		goto error;
	} else {
		/* locate assciated enclosure dev_di_node. */
		for (sesdnode = topo_list_next(&sdp->sed_devs);
		    sesdnode != NULL; sesdnode = topo_list_next(sesdnode)) {
			for (i = 0; i < sesdnode->ddn_ppath_count; i++) {
				/*
				 * check if attached port exists and
				 * its node type is enclosure and
				 * attached port is same as sas address of
				 * the expander and
				 * bridge port for virtual phy indication
				 * exist.
				 */
				if ((sesdnode->ddn_attached_port[i] != NULL) &&
				    (sesdnode->ddn_dtype == DTYPE_ESI) &&
				    (strstr(sesdnode->ddn_attached_port[i],
				    sasaddr_str) != NULL) &&
				    (sesdnode->ddn_bridge_port[i] != NULL)) {
					ses_found = B_TRUE;
					break;
				}
			}
			if (ses_found) break;
		}

		if (ses_found) {
			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
			    TOPO_PROP_SES_TARGET_PORT, TOPO_PROP_IMMUTABLE,
			    sesdnode->ddn_target_port[i], &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set ses %S error %s\n", TOPO_PROP_SAS_ADDR,
				    topo_strerror(err));
			}
			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
			    TOPO_PROP_SES_DEV_PATH, TOPO_PROP_IMMUTABLE,
			    sesdnode->ddn_dpath, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set ses dev error %s\n",
				    topo_strerror(err));
			}
			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
			    TOPO_PROP_SES_DEVID, TOPO_PROP_IMMUTABLE,
			    sesdnode->ddn_devid, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set ses devid error %s\n",
				    topo_strerror(err));
			}
			if (sesdnode->ddn_ppath_count != 0 &&
			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SES,
			    TOPO_PROP_SES_PHYS_PATH, TOPO_PROP_IMMUTABLE,
			    (const char **)sesdnode->ddn_ppath,
			    sesdnode->ddn_ppath_count, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set ses phys-path error %s\n",
				    topo_strerror(err));
			}

		}
	}

	/* create the storage group */
	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
		topo_mod_dprintf(mod, "ses_set_expander_props: "
		    "create storage error %s\n", topo_strerror(err));
		goto error;
	} else {
		/* set the SAS address prop out of expander element status. */
		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
		    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE, sasaddr_str,
		    &err) != 0) {
			topo_mod_dprintf(mod, "ses_set_expander_props: "
			    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
			    topo_strerror(err));
		}

		/* Get the phy information for the expander */
		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
		    &phylist, &pcount) != 0) {
			topo_mod_dprintf(mod,
			    "Failed to get prop %s.", SES_SAS_PROP_PHYS);
		} else {
			/*
			 * For each phy, get the connector element index and
			 * stores into connector element index array.
			 */
			*phycount = pcount;
			for (i = 0; i < pcount; i++) {
				if (nvlist_lookup_uint64(phylist[i],
				    SES_PROP_CE_IDX, &connidx) == 0) {
					if (connidx != 0xff) {
						connlist[i] = connidx;
					} else {
						connlist[i] = -1;
					}
				} else {
					/* Fail to get the index. set to -1. */
					connlist[i] = -1;
				}
			}

			/* set the phy count prop of the expander. */
			if (topo_prop_set_uint64(tnode, TOPO_PGROUP_STORAGE,
			    TOPO_PROP_PHY_COUNT, TOPO_PROP_IMMUTABLE, pcount,
			    &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
				    topo_strerror(err));
			}

			/*
			 * set the connector element index of
			 * the expander phys.
			 */
		}

		/* populate other misc storage group properties */
		if (found) {
			if (dnode->ddn_mfg && (topo_prop_set_string(tnode,
			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MANUFACTURER,
			    TOPO_PROP_IMMUTABLE, dnode->ddn_mfg, &err) != 0)) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set mfg error %s\n", topo_strerror(err));
			}

			if (dnode->ddn_model && (topo_prop_set_string(tnode,
			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MODEL,
			    TOPO_PROP_IMMUTABLE,
			    dnode->ddn_model, &err) != 0)) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set model error %s\n", topo_strerror(err));
			}

			if (dnode->ddn_serial && (topo_prop_set_string(tnode,
			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_SERIAL_NUM,
			    TOPO_PROP_IMMUTABLE,
			    dnode->ddn_serial, &err) != 0)) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set serial error %s\n",
				    topo_strerror(err));
			}

			if (dnode->ddn_firm && (topo_prop_set_string(tnode,
			    TOPO_PGROUP_STORAGE,
			    TOPO_STORAGE_FIRMWARE_REV, TOPO_PROP_IMMUTABLE,
			    dnode->ddn_firm, &err) != 0)) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set firm error %s\n", topo_strerror(err));
			}
		}
	}

	return (0);

error:
	nvlist_free(auth);
	nvlist_free(fmri);
	return (-1);
}

/*
 * Create SAS expander specific props.
 */
/*ARGSUSED*/
static int
ses_set_connector_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
    tnode_t *tnode, int64_t phy_mask)
{
	ses_node_t *np = snp->sen_node;
	topo_mod_t *mod = sdp->sed_mod;
	nvlist_t *props;
	int err, i;
	uint64_t conntype;
	char phymask_str[17], *conntype_str;
	boolean_t   found;

	props = ses_node_props(np);

	/*
	 * convert phy mask to string.
	 */
	(void) snprintf(phymask_str, 17, "%llx", phy_mask);

	/* create the storage group */
	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
		topo_mod_dprintf(mod, "ses_set_expander_props: "
		    "create storage error %s\n", topo_strerror(err));
		return (-1);
	} else {
		/* set the SAS address prop of the expander. */
		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
		    TOPO_STORAGE_SAS_PHY_MASK, TOPO_PROP_IMMUTABLE,
		    phymask_str, &err) != 0) {
			topo_mod_dprintf(mod, "ses_set_expander_props: "
			    "set %S error %s\n", TOPO_STORAGE_SAS_PHY_MASK,
			    topo_strerror(err));
		}

		/* Get the connector type information for the expander */
		if (nvlist_lookup_uint64(props,
		    SES_SC_PROP_CONNECTOR_TYPE, &conntype) != 0) {
			topo_mod_dprintf(mod, "Failed to get prop %s.",
			    TOPO_STORAGE_SAS_PHY_MASK);
		} else {
			found = B_FALSE;
			for (i = 0; ; i++) {
				if (sas_connector_type_list[i].sct_type ==
				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
					break;
				}
				if (sas_connector_type_list[i].sct_type ==
				    conntype) {
					conntype_str =
					    sas_connector_type_list[i].sct_name;
					found = B_TRUE;
					break;
				}
			}

			if (!found) {
				if (conntype <
				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
					conntype_str =
					    SAS_CONNECTOR_TYPE_RESERVED;
				} else {
					conntype_str =
					    SAS_CONNECTOR_TYPE_NOT_DEFINED;
				}
			}

			/* set the phy count prop of the expander. */
			if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
			    TOPO_STORAGE_SAS_CONNECTOR_TYPE,
			    TOPO_PROP_IMMUTABLE, conntype_str, &err) != 0) {
				topo_mod_dprintf(mod, "ses_set_expander_props: "
				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
				    topo_strerror(err));
			}
		}
	}

	return (0);
}

/*
 * Instantiate SAS expander nodes for a given ESC Electronics node(controller)
 * nodes.
 */
/*ARGSUSED*/
static int
ses_create_esc_sasspecific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
    tnode_t *pnode, ses_enum_chassis_t *cp,
    boolean_t dorange)
{
	topo_mod_t *mod = sdp->sed_mod;
	tnode_t	*exptn, *contn;
	boolean_t found;
	sas_connector_phy_data_t connectors[64] = {0};
	uint64_t max;
	ses_enum_node_t *ctlsnp, *xsnp, *consnp;
	ses_node_t *np = snp->sen_node;
	nvlist_t *props, *psprops;
	uint64_t index, psindex, conindex, psstatus, i, j, count;
	int64_t cidxlist[256] = {0};
	int phycount;

	props = ses_node_props(np);

	if (nvlist_lookup_uint64(props, SES_PROP_ELEMENT_ONLY_INDEX,
	    &index) != 0)
		return (-1);

	/*
	 * For SES constroller node, check to see if there are
	 * associated SAS expanders.
	 */
	found = B_FALSE;
	max = 0;
	for (ctlsnp = topo_list_next(&cp->sec_nodes); ctlsnp != NULL;
	    ctlsnp = topo_list_next(ctlsnp)) {
		if (ctlsnp->sen_type == SES_ET_SAS_EXPANDER) {
			found = B_TRUE;
			if (ctlsnp->sen_instance > max)
				max = ctlsnp->sen_instance;
		}
	}

	/*
	 * No SAS expander found notthing to process.
	 */
	if (!found)
		return (0);

	topo_mod_dprintf(mod, "%s Controller %d: creating "
	    "%llu %s nodes", cp->sec_csn, index, max + 1, SASEXPANDER);

	/*
	 * The max number represent the number of elements
	 * deducted from the highest SES_PROP_ELEMENT_CLASS_INDEX
	 * of SET_ET_SAS_EXPANDER type element.
	 *
	 * There may be multiple ESC Electronics element(controllers)
	 * within JBOD(typicall two for redundancy) and SAS expander
	 * elements are associated with only one of them.  We are
	 * still creating the range based max number here.
	 * That will cover the case that all expanders are associated
	 * with one SES controller.
	 */
	if (dorange && topo_node_range_create(mod, pnode,
	    SASEXPANDER, 0, max) != 0) {
		topo_mod_dprintf(mod,
		    "topo_node_create_range() failed: %s",
		    topo_mod_errmsg(mod));
		return (-1);
	}

	/*
	 * Search exapnders with the parent index matching with
	 * ESC Electronics element index.
	 * Note the index used here is a global index across
	 * SES elements.
	 */
	for (xsnp = topo_list_next(&cp->sec_nodes); xsnp != NULL;
	    xsnp = topo_list_next(xsnp)) {
		if (xsnp->sen_type == SES_ET_SAS_EXPANDER) {
			/*
			 * get the parent ESC controller.
			 */
			psprops = ses_node_props(xsnp->sen_node);
			if (nvlist_lookup_uint64(psprops,
			    SES_PROP_STATUS_CODE, &psstatus) == 0) {
				if (psstatus == SES_ESC_NOT_INSTALLED) {
					/*
					 * Not installed.
					 * Don't create a ndoe.
					 */
					continue;
				}
			} else {
				/*
				 * The element should have status code.
				 * If not there is no way to find
				 * out if the expander element exist or
				 * not.
				 */
				continue;
			}

			/* Get the physical parent index to compare. */
			if (nvlist_lookup_uint64(psprops,
			    LIBSES_PROP_PHYS_PARENT, &psindex) == 0) {
				if (index == psindex) {
		/* indentation moved forward */
		/*
		 * Handle basic node information of SAS expander
		 * element - binding to parent node and
		 * allocating FMRI...
		 */
		if (ses_create_generic(sdp, xsnp, pnode, pnode, SASEXPANDER,
		    "SAS-EXPANDER", &exptn) != 0)
			continue;
		/*
		 * Now handle SAS expander unique portion of node creation.
		 * The max nubmer of the phy count is 256 since SES-2
		 * defines as 1 byte field.  The cidxlist has the same
		 * number of elements.
		 *
		 * We use size 64 array to store the connectors.
		 * Typically a connectors associated with 4 phys so that
		 * matches with the max number of connecters associated
		 * with an expander.
		 * The phy count goes up to 38 for Sun supported
		 * JBOD.
		 */
		(void) memset(cidxlist, 0, sizeof (int64_t) * 64);
		if (ses_set_expander_props(sdp, xsnp, pnode, exptn, &phycount,
		    cidxlist) != 0) {
			/*
			 * error on getting specific prop failed.
			 * continue on.  Note that the node is
			 * left bound.
			 */
			continue;
		}

		/*
		 * count represetns the number of connectors discovered so far.
		 */
		count = 0;
		(void) memset(connectors, 0,
		    sizeof (sas_connector_phy_data_t) * 64);
		for (i = 0; i < phycount; i++) {
			if (cidxlist[i] != -1) {
				/* connector index is valid. */
				for (j = 0; j < count; j++) {
					if (connectors[j].scpd_index ==
					    cidxlist[i]) {
						/*
						 * Just update phy mask.
						 * The postion for connector
						 * index lists(cidxlist index)
						 * is set.
						 */
						connectors[j].scpd_pm =
						    connectors[j].scpd_pm |
						    (1ULL << i);
						break;
					}
				}
				/*
				 * If j and count matche a  new connector
				 * index is found.
				 */
				if (j == count) {
					/* add a new index and phy mask. */
					connectors[count].scpd_index =
					    cidxlist[i];
					connectors[count].scpd_pm =
					    connectors[count].scpd_pm |
					    (1ULL << i);
					count++;
				}
			}
		}

		/*
		 * create range for the connector nodes.
		 * The class index of the ses connector element
		 * is set as the instance nubmer for the node.
		 * Even though one expander may not have all connectors
		 * are associated with we are creating the range with
		 * max possible instance number.
		 */
		found = B_FALSE;
		max = 0;
		for (consnp = topo_list_next(&cp->sec_nodes);
		    consnp != NULL; consnp = topo_list_next(consnp)) {
			if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
				psprops = ses_node_props(consnp->sen_node);
				found = B_TRUE;
				if (consnp->sen_instance > max)
					max = consnp->sen_instance;
			}
		}

		/*
		 * No SAS connector found nothing to process.
		 */
		if (!found)
			return (0);

		if (dorange && topo_node_range_create(mod, exptn,
		    RECEPTACLE, 0, max) != 0) {
			topo_mod_dprintf(mod,
			    "topo_node_create_range() failed: %s",
			    topo_mod_errmsg(mod));
			return (-1);
		}

		/* search matching connector element using the index. */
		for (i = 0; i < count; i++) {
			found = B_FALSE;
			for (consnp = topo_list_next(&cp->sec_nodes);
			    consnp != NULL; consnp = topo_list_next(consnp)) {
				if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
					psprops = ses_node_props(
					    consnp->sen_node);
					/*
					 * Get the physical parent index to
					 * compare.
					 * The connector elements are children
					 * of ESC Electronics element even
					 * though we enumerate them under
					 * an expander in libtopo.
					 */
					if (nvlist_lookup_uint64(psprops,
					    SES_PROP_ELEMENT_ONLY_INDEX,
					    &conindex) == 0) {
						if (conindex ==
						    connectors[i].scpd_index) {
							found = B_TRUE;
							break;
						}
					}
				}
			}

			/* now create a libtopo node. */
			if (found) {
				/* Create generic props. */
				if (ses_create_generic(sdp, consnp, exptn,
				    topo_node_parent(exptn),
				    RECEPTACLE, "RECEPTACLE", &contn) !=
				    0) {
					continue;
				}
				/* Create connector specific props. */
				if (ses_set_connector_props(sdp, consnp,
				    contn, connectors[i].scpd_pm) != 0) {
					continue;
				}
				if (topo_method_register(mod, contn,
				    ses_recep_methods) != 0) {
					topo_mod_dprintf(mod,
					    "topo_method_register() failed: "
					    "%s",
					    topo_mod_errmsg(mod));
					continue;
				}

			}
		}
		/* end indentation change */
				}
			}
		}
	}

	return (0);
}

/*
 * Instantiate any protocol specific portion of a node.
 */
/*ARGSUSED*/
static int
ses_create_protocol_specific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
    tnode_t *pnode, uint64_t type, ses_enum_chassis_t *cp,
    boolean_t dorange)
{

	if (type == SES_ET_ESC_ELECTRONICS) {
		/* create SAS specific children(expanders and connectors. */
		return (ses_create_esc_sasspecific(sdp, snp, pnode, cp,
		    dorange));
	}

	return (0);
}

/*
 * Instantiate any children of a given type.
 */
static int
ses_create_children(ses_enum_data_t *sdp, tnode_t *pnode, uint64_t type,
    const char *nodename, const char *defaultlabel, ses_enum_chassis_t *cp,
    boolean_t dorange)
{
	topo_mod_t *mod = sdp->sed_mod;
	boolean_t found;
	uint64_t max;
	ses_enum_node_t *snp;
	tnode_t	*tn;

	/*
	 * First go through and count how many matching nodes we have.
	 */
	max = 0;
	found = B_FALSE;
	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
	    snp = topo_list_next(snp)) {
		if (snp->sen_type == type) {
			found = B_TRUE;
			if (snp->sen_instance > max)
				max = snp->sen_instance;
		}
	}

	/*
	 * No enclosure should export both DEVICE and ARRAY_DEVICE elements.
	 * Since we map both of these to 'disk', if an enclosure does this, we
	 * just ignore the array elements.
	 */
	if (!found ||
	    (type == SES_ET_ARRAY_DEVICE && cp->sec_hasdev))
		return (0);

	topo_mod_dprintf(mod, "%s: creating %llu %s nodes",
	    cp->sec_csn, max + 1, nodename);

	if (dorange && topo_node_range_create(mod, pnode,
	    nodename, 0, max) != 0) {
		topo_mod_dprintf(mod,
		    "topo_node_create_range() failed: %s",
		    topo_mod_errmsg(mod));
		return (-1);
	}

	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
	    snp = topo_list_next(snp)) {
		if (snp->sen_type == type) {
			/*
			 * With flat layout of ses nodes there is no
			 * way to find out the direct FRU for a node.
			 * Passing NULL for fru topo node.  Note that
			 * ses_create_children_from_phys_tree() provides
			 * the actual direct FRU for a node.
			 */
			if (ses_create_generic(sdp, snp, pnode, NULL,
			    nodename, defaultlabel, &tn) != 0)
				return (-1);
			/*
			 * For some SES element there may be protocol specific
			 * information to process.   Here we are processing
			 * the association between enclosure controller and
			 * SAS expanders.
			 */
			if (type == SES_ET_ESC_ELECTRONICS) {
				/* create SAS expander node */
				if (ses_create_protocol_specific(sdp, snp,
				    tn, type, cp, dorange) != 0) {
					return (-1);
				}
			}

		}
	}

	return (0);
}

/*
 * Instantiate a new subchassis instance in the topology.
 */
static int
ses_create_subchassis(ses_enum_data_t *sdp, tnode_t *pnode,
    ses_enum_chassis_t *scp)
{
	topo_mod_t *mod = sdp->sed_mod;
	tnode_t *tn;
	nvlist_t *props;
	nvlist_t *auth = NULL, *fmri = NULL;
	uint64_t instance = scp->sec_instance;
	char *desc;
	char *clean_label = NULL, label[128];
	char **paths;
	int i, err;
	ses_enum_target_t *stp;
	int ret = -1;

	/*
	 * Copy authority information from parent enclosure node
	 */
	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
		goto error;

	/*
	 * Record the subchassis serial number in the FMRI.
	 * For now, we assume that logical id is the subchassis serial number.
	 * If this assumption changes in future, then the following
	 * piece of code will need to be updated via an RFE.
	 */
	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
	    SUBCHASSIS, (topo_instance_t)instance, NULL, auth, NULL, NULL,
	    NULL)) == NULL) {
		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	if ((tn = topo_node_bind(mod, pnode, SUBCHASSIS,
	    instance, fmri)) == NULL) {
		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	props = ses_node_props(scp->sec_enclosure);

	/*
	 * Look for the subchassis label in the following order:
	 *	<ses-description>
	 *	<ses-class-description> <instance>
	 *	<default-type-label> <instance>
	 *
	 * For subchassis, the default label is "SUBCHASSIS"
	 */
	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
	    desc[0] == '\0') {
		if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION,
		    &desc) == 0 && desc[0] != '\0')
			(void) snprintf(label, sizeof (label), "%s %llu", desc,
			    instance);
		else
			(void) snprintf(label, sizeof (label),
			    "SUBCHASSIS %llu", instance);
		desc = label;
	}

	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
		goto error;

	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
		goto error;

	if (ses_set_standard_props(mod, NULL, tn, NULL,
	    ses_node_id(scp->sec_enclosure), scp->sec_target->set_devpath) != 0)
		goto error;

	/*
	 * Set the 'chassis-type' property for this subchassis.  This is either
	 * 'ses-class-description' or 'subchassis'.
	 */
	if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION, &desc) != 0)
		desc = "subchassis";

	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_CHASSIS_TYPE, TOPO_PROP_IMMUTABLE, desc, &err) != 0) {
		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
		    TOPO_PROP_CHASSIS_TYPE, topo_strerror(err));
		goto error;
	}

	/*
	 * For enclosures, we want to include all possible targets (for upgrade
	 * purposes).
	 */
	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
	    stp = topo_list_next(stp), i++)
		;

	verify(i != 0);
	paths = alloca(i * sizeof (char *));

	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
	    stp = topo_list_next(stp), i++)
		paths[i] = stp->set_devpath;

	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
	    i, &err) != 0) {
		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
		    TOPO_PROP_PATHS, topo_strerror(err));
		goto error;
	}

	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
		topo_mod_dprintf(mod, "topo_method_register() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	/*
	 * Create the nodes for controllers and bays.
	 */
	if (ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
	    CONTROLLER, "CONTROLLER", scp, B_TRUE) != 0 ||
	    ses_create_children(sdp, tn, SES_ET_DEVICE,
	    BAY, "BAY", scp, B_TRUE) != 0 ||
	    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
	    BAY, "BAY", scp, B_TRUE) != 0)
		goto error;

	ret = 0;

error:
	nvlist_free(auth);
	nvlist_free(fmri);
	topo_mod_strfree(mod, clean_label);
	return (ret);
}

/*
 * Function we use to insert a node.
 */
static int
ses_phys_tree_insert(topo_mod_t *mod, ses_phys_tree_t **sproot,
    ses_phys_tree_t *child)
{
	uint64_t ppindex, eindex, pindex;
	ses_phys_tree_t *node_ptr;
	int ret = 0;

	assert(sproot != NULL);
	assert(child != NULL);

	if (*sproot == NULL) {
		*sproot = child;
		return (0);
	}

	pindex = child->spt_pindex;
	ppindex = (*sproot)->spt_pindex;
	eindex = (*sproot)->spt_eonlyindex;

	/*
	 * If the element only index of the root is same as the physical
	 * parent index of a node to be added, add the node as a child of
	 * the current root.
	 */
	if (eindex == pindex) {
		(void) ses_phys_tree_insert(mod, &(*sproot)->spt_child, child);
		child->spt_parent = *sproot;
	} else if (ppindex == pindex) {
		/*
		 * if the physical parent of the current root and the child
		 * is same, then this should be a sibling node.
		 * Siblings can be different element types and arrange
		 * them by group.
		 */
		if ((*sproot)->spt_senumnode->sen_type ==
		    child->spt_senumnode->sen_type) {
			child->spt_sibling = *sproot;
			*sproot = child;
		} else {
			/* add a node in front of matching element type. */
			node_ptr = *sproot;
			while (node_ptr->spt_sibling != NULL) {
				if (node_ptr->spt_sibling->
				    spt_senumnode->sen_type ==
				    child->spt_senumnode->sen_type) {
					child->spt_sibling =
					    node_ptr->spt_sibling;
					node_ptr->spt_sibling = child;
					break;
				}
				node_ptr = node_ptr->spt_sibling;
			}
			/* no matching.  Add the child at the end. */
			if (node_ptr->spt_sibling == NULL) {
				node_ptr->spt_sibling = child;
			}
		}
		child->spt_parent = (*sproot)->spt_parent;
	} else {
		/*
		 * The root and the node is not directly related.
		 * Try to insert to the child sub-tree first and then try to
		 * insert to the sibling sub-trees.  If fails for both
		 * the caller will retry insertion later.
		 */
		if ((*sproot)->spt_child) {
			ret = ses_phys_tree_insert(mod, &(*sproot)->spt_child,
			    child);
		}
		if ((*sproot)->spt_child == NULL || ret != 0) {
			if ((*sproot)->spt_sibling) {
				ret = ses_phys_tree_insert(mod,
				    &(*sproot)->spt_sibling, child);
			} else {
				ret = 1;
			}
		}
		return (ret);
	}
	return (0);
}

/*
 * Construct tree view of ses elements through parent phyiscal element index.
 * The root of tree is already constructed using the enclosure element.
 */
static int
ses_construct_phys_tree(ses_enum_data_t *sdp, ses_enum_chassis_t *cp,
    ses_phys_tree_t *sproot)
{
	ses_enum_node_t *snp;
	ses_phys_tree_t	*child;
	ses_phys_tree_t	*u_watch = NULL;
	ses_phys_tree_t	*u_head = NULL;
	ses_phys_tree_t	*u_tail = NULL;
	int u_inserted = 0, u_left = 0;
	nvlist_t *props;
	topo_mod_t *mod = sdp->sed_mod;

	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
	    snp = topo_list_next(snp)) {
		if ((child = topo_mod_zalloc(mod,
		    sizeof (ses_phys_tree_t))) == NULL) {
			topo_mod_dprintf(mod,
			    "failed to allocate root.");
			return (-1);
		}
		child->spt_snode = snp->sen_node;
		props = ses_node_props(snp->sen_node);
		if (nvlist_lookup_uint64(props,
		    LIBSES_PROP_PHYS_PARENT, &child->spt_pindex) != 0) {
			/*
			 * the prop should exist. continue to see if
			 * we can build a partial tree with other elements.
			 */
			topo_mod_dprintf(mod,
			    "ses_construct_phys_tree(): Failed to find prop %s "
			    "on ses element type %d and instance %d "
			    "(CSN %s).", LIBSES_PROP_PHYS_PARENT,
			    snp->sen_type, snp->sen_instance, cp->sec_csn);
			topo_mod_free(mod, child, sizeof (ses_phys_tree_t));
			continue;
		} else {
			if (nvlist_lookup_boolean_value(props,
			    LIBSES_PROP_FRU, &child->spt_isfru) != 0) {
				topo_mod_dprintf(mod,
				    "ses_construct_phys_tree(): Failed to "
				    "find prop %s on ses element type %d "
				    "and instance %d (CSN %s).",
				    LIBSES_PROP_FRU,
				    snp->sen_type, snp->sen_instance,
				    cp->sec_csn);
				/*
				 * Ignore if the prop doesn't exist.
				 * Note that the enclosure itself should be
				 * a FRU so if no FRU found the enclosure FRU
				 * can be a direct FRU.
				 */
			}
			verify(nvlist_lookup_uint64(props,
			    SES_PROP_ELEMENT_ONLY_INDEX,
			    &child->spt_eonlyindex) == 0);
			verify(nvlist_lookup_uint64(props,
			    SES_PROP_ELEMENT_CLASS_INDEX,
			    &child->spt_cindex) == 0);
		}
		child->spt_senumnode = snp;
		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
			/* collect unresolved element to process later. */
			if (u_head == NULL) {
				u_head = child;
				u_tail = child;
			} else {
				child->spt_sibling = u_head;
				u_head = child;
			}
		}
	}

	/*
	 * The parent of a child node may not be inserted yet.
	 * Trying to insert the child until no child is left or
	 * no child is not added further.  For the latter
	 * the hierarchical relationship between elements
	 * should be checked through SUNW,FRUID page.
	 * u_watch is a watch dog to check the prgress of unresolved
	 * node.
	 */
	u_watch = u_tail;
	while (u_head) {
		child = u_head;
		u_head = u_head->spt_sibling;
		if (u_head == NULL)
			u_tail = NULL;
		child->spt_sibling = NULL;
		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
			u_tail->spt_sibling = child;
			u_tail = child;
			if (child == u_watch) {
				/*
				 * We just scanned one round for the
				 * unresolved list. Check to see whether we
				 * have nodes inserted, if none, we should
				 * break in case of an indefinite loop.
				 */
				if (u_inserted == 0) {
					/*
					 * Indicate there is unhandled node.
					 * Chain free the whole unsolved
					 * list here.
					 */
					u_left++;
					break;
				} else {
					u_inserted = 0;
					u_watch = u_tail;
				}
			}
		} else {
			/*
			 * We just inserted one rpnode, increment the
			 * unsolved_inserted counter. We will utilize this
			 * counter to detect an indefinite insertion loop.
			 */
			u_inserted++;
			if (child == u_watch) {
				/*
				 * watch dog node itself is inserted.
				 * Set it to the tail and refresh the watching.
				 */
				u_watch = u_tail;
				u_inserted = 0;
				u_left = 0;
			}
		}
	}

	/* check if there is left out unresolved nodes. */
	if (u_left) {
		topo_mod_dprintf(mod, "ses_construct_phys_tree(): "
		    "Failed to construct physical view of the following "
		    "ses elements of Chassis CSN %s.", cp->sec_csn);
		while (u_head) {
			u_tail = u_head->spt_sibling;
			topo_mod_dprintf(mod,
			    "\telement type (%d) and instance (%d)",
			    u_head->spt_senumnode->sen_type,
			    u_head->spt_senumnode->sen_instance);
			topo_mod_free(mod, u_head, sizeof (ses_phys_tree_t));
			u_head = u_tail;
		}
		return (-1);
	}

	return (0);
}

/*
 * Free the whole phys tree.
 */
static void ses_phys_tree_free(topo_mod_t *mod, ses_phys_tree_t *sproot)
{
	if (sproot == NULL)
		return;

	/* Free child tree. */
	if (sproot->spt_child) {
		ses_phys_tree_free(mod, sproot->spt_child);
	}

	/* Free sibling trees. */
	if (sproot->spt_sibling) {
		ses_phys_tree_free(mod, sproot->spt_sibling);
	}

	/* Free root node itself. */
	topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
}

/*
 * Parses phys_enum_type table to get the index of the given type.
 */
static boolean_t
is_type_enumerated(ses_phys_tree_t *node, int *index)
{
	int i;

	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
		if (node->spt_senumnode->sen_type ==
		    phys_enum_type_list[i].pet_type) {
			*index = i;
			return (B_TRUE);
		}
	}
	return (B_FALSE);
}

/*
 * Recusrive routine for top-down enumeration of the tree.
 */
static int
ses_enumerate_node(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp,
    ses_phys_tree_t *parent, int mrange[])
{
	topo_mod_t *mod = sdp->sed_mod;
	ses_phys_tree_t *child = NULL;
	int i, ret = 0, ret_ch;
	uint64_t prevtype = SES_ET_UNSPECIFIED;
	ses_phys_tree_t *dirfru = NULL;
	tnode_t *tn = NULL, *frutn = NULL;

	if (parent == NULL) {
		return (0);
	}

	for (child = parent->spt_child; child != NULL;
	    child = child->spt_sibling) {
		if (is_type_enumerated(child, &i)) {
			if (prevtype != phys_enum_type_list[i].pet_type) {
				/* check if range needs to be created. */
				if (phys_enum_type_list[i].pet_dorange &&
				    topo_node_range_create(mod, pnode,
				    phys_enum_type_list[i].pet_nodename, 0,
				    mrange[i]) != 0) {
					topo_mod_dprintf(mod,
					    "topo_node_create_range() failed: "
					    "%s", topo_mod_errmsg(mod));
					return (-1);
				}
				prevtype = phys_enum_type_list[i].pet_type;
			}

			if (!(child->spt_isfru)) {
				for (dirfru = parent; dirfru != NULL;
				    dirfru = dirfru->spt_parent) {
					if (dirfru->spt_isfru) {
						break;
					}
				}
				/* found direct FRU node. */
				if (dirfru) {
					frutn = dirfru->spt_tnode;
				} else {
					frutn = NULL;
				}
			} else {
				frutn = NULL;
			}

			if (ses_create_generic(sdp, child->spt_senumnode,
			    pnode, frutn, phys_enum_type_list[i].pet_nodename,
			    phys_enum_type_list[i].pet_defaultlabel, &tn) != 0)
				return (-1);

			child->spt_tnode = tn;
			/*
			 * For some SES element there may be protocol specific
			 * information to process.   Here we are processing
			 * the association between enclosure controller and
			 * SAS expanders.
			 */
			if (phys_enum_type_list[i].pet_type ==
			    SES_ET_ESC_ELECTRONICS) {
				/* create SAS expander node */
				if (ses_create_protocol_specific(sdp,
				    child->spt_senumnode, tn,
				    phys_enum_type_list[i].pet_type,
				    cp, phys_enum_type_list[i].pet_dorange) !=
				    0) {
					return (-1);
				}
			}
		} else {
			continue;
		}
		ret_ch = ses_enumerate_node(sdp, tn, cp, child, mrange);
		if (ret_ch)
			ret = ret_ch; /* there was an error and set the ret. */
	}

	return (ret);
}

/*
 * Instantiate types of nodes that are specified in the hierarchy
 * element type list.
 */
static int
ses_create_children_from_phys_tree(ses_enum_data_t *sdp, tnode_t *pnode,
    ses_enum_chassis_t *cp, ses_phys_tree_t *phys_tree)
{
	topo_mod_t *mod = sdp->sed_mod;
	int mrange[N_PHYS_ENUM_TYPES] = { 0 };
	ses_enum_node_t *snp;
	int i, ret;

	/*
	 * First get max range for each type of element to be enumerated.
	 */
	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
		if (phys_enum_type_list[i].pet_dorange) {
			for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
			    snp = topo_list_next(snp)) {
				if (snp->sen_type ==
				    phys_enum_type_list[i].pet_type) {
					if (snp->sen_instance > mrange[i])
						mrange[i] =
						    snp->sen_instance;
				}
			}
		}
	}

	topo_mod_dprintf(mod, "%s: creating nodes from FRU hierarchy tree.",
	    cp->sec_csn);

	if ((ret = ses_enumerate_node(sdp, pnode, cp, phys_tree, mrange)) !=
	    0) {
		topo_mod_dprintf(mod,
		    "ses_create_children_from_phys_tree() failed: ");
		return (ret);
	}

	return (0);
}

/*
 * Instantiate a new chassis instance in the topology.
 */
static int
ses_create_chassis(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp)
{
	topo_mod_t *mod = sdp->sed_mod;
	nvlist_t *props;
	char *raw_manufacturer, *raw_model, *raw_revision;
	char *manufacturer = NULL, *model = NULL, *product = NULL;
	char *revision = NULL;
	char *serial;
	char **paths;
	size_t prodlen;
	tnode_t *tn;
	nvlist_t *fmri = NULL, *auth = NULL;
	int ret = -1;
	ses_enum_node_t *snp;
	ses_enum_target_t *stp;
	ses_enum_chassis_t *scp;
	int i, err;
	uint64_t sc_count = 0, pindex;
	ses_phys_tree_t	*sproot = NULL;
	hrtime_t start;
	hrtime_t end;
	double duration;

	/*
	 * Ignore any internal enclosures.
	 */
	if (cp->sec_internal)
		return (0);

	/*
	 * Check to see if there are any devices presennt in the chassis.  If
	 * not, ignore the chassis alltogether.  This is most useful for
	 * ignoring internal HBAs that present a SES target but don't actually
	 * manage any of the devices.
	 */
	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
	    snp = topo_list_next(snp)) {
		if (snp->sen_type == SES_ET_DEVICE ||
		    snp->sen_type == SES_ET_ARRAY_DEVICE)
			break;
	}

	if (snp == NULL)
		return (0);

	props = ses_node_props(cp->sec_enclosure);

	/*
	 * We use the following property mappings:
	 *
	 * manufacturer		vendor-id
	 * model		product-id
	 * serial-number	libses-chassis-serial
	 */
	verify(nvlist_lookup_string(props, SES_EN_PROP_VID,
	    &raw_manufacturer) == 0);
	verify(nvlist_lookup_string(props, SES_EN_PROP_PID, &raw_model) == 0);
	verify(nvlist_lookup_string(props, SES_EN_PROP_REV,
	    &raw_revision) == 0);
	verify(nvlist_lookup_string(props, LIBSES_EN_PROP_CSN, &serial) == 0);

	/*
	 * To construct the authority information, we 'clean' each string by
	 * removing any offensive characters and trimmming whitespace.  For the
	 * 'product-id', we use a concatenation of 'manufacturer-model'.  We
	 * also take the numerical serial number and convert it to a string.
	 */
	if ((manufacturer = topo_mod_clean_str(mod, raw_manufacturer)) ==
	    NULL || (model = topo_mod_clean_str(mod, raw_model)) == NULL ||
	    (revision = topo_mod_clean_str(mod, raw_revision)) == NULL) {
		goto error;
	}

	prodlen = strlen(manufacturer) + strlen(model) + 2;
	if ((product = topo_mod_alloc(mod, prodlen)) == NULL)
		goto error;

	(void) snprintf(product, prodlen, "%s-%s", manufacturer, model);

	/*
	 * Construct the topo node and bind it to our parent.
	 */
	if (topo_mod_nvalloc(mod, &auth, NV_UNIQUE_NAME) != 0)
		goto error;

	if (nvlist_add_string(auth, FM_FMRI_AUTH_PRODUCT, product) != 0 ||
	    nvlist_add_string(auth, FM_FMRI_AUTH_CHASSIS, serial) != 0) {
		(void) topo_mod_seterrno(mod, EMOD_NVL_INVAL);
		goto error;
	}

	/*
	 * We pass NULL for the parent FMRI because there is no resource
	 * associated with it.  For the toplevel enclosure, we leave the
	 * serial/part/revision portions empty, which are reserved for
	 * individual components within the chassis.
	 */
	if ((fmri = topo_mod_hcfmri(mod, NULL, FM_HC_SCHEME_VERSION,
	    SES_ENCLOSURE, cp->sec_instance, NULL, auth,
	    model, revision, serial)) == NULL) {
		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	if ((tn = topo_node_bind(mod, pnode, SES_ENCLOSURE,
	    cp->sec_instance, fmri)) == NULL) {
		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
		topo_mod_dprintf(mod,
		    "topo_method_register() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	if (ses_set_standard_props(mod, NULL, tn, auth,
	    ses_node_id(cp->sec_enclosure), cp->sec_target->set_devpath) != 0)
		goto error;

	/*
	 * For enclosures, we want to include all possible targets (for upgrade
	 * purposes).
	 */
	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
	    stp = topo_list_next(stp), i++)
		;

	verify(i != 0);
	paths = alloca(i * sizeof (char *));

	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
	    stp = topo_list_next(stp), i++)
		paths[i] = stp->set_devpath;


	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
	    i, &err) != 0) {
		topo_mod_dprintf(mod,
		    "failed to create property %s: %s\n",
		    TOPO_PROP_PATHS, topo_strerror(err));
		goto error;
	}

	if (nvlist_lookup_uint64(props,
	    LIBSES_PROP_PHYS_PARENT, &pindex) == 0) {
		start = gethrtime(); /* to mearusre performance */
		/*
		 * The enclosure is supported through SUNW,FRUID.
		 * Need to enumerate the nodes through hierarchical order.
		 */
		if ((sproot = topo_mod_zalloc(mod,
		    sizeof (ses_phys_tree_t))) == NULL) {
			topo_mod_dprintf(mod,
			    "failed to allocate root: %s\n",
			    topo_strerror(err));
			goto error;
		}
		sproot->spt_pindex = pindex;
		if (nvlist_lookup_boolean_value(props,
		    LIBSES_PROP_FRU, &sproot->spt_isfru) != 0) {
			topo_mod_dprintf(mod,
			    "ses_create_chassis(): Failed to find prop %s "
			    "on enclosure element (CSN %s).",
			    LIBSES_PROP_FRU, cp->sec_csn);
			/* an enclosure should be a FRU. continue to process. */
			sproot->spt_isfru = B_TRUE;
		}
		if (nvlist_lookup_uint64(props,
		    SES_PROP_ELEMENT_ONLY_INDEX,
		    &sproot->spt_eonlyindex) != 0) {
			topo_mod_dprintf(mod,
			    "ses_create_chassis(): Failed to find prop %s "
			    "on enclosure element (CSN %s).",
			    LIBSES_PROP_PHYS_PARENT, cp->sec_csn);
			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
			goto error;
		}
		if (sproot->spt_pindex != sproot->spt_eonlyindex) {
			topo_mod_dprintf(mod, "ses_create_chassis(): "
			    "Enclosure element(CSN %s) should have "
			    "itself as the parent to be the root node "
			    "of FRU hierarchical tree.)", cp->sec_csn);
			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
			goto error;
		} else {
			sproot->spt_snode = cp->sec_enclosure;
			sproot->spt_tnode = tn;
			/* construct a tree. */
			if (ses_construct_phys_tree(sdp, cp, sproot) != 0) {
				topo_mod_dprintf(mod, "ses_create_chassis(): "
				    "Failed to construct FRU hierarchical "
				    "tree on enclosure (CSN %s.)",
				    cp->sec_csn);
			}

			/* enumerate elements from the tree. */
			if (ses_create_children_from_phys_tree(sdp, tn, cp,
			    sproot) != 0) {
				topo_mod_dprintf(mod, "ses_create_chassis(): "
				    "Failed to create children topo nodes out "
				    "of FRU hierarchical tree on enclosure "
				    "(CSN %s).", cp->sec_csn);
			}
			/* destroy the phys tree. */
			ses_phys_tree_free(mod, sproot);
		}

		end = gethrtime();
		duration = end - start;
		duration /= HR_SECOND;
		topo_mod_dprintf(mod,
		    "FRU boundary tree based enumeration: %.6f seconds",
		    duration);
	} else {
		/*
		 * Create the nodes for power supplies, fans, controllers and
		 * devices.  Note that SAS exopander nodes and connector nodes
		 * are handled through protocol specific processing of
		 * controllers.
		 */
		if (ses_create_children(sdp, tn, SES_ET_POWER_SUPPLY,
		    PSU, "PSU", cp, B_TRUE) != 0 ||
		    ses_create_children(sdp, tn, SES_ET_COOLING,
		    FAN, "FAN", cp, B_TRUE) != 0 ||
		    ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
		    CONTROLLER, "CONTROLLER", cp, B_TRUE) != 0 ||
		    ses_create_children(sdp, tn, SES_ET_DEVICE,
		    BAY, "BAY", cp, B_TRUE) != 0 ||
		    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
		    BAY, "BAY", cp, B_TRUE) != 0)
			goto error;
	}

	if (cp->sec_maxinstance != SES_INST_NOTSET &&
	    (topo_node_range_create(mod, tn, SUBCHASSIS, 0,
	    cp->sec_maxinstance) != 0)) {
		topo_mod_dprintf(mod, "topo_node_create_range() failed: %s",
		    topo_mod_errmsg(mod));
		goto error;
	}

	for (scp = topo_list_next(&cp->sec_subchassis); scp != NULL;
	    scp = topo_list_next(scp)) {

		if (ses_create_subchassis(sdp, tn, scp) != 0)
			goto error;

		topo_mod_dprintf(mod, "created Subchassis node with "
		    "instance %u\nand target (%s) under Chassis with CSN %s",
		    scp->sec_instance, scp->sec_target->set_devpath,
		    cp->sec_csn);

		sc_count++;
	}

	topo_mod_dprintf(mod, "%s: created %llu %s nodes",
	    cp->sec_csn, sc_count, SUBCHASSIS);

	cp->sec_target->set_refcount++;
	topo_node_setspecific(tn, cp->sec_target);

	ret = 0;
error:
	topo_mod_strfree(mod, manufacturer);
	topo_mod_strfree(mod, model);
	topo_mod_strfree(mod, revision);
	topo_mod_strfree(mod, product);

	nvlist_free(fmri);
	nvlist_free(auth);
	return (ret);
}

/*
 * Create a bay node explicitly enumerated via XML.
 */
static int
ses_create_bays(ses_enum_data_t *sdp, tnode_t *pnode)
{
	topo_mod_t *mod = sdp->sed_mod;
	ses_enum_chassis_t *cp;

	/*
	 * Iterate over chassis looking for an internal enclosure.  This
	 * property is set via a vendor-specific plugin, and there should only
	 * ever be a single internal chassis in a system.
	 */
	for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
	    cp = topo_list_next(cp)) {
		if (cp->sec_internal)
			break;
	}

	if (cp == NULL) {
		topo_mod_dprintf(mod, "failed to find internal chassis\n");
		return (-1);
	}

	if (ses_create_children(sdp, pnode, SES_ET_DEVICE,
	    BAY, "BAY", cp, B_FALSE) != 0 ||
	    ses_create_children(sdp, pnode, SES_ET_ARRAY_DEVICE,
	    BAY, "BAY", cp, B_FALSE) != 0)
		return (-1);

	return (0);
}

/*
 * Initialize chassis or subchassis.
 */
static int
ses_init_chassis(topo_mod_t *mod, ses_enum_data_t *sdp, ses_enum_chassis_t *pcp,
    ses_enum_chassis_t *cp, ses_node_t *np, nvlist_t *props,
    uint64_t subchassis, ses_chassis_type_e flags)
{
	boolean_t internal, ident;

	assert((flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS |
	    SES_DUP_CHASSIS | SES_DUP_SUBCHASSIS)) != 0);

	assert(cp != NULL);
	assert(np != NULL);
	assert(props != NULL);

	if (flags & (SES_NEW_SUBCHASSIS | SES_DUP_SUBCHASSIS))
		assert(pcp != NULL);

	topo_mod_dprintf(mod, "ses_init_chassis: %s: index %llu, flags (%d)",
	    sdp->sed_name, subchassis, flags);

	if (flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS)) {

		topo_mod_dprintf(mod, "new chassis/subchassis");
		if (nvlist_lookup_boolean_value(props,
		    LIBSES_EN_PROP_INTERNAL, &internal) == 0)
			cp->sec_internal = internal;

		cp->sec_enclosure = np;
		cp->sec_target = sdp->sed_target;

		if (flags & SES_NEW_CHASSIS) {
			if (!cp->sec_internal)
				cp->sec_instance = sdp->sed_instance++;
			topo_list_append(&sdp->sed_chassis, cp);
		} else {
			if (subchassis != NO_SUBCHASSIS)
				cp->sec_instance = subchassis;
			else
				cp->sec_instance = pcp->sec_scinstance++;

			if (cp->sec_instance > pcp->sec_maxinstance)
				pcp->sec_maxinstance = cp->sec_instance;

			topo_list_append(&pcp->sec_subchassis, cp);
		}

	} else {
		topo_mod_dprintf(mod, "dup chassis/subchassis");
		if (nvlist_lookup_boolean_value(props,
		    SES_PROP_IDENT, &ident) == 0) {
			topo_mod_dprintf(mod,  "overriding enclosure node");

			cp->sec_enclosure = np;
			cp->sec_target = sdp->sed_target;
		}
	}

	topo_list_append(&cp->sec_targets, sdp->sed_target);
	sdp->sed_current = cp;

	return (0);
}

/*
 * Gather nodes from the current SES target into our chassis list, merging the
 * results if necessary.
 */
static ses_walk_action_t
ses_enum_gather(ses_node_t *np, void *data)
{
	nvlist_t *props = ses_node_props(np);
	ses_enum_data_t *sdp = data;
	topo_mod_t *mod = sdp->sed_mod;
	ses_enum_chassis_t *cp, *scp;
	ses_enum_node_t *snp;
	ses_alt_node_t *sap;
	char *csn;
	uint64_t instance, type;
	uint64_t prevstatus, status;
	boolean_t report;
	uint64_t subchassis = NO_SUBCHASSIS;

	if (ses_node_type(np) == SES_NODE_ENCLOSURE) {
		/*
		 * If we have already identified the chassis for this target,
		 * then this is a secondary enclosure and we should ignore it,
		 * along with the rest of the tree (since this is depth-first).
		 */
		if (sdp->sed_current != NULL)
			return (SES_WALK_ACTION_TERMINATE);

		/*
		 * Go through the list of chassis we have seen so far and see
		 * if this serial number matches one of the known values.
		 * If so, check whether this enclosure is a subchassis.
		 */
		if (nvlist_lookup_string(props, LIBSES_EN_PROP_CSN,
		    &csn) != 0)
			return (SES_WALK_ACTION_TERMINATE);

		(void) nvlist_lookup_uint64(props, LIBSES_EN_PROP_SUBCHASSIS_ID,
		    &subchassis);

		topo_mod_dprintf(mod, "ses_enum_gather: Enclosure Node (%s) "
		    "CSN (%s), subchassis (%llu)", sdp->sed_name, csn,
		    subchassis);

		/*
		 * We need to determine whether this enclosure node
		 * represents a chassis or a subchassis. Since we may
		 * receive the enclosure nodes in a non-deterministic
		 * manner, we need to account for all possible combinations:
		 *	1. Chassis for the current CSN has not yet been
		 *	   allocated
		 *		1.1 This is a new chassis:
		 *			allocate and instantiate the chassis
		 *		1.2 This is a new subchassis:
		 *			allocate a placeholder chassis
		 *			allocate and instantiate the subchassis
		 *			link the subchassis to the chassis
		 *	2. Chassis for the current CSN has been allocated
		 *		2.1 This is a duplicate chassis enclosure
		 *			check whether to override old chassis
		 *			append to chassis' target list
		 *		2.2 Only placeholder chassis exists
		 *			fill in the chassis fields
		 *		2.3 This is a new subchassis
		 *			allocate and instantiate the subchassis
		 *			link the subchassis to the chassis
		 *		2.4 This is a duplicate subchassis enclosure
		 *			 check whether to override old chassis
		 *			 append to chassis' target list
		 */

		for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
		    cp = topo_list_next(cp))
			if (strcmp(cp->sec_csn, csn) == 0)
				break;

		if (cp == NULL) {
			/* 1. Haven't seen a chassis with this CSN before */

			if ((cp = topo_mod_zalloc(mod,
			    sizeof (ses_enum_chassis_t))) == NULL)
				goto error;

			cp->sec_scinstance = SES_STARTING_SUBCHASSIS;
			cp->sec_maxinstance = SES_INST_NOTSET;
			cp->sec_csn = csn;

			if (subchassis == NO_SUBCHASSIS) {
				/* 1.1 This is a new chassis */

				topo_mod_dprintf(mod, "%s: Initialize new "
				    "chassis with CSN %s", sdp->sed_name, csn);

				if (ses_init_chassis(mod, sdp, NULL, cp,
				    np, props, NO_SUBCHASSIS,
				    SES_NEW_CHASSIS) < 0)
					goto error;
			} else {
				/* 1.2 This is a new subchassis */

				topo_mod_dprintf(mod, "%s: Initialize new "
				    "subchassis with CSN %s and index %llu",
				    sdp->sed_name, csn, subchassis);

				if ((scp = topo_mod_zalloc(mod,
				    sizeof (ses_enum_chassis_t))) == NULL)
					goto error;

				scp->sec_csn = csn;

				if (ses_init_chassis(mod, sdp, cp, scp, np,
				    props, subchassis, SES_NEW_SUBCHASSIS) < 0)
					goto error;
			}
		} else {
			/*
			 * We have a chassis or subchassis with this CSN.  If
			 * it's a chassis, we must check to see whether it is
			 * a placeholder previously created because we found a
			 * subchassis with this CSN.  We will know that because
			 * the sec_target value will not be set; it is set only
			 * in ses_init_chassis().  In that case, initialise it
			 * as a new chassis; otherwise, it's a duplicate and we
			 * need to append only.
			 */
			if (subchassis == NO_SUBCHASSIS) {
				if (cp->sec_target != NULL) {
					/* 2.1 This is a duplicate chassis */

					topo_mod_dprintf(mod, "%s: Append "
					    "duplicate chassis with CSN (%s)",
					    sdp->sed_name, csn);

					if (ses_init_chassis(mod, sdp, NULL, cp,
					    np, props, NO_SUBCHASSIS,
					    SES_DUP_CHASSIS) < 0)
						goto error;
				} else {
					/* Placeholder chassis - init it up */
					topo_mod_dprintf(mod, "%s: Initialize"
					    "placeholder chassis with CSN %s",
					    sdp->sed_name, csn);

					if (ses_init_chassis(mod, sdp, NULL,
					    cp, np, props, NO_SUBCHASSIS,
					    SES_NEW_CHASSIS) < 0)
						goto error;

				}
			} else {
				/* This is a subchassis */

				for (scp = topo_list_next(&cp->sec_subchassis);
				    scp != NULL; scp = topo_list_next(scp))
					if (scp->sec_instance == subchassis)
						break;

				if (scp == NULL) {
					/* 2.3 This is a new subchassis */

					topo_mod_dprintf(mod, "%s: Initialize "
					    "new subchassis with CSN (%s) "
					    "and LID (%s)",
					    sdp->sed_name, csn);

					if ((scp = topo_mod_zalloc(mod,
					    sizeof (ses_enum_chassis_t)))
					    == NULL)
						goto error;

					scp->sec_csn = csn;

					if (ses_init_chassis(mod, sdp, cp, scp,
					    np, props, subchassis,
					    SES_NEW_SUBCHASSIS) < 0)
						goto error;
				} else {
					/* 2.4 This is a duplicate subchassis */

					topo_mod_dprintf(mod, "%s: Append "
					    "duplicate subchassis with "
					    "CSN (%s)", sdp->sed_name, csn);

					if (ses_init_chassis(mod, sdp, cp, scp,
					    np, props, subchassis,
					    SES_DUP_SUBCHASSIS) < 0)
						goto error;
				}
			}
		}
	} else if (ses_node_type(np) == SES_NODE_ELEMENT) {
		/*
		 * If we haven't yet seen an enclosure node and identified the
		 * current chassis, something is very wrong; bail out.
		 */
		if (sdp->sed_current == NULL)
			return (SES_WALK_ACTION_TERMINATE);

		/*
		 * If this isn't one of the element types we care about, then
		 * ignore it.
		 */
		verify(nvlist_lookup_uint64(props, SES_PROP_ELEMENT_TYPE,
		    &type) == 0);
		if (type != SES_ET_DEVICE &&
		    type != SES_ET_ARRAY_DEVICE &&
		    type != SES_ET_SUNW_FANBOARD &&
		    type != SES_ET_SUNW_FANMODULE &&
		    type != SES_ET_COOLING &&
		    type != SES_ET_SUNW_POWERBOARD &&
		    type != SES_ET_SUNW_POWERMODULE &&
		    type != SES_ET_POWER_SUPPLY &&
		    type != SES_ET_ESC_ELECTRONICS &&
		    type != SES_ET_SAS_EXPANDER &&
		    type != SES_ET_SAS_CONNECTOR)
			return (SES_WALK_ACTION_CONTINUE);

		/*
		 * Get the current instance number and see if we already know
		 * about this element.  If so, it means we have multiple paths
		 * to the same elements, and we should ignore the current path.
		 */
		verify(nvlist_lookup_uint64(props, SES_PROP_ELEMENT_CLASS_INDEX,
		    &instance) == 0);
		if (type == SES_ET_DEVICE || type == SES_ET_ARRAY_DEVICE)
			(void) nvlist_lookup_uint64(props, SES_PROP_BAY_NUMBER,
			    &instance);

		cp = sdp->sed_current;

		for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
		    snp = topo_list_next(snp)) {
			if (snp->sen_type == type &&
			    snp->sen_instance == instance)
				break;
		}

		/*
		 * We prefer the new element under the following circumstances:
		 *
		 * - The currently known element's status is unknown or not
		 *   available, but the new element has a known status.  This
		 *   occurs if a given element is only available through a
		 *   particular target.
		 *
		 * - This is an ESC_ELECTRONICS element, and the 'reported-via'
		 *   property is set.  This allows us to get reliable firmware
		 *   revision information from the enclosure node.
		 */
		if (snp != NULL) {
			if (nvlist_lookup_uint64(
			    ses_node_props(snp->sen_node),
			    SES_PROP_STATUS_CODE, &prevstatus) != 0)
				prevstatus = SES_ESC_UNSUPPORTED;
			if (nvlist_lookup_uint64(
			    props, SES_PROP_STATUS_CODE, &status) != 0)
				status = SES_ESC_UNSUPPORTED;
			if (nvlist_lookup_boolean_value(
			    props, SES_PROP_REPORT, &report) != 0)
				report = B_FALSE;

			if ((SES_STATUS_UNAVAIL(prevstatus) &&
			    !SES_STATUS_UNAVAIL(status)) ||
			    (type == SES_ET_ESC_ELECTRONICS &&
			    report)) {
				snp->sen_node = np;
				snp->sen_target = sdp->sed_target;
			}

			if ((sap = topo_mod_zalloc(mod,
			    sizeof (ses_alt_node_t))) == NULL)
				goto error;

			sap->san_node = np;
			topo_list_append(&snp->sen_alt_nodes, sap);

			return (SES_WALK_ACTION_CONTINUE);
		}

		if ((snp = topo_mod_zalloc(mod,
		    sizeof (ses_enum_node_t))) == NULL)
			goto error;

		if ((sap = topo_mod_zalloc(mod,
		    sizeof (ses_alt_node_t))) == NULL) {
			topo_mod_free(mod, snp, sizeof (ses_enum_node_t));
			goto error;
		}

		topo_mod_dprintf(mod, "%s: adding node (%llu, %llu)",
		    sdp->sed_name, type, instance);
		snp->sen_node = np;
		snp->sen_type = type;
		snp->sen_instance = instance;
		snp->sen_target = sdp->sed_target;
		sap->san_node = np;
		topo_list_append(&snp->sen_alt_nodes, sap);
		topo_list_append(&cp->sec_nodes, snp);

		if (type == SES_ET_DEVICE)
			cp->sec_hasdev = B_TRUE;
	}

	return (SES_WALK_ACTION_CONTINUE);

error:
	sdp->sed_errno = -1;
	return (SES_WALK_ACTION_TERMINATE);
}

static int
ses_process_dir(const char *dirpath, ses_enum_data_t *sdp)
{
	topo_mod_t *mod = sdp->sed_mod;
	DIR *dir;
	struct dirent *dp;
	char path[PATH_MAX];
	ses_enum_target_t *stp;
	int err = -1;

	/*
	 * Open the SES target directory and iterate over any available
	 * targets.
	 */
	if ((dir = opendir(dirpath)) == NULL) {
		/*
		 * If the SES target directory does not exist, then return as if
		 * there are no active targets.
		 */
		topo_mod_dprintf(mod, "failed to open ses "
		    "directory '%s'", dirpath);
		return (0);
	}

	while ((dp = readdir(dir)) != NULL) {
		if (strcmp(dp->d_name, ".") == 0 ||
		    strcmp(dp->d_name, "..") == 0)
			continue;

		/*
		 * Create a new target instance and take a snapshot.
		 */
		if ((stp = topo_mod_zalloc(mod,
		    sizeof (ses_enum_target_t))) == NULL)
			goto error;

		(void) pthread_mutex_init(&stp->set_lock, NULL);

		(void) snprintf(path, sizeof (path), "%s/%s", dirpath,
		    dp->d_name);

		/*
		 * We keep track of the SES device path and export it on a
		 * per-node basis to allow higher level software to get to the
		 * corresponding SES state.
		 */
		if ((stp->set_devpath = topo_mod_strdup(mod, path)) == NULL) {
			topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
			goto error;
		}

		if ((stp->set_target =
		    ses_open(LIBSES_VERSION, path)) == NULL) {
			topo_mod_dprintf(mod, "failed to open ses target "
			    "'%s': %s", dp->d_name, ses_errmsg());
			ses_sof_alloc(mod, stp->set_devpath);
			topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
			continue;
		}
		topo_mod_dprintf(mod, "open contract");
		ses_ssl_alloc(mod, stp);
		ses_create_contract(mod, stp);

		stp->set_refcount = 1;
		sdp->sed_target = stp;
		stp->set_snap = ses_snap_hold(stp->set_target);
		stp->set_snaptime = gethrtime();

		/*
		 * Enumerate over all SES elements and merge them into the
		 * correct ses_enum_chassis_t.
		 */
		sdp->sed_current = NULL;
		sdp->sed_errno = 0;
		sdp->sed_name = dp->d_name;
		(void) ses_walk(stp->set_snap, ses_enum_gather, sdp);

		if (sdp->sed_errno != 0)
			goto error;
	}

	err = 0;
error:
	(void) closedir(dir);
	return (err);
}

/*
 * Different generations of SMCI's 4U36 storage servers used different models
 * of front and rear SAS expanders.
 */
#define	SMCI4U36_FRONT_EXPANDER_PID1	"LSI-SAS2X36"
#define	SMCI4U36_FRONT_EXPANDER_PID2	"LSI-SAS3x40"
#define	SMCI4U36_FRONT_EXPANDER_PID3	"SMC-SC846P"

#define	SMCI4U36_REAR_EXPANDER_PID1	"LSI-CORP-SAS2X28"
#define	SMCI4U36_REAR_EXPANDER_PID2	"LSI-SAS3x28"

static int
smci_4u36_bay_label(topo_mod_t *mod, tnode_t *node, topo_version_t version,
    nvlist_t *in, nvlist_t **out)
{
	int err, ret = -1;
	nvlist_t *pargs, *auth, *nvl = NULL, *fmri;
	char *label = NULL, *product_id;

	/*
	 * Now look for a private argument list to determine if the invoker is
	 * trying to do a set operation and if so, return an error as this
	 * method only supports get operations.
	 */
	if ((nvlist_lookup_nvlist(in, TOPO_PROP_PARGS, &pargs) == 0) &&
	    nvlist_exists(pargs, TOPO_PROP_VAL_VAL)) {
		topo_mod_dprintf(mod, "%s: set operation not suppported",
		    __func__);
		return (topo_mod_seterrno(mod, EMOD_NVL_INVAL));
	}

	if (topo_node_resource(node, &fmri, &err) != 0) {
		(void) topo_mod_seterrno(mod, err);
		goto err;
	}

	if (nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) != 0 ||
	    nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT, &product_id) !=
	    0) {
		topo_mod_dprintf(mod, "%s: malformed FMRI", __func__);
		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
		nvlist_free(fmri);
		goto err;
	}
	nvlist_free(fmri);

	if (strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID1) == 0 ||
	    strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID2) == 0 ||
	    strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID3) == 0) {
		err = asprintf(&label, "Front Slot %" PRIu64,
		    topo_node_instance(node));
	} else if (strcmp(product_id, SMCI4U36_REAR_EXPANDER_PID1) == 0 ||
	    strcmp(product_id, SMCI4U36_REAR_EXPANDER_PID2) == 0) {
		err = asprintf(&label, "Rear Slot %" PRIu64,
		    topo_node_instance(node));
	} else {
		topo_mod_dprintf(mod, "%s: unexpected expander product id: %s",
		    __func__, product_id);
		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
		goto err;
	}

	if (err < 0) {
		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
		goto err;
	}

	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0 ||
	    nvlist_add_string(nvl, TOPO_PROP_VAL_NAME, TOPO_PROP_LABEL) != 0 ||
	    nvlist_add_uint32(nvl, TOPO_PROP_VAL_TYPE, TOPO_TYPE_STRING)
	    != 0 ||
	    nvlist_add_string(nvl, TOPO_PROP_VAL_VAL, label)
	    != 0) {
		topo_mod_dprintf(mod, "Failed to allocate 'out' nvlist");
		nvlist_free(nvl);
		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
		goto err;
	}
	*out = nvl;
	ret = 0;
err:
	free(label);
	return (ret);

}

static void
ses_release(topo_mod_t *mod, tnode_t *tn)
{
	ses_enum_target_t *stp;

	if ((stp = topo_node_getspecific(tn)) != NULL) {
		topo_node_setspecific(tn, NULL);
		ses_target_free(mod, stp);
	}
}

/*ARGSUSED*/
static int
ses_enum(topo_mod_t *mod, tnode_t *rnode, const char *name,
    topo_instance_t min, topo_instance_t max, void *arg, void *notused)
{
	ses_enum_chassis_t *cp;
	ses_enum_data_t *data;

	/*
	 * Check to make sure we're being invoked sensibly, and that we're not
	 * being invoked as part of a post-processing step.
	 */
	if (strcmp(name, SES_ENCLOSURE) != 0 && strcmp(name, BAY) != 0)
		return (0);

	/*
	 * If this is the first time we've called our enumeration method, then
	 * gather information about any available enclosures.
	 */
	if ((data = topo_mod_getspecific(mod)) == NULL) {
		ses_sof_freeall(mod);
		if ((data = topo_mod_zalloc(mod, sizeof (ses_enum_data_t))) ==
		    NULL)
			return (-1);

		data->sed_mod = mod;
		topo_mod_setspecific(mod, data);

		if (dev_list_gather(mod, &data->sed_devs) != 0)
			goto error;

		/*
		 * We search both the ses(7D) and sgen(7D) locations, so we are
		 * independent of any particular driver class bindings.
		 */
		if (ses_process_dir("/dev/es", data) != 0 ||
		    ses_process_dir("/dev/scsi/ses", data) != 0)
			goto error;
	}

	if (strcmp(name, SES_ENCLOSURE) == 0) {
		/*
		 * This is a request to enumerate external enclosures.  Go
		 * through all the targets and create chassis nodes where
		 * necessary.
		 */
		for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
		    cp = topo_list_next(cp)) {
			if (ses_create_chassis(data, rnode, cp) != 0)
				goto error;
		}
	} else {
		/*
		 * This is a request to enumerate a specific bay underneath the
		 * root chassis (for internal disks).
		 */
		if (ses_create_bays(data, rnode) != 0)
			goto error;
	}

	/*
	 * This is a bit of a kludge.  In order to allow internal disks to be
	 * enumerated and share snapshot-specific information with the external
	 * enclosure enumeration, we rely on the fact that we will be invoked
	 * for the 'ses-enclosure' node last.
	 */
	if (strcmp(name, SES_ENCLOSURE) == 0) {
		for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
		    cp = topo_list_next(cp))
			ses_data_free(data, cp);
		ses_data_free(data, NULL);
		topo_mod_setspecific(mod, NULL);
	}
	return (0);

error:
	for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
	    cp = topo_list_next(cp))
		ses_data_free(data, cp);
	ses_data_free(data, NULL);
	topo_mod_setspecific(mod, NULL);
	return (-1);
}

static const topo_modops_t ses_ops =
	{ ses_enum, ses_release };

static topo_modinfo_t ses_info =
	{ SES_ENCLOSURE, FM_FMRI_SCHEME_HC, SES_VERSION, &ses_ops };

/*ARGSUSED*/
int
_topo_init(topo_mod_t *mod, topo_version_t version)
{
	int rval;

	if (getenv("TOPOSESDEBUG") != NULL)
		topo_mod_setdebug(mod);

	topo_mod_dprintf(mod, "initializing %s enumerator\n",
	    SES_ENCLOSURE);

	if ((rval = topo_mod_register(mod, &ses_info, TOPO_VERSION)) == 0)
		ses_thread_init(mod);

	return (rval);
}

void
_topo_fini(topo_mod_t *mod)
{
	ses_thread_fini(mod);
	ses_sof_freeall(mod);
	topo_mod_unregister(mod);
}