xref: /illumos-gate/usr/src/lib/fm/topo/modules/common/ses/ses.c (revision 8a2b682e57a046b828f37bcde1776f131ef4629f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2012 Milan Jurik. All rights reserved.
25  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2020, Joyent, Inc.
27  */
28 
29 #include <alloca.h>
30 #include <dirent.h>
31 #include <devid.h>
32 #include <fm/libdiskstatus.h>
33 #include <inttypes.h>
34 #include <pthread.h>
35 #include <strings.h>
36 #include <string.h>
37 #include <unistd.h>
38 #include <sys/dkio.h>
39 #include <sys/fm/protocol.h>
40 #include <sys/libdevid.h>
41 #include <sys/scsi/scsi_types.h>
42 #include <sys/byteorder.h>
43 #include <pthread.h>
44 #include <signal.h>
45 #include <fcntl.h>
46 #include <sys/ctfs.h>
47 #include <libcontract.h>
48 #include <poll.h>
49 #include <sys/contract/device.h>
50 #include <libsysevent.h>
51 #include <sys/sysevent/eventdefs.h>
52 #include <scsi/plugins/ses/vendor/sun.h>
53 
54 #include "disk.h"
55 #include "ses.h"
56 
57 #define	SES_VERSION	1
58 
59 #define	SES_STARTING_SUBCHASSIS 256	/* valid subchassis IDs are uint8_t */
60 #define	NO_SUBCHASSIS	((uint64_t)-1)
61 
62 static int ses_snap_freq = 250;		/* in milliseconds */
63 
64 #define	SES_STATUS_UNAVAIL(s)	\
65 	((s) == SES_ESC_UNSUPPORTED || (s) >= SES_ESC_NOT_INSTALLED)
66 
67 #define	HR_SECOND   1000000000
68 
69 /*
70  * Because multiple SES targets can be part of a single chassis, we construct
71  * our own hierarchy that takes this into account.  These SES targets may refer
72  * to the same devices (multiple paths) or to different devices (managing
73  * different portions of the space).  We arrange things into a
74  * ses_enum_enclosure_t, which contains a set of ses targets, and a list of all
75  * nodes found so far.
76  */
77 typedef struct ses_alt_node {
78 	topo_list_t		san_link;
79 	ses_node_t		*san_node;
80 } ses_alt_node_t;
81 
82 typedef struct ses_enum_node {
83 	topo_list_t		sen_link;
84 	ses_node_t		*sen_node;
85 	topo_list_t		sen_alt_nodes;
86 	uint64_t		sen_type;
87 	uint64_t		sen_instance;
88 	ses_enum_target_t	*sen_target;
89 } ses_enum_node_t;
90 
91 typedef struct ses_enum_chassis {
92 	topo_list_t		sec_link;
93 	topo_list_t		sec_subchassis;
94 	topo_list_t		sec_nodes;
95 	topo_list_t		sec_targets;
96 	const char		*sec_csn;
97 	ses_node_t		*sec_enclosure;
98 	ses_enum_target_t	*sec_target;
99 	topo_instance_t		sec_instance;
100 	topo_instance_t		sec_scinstance;
101 	topo_instance_t		sec_maxinstance;
102 	boolean_t		sec_hasdev;
103 	boolean_t		sec_internal;
104 } ses_enum_chassis_t;
105 
106 typedef struct ses_enum_data {
107 	topo_list_t		sed_devs;
108 	topo_list_t		sed_chassis;
109 	ses_enum_chassis_t	*sed_current;
110 	ses_enum_target_t	*sed_target;
111 	int			sed_errno;
112 	char			*sed_name;
113 	topo_mod_t		*sed_mod;
114 	topo_instance_t		sed_instance;
115 } ses_enum_data_t;
116 
117 typedef struct sas_connector_phy_data {
118 	uint64_t    scpd_index;
119 	uint64_t    scpd_pm;
120 } sas_connector_phy_data_t;
121 
122 typedef struct sas_connector_type {
123 	uint64_t    sct_type;
124 	char	    *sct_name;
125 } sas_connector_type_t;
126 
127 static const sas_connector_type_t sas_connector_type_list[] = {
128 	{   0x0, "Information unknown"  },
129 	{   0x1, "External SAS 4x receptacle (see SAS-2 and SFF-8470)"	},
130 	{   0x2, "Exteranl Mini SAS 4x receptacle (see SAS-2 and SFF-8088)" },
131 	{   0x3, "QSFP+ receptacle (see SAS-2.1 and SFF-8436)" },
132 	{   0x4, "Mini SAS 4x active receptacle (see SAS-2.1 and SFF-8088)" },
133 	{   0x5, "Mini SAS HD 4x receptacle (see SAS-2.1 and SFF-8644)" },
134 	{   0x6, "Mini SAS HD 8x receptacle (see SAS-2.1 and SFF-8644)" },
135 	{   0x7, "Mini SAS HD 16x receptacle (see SAS-2.1 and SFF-8644)" },
136 	{   0xF, "Vendor-specific external connector"	},
137 	{   0x10, "Internal wide SAS 4i plug (see SAS-2 and SFF-8484)"	},
138 	{   0x11,
139 	"Internal wide Mini SAS 4i receptacle (see SAS-2 and SFF-8087)"	},
140 	{   0x12, "Mini SAS HD 4i receptacle (see SAS-2.1 and SFF-8643)" },
141 	{   0x20, "Internal SAS Drive receptacle (see SAS-2 and SFF-8482)" },
142 	{   0x21, "Internal SATA host plug (see SAS-2 and SATA-2)"	},
143 	{   0x22, "Internal SAS Drive plug (see SAS-2 and SFF-8482)"	},
144 	{   0x23, "Internal SATA device plug (see SAS-2 and SATA-2)"	},
145 	{   0x24, "Micro SAS receptacle (see SAS-2.14)"	},
146 	{   0x25, "Micro SATA device plug (see SAS-2.1 and SATA)"	},
147 	{   0x26, "Micro SAS plug (see SAS-2.1 and SFF-8486)"	},
148 	{   0x27, "Micro SAS/SATA plug (see SAS-2.1 and SFF-8486)"	},
149 	{   0x28,
150 	"12 Gb/s SAS Drive backplane receptacle (see SAS-34 and SFF-8680)" },
151 	{   0x29, "12Gb/s SAS Drive Plug (see SAS-3 and SFF-8680)"	},
152 	{   0x2A, "Multifunction 12 Gb/s 6x Unshielded receptacle connector "
153 		"receptacle (see SAS-3 and SFF-8639)"	},
154 	{   0x2B, "Multifunction 12 Gb/s 6x Unshielded receptable connector "
155 		"plug (see SAS-3 and SFF-8639)"	},
156 	{   0x2F, "Internal SAS virtual connector"  },
157 	{   0x3F, "Vendor-specific internal connector"	},
158 	{   0x70, "Other Vendor-specific connector"	},
159 	{   0x71, "Other Vendor-specific connector"	},
160 	{   0x72, "Other Vendor-specific connector"	},
161 	{   0x73, "Other Vendor-specific connector"	},
162 	{   0x74, "Other Vendor-specific connector"	},
163 	{   0x75, "Other Vendor-specific connector"	},
164 	{   0x76, "Other Vendor-specific connector"	},
165 	{   0x77, "Other Vendor-specific connector"	},
166 	{   0x78, "Other Vendor-specific connector"	},
167 	{   0x79, "Other Vendor-specific connector"	},
168 	{   0x7A, "Other Vendor-specific connector"	},
169 	{   0x7B, "Other Vendor-specific connector"	},
170 	{   0x7C, "Other Vendor-specific connector"	},
171 	{   0x7D, "Other Vendor-specific connector"	},
172 	{   0x7E, "Other Vendor-specific connector"	},
173 	{   0x7F, "Other Vendor-specific connector"	},
174 	{   0x80, "Not Defined"	}
175 };
176 
177 #define	SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED  0x80
178 #define	SAS_CONNECTOR_TYPE_NOT_DEFINED \
179 	"Connector type not defined by SES-2 standard"
180 #define	SAS_CONNECTOR_TYPE_RESERVED \
181 	"Connector type reserved by SES-2 standard"
182 
183 typedef struct phys_enum_type {
184 	uint64_t    pet_type;
185 	char	    *pet_nodename;
186 	char	    *pet_defaultlabel;
187 	boolean_t   pet_dorange;
188 } phys_enum_type_t;
189 
190 static const phys_enum_type_t phys_enum_type_list[] = {
191 	{   SES_ET_ARRAY_DEVICE, BAY, "BAY", B_TRUE  },
192 	{   SES_ET_COOLING, FAN, "FAN", B_TRUE  },
193 	{   SES_ET_DEVICE, BAY, "BAY", B_TRUE  },
194 	{   SES_ET_ESC_ELECTRONICS, CONTROLLER, "CONTROLLER", B_TRUE  },
195 	{   SES_ET_POWER_SUPPLY, PSU, "PSU", B_TRUE  },
196 	{   SES_ET_SUNW_FANBOARD, FANBOARD, "FANBOARD", B_TRUE  },
197 	{   SES_ET_SUNW_FANMODULE, FANMODULE, "FANMODULE", B_TRUE  },
198 	{   SES_ET_SUNW_POWERBOARD, POWERBOARD, "POWERBOARD", B_TRUE  },
199 	{   SES_ET_SUNW_POWERMODULE, POWERMODULE, "POWERMODULE", B_TRUE  }
200 };
201 
202 #define	N_PHYS_ENUM_TYPES (sizeof (phys_enum_type_list) / \
203 	sizeof (phys_enum_type_list[0]))
204 
205 /*
206  * Structure for the hierarchical tree for element nodes.
207  */
208 typedef struct ses_phys_tree {
209     ses_node_t	*spt_snode;
210     ses_enum_node_t	*spt_senumnode;
211     boolean_t	spt_isfru;
212     uint64_t	spt_eonlyindex;
213     uint64_t	spt_cindex;
214     uint64_t	spt_pindex;
215     uint64_t	spt_maxinst;
216     struct ses_phys_tree    *spt_parent;
217     struct ses_phys_tree    *spt_child;
218     struct ses_phys_tree    *spt_sibling;
219     tnode_t	*spt_tnode;
220 } ses_phys_tree_t;
221 
222 typedef enum {
223 	SES_NEW_CHASSIS		= 0x1,
224 	SES_NEW_SUBCHASSIS	= 0x2,
225 	SES_DUP_CHASSIS		= 0x4,
226 	SES_DUP_SUBCHASSIS	= 0x8
227 } ses_chassis_type_e;
228 
229 
230 static const topo_pgroup_info_t storage_pgroup = {
231 	TOPO_PGROUP_STORAGE,
232 	TOPO_STABILITY_PRIVATE,
233 	TOPO_STABILITY_PRIVATE,
234 	1
235 };
236 
237 static const topo_pgroup_info_t smp_pgroup = {
238 	TOPO_PGROUP_SMP,
239 	TOPO_STABILITY_PRIVATE,
240 	TOPO_STABILITY_PRIVATE,
241 	1
242 };
243 
244 static const topo_pgroup_info_t ses_pgroup = {
245 	TOPO_PGROUP_SES,
246 	TOPO_STABILITY_PRIVATE,
247 	TOPO_STABILITY_PRIVATE,
248 	1
249 };
250 
251 static int ses_present(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
252     nvlist_t **);
253 static int ses_contains(topo_mod_t *, tnode_t *, topo_version_t, nvlist_t *,
254     nvlist_t **);
255 
256 static const topo_method_t ses_component_methods[] = {
257 	{ TOPO_METH_PRESENT, TOPO_METH_PRESENT_DESC,
258 	    TOPO_METH_PRESENT_VERSION0, TOPO_STABILITY_INTERNAL, ses_present },
259 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
260 	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
261 	{ TOPO_METH_SENSOR_FAILURE, TOPO_METH_SENSOR_FAILURE_DESC,
262 	    TOPO_METH_SENSOR_FAILURE_VERSION, TOPO_STABILITY_INTERNAL,
263 	    topo_method_sensor_failure },
264 	{ NULL }
265 };
266 
267 #define	TOPO_METH_SMCI_4U36_LABEL		"smci_4u36_bay_label"
268 #define	TOPO_METH_SMCI_4U36_LABEL_DESC	\
269 	"compute bay labels on SMCI 4U36 storage platform variants"
270 #define	TOPO_METH_SMCI_4U36_LABEL_VERSION	0
271 static int smci_4u36_bay_label(topo_mod_t *, tnode_t *, topo_version_t,
272     nvlist_t *, nvlist_t **);
273 
274 static const topo_method_t ses_bay_methods[] = {
275 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
276 	    TOPO_STABILITY_INTERNAL, ses_node_enum_facility },
277 	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
278 	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
279 	    topo_mod_hc_occupied },
280 	{ TOPO_METH_SMCI_4U36_LABEL, TOPO_METH_SMCI_4U36_LABEL_DESC,
281 	    TOPO_METH_SMCI_4U36_LABEL_VERSION, TOPO_STABILITY_INTERNAL,
282 	    smci_4u36_bay_label },
283 	{ NULL }
284 };
285 
286 static const topo_method_t ses_recep_methods[] = {
287 	{ TOPO_METH_OCCUPIED, TOPO_METH_OCCUPIED_DESC,
288 	    TOPO_METH_OCCUPIED_VERSION, TOPO_STABILITY_INTERNAL,
289 	    topo_mod_hc_occupied },
290 	{ NULL }
291 };
292 
293 static const topo_method_t ses_enclosure_methods[] = {
294 	{ TOPO_METH_CONTAINS, TOPO_METH_CONTAINS_DESC,
295 	    TOPO_METH_CONTAINS_VERSION, TOPO_STABILITY_INTERNAL, ses_contains },
296 	{ TOPO_METH_FAC_ENUM, TOPO_METH_FAC_ENUM_DESC, 0,
297 	    TOPO_STABILITY_INTERNAL, ses_enc_enum_facility },
298 	{ NULL }
299 };
300 
301 /*
302  * The bay_label_overrides table can be used to map a server product ID to a
303  * topo method that will be invoked to override the value of the label property
304  * for all bay nodes.  By default the property value is static, derived from
305  * the corresponding SES array device element's descriptor string.
306  */
307 typedef struct ses_label_overrides {
308 	const char *slbl_product;
309 	const char *slbl_mname;
310 } ses_label_overrides_t;
311 
312 /*
313  * This table covers three generations of SMCI's 4U 36-bay storage server
314  * (and the Joyent-branded versions).  There was also an Ivy Bridge variant
315  * which has been omitted due to an inability to find one to test on.
316  */
317 static const ses_label_overrides_t bay_label_overrides[] = {
318 	/* Sandy Bridge variant */
319 	{ "SSG-6047R-E1R36L", TOPO_METH_SMCI_4U36_LABEL },
320 	{ "Joyent-Storage-Platform-5001", TOPO_METH_SMCI_4U36_LABEL },
321 
322 	/* Broadwell variant */
323 	{ "SSG-6048R-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
324 	{ "Joyent-Storage-Platform-7001", TOPO_METH_SMCI_4U36_LABEL },
325 
326 	/* Skylake variant */
327 	{ "SSG-6049P-E1CR36L", TOPO_METH_SMCI_4U36_LABEL },
328 	{ "Joyent-S10G5", TOPO_METH_SMCI_4U36_LABEL }
329 };
330 
331 #define	N_BAY_LBL_OVERRIDES (sizeof (bay_label_overrides) / \
332 	sizeof (bay_label_overrides[0]))
333 
334 /*
335  * Functions for tracking ses devices which we were unable to open. We retry
336  * these at regular intervals using ses_recheck_dir() and if we find that we
337  * can now open any of them then we send a sysevent to indicate that a new topo
338  * snapshot should be taken.
339  */
340 typedef struct ses_open_fail_list {
341 	struct ses_open_fail_list	*sof_next;
342 	char				*sof_path;
343 } ses_open_fail_list_t;
344 
345 static ses_open_fail_list_t *ses_sofh;
346 static pthread_mutex_t ses_sofmt;
347 static void ses_ct_print(char *ptr);
348 
349 static void
350 ses_recheck_dir()
351 {
352 	ses_target_t *target;
353 	sysevent_id_t eid;
354 	char buf[80];
355 	ses_open_fail_list_t *sof;
356 
357 	/*
358 	 * check list of "unable to open" devices
359 	 */
360 	(void) pthread_mutex_lock(&ses_sofmt);
361 	for (sof = ses_sofh; sof != NULL; sof = sof->sof_next) {
362 		/*
363 		 * see if we can open it now
364 		 */
365 		if ((target = ses_open(LIBSES_VERSION,
366 		    sof->sof_path)) == NULL) {
367 			(void) snprintf(buf, sizeof (buf),
368 			    "recheck_dir - still can't open %s", sof->sof_path);
369 			ses_ct_print(buf);
370 			continue;
371 		}
372 
373 		/*
374 		 * ok - better force a new snapshot
375 		 */
376 		(void) snprintf(buf, sizeof (buf),
377 		    "recheck_dir - can now open %s", sof->sof_path);
378 		ses_ct_print(buf);
379 		(void) sysevent_post_event(EC_PLATFORM, ESC_PLATFORM_SP_RESET,
380 		    SUNW_VENDOR, "fmd", NULL, &eid);
381 		ses_close(target);
382 		break;
383 	}
384 	(void) pthread_mutex_unlock(&ses_sofmt);
385 }
386 
387 static void
388 ses_sof_alloc(topo_mod_t *mod, char *path)
389 {
390 	ses_open_fail_list_t *sof;
391 
392 	(void) pthread_mutex_lock(&ses_sofmt);
393 	sof = topo_mod_zalloc(mod, sizeof (*sof));
394 	topo_mod_dprintf(mod, "sof_alloc %s", path);
395 	sof->sof_path = path;
396 	sof->sof_next = ses_sofh;
397 	ses_sofh = sof;
398 	(void) pthread_mutex_unlock(&ses_sofmt);
399 }
400 
401 static void
402 ses_sof_freeall(topo_mod_t *mod)
403 {
404 	ses_open_fail_list_t *sof, *next_sof;
405 
406 	(void) pthread_mutex_lock(&ses_sofmt);
407 	for (sof = ses_sofh; sof != NULL; sof = next_sof) {
408 		next_sof = sof->sof_next;
409 		topo_mod_dprintf(mod, "sof_freeall %s", sof->sof_path);
410 		topo_mod_strfree(mod, sof->sof_path);
411 		topo_mod_free(mod, sof, sizeof (*sof));
412 	}
413 	ses_sofh = NULL;
414 	(void) pthread_mutex_unlock(&ses_sofmt);
415 }
416 
417 /*
418  * functions for verifying that the ses_enum_target_t held in a device
419  * contract's cookie field is still valid (it may have been freed by
420  * ses_release()).
421  */
422 typedef struct ses_stp_list {
423 	struct ses_stp_list	*ssl_next;
424 	ses_enum_target_t	*ssl_tgt;
425 } ses_stp_list_t;
426 
427 static ses_stp_list_t *ses_sslh;
428 static pthread_mutex_t ses_sslmt;
429 
430 static void
431 ses_ssl_alloc(topo_mod_t *mod, ses_enum_target_t *stp)
432 {
433 	ses_stp_list_t *ssl;
434 
435 	(void) pthread_mutex_lock(&ses_sslmt);
436 	ssl = topo_mod_zalloc(mod, sizeof (*ssl));
437 	topo_mod_dprintf(mod, "ssl_alloc %p", stp);
438 	ssl->ssl_tgt = stp;
439 	ssl->ssl_next = ses_sslh;
440 	ses_sslh = ssl;
441 	(void) pthread_mutex_unlock(&ses_sslmt);
442 }
443 
444 static void
445 ses_ssl_free(topo_mod_t *mod, ses_enum_target_t *stp)
446 {
447 	ses_stp_list_t *ssl, *prev_ssl;
448 
449 	(void) pthread_mutex_lock(&ses_sslmt);
450 	prev_ssl = NULL;
451 	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next) {
452 		if (ssl->ssl_tgt == stp) {
453 			topo_mod_dprintf(mod, "ssl_free %p", ssl->ssl_tgt);
454 			if (prev_ssl == NULL)
455 				ses_sslh = ssl->ssl_next;
456 			else
457 				prev_ssl->ssl_next = ssl->ssl_next;
458 			topo_mod_free(mod, ssl, sizeof (*ssl));
459 			break;
460 		}
461 		prev_ssl = ssl;
462 	}
463 	(void) pthread_mutex_unlock(&ses_sslmt);
464 }
465 
466 static int
467 ses_ssl_valid(ses_enum_target_t *stp)
468 {
469 	ses_stp_list_t *ssl;
470 
471 	for (ssl = ses_sslh; ssl != NULL; ssl = ssl->ssl_next)
472 		if (ssl->ssl_tgt == stp)
473 			return (1);
474 	return (0);
475 }
476 
477 /*
478  * Functions for creating and destroying a background thread
479  * (ses_contract_thread) used for detecting when ses devices have been
480  * retired/unretired.
481  */
482 static struct ses_thread_s {
483 	pthread_mutex_t mt;
484 	pthread_t tid;
485 	int thr_sig;
486 	int doexit;
487 	int count;
488 } sesthread = {
489 	PTHREAD_MUTEX_INITIALIZER,
490 	0,
491 	SIGTERM,
492 	0,
493 	0
494 };
495 
496 typedef struct ses_mod_list {
497 	struct ses_mod_list	*smod_next;
498 	topo_mod_t		*smod_mod;
499 } ses_mod_list_t;
500 
501 static ses_mod_list_t *ses_smod;
502 
503 static void
504 ses_ct_print(char *ptr)
505 {
506 	(void) pthread_mutex_lock(&sesthread.mt);
507 	if (ses_smod != NULL && ses_smod->smod_mod != NULL)
508 		topo_mod_dprintf(ses_smod->smod_mod, ptr);
509 	(void) pthread_mutex_unlock(&sesthread.mt);
510 }
511 
512 /*ARGSUSED*/
513 static void *
514 ses_contract_thread(void *arg)
515 {
516 	int efd, ctlfd, statfd;
517 	ct_evthdl_t ev;
518 	ctevid_t evid;
519 	uint_t event;
520 	char path[PATH_MAX];
521 	char buf[80];
522 	ses_enum_target_t *stp;
523 	ct_stathdl_t stathdl;
524 	ctid_t ctid;
525 	struct pollfd fds;
526 	int pollret;
527 	sigset_t sigset;
528 
529 	ses_ct_print("start contract event thread");
530 	efd = open64(CTFS_ROOT "/device/pbundle", O_RDONLY);
531 	fds.fd = efd;
532 	fds.events = POLLIN;
533 	fds.revents = 0;
534 	(void) sigaddset(&sigset, sesthread.thr_sig);
535 	(void) pthread_sigmask(SIG_UNBLOCK, &sigset, NULL);
536 	for (;;) {
537 		/* check if we've been asked to exit */
538 		(void) pthread_mutex_lock(&sesthread.mt);
539 		if (sesthread.doexit) {
540 			(void) pthread_mutex_unlock(&sesthread.mt);
541 			break;
542 		}
543 		(void) pthread_mutex_unlock(&sesthread.mt);
544 
545 		/* poll until an event arrives */
546 		if ((pollret = poll(&fds, 1, 10000)) <= 0) {
547 			if (pollret == 0)
548 				ses_recheck_dir();
549 			continue;
550 		}
551 
552 		/* read the event */
553 		(void) pthread_mutex_lock(&ses_sslmt);
554 		ses_ct_print("read contract event");
555 		if (ct_event_read(efd, &ev) != 0) {
556 			(void) pthread_mutex_unlock(&ses_sslmt);
557 			continue;
558 		}
559 
560 		/* see if it is an event we are expecting */
561 		ctid = ct_event_get_ctid(ev);
562 		(void) snprintf(buf, sizeof (buf),
563 		    "got contract event ctid=%d", ctid);
564 		ses_ct_print(buf);
565 		event = ct_event_get_type(ev);
566 		if (event != CT_DEV_EV_OFFLINE && event != CT_EV_NEGEND) {
567 			(void) snprintf(buf, sizeof (buf),
568 			    "bad contract event %x", event);
569 			ses_ct_print(buf);
570 			ct_event_free(ev);
571 			(void) pthread_mutex_unlock(&ses_sslmt);
572 			continue;
573 		}
574 
575 		/* find target pointer saved in cookie */
576 		evid = ct_event_get_evid(ev);
577 		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/status",
578 		    ctid);
579 		statfd = open64(path, O_RDONLY);
580 		(void) ct_status_read(statfd, CTD_COMMON, &stathdl);
581 		stp = (ses_enum_target_t *)(uintptr_t)
582 		    ct_status_get_cookie(stathdl);
583 		ct_status_free(stathdl);
584 		(void) close(statfd);
585 
586 		/* check if target pointer is still valid */
587 		if (ses_ssl_valid(stp) == 0) {
588 			(void) snprintf(buf, sizeof (buf),
589 			    "contract already abandoned %x", event);
590 			ses_ct_print(buf);
591 			(void) snprintf(path, PATH_MAX,
592 			    CTFS_ROOT "/device/%ld/ctl", ctid);
593 			ctlfd = open64(path, O_WRONLY);
594 			if (event != CT_EV_NEGEND)
595 				(void) ct_ctl_ack(ctlfd, evid);
596 			else
597 				(void) ct_ctl_abandon(ctlfd);
598 			(void) close(ctlfd);
599 			ct_event_free(ev);
600 			(void) pthread_mutex_unlock(&ses_sslmt);
601 			continue;
602 		}
603 
604 		/* find control device for ack/abandon */
605 		(void) pthread_mutex_lock(&stp->set_lock);
606 		(void) snprintf(path, PATH_MAX, CTFS_ROOT "/device/%ld/ctl",
607 		    ctid);
608 		ctlfd = open64(path, O_WRONLY);
609 		if (event != CT_EV_NEGEND) {
610 			/* if this is an offline event, do the offline */
611 			ses_ct_print("got contract offline event");
612 			if (stp->set_target) {
613 				ses_ct_print("contract thread rele");
614 				ses_snap_rele(stp->set_snap);
615 				ses_close(stp->set_target);
616 				stp->set_target = NULL;
617 			}
618 			(void) ct_ctl_ack(ctlfd, evid);
619 		} else {
620 			/* if this is the negend, then abandon the contract */
621 			ses_ct_print("got contract negend");
622 			if (stp->set_ctid) {
623 				(void) snprintf(buf, sizeof (buf),
624 				    "abandon old contract %d", stp->set_ctid);
625 				ses_ct_print(buf);
626 				stp->set_ctid = 0;
627 			}
628 			(void) ct_ctl_abandon(ctlfd);
629 		}
630 		(void) close(ctlfd);
631 		(void) pthread_mutex_unlock(&stp->set_lock);
632 		ct_event_free(ev);
633 		(void) pthread_mutex_unlock(&ses_sslmt);
634 	}
635 	(void) close(efd);
636 	return (NULL);
637 }
638 
639 int
640 find_thr_sig(void)
641 {
642 	int i;
643 	sigset_t oset, rset;
644 	int sig[] = {SIGTERM, SIGUSR1, SIGUSR2};
645 	int sig_sz = sizeof (sig) / sizeof (int);
646 	int rc = SIGTERM;
647 
648 	/* prefered set of signals that are likely used to terminate threads */
649 	(void) sigemptyset(&oset);
650 	(void) pthread_sigmask(SIG_SETMASK, NULL, &oset);
651 	for (i = 0; i < sig_sz; i++) {
652 		if (sigismember(&oset, sig[i]) == 0) {
653 			return (sig[i]);
654 		}
655 	}
656 
657 	/* reserved set of signals that are not allowed to terminate thread */
658 	(void) sigemptyset(&rset);
659 	(void) sigaddset(&rset, SIGABRT);
660 	(void) sigaddset(&rset, SIGKILL);
661 	(void) sigaddset(&rset, SIGSTOP);
662 	(void) sigaddset(&rset, SIGCANCEL);
663 
664 	/* Find signal that is not masked and not in the reserved list. */
665 	for (i = 1; i < MAXSIG; i++) {
666 		if (sigismember(&rset, i) == 1) {
667 			continue;
668 		}
669 		if (sigismember(&oset, i) == 0) {
670 			return (i);
671 		}
672 	}
673 
674 	return (rc);
675 }
676 
677 /*ARGSUSED*/
678 static void
679 ses_handler(int sig)
680 {
681 }
682 
683 static void
684 ses_thread_init(topo_mod_t *mod)
685 {
686 	pthread_attr_t *attr = NULL;
687 	struct sigaction act;
688 	ses_mod_list_t *smod;
689 
690 	(void) pthread_mutex_lock(&sesthread.mt);
691 	sesthread.count++;
692 	smod = topo_mod_zalloc(mod, sizeof (*smod));
693 	smod->smod_mod = mod;
694 	smod->smod_next = ses_smod;
695 	ses_smod = smod;
696 	if (sesthread.tid == 0) {
697 		/* find a suitable signal to use for killing the thread below */
698 		sesthread.thr_sig = find_thr_sig();
699 
700 		/* if don't have a handler for this signal, create one */
701 		(void) sigaction(sesthread.thr_sig, NULL, &act);
702 		if (act.sa_handler == SIG_DFL || act.sa_handler == SIG_IGN)
703 			act.sa_handler = ses_handler;
704 		(void) sigaction(sesthread.thr_sig, &act, NULL);
705 
706 		/* create a thread to listen for offline events */
707 		(void) pthread_create(&sesthread.tid,
708 		    attr, ses_contract_thread, NULL);
709 	}
710 	(void) pthread_mutex_unlock(&sesthread.mt);
711 }
712 
713 static void
714 ses_thread_fini(topo_mod_t *mod)
715 {
716 	ses_mod_list_t *smod, *prev_smod;
717 
718 	(void) pthread_mutex_lock(&sesthread.mt);
719 	prev_smod = NULL;
720 	for (smod = ses_smod; smod != NULL; smod = smod->smod_next) {
721 		if (smod->smod_mod == mod) {
722 			if (prev_smod == NULL)
723 				ses_smod = smod->smod_next;
724 			else
725 				prev_smod->smod_next = smod->smod_next;
726 			topo_mod_free(mod, smod, sizeof (*smod));
727 			break;
728 		}
729 		prev_smod = smod;
730 	}
731 	if (--sesthread.count > 0) {
732 		(void) pthread_mutex_unlock(&sesthread.mt);
733 		return;
734 	}
735 	sesthread.doexit = 1;
736 	(void) pthread_mutex_unlock(&sesthread.mt);
737 	(void) pthread_kill(sesthread.tid, sesthread.thr_sig);
738 	(void) pthread_join(sesthread.tid, NULL);
739 	sesthread.tid = 0;
740 }
741 
742 static void
743 ses_create_contract(topo_mod_t *mod, ses_enum_target_t *stp)
744 {
745 	int tfd, len, rval;
746 	char link_path[PATH_MAX];
747 
748 	stp->set_ctid = 0;
749 
750 	/* convert "/dev" path into "/devices" path */
751 	if ((len = readlink(stp->set_devpath, link_path, PATH_MAX)) < 0) {
752 		topo_mod_dprintf(mod, "readlink failed");
753 		return;
754 	}
755 	link_path[len] = '\0';
756 
757 	/* set up template to create new contract */
758 	tfd = open64(CTFS_ROOT "/device/template", O_RDWR);
759 	(void) ct_tmpl_set_critical(tfd, CT_DEV_EV_OFFLINE);
760 	(void) ct_tmpl_set_cookie(tfd, (uint64_t)(uintptr_t)stp);
761 
762 	/* strip "../../devices" off the front and create the contract */
763 	if ((rval = ct_dev_tmpl_set_minor(tfd, &link_path[13])) != 0)
764 		topo_mod_dprintf(mod, "failed to set minor %s rval = %d",
765 		    &link_path[13], rval);
766 	else if ((rval = ct_tmpl_create(tfd, &stp->set_ctid)) != 0)
767 		topo_mod_dprintf(mod, "failed to create ctid rval = %d", rval);
768 	else
769 		topo_mod_dprintf(mod, "created ctid=%d", stp->set_ctid);
770 	(void) close(tfd);
771 }
772 
773 static void
774 ses_target_free(topo_mod_t *mod, ses_enum_target_t *stp)
775 {
776 	if (--stp->set_refcount == 0) {
777 		/* check if already closed due to contract offline request */
778 		(void) pthread_mutex_lock(&stp->set_lock);
779 		if (stp->set_target) {
780 			ses_snap_rele(stp->set_snap);
781 			ses_close(stp->set_target);
782 			stp->set_target = NULL;
783 		}
784 		if (stp->set_ctid) {
785 			int ctlfd;
786 			char path[PATH_MAX];
787 
788 			topo_mod_dprintf(mod, "abandon old contract %d",
789 			    stp->set_ctid);
790 			(void) snprintf(path, PATH_MAX,
791 			    CTFS_ROOT "/device/%ld/ctl", stp->set_ctid);
792 			ctlfd = open64(path, O_WRONLY);
793 			(void) ct_ctl_abandon(ctlfd);
794 			(void) close(ctlfd);
795 			stp->set_ctid = 0;
796 		}
797 		(void) pthread_mutex_unlock(&stp->set_lock);
798 		ses_ssl_free(mod, stp);
799 		topo_mod_strfree(mod, stp->set_devpath);
800 		topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
801 	}
802 }
803 
804 static void
805 ses_data_free(ses_enum_data_t *sdp, ses_enum_chassis_t *pcp)
806 {
807 	topo_mod_t *mod = sdp->sed_mod;
808 	ses_enum_chassis_t *cp;
809 	ses_enum_node_t *np;
810 	ses_enum_target_t *tp;
811 	ses_alt_node_t *ap;
812 	topo_list_t *cpl;
813 
814 
815 	if (pcp != NULL)
816 		cpl = &pcp->sec_subchassis;
817 	else
818 		cpl = &sdp->sed_chassis;
819 
820 	while ((cp = topo_list_next(cpl)) != NULL) {
821 		topo_list_delete(cpl, cp);
822 
823 		while ((np = topo_list_next(&cp->sec_nodes)) != NULL) {
824 			while ((ap = topo_list_next(&np->sen_alt_nodes)) !=
825 			    NULL) {
826 				topo_list_delete(&np->sen_alt_nodes, ap);
827 				topo_mod_free(mod, ap, sizeof (ses_alt_node_t));
828 			}
829 			topo_list_delete(&cp->sec_nodes, np);
830 			topo_mod_free(mod, np, sizeof (ses_enum_node_t));
831 		}
832 
833 		while ((tp = topo_list_next(&cp->sec_targets)) != NULL) {
834 			topo_list_delete(&cp->sec_targets, tp);
835 			ses_target_free(mod, tp);
836 		}
837 
838 		topo_mod_free(mod, cp, sizeof (ses_enum_chassis_t));
839 	}
840 
841 	if (pcp == NULL) {
842 		dev_list_free(mod, &sdp->sed_devs);
843 		topo_mod_free(mod, sdp, sizeof (ses_enum_data_t));
844 	}
845 }
846 
847 /*
848  * For enclosure nodes, we have a special contains method.  By default, the hc
849  * walker will compare the node name and instance number to determine if an
850  * FMRI matches.  For enclosures where the enumeration order is impossible to
851  * predict, we instead use the chassis-id as a unique identifier, and ignore
852  * the instance number.
853  */
854 static int
855 fmri_contains(topo_mod_t *mod, nvlist_t *nv1, nvlist_t *nv2)
856 {
857 	uint8_t v1, v2;
858 	nvlist_t **hcp1, **hcp2;
859 	int err, i;
860 	uint_t nhcp1, nhcp2;
861 	nvlist_t *a1, *a2;
862 	char *c1, *c2;
863 	int mindepth;
864 
865 	if (nvlist_lookup_uint8(nv1, FM_VERSION, &v1) != 0 ||
866 	    nvlist_lookup_uint8(nv2, FM_VERSION, &v2) != 0 ||
867 	    v1 > FM_HC_SCHEME_VERSION || v2 > FM_HC_SCHEME_VERSION)
868 		return (topo_mod_seterrno(mod, EMOD_FMRI_VERSION));
869 
870 	err = nvlist_lookup_nvlist_array(nv1, FM_FMRI_HC_LIST, &hcp1, &nhcp1);
871 	err |= nvlist_lookup_nvlist_array(nv2, FM_FMRI_HC_LIST, &hcp2, &nhcp2);
872 	if (err != 0)
873 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
874 
875 	/*
876 	 * If the chassis-id doesn't match, then these FMRIs are not
877 	 * equivalent.  If one of the FMRIs doesn't have a chassis ID, then we
878 	 * have no choice but to fall back to the instance ID.
879 	 */
880 	if (nvlist_lookup_nvlist(nv1, FM_FMRI_AUTHORITY, &a1) == 0 &&
881 	    nvlist_lookup_nvlist(nv2, FM_FMRI_AUTHORITY, &a2) == 0 &&
882 	    nvlist_lookup_string(a1, FM_FMRI_AUTH_CHASSIS, &c1) == 0 &&
883 	    nvlist_lookup_string(a2, FM_FMRI_AUTH_CHASSIS, &c2) == 0) {
884 		if (strcmp(c1, c2) != 0)
885 			return (0);
886 
887 		mindepth = 1;
888 	} else {
889 		mindepth = 0;
890 	}
891 
892 	if (nhcp2 < nhcp1)
893 		return (0);
894 
895 	for (i = 0; i < nhcp1; i++) {
896 		char *nm1 = NULL;
897 		char *nm2 = NULL;
898 		char *id1 = NULL;
899 		char *id2 = NULL;
900 
901 		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_NAME, &nm1);
902 		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_NAME, &nm2);
903 		(void) nvlist_lookup_string(hcp1[i], FM_FMRI_HC_ID, &id1);
904 		(void) nvlist_lookup_string(hcp2[i], FM_FMRI_HC_ID, &id2);
905 		if (nm1 == NULL || nm2 == NULL || id1 == NULL || id2 == NULL)
906 			return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
907 
908 		if (strcmp(nm1, nm2) == 0 &&
909 		    (i < mindepth || strcmp(id1, id2) == 0))
910 			continue;
911 
912 		return (0);
913 	}
914 
915 	return (1);
916 }
917 
918 /*ARGSUSED*/
919 static int
920 ses_contains(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
921     nvlist_t *in, nvlist_t **out)
922 {
923 	int ret;
924 	nvlist_t *nv1, *nv2;
925 
926 	if (version > TOPO_METH_CONTAINS_VERSION)
927 		return (topo_mod_seterrno(mod, EMOD_VER_NEW));
928 
929 	if (nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_FMRI, &nv1) != 0 ||
930 	    nvlist_lookup_nvlist(in, TOPO_METH_FMRI_ARG_SUBFMRI, &nv2) != 0)
931 		return (topo_mod_seterrno(mod, EMOD_METHOD_INVAL));
932 
933 	ret = fmri_contains(mod, nv1, nv2);
934 	if (ret < 0)
935 		return (-1);
936 
937 	if (topo_mod_nvalloc(mod, out, NV_UNIQUE_NAME) == 0) {
938 		if (nvlist_add_uint32(*out, TOPO_METH_CONTAINS_RET,
939 		    ret) == 0)
940 			return (0);
941 		else
942 			nvlist_free(*out);
943 	}
944 
945 	return (-1);
946 
947 }
948 
949 /*
950  * Return a current instance of the node.  This is somewhat complicated because
951  * we need to take a new snapshot in order to get the new data, but we don't
952  * want to be constantly taking SES snapshots if the consumer is going to do a
953  * series of queries.  So we adopt the strategy of assuming that the SES state
954  * is not going to be rapidly changing, and limit our snapshot frequency to
955  * some defined bounds.
956  */
957 ses_node_t *
958 ses_node_lock(topo_mod_t *mod, tnode_t *tn)
959 {
960 	ses_enum_target_t *tp = topo_node_getspecific(tn);
961 	hrtime_t now;
962 	ses_snap_t *snap;
963 	int err;
964 	uint64_t nodeid;
965 	ses_node_t *np;
966 
967 	if (tp == NULL) {
968 		(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
969 		return (NULL);
970 	}
971 
972 	(void) pthread_mutex_lock(&tp->set_lock);
973 
974 	/*
975 	 * Determine if we need to take a new snapshot.
976 	 */
977 	now = gethrtime();
978 
979 	if (tp->set_target == NULL) {
980 		/*
981 		 * We may have closed the device but not yet abandoned the
982 		 * contract (ie we've had the offline event but not yet the
983 		 * negend). If so, just return failure.
984 		 */
985 		if (tp->set_ctid != 0) {
986 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
987 			(void) pthread_mutex_unlock(&tp->set_lock);
988 			return (NULL);
989 		}
990 
991 		/*
992 		 * The device has been closed due to a contract offline
993 		 * request, then we need to reopen it and create a new contract.
994 		 */
995 		if ((tp->set_target =
996 		    ses_open(LIBSES_VERSION, tp->set_devpath)) == NULL) {
997 			sysevent_id_t eid;
998 
999 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
1000 			(void) pthread_mutex_unlock(&tp->set_lock);
1001 			topo_mod_dprintf(mod, "recheck_dir - "
1002 			    "can no longer open %s", tp->set_devpath);
1003 			(void) sysevent_post_event(EC_PLATFORM,
1004 			    ESC_PLATFORM_SP_RESET, SUNW_VENDOR, "fmd", NULL,
1005 			    &eid);
1006 			return (NULL);
1007 		}
1008 		topo_mod_dprintf(mod, "reopen contract");
1009 		ses_create_contract(mod, tp);
1010 		tp->set_snap = ses_snap_hold(tp->set_target);
1011 		tp->set_snaptime = gethrtime();
1012 	} else if (now - tp->set_snaptime > (ses_snap_freq * 1000 * 1000) &&
1013 	    (snap = ses_snap_new(tp->set_target)) != NULL) {
1014 		if (ses_snap_generation(snap) !=
1015 		    ses_snap_generation(tp->set_snap)) {
1016 			/*
1017 			 * If we find ourselves in this situation, we're in
1018 			 * trouble.  The generation count has changed, which
1019 			 * indicates that our current topology is out of date.
1020 			 * But we need to consult the new topology in order to
1021 			 * determine presence at this moment in time.  We can't
1022 			 * go back and change the topo snapshot in situ, so
1023 			 * we'll just have to fail the call in this unlikely
1024 			 * scenario.
1025 			 */
1026 			ses_snap_rele(snap);
1027 			(void) topo_mod_seterrno(mod, EMOD_METHOD_NOTSUP);
1028 			(void) pthread_mutex_unlock(&tp->set_lock);
1029 			return (NULL);
1030 		} else {
1031 			ses_snap_rele(tp->set_snap);
1032 			tp->set_snap = snap;
1033 		}
1034 		tp->set_snaptime = gethrtime();
1035 	}
1036 
1037 	snap = tp->set_snap;
1038 
1039 	verify(topo_prop_get_uint64(tn, TOPO_PGROUP_SES,
1040 	    TOPO_PROP_NODE_ID, &nodeid, &err) == 0);
1041 	verify((np = ses_node_lookup(snap, nodeid)) != NULL);
1042 
1043 	return (np);
1044 }
1045 
1046 /*ARGSUSED*/
1047 void
1048 ses_node_unlock(topo_mod_t *mod, tnode_t *tn)
1049 {
1050 	ses_enum_target_t *tp = topo_node_getspecific(tn);
1051 
1052 	verify(tp != NULL);
1053 
1054 	(void) pthread_mutex_unlock(&tp->set_lock);
1055 }
1056 
1057 /*
1058  * Determine if the element is present.
1059  */
1060 /*ARGSUSED*/
1061 static int
1062 ses_present(topo_mod_t *mod, tnode_t *tn, topo_version_t version,
1063     nvlist_t *in, nvlist_t **out)
1064 {
1065 	boolean_t present;
1066 	ses_node_t *np;
1067 	nvlist_t *props, *nvl;
1068 	uint64_t status;
1069 
1070 	if ((np = ses_node_lock(mod, tn)) == NULL)
1071 		return (-1);
1072 
1073 	verify((props = ses_node_props(np)) != NULL);
1074 	verify(nvlist_lookup_uint64(props,
1075 	    SES_PROP_STATUS_CODE, &status) == 0);
1076 
1077 	ses_node_unlock(mod, tn);
1078 
1079 	present = (status != SES_ESC_NOT_INSTALLED);
1080 
1081 	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0)
1082 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
1083 
1084 	if (nvlist_add_uint32(nvl, TOPO_METH_PRESENT_RET,
1085 	    present) != 0) {
1086 		nvlist_free(nvl);
1087 		return (topo_mod_seterrno(mod, EMOD_FMRI_NVL));
1088 	}
1089 
1090 	*out = nvl;
1091 
1092 	return (0);
1093 }
1094 
1095 /*
1096  * Sets standard properties for a ses node (enclosure, bay, controller
1097  * or expander).
1098  * This includes setting the FRU, as well as setting the
1099  * authority information.  When  the fru topo node(frutn) is not NULL
1100  * its resouce should be used as FRU.
1101  */
1102 static int
1103 ses_set_standard_props(topo_mod_t *mod, tnode_t *frutn, tnode_t *tn,
1104     nvlist_t *auth, uint64_t nodeid, const char *path)
1105 {
1106 	int err;
1107 	char *product, *chassis;
1108 	nvlist_t *fmri;
1109 
1110 	/*
1111 	 * Set the authority explicitly if specified.
1112 	 */
1113 	if (auth) {
1114 		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
1115 		    &product) == 0);
1116 		verify(nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
1117 		    &chassis) == 0);
1118 		if (topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1119 		    FM_FMRI_AUTH_PRODUCT, TOPO_PROP_IMMUTABLE, product,
1120 		    &err) != 0 ||
1121 		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1122 		    FM_FMRI_AUTH_CHASSIS, TOPO_PROP_IMMUTABLE, chassis,
1123 		    &err) != 0 ||
1124 		    topo_prop_set_string(tn, FM_FMRI_AUTHORITY,
1125 		    FM_FMRI_AUTH_SERVER, TOPO_PROP_IMMUTABLE, "",
1126 		    &err) != 0) {
1127 			topo_mod_dprintf(mod, "failed to add authority "
1128 			    "properties: %s\n", topo_strerror(err));
1129 			return (topo_mod_seterrno(mod, err));
1130 		}
1131 	}
1132 
1133 	/*
1134 	 * Copy the resource and set that as the FRU.
1135 	 */
1136 	if (frutn != NULL) {
1137 		if (topo_node_resource(frutn, &fmri, &err) != 0) {
1138 			topo_mod_dprintf(mod,
1139 			    "topo_node_resource() failed : %s\n",
1140 			    topo_strerror(err));
1141 			return (topo_mod_seterrno(mod, err));
1142 		}
1143 	} else {
1144 		if (topo_node_resource(tn, &fmri, &err) != 0) {
1145 			topo_mod_dprintf(mod,
1146 			    "topo_node_resource() failed : %s\n",
1147 			    topo_strerror(err));
1148 			return (topo_mod_seterrno(mod, err));
1149 		}
1150 	}
1151 
1152 	if (topo_node_fru_set(tn, fmri, 0, &err) != 0) {
1153 		topo_mod_dprintf(mod,
1154 		    "topo_node_fru_set() failed : %s\n",
1155 		    topo_strerror(err));
1156 		nvlist_free(fmri);
1157 		return (topo_mod_seterrno(mod, err));
1158 	}
1159 
1160 	nvlist_free(fmri);
1161 
1162 	/*
1163 	 * Set the SES-specific properties so that consumers can query
1164 	 * additional information about the particular SES element.
1165 	 */
1166 	if (topo_pgroup_create(tn, &ses_pgroup, &err) != 0) {
1167 		topo_mod_dprintf(mod, "failed to create propgroup "
1168 		    "%s: %s\n", TOPO_PGROUP_SES, topo_strerror(err));
1169 		return (-1);
1170 	}
1171 
1172 	if (topo_prop_set_uint64(tn, TOPO_PGROUP_SES,
1173 	    TOPO_PROP_NODE_ID, TOPO_PROP_IMMUTABLE,
1174 	    nodeid, &err) != 0) {
1175 		topo_mod_dprintf(mod,
1176 		    "failed to create property %s: %s\n",
1177 		    TOPO_PROP_NODE_ID, topo_strerror(err));
1178 		return (-1);
1179 	}
1180 
1181 	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
1182 	    TOPO_PROP_TARGET_PATH, TOPO_PROP_IMMUTABLE,
1183 	    path, &err) != 0) {
1184 		topo_mod_dprintf(mod,
1185 		    "failed to create property %s: %s\n",
1186 		    TOPO_PROP_TARGET_PATH, topo_strerror(err));
1187 		return (-1);
1188 	}
1189 
1190 	return (0);
1191 }
1192 
1193 /*
1194  * Iterate over the SES phy information. If any of the ports indicates that it's
1195  * a SATA device and we haven't matched any disk devices yet, that means
1196  * that the HBA was able to create a WWN for the SATA device based on its GUID,
1197  * which is good. However, SES includes the WWN for the device's STP bridge. In
1198  * theory, if the driver includes the WWN based on the SATA guid then it should
1199  * also set the bridge-port property indicating the WWN that should match the
1200  * SATA device.
1201  */
1202 static int
1203 ses_create_disk_bridge(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props,
1204     tnode_t **child)
1205 {
1206 	nvlist_t **phys;
1207 	uint_t i, n_phys;
1208 	topo_mod_t *mod = sdp->sed_mod;
1209 
1210 	if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS, &phys,
1211 	    &n_phys) != 0)
1212 		return (1);
1213 
1214 	for (i = 0; i < n_phys; i++) {
1215 		uint64_t wwn;
1216 		boolean_t sata;
1217 		char wwnstr[64];
1218 
1219 		if (nvlist_lookup_uint64(phys[i], SES_SAS_PROP_ADDR,
1220 		    &wwn) != 0 || wwn == 0) {
1221 			continue;
1222 		}
1223 
1224 		if (nvlist_lookup_boolean_value(phys[i],
1225 		    SES_SAS_PROP_SATA_DEVICE, &sata) != 0 || !sata) {
1226 			continue;
1227 		}
1228 
1229 		if (scsi_wwn_to_wwnstr(wwn, 0, wwnstr) == NULL)
1230 			continue;
1231 
1232 		if (disk_declare_bridge(mod, pnode, &sdp->sed_devs,
1233 		    wwnstr, child) == 0) {
1234 			return (0);
1235 		}
1236 
1237 	}
1238 
1239 	return (1);
1240 }
1241 
1242 /*
1243  * Callback to add a disk to a given bay.  We first check the status-code to
1244  * determine if a disk is present, ignoring those that aren't in an appropriate
1245  * state.  We then scan the parent bay node's SAS address array to determine
1246  * possible attached SAS addresses.  We create a disk node if the disk is not
1247  * SAS or the SES target does not support the necessary pages for this; if we
1248  * find the SAS address, we create a disk node and also correlate it with
1249  * the corresponding Solaris device node to fill in the rest of the data.
1250  */
1251 static int
1252 ses_create_disk(ses_enum_data_t *sdp, tnode_t *pnode, nvlist_t *props)
1253 {
1254 	topo_mod_t *mod = sdp->sed_mod;
1255 	uint64_t status;
1256 	uint_t s, nsas;
1257 	char **paths;
1258 	int err, ret;
1259 	tnode_t *child = NULL;
1260 
1261 	/*
1262 	 * Skip devices that are not in a present (and possibly damaged) state.
1263 	 * Also, skip devices that this expander is either not fully wired to,
1264 	 * or are hidden due to SAS zoning, as indicated by the
1265 	 * SES_ESC_NO_ACCESS state.
1266 	 */
1267 	if (nvlist_lookup_uint64(props, SES_PROP_STATUS_CODE, &status) != 0)
1268 		return (0);
1269 
1270 	if (status != SES_ESC_UNSUPPORTED &&
1271 	    status != SES_ESC_OK &&
1272 	    status != SES_ESC_CRITICAL &&
1273 	    status != SES_ESC_NONCRITICAL &&
1274 	    status != SES_ESC_UNRECOVERABLE &&
1275 	    status != SES_ESC_UNKNOWN)
1276 		return (0);
1277 
1278 	topo_mod_dprintf(mod, "found attached disk");
1279 
1280 	/*
1281 	 * Create the disk range.
1282 	 */
1283 	if (topo_node_range_create(mod, pnode, DISK, 0, 0) != 0) {
1284 		topo_mod_dprintf(mod,
1285 		    "topo_node_create_range() failed: %s",
1286 		    topo_mod_errmsg(mod));
1287 		return (-1);
1288 	}
1289 
1290 	/*
1291 	 * Look through all SAS addresses and attempt to correlate them to a
1292 	 * known Solaris device.  If we don't find a matching node, then we
1293 	 * don't enumerate the disk node.
1294 	 * Note that TOPO_PROP_SAS_ADDR prop includes SAS address from
1295 	 * alternate elements that represent the same device.
1296 	 */
1297 	if (topo_prop_get_string_array(pnode, TOPO_PGROUP_SES,
1298 	    TOPO_PROP_SAS_ADDR, &paths, &nsas, &err) != 0)
1299 		return (0);
1300 
1301 	err = 0;
1302 
1303 	for (s = 0; s < nsas; s++) {
1304 		ret = disk_declare_addr(mod, pnode, &sdp->sed_devs, paths[s],
1305 		    &child);
1306 		if (ret == 0) {
1307 			break;
1308 		} else if (ret < 0) {
1309 			err = -1;
1310 			break;
1311 		}
1312 	}
1313 
1314 	/*
1315 	 * We need to take another pass through the properties for this bay by
1316 	 * iterating over the phys and noting if any of these are SATA. Note,
1317 	 * this information isn't commonly part of the topo tree at this time,
1318 	 * hence why we end up going back and iterating over the properties
1319 	 * ourselves.
1320 	 */
1321 	if (s == nsas) {
1322 		if (ses_create_disk_bridge(sdp, pnode, props, &child) != 0)
1323 			(void) disk_declare_non_enumerated(mod, pnode, &child);
1324 	}
1325 
1326 	/* copy sas_addresses (target-ports) from parent (with 'w'added) */
1327 	if (child != NULL) {
1328 		int i;
1329 		char **tports;
1330 		uint64_t wwn;
1331 
1332 		tports = topo_mod_zalloc(mod, sizeof (char *) * nsas);
1333 		if (tports != NULL) {
1334 			for (i = 0; i < nsas; i++) {
1335 				if (scsi_wwnstr_to_wwn(paths[i], &wwn) !=
1336 				    DDI_SUCCESS)
1337 					break;
1338 				tports[i] = scsi_wwn_to_wwnstr(wwn, 1, NULL);
1339 				if (tports[i] == NULL)
1340 					break;
1341 			}
1342 			/* if they all worked then create the property */
1343 			if (i == nsas)
1344 				(void) topo_prop_set_string_array(child,
1345 				    TOPO_PGROUP_STORAGE,
1346 				    TOPO_STORAGE_TARGET_PORT_L0IDS,
1347 				    TOPO_PROP_IMMUTABLE, (const char **)tports,
1348 				    nsas, &err);
1349 
1350 			for (i = 0; i < nsas; i++)
1351 				if (tports[i] != NULL)
1352 					scsi_free_wwnstr(tports[i]);
1353 			topo_mod_free(mod, tports, sizeof (char *) * nsas);
1354 		}
1355 	}
1356 
1357 	for (s = 0; s < nsas; s++)
1358 		topo_mod_free(mod, paths[s], strlen(paths[s]) + 1);
1359 	topo_mod_free(mod, paths, nsas * sizeof (char *));
1360 
1361 	return (err);
1362 }
1363 
1364 static int
1365 ses_add_bay_props(topo_mod_t *mod, tnode_t *tn, ses_enum_node_t *snp)
1366 {
1367 	ses_alt_node_t *ap;
1368 	ses_node_t *np;
1369 	nvlist_t *props;
1370 
1371 	nvlist_t **phys;
1372 	uint_t i, j, n_phys, all_phys = 0;
1373 	char **paths;
1374 	uint64_t addr;
1375 	size_t len;
1376 	int terr, err = -1;
1377 
1378 	for (ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
1379 	    ap = topo_list_next(ap)) {
1380 		np = ap->san_node;
1381 		props = ses_node_props(np);
1382 
1383 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1384 		    &phys, &n_phys) != 0)
1385 			continue;
1386 
1387 		all_phys += n_phys;
1388 	}
1389 
1390 	if (all_phys == 0)
1391 		return (0);
1392 
1393 	if ((paths = topo_mod_zalloc(mod, all_phys * sizeof (char *))) == NULL)
1394 		return (-1);
1395 
1396 	for (i = 0, ap = topo_list_next(&snp->sen_alt_nodes); ap != NULL;
1397 	    ap = topo_list_next(ap)) {
1398 		np = ap->san_node;
1399 		props = ses_node_props(np);
1400 
1401 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1402 		    &phys, &n_phys) != 0)
1403 			continue;
1404 
1405 		for (j = 0; j < n_phys; j++) {
1406 			if (nvlist_lookup_uint64(phys[j], SES_SAS_PROP_ADDR,
1407 			    &addr) != 0)
1408 				continue;
1409 
1410 			len = snprintf(NULL, 0, "%016llx", addr) + 1;
1411 			if ((paths[i] = topo_mod_alloc(mod, len)) == NULL)
1412 				goto error;
1413 
1414 			(void) snprintf(paths[i], len, "%016llx", addr);
1415 
1416 			++i;
1417 		}
1418 	}
1419 
1420 	err = topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
1421 	    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE,
1422 	    (const char **)paths, i, &terr);
1423 	if (err != 0)
1424 		err = topo_mod_seterrno(mod, terr);
1425 
1426 error:
1427 	for (i = 0; i < all_phys && paths[i] != NULL; i++)
1428 		topo_mod_free(mod, paths[i], strlen(paths[i]) + 1);
1429 	topo_mod_free(mod, paths, all_phys * sizeof (char *));
1430 
1431 	return (err);
1432 }
1433 
1434 static const char *
1435 lookup_bay_override(const char *product_id)
1436 {
1437 	for (uint_t i = 0; i < N_BAY_LBL_OVERRIDES; i++) {
1438 		if (strcmp(product_id,
1439 		    bay_label_overrides[i].slbl_product) == 0) {
1440 			return (bay_label_overrides[i].slbl_mname);
1441 		}
1442 	}
1443 	return (NULL);
1444 }
1445 
1446 /*
1447  * Callback to create a basic node (bay, psu, fan, or controller and expander).
1448  */
1449 static int
1450 ses_create_generic(ses_enum_data_t *sdp, ses_enum_node_t *snp, tnode_t *pnode,
1451     tnode_t *frutn, const char *nodename, const char *labelname,
1452     tnode_t **node)
1453 {
1454 	ses_node_t *np = snp->sen_node;
1455 	ses_node_t *parent;
1456 	uint64_t instance = snp->sen_instance;
1457 	topo_mod_t *mod = sdp->sed_mod;
1458 	nvlist_t *props, *aprops;
1459 	nvlist_t *auth = NULL, *fmri = NULL;
1460 	tnode_t *tn = NULL;
1461 	char *clean_label = NULL, label[128];
1462 	int err;
1463 	char *part = NULL, *serial = NULL, *revision = NULL;
1464 	char *desc;
1465 	boolean_t report;
1466 
1467 	props = ses_node_props(np);
1468 
1469 	(void) nvlist_lookup_string(props, LIBSES_PROP_PART, &part);
1470 	(void) nvlist_lookup_string(props, LIBSES_PROP_SERIAL, &serial);
1471 
1472 	topo_mod_dprintf(mod, "adding %s %llu", nodename, instance);
1473 
1474 	/*
1475 	 * Create the node.  The interesting information is all copied from the
1476 	 * parent enclosure node, so there is not much to do.
1477 	 */
1478 	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
1479 		goto error;
1480 
1481 	/*
1482 	 * We want to report revision information for the controller nodes, but
1483 	 * we do not get per-element revision information.  However, we do have
1484 	 * revision information for the entire enclosure, and we can use the
1485 	 * 'reported-via' property to know that this controller corresponds to
1486 	 * the given revision information.  This means we cannot get revision
1487 	 * information for targets we are not explicitly connected to, but
1488 	 * there is little we can do about the situation.
1489 	 */
1490 	if (strcmp(nodename, CONTROLLER) == 0 &&
1491 	    nvlist_lookup_boolean_value(props, SES_PROP_REPORT, &report) == 0 &&
1492 	    report) {
1493 		for (parent = ses_node_parent(np); parent != NULL;
1494 		    parent = ses_node_parent(parent)) {
1495 			if (ses_node_type(parent) == SES_NODE_ENCLOSURE) {
1496 				(void) nvlist_lookup_string(
1497 				    ses_node_props(parent),
1498 				    SES_EN_PROP_REV, &revision);
1499 				break;
1500 			}
1501 		}
1502 	}
1503 
1504 	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
1505 	    nodename, (topo_instance_t)instance, NULL, auth, part, revision,
1506 	    serial)) == NULL) {
1507 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
1508 		    topo_mod_errmsg(mod));
1509 		goto error;
1510 	}
1511 
1512 	if ((tn = topo_node_bind(mod, pnode, nodename,
1513 	    instance, fmri)) == NULL) {
1514 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
1515 		    topo_mod_errmsg(mod));
1516 		goto error;
1517 	}
1518 
1519 	/*
1520 	 * For the node label, we look for the following in order:
1521 	 *
1522 	 * <ses-description>
1523 	 * <ses-class-description> <instance>
1524 	 * <default-type-label> <instance>
1525 	 */
1526 	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
1527 	    desc[0] == '\0') {
1528 		parent = ses_node_parent(np);
1529 		aprops = ses_node_props(parent);
1530 		if (nvlist_lookup_string(aprops, SES_PROP_CLASS_DESCRIPTION,
1531 		    &desc) != 0 || desc[0] == '\0')
1532 			desc = (char *)labelname;
1533 		(void) snprintf(label, sizeof (label), "%s %llu", desc,
1534 		    instance);
1535 		desc = label;
1536 	}
1537 
1538 	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
1539 		goto error;
1540 
1541 	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
1542 	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
1543 		goto error;
1544 
1545 	if (ses_set_standard_props(mod, frutn, tn, NULL, ses_node_id(np),
1546 	    snp->sen_target->set_devpath) != 0)
1547 		goto error;
1548 
1549 	if (strcmp(nodename, BAY) == 0) {
1550 		const char *label_method;
1551 		char *product;
1552 		nvlist_t *args = NULL;
1553 
1554 		if (ses_add_bay_props(mod, tn, snp) != 0)
1555 			goto error;
1556 
1557 		if (topo_method_register(mod, tn, ses_bay_methods) != 0) {
1558 			topo_mod_dprintf(mod,
1559 			    "topo_method_register() failed: %s",
1560 			    topo_mod_errmsg(mod));
1561 			goto error;
1562 		}
1563 
1564 		/*
1565 		 * Ideally we'd perform this sort of override with a platform
1566 		 * specific XML map file, and that would work here if we only
1567 		 * wanted to override the bay node label.  However, we'd also
1568 		 * like the disk node label (if the bay is occupied) to inherit
1569 		 * the overriden bay label.  So we need to ensure the
1570 		 * propmethod is registered before we create the child disk
1571 		 * node.
1572 		 */
1573 		if ((product = topo_mod_product(mod)) == NULL) {
1574 			(void) topo_mod_seterrno(mod, EMOD_NOMEM);
1575 			goto error;
1576 		}
1577 		if ((label_method = lookup_bay_override(product)) != NULL) {
1578 			if (topo_mod_nvalloc(mod, &args, NV_UNIQUE_NAME) != 0 ||
1579 			    topo_prop_method_register(tn, TOPO_PGROUP_PROTOCOL,
1580 			    TOPO_PROP_LABEL, TOPO_TYPE_STRING, label_method,
1581 			    args, &err)) {
1582 				topo_mod_dprintf(mod,
1583 				    "Failed to register method: %s on %s=%"
1584 				    PRIu64, label_method, BAY,
1585 				    topo_node_instance(tn));
1586 				topo_mod_strfree(mod, product);
1587 				nvlist_free(args);
1588 				goto error;
1589 			}
1590 			nvlist_free(args);
1591 		}
1592 		topo_mod_strfree(mod, product);
1593 
1594 		if (ses_create_disk(sdp, tn, props) != 0)
1595 			goto error;
1596 	} else if ((strcmp(nodename, FAN) == 0) ||
1597 	    (strcmp(nodename, PSU) == 0) ||
1598 	    (strcmp(nodename, CONTROLLER) == 0)) {
1599 		/*
1600 		 * Only fan, psu, and controller nodes have a 'present' method.
1601 		 * Bay nodes are always present, and disk nodes are present by
1602 		 * virtue of being enumerated and SAS expander nodes and
1603 		 * SAS connector nodes are also always present once
1604 		 * the parent controller is found.
1605 		 */
1606 		if (topo_method_register(mod, tn, ses_component_methods) != 0) {
1607 			topo_mod_dprintf(mod,
1608 			    "topo_method_register() failed: %s",
1609 			    topo_mod_errmsg(mod));
1610 			goto error;
1611 		}
1612 
1613 	}
1614 
1615 	snp->sen_target->set_refcount++;
1616 	topo_node_setspecific(tn, snp->sen_target);
1617 
1618 	nvlist_free(auth);
1619 	nvlist_free(fmri);
1620 	topo_mod_strfree(mod, clean_label);
1621 	if (node != NULL) *node = tn;
1622 	return (0);
1623 
1624 error:
1625 	nvlist_free(auth);
1626 	nvlist_free(fmri);
1627 	topo_mod_strfree(mod, clean_label);
1628 	return (-1);
1629 }
1630 
1631 /*
1632  * Create SAS expander specific props.
1633  */
1634 /*ARGSUSED*/
1635 static int
1636 ses_set_expander_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1637     tnode_t *ptnode, tnode_t *tnode, int *phycount, int64_t *connlist)
1638 {
1639 	ses_node_t *np = snp->sen_node;
1640 	topo_mod_t *mod = sdp->sed_mod;
1641 	nvlist_t *auth = NULL, *fmri = NULL;
1642 	nvlist_t *props, **phylist;
1643 	int err, i;
1644 	uint_t pcount;
1645 	uint64_t sasaddr, connidx;
1646 	char sasaddr_str[17];
1647 	boolean_t found = B_FALSE, ses_found = B_FALSE;
1648 	dev_di_node_t *dnode, *sesdnode;
1649 
1650 	props = ses_node_props(np);
1651 
1652 	/*
1653 	 * the uninstalled expander is not enumerated by checking
1654 	 * the element status code.  No present present' method provided.
1655 	 */
1656 	/*
1657 	 * Get the Expander SAS address.  It should exist.
1658 	 */
1659 	if (nvlist_lookup_uint64(props, SES_EXP_PROP_SAS_ADDR,
1660 	    &sasaddr) != 0) {
1661 		topo_mod_dprintf(mod,
1662 		    "Failed to get prop %s.", SES_EXP_PROP_SAS_ADDR);
1663 		goto error;
1664 	}
1665 
1666 	(void) sprintf(sasaddr_str, "%llx", sasaddr);
1667 
1668 	/* search matching dev_di_node. */
1669 	for (dnode = topo_list_next(&sdp->sed_devs); dnode != NULL;
1670 	    dnode = topo_list_next(dnode)) {
1671 		for (i = 0; i < dnode->ddn_ppath_count; i++) {
1672 			if ((dnode->ddn_target_port[i] != NULL) &&
1673 			    (strstr(dnode->ddn_target_port[i],
1674 			    sasaddr_str) != NULL)) {
1675 				found = B_TRUE;
1676 				break;
1677 			}
1678 		}
1679 		if (found)
1680 			break;
1681 	}
1682 
1683 	if (!found) {
1684 		topo_mod_dprintf(mod,
1685 		    "ses_set_expander_props: Failed to find matching "
1686 		    "devinfo node for Exapnder SAS address %s",
1687 		    SES_EXP_PROP_SAS_ADDR);
1688 		/* continue on to get storage group props. */
1689 	} else {
1690 		/* create/set the devfs-path and devid in the smp group */
1691 		if (topo_pgroup_create(tnode, &smp_pgroup, &err) != 0) {
1692 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1693 			    "failed to create smp property group %s\n",
1694 			    topo_strerror(err));
1695 			goto error;
1696 		} else {
1697 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1698 			    TOPO_PROP_SMP_TARGET_PORT, TOPO_PROP_IMMUTABLE,
1699 			    dnode->ddn_target_port[i], &err) != 0) {
1700 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1701 				    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
1702 				    topo_strerror(err));
1703 			}
1704 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1705 			    TOPO_PROP_SMP_DEV_PATH, TOPO_PROP_IMMUTABLE,
1706 			    dnode->ddn_dpath, &err) != 0) {
1707 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1708 				    "set dev error %s\n", topo_strerror(err));
1709 			}
1710 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SMP,
1711 			    TOPO_PROP_SMP_DEVID, TOPO_PROP_IMMUTABLE,
1712 			    dnode->ddn_devid, &err) != 0) {
1713 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1714 				    "set devid error %s\n", topo_strerror(err));
1715 			}
1716 			if (dnode->ddn_ppath_count != 0 &&
1717 			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SMP,
1718 			    TOPO_PROP_SMP_PHYS_PATH, TOPO_PROP_IMMUTABLE,
1719 			    (const char **)dnode->ddn_ppath,
1720 			    dnode->ddn_ppath_count, &err) != 0) {
1721 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1722 				    "set phys-path error %s\n",
1723 				    topo_strerror(err));
1724 			}
1725 		}
1726 	}
1727 
1728 	/* update the ses property group with SES target info */
1729 	if ((topo_pgroup_create(tnode, &ses_pgroup, &err) != 0) &&
1730 	    (err != ETOPO_PROP_DEFD)) {
1731 		/* SES prop group doesn't exist but failed to be created. */
1732 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1733 		    "ses pgroup create error %s\n", topo_strerror(err));
1734 		goto error;
1735 	} else {
1736 		/* locate assciated enclosure dev_di_node. */
1737 		for (sesdnode = topo_list_next(&sdp->sed_devs);
1738 		    sesdnode != NULL; sesdnode = topo_list_next(sesdnode)) {
1739 			for (i = 0; i < sesdnode->ddn_ppath_count; i++) {
1740 				/*
1741 				 * check if attached port exists and
1742 				 * its node type is enclosure and
1743 				 * attached port is same as sas address of
1744 				 * the expander and
1745 				 * bridge port for virtual phy indication
1746 				 * exist.
1747 				 */
1748 				if ((sesdnode->ddn_attached_port[i] != NULL) &&
1749 				    (sesdnode->ddn_dtype == DTYPE_ESI) &&
1750 				    (strstr(sesdnode->ddn_attached_port[i],
1751 				    sasaddr_str) != NULL) &&
1752 				    (sesdnode->ddn_bridge_port[i] != NULL)) {
1753 					ses_found = B_TRUE;
1754 					break;
1755 				}
1756 			}
1757 			if (ses_found) break;
1758 		}
1759 
1760 		if (ses_found) {
1761 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1762 			    TOPO_PROP_SES_TARGET_PORT, TOPO_PROP_IMMUTABLE,
1763 			    sesdnode->ddn_target_port[i], &err) != 0) {
1764 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1765 				    "set ses %S error %s\n", TOPO_PROP_SAS_ADDR,
1766 				    topo_strerror(err));
1767 			}
1768 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1769 			    TOPO_PROP_SES_DEV_PATH, TOPO_PROP_IMMUTABLE,
1770 			    sesdnode->ddn_dpath, &err) != 0) {
1771 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1772 				    "set ses dev error %s\n",
1773 				    topo_strerror(err));
1774 			}
1775 			if (topo_prop_set_string(tnode, TOPO_PGROUP_SES,
1776 			    TOPO_PROP_SES_DEVID, TOPO_PROP_IMMUTABLE,
1777 			    sesdnode->ddn_devid, &err) != 0) {
1778 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1779 				    "set ses devid error %s\n",
1780 				    topo_strerror(err));
1781 			}
1782 			if (sesdnode->ddn_ppath_count != 0 &&
1783 			    topo_prop_set_string_array(tnode, TOPO_PGROUP_SES,
1784 			    TOPO_PROP_SES_PHYS_PATH, TOPO_PROP_IMMUTABLE,
1785 			    (const char **)sesdnode->ddn_ppath,
1786 			    sesdnode->ddn_ppath_count, &err) != 0) {
1787 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1788 				    "set ses phys-path error %s\n",
1789 				    topo_strerror(err));
1790 			}
1791 
1792 		}
1793 	}
1794 
1795 	/* create the storage group */
1796 	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
1797 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1798 		    "create storage error %s\n", topo_strerror(err));
1799 		goto error;
1800 	} else {
1801 		/* set the SAS address prop out of expander element status. */
1802 		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1803 		    TOPO_PROP_SAS_ADDR, TOPO_PROP_IMMUTABLE, sasaddr_str,
1804 		    &err) != 0) {
1805 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1806 			    "set %S error %s\n", TOPO_PROP_SAS_ADDR,
1807 			    topo_strerror(err));
1808 		}
1809 
1810 		/* Get the phy information for the expander */
1811 		if (nvlist_lookup_nvlist_array(props, SES_SAS_PROP_PHYS,
1812 		    &phylist, &pcount) != 0) {
1813 			topo_mod_dprintf(mod,
1814 			    "Failed to get prop %s.", SES_SAS_PROP_PHYS);
1815 		} else {
1816 			/*
1817 			 * For each phy, get the connector element index and
1818 			 * stores into connector element index array.
1819 			 */
1820 			*phycount = pcount;
1821 			for (i = 0; i < pcount; i++) {
1822 				if (nvlist_lookup_uint64(phylist[i],
1823 				    SES_PROP_CE_IDX, &connidx) == 0) {
1824 					if (connidx != 0xff) {
1825 						connlist[i] = connidx;
1826 					} else {
1827 						connlist[i] = -1;
1828 					}
1829 				} else {
1830 					/* Fail to get the index. set to -1. */
1831 					connlist[i] = -1;
1832 				}
1833 			}
1834 
1835 			/* set the phy count prop of the expander. */
1836 			if (topo_prop_set_uint64(tnode, TOPO_PGROUP_STORAGE,
1837 			    TOPO_PROP_PHY_COUNT, TOPO_PROP_IMMUTABLE, pcount,
1838 			    &err) != 0) {
1839 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1840 				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
1841 				    topo_strerror(err));
1842 			}
1843 
1844 			/*
1845 			 * set the connector element index of
1846 			 * the expander phys.
1847 			 */
1848 		}
1849 
1850 		/* populate other misc storage group properties */
1851 		if (found) {
1852 			if (dnode->ddn_mfg && (topo_prop_set_string(tnode,
1853 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MANUFACTURER,
1854 			    TOPO_PROP_IMMUTABLE, dnode->ddn_mfg, &err) != 0)) {
1855 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1856 				    "set mfg error %s\n", topo_strerror(err));
1857 			}
1858 
1859 			if (dnode->ddn_model && (topo_prop_set_string(tnode,
1860 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_MODEL,
1861 			    TOPO_PROP_IMMUTABLE,
1862 			    dnode->ddn_model, &err) != 0)) {
1863 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1864 				    "set model error %s\n", topo_strerror(err));
1865 			}
1866 
1867 			if (dnode->ddn_serial && (topo_prop_set_string(tnode,
1868 			    TOPO_PGROUP_STORAGE, TOPO_STORAGE_SERIAL_NUM,
1869 			    TOPO_PROP_IMMUTABLE,
1870 			    dnode->ddn_serial, &err) != 0)) {
1871 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1872 				    "set serial error %s\n",
1873 				    topo_strerror(err));
1874 			}
1875 
1876 			if (dnode->ddn_firm && (topo_prop_set_string(tnode,
1877 			    TOPO_PGROUP_STORAGE,
1878 			    TOPO_STORAGE_FIRMWARE_REV, TOPO_PROP_IMMUTABLE,
1879 			    dnode->ddn_firm, &err) != 0)) {
1880 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1881 				    "set firm error %s\n", topo_strerror(err));
1882 			}
1883 		}
1884 	}
1885 
1886 	return (0);
1887 
1888 error:
1889 	nvlist_free(auth);
1890 	nvlist_free(fmri);
1891 	return (-1);
1892 }
1893 
1894 /*
1895  * Create SAS expander specific props.
1896  */
1897 /*ARGSUSED*/
1898 static int
1899 ses_set_connector_props(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1900     tnode_t *tnode, int64_t phy_mask)
1901 {
1902 	ses_node_t *np = snp->sen_node;
1903 	topo_mod_t *mod = sdp->sed_mod;
1904 	nvlist_t *props;
1905 	int err, i;
1906 	uint64_t conntype;
1907 	char phymask_str[17], *conntype_str;
1908 	boolean_t   found;
1909 
1910 	props = ses_node_props(np);
1911 
1912 	/*
1913 	 * convert phy mask to string.
1914 	 */
1915 	(void) snprintf(phymask_str, 17, "%llx", phy_mask);
1916 
1917 	/* create the storage group */
1918 	if (topo_pgroup_create(tnode, &storage_pgroup, &err) != 0) {
1919 		topo_mod_dprintf(mod, "ses_set_expander_props: "
1920 		    "create storage error %s\n", topo_strerror(err));
1921 		return (-1);
1922 	} else {
1923 		/* set the SAS address prop of the expander. */
1924 		if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1925 		    TOPO_STORAGE_SAS_PHY_MASK, TOPO_PROP_IMMUTABLE,
1926 		    phymask_str, &err) != 0) {
1927 			topo_mod_dprintf(mod, "ses_set_expander_props: "
1928 			    "set %S error %s\n", TOPO_STORAGE_SAS_PHY_MASK,
1929 			    topo_strerror(err));
1930 		}
1931 
1932 		/* Get the connector type information for the expander */
1933 		if (nvlist_lookup_uint64(props,
1934 		    SES_SC_PROP_CONNECTOR_TYPE, &conntype) != 0) {
1935 			topo_mod_dprintf(mod, "Failed to get prop %s.",
1936 			    TOPO_STORAGE_SAS_PHY_MASK);
1937 		} else {
1938 			found = B_FALSE;
1939 			for (i = 0; ; i++) {
1940 				if (sas_connector_type_list[i].sct_type ==
1941 				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
1942 					break;
1943 				}
1944 				if (sas_connector_type_list[i].sct_type ==
1945 				    conntype) {
1946 					conntype_str =
1947 					    sas_connector_type_list[i].sct_name;
1948 					found = B_TRUE;
1949 					break;
1950 				}
1951 			}
1952 
1953 			if (!found) {
1954 				if (conntype <
1955 				    SAS_CONNECTOR_TYPE_CODE_NOT_DEFINED) {
1956 					conntype_str =
1957 					    SAS_CONNECTOR_TYPE_RESERVED;
1958 				} else {
1959 					conntype_str =
1960 					    SAS_CONNECTOR_TYPE_NOT_DEFINED;
1961 				}
1962 			}
1963 
1964 			/* set the phy count prop of the expander. */
1965 			if (topo_prop_set_string(tnode, TOPO_PGROUP_STORAGE,
1966 			    TOPO_STORAGE_SAS_CONNECTOR_TYPE,
1967 			    TOPO_PROP_IMMUTABLE, conntype_str, &err) != 0) {
1968 				topo_mod_dprintf(mod, "ses_set_expander_props: "
1969 				    "set %S error %s\n", TOPO_PROP_PHY_COUNT,
1970 				    topo_strerror(err));
1971 			}
1972 		}
1973 	}
1974 
1975 	return (0);
1976 }
1977 
1978 /*
1979  * Instantiate SAS expander nodes for a given ESC Electronics node(controller)
1980  * nodes.
1981  */
1982 /*ARGSUSED*/
1983 static int
1984 ses_create_esc_sasspecific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
1985     tnode_t *pnode, ses_enum_chassis_t *cp,
1986     boolean_t dorange)
1987 {
1988 	topo_mod_t *mod = sdp->sed_mod;
1989 	tnode_t	*exptn, *contn;
1990 	boolean_t found;
1991 	sas_connector_phy_data_t connectors[64] = {0};
1992 	uint64_t max;
1993 	ses_enum_node_t *ctlsnp, *xsnp, *consnp;
1994 	ses_node_t *np = snp->sen_node;
1995 	nvlist_t *props, *psprops;
1996 	uint64_t index, psindex, conindex, psstatus, i, j, count;
1997 	int64_t cidxlist[256] = {0};
1998 	int phycount;
1999 
2000 	props = ses_node_props(np);
2001 
2002 	if (nvlist_lookup_uint64(props, SES_PROP_ELEMENT_ONLY_INDEX,
2003 	    &index) != 0)
2004 		return (-1);
2005 
2006 	/*
2007 	 * For SES constroller node, check to see if there are
2008 	 * associated SAS expanders.
2009 	 */
2010 	found = B_FALSE;
2011 	max = 0;
2012 	for (ctlsnp = topo_list_next(&cp->sec_nodes); ctlsnp != NULL;
2013 	    ctlsnp = topo_list_next(ctlsnp)) {
2014 		if (ctlsnp->sen_type == SES_ET_SAS_EXPANDER) {
2015 			found = B_TRUE;
2016 			if (ctlsnp->sen_instance > max)
2017 				max = ctlsnp->sen_instance;
2018 		}
2019 	}
2020 
2021 	/*
2022 	 * No SAS expander found notthing to process.
2023 	 */
2024 	if (!found)
2025 		return (0);
2026 
2027 	topo_mod_dprintf(mod, "%s Controller %d: creating "
2028 	    "%llu %s nodes", cp->sec_csn, index, max + 1, SASEXPANDER);
2029 
2030 	/*
2031 	 * The max number represent the number of elements
2032 	 * deducted from the highest SES_PROP_ELEMENT_CLASS_INDEX
2033 	 * of SET_ET_SAS_EXPANDER type element.
2034 	 *
2035 	 * There may be multiple ESC Electronics element(controllers)
2036 	 * within JBOD(typicall two for redundancy) and SAS expander
2037 	 * elements are associated with only one of them.  We are
2038 	 * still creating the range based max number here.
2039 	 * That will cover the case that all expanders are associated
2040 	 * with one SES controller.
2041 	 */
2042 	if (dorange && topo_node_range_create(mod, pnode,
2043 	    SASEXPANDER, 0, max) != 0) {
2044 		topo_mod_dprintf(mod,
2045 		    "topo_node_create_range() failed: %s",
2046 		    topo_mod_errmsg(mod));
2047 		return (-1);
2048 	}
2049 
2050 	/*
2051 	 * Search exapnders with the parent index matching with
2052 	 * ESC Electronics element index.
2053 	 * Note the index used here is a global index across
2054 	 * SES elements.
2055 	 */
2056 	for (xsnp = topo_list_next(&cp->sec_nodes); xsnp != NULL;
2057 	    xsnp = topo_list_next(xsnp)) {
2058 		if (xsnp->sen_type == SES_ET_SAS_EXPANDER) {
2059 			/*
2060 			 * get the parent ESC controller.
2061 			 */
2062 			psprops = ses_node_props(xsnp->sen_node);
2063 			if (nvlist_lookup_uint64(psprops,
2064 			    SES_PROP_STATUS_CODE, &psstatus) == 0) {
2065 				if (psstatus == SES_ESC_NOT_INSTALLED) {
2066 					/*
2067 					 * Not installed.
2068 					 * Don't create a ndoe.
2069 					 */
2070 					continue;
2071 				}
2072 			} else {
2073 				/*
2074 				 * The element should have status code.
2075 				 * If not there is no way to find
2076 				 * out if the expander element exist or
2077 				 * not.
2078 				 */
2079 				continue;
2080 			}
2081 
2082 			/* Get the physical parent index to compare. */
2083 			if (nvlist_lookup_uint64(psprops,
2084 			    LIBSES_PROP_PHYS_PARENT, &psindex) == 0) {
2085 				if (index == psindex) {
2086 		/* indentation moved forward */
2087 		/*
2088 		 * Handle basic node information of SAS expander
2089 		 * element - binding to parent node and
2090 		 * allocating FMRI...
2091 		 */
2092 		if (ses_create_generic(sdp, xsnp, pnode, pnode, SASEXPANDER,
2093 		    "SAS-EXPANDER", &exptn) != 0)
2094 			continue;
2095 		/*
2096 		 * Now handle SAS expander unique portion of node creation.
2097 		 * The max nubmer of the phy count is 256 since SES-2
2098 		 * defines as 1 byte field.  The cidxlist has the same
2099 		 * number of elements.
2100 		 *
2101 		 * We use size 64 array to store the connectors.
2102 		 * Typically a connectors associated with 4 phys so that
2103 		 * matches with the max number of connecters associated
2104 		 * with an expander.
2105 		 * The phy count goes up to 38 for Sun supported
2106 		 * JBOD.
2107 		 */
2108 		(void) memset(cidxlist, 0, sizeof (int64_t) * 64);
2109 		if (ses_set_expander_props(sdp, xsnp, pnode, exptn, &phycount,
2110 		    cidxlist) != 0) {
2111 			/*
2112 			 * error on getting specific prop failed.
2113 			 * continue on.  Note that the node is
2114 			 * left bound.
2115 			 */
2116 			continue;
2117 		}
2118 
2119 		/*
2120 		 * count represetns the number of connectors discovered so far.
2121 		 */
2122 		count = 0;
2123 		(void) memset(connectors, 0,
2124 		    sizeof (sas_connector_phy_data_t) * 64);
2125 		for (i = 0; i < phycount; i++) {
2126 			if (cidxlist[i] != -1) {
2127 				/* connector index is valid. */
2128 				for (j = 0; j < count; j++) {
2129 					if (connectors[j].scpd_index ==
2130 					    cidxlist[i]) {
2131 						/*
2132 						 * Just update phy mask.
2133 						 * The postion for connector
2134 						 * index lists(cidxlist index)
2135 						 * is set.
2136 						 */
2137 						connectors[j].scpd_pm =
2138 						    connectors[j].scpd_pm |
2139 						    (1ULL << i);
2140 						break;
2141 					}
2142 				}
2143 				/*
2144 				 * If j and count matche a  new connector
2145 				 * index is found.
2146 				 */
2147 				if (j == count) {
2148 					/* add a new index and phy mask. */
2149 					connectors[count].scpd_index =
2150 					    cidxlist[i];
2151 					connectors[count].scpd_pm =
2152 					    connectors[count].scpd_pm |
2153 					    (1ULL << i);
2154 					count++;
2155 				}
2156 			}
2157 		}
2158 
2159 		/*
2160 		 * create range for the connector nodes.
2161 		 * The class index of the ses connector element
2162 		 * is set as the instance nubmer for the node.
2163 		 * Even though one expander may not have all connectors
2164 		 * are associated with we are creating the range with
2165 		 * max possible instance number.
2166 		 */
2167 		found = B_FALSE;
2168 		max = 0;
2169 		for (consnp = topo_list_next(&cp->sec_nodes);
2170 		    consnp != NULL; consnp = topo_list_next(consnp)) {
2171 			if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
2172 				psprops = ses_node_props(consnp->sen_node);
2173 				found = B_TRUE;
2174 				if (consnp->sen_instance > max)
2175 					max = consnp->sen_instance;
2176 			}
2177 		}
2178 
2179 		/*
2180 		 * No SAS connector found nothing to process.
2181 		 */
2182 		if (!found)
2183 			return (0);
2184 
2185 		if (dorange && topo_node_range_create(mod, exptn,
2186 		    RECEPTACLE, 0, max) != 0) {
2187 			topo_mod_dprintf(mod,
2188 			    "topo_node_create_range() failed: %s",
2189 			    topo_mod_errmsg(mod));
2190 			return (-1);
2191 		}
2192 
2193 		/* search matching connector element using the index. */
2194 		for (i = 0; i < count; i++) {
2195 			found = B_FALSE;
2196 			for (consnp = topo_list_next(&cp->sec_nodes);
2197 			    consnp != NULL; consnp = topo_list_next(consnp)) {
2198 				if (consnp->sen_type == SES_ET_SAS_CONNECTOR) {
2199 					psprops = ses_node_props(
2200 					    consnp->sen_node);
2201 					/*
2202 					 * Get the physical parent index to
2203 					 * compare.
2204 					 * The connector elements are children
2205 					 * of ESC Electronics element even
2206 					 * though we enumerate them under
2207 					 * an expander in libtopo.
2208 					 */
2209 					if (nvlist_lookup_uint64(psprops,
2210 					    SES_PROP_ELEMENT_ONLY_INDEX,
2211 					    &conindex) == 0) {
2212 						if (conindex ==
2213 						    connectors[i].scpd_index) {
2214 							found = B_TRUE;
2215 							break;
2216 						}
2217 					}
2218 				}
2219 			}
2220 
2221 			/* now create a libtopo node. */
2222 			if (found) {
2223 				/* Create generic props. */
2224 				if (ses_create_generic(sdp, consnp, exptn,
2225 				    topo_node_parent(exptn),
2226 				    RECEPTACLE, "RECEPTACLE", &contn) !=
2227 				    0) {
2228 					continue;
2229 				}
2230 				/* Create connector specific props. */
2231 				if (ses_set_connector_props(sdp, consnp,
2232 				    contn, connectors[i].scpd_pm) != 0) {
2233 					continue;
2234 				}
2235 				if (topo_method_register(mod, contn,
2236 				    ses_recep_methods) != 0) {
2237 					topo_mod_dprintf(mod,
2238 					    "topo_method_register() failed: "
2239 					    "%s",
2240 					    topo_mod_errmsg(mod));
2241 					continue;
2242 				}
2243 
2244 			}
2245 		}
2246 		/* end indentation change */
2247 				}
2248 			}
2249 		}
2250 	}
2251 
2252 	return (0);
2253 }
2254 
2255 /*
2256  * Instantiate any protocol specific portion of a node.
2257  */
2258 /*ARGSUSED*/
2259 static int
2260 ses_create_protocol_specific(ses_enum_data_t *sdp, ses_enum_node_t *snp,
2261     tnode_t *pnode, uint64_t type, ses_enum_chassis_t *cp,
2262     boolean_t dorange)
2263 {
2264 
2265 	if (type == SES_ET_ESC_ELECTRONICS) {
2266 		/* create SAS specific children(expanders and connectors. */
2267 		return (ses_create_esc_sasspecific(sdp, snp, pnode, cp,
2268 		    dorange));
2269 	}
2270 
2271 	return (0);
2272 }
2273 
2274 /*
2275  * Instantiate any children of a given type.
2276  */
2277 static int
2278 ses_create_children(ses_enum_data_t *sdp, tnode_t *pnode, uint64_t type,
2279     const char *nodename, const char *defaultlabel, ses_enum_chassis_t *cp,
2280     boolean_t dorange)
2281 {
2282 	topo_mod_t *mod = sdp->sed_mod;
2283 	boolean_t found;
2284 	uint64_t max;
2285 	ses_enum_node_t *snp;
2286 	tnode_t	*tn;
2287 
2288 	/*
2289 	 * First go through and count how many matching nodes we have.
2290 	 */
2291 	max = 0;
2292 	found = B_FALSE;
2293 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2294 	    snp = topo_list_next(snp)) {
2295 		if (snp->sen_type == type) {
2296 			found = B_TRUE;
2297 			if (snp->sen_instance > max)
2298 				max = snp->sen_instance;
2299 		}
2300 	}
2301 
2302 	/*
2303 	 * No enclosure should export both DEVICE and ARRAY_DEVICE elements.
2304 	 * Since we map both of these to 'disk', if an enclosure does this, we
2305 	 * just ignore the array elements.
2306 	 */
2307 	if (!found ||
2308 	    (type == SES_ET_ARRAY_DEVICE && cp->sec_hasdev))
2309 		return (0);
2310 
2311 	topo_mod_dprintf(mod, "%s: creating %llu %s nodes",
2312 	    cp->sec_csn, max + 1, nodename);
2313 
2314 	if (dorange && topo_node_range_create(mod, pnode,
2315 	    nodename, 0, max) != 0) {
2316 		topo_mod_dprintf(mod,
2317 		    "topo_node_create_range() failed: %s",
2318 		    topo_mod_errmsg(mod));
2319 		return (-1);
2320 	}
2321 
2322 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2323 	    snp = topo_list_next(snp)) {
2324 		if (snp->sen_type == type) {
2325 			/*
2326 			 * With flat layout of ses nodes there is no
2327 			 * way to find out the direct FRU for a node.
2328 			 * Passing NULL for fru topo node.  Note that
2329 			 * ses_create_children_from_phys_tree() provides
2330 			 * the actual direct FRU for a node.
2331 			 */
2332 			if (ses_create_generic(sdp, snp, pnode, NULL,
2333 			    nodename, defaultlabel, &tn) != 0)
2334 				return (-1);
2335 			/*
2336 			 * For some SES element there may be protocol specific
2337 			 * information to process.   Here we are processing
2338 			 * the association between enclosure controller and
2339 			 * SAS expanders.
2340 			 */
2341 			if (type == SES_ET_ESC_ELECTRONICS) {
2342 				/* create SAS expander node */
2343 				if (ses_create_protocol_specific(sdp, snp,
2344 				    tn, type, cp, dorange) != 0) {
2345 					return (-1);
2346 				}
2347 			}
2348 
2349 		}
2350 	}
2351 
2352 	return (0);
2353 }
2354 
2355 /*
2356  * Instantiate a new subchassis instance in the topology.
2357  */
2358 static int
2359 ses_create_subchassis(ses_enum_data_t *sdp, tnode_t *pnode,
2360     ses_enum_chassis_t *scp)
2361 {
2362 	topo_mod_t *mod = sdp->sed_mod;
2363 	tnode_t *tn;
2364 	nvlist_t *props;
2365 	nvlist_t *auth = NULL, *fmri = NULL;
2366 	uint64_t instance = scp->sec_instance;
2367 	char *desc;
2368 	char *clean_label = NULL, label[128];
2369 	char **paths;
2370 	int i, err;
2371 	ses_enum_target_t *stp;
2372 	int ret = -1;
2373 
2374 	/*
2375 	 * Copy authority information from parent enclosure node
2376 	 */
2377 	if ((auth = topo_mod_auth(mod, pnode)) == NULL)
2378 		goto error;
2379 
2380 	/*
2381 	 * Record the subchassis serial number in the FMRI.
2382 	 * For now, we assume that logical id is the subchassis serial number.
2383 	 * If this assumption changes in future, then the following
2384 	 * piece of code will need to be updated via an RFE.
2385 	 */
2386 	if ((fmri = topo_mod_hcfmri(mod, pnode, FM_HC_SCHEME_VERSION,
2387 	    SUBCHASSIS, (topo_instance_t)instance, NULL, auth, NULL, NULL,
2388 	    NULL)) == NULL) {
2389 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
2390 		    topo_mod_errmsg(mod));
2391 		goto error;
2392 	}
2393 
2394 	if ((tn = topo_node_bind(mod, pnode, SUBCHASSIS,
2395 	    instance, fmri)) == NULL) {
2396 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
2397 		    topo_mod_errmsg(mod));
2398 		goto error;
2399 	}
2400 
2401 	props = ses_node_props(scp->sec_enclosure);
2402 
2403 	/*
2404 	 * Look for the subchassis label in the following order:
2405 	 *	<ses-description>
2406 	 *	<ses-class-description> <instance>
2407 	 *	<default-type-label> <instance>
2408 	 *
2409 	 * For subchassis, the default label is "SUBCHASSIS"
2410 	 */
2411 	if (nvlist_lookup_string(props, SES_PROP_DESCRIPTION, &desc) != 0 ||
2412 	    desc[0] == '\0') {
2413 		if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION,
2414 		    &desc) == 0 && desc[0] != '\0')
2415 			(void) snprintf(label, sizeof (label), "%s %llu", desc,
2416 			    instance);
2417 		else
2418 			(void) snprintf(label, sizeof (label),
2419 			    "SUBCHASSIS %llu", instance);
2420 		desc = label;
2421 	}
2422 
2423 	if ((clean_label = topo_mod_clean_str(mod, desc)) == NULL)
2424 		goto error;
2425 
2426 	if (topo_prop_set_string(tn, TOPO_PGROUP_PROTOCOL, TOPO_PROP_LABEL,
2427 	    TOPO_PROP_MUTABLE, clean_label, &err) < 0)
2428 		goto error;
2429 
2430 	if (ses_set_standard_props(mod, NULL, tn, NULL,
2431 	    ses_node_id(scp->sec_enclosure), scp->sec_target->set_devpath) != 0)
2432 		goto error;
2433 
2434 	/*
2435 	 * Set the 'chassis-type' property for this subchassis.  This is either
2436 	 * 'ses-class-description' or 'subchassis'.
2437 	 */
2438 	if (nvlist_lookup_string(props, SES_PROP_CLASS_DESCRIPTION, &desc) != 0)
2439 		desc = "subchassis";
2440 
2441 	if (topo_prop_set_string(tn, TOPO_PGROUP_SES,
2442 	    TOPO_PROP_CHASSIS_TYPE, TOPO_PROP_IMMUTABLE, desc, &err) != 0) {
2443 		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
2444 		    TOPO_PROP_CHASSIS_TYPE, topo_strerror(err));
2445 		goto error;
2446 	}
2447 
2448 	/*
2449 	 * For enclosures, we want to include all possible targets (for upgrade
2450 	 * purposes).
2451 	 */
2452 	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
2453 	    stp = topo_list_next(stp), i++)
2454 		;
2455 
2456 	verify(i != 0);
2457 	paths = alloca(i * sizeof (char *));
2458 
2459 	for (i = 0, stp = topo_list_next(&scp->sec_targets); stp != NULL;
2460 	    stp = topo_list_next(stp), i++)
2461 		paths[i] = stp->set_devpath;
2462 
2463 	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
2464 	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
2465 	    i, &err) != 0) {
2466 		topo_mod_dprintf(mod, "failed to create property %s: %s\n",
2467 		    TOPO_PROP_PATHS, topo_strerror(err));
2468 		goto error;
2469 	}
2470 
2471 	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
2472 		topo_mod_dprintf(mod, "topo_method_register() failed: %s",
2473 		    topo_mod_errmsg(mod));
2474 		goto error;
2475 	}
2476 
2477 	/*
2478 	 * Create the nodes for controllers and bays.
2479 	 */
2480 	if (ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
2481 	    CONTROLLER, "CONTROLLER", scp, B_TRUE) != 0 ||
2482 	    ses_create_children(sdp, tn, SES_ET_DEVICE,
2483 	    BAY, "BAY", scp, B_TRUE) != 0 ||
2484 	    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
2485 	    BAY, "BAY", scp, B_TRUE) != 0)
2486 		goto error;
2487 
2488 	ret = 0;
2489 
2490 error:
2491 	nvlist_free(auth);
2492 	nvlist_free(fmri);
2493 	topo_mod_strfree(mod, clean_label);
2494 	return (ret);
2495 }
2496 
2497 /*
2498  * Function we use to insert a node.
2499  */
2500 static int
2501 ses_phys_tree_insert(topo_mod_t *mod, ses_phys_tree_t **sproot,
2502     ses_phys_tree_t *child)
2503 {
2504 	uint64_t ppindex, eindex, pindex;
2505 	ses_phys_tree_t *node_ptr;
2506 	int ret = 0;
2507 
2508 	assert(sproot != NULL);
2509 	assert(child != NULL);
2510 
2511 	if (*sproot == NULL) {
2512 		*sproot = child;
2513 		return (0);
2514 	}
2515 
2516 	pindex = child->spt_pindex;
2517 	ppindex = (*sproot)->spt_pindex;
2518 	eindex = (*sproot)->spt_eonlyindex;
2519 
2520 	/*
2521 	 * If the element only index of the root is same as the physical
2522 	 * parent index of a node to be added, add the node as a child of
2523 	 * the current root.
2524 	 */
2525 	if (eindex == pindex) {
2526 		(void) ses_phys_tree_insert(mod, &(*sproot)->spt_child, child);
2527 		child->spt_parent = *sproot;
2528 	} else if (ppindex == pindex) {
2529 		/*
2530 		 * if the physical parent of the current root and the child
2531 		 * is same, then this should be a sibling node.
2532 		 * Siblings can be different element types and arrange
2533 		 * them by group.
2534 		 */
2535 		if ((*sproot)->spt_senumnode->sen_type ==
2536 		    child->spt_senumnode->sen_type) {
2537 			child->spt_sibling = *sproot;
2538 			*sproot = child;
2539 		} else {
2540 			/* add a node in front of matching element type. */
2541 			node_ptr = *sproot;
2542 			while (node_ptr->spt_sibling != NULL) {
2543 				if (node_ptr->spt_sibling->
2544 				    spt_senumnode->sen_type ==
2545 				    child->spt_senumnode->sen_type) {
2546 					child->spt_sibling =
2547 					    node_ptr->spt_sibling;
2548 					node_ptr->spt_sibling = child;
2549 					break;
2550 				}
2551 				node_ptr = node_ptr->spt_sibling;
2552 			}
2553 			/* no matching.  Add the child at the end. */
2554 			if (node_ptr->spt_sibling == NULL) {
2555 				node_ptr->spt_sibling = child;
2556 			}
2557 		}
2558 		child->spt_parent = (*sproot)->spt_parent;
2559 	} else {
2560 		/*
2561 		 * The root and the node is not directly related.
2562 		 * Try to insert to the child sub-tree first and then try to
2563 		 * insert to the sibling sub-trees.  If fails for both
2564 		 * the caller will retry insertion later.
2565 		 */
2566 		if ((*sproot)->spt_child) {
2567 			ret = ses_phys_tree_insert(mod, &(*sproot)->spt_child,
2568 			    child);
2569 		}
2570 		if ((*sproot)->spt_child == NULL || ret != 0) {
2571 			if ((*sproot)->spt_sibling) {
2572 				ret = ses_phys_tree_insert(mod,
2573 				    &(*sproot)->spt_sibling, child);
2574 			} else {
2575 				ret = 1;
2576 			}
2577 		}
2578 		return (ret);
2579 	}
2580 	return (0);
2581 }
2582 
2583 /*
2584  * Construct tree view of ses elements through parent phyiscal element index.
2585  * The root of tree is already constructed using the enclosure element.
2586  */
2587 static int
2588 ses_construct_phys_tree(ses_enum_data_t *sdp, ses_enum_chassis_t *cp,
2589     ses_phys_tree_t *sproot)
2590 {
2591 	ses_enum_node_t *snp;
2592 	ses_phys_tree_t	*child;
2593 	ses_phys_tree_t	*u_watch = NULL;
2594 	ses_phys_tree_t	*u_head = NULL;
2595 	ses_phys_tree_t	*u_tail = NULL;
2596 	int u_inserted = 0, u_left = 0;
2597 	nvlist_t *props;
2598 	topo_mod_t *mod = sdp->sed_mod;
2599 
2600 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2601 	    snp = topo_list_next(snp)) {
2602 		if ((child = topo_mod_zalloc(mod,
2603 		    sizeof (ses_phys_tree_t))) == NULL) {
2604 			topo_mod_dprintf(mod,
2605 			    "failed to allocate root.");
2606 			return (-1);
2607 		}
2608 		child->spt_snode = snp->sen_node;
2609 		props = ses_node_props(snp->sen_node);
2610 		if (nvlist_lookup_uint64(props,
2611 		    LIBSES_PROP_PHYS_PARENT, &child->spt_pindex) != 0) {
2612 			/*
2613 			 * the prop should exist. continue to see if
2614 			 * we can build a partial tree with other elements.
2615 			 */
2616 			topo_mod_dprintf(mod,
2617 			    "ses_construct_phys_tree(): Failed to find prop %s "
2618 			    "on ses element type %d and instance %d "
2619 			    "(CSN %s).", LIBSES_PROP_PHYS_PARENT,
2620 			    snp->sen_type, snp->sen_instance, cp->sec_csn);
2621 			topo_mod_free(mod, child, sizeof (ses_phys_tree_t));
2622 			continue;
2623 		} else {
2624 			if (nvlist_lookup_boolean_value(props,
2625 			    LIBSES_PROP_FRU, &child->spt_isfru) != 0) {
2626 				topo_mod_dprintf(mod,
2627 				    "ses_construct_phys_tree(): Failed to "
2628 				    "find prop %s on ses element type %d "
2629 				    "and instance %d (CSN %s).",
2630 				    LIBSES_PROP_FRU,
2631 				    snp->sen_type, snp->sen_instance,
2632 				    cp->sec_csn);
2633 				/*
2634 				 * Ignore if the prop doesn't exist.
2635 				 * Note that the enclosure itself should be
2636 				 * a FRU so if no FRU found the enclosure FRU
2637 				 * can be a direct FRU.
2638 				 */
2639 			}
2640 			verify(nvlist_lookup_uint64(props,
2641 			    SES_PROP_ELEMENT_ONLY_INDEX,
2642 			    &child->spt_eonlyindex) == 0);
2643 			verify(nvlist_lookup_uint64(props,
2644 			    SES_PROP_ELEMENT_CLASS_INDEX,
2645 			    &child->spt_cindex) == 0);
2646 		}
2647 		child->spt_senumnode = snp;
2648 		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
2649 			/* collect unresolved element to process later. */
2650 			if (u_head == NULL) {
2651 				u_head = child;
2652 				u_tail = child;
2653 			} else {
2654 				child->spt_sibling = u_head;
2655 				u_head = child;
2656 			}
2657 		}
2658 	}
2659 
2660 	/*
2661 	 * The parent of a child node may not be inserted yet.
2662 	 * Trying to insert the child until no child is left or
2663 	 * no child is not added further.  For the latter
2664 	 * the hierarchical relationship between elements
2665 	 * should be checked through SUNW,FRUID page.
2666 	 * u_watch is a watch dog to check the prgress of unresolved
2667 	 * node.
2668 	 */
2669 	u_watch = u_tail;
2670 	while (u_head) {
2671 		child = u_head;
2672 		u_head = u_head->spt_sibling;
2673 		if (u_head == NULL)
2674 			u_tail = NULL;
2675 		child->spt_sibling = NULL;
2676 		if (ses_phys_tree_insert(mod, &sproot, child) != 0) {
2677 			u_tail->spt_sibling = child;
2678 			u_tail = child;
2679 			if (child == u_watch) {
2680 				/*
2681 				 * We just scanned one round for the
2682 				 * unresolved list. Check to see whether we
2683 				 * have nodes inserted, if none, we should
2684 				 * break in case of an indefinite loop.
2685 				 */
2686 				if (u_inserted == 0) {
2687 					/*
2688 					 * Indicate there is unhandled node.
2689 					 * Chain free the whole unsolved
2690 					 * list here.
2691 					 */
2692 					u_left++;
2693 					break;
2694 				} else {
2695 					u_inserted = 0;
2696 					u_watch = u_tail;
2697 				}
2698 			}
2699 		} else {
2700 			/*
2701 			 * We just inserted one rpnode, increment the
2702 			 * unsolved_inserted counter. We will utilize this
2703 			 * counter to detect an indefinite insertion loop.
2704 			 */
2705 			u_inserted++;
2706 			if (child == u_watch) {
2707 				/*
2708 				 * watch dog node itself is inserted.
2709 				 * Set it to the tail and refresh the watching.
2710 				 */
2711 				u_watch = u_tail;
2712 				u_inserted = 0;
2713 				u_left = 0;
2714 			}
2715 		}
2716 	}
2717 
2718 	/* check if there is left out unresolved nodes. */
2719 	if (u_left) {
2720 		topo_mod_dprintf(mod, "ses_construct_phys_tree(): "
2721 		    "Failed to construct physical view of the following "
2722 		    "ses elements of Chassis CSN %s.", cp->sec_csn);
2723 		while (u_head) {
2724 			u_tail = u_head->spt_sibling;
2725 			topo_mod_dprintf(mod,
2726 			    "\telement type (%d) and instance (%d)",
2727 			    u_head->spt_senumnode->sen_type,
2728 			    u_head->spt_senumnode->sen_instance);
2729 			topo_mod_free(mod, u_head, sizeof (ses_phys_tree_t));
2730 			u_head = u_tail;
2731 		}
2732 		return (-1);
2733 	}
2734 
2735 	return (0);
2736 }
2737 
2738 /*
2739  * Free the whole phys tree.
2740  */
2741 static void ses_phys_tree_free(topo_mod_t *mod, ses_phys_tree_t *sproot)
2742 {
2743 	if (sproot == NULL)
2744 		return;
2745 
2746 	/* Free child tree. */
2747 	if (sproot->spt_child) {
2748 		ses_phys_tree_free(mod, sproot->spt_child);
2749 	}
2750 
2751 	/* Free sibling trees. */
2752 	if (sproot->spt_sibling) {
2753 		ses_phys_tree_free(mod, sproot->spt_sibling);
2754 	}
2755 
2756 	/* Free root node itself. */
2757 	topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
2758 }
2759 
2760 /*
2761  * Parses phys_enum_type table to get the index of the given type.
2762  */
2763 static boolean_t
2764 is_type_enumerated(ses_phys_tree_t *node, int *index)
2765 {
2766 	int i;
2767 
2768 	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
2769 		if (node->spt_senumnode->sen_type ==
2770 		    phys_enum_type_list[i].pet_type) {
2771 			*index = i;
2772 			return (B_TRUE);
2773 		}
2774 	}
2775 	return (B_FALSE);
2776 }
2777 
2778 /*
2779  * Recusrive routine for top-down enumeration of the tree.
2780  */
2781 static int
2782 ses_enumerate_node(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp,
2783     ses_phys_tree_t *parent, int mrange[])
2784 {
2785 	topo_mod_t *mod = sdp->sed_mod;
2786 	ses_phys_tree_t *child = NULL;
2787 	int i, ret = 0, ret_ch;
2788 	uint64_t prevtype = SES_ET_UNSPECIFIED;
2789 	ses_phys_tree_t *dirfru = NULL;
2790 	tnode_t *tn = NULL, *frutn = NULL;
2791 
2792 	if (parent == NULL) {
2793 		return (0);
2794 	}
2795 
2796 	for (child = parent->spt_child; child != NULL;
2797 	    child = child->spt_sibling) {
2798 		if (is_type_enumerated(child, &i)) {
2799 			if (prevtype != phys_enum_type_list[i].pet_type) {
2800 				/* check if range needs to be created. */
2801 				if (phys_enum_type_list[i].pet_dorange &&
2802 				    topo_node_range_create(mod, pnode,
2803 				    phys_enum_type_list[i].pet_nodename, 0,
2804 				    mrange[i]) != 0) {
2805 					topo_mod_dprintf(mod,
2806 					    "topo_node_create_range() failed: "
2807 					    "%s", topo_mod_errmsg(mod));
2808 					return (-1);
2809 				}
2810 				prevtype = phys_enum_type_list[i].pet_type;
2811 			}
2812 
2813 			if (!(child->spt_isfru)) {
2814 				for (dirfru = parent; dirfru != NULL;
2815 				    dirfru = dirfru->spt_parent) {
2816 					if (dirfru->spt_isfru) {
2817 						break;
2818 					}
2819 				}
2820 				/* found direct FRU node. */
2821 				if (dirfru) {
2822 					frutn = dirfru->spt_tnode;
2823 				} else {
2824 					frutn = NULL;
2825 				}
2826 			} else {
2827 				frutn = NULL;
2828 			}
2829 
2830 			if (ses_create_generic(sdp, child->spt_senumnode,
2831 			    pnode, frutn, phys_enum_type_list[i].pet_nodename,
2832 			    phys_enum_type_list[i].pet_defaultlabel, &tn) != 0)
2833 				return (-1);
2834 
2835 			child->spt_tnode = tn;
2836 			/*
2837 			 * For some SES element there may be protocol specific
2838 			 * information to process.   Here we are processing
2839 			 * the association between enclosure controller and
2840 			 * SAS expanders.
2841 			 */
2842 			if (phys_enum_type_list[i].pet_type ==
2843 			    SES_ET_ESC_ELECTRONICS) {
2844 				/* create SAS expander node */
2845 				if (ses_create_protocol_specific(sdp,
2846 				    child->spt_senumnode, tn,
2847 				    phys_enum_type_list[i].pet_type,
2848 				    cp, phys_enum_type_list[i].pet_dorange) !=
2849 				    0) {
2850 					return (-1);
2851 				}
2852 			}
2853 		} else {
2854 			continue;
2855 		}
2856 		ret_ch = ses_enumerate_node(sdp, tn, cp, child, mrange);
2857 		if (ret_ch)
2858 			ret = ret_ch; /* there was an error and set the ret. */
2859 	}
2860 
2861 	return (ret);
2862 }
2863 
2864 /*
2865  * Instantiate types of nodes that are specified in the hierarchy
2866  * element type list.
2867  */
2868 static int
2869 ses_create_children_from_phys_tree(ses_enum_data_t *sdp, tnode_t *pnode,
2870     ses_enum_chassis_t *cp, ses_phys_tree_t *phys_tree)
2871 {
2872 	topo_mod_t *mod = sdp->sed_mod;
2873 	int mrange[N_PHYS_ENUM_TYPES] = { 0 };
2874 	ses_enum_node_t *snp;
2875 	int i, ret;
2876 
2877 	/*
2878 	 * First get max range for each type of element to be enumerated.
2879 	 */
2880 	for (i = 0; i < N_PHYS_ENUM_TYPES; i++) {
2881 		if (phys_enum_type_list[i].pet_dorange) {
2882 			for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2883 			    snp = topo_list_next(snp)) {
2884 				if (snp->sen_type ==
2885 				    phys_enum_type_list[i].pet_type) {
2886 					if (snp->sen_instance > mrange[i])
2887 						mrange[i] =
2888 						    snp->sen_instance;
2889 				}
2890 			}
2891 		}
2892 	}
2893 
2894 	topo_mod_dprintf(mod, "%s: creating nodes from FRU hierarchy tree.",
2895 	    cp->sec_csn);
2896 
2897 	if ((ret = ses_enumerate_node(sdp, pnode, cp, phys_tree, mrange)) !=
2898 	    0) {
2899 		topo_mod_dprintf(mod,
2900 		    "ses_create_children_from_phys_tree() failed: ");
2901 		return (ret);
2902 	}
2903 
2904 	return (0);
2905 }
2906 
2907 /*
2908  * Instantiate a new chassis instance in the topology.
2909  */
2910 static int
2911 ses_create_chassis(ses_enum_data_t *sdp, tnode_t *pnode, ses_enum_chassis_t *cp)
2912 {
2913 	topo_mod_t *mod = sdp->sed_mod;
2914 	nvlist_t *props;
2915 	char *raw_manufacturer, *raw_model, *raw_revision;
2916 	char *manufacturer = NULL, *model = NULL, *product = NULL;
2917 	char *revision = NULL;
2918 	char *serial;
2919 	char **paths;
2920 	size_t prodlen;
2921 	tnode_t *tn;
2922 	nvlist_t *fmri = NULL, *auth = NULL;
2923 	int ret = -1;
2924 	ses_enum_node_t *snp;
2925 	ses_enum_target_t *stp;
2926 	ses_enum_chassis_t *scp;
2927 	int i, err;
2928 	uint64_t sc_count = 0, pindex;
2929 	ses_phys_tree_t	*sproot = NULL;
2930 	hrtime_t start;
2931 	hrtime_t end;
2932 	double duration;
2933 
2934 	/*
2935 	 * Ignore any internal enclosures.
2936 	 */
2937 	if (cp->sec_internal)
2938 		return (0);
2939 
2940 	/*
2941 	 * Check to see if there are any devices presennt in the chassis.  If
2942 	 * not, ignore the chassis alltogether.  This is most useful for
2943 	 * ignoring internal HBAs that present a SES target but don't actually
2944 	 * manage any of the devices.
2945 	 */
2946 	for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
2947 	    snp = topo_list_next(snp)) {
2948 		if (snp->sen_type == SES_ET_DEVICE ||
2949 		    snp->sen_type == SES_ET_ARRAY_DEVICE)
2950 			break;
2951 	}
2952 
2953 	if (snp == NULL)
2954 		return (0);
2955 
2956 	props = ses_node_props(cp->sec_enclosure);
2957 
2958 	/*
2959 	 * We use the following property mappings:
2960 	 *
2961 	 * manufacturer		vendor-id
2962 	 * model		product-id
2963 	 * serial-number	libses-chassis-serial
2964 	 */
2965 	verify(nvlist_lookup_string(props, SES_EN_PROP_VID,
2966 	    &raw_manufacturer) == 0);
2967 	verify(nvlist_lookup_string(props, SES_EN_PROP_PID, &raw_model) == 0);
2968 	verify(nvlist_lookup_string(props, SES_EN_PROP_REV,
2969 	    &raw_revision) == 0);
2970 	verify(nvlist_lookup_string(props, LIBSES_EN_PROP_CSN, &serial) == 0);
2971 
2972 	/*
2973 	 * To construct the authority information, we 'clean' each string by
2974 	 * removing any offensive characters and trimmming whitespace.  For the
2975 	 * 'product-id', we use a concatenation of 'manufacturer-model'.  We
2976 	 * also take the numerical serial number and convert it to a string.
2977 	 */
2978 	if ((manufacturer = topo_mod_clean_str(mod, raw_manufacturer)) ==
2979 	    NULL || (model = topo_mod_clean_str(mod, raw_model)) == NULL ||
2980 	    (revision = topo_mod_clean_str(mod, raw_revision)) == NULL) {
2981 		goto error;
2982 	}
2983 
2984 	prodlen = strlen(manufacturer) + strlen(model) + 2;
2985 	if ((product = topo_mod_alloc(mod, prodlen)) == NULL)
2986 		goto error;
2987 
2988 	(void) snprintf(product, prodlen, "%s-%s", manufacturer, model);
2989 
2990 	/*
2991 	 * Construct the topo node and bind it to our parent.
2992 	 */
2993 	if (topo_mod_nvalloc(mod, &auth, NV_UNIQUE_NAME) != 0)
2994 		goto error;
2995 
2996 	if (nvlist_add_string(auth, FM_FMRI_AUTH_PRODUCT, product) != 0 ||
2997 	    nvlist_add_string(auth, FM_FMRI_AUTH_CHASSIS, serial) != 0) {
2998 		(void) topo_mod_seterrno(mod, EMOD_NVL_INVAL);
2999 		goto error;
3000 	}
3001 
3002 	/*
3003 	 * We pass NULL for the parent FMRI because there is no resource
3004 	 * associated with it.  For the toplevel enclosure, we leave the
3005 	 * serial/part/revision portions empty, which are reserved for
3006 	 * individual components within the chassis.
3007 	 */
3008 	if ((fmri = topo_mod_hcfmri(mod, NULL, FM_HC_SCHEME_VERSION,
3009 	    SES_ENCLOSURE, cp->sec_instance, NULL, auth,
3010 	    model, revision, serial)) == NULL) {
3011 		topo_mod_dprintf(mod, "topo_mod_hcfmri() failed: %s",
3012 		    topo_mod_errmsg(mod));
3013 		goto error;
3014 	}
3015 
3016 	if ((tn = topo_node_bind(mod, pnode, SES_ENCLOSURE,
3017 	    cp->sec_instance, fmri)) == NULL) {
3018 		topo_mod_dprintf(mod, "topo_node_bind() failed: %s",
3019 		    topo_mod_errmsg(mod));
3020 		goto error;
3021 	}
3022 
3023 	if (topo_method_register(mod, tn, ses_enclosure_methods) != 0) {
3024 		topo_mod_dprintf(mod,
3025 		    "topo_method_register() failed: %s",
3026 		    topo_mod_errmsg(mod));
3027 		goto error;
3028 	}
3029 
3030 	if (ses_set_standard_props(mod, NULL, tn, auth,
3031 	    ses_node_id(cp->sec_enclosure), cp->sec_target->set_devpath) != 0)
3032 		goto error;
3033 
3034 	/*
3035 	 * For enclosures, we want to include all possible targets (for upgrade
3036 	 * purposes).
3037 	 */
3038 	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
3039 	    stp = topo_list_next(stp), i++)
3040 		;
3041 
3042 	verify(i != 0);
3043 	paths = alloca(i * sizeof (char *));
3044 
3045 	for (i = 0, stp = topo_list_next(&cp->sec_targets); stp != NULL;
3046 	    stp = topo_list_next(stp), i++)
3047 		paths[i] = stp->set_devpath;
3048 
3049 
3050 	if (topo_prop_set_string_array(tn, TOPO_PGROUP_SES,
3051 	    TOPO_PROP_PATHS, TOPO_PROP_IMMUTABLE, (const char **)paths,
3052 	    i, &err) != 0) {
3053 		topo_mod_dprintf(mod,
3054 		    "failed to create property %s: %s\n",
3055 		    TOPO_PROP_PATHS, topo_strerror(err));
3056 		goto error;
3057 	}
3058 
3059 	if (nvlist_lookup_uint64(props,
3060 	    LIBSES_PROP_PHYS_PARENT, &pindex) == 0) {
3061 		start = gethrtime(); /* to mearusre performance */
3062 		/*
3063 		 * The enclosure is supported through SUNW,FRUID.
3064 		 * Need to enumerate the nodes through hierarchical order.
3065 		 */
3066 		if ((sproot = topo_mod_zalloc(mod,
3067 		    sizeof (ses_phys_tree_t))) == NULL) {
3068 			topo_mod_dprintf(mod,
3069 			    "failed to allocate root: %s\n",
3070 			    topo_strerror(err));
3071 			goto error;
3072 		}
3073 		sproot->spt_pindex = pindex;
3074 		if (nvlist_lookup_boolean_value(props,
3075 		    LIBSES_PROP_FRU, &sproot->spt_isfru) != 0) {
3076 			topo_mod_dprintf(mod,
3077 			    "ses_create_chassis(): Failed to find prop %s "
3078 			    "on enclosure element (CSN %s).",
3079 			    LIBSES_PROP_FRU, cp->sec_csn);
3080 			/* an enclosure should be a FRU. continue to process. */
3081 			sproot->spt_isfru = B_TRUE;
3082 		}
3083 		if (nvlist_lookup_uint64(props,
3084 		    SES_PROP_ELEMENT_ONLY_INDEX,
3085 		    &sproot->spt_eonlyindex) != 0) {
3086 			topo_mod_dprintf(mod,
3087 			    "ses_create_chassis(): Failed to find prop %s "
3088 			    "on enclosure element (CSN %s).",
3089 			    LIBSES_PROP_PHYS_PARENT, cp->sec_csn);
3090 			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
3091 			goto error;
3092 		}
3093 		if (sproot->spt_pindex != sproot->spt_eonlyindex) {
3094 			topo_mod_dprintf(mod, "ses_create_chassis(): "
3095 			    "Enclosure element(CSN %s) should have "
3096 			    "itself as the parent to be the root node "
3097 			    "of FRU hierarchical tree.)", cp->sec_csn);
3098 			topo_mod_free(mod, sproot, sizeof (ses_phys_tree_t));
3099 			goto error;
3100 		} else {
3101 			sproot->spt_snode = cp->sec_enclosure;
3102 			sproot->spt_tnode = tn;
3103 			/* construct a tree. */
3104 			if (ses_construct_phys_tree(sdp, cp, sproot) != 0) {
3105 				topo_mod_dprintf(mod, "ses_create_chassis(): "
3106 				    "Failed to construct FRU hierarchical "
3107 				    "tree on enclosure (CSN %s.)",
3108 				    cp->sec_csn);
3109 			}
3110 
3111 			/* enumerate elements from the tree. */
3112 			if (ses_create_children_from_phys_tree(sdp, tn, cp,
3113 			    sproot) != 0) {
3114 				topo_mod_dprintf(mod, "ses_create_chassis(): "
3115 				    "Failed to create children topo nodes out "
3116 				    "of FRU hierarchical tree on enclosure "
3117 				    "(CSN %s).", cp->sec_csn);
3118 			}
3119 			/* destroy the phys tree. */
3120 			ses_phys_tree_free(mod, sproot);
3121 		}
3122 
3123 		end = gethrtime();
3124 		duration = end - start;
3125 		duration /= HR_SECOND;
3126 		topo_mod_dprintf(mod,
3127 		    "FRU boundary tree based enumeration: %.6f seconds",
3128 		    duration);
3129 	} else {
3130 		/*
3131 		 * Create the nodes for power supplies, fans, controllers and
3132 		 * devices.  Note that SAS exopander nodes and connector nodes
3133 		 * are handled through protocol specific processing of
3134 		 * controllers.
3135 		 */
3136 		if (ses_create_children(sdp, tn, SES_ET_POWER_SUPPLY,
3137 		    PSU, "PSU", cp, B_TRUE) != 0 ||
3138 		    ses_create_children(sdp, tn, SES_ET_COOLING,
3139 		    FAN, "FAN", cp, B_TRUE) != 0 ||
3140 		    ses_create_children(sdp, tn, SES_ET_ESC_ELECTRONICS,
3141 		    CONTROLLER, "CONTROLLER", cp, B_TRUE) != 0 ||
3142 		    ses_create_children(sdp, tn, SES_ET_DEVICE,
3143 		    BAY, "BAY", cp, B_TRUE) != 0 ||
3144 		    ses_create_children(sdp, tn, SES_ET_ARRAY_DEVICE,
3145 		    BAY, "BAY", cp, B_TRUE) != 0)
3146 			goto error;
3147 	}
3148 
3149 	if (cp->sec_maxinstance >= 0 &&
3150 	    (topo_node_range_create(mod, tn, SUBCHASSIS, 0,
3151 	    cp->sec_maxinstance) != 0)) {
3152 		topo_mod_dprintf(mod, "topo_node_create_range() failed: %s",
3153 		    topo_mod_errmsg(mod));
3154 		goto error;
3155 	}
3156 
3157 	for (scp = topo_list_next(&cp->sec_subchassis); scp != NULL;
3158 	    scp = topo_list_next(scp)) {
3159 
3160 		if (ses_create_subchassis(sdp, tn, scp) != 0)
3161 			goto error;
3162 
3163 		topo_mod_dprintf(mod, "created Subchassis node with "
3164 		    "instance %u\nand target (%s) under Chassis with CSN %s",
3165 		    scp->sec_instance, scp->sec_target->set_devpath,
3166 		    cp->sec_csn);
3167 
3168 		sc_count++;
3169 	}
3170 
3171 	topo_mod_dprintf(mod, "%s: created %llu %s nodes",
3172 	    cp->sec_csn, sc_count, SUBCHASSIS);
3173 
3174 	cp->sec_target->set_refcount++;
3175 	topo_node_setspecific(tn, cp->sec_target);
3176 
3177 	ret = 0;
3178 error:
3179 	topo_mod_strfree(mod, manufacturer);
3180 	topo_mod_strfree(mod, model);
3181 	topo_mod_strfree(mod, revision);
3182 	topo_mod_strfree(mod, product);
3183 
3184 	nvlist_free(fmri);
3185 	nvlist_free(auth);
3186 	return (ret);
3187 }
3188 
3189 /*
3190  * Create a bay node explicitly enumerated via XML.
3191  */
3192 static int
3193 ses_create_bays(ses_enum_data_t *sdp, tnode_t *pnode)
3194 {
3195 	topo_mod_t *mod = sdp->sed_mod;
3196 	ses_enum_chassis_t *cp;
3197 
3198 	/*
3199 	 * Iterate over chassis looking for an internal enclosure.  This
3200 	 * property is set via a vendor-specific plugin, and there should only
3201 	 * ever be a single internal chassis in a system.
3202 	 */
3203 	for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
3204 	    cp = topo_list_next(cp)) {
3205 		if (cp->sec_internal)
3206 			break;
3207 	}
3208 
3209 	if (cp == NULL) {
3210 		topo_mod_dprintf(mod, "failed to find internal chassis\n");
3211 		return (-1);
3212 	}
3213 
3214 	if (ses_create_children(sdp, pnode, SES_ET_DEVICE,
3215 	    BAY, "BAY", cp, B_FALSE) != 0 ||
3216 	    ses_create_children(sdp, pnode, SES_ET_ARRAY_DEVICE,
3217 	    BAY, "BAY", cp, B_FALSE) != 0)
3218 		return (-1);
3219 
3220 	return (0);
3221 }
3222 
3223 /*
3224  * Initialize chassis or subchassis.
3225  */
3226 static int
3227 ses_init_chassis(topo_mod_t *mod, ses_enum_data_t *sdp, ses_enum_chassis_t *pcp,
3228     ses_enum_chassis_t *cp, ses_node_t *np, nvlist_t *props,
3229     uint64_t subchassis, ses_chassis_type_e flags)
3230 {
3231 	boolean_t internal, ident;
3232 
3233 	assert((flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS |
3234 	    SES_DUP_CHASSIS | SES_DUP_SUBCHASSIS)) != 0);
3235 
3236 	assert(cp != NULL);
3237 	assert(np != NULL);
3238 	assert(props != NULL);
3239 
3240 	if (flags & (SES_NEW_SUBCHASSIS | SES_DUP_SUBCHASSIS))
3241 		assert(pcp != NULL);
3242 
3243 	topo_mod_dprintf(mod, "ses_init_chassis: %s: index %llu, flags (%d)",
3244 	    sdp->sed_name, subchassis, flags);
3245 
3246 	if (flags & (SES_NEW_CHASSIS | SES_NEW_SUBCHASSIS)) {
3247 
3248 		topo_mod_dprintf(mod, "new chassis/subchassis");
3249 		if (nvlist_lookup_boolean_value(props,
3250 		    LIBSES_EN_PROP_INTERNAL, &internal) == 0)
3251 			cp->sec_internal = internal;
3252 
3253 		cp->sec_enclosure = np;
3254 		cp->sec_target = sdp->sed_target;
3255 
3256 		if (flags & SES_NEW_CHASSIS) {
3257 			if (!cp->sec_internal)
3258 				cp->sec_instance = sdp->sed_instance++;
3259 			topo_list_append(&sdp->sed_chassis, cp);
3260 		} else {
3261 			if (subchassis != NO_SUBCHASSIS)
3262 				cp->sec_instance = subchassis;
3263 			else
3264 				cp->sec_instance = pcp->sec_scinstance++;
3265 
3266 			if (cp->sec_instance > pcp->sec_maxinstance)
3267 				pcp->sec_maxinstance = cp->sec_instance;
3268 
3269 			topo_list_append(&pcp->sec_subchassis, cp);
3270 		}
3271 
3272 	} else {
3273 		topo_mod_dprintf(mod, "dup chassis/subchassis");
3274 		if (nvlist_lookup_boolean_value(props,
3275 		    SES_PROP_IDENT, &ident) == 0) {
3276 			topo_mod_dprintf(mod,  "overriding enclosure node");
3277 
3278 			cp->sec_enclosure = np;
3279 			cp->sec_target = sdp->sed_target;
3280 		}
3281 	}
3282 
3283 	topo_list_append(&cp->sec_targets, sdp->sed_target);
3284 	sdp->sed_current = cp;
3285 
3286 	return (0);
3287 }
3288 
3289 /*
3290  * Gather nodes from the current SES target into our chassis list, merging the
3291  * results if necessary.
3292  */
3293 static ses_walk_action_t
3294 ses_enum_gather(ses_node_t *np, void *data)
3295 {
3296 	nvlist_t *props = ses_node_props(np);
3297 	ses_enum_data_t *sdp = data;
3298 	topo_mod_t *mod = sdp->sed_mod;
3299 	ses_enum_chassis_t *cp, *scp;
3300 	ses_enum_node_t *snp;
3301 	ses_alt_node_t *sap;
3302 	char *csn;
3303 	uint64_t instance, type;
3304 	uint64_t prevstatus, status;
3305 	boolean_t report;
3306 	uint64_t subchassis = NO_SUBCHASSIS;
3307 
3308 	if (ses_node_type(np) == SES_NODE_ENCLOSURE) {
3309 		/*
3310 		 * If we have already identified the chassis for this target,
3311 		 * then this is a secondary enclosure and we should ignore it,
3312 		 * along with the rest of the tree (since this is depth-first).
3313 		 */
3314 		if (sdp->sed_current != NULL)
3315 			return (SES_WALK_ACTION_TERMINATE);
3316 
3317 		/*
3318 		 * Go through the list of chassis we have seen so far and see
3319 		 * if this serial number matches one of the known values.
3320 		 * If so, check whether this enclosure is a subchassis.
3321 		 */
3322 		if (nvlist_lookup_string(props, LIBSES_EN_PROP_CSN,
3323 		    &csn) != 0)
3324 			return (SES_WALK_ACTION_TERMINATE);
3325 
3326 		(void) nvlist_lookup_uint64(props, LIBSES_EN_PROP_SUBCHASSIS_ID,
3327 		    &subchassis);
3328 
3329 		topo_mod_dprintf(mod, "ses_enum_gather: Enclosure Node (%s) "
3330 		    "CSN (%s), subchassis (%llu)", sdp->sed_name, csn,
3331 		    subchassis);
3332 
3333 		/*
3334 		 * We need to determine whether this enclosure node
3335 		 * represents a chassis or a subchassis. Since we may
3336 		 * receive the enclosure nodes in a non-deterministic
3337 		 * manner, we need to account for all possible combinations:
3338 		 *	1. Chassis for the current CSN has not yet been
3339 		 *	   allocated
3340 		 *		1.1 This is a new chassis:
3341 		 *			allocate and instantiate the chassis
3342 		 *		1.2 This is a new subchassis:
3343 		 *			allocate a placeholder chassis
3344 		 *			allocate and instantiate the subchassis
3345 		 *			link the subchassis to the chassis
3346 		 *	2. Chassis for the current CSN has been allocated
3347 		 *		2.1 This is a duplicate chassis enclosure
3348 		 *			check whether to override old chassis
3349 		 *			append to chassis' target list
3350 		 *		2.2 Only placeholder chassis exists
3351 		 *			fill in the chassis fields
3352 		 *		2.3 This is a new subchassis
3353 		 *			allocate and instantiate the subchassis
3354 		 *			link the subchassis to the chassis
3355 		 *		2.4 This is a duplicate subchassis enclosure
3356 		 *			 check whether to override old chassis
3357 		 *			 append to chassis' target list
3358 		 */
3359 
3360 		for (cp = topo_list_next(&sdp->sed_chassis); cp != NULL;
3361 		    cp = topo_list_next(cp))
3362 			if (strcmp(cp->sec_csn, csn) == 0)
3363 				break;
3364 
3365 		if (cp == NULL) {
3366 			/* 1. Haven't seen a chassis with this CSN before */
3367 
3368 			if ((cp = topo_mod_zalloc(mod,
3369 			    sizeof (ses_enum_chassis_t))) == NULL)
3370 				goto error;
3371 
3372 			cp->sec_scinstance = SES_STARTING_SUBCHASSIS;
3373 			cp->sec_maxinstance = -1;
3374 			cp->sec_csn = csn;
3375 
3376 			if (subchassis == NO_SUBCHASSIS) {
3377 				/* 1.1 This is a new chassis */
3378 
3379 				topo_mod_dprintf(mod, "%s: Initialize new "
3380 				    "chassis with CSN %s", sdp->sed_name, csn);
3381 
3382 				if (ses_init_chassis(mod, sdp, NULL, cp,
3383 				    np, props, NO_SUBCHASSIS,
3384 				    SES_NEW_CHASSIS) < 0)
3385 					goto error;
3386 			} else {
3387 				/* 1.2 This is a new subchassis */
3388 
3389 				topo_mod_dprintf(mod, "%s: Initialize new "
3390 				    "subchassis with CSN %s and index %llu",
3391 				    sdp->sed_name, csn, subchassis);
3392 
3393 				if ((scp = topo_mod_zalloc(mod,
3394 				    sizeof (ses_enum_chassis_t))) == NULL)
3395 					goto error;
3396 
3397 				scp->sec_csn = csn;
3398 
3399 				if (ses_init_chassis(mod, sdp, cp, scp, np,
3400 				    props, subchassis, SES_NEW_SUBCHASSIS) < 0)
3401 					goto error;
3402 			}
3403 		} else {
3404 			/*
3405 			 * We have a chassis or subchassis with this CSN.  If
3406 			 * it's a chassis, we must check to see whether it is
3407 			 * a placeholder previously created because we found a
3408 			 * subchassis with this CSN.  We will know that because
3409 			 * the sec_target value will not be set; it is set only
3410 			 * in ses_init_chassis().  In that case, initialise it
3411 			 * as a new chassis; otherwise, it's a duplicate and we
3412 			 * need to append only.
3413 			 */
3414 			if (subchassis == NO_SUBCHASSIS) {
3415 				if (cp->sec_target != NULL) {
3416 					/* 2.1 This is a duplicate chassis */
3417 
3418 					topo_mod_dprintf(mod, "%s: Append "
3419 					    "duplicate chassis with CSN (%s)",
3420 					    sdp->sed_name, csn);
3421 
3422 					if (ses_init_chassis(mod, sdp, NULL, cp,
3423 					    np, props, NO_SUBCHASSIS,
3424 					    SES_DUP_CHASSIS) < 0)
3425 						goto error;
3426 				} else {
3427 					/* Placeholder chassis - init it up */
3428 					topo_mod_dprintf(mod, "%s: Initialize"
3429 					    "placeholder chassis with CSN %s",
3430 					    sdp->sed_name, csn);
3431 
3432 					if (ses_init_chassis(mod, sdp, NULL,
3433 					    cp, np, props, NO_SUBCHASSIS,
3434 					    SES_NEW_CHASSIS) < 0)
3435 						goto error;
3436 
3437 				}
3438 			} else {
3439 				/* This is a subchassis */
3440 
3441 				for (scp = topo_list_next(&cp->sec_subchassis);
3442 				    scp != NULL; scp = topo_list_next(scp))
3443 					if (scp->sec_instance == subchassis)
3444 						break;
3445 
3446 				if (scp == NULL) {
3447 					/* 2.3 This is a new subchassis */
3448 
3449 					topo_mod_dprintf(mod, "%s: Initialize "
3450 					    "new subchassis with CSN (%s) "
3451 					    "and LID (%s)",
3452 					    sdp->sed_name, csn);
3453 
3454 					if ((scp = topo_mod_zalloc(mod,
3455 					    sizeof (ses_enum_chassis_t)))
3456 					    == NULL)
3457 						goto error;
3458 
3459 					scp->sec_csn = csn;
3460 
3461 					if (ses_init_chassis(mod, sdp, cp, scp,
3462 					    np, props, subchassis,
3463 					    SES_NEW_SUBCHASSIS) < 0)
3464 						goto error;
3465 				} else {
3466 					/* 2.4 This is a duplicate subchassis */
3467 
3468 					topo_mod_dprintf(mod, "%s: Append "
3469 					    "duplicate subchassis with "
3470 					    "CSN (%s)", sdp->sed_name, csn);
3471 
3472 					if (ses_init_chassis(mod, sdp, cp, scp,
3473 					    np, props, subchassis,
3474 					    SES_DUP_SUBCHASSIS) < 0)
3475 						goto error;
3476 				}
3477 			}
3478 		}
3479 	} else if (ses_node_type(np) == SES_NODE_ELEMENT) {
3480 		/*
3481 		 * If we haven't yet seen an enclosure node and identified the
3482 		 * current chassis, something is very wrong; bail out.
3483 		 */
3484 		if (sdp->sed_current == NULL)
3485 			return (SES_WALK_ACTION_TERMINATE);
3486 
3487 		/*
3488 		 * If this isn't one of the element types we care about, then
3489 		 * ignore it.
3490 		 */
3491 		verify(nvlist_lookup_uint64(props, SES_PROP_ELEMENT_TYPE,
3492 		    &type) == 0);
3493 		if (type != SES_ET_DEVICE &&
3494 		    type != SES_ET_ARRAY_DEVICE &&
3495 		    type != SES_ET_SUNW_FANBOARD &&
3496 		    type != SES_ET_SUNW_FANMODULE &&
3497 		    type != SES_ET_COOLING &&
3498 		    type != SES_ET_SUNW_POWERBOARD &&
3499 		    type != SES_ET_SUNW_POWERMODULE &&
3500 		    type != SES_ET_POWER_SUPPLY &&
3501 		    type != SES_ET_ESC_ELECTRONICS &&
3502 		    type != SES_ET_SAS_EXPANDER &&
3503 		    type != SES_ET_SAS_CONNECTOR)
3504 			return (SES_WALK_ACTION_CONTINUE);
3505 
3506 		/*
3507 		 * Get the current instance number and see if we already know
3508 		 * about this element.  If so, it means we have multiple paths
3509 		 * to the same elements, and we should ignore the current path.
3510 		 */
3511 		verify(nvlist_lookup_uint64(props, SES_PROP_ELEMENT_CLASS_INDEX,
3512 		    &instance) == 0);
3513 		if (type == SES_ET_DEVICE || type == SES_ET_ARRAY_DEVICE)
3514 			(void) nvlist_lookup_uint64(props, SES_PROP_BAY_NUMBER,
3515 			    &instance);
3516 
3517 		cp = sdp->sed_current;
3518 
3519 		for (snp = topo_list_next(&cp->sec_nodes); snp != NULL;
3520 		    snp = topo_list_next(snp)) {
3521 			if (snp->sen_type == type &&
3522 			    snp->sen_instance == instance)
3523 				break;
3524 		}
3525 
3526 		/*
3527 		 * We prefer the new element under the following circumstances:
3528 		 *
3529 		 * - The currently known element's status is unknown or not
3530 		 *   available, but the new element has a known status.  This
3531 		 *   occurs if a given element is only available through a
3532 		 *   particular target.
3533 		 *
3534 		 * - This is an ESC_ELECTRONICS element, and the 'reported-via'
3535 		 *   property is set.  This allows us to get reliable firmware
3536 		 *   revision information from the enclosure node.
3537 		 */
3538 		if (snp != NULL) {
3539 			if (nvlist_lookup_uint64(
3540 			    ses_node_props(snp->sen_node),
3541 			    SES_PROP_STATUS_CODE, &prevstatus) != 0)
3542 				prevstatus = SES_ESC_UNSUPPORTED;
3543 			if (nvlist_lookup_uint64(
3544 			    props, SES_PROP_STATUS_CODE, &status) != 0)
3545 				status = SES_ESC_UNSUPPORTED;
3546 			if (nvlist_lookup_boolean_value(
3547 			    props, SES_PROP_REPORT, &report) != 0)
3548 				report = B_FALSE;
3549 
3550 			if ((SES_STATUS_UNAVAIL(prevstatus) &&
3551 			    !SES_STATUS_UNAVAIL(status)) ||
3552 			    (type == SES_ET_ESC_ELECTRONICS &&
3553 			    report)) {
3554 				snp->sen_node = np;
3555 				snp->sen_target = sdp->sed_target;
3556 			}
3557 
3558 			if ((sap = topo_mod_zalloc(mod,
3559 			    sizeof (ses_alt_node_t))) == NULL)
3560 				goto error;
3561 
3562 			sap->san_node = np;
3563 			topo_list_append(&snp->sen_alt_nodes, sap);
3564 
3565 			return (SES_WALK_ACTION_CONTINUE);
3566 		}
3567 
3568 		if ((snp = topo_mod_zalloc(mod,
3569 		    sizeof (ses_enum_node_t))) == NULL)
3570 			goto error;
3571 
3572 		if ((sap = topo_mod_zalloc(mod,
3573 		    sizeof (ses_alt_node_t))) == NULL) {
3574 			topo_mod_free(mod, snp, sizeof (ses_enum_node_t));
3575 			goto error;
3576 		}
3577 
3578 		topo_mod_dprintf(mod, "%s: adding node (%llu, %llu)",
3579 		    sdp->sed_name, type, instance);
3580 		snp->sen_node = np;
3581 		snp->sen_type = type;
3582 		snp->sen_instance = instance;
3583 		snp->sen_target = sdp->sed_target;
3584 		sap->san_node = np;
3585 		topo_list_append(&snp->sen_alt_nodes, sap);
3586 		topo_list_append(&cp->sec_nodes, snp);
3587 
3588 		if (type == SES_ET_DEVICE)
3589 			cp->sec_hasdev = B_TRUE;
3590 	}
3591 
3592 	return (SES_WALK_ACTION_CONTINUE);
3593 
3594 error:
3595 	sdp->sed_errno = -1;
3596 	return (SES_WALK_ACTION_TERMINATE);
3597 }
3598 
3599 static int
3600 ses_process_dir(const char *dirpath, ses_enum_data_t *sdp)
3601 {
3602 	topo_mod_t *mod = sdp->sed_mod;
3603 	DIR *dir;
3604 	struct dirent *dp;
3605 	char path[PATH_MAX];
3606 	ses_enum_target_t *stp;
3607 	int err = -1;
3608 
3609 	/*
3610 	 * Open the SES target directory and iterate over any available
3611 	 * targets.
3612 	 */
3613 	if ((dir = opendir(dirpath)) == NULL) {
3614 		/*
3615 		 * If the SES target directory does not exist, then return as if
3616 		 * there are no active targets.
3617 		 */
3618 		topo_mod_dprintf(mod, "failed to open ses "
3619 		    "directory '%s'", dirpath);
3620 		return (0);
3621 	}
3622 
3623 	while ((dp = readdir(dir)) != NULL) {
3624 		if (strcmp(dp->d_name, ".") == 0 ||
3625 		    strcmp(dp->d_name, "..") == 0)
3626 			continue;
3627 
3628 		/*
3629 		 * Create a new target instance and take a snapshot.
3630 		 */
3631 		if ((stp = topo_mod_zalloc(mod,
3632 		    sizeof (ses_enum_target_t))) == NULL)
3633 			goto error;
3634 
3635 		(void) pthread_mutex_init(&stp->set_lock, NULL);
3636 
3637 		(void) snprintf(path, sizeof (path), "%s/%s", dirpath,
3638 		    dp->d_name);
3639 
3640 		/*
3641 		 * We keep track of the SES device path and export it on a
3642 		 * per-node basis to allow higher level software to get to the
3643 		 * corresponding SES state.
3644 		 */
3645 		if ((stp->set_devpath = topo_mod_strdup(mod, path)) == NULL) {
3646 			topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
3647 			goto error;
3648 		}
3649 
3650 		if ((stp->set_target =
3651 		    ses_open(LIBSES_VERSION, path)) == NULL) {
3652 			topo_mod_dprintf(mod, "failed to open ses target "
3653 			    "'%s': %s", dp->d_name, ses_errmsg());
3654 			ses_sof_alloc(mod, stp->set_devpath);
3655 			topo_mod_free(mod, stp, sizeof (ses_enum_target_t));
3656 			continue;
3657 		}
3658 		topo_mod_dprintf(mod, "open contract");
3659 		ses_ssl_alloc(mod, stp);
3660 		ses_create_contract(mod, stp);
3661 
3662 		stp->set_refcount = 1;
3663 		sdp->sed_target = stp;
3664 		stp->set_snap = ses_snap_hold(stp->set_target);
3665 		stp->set_snaptime = gethrtime();
3666 
3667 		/*
3668 		 * Enumerate over all SES elements and merge them into the
3669 		 * correct ses_enum_chassis_t.
3670 		 */
3671 		sdp->sed_current = NULL;
3672 		sdp->sed_errno = 0;
3673 		sdp->sed_name = dp->d_name;
3674 		(void) ses_walk(stp->set_snap, ses_enum_gather, sdp);
3675 
3676 		if (sdp->sed_errno != 0)
3677 			goto error;
3678 	}
3679 
3680 	err = 0;
3681 error:
3682 	(void) closedir(dir);
3683 	return (err);
3684 }
3685 
3686 /*
3687  * Different generations of SMCI's 4U36 storage servers used different models
3688  * of front and rear SAS expanders.
3689  */
3690 #define	SMCI4U36_FRONT_EXPANDER_PID1	"LSI-SAS2X36"
3691 #define	SMCI4U36_FRONT_EXPANDER_PID2	"LSI-SAS3x40"
3692 #define	SMCI4U36_FRONT_EXPANDER_PID3	"SMC-SC846P"
3693 
3694 #define	SMCI4U36_REAR_EXPANDER_PID1	"LSI-CORP-SAS2X28"
3695 #define	SMCI4U36_REAR_EXPANDER_PID2	"LSI-SAS3x28"
3696 
3697 static int
3698 smci_4u36_bay_label(topo_mod_t *mod, tnode_t *node, topo_version_t version,
3699     nvlist_t *in, nvlist_t **out)
3700 {
3701 	int err, ret = -1;
3702 	nvlist_t *pargs, *auth, *nvl = NULL, *fmri;
3703 	char *label = NULL, *product_id;
3704 
3705 	/*
3706 	 * Now look for a private argument list to determine if the invoker is
3707 	 * trying to do a set operation and if so, return an error as this
3708 	 * method only supports get operations.
3709 	 */
3710 	if ((nvlist_lookup_nvlist(in, TOPO_PROP_PARGS, &pargs) == 0) &&
3711 	    nvlist_exists(pargs, TOPO_PROP_VAL_VAL)) {
3712 		topo_mod_dprintf(mod, "%s: set operation not suppported",
3713 		    __func__);
3714 		return (topo_mod_seterrno(mod, EMOD_NVL_INVAL));
3715 	}
3716 
3717 	if (topo_node_resource(node, &fmri, &err) != 0) {
3718 		(void) topo_mod_seterrno(mod, err);
3719 		goto err;
3720 	}
3721 
3722 	if (nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) != 0 ||
3723 	    nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT, &product_id) !=
3724 	    0) {
3725 		topo_mod_dprintf(mod, "%s: malformed FMRI", __func__);
3726 		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
3727 		nvlist_free(fmri);
3728 		goto err;
3729 	}
3730 	nvlist_free(fmri);
3731 
3732 	if (strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID1) == 0 ||
3733 	    strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID2) == 0 ||
3734 	    strcmp(product_id, SMCI4U36_FRONT_EXPANDER_PID3) == 0) {
3735 		err = asprintf(&label, "Front Slot %" PRIu64,
3736 		    topo_node_instance(node));
3737 	} else if (strcmp(product_id, SMCI4U36_REAR_EXPANDER_PID1) == 0 ||
3738 	    strcmp(product_id, SMCI4U36_REAR_EXPANDER_PID2) == 0) {
3739 		err = asprintf(&label, "Rear Slot %" PRIu64,
3740 		    topo_node_instance(node));
3741 	} else {
3742 		topo_mod_dprintf(mod, "%s: unexpected expander product id: %s",
3743 		    __func__, product_id);
3744 		(void) topo_mod_seterrno(mod, EMOD_UNKNOWN);
3745 		goto err;
3746 	}
3747 
3748 	if (err < 0) {
3749 		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
3750 		goto err;
3751 	}
3752 
3753 	if (topo_mod_nvalloc(mod, &nvl, NV_UNIQUE_NAME) != 0 ||
3754 	    nvlist_add_string(nvl, TOPO_PROP_VAL_NAME, TOPO_PROP_LABEL) != 0 ||
3755 	    nvlist_add_uint32(nvl, TOPO_PROP_VAL_TYPE, TOPO_TYPE_STRING)
3756 	    != 0 ||
3757 	    nvlist_add_string(nvl, TOPO_PROP_VAL_VAL, label)
3758 	    != 0) {
3759 		topo_mod_dprintf(mod, "Failed to allocate 'out' nvlist");
3760 		nvlist_free(nvl);
3761 		(void) topo_mod_seterrno(mod, EMOD_NOMEM);
3762 		goto err;
3763 	}
3764 	*out = nvl;
3765 	ret = 0;
3766 err:
3767 	free(label);
3768 	return (ret);
3769 
3770 }
3771 
3772 static void
3773 ses_release(topo_mod_t *mod, tnode_t *tn)
3774 {
3775 	ses_enum_target_t *stp;
3776 
3777 	if ((stp = topo_node_getspecific(tn)) != NULL) {
3778 		topo_node_setspecific(tn, NULL);
3779 		ses_target_free(mod, stp);
3780 	}
3781 }
3782 
3783 /*ARGSUSED*/
3784 static int
3785 ses_enum(topo_mod_t *mod, tnode_t *rnode, const char *name,
3786     topo_instance_t min, topo_instance_t max, void *arg, void *notused)
3787 {
3788 	ses_enum_chassis_t *cp;
3789 	ses_enum_data_t *data;
3790 
3791 	/*
3792 	 * Check to make sure we're being invoked sensibly, and that we're not
3793 	 * being invoked as part of a post-processing step.
3794 	 */
3795 	if (strcmp(name, SES_ENCLOSURE) != 0 && strcmp(name, BAY) != 0)
3796 		return (0);
3797 
3798 	/*
3799 	 * If this is the first time we've called our enumeration method, then
3800 	 * gather information about any available enclosures.
3801 	 */
3802 	if ((data = topo_mod_getspecific(mod)) == NULL) {
3803 		ses_sof_freeall(mod);
3804 		if ((data = topo_mod_zalloc(mod, sizeof (ses_enum_data_t))) ==
3805 		    NULL)
3806 			return (-1);
3807 
3808 		data->sed_mod = mod;
3809 		topo_mod_setspecific(mod, data);
3810 
3811 		if (dev_list_gather(mod, &data->sed_devs) != 0)
3812 			goto error;
3813 
3814 		/*
3815 		 * We search both the ses(7D) and sgen(7D) locations, so we are
3816 		 * independent of any particular driver class bindings.
3817 		 */
3818 		if (ses_process_dir("/dev/es", data) != 0 ||
3819 		    ses_process_dir("/dev/scsi/ses", data) != 0)
3820 			goto error;
3821 	}
3822 
3823 	if (strcmp(name, SES_ENCLOSURE) == 0) {
3824 		/*
3825 		 * This is a request to enumerate external enclosures.  Go
3826 		 * through all the targets and create chassis nodes where
3827 		 * necessary.
3828 		 */
3829 		for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
3830 		    cp = topo_list_next(cp)) {
3831 			if (ses_create_chassis(data, rnode, cp) != 0)
3832 				goto error;
3833 		}
3834 	} else {
3835 		/*
3836 		 * This is a request to enumerate a specific bay underneath the
3837 		 * root chassis (for internal disks).
3838 		 */
3839 		if (ses_create_bays(data, rnode) != 0)
3840 			goto error;
3841 	}
3842 
3843 	/*
3844 	 * This is a bit of a kludge.  In order to allow internal disks to be
3845 	 * enumerated and share snapshot-specific information with the external
3846 	 * enclosure enumeration, we rely on the fact that we will be invoked
3847 	 * for the 'ses-enclosure' node last.
3848 	 */
3849 	if (strcmp(name, SES_ENCLOSURE) == 0) {
3850 		for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
3851 		    cp = topo_list_next(cp))
3852 			ses_data_free(data, cp);
3853 		ses_data_free(data, NULL);
3854 		topo_mod_setspecific(mod, NULL);
3855 	}
3856 	return (0);
3857 
3858 error:
3859 	for (cp = topo_list_next(&data->sed_chassis); cp != NULL;
3860 	    cp = topo_list_next(cp))
3861 		ses_data_free(data, cp);
3862 	ses_data_free(data, NULL);
3863 	topo_mod_setspecific(mod, NULL);
3864 	return (-1);
3865 }
3866 
3867 static const topo_modops_t ses_ops =
3868 	{ ses_enum, ses_release };
3869 
3870 static topo_modinfo_t ses_info =
3871 	{ SES_ENCLOSURE, FM_FMRI_SCHEME_HC, SES_VERSION, &ses_ops };
3872 
3873 /*ARGSUSED*/
3874 int
3875 _topo_init(topo_mod_t *mod, topo_version_t version)
3876 {
3877 	int rval;
3878 
3879 	if (getenv("TOPOSESDEBUG") != NULL)
3880 		topo_mod_setdebug(mod);
3881 
3882 	topo_mod_dprintf(mod, "initializing %s enumerator\n",
3883 	    SES_ENCLOSURE);
3884 
3885 	if ((rval = topo_mod_register(mod, &ses_info, TOPO_VERSION)) == 0)
3886 		ses_thread_init(mod);
3887 
3888 	return (rval);
3889 }
3890 
3891 void
3892 _topo_fini(topo_mod_t *mod)
3893 {
3894 	ses_thread_fini(mod);
3895 	ses_sof_freeall(mod);
3896 	topo_mod_unregister(mod);
3897 }
3898