xref: /illumos-gate/usr/src/cmd/picl/plugins/sun4u/daktari/psvcpolicy/psvcpolicy.c (revision 7f7322febbcfe774b7270abc3b191c094bfcc517)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Daktari platform platform specific environment monitoring policies
31  */
32 #include	<poll.h>
33 #include	<syslog.h>
34 #include	<unistd.h>
35 #include	<stdio.h>
36 #include	<stdlib.h>
37 #include	<errno.h>
38 #include	<fcntl.h>
39 #include	<strings.h>
40 #include	<libintl.h>
41 #include	<sys/types.h>
42 #include	<sys/param.h>
43 #include	<config_admin.h>
44 #include	<libdevice.h>
45 #include	<picl.h>
46 #include	<picltree.h>
47 #include	<psvc_objects.h>
48 #include	<sys/i2c/clients/i2c_client.h>
49 #include	<sys/daktari.h>
50 #include	<sys/hpc3130_events.h>
51 #include	<assert.h>
52 #include	<limits.h>
53 #include	<sys/systeminfo.h>
54 
55 /*LINTLIBRARY*/
56 
57 /* resides in libcfgadm */
58 extern cfga_err_t config_change_state(cfga_cmd_t, int, char *const *,
59 	const char *, struct cfga_confirm *, struct cfga_msg *, char **,
60 	cfga_flags_t);
61 /* Local Routine */
62 static int32_t update_gen_fault_led(psvc_opaque_t, char *);
63 static void shutdown_routine(void);
64 static int32_t update_thresholds(psvc_opaque_t hdlp, char *id, int offset);
65 
66 
67 #ifdef DEBUG
68 
69 static int dak_policy_debug = 0;
70 
71 #define	D1SYS_ERR(ARGS) if (dak_policy_debug & 0x1) syslog ARGS;
72 #define	D2SYS_ERR(ARGS) if (dak_policy_debug & 0x2) syslog ARGS;
73 
74 #else
75 
76 #define	D1SYS_ERR(ARGS)
77 #define	D2SYS_ERR(ARGS)
78 
79 #endif
80 
81 #define	I2C_PATH	"/devices/pci@9,700000/ebus@1/i2c@1,30"
82 #define	I2C_NODE	I2C_PATH ":devctl"
83 #define	PCF8574		I2C_PATH "/ioexp@0,%x:pcf8574"
84 #define	PCF8591		I2C_PATH "/adio@0,%x:port_0"
85 #define	FRU		I2C_PATH "/fru@0,%x:fru"
86 #define	HPC3130_DEV	I2C_PATH "/hotplug-controller@0,%2x:port_%1x"
87 #define	GEN_FAULT_LED	"FSP_GEN_FAULT_LED"
88 #define	EMPTY_STRING	"EMPTY"
89 #define	DEVICE_FAILURE_MSG	gettext("WARNING: Device %s failure detected")
90 #define	DEVICE_INSERTED_MSG	gettext("Device %s inserted")
91 #define	DEVICE_REMOVED_MSG	gettext("Device %s removed")
92 #define	PS_UNPLUGGED_MSG	gettext("Device %s unplugged")
93 #define	PS_PLUGGED_MSG		gettext("Device %s Plugged in")
94 #define	DEVICE_OK_MSG		gettext("Device %s OK")
95 #define	SET_LED_FAILED_MSG		\
96 	gettext("Failed to set LED state, id = %s, errno = %d\n")
97 #define	GET_PRESENCE_FAILED_MSG		\
98 	gettext("Failed to get presence attribute, id = %s, errno = %d\n")
99 #define	GET_SENSOR_FAILED_MSG		\
100 	gettext("Failed to get sensor value, id = %s, errno = %d\n")
101 #define	ADD_PS_MSG			\
102 gettext("WARNING: Only 1 Power Supply in system. ADD a 2nd Power Supply.\n")
103 #define	REMOVE_LOAD_MSG			\
104 	gettext("WARNING: Power Supply at 95%% current. Remove some load.\n")
105 #define	PS_OVER_CURRENT_MSG		\
106 	gettext("WARNING: Power Supply overcurrent detected\n")
107 #define	DEVICE_UNKNOWN_MSG	gettext("Unknown device %s instance %d\n")
108 #define	DEVICE_HANDLE_FAIL_MSG		\
109 	gettext("Failed to get device handle for %s, errno = %d\n")
110 #define	DEVTREE_NODE_CREATE_FAILED	\
111 	gettext("psvc PICL plugin: Failed to create node for %s, errno = %d")
112 #define	DEVTREE_NODE_DELETE_FAILED	\
113 	gettext("psvc PICL plugin: Failed to delete node for %s, errno = %d")
114 #define	DISK_FAULT_MSG		gettext("%s: Error Reported\n")
115 #define	DISK_OK_MSG		gettext("%s: Error Cleared\n")
116 #define	SET_FANSPEED_FAILED_MSG		\
117 	gettext("Failed to set fan speed, id = %s, errno = %d\n")
118 #define	GET_ATTR_FRU_FAILED_MSG	gettext("Failed psvc_get_attr for FRU info\n")
119 #define	NO_FRU_INFO_MSG			\
120 	gettext("No FRU Information for %s using default module card\n")
121 
122 #define	DAKTARI_MAX_PS	3
123 #define	DAK_MAX_PS_I_SENSORS 4
124 #define	DAK_MAX_DISKS	12
125 #define	DAK_MAX_CPU_MOD	4
126 #define	DAK_MAX_FAULT_SENSORS 3
127 #define	DAK_MAX_FANS 10
128 
129 static int co_ps = 0;
130 static char *shutdown_string = "shutdown -y -g 60 -i 5 \"OVERTEMP condition\"";
131 
132 typedef struct i2c_hp {
133 	int32_t		addr[2];
134 	char		name[256];
135 	char		compatible[256];
136 } i2c_hp_t;
137 
138 typedef struct seg_desc {
139 	int32_t segdesc;
140 	int16_t segoffset;
141 	int16_t seglength;
142 } seg_desc_t;
143 
144 static int32_t threshold_names[] = {
145 	PSVC_HW_LO_SHUT_ATTR,
146 	PSVC_LO_SHUT_ATTR,
147 	PSVC_LO_WARN_ATTR,
148 	PSVC_NOT_USED,			/* LOW MODE which is not used */
149 	PSVC_OPTIMAL_TEMP_ATTR,
150 	PSVC_HI_WARN_ATTR,
151 	PSVC_HI_SHUT_ATTR,
152 	PSVC_HW_HI_SHUT_ATTR
153 };
154 
155 /*
156  * The I2C bus is noisy, and the state may be incorrectly reported as
157  * having changed.  When the state changes, we attempt to confirm by
158  * retrying.  If any retries indicate that the state has not changed, we
159  * assume the state change(s) were incorrect and the state has not changed.
160  * The following variables are used to store the tuneable values read in
161  * from the optional i2cparam.conf file for this shared object library.
162  */
163 static int n_retry_pshp_status = PSVC_NUM_OF_RETRIES;
164 static int retry_sleep_pshp_status = 1;
165 static int n_read_overcurrent = PSVC_THRESHOLD_COUNTER;
166 static int n_retry_devicefail = PSVC_NUM_OF_RETRIES;
167 static int retry_sleep_devicefail = 1;
168 static int n_read_fanfault = PSVC_THRESHOLD_COUNTER;
169 static int n_retry_pshp = PSVC_NUM_OF_RETRIES;
170 static int retry_sleep_pshp = 1;
171 static int n_retry_diskfault = PSVC_NUM_OF_RETRIES;
172 static int retry_sleep_diskfault = 1;
173 static int n_retry_temp_shutdown = PSVC_NUM_OF_RETRIES;
174 static int retry_sleep_temp_shutdown = 1;
175 
176 typedef struct {
177 	int *pvar;
178 	char *texttag;
179 } i2c_noise_param_t;
180 
181 static i2c_noise_param_t i2cparams[] = {
182 	&n_retry_pshp_status, "n_retry_pshp_status",
183 	&retry_sleep_pshp_status, "retry_sleep_pshp_status",
184 	&n_read_overcurrent, "n_read_overcurrent",
185 	&n_retry_devicefail, "n_retry_devicefail",
186 	&retry_sleep_devicefail, "retry_sleep_devicefail",
187 	&n_read_fanfault, "n_read_fanfault",
188 	&n_retry_pshp, "n_retry_pshp",
189 	&retry_sleep_pshp, "retry_sleep_pshp",
190 	&n_retry_diskfault, "n_retry_diskfault",
191 	&retry_sleep_diskfault, "retry_sleep_diskfault",
192 	&n_retry_temp_shutdown, "n_retry_temp_shutdown",
193 	&retry_sleep_temp_shutdown, "retry_sleep_temp_shutdown",
194 	NULL, NULL
195 };
196 
197 #pragma init(i2cparams_load)
198 
199 static void
200 i2cparams_debug(i2c_noise_param_t *pi2cparams, char *platform,
201 	int usingDefaults)
202 {
203 	char s[128];
204 	i2c_noise_param_t *p;
205 
206 	if (!usingDefaults) {
207 		(void) snprintf(s, sizeof (s),
208 		    "# Values from /usr/platform/%s/lib/i2cparam.conf\n",
209 		    platform);
210 		syslog(LOG_WARNING, "%s", s);
211 	} else {
212 		/* no file - we're using the defaults */
213 		(void) snprintf(s, sizeof (s),
214 "# No /usr/platform/%s/lib/i2cparam.conf file, using defaults\n",
215 		    platform);
216 	}
217 	(void) fputs(s, stdout);
218 	p = pi2cparams;
219 	while (p->pvar != NULL) {
220 		(void) snprintf(s, sizeof (s), "%s %d\n", p->texttag,
221 		    *(p->pvar));
222 		if (!usingDefaults)
223 			syslog(LOG_WARNING, "%s", s);
224 		(void) fputs(s, stdout);
225 		p++;
226 	}
227 }
228 
229 static void
230 i2cparams_load(void)
231 {
232 	FILE *fp;
233 	char filename[PATH_MAX];
234 	char platform[64];
235 	char s[128];
236 	char var[128];
237 	int val;
238 	i2c_noise_param_t *p;
239 
240 	if (sysinfo(SI_PLATFORM, platform, sizeof (platform)) == -1) {
241 		syslog(LOG_ERR, "sysinfo error %s\n", strerror(errno));
242 		return;
243 	}
244 	(void) snprintf(filename, sizeof (filename),
245 	    "/usr/platform/%s/lib/i2cparam.conf", platform);
246 	/* read thru the i2cparam.conf file and set variables */
247 	if ((fp = fopen(filename, "r")) != NULL) {
248 		while (fgets(s, sizeof (s), fp) != NULL) {
249 			if (s[0] == '#') /* skip comment lines */
250 				continue;
251 			/* try to find a string match and get the value */
252 			if (sscanf(s, "%127s %d", var, &val) != 2)
253 				continue;
254 			if (val < 1)
255 				val = 1;  /* clamp min value */
256 			p = &(i2cparams[0]);
257 			while (p->pvar != NULL) {
258 				if (strncmp(p->texttag, var, sizeof (var)) ==
259 				    0) {
260 					*(p->pvar) = val;
261 					break;
262 				}
263 				p++;
264 			}
265 		}
266 		(void) fclose(fp);
267 	}
268 	/* output the values of the parameters */
269 	i2cparams_debug(&(i2cparams[0]), platform, ((fp == NULL)? 1 : 0));
270 }
271 
272 int32_t
273 psvc_MB_update_thresholds_0(psvc_opaque_t hdlp, char *id, int offset)
274 {
275 	int IO_offset = 0xd;
276 	int32_t err;
277 
278 	err = update_thresholds(hdlp, id, IO_offset);
279 
280 	return (err);
281 }
282 
283 int32_t
284 psvc_IO_update_thresholds_0(psvc_opaque_t hdlp, char *id, int offset)
285 {
286 	int IO_offset = 0x8;
287 	int32_t err;
288 
289 	err = update_thresholds(hdlp, id, IO_offset);
290 
291 	return (err);
292 }
293 
294 int32_t
295 psvc_DBP_update_thresholds_0(psvc_opaque_t hdlp, char *id, int offset)
296 {
297 	int IO_offset = 0x7;
298 	int32_t err;
299 
300 	err = update_thresholds(hdlp, id, IO_offset);
301 
302 	return (err);
303 }
304 
305 /*
306  * used to determine if a change of state occured. valid when states
307  * are strings.
308  */
309 static int8_t
310 change_of_state_str(char *state1, char *check1, char *state2, char *check2)
311 {
312 	int change = 0;
313 
314 	if ((strcmp(state1, check1) == 0) && (strcmp(state2, check2) != 0))
315 		change = 1;
316 	if ((strcmp(state1, check1) != 0) && (strcmp(state2, check2) == 0))
317 		change = 1;
318 
319 	return (change);
320 }
321 
322 /*
323  * Update thresholds tries to read the temperature thresholds from the FRU
324  * SEEproms and then updates the thresholds in the object by overriding the
325  * hardcoded thresholds.  For Daktari it is an Error if the FRU does not
326  * contain the segment that had the temperature thresholds.
327  */
328 static int32_t
329 update_thresholds(psvc_opaque_t hdlp, char *id, int offset)
330 {
331 	int32_t status = PSVC_SUCCESS;
332 	fru_info_t fru_data;
333 	char *fru, seg_name[2];
334 	int8_t seg_count, temp_array[8];
335 	int32_t match_count, i, j, seg_desc_start = 0x1806, temp_address;
336 	int32_t seg_found, temp;
337 	boolean_t present;
338 	seg_desc_t segment;
339 
340 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
341 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT))
342 		return (status);
343 
344 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &match_count,
345 	    PSVC_FRU);
346 	if (status == PSVC_FAILURE)
347 		return (status);
348 
349 	for (i = 0; i < match_count; i++) {
350 		seg_found = 0;
351 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
352 		    &fru, PSVC_FRU, i);
353 		if (status != PSVC_SUCCESS)
354 			return (status);
355 
356 		fru_data.buf_start = 0x1805;
357 		fru_data.buf = (char *)&seg_count;
358 		fru_data.read_size = 1;
359 
360 		status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
361 		    &fru_data);
362 		if (status != PSVC_SUCCESS) {
363 			return (status);
364 		}
365 		for (j = 0; (j < seg_count) && (!seg_found); j++) {
366 			fru_data.buf_start = seg_desc_start;
367 			fru_data.buf = seg_name;
368 			fru_data.read_size = 2;
369 
370 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
371 			    &fru_data);
372 
373 			seg_desc_start = seg_desc_start + 2;
374 			fru_data.buf_start = seg_desc_start;
375 			fru_data.buf = (char *)&segment;
376 			fru_data.read_size = sizeof (seg_desc_t);
377 
378 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
379 			    &fru_data);
380 			if (status != PSVC_SUCCESS) {
381 				syslog(LOG_ERR,
382 				    "Failed psvc_get_attr for FRU info\n");
383 				return (status);
384 			}
385 			seg_desc_start = seg_desc_start + sizeof (seg_desc_t);
386 			if (memcmp(seg_name, "SC", 2) == 0)
387 				seg_found = 1;
388 		}
389 		if (seg_found) {
390 			temp_address = segment.segoffset + offset;
391 			fru_data.buf_start = temp_address;
392 			fru_data.buf = (char *)&temp_array;
393 			fru_data.read_size = sizeof (temp_array);
394 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
395 			    &fru_data);
396 			if (status != PSVC_SUCCESS) {
397 				syslog(LOG_ERR,
398 				    "Failed psvc_get_attr for FRU info\n");
399 				return (status);
400 			} else {
401 				for (j = 0; j < sizeof (temp_array); j++) {
402 					if (threshold_names[j] ==
403 					    PSVC_NOT_USED)
404 						continue;
405 					temp = temp_array[j];
406 					status = psvc_set_attr(hdlp, id,
407 					    threshold_names[j], &temp);
408 					if (status != PSVC_SUCCESS) {
409 						return (status);
410 					}
411 				}
412 			}
413 		} else {
414 			syslog(LOG_ERR, "No FRU Information for %s"
415 			    " using default temperatures\n", id);
416 		}
417 	}
418 	return (status);
419 }
420 
421 int32_t
422 psvc_fan_init_speed_0(psvc_opaque_t hdlp, char *id)
423 {
424 	int32_t status = PSVC_SUCCESS;
425 	boolean_t present;
426 	char *control_id;
427 	int32_t init_speed = 0;
428 
429 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
430 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT))
431 		return (status);
432 
433 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR, &control_id,
434 		PSVC_FAN_DRIVE_CONTROL, 0);
435 	if (status != PSVC_SUCCESS)
436 		return (status);
437 
438 	status = psvc_set_attr(hdlp, control_id, PSVC_CONTROL_VALUE_ATTR,
439 	    &init_speed);
440 	if (status == PSVC_FAILURE) {
441 		syslog(LOG_ERR, SET_FANSPEED_FAILED_MSG, control_id, errno);
442 		return (status);
443 	}
444 
445 	return (status);
446 }
447 
448 int32_t
449 psvc_update_setpoint_0(psvc_opaque_t hdlp, char *id)
450 {
451 	int32_t status = PSVC_SUCCESS;
452 	char *temp_sensor;
453 	int32_t match_count, i, temp;
454 	int16_t lowest_temp = 500;
455 	boolean_t present;
456 
457 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
458 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT))
459 		return (status);
460 
461 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &match_count,
462 	    PSVC_DEV_TEMP_SENSOR);
463 	if (status == PSVC_FAILURE)
464 		return (status);
465 
466 	for (i = 0; i < match_count; i++) {
467 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
468 		    &temp_sensor, PSVC_DEV_TEMP_SENSOR, i);
469 		if (status != PSVC_SUCCESS)
470 			return (status);
471 		status = psvc_get_attr(hdlp, temp_sensor,
472 		    PSVC_OPTIMAL_TEMP_ATTR, &temp);
473 		if (status != PSVC_SUCCESS) {
474 			syslog(LOG_ERR, "Failed to get Optimal temp for %s\n",
475 			    temp_sensor);
476 			return (status);
477 		}
478 		if (temp < lowest_temp)
479 			lowest_temp = temp;
480 	}
481 	status = psvc_set_attr(hdlp, id, PSVC_SETPOINT_ATTR, &lowest_temp);
482 	if (status == PSVC_FAILURE) {
483 		syslog(LOG_ERR, "Failed to change setpoint for %s\n", id);
484 		return (status);
485 	}
486 	return (status);
487 }
488 
489 int32_t
490 psvc_remove_missing_nodes_0(psvc_opaque_t hdlp, char *id)
491 {
492 	int32_t status = PSVC_SUCCESS;
493 	char state[32];
494 	char *physical_dev;
495 	int32_t i, device_count;
496 	char parent_path[256];
497 	picl_nodehdl_t child_node;
498 	boolean_t present;
499 
500 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR,
501 	    &device_count, PSVC_PHYSICAL_DEVICE);
502 	if (status == PSVC_FAILURE)
503 		return (status);
504 
505 	for (i = 0; i < device_count; i++) {
506 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
507 		    &physical_dev, PSVC_PHYSICAL_DEVICE, i);
508 		if (status != PSVC_SUCCESS)
509 			return (status);
510 		if (strncmp(physical_dev, "LTC1427", 7) == 0)
511 			continue;
512 		status = psvc_get_attr(hdlp, physical_dev,
513 		    PSVC_PROBE_RESULT_ATTR, state);
514 		if (status != PSVC_SUCCESS)
515 			continue;
516 		status = psvc_get_attr(hdlp, physical_dev, PSVC_PRESENCE_ATTR,
517 		    &present);
518 		if (status == PSVC_FAILURE) {
519 			syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG, physical_dev,
520 			    errno);
521 			return (status);
522 		}
523 
524 		if ((strcmp(state, PSVC_ERROR) == 0) &&
525 		    (present == PSVC_PRESENT)) {
526 			/* convert name to node, and parent path */
527 			psvcplugin_lookup(physical_dev, parent_path,
528 			    &child_node);
529 			/* Device removed */
530 			ptree_delete_node(child_node);
531 		}
532 	}
533 	return (status);
534 }
535 
536 int32_t
537 psvc_check_ps_hotplug_status_0(psvc_opaque_t hdlp, char *id)
538 {
539 	char		fail_valid_switch_id[PICL_PROPNAMELEN_MAX];
540 	int32_t		status = PSVC_SUCCESS;
541 	char		valid_switch_state[32];
542 	char		state[32], fault[32];
543 	int32_t		led_count, j;
544 	char		*led_id;
545 	char		led_state[32];
546 	boolean_t	present;
547 	static int8_t	hotplug_failed_count = 0;
548 	int	retry;
549 
550 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
551 	if (status == PSVC_FAILURE) {
552 		syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG, id, errno);
553 		return (status);
554 	}
555 
556 	if (present == PSVC_ABSENT) {
557 		errno = ENODEV;
558 		return (PSVC_FAILURE);
559 	}
560 
561 	snprintf(fail_valid_switch_id, sizeof (fail_valid_switch_id), "%s%s",
562 	    id, "_SENSOR_VALID_SWITCH");
563 
564 	retry = 0;
565 	do {
566 		if (retry)
567 			(void) sleep(retry_sleep_pshp_status);
568 		status = psvc_get_attr(hdlp, fail_valid_switch_id,
569 		    PSVC_STATE_ATTR, valid_switch_state);
570 		if (status == PSVC_FAILURE) {
571 			if (hotplug_failed_count == 0) {
572 				/*
573 				 * First time the get_attr call failed
574 				 * set count so that if we fail again
575 				 * we will know
576 				 */
577 				hotplug_failed_count = 1;
578 				/*
579 				 * We probably failed because the power
580 				 * supply was just insterted or removed
581 				 * before the get_attr call. We then
582 				 * return from this policy successfully
583 				 * knowing it will be run again shortly
584 				 * with the right PS state.
585 				 */
586 				return (PSVC_SUCCESS);
587 			} else {
588 				/*
589 				 * We have failed before and so this
590 				 * we will consider a hardware problem
591 				 * and it should be reported
592 				 */
593 				syslog(LOG_ERR,
594 				    "Failed getting %s State: ",
595 				    "ps_hotplug_status_0\n",
596 				    fail_valid_switch_id);
597 				return (status);
598 			}
599 		}
600 		/*
601 		 * Because we have successfully gotten a value from
602 		 * the i2c device on the PS we will set the
603 		 * failed_count to 0
604 		 */
605 		hotplug_failed_count = 0;
606 
607 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, state);
608 		if (status == PSVC_FAILURE)
609 			return (status);
610 		retry++;
611 		/*
612 		 * check to see if we need to retry. the conditions are:
613 		 *
614 		 * valid_switch_state	state			retry
615 		 * --------------------------------------------------
616 		 *	PSVC_OFF	!PSVC_HOTPLUGGED	yes
617 		 *	PSVC_ON		PSVC_HOTPLUGGED		yes
618 		 *	PSVC_OFF	PSVC_HOTPLUGGED		no
619 		 *	PSVC_ON		!PSVC_HOTPLUGGED	no
620 		 */
621 	} while ((retry < n_retry_pshp_status) &&
622 	    change_of_state_str(valid_switch_state, PSVC_OFF,
623 	    state, PSVC_HOTPLUGGED));
624 
625 	if ((strcmp(valid_switch_state, PSVC_OFF) == 0) &&
626 		(strcmp(state, PSVC_HOTPLUGGED) != 0)) {
627 		strcpy(state, PSVC_HOTPLUGGED);
628 		strcpy(fault, PSVC_NO_FAULT);
629 		strcpy(led_state, PSVC_LED_OFF);
630 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
631 			state);
632 		if (status == PSVC_FAILURE)
633 			return (status);
634 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR,
635 			&led_count, PSVC_DEV_FAULT_LED);
636 		if (status == PSVC_FAILURE)
637 			return (status);
638 
639 		for (j = 0; j < led_count; j++) {
640 
641 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
642 				&led_id, PSVC_DEV_FAULT_LED, j);
643 			if (status != PSVC_SUCCESS)
644 				return (status);
645 
646 			status = psvc_set_attr(hdlp, led_id,
647 				PSVC_LED_STATE_ATTR, led_state);
648 			if (status != PSVC_SUCCESS) {
649 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
650 				    errno);
651 				return (status);
652 			}
653 
654 		}
655 		syslog(LOG_ERR, PS_UNPLUGGED_MSG, id);
656 		return (status);
657 	}
658 
659 	if ((strcmp(valid_switch_state, PSVC_ON) == 0) &&
660 		(strcmp(state, PSVC_HOTPLUGGED) == 0)) {
661 		strcpy(state, PSVC_OK);
662 		strcpy(fault, PSVC_NO_FAULT);
663 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
664 		if (status == PSVC_FAILURE)
665 			return (status);
666 		syslog(LOG_ERR, PS_PLUGGED_MSG, id);
667 	}
668 
669 	return (status);
670 }
671 
672 int32_t
673 psvc_ps_overcurrent_check_policy_0(psvc_opaque_t hdlp, char *system)
674 {
675 	int32_t status = PSVC_SUCCESS;
676 	boolean_t present;
677 	static char *sensor_id[DAKTARI_MAX_PS][DAK_MAX_PS_I_SENSORS];
678 	static char *power_supply_id[DAKTARI_MAX_PS] = {NULL};
679 	int32_t i, j;
680 	int32_t amps, oc_flag = 0, ps_present = 0;
681 	static int32_t hi_warn[DAKTARI_MAX_PS][DAK_MAX_PS_I_SENSORS];
682 	char state[32];
683 	static int8_t overcurrent_failed_check = 0;
684 	static int threshold_counter = 0;
685 
686 	if (power_supply_id[0] == NULL) {
687 		for (i = 0; i < DAKTARI_MAX_PS; i++) {
688 			status = psvc_get_attr(hdlp, system,
689 			    PSVC_ASSOC_ID_ATTR, &(power_supply_id[i]),
690 			    PSVC_PS, i);
691 			if (status != PSVC_SUCCESS)
692 				return (status);
693 			for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
694 				status = psvc_get_attr(hdlp,
695 				    power_supply_id[i], PSVC_ASSOC_ID_ATTR,
696 				    &(sensor_id[i][j]), PSVC_PS_I_SENSOR, j);
697 				if (status != PSVC_SUCCESS)
698 					return (status);
699 				status = psvc_get_attr(hdlp, sensor_id[i][j],
700 				    PSVC_HI_WARN_ATTR, &(hi_warn[i][j]));
701 				if (status != PSVC_SUCCESS)
702 					return (status);
703 			}
704 		}
705 	}
706 
707 	for (i = 0; i < DAKTARI_MAX_PS; i++) {
708 		status = psvc_get_attr(hdlp, power_supply_id[i],
709 			PSVC_PRESENCE_ATTR, &present);
710 		if (status == PSVC_FAILURE) {
711 			syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG,
712 				power_supply_id[i], errno);
713 			return (status);
714 		}
715 
716 		if (present == PSVC_ABSENT) {
717 			continue;
718 		}
719 
720 		status = psvc_check_ps_hotplug_status_0(hdlp,
721 		    power_supply_id[i]);
722 		if (status == PSVC_FAILURE)
723 			return (status);
724 
725 		status = psvc_get_attr(hdlp, power_supply_id[i],
726 		    PSVC_STATE_ATTR, state);
727 		if (status == PSVC_FAILURE)
728 			return (status);
729 
730 		if (strcmp(state, PSVC_HOTPLUGGED) == 0) {
731 			continue;
732 		} else {
733 			ps_present++;
734 		}
735 
736 		for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
737 			status = psvc_get_attr(hdlp, sensor_id[i][j],
738 				PSVC_SENSOR_VALUE_ATTR, &amps);
739 			if (status != PSVC_SUCCESS) {
740 				if (overcurrent_failed_check == 0) {
741 					/*
742 					 * First time the get_attr call
743 					 * failed  set count so that if we
744 					 * fail again we will know
745 					 */
746 					overcurrent_failed_check = 1;
747 					/*
748 					 * We probably failed because the power
749 					 * supply was just insterted or removed
750 					 * before the get_attr call. We then
751 					 * return from this policy successfully
752 					 * knowing it will be run again shortly
753 					 * with the right PS state.
754 					 */
755 					return (PSVC_SUCCESS);
756 				} else {
757 					/*
758 					 * We have failed before and so this we
759 					 * will consider a hardware problem and
760 					 * it should be reported.
761 					 */
762 					syslog(LOG_ERR,
763 					    "Failed getting %s sensor value",
764 					    sensor_id[i][j]);
765 					return (status);
766 				}
767 			}
768 			/*
769 			 * Because we have successfully gotten a value from the
770 			 * i2c device on the PS we will set the failed_count
771 			 * to 0.
772 			 */
773 			overcurrent_failed_check = 0;
774 
775 			if (amps >= hi_warn[i][j]) {
776 				oc_flag = 1;
777 			}
778 		}
779 	}
780 
781 	if (oc_flag) {
782 		/*
783 		 * Because we observed an overcurrent
784 		 * condition, we increment threshold_counter.
785 		 * Once threshold_counter reaches the value
786 		 * of n_read_overcurrent we log the event.
787 		 */
788 		threshold_counter++;
789 		if (threshold_counter == n_read_overcurrent) {
790 			threshold_counter = 0;
791 			if (ps_present == 1) {
792 				syslog(LOG_ERR, PS_OVER_CURRENT_MSG);
793 				syslog(LOG_ERR, ADD_PS_MSG);
794 			} else {
795 				syslog(LOG_ERR, PS_OVER_CURRENT_MSG);
796 				syslog(LOG_ERR, REMOVE_LOAD_MSG);
797 			}
798 		}
799 	} else {
800 		threshold_counter = 0;
801 	}
802 
803 	return (PSVC_SUCCESS);
804 }
805 
806 int32_t
807 psvc_ps_device_fail_notifier_policy_0(psvc_opaque_t hdlp, char *system)
808 {
809 	static char *ps_id[DAKTARI_MAX_PS] = {NULL};
810 	static char *sensor_id[DAKTARI_MAX_PS][DAK_MAX_FAULT_SENSORS];
811 	char *led_id = "FSP_POWER_FAULT_LED";
812 	int i, j;
813 	char state[32], fault[32], previous_state[32], past_state[32];
814 	char led_state[32];
815 	char bad_sensors[DAK_MAX_FAULT_SENSORS][256];
816 	int32_t status = PSVC_SUCCESS;
817 	boolean_t present;
818 	int fail_state;
819 	static int8_t device_fail_failed_check = 0;
820 	int retry, should_retry;
821 
822 	if (ps_id[0] == NULL) {
823 		for (i = 0; i < DAKTARI_MAX_PS; i++) {
824 			status = psvc_get_attr(hdlp, system,
825 			    PSVC_ASSOC_ID_ATTR, &(ps_id[i]), PSVC_PS, i);
826 			if (status != PSVC_SUCCESS)
827 				return (status);
828 			for (j = 0; j < DAK_MAX_FAULT_SENSORS; j++) {
829 				status = psvc_get_attr(hdlp, ps_id[i],
830 				    PSVC_ASSOC_ID_ATTR, &(sensor_id[i][j]),
831 				    PSVC_DEV_FAULT_SENSOR, j);
832 				if (status != PSVC_SUCCESS)
833 					return (status);
834 			}
835 		}
836 	}
837 
838 	for (i = 0; i < DAKTARI_MAX_PS; i++) {
839 		fail_state = 0;
840 		status = psvc_get_attr(hdlp, ps_id[i], PSVC_PRESENCE_ATTR,
841 		    &present);
842 		if (status == PSVC_FAILURE)
843 			return (status);
844 
845 		if (present == PSVC_ABSENT) {
846 			errno = ENODEV;
847 			return (PSVC_FAILURE);
848 		}
849 
850 		status = psvc_check_ps_hotplug_status_0(hdlp, ps_id[i]);
851 		if (status == PSVC_FAILURE)
852 			return (status);
853 
854 		status = psvc_get_attr(hdlp, ps_id[i], PSVC_STATE_ATTR,
855 		    past_state);
856 		if (status == PSVC_FAILURE)
857 			return (status);
858 
859 		if (strcmp(past_state, PSVC_HOTPLUGGED) == 0) {
860 			return (PICL_SUCCESS);
861 		}
862 
863 		retry = 0;
864 		do {
865 			if (retry)
866 				(void) sleep(retry_sleep_devicefail);
867 			fail_state = 0;
868 			should_retry = 0;
869 			for (j = 0; j < DAK_MAX_FAULT_SENSORS; ++j) {
870 				status = psvc_get_attr(hdlp, sensor_id[i][j],
871 				    PSVC_SWITCH_STATE_ATTR, state);
872 				if (status != PSVC_SUCCESS) {
873 					if (device_fail_failed_check == 0) {
874 						/*
875 						 * First time the get_attr call
876 						 * failed  set count so that
877 						 * if we fail again we will know
878 						 */
879 						device_fail_failed_check = 1;
880 						/*
881 						 * We probably failed because
882 						 * the power supply was just
883 						 * insterted or removed before
884 						 * the get_attr call. We then
885 						 * return from this policy
886 						 * successfully knowing it will
887 						 * be run again shortly
888 						 * with the right PS state.
889 						 */
890 						return (PSVC_SUCCESS);
891 					} else {
892 						/*
893 						 * We have failed before and
894 						 * so this we will consider a
895 						 * hardware problem and
896 						 * it should be reported.
897 						 */
898 						syslog(LOG_ERR, "Failed in "
899 						    "getting sensor state for "
900 						    "%s\n", sensor_id[i][j]);
901 
902 						return (status);
903 					}
904 				}
905 
906 				/*
907 				 * Because we have successfully gotten
908 				 * a value from the i2c device on the
909 				 * PS we will set the failed_count to 0.
910 				 */
911 				device_fail_failed_check = 0;
912 
913 				/*
914 				 * If we find that the sensor is on we
915 				 * fill in the name of the sensor in
916 				 * the bad_sensor array. If the sensor
917 				 * is off we use EMPTY_STRING as a check
918 				 * later on as to when NOT to print out
919 				 * what is in bad_sensor[].
920 				 */
921 				if (strcmp(state, PSVC_SWITCH_ON) == 0) {
922 					fail_state++;
923 					strlcpy(bad_sensors[j], sensor_id[i][j],
924 					    sizeof (bad_sensors[j]));
925 				} else {
926 					strcpy(bad_sensors[j], EMPTY_STRING);
927 				}
928 			}
929 			retry++;
930 			/*
931 			 * check to see if we need to retry. the conditions are:
932 			 *
933 			 * fail_state		past_state		retry
934 			 * --------------------------------------------------
935 			 *	+		PSVC_OK			yes
936 			 *	0		PSVC_ERROR		yes
937 			 *	+		PSVC_ERROR		no
938 			 *	0		PSVC_OK			no
939 			 */
940 			if ((fail_state > 0) &&
941 			    (strcmp(past_state, PSVC_OK) == 0)) {
942 				should_retry = 1;
943 			} else if ((fail_state == 0) &&
944 			    (strcmp(past_state, PSVC_ERROR) == 0)) {
945 				should_retry = 1;
946 			}
947 		} while ((retry < n_retry_devicefail) && should_retry);
948 
949 		if (fail_state != 0) {
950 			strcpy(state, PSVC_ERROR);
951 			strcpy(fault, PSVC_GEN_FAULT);
952 		} else {
953 			strcpy(state, PSVC_OK);
954 			strcpy(fault, PSVC_NO_FAULT);
955 		}
956 
957 		status = psvc_set_attr(hdlp, ps_id[i], PSVC_STATE_ATTR, state);
958 		if (status != PSVC_SUCCESS)
959 			return (status);
960 
961 		status = psvc_set_attr(hdlp, ps_id[i], PSVC_FAULTID_ATTR,
962 		    fault);
963 		if (status != PSVC_SUCCESS)
964 			return (status);
965 
966 		status = psvc_get_attr(hdlp, ps_id[i], PSVC_PREV_STATE_ATTR,
967 		    previous_state);
968 		if (status != PSVC_SUCCESS)
969 			return (status);
970 
971 		if (strcmp(state, previous_state) != 0) {
972 			char dev_label[32];
973 
974 			psvc_get_attr(hdlp, ps_id[i], PSVC_LABEL_ATTR,
975 			    dev_label);
976 
977 			if (strcmp(state, PSVC_ERROR) == 0) {
978 				syslog(LOG_ERR, DEVICE_FAILURE_MSG, dev_label);
979 				for (j = 0; j < DAK_MAX_FAULT_SENSORS; ++j) {
980 					if (strcmp(bad_sensors[j],
981 						EMPTY_STRING) != 0)
982 						syslog(LOG_ERR, "%s\n",
983 						    bad_sensors[j]);
984 				}
985 				strcpy(led_state, PSVC_LED_ON);
986 			} else {
987 				syslog(LOG_ERR, DEVICE_OK_MSG, dev_label);
988 				strcpy(led_state, PSVC_LED_OFF);
989 			}
990 
991 			status = psvc_set_attr(hdlp, led_id,
992 			    PSVC_LED_STATE_ATTR, led_state);
993 			if (status != PSVC_SUCCESS) {
994 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
995 				    errno);
996 				return (status);
997 			}
998 		}
999 	}
1000 
1001 	return (PSVC_SUCCESS);
1002 }
1003 
1004 int32_t
1005 psvc_ps_check_and_disable_dr_policy_0(psvc_opaque_t hdlp, char *id)
1006 {
1007 	char		state[32];
1008 	static char	*name[DAKTARI_MAX_PS] = {NULL};
1009 	int		ps_cnt = 0;
1010 	int		i, j;
1011 	int		dr_conf;
1012 	int		fd, rv;
1013 	boolean_t	present;
1014 	char		dev_path[sizeof (HPC3130_DEV)+8];
1015 	unsigned char	controller_names[HPC3130_CONTROLLERS] =
1016 		{ 0xe2, 0xe6, 0xe8, 0xec };
1017 
1018 	if (name[0] == NULL) {
1019 		for (i = 0; i < DAKTARI_MAX_PS; i++) {
1020 			rv = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1021 			    &(name[i]), PSVC_PS, i);
1022 			if (rv != PSVC_SUCCESS)
1023 				return (rv);
1024 		}
1025 	}
1026 
1027 	/*
1028 	 * Go through the power supplies to make sure they're present
1029 	 * and OK.
1030 	 */
1031 	ps_cnt = DAKTARI_MAX_PS;
1032 	for (i = 0; i < DAKTARI_MAX_PS; i++) {
1033 		rv = psvc_get_attr(hdlp, name[i], PSVC_PRESENCE_ATTR,
1034 		    &present);
1035 		if (rv != PSVC_SUCCESS)
1036 			return (rv);
1037 
1038 		if (present != PSVC_PRESENT) {
1039 			ps_cnt--;
1040 			continue;
1041 		} else {
1042 			rv = psvc_get_attr(hdlp, name[i], PSVC_STATE_ATTR,
1043 				state);
1044 			if (rv != PSVC_SUCCESS)
1045 				return (rv);
1046 
1047 			if (strcmp(state, PSVC_OK))
1048 				ps_cnt--;
1049 		}
1050 	}
1051 
1052 	/*
1053 	 * No change in DR configuration is needed if the new power supply
1054 	 * count equals the current count.
1055 	 */
1056 	if (ps_cnt == co_ps)
1057 		return (PSVC_SUCCESS);
1058 
1059 	/*
1060 	 * Disable DR when hotplugged down to 1 power supply; enable DR when
1061 	 * hotplugged up from 1 supply.
1062 	 */
1063 	assert(ps_cnt);
1064 	if ((co_ps == 0 || co_ps > 1) && ps_cnt != 1) {
1065 		co_ps = ps_cnt;
1066 		return (PSVC_SUCCESS);
1067 	}
1068 	dr_conf = (ps_cnt == 1 ? HPC3130_DR_DISABLE : HPC3130_DR_ENABLE);
1069 	co_ps = ps_cnt;
1070 
1071 	for (i = 0; i < HPC3130_CONTROLLERS; i++) {
1072 		for (j = 0; j < HPC3130_SLOTS; j++) {
1073 			(void) snprintf(dev_path, sizeof (dev_path),
1074 			    HPC3130_DEV, controller_names[i], j);
1075 			fd = open(dev_path, O_RDWR);
1076 			if (fd == -1)
1077 				return (PSVC_FAILURE);
1078 
1079 			rv = ioctl(fd, HPC3130_CONF_DR, &dr_conf);
1080 			close(fd);
1081 			if (rv == -1)
1082 				return (PSVC_FAILURE);
1083 		}
1084 	}
1085 
1086 	return (PSVC_SUCCESS);
1087 }
1088 
1089 int32_t
1090 psvc_fan_blast_shutoff_policy_0(psvc_opaque_t hdlp, char *id)
1091 {
1092 	char		switch_status[32];
1093 	int32_t		status = PSVC_SUCCESS;
1094 
1095 	status = psvc_get_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR, switch_status);
1096 	if (status != PSVC_SUCCESS)
1097 		return (status);
1098 	status = psvc_set_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR,
1099 		PSVC_SWITCH_OFF);
1100 	if (status != PSVC_SUCCESS)
1101 		return (status);
1102 	status = psvc_set_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR,
1103 		PSVC_SWITCH_ON);
1104 	if (status != PSVC_SUCCESS)
1105 		return (status);
1106 	status = psvc_set_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR,
1107 		PSVC_SWITCH_OFF);
1108 
1109 	return (status);
1110 }
1111 
1112 int32_t
1113 psvc_fan_fault_check_policy_0(psvc_opaque_t hdlp, char *system)
1114 {
1115 	static char *fan_id[DAK_MAX_FANS] = {NULL};
1116 	boolean_t enabled;
1117 	int32_t speed;
1118 	int32_t status = PSVC_SUCCESS;
1119 	int r;
1120 	static int threshold_counter = 0;
1121 
1122 	if (fan_id[0] == NULL) {
1123 		for (r = 0; r < DAK_MAX_FANS; r++) {
1124 			status = psvc_get_attr(hdlp, system,
1125 			    PSVC_ASSOC_ID_ATTR, &(fan_id[r]), PSVC_FAN, r);
1126 			if (status != PSVC_SUCCESS)
1127 				return (status);
1128 		}
1129 	}
1130 
1131 	for (r = 0; r < DAK_MAX_FANS; r++) {
1132 		status = psvc_get_attr(hdlp, fan_id[r], PSVC_ENABLE_ATTR,
1133 		    &enabled);
1134 		if (status != PSVC_SUCCESS)
1135 			return (status);
1136 
1137 		if (enabled == PSVC_ENABLED) {
1138 			uint64_t features;
1139 			char *switch_id;
1140 			char switch_state[32], fan_state[32];
1141 			int fan_count, fans;
1142 			char *other_fan_id;
1143 			char fstate[32], ffault[32];
1144 
1145 			/*
1146 			 * If any other fan on the fan tray has an ERROR state,
1147 			 * mark this fan bad and return
1148 			 */
1149 			psvc_get_attr(hdlp, fan_id[r], PSVC_ASSOC_MATCHES_ATTR,
1150 			    &fan_count, PSVC_FAN_TRAY_FANS);
1151 			for (fans = 0; fans < fan_count; ++fans) {
1152 				status = psvc_get_attr(hdlp, fan_id[r],
1153 				    PSVC_ASSOC_ID_ATTR, &other_fan_id,
1154 				    PSVC_FAN_TRAY_FANS, fans);
1155 				if (status == PSVC_FAILURE)
1156 					return (status);
1157 				status = psvc_get_attr(hdlp, other_fan_id,
1158 				    PSVC_STATE_ATTR, fan_state);
1159 				if (status != PSVC_SUCCESS)
1160 					return (status);
1161 
1162 				if (strcmp(fan_state, PSVC_ERROR) == 0) {
1163 					strlcpy(ffault, PSVC_GEN_FAULT,
1164 					    sizeof (ffault));
1165 					status = psvc_set_attr(hdlp, fan_id[r],
1166 					    PSVC_FAULTID_ATTR, ffault);
1167 					if (status != PSVC_SUCCESS)
1168 						return (status);
1169 
1170 					strlcpy(fstate, PSVC_ERROR,
1171 					    sizeof (fstate));
1172 					status = psvc_set_attr(hdlp, fan_id[r],
1173 					    PSVC_STATE_ATTR, fstate);
1174 
1175 					return (status);
1176 				}
1177 			}
1178 
1179 			/*
1180 			 * Select tachometer for IO or CPU primary/secondary
1181 			 * fans.
1182 			 */
1183 			pthread_mutex_lock(&fan_mutex);
1184 
1185 			status = psvc_get_attr(hdlp, fan_id[r],
1186 			    PSVC_ASSOC_ID_ATTR, &switch_id,
1187 			    PSVC_FAN_PRIM_SEC_SELECTOR, 0);
1188 
1189 			if (status != PSVC_FAILURE) {
1190 				status = psvc_get_attr(hdlp, fan_id[r],
1191 				    PSVC_FEATURES_ATTR,	&features);
1192 				if (status == PSVC_FAILURE) {
1193 					pthread_mutex_unlock(&fan_mutex);
1194 					return (status);
1195 				}
1196 
1197 				if (features & PSVC_DEV_PRIMARY)
1198 					strlcpy(switch_state, PSVC_SWITCH_ON,
1199 					    sizeof (switch_state));
1200 				else
1201 					strlcpy(switch_state, PSVC_SWITCH_OFF,
1202 					    sizeof (switch_state));
1203 				status = psvc_set_attr(hdlp, switch_id,
1204 				    PSVC_SWITCH_STATE_ATTR, switch_state);
1205 				if (status == PSVC_FAILURE) {
1206 					pthread_mutex_unlock(&fan_mutex);
1207 					return (status);
1208 				}
1209 
1210 				/* allow time for speed to be determined */
1211 				(void) poll(NULL, 0, 250);
1212 			}
1213 
1214 			status = psvc_get_attr(hdlp, fan_id[r],
1215 			    PSVC_SENSOR_VALUE_ATTR, &speed);
1216 			if (status != PSVC_SUCCESS) {
1217 				pthread_mutex_unlock(&fan_mutex);
1218 				return (status);
1219 			}
1220 
1221 			pthread_mutex_unlock(&fan_mutex);
1222 
1223 			if (speed == 0) {
1224 				threshold_counter++;
1225 				if (threshold_counter ==
1226 				    n_read_fanfault) {
1227 					int32_t i;
1228 					int32_t led_count;
1229 					char led_state[32];
1230 					char *led_id;
1231 					char *slot_id;
1232 					char label[32];
1233 					char state[32], fault[32];
1234 
1235 					threshold_counter = 0;
1236 					strlcpy(fault, PSVC_GEN_FAULT,
1237 					    sizeof (fault));
1238 					status = psvc_set_attr(hdlp, fan_id[r],
1239 					    PSVC_FAULTID_ATTR, fault);
1240 					if (status != PSVC_SUCCESS)
1241 						return (status);
1242 
1243 					strlcpy(state, PSVC_ERROR,
1244 					    sizeof (state));
1245 					status = psvc_set_attr(hdlp, fan_id[r],
1246 					    PSVC_STATE_ATTR, state);
1247 					if (status != PSVC_SUCCESS)
1248 						return (status);
1249 
1250 					status = psvc_get_attr(hdlp, fan_id[r],
1251 					    PSVC_LABEL_ATTR, label);
1252 					if (status != PSVC_SUCCESS)
1253 						return (status);
1254 
1255 					syslog(LOG_ERR, DEVICE_FAILURE_MSG,
1256 					    label);
1257 
1258 					/* turn on fault LEDs */
1259 					psvc_get_attr(hdlp, fan_id[r],
1260 					    PSVC_ASSOC_MATCHES_ATTR, &led_count,
1261 					    PSVC_DEV_FAULT_LED);
1262 					strlcpy(led_state, PSVC_LED_ON,
1263 					    sizeof (led_state));
1264 					for (i = 0; i < led_count; ++i) {
1265 						status = psvc_get_attr(hdlp,
1266 						    fan_id[r],
1267 						    PSVC_ASSOC_ID_ATTR, &led_id,
1268 						    PSVC_DEV_FAULT_LED, i);
1269 						if (status == PSVC_FAILURE)
1270 							return (status);
1271 
1272 						status = psvc_set_attr(hdlp,
1273 						    led_id, PSVC_LED_STATE_ATTR,
1274 						    led_state);
1275 						if (status == PSVC_FAILURE)
1276 							return (status);
1277 					}
1278 
1279 					/* turn on OK to remove LEDs */
1280 
1281 					status = psvc_get_attr(hdlp, fan_id[r],
1282 					    PSVC_ASSOC_ID_ATTR, &slot_id,
1283 					    PSVC_PARENT, 0);
1284 					if (status != PSVC_SUCCESS)
1285 						return (status);
1286 
1287 					psvc_get_attr(hdlp, slot_id,
1288 					    PSVC_ASSOC_MATCHES_ATTR, &led_count,
1289 					    PSVC_SLOT_REMOVE_LED);
1290 					strlcpy(led_state, PSVC_LED_ON,
1291 					    sizeof (led_state));
1292 					for (i = 0; i < led_count; ++i) {
1293 						status = psvc_get_attr(hdlp,
1294 						    slot_id,
1295 						    PSVC_ASSOC_ID_ATTR, &led_id,
1296 						    PSVC_SLOT_REMOVE_LED, i);
1297 						if (status == PSVC_FAILURE)
1298 							return (status);
1299 
1300 						status = psvc_set_attr(hdlp,
1301 						    led_id, PSVC_LED_STATE_ATTR,
1302 						    led_state);
1303 						if (status == PSVC_FAILURE)
1304 							return (status);
1305 					}
1306 				}
1307 			}
1308 		}
1309 	}
1310 
1311 	return (PSVC_SUCCESS);
1312 }
1313 
1314 /*
1315  * This routine takes in the PSVC handle pointer, the PS name, and the
1316  * instance number (0, 1, or 2). It simply make a psvc_get call to get the
1317  * presence of each of the children under the PS. This call will set the
1318  * presence state of the child device if it was not there when the system
1319  * was booted.
1320  */
1321 static int
1322 handle_ps_hotplug_children_presence(psvc_opaque_t hdlp, char *id)
1323 {
1324 	char *sensor_id;
1325 	char fail_valid_switch_id[PICL_PROPNAMELEN_MAX];
1326 	int32_t	status = PSVC_SUCCESS;
1327 	boolean_t presence;
1328 	int j;
1329 
1330 	/* Get the Sensor Valid Switch presence */
1331 	snprintf(fail_valid_switch_id, sizeof (fail_valid_switch_id), "%s%s",
1332 	    id, "_SENSOR_VALID_SWITCH");
1333 
1334 	status = psvc_get_attr(hdlp, fail_valid_switch_id, PSVC_PRESENCE_ATTR,
1335 	    &presence);
1336 	if (status != PSVC_SUCCESS)
1337 		return (status);
1338 
1339 	/* Go through each PS's fault sensors */
1340 	for (j = 0; j < DAK_MAX_FAULT_SENSORS; j++) {
1341 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1342 		    &(sensor_id), PSVC_DEV_FAULT_SENSOR, j);
1343 		if (status != PSVC_SUCCESS)
1344 			return (status);
1345 		status = psvc_get_attr(hdlp, sensor_id, PSVC_PRESENCE_ATTR,
1346 		    &presence);
1347 		if (status != PSVC_SUCCESS)
1348 			return (status);
1349 	}
1350 
1351 	/* Go through each PS's current sensors */
1352 	for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
1353 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1354 		    &(sensor_id), PSVC_PS_I_SENSOR, j);
1355 		if (status != PSVC_SUCCESS)
1356 			return (status);
1357 		status = psvc_get_attr(hdlp, sensor_id, PSVC_PRESENCE_ATTR,
1358 		    &presence);
1359 		if (status != PSVC_SUCCESS)
1360 			return (status);
1361 
1362 	}
1363 
1364 	/* Go through each PS's onboard i2c hardware */
1365 	for (j = 0; j < 3; j++) {
1366 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1367 		    &(sensor_id), PSVC_PHYSICAL_DEVICE, j);
1368 		if (status != PSVC_SUCCESS)
1369 			return (status);
1370 		status = psvc_get_attr(hdlp, sensor_id, PSVC_PRESENCE_ATTR,
1371 		    &presence);
1372 		if (status != PSVC_SUCCESS)
1373 			return (status);
1374 	}
1375 
1376 	return (status);
1377 }
1378 
1379 static i2c_hp_t devices[3][3] = {
1380 {{{0, 0x90}, "adio", "i2c-pcf8591"}, {{0, 0x70}, "ioexp", "i2c-pcf8574"},
1381 	{{0, 0xa0}, "fru", "i2c-at24c64"}},
1382 {{{0, 0x92}, "adio", "i2c-pcf8591"}, {{0, 0x72}, "ioexp", "i2c-pcf8574"},
1383 	{{0, 0xa2}, "fru", "i2c-at24c64"}},
1384 {{{0, 0x94}, "adio", "i2c-pcf8591"}, {{0, 0x74}, "ioexp", "i2c-pcf8574"},
1385 	{{0, 0xa4}, "fru", "i2c-at24c64"}},
1386 };
1387 
1388 int32_t
1389 psvc_ps_hotplug_policy_0(psvc_opaque_t hdlp, char *id)
1390 {
1391 	boolean_t presence, previous_presence;
1392 	int32_t status = PSVC_SUCCESS;
1393 	char label[32], state[32], fault[32];
1394 	int32_t ps_instance, led_count;
1395 	char *switch_id, *led_id;
1396 	int i;
1397 	picl_nodehdl_t parent_node;
1398 	char parent_path[256], ps_path[256];
1399 	picl_nodehdl_t child_node;
1400 	devctl_hdl_t bus_handle, dev_handle;
1401 	devctl_ddef_t ddef_hdl;
1402 	char pcf8574_devpath[256], pcf8591_devpath[256], fru_devpath[256];
1403 	int retry;
1404 
1405 	status = psvc_get_attr(hdlp, id, PSVC_PREV_PRESENCE_ATTR,
1406 		&previous_presence);
1407 	if (status != PSVC_SUCCESS)
1408 		return (status);
1409 
1410 	retry = 0;
1411 	do {
1412 		if (retry)
1413 			(void) sleep(retry_sleep_pshp);
1414 
1415 		status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &presence);
1416 		if (status != PSVC_SUCCESS)
1417 			return (status);
1418 		retry++;
1419 	} while ((retry < n_retry_pshp) &&
1420 	    (presence != previous_presence));
1421 
1422 	if (presence == previous_presence) {
1423 		/* No change */
1424 		return (status);
1425 	}
1426 
1427 	status = psvc_get_attr(hdlp, id, PSVC_LABEL_ATTR, label);
1428 	if (status != PSVC_SUCCESS)
1429 		return (status);
1430 
1431 	/* convert name to node, and parent path */
1432 	psvcplugin_lookup(id, parent_path, &child_node);
1433 
1434 	if (presence == PSVC_PRESENT) {
1435 		/*
1436 		 * Run this code if Power Supply was just added into the
1437 		 * System.  This code toggles hotplug switch and adds the
1438 		 * PS and it's children to the picl tree. We then goto adding
1439 		 * device drivers at bottom of the routine.
1440 		 */
1441 		int32_t switch_count;
1442 		char state[32], fault[32];
1443 		char switch_state[32];
1444 
1445 		/* may detect presence before all connections are made */
1446 		(void) poll(NULL, 0, 500);
1447 
1448 		/* Device added */
1449 		syslog(LOG_ERR, DEVICE_INSERTED_MSG, label);
1450 
1451 		strcpy(state, PSVC_OK);
1452 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
1453 		if (status != PSVC_SUCCESS)
1454 			return (status);
1455 
1456 		strcpy(fault, PSVC_NO_FAULT);
1457 		status = psvc_set_attr(hdlp, id, PSVC_FAULTID_ATTR, fault);
1458 		if (status != PSVC_SUCCESS)
1459 			return (status);
1460 
1461 		/* Enable i2c bus */
1462 		psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR,
1463 			&switch_count, PSVC_HOTPLUG_ENABLE_SWITCH);
1464 		for (i = 0; i < switch_count; ++i) {
1465 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1466 				&switch_id, PSVC_HOTPLUG_ENABLE_SWITCH, i);
1467 			if (status == PSVC_FAILURE)
1468 				return (status);
1469 
1470 			strcpy(switch_state, PSVC_SWITCH_OFF);
1471 			status = psvc_set_attr(hdlp, switch_id,
1472 				PSVC_SWITCH_STATE_ATTR, switch_state);
1473 			if (status == PSVC_FAILURE)
1474 				return (status);
1475 
1476 			strcpy(switch_state, PSVC_SWITCH_ON);
1477 			status = psvc_set_attr(hdlp, switch_id,
1478 				PSVC_SWITCH_STATE_ATTR, switch_state);
1479 			if (status == PSVC_FAILURE)
1480 				return (status);
1481 		}
1482 		ptree_get_node_by_path(parent_path, &parent_node);
1483 		ptree_add_node(parent_node, child_node);
1484 		snprintf(ps_path, sizeof (ps_path), "%s/%s", parent_path, id);
1485 		psvcplugin_add_children(ps_path);
1486 	} else {
1487 		/*
1488 		 * Run this code if PS was just removed from the system. We
1489 		 * delete the device from the picl tree and then shut off
1490 		 * all fault lights associated with the PS.  We also set the
1491 		 * device state to PSVC_REMOVED so that if we hit overcurrent
1492 		 * or fault checking code we can do a psvc call to see that
1493 		 * the device has not offically been added into the system.
1494 		 * We then will drop to code lower in the routine to remove
1495 		 * the device drivers for this PS.
1496 		 */
1497 
1498 		/* Device removed */
1499 		syslog(LOG_ERR, DEVICE_REMOVED_MSG, label);
1500 		ptree_delete_node(child_node);
1501 		psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &led_count,
1502 			PSVC_DEV_FAULT_LED);
1503 
1504 		for (i = 0; i < led_count; i++) {
1505 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1506 				&led_id, PSVC_DEV_FAULT_LED, i);
1507 			if (status != PSVC_SUCCESS) {
1508 				return (status);
1509 			}
1510 
1511 			status = psvc_set_attr(hdlp, led_id,
1512 			    PSVC_LED_STATE_ATTR, PSVC_OFF);
1513 			if (status != PSVC_SUCCESS) {
1514 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
1515 					errno);
1516 				return (status);
1517 			}
1518 
1519 		}
1520 
1521 		strcpy(state, PSVC_OK);
1522 		strcpy(fault, PSVC_NO_FAULT);
1523 
1524 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
1525 		if (status != PSVC_SUCCESS)
1526 			return (status);
1527 		status = psvc_set_attr(hdlp, id, PSVC_FAULTID_ATTR, fault);
1528 		if (status != PSVC_SUCCESS)
1529 			return (status);
1530 	}
1531 
1532 	status = psvc_set_attr(hdlp, id, PSVC_PREV_PRESENCE_ATTR, &presence);
1533 	if (status != PSVC_SUCCESS)
1534 		return (status);
1535 
1536 	status = psvc_get_attr(hdlp, id, PSVC_INSTANCE_ATTR, &ps_instance);
1537 	if (status != PSVC_SUCCESS)
1538 		return (status);
1539 
1540 	if (presence != PSVC_PRESENT) {
1541 		/*
1542 		 * This is the additional code needed to remove the PS from
1543 		 * the system.  It removes the device drivers from the
1544 		 * device tree.
1545 		 */
1546 		snprintf(pcf8574_devpath, sizeof (pcf8574_devpath), PCF8574,
1547 			devices[ps_instance][1].addr[1]);
1548 		snprintf(pcf8591_devpath, sizeof (pcf8591_devpath), PCF8591,
1549 			devices[ps_instance][0].addr[1]);
1550 		snprintf(fru_devpath, sizeof (fru_devpath), FRU,
1551 			devices[ps_instance][2].addr[1]);
1552 
1553 		dev_handle = devctl_device_acquire(pcf8591_devpath, 0);
1554 		if (dev_handle == NULL) {
1555 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1556 				pcf8591_devpath, errno);
1557 			devctl_release(dev_handle);
1558 			return (PSVC_FAILURE);
1559 		} else if ((devctl_device_remove(dev_handle)) &&
1560 			(errno != ENXIO)) {
1561 				syslog(LOG_ERR, DEVTREE_NODE_DELETE_FAILED,
1562 					pcf8591_devpath, errno);
1563 				devctl_release(dev_handle);
1564 				return (PSVC_FAILURE);
1565 			} else {
1566 				devctl_release(dev_handle);
1567 				status = PSVC_SUCCESS;
1568 			}
1569 
1570 		dev_handle = devctl_device_acquire(pcf8574_devpath, 0);
1571 		if (dev_handle == NULL) {
1572 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1573 				pcf8574_devpath, errno);
1574 			devctl_release(dev_handle);
1575 			return (PSVC_FAILURE);
1576 		} else if ((devctl_device_remove(dev_handle)) &&
1577 			(errno != ENXIO)) {
1578 				syslog(LOG_ERR, DEVTREE_NODE_DELETE_FAILED,
1579 					pcf8574_devpath, errno);
1580 				devctl_release(dev_handle);
1581 				return (PSVC_FAILURE);
1582 			} else {
1583 				devctl_release(dev_handle);
1584 				status = PSVC_SUCCESS;
1585 			}
1586 
1587 		dev_handle = devctl_device_acquire(fru_devpath, 0);
1588 		if (dev_handle == NULL) {
1589 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1590 				fru_devpath, errno);
1591 			devctl_release(dev_handle);
1592 			return (PSVC_FAILURE);
1593 		} else if ((devctl_device_remove(dev_handle)) &&
1594 			(errno != ENXIO)) {
1595 				syslog(LOG_ERR, DEVTREE_NODE_DELETE_FAILED,
1596 					fru_devpath, errno);
1597 				devctl_release(dev_handle);
1598 				return (PSVC_FAILURE);
1599 			} else {
1600 				devctl_release(dev_handle);
1601 				status = PSVC_SUCCESS;
1602 			}
1603 
1604 		return (status);
1605 	}
1606 
1607 	/*
1608 	 * This code is to update the presences of power supply child
1609 	 * devices in the event that picld was started without a power
1610 	 * supply present.  This call makes the devices available
1611 	 * after that initial insertion.
1612 	 */
1613 	status = handle_ps_hotplug_children_presence(hdlp, id);
1614 	if (status == PSVC_FAILURE) {
1615 		return (status);
1616 	}
1617 
1618 	/*
1619 	 * We fall through to here if the device has been inserted.
1620 	 * Add the devinfo tree node entry for the seeprom and attach
1621 	 * the i2c seeprom driver
1622 	 */
1623 
1624 	bus_handle = devctl_bus_acquire(I2C_NODE, 0);
1625 	if (bus_handle == NULL) {
1626 		syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG, I2C_NODE, errno);
1627 		return (PSVC_FAILURE);
1628 	}
1629 	/* Create the deivce nodes for all 3 i2c parts on the PS */
1630 	for (i = 0; i < 3; i++) {
1631 		ddef_hdl = devctl_ddef_alloc(devices[ps_instance][i].name, 0);
1632 		if (ddef_hdl == NULL) {
1633 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1634 				devices[ps_instance][i].name, errno);
1635 			return (PSVC_FAILURE);
1636 		}
1637 		status = devctl_ddef_string(ddef_hdl, "compatible",
1638 			devices[ps_instance][i].compatible);
1639 		if (status == -1) {
1640 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1641 				devices[ps_instance][i].name, errno);
1642 			return (PSVC_FAILURE);
1643 		}
1644 		status = devctl_ddef_int_array(ddef_hdl, "reg", 2,
1645 			devices[ps_instance][i].addr);
1646 		if (status == -1) {
1647 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1648 				devices[ps_instance][i].name, errno);
1649 			return (PSVC_FAILURE);
1650 		}
1651 		if (devctl_bus_dev_create(bus_handle, ddef_hdl, 0,
1652 			&dev_handle)) {
1653 			syslog(LOG_ERR, DEVTREE_NODE_CREATE_FAILED,
1654 				devices[ps_instance][i].name, errno);
1655 			return (PSVC_FAILURE);
1656 		} else
1657 			devctl_release(dev_handle);
1658 		devctl_ddef_free(ddef_hdl);
1659 	}
1660 	devctl_release(bus_handle);
1661 
1662 	return (status);
1663 }
1664 
1665 static void
1666 shutdown_routine()
1667 {
1668 	static boolean_t shutdown_flag = 0;
1669 
1670 	if (!(shutdown_flag)) {
1671 		system(shutdown_string);
1672 		shutdown_flag = 1;
1673 	}
1674 }
1675 
1676 /*
1677  * This policy checks temperature sensors to see if the fault attribute
1678  * is set to either High or Low Shutdown. If so then it shuts the system
1679  * down with a 1 minute warning period
1680  */
1681 int32_t
1682 psvc_shutdown_policy(psvc_opaque_t hdlp, char *id)
1683 {
1684 	int32_t	status;
1685 	char	fault[32] = {0};
1686 	boolean_t	pr;
1687 	int	retry;
1688 
1689 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &pr);
1690 	if ((status != PSVC_SUCCESS) || (pr != PSVC_PRESENT)) {
1691 		return (status);
1692 	}
1693 
1694 	retry = 0;
1695 	do {
1696 		if (retry)
1697 			(void) sleep(retry_sleep_temp_shutdown);
1698 		status = psvc_get_attr(hdlp, id, PSVC_FAULTID_ATTR, fault);
1699 		if (status != PSVC_SUCCESS)
1700 			return (status);
1701 		retry++;
1702 	} while (((strcmp(fault, PSVC_TEMP_LO_SHUT) == 0) ||
1703 	    (strcmp(fault, PSVC_TEMP_HI_SHUT) == 0)) &&
1704 		(retry < n_retry_temp_shutdown));
1705 	if ((strcmp(fault, PSVC_TEMP_LO_SHUT) == 0) ||
1706 	    (strcmp(fault, PSVC_TEMP_HI_SHUT) == 0)) {
1707 		shutdown_routine();
1708 	}
1709 
1710 	return (PSVC_SUCCESS);
1711 }
1712 
1713 int32_t
1714 psvc_check_disk_fault_policy_0(psvc_opaque_t hdlp, char *id)
1715 {
1716 	int32_t		status = PSVC_SUCCESS;
1717 	int32_t		i;
1718 	char		curr_state[32], prev_state[32], led_state[32];
1719 	char		disk_fault[32], disk_state[32];
1720 	static char	*disk_id[DAK_MAX_DISKS] = {NULL};
1721 	static char	*led_id[DAK_MAX_DISKS] = {NULL};
1722 	static char	*parent_id[DAK_MAX_DISKS] = {NULL};
1723 	boolean_t	present;
1724 	int		retry;
1725 
1726 	/*
1727 	 * Check which disk faulted, now get the disks.
1728 	 * We are now going to get disk, disk parent,
1729 	 * parent's leds, and check to see if parent's leds are on
1730 	 */
1731 
1732 	if (disk_id[0] == NULL) {
1733 		for (i = 0; i < DAK_MAX_DISKS; i++) {
1734 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1735 			    &(disk_id[i]), PSVC_DISK, i);
1736 			if (status != PSVC_SUCCESS)
1737 				return (status);
1738 			status = psvc_get_attr(hdlp, disk_id[i],
1739 			    PSVC_ASSOC_ID_ATTR, &(parent_id[i]),
1740 			    PSVC_PARENT, 0);
1741 			if (status != PSVC_SUCCESS)
1742 				return (status);
1743 			status = psvc_get_attr(hdlp, parent_id[i],
1744 			    PSVC_ASSOC_ID_ATTR, &(led_id[i]),
1745 			    PSVC_SLOT_FAULT_LED, 0);
1746 			if (status != PSVC_SUCCESS)
1747 				return (status);
1748 
1749 		}
1750 	}
1751 
1752 	for (i = 0; i < DAK_MAX_DISKS; i++) {
1753 		curr_state[0] = 0;
1754 		prev_state[0] = 0;
1755 
1756 		status = psvc_get_attr(hdlp, disk_id[i], PSVC_PRESENCE_ATTR,
1757 			&present);
1758 		if (status != PSVC_SUCCESS)
1759 			return (status);
1760 
1761 		if (present == PSVC_ABSENT)
1762 			continue;
1763 
1764 		/*
1765 		 * Check if whether or not the led is on.
1766 		 * If so, then this disk has a problem and
1767 		 * set its fault and error states to bad.
1768 		 * If not, then set fault and error states to good.
1769 		 * If the disk underwent a change in state, then
1770 		 * print out what state it's now in.
1771 		 */
1772 
1773 		status = psvc_get_attr(hdlp, disk_id[i], PSVC_STATE_ATTR,
1774 		    prev_state);
1775 		if (status != PSVC_SUCCESS)
1776 			return (status);
1777 
1778 		retry = 0;
1779 		do {
1780 			if (retry)
1781 				(void) sleep(retry_sleep_diskfault);
1782 			status = psvc_get_attr(hdlp, led_id[i], PSVC_STATE_ATTR,
1783 			    led_state);
1784 			if (status != PSVC_SUCCESS)
1785 				return (status);
1786 			retry++;
1787 			/*
1788 			 * check to see if we need to retry. the conditions are:
1789 			 *
1790 			 * prev_state		led_state		retry
1791 			 * --------------------------------------------------
1792 			 * PSVC_ERROR		PSVC_LED_ON		yes
1793 			 * PSVC_OK		PSVC_LED_OFF		yes
1794 			 * PSVC_ERROR		PSVC_LED_OFF		no
1795 			 * PSVC_OK		PSVC_LED_ON		no
1796 			 */
1797 		} while ((retry < n_retry_diskfault) &&
1798 		    change_of_state_str(prev_state, PSVC_OK,
1799 		    led_state, PSVC_LED_ON));
1800 
1801 		/*
1802 		 * Set the disk's state and fault id according to
1803 		 * what we found the disk fault sensor (disk_slot_fault_led)
1804 		 * to be.
1805 		 */
1806 		if (strcmp(led_state, PSVC_LED_ON) == 0) {
1807 			strcpy(disk_fault, PSVC_GEN_FAULT);
1808 			strcpy(disk_state, PSVC_ERROR);
1809 		} else {
1810 			strcpy(disk_fault, PSVC_NO_FAULT);
1811 			strcpy(disk_state, PSVC_OK);
1812 		}
1813 		status = psvc_set_attr(hdlp, disk_id[i], PSVC_STATE_ATTR,
1814 			disk_state);
1815 		if (status != PSVC_SUCCESS)
1816 			return (status);
1817 		status = psvc_set_attr(hdlp, disk_id[i], PSVC_FAULTID_ATTR,
1818 			disk_fault);
1819 		if (status != PSVC_SUCCESS)
1820 			return (status);
1821 		/*
1822 		 * Check disk states.  If they differ, then print out
1823 		 * the current state of the disk
1824 		 */
1825 		status = psvc_get_attr(hdlp, disk_id[i], PSVC_PREV_STATE_ATTR,
1826 			prev_state);
1827 		if (status != PSVC_SUCCESS)
1828 			return (status);
1829 
1830 		if (strcmp(disk_state, prev_state) != 0) {
1831 			if (strcmp(disk_state, PSVC_ERROR) == 0) {
1832 				syslog(LOG_ERR, DISK_FAULT_MSG, disk_id[i]);
1833 			} else {
1834 				syslog(LOG_ERR, DISK_OK_MSG, disk_id[i]);
1835 			}
1836 		}
1837 	}
1838 	return (PSVC_SUCCESS);
1839 }
1840 
1841 int32_t
1842 psvc_update_FSP_fault_led_policy_0(psvc_opaque_t hdlp, char *id)
1843 {
1844 	int32_t status = PSVC_SUCCESS;
1845 	int32_t i;
1846 	int32_t dev_count, fault_state = 0;
1847 	char	*dev_id;
1848 	char	dev_state[32], led_state[32];
1849 	boolean_t	present;
1850 
1851 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &dev_count,
1852 		PSVC_DEV_FAULT_SENSOR);
1853 	if (status != PSVC_SUCCESS)
1854 		return (status);
1855 
1856 	fault_state = 0;
1857 
1858 	for (i = 0; i < dev_count; i++) {
1859 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1860 			&dev_id, PSVC_DEV_FAULT_SENSOR, i);
1861 		if (status != PSVC_SUCCESS)
1862 			return (status);
1863 		status = psvc_get_attr(hdlp, dev_id, PSVC_PRESENCE_ATTR,
1864 			&present);
1865 		if (status != PSVC_SUCCESS)
1866 			return (status);
1867 
1868 		if (present == PSVC_ABSENT)
1869 			continue;
1870 
1871 		status = psvc_get_attr(hdlp, dev_id, PSVC_STATE_ATTR,
1872 			dev_state);
1873 		if (status != PSVC_SUCCESS)
1874 			return (status);
1875 
1876 		if (strcmp(dev_state, PSVC_ERROR) == 0) {
1877 			fault_state = 1;
1878 		}
1879 	}
1880 	if (fault_state == 1) {
1881 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
1882 		if (status != PSVC_SUCCESS)
1883 			return (status);
1884 		if (strcmp(led_state, PSVC_OFF) == 0) {
1885 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
1886 			    PSVC_ON);
1887 			if (status != PSVC_SUCCESS)
1888 				return (status);
1889 		}
1890 	} else {
1891 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
1892 		if (status != PSVC_SUCCESS)
1893 			return (status);
1894 		if (strcmp(led_state, PSVC_ON) == 0) {
1895 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
1896 			    PSVC_OFF);
1897 			if (status != PSVC_SUCCESS)
1898 				return (status);
1899 		}
1900 	}
1901 	status = update_gen_fault_led(hdlp, GEN_FAULT_LED);
1902 
1903 	return (status);
1904 }
1905 
1906 int32_t
1907 update_gen_fault_led(psvc_opaque_t hdlp, char *id)
1908 {
1909 	int32_t status = PSVC_SUCCESS;
1910 	int32_t i;
1911 	int32_t led_count, fault_state;
1912 	char	*led_id;
1913 	char	led_state[32];
1914 
1915 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &led_count,
1916 		PSVC_DEV_FAULT_SENSOR);
1917 	if (status != PSVC_SUCCESS)
1918 		return (status);
1919 
1920 	fault_state = 0;
1921 
1922 	for (i = 0; i < led_count; i++) {
1923 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1924 			&led_id, PSVC_DEV_FAULT_SENSOR, i);
1925 		if (status != PSVC_SUCCESS)
1926 			return (status);
1927 		status = psvc_get_attr(hdlp, led_id, PSVC_STATE_ATTR,
1928 			led_state);
1929 		if (status != PSVC_SUCCESS)
1930 			return (status);
1931 
1932 		if (strcmp(led_state, PSVC_ON) == 0) {
1933 			fault_state = 1;
1934 		}
1935 	}
1936 
1937 	if (fault_state == 1) {
1938 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
1939 		if (status != PSVC_SUCCESS)
1940 			return (status);
1941 		if (strcmp(led_state, PSVC_OFF) == 0) {
1942 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
1943 			    PSVC_ON);
1944 			if (status != PSVC_SUCCESS)
1945 				return (status);
1946 		}
1947 	} else {
1948 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
1949 		if (status != PSVC_SUCCESS)
1950 			return (status);
1951 		if (strcmp(led_state, PSVC_ON) == 0) {
1952 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
1953 			    PSVC_OFF);
1954 			if (status != PSVC_SUCCESS)
1955 				return (status);
1956 		}
1957 	}
1958 
1959 	return (status);
1960 }
1961 
1962 
1963 /*
1964  * This function detects whether the module present in the dakatari's
1965  * CPU slot is a CPU module or a Zulu (XVR-4000).
1966  * Based on this detection it also sets the appropriate temperature sensors
1967  * to HOTPLUGGED, so that it works properly with check_temp() function
1968  */
1969 #define	MAX_MODULE_SIZE		20
1970 #define	MAX_TEMP_SENSOR_SIZE	30
1971 
1972 int32_t
1973 psvc_update_cpu_module_card_node_0(psvc_opaque_t hdlp, char *id)
1974 {
1975 	int32_t	set_temp_sensor_properties(psvc_opaque_t, char *);
1976 	int32_t	remove_module_node(psvc_opaque_t, char *);
1977 	int32_t status = PSVC_SUCCESS;
1978 	fru_info_t fru_data;
1979 	char *fru, seg_name[2];
1980 	int8_t seg_count, module_card;
1981 	int32_t match_count, i, j, seg_desc_start = 0x1806, module_address;
1982 	int32_t seg_found;
1983 	boolean_t present;
1984 	seg_desc_t segment;
1985 	char other_module_id[MAX_MODULE_SIZE];
1986 	char cpu_temp_sensor1[MAX_TEMP_SENSOR_SIZE];
1987 	char cpu_temp_sensor2[MAX_TEMP_SENSOR_SIZE];
1988 	char zulu_temp_sensor1[MAX_TEMP_SENSOR_SIZE];
1989 	char zulu_temp_sensor2[MAX_TEMP_SENSOR_SIZE];
1990 	int offset = 0x7;
1991 
1992 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
1993 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT)) {
1994 		return (status);
1995 	}
1996 
1997 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &match_count,
1998 	    PSVC_FRU);
1999 	if (status == PSVC_FAILURE) {
2000 		return (status);
2001 	}
2002 
2003 	for (i = 0; i < match_count; i++) {
2004 		seg_found = 0;
2005 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR, &fru,
2006 		    PSVC_FRU, i);
2007 		if (status != PSVC_SUCCESS)
2008 			return (status);
2009 
2010 		fru_data.buf_start = 0x1805;
2011 		fru_data.buf = (char *)&seg_count;
2012 		fru_data.read_size = 1;
2013 
2014 		status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2015 		    &fru_data);
2016 		if (status != PSVC_SUCCESS) {
2017 			return (status);
2018 		}
2019 
2020 		for (j = 0; (j < seg_count) && (!seg_found); j++) {
2021 			fru_data.buf_start = seg_desc_start;
2022 			fru_data.buf = seg_name;
2023 			fru_data.read_size = 2;
2024 
2025 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2026 			    &fru_data);
2027 			if (status != PSVC_SUCCESS) {
2028 				syslog(LOG_ERR, GET_ATTR_FRU_FAILED_MSG);
2029 				return (status);
2030 			}
2031 
2032 			seg_desc_start = seg_desc_start + 2;
2033 			fru_data.buf_start = seg_desc_start;
2034 			fru_data.buf = (char *)&segment;
2035 			fru_data.read_size = sizeof (seg_desc_t);
2036 
2037 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2038 			    &fru_data);
2039 			if (status != PSVC_SUCCESS) {
2040 				syslog(LOG_ERR, GET_ATTR_FRU_FAILED_MSG);
2041 				return (status);
2042 			}
2043 			seg_desc_start = seg_desc_start + sizeof (seg_desc_t);
2044 			if (memcmp(seg_name, "SC", 2) == 0)
2045 				seg_found = 1;
2046 		}
2047 
2048 		if (seg_found) {
2049 			module_address = segment.segoffset + offset;
2050 			fru_data.buf_start = module_address;
2051 			fru_data.buf = (char *)&module_card;
2052 			fru_data.read_size = 1;
2053 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2054 			    &fru_data);
2055 			if (status != PSVC_SUCCESS) {
2056 				syslog(LOG_ERR, GET_ATTR_FRU_FAILED_MSG);
2057 				return (status);
2058 			}
2059 		} else {
2060 			syslog(LOG_ERR, NO_FRU_INFO_MSG, id);
2061 		}
2062 	}
2063 
2064 	if (strcmp(id, "ZULU_1_3_MOD_CARD") == 0) {
2065 		strlcpy(other_module_id, "CPU_1_3_MOD_CARD", MAX_MODULE_SIZE);
2066 
2067 		strlcpy(cpu_temp_sensor1, "CPU1_DIE_TEMPERATURE_SENSOR",
2068 		    MAX_TEMP_SENSOR_SIZE);
2069 		strlcpy(cpu_temp_sensor2, "CPU3_DIE_TEMPERATURE_SENSOR",
2070 		    MAX_TEMP_SENSOR_SIZE);
2071 
2072 		strlcpy(zulu_temp_sensor1, "ZULU1_DIE_TEMPERATURE_SENSOR",
2073 		    MAX_TEMP_SENSOR_SIZE);
2074 		strlcpy(zulu_temp_sensor2, "ZULU3_DIE_TEMPERATURE_SENSOR",
2075 		    MAX_TEMP_SENSOR_SIZE);
2076 	}
2077 
2078 	if (strcmp(id, "ZULU_4_6_MOD_CARD") == 0) {
2079 		strlcpy(other_module_id, "CPU_4_6_MOD_CARD", MAX_MODULE_SIZE);
2080 
2081 		strlcpy(cpu_temp_sensor1, "CPU4_DIE_TEMPERATURE_SENSOR",
2082 		    MAX_TEMP_SENSOR_SIZE);
2083 		strlcpy(cpu_temp_sensor2, "CPU6_DIE_TEMPERATURE_SENSOR",
2084 		    MAX_TEMP_SENSOR_SIZE);
2085 
2086 		strlcpy(zulu_temp_sensor1, "ZULU4_DIE_TEMPERATURE_SENSOR",
2087 		    MAX_TEMP_SENSOR_SIZE);
2088 		strlcpy(zulu_temp_sensor2, "ZULU6_DIE_TEMPERATURE_SENSOR",
2089 		    MAX_TEMP_SENSOR_SIZE);
2090 	}
2091 
2092 
2093 	/*
2094 	 * If the module in the CPU slot is a Zulu (XVR-4000), then
2095 	 * location 0x1EB0 in its FRUid prom has a value 0xFB.
2096 	 * If Zulu (XVR-4000) is detected, delete the CPU node, otherwise
2097 	 * delete the Zulu node. Also set the temperature sensor value to
2098 	 * HOTPLUGGED for absent temperature sensors.
2099 	 */
2100 	if ((module_card & 0xff) == 0xfb) {
2101 		status = set_temp_sensor_properties(hdlp, cpu_temp_sensor1);
2102 		if (status == PSVC_FAILURE) {
2103 			return (status);
2104 		}
2105 
2106 		status = set_temp_sensor_properties(hdlp, cpu_temp_sensor2);
2107 		if (status == PSVC_FAILURE) {
2108 			return (status);
2109 		}
2110 
2111 		/*
2112 		 * Remove CPU node
2113 		 */
2114 		status = remove_module_node(hdlp, other_module_id);
2115 		if (status == PSVC_FAILURE) {
2116 			return (status);
2117 		}
2118 	} else {
2119 		status = set_temp_sensor_properties(hdlp, zulu_temp_sensor1);
2120 		if (status == PSVC_FAILURE) {
2121 			return (status);
2122 		}
2123 		status = set_temp_sensor_properties(hdlp, zulu_temp_sensor2);
2124 		if (status == PSVC_FAILURE) {
2125 			return (status);
2126 		}
2127 
2128 		/*
2129 		 * Remove Zulu (XVR-4000) node
2130 		 */
2131 		status = remove_module_node(hdlp, id);
2132 		if (status == PSVC_FAILURE) {
2133 			return (status);
2134 		}
2135 	}
2136 
2137 	return (PSVC_SUCCESS);
2138 }
2139 
2140 
2141 /*
2142  * Remove the CPU slot's module node
2143  */
2144 int32_t
2145 remove_module_node(psvc_opaque_t hdlp, char *id)
2146 {
2147 	char parent_path[256];
2148 	picl_nodehdl_t child_node;
2149 
2150 	/* convert name to node, and parent path */
2151 	psvcplugin_lookup(id, parent_path, &child_node);
2152 	/* Device removed */
2153 	ptree_delete_node(child_node);
2154 
2155 	return (PSVC_SUCCESS);
2156 }
2157 
2158 
2159 /*
2160  * Set absent temperature sensor values to HOTPLUGGED
2161  */
2162 int32_t
2163 set_temp_sensor_properties(psvc_opaque_t hdlp, char *id)
2164 {
2165 	char state[32];
2166 	int32_t status = PSVC_SUCCESS;
2167 
2168 	status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, state);
2169 	if (status == PSVC_FAILURE) {
2170 		return (status);
2171 	}
2172 
2173 	if (strcmp(state, PSVC_HOTPLUGGED) != 0) {
2174 		strcpy(state, PSVC_HOTPLUGGED);
2175 
2176 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
2177 		if (status == PSVC_FAILURE) {
2178 			return (status);
2179 		}
2180 	}
2181 
2182 	return (PSVC_SUCCESS);
2183 }
2184