xref: /illumos-gate/usr/src/cmd/picl/plugins/sun4u/daktari/psvcpolicy/psvcpolicy.c (revision 69a119caa6570c7077699161b7c28b6ee9f8b0f4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * Daktari platform platform specific environment monitoring policies
30  */
31 #include	<poll.h>
32 #include	<syslog.h>
33 #include	<unistd.h>
34 #include	<stdio.h>
35 #include	<stdlib.h>
36 #include	<errno.h>
37 #include	<fcntl.h>
38 #include	<strings.h>
39 #include	<libintl.h>
40 #include	<sys/types.h>
41 #include	<sys/param.h>
42 #include	<config_admin.h>
43 #include	<libdevice.h>
44 #include	<picl.h>
45 #include	<picltree.h>
46 #include	<psvc_objects.h>
47 #include	<sys/i2c/clients/i2c_client.h>
48 #include	<sys/daktari.h>
49 #include	<sys/hpc3130_events.h>
50 #include	<assert.h>
51 #include	<limits.h>
52 #include	<sys/systeminfo.h>
53 
54 /*LINTLIBRARY*/
55 
56 /* resides in libcfgadm */
57 extern cfga_err_t config_change_state(cfga_cmd_t, int, char *const *,
58 	const char *, struct cfga_confirm *, struct cfga_msg *, char **,
59 	cfga_flags_t);
60 /* Local Routine */
61 static int32_t update_gen_fault_led(psvc_opaque_t, char *);
62 static void shutdown_routine(void);
63 static int32_t update_thresholds(psvc_opaque_t hdlp, char *id, int offset);
64 
65 
66 #ifdef DEBUG
67 
68 static int dak_policy_debug = 0;
69 
70 #define	D1SYS_ERR(ARGS) if (dak_policy_debug & 0x1) syslog ARGS;
71 #define	D2SYS_ERR(ARGS) if (dak_policy_debug & 0x2) syslog ARGS;
72 
73 #else
74 
75 #define	D1SYS_ERR(ARGS)
76 #define	D2SYS_ERR(ARGS)
77 
78 #endif
79 
80 #define	I2C_PATH	"/devices/pci@9,700000/ebus@1/i2c@1,30"
81 #define	I2C_NODE	I2C_PATH ":devctl"
82 #define	PCF8574		I2C_PATH "/ioexp@0,%x:pcf8574"
83 #define	PCF8591		I2C_PATH "/adio@0,%x:port_0"
84 #define	FRU		I2C_PATH "/fru@0,%x:fru"
85 #define	HPC3130_DEV	I2C_PATH "/hotplug-controller@0,%2x:port_%1x"
86 #define	GEN_FAULT_LED	"FSP_GEN_FAULT_LED"
87 #define	EMPTY_STRING	"EMPTY"
88 #define	DEVICE_FAILURE_MSG	gettext("WARNING: Device %s failure detected")
89 #define	DEVICE_INSERTED_MSG	gettext("Device %s inserted")
90 #define	DEVICE_REMOVED_MSG	gettext("Device %s removed")
91 #define	PS_UNPLUGGED_MSG	gettext("Device %s unplugged")
92 #define	PS_PLUGGED_MSG		gettext("Device %s Plugged in")
93 #define	DEVICE_OK_MSG		gettext("Device %s OK")
94 #define	SET_LED_FAILED_MSG		\
95 	gettext("Failed to set LED state, id = %s, errno = %d\n")
96 #define	GET_PRESENCE_FAILED_MSG		\
97 	gettext("Failed to get presence attribute, id = %s, errno = %d\n")
98 #define	GET_SENSOR_FAILED_MSG		\
99 	gettext("Failed to get sensor value, id = %s, errno = %d\n")
100 #define	ADD_PS_MSG			\
101 gettext("WARNING: Only 1 Power Supply in system. ADD a 2nd Power Supply.\n")
102 #define	REMOVE_LOAD_MSG			\
103 	gettext("WARNING: Power Supply at 95%% current. Remove some load.\n")
104 #define	PS_OVER_CURRENT_MSG		\
105 	gettext("WARNING: Power Supply overcurrent detected\n")
106 #define	PS_UNDER_CURRENT_MSG		\
107 	gettext("WARNING: PS%d Undercurrent on one or more DC lines\n")
108 #define	DEVICE_UNKNOWN_MSG	gettext("Unknown device %s instance %d\n")
109 #define	DEVICE_HANDLE_FAIL_MSG		\
110 	gettext("Failed to get device handle for %s, errno = %d\n")
111 #define	DEVTREE_NODE_CREATE_FAILED	\
112 	gettext("psvc PICL plugin: Failed to create node for %s, errno = %d")
113 #define	DEVTREE_NODE_DELETE_FAILED	\
114 	gettext("psvc PICL plugin: Failed to delete node for %s, errno = %d")
115 #define	DISK_FAULT_MSG		gettext("%s: Error Reported\n")
116 #define	DISK_OK_MSG		gettext("%s: Error Cleared\n")
117 #define	SET_FANSPEED_FAILED_MSG		\
118 	gettext("Failed to set fan speed, id = %s, errno = %d\n")
119 #define	GET_ATTR_FRU_FAILED_MSG	gettext("Failed psvc_get_attr for FRU info\n")
120 #define	NO_FRU_INFO_MSG			\
121 	gettext("No FRU Information for %s using default module card\n")
122 
123 #define	DAKTARI_MAX_PS	3
124 #define	DAK_MAX_PS_I_SENSORS 4
125 #define	DAK_MAX_DISKS	12
126 #define	DAK_MAX_CPU_MOD	4
127 #define	DAK_MAX_FAULT_SENSORS 3
128 #define	DAK_MAX_FANS 10
129 
130 static int co_ps = 0;
131 static char *shutdown_string = "shutdown -y -g 60 -i 5 \"OVERTEMP condition\"";
132 
133 typedef struct i2c_hp {
134 	int32_t		addr[2];
135 	char		name[256];
136 	char		compatible[256];
137 } i2c_hp_t;
138 
139 typedef struct seg_desc {
140 	int32_t segdesc;
141 	int16_t segoffset;
142 	int16_t seglength;
143 } seg_desc_t;
144 
145 static int32_t threshold_names[] = {
146 	PSVC_HW_LO_SHUT_ATTR,
147 	PSVC_LO_SHUT_ATTR,
148 	PSVC_LO_WARN_ATTR,
149 	PSVC_NOT_USED,			/* LOW MODE which is not used */
150 	PSVC_OPTIMAL_TEMP_ATTR,
151 	PSVC_HI_WARN_ATTR,
152 	PSVC_HI_SHUT_ATTR,
153 	PSVC_HW_HI_SHUT_ATTR
154 };
155 
156 /*
157  * The I2C bus is noisy, and the state may be incorrectly reported as
158  * having changed.  When the state changes, we attempt to confirm by
159  * retrying.  If any retries indicate that the state has not changed, we
160  * assume the state change(s) were incorrect and the state has not changed.
161  * The following variables are used to store the tuneable values read in
162  * from the optional i2cparam.conf file for this shared object library.
163  */
164 static int n_retry_pshp_status = PSVC_NUM_OF_RETRIES;
165 static int retry_sleep_pshp_status = 1;
166 static int n_read_overcurrent = PSVC_THRESHOLD_COUNTER;
167 static int n_read_undercurrent = PSVC_THRESHOLD_COUNTER;
168 static int n_retry_devicefail = PSVC_NUM_OF_RETRIES;
169 static int retry_sleep_devicefail = 1;
170 static int n_read_fanfault = PSVC_THRESHOLD_COUNTER;
171 static int n_retry_pshp = PSVC_NUM_OF_RETRIES;
172 static int retry_sleep_pshp = 1;
173 static int n_retry_diskfault = PSVC_NUM_OF_RETRIES;
174 static int retry_sleep_diskfault = 1;
175 static int n_retry_temp_shutdown = PSVC_NUM_OF_RETRIES;
176 static int retry_sleep_temp_shutdown = 1;
177 
178 typedef struct {
179 	int *pvar;
180 	char *texttag;
181 } i2c_noise_param_t;
182 
183 static i2c_noise_param_t i2cparams[] = {
184 	&n_retry_pshp_status, "n_retry_pshp_status",
185 	&retry_sleep_pshp_status, "retry_sleep_pshp_status",
186 	&n_read_overcurrent, "n_read_overcurrent",
187 	&n_read_undercurrent, "n_read_undercurrent",
188 	&n_retry_devicefail, "n_retry_devicefail",
189 	&retry_sleep_devicefail, "retry_sleep_devicefail",
190 	&n_read_fanfault, "n_read_fanfault",
191 	&n_retry_pshp, "n_retry_pshp",
192 	&retry_sleep_pshp, "retry_sleep_pshp",
193 	&n_retry_diskfault, "n_retry_diskfault",
194 	&retry_sleep_diskfault, "retry_sleep_diskfault",
195 	&n_retry_temp_shutdown, "n_retry_temp_shutdown",
196 	&retry_sleep_temp_shutdown, "retry_sleep_temp_shutdown",
197 	NULL, NULL
198 };
199 
200 #pragma init(i2cparams_load)
201 
202 static void
203 i2cparams_debug(i2c_noise_param_t *pi2cparams, char *platform,
204 	int usingDefaults)
205 {
206 	char s[128];
207 	i2c_noise_param_t *p;
208 
209 	if (!usingDefaults) {
210 		(void) snprintf(s, sizeof (s),
211 		    "# Values from /usr/platform/%s/lib/i2cparam.conf\n",
212 		    platform);
213 		syslog(LOG_WARNING, "%s", s);
214 	} else {
215 		/* no file - we're using the defaults */
216 		(void) snprintf(s, sizeof (s),
217 "# No /usr/platform/%s/lib/i2cparam.conf file, using defaults\n",
218 		    platform);
219 	}
220 	(void) fputs(s, stdout);
221 	p = pi2cparams;
222 	while (p->pvar != NULL) {
223 		(void) snprintf(s, sizeof (s), "%s %d\n", p->texttag,
224 		    *(p->pvar));
225 		if (!usingDefaults)
226 			syslog(LOG_WARNING, "%s", s);
227 		(void) fputs(s, stdout);
228 		p++;
229 	}
230 }
231 
232 static void
233 i2cparams_load(void)
234 {
235 	FILE *fp;
236 	char filename[PATH_MAX];
237 	char platform[64];
238 	char s[128];
239 	char var[128];
240 	int val;
241 	i2c_noise_param_t *p;
242 
243 	if (sysinfo(SI_PLATFORM, platform, sizeof (platform)) == -1) {
244 		syslog(LOG_ERR, "sysinfo error %s\n", strerror(errno));
245 		return;
246 	}
247 	(void) snprintf(filename, sizeof (filename),
248 	    "/usr/platform/%s/lib/i2cparam.conf", platform);
249 	/* read thru the i2cparam.conf file and set variables */
250 	if ((fp = fopen(filename, "r")) != NULL) {
251 		while (fgets(s, sizeof (s), fp) != NULL) {
252 			if (s[0] == '#') /* skip comment lines */
253 				continue;
254 			/* try to find a string match and get the value */
255 			if (sscanf(s, "%127s %d", var, &val) != 2)
256 				continue;
257 			if (val < 1)
258 				val = 1;  /* clamp min value */
259 			p = &(i2cparams[0]);
260 			while (p->pvar != NULL) {
261 				if (strncmp(p->texttag, var, sizeof (var)) ==
262 				    0) {
263 					*(p->pvar) = val;
264 					break;
265 				}
266 				p++;
267 			}
268 		}
269 		(void) fclose(fp);
270 	}
271 	/* output the values of the parameters */
272 	i2cparams_debug(&(i2cparams[0]), platform, ((fp == NULL)? 1 : 0));
273 }
274 
275 int32_t
276 psvc_MB_update_thresholds_0(psvc_opaque_t hdlp, char *id, int offset)
277 {
278 	int IO_offset = 0xd;
279 	int32_t err;
280 
281 	err = update_thresholds(hdlp, id, IO_offset);
282 
283 	return (err);
284 }
285 
286 int32_t
287 psvc_IO_update_thresholds_0(psvc_opaque_t hdlp, char *id, int offset)
288 {
289 	int IO_offset = 0x8;
290 	int32_t err;
291 
292 	err = update_thresholds(hdlp, id, IO_offset);
293 
294 	return (err);
295 }
296 
297 int32_t
298 psvc_DBP_update_thresholds_0(psvc_opaque_t hdlp, char *id, int offset)
299 {
300 	int IO_offset = 0x7;
301 	int32_t err;
302 
303 	err = update_thresholds(hdlp, id, IO_offset);
304 
305 	return (err);
306 }
307 
308 /*
309  * used to determine if a change of state occured. valid when states
310  * are strings.
311  */
312 static int8_t
313 change_of_state_str(char *state1, char *check1, char *state2, char *check2)
314 {
315 	int change = 0;
316 
317 	if ((strcmp(state1, check1) == 0) && (strcmp(state2, check2) != 0))
318 		change = 1;
319 	if ((strcmp(state1, check1) != 0) && (strcmp(state2, check2) == 0))
320 		change = 1;
321 
322 	return (change);
323 }
324 
325 /*
326  * Update thresholds tries to read the temperature thresholds from the FRU
327  * SEEproms and then updates the thresholds in the object by overriding the
328  * hardcoded thresholds.  For Daktari it is an Error if the FRU does not
329  * contain the segment that had the temperature thresholds.
330  */
331 static int32_t
332 update_thresholds(psvc_opaque_t hdlp, char *id, int offset)
333 {
334 	int32_t status = PSVC_SUCCESS;
335 	fru_info_t fru_data;
336 	char *fru, seg_name[2];
337 	int8_t seg_count, temp_array[8];
338 	int32_t match_count, i, j, seg_desc_start = 0x1806, temp_address;
339 	int32_t seg_found, temp;
340 	boolean_t present;
341 	seg_desc_t segment;
342 
343 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
344 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT))
345 		return (status);
346 
347 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &match_count,
348 	    PSVC_FRU);
349 	if (status == PSVC_FAILURE)
350 		return (status);
351 
352 	for (i = 0; i < match_count; i++) {
353 		seg_found = 0;
354 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
355 		    &fru, PSVC_FRU, i);
356 		if (status != PSVC_SUCCESS)
357 			return (status);
358 
359 		fru_data.buf_start = 0x1805;
360 		fru_data.buf = (char *)&seg_count;
361 		fru_data.read_size = 1;
362 
363 		status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
364 		    &fru_data);
365 		if (status != PSVC_SUCCESS) {
366 			return (status);
367 		}
368 		for (j = 0; (j < seg_count) && (!seg_found); j++) {
369 			fru_data.buf_start = seg_desc_start;
370 			fru_data.buf = seg_name;
371 			fru_data.read_size = 2;
372 
373 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
374 			    &fru_data);
375 
376 			seg_desc_start = seg_desc_start + 2;
377 			fru_data.buf_start = seg_desc_start;
378 			fru_data.buf = (char *)&segment;
379 			fru_data.read_size = sizeof (seg_desc_t);
380 
381 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
382 			    &fru_data);
383 			if (status != PSVC_SUCCESS) {
384 				syslog(LOG_ERR,
385 				    "Failed psvc_get_attr for FRU info\n");
386 				return (status);
387 			}
388 			seg_desc_start = seg_desc_start + sizeof (seg_desc_t);
389 			if (memcmp(seg_name, "SC", 2) == 0)
390 				seg_found = 1;
391 		}
392 		if (seg_found) {
393 			temp_address = segment.segoffset + offset;
394 			fru_data.buf_start = temp_address;
395 			fru_data.buf = (char *)&temp_array;
396 			fru_data.read_size = sizeof (temp_array);
397 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
398 			    &fru_data);
399 			if (status != PSVC_SUCCESS) {
400 				syslog(LOG_ERR,
401 				    "Failed psvc_get_attr for FRU info\n");
402 				return (status);
403 			} else {
404 				for (j = 0; j < sizeof (temp_array); j++) {
405 					if (threshold_names[j] ==
406 					    PSVC_NOT_USED)
407 						continue;
408 					temp = temp_array[j];
409 					status = psvc_set_attr(hdlp, id,
410 					    threshold_names[j], &temp);
411 					if (status != PSVC_SUCCESS) {
412 						return (status);
413 					}
414 				}
415 			}
416 		} else {
417 			syslog(LOG_ERR, "No FRU Information for %s"
418 			    " using default temperatures\n", id);
419 		}
420 	}
421 	return (status);
422 }
423 
424 int32_t
425 psvc_fan_init_speed_0(psvc_opaque_t hdlp, char *id)
426 {
427 	int32_t status = PSVC_SUCCESS;
428 	boolean_t present;
429 	char *control_id;
430 	int32_t init_speed = 0;
431 
432 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
433 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT))
434 		return (status);
435 
436 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR, &control_id,
437 	    PSVC_FAN_DRIVE_CONTROL, 0);
438 	if (status != PSVC_SUCCESS)
439 		return (status);
440 
441 	status = psvc_set_attr(hdlp, control_id, PSVC_CONTROL_VALUE_ATTR,
442 	    &init_speed);
443 	if (status == PSVC_FAILURE) {
444 		syslog(LOG_ERR, SET_FANSPEED_FAILED_MSG, control_id, errno);
445 		return (status);
446 	}
447 
448 	return (status);
449 }
450 
451 int32_t
452 psvc_update_setpoint_0(psvc_opaque_t hdlp, char *id)
453 {
454 	int32_t status = PSVC_SUCCESS;
455 	char *temp_sensor;
456 	int32_t match_count, i, temp;
457 	int16_t lowest_temp = 500;
458 	boolean_t present;
459 
460 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
461 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT))
462 		return (status);
463 
464 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &match_count,
465 	    PSVC_DEV_TEMP_SENSOR);
466 	if (status == PSVC_FAILURE)
467 		return (status);
468 
469 	for (i = 0; i < match_count; i++) {
470 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
471 		    &temp_sensor, PSVC_DEV_TEMP_SENSOR, i);
472 		if (status != PSVC_SUCCESS)
473 			return (status);
474 		status = psvc_get_attr(hdlp, temp_sensor,
475 		    PSVC_OPTIMAL_TEMP_ATTR, &temp);
476 		if (status != PSVC_SUCCESS) {
477 			syslog(LOG_ERR, "Failed to get Optimal temp for %s\n",
478 			    temp_sensor);
479 			return (status);
480 		}
481 		if (temp < lowest_temp)
482 			lowest_temp = temp;
483 	}
484 	status = psvc_set_attr(hdlp, id, PSVC_SETPOINT_ATTR, &lowest_temp);
485 	if (status == PSVC_FAILURE) {
486 		syslog(LOG_ERR, "Failed to change setpoint for %s\n", id);
487 		return (status);
488 	}
489 	return (status);
490 }
491 
492 int32_t
493 psvc_remove_missing_nodes_0(psvc_opaque_t hdlp, char *id)
494 {
495 	int32_t status = PSVC_SUCCESS;
496 	char state[32];
497 	char *physical_dev;
498 	int32_t i, device_count;
499 	char parent_path[256];
500 	picl_nodehdl_t child_node;
501 	boolean_t present;
502 
503 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR,
504 	    &device_count, PSVC_PHYSICAL_DEVICE);
505 	if (status == PSVC_FAILURE)
506 		return (status);
507 
508 	for (i = 0; i < device_count; i++) {
509 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
510 		    &physical_dev, PSVC_PHYSICAL_DEVICE, i);
511 		if (status != PSVC_SUCCESS)
512 			return (status);
513 		if (strncmp(physical_dev, "LTC1427", 7) == 0)
514 			continue;
515 		status = psvc_get_attr(hdlp, physical_dev,
516 		    PSVC_PROBE_RESULT_ATTR, state);
517 		if (status != PSVC_SUCCESS)
518 			continue;
519 		status = psvc_get_attr(hdlp, physical_dev, PSVC_PRESENCE_ATTR,
520 		    &present);
521 		if (status == PSVC_FAILURE) {
522 			syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG, physical_dev,
523 			    errno);
524 			return (status);
525 		}
526 
527 		if ((strcmp(state, PSVC_ERROR) == 0) &&
528 		    (present == PSVC_PRESENT)) {
529 			/* convert name to node, and parent path */
530 			psvcplugin_lookup(physical_dev, parent_path,
531 			    &child_node);
532 			/* Device removed */
533 			ptree_delete_node(child_node);
534 		}
535 	}
536 	return (status);
537 }
538 
539 int32_t
540 psvc_check_ps_hotplug_status_0(psvc_opaque_t hdlp, char *id)
541 {
542 	char		fail_valid_switch_id[PICL_PROPNAMELEN_MAX];
543 	int32_t		status = PSVC_SUCCESS;
544 	char		valid_switch_state[32];
545 	char		state[32], fault[32];
546 	int32_t		led_count, j;
547 	char		*led_id;
548 	char		led_state[32];
549 	boolean_t	present;
550 	static int8_t	hotplug_failed_count = 0;
551 	static int	unplugged_ps = 0;
552 	int	retry;
553 	char		*unplugged_id;
554 
555 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
556 	if (status == PSVC_FAILURE) {
557 		syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG, id, errno);
558 		return (status);
559 	}
560 
561 	if (present == PSVC_ABSENT) {
562 		errno = ENODEV;
563 		return (PSVC_FAILURE);
564 	}
565 
566 	snprintf(fail_valid_switch_id, sizeof (fail_valid_switch_id), "%s%s",
567 	    id, "_SENSOR_VALID_SWITCH");
568 
569 	retry = 0;
570 	do {
571 		if (retry)
572 			(void) sleep(retry_sleep_pshp_status);
573 		status = psvc_get_attr(hdlp, fail_valid_switch_id,
574 		    PSVC_STATE_ATTR, valid_switch_state);
575 		if (status == PSVC_FAILURE) {
576 			if (hotplug_failed_count == 0) {
577 				/*
578 				 * First time the get_attr call failed
579 				 * set count so that if we fail again
580 				 * we will know
581 				 */
582 				hotplug_failed_count = 1;
583 				/*
584 				 * We probably failed because the power
585 				 * supply was just insterted or removed
586 				 * before the get_attr call. We then
587 				 * return from this policy successfully
588 				 * knowing it will be run again shortly
589 				 * with the right PS state.
590 				 */
591 				return (PSVC_SUCCESS);
592 			} else {
593 				/*
594 				 * We have failed before and so this
595 				 * we will consider a hardware problem
596 				 * and it should be reported
597 				 */
598 				syslog(LOG_ERR,
599 				    "Failed getting %s State: ",
600 				    "ps_hotplug_status_0\n",
601 				    fail_valid_switch_id);
602 				return (status);
603 			}
604 		}
605 		/*
606 		 * Because we have successfully gotten a value from
607 		 * the i2c device on the PS we will set the
608 		 * failed_count to 0
609 		 */
610 		hotplug_failed_count = 0;
611 
612 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, state);
613 		if (status == PSVC_FAILURE)
614 			return (status);
615 		retry++;
616 		/*
617 		 * check to see if we need to retry. the conditions are:
618 		 *
619 		 * valid_switch_state	state			retry
620 		 * --------------------------------------------------
621 		 *	PSVC_OFF	!PSVC_HOTPLUGGED	yes
622 		 *	PSVC_ON		PSVC_HOTPLUGGED		yes
623 		 *	PSVC_OFF	PSVC_HOTPLUGGED		no
624 		 *	PSVC_ON		!PSVC_HOTPLUGGED	no
625 		 */
626 	} while ((retry < n_retry_pshp_status) &&
627 	    change_of_state_str(valid_switch_state, PSVC_OFF,
628 	    state, PSVC_HOTPLUGGED));
629 
630 	if ((strcmp(valid_switch_state, PSVC_OFF) == 0) &&
631 	    (strcmp(state, PSVC_HOTPLUGGED) != 0)) {
632 		strcpy(state, PSVC_HOTPLUGGED);
633 		strcpy(fault, PSVC_NO_FAULT);
634 		strcpy(led_state, PSVC_LED_OFF);
635 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
636 		    state);
637 		if (status == PSVC_FAILURE)
638 			return (status);
639 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR,
640 		    &led_count, PSVC_DEV_FAULT_LED);
641 		if (status == PSVC_FAILURE)
642 			return (status);
643 
644 		for (j = 0; j < led_count; j++) {
645 
646 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
647 			    &led_id, PSVC_DEV_FAULT_LED, j);
648 			if (status != PSVC_SUCCESS)
649 				return (status);
650 
651 			status = psvc_set_attr(hdlp, led_id,
652 			    PSVC_LED_STATE_ATTR, led_state);
653 			if (status != PSVC_SUCCESS) {
654 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
655 				    errno);
656 				return (status);
657 			}
658 
659 		}
660 		strcpy(led_state, PSVC_LED_ON);
661 		status = psvc_set_attr(hdlp, "FSP_POWER_FAULT_LED",
662 		    PSVC_LED_STATE_ATTR, led_state);
663 		if (status != PSVC_SUCCESS) {
664 			syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id, errno);
665 			return (status);
666 		}
667 		unplugged_id = id + 2;
668 		unplugged_ps = unplugged_ps | (1 << (int)strtol(unplugged_id,
669 		    (char **)NULL, 10));
670 		status = update_gen_fault_led(hdlp, GEN_FAULT_LED);
671 		syslog(LOG_ERR, PS_UNPLUGGED_MSG, id);
672 		return (status);
673 	}
674 
675 	if ((strcmp(valid_switch_state, PSVC_ON) == 0) &&
676 	    (strcmp(state, PSVC_HOTPLUGGED) == 0)) {
677 		strcpy(state, PSVC_OK);
678 		strcpy(fault, PSVC_NO_FAULT);
679 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
680 		if (status == PSVC_FAILURE)
681 			return (status);
682 		unplugged_id = id + 2;
683 		unplugged_ps = unplugged_ps ^ (1 << (int)strtol(unplugged_id,
684 		    (char **)NULL, 10));
685 		if (unplugged_ps == 0) {
686 			strcpy(led_state, PSVC_LED_OFF);
687 			status = psvc_set_attr(hdlp, "FSP_POWER_FAULT_LED",
688 			    PSVC_LED_STATE_ATTR, led_state);
689 			if (status != PSVC_SUCCESS) {
690 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
691 				    errno);
692 				return (status);
693 			}
694 			status = update_gen_fault_led(hdlp, GEN_FAULT_LED);
695 		}
696 		syslog(LOG_ERR, PS_PLUGGED_MSG, id);
697 	}
698 
699 	return (status);
700 }
701 
702 int32_t
703 psvc_ps_overcurrent_check_policy_0(psvc_opaque_t hdlp, char *system)
704 {
705 	int32_t status = PSVC_SUCCESS;
706 	boolean_t present;
707 	static char *sensor_id[DAKTARI_MAX_PS][DAK_MAX_PS_I_SENSORS];
708 	static char *power_supply_id[DAKTARI_MAX_PS] = {NULL};
709 	int32_t i, j;
710 	int32_t amps, oc_flag = 0, ps_present = 0;
711 	static int32_t hi_warn[DAKTARI_MAX_PS][DAK_MAX_PS_I_SENSORS];
712 	char state[32];
713 	static int8_t overcurrent_failed_check = 0;
714 	static int threshold_counter = 0;
715 
716 	if (power_supply_id[0] == NULL) {
717 		for (i = 0; i < DAKTARI_MAX_PS; i++) {
718 			status = psvc_get_attr(hdlp, system,
719 			    PSVC_ASSOC_ID_ATTR, &(power_supply_id[i]),
720 			    PSVC_PS, i);
721 			if (status != PSVC_SUCCESS)
722 				return (status);
723 			for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
724 				status = psvc_get_attr(hdlp,
725 				    power_supply_id[i], PSVC_ASSOC_ID_ATTR,
726 				    &(sensor_id[i][j]), PSVC_PS_I_SENSOR, j);
727 				if (status != PSVC_SUCCESS)
728 					return (status);
729 				status = psvc_get_attr(hdlp, sensor_id[i][j],
730 				    PSVC_HI_WARN_ATTR, &(hi_warn[i][j]));
731 				if (status != PSVC_SUCCESS)
732 					return (status);
733 			}
734 		}
735 	}
736 
737 	for (i = 0; i < DAKTARI_MAX_PS; i++) {
738 		status = psvc_get_attr(hdlp, power_supply_id[i],
739 		    PSVC_PRESENCE_ATTR, &present);
740 		if (status == PSVC_FAILURE) {
741 			syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG,
742 			    power_supply_id[i], errno);
743 			return (status);
744 		}
745 
746 		if (present == PSVC_ABSENT) {
747 			continue;
748 		}
749 
750 		status = psvc_check_ps_hotplug_status_0(hdlp,
751 		    power_supply_id[i]);
752 		if (status == PSVC_FAILURE)
753 			return (status);
754 
755 		status = psvc_get_attr(hdlp, power_supply_id[i],
756 		    PSVC_STATE_ATTR, state);
757 		if (status == PSVC_FAILURE)
758 			return (status);
759 
760 		if (strcmp(state, PSVC_HOTPLUGGED) == 0) {
761 			continue;
762 		} else {
763 			ps_present++;
764 		}
765 
766 		for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
767 			status = psvc_get_attr(hdlp, sensor_id[i][j],
768 			    PSVC_SENSOR_VALUE_ATTR, &amps);
769 			if (status != PSVC_SUCCESS) {
770 				if (overcurrent_failed_check == 0) {
771 					/*
772 					 * First time the get_attr call
773 					 * failed  set count so that if we
774 					 * fail again we will know
775 					 */
776 					overcurrent_failed_check = 1;
777 					/*
778 					 * We probably failed because the power
779 					 * supply was just insterted or removed
780 					 * before the get_attr call. We then
781 					 * return from this policy successfully
782 					 * knowing it will be run again shortly
783 					 * with the right PS state.
784 					 */
785 					return (PSVC_SUCCESS);
786 				} else {
787 					/*
788 					 * We have failed before and so this we
789 					 * will consider a hardware problem and
790 					 * it should be reported.
791 					 */
792 					syslog(LOG_ERR,
793 					    "Failed getting %s sensor value",
794 					    sensor_id[i][j]);
795 					return (status);
796 				}
797 			}
798 			/*
799 			 * Because we have successfully gotten a value from the
800 			 * i2c device on the PS we will set the failed_count
801 			 * to 0.
802 			 */
803 			overcurrent_failed_check = 0;
804 
805 			if (amps >= hi_warn[i][j]) {
806 				oc_flag = 1;
807 			}
808 		}
809 	}
810 
811 	if (oc_flag) {
812 		/*
813 		 * Because we observed an overcurrent
814 		 * condition, we increment threshold_counter.
815 		 * Once threshold_counter reaches the value
816 		 * of n_read_overcurrent we log the event.
817 		 */
818 		threshold_counter++;
819 		if (threshold_counter == n_read_overcurrent) {
820 			threshold_counter = 0;
821 			if (ps_present == 1) {
822 				syslog(LOG_ERR, PS_OVER_CURRENT_MSG);
823 				syslog(LOG_ERR, ADD_PS_MSG);
824 			} else {
825 				syslog(LOG_ERR, PS_OVER_CURRENT_MSG);
826 				syslog(LOG_ERR, REMOVE_LOAD_MSG);
827 			}
828 		}
829 	} else {
830 		threshold_counter = 0;
831 	}
832 
833 	return (PSVC_SUCCESS);
834 }
835 
836 int32_t
837 psvc_ps_undercurrent_check(psvc_opaque_t hdlp, char *id, int32_t *uc_flag)
838 {
839 	int32_t status = PSVC_SUCCESS;
840 	boolean_t present;
841 	static char *sensor_id[DAK_MAX_PS_I_SENSORS];
842 	int32_t j;
843 	int32_t amps;
844 	static int32_t lo_warn[DAK_MAX_PS_I_SENSORS];
845 	static int8_t undercurrent_failed_check = 0;
846 
847 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
848 	if (status == PSVC_FAILURE) {
849 		syslog(LOG_ERR, GET_PRESENCE_FAILED_MSG, id, errno);
850 		return (status);
851 	}
852 
853 	if (present == PSVC_ABSENT) {
854 		errno = ENODEV;
855 		return (PSVC_FAILURE);
856 	}
857 
858 	for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
859 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
860 		    &(sensor_id[j]), PSVC_PS_I_SENSOR, j);
861 		if (status != PSVC_SUCCESS)
862 			return (status);
863 		status = psvc_get_attr(hdlp, sensor_id[j],
864 		    PSVC_LO_WARN_ATTR, &(lo_warn[j]));
865 		if (status != PSVC_SUCCESS)
866 			return (status);
867 	}
868 
869 	*uc_flag = 0;
870 	for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
871 		status = psvc_get_attr(hdlp, sensor_id[j],
872 		    PSVC_SENSOR_VALUE_ATTR, &amps);
873 		if (status != PSVC_SUCCESS) {
874 			if (undercurrent_failed_check == 0) {
875 				/*
876 				 * First time the get_attr call
877 				 * failed  set count so that if we
878 				 * fail again we will know.
879 				 */
880 				undercurrent_failed_check = 1;
881 				/*
882 				 * We probably failed because the power
883 				 * supply was just inserted or removed
884 				 * before the get_attr call. We then
885 				 * return from this policy successfully
886 				 * knowing it will be run again shortly
887 				 * with the right PS state.
888 				 */
889 				return (PSVC_SUCCESS);
890 			} else {
891 				/*
892 				 * Repeated failures are logged.
893 				 */
894 				syslog(LOG_ERR,
895 				    "Failed getting %s sensor value",
896 				    sensor_id[j]);
897 				return (status);
898 			}
899 		}
900 		/*
901 		 * Because we have successfully gotten a value from the
902 		 * i2c device on the PS we will set the failed_count
903 		 * to 0.
904 		 */
905 		undercurrent_failed_check = 0;
906 
907 		if (amps <= lo_warn[j]) {
908 			*uc_flag = 1;
909 			return (PSVC_SUCCESS);
910 		}
911 	}
912 
913 	return (PSVC_SUCCESS);
914 }
915 
916 int32_t
917 psvc_ps_device_fail_notifier_policy_0(psvc_opaque_t hdlp, char *system)
918 {
919 	static char *ps_id[DAKTARI_MAX_PS] = {NULL};
920 	static char *sensor_id[DAKTARI_MAX_PS][DAK_MAX_FAULT_SENSORS];
921 	char *led_id = "FSP_POWER_FAULT_LED";
922 	int i, j, uc_flag;
923 	char state[32], fault[32], previous_state[32], past_state[32];
924 	char led_state[32];
925 	char bad_sensors[DAK_MAX_FAULT_SENSORS][256];
926 	static int threshold_counter[DAKTARI_MAX_PS];
927 	int32_t status = PSVC_SUCCESS;
928 	boolean_t present;
929 	int fail_state;
930 	static int8_t device_fail_failed_check = 0;
931 	int retry, should_retry;
932 
933 	if (ps_id[0] == NULL) {
934 		for (i = 0; i < DAKTARI_MAX_PS; i++) {
935 			status = psvc_get_attr(hdlp, system,
936 			    PSVC_ASSOC_ID_ATTR, &(ps_id[i]), PSVC_PS, i);
937 			if (status != PSVC_SUCCESS)
938 				return (status);
939 			for (j = 0; j < DAK_MAX_FAULT_SENSORS; j++) {
940 				status = psvc_get_attr(hdlp, ps_id[i],
941 				    PSVC_ASSOC_ID_ATTR, &(sensor_id[i][j]),
942 				    PSVC_DEV_FAULT_SENSOR, j);
943 				if (status != PSVC_SUCCESS)
944 					return (status);
945 			}
946 		}
947 	}
948 
949 	for (i = 0; i < DAKTARI_MAX_PS; i++) {
950 		fail_state = 0;
951 		status = psvc_get_attr(hdlp, ps_id[i], PSVC_PRESENCE_ATTR,
952 		    &present);
953 		if (status == PSVC_FAILURE)
954 			return (status);
955 
956 		if (present == PSVC_ABSENT) {
957 			errno = ENODEV;
958 			return (PSVC_FAILURE);
959 		}
960 
961 		status = psvc_check_ps_hotplug_status_0(hdlp, ps_id[i]);
962 		if (status == PSVC_FAILURE)
963 			return (status);
964 
965 		status = psvc_get_attr(hdlp, ps_id[i], PSVC_STATE_ATTR,
966 		    past_state);
967 		if (status == PSVC_FAILURE)
968 			return (status);
969 
970 		if (strcmp(past_state, PSVC_HOTPLUGGED) == 0) {
971 			return (PICL_SUCCESS);
972 		}
973 
974 		retry = 0;
975 		do {
976 			if (retry)
977 				(void) sleep(retry_sleep_devicefail);
978 			fail_state = 0;
979 			should_retry = 0;
980 			for (j = 0; j < DAK_MAX_FAULT_SENSORS; ++j) {
981 				status = psvc_get_attr(hdlp, sensor_id[i][j],
982 				    PSVC_SWITCH_STATE_ATTR, state);
983 				if (status != PSVC_SUCCESS) {
984 					if (device_fail_failed_check == 0) {
985 						/*
986 						 * First time the get_attr call
987 						 * failed  set count so that
988 						 * if we fail again we will know
989 						 */
990 						device_fail_failed_check = 1;
991 						/*
992 						 * We probably failed because
993 						 * the power supply was just
994 						 * insterted or removed before
995 						 * the get_attr call. We then
996 						 * return from this policy
997 						 * successfully knowing it will
998 						 * be run again shortly
999 						 * with the right PS state.
1000 						 */
1001 						return (PSVC_SUCCESS);
1002 					} else {
1003 						/*
1004 						 * We have failed before and
1005 						 * so this we will consider a
1006 						 * hardware problem and
1007 						 * it should be reported.
1008 						 */
1009 						syslog(LOG_ERR, "Failed in "
1010 						    "getting sensor state for "
1011 						    "%s\n", sensor_id[i][j]);
1012 
1013 						return (status);
1014 					}
1015 				}
1016 
1017 				/*
1018 				 * Because we have successfully gotten
1019 				 * a value from the i2c device on the
1020 				 * PS we will set the failed_count to 0.
1021 				 */
1022 				device_fail_failed_check = 0;
1023 
1024 				/*
1025 				 * If we find that the sensor is on we
1026 				 * fill in the name of the sensor in
1027 				 * the bad_sensor array. If the sensor
1028 				 * is off we use EMPTY_STRING as a check
1029 				 * later on as to when NOT to print out
1030 				 * what is in bad_sensor[].
1031 				 */
1032 				if (strcmp(state, PSVC_SWITCH_ON) == 0) {
1033 					fail_state++;
1034 					strlcpy(bad_sensors[j], sensor_id[i][j],
1035 					    sizeof (bad_sensors[j]));
1036 				} else {
1037 					strcpy(bad_sensors[j], EMPTY_STRING);
1038 				}
1039 			}
1040 			retry++;
1041 			/*
1042 			 * check to see if we need to retry. the conditions are:
1043 			 *
1044 			 * fail_state		past_state		retry
1045 			 * --------------------------------------------------
1046 			 *	+		PSVC_OK			yes
1047 			 *	0		PSVC_ERROR		yes
1048 			 *	+		PSVC_ERROR		no
1049 			 *	0		PSVC_OK			no
1050 			 */
1051 			if ((fail_state > 0) &&
1052 			    (strcmp(past_state, PSVC_OK) == 0)) {
1053 				should_retry = 1;
1054 			} else if ((fail_state == 0) &&
1055 			    (strcmp(past_state, PSVC_ERROR) == 0)) {
1056 				should_retry = 1;
1057 			}
1058 		} while ((retry < n_retry_devicefail) && should_retry);
1059 
1060 		/* Under current check */
1061 		status = psvc_ps_undercurrent_check(hdlp, ps_id[i], &uc_flag);
1062 
1063 		if (status != PSVC_FAILURE) {
1064 			if (uc_flag) {
1065 				/*
1066 				 * Because we observed an undercurrent
1067 				 * condition, we increment threshold counter.
1068 				 * Once threshold counter reaches the value
1069 				 * of n_read_undercurrent we log the event.
1070 				 */
1071 				threshold_counter[i]++;
1072 				if (threshold_counter[i] >=
1073 				    n_read_undercurrent) {
1074 					fail_state++;
1075 					syslog(LOG_ERR, PS_UNDER_CURRENT_MSG,
1076 					    i);
1077 				}
1078 			} else {
1079 				threshold_counter[i] = 0;
1080 			}
1081 		}
1082 
1083 		if (fail_state != 0) {
1084 			strcpy(state, PSVC_ERROR);
1085 			strcpy(fault, PSVC_GEN_FAULT);
1086 		} else {
1087 			strcpy(state, PSVC_OK);
1088 			strcpy(fault, PSVC_NO_FAULT);
1089 		}
1090 
1091 		status = psvc_set_attr(hdlp, ps_id[i], PSVC_STATE_ATTR, state);
1092 		if (status != PSVC_SUCCESS)
1093 			return (status);
1094 
1095 		status = psvc_set_attr(hdlp, ps_id[i], PSVC_FAULTID_ATTR,
1096 		    fault);
1097 		if (status != PSVC_SUCCESS)
1098 			return (status);
1099 
1100 		status = psvc_get_attr(hdlp, ps_id[i], PSVC_PREV_STATE_ATTR,
1101 		    previous_state);
1102 		if (status != PSVC_SUCCESS)
1103 			return (status);
1104 
1105 		if (strcmp(state, previous_state) != 0) {
1106 			char dev_label[32];
1107 
1108 			psvc_get_attr(hdlp, ps_id[i], PSVC_LABEL_ATTR,
1109 			    dev_label);
1110 
1111 			if (strcmp(state, PSVC_ERROR) == 0) {
1112 				syslog(LOG_ERR, DEVICE_FAILURE_MSG, dev_label);
1113 				for (j = 0; j < DAK_MAX_FAULT_SENSORS; ++j) {
1114 					if (strcmp(bad_sensors[j],
1115 					    EMPTY_STRING) != 0)
1116 						syslog(LOG_ERR, "%s\n",
1117 						    bad_sensors[j]);
1118 				}
1119 				strcpy(led_state, PSVC_LED_ON);
1120 			} else {
1121 				syslog(LOG_ERR, DEVICE_OK_MSG, dev_label);
1122 				strcpy(led_state, PSVC_LED_OFF);
1123 			}
1124 
1125 			status = psvc_set_attr(hdlp, led_id,
1126 			    PSVC_LED_STATE_ATTR, led_state);
1127 			if (status != PSVC_SUCCESS) {
1128 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
1129 				    errno);
1130 				return (status);
1131 			}
1132 		}
1133 	}
1134 
1135 	return (PSVC_SUCCESS);
1136 }
1137 
1138 int32_t
1139 psvc_ps_check_and_disable_dr_policy_0(psvc_opaque_t hdlp, char *id)
1140 {
1141 	char		state[32];
1142 	static char	*name[DAKTARI_MAX_PS] = {NULL};
1143 	int		ps_cnt = 0;
1144 	int		i, j;
1145 	int		dr_conf;
1146 	int		fd, rv;
1147 	boolean_t	present;
1148 	char		dev_path[sizeof (HPC3130_DEV)+8];
1149 	unsigned char	controller_names[HPC3130_CONTROLLERS] =
1150 		{ 0xe2, 0xe6, 0xe8, 0xec };
1151 
1152 	if (name[0] == NULL) {
1153 		for (i = 0; i < DAKTARI_MAX_PS; i++) {
1154 			rv = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1155 			    &(name[i]), PSVC_PS, i);
1156 			if (rv != PSVC_SUCCESS)
1157 				return (rv);
1158 		}
1159 	}
1160 
1161 	/*
1162 	 * Go through the power supplies to make sure they're present
1163 	 * and OK.
1164 	 */
1165 	ps_cnt = DAKTARI_MAX_PS;
1166 	for (i = 0; i < DAKTARI_MAX_PS; i++) {
1167 		rv = psvc_get_attr(hdlp, name[i], PSVC_PRESENCE_ATTR,
1168 		    &present);
1169 		if (rv != PSVC_SUCCESS)
1170 			return (rv);
1171 
1172 		if (present != PSVC_PRESENT) {
1173 			ps_cnt--;
1174 			continue;
1175 		} else {
1176 			rv = psvc_get_attr(hdlp, name[i], PSVC_STATE_ATTR,
1177 			    state);
1178 			if (rv != PSVC_SUCCESS)
1179 				return (rv);
1180 
1181 			if (strcmp(state, PSVC_OK))
1182 				ps_cnt--;
1183 		}
1184 	}
1185 
1186 	/*
1187 	 * No change in DR configuration is needed if the new power supply
1188 	 * count equals the current count.
1189 	 */
1190 	if (ps_cnt == co_ps)
1191 		return (PSVC_SUCCESS);
1192 
1193 	/*
1194 	 * Disable DR when hotplugged down to 1 power supply; enable DR when
1195 	 * hotplugged up from 1 supply.
1196 	 */
1197 	assert(ps_cnt);
1198 	if ((co_ps == 0 || co_ps > 1) && ps_cnt != 1) {
1199 		co_ps = ps_cnt;
1200 		return (PSVC_SUCCESS);
1201 	}
1202 	dr_conf = (ps_cnt == 1 ? HPC3130_DR_DISABLE : HPC3130_DR_ENABLE);
1203 	co_ps = ps_cnt;
1204 
1205 	for (i = 0; i < HPC3130_CONTROLLERS; i++) {
1206 		for (j = 0; j < HPC3130_SLOTS; j++) {
1207 			(void) snprintf(dev_path, sizeof (dev_path),
1208 			    HPC3130_DEV, controller_names[i], j);
1209 			fd = open(dev_path, O_RDWR);
1210 			if (fd == -1)
1211 				return (PSVC_FAILURE);
1212 
1213 			rv = ioctl(fd, HPC3130_CONF_DR, &dr_conf);
1214 			close(fd);
1215 			if (rv == -1)
1216 				return (PSVC_FAILURE);
1217 		}
1218 	}
1219 
1220 	return (PSVC_SUCCESS);
1221 }
1222 
1223 int32_t
1224 psvc_fan_blast_shutoff_policy_0(psvc_opaque_t hdlp, char *id)
1225 {
1226 	char		switch_status[32];
1227 	int32_t		status = PSVC_SUCCESS;
1228 
1229 	status = psvc_get_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR, switch_status);
1230 	if (status != PSVC_SUCCESS)
1231 		return (status);
1232 	status = psvc_set_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR,
1233 	    PSVC_SWITCH_OFF);
1234 	if (status != PSVC_SUCCESS)
1235 		return (status);
1236 	status = psvc_set_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR,
1237 	    PSVC_SWITCH_ON);
1238 	if (status != PSVC_SUCCESS)
1239 		return (status);
1240 	status = psvc_set_attr(hdlp, id, PSVC_SWITCH_STATE_ATTR,
1241 	    PSVC_SWITCH_OFF);
1242 
1243 	return (status);
1244 }
1245 
1246 int32_t
1247 psvc_fan_fault_check_policy_0(psvc_opaque_t hdlp, char *system)
1248 {
1249 	static char *fan_id[DAK_MAX_FANS] = {NULL};
1250 	boolean_t enabled;
1251 	int32_t speed;
1252 	int32_t status = PSVC_SUCCESS;
1253 	int r;
1254 	static int threshold_counter = 0;
1255 
1256 	if (fan_id[0] == NULL) {
1257 		for (r = 0; r < DAK_MAX_FANS; r++) {
1258 			status = psvc_get_attr(hdlp, system,
1259 			    PSVC_ASSOC_ID_ATTR, &(fan_id[r]), PSVC_FAN, r);
1260 			if (status != PSVC_SUCCESS)
1261 				return (status);
1262 		}
1263 	}
1264 
1265 	for (r = 0; r < DAK_MAX_FANS; r++) {
1266 		status = psvc_get_attr(hdlp, fan_id[r], PSVC_ENABLE_ATTR,
1267 		    &enabled);
1268 		if (status != PSVC_SUCCESS)
1269 			return (status);
1270 
1271 		if (enabled == PSVC_ENABLED) {
1272 			uint64_t features;
1273 			char *switch_id;
1274 			char switch_state[32], fan_state[32];
1275 			int fan_count, fans;
1276 			char *other_fan_id;
1277 			char fstate[32], ffault[32];
1278 
1279 			/*
1280 			 * If any other fan on the fan tray has an ERROR state,
1281 			 * mark this fan bad and return
1282 			 */
1283 			psvc_get_attr(hdlp, fan_id[r], PSVC_ASSOC_MATCHES_ATTR,
1284 			    &fan_count, PSVC_FAN_TRAY_FANS);
1285 			for (fans = 0; fans < fan_count; ++fans) {
1286 				status = psvc_get_attr(hdlp, fan_id[r],
1287 				    PSVC_ASSOC_ID_ATTR, &other_fan_id,
1288 				    PSVC_FAN_TRAY_FANS, fans);
1289 				if (status == PSVC_FAILURE)
1290 					return (status);
1291 				status = psvc_get_attr(hdlp, other_fan_id,
1292 				    PSVC_STATE_ATTR, fan_state);
1293 				if (status != PSVC_SUCCESS)
1294 					return (status);
1295 
1296 				if (strcmp(fan_state, PSVC_ERROR) == 0) {
1297 					strlcpy(ffault, PSVC_GEN_FAULT,
1298 					    sizeof (ffault));
1299 					status = psvc_set_attr(hdlp, fan_id[r],
1300 					    PSVC_FAULTID_ATTR, ffault);
1301 					if (status != PSVC_SUCCESS)
1302 						return (status);
1303 
1304 					strlcpy(fstate, PSVC_ERROR,
1305 					    sizeof (fstate));
1306 					status = psvc_set_attr(hdlp, fan_id[r],
1307 					    PSVC_STATE_ATTR, fstate);
1308 
1309 					return (status);
1310 				}
1311 			}
1312 
1313 			/*
1314 			 * Select tachometer for IO or CPU primary/secondary
1315 			 * fans.
1316 			 */
1317 			pthread_mutex_lock(&fan_mutex);
1318 
1319 			status = psvc_get_attr(hdlp, fan_id[r],
1320 			    PSVC_ASSOC_ID_ATTR, &switch_id,
1321 			    PSVC_FAN_PRIM_SEC_SELECTOR, 0);
1322 
1323 			if (status != PSVC_FAILURE) {
1324 				status = psvc_get_attr(hdlp, fan_id[r],
1325 				    PSVC_FEATURES_ATTR,	&features);
1326 				if (status == PSVC_FAILURE) {
1327 					pthread_mutex_unlock(&fan_mutex);
1328 					return (status);
1329 				}
1330 
1331 				if (features & PSVC_DEV_PRIMARY)
1332 					strlcpy(switch_state, PSVC_SWITCH_ON,
1333 					    sizeof (switch_state));
1334 				else
1335 					strlcpy(switch_state, PSVC_SWITCH_OFF,
1336 					    sizeof (switch_state));
1337 				status = psvc_set_attr(hdlp, switch_id,
1338 				    PSVC_SWITCH_STATE_ATTR, switch_state);
1339 				if (status == PSVC_FAILURE) {
1340 					pthread_mutex_unlock(&fan_mutex);
1341 					return (status);
1342 				}
1343 
1344 				/* allow time for speed to be determined */
1345 				(void) poll(NULL, 0, 250);
1346 			}
1347 
1348 			status = psvc_get_attr(hdlp, fan_id[r],
1349 			    PSVC_SENSOR_VALUE_ATTR, &speed);
1350 			if (status != PSVC_SUCCESS) {
1351 				pthread_mutex_unlock(&fan_mutex);
1352 				return (status);
1353 			}
1354 
1355 			pthread_mutex_unlock(&fan_mutex);
1356 
1357 			if (speed == 0) {
1358 				threshold_counter++;
1359 				if (threshold_counter ==
1360 				    n_read_fanfault) {
1361 					int32_t i;
1362 					int32_t led_count;
1363 					char led_state[32];
1364 					char *led_id;
1365 					char *slot_id;
1366 					char label[32];
1367 					char state[32], fault[32];
1368 
1369 					threshold_counter = 0;
1370 					strlcpy(fault, PSVC_GEN_FAULT,
1371 					    sizeof (fault));
1372 					status = psvc_set_attr(hdlp, fan_id[r],
1373 					    PSVC_FAULTID_ATTR, fault);
1374 					if (status != PSVC_SUCCESS)
1375 						return (status);
1376 
1377 					strlcpy(state, PSVC_ERROR,
1378 					    sizeof (state));
1379 					status = psvc_set_attr(hdlp, fan_id[r],
1380 					    PSVC_STATE_ATTR, state);
1381 					if (status != PSVC_SUCCESS)
1382 						return (status);
1383 
1384 					status = psvc_get_attr(hdlp, fan_id[r],
1385 					    PSVC_LABEL_ATTR, label);
1386 					if (status != PSVC_SUCCESS)
1387 						return (status);
1388 
1389 					syslog(LOG_ERR, DEVICE_FAILURE_MSG,
1390 					    label);
1391 
1392 					/* turn on fault LEDs */
1393 					psvc_get_attr(hdlp, fan_id[r],
1394 					    PSVC_ASSOC_MATCHES_ATTR, &led_count,
1395 					    PSVC_DEV_FAULT_LED);
1396 					strlcpy(led_state, PSVC_LED_ON,
1397 					    sizeof (led_state));
1398 					for (i = 0; i < led_count; ++i) {
1399 						status = psvc_get_attr(hdlp,
1400 						    fan_id[r],
1401 						    PSVC_ASSOC_ID_ATTR, &led_id,
1402 						    PSVC_DEV_FAULT_LED, i);
1403 						if (status == PSVC_FAILURE)
1404 							return (status);
1405 
1406 						status = psvc_set_attr(hdlp,
1407 						    led_id, PSVC_LED_STATE_ATTR,
1408 						    led_state);
1409 						if (status == PSVC_FAILURE)
1410 							return (status);
1411 					}
1412 
1413 					/* turn on OK to remove LEDs */
1414 
1415 					status = psvc_get_attr(hdlp, fan_id[r],
1416 					    PSVC_ASSOC_ID_ATTR, &slot_id,
1417 					    PSVC_PARENT, 0);
1418 					if (status != PSVC_SUCCESS)
1419 						return (status);
1420 
1421 					psvc_get_attr(hdlp, slot_id,
1422 					    PSVC_ASSOC_MATCHES_ATTR, &led_count,
1423 					    PSVC_SLOT_REMOVE_LED);
1424 					strlcpy(led_state, PSVC_LED_ON,
1425 					    sizeof (led_state));
1426 					for (i = 0; i < led_count; ++i) {
1427 						status = psvc_get_attr(hdlp,
1428 						    slot_id,
1429 						    PSVC_ASSOC_ID_ATTR, &led_id,
1430 						    PSVC_SLOT_REMOVE_LED, i);
1431 						if (status == PSVC_FAILURE)
1432 							return (status);
1433 
1434 						status = psvc_set_attr(hdlp,
1435 						    led_id, PSVC_LED_STATE_ATTR,
1436 						    led_state);
1437 						if (status == PSVC_FAILURE)
1438 							return (status);
1439 					}
1440 				}
1441 			}
1442 		}
1443 	}
1444 
1445 	return (PSVC_SUCCESS);
1446 }
1447 
1448 /*
1449  * This routine takes in the PSVC handle pointer, the PS name, and the
1450  * instance number (0, 1, or 2). It simply make a psvc_get call to get the
1451  * presence of each of the children under the PS. This call will set the
1452  * presence state of the child device if it was not there when the system
1453  * was booted.
1454  */
1455 static int
1456 handle_ps_hotplug_children_presence(psvc_opaque_t hdlp, char *id)
1457 {
1458 	char *sensor_id;
1459 	char fail_valid_switch_id[PICL_PROPNAMELEN_MAX];
1460 	int32_t	status = PSVC_SUCCESS;
1461 	boolean_t presence;
1462 	int j;
1463 
1464 	/* Get the Sensor Valid Switch presence */
1465 	snprintf(fail_valid_switch_id, sizeof (fail_valid_switch_id), "%s%s",
1466 	    id, "_SENSOR_VALID_SWITCH");
1467 
1468 	status = psvc_get_attr(hdlp, fail_valid_switch_id, PSVC_PRESENCE_ATTR,
1469 	    &presence);
1470 	if (status != PSVC_SUCCESS)
1471 		return (status);
1472 
1473 	/* Go through each PS's fault sensors */
1474 	for (j = 0; j < DAK_MAX_FAULT_SENSORS; j++) {
1475 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1476 		    &(sensor_id), PSVC_DEV_FAULT_SENSOR, j);
1477 		if (status != PSVC_SUCCESS)
1478 			return (status);
1479 		status = psvc_get_attr(hdlp, sensor_id, PSVC_PRESENCE_ATTR,
1480 		    &presence);
1481 		if (status != PSVC_SUCCESS)
1482 			return (status);
1483 	}
1484 
1485 	/* Go through each PS's current sensors */
1486 	for (j = 0; j < DAK_MAX_PS_I_SENSORS; ++j) {
1487 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1488 		    &(sensor_id), PSVC_PS_I_SENSOR, j);
1489 		if (status != PSVC_SUCCESS)
1490 			return (status);
1491 		status = psvc_get_attr(hdlp, sensor_id, PSVC_PRESENCE_ATTR,
1492 		    &presence);
1493 		if (status != PSVC_SUCCESS)
1494 			return (status);
1495 
1496 	}
1497 
1498 	/* Go through each PS's onboard i2c hardware */
1499 	for (j = 0; j < 3; j++) {
1500 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1501 		    &(sensor_id), PSVC_PHYSICAL_DEVICE, j);
1502 		if (status != PSVC_SUCCESS)
1503 			return (status);
1504 		status = psvc_get_attr(hdlp, sensor_id, PSVC_PRESENCE_ATTR,
1505 		    &presence);
1506 		if (status != PSVC_SUCCESS)
1507 			return (status);
1508 	}
1509 
1510 	return (status);
1511 }
1512 
1513 static i2c_hp_t devices[3][3] = {
1514 {{{0, 0x90}, "adio", "i2c-pcf8591"}, {{0, 0x70}, "ioexp", "i2c-pcf8574"},
1515 	{{0, 0xa0}, "fru", "i2c-at24c64"}},
1516 {{{0, 0x92}, "adio", "i2c-pcf8591"}, {{0, 0x72}, "ioexp", "i2c-pcf8574"},
1517 	{{0, 0xa2}, "fru", "i2c-at24c64"}},
1518 {{{0, 0x94}, "adio", "i2c-pcf8591"}, {{0, 0x74}, "ioexp", "i2c-pcf8574"},
1519 	{{0, 0xa4}, "fru", "i2c-at24c64"}},
1520 };
1521 
1522 int32_t
1523 psvc_ps_hotplug_policy_0(psvc_opaque_t hdlp, char *id)
1524 {
1525 	boolean_t presence, previous_presence;
1526 	int32_t status = PSVC_SUCCESS;
1527 	char label[32], state[32], fault[32];
1528 	int32_t ps_instance, led_count;
1529 	char *switch_id, *led_id;
1530 	int i;
1531 	picl_nodehdl_t parent_node;
1532 	char parent_path[256], ps_path[256];
1533 	picl_nodehdl_t child_node;
1534 	devctl_hdl_t bus_handle, dev_handle;
1535 	devctl_ddef_t ddef_hdl;
1536 	char pcf8574_devpath[256], pcf8591_devpath[256], fru_devpath[256];
1537 	int retry;
1538 
1539 	status = psvc_get_attr(hdlp, id, PSVC_PREV_PRESENCE_ATTR,
1540 	    &previous_presence);
1541 	if (status != PSVC_SUCCESS)
1542 		return (status);
1543 
1544 	retry = 0;
1545 	do {
1546 		if (retry)
1547 			(void) sleep(retry_sleep_pshp);
1548 
1549 		status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &presence);
1550 		if (status != PSVC_SUCCESS)
1551 			return (status);
1552 		retry++;
1553 	} while ((retry < n_retry_pshp) &&
1554 	    (presence != previous_presence));
1555 
1556 	if (presence == previous_presence) {
1557 		/* No change */
1558 		return (status);
1559 	}
1560 
1561 	status = psvc_get_attr(hdlp, id, PSVC_LABEL_ATTR, label);
1562 	if (status != PSVC_SUCCESS)
1563 		return (status);
1564 
1565 	/* convert name to node, and parent path */
1566 	psvcplugin_lookup(id, parent_path, &child_node);
1567 
1568 	if (presence == PSVC_PRESENT) {
1569 		/*
1570 		 * Run this code if Power Supply was just added into the
1571 		 * System.  This code toggles hotplug switch and adds the
1572 		 * PS and it's children to the picl tree. We then goto adding
1573 		 * device drivers at bottom of the routine.
1574 		 */
1575 		int32_t switch_count;
1576 		char state[32], fault[32];
1577 		char switch_state[32];
1578 
1579 		/* may detect presence before all connections are made */
1580 		(void) poll(NULL, 0, 500);
1581 
1582 		/* Device added */
1583 		syslog(LOG_ERR, DEVICE_INSERTED_MSG, label);
1584 
1585 		strcpy(state, PSVC_OK);
1586 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
1587 		if (status != PSVC_SUCCESS)
1588 			return (status);
1589 
1590 		strcpy(fault, PSVC_NO_FAULT);
1591 		status = psvc_set_attr(hdlp, id, PSVC_FAULTID_ATTR, fault);
1592 		if (status != PSVC_SUCCESS)
1593 			return (status);
1594 
1595 		/* Enable i2c bus */
1596 		psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR,
1597 		    &switch_count, PSVC_HOTPLUG_ENABLE_SWITCH);
1598 		for (i = 0; i < switch_count; ++i) {
1599 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1600 			    &switch_id, PSVC_HOTPLUG_ENABLE_SWITCH, i);
1601 			if (status == PSVC_FAILURE)
1602 				return (status);
1603 
1604 			strcpy(switch_state, PSVC_SWITCH_OFF);
1605 			status = psvc_set_attr(hdlp, switch_id,
1606 			    PSVC_SWITCH_STATE_ATTR, switch_state);
1607 			if (status == PSVC_FAILURE)
1608 				return (status);
1609 
1610 			strcpy(switch_state, PSVC_SWITCH_ON);
1611 			status = psvc_set_attr(hdlp, switch_id,
1612 			    PSVC_SWITCH_STATE_ATTR, switch_state);
1613 			if (status == PSVC_FAILURE)
1614 				return (status);
1615 		}
1616 		ptree_get_node_by_path(parent_path, &parent_node);
1617 		ptree_add_node(parent_node, child_node);
1618 		snprintf(ps_path, sizeof (ps_path), "%s/%s", parent_path, id);
1619 		psvcplugin_add_children(ps_path);
1620 	} else {
1621 		/*
1622 		 * Run this code if PS was just removed from the system. We
1623 		 * delete the device from the picl tree and then shut off
1624 		 * all fault lights associated with the PS.  We also set the
1625 		 * device state to PSVC_REMOVED so that if we hit overcurrent
1626 		 * or fault checking code we can do a psvc call to see that
1627 		 * the device has not offically been added into the system.
1628 		 * We then will drop to code lower in the routine to remove
1629 		 * the device drivers for this PS.
1630 		 */
1631 
1632 		/* Device removed */
1633 		syslog(LOG_ERR, DEVICE_REMOVED_MSG, label);
1634 		ptree_delete_node(child_node);
1635 		psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &led_count,
1636 		    PSVC_DEV_FAULT_LED);
1637 
1638 		for (i = 0; i < led_count; i++) {
1639 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1640 			    &led_id, PSVC_DEV_FAULT_LED, i);
1641 			if (status != PSVC_SUCCESS) {
1642 				return (status);
1643 			}
1644 
1645 			status = psvc_set_attr(hdlp, led_id,
1646 			    PSVC_LED_STATE_ATTR, PSVC_OFF);
1647 			if (status != PSVC_SUCCESS) {
1648 				syslog(LOG_ERR, SET_LED_FAILED_MSG, led_id,
1649 				    errno);
1650 				return (status);
1651 			}
1652 
1653 		}
1654 
1655 		strcpy(state, PSVC_OK);
1656 		strcpy(fault, PSVC_NO_FAULT);
1657 
1658 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
1659 		if (status != PSVC_SUCCESS)
1660 			return (status);
1661 		status = psvc_set_attr(hdlp, id, PSVC_FAULTID_ATTR, fault);
1662 		if (status != PSVC_SUCCESS)
1663 			return (status);
1664 	}
1665 
1666 	status = psvc_set_attr(hdlp, id, PSVC_PREV_PRESENCE_ATTR, &presence);
1667 	if (status != PSVC_SUCCESS)
1668 		return (status);
1669 
1670 	status = psvc_get_attr(hdlp, id, PSVC_INSTANCE_ATTR, &ps_instance);
1671 	if (status != PSVC_SUCCESS)
1672 		return (status);
1673 
1674 	if (presence != PSVC_PRESENT) {
1675 		/*
1676 		 * This is the additional code needed to remove the PS from
1677 		 * the system.  It removes the device drivers from the
1678 		 * device tree.
1679 		 */
1680 		snprintf(pcf8574_devpath, sizeof (pcf8574_devpath), PCF8574,
1681 		    devices[ps_instance][1].addr[1]);
1682 		snprintf(pcf8591_devpath, sizeof (pcf8591_devpath), PCF8591,
1683 		    devices[ps_instance][0].addr[1]);
1684 		snprintf(fru_devpath, sizeof (fru_devpath), FRU,
1685 		    devices[ps_instance][2].addr[1]);
1686 
1687 		dev_handle = devctl_device_acquire(pcf8591_devpath, 0);
1688 		if (dev_handle == NULL) {
1689 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1690 			    pcf8591_devpath, errno);
1691 			devctl_release(dev_handle);
1692 			return (PSVC_FAILURE);
1693 		} else if ((devctl_device_remove(dev_handle)) &&
1694 		    (errno != ENXIO)) {
1695 				syslog(LOG_ERR, DEVTREE_NODE_DELETE_FAILED,
1696 				    pcf8591_devpath, errno);
1697 				devctl_release(dev_handle);
1698 				return (PSVC_FAILURE);
1699 			} else {
1700 				devctl_release(dev_handle);
1701 				status = PSVC_SUCCESS;
1702 			}
1703 
1704 		dev_handle = devctl_device_acquire(pcf8574_devpath, 0);
1705 		if (dev_handle == NULL) {
1706 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1707 			    pcf8574_devpath, errno);
1708 			devctl_release(dev_handle);
1709 			return (PSVC_FAILURE);
1710 		} else if ((devctl_device_remove(dev_handle)) &&
1711 		    (errno != ENXIO)) {
1712 				syslog(LOG_ERR, DEVTREE_NODE_DELETE_FAILED,
1713 				    pcf8574_devpath, errno);
1714 				devctl_release(dev_handle);
1715 				return (PSVC_FAILURE);
1716 			} else {
1717 				devctl_release(dev_handle);
1718 				status = PSVC_SUCCESS;
1719 			}
1720 
1721 		dev_handle = devctl_device_acquire(fru_devpath, 0);
1722 		if (dev_handle == NULL) {
1723 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1724 			    fru_devpath, errno);
1725 			devctl_release(dev_handle);
1726 			return (PSVC_FAILURE);
1727 		} else if ((devctl_device_remove(dev_handle)) &&
1728 		    (errno != ENXIO)) {
1729 				syslog(LOG_ERR, DEVTREE_NODE_DELETE_FAILED,
1730 				    fru_devpath, errno);
1731 				devctl_release(dev_handle);
1732 				return (PSVC_FAILURE);
1733 			} else {
1734 				devctl_release(dev_handle);
1735 				status = PSVC_SUCCESS;
1736 			}
1737 
1738 		return (status);
1739 	}
1740 
1741 	/*
1742 	 * This code is to update the presences of power supply child
1743 	 * devices in the event that picld was started without a power
1744 	 * supply present.  This call makes the devices available
1745 	 * after that initial insertion.
1746 	 */
1747 	status = handle_ps_hotplug_children_presence(hdlp, id);
1748 	if (status == PSVC_FAILURE) {
1749 		return (status);
1750 	}
1751 
1752 	/*
1753 	 * We fall through to here if the device has been inserted.
1754 	 * Add the devinfo tree node entry for the seeprom and attach
1755 	 * the i2c seeprom driver
1756 	 */
1757 
1758 	bus_handle = devctl_bus_acquire(I2C_NODE, 0);
1759 	if (bus_handle == NULL) {
1760 		syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG, I2C_NODE, errno);
1761 		return (PSVC_FAILURE);
1762 	}
1763 	/* Create the deivce nodes for all 3 i2c parts on the PS */
1764 	for (i = 0; i < 3; i++) {
1765 		ddef_hdl = devctl_ddef_alloc(devices[ps_instance][i].name, 0);
1766 		if (ddef_hdl == NULL) {
1767 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1768 			    devices[ps_instance][i].name, errno);
1769 			return (PSVC_FAILURE);
1770 		}
1771 		status = devctl_ddef_string(ddef_hdl, "compatible",
1772 		    devices[ps_instance][i].compatible);
1773 		if (status == -1) {
1774 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1775 			    devices[ps_instance][i].name, errno);
1776 			return (PSVC_FAILURE);
1777 		}
1778 		status = devctl_ddef_int_array(ddef_hdl, "reg", 2,
1779 		    devices[ps_instance][i].addr);
1780 		if (status == -1) {
1781 			syslog(LOG_ERR, DEVICE_HANDLE_FAIL_MSG,
1782 			    devices[ps_instance][i].name, errno);
1783 			return (PSVC_FAILURE);
1784 		}
1785 		if (devctl_bus_dev_create(bus_handle, ddef_hdl, 0,
1786 		    &dev_handle)) {
1787 			syslog(LOG_ERR, DEVTREE_NODE_CREATE_FAILED,
1788 			    devices[ps_instance][i].name, errno);
1789 			return (PSVC_FAILURE);
1790 		} else
1791 			devctl_release(dev_handle);
1792 		devctl_ddef_free(ddef_hdl);
1793 	}
1794 	devctl_release(bus_handle);
1795 
1796 	return (status);
1797 }
1798 
1799 static void
1800 shutdown_routine()
1801 {
1802 	static boolean_t shutdown_flag = 0;
1803 
1804 	if (!(shutdown_flag)) {
1805 		system(shutdown_string);
1806 		shutdown_flag = 1;
1807 	}
1808 }
1809 
1810 /*
1811  * This policy checks temperature sensors to see if the fault attribute
1812  * is set to either High or Low Shutdown. If so then it shuts the system
1813  * down with a 1 minute warning period
1814  */
1815 int32_t
1816 psvc_shutdown_policy(psvc_opaque_t hdlp, char *id)
1817 {
1818 	int32_t	status;
1819 	char	fault[32] = {0};
1820 	boolean_t	pr;
1821 	int	retry;
1822 
1823 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &pr);
1824 	if ((status != PSVC_SUCCESS) || (pr != PSVC_PRESENT)) {
1825 		return (status);
1826 	}
1827 
1828 	retry = 0;
1829 	do {
1830 		if (retry)
1831 			(void) sleep(retry_sleep_temp_shutdown);
1832 		status = psvc_get_attr(hdlp, id, PSVC_FAULTID_ATTR, fault);
1833 		if (status != PSVC_SUCCESS)
1834 			return (status);
1835 		retry++;
1836 	} while (((strcmp(fault, PSVC_TEMP_LO_SHUT) == 0) ||
1837 	    (strcmp(fault, PSVC_TEMP_HI_SHUT) == 0)) &&
1838 	    (retry < n_retry_temp_shutdown));
1839 	if ((strcmp(fault, PSVC_TEMP_LO_SHUT) == 0) ||
1840 	    (strcmp(fault, PSVC_TEMP_HI_SHUT) == 0)) {
1841 		shutdown_routine();
1842 	}
1843 
1844 	return (PSVC_SUCCESS);
1845 }
1846 
1847 int32_t
1848 psvc_check_disk_fault_policy_0(psvc_opaque_t hdlp, char *id)
1849 {
1850 	int32_t		status = PSVC_SUCCESS;
1851 	int32_t		i;
1852 	char		curr_state[32], prev_state[32], led_state[32];
1853 	char		disk_fault[32], disk_state[32];
1854 	static char	*disk_id[DAK_MAX_DISKS] = {NULL};
1855 	static char	*led_id[DAK_MAX_DISKS] = {NULL};
1856 	static char	*parent_id[DAK_MAX_DISKS] = {NULL};
1857 	boolean_t	present;
1858 	int		retry;
1859 
1860 	/*
1861 	 * Check which disk faulted, now get the disks.
1862 	 * We are now going to get disk, disk parent,
1863 	 * parent's leds, and check to see if parent's leds are on
1864 	 */
1865 
1866 	if (disk_id[0] == NULL) {
1867 		for (i = 0; i < DAK_MAX_DISKS; i++) {
1868 			status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1869 			    &(disk_id[i]), PSVC_DISK, i);
1870 			if (status != PSVC_SUCCESS)
1871 				return (status);
1872 			status = psvc_get_attr(hdlp, disk_id[i],
1873 			    PSVC_ASSOC_ID_ATTR, &(parent_id[i]),
1874 			    PSVC_PARENT, 0);
1875 			if (status != PSVC_SUCCESS)
1876 				return (status);
1877 			status = psvc_get_attr(hdlp, parent_id[i],
1878 			    PSVC_ASSOC_ID_ATTR, &(led_id[i]),
1879 			    PSVC_SLOT_FAULT_LED, 0);
1880 			if (status != PSVC_SUCCESS)
1881 				return (status);
1882 
1883 		}
1884 	}
1885 
1886 	for (i = 0; i < DAK_MAX_DISKS; i++) {
1887 		curr_state[0] = 0;
1888 		prev_state[0] = 0;
1889 
1890 		status = psvc_get_attr(hdlp, disk_id[i], PSVC_PRESENCE_ATTR,
1891 		    &present);
1892 		if (status != PSVC_SUCCESS)
1893 			return (status);
1894 
1895 		if (present == PSVC_ABSENT)
1896 			continue;
1897 
1898 		/*
1899 		 * Check if whether or not the led is on.
1900 		 * If so, then this disk has a problem and
1901 		 * set its fault and error states to bad.
1902 		 * If not, then set fault and error states to good.
1903 		 * If the disk underwent a change in state, then
1904 		 * print out what state it's now in.
1905 		 */
1906 
1907 		status = psvc_get_attr(hdlp, disk_id[i], PSVC_STATE_ATTR,
1908 		    prev_state);
1909 		if (status != PSVC_SUCCESS)
1910 			return (status);
1911 
1912 		retry = 0;
1913 		do {
1914 			if (retry)
1915 				(void) sleep(retry_sleep_diskfault);
1916 			status = psvc_get_attr(hdlp, led_id[i], PSVC_STATE_ATTR,
1917 			    led_state);
1918 			if (status != PSVC_SUCCESS)
1919 				return (status);
1920 			retry++;
1921 			/*
1922 			 * check to see if we need to retry. the conditions are:
1923 			 *
1924 			 * prev_state		led_state		retry
1925 			 * --------------------------------------------------
1926 			 * PSVC_ERROR		PSVC_LED_ON		yes
1927 			 * PSVC_OK		PSVC_LED_OFF		yes
1928 			 * PSVC_ERROR		PSVC_LED_OFF		no
1929 			 * PSVC_OK		PSVC_LED_ON		no
1930 			 */
1931 		} while ((retry < n_retry_diskfault) &&
1932 		    change_of_state_str(prev_state, PSVC_OK,
1933 		    led_state, PSVC_LED_ON));
1934 
1935 		/*
1936 		 * Set the disk's state and fault id according to
1937 		 * what we found the disk fault sensor (disk_slot_fault_led)
1938 		 * to be.
1939 		 */
1940 		if (strcmp(led_state, PSVC_LED_ON) == 0) {
1941 			strcpy(disk_fault, PSVC_GEN_FAULT);
1942 			strcpy(disk_state, PSVC_ERROR);
1943 		} else {
1944 			strcpy(disk_fault, PSVC_NO_FAULT);
1945 			strcpy(disk_state, PSVC_OK);
1946 		}
1947 		status = psvc_set_attr(hdlp, disk_id[i], PSVC_STATE_ATTR,
1948 		    disk_state);
1949 		if (status != PSVC_SUCCESS)
1950 			return (status);
1951 		status = psvc_set_attr(hdlp, disk_id[i], PSVC_FAULTID_ATTR,
1952 		    disk_fault);
1953 		if (status != PSVC_SUCCESS)
1954 			return (status);
1955 		/*
1956 		 * Check disk states.  If they differ, then print out
1957 		 * the current state of the disk
1958 		 */
1959 		status = psvc_get_attr(hdlp, disk_id[i], PSVC_PREV_STATE_ATTR,
1960 		    prev_state);
1961 		if (status != PSVC_SUCCESS)
1962 			return (status);
1963 
1964 		if (strcmp(disk_state, prev_state) != 0) {
1965 			if (strcmp(disk_state, PSVC_ERROR) == 0) {
1966 				syslog(LOG_ERR, DISK_FAULT_MSG, disk_id[i]);
1967 			} else {
1968 				syslog(LOG_ERR, DISK_OK_MSG, disk_id[i]);
1969 			}
1970 		}
1971 	}
1972 	return (PSVC_SUCCESS);
1973 }
1974 
1975 int32_t
1976 psvc_update_FSP_fault_led_policy_0(psvc_opaque_t hdlp, char *id)
1977 {
1978 	int32_t status = PSVC_SUCCESS;
1979 	int32_t i;
1980 	int32_t dev_count, fault_state = 0;
1981 	char	*dev_id;
1982 	char	dev_state[32], led_state[32];
1983 	boolean_t	present;
1984 
1985 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &dev_count,
1986 	    PSVC_DEV_FAULT_SENSOR);
1987 	if (status != PSVC_SUCCESS)
1988 		return (status);
1989 
1990 	fault_state = 0;
1991 
1992 	for (i = 0; i < dev_count; i++) {
1993 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
1994 		    &dev_id, PSVC_DEV_FAULT_SENSOR, i);
1995 		if (status != PSVC_SUCCESS)
1996 			return (status);
1997 		status = psvc_get_attr(hdlp, dev_id, PSVC_PRESENCE_ATTR,
1998 		    &present);
1999 		if (status != PSVC_SUCCESS)
2000 			return (status);
2001 
2002 		if (present == PSVC_ABSENT)
2003 			continue;
2004 
2005 		status = psvc_get_attr(hdlp, dev_id, PSVC_STATE_ATTR,
2006 		    dev_state);
2007 		if (status != PSVC_SUCCESS)
2008 			return (status);
2009 
2010 		if (strcmp(dev_state, PSVC_ERROR) == 0) {
2011 			fault_state = 1;
2012 		}
2013 	}
2014 	if (fault_state == 1) {
2015 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
2016 		if (status != PSVC_SUCCESS)
2017 			return (status);
2018 		if (strcmp(led_state, PSVC_OFF) == 0) {
2019 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
2020 			    PSVC_ON);
2021 			if (status != PSVC_SUCCESS)
2022 				return (status);
2023 		}
2024 	} else {
2025 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
2026 		if (status != PSVC_SUCCESS)
2027 			return (status);
2028 		if (strcmp(led_state, PSVC_ON) == 0) {
2029 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
2030 			    PSVC_OFF);
2031 			if (status != PSVC_SUCCESS)
2032 				return (status);
2033 		}
2034 	}
2035 	status = update_gen_fault_led(hdlp, GEN_FAULT_LED);
2036 
2037 	return (status);
2038 }
2039 
2040 int32_t
2041 update_gen_fault_led(psvc_opaque_t hdlp, char *id)
2042 {
2043 	int32_t status = PSVC_SUCCESS;
2044 	int32_t i;
2045 	int32_t led_count, fault_state;
2046 	char	*led_id;
2047 	char	led_state[32];
2048 
2049 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &led_count,
2050 	    PSVC_DEV_FAULT_SENSOR);
2051 	if (status != PSVC_SUCCESS)
2052 		return (status);
2053 
2054 	fault_state = 0;
2055 
2056 	for (i = 0; i < led_count; i++) {
2057 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR,
2058 		    &led_id, PSVC_DEV_FAULT_SENSOR, i);
2059 		if (status != PSVC_SUCCESS)
2060 			return (status);
2061 		status = psvc_get_attr(hdlp, led_id, PSVC_STATE_ATTR,
2062 		    led_state);
2063 		if (status != PSVC_SUCCESS)
2064 			return (status);
2065 
2066 		if (strcmp(led_state, PSVC_ON) == 0) {
2067 			fault_state = 1;
2068 		}
2069 	}
2070 
2071 	if (fault_state == 1) {
2072 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
2073 		if (status != PSVC_SUCCESS)
2074 			return (status);
2075 		if (strcmp(led_state, PSVC_OFF) == 0) {
2076 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
2077 			    PSVC_ON);
2078 			if (status != PSVC_SUCCESS)
2079 				return (status);
2080 		}
2081 	} else {
2082 		status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, led_state);
2083 		if (status != PSVC_SUCCESS)
2084 			return (status);
2085 		if (strcmp(led_state, PSVC_ON) == 0) {
2086 			status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR,
2087 			    PSVC_OFF);
2088 			if (status != PSVC_SUCCESS)
2089 				return (status);
2090 		}
2091 	}
2092 
2093 	return (status);
2094 }
2095 
2096 
2097 /*
2098  * This function detects whether the module present in the dakatari's
2099  * CPU slot is a CPU module or a Zulu (XVR-4000).
2100  * Based on this detection it also sets the appropriate temperature sensors
2101  * to HOTPLUGGED, so that it works properly with check_temp() function
2102  */
2103 #define	MAX_MODULE_SIZE		20
2104 #define	MAX_TEMP_SENSOR_SIZE	30
2105 
2106 int32_t
2107 psvc_update_cpu_module_card_node_0(psvc_opaque_t hdlp, char *id)
2108 {
2109 	int32_t	set_temp_sensor_properties(psvc_opaque_t, char *);
2110 	int32_t	remove_module_node(psvc_opaque_t, char *);
2111 	int32_t status = PSVC_SUCCESS;
2112 	fru_info_t fru_data;
2113 	char *fru, seg_name[2];
2114 	int8_t seg_count, module_card;
2115 	int32_t match_count, i, j, seg_desc_start = 0x1806, module_address;
2116 	int32_t seg_found;
2117 	boolean_t present;
2118 	seg_desc_t segment;
2119 	char other_module_id[MAX_MODULE_SIZE];
2120 	char cpu_temp_sensor1[MAX_TEMP_SENSOR_SIZE];
2121 	char cpu_temp_sensor2[MAX_TEMP_SENSOR_SIZE];
2122 	char zulu_temp_sensor1[MAX_TEMP_SENSOR_SIZE];
2123 	char zulu_temp_sensor2[MAX_TEMP_SENSOR_SIZE];
2124 	int offset = 0x7;
2125 
2126 	status = psvc_get_attr(hdlp, id, PSVC_PRESENCE_ATTR, &present);
2127 	if ((status != PSVC_SUCCESS) || (present != PSVC_PRESENT)) {
2128 		return (status);
2129 	}
2130 
2131 	status = psvc_get_attr(hdlp, id, PSVC_ASSOC_MATCHES_ATTR, &match_count,
2132 	    PSVC_FRU);
2133 	if (status == PSVC_FAILURE) {
2134 		return (status);
2135 	}
2136 
2137 	for (i = 0; i < match_count; i++) {
2138 		seg_found = 0;
2139 		status = psvc_get_attr(hdlp, id, PSVC_ASSOC_ID_ATTR, &fru,
2140 		    PSVC_FRU, i);
2141 		if (status != PSVC_SUCCESS)
2142 			return (status);
2143 
2144 		fru_data.buf_start = 0x1805;
2145 		fru_data.buf = (char *)&seg_count;
2146 		fru_data.read_size = 1;
2147 
2148 		status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2149 		    &fru_data);
2150 		if (status != PSVC_SUCCESS) {
2151 			return (status);
2152 		}
2153 
2154 		for (j = 0; (j < seg_count) && (!seg_found); j++) {
2155 			fru_data.buf_start = seg_desc_start;
2156 			fru_data.buf = seg_name;
2157 			fru_data.read_size = 2;
2158 
2159 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2160 			    &fru_data);
2161 			if (status != PSVC_SUCCESS) {
2162 				syslog(LOG_ERR, GET_ATTR_FRU_FAILED_MSG);
2163 				return (status);
2164 			}
2165 
2166 			seg_desc_start = seg_desc_start + 2;
2167 			fru_data.buf_start = seg_desc_start;
2168 			fru_data.buf = (char *)&segment;
2169 			fru_data.read_size = sizeof (seg_desc_t);
2170 
2171 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2172 			    &fru_data);
2173 			if (status != PSVC_SUCCESS) {
2174 				syslog(LOG_ERR, GET_ATTR_FRU_FAILED_MSG);
2175 				return (status);
2176 			}
2177 			seg_desc_start = seg_desc_start + sizeof (seg_desc_t);
2178 			if (memcmp(seg_name, "SC", 2) == 0)
2179 				seg_found = 1;
2180 		}
2181 
2182 		if (seg_found) {
2183 			module_address = segment.segoffset + offset;
2184 			fru_data.buf_start = module_address;
2185 			fru_data.buf = (char *)&module_card;
2186 			fru_data.read_size = 1;
2187 			status = psvc_get_attr(hdlp, fru, PSVC_FRU_INFO_ATTR,
2188 			    &fru_data);
2189 			if (status != PSVC_SUCCESS) {
2190 				syslog(LOG_ERR, GET_ATTR_FRU_FAILED_MSG);
2191 				return (status);
2192 			}
2193 		} else {
2194 			syslog(LOG_ERR, NO_FRU_INFO_MSG, id);
2195 		}
2196 	}
2197 
2198 	if (strcmp(id, "ZULU_1_3_MOD_CARD") == 0) {
2199 		strlcpy(other_module_id, "CPU_1_3_MOD_CARD", MAX_MODULE_SIZE);
2200 
2201 		strlcpy(cpu_temp_sensor1, "CPU1_DIE_TEMPERATURE_SENSOR",
2202 		    MAX_TEMP_SENSOR_SIZE);
2203 		strlcpy(cpu_temp_sensor2, "CPU3_DIE_TEMPERATURE_SENSOR",
2204 		    MAX_TEMP_SENSOR_SIZE);
2205 
2206 		strlcpy(zulu_temp_sensor1, "ZULU1_DIE_TEMPERATURE_SENSOR",
2207 		    MAX_TEMP_SENSOR_SIZE);
2208 		strlcpy(zulu_temp_sensor2, "ZULU3_DIE_TEMPERATURE_SENSOR",
2209 		    MAX_TEMP_SENSOR_SIZE);
2210 	}
2211 
2212 	if (strcmp(id, "ZULU_4_6_MOD_CARD") == 0) {
2213 		strlcpy(other_module_id, "CPU_4_6_MOD_CARD", MAX_MODULE_SIZE);
2214 
2215 		strlcpy(cpu_temp_sensor1, "CPU4_DIE_TEMPERATURE_SENSOR",
2216 		    MAX_TEMP_SENSOR_SIZE);
2217 		strlcpy(cpu_temp_sensor2, "CPU6_DIE_TEMPERATURE_SENSOR",
2218 		    MAX_TEMP_SENSOR_SIZE);
2219 
2220 		strlcpy(zulu_temp_sensor1, "ZULU4_DIE_TEMPERATURE_SENSOR",
2221 		    MAX_TEMP_SENSOR_SIZE);
2222 		strlcpy(zulu_temp_sensor2, "ZULU6_DIE_TEMPERATURE_SENSOR",
2223 		    MAX_TEMP_SENSOR_SIZE);
2224 	}
2225 
2226 
2227 	/*
2228 	 * If the module in the CPU slot is a Zulu (XVR-4000), then
2229 	 * location 0x1EB0 in its FRUid prom has a value 0xFB.
2230 	 * If Zulu (XVR-4000) is detected, delete the CPU node, otherwise
2231 	 * delete the Zulu node. Also set the temperature sensor value to
2232 	 * HOTPLUGGED for absent temperature sensors.
2233 	 */
2234 	if ((module_card & 0xff) == 0xfb) {
2235 		status = set_temp_sensor_properties(hdlp, cpu_temp_sensor1);
2236 		if (status == PSVC_FAILURE) {
2237 			return (status);
2238 		}
2239 
2240 		status = set_temp_sensor_properties(hdlp, cpu_temp_sensor2);
2241 		if (status == PSVC_FAILURE) {
2242 			return (status);
2243 		}
2244 
2245 		/*
2246 		 * Remove CPU node
2247 		 */
2248 		status = remove_module_node(hdlp, other_module_id);
2249 		if (status == PSVC_FAILURE) {
2250 			return (status);
2251 		}
2252 	} else {
2253 		status = set_temp_sensor_properties(hdlp, zulu_temp_sensor1);
2254 		if (status == PSVC_FAILURE) {
2255 			return (status);
2256 		}
2257 		status = set_temp_sensor_properties(hdlp, zulu_temp_sensor2);
2258 		if (status == PSVC_FAILURE) {
2259 			return (status);
2260 		}
2261 
2262 		/*
2263 		 * Remove Zulu (XVR-4000) node
2264 		 */
2265 		status = remove_module_node(hdlp, id);
2266 		if (status == PSVC_FAILURE) {
2267 			return (status);
2268 		}
2269 	}
2270 
2271 	return (PSVC_SUCCESS);
2272 }
2273 
2274 
2275 /*
2276  * Remove the CPU slot's module node
2277  */
2278 int32_t
2279 remove_module_node(psvc_opaque_t hdlp, char *id)
2280 {
2281 	char parent_path[256];
2282 	picl_nodehdl_t child_node;
2283 
2284 	/* convert name to node, and parent path */
2285 	psvcplugin_lookup(id, parent_path, &child_node);
2286 	/* Device removed */
2287 	ptree_delete_node(child_node);
2288 
2289 	return (PSVC_SUCCESS);
2290 }
2291 
2292 
2293 /*
2294  * Set absent temperature sensor values to HOTPLUGGED
2295  */
2296 int32_t
2297 set_temp_sensor_properties(psvc_opaque_t hdlp, char *id)
2298 {
2299 	char state[32];
2300 	int32_t status = PSVC_SUCCESS;
2301 
2302 	status = psvc_get_attr(hdlp, id, PSVC_STATE_ATTR, state);
2303 	if (status == PSVC_FAILURE) {
2304 		return (status);
2305 	}
2306 
2307 	if (strcmp(state, PSVC_HOTPLUGGED) != 0) {
2308 		strcpy(state, PSVC_HOTPLUGGED);
2309 
2310 		status = psvc_set_attr(hdlp, id, PSVC_STATE_ATTR, state);
2311 		if (status == PSVC_FAILURE) {
2312 			return (status);
2313 		}
2314 	}
2315 
2316 	return (PSVC_SUCCESS);
2317 }
2318