xref: /titanic_52/usr/src/cmd/fm/fmadm/common/faulty.c (revision aa041649636a335c2536daf3e1b2903e5edbb343)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <fmadm.h>
28 #include <errno.h>
29 #include <limits.h>
30 #include <strings.h>
31 #include <stdio.h>
32 #include <unistd.h>
33 #include <sys/wait.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <fm/fmd_log.h>
37 #include <sys/fm/protocol.h>
38 #include <fm/libtopo.h>
39 #include <fm/fmd_adm.h>
40 #include <fm/fmd_msg.h>
41 #include <dlfcn.h>
42 #include <sys/systeminfo.h>
43 #include <sys/utsname.h>
44 #include <libintl.h>
45 #include <locale.h>
46 #include <sys/smbios.h>
47 #include <libdevinfo.h>
48 #include <stdlib.h>
49 
50 #define	offsetof(s, m)	((size_t)(&(((s*)0)->m)))
51 
52 /*
53  * Fault records are added to catalog by calling add_fault_record_to_catalog()
54  * records are stored in order of importance to the system.
55  * If -g flag is set or not_suppressed is not set and the class fru, fault,
56  * type are the same then details are merged into an existing record, with uuid
57  * records are stored in time order.
58  * For each record information is extracted from nvlist and merged into linked
59  * list each is checked for identical records for which percentage certainty are
60  * added together.
61  * print_catalog() is called to print out catalog and release external resources
62  *
63  *                         /---------------\
64  *	status_rec_list -> |               | -|
65  *                         \---------------/
66  *                                \/
67  *                         /---------------\    /-------\    /-------\
68  *      status_fru_list    | status_record | -> | uurec | -> | uurec | -|
69  *            \/           |               | |- |       | <- |       |
70  *      /-------------\    |               |    \-------/    \-------/
71  *      |             | -> |               |       \/           \/
72  *      \-------------/    |               |    /-------\    /-------\
73  *            \/           |               | -> | asru  | -> | asru  |
74  *            ---          |               |    |       | <- |       |
75  *                         |               |    \-------/    \-------/
76  *      status_asru_list   |  class        |
77  *            \/           |  resource     |    /-------\    /-------\
78  *      /-------------\    |  fru          | -> | list  | -> | list  |
79  *      |             | -> |  serial       |    |       | <- |       |
80  *      \-------------/    |               |    \-------/    \-------/
81  *            \/           \---------------/
82  *            ---               \/    /\
83  *                         /---------------\
84  *                         | status_record |
85  *                         \---------------/
86  *
87  * Fmadm faulty takes a number of options which affect the format of the
88  * output displayed. By default, the display reports the FRU and ASRU along
89  * with other information on per-case basis as in the example below.
90  *
91  * --------------- ------------------------------------  -------------- -------
92  * TIME            EVENT-ID                              MSG-ID         SEVERITY
93  * --------------- ------------------------------------  -------------- -------
94  * Sep 21 10:01:36 d482f935-5c8f-e9ab-9f25-d0aaafec1e6c  AMD-8000-2F    Major
95  *
96  * Fault class	: fault.memory.dimm_sb
97  * Affects	: mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0
98  *		    faulted but still in service
99  * FRU		: "CPU 0 DIMM 0" (hc://.../memory-controller=0/dimm=0)
100  *		    faulty
101  *
102  * Description	: The number of errors associated with this memory module has
103  *		exceeded acceptable levels.  Refer to
104  *		http://sun.com/msg/AMD-8000-2F for more information.
105  *
106  * Response	: Pages of memory associated with this memory module are being
107  *		removed from service as errors are reported.
108  *
109  * Impact	: Total system memory capacity will be reduced as pages are
110  *		retired.
111  *
112  * Action	: Schedule a repair procedure to replace the affected memory
113  *		module.  Use fmdump -v -u <EVENT_ID> to identify the module.
114  *
115  * The -v flag is similar, but adds some additonal information such as the
116  * resource. The -s flag is also similar but just gives the top line summary.
117  * All these options (ie without the -f or -r flags) use the print_catalog()
118  * function to do the display.
119  *
120  * The -f flag changes the output so that it appears sorted on a per-fru basis.
121  * The output is somewhat cut down compared to the default output. If -f is
122  * used, then print_fru() is used to print the output.
123  *
124  * -----------------------------------------------------------------------------
125  * "SLOT 2" (hc://.../hostbridge=3/pciexrc=3/pciexbus=4/pciexdev=0) faulty
126  * 5ca4aeb3-36...f6be-c2e8166dc484 2 suspects in this FRU total certainty 100%
127  *
128  * Description	: A problem was detected for a PCI device.
129  *		Refer to http://sun.com/msg/PCI-8000-7J for more information.
130  *
131  * Response	: One or more device instances may be disabled
132  *
133  * Impact	: Possible loss of services provided by the device instances
134  *		associated with this fault
135  *
136  * Action	: Schedule a repair procedure to replace the affected device.
137  * 		Use fmdump -v -u <EVENT_ID> to identify the device or contact
138  *		Sun for support.
139  *
140  * The -r flag changes the output so that it appears sorted on a per-asru basis.
141  * The output is very much cut down compared to the default output, just giving
142  * the asru fmri and state. Here print_asru() is used to print the output.
143  *
144  * mem:///motherboard=0/chip=0/memory-controller=0/dimm=0/rank=0	degraded
145  *
146  * For all fmadm faulty options, the sequence of events is
147  *
148  * 1) Walk through all the cases in the system using fmd_adm_case_iter() and
149  * for each case call dfault_rec(). This will call add_fault_record_to_catalog()
150  * This will extract the data from the nvlist and call catalog_new_record() to
151  * save the data away in various linked lists in the catalogue.
152  *
153  * 2) Once this is done, the data can be supplemented by using
154  * fmd_adm_rsrc_iter(). However this is now only necessary for the -i option.
155  *
156  * 3) Finally print_catalog(), print_fru() or print_asru() are called as
157  * appropriate to display the information from the catalogue sorted in the
158  * requested way.
159  *
160  */
161 
162 typedef struct name_list {
163 	struct name_list *next;
164 	struct name_list *prev;
165 	char *name;
166 	uint8_t pct;
167 	uint8_t max_pct;
168 	ushort_t count;
169 	int status;
170 	char *label;
171 } name_list_t;
172 
173 typedef struct ari_list {
174 	char *ari_uuid;
175 	struct ari_list *next;
176 } ari_list_t;
177 
178 typedef struct uurec {
179 	struct uurec *next;
180 	struct uurec *prev;
181 	char *uuid;
182 	ari_list_t *ari_uuid_list;
183 	name_list_t *asru;
184 	uint64_t sec;
185 	nvlist_t *event;
186 } uurec_t;
187 
188 typedef struct uurec_select {
189 	struct uurec_select *next;
190 	char *uuid;
191 } uurec_select_t;
192 
193 typedef struct host_id {
194 	char *chassis;
195 	char *server;
196 	char *platform;
197 	char *domain;
198 	char *product_sn;
199 } hostid_t;
200 
201 typedef struct host_id_list {
202 	hostid_t hostid;
203 	struct host_id_list *next;
204 } host_id_list_t;
205 
206 typedef struct status_record {
207 	hostid_t *host;
208 	int nrecs;
209 	uurec_t *uurec;
210 	char *severity;			/* in C locale */
211 	char *msgid;
212 	name_list_t *class;
213 	name_list_t *resource;
214 	name_list_t *asru;
215 	name_list_t *fru;
216 	name_list_t *serial;
217 	uint8_t not_suppressed;
218 } status_record_t;
219 
220 typedef struct sr_list {
221 	struct sr_list *next;
222 	struct sr_list *prev;
223 	struct status_record *status_record;
224 } sr_list_t;
225 
226 typedef struct resource_list {
227 	struct resource_list *next;
228 	struct resource_list *prev;
229 	sr_list_t *status_rec_list;
230 	char *resource;
231 	uint8_t not_suppressed;
232 	uint8_t max_pct;
233 } resource_list_t;
234 
235 typedef struct tgetlabel_data {
236 	char *label;
237 	char *fru;
238 } tgetlabel_data_t;
239 
240 sr_list_t *status_rec_list;
241 resource_list_t *status_fru_list;
242 resource_list_t *status_asru_list;
243 
244 static int max_display;
245 static int max_fault = 0;
246 static topo_hdl_t *topo_handle;
247 static char *topo_handle_uuid;
248 static host_id_list_t *host_list;
249 static int n_server;
250 static int opt_g;
251 static fmd_msg_hdl_t *fmadm_msghdl = NULL; /* handle for libfmd_msg calls */
252 
253 static char *
254 format_date(char *buf, size_t len, uint64_t sec)
255 {
256 	if (sec > LONG_MAX) {
257 		(void) fprintf(stderr,
258 		    "record time is too large for 32-bit utility\n");
259 		(void) snprintf(buf, len, "0x%llx", sec);
260 	} else {
261 		time_t tod = (time_t)sec;
262 		(void) strftime(buf, len, "%b %d %T", localtime(&tod));
263 	}
264 
265 	return (buf);
266 }
267 
268 static hostid_t *
269 find_hostid_in_list(char *platform, char *chassis, char *server, char *domain,
270     char *product_sn)
271 {
272 	hostid_t *rt = NULL;
273 	host_id_list_t *hostp;
274 
275 	if (platform == NULL)
276 		platform = "-";
277 	if (server == NULL)
278 		server = "-";
279 	hostp = host_list;
280 	while (hostp) {
281 		if (hostp->hostid.platform &&
282 		    strcmp(hostp->hostid.platform, platform) == 0 &&
283 		    hostp->hostid.server &&
284 		    strcmp(hostp->hostid.server, server) == 0 &&
285 		    (chassis == NULL || hostp->hostid.chassis == NULL ||
286 		    strcmp(chassis, hostp->hostid.chassis) == 0) &&
287 		    (product_sn == NULL || hostp->hostid.product_sn == NULL ||
288 		    strcmp(product_sn, hostp->hostid.product_sn) == 0) &&
289 		    (domain == NULL || hostp->hostid.domain == NULL ||
290 		    strcmp(domain, hostp->hostid.domain) == 0)) {
291 			rt = &hostp->hostid;
292 			break;
293 		}
294 		hostp = hostp->next;
295 	}
296 	if (rt == NULL) {
297 		hostp = malloc(sizeof (host_id_list_t));
298 		hostp->hostid.platform = strdup(platform);
299 		hostp->hostid.product_sn =
300 		    product_sn ? strdup(product_sn) : NULL;
301 		hostp->hostid.server = strdup(server);
302 		hostp->hostid.chassis = chassis ? strdup(chassis) : NULL;
303 		hostp->hostid.domain = domain ? strdup(domain) : NULL;
304 		hostp->next = host_list;
305 		host_list = hostp;
306 		rt = &hostp->hostid;
307 		n_server++;
308 	}
309 	return (rt);
310 }
311 
312 static hostid_t *
313 find_hostid(nvlist_t *nvl)
314 {
315 	char *platform = NULL, *chassis = NULL, *server = NULL, *domain = NULL;
316 	char *product_sn = NULL;
317 	nvlist_t *auth, *fmri;
318 	hostid_t *rt = NULL;
319 
320 	if (nvlist_lookup_nvlist(nvl, FM_SUSPECT_DE, &fmri) == 0 &&
321 	    nvlist_lookup_nvlist(fmri, FM_FMRI_AUTHORITY, &auth) == 0) {
322 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT,
323 		    &platform);
324 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_PRODUCT_SN,
325 		    &product_sn);
326 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_SERVER, &server);
327 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_CHASSIS,
328 		    &chassis);
329 		(void) nvlist_lookup_string(auth, FM_FMRI_AUTH_DOMAIN, &domain);
330 		rt = find_hostid_in_list(platform, chassis, server,
331 		    domain, product_sn);
332 	}
333 	return (rt);
334 }
335 
336 /*
337  * compare two fru strings which are made up of substrings seperated by '/'
338  * return true if every substring is the same in the two strings, or if a
339  * substring is null in one.
340  */
341 
342 static int
343 frucmp(char *f1, char *f2)
344 {
345 	char c1, c2;
346 	int i = 0;
347 
348 	for (;;) {
349 		c1 = *f1;
350 		c2 = *f2;
351 		if (c1 == c2) {
352 			i = (c1 == '/') ? 0 : i + 1;
353 		} else if (i == 0) {
354 			if (c1 == '/') {
355 				do {
356 					f2++;
357 				} while ((c2 = *f2) != 0 && c2 != '/');
358 				if (c2 == NULL)
359 					break;
360 			} else if (c2 == '/') {
361 				do {
362 					f1++;
363 				} while ((c1 = *f1) != 0 && c1 != '/');
364 				if (c1 == NULL)
365 					break;
366 			} else
367 				break;
368 		} else
369 			break;
370 		if (c1 == NULL)
371 			return (0);
372 		f1++;
373 		f2++;
374 	}
375 	return (1);
376 }
377 
378 static int
379 tgetlabel(topo_hdl_t *thp, tnode_t *node, void *arg)
380 {
381 	int err;
382 	char *fru_name, *lname;
383 	nvlist_t *fru = NULL;
384 	int rt = TOPO_WALK_NEXT;
385 	tgetlabel_data_t *tdp = (tgetlabel_data_t *)arg;
386 
387 	if (topo_node_fru(node, &fru, NULL, &err) == 0) {
388 		if (topo_fmri_nvl2str(thp, fru, &fru_name, &err) == 0) {
389 			if (frucmp(tdp->fru, fru_name) == 0 &&
390 			    topo_node_label(node, &lname, &err) == 0) {
391 				tdp->label = strdup(lname);
392 				topo_hdl_strfree(thp, lname);
393 				rt = TOPO_WALK_TERMINATE;
394 			}
395 			topo_hdl_strfree(thp, fru_name);
396 		}
397 		nvlist_free(fru);
398 	}
399 	return (rt);
400 }
401 
402 static void
403 label_get_topo(void)
404 {
405 	int err;
406 
407 	topo_handle = topo_open(TOPO_VERSION, 0, &err);
408 	if (topo_handle) {
409 		topo_handle_uuid = topo_snap_hold(topo_handle, NULL, &err);
410 	}
411 }
412 
413 static void
414 label_release_topo(void)
415 {
416 	if (topo_handle_uuid)
417 		topo_hdl_strfree(topo_handle, topo_handle_uuid);
418 	if (topo_handle) {
419 		topo_snap_release(topo_handle);
420 		topo_close(topo_handle);
421 	}
422 }
423 
424 static char *
425 get_fmri_label(char *fru)
426 {
427 	topo_walk_t *twp;
428 	tgetlabel_data_t td;
429 	int err;
430 
431 	td.label = NULL;
432 	td.fru = fru;
433 	if (topo_handle == NULL)
434 		label_get_topo();
435 	if (topo_handle_uuid) {
436 		twp = topo_walk_init(topo_handle, FM_FMRI_SCHEME_HC,
437 		    tgetlabel, &td, &err);
438 		if (twp) {
439 			topo_walk_step(twp, TOPO_WALK_CHILD);
440 			topo_walk_fini(twp);
441 		}
442 	}
443 	return (td.label);
444 }
445 
446 static char *
447 get_nvl2str_topo(nvlist_t *nvl)
448 {
449 	char *name = NULL;
450 	char *tname;
451 	int err;
452 	char *scheme = NULL;
453 	char *mod_name = NULL;
454 	char buf[128];
455 
456 	if (topo_handle == NULL)
457 		label_get_topo();
458 	if (topo_fmri_nvl2str(topo_handle, nvl, &tname, &err) == 0) {
459 		name = strdup(tname);
460 		topo_hdl_strfree(topo_handle, tname);
461 	} else {
462 		(void) nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &scheme);
463 		(void) nvlist_lookup_string(nvl, FM_FMRI_MOD_NAME, &mod_name);
464 		if (scheme && strcmp(scheme, FM_FMRI_SCHEME_FMD) == 0 &&
465 		    mod_name) {
466 			(void) snprintf(buf, sizeof (buf), "%s:///module/%s",
467 			    scheme, mod_name);
468 			name = strdup(buf);
469 		}
470 	}
471 	return (name);
472 }
473 
474 static int
475 set_priority(char *s)
476 {
477 	int rt = 0;
478 
479 	if (s) {
480 		if (strcmp(s, "Minor") == 0)
481 			rt = 1;
482 		else if (strcmp(s, "Major") == 0)
483 			rt = 10;
484 		else if (strcmp(s, "Critical") == 0)
485 			rt = 100;
486 	}
487 	return (rt);
488 }
489 
490 static int
491 cmp_priority(char *s1, char *s2, uint64_t t1, uint64_t t2, uint8_t p1,
492     uint8_t p2)
493 {
494 	int r1, r2;
495 	int rt;
496 
497 	r1 = set_priority(s1);
498 	r2 = set_priority(s2);
499 	rt = r1 - r2;
500 	if (rt == 0) {
501 		if (t1 > t2)
502 			rt = 1;
503 		else if (t1 < t2)
504 			rt = -1;
505 		else
506 			rt = p1 - p2;
507 	}
508 	return (rt);
509 }
510 
511 /*
512  * merge two lists into one, by comparing enties in new and moving into list if
513  * name is not there or free off memory for names which are already there
514  * add_pct indicates if pct is the sum or highest pct
515  */
516 static name_list_t *
517 merge_name_list(name_list_t **list, name_list_t *new, int add_pct)
518 {
519 	name_list_t *lp, *np, *sp, *rt = NULL;
520 	int max_pct;
521 
522 	rt = *list;
523 	np = new;
524 	while (np) {
525 		lp = *list;
526 		while (lp) {
527 			if (strcmp(lp->name, np->name) == 0)
528 				break;
529 			lp = lp->next;
530 			if (lp == *list)
531 				lp = NULL;
532 		}
533 		if (np->next == new)
534 			sp = NULL;
535 		else
536 			sp = np->next;
537 		if (lp) {
538 			lp->status |= (np->status & FM_SUSPECT_FAULTY);
539 			if (add_pct) {
540 				lp->pct += np->pct;
541 				lp->count += np->count;
542 			} else if (np->pct > lp->pct) {
543 				lp->pct = np->pct;
544 			}
545 			max_pct = np->max_pct;
546 			if (np->label)
547 				free(np->label);
548 			free(np->name);
549 			free(np);
550 			np = NULL;
551 			if (max_pct > lp->max_pct) {
552 				lp->max_pct = max_pct;
553 				if (lp->max_pct > lp->prev->max_pct &&
554 				    lp != *list) {
555 					lp->prev->next = lp->next;
556 					lp->next->prev = lp->prev;
557 					np = lp;
558 				}
559 			}
560 		}
561 		if (np) {
562 			lp = *list;
563 			if (lp) {
564 				if (np->max_pct > lp->max_pct) {
565 					np->next = lp;
566 					np->prev = lp->prev;
567 					lp->prev->next = np;
568 					lp->prev = np;
569 					*list = np;
570 					rt = np;
571 				} else {
572 					lp = lp->next;
573 					while (lp != *list &&
574 					    np->max_pct < lp->max_pct) {
575 						lp = lp->next;
576 					}
577 					np->next = lp;
578 					np->prev = lp->prev;
579 					lp->prev->next = np;
580 					lp->prev = np;
581 				}
582 			} else {
583 				*list = np;
584 				np->next = np;
585 				np->prev = np;
586 				rt = np;
587 			}
588 		}
589 		np = sp;
590 	}
591 	return (rt);
592 }
593 
594 /*
595  * compare entries in two lists return true if the two lists have identical
596  * content. The two lists may not have entries in the same order, so we compare
597  * the size of the list as well as trying to find every entry from one list in
598  * the other.
599  */
600 static int
601 cmp_name_list(name_list_t *lxp1, name_list_t *lxp2)
602 {
603 	name_list_t *lp1, *lp2;
604 	int l1 = 0, l2 = 0, common = 0;
605 
606 	lp2 = lxp2;
607 	while (lp2) {
608 		l2++;
609 		lp2 = lp2->next;
610 		if (lp2 == lxp2)
611 			break;
612 	}
613 	lp1 = lxp1;
614 	while (lp1) {
615 		l1++;
616 		lp2 = lxp2;
617 		while (lp2) {
618 			if (strcmp(lp2->name, lp1->name) == 0) {
619 				common++;
620 				break;
621 			}
622 			lp2 = lp2->next;
623 			if (lp2 == lxp2)
624 				break;
625 		}
626 		lp1 = lp1->next;
627 		if (lp1 == lxp1)
628 			break;
629 	}
630 	if (l1 == l2 && l2 == common)
631 		return (0);
632 	else
633 		return (1);
634 }
635 
636 static name_list_t *
637 alloc_name_list(char *name, uint8_t pct)
638 {
639 	name_list_t *nlp;
640 
641 	nlp = malloc(sizeof (*nlp));
642 	nlp->name = strdup(name);
643 	nlp->pct = pct;
644 	nlp->max_pct = pct;
645 	nlp->count = 1;
646 	nlp->next = nlp;
647 	nlp->prev = nlp;
648 	nlp->status = 0;
649 	nlp->label = NULL;
650 	return (nlp);
651 }
652 
653 static void
654 free_name_list(name_list_t *list)
655 {
656 	name_list_t *next = list;
657 	name_list_t *lp;
658 
659 	if (list) {
660 		do {
661 			lp = next;
662 			next = lp->next;
663 			if (lp->label)
664 				free(lp->label);
665 			free(lp->name);
666 			free(lp);
667 		} while (next != list);
668 	}
669 }
670 
671 static status_record_t *
672 new_record_init(uurec_t *uurec_p, char *msgid, name_list_t *class,
673     name_list_t *fru, name_list_t *asru, name_list_t *resource,
674     name_list_t *serial, boolean_t not_suppressed,
675     hostid_t *hostid)
676 {
677 	status_record_t *status_rec_p;
678 
679 	status_rec_p = (status_record_t *)malloc(sizeof (status_record_t));
680 	status_rec_p->nrecs = 1;
681 	status_rec_p->host = hostid;
682 	status_rec_p->uurec = uurec_p;
683 	uurec_p->next = NULL;
684 	uurec_p->prev = NULL;
685 	uurec_p->asru = asru;
686 	if ((status_rec_p->severity = fmd_msg_getitem_id(fmadm_msghdl, NULL,
687 	    msgid, FMD_MSG_ITEM_SEVERITY)) == NULL)
688 		status_rec_p->severity = strdup("unknown");
689 	status_rec_p->class = class;
690 	status_rec_p->fru = fru;
691 	status_rec_p->asru = asru;
692 	status_rec_p->resource = resource;
693 	status_rec_p->serial = serial;
694 	status_rec_p->msgid = strdup(msgid);
695 	status_rec_p->not_suppressed = not_suppressed;
696 	return (status_rec_p);
697 }
698 
699 /*
700  * add record to given list maintaining order higher priority first.
701  */
702 static void
703 add_rec_list(status_record_t *status_rec_p, sr_list_t **list_pp)
704 {
705 	sr_list_t *tp, *np, *sp;
706 	int order;
707 	uint64_t sec;
708 
709 	np = malloc(sizeof (sr_list_t));
710 	np->status_record = status_rec_p;
711 	sec = status_rec_p->uurec->sec;
712 	if ((sp = *list_pp) == NULL) {
713 		*list_pp = np;
714 		np->next = np;
715 		np->prev = np;
716 	} else {
717 		/* insert new record in front of lower priority */
718 		tp = sp;
719 		order = cmp_priority(status_rec_p->severity,
720 		    sp->status_record->severity, sec,
721 		    tp->status_record->uurec->sec, 0, 0);
722 		if (order > 0) {
723 			*list_pp = np;
724 		} else {
725 			tp = sp->next;
726 			while (tp != sp &&
727 			    cmp_priority(status_rec_p->severity,
728 			    tp->status_record->severity, sec,
729 			    tp->status_record->uurec->sec, 0, 0)) {
730 				tp = tp->next;
731 			}
732 		}
733 		np->next = tp;
734 		np->prev = tp->prev;
735 		tp->prev->next = np;
736 		tp->prev = np;
737 	}
738 }
739 
740 static void
741 add_resource(status_record_t *status_rec_p, resource_list_t **rp,
742     resource_list_t *np)
743 {
744 	int order;
745 	uint64_t sec;
746 	resource_list_t *sp, *tp;
747 	status_record_t *srp;
748 	char *severity = status_rec_p->severity;
749 
750 	add_rec_list(status_rec_p, &np->status_rec_list);
751 	if ((sp = *rp) == NULL) {
752 		np->next = np;
753 		np->prev = np;
754 		*rp = np;
755 	} else {
756 		/*
757 		 * insert new record in front of lower priority
758 		 */
759 		tp = sp->next;
760 		srp = sp->status_rec_list->status_record;
761 		sec = status_rec_p->uurec->sec;
762 		order = cmp_priority(severity, srp->severity, sec,
763 		    srp->uurec->sec, np->max_pct, sp->max_pct);
764 		if (order > 0) {
765 			*rp = np;
766 		} else {
767 			srp = tp->status_rec_list->status_record;
768 			while (tp != sp &&
769 			    cmp_priority(severity, srp->severity, sec,
770 			    srp->uurec->sec, np->max_pct, sp->max_pct) < 0) {
771 				tp = tp->next;
772 				srp = tp->status_rec_list->status_record;
773 			}
774 		}
775 		np->next = tp;
776 		np->prev = tp->prev;
777 		tp->prev->next = np;
778 		tp->prev = np;
779 	}
780 }
781 
782 static void
783 add_resource_list(status_record_t *status_rec_p, name_list_t *fp,
784     resource_list_t **rpp)
785 {
786 	int order;
787 	resource_list_t *np, *end;
788 	status_record_t *srp;
789 
790 	np = *rpp;
791 	end = np;
792 	while (np) {
793 		if (strcmp(fp->name, np->resource) == 0) {
794 			np->not_suppressed |= status_rec_p->not_suppressed;
795 			srp = np->status_rec_list->status_record;
796 			order = cmp_priority(status_rec_p->severity,
797 			    srp->severity, status_rec_p->uurec->sec,
798 			    srp->uurec->sec, fp->max_pct, np->max_pct);
799 			if (order > 0 && np != end) {
800 				/*
801 				 * remove from list and add again using
802 				 * new priority
803 				 */
804 				np->prev->next = np->next;
805 				np->next->prev = np->prev;
806 				add_resource(status_rec_p,
807 				    rpp, np);
808 			} else {
809 				add_rec_list(status_rec_p,
810 				    &np->status_rec_list);
811 			}
812 			break;
813 		}
814 		np = np->next;
815 		if (np == end) {
816 			np = NULL;
817 			break;
818 		}
819 	}
820 	if (np == NULL) {
821 		np = malloc(sizeof (resource_list_t));
822 		np->resource = fp->name;
823 		np->not_suppressed = status_rec_p->not_suppressed;
824 		np->status_rec_list = NULL;
825 		np->max_pct = fp->max_pct;
826 		add_resource(status_rec_p, rpp, np);
827 	}
828 }
829 
830 static void
831 add_list(status_record_t *status_rec_p, name_list_t *listp,
832     resource_list_t **glistp)
833 {
834 	name_list_t *fp, *end;
835 
836 	fp = listp;
837 	end = fp;
838 	while (fp) {
839 		add_resource_list(status_rec_p, fp, glistp);
840 		fp = fp->next;
841 		if (fp == end)
842 			break;
843 	}
844 }
845 
846 /*
847  * add record to rec, fru and asru lists.
848  */
849 static void
850 catalog_new_record(uurec_t *uurec_p, char *msgid, name_list_t *class,
851     name_list_t *fru, name_list_t *asru, name_list_t *resource,
852     name_list_t *serial, boolean_t not_suppressed,
853     hostid_t *hostid)
854 {
855 	status_record_t *status_rec_p;
856 
857 	status_rec_p = new_record_init(uurec_p, msgid, class, fru, asru,
858 	    resource, serial, not_suppressed, hostid);
859 	add_rec_list(status_rec_p, &status_rec_list);
860 	if (status_rec_p->fru)
861 		add_list(status_rec_p, status_rec_p->fru, &status_fru_list);
862 	if (status_rec_p->asru)
863 		add_list(status_rec_p, status_rec_p->asru, &status_asru_list);
864 }
865 
866 /*
867  * add uuid and diagnoses time to an existing record for similar fault on the
868  * same fru
869  */
870 static void
871 catalog_merge_record(status_record_t *status_rec_p, uurec_t *uurec_p,
872     name_list_t *asru, name_list_t *resource, name_list_t *serial,
873     boolean_t not_suppressed)
874 {
875 	uurec_t *uurec1_p;
876 
877 	status_rec_p->nrecs++;
878 	/* add uurec in time order */
879 	if (status_rec_p->uurec->sec > uurec_p->sec) {
880 		uurec_p->next = status_rec_p->uurec;
881 		uurec_p->prev = NULL;
882 		status_rec_p->uurec = uurec_p;
883 	} else {
884 		uurec1_p = status_rec_p->uurec;
885 		while (uurec1_p->next && uurec1_p->next->sec <= uurec_p->sec)
886 			uurec1_p = uurec1_p->next;
887 		if (uurec1_p->next)
888 			uurec1_p->next->prev = uurec_p;
889 		uurec_p->next = uurec1_p->next;
890 		uurec_p->prev = uurec1_p;
891 		uurec1_p->next = uurec_p;
892 	}
893 	status_rec_p->not_suppressed |= not_suppressed;
894 	uurec_p->asru = merge_name_list(&status_rec_p->asru, asru, 0);
895 	(void) merge_name_list(&status_rec_p->resource, resource, 0);
896 	(void) merge_name_list(&status_rec_p->serial, serial, 0);
897 }
898 
899 static status_record_t *
900 record_in_catalog(name_list_t *class, name_list_t *fru,
901     char *msgid, hostid_t *host)
902 {
903 	sr_list_t *status_rec_p;
904 	status_record_t *srp = NULL;
905 
906 	status_rec_p = status_rec_list;
907 	while (status_rec_p) {
908 		srp = status_rec_p->status_record;
909 		if (host == srp->host &&
910 		    cmp_name_list(class, srp->class) == 0 &&
911 		    cmp_name_list(fru, srp->fru) == 0 &&
912 		    strcmp(msgid, srp->msgid) == 0)
913 			break;
914 		if (status_rec_p->next == status_rec_list) {
915 			srp = NULL;
916 			break;
917 		} else {
918 			status_rec_p = status_rec_p->next;
919 		}
920 	}
921 	return (srp);
922 }
923 
924 static void
925 get_serial_no(nvlist_t *nvl, name_list_t **serial_p, uint8_t pct)
926 {
927 	char *name;
928 	char *serial = NULL;
929 	char **lserial = NULL;
930 	uint64_t serint;
931 	name_list_t *nlp;
932 	int j;
933 	uint_t nelem;
934 	char buf[64];
935 
936 	if (nvlist_lookup_string(nvl, FM_FMRI_SCHEME, &name) == 0) {
937 		if (strcmp(name, FM_FMRI_SCHEME_CPU) == 0) {
938 			if (nvlist_lookup_uint64(nvl, FM_FMRI_CPU_SERIAL_ID,
939 			    &serint) == 0) {
940 				(void) snprintf(buf, sizeof (buf), "%llX",
941 				    serint);
942 				nlp = alloc_name_list(buf, pct);
943 				(void) merge_name_list(serial_p, nlp, 1);
944 			}
945 		} else if (strcmp(name, FM_FMRI_SCHEME_MEM) == 0) {
946 			if (nvlist_lookup_string_array(nvl,
947 			    FM_FMRI_MEM_SERIAL_ID, &lserial, &nelem) == 0) {
948 				nlp = alloc_name_list(lserial[0], pct);
949 				for (j = 1; j < nelem; j++) {
950 					name_list_t *n1lp;
951 					n1lp = alloc_name_list(lserial[j], pct);
952 					(void) merge_name_list(&nlp, n1lp, 1);
953 				}
954 				(void) merge_name_list(serial_p, nlp, 1);
955 			}
956 		} else if (strcmp(name, FM_FMRI_SCHEME_HC) == 0) {
957 			if (nvlist_lookup_string(nvl, FM_FMRI_HC_SERIAL_ID,
958 			    &serial) == 0) {
959 				nlp = alloc_name_list(serial, pct);
960 				(void) merge_name_list(serial_p, nlp, 1);
961 			}
962 		}
963 	}
964 }
965 
966 static void
967 extract_record_info(nvlist_t *nvl, name_list_t **class_p,
968     name_list_t **fru_p, name_list_t **serial_p,
969     name_list_t **resource_p, name_list_t **asru_p, uint8_t status)
970 {
971 	nvlist_t *lfru, *lasru, *rsrc;
972 	name_list_t *nlp;
973 	char *name;
974 	uint8_t lpct = 0;
975 	char *lclass = NULL;
976 	char *label;
977 
978 	(void) nvlist_lookup_uint8(nvl, FM_FAULT_CERTAINTY, &lpct);
979 	if (nvlist_lookup_string(nvl, FM_CLASS, &lclass) == 0) {
980 		nlp = alloc_name_list(lclass, lpct);
981 		(void) merge_name_list(class_p, nlp, 1);
982 	}
983 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_FRU, &lfru) == 0) {
984 		name = get_nvl2str_topo(lfru);
985 		if (name != NULL) {
986 			nlp = alloc_name_list(name, lpct);
987 			nlp->status = status & ~(FM_SUSPECT_UNUSABLE |
988 			    FM_SUSPECT_DEGRADED);
989 			free(name);
990 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
991 			    &label) == 0)
992 				nlp->label = strdup(label);
993 			(void) merge_name_list(fru_p, nlp, 1);
994 		}
995 		get_serial_no(lfru, serial_p, lpct);
996 	}
997 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &lasru) == 0) {
998 		name = get_nvl2str_topo(lasru);
999 		if (name != NULL) {
1000 			nlp = alloc_name_list(name, lpct);
1001 			nlp->status = status & ~(FM_SUSPECT_NOT_PRESENT |
1002 			    FM_SUSPECT_REPAIRED | FM_SUSPECT_REPLACED |
1003 			    FM_SUSPECT_ACQUITTED);
1004 			free(name);
1005 			(void) merge_name_list(asru_p, nlp, 1);
1006 		}
1007 		get_serial_no(lasru, serial_p, lpct);
1008 	}
1009 	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
1010 		name = get_nvl2str_topo(rsrc);
1011 		if (name != NULL) {
1012 			nlp = alloc_name_list(name, lpct);
1013 			nlp->status = status;
1014 			free(name);
1015 			if (nvlist_lookup_string(nvl, FM_FAULT_LOCATION,
1016 			    &label) == 0)
1017 				nlp->label = strdup(label);
1018 			(void) merge_name_list(resource_p, nlp, 1);
1019 		}
1020 	}
1021 }
1022 
1023 static void
1024 add_fault_record_to_catalog(nvlist_t *nvl, uint64_t sec, char *uuid)
1025 {
1026 	char *msgid = "-";
1027 	uint_t i, size = 0;
1028 	name_list_t *class = NULL, *resource = NULL;
1029 	name_list_t *asru = NULL, *fru = NULL, *serial = NULL;
1030 	nvlist_t **nva;
1031 	uint8_t *ba;
1032 	status_record_t *status_rec_p;
1033 	uurec_t *uurec_p;
1034 	hostid_t *host;
1035 	boolean_t not_suppressed = 1;
1036 	boolean_t any_present = 0;
1037 
1038 	(void) nvlist_lookup_string(nvl, FM_SUSPECT_DIAG_CODE, &msgid);
1039 	(void) nvlist_lookup_uint32(nvl, FM_SUSPECT_FAULT_SZ, &size);
1040 	(void) nvlist_lookup_boolean_value(nvl, FM_SUSPECT_MESSAGE,
1041 	    &not_suppressed);
1042 
1043 	if (size != 0) {
1044 		(void) nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
1045 		    &nva, &size);
1046 		(void) nvlist_lookup_uint8_array(nvl, FM_SUSPECT_FAULT_STATUS,
1047 		    &ba, &size);
1048 		for (i = 0; i < size; i++) {
1049 			extract_record_info(nva[i], &class, &fru, &serial,
1050 			    &resource, &asru, ba[i]);
1051 			if (!(ba[i] & FM_SUSPECT_NOT_PRESENT) &&
1052 			    (ba[i] & FM_SUSPECT_FAULTY))
1053 				any_present = 1;
1054 		}
1055 		/*
1056 		 * also suppress if no resources present
1057 		 */
1058 		if (any_present == 0)
1059 			not_suppressed = 0;
1060 	}
1061 
1062 	uurec_p = (uurec_t *)malloc(sizeof (uurec_t));
1063 	uurec_p->uuid = strdup(uuid);
1064 	uurec_p->sec = sec;
1065 	uurec_p->ari_uuid_list = NULL;
1066 	uurec_p->event = NULL;
1067 	(void) nvlist_dup(nvl, &uurec_p->event, 0);
1068 	host = find_hostid(nvl);
1069 	if (not_suppressed && !opt_g)
1070 		status_rec_p = NULL;
1071 	else
1072 		status_rec_p = record_in_catalog(class, fru, msgid, host);
1073 	if (status_rec_p) {
1074 		catalog_merge_record(status_rec_p, uurec_p, asru, resource,
1075 		    serial, not_suppressed);
1076 		free_name_list(class);
1077 		free_name_list(fru);
1078 	} else {
1079 		catalog_new_record(uurec_p, msgid, class, fru, asru,
1080 		    resource, serial, not_suppressed, host);
1081 	}
1082 }
1083 
1084 static void
1085 update_asru_state_in_catalog(const char *uuid, const char *ari_uuid)
1086 {
1087 	sr_list_t *srp;
1088 	uurec_t *uurp;
1089 	ari_list_t *ari_list;
1090 
1091 	srp = status_rec_list;
1092 	if (srp) {
1093 		for (;;) {
1094 			uurp = srp->status_record->uurec;
1095 			while (uurp) {
1096 				if (strcmp(uuid, uurp->uuid) == 0) {
1097 					ari_list = (ari_list_t *)
1098 					    malloc(sizeof (ari_list_t));
1099 					ari_list->ari_uuid = strdup(ari_uuid);
1100 					ari_list->next = uurp->ari_uuid_list;
1101 					uurp->ari_uuid_list = ari_list;
1102 					return;
1103 				}
1104 				uurp = uurp->next;
1105 			}
1106 			if (srp->next == status_rec_list)
1107 				break;
1108 			srp = srp->next;
1109 		}
1110 	}
1111 }
1112 
1113 static void
1114 print_line(char *label, char *buf)
1115 {
1116 	char *cp, *ep, *wp;
1117 	char c;
1118 	int i;
1119 	int lsz;
1120 	char *padding;
1121 
1122 	lsz = strlen(label);
1123 	padding = malloc(lsz + 1);
1124 	for (i = 0; i < lsz; i++)
1125 		padding[i] = ' ';
1126 	padding[i] = 0;
1127 	cp = buf;
1128 	ep = buf;
1129 	c = *ep;
1130 	(void) printf("\n");
1131 	while (c) {
1132 		i = lsz;
1133 		wp = NULL;
1134 		while ((c = *ep) != NULL && (wp == NULL || i < 80)) {
1135 			if (c == ' ')
1136 				wp = ep;
1137 			else if (c == '\n') {
1138 				i = 0;
1139 				*ep = 0;
1140 				do {
1141 					ep++;
1142 				} while ((c = *ep) != NULL && c == ' ');
1143 				break;
1144 			}
1145 			ep++;
1146 			i++;
1147 		}
1148 		if (i >= 80 && wp) {
1149 			*wp = 0;
1150 			ep = wp + 1;
1151 			c = *ep;
1152 		}
1153 		(void) printf("%s%s\n", label, cp);
1154 		cp = ep;
1155 		label = padding;
1156 	}
1157 	free(padding);
1158 }
1159 
1160 static void
1161 print_dict_info_line(nvlist_t *e, fmd_msg_item_t what, const char *linehdr)
1162 {
1163 	char *cp = fmd_msg_getitem_nv(fmadm_msghdl, NULL, e, what);
1164 
1165 	if (cp) {
1166 		print_line(dgettext("FMD", linehdr), cp);
1167 		free(cp);
1168 	}
1169 }
1170 
1171 static void
1172 print_dict_info(nvlist_t *nvl)
1173 {
1174 	print_dict_info_line(nvl, FMD_MSG_ITEM_DESC, "Description : ");
1175 	print_dict_info_line(nvl, FMD_MSG_ITEM_RESPONSE, "Response    : ");
1176 	print_dict_info_line(nvl, FMD_MSG_ITEM_IMPACT, "Impact      : ");
1177 	print_dict_info_line(nvl, FMD_MSG_ITEM_ACTION, "Action      : ");
1178 }
1179 
1180 static void
1181 print_name(name_list_t *list, char *(func)(char *), char *padding, int *np,
1182     int pct, int full)
1183 {
1184 	char *name, *fru_label = NULL;
1185 
1186 	name = list->name;
1187 	if (list->label) {
1188 		(void) printf("%s \"%s\" (%s)", padding, list->label, name);
1189 		*np += 1;
1190 	} else if (func && (fru_label = func(list->name)) != NULL) {
1191 		(void) printf("%s \"%s\" (%s)", padding, fru_label, name);
1192 		*np += 1;
1193 		free(fru_label);
1194 	} else {
1195 		(void) printf("%s %s", padding, name);
1196 		*np += 1;
1197 	}
1198 	if (list->pct && pct > 0 && pct < 100) {
1199 		if (list->count > 1) {
1200 			if (full) {
1201 				(void) printf(" %d @ %s %d%%\n", list->count,
1202 				    dgettext("FMD", "max"),
1203 				    list->max_pct);
1204 			} else {
1205 				(void) printf(" %s %d%%\n",
1206 				    dgettext("FMD", "max"),
1207 				    list->max_pct);
1208 			}
1209 		} else {
1210 			(void) printf(" %d%%\n", list->pct);
1211 		}
1212 	} else {
1213 		(void) printf("\n");
1214 	}
1215 }
1216 
1217 static void
1218 print_asru_status(int status, char *label)
1219 {
1220 	char *msg = NULL;
1221 
1222 	switch (status) {
1223 	case 0:
1224 		msg = dgettext("FMD", "ok and in service");
1225 		break;
1226 	case FM_SUSPECT_DEGRADED:
1227 		msg = dgettext("FMD", "service degraded, "
1228 		    "but associated components no longer faulty");
1229 		break;
1230 	case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1231 		msg = dgettext("FMD", "faulted but still "
1232 		    "providing degraded service");
1233 		break;
1234 	case FM_SUSPECT_FAULTY:
1235 		msg = dgettext("FMD", "faulted but still in service");
1236 		break;
1237 	case FM_SUSPECT_UNUSABLE:
1238 		msg = dgettext("FMD", "out of service, "
1239 		    "but associated components no longer faulty");
1240 		break;
1241 	case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1242 		msg = dgettext("FMD", "faulted and taken out of service");
1243 		break;
1244 	default:
1245 		break;
1246 	}
1247 	if (msg) {
1248 		(void) printf("%s     %s\n", label, msg);
1249 	}
1250 }
1251 
1252 static void
1253 print_fru_status(int status, char *label)
1254 {
1255 	char *msg = NULL;
1256 
1257 	if (status & FM_SUSPECT_NOT_PRESENT)
1258 		msg = dgettext("FMD", "not present");
1259 	else if (status & FM_SUSPECT_FAULTY)
1260 		msg = dgettext("FMD", "faulty");
1261 	else if (status & FM_SUSPECT_REPLACED)
1262 		msg = dgettext("FMD", "replaced");
1263 	else if (status & FM_SUSPECT_REPAIRED)
1264 		msg = dgettext("FMD", "repair attempted");
1265 	else if (status & FM_SUSPECT_ACQUITTED)
1266 		msg = dgettext("FMD", "acquitted");
1267 	else
1268 		msg = dgettext("FMD", "removed");
1269 	(void) printf("%s     %s\n", label, msg);
1270 }
1271 
1272 static void
1273 print_rsrc_status(int status, char *label)
1274 {
1275 	char *msg = "";
1276 
1277 	if (status & FM_SUSPECT_NOT_PRESENT)
1278 		msg = dgettext("FMD", "not present");
1279 	else if (status & FM_SUSPECT_FAULTY) {
1280 		if (status & FM_SUSPECT_DEGRADED)
1281 			msg = dgettext("FMD",
1282 			    "faulted but still providing degraded service");
1283 		else if (status & FM_SUSPECT_UNUSABLE)
1284 			msg = dgettext("FMD",
1285 			    "faulted and taken out of service");
1286 		else
1287 			msg = dgettext("FMD", "faulted but still in service");
1288 	} else if (status & FM_SUSPECT_REPLACED)
1289 		msg = dgettext("FMD", "replaced");
1290 	else if (status & FM_SUSPECT_REPAIRED)
1291 		msg = dgettext("FMD", "repair attempted");
1292 	else if (status & FM_SUSPECT_ACQUITTED)
1293 		msg = dgettext("FMD", "acquitted");
1294 	else
1295 		msg = dgettext("FMD", "removed");
1296 	(void) printf("%s     %s\n", label, msg);
1297 }
1298 
1299 static void
1300 print_name_list(name_list_t *list, char *label, char *(func)(char *),
1301     int limit, int pct, void (func1)(int, char *), int full)
1302 {
1303 	char *name, *fru_label = NULL;
1304 	char *padding;
1305 	int i, j, l, n;
1306 	name_list_t *end = list;
1307 
1308 	l = strlen(label);
1309 	padding = malloc(l + 1);
1310 	for (i = 0; i < l; i++)
1311 		padding[i] = ' ';
1312 	padding[l] = 0;
1313 	(void) printf("%s", label);
1314 	name = list->name;
1315 	if (list->label)
1316 		(void) printf(" \"%s\" (%s)", list->label, name);
1317 	else if (func && (fru_label = func(list->name)) != NULL) {
1318 		(void) printf(" \"%s\" (%s)", fru_label, name);
1319 		free(fru_label);
1320 	} else
1321 		(void) printf(" %s", name);
1322 	if (list->pct && pct > 0 && pct < 100) {
1323 		if (list->count > 1) {
1324 			if (full) {
1325 				(void) printf(" %d @ %s %d%%\n", list->count,
1326 				    dgettext("FMD", "max"), list->max_pct);
1327 			} else {
1328 				(void) printf(" %s %d%%\n",
1329 				    dgettext("FMD", "max"), list->max_pct);
1330 			}
1331 		} else {
1332 			(void) printf(" %d%%\n", list->pct);
1333 		}
1334 	} else {
1335 		(void) printf("\n");
1336 	}
1337 	if (func1)
1338 		func1(list->status, padding);
1339 	n = 1;
1340 	j = 0;
1341 	while ((list = list->next) != end) {
1342 		if (limit == 0 || n < limit) {
1343 			print_name(list, func, padding, &n, pct, full);
1344 			if (func1)
1345 				func1(list->status, padding);
1346 		} else
1347 			j++;
1348 	}
1349 	if (j == 1) {
1350 		print_name(list->prev, func, padding, &n, pct, full);
1351 	} else if (j > 1) {
1352 		(void) printf("%s... %d %s\n", padding, j,
1353 		    dgettext("FMD", "more entries suppressed,"
1354 		    " use -v option for full list"));
1355 	}
1356 	free(padding);
1357 }
1358 
1359 static int
1360 asru_same_status(name_list_t *list)
1361 {
1362 	name_list_t *end = list;
1363 	int status = list->status;
1364 
1365 	while ((list = list->next) != end) {
1366 		if (status == -1) {
1367 			status = list->status;
1368 			continue;
1369 		}
1370 		if (list->status != -1 && status != list->status) {
1371 			status = -1;
1372 			break;
1373 		}
1374 	}
1375 	return (status);
1376 }
1377 
1378 static int
1379 serial_in_fru(name_list_t *fru, name_list_t *serial)
1380 {
1381 	name_list_t *sp = serial;
1382 	name_list_t *fp;
1383 	int nserial = 0;
1384 	int found = 0;
1385 	char buf[128];
1386 
1387 	while (sp) {
1388 		fp = fru;
1389 		nserial++;
1390 		(void) snprintf(buf, sizeof (buf), "serial=%s", sp->name);
1391 		buf[sizeof (buf) - 1] = 0;
1392 		while (fp) {
1393 			if (strstr(fp->name, buf) != NULL) {
1394 				found++;
1395 				break;
1396 			}
1397 			fp = fp->next;
1398 			if (fp == fru)
1399 				break;
1400 		}
1401 		sp = sp->next;
1402 		if (sp == serial)
1403 			break;
1404 	}
1405 	return (found == nserial ? 1 : 0);
1406 }
1407 
1408 static void
1409 print_sup_record(status_record_t *srp, int opt_i, int full)
1410 {
1411 	char buf[32];
1412 	uurec_t *uurp = srp->uurec;
1413 	int n, j, k, max;
1414 	int status;
1415 	ari_list_t *ari_list;
1416 
1417 	n = 0;
1418 	max = max_fault;
1419 	if (max < 0) {
1420 		max = 0;
1421 	}
1422 	j = max / 2;
1423 	max -= j;
1424 	k = srp->nrecs - max;
1425 	while ((uurp = uurp->next) != NULL) {
1426 		if (full || n < j || n >= k || max_fault == 0 ||
1427 		    srp->nrecs == max_fault+1) {
1428 			if (opt_i) {
1429 				ari_list = uurp->ari_uuid_list;
1430 				while (ari_list) {
1431 					(void) printf("%-15s %s\n",
1432 					    format_date(buf, sizeof (buf),
1433 					    uurp->sec), ari_list->ari_uuid);
1434 					ari_list = ari_list->next;
1435 				}
1436 			} else {
1437 				(void) printf("%-15s %s\n",
1438 				    format_date(buf, sizeof (buf), uurp->sec),
1439 				    uurp->uuid);
1440 			}
1441 		} else if (n == j)
1442 			(void) printf("... %d %s\n", srp->nrecs - max_fault,
1443 			    dgettext("FMD", "more entries suppressed"));
1444 		n++;
1445 	}
1446 	(void) printf("\n");
1447 	(void) printf("%s %s", dgettext("FMD", "Host        :"),
1448 	    srp->host->server);
1449 	if (srp->host->domain)
1450 		(void) printf("\t%s %s", dgettext("FMD", "Domain      :"),
1451 		    srp->host->domain);
1452 	(void) printf("\n%s %s", dgettext("FMD", "Platform    :"),
1453 	    srp->host->platform);
1454 	(void) printf("\t%s %s", dgettext("FMD", "Chassis_id  :"),
1455 	    srp->host->chassis ? srp->host->chassis : "");
1456 	(void) printf("\n%s %s\n\n", dgettext("FMD", "Product_sn  :"),
1457 	    srp->host->product_sn? srp->host->product_sn : "");
1458 	if (srp->class)
1459 		print_name_list(srp->class,
1460 		    dgettext("FMD", "Fault class :"), NULL, 0, srp->class->pct,
1461 		    NULL, full);
1462 	if (srp->asru) {
1463 		status = asru_same_status(srp->asru);
1464 		if (status != -1) {
1465 			print_name_list(srp->asru,
1466 			    dgettext("FMD", "Affects     :"), NULL,
1467 			    full ? 0 : max_display, 0, NULL, full);
1468 			print_asru_status(status, "             ");
1469 		} else
1470 			print_name_list(srp->asru,
1471 			    dgettext("FMD", "Affects     :"), NULL,
1472 			    full ? 0 : max_display, 0, print_asru_status, full);
1473 	}
1474 	if (full || srp->fru == NULL || srp->asru == NULL) {
1475 		if (srp->resource) {
1476 			print_name_list(srp->resource,
1477 			    dgettext("FMD", "Problem in  :"),
1478 			    NULL, full ? 0 : max_display, 0, print_rsrc_status,
1479 			    full);
1480 		}
1481 	}
1482 	if (srp->fru) {
1483 		status = asru_same_status(srp->fru);
1484 		if (status != -1) {
1485 			print_name_list(srp->fru, dgettext("FMD",
1486 			    "FRU         :"), get_fmri_label, 0,
1487 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1488 			    NULL, full);
1489 			print_fru_status(status, "             ");
1490 		} else
1491 			print_name_list(srp->fru, dgettext("FMD",
1492 			    "FRU         :"), get_fmri_label, 0,
1493 			    srp->fru->pct == 100 ? 100 : srp->fru->max_pct,
1494 			    print_fru_status, full);
1495 	}
1496 	if (srp->serial && !serial_in_fru(srp->fru, srp->serial) &&
1497 	    !serial_in_fru(srp->asru, srp->serial)) {
1498 		print_name_list(srp->serial, dgettext("FMD", "Serial ID.  :"),
1499 		    NULL, 0, 0, NULL, full);
1500 	}
1501 	print_dict_info(srp->uurec->event);
1502 	(void) printf("\n");
1503 }
1504 
1505 static void
1506 print_status_record(status_record_t *srp, int summary, int opt_i, int full)
1507 {
1508 	char buf[32];
1509 	uurec_t *uurp = srp->uurec;
1510 	static int header = 0;
1511 	char *head;
1512 	ari_list_t *ari_list;
1513 
1514 	if (!summary || !header) {
1515 		if (opt_i) {
1516 			head = "--------------- "
1517 			    "------------------------------------  "
1518 			    "-------------- ---------\n"
1519 			    "TIME            CACHE-ID"
1520 			    "                              MSG-ID"
1521 			    "         SEVERITY\n--------------- "
1522 			    "------------------------------------ "
1523 			    " -------------- ---------";
1524 		} else {
1525 			head = "--------------- "
1526 			    "------------------------------------  "
1527 			    "-------------- ---------\n"
1528 			    "TIME            EVENT-ID"
1529 			    "                              MSG-ID"
1530 			    "         SEVERITY\n--------------- "
1531 			    "------------------------------------ "
1532 			    " -------------- ---------";
1533 		}
1534 		(void) printf("%s\n", dgettext("FMD", head));
1535 		header = 1;
1536 	}
1537 	if (opt_i) {
1538 		ari_list = uurp->ari_uuid_list;
1539 		while (ari_list) {
1540 			(void) printf("%-15s %-37s %-14s %-9s\n",
1541 			    format_date(buf, sizeof (buf), uurp->sec),
1542 			    ari_list->ari_uuid, srp->msgid, srp->severity);
1543 			ari_list = ari_list->next;
1544 		}
1545 	} else {
1546 		(void) printf("%-15s %-37s %-14s %-9s\n",
1547 		    format_date(buf, sizeof (buf), uurp->sec),
1548 		    uurp->uuid, srp->msgid, srp->severity);
1549 	}
1550 
1551 	if (!summary)
1552 		print_sup_record(srp, opt_i, full);
1553 }
1554 
1555 static void
1556 print_catalog(int summary, int opt_a, int full, int opt_i, int page_feed)
1557 {
1558 	status_record_t *srp;
1559 	sr_list_t *slp;
1560 
1561 	slp = status_rec_list;
1562 	if (slp) {
1563 		for (;;) {
1564 			srp = slp->status_record;
1565 			if (opt_a || srp->not_suppressed) {
1566 				if (page_feed)
1567 					(void) printf("\f\n");
1568 				print_status_record(srp, summary, opt_i, full);
1569 			}
1570 			if (slp->next == status_rec_list)
1571 				break;
1572 			slp = slp->next;
1573 		}
1574 	}
1575 }
1576 
1577 static name_list_t *
1578 find_fru(status_record_t *srp, char *resource)
1579 {
1580 	name_list_t *rt = NULL;
1581 	name_list_t *fru = srp->fru;
1582 
1583 	while (fru) {
1584 		if (strcmp(resource, fru->name) == 0) {
1585 			rt = fru;
1586 			break;
1587 		}
1588 		fru = fru->next;
1589 		if (fru == srp->fru)
1590 			break;
1591 	}
1592 	return (rt);
1593 }
1594 
1595 static void
1596 print_fru_line(name_list_t *fru, char *uuid)
1597 {
1598 	if (fru->pct == 100) {
1599 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1600 		    dgettext("FMD", "suspects in this FRU total certainty"),
1601 		    100);
1602 	} else {
1603 		(void) printf("%s %d %s %d%%\n", uuid, fru->count,
1604 		    dgettext("FMD", "suspects in this FRU max certainty"),
1605 		    fru->max_pct);
1606 	}
1607 }
1608 
1609 static void
1610 print_fru(int summary, int opt_a, int opt_i, int page_feed)
1611 {
1612 	resource_list_t *tp = status_fru_list;
1613 	status_record_t *srp;
1614 	sr_list_t *slp, *end;
1615 	char *msgid, *fru_label;
1616 	uurec_t *uurp;
1617 	name_list_t *fru;
1618 	int status;
1619 	ari_list_t *ari_list;
1620 
1621 	while (tp) {
1622 		if (opt_a || tp->not_suppressed) {
1623 			if (page_feed)
1624 				(void) printf("\f\n");
1625 			if (!summary)
1626 				(void) printf("-----------------------------"
1627 				    "---------------------------------------"
1628 				    "----------\n");
1629 			slp = tp->status_rec_list;
1630 			end = slp;
1631 			do {
1632 				srp = slp->status_record;
1633 				fru = find_fru(srp, tp->resource);
1634 				if (fru) {
1635 					if (fru->label)
1636 						(void) printf("\"%s\" (%s) ",
1637 						    fru->label, fru->name);
1638 					else if ((fru_label = get_fmri_label(
1639 					    fru->name)) != NULL) {
1640 						(void) printf("\"%s\" (%s) ",
1641 						    fru_label, fru->name);
1642 						free(fru_label);
1643 					} else
1644 						(void) printf("%s ",
1645 						    fru->name);
1646 					break;
1647 				}
1648 				slp = slp->next;
1649 			} while (slp != end);
1650 
1651 			slp = tp->status_rec_list;
1652 			end = slp;
1653 			status = 0;
1654 			do {
1655 				srp = slp->status_record;
1656 				fru = srp->fru;
1657 				while (fru) {
1658 					if (strcmp(tp->resource,
1659 					    fru->name) == 0)
1660 						status |= fru->status;
1661 					fru = fru->next;
1662 					if (fru == srp->fru)
1663 						break;
1664 				}
1665 				slp = slp->next;
1666 			} while (slp != end);
1667 			if (status & FM_SUSPECT_NOT_PRESENT)
1668 				(void) printf(dgettext("FMD", "not present\n"));
1669 			else if (status & FM_SUSPECT_FAULTY)
1670 				(void) printf(dgettext("FMD", "faulty\n"));
1671 			else if (status & FM_SUSPECT_REPLACED)
1672 				(void) printf(dgettext("FMD", "replaced\n"));
1673 			else if (status & FM_SUSPECT_REPAIRED)
1674 				(void) printf(dgettext("FMD",
1675 				    "repair attempted\n"));
1676 			else if (status & FM_SUSPECT_ACQUITTED)
1677 				(void) printf(dgettext("FMD", "acquitted\n"));
1678 			else
1679 				(void) printf(dgettext("FMD", "removed\n"));
1680 
1681 			slp = tp->status_rec_list;
1682 			end = slp;
1683 			do {
1684 				srp = slp->status_record;
1685 				uurp = srp->uurec;
1686 				fru = find_fru(srp, tp->resource);
1687 				if (fru) {
1688 					if (opt_i) {
1689 						ari_list = uurp->ari_uuid_list;
1690 						while (ari_list) {
1691 							print_fru_line(fru,
1692 							    ari_list->ari_uuid);
1693 							ari_list =
1694 							    ari_list->next;
1695 						}
1696 					} else {
1697 						print_fru_line(fru, uurp->uuid);
1698 					}
1699 				}
1700 				slp = slp->next;
1701 			} while (slp != end);
1702 			if (!summary) {
1703 				slp = tp->status_rec_list;
1704 				end = slp;
1705 				srp = slp->status_record;
1706 				if (srp->serial &&
1707 				    !serial_in_fru(srp->fru, srp->serial)) {
1708 					print_name_list(srp->serial,
1709 					    dgettext("FMD", "Serial ID.  :"),
1710 					    NULL, 0, 0, NULL, 1);
1711 				}
1712 				msgid = NULL;
1713 				do {
1714 					if (msgid == NULL ||
1715 					    strcmp(msgid, srp->msgid) != 0) {
1716 						msgid = srp->msgid;
1717 						print_dict_info(uurp->event);
1718 					}
1719 					slp = slp->next;
1720 				} while (slp != end);
1721 			}
1722 		}
1723 		tp = tp->next;
1724 		if (tp == status_fru_list)
1725 			break;
1726 	}
1727 }
1728 
1729 static void
1730 print_asru(int opt_a)
1731 {
1732 	resource_list_t *tp = status_asru_list;
1733 	status_record_t *srp;
1734 	sr_list_t *slp, *end;
1735 	char *msg;
1736 	int status;
1737 	name_list_t *asru;
1738 
1739 	while (tp) {
1740 		if (opt_a || tp->not_suppressed) {
1741 			status = 0;
1742 			slp = tp->status_rec_list;
1743 			end = slp;
1744 			do {
1745 				srp = slp->status_record;
1746 				asru = srp->asru;
1747 				while (asru) {
1748 					if (strcmp(tp->resource,
1749 					    asru->name) == 0)
1750 						status |= asru->status;
1751 					asru = asru->next;
1752 					if (asru == srp->asru)
1753 						break;
1754 				}
1755 				slp = slp->next;
1756 			} while (slp != end);
1757 			switch (status) {
1758 			case 0:
1759 				msg = dgettext("FMD", "ok");
1760 				break;
1761 			case FM_SUSPECT_DEGRADED:
1762 				msg = dgettext("FMD", "degraded");
1763 				break;
1764 			case FM_SUSPECT_FAULTY | FM_SUSPECT_DEGRADED:
1765 				msg = dgettext("FMD", "degraded");
1766 				break;
1767 			case FM_SUSPECT_FAULTY:
1768 				msg = dgettext("FMD", "degraded");
1769 				break;
1770 			case FM_SUSPECT_UNUSABLE:
1771 				msg = dgettext("FMD", "unknown");
1772 				break;
1773 			case FM_SUSPECT_FAULTY | FM_SUSPECT_UNUSABLE:
1774 				msg = dgettext("FMD", "faulted");
1775 				break;
1776 			default:
1777 				msg = "";
1778 				break;
1779 			}
1780 			(void) printf("%-69s %s\n", tp->resource, msg);
1781 		}
1782 		tp = tp->next;
1783 		if (tp == status_asru_list)
1784 			break;
1785 	}
1786 }
1787 
1788 static int
1789 uuid_in_list(char *uuid, uurec_select_t *uurecp)
1790 {
1791 	while (uurecp) {
1792 		if (strcmp(uuid, uurecp->uuid) == 0)
1793 			return (1);
1794 		uurecp = uurecp->next;
1795 	}
1796 	return (0);
1797 }
1798 
1799 static int
1800 dfault_rec(const fmd_adm_caseinfo_t *acp, void *arg)
1801 {
1802 	int64_t *diag_time;
1803 	uint_t nelem;
1804 	int rt = 0;
1805 	char *uuid = "-";
1806 	uurec_select_t *uurecp = (uurec_select_t *)arg;
1807 
1808 	if (nvlist_lookup_int64_array(acp->aci_event, FM_SUSPECT_DIAG_TIME,
1809 	    &diag_time, &nelem) == 0 && nelem >= 2) {
1810 		(void) nvlist_lookup_string(acp->aci_event, FM_SUSPECT_UUID,
1811 		    &uuid);
1812 		if (uurecp == NULL || uuid_in_list(uuid, uurecp))
1813 			add_fault_record_to_catalog(acp->aci_event, *diag_time,
1814 			    uuid);
1815 	} else {
1816 		rt = -1;
1817 	}
1818 	return (rt);
1819 }
1820 
1821 /*ARGSUSED*/
1822 static int
1823 dstatus_rec(const fmd_adm_rsrcinfo_t *ari, void *unused)
1824 {
1825 	update_asru_state_in_catalog(ari->ari_case, ari->ari_uuid);
1826 	return (0);
1827 }
1828 
1829 static int
1830 get_cases_from_fmd(fmd_adm_t *adm, uurec_select_t *uurecp, int opt_i)
1831 {
1832 	int rt = FMADM_EXIT_SUCCESS;
1833 
1834 	/*
1835 	 * These calls may fail with Protocol error if message payload is
1836 	 * too big
1837 	 */
1838 	if (fmd_adm_case_iter(adm, NULL, dfault_rec, uurecp) != 0)
1839 		die("failed to get case list from fmd");
1840 	if (opt_i && fmd_adm_rsrc_iter(adm, 1, dstatus_rec, NULL) != 0)
1841 		die("failed to get case status from fmd");
1842 	return (rt);
1843 }
1844 
1845 /*
1846  * fmadm faulty command
1847  *
1848  *	-a		show hidden fault records
1849  *	-f		show faulty fru's
1850  *	-g		force grouping of similar faults on the same fru
1851  *	-n		number of fault records to display
1852  *	-p		pipe output through pager
1853  *	-r		show faulty asru's
1854  *	-s		print summary of first fault
1855  *	-u		print listed uuid's only
1856  *	-v		full output
1857  */
1858 
1859 int
1860 cmd_faulty(fmd_adm_t *adm, int argc, char *argv[])
1861 {
1862 	int opt_a = 0, opt_v = 0, opt_p = 0, opt_s = 0, opt_r = 0, opt_f = 0;
1863 	int opt_i = 0;
1864 	char *pager;
1865 	FILE *fp;
1866 	int rt, c, stat;
1867 	uurec_select_t *tp;
1868 	uurec_select_t *uurecp = NULL;
1869 
1870 	while ((c = getopt(argc, argv, "afgin:prsu:v")) != EOF) {
1871 		switch (c) {
1872 		case 'a':
1873 			opt_a++;
1874 			break;
1875 		case 'f':
1876 			opt_f++;
1877 			break;
1878 		case 'g':
1879 			opt_g++;
1880 			break;
1881 		case 'i':
1882 			opt_i++;
1883 			break;
1884 		case 'n':
1885 			max_fault = atoi(optarg);
1886 			break;
1887 		case 'p':
1888 			opt_p++;
1889 			break;
1890 		case 'r':
1891 			opt_r++;
1892 			break;
1893 		case 's':
1894 			opt_s++;
1895 			break;
1896 		case 'u':
1897 			tp = (uurec_select_t *)malloc(sizeof (uurec_select_t));
1898 			tp->uuid = optarg;
1899 			tp->next = uurecp;
1900 			uurecp = tp;
1901 			opt_a = 1;
1902 			break;
1903 		case 'v':
1904 			opt_v++;
1905 			break;
1906 		default:
1907 			return (FMADM_EXIT_USAGE);
1908 		}
1909 	}
1910 	if (optind < argc)
1911 		return (FMADM_EXIT_USAGE);
1912 
1913 	if ((fmadm_msghdl = fmd_msg_init(NULL, FMD_MSG_VERSION)) == NULL)
1914 		return (FMADM_EXIT_ERROR);
1915 	rt = get_cases_from_fmd(adm, uurecp, opt_i);
1916 	if (opt_p) {
1917 		if ((pager = getenv("PAGER")) == NULL)
1918 			pager = "/usr/bin/more";
1919 		fp = popen(pager, "w");
1920 		if (fp == NULL) {
1921 			rt = FMADM_EXIT_ERROR;
1922 			opt_p = 0;
1923 		} else {
1924 			dup2(fileno(fp), 1);
1925 			setbuf(stdout, NULL);
1926 			(void) fclose(fp);
1927 		}
1928 	}
1929 	max_display = max_fault;
1930 	if (opt_f)
1931 		print_fru(opt_s, opt_a, opt_i, opt_p && !opt_s);
1932 	if (opt_r)
1933 		print_asru(opt_a);
1934 	if (opt_f == 0 && opt_r == 0)
1935 		print_catalog(opt_s, opt_a, opt_v, opt_i, opt_p && !opt_s);
1936 	fmd_msg_fini(fmadm_msghdl);
1937 	label_release_topo();
1938 	if (opt_p) {
1939 		(void) fclose(stdout);
1940 		(void) wait(&stat);
1941 	}
1942 	return (rt);
1943 }
1944 
1945 int
1946 cmd_flush(fmd_adm_t *adm, int argc, char *argv[])
1947 {
1948 	int i, status = FMADM_EXIT_SUCCESS;
1949 
1950 	if (argc < 2 || (i = getopt(argc, argv, "")) != EOF)
1951 		return (FMADM_EXIT_USAGE);
1952 
1953 	for (i = 1; i < argc; i++) {
1954 		if (fmd_adm_rsrc_flush(adm, argv[i]) != 0) {
1955 			warn("failed to flush %s", argv[i]);
1956 			status = FMADM_EXIT_ERROR;
1957 		} else
1958 			note("flushed resource history for %s\n", argv[i]);
1959 	}
1960 
1961 	return (status);
1962 }
1963 
1964 int
1965 cmd_repair(fmd_adm_t *adm, int argc, char *argv[])
1966 {
1967 	int err;
1968 
1969 	if (getopt(argc, argv, "") != EOF)
1970 		return (FMADM_EXIT_USAGE);
1971 
1972 	if (argc - optind != 1)
1973 		return (FMADM_EXIT_USAGE);
1974 
1975 	/*
1976 	 * argument could be a uuid, an fmri (asru, fru or resource)
1977 	 * or a label. Try uuid first, If that fails try the others.
1978 	 */
1979 	err = fmd_adm_case_repair(adm, argv[optind]);
1980 	if (err != 0)
1981 		err = fmd_adm_rsrc_repaired(adm, argv[optind]);
1982 
1983 	if (err != 0)
1984 		die("failed to record repair to %s", argv[optind]);
1985 
1986 	note("recorded repair to %s\n", argv[optind]);
1987 	return (FMADM_EXIT_SUCCESS);
1988 }
1989 
1990 int
1991 cmd_repaired(fmd_adm_t *adm, int argc, char *argv[])
1992 {
1993 	int err;
1994 
1995 	if (getopt(argc, argv, "") != EOF)
1996 		return (FMADM_EXIT_USAGE);
1997 
1998 	if (argc - optind != 1)
1999 		return (FMADM_EXIT_USAGE);
2000 
2001 	/*
2002 	 * argument could be an fmri (asru, fru or resource) or a label.
2003 	 */
2004 	err = fmd_adm_rsrc_repaired(adm, argv[optind]);
2005 	if (err != 0)
2006 		die("failed to record repair to %s", argv[optind]);
2007 
2008 	note("recorded repair to of %s\n", argv[optind]);
2009 	return (FMADM_EXIT_SUCCESS);
2010 }
2011 
2012 int
2013 cmd_replaced(fmd_adm_t *adm, int argc, char *argv[])
2014 {
2015 	int err;
2016 
2017 	if (getopt(argc, argv, "") != EOF)
2018 		return (FMADM_EXIT_USAGE);
2019 
2020 	if (argc - optind != 1)
2021 		return (FMADM_EXIT_USAGE);
2022 
2023 	/*
2024 	 * argument could be an fmri (asru, fru or resource) or a label.
2025 	 */
2026 	err = fmd_adm_rsrc_replaced(adm, argv[optind]);
2027 	if (err != 0)
2028 		die("failed to record replacement of %s", argv[optind]);
2029 
2030 	note("recorded replacement of %s\n", argv[optind]);
2031 	return (FMADM_EXIT_SUCCESS);
2032 }
2033 
2034 int
2035 cmd_acquit(fmd_adm_t *adm, int argc, char *argv[])
2036 {
2037 	int err;
2038 
2039 	if (getopt(argc, argv, "") != EOF)
2040 		return (FMADM_EXIT_USAGE);
2041 
2042 	if (argc - optind != 1 && argc - optind != 2)
2043 		return (FMADM_EXIT_USAGE);
2044 
2045 	/*
2046 	 * argument could be a uuid, an fmri (asru, fru or resource)
2047 	 * or a label. Or it could be a uuid and an fmri or label.
2048 	 */
2049 	if (argc - optind == 2) {
2050 		err = fmd_adm_rsrc_acquit(adm, argv[optind], argv[optind + 1]);
2051 		if (err != 0)
2052 			err = fmd_adm_rsrc_acquit(adm, argv[optind + 1],
2053 			    argv[optind]);
2054 	} else {
2055 		err = fmd_adm_case_acquit(adm, argv[optind]);
2056 		if (err != 0)
2057 			err = fmd_adm_rsrc_acquit(adm, argv[optind], "");
2058 	}
2059 
2060 	if (err != 0)
2061 		die("failed to record acquital of %s", argv[optind]);
2062 
2063 	note("recorded acquital of %s\n", argv[optind]);
2064 	return (FMADM_EXIT_SUCCESS);
2065 }
2066