xref: /titanic_50/usr/src/cmd/fm/fmdump/common/fmdump.c (revision a8449b6b91b07447ea0dd49293b52ffc26e07fe8)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <alloca.h>
26 #include <unistd.h>
27 #include <limits.h>
28 #include <strings.h>
29 #include <stdlib.h>
30 #include <stdarg.h>
31 #include <stdio.h>
32 #include <errno.h>
33 #include <time.h>
34 #include <ctype.h>
35 #include <regex.h>
36 #include <dirent.h>
37 #include <pthread.h>
38 
39 #include <fmdump.h>
40 
41 #define	FMDUMP_EXIT_SUCCESS	0
42 #define	FMDUMP_EXIT_FATAL	1
43 #define	FMDUMP_EXIT_USAGE	2
44 #define	FMDUMP_EXIT_ERROR	3
45 
46 const char *g_pname;
47 ulong_t g_errs;
48 ulong_t g_recs;
49 char *g_root;
50 
51 struct topo_hdl *g_thp;
52 fmd_msg_hdl_t *g_msg;
53 
54 /*PRINTFLIKE2*/
55 void
56 fmdump_printf(FILE *fp, const char *format, ...)
57 {
58 	va_list ap;
59 
60 	va_start(ap, format);
61 
62 	if (vfprintf(fp, format, ap) < 0) {
63 		(void) fprintf(stderr, "%s: failed to print record: %s\n",
64 		    g_pname, strerror(errno));
65 		g_errs++;
66 	}
67 
68 	va_end(ap);
69 }
70 
71 void
72 fmdump_vwarn(const char *format, va_list ap)
73 {
74 	int err = errno;
75 
76 	(void) fprintf(stderr, "%s: warning: ", g_pname);
77 	(void) vfprintf(stderr, format, ap);
78 
79 	if (strchr(format, '\n') == NULL)
80 		(void) fprintf(stderr, ": %s\n", strerror(err));
81 
82 	g_errs++;
83 }
84 
85 /*PRINTFLIKE1*/
86 void
87 fmdump_warn(const char *format, ...)
88 {
89 	va_list ap;
90 
91 	va_start(ap, format);
92 	fmdump_vwarn(format, ap);
93 	va_end(ap);
94 }
95 
96 static void
97 fmdump_exit(int err, int exitcode, const char *format, va_list ap)
98 {
99 	(void) fprintf(stderr, "%s: ", g_pname);
100 
101 	(void) vfprintf(stderr, format, ap);
102 
103 	if (strchr(format, '\n') == NULL)
104 		(void) fprintf(stderr, ": %s\n", strerror(err));
105 
106 	exit(exitcode);
107 }
108 
109 /*PRINTFLIKE1*/
110 static void
111 fmdump_fatal(const char *format, ...)
112 {
113 	int err = errno;
114 
115 	va_list ap;
116 
117 	va_start(ap, format);
118 	fmdump_exit(err, FMDUMP_EXIT_FATAL, format, ap);
119 	va_end(ap);
120 }
121 
122 /*PRINTFLIKE1*/
123 static void
124 fmdump_usage(const char *format, ...)
125 {
126 
127 	int err = errno;
128 
129 	va_list ap;
130 
131 	va_start(ap, format);
132 	fmdump_exit(err, FMDUMP_EXIT_USAGE, format, ap);
133 	va_end(ap);
134 }
135 
136 char *
137 fmdump_date(char *buf, size_t len, const fmd_log_record_t *rp)
138 {
139 	if (rp->rec_sec > LONG_MAX) {
140 		fmdump_warn("record time is too large for 32-bit utility\n");
141 		(void) snprintf(buf, len, "0x%llx", rp->rec_sec);
142 	} else {
143 		time_t tod = (time_t)rp->rec_sec;
144 		time_t now = time(NULL);
145 		if (tod > now+60 ||
146 		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
147 			(void) strftime(buf, len, "%b %d %Y %T",
148 			    localtime(&tod));
149 		} else {
150 			size_t sz;
151 			sz = strftime(buf, len, "%b %d %T", localtime(&tod));
152 			(void) snprintf(buf + sz, len - sz, ".%4.4llu",
153 			    rp->rec_nsec / (NANOSEC / 10000));
154 		}
155 	}
156 
157 	return (buf);
158 }
159 
160 char *
161 fmdump_year(char *buf, size_t len, const fmd_log_record_t *rp)
162 {
163 #ifdef _ILP32
164 	if (rp->rec_sec > LONG_MAX) {
165 		fmdump_warn("record time is too large for 32-bit utility\n");
166 		(void) snprintf(buf, len, "0x%llx", rp->rec_sec);
167 	} else {
168 #endif
169 		time_t tod = (time_t)rp->rec_sec;
170 		(void) strftime(buf, len, "%b %d %Y %T", localtime(&tod));
171 #ifdef _ILP32
172 	}
173 #endif
174 	return (buf);
175 }
176 
177 /* BEGIN CSTYLED */
178 static const char *synopsis =
179 "Usage: %s [[-e | -i | -I] | -A ] [-f] [-mvVp] [-c class] [-R root]\n"
180 	"\t      [-t time ][-T time] [-u uuid] [-n name[.name]*[=value]] "
181 							"[file]...\n    "
182     "Log selection: [-e | -i | -I] or one [file]; default is the fault log\n"
183 	"\t-e  display error log content\n"
184 	"\t-i  display infolog content\n"
185 	"\t-I  display the high-value-infolog content\n"
186 	"\t-R  set root directory for pathname expansions\n    "
187     "Command behaviour:\n"
188 	"\t-A  Aggregate specified [file]s or, if no [file], all known logs\n"
189 	"\t-f  follow growth of log file by waiting for additional data\n    "
190     "Output options:\n"
191 	"\t-m  display human-readable messages (only for fault logs)\n"
192 	"\t-v  set verbose mode: display additional event detail\n"
193 	"\t-V  set very verbose mode: display complete event contents\n"
194 	"\t-p  Used with -V: apply some output prettification\n    "
195     "Selection filters:\n"
196 	"\t-c  select events that match the specified class\n"
197 	"\t-t  select events that occurred after the specified time\n"
198 	"\t-T  select events that occurred before the specified time\n"
199 	"\t-u  select events that match the specified diagnosis uuid\n"
200 	"\t-n  select events containing named nvpair (with matching value)\n";
201 /* END CSTYLED */
202 
203 static int
204 usage(FILE *fp)
205 {
206 	(void) fprintf(fp, synopsis, g_pname);
207 	return (FMDUMP_EXIT_USAGE);
208 }
209 
210 /*ARGSUSED*/
211 static int
212 error(fmd_log_t *lp, void *private)
213 {
214 	fmdump_warn("skipping record: %s\n",
215 	    fmd_log_errmsg(lp, fmd_log_errno(lp)));
216 	return (0);
217 }
218 
219 /*
220  * Yet another disgusting argument parsing function (TM).  We attempt to parse
221  * a time argument in a variety of strptime(3C) formats, in which case it is
222  * interpreted as a local time and is converted to a timeval using mktime(3C).
223  * If those formats fail, we look to see if the time is a decimal integer
224  * followed by one of our magic suffixes, in which case the time is interpreted
225  * as a time delta *before* the current time-of-day (i.e. "1h" = "1 hour ago").
226  */
227 static struct timeval *
228 gettimeopt(const char *arg)
229 {
230 	const struct {
231 		const char *name;
232 		hrtime_t mul;
233 	} suffix[] = {
234 		{ "ns",		NANOSEC / NANOSEC },
235 		{ "nsec",	NANOSEC / NANOSEC },
236 		{ "us",		NANOSEC / MICROSEC },
237 		{ "usec",	NANOSEC / MICROSEC },
238 		{ "ms",		NANOSEC / MILLISEC },
239 		{ "msec",	NANOSEC / MILLISEC },
240 		{ "s",		NANOSEC / SEC },
241 		{ "sec",	NANOSEC / SEC },
242 		{ "m",		NANOSEC * (hrtime_t)60 },
243 		{ "min",	NANOSEC * (hrtime_t)60 },
244 		{ "h",		NANOSEC * (hrtime_t)(60 * 60) },
245 		{ "hour",	NANOSEC * (hrtime_t)(60 * 60) },
246 		{ "d",		NANOSEC * (hrtime_t)(24 * 60 * 60) },
247 		{ "day",	NANOSEC * (hrtime_t)(24 * 60 * 60) },
248 		{ NULL }
249 	};
250 
251 	struct timeval *tvp = malloc(sizeof (struct timeval));
252 	struct timeval tod;
253 	struct tm tm;
254 	char *p;
255 
256 	if (tvp == NULL)
257 		fmdump_fatal("failed to allocate memory");
258 
259 	if (gettimeofday(&tod, NULL) != 0)
260 		fmdump_fatal("failed to get tod");
261 
262 	/*
263 	 * First try a variety of strptime() calls.  If these all fail, we'll
264 	 * try parsing an integer followed by one of our suffix[] strings.
265 	 * NOTE: any form using %y must appear *before* the equivalent %Y form;
266 	 * otherwise %Y will accept the two year digits but infer century zero.
267 	 * Any form ending in %y must additionally check isdigit(*p) to ensure
268 	 * that it does not inadvertently match 2 digits of a 4-digit year.
269 	 *
270 	 * Beware: Any strptime() sequence containing consecutive %x sequences
271 	 * may fall victim to SCCS expanding it as a keyword!  If this happens
272 	 * we use separate string constant that ANSI C will concatenate.
273 	 */
274 	if ((p = strptime(arg, "%m/%d/%y" "%t" "%H:%M:%S", &tm)) == NULL &&
275 	    (p = strptime(arg, "%m/%d/%Y" "%t" "%H:%M:%S", &tm)) == NULL &&
276 	    (p = strptime(arg, "%m/%d/%y" "%t" "%H:%M", &tm)) == NULL &&
277 	    (p = strptime(arg, "%m/%d/%Y" "%t" "%H:%M", &tm)) == NULL &&
278 	    ((p = strptime(arg, "%m/%d/%y", &tm)) == NULL || isdigit(*p)) &&
279 	    (p = strptime(arg, "%m/%d/%Y", &tm)) == NULL &&
280 	    (p = strptime(arg, "%y-%m-%dT%H:%M:%S", &tm)) == NULL &&
281 	    (p = strptime(arg, "%Y-%m-%dT%H:%M:%S", &tm)) == NULL &&
282 	    (p = strptime(arg, "%y-%m-%dT%H:%M", &tm)) == NULL &&
283 	    (p = strptime(arg, "%Y-%m-%dT%H:%M", &tm)) == NULL &&
284 	    (p = strptime(arg, "%y-%m-%d", &tm)) == NULL &&
285 	    (p = strptime(arg, "%Y-%m-%d", &tm)) == NULL &&
286 	    (p = strptime(arg, "%d%b%y" "%t" "%H:%M:%S", &tm)) == NULL &&
287 	    (p = strptime(arg, "%d%b%Y" "%t" "%H:%M:%S", &tm)) == NULL &&
288 	    (p = strptime(arg, "%d%b%y" "%t" "%H:%M", &tm)) == NULL &&
289 	    (p = strptime(arg, "%d%b%Y" "%t" "%H:%M", &tm)) == NULL &&
290 	    ((p = strptime(arg, "%d%b%y", &tm)) == NULL || isdigit(*p)) &&
291 	    (p = strptime(arg, "%d%b%Y", &tm)) == NULL &&
292 	    (p = strptime(arg, "%b%t%d" "%t" "%H:%M:%S", &tm)) == NULL &&
293 	    (p = strptime(arg, "%b%t%d" "%t" "%H:%M:%S", &tm)) == NULL &&
294 	    (p = strptime(arg, "%H:%M:%S", &tm)) == NULL &&
295 	    (p = strptime(arg, "%H:%M", &tm)) == NULL) {
296 
297 		hrtime_t nsec;
298 		int i;
299 
300 		errno = 0;
301 		nsec = strtol(arg, (char **)&p, 10);
302 
303 		if (errno != 0 || nsec == 0 || p == arg || *p == '\0')
304 			fmdump_usage("illegal time format -- %s\n", arg);
305 
306 		for (i = 0; suffix[i].name != NULL; i++) {
307 			if (strcasecmp(suffix[i].name, p) == 0) {
308 				nsec *= suffix[i].mul;
309 				break;
310 			}
311 		}
312 
313 		if (suffix[i].name == NULL)
314 			fmdump_usage("illegal time format -- %s\n", arg);
315 
316 		tvp->tv_sec = nsec / NANOSEC;
317 		tvp->tv_usec = (nsec % NANOSEC) / (NANOSEC / MICROSEC);
318 
319 		if (tvp->tv_sec > tod.tv_sec)
320 			fmdump_usage("time delta precedes UTC time origin "
321 			    "-- %s\n", arg);
322 
323 		tvp->tv_sec = tod.tv_sec - tvp->tv_sec;
324 
325 	} else if (*p == '\0' || *p == '.') {
326 		/*
327 		 * If tm_year is zero, we matched [%b %d] %H:%M[:%S]; use
328 		 * the result of localtime(&tod.tv_sec) to fill in the rest.
329 		 */
330 		if (tm.tm_year == 0) {
331 			int h = tm.tm_hour;
332 			int m = tm.tm_min;
333 			int s = tm.tm_sec;
334 			int b = tm.tm_mon;
335 			int d = tm.tm_mday;
336 
337 			bcopy(localtime(&tod.tv_sec), &tm, sizeof (tm));
338 			tm.tm_isdst = 0; /* see strptime(3C) and below */
339 
340 			if (d > 0) {
341 				tm.tm_mon = b;
342 				tm.tm_mday = d;
343 			}
344 
345 			tm.tm_hour = h;
346 			tm.tm_min = m;
347 			tm.tm_sec = s;
348 		}
349 
350 		errno = 0;
351 		tvp->tv_sec = mktime(&tm);
352 		tvp->tv_usec = 0;
353 
354 		if (tvp->tv_sec == -1L && errno != 0)
355 			fmdump_fatal("failed to compose time %s", arg);
356 
357 		/*
358 		 * If our mktime() set tm_isdst, adjust the result for DST by
359 		 * subtracting the offset between the main and alternate zones.
360 		 */
361 		if (tm.tm_isdst)
362 			tvp->tv_sec -= timezone - altzone;
363 
364 		if (p[0] == '.') {
365 			arg = p;
366 			errno = 0;
367 			tvp->tv_usec =
368 			    (suseconds_t)(strtod(arg, &p) * (double)MICROSEC);
369 
370 			if (errno != 0 || p == arg || *p != '\0')
371 				fmdump_usage("illegal time suffix -- .%s\n",
372 				    arg);
373 		}
374 
375 	} else {
376 		fmdump_usage("unexpected suffix after time %s -- %s\n", arg, p);
377 	}
378 
379 	return (tvp);
380 }
381 
382 /*
383  * If the -u option is specified in combination with the -e option, we iterate
384  * over each record in the fault log with a matching UUID finding xrefs to the
385  * error log, and then use this function to iterate over every xref'd record.
386  */
387 int
388 xref_iter(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
389 {
390 	const fmd_log_record_t *xrp = rp->rec_xrefs;
391 	fmdump_arg_t *dap = arg;
392 	int i, rv = 0;
393 
394 	for (i = 0; rv == 0 && i < rp->rec_nrefs; i++, xrp++) {
395 		if (fmd_log_filter(lp, dap->da_fc, dap->da_fv, xrp))
396 			rv = dap->da_fmt->do_func(lp, xrp, dap->da_fp);
397 	}
398 
399 	return (rv);
400 }
401 
402 int
403 xoff_iter(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
404 {
405 	fmdump_lyr_t *dyp = arg;
406 
407 	fmdump_printf(dyp->dy_fp, "%16llx ", (u_longlong_t)rp->rec_off);
408 	return (dyp->dy_func(lp, rp, dyp->dy_arg));
409 }
410 
411 /*
412  * Initialize fmd_log_filter_nvarg_t from -n name=value argument string.
413  */
414 static fmd_log_filter_nvarg_t *
415 setupnamevalue(char *namevalue)
416 {
417 	fmd_log_filter_nvarg_t	*argt;
418 	char			*value;
419 	regex_t			*value_regex = NULL;
420 	char			errstr[128];
421 	int			rv;
422 
423 	if ((value = strchr(namevalue, '=')) == NULL) {
424 		value_regex = NULL;
425 	} else {
426 		*value++ = '\0';	/* separate name and value string */
427 
428 		/*
429 		 * Skip white space before value to facilitate direct
430 		 * cut/paste from previous fmdump output.
431 		 */
432 		while (isspace(*value))
433 			value++;
434 
435 		if ((value_regex = malloc(sizeof (regex_t))) == NULL)
436 			fmdump_fatal("failed to allocate memory");
437 
438 		/* compile regular expression for possible string match */
439 		if ((rv = regcomp(value_regex, value,
440 		    REG_NOSUB|REG_NEWLINE)) != 0) {
441 			(void) regerror(rv, value_regex, errstr,
442 			    sizeof (errstr));
443 			free(value_regex);
444 			fmdump_usage("unexpected regular expression in "
445 			    "%s: %s\n", value, errstr);
446 		}
447 	}
448 
449 	if ((argt = malloc(sizeof (fmd_log_filter_nvarg_t))) == NULL)
450 		fmdump_fatal("failed to allocate memory");
451 
452 	argt->nvarg_name = namevalue;		/* now just name */
453 	argt->nvarg_value = value;
454 	argt->nvarg_value_regex = value_regex;
455 	return (argt);
456 }
457 
458 /*
459  * If the -a option is not present, filter out fault records that correspond
460  * to events that the producer requested not be messaged for administrators.
461  */
462 /*ARGSUSED*/
463 int
464 log_filter_silent(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
465 {
466 	int opt_A = (arg != NULL);
467 	boolean_t msg;
468 	char *class;
469 
470 	/*
471 	 * If -A was used then apply this filter only to events of list class
472 	 */
473 	if (opt_A) {
474 		if (nvlist_lookup_string(rp->rec_nvl, FM_CLASS, &class) != 0 ||
475 		    strncmp(class, FM_LIST_EVENT ".",
476 		    sizeof (FM_LIST_EVENT)) != 0)
477 			return (1);
478 	}
479 
480 	return (nvlist_lookup_boolean_value(rp->rec_nvl,
481 	    FM_SUSPECT_MESSAGE, &msg) != 0 || msg != 0);
482 }
483 
484 struct loglink {
485 	char		*path;
486 	long		suffix;
487 	struct loglink	*next;
488 };
489 
490 static void
491 addlink(struct loglink **llp, char *dirname, char *logname, long suffix)
492 {
493 	struct loglink *newp;
494 	size_t len;
495 	char *str;
496 
497 	newp = malloc(sizeof (struct loglink));
498 	len = strlen(dirname) + strlen(logname) + 2;
499 	str = malloc(len);
500 	if (newp == NULL || str == NULL)
501 		fmdump_fatal("failed to allocate memory");
502 
503 	(void) snprintf(str, len, "%s/%s", dirname, logname);
504 	newp->path = str;
505 	newp->suffix = suffix;
506 
507 	while (*llp != NULL && suffix < (*llp)->suffix)
508 		llp = &(*llp)->next;
509 
510 	newp->next = *llp;
511 	*llp = newp;
512 }
513 
514 /*
515  * Find and return all the rotated logs.
516  */
517 static struct loglink *
518 get_rotated_logs(char *logpath)
519 {
520 	char dirname[PATH_MAX], *logname, *endptr;
521 	DIR *dirp;
522 	struct dirent *dp;
523 	long len, suffix;
524 	struct loglink *head = NULL;
525 
526 	(void) strlcpy(dirname, logpath, sizeof (dirname));
527 	logname = strrchr(dirname, '/');
528 	*logname++ = '\0';
529 	len = strlen(logname);
530 
531 	if ((dirp = opendir(dirname)) == NULL) {
532 		fmdump_warn("failed to opendir `%s'", dirname);
533 		g_errs++;
534 		return (NULL);
535 	}
536 
537 	while ((dp = readdir(dirp)) != NULL) {
538 		/*
539 		 * Search the log directory for logs named "<logname>.0",
540 		 * "<logname>.1", etc and add to the link in the
541 		 * reverse numeric order.
542 		 */
543 		if (strlen(dp->d_name) < len + 2 ||
544 		    strncmp(dp->d_name, logname, len) != 0 ||
545 		    dp->d_name[len] != '.')
546 			continue;
547 
548 		/*
549 		 * "*.0-" file normally should not be seen.  It may
550 		 * exist when user manually run 'fmadm rotate'.
551 		 * In such case, we put it at the end of the list so
552 		 * it'll be dumped after all the rotated logs, before
553 		 * the current one.
554 		 */
555 		if (strcmp(dp->d_name + len + 1, "0-") == 0)
556 			addlink(&head, dirname, dp->d_name, -1);
557 		else if ((suffix = strtol(dp->d_name + len + 1,
558 		    &endptr, 10)) >= 0 && *endptr == '\0')
559 			addlink(&head, dirname, dp->d_name, suffix);
560 	}
561 
562 	(void) closedir(dirp);
563 
564 	return (head);
565 }
566 
567 /*
568  * Aggregate log files.  If ifiles is not NULL then one or more files
569  * were listed on the command line, and we will merge just those files.
570  * Otherwise we will merge all known log file types, and include the
571  * rotated logs for each type (you can suppress the inclusion of
572  * some logtypes through use of FMDUMP_AGGREGATE_IGNORE in the process
573  * environment, setting it to a comma-separated list of log labels and/or
574  * log filenames to ignore).
575  *
576  * We will not attempt to perform a chronological sort across all log records
577  * of all files.  Indeed, we won't even sort individual log files -
578  * we will not re-order events differently to how they appeared in their
579  * original log file.  This is because log files are already inherently
580  * ordered by the order in which fmd receives and processes events.
581  * So we determine the output order by comparing the "next" record
582  * off the top of each log file.
583  *
584  * We will construct a number of log record source "pipelines".  As above,
585  * the next record to render in the overall output is that from the
586  * pipeline with the oldest event.
587  *
588  * For the case that input logfiles were listed on the command line, each
589  * pipeline will process exactly one of those logfiles.  Distinct pipelines
590  * may process logfiles of the same "type" - eg if two "error" logs and
591  * one "fault" logs are specified then there'll be two pipelines producing
592  * events from "error" logs.
593  *
594  * If we are merging all known log types then we will construct exactly
595  * one pipeline for each known log type - one for error, one for fault, etc.
596  * Each pipeline will process first the rotated logs of that type and then
597  * move on to the current log of that type.
598  *
599  * The output from all pipelines flows into a serializer which selects
600  * the next record once all pipelines have asserted their output state.
601  * The output state of a pipeline is one of:
602  *
603  *	- record available: the next record from this pipeline is available
604  *	  for comparison and consumption
605  *
606  *	- done: this pipeline will produce no more records
607  *
608  *	- polling: this pipeline is polling for new records and will
609  *	  make them available as output if/when any are observed
610  *
611  *	- processing: output state will be updated shortly
612  *
613  * A pipeline iterates over each file queued to it using fmd_log_xiter.
614  * We do this in a separate thread for each pipeline.  The callback on
615  * each iteration must update the serializer to let it know that
616  * a new record is available.  In the serializer thread we decide whether
617  * we have all records expected have arrived and it is time to choose
618  * the next output record.
619  */
620 
621 /*
622  * A pipeline descriptor.  The pl_cv condition variable is used together
623  * with pl_lock for initial synchronisation, and thereafter with the
624  * lock for the serializer for pausing and continuing this pipeline.
625  */
626 struct fmdump_pipeline {
627 	pthread_mutex_t pl_lock;	/* used only in pipeline startup */
628 	int pl_started;			/* sync with main thread on startup */
629 	pthread_t pl_thr;		/* our processing thread */
630 	pthread_cond_t pl_cv;		/* see above */
631 	struct loglink *pl_rotated;	/* rotated logs to process first */
632 	char *pl_logpath;		/* target path to process */
633 	char *pl_processing;		/* path currently being processed */
634 	struct fmdump_srlzer *pl_srlzer;	/* link to serializer */
635 	int pl_srlzeridx;		/* serializer index for this pipeline */
636 	const fmdump_ops_t *pl_ops;	/* ops for the log type we're given */
637 	int pl_fmt;			/* FMDUMP_{SHORT,VERB1,VERB2,PRETTY} */
638 	boolean_t pl_follow;		/* go into poll mode at log end */
639 	fmdump_arg_t pl_arg;		/* arguments */
640 };
641 
642 enum fmdump_pipestate {
643 	FMDUMP_PIPE_PROCESSING = 0x1000,
644 	FMDUMP_PIPE_RECORDAVAIL,
645 	FMDUMP_PIPE_POLLING,
646 	FMDUMP_PIPE_DONE
647 };
648 
649 /*
650  * Each pipeline has an associated output slot in the serializer.  This
651  * must be updated with the serializer locked.  After update evaluate
652  * whether there are enough slots decided that we should select a
653  * record to output.
654  */
655 struct fmdump_srlzer_slot {
656 	enum fmdump_pipestate ss_state;
657 	uint64_t ss_sec;
658 	uint64_t ss_nsec;
659 };
660 
661 /*
662  * All pipelines are linked to a single serializer.  The serializer
663  * structure must be updated under the ds_lock; this mutex is also
664  * paired with the pl_cv of individual pipelines (one mutex, many condvars)
665  * in pausing and continuing individual pipelines.
666  */
667 struct fmdump_srlzer {
668 	struct fmdump_pipeline *ds_pipearr;	/* pipeline array */
669 	pthread_mutex_t ds_lock;		/* see above */
670 	uint32_t ds_pipecnt;			/* number of pipelines */
671 	uint32_t ds_pollcnt;			/* pipelines in poll mode */
672 	uint32_t ds_nrecordavail;		/* pipelines with a record */
673 	uint32_t ds_ndone;			/* completed pipelines */
674 	struct fmdump_srlzer_slot *ds_slot;	/* slot array */
675 };
676 
677 /*
678  * All known log types.  When aggregation is requested an no file list
679  * is provided we will process the logs identified here (if lt_enabled
680  * is true and not over-ridden by environment settings).  We also
681  * use this in determining the appropriate ops structure for each distinct
682  * label.
683  */
684 static struct fmdump_logtype {
685 	const char *lt_label;		/* label from log header */
686 	boolean_t lt_enabled;		/* include in merge? */
687 	const char *lt_logname;		/* var/fm/fmd/%s */
688 	const fmdump_ops_t *lt_ops;
689 } logtypes[] = {
690 	{
691 		"error",
692 		B_TRUE,
693 		"errlog",
694 		&fmdump_err_ops
695 	},
696 	{
697 		"fault",
698 		B_TRUE,
699 		"fltlog",
700 		&fmdump_flt_ops
701 	},
702 	{
703 		"info",
704 		B_TRUE,
705 		"infolog",
706 		&fmdump_info_ops
707 	},
708 	{
709 		"info",
710 		B_TRUE,
711 		"infolog_hival",
712 		&fmdump_info_ops
713 	},
714 	{
715 		"asru",
716 		B_FALSE,		/* not included unless in file list */
717 		NULL,
718 		&fmdump_asru_ops	/* but we need ops when it is */
719 	}
720 };
721 
722 /*
723  * Disable logtypes per environment setting.  Does not apply when a list
724  * of logs is provided on the command line.
725  */
726 static void
727 do_disables(void)
728 {
729 	char *env = getenv("FMDUMP_AGGREGATE_IGNORE");
730 	char *dup, *start, *tofree;
731 	int i;
732 
733 	if (env == NULL)
734 		return;
735 
736 	tofree = dup = strdup(env);
737 
738 	while (dup != NULL) {
739 		start = strsep(&dup, ",");
740 		for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) {
741 			if (logtypes[i].lt_logname == NULL)
742 				continue;
743 
744 			if (strcmp(start, logtypes[i].lt_label) == 0 ||
745 			    strcmp(start, logtypes[i].lt_logname) == 0) {
746 				logtypes[i].lt_enabled = B_FALSE;
747 			}
748 		}
749 	}
750 
751 	free(tofree);
752 }
753 
754 static void
755 srlzer_enter(struct fmdump_pipeline *pl)
756 {
757 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
758 
759 	(void) pthread_mutex_lock(&srlzer->ds_lock);
760 }
761 
762 static void
763 srlzer_exit(struct fmdump_pipeline *pl)
764 {
765 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
766 
767 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
768 	(void) pthread_mutex_unlock(&srlzer->ds_lock);
769 }
770 
771 static struct fmdump_pipeline *
772 srlzer_choose(struct fmdump_srlzer *srlzer)
773 {
774 	struct fmdump_srlzer_slot *slot, *oldest;
775 	int oldestidx = -1;
776 	int first = 1;
777 	int i;
778 
779 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
780 
781 	for (i = 0, slot = &srlzer->ds_slot[0]; i < srlzer->ds_pipecnt;
782 	    i++, slot++) {
783 		if (slot->ss_state != FMDUMP_PIPE_RECORDAVAIL)
784 			continue;
785 
786 		if (first) {
787 			oldest = slot;
788 			oldestidx = i;
789 			first = 0;
790 			continue;
791 		}
792 
793 		if (slot->ss_sec < oldest->ss_sec ||
794 		    slot->ss_sec == oldest->ss_sec &&
795 		    slot->ss_nsec < oldest->ss_nsec) {
796 			oldest = slot;
797 			oldestidx = i;
798 		}
799 	}
800 
801 	return (oldestidx >= 0 ? &srlzer->ds_pipearr[oldestidx] : NULL);
802 }
803 
804 static void
805 pipeline_stall(struct fmdump_pipeline *pl)
806 {
807 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
808 
809 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
810 	(void) pthread_cond_wait(&pl->pl_cv, &srlzer->ds_lock);
811 }
812 
813 static void
814 pipeline_continue(struct fmdump_pipeline *pl)
815 {
816 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
817 
818 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
819 	(void) pthread_cond_signal(&srlzer->ds_pipearr[pl->pl_srlzeridx].pl_cv);
820 }
821 
822 /*
823  * Called on each pipeline record iteration to make a new record
824  * available for input to the serializer.  Returns 0 to indicate that
825  * the caller must stall the pipeline, or 1 to indicate that the
826  * caller should go ahead and render their record.  If this record
827  * addition fills the serializer then choose a pipeline that must
828  * render output.
829  */
830 static int
831 pipeline_output(struct fmdump_pipeline *pl, const fmd_log_record_t *rp)
832 {
833 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
834 	struct fmdump_srlzer_slot *slot;
835 	struct fmdump_pipeline *wpl;
836 	int thisidx = pl->pl_srlzeridx;
837 
838 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
839 
840 	slot = &srlzer->ds_slot[thisidx];
841 	slot->ss_state = FMDUMP_PIPE_RECORDAVAIL;
842 	slot->ss_sec = rp->rec_sec;
843 	slot->ss_nsec = rp->rec_nsec;
844 	srlzer->ds_nrecordavail++;
845 
846 	/*
847 	 * Once all pipelines are polling we just render in arrival order.
848 	 */
849 	if (srlzer->ds_pollcnt == srlzer->ds_pipecnt)
850 		return (1);
851 
852 	/*
853 	 * If not all pipelines have asserted an output yet then the
854 	 * caller must block.
855 	 */
856 	if (srlzer->ds_nrecordavail + srlzer->ds_ndone +
857 	    srlzer->ds_pollcnt < srlzer->ds_pipecnt)
858 		return (0);
859 
860 	/*
861 	 * Right so it's time to turn the crank by choosing which of the
862 	 * filled line of slots should produce output.  If it is the slot
863 	 * for our caller then return their index to them, otherwise return
864 	 * -1 to the caller to make them block and cv_signal the winner.
865 	 */
866 	wpl = srlzer_choose(srlzer);
867 	ASSERT(wpl != NULL);
868 
869 	if (wpl == pl)
870 		return (1);
871 
872 	/* Wake the oldest, and return 0 to put the caller to sleep */
873 	pipeline_continue(wpl);
874 
875 	return (0);
876 }
877 
878 static void
879 pipeline_mark_consumed(struct fmdump_pipeline *pl)
880 {
881 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
882 
883 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
884 	srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_PROCESSING;
885 	srlzer->ds_nrecordavail--;
886 }
887 
888 static void
889 pipeline_done(struct fmdump_pipeline *pl)
890 {
891 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
892 	struct fmdump_pipeline *wpl;
893 
894 	srlzer_enter(pl);
895 
896 	srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_DONE;
897 	srlzer->ds_ndone++;
898 	wpl = srlzer_choose(srlzer);
899 	if (wpl != NULL)
900 		pipeline_continue(wpl);
901 
902 	srlzer_exit(pl);
903 }
904 
905 static void
906 pipeline_pollmode(struct fmdump_pipeline *pl)
907 {
908 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
909 	struct fmdump_pipeline *wpl;
910 
911 	if (srlzer->ds_slot[pl->pl_srlzeridx].ss_state == FMDUMP_PIPE_POLLING)
912 		return;
913 
914 	srlzer_enter(pl);
915 
916 	srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_POLLING;
917 	if (++srlzer->ds_pollcnt + srlzer->ds_nrecordavail ==
918 	    srlzer->ds_pipecnt && (wpl = srlzer_choose(srlzer)) != NULL)
919 		pipeline_continue(wpl);
920 
921 	srlzer_exit(pl);
922 }
923 
924 static int
925 pipeline_err(fmd_log_t *lp, void *arg)
926 {
927 	struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg;
928 
929 	fmdump_warn("skipping record in %s: %s\n", pl->pl_processing,
930 	    fmd_log_errmsg(lp, fmd_log_errno(lp)));
931 	g_errs++;
932 
933 	return (0);
934 }
935 
936 static int
937 pipeline_cb(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
938 {
939 	struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg;
940 	int rc;
941 
942 	fmd_log_rec_f *func = pl->pl_arg.da_fmt->do_func;
943 
944 	srlzer_enter(pl);
945 
946 	if (!pipeline_output(pl, rp))
947 		pipeline_stall(pl);
948 
949 	rc = func(lp, rp, pl->pl_arg.da_fp);
950 	pipeline_mark_consumed(pl);
951 
952 	srlzer_exit(pl);
953 
954 	return (rc);
955 }
956 
957 static void
958 pipeline_process(struct fmdump_pipeline *pl, char *logpath, boolean_t follow)
959 {
960 	fmd_log_header_t log;
961 	fmd_log_t *lp;
962 	int err;
963 	int i;
964 
965 	pl->pl_processing = logpath;
966 
967 	if ((lp = fmd_log_open(FMD_LOG_VERSION, logpath, &err)) == NULL) {
968 		fmdump_warn("failed to open %s: %s\n",
969 		    logpath, fmd_log_errmsg(NULL, err));
970 		g_errs++;
971 		return;
972 	}
973 
974 	fmd_log_header(lp, &log);
975 	for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) {
976 		if (strcmp(log.log_label, logtypes[i].lt_label) == 0) {
977 			pl->pl_ops = logtypes[i].lt_ops;
978 			pl->pl_arg.da_fmt =
979 			    &pl->pl_ops->do_formats[pl->pl_fmt];
980 			break;
981 		}
982 	}
983 
984 	if (pl->pl_ops == NULL) {
985 		fmdump_warn("unknown log type %s for %s\n",
986 		    log.log_label, logpath);
987 		g_errs++;
988 		return;
989 	}
990 
991 	do {
992 		if (fmd_log_xiter(lp, FMD_LOG_XITER_REFS, pl->pl_arg.da_fc,
993 		    pl->pl_arg.da_fv, pipeline_cb, pipeline_err, (void *)pl,
994 		    NULL) != 0) {
995 			fmdump_warn("failed to dump %s: %s\n",
996 			    logpath, fmd_log_errmsg(lp, fmd_log_errno(lp)));
997 			g_errs++;
998 			fmd_log_close(lp);
999 			return;
1000 		}
1001 
1002 		if (follow) {
1003 			pipeline_pollmode(pl);
1004 			(void) sleep(1);
1005 		}
1006 
1007 	} while (follow);
1008 
1009 	fmd_log_close(lp);
1010 }
1011 
1012 static void *
1013 pipeline_thr(void *arg)
1014 {
1015 	struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg;
1016 	struct loglink *ll;
1017 
1018 	(void) pthread_mutex_lock(&pl->pl_lock);
1019 	pl->pl_started = 1;
1020 	(void) pthread_mutex_unlock(&pl->pl_lock);
1021 	(void) pthread_cond_signal(&pl->pl_cv);
1022 
1023 	for (ll = pl->pl_rotated; ll != NULL; ll = ll->next)
1024 		pipeline_process(pl, ll->path, B_FALSE);
1025 
1026 	pipeline_process(pl, pl->pl_logpath, pl->pl_follow);
1027 	pipeline_done(pl);
1028 
1029 	return (NULL);
1030 }
1031 
1032 
1033 static int
1034 aggregate(char **ifiles, int n_ifiles, int opt_f,
1035     fmd_log_filter_t *fv, uint_t fc,
1036     int opt_v, int opt_V, int opt_p)
1037 {
1038 	struct fmdump_pipeline *pipeline, *pl;
1039 	struct fmdump_srlzer srlzer;
1040 	uint32_t npipe;
1041 	int fmt;
1042 	int i;
1043 
1044 	if (ifiles != NULL) {
1045 		npipe = n_ifiles;
1046 		pipeline = calloc(npipe, sizeof (struct fmdump_pipeline));
1047 		if (!pipeline)
1048 			fmdump_fatal("failed to allocate memory");
1049 
1050 		for (i = 0; i < n_ifiles; i++)
1051 			pipeline[i].pl_logpath = ifiles[i];
1052 	} else {
1053 		pipeline = calloc(sizeof (logtypes) / sizeof (logtypes[0]),
1054 		    sizeof (struct fmdump_pipeline));
1055 		if (!pipeline)
1056 			fmdump_fatal("failed to allocate memory");
1057 
1058 		do_disables();
1059 
1060 		npipe = 0;
1061 		for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) {
1062 			struct fmdump_logtype *ltp = &logtypes[i];
1063 			char *logpath;
1064 
1065 			if (ltp->lt_enabled == B_FALSE)
1066 				continue;
1067 
1068 			if ((logpath = malloc(PATH_MAX)) == NULL)
1069 				fmdump_fatal("failed to allocate memory");
1070 
1071 			(void) snprintf(logpath, PATH_MAX,
1072 			    "%s/var/fm/fmd/%s",
1073 			    g_root ? g_root : "", ltp->lt_logname);
1074 
1075 			pipeline[npipe].pl_rotated =
1076 			    get_rotated_logs(logpath);
1077 
1078 			pipeline[npipe++].pl_logpath = logpath;
1079 		}
1080 	}
1081 
1082 	if (opt_V)
1083 		fmt = opt_p ? FMDUMP_PRETTY : FMDUMP_VERB2;
1084 	else if (opt_v)
1085 		fmt = FMDUMP_VERB1;
1086 	else
1087 		fmt = FMDUMP_SHORT;
1088 
1089 	bzero(&srlzer, sizeof (srlzer));
1090 	srlzer.ds_pipearr = pipeline;
1091 	srlzer.ds_pipecnt = npipe;
1092 	srlzer.ds_slot = calloc(npipe, sizeof (struct fmdump_srlzer_slot));
1093 	if (!srlzer.ds_slot)
1094 		fmdump_fatal("failed to allocate memory");
1095 	(void) pthread_mutex_init(&srlzer.ds_lock, NULL);
1096 
1097 	for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) {
1098 		(void) pthread_mutex_init(&pl->pl_lock, NULL);
1099 		(void) pthread_cond_init(&pl->pl_cv, NULL);
1100 		srlzer.ds_slot[i].ss_state = FMDUMP_PIPE_PROCESSING;
1101 		pl->pl_srlzer = &srlzer;
1102 		pl->pl_srlzeridx = i;
1103 		pl->pl_follow = opt_f ? B_TRUE : B_FALSE;
1104 		pl->pl_fmt = fmt;
1105 		pl->pl_arg.da_fv = fv;
1106 		pl->pl_arg.da_fc = fc;
1107 		pl->pl_arg.da_fp = stdout;
1108 
1109 		(void) pthread_mutex_lock(&pl->pl_lock);
1110 
1111 		if (pthread_create(&pl->pl_thr, NULL,
1112 		    pipeline_thr, (void *)pl) != 0)
1113 			fmdump_fatal("pthread_create for pipeline %d failed",
1114 			    i);
1115 	}
1116 
1117 	for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) {
1118 		while (!pl->pl_started)
1119 			(void) pthread_cond_wait(&pl->pl_cv, &pl->pl_lock);
1120 
1121 		(void) pthread_mutex_unlock(&pl->pl_lock);
1122 	}
1123 
1124 	for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++)
1125 		(void) pthread_join(pl->pl_thr, NULL);
1126 
1127 	if (ifiles == NULL) {
1128 		for (i = 0; i < npipe; i++)
1129 			free(pipeline[i].pl_logpath);
1130 	}
1131 
1132 	free(srlzer.ds_slot);
1133 
1134 	free(pipeline);
1135 
1136 	return (FMDUMP_EXIT_SUCCESS);
1137 }
1138 
1139 static void
1140 cleanup(char **ifiles, int n_ifiles)
1141 {
1142 	int i;
1143 
1144 	if (ifiles == NULL)
1145 		return;
1146 
1147 	for (i = 0; i < n_ifiles; i++) {
1148 		if (ifiles[i] != NULL) {
1149 			free(ifiles[i]);
1150 			ifiles[i] = NULL;
1151 		}
1152 	}
1153 
1154 	free(ifiles);
1155 }
1156 
1157 int
1158 main(int argc, char *argv[])
1159 {
1160 	int opt_a = 0, opt_e = 0, opt_f = 0, opt_H = 0, opt_m = 0, opt_p = 0;
1161 	int opt_u = 0, opt_v = 0, opt_V = 0;
1162 	int opt_i = 0, opt_I = 0;
1163 	int opt_A = 0;
1164 	char **ifiles = NULL;
1165 	char *ifile = NULL;
1166 	int n_ifiles;
1167 	int ifileidx = 0;
1168 	int iflags = 0;
1169 
1170 	fmdump_arg_t arg;
1171 	fmdump_lyr_t lyr;
1172 	const fmdump_ops_t *ops;
1173 	fmd_log_filter_t *filtv;
1174 	uint_t filtc;
1175 
1176 	fmd_log_filter_t *errfv, *fltfv, *allfv;
1177 	uint_t errfc = 0, fltfc = 0, allfc = 0;
1178 
1179 	fmd_log_header_t log;
1180 	fmd_log_rec_f *func;
1181 	void *farg;
1182 	fmd_log_t *lp;
1183 	int c, err;
1184 	off64_t off = 0;
1185 	ulong_t recs;
1186 	struct loglink *rotated_logs = NULL, *llp;
1187 
1188 	g_pname = argv[0];
1189 
1190 	errfv = alloca(sizeof (fmd_log_filter_t) * argc);
1191 	fltfv = alloca(sizeof (fmd_log_filter_t) * argc);
1192 	allfv = alloca(sizeof (fmd_log_filter_t) * argc);
1193 
1194 	while (optind < argc) {
1195 		while ((c =
1196 		    getopt(argc, argv, "Aac:efHiImn:O:pR:t:T:u:vV")) != EOF) {
1197 			switch (c) {
1198 			case 'A':
1199 				opt_A++;
1200 				break;
1201 			case 'a':
1202 				opt_a++;
1203 				break;
1204 			case 'c':
1205 				errfv[errfc].filt_func = fmd_log_filter_class;
1206 				errfv[errfc].filt_arg = optarg;
1207 				allfv[allfc++] = errfv[errfc++];
1208 				break;
1209 			case 'e':
1210 				if (opt_i)
1211 					return (usage(stderr));
1212 				opt_e++;
1213 				break;
1214 			case 'f':
1215 				opt_f++;
1216 				break;
1217 			case 'H':
1218 				opt_H++;
1219 				break;
1220 			case 'i':
1221 				if (opt_e || opt_I)
1222 					return (usage(stderr));
1223 				opt_i++;
1224 				break;
1225 			case 'I':
1226 				if (opt_e || opt_i)
1227 					return (usage(stderr));
1228 				opt_I++;
1229 				break;
1230 			case 'm':
1231 				opt_m++;
1232 				break;
1233 			case 'O':
1234 				off = strtoull(optarg, NULL, 16);
1235 				iflags |= FMD_LOG_XITER_OFFS;
1236 				break;
1237 			case 'p':
1238 				opt_p++;
1239 				break;
1240 			case 'R':
1241 				g_root = optarg;
1242 				break;
1243 			case 't':
1244 				errfv[errfc].filt_func = fmd_log_filter_after;
1245 				errfv[errfc].filt_arg = gettimeopt(optarg);
1246 				allfv[allfc++] = errfv[errfc++];
1247 				break;
1248 			case 'T':
1249 				errfv[errfc].filt_func = fmd_log_filter_before;
1250 				errfv[errfc].filt_arg = gettimeopt(optarg);
1251 				allfv[allfc++] = errfv[errfc++];
1252 				break;
1253 			case 'u':
1254 				fltfv[fltfc].filt_func = fmd_log_filter_uuid;
1255 				fltfv[fltfc].filt_arg = optarg;
1256 				allfv[allfc++] = fltfv[fltfc++];
1257 				opt_u++;
1258 				opt_a++; /* -u implies -a */
1259 				break;
1260 			case 'n': {
1261 				fltfv[fltfc].filt_func = fmd_log_filter_nv;
1262 				fltfv[fltfc].filt_arg = setupnamevalue(optarg);
1263 				allfv[allfc++] = fltfv[fltfc++];
1264 				break;
1265 			}
1266 			case 'v':
1267 				opt_v++;
1268 				break;
1269 			case 'V':
1270 				opt_V++;
1271 				break;
1272 			default:
1273 				return (usage(stderr));
1274 			}
1275 		}
1276 
1277 		if (opt_A && (opt_e || opt_i || opt_I || opt_m || opt_u))
1278 			fmdump_usage("-A excludes all of "
1279 			    "-e, -i, -I, -m and -u\n");
1280 
1281 		if (optind < argc) {
1282 			char *dest;
1283 
1284 			if (ifiles == NULL) {
1285 				n_ifiles = argc - optind;
1286 				ifiles = calloc(n_ifiles, sizeof (char *));
1287 				if (ifiles == NULL) {
1288 					fmdump_fatal(
1289 					    "failed to allocate memory for "
1290 					    "%d input file%s", n_ifiles,
1291 					    n_ifiles > 1 ? "s" : "");
1292 				}
1293 			}
1294 
1295 			if (ifileidx > 0 && !opt_A)
1296 				fmdump_usage("illegal argument -- %s\n",
1297 				    argv[optind]);
1298 
1299 			if ((dest = malloc(PATH_MAX)) == NULL)
1300 				fmdump_fatal("failed to allocate memory");
1301 
1302 			(void) strlcpy(dest, argv[optind++], PATH_MAX);
1303 			ifiles[ifileidx++] = dest;
1304 		}
1305 	}
1306 
1307 	if (opt_A) {
1308 		int rc;
1309 
1310 		if (!opt_a) {
1311 			fltfv[fltfc].filt_func = log_filter_silent;
1312 			fltfv[fltfc].filt_arg = (void *)1;
1313 			allfv[allfc++] = fltfv[fltfc++];
1314 		}
1315 
1316 		rc = aggregate(ifiles, n_ifiles, opt_f,
1317 		    allfv, allfc,
1318 		    opt_v, opt_V, opt_p);
1319 
1320 		cleanup(ifiles, n_ifiles);
1321 		return (rc);
1322 	} else {
1323 		if (ifiles == NULL) {
1324 			if ((ifile = calloc(1, PATH_MAX)) == NULL)
1325 				fmdump_fatal("failed to allocate memory");
1326 		} else {
1327 			ifile = ifiles[0];
1328 		}
1329 	}
1330 
1331 
1332 	if (*ifile == '\0') {
1333 		const char *pfx, *sfx;
1334 
1335 		if (opt_u || (!opt_e && !opt_i && !opt_I)) {
1336 			pfx = "flt";
1337 			sfx = "";
1338 		} else {
1339 			if (opt_e) {
1340 				pfx = "err";
1341 				sfx = "";
1342 			} else {
1343 				pfx = "info";
1344 				sfx = opt_I ? "_hival" : "";
1345 			}
1346 		}
1347 
1348 		(void) snprintf(ifile, PATH_MAX, "%s/var/fm/fmd/%slog%s",
1349 		    g_root ? g_root : "", pfx, sfx);
1350 		/*
1351 		 * logadm may rotate the logs.  When no input file is specified,
1352 		 * we try to dump all the rotated logs as well in the right
1353 		 * order.
1354 		 */
1355 		if (!opt_H && off == 0)
1356 			rotated_logs = get_rotated_logs(ifile);
1357 	} else if (g_root != NULL) {
1358 		fmdump_usage("-R option is not appropriate "
1359 		    "when file operand is present\n");
1360 	}
1361 
1362 	if ((g_msg = fmd_msg_init(g_root, FMD_MSG_VERSION)) == NULL)
1363 		fmdump_fatal("failed to initialize libfmd_msg");
1364 
1365 	if ((lp = fmd_log_open(FMD_LOG_VERSION, ifile, &err)) == NULL) {
1366 		fmdump_fatal("failed to open %s: %s\n", ifile,
1367 		    fmd_log_errmsg(NULL, err));
1368 	}
1369 
1370 	if (opt_H) {
1371 		fmd_log_header(lp, &log);
1372 
1373 		(void) printf("EXD_CREATOR = %s\n", log.log_creator);
1374 		(void) printf("EXD_HOSTNAME = %s\n", log.log_hostname);
1375 		(void) printf("EXD_FMA_LABEL = %s\n", log.log_label);
1376 		(void) printf("EXD_FMA_VERSION = %s\n", log.log_version);
1377 		(void) printf("EXD_FMA_OSREL = %s\n", log.log_osrelease);
1378 		(void) printf("EXD_FMA_OSVER = %s\n", log.log_osversion);
1379 		(void) printf("EXD_FMA_PLAT = %s\n", log.log_platform);
1380 		(void) printf("EXD_FMA_UUID = %s\n", log.log_uuid);
1381 
1382 		return (FMDUMP_EXIT_SUCCESS);
1383 	}
1384 
1385 	if (off != 0 && fmd_log_seek(lp, off) != 0) {
1386 		fmdump_fatal("failed to seek %s: %s\n", ifile,
1387 		    fmd_log_errmsg(lp, fmd_log_errno(lp)));
1388 	}
1389 
1390 	if (opt_e && opt_u)
1391 		ops = &fmdump_err_ops;
1392 	else if (strcmp(fmd_log_label(lp), fmdump_flt_ops.do_label) == 0)
1393 		ops = &fmdump_flt_ops;
1394 	else if (strcmp(fmd_log_label(lp), fmdump_asru_ops.do_label) == 0)
1395 		ops = &fmdump_asru_ops;
1396 	else if (strcmp(fmd_log_label(lp), fmdump_info_ops.do_label) == 0)
1397 		ops = &fmdump_info_ops;
1398 	else
1399 		ops = &fmdump_err_ops;
1400 
1401 	if (!opt_a && ops == &fmdump_flt_ops) {
1402 		fltfv[fltfc].filt_func = log_filter_silent;
1403 		fltfv[fltfc].filt_arg = NULL;
1404 		allfv[allfc++] = fltfv[fltfc++];
1405 	}
1406 
1407 	if (opt_V) {
1408 		arg.da_fmt =
1409 		    &ops->do_formats[opt_p ? FMDUMP_PRETTY : FMDUMP_VERB2];
1410 		iflags |= FMD_LOG_XITER_REFS;
1411 	} else if (opt_v) {
1412 		arg.da_fmt = &ops->do_formats[FMDUMP_VERB1];
1413 	} else if (opt_m) {
1414 		arg.da_fmt = &ops->do_formats[FMDUMP_MSG];
1415 	} else
1416 		arg.da_fmt = &ops->do_formats[FMDUMP_SHORT];
1417 
1418 	if (opt_m && arg.da_fmt->do_func == NULL) {
1419 		fmdump_usage("-m mode is not supported for "
1420 		    "log of type %s: %s\n", fmd_log_label(lp), ifile);
1421 	}
1422 
1423 	arg.da_fv = errfv;
1424 	arg.da_fc = errfc;
1425 	arg.da_fp = stdout;
1426 
1427 	if (iflags & FMD_LOG_XITER_OFFS)
1428 		fmdump_printf(arg.da_fp, "%16s ", "OFFSET");
1429 
1430 	if (arg.da_fmt->do_hdr && !(opt_V && ops == &fmdump_flt_ops))
1431 		fmdump_printf(arg.da_fp, "%s\n", arg.da_fmt->do_hdr);
1432 
1433 	if (opt_e && opt_u) {
1434 		iflags |= FMD_LOG_XITER_REFS;
1435 		func = xref_iter;
1436 		farg = &arg;
1437 		filtc = fltfc;
1438 		filtv = fltfv;
1439 	} else {
1440 		func = arg.da_fmt->do_func;
1441 		farg = arg.da_fp;
1442 		filtc = allfc;
1443 		filtv = allfv;
1444 	}
1445 
1446 	if (iflags & FMD_LOG_XITER_OFFS) {
1447 		lyr.dy_func = func;
1448 		lyr.dy_arg = farg;
1449 		lyr.dy_fp = arg.da_fp;
1450 		func = xoff_iter;
1451 		farg = &lyr;
1452 	}
1453 
1454 	for (llp = rotated_logs; llp != NULL; llp = llp->next) {
1455 		fmd_log_t *rlp;
1456 
1457 		if ((rlp = fmd_log_open(FMD_LOG_VERSION, llp->path, &err))
1458 		    == NULL) {
1459 			fmdump_warn("failed to open %s: %s\n",
1460 			    llp->path, fmd_log_errmsg(NULL, err));
1461 			g_errs++;
1462 			continue;
1463 		}
1464 
1465 		recs = 0;
1466 		if (fmd_log_xiter(rlp, iflags, filtc, filtv,
1467 		    func, error, farg, &recs) != 0) {
1468 			fmdump_warn("failed to dump %s: %s\n", llp->path,
1469 			    fmd_log_errmsg(rlp, fmd_log_errno(rlp)));
1470 			g_errs++;
1471 		}
1472 		g_recs += recs;
1473 
1474 		fmd_log_close(rlp);
1475 	}
1476 
1477 	do {
1478 		recs = 0;
1479 		if (fmd_log_xiter(lp, iflags, filtc, filtv,
1480 		    func, error, farg, &recs) != 0) {
1481 			fmdump_warn("failed to dump %s: %s\n", ifile,
1482 			    fmd_log_errmsg(lp, fmd_log_errno(lp)));
1483 			g_errs++;
1484 		}
1485 		g_recs += recs;
1486 
1487 		if (opt_f)
1488 			(void) sleep(1);
1489 
1490 	} while (opt_f);
1491 
1492 	if (!opt_f && g_recs == 0 && isatty(STDOUT_FILENO))
1493 		fmdump_warn("%s is empty\n", ifile);
1494 
1495 	if (g_thp != NULL)
1496 		topo_close(g_thp);
1497 
1498 	fmd_log_close(lp);
1499 	fmd_msg_fini(g_msg);
1500 
1501 	if (ifiles == NULL)
1502 		free(ifile);
1503 	else
1504 		cleanup(ifiles, n_ifiles);
1505 
1506 	return (g_errs ? FMDUMP_EXIT_ERROR : FMDUMP_EXIT_SUCCESS);
1507 }
1508