xref: /titanic_50/usr/src/cmd/fm/fmdump/common/fmdump.c (revision 2db6d663182655cb393dc2c15668bc9293364594)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
25  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
26  */
27 
28 #include <alloca.h>
29 #include <unistd.h>
30 #include <limits.h>
31 #include <strings.h>
32 #include <stdlib.h>
33 #include <stdarg.h>
34 #include <stdio.h>
35 #include <errno.h>
36 #include <time.h>
37 #include <ctype.h>
38 #include <regex.h>
39 #include <dirent.h>
40 #include <pthread.h>
41 
42 #include <fmdump.h>
43 
44 #define	FMDUMP_EXIT_SUCCESS	0
45 #define	FMDUMP_EXIT_FATAL	1
46 #define	FMDUMP_EXIT_USAGE	2
47 #define	FMDUMP_EXIT_ERROR	3
48 
49 const char *g_pname;
50 ulong_t g_errs;
51 ulong_t g_recs;
52 char *g_root;
53 
54 struct topo_hdl *g_thp;
55 fmd_msg_hdl_t *g_msg;
56 
57 /*PRINTFLIKE2*/
58 void
fmdump_printf(FILE * fp,const char * format,...)59 fmdump_printf(FILE *fp, const char *format, ...)
60 {
61 	va_list ap;
62 
63 	va_start(ap, format);
64 
65 	if (vfprintf(fp, format, ap) < 0) {
66 		(void) fprintf(stderr, "%s: failed to print record: %s\n",
67 		    g_pname, strerror(errno));
68 		g_errs++;
69 	}
70 
71 	va_end(ap);
72 }
73 
74 void
fmdump_vwarn(const char * format,va_list ap)75 fmdump_vwarn(const char *format, va_list ap)
76 {
77 	int err = errno;
78 
79 	(void) fprintf(stderr, "%s: warning: ", g_pname);
80 	(void) vfprintf(stderr, format, ap);
81 
82 	if (strchr(format, '\n') == NULL)
83 		(void) fprintf(stderr, ": %s\n", strerror(err));
84 
85 	g_errs++;
86 }
87 
88 /*PRINTFLIKE1*/
89 void
fmdump_warn(const char * format,...)90 fmdump_warn(const char *format, ...)
91 {
92 	va_list ap;
93 
94 	va_start(ap, format);
95 	fmdump_vwarn(format, ap);
96 	va_end(ap);
97 }
98 
99 static void
fmdump_exit(int err,int exitcode,const char * format,va_list ap)100 fmdump_exit(int err, int exitcode, const char *format, va_list ap)
101 {
102 	(void) fprintf(stderr, "%s: ", g_pname);
103 
104 	(void) vfprintf(stderr, format, ap);
105 
106 	if (strchr(format, '\n') == NULL)
107 		(void) fprintf(stderr, ": %s\n", strerror(err));
108 
109 	exit(exitcode);
110 }
111 
112 /*PRINTFLIKE1*/
113 static void
fmdump_fatal(const char * format,...)114 fmdump_fatal(const char *format, ...)
115 {
116 	int err = errno;
117 
118 	va_list ap;
119 
120 	va_start(ap, format);
121 	fmdump_exit(err, FMDUMP_EXIT_FATAL, format, ap);
122 	va_end(ap);
123 }
124 
125 /*PRINTFLIKE1*/
126 static void
fmdump_usage(const char * format,...)127 fmdump_usage(const char *format, ...)
128 {
129 
130 	int err = errno;
131 
132 	va_list ap;
133 
134 	va_start(ap, format);
135 	fmdump_exit(err, FMDUMP_EXIT_USAGE, format, ap);
136 	va_end(ap);
137 }
138 
139 char *
fmdump_date(char * buf,size_t len,const fmd_log_record_t * rp)140 fmdump_date(char *buf, size_t len, const fmd_log_record_t *rp)
141 {
142 	if (rp->rec_sec > LONG_MAX) {
143 		fmdump_warn("record time is too large for 32-bit utility\n");
144 		(void) snprintf(buf, len, "0x%llx", rp->rec_sec);
145 	} else {
146 		time_t tod = (time_t)rp->rec_sec;
147 		time_t now = time(NULL);
148 		if (tod > now+60 ||
149 		    tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */
150 			(void) strftime(buf, len, "%b %d %Y %T",
151 			    localtime(&tod));
152 		} else {
153 			size_t sz;
154 			sz = strftime(buf, len, "%b %d %T", localtime(&tod));
155 			(void) snprintf(buf + sz, len - sz, ".%4.4llu",
156 			    rp->rec_nsec / (NANOSEC / 10000));
157 		}
158 	}
159 
160 	return (buf);
161 }
162 
163 char *
fmdump_year(char * buf,size_t len,const fmd_log_record_t * rp)164 fmdump_year(char *buf, size_t len, const fmd_log_record_t *rp)
165 {
166 #ifdef _ILP32
167 	if (rp->rec_sec > LONG_MAX) {
168 		fmdump_warn("record time is too large for 32-bit utility\n");
169 		(void) snprintf(buf, len, "0x%llx", rp->rec_sec);
170 	} else {
171 #endif
172 		time_t tod = (time_t)rp->rec_sec;
173 		(void) strftime(buf, len, "%b %d %Y %T", localtime(&tod));
174 #ifdef _ILP32
175 	}
176 #endif
177 	return (buf);
178 }
179 
180 /* BEGIN CSTYLED */
181 static const char *synopsis =
182 "Usage: %s [[-e | -i | -I] | -A ] [-f] [-mvVp] [-c class] [-R root]\n"
183 	"\t      [-t time ][-T time] [-u uuid] [-n name[.name]*[=value]] "
184 							"[file]...\n    "
185     "Log selection: [-e | -i | -I] or one [file]; default is the fault log\n"
186 	"\t-e  display error log content\n"
187 	"\t-i  display infolog content\n"
188 	"\t-I  display the high-value-infolog content\n"
189 	"\t-R  set root directory for pathname expansions\n    "
190     "Command behaviour:\n"
191 	"\t-A  Aggregate specified [file]s or, if no [file], all known logs\n"
192 	"\t-f  follow growth of log file by waiting for additional data\n    "
193     "Output options:\n"
194 	"\t-m  display human-readable messages (only for fault logs)\n"
195 	"\t-v  set verbose mode: display additional event detail\n"
196 	"\t-V  set very verbose mode: display complete event contents\n"
197 	"\t-p  Used with -V: apply some output prettification\n"
198 	"\t-j  Used with -V: emit JSON-formatted output\n    "
199     "Selection filters:\n"
200 	"\t-c  select events that match the specified class\n"
201 	"\t-t  select events that occurred after the specified time\n"
202 	"\t-T  select events that occurred before the specified time\n"
203 	"\t-u  select events that match the specified diagnosis uuid\n"
204 	"\t-n  select events containing named nvpair (with matching value)\n";
205 /* END CSTYLED */
206 
207 static int
usage(FILE * fp)208 usage(FILE *fp)
209 {
210 	(void) fprintf(fp, synopsis, g_pname);
211 	return (FMDUMP_EXIT_USAGE);
212 }
213 
214 /*ARGSUSED*/
215 static int
error(fmd_log_t * lp,void * private)216 error(fmd_log_t *lp, void *private)
217 {
218 	fmdump_warn("skipping record: %s\n",
219 	    fmd_log_errmsg(lp, fmd_log_errno(lp)));
220 	return (0);
221 }
222 
223 /*
224  * Yet another disgusting argument parsing function (TM).  We attempt to parse
225  * a time argument in a variety of strptime(3C) formats, in which case it is
226  * interpreted as a local time and is converted to a timeval using mktime(3C).
227  * If those formats fail, we look to see if the time is a decimal integer
228  * followed by one of our magic suffixes, in which case the time is interpreted
229  * as a time delta *before* the current time-of-day (i.e. "1h" = "1 hour ago").
230  */
231 static struct timeval *
gettimeopt(const char * arg)232 gettimeopt(const char *arg)
233 {
234 	const struct {
235 		const char *name;
236 		hrtime_t mul;
237 	} suffix[] = {
238 		{ "ns",		NANOSEC / NANOSEC },
239 		{ "nsec",	NANOSEC / NANOSEC },
240 		{ "us",		NANOSEC / MICROSEC },
241 		{ "usec",	NANOSEC / MICROSEC },
242 		{ "ms",		NANOSEC / MILLISEC },
243 		{ "msec",	NANOSEC / MILLISEC },
244 		{ "s",		NANOSEC / SEC },
245 		{ "sec",	NANOSEC / SEC },
246 		{ "m",		NANOSEC * (hrtime_t)60 },
247 		{ "min",	NANOSEC * (hrtime_t)60 },
248 		{ "h",		NANOSEC * (hrtime_t)(60 * 60) },
249 		{ "hour",	NANOSEC * (hrtime_t)(60 * 60) },
250 		{ "d",		NANOSEC * (hrtime_t)(24 * 60 * 60) },
251 		{ "day",	NANOSEC * (hrtime_t)(24 * 60 * 60) },
252 		{ NULL }
253 	};
254 
255 	struct timeval *tvp = malloc(sizeof (struct timeval));
256 	struct timeval tod;
257 	struct tm tm;
258 	char *p;
259 
260 	if (tvp == NULL)
261 		fmdump_fatal("failed to allocate memory");
262 
263 	if (gettimeofday(&tod, NULL) != 0)
264 		fmdump_fatal("failed to get tod");
265 
266 	/*
267 	 * First try a variety of strptime() calls.  If these all fail, we'll
268 	 * try parsing an integer followed by one of our suffix[] strings.
269 	 */
270 	if ((p = strptime(arg, "%m/%d/%Y %H:%M:%S", &tm)) == NULL &&
271 	    (p = strptime(arg, "%m/%d/%y %H:%M:%S", &tm)) == NULL &&
272 	    (p = strptime(arg, "%m/%d/%Y %H:%M", &tm)) == NULL &&
273 	    (p = strptime(arg, "%m/%d/%y %H:%M", &tm)) == NULL &&
274 	    (p = strptime(arg, "%m/%d/%Y", &tm)) == NULL &&
275 	    (p = strptime(arg, "%m/%d/%y", &tm)) == NULL &&
276 	    (p = strptime(arg, "%Y-%m-%dT%H:%M:%S", &tm)) == NULL &&
277 	    (p = strptime(arg, "%y-%m-%dT%H:%M:%S", &tm)) == NULL &&
278 	    (p = strptime(arg, "%Y-%m-%dT%H:%M", &tm)) == NULL &&
279 	    (p = strptime(arg, "%y-%m-%dT%H:%M", &tm)) == NULL &&
280 	    (p = strptime(arg, "%Y-%m-%d", &tm)) == NULL &&
281 	    (p = strptime(arg, "%y-%m-%d", &tm)) == NULL &&
282 	    (p = strptime(arg, "%d%b%Y %H:%M:%S", &tm)) == NULL &&
283 	    (p = strptime(arg, "%d%b%y %H:%M:%S", &tm)) == NULL &&
284 	    (p = strptime(arg, "%d%b%Y %H:%M", &tm)) == NULL &&
285 	    (p = strptime(arg, "%d%b%y %H:%M", &tm)) == NULL &&
286 	    (p = strptime(arg, "%d%b%Y", &tm)) == NULL &&
287 	    (p = strptime(arg, "%d%b%y", &tm)) == NULL &&
288 	    (p = strptime(arg, "%b %d %H:%M:%S", &tm)) == NULL &&
289 	    (p = strptime(arg, "%b %d %H:%M:%S", &tm)) == NULL &&
290 	    (p = strptime(arg, "%H:%M:%S", &tm)) == NULL &&
291 	    (p = strptime(arg, "%H:%M", &tm)) == NULL) {
292 
293 		hrtime_t nsec;
294 		int i;
295 
296 		errno = 0;
297 		nsec = strtol(arg, (char **)&p, 10);
298 
299 		if (errno != 0 || nsec == 0 || p == arg || *p == '\0')
300 			fmdump_usage("illegal time format -- %s\n", arg);
301 
302 		for (i = 0; suffix[i].name != NULL; i++) {
303 			if (strcasecmp(suffix[i].name, p) == 0) {
304 				nsec *= suffix[i].mul;
305 				break;
306 			}
307 		}
308 
309 		if (suffix[i].name == NULL)
310 			fmdump_usage("illegal time format -- %s\n", arg);
311 
312 		tvp->tv_sec = nsec / NANOSEC;
313 		tvp->tv_usec = (nsec % NANOSEC) / (NANOSEC / MICROSEC);
314 
315 		if (tvp->tv_sec > tod.tv_sec)
316 			fmdump_usage("time delta precedes UTC time origin "
317 			    "-- %s\n", arg);
318 
319 		tvp->tv_sec = tod.tv_sec - tvp->tv_sec;
320 
321 	} else if (*p == '\0' || *p == '.') {
322 		/*
323 		 * If tm_year is zero, we matched [%b %d] %H:%M[:%S]; use
324 		 * the result of localtime(&tod.tv_sec) to fill in the rest.
325 		 */
326 		if (tm.tm_year == 0) {
327 			int h = tm.tm_hour;
328 			int m = tm.tm_min;
329 			int s = tm.tm_sec;
330 			int b = tm.tm_mon;
331 			int d = tm.tm_mday;
332 
333 			bcopy(localtime(&tod.tv_sec), &tm, sizeof (tm));
334 			tm.tm_isdst = 0; /* see strptime(3C) and below */
335 
336 			if (d > 0) {
337 				tm.tm_mon = b;
338 				tm.tm_mday = d;
339 			}
340 
341 			tm.tm_hour = h;
342 			tm.tm_min = m;
343 			tm.tm_sec = s;
344 		}
345 
346 		errno = 0;
347 		tvp->tv_sec = mktime(&tm);
348 		tvp->tv_usec = 0;
349 
350 		if (tvp->tv_sec == -1L && errno != 0)
351 			fmdump_fatal("failed to compose time %s", arg);
352 
353 		/*
354 		 * If our mktime() set tm_isdst, adjust the result for DST by
355 		 * subtracting the offset between the main and alternate zones.
356 		 */
357 		if (tm.tm_isdst)
358 			tvp->tv_sec -= timezone - altzone;
359 
360 		if (p[0] == '.') {
361 			arg = p;
362 			errno = 0;
363 			tvp->tv_usec =
364 			    (suseconds_t)(strtod(arg, &p) * (double)MICROSEC);
365 
366 			if (errno != 0 || p == arg || *p != '\0')
367 				fmdump_usage("illegal time suffix -- .%s\n",
368 				    arg);
369 		}
370 
371 	} else {
372 		fmdump_usage("unexpected suffix after time %s -- %s\n", arg, p);
373 	}
374 
375 	return (tvp);
376 }
377 
378 /*
379  * If the -u option is specified in combination with the -e option, we iterate
380  * over each record in the fault log with a matching UUID finding xrefs to the
381  * error log, and then use this function to iterate over every xref'd record.
382  */
383 int
xref_iter(fmd_log_t * lp,const fmd_log_record_t * rp,void * arg)384 xref_iter(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
385 {
386 	const fmd_log_record_t *xrp = rp->rec_xrefs;
387 	fmdump_arg_t *dap = arg;
388 	int i, rv = 0;
389 
390 	for (i = 0; rv == 0 && i < rp->rec_nrefs; i++, xrp++) {
391 		if (fmd_log_filter(lp, dap->da_fc, dap->da_fv, xrp))
392 			rv = dap->da_fmt->do_func(lp, xrp, dap->da_fp);
393 	}
394 
395 	return (rv);
396 }
397 
398 int
xoff_iter(fmd_log_t * lp,const fmd_log_record_t * rp,void * arg)399 xoff_iter(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
400 {
401 	fmdump_lyr_t *dyp = arg;
402 
403 	fmdump_printf(dyp->dy_fp, "%16llx ", (u_longlong_t)rp->rec_off);
404 	return (dyp->dy_func(lp, rp, dyp->dy_arg));
405 }
406 
407 /*
408  * Initialize fmd_log_filter_nvarg_t from -n name=value argument string.
409  */
410 static fmd_log_filter_nvarg_t *
setupnamevalue(char * namevalue)411 setupnamevalue(char *namevalue)
412 {
413 	fmd_log_filter_nvarg_t	*argt;
414 	char			*value;
415 	regex_t			*value_regex = NULL;
416 	char			errstr[128];
417 	int			rv;
418 
419 	if ((value = strchr(namevalue, '=')) == NULL) {
420 		value_regex = NULL;
421 	} else {
422 		*value++ = '\0';	/* separate name and value string */
423 
424 		/*
425 		 * Skip white space before value to facilitate direct
426 		 * cut/paste from previous fmdump output.
427 		 */
428 		while (isspace(*value))
429 			value++;
430 
431 		if ((value_regex = malloc(sizeof (regex_t))) == NULL)
432 			fmdump_fatal("failed to allocate memory");
433 
434 		/* compile regular expression for possible string match */
435 		if ((rv = regcomp(value_regex, value,
436 		    REG_NOSUB|REG_NEWLINE)) != 0) {
437 			(void) regerror(rv, value_regex, errstr,
438 			    sizeof (errstr));
439 			free(value_regex);
440 			fmdump_usage("unexpected regular expression in "
441 			    "%s: %s\n", value, errstr);
442 		}
443 	}
444 
445 	if ((argt = malloc(sizeof (fmd_log_filter_nvarg_t))) == NULL)
446 		fmdump_fatal("failed to allocate memory");
447 
448 	argt->nvarg_name = namevalue;		/* now just name */
449 	argt->nvarg_value = value;
450 	argt->nvarg_value_regex = value_regex;
451 	return (argt);
452 }
453 
454 /*
455  * If the -a option is not present, filter out fault records that correspond
456  * to events that the producer requested not be messaged for administrators.
457  */
458 /*ARGSUSED*/
459 int
log_filter_silent(fmd_log_t * lp,const fmd_log_record_t * rp,void * arg)460 log_filter_silent(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
461 {
462 	int opt_A = (arg != NULL);
463 	boolean_t msg;
464 	char *class;
465 
466 	/*
467 	 * If -A was used then apply this filter only to events of list class
468 	 */
469 	if (opt_A) {
470 		if (nvlist_lookup_string(rp->rec_nvl, FM_CLASS, &class) != 0 ||
471 		    strncmp(class, FM_LIST_EVENT ".",
472 		    sizeof (FM_LIST_EVENT)) != 0)
473 			return (1);
474 	}
475 
476 	return (nvlist_lookup_boolean_value(rp->rec_nvl,
477 	    FM_SUSPECT_MESSAGE, &msg) != 0 || msg != 0);
478 }
479 
480 struct loglink {
481 	char		*path;
482 	long		suffix;
483 	struct loglink	*next;
484 };
485 
486 static void
addlink(struct loglink ** llp,char * dirname,char * logname,long suffix)487 addlink(struct loglink **llp, char *dirname, char *logname, long suffix)
488 {
489 	struct loglink *newp;
490 	size_t len;
491 	char *str;
492 
493 	newp = malloc(sizeof (struct loglink));
494 	len = strlen(dirname) + strlen(logname) + 2;
495 	str = malloc(len);
496 	if (newp == NULL || str == NULL)
497 		fmdump_fatal("failed to allocate memory");
498 
499 	(void) snprintf(str, len, "%s/%s", dirname, logname);
500 	newp->path = str;
501 	newp->suffix = suffix;
502 
503 	while (*llp != NULL && suffix < (*llp)->suffix)
504 		llp = &(*llp)->next;
505 
506 	newp->next = *llp;
507 	*llp = newp;
508 }
509 
510 /*
511  * Find and return all the rotated logs.
512  */
513 static struct loglink *
get_rotated_logs(char * logpath)514 get_rotated_logs(char *logpath)
515 {
516 	char dirname[PATH_MAX], *logname, *endptr;
517 	DIR *dirp;
518 	struct dirent *dp;
519 	long len, suffix;
520 	struct loglink *head = NULL;
521 
522 	(void) strlcpy(dirname, logpath, sizeof (dirname));
523 	logname = strrchr(dirname, '/');
524 	*logname++ = '\0';
525 	len = strlen(logname);
526 
527 	if ((dirp = opendir(dirname)) == NULL) {
528 		fmdump_warn("failed to opendir `%s'", dirname);
529 		g_errs++;
530 		return (NULL);
531 	}
532 
533 	while ((dp = readdir(dirp)) != NULL) {
534 		/*
535 		 * Search the log directory for logs named "<logname>.0",
536 		 * "<logname>.1", etc and add to the link in the
537 		 * reverse numeric order.
538 		 */
539 		if (strlen(dp->d_name) < len + 2 ||
540 		    strncmp(dp->d_name, logname, len) != 0 ||
541 		    dp->d_name[len] != '.')
542 			continue;
543 
544 		/*
545 		 * "*.0-" file normally should not be seen.  It may
546 		 * exist when user manually run 'fmadm rotate'.
547 		 * In such case, we put it at the end of the list so
548 		 * it'll be dumped after all the rotated logs, before
549 		 * the current one.
550 		 */
551 		if (strcmp(dp->d_name + len + 1, "0-") == 0)
552 			addlink(&head, dirname, dp->d_name, -1);
553 		else if ((suffix = strtol(dp->d_name + len + 1,
554 		    &endptr, 10)) >= 0 && *endptr == '\0')
555 			addlink(&head, dirname, dp->d_name, suffix);
556 	}
557 
558 	(void) closedir(dirp);
559 
560 	return (head);
561 }
562 
563 /*
564  * Aggregate log files.  If ifiles is not NULL then one or more files
565  * were listed on the command line, and we will merge just those files.
566  * Otherwise we will merge all known log file types, and include the
567  * rotated logs for each type (you can suppress the inclusion of
568  * some logtypes through use of FMDUMP_AGGREGATE_IGNORE in the process
569  * environment, setting it to a comma-separated list of log labels and/or
570  * log filenames to ignore).
571  *
572  * We will not attempt to perform a chronological sort across all log records
573  * of all files.  Indeed, we won't even sort individual log files -
574  * we will not re-order events differently to how they appeared in their
575  * original log file.  This is because log files are already inherently
576  * ordered by the order in which fmd receives and processes events.
577  * So we determine the output order by comparing the "next" record
578  * off the top of each log file.
579  *
580  * We will construct a number of log record source "pipelines".  As above,
581  * the next record to render in the overall output is that from the
582  * pipeline with the oldest event.
583  *
584  * For the case that input logfiles were listed on the command line, each
585  * pipeline will process exactly one of those logfiles.  Distinct pipelines
586  * may process logfiles of the same "type" - eg if two "error" logs and
587  * one "fault" logs are specified then there'll be two pipelines producing
588  * events from "error" logs.
589  *
590  * If we are merging all known log types then we will construct exactly
591  * one pipeline for each known log type - one for error, one for fault, etc.
592  * Each pipeline will process first the rotated logs of that type and then
593  * move on to the current log of that type.
594  *
595  * The output from all pipelines flows into a serializer which selects
596  * the next record once all pipelines have asserted their output state.
597  * The output state of a pipeline is one of:
598  *
599  *	- record available: the next record from this pipeline is available
600  *	  for comparison and consumption
601  *
602  *	- done: this pipeline will produce no more records
603  *
604  *	- polling: this pipeline is polling for new records and will
605  *	  make them available as output if/when any are observed
606  *
607  *	- processing: output state will be updated shortly
608  *
609  * A pipeline iterates over each file queued to it using fmd_log_xiter.
610  * We do this in a separate thread for each pipeline.  The callback on
611  * each iteration must update the serializer to let it know that
612  * a new record is available.  In the serializer thread we decide whether
613  * we have all records expected have arrived and it is time to choose
614  * the next output record.
615  */
616 
617 /*
618  * A pipeline descriptor.  The pl_cv condition variable is used together
619  * with pl_lock for initial synchronisation, and thereafter with the
620  * lock for the serializer for pausing and continuing this pipeline.
621  */
622 struct fmdump_pipeline {
623 	pthread_mutex_t pl_lock;	/* used only in pipeline startup */
624 	int pl_started;			/* sync with main thread on startup */
625 	pthread_t pl_thr;		/* our processing thread */
626 	pthread_cond_t pl_cv;		/* see above */
627 	struct loglink *pl_rotated;	/* rotated logs to process first */
628 	char *pl_logpath;		/* target path to process */
629 	char *pl_processing;		/* path currently being processed */
630 	struct fmdump_srlzer *pl_srlzer;	/* link to serializer */
631 	int pl_srlzeridx;		/* serializer index for this pipeline */
632 	const fmdump_ops_t *pl_ops;	/* ops for the log type we're given */
633 	int pl_fmt;			/* FMDUMP_{SHORT,VERB1,VERB2,PRETTY} */
634 	boolean_t pl_follow;		/* go into poll mode at log end */
635 	fmdump_arg_t pl_arg;		/* arguments */
636 };
637 
638 enum fmdump_pipestate {
639 	FMDUMP_PIPE_PROCESSING = 0x1000,
640 	FMDUMP_PIPE_RECORDAVAIL,
641 	FMDUMP_PIPE_POLLING,
642 	FMDUMP_PIPE_DONE
643 };
644 
645 /*
646  * Each pipeline has an associated output slot in the serializer.  This
647  * must be updated with the serializer locked.  After update evaluate
648  * whether there are enough slots decided that we should select a
649  * record to output.
650  */
651 struct fmdump_srlzer_slot {
652 	enum fmdump_pipestate ss_state;
653 	uint64_t ss_sec;
654 	uint64_t ss_nsec;
655 };
656 
657 /*
658  * All pipelines are linked to a single serializer.  The serializer
659  * structure must be updated under the ds_lock; this mutex is also
660  * paired with the pl_cv of individual pipelines (one mutex, many condvars)
661  * in pausing and continuing individual pipelines.
662  */
663 struct fmdump_srlzer {
664 	struct fmdump_pipeline *ds_pipearr;	/* pipeline array */
665 	pthread_mutex_t ds_lock;		/* see above */
666 	uint32_t ds_pipecnt;			/* number of pipelines */
667 	uint32_t ds_pollcnt;			/* pipelines in poll mode */
668 	uint32_t ds_nrecordavail;		/* pipelines with a record */
669 	uint32_t ds_ndone;			/* completed pipelines */
670 	struct fmdump_srlzer_slot *ds_slot;	/* slot array */
671 };
672 
673 /*
674  * All known log types.  When aggregation is requested an no file list
675  * is provided we will process the logs identified here (if lt_enabled
676  * is true and not over-ridden by environment settings).  We also
677  * use this in determining the appropriate ops structure for each distinct
678  * label.
679  */
680 static struct fmdump_logtype {
681 	const char *lt_label;		/* label from log header */
682 	boolean_t lt_enabled;		/* include in merge? */
683 	const char *lt_logname;		/* var/fm/fmd/%s */
684 	const fmdump_ops_t *lt_ops;
685 } logtypes[] = {
686 	{
687 		"error",
688 		B_TRUE,
689 		"errlog",
690 		&fmdump_err_ops
691 	},
692 	{
693 		"fault",
694 		B_TRUE,
695 		"fltlog",
696 		&fmdump_flt_ops
697 	},
698 	{
699 		"info",
700 		B_TRUE,
701 		"infolog",
702 		&fmdump_info_ops
703 	},
704 	{
705 		"info",
706 		B_TRUE,
707 		"infolog_hival",
708 		&fmdump_info_ops
709 	},
710 	{
711 		"asru",
712 		B_FALSE,		/* not included unless in file list */
713 		NULL,
714 		&fmdump_asru_ops	/* but we need ops when it is */
715 	}
716 };
717 
718 /*
719  * Disable logtypes per environment setting.  Does not apply when a list
720  * of logs is provided on the command line.
721  */
722 static void
do_disables(void)723 do_disables(void)
724 {
725 	char *env = getenv("FMDUMP_AGGREGATE_IGNORE");
726 	char *dup, *start, *tofree;
727 	int i;
728 
729 	if (env == NULL)
730 		return;
731 
732 	tofree = dup = strdup(env);
733 
734 	while (dup != NULL) {
735 		start = strsep(&dup, ",");
736 		for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) {
737 			if (logtypes[i].lt_logname == NULL)
738 				continue;
739 
740 			if (strcmp(start, logtypes[i].lt_label) == 0 ||
741 			    strcmp(start, logtypes[i].lt_logname) == 0) {
742 				logtypes[i].lt_enabled = B_FALSE;
743 			}
744 		}
745 	}
746 
747 	free(tofree);
748 }
749 
750 static void
srlzer_enter(struct fmdump_pipeline * pl)751 srlzer_enter(struct fmdump_pipeline *pl)
752 {
753 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
754 
755 	(void) pthread_mutex_lock(&srlzer->ds_lock);
756 }
757 
758 static void
srlzer_exit(struct fmdump_pipeline * pl)759 srlzer_exit(struct fmdump_pipeline *pl)
760 {
761 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
762 
763 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
764 	(void) pthread_mutex_unlock(&srlzer->ds_lock);
765 }
766 
767 static struct fmdump_pipeline *
srlzer_choose(struct fmdump_srlzer * srlzer)768 srlzer_choose(struct fmdump_srlzer *srlzer)
769 {
770 	struct fmdump_srlzer_slot *slot, *oldest;
771 	int oldestidx = -1;
772 	int first = 1;
773 	int i;
774 
775 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
776 
777 	for (i = 0, slot = &srlzer->ds_slot[0]; i < srlzer->ds_pipecnt;
778 	    i++, slot++) {
779 		if (slot->ss_state != FMDUMP_PIPE_RECORDAVAIL)
780 			continue;
781 
782 		if (first) {
783 			oldest = slot;
784 			oldestidx = i;
785 			first = 0;
786 			continue;
787 		}
788 
789 		if (slot->ss_sec < oldest->ss_sec ||
790 		    slot->ss_sec == oldest->ss_sec &&
791 		    slot->ss_nsec < oldest->ss_nsec) {
792 			oldest = slot;
793 			oldestidx = i;
794 		}
795 	}
796 
797 	return (oldestidx >= 0 ? &srlzer->ds_pipearr[oldestidx] : NULL);
798 }
799 
800 static void
pipeline_stall(struct fmdump_pipeline * pl)801 pipeline_stall(struct fmdump_pipeline *pl)
802 {
803 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
804 
805 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
806 	(void) pthread_cond_wait(&pl->pl_cv, &srlzer->ds_lock);
807 }
808 
809 static void
pipeline_continue(struct fmdump_pipeline * pl)810 pipeline_continue(struct fmdump_pipeline *pl)
811 {
812 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
813 
814 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
815 	(void) pthread_cond_signal(&srlzer->ds_pipearr[pl->pl_srlzeridx].pl_cv);
816 }
817 
818 /*
819  * Called on each pipeline record iteration to make a new record
820  * available for input to the serializer.  Returns 0 to indicate that
821  * the caller must stall the pipeline, or 1 to indicate that the
822  * caller should go ahead and render their record.  If this record
823  * addition fills the serializer then choose a pipeline that must
824  * render output.
825  */
826 static int
pipeline_output(struct fmdump_pipeline * pl,const fmd_log_record_t * rp)827 pipeline_output(struct fmdump_pipeline *pl, const fmd_log_record_t *rp)
828 {
829 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
830 	struct fmdump_srlzer_slot *slot;
831 	struct fmdump_pipeline *wpl;
832 	int thisidx = pl->pl_srlzeridx;
833 
834 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
835 
836 	slot = &srlzer->ds_slot[thisidx];
837 	slot->ss_state = FMDUMP_PIPE_RECORDAVAIL;
838 	slot->ss_sec = rp->rec_sec;
839 	slot->ss_nsec = rp->rec_nsec;
840 	srlzer->ds_nrecordavail++;
841 
842 	/*
843 	 * Once all pipelines are polling we just render in arrival order.
844 	 */
845 	if (srlzer->ds_pollcnt == srlzer->ds_pipecnt)
846 		return (1);
847 
848 	/*
849 	 * If not all pipelines have asserted an output yet then the
850 	 * caller must block.
851 	 */
852 	if (srlzer->ds_nrecordavail + srlzer->ds_ndone +
853 	    srlzer->ds_pollcnt < srlzer->ds_pipecnt)
854 		return (0);
855 
856 	/*
857 	 * Right so it's time to turn the crank by choosing which of the
858 	 * filled line of slots should produce output.  If it is the slot
859 	 * for our caller then return their index to them, otherwise return
860 	 * -1 to the caller to make them block and cv_signal the winner.
861 	 */
862 	wpl = srlzer_choose(srlzer);
863 	ASSERT(wpl != NULL);
864 
865 	if (wpl == pl)
866 		return (1);
867 
868 	/* Wake the oldest, and return 0 to put the caller to sleep */
869 	pipeline_continue(wpl);
870 
871 	return (0);
872 }
873 
874 static void
pipeline_mark_consumed(struct fmdump_pipeline * pl)875 pipeline_mark_consumed(struct fmdump_pipeline *pl)
876 {
877 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
878 
879 	ASSERT(MUTEX_HELD(&srlzer->ds_lock));
880 	srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_PROCESSING;
881 	srlzer->ds_nrecordavail--;
882 }
883 
884 static void
pipeline_done(struct fmdump_pipeline * pl)885 pipeline_done(struct fmdump_pipeline *pl)
886 {
887 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
888 	struct fmdump_pipeline *wpl;
889 
890 	srlzer_enter(pl);
891 
892 	srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_DONE;
893 	srlzer->ds_ndone++;
894 	wpl = srlzer_choose(srlzer);
895 	if (wpl != NULL)
896 		pipeline_continue(wpl);
897 
898 	srlzer_exit(pl);
899 }
900 
901 static void
pipeline_pollmode(struct fmdump_pipeline * pl)902 pipeline_pollmode(struct fmdump_pipeline *pl)
903 {
904 	struct fmdump_srlzer *srlzer = pl->pl_srlzer;
905 	struct fmdump_pipeline *wpl;
906 
907 	if (srlzer->ds_slot[pl->pl_srlzeridx].ss_state == FMDUMP_PIPE_POLLING)
908 		return;
909 
910 	srlzer_enter(pl);
911 
912 	srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_POLLING;
913 	if (++srlzer->ds_pollcnt + srlzer->ds_nrecordavail ==
914 	    srlzer->ds_pipecnt && (wpl = srlzer_choose(srlzer)) != NULL)
915 		pipeline_continue(wpl);
916 
917 	srlzer_exit(pl);
918 }
919 
920 static int
pipeline_err(fmd_log_t * lp,void * arg)921 pipeline_err(fmd_log_t *lp, void *arg)
922 {
923 	struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg;
924 
925 	fmdump_warn("skipping record in %s: %s\n", pl->pl_processing,
926 	    fmd_log_errmsg(lp, fmd_log_errno(lp)));
927 	g_errs++;
928 
929 	return (0);
930 }
931 
932 static int
pipeline_cb(fmd_log_t * lp,const fmd_log_record_t * rp,void * arg)933 pipeline_cb(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg)
934 {
935 	struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg;
936 	int rc;
937 
938 	fmd_log_rec_f *func = pl->pl_arg.da_fmt->do_func;
939 
940 	srlzer_enter(pl);
941 
942 	if (!pipeline_output(pl, rp))
943 		pipeline_stall(pl);
944 
945 	rc = func(lp, rp, pl->pl_arg.da_fp);
946 	pipeline_mark_consumed(pl);
947 
948 	srlzer_exit(pl);
949 
950 	return (rc);
951 }
952 
953 static void
pipeline_process(struct fmdump_pipeline * pl,char * logpath,boolean_t follow)954 pipeline_process(struct fmdump_pipeline *pl, char *logpath, boolean_t follow)
955 {
956 	fmd_log_header_t log;
957 	fmd_log_t *lp;
958 	int err;
959 	int i;
960 
961 	pl->pl_processing = logpath;
962 
963 	if ((lp = fmd_log_open(FMD_LOG_VERSION, logpath, &err)) == NULL) {
964 		fmdump_warn("failed to open %s: %s\n",
965 		    logpath, fmd_log_errmsg(NULL, err));
966 		g_errs++;
967 		return;
968 	}
969 
970 	fmd_log_header(lp, &log);
971 	for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) {
972 		if (strcmp(log.log_label, logtypes[i].lt_label) == 0) {
973 			pl->pl_ops = logtypes[i].lt_ops;
974 			pl->pl_arg.da_fmt =
975 			    &pl->pl_ops->do_formats[pl->pl_fmt];
976 			break;
977 		}
978 	}
979 
980 	if (pl->pl_ops == NULL) {
981 		fmdump_warn("unknown log type %s for %s\n",
982 		    log.log_label, logpath);
983 		g_errs++;
984 		return;
985 	}
986 
987 	do {
988 		if (fmd_log_xiter(lp, FMD_LOG_XITER_REFS, pl->pl_arg.da_fc,
989 		    pl->pl_arg.da_fv, pipeline_cb, pipeline_err, (void *)pl,
990 		    NULL) != 0) {
991 			fmdump_warn("failed to dump %s: %s\n",
992 			    logpath, fmd_log_errmsg(lp, fmd_log_errno(lp)));
993 			g_errs++;
994 			fmd_log_close(lp);
995 			return;
996 		}
997 
998 		if (follow) {
999 			pipeline_pollmode(pl);
1000 			(void) sleep(1);
1001 		}
1002 
1003 	} while (follow);
1004 
1005 	fmd_log_close(lp);
1006 }
1007 
1008 static void *
pipeline_thr(void * arg)1009 pipeline_thr(void *arg)
1010 {
1011 	struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg;
1012 	struct loglink *ll;
1013 
1014 	(void) pthread_mutex_lock(&pl->pl_lock);
1015 	pl->pl_started = 1;
1016 	(void) pthread_mutex_unlock(&pl->pl_lock);
1017 	(void) pthread_cond_signal(&pl->pl_cv);
1018 
1019 	for (ll = pl->pl_rotated; ll != NULL; ll = ll->next)
1020 		pipeline_process(pl, ll->path, B_FALSE);
1021 
1022 	pipeline_process(pl, pl->pl_logpath, pl->pl_follow);
1023 	pipeline_done(pl);
1024 
1025 	return (NULL);
1026 }
1027 
1028 
1029 static int
aggregate(char ** ifiles,int n_ifiles,int opt_f,fmd_log_filter_t * fv,uint_t fc,int opt_v,int opt_V,int opt_p,int opt_j)1030 aggregate(char **ifiles, int n_ifiles, int opt_f,
1031     fmd_log_filter_t *fv, uint_t fc,
1032     int opt_v, int opt_V, int opt_p, int opt_j)
1033 {
1034 	struct fmdump_pipeline *pipeline, *pl;
1035 	struct fmdump_srlzer srlzer;
1036 	uint32_t npipe;
1037 	int fmt;
1038 	int i;
1039 
1040 	if (ifiles != NULL) {
1041 		npipe = n_ifiles;
1042 		pipeline = calloc(npipe, sizeof (struct fmdump_pipeline));
1043 		if (!pipeline)
1044 			fmdump_fatal("failed to allocate memory");
1045 
1046 		for (i = 0; i < n_ifiles; i++)
1047 			pipeline[i].pl_logpath = ifiles[i];
1048 	} else {
1049 		pipeline = calloc(sizeof (logtypes) / sizeof (logtypes[0]),
1050 		    sizeof (struct fmdump_pipeline));
1051 		if (!pipeline)
1052 			fmdump_fatal("failed to allocate memory");
1053 
1054 		do_disables();
1055 
1056 		npipe = 0;
1057 		for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) {
1058 			struct fmdump_logtype *ltp = &logtypes[i];
1059 			char *logpath;
1060 
1061 			if (ltp->lt_enabled == B_FALSE)
1062 				continue;
1063 
1064 			if ((logpath = malloc(PATH_MAX)) == NULL)
1065 				fmdump_fatal("failed to allocate memory");
1066 
1067 			(void) snprintf(logpath, PATH_MAX,
1068 			    "%s/var/fm/fmd/%s",
1069 			    g_root ? g_root : "", ltp->lt_logname);
1070 
1071 			pipeline[npipe].pl_rotated =
1072 			    get_rotated_logs(logpath);
1073 
1074 			pipeline[npipe++].pl_logpath = logpath;
1075 		}
1076 	}
1077 
1078 	if (opt_V)
1079 		fmt = opt_p ? FMDUMP_PRETTY : opt_j ? FMDUMP_JSON :
1080 		    FMDUMP_VERB2;
1081 	else if (opt_v)
1082 		fmt = FMDUMP_VERB1;
1083 	else
1084 		fmt = FMDUMP_SHORT;
1085 
1086 	bzero(&srlzer, sizeof (srlzer));
1087 	srlzer.ds_pipearr = pipeline;
1088 	srlzer.ds_pipecnt = npipe;
1089 	srlzer.ds_slot = calloc(npipe, sizeof (struct fmdump_srlzer_slot));
1090 	if (!srlzer.ds_slot)
1091 		fmdump_fatal("failed to allocate memory");
1092 	(void) pthread_mutex_init(&srlzer.ds_lock, NULL);
1093 
1094 	for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) {
1095 		(void) pthread_mutex_init(&pl->pl_lock, NULL);
1096 		(void) pthread_cond_init(&pl->pl_cv, NULL);
1097 		srlzer.ds_slot[i].ss_state = FMDUMP_PIPE_PROCESSING;
1098 		pl->pl_srlzer = &srlzer;
1099 		pl->pl_srlzeridx = i;
1100 		pl->pl_follow = opt_f ? B_TRUE : B_FALSE;
1101 		pl->pl_fmt = fmt;
1102 		pl->pl_arg.da_fv = fv;
1103 		pl->pl_arg.da_fc = fc;
1104 		pl->pl_arg.da_fp = stdout;
1105 
1106 		(void) pthread_mutex_lock(&pl->pl_lock);
1107 
1108 		if (pthread_create(&pl->pl_thr, NULL,
1109 		    pipeline_thr, (void *)pl) != 0)
1110 			fmdump_fatal("pthread_create for pipeline %d failed",
1111 			    i);
1112 	}
1113 
1114 	for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) {
1115 		while (!pl->pl_started)
1116 			(void) pthread_cond_wait(&pl->pl_cv, &pl->pl_lock);
1117 
1118 		(void) pthread_mutex_unlock(&pl->pl_lock);
1119 	}
1120 
1121 	for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++)
1122 		(void) pthread_join(pl->pl_thr, NULL);
1123 
1124 	if (ifiles == NULL) {
1125 		for (i = 0; i < npipe; i++)
1126 			free(pipeline[i].pl_logpath);
1127 	}
1128 
1129 	free(srlzer.ds_slot);
1130 
1131 	free(pipeline);
1132 
1133 	return (FMDUMP_EXIT_SUCCESS);
1134 }
1135 
1136 static void
cleanup(char ** ifiles,int n_ifiles)1137 cleanup(char **ifiles, int n_ifiles)
1138 {
1139 	int i;
1140 
1141 	if (ifiles == NULL)
1142 		return;
1143 
1144 	for (i = 0; i < n_ifiles; i++) {
1145 		if (ifiles[i] != NULL) {
1146 			free(ifiles[i]);
1147 			ifiles[i] = NULL;
1148 		}
1149 	}
1150 
1151 	free(ifiles);
1152 }
1153 
1154 int
main(int argc,char * argv[])1155 main(int argc, char *argv[])
1156 {
1157 	int opt_a = 0, opt_e = 0, opt_f = 0, opt_H = 0, opt_m = 0, opt_p = 0;
1158 	int opt_u = 0, opt_v = 0, opt_V = 0, opt_j = 0;
1159 	int opt_i = 0, opt_I = 0;
1160 	int opt_A = 0;
1161 	char **ifiles = NULL;
1162 	char *ifile = NULL;
1163 	int n_ifiles;
1164 	int ifileidx = 0;
1165 	int iflags = 0;
1166 
1167 	fmdump_arg_t arg;
1168 	fmdump_lyr_t lyr;
1169 	const fmdump_ops_t *ops;
1170 	fmd_log_filter_t *filtv;
1171 	uint_t filtc;
1172 
1173 	fmd_log_filter_t *errfv, *fltfv, *allfv;
1174 	uint_t errfc = 0, fltfc = 0, allfc = 0;
1175 
1176 	fmd_log_header_t log;
1177 	fmd_log_rec_f *func;
1178 	void *farg;
1179 	fmd_log_t *lp;
1180 	int c, err;
1181 	off64_t off = 0;
1182 	ulong_t recs;
1183 	struct loglink *rotated_logs = NULL, *llp;
1184 
1185 	g_pname = argv[0];
1186 
1187 	errfv = alloca(sizeof (fmd_log_filter_t) * argc);
1188 	fltfv = alloca(sizeof (fmd_log_filter_t) * argc);
1189 	allfv = alloca(sizeof (fmd_log_filter_t) * argc);
1190 
1191 	while (optind < argc) {
1192 		while ((c =
1193 		    getopt(argc, argv, "Aac:efHiIjmn:O:pR:t:T:u:vV")) != EOF) {
1194 			switch (c) {
1195 			case 'A':
1196 				opt_A++;
1197 				break;
1198 			case 'a':
1199 				opt_a++;
1200 				break;
1201 			case 'c':
1202 				errfv[errfc].filt_func = fmd_log_filter_class;
1203 				errfv[errfc].filt_arg = optarg;
1204 				allfv[allfc++] = errfv[errfc++];
1205 				break;
1206 			case 'e':
1207 				if (opt_i)
1208 					return (usage(stderr));
1209 				opt_e++;
1210 				break;
1211 			case 'f':
1212 				opt_f++;
1213 				break;
1214 			case 'H':
1215 				opt_H++;
1216 				break;
1217 			case 'i':
1218 				if (opt_e || opt_I)
1219 					return (usage(stderr));
1220 				opt_i++;
1221 				break;
1222 			case 'I':
1223 				if (opt_e || opt_i)
1224 					return (usage(stderr));
1225 				opt_I++;
1226 				break;
1227 			case 'j':
1228 				if (opt_p)
1229 					return (usage(stderr));
1230 				opt_j++;
1231 				break;
1232 			case 'm':
1233 				opt_m++;
1234 				break;
1235 			case 'O':
1236 				off = strtoull(optarg, NULL, 16);
1237 				iflags |= FMD_LOG_XITER_OFFS;
1238 				break;
1239 			case 'p':
1240 				if (opt_j)
1241 					return (usage(stderr));
1242 				opt_p++;
1243 				break;
1244 			case 'R':
1245 				g_root = optarg;
1246 				break;
1247 			case 't':
1248 				errfv[errfc].filt_func = fmd_log_filter_after;
1249 				errfv[errfc].filt_arg = gettimeopt(optarg);
1250 				allfv[allfc++] = errfv[errfc++];
1251 				break;
1252 			case 'T':
1253 				errfv[errfc].filt_func = fmd_log_filter_before;
1254 				errfv[errfc].filt_arg = gettimeopt(optarg);
1255 				allfv[allfc++] = errfv[errfc++];
1256 				break;
1257 			case 'u':
1258 				fltfv[fltfc].filt_func = fmd_log_filter_uuid;
1259 				fltfv[fltfc].filt_arg = optarg;
1260 				allfv[allfc++] = fltfv[fltfc++];
1261 				opt_u++;
1262 				opt_a++; /* -u implies -a */
1263 				break;
1264 			case 'n': {
1265 				fltfv[fltfc].filt_func = fmd_log_filter_nv;
1266 				fltfv[fltfc].filt_arg = setupnamevalue(optarg);
1267 				allfv[allfc++] = fltfv[fltfc++];
1268 				break;
1269 			}
1270 			case 'v':
1271 				opt_v++;
1272 				break;
1273 			case 'V':
1274 				opt_V++;
1275 				break;
1276 			default:
1277 				return (usage(stderr));
1278 			}
1279 		}
1280 
1281 		if (opt_A && (opt_e || opt_i || opt_I || opt_m || opt_u))
1282 			fmdump_usage("-A excludes all of "
1283 			    "-e, -i, -I, -m and -u\n");
1284 
1285 		if (optind < argc) {
1286 			char *dest;
1287 
1288 			if (ifiles == NULL) {
1289 				n_ifiles = argc - optind;
1290 				ifiles = calloc(n_ifiles, sizeof (char *));
1291 				if (ifiles == NULL) {
1292 					fmdump_fatal(
1293 					    "failed to allocate memory for "
1294 					    "%d input file%s", n_ifiles,
1295 					    n_ifiles > 1 ? "s" : "");
1296 				}
1297 			}
1298 
1299 			if (ifileidx > 0 && !opt_A)
1300 				fmdump_usage("illegal argument -- %s\n",
1301 				    argv[optind]);
1302 
1303 			if ((dest = malloc(PATH_MAX)) == NULL)
1304 				fmdump_fatal("failed to allocate memory");
1305 
1306 			(void) strlcpy(dest, argv[optind++], PATH_MAX);
1307 			ifiles[ifileidx++] = dest;
1308 		}
1309 	}
1310 
1311 	if (opt_A) {
1312 		int rc;
1313 
1314 		if (!opt_a) {
1315 			fltfv[fltfc].filt_func = log_filter_silent;
1316 			fltfv[fltfc].filt_arg = (void *)1;
1317 			allfv[allfc++] = fltfv[fltfc++];
1318 		}
1319 
1320 		rc = aggregate(ifiles, n_ifiles, opt_f,
1321 		    allfv, allfc,
1322 		    opt_v, opt_V, opt_p, opt_j);
1323 
1324 		cleanup(ifiles, n_ifiles);
1325 		return (rc);
1326 	} else {
1327 		if (ifiles == NULL) {
1328 			if ((ifile = calloc(1, PATH_MAX)) == NULL)
1329 				fmdump_fatal("failed to allocate memory");
1330 		} else {
1331 			ifile = ifiles[0];
1332 		}
1333 	}
1334 
1335 
1336 	if (*ifile == '\0') {
1337 		const char *pfx, *sfx;
1338 
1339 		if (opt_u || (!opt_e && !opt_i && !opt_I)) {
1340 			pfx = "flt";
1341 			sfx = "";
1342 		} else {
1343 			if (opt_e) {
1344 				pfx = "err";
1345 				sfx = "";
1346 			} else {
1347 				pfx = "info";
1348 				sfx = opt_I ? "_hival" : "";
1349 			}
1350 		}
1351 
1352 		(void) snprintf(ifile, PATH_MAX, "%s/var/fm/fmd/%slog%s",
1353 		    g_root ? g_root : "", pfx, sfx);
1354 		/*
1355 		 * logadm may rotate the logs.  When no input file is specified,
1356 		 * we try to dump all the rotated logs as well in the right
1357 		 * order.
1358 		 */
1359 		if (!opt_H && off == 0)
1360 			rotated_logs = get_rotated_logs(ifile);
1361 	} else if (g_root != NULL) {
1362 		fmdump_usage("-R option is not appropriate "
1363 		    "when file operand is present\n");
1364 	}
1365 
1366 	if ((g_msg = fmd_msg_init(g_root, FMD_MSG_VERSION)) == NULL)
1367 		fmdump_fatal("failed to initialize libfmd_msg");
1368 
1369 	if ((lp = fmd_log_open(FMD_LOG_VERSION, ifile, &err)) == NULL) {
1370 		fmdump_fatal("failed to open %s: %s\n", ifile,
1371 		    fmd_log_errmsg(NULL, err));
1372 	}
1373 
1374 	if (opt_H) {
1375 		fmd_log_header(lp, &log);
1376 
1377 		(void) printf("EXD_CREATOR = %s\n", log.log_creator);
1378 		(void) printf("EXD_HOSTNAME = %s\n", log.log_hostname);
1379 		(void) printf("EXD_FMA_LABEL = %s\n", log.log_label);
1380 		(void) printf("EXD_FMA_VERSION = %s\n", log.log_version);
1381 		(void) printf("EXD_FMA_OSREL = %s\n", log.log_osrelease);
1382 		(void) printf("EXD_FMA_OSVER = %s\n", log.log_osversion);
1383 		(void) printf("EXD_FMA_PLAT = %s\n", log.log_platform);
1384 		(void) printf("EXD_FMA_UUID = %s\n", log.log_uuid);
1385 
1386 		return (FMDUMP_EXIT_SUCCESS);
1387 	}
1388 
1389 	if (off != 0 && fmd_log_seek(lp, off) != 0) {
1390 		fmdump_fatal("failed to seek %s: %s\n", ifile,
1391 		    fmd_log_errmsg(lp, fmd_log_errno(lp)));
1392 	}
1393 
1394 	if (opt_e && opt_u)
1395 		ops = &fmdump_err_ops;
1396 	else if (strcmp(fmd_log_label(lp), fmdump_flt_ops.do_label) == 0)
1397 		ops = &fmdump_flt_ops;
1398 	else if (strcmp(fmd_log_label(lp), fmdump_asru_ops.do_label) == 0)
1399 		ops = &fmdump_asru_ops;
1400 	else if (strcmp(fmd_log_label(lp), fmdump_info_ops.do_label) == 0)
1401 		ops = &fmdump_info_ops;
1402 	else
1403 		ops = &fmdump_err_ops;
1404 
1405 	if (!opt_a && ops == &fmdump_flt_ops) {
1406 		fltfv[fltfc].filt_func = log_filter_silent;
1407 		fltfv[fltfc].filt_arg = NULL;
1408 		allfv[allfc++] = fltfv[fltfc++];
1409 	}
1410 
1411 	if (opt_V) {
1412 		arg.da_fmt =
1413 		    &ops->do_formats[opt_p ? FMDUMP_PRETTY :
1414 		    opt_j ? FMDUMP_JSON : FMDUMP_VERB2];
1415 		iflags |= FMD_LOG_XITER_REFS;
1416 	} else if (opt_v) {
1417 		arg.da_fmt = &ops->do_formats[FMDUMP_VERB1];
1418 	} else if (opt_m) {
1419 		arg.da_fmt = &ops->do_formats[FMDUMP_MSG];
1420 	} else
1421 		arg.da_fmt = &ops->do_formats[FMDUMP_SHORT];
1422 
1423 	if (opt_m && arg.da_fmt->do_func == NULL) {
1424 		fmdump_usage("-m mode is not supported for "
1425 		    "log of type %s: %s\n", fmd_log_label(lp), ifile);
1426 	}
1427 
1428 	arg.da_fv = errfv;
1429 	arg.da_fc = errfc;
1430 	arg.da_fp = stdout;
1431 
1432 	if (iflags & FMD_LOG_XITER_OFFS)
1433 		fmdump_printf(arg.da_fp, "%16s ", "OFFSET");
1434 
1435 	if (arg.da_fmt->do_hdr && !(opt_V && ops == &fmdump_flt_ops))
1436 		fmdump_printf(arg.da_fp, "%s\n", arg.da_fmt->do_hdr);
1437 
1438 	if (opt_e && opt_u) {
1439 		iflags |= FMD_LOG_XITER_REFS;
1440 		func = xref_iter;
1441 		farg = &arg;
1442 		filtc = fltfc;
1443 		filtv = fltfv;
1444 	} else {
1445 		func = arg.da_fmt->do_func;
1446 		farg = arg.da_fp;
1447 		filtc = allfc;
1448 		filtv = allfv;
1449 	}
1450 
1451 	if (iflags & FMD_LOG_XITER_OFFS) {
1452 		lyr.dy_func = func;
1453 		lyr.dy_arg = farg;
1454 		lyr.dy_fp = arg.da_fp;
1455 		func = xoff_iter;
1456 		farg = &lyr;
1457 	}
1458 
1459 	for (llp = rotated_logs; llp != NULL; llp = llp->next) {
1460 		fmd_log_t *rlp;
1461 
1462 		if ((rlp = fmd_log_open(FMD_LOG_VERSION, llp->path, &err))
1463 		    == NULL) {
1464 			fmdump_warn("failed to open %s: %s\n",
1465 			    llp->path, fmd_log_errmsg(NULL, err));
1466 			g_errs++;
1467 			continue;
1468 		}
1469 
1470 		recs = 0;
1471 		if (fmd_log_xiter(rlp, iflags, filtc, filtv,
1472 		    func, error, farg, &recs) != 0) {
1473 			fmdump_warn("failed to dump %s: %s\n", llp->path,
1474 			    fmd_log_errmsg(rlp, fmd_log_errno(rlp)));
1475 			g_errs++;
1476 		}
1477 		g_recs += recs;
1478 
1479 		fmd_log_close(rlp);
1480 	}
1481 
1482 	do {
1483 		recs = 0;
1484 		if (fmd_log_xiter(lp, iflags, filtc, filtv,
1485 		    func, error, farg, &recs) != 0) {
1486 			fmdump_warn("failed to dump %s: %s\n", ifile,
1487 			    fmd_log_errmsg(lp, fmd_log_errno(lp)));
1488 			g_errs++;
1489 		}
1490 		g_recs += recs;
1491 
1492 		if (opt_f)
1493 			(void) sleep(1);
1494 
1495 	} while (opt_f);
1496 
1497 	if (!opt_f && g_recs == 0 && isatty(STDOUT_FILENO))
1498 		fmdump_warn("%s is empty\n", ifile);
1499 
1500 	if (g_thp != NULL)
1501 		topo_close(g_thp);
1502 
1503 	fmd_log_close(lp);
1504 	fmd_msg_fini(g_msg);
1505 
1506 	if (ifiles == NULL)
1507 		free(ifile);
1508 	else
1509 		cleanup(ifiles, n_ifiles);
1510 
1511 	return (g_errs ? FMDUMP_EXIT_ERROR : FMDUMP_EXIT_SUCCESS);
1512 }
1513