xref: /illumos-gate/usr/src/cmd/stat/kstat/kstat.c (revision c160bf3613805cfb4a89a0433ae896d3594f551f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2013 David Hoeppner. All rights reserved.
25  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright 2016 Joyent, Inc.
27  */
28 
29 /*
30  * Display kernel statistics
31  *
32  * This is a reimplementation of the perl kstat command originally found
33  * under usr/src/cmd/kstat/kstat.pl
34  *
35  * Incompatibilities:
36  *	- perl regular expressions replaced with extended REs bracketed by '/'
37  *
38  * Flags added:
39  *	-C	similar to the -p option but value is separated by a colon
40  *	-h	display help
41  *	-j	json format
42  */
43 
44 #include <assert.h>
45 #include <ctype.h>
46 #include <errno.h>
47 #include <kstat.h>
48 #include <langinfo.h>
49 #include <libgen.h>
50 #include <limits.h>
51 #include <locale.h>
52 #include <signal.h>
53 #include <stddef.h>
54 #include <stdio.h>
55 #include <stdlib.h>
56 #include <string.h>
57 #include <strings.h>
58 #include <time.h>
59 #include <unistd.h>
60 #include <sys/list.h>
61 #include <sys/time.h>
62 #include <sys/types.h>
63 
64 #include "kstat.h"
65 #include "statcommon.h"
66 
67 char	*cmdname = "kstat";	/* Name of this command */
68 int	caught_cont = 0;	/* Have caught a SIGCONT */
69 
70 static uint_t	g_timestamp_fmt = NODATE;
71 
72 /* Helper flag - header was printed already? */
73 static boolean_t g_headerflg;
74 
75 /* Saved command line options */
76 static boolean_t g_cflg = B_FALSE;
77 static boolean_t g_jflg = B_FALSE;
78 static boolean_t g_lflg = B_FALSE;
79 static boolean_t g_pflg = B_FALSE;
80 static boolean_t g_qflg = B_FALSE;
81 static ks_pattern_t	g_ks_class = {"*", 0};
82 
83 static boolean_t g_matched = B_FALSE;
84 
85 /* Sorted list of kstat instances */
86 static list_t	instances_list;
87 static list_t	selector_list;
88 
89 int
90 main(int argc, char **argv)
91 {
92 	ks_selector_t	*nselector;
93 	ks_selector_t	*uselector;
94 	kstat_ctl_t	*kc;
95 	hrtime_t	start_n;
96 	hrtime_t	period_n;
97 	boolean_t	errflg = B_FALSE;
98 	boolean_t	nselflg = B_FALSE;
99 	boolean_t	uselflg = B_FALSE;
100 	char		*q;
101 	int		count = 1;
102 	int		infinite_cycles = 0;
103 	int		interval = 0;
104 	int		n = 0;
105 	int		c, m, tmp;
106 
107 	(void) setlocale(LC_ALL, "");
108 #if !defined(TEXT_DOMAIN)		/* Should be defined by cc -D */
109 #define	TEXT_DOMAIN "SYS_TEST"		/* Use this only if it wasn't */
110 #endif
111 	(void) textdomain(TEXT_DOMAIN);
112 
113 	/*
114 	 * Create the selector list and a dummy default selector to match
115 	 * everything. While we process the cmdline options we will add
116 	 * selectors to this list.
117 	 */
118 	list_create(&selector_list, sizeof (ks_selector_t),
119 	    offsetof(ks_selector_t, ks_next));
120 
121 	nselector = new_selector();
122 
123 	/*
124 	 * Parse named command line arguments.
125 	 */
126 	while ((c = getopt(argc, argv, "h?CqjlpT:m:i:n:s:c:")) != EOF)
127 		switch (c) {
128 		case 'h':
129 		case '?':
130 			usage();
131 			exit(0);
132 			break;
133 		case 'C':
134 			g_pflg = g_cflg = B_TRUE;
135 			break;
136 		case 'q':
137 			g_qflg = B_TRUE;
138 			break;
139 		case 'j':
140 			/*
141 			 * If we're printing JSON, we're going to force numeric
142 			 * representation to be in the C locale to assure that
143 			 * the decimal point is compliant with RFC 7159 (i.e.,
144 			 * ASCII 0x2e).
145 			 */
146 			(void) setlocale(LC_NUMERIC, "C");
147 			g_jflg = B_TRUE;
148 			break;
149 		case 'l':
150 			g_pflg = g_lflg = B_TRUE;
151 			break;
152 		case 'p':
153 			g_pflg = B_TRUE;
154 			break;
155 		case 'T':
156 			switch (*optarg) {
157 			case 'd':
158 				g_timestamp_fmt = DDATE;
159 				break;
160 			case 'u':
161 				g_timestamp_fmt = UDATE;
162 				break;
163 			default:
164 				errflg = B_TRUE;
165 			}
166 			break;
167 		case 'm':
168 			nselflg = B_TRUE;
169 			nselector->ks_module.pstr =
170 			    (char *)ks_safe_strdup(optarg);
171 			break;
172 		case 'i':
173 			nselflg = B_TRUE;
174 			nselector->ks_instance.pstr =
175 			    (char *)ks_safe_strdup(optarg);
176 			break;
177 		case 'n':
178 			nselflg = B_TRUE;
179 			nselector->ks_name.pstr =
180 			    (char *)ks_safe_strdup(optarg);
181 			break;
182 		case 's':
183 			nselflg = B_TRUE;
184 			nselector->ks_statistic.pstr =
185 			    (char *)ks_safe_strdup(optarg);
186 			break;
187 		case 'c':
188 			g_ks_class.pstr =
189 			    (char *)ks_safe_strdup(optarg);
190 			break;
191 		default:
192 			errflg = B_TRUE;
193 			break;
194 		}
195 
196 	if (g_qflg && (g_jflg || g_pflg)) {
197 		(void) fprintf(stderr, gettext(
198 		    "-q and -lpj are mutually exclusive\n"));
199 		errflg = B_TRUE;
200 	}
201 
202 	if (errflg) {
203 		usage();
204 		exit(2);
205 	}
206 
207 	argc -= optind;
208 	argv += optind;
209 
210 	/*
211 	 * Consume the rest of the command line. Parsing the
212 	 * unnamed command line arguments.
213 	 */
214 	while (argc--) {
215 		errno = 0;
216 		tmp = strtoul(*argv, &q, 10);
217 		if (tmp == ULONG_MAX && errno == ERANGE) {
218 			if (n == 0) {
219 				(void) fprintf(stderr, gettext(
220 				    "Interval is too large\n"));
221 			} else if (n == 1) {
222 				(void) fprintf(stderr, gettext(
223 				    "Count is too large\n"));
224 			}
225 			usage();
226 			exit(2);
227 		}
228 
229 		if (errno != 0 || *q != '\0') {
230 			m = 0;
231 			uselector = new_selector();
232 			while ((q = (char *)strsep(argv, ":")) != NULL) {
233 				m++;
234 				if (m > 4) {
235 					free(uselector);
236 					usage();
237 					exit(2);
238 				}
239 
240 				if (*q != '\0') {
241 					switch (m) {
242 					case 1:
243 						uselector->ks_module.pstr =
244 						    (char *)ks_safe_strdup(q);
245 						break;
246 					case 2:
247 						uselector->ks_instance.pstr =
248 						    (char *)ks_safe_strdup(q);
249 						break;
250 					case 3:
251 						uselector->ks_name.pstr =
252 						    (char *)ks_safe_strdup(q);
253 						break;
254 					case 4:
255 						uselector->ks_statistic.pstr =
256 						    (char *)ks_safe_strdup(q);
257 						break;
258 					default:
259 						assert(B_FALSE);
260 					}
261 				}
262 			}
263 
264 			uselflg = B_TRUE;
265 			list_insert_tail(&selector_list, uselector);
266 		} else {
267 			if (tmp < 1) {
268 				if (n == 0) {
269 					(void) fprintf(stderr, gettext(
270 					    "Interval must be an "
271 					    "integer >= 1"));
272 				} else if (n == 1) {
273 					(void) fprintf(stderr, gettext(
274 					    "Count must be an integer >= 1"));
275 				}
276 				usage();
277 				exit(2);
278 			} else {
279 				if (n == 0) {
280 					interval = tmp;
281 					count = -1;
282 				} else if (n == 1) {
283 					count = tmp;
284 				} else {
285 					usage();
286 					exit(2);
287 				}
288 			}
289 			n++;
290 		}
291 		argv++;
292 	}
293 
294 	/*
295 	 * Check if we founded a named selector on the cmdline.
296 	 */
297 	if (uselflg) {
298 		if (nselflg) {
299 			(void) fprintf(stderr, gettext(
300 			    "[module[:instance[:name[:statistic]]]] and "
301 			    "-m -i -n -s are mutually exclusive"));
302 			usage();
303 			exit(2);
304 		} else {
305 			free(nselector);
306 		}
307 	} else {
308 		list_insert_tail(&selector_list, nselector);
309 	}
310 
311 	assert(!list_is_empty(&selector_list));
312 
313 	list_create(&instances_list, sizeof (ks_instance_t),
314 	    offsetof(ks_instance_t, ks_next));
315 
316 	while ((kc = kstat_open()) == NULL) {
317 		if (errno == EAGAIN) {
318 			(void) poll(NULL, 0, 200);
319 		} else {
320 			perror("kstat_open");
321 			exit(3);
322 		}
323 	}
324 
325 	if (count > 1) {
326 		if (signal(SIGCONT, cont_handler) == SIG_ERR) {
327 			(void) fprintf(stderr, gettext(
328 			    "signal failed"));
329 			exit(3);
330 		}
331 	}
332 
333 	period_n = (hrtime_t)interval * NANOSEC;
334 	start_n = gethrtime();
335 
336 	while (count == -1 || count-- > 0) {
337 		ks_instances_read(kc);
338 		ks_instances_print();
339 
340 		if (interval && count) {
341 			ks_sleep_until(&start_n, period_n, infinite_cycles,
342 			    &caught_cont);
343 			(void) kstat_chain_update(kc);
344 			(void) putchar('\n');
345 		}
346 	}
347 
348 	(void) kstat_close(kc);
349 
350 	/*
351 	 * Return a non-zero exit code if we didn't match anything.
352 	 */
353 	return (g_matched ? 0 : 1);
354 }
355 
356 /*
357  * Print usage.
358  */
359 static void
360 usage(void)
361 {
362 	(void) fprintf(stderr, gettext(
363 	    "Usage:\n"
364 	    "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ]\n"
365 	    "      [ -m module ] [ -i instance ] [ -n name ] [ -s statistic ]\n"
366 	    "      [ interval [ count ] ]\n"
367 	    "kstat [ -Cjlpq ] [ -T d|u ] [ -c class ]\n"
368 	    "      [ module[:instance[:name[:statistic]]] ... ]\n"
369 	    "      [ interval [ count ] ]\n"));
370 }
371 
372 /*
373  * Sort compare function.
374  */
375 static int
376 compare_instances(ks_instance_t *l_arg, ks_instance_t *r_arg)
377 {
378 	int	rval;
379 
380 	rval = strcasecmp(l_arg->ks_module, r_arg->ks_module);
381 	if (rval == 0) {
382 		if (l_arg->ks_instance == r_arg->ks_instance) {
383 			return (strcasecmp(l_arg->ks_name, r_arg->ks_name));
384 		} else if (l_arg->ks_instance < r_arg->ks_instance) {
385 			return (-1);
386 		} else {
387 			return (1);
388 		}
389 	} else {
390 		return (rval);
391 	}
392 }
393 
394 static char *
395 ks_safe_strdup(char *str)
396 {
397 	char	*ret;
398 
399 	if (str == NULL) {
400 		return (NULL);
401 	}
402 
403 	while ((ret = strdup(str)) == NULL) {
404 		if (errno == EAGAIN) {
405 			(void) poll(NULL, 0, 200);
406 		} else {
407 			perror("strdup");
408 			exit(3);
409 		}
410 	}
411 
412 	return (ret);
413 }
414 
415 static void
416 ks_sleep_until(hrtime_t *wakeup, hrtime_t interval, int forever,
417     int *caught_cont)
418 {
419 	hrtime_t	now, pause, pause_left;
420 	struct timespec	pause_tv;
421 	int		status;
422 
423 	now = gethrtime();
424 	pause = *wakeup + interval - now;
425 
426 	if (pause <= 0 || pause < (interval / 4)) {
427 		if (forever || *caught_cont) {
428 			*wakeup = now + interval;
429 			pause = interval;
430 		} else {
431 			pause = interval / 2;
432 			*wakeup += interval;
433 		}
434 	} else {
435 		*wakeup += interval;
436 	}
437 
438 	if (pause < 1000) {
439 		return;
440 	}
441 
442 	pause_left = pause;
443 	do {
444 		pause_tv.tv_sec = pause_left / NANOSEC;
445 		pause_tv.tv_nsec = pause_left % NANOSEC;
446 		status = nanosleep(&pause_tv, (struct timespec *)NULL);
447 		if (status < 0) {
448 			if (errno == EINTR) {
449 				now = gethrtime();
450 				pause_left = *wakeup - now;
451 				if (pause_left < 1000) {
452 					return;
453 				}
454 			} else {
455 				perror("nanosleep");
456 				exit(3);
457 			}
458 		}
459 	} while (status != 0);
460 }
461 
462 /*
463  * Inserts an instance in the per selector list.
464  */
465 static void
466 nvpair_insert(ks_instance_t *ksi, char *name, ks_value_t *value,
467     uchar_t data_type)
468 {
469 	ks_nvpair_t	*instance;
470 	ks_nvpair_t	*tmp;
471 
472 	instance = (ks_nvpair_t *)malloc(sizeof (ks_nvpair_t));
473 	if (instance == NULL) {
474 		perror("malloc");
475 		exit(3);
476 	}
477 
478 	(void) strlcpy(instance->name, name, KSTAT_STRLEN);
479 	(void) memcpy(&instance->value, value, sizeof (ks_value_t));
480 	instance->data_type = data_type;
481 
482 	tmp = list_head(&ksi->ks_nvlist);
483 	while (tmp != NULL && strcasecmp(instance->name, tmp->name) > 0)
484 		tmp = list_next(&ksi->ks_nvlist, tmp);
485 
486 	list_insert_before(&ksi->ks_nvlist, tmp, instance);
487 }
488 
489 /*
490  * Allocates a new all-matching selector.
491  */
492 static ks_selector_t *
493 new_selector(void)
494 {
495 	ks_selector_t	*selector;
496 
497 	selector = (ks_selector_t *)malloc(sizeof (ks_selector_t));
498 	if (selector == NULL) {
499 		perror("malloc");
500 		exit(3);
501 	}
502 
503 	list_link_init(&selector->ks_next);
504 
505 	selector->ks_module.pstr = "*";
506 	selector->ks_instance.pstr = "*";
507 	selector->ks_name.pstr = "*";
508 	selector->ks_statistic.pstr = "*";
509 
510 	return (selector);
511 }
512 
513 /*
514  * This function was taken from the perl kstat module code - please
515  * see for further comments there.
516  */
517 static kstat_raw_reader_t
518 lookup_raw_kstat_fn(char *module, char *name)
519 {
520 	char		key[KSTAT_STRLEN * 2];
521 	register char 	*f, *t;
522 	int		n = 0;
523 
524 	for (f = module, t = key; *f != '\0'; f++, t++) {
525 		while (*f != '\0' && isdigit(*f))
526 			f++;
527 		*t = *f;
528 	}
529 	*t++ = ':';
530 
531 	for (f = name; *f != '\0'; f++, t++) {
532 		while (*f != '\0' && isdigit(*f))
533 			f++;
534 		*t = *f;
535 	}
536 	*t = '\0';
537 
538 	while (ks_raw_lookup[n].fn != NULL) {
539 		if (strncmp(ks_raw_lookup[n].name, key, strlen(key)) == 0)
540 			return (ks_raw_lookup[n].fn);
541 		n++;
542 	}
543 
544 	return (0);
545 }
546 
547 /*
548  * Match a string against a shell glob or extended regular expression.
549  */
550 static boolean_t
551 ks_match(const char *str, ks_pattern_t *pattern)
552 {
553 	int	regcode;
554 	char	*regstr;
555 	char	*errbuf;
556 	size_t	bufsz;
557 
558 	if (pattern->pstr != NULL && gmatch(pattern->pstr, "/*/") != 0) {
559 		/* All regex patterns are strdup'd copies */
560 		regstr = pattern->pstr + 1;
561 		*(strrchr(regstr, '/')) = '\0';
562 
563 		regcode = regcomp(&pattern->preg, regstr,
564 		    REG_EXTENDED | REG_NOSUB);
565 		if (regcode != 0) {
566 			bufsz = regerror(regcode, NULL, NULL, 0);
567 			if (bufsz != 0) {
568 				errbuf = malloc(bufsz);
569 				if (errbuf == NULL) {
570 					perror("malloc");
571 					exit(3);
572 				}
573 				(void) regerror(regcode, NULL, errbuf, bufsz);
574 				(void) fprintf(stderr, "kstat: %s\n", errbuf);
575 			}
576 			usage();
577 			exit(2);
578 		}
579 
580 		pattern->pstr = NULL;
581 	}
582 
583 	if (pattern->pstr == NULL) {
584 		return (regexec(&pattern->preg, str, 0, NULL, 0) == 0);
585 	}
586 
587 	return ((gmatch(str, pattern->pstr) != 0));
588 }
589 
590 /*
591  * Iterate over all kernel statistics and save matches.
592  */
593 static void
594 ks_instances_read(kstat_ctl_t *kc)
595 {
596 	kstat_raw_reader_t save_raw = NULL;
597 	kid_t		id;
598 	ks_selector_t	*selector;
599 	ks_instance_t	*ksi;
600 	ks_instance_t	*tmp;
601 	kstat_t		*kp;
602 	boolean_t	skip;
603 
604 	for (kp = kc->kc_chain; kp != NULL; kp = kp->ks_next) {
605 		/* Don't bother storing the kstat headers */
606 		if (strncmp(kp->ks_name, "kstat_", 6) == 0) {
607 			continue;
608 		}
609 
610 		/* Don't bother storing raw stats we don't understand */
611 		if (kp->ks_type == KSTAT_TYPE_RAW) {
612 			save_raw = lookup_raw_kstat_fn(kp->ks_module,
613 			    kp->ks_name);
614 			if (save_raw == NULL) {
615 #ifdef REPORT_UNKNOWN
616 				(void) fprintf(stderr,
617 				    "Unknown kstat type %s:%d:%s - "
618 				    "%d of size %d\n", kp->ks_module,
619 				    kp->ks_instance, kp->ks_name,
620 				    kp->ks_ndata, kp->ks_data_size);
621 #endif
622 				continue;
623 			}
624 		}
625 
626 		/*
627 		 * Iterate over the list of selectors and skip
628 		 * instances we dont want. We filter for statistics
629 		 * later, as we dont know them yet.
630 		 */
631 		skip = B_TRUE;
632 		selector = list_head(&selector_list);
633 		while (selector != NULL) {
634 			if (ks_match(kp->ks_module, &selector->ks_module) &&
635 			    ks_match(kp->ks_name, &selector->ks_name)) {
636 				skip = B_FALSE;
637 				break;
638 			}
639 			selector = list_next(&selector_list, selector);
640 		}
641 
642 		if (skip) {
643 			continue;
644 		}
645 
646 		/*
647 		 * Allocate a new instance and fill in the values
648 		 * we know so far.
649 		 */
650 		ksi = (ks_instance_t *)malloc(sizeof (ks_instance_t));
651 		if (ksi == NULL) {
652 			perror("malloc");
653 			exit(3);
654 		}
655 
656 		list_link_init(&ksi->ks_next);
657 
658 		(void) strlcpy(ksi->ks_module, kp->ks_module, KSTAT_STRLEN);
659 		(void) strlcpy(ksi->ks_name, kp->ks_name, KSTAT_STRLEN);
660 		(void) strlcpy(ksi->ks_class, kp->ks_class, KSTAT_STRLEN);
661 
662 		ksi->ks_instance = kp->ks_instance;
663 		ksi->ks_snaptime = kp->ks_snaptime;
664 		ksi->ks_type = kp->ks_type;
665 
666 		list_create(&ksi->ks_nvlist, sizeof (ks_nvpair_t),
667 		    offsetof(ks_nvpair_t, nv_next));
668 
669 		SAVE_HRTIME_X(ksi, "crtime", kp->ks_crtime);
670 		if (g_pflg) {
671 			SAVE_STRING_X(ksi, "class", kp->ks_class);
672 		}
673 
674 		/* Insert this instance into a sorted list */
675 		tmp = list_head(&instances_list);
676 		while (tmp != NULL && compare_instances(ksi, tmp) > 0)
677 			tmp = list_next(&instances_list, tmp);
678 
679 		list_insert_before(&instances_list, tmp, ksi);
680 
681 		/* Read the actual statistics */
682 		id = kstat_read(kc, kp, NULL);
683 		if (id == -1) {
684 #ifdef REPORT_UNKNOWN
685 			perror("kstat_read");
686 #endif
687 			continue;
688 		}
689 
690 		SAVE_HRTIME_X(ksi, "snaptime", kp->ks_snaptime);
691 
692 		switch (kp->ks_type) {
693 		case KSTAT_TYPE_RAW:
694 			save_raw(kp, ksi);
695 			break;
696 		case KSTAT_TYPE_NAMED:
697 			save_named(kp, ksi);
698 			break;
699 		case KSTAT_TYPE_INTR:
700 			save_intr(kp, ksi);
701 			break;
702 		case KSTAT_TYPE_IO:
703 			save_io(kp, ksi);
704 			break;
705 		case KSTAT_TYPE_TIMER:
706 			save_timer(kp, ksi);
707 			break;
708 		default:
709 			assert(B_FALSE); /* Invalid type */
710 			break;
711 		}
712 	}
713 }
714 
715 /*
716  * Print the value of a name-value pair.
717  */
718 static void
719 ks_value_print(ks_nvpair_t *nvpair)
720 {
721 	switch (nvpair->data_type) {
722 	case KSTAT_DATA_CHAR:
723 		(void) fprintf(stdout, "%s", nvpair->value.c);
724 		break;
725 	case KSTAT_DATA_INT32:
726 		(void) fprintf(stdout, "%d", nvpair->value.i32);
727 		break;
728 	case KSTAT_DATA_UINT32:
729 		(void) fprintf(stdout, "%u", nvpair->value.ui32);
730 		break;
731 	case KSTAT_DATA_INT64:
732 		(void) fprintf(stdout, "%lld", nvpair->value.i64);
733 		break;
734 	case KSTAT_DATA_UINT64:
735 		(void) fprintf(stdout, "%llu", nvpair->value.ui64);
736 		break;
737 	case KSTAT_DATA_STRING:
738 		(void) fprintf(stdout, "%s", KSTAT_NAMED_STR_PTR(nvpair));
739 		break;
740 	case KSTAT_DATA_HRTIME:
741 		if (nvpair->value.ui64 == 0)
742 			(void) fprintf(stdout, "0");
743 		else
744 			(void) fprintf(stdout, "%.9f",
745 			    nvpair->value.ui64 / 1000000000.0);
746 		break;
747 	default:
748 		assert(B_FALSE);
749 	}
750 }
751 
752 /*
753  * Print a single instance.
754  */
755 /*ARGSUSED*/
756 static void
757 ks_instance_print(ks_instance_t *ksi, ks_nvpair_t *nvpair, boolean_t last)
758 {
759 	if (g_headerflg) {
760 		if (!g_pflg) {
761 			(void) fprintf(stdout, DFLT_FMT,
762 			    ksi->ks_module, ksi->ks_instance,
763 			    ksi->ks_name, ksi->ks_class);
764 		}
765 		g_headerflg = B_FALSE;
766 	}
767 
768 	if (g_pflg) {
769 		(void) fprintf(stdout, KS_PFMT,
770 		    ksi->ks_module, ksi->ks_instance,
771 		    ksi->ks_name, nvpair->name);
772 		if (!g_lflg) {
773 			(void) putchar(g_cflg ? ':': '\t');
774 			ks_value_print(nvpair);
775 		}
776 	} else {
777 		(void) fprintf(stdout, KS_DFMT, nvpair->name);
778 		ks_value_print(nvpair);
779 	}
780 
781 	(void) putchar('\n');
782 }
783 
784 /*
785  * Print a C string as a JSON string.
786  */
787 static void
788 ks_print_json_string(const char *str)
789 {
790 	char c;
791 
792 	(void) putchar('"');
793 
794 	while ((c = *str++) != '\0') {
795 		/*
796 		 * For readability, we use the allowed alternate escape
797 		 * sequence for quote, question mark, reverse solidus (look
798 		 * it up!), newline and tab -- and use the universal escape
799 		 * sequence for all other control characters.
800 		 */
801 		switch (c) {
802 		case '"':
803 		case '?':
804 		case '\\':
805 			(void) fprintf(stdout, "\\%c", c);
806 			break;
807 
808 		case '\n':
809 			(void) fprintf(stdout, "\\n");
810 			break;
811 
812 		case '\t':
813 			(void) fprintf(stdout, "\\t");
814 			break;
815 
816 		default:
817 			/*
818 			 * By escaping those characters for which isprint(3C)
819 			 * is false, we escape both the RFC 7159 mandated
820 			 * escaped range of 0x01 through 0x1f as well as DEL
821 			 * (0x7f -- the control character that RFC 7159 forgot)
822 			 * and then everything else that's unprintable for
823 			 * good measure.
824 			 */
825 			if (!isprint(c)) {
826 				(void) fprintf(stdout, "\\u%04hhx", (uint8_t)c);
827 				break;
828 			}
829 
830 			(void) putchar(c);
831 			break;
832 		}
833 	}
834 
835 	(void) putchar('"');
836 }
837 
838 /*
839  * Print a single instance in JSON format.
840  */
841 static void
842 ks_instance_print_json(ks_instance_t *ksi, ks_nvpair_t *nvpair, boolean_t last)
843 {
844 	static int headers;
845 
846 	if (g_headerflg) {
847 		if (headers++ > 0)
848 			(void) fprintf(stdout, ", ");
849 
850 		(void) fprintf(stdout, "{\n\t\"module\": ");
851 		ks_print_json_string(ksi->ks_module);
852 
853 		(void) fprintf(stdout,
854 		    ",\n\t\"instance\": %d,\n\t\"name\": ", ksi->ks_instance);
855 		ks_print_json_string(ksi->ks_name);
856 
857 		(void) fprintf(stdout, ",\n\t\"class\": ");
858 		ks_print_json_string(ksi->ks_class);
859 
860 		(void) fprintf(stdout, ",\n\t\"type\": %d,\n", ksi->ks_type);
861 
862 		if (ksi->ks_snaptime == 0)
863 			(void) fprintf(stdout, "\t\"snaptime\": 0,\n");
864 		else
865 			(void) fprintf(stdout, "\t\"snaptime\": %.9f,\n",
866 			    ksi->ks_snaptime / 1000000000.0);
867 
868 		(void) fprintf(stdout, "\t\"data\": {\n");
869 
870 		g_headerflg = B_FALSE;
871 	}
872 
873 	(void) fprintf(stdout, "\t\t");
874 	ks_print_json_string(nvpair->name);
875 	(void) fprintf(stdout, ": ");
876 
877 	switch (nvpair->data_type) {
878 	case KSTAT_DATA_CHAR:
879 		ks_print_json_string(nvpair->value.c);
880 		break;
881 
882 	case KSTAT_DATA_STRING:
883 		ks_print_json_string(KSTAT_NAMED_STR_PTR(nvpair));
884 		break;
885 
886 	default:
887 		ks_value_print(nvpair);
888 		break;
889 	}
890 
891 	if (!last)
892 		(void) putchar(',');
893 
894 	(void) putchar('\n');
895 }
896 
897 /*
898  * Print all instances.
899  */
900 static void
901 ks_instances_print(void)
902 {
903 	ks_selector_t *selector;
904 	ks_instance_t *ksi, *ktmp;
905 	ks_nvpair_t *nvpair, *ntmp, *next;
906 	void (*ks_print_fn)(ks_instance_t *, ks_nvpair_t *, boolean_t);
907 	char *ks_number;
908 
909 	if (g_timestamp_fmt != NODATE)
910 		print_timestamp(g_timestamp_fmt);
911 
912 	if (g_jflg) {
913 		ks_print_fn = &ks_instance_print_json;
914 		(void) putchar('[');
915 	} else {
916 		ks_print_fn = &ks_instance_print;
917 	}
918 
919 	/* Iterate over each selector */
920 	selector = list_head(&selector_list);
921 	while (selector != NULL) {
922 
923 		/* Iterate over each instance */
924 		for (ksi = list_head(&instances_list); ksi != NULL;
925 		    ksi = list_next(&instances_list, ksi)) {
926 
927 			(void) asprintf(&ks_number, "%d", ksi->ks_instance);
928 			if (!(ks_match(ksi->ks_module, &selector->ks_module) &&
929 			    ks_match(ksi->ks_name, &selector->ks_name) &&
930 			    ks_match(ks_number, &selector->ks_instance) &&
931 			    ks_match(ksi->ks_class, &g_ks_class))) {
932 				free(ks_number);
933 				continue;
934 			}
935 
936 			free(ks_number);
937 
938 			g_headerflg = B_TRUE;
939 
940 			/*
941 			 * Find our first statistic to print.
942 			 */
943 			for (nvpair = list_head(&ksi->ks_nvlist);
944 			    nvpair != NULL;
945 			    nvpair = list_next(&ksi->ks_nvlist, nvpair)) {
946 				if (ks_match(nvpair->name,
947 				    &selector->ks_statistic))
948 					break;
949 			}
950 
951 			while (nvpair != NULL) {
952 				boolean_t last;
953 
954 				/*
955 				 * Find the next statistic to print so we can
956 				 * indicate to the print function if this
957 				 * statistic is the last to be printed for
958 				 * this instance.
959 				 */
960 				for (next = list_next(&ksi->ks_nvlist, nvpair);
961 				    next != NULL;
962 				    next = list_next(&ksi->ks_nvlist, next)) {
963 					if (ks_match(next->name,
964 					    &selector->ks_statistic))
965 						break;
966 				}
967 
968 				g_matched = B_TRUE;
969 				last = next == NULL ? B_TRUE : B_FALSE;
970 
971 				if (!g_qflg)
972 					(*ks_print_fn)(ksi, nvpair, last);
973 
974 				nvpair = next;
975 			}
976 
977 			if (!g_headerflg) {
978 				if (g_jflg) {
979 					(void) fprintf(stdout, "\t}\n}");
980 				} else if (!g_pflg) {
981 					(void) putchar('\n');
982 				}
983 			}
984 		}
985 
986 		selector = list_next(&selector_list, selector);
987 	}
988 
989 	if (g_jflg)
990 		(void) fprintf(stdout, "]\n");
991 
992 	(void) fflush(stdout);
993 
994 	/* Free the instances list */
995 	ksi = list_head(&instances_list);
996 	while (ksi != NULL) {
997 		nvpair = list_head(&ksi->ks_nvlist);
998 		while (nvpair != NULL) {
999 			ntmp = nvpair;
1000 			nvpair = list_next(&ksi->ks_nvlist, nvpair);
1001 			list_remove(&ksi->ks_nvlist, ntmp);
1002 			if (ntmp->data_type == KSTAT_DATA_STRING)
1003 				free(ntmp->value.str.addr.ptr);
1004 			free(ntmp);
1005 		}
1006 
1007 		ktmp = ksi;
1008 		ksi = list_next(&instances_list, ksi);
1009 		list_remove(&instances_list, ktmp);
1010 		list_destroy(&ktmp->ks_nvlist);
1011 		free(ktmp);
1012 	}
1013 }
1014 
1015 static void
1016 save_cpu_stat(kstat_t *kp, ks_instance_t *ksi)
1017 {
1018 	cpu_stat_t	*stat;
1019 	cpu_sysinfo_t	*sysinfo;
1020 	cpu_syswait_t	*syswait;
1021 	cpu_vminfo_t	*vminfo;
1022 
1023 	stat = (cpu_stat_t *)(kp->ks_data);
1024 	sysinfo = &stat->cpu_sysinfo;
1025 	syswait = &stat->cpu_syswait;
1026 	vminfo  = &stat->cpu_vminfo;
1027 
1028 	SAVE_UINT32_X(ksi, "idle", sysinfo->cpu[CPU_IDLE]);
1029 	SAVE_UINT32_X(ksi, "user", sysinfo->cpu[CPU_USER]);
1030 	SAVE_UINT32_X(ksi, "kernel", sysinfo->cpu[CPU_KERNEL]);
1031 	SAVE_UINT32_X(ksi, "wait", sysinfo->cpu[CPU_WAIT]);
1032 	SAVE_UINT32_X(ksi, "wait_io", sysinfo->wait[W_IO]);
1033 	SAVE_UINT32_X(ksi, "wait_swap", sysinfo->wait[W_SWAP]);
1034 	SAVE_UINT32_X(ksi, "wait_pio", sysinfo->wait[W_PIO]);
1035 	SAVE_UINT32(ksi, sysinfo, bread);
1036 	SAVE_UINT32(ksi, sysinfo, bwrite);
1037 	SAVE_UINT32(ksi, sysinfo, lread);
1038 	SAVE_UINT32(ksi, sysinfo, lwrite);
1039 	SAVE_UINT32(ksi, sysinfo, phread);
1040 	SAVE_UINT32(ksi, sysinfo, phwrite);
1041 	SAVE_UINT32(ksi, sysinfo, pswitch);
1042 	SAVE_UINT32(ksi, sysinfo, trap);
1043 	SAVE_UINT32(ksi, sysinfo, intr);
1044 	SAVE_UINT32(ksi, sysinfo, syscall);
1045 	SAVE_UINT32(ksi, sysinfo, sysread);
1046 	SAVE_UINT32(ksi, sysinfo, syswrite);
1047 	SAVE_UINT32(ksi, sysinfo, sysfork);
1048 	SAVE_UINT32(ksi, sysinfo, sysvfork);
1049 	SAVE_UINT32(ksi, sysinfo, sysexec);
1050 	SAVE_UINT32(ksi, sysinfo, readch);
1051 	SAVE_UINT32(ksi, sysinfo, writech);
1052 	SAVE_UINT32(ksi, sysinfo, rcvint);
1053 	SAVE_UINT32(ksi, sysinfo, xmtint);
1054 	SAVE_UINT32(ksi, sysinfo, mdmint);
1055 	SAVE_UINT32(ksi, sysinfo, rawch);
1056 	SAVE_UINT32(ksi, sysinfo, canch);
1057 	SAVE_UINT32(ksi, sysinfo, outch);
1058 	SAVE_UINT32(ksi, sysinfo, msg);
1059 	SAVE_UINT32(ksi, sysinfo, sema);
1060 	SAVE_UINT32(ksi, sysinfo, namei);
1061 	SAVE_UINT32(ksi, sysinfo, ufsiget);
1062 	SAVE_UINT32(ksi, sysinfo, ufsdirblk);
1063 	SAVE_UINT32(ksi, sysinfo, ufsipage);
1064 	SAVE_UINT32(ksi, sysinfo, ufsinopage);
1065 	SAVE_UINT32(ksi, sysinfo, inodeovf);
1066 	SAVE_UINT32(ksi, sysinfo, fileovf);
1067 	SAVE_UINT32(ksi, sysinfo, procovf);
1068 	SAVE_UINT32(ksi, sysinfo, intrthread);
1069 	SAVE_UINT32(ksi, sysinfo, intrblk);
1070 	SAVE_UINT32(ksi, sysinfo, idlethread);
1071 	SAVE_UINT32(ksi, sysinfo, inv_swtch);
1072 	SAVE_UINT32(ksi, sysinfo, nthreads);
1073 	SAVE_UINT32(ksi, sysinfo, cpumigrate);
1074 	SAVE_UINT32(ksi, sysinfo, xcalls);
1075 	SAVE_UINT32(ksi, sysinfo, mutex_adenters);
1076 	SAVE_UINT32(ksi, sysinfo, rw_rdfails);
1077 	SAVE_UINT32(ksi, sysinfo, rw_wrfails);
1078 	SAVE_UINT32(ksi, sysinfo, modload);
1079 	SAVE_UINT32(ksi, sysinfo, modunload);
1080 	SAVE_UINT32(ksi, sysinfo, bawrite);
1081 #ifdef	STATISTICS	/* see header file */
1082 	SAVE_UINT32(ksi, sysinfo, rw_enters);
1083 	SAVE_UINT32(ksi, sysinfo, win_uo_cnt);
1084 	SAVE_UINT32(ksi, sysinfo, win_uu_cnt);
1085 	SAVE_UINT32(ksi, sysinfo, win_so_cnt);
1086 	SAVE_UINT32(ksi, sysinfo, win_su_cnt);
1087 	SAVE_UINT32(ksi, sysinfo, win_suo_cnt);
1088 #endif
1089 
1090 	SAVE_INT32(ksi, syswait, iowait);
1091 	SAVE_INT32(ksi, syswait, swap);
1092 	SAVE_INT32(ksi, syswait, physio);
1093 
1094 	SAVE_UINT32(ksi, vminfo, pgrec);
1095 	SAVE_UINT32(ksi, vminfo, pgfrec);
1096 	SAVE_UINT32(ksi, vminfo, pgin);
1097 	SAVE_UINT32(ksi, vminfo, pgpgin);
1098 	SAVE_UINT32(ksi, vminfo, pgout);
1099 	SAVE_UINT32(ksi, vminfo, pgpgout);
1100 	SAVE_UINT32(ksi, vminfo, swapin);
1101 	SAVE_UINT32(ksi, vminfo, pgswapin);
1102 	SAVE_UINT32(ksi, vminfo, swapout);
1103 	SAVE_UINT32(ksi, vminfo, pgswapout);
1104 	SAVE_UINT32(ksi, vminfo, zfod);
1105 	SAVE_UINT32(ksi, vminfo, dfree);
1106 	SAVE_UINT32(ksi, vminfo, scan);
1107 	SAVE_UINT32(ksi, vminfo, rev);
1108 	SAVE_UINT32(ksi, vminfo, hat_fault);
1109 	SAVE_UINT32(ksi, vminfo, as_fault);
1110 	SAVE_UINT32(ksi, vminfo, maj_fault);
1111 	SAVE_UINT32(ksi, vminfo, cow_fault);
1112 	SAVE_UINT32(ksi, vminfo, prot_fault);
1113 	SAVE_UINT32(ksi, vminfo, softlock);
1114 	SAVE_UINT32(ksi, vminfo, kernel_asflt);
1115 	SAVE_UINT32(ksi, vminfo, pgrrun);
1116 	SAVE_UINT32(ksi, vminfo, execpgin);
1117 	SAVE_UINT32(ksi, vminfo, execpgout);
1118 	SAVE_UINT32(ksi, vminfo, execfree);
1119 	SAVE_UINT32(ksi, vminfo, anonpgin);
1120 	SAVE_UINT32(ksi, vminfo, anonpgout);
1121 	SAVE_UINT32(ksi, vminfo, anonfree);
1122 	SAVE_UINT32(ksi, vminfo, fspgin);
1123 	SAVE_UINT32(ksi, vminfo, fspgout);
1124 	SAVE_UINT32(ksi, vminfo, fsfree);
1125 }
1126 
1127 static void
1128 save_var(kstat_t *kp, ks_instance_t *ksi)
1129 {
1130 	struct var	*var = (struct var *)(kp->ks_data);
1131 
1132 	assert(kp->ks_data_size == sizeof (struct var));
1133 
1134 	SAVE_INT32(ksi, var, v_buf);
1135 	SAVE_INT32(ksi, var, v_call);
1136 	SAVE_INT32(ksi, var, v_proc);
1137 	SAVE_INT32(ksi, var, v_maxupttl);
1138 	SAVE_INT32(ksi, var, v_nglobpris);
1139 	SAVE_INT32(ksi, var, v_maxsyspri);
1140 	SAVE_INT32(ksi, var, v_clist);
1141 	SAVE_INT32(ksi, var, v_maxup);
1142 	SAVE_INT32(ksi, var, v_hbuf);
1143 	SAVE_INT32(ksi, var, v_hmask);
1144 	SAVE_INT32(ksi, var, v_pbuf);
1145 	SAVE_INT32(ksi, var, v_sptmap);
1146 	SAVE_INT32(ksi, var, v_maxpmem);
1147 	SAVE_INT32(ksi, var, v_autoup);
1148 	SAVE_INT32(ksi, var, v_bufhwm);
1149 }
1150 
1151 static void
1152 save_ncstats(kstat_t *kp, ks_instance_t *ksi)
1153 {
1154 	struct ncstats	*ncstats = (struct ncstats *)(kp->ks_data);
1155 
1156 	assert(kp->ks_data_size == sizeof (struct ncstats));
1157 
1158 	SAVE_INT32(ksi, ncstats, hits);
1159 	SAVE_INT32(ksi, ncstats, misses);
1160 	SAVE_INT32(ksi, ncstats, enters);
1161 	SAVE_INT32(ksi, ncstats, dbl_enters);
1162 	SAVE_INT32(ksi, ncstats, long_enter);
1163 	SAVE_INT32(ksi, ncstats, long_look);
1164 	SAVE_INT32(ksi, ncstats, move_to_front);
1165 	SAVE_INT32(ksi, ncstats, purges);
1166 }
1167 
1168 static void
1169 save_sysinfo(kstat_t *kp, ks_instance_t *ksi)
1170 {
1171 	sysinfo_t	*sysinfo = (sysinfo_t *)(kp->ks_data);
1172 
1173 	assert(kp->ks_data_size == sizeof (sysinfo_t));
1174 
1175 	SAVE_UINT32(ksi, sysinfo, updates);
1176 	SAVE_UINT32(ksi, sysinfo, runque);
1177 	SAVE_UINT32(ksi, sysinfo, runocc);
1178 	SAVE_UINT32(ksi, sysinfo, swpque);
1179 	SAVE_UINT32(ksi, sysinfo, swpocc);
1180 	SAVE_UINT32(ksi, sysinfo, waiting);
1181 }
1182 
1183 static void
1184 save_vminfo(kstat_t *kp, ks_instance_t *ksi)
1185 {
1186 	vminfo_t	*vminfo = (vminfo_t *)(kp->ks_data);
1187 
1188 	assert(kp->ks_data_size == sizeof (vminfo_t));
1189 
1190 	SAVE_UINT64(ksi, vminfo, freemem);
1191 	SAVE_UINT64(ksi, vminfo, swap_resv);
1192 	SAVE_UINT64(ksi, vminfo, swap_alloc);
1193 	SAVE_UINT64(ksi, vminfo, swap_avail);
1194 	SAVE_UINT64(ksi, vminfo, swap_free);
1195 	SAVE_UINT64(ksi, vminfo, updates);
1196 }
1197 
1198 static void
1199 save_nfs(kstat_t *kp, ks_instance_t *ksi)
1200 {
1201 	struct mntinfo_kstat *mntinfo = (struct mntinfo_kstat *)(kp->ks_data);
1202 
1203 	assert(kp->ks_data_size == sizeof (struct mntinfo_kstat));
1204 
1205 	SAVE_STRING(ksi, mntinfo, mik_proto);
1206 	SAVE_UINT32(ksi, mntinfo, mik_vers);
1207 	SAVE_UINT32(ksi, mntinfo, mik_flags);
1208 	SAVE_UINT32(ksi, mntinfo, mik_secmod);
1209 	SAVE_UINT32(ksi, mntinfo, mik_curread);
1210 	SAVE_UINT32(ksi, mntinfo, mik_curwrite);
1211 	SAVE_INT32(ksi, mntinfo, mik_timeo);
1212 	SAVE_INT32(ksi, mntinfo, mik_retrans);
1213 	SAVE_UINT32(ksi, mntinfo, mik_acregmin);
1214 	SAVE_UINT32(ksi, mntinfo, mik_acregmax);
1215 	SAVE_UINT32(ksi, mntinfo, mik_acdirmin);
1216 	SAVE_UINT32(ksi, mntinfo, mik_acdirmax);
1217 	SAVE_UINT32_X(ksi, "lookup_srtt", mntinfo->mik_timers[0].srtt);
1218 	SAVE_UINT32_X(ksi, "lookup_deviate", mntinfo->mik_timers[0].deviate);
1219 	SAVE_UINT32_X(ksi, "lookup_rtxcur", mntinfo->mik_timers[0].rtxcur);
1220 	SAVE_UINT32_X(ksi, "read_srtt", mntinfo->mik_timers[1].srtt);
1221 	SAVE_UINT32_X(ksi, "read_deviate", mntinfo->mik_timers[1].deviate);
1222 	SAVE_UINT32_X(ksi, "read_rtxcur", mntinfo->mik_timers[1].rtxcur);
1223 	SAVE_UINT32_X(ksi, "write_srtt", mntinfo->mik_timers[2].srtt);
1224 	SAVE_UINT32_X(ksi, "write_deviate", mntinfo->mik_timers[2].deviate);
1225 	SAVE_UINT32_X(ksi, "write_rtxcur", mntinfo->mik_timers[2].rtxcur);
1226 	SAVE_UINT32(ksi, mntinfo, mik_noresponse);
1227 	SAVE_UINT32(ksi, mntinfo, mik_failover);
1228 	SAVE_UINT32(ksi, mntinfo, mik_remap);
1229 	SAVE_STRING(ksi, mntinfo, mik_curserver);
1230 }
1231 
1232 #ifdef __sparc
1233 static void
1234 save_sfmmu_global_stat(kstat_t *kp, ks_instance_t *ksi)
1235 {
1236 	struct sfmmu_global_stat *sfmmug =
1237 	    (struct sfmmu_global_stat *)(kp->ks_data);
1238 
1239 	assert(kp->ks_data_size == sizeof (struct sfmmu_global_stat));
1240 
1241 	SAVE_INT32(ksi, sfmmug, sf_tsb_exceptions);
1242 	SAVE_INT32(ksi, sfmmug, sf_tsb_raise_exception);
1243 	SAVE_INT32(ksi, sfmmug, sf_pagefaults);
1244 	SAVE_INT32(ksi, sfmmug, sf_uhash_searches);
1245 	SAVE_INT32(ksi, sfmmug, sf_uhash_links);
1246 	SAVE_INT32(ksi, sfmmug, sf_khash_searches);
1247 	SAVE_INT32(ksi, sfmmug, sf_khash_links);
1248 	SAVE_INT32(ksi, sfmmug, sf_swapout);
1249 	SAVE_INT32(ksi, sfmmug, sf_tsb_alloc);
1250 	SAVE_INT32(ksi, sfmmug, sf_tsb_allocfail);
1251 	SAVE_INT32(ksi, sfmmug, sf_tsb_sectsb_create);
1252 	SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_alloc);
1253 	SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_alloc);
1254 	SAVE_INT32(ksi, sfmmug, sf_scd_1sttsb_allocfail);
1255 	SAVE_INT32(ksi, sfmmug, sf_scd_2ndtsb_allocfail);
1256 	SAVE_INT32(ksi, sfmmug, sf_tteload8k);
1257 	SAVE_INT32(ksi, sfmmug, sf_tteload64k);
1258 	SAVE_INT32(ksi, sfmmug, sf_tteload512k);
1259 	SAVE_INT32(ksi, sfmmug, sf_tteload4m);
1260 	SAVE_INT32(ksi, sfmmug, sf_tteload32m);
1261 	SAVE_INT32(ksi, sfmmug, sf_tteload256m);
1262 	SAVE_INT32(ksi, sfmmug, sf_tsb_load8k);
1263 	SAVE_INT32(ksi, sfmmug, sf_tsb_load4m);
1264 	SAVE_INT32(ksi, sfmmug, sf_hblk_hit);
1265 	SAVE_INT32(ksi, sfmmug, sf_hblk8_ncreate);
1266 	SAVE_INT32(ksi, sfmmug, sf_hblk8_nalloc);
1267 	SAVE_INT32(ksi, sfmmug, sf_hblk1_ncreate);
1268 	SAVE_INT32(ksi, sfmmug, sf_hblk1_nalloc);
1269 	SAVE_INT32(ksi, sfmmug, sf_hblk_slab_cnt);
1270 	SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_cnt);
1271 	SAVE_INT32(ksi, sfmmug, sf_hblk_recurse_cnt);
1272 	SAVE_INT32(ksi, sfmmug, sf_hblk_reserve_hit);
1273 	SAVE_INT32(ksi, sfmmug, sf_get_free_success);
1274 	SAVE_INT32(ksi, sfmmug, sf_get_free_throttle);
1275 	SAVE_INT32(ksi, sfmmug, sf_get_free_fail);
1276 	SAVE_INT32(ksi, sfmmug, sf_put_free_success);
1277 	SAVE_INT32(ksi, sfmmug, sf_put_free_fail);
1278 	SAVE_INT32(ksi, sfmmug, sf_pgcolor_conflict);
1279 	SAVE_INT32(ksi, sfmmug, sf_uncache_conflict);
1280 	SAVE_INT32(ksi, sfmmug, sf_unload_conflict);
1281 	SAVE_INT32(ksi, sfmmug, sf_ism_uncache);
1282 	SAVE_INT32(ksi, sfmmug, sf_ism_recache);
1283 	SAVE_INT32(ksi, sfmmug, sf_recache);
1284 	SAVE_INT32(ksi, sfmmug, sf_steal_count);
1285 	SAVE_INT32(ksi, sfmmug, sf_pagesync);
1286 	SAVE_INT32(ksi, sfmmug, sf_clrwrt);
1287 	SAVE_INT32(ksi, sfmmug, sf_pagesync_invalid);
1288 	SAVE_INT32(ksi, sfmmug, sf_kernel_xcalls);
1289 	SAVE_INT32(ksi, sfmmug, sf_user_xcalls);
1290 	SAVE_INT32(ksi, sfmmug, sf_tsb_grow);
1291 	SAVE_INT32(ksi, sfmmug, sf_tsb_shrink);
1292 	SAVE_INT32(ksi, sfmmug, sf_tsb_resize_failures);
1293 	SAVE_INT32(ksi, sfmmug, sf_tsb_reloc);
1294 	SAVE_INT32(ksi, sfmmug, sf_user_vtop);
1295 	SAVE_INT32(ksi, sfmmug, sf_ctx_inv);
1296 	SAVE_INT32(ksi, sfmmug, sf_tlb_reprog_pgsz);
1297 	SAVE_INT32(ksi, sfmmug, sf_region_remap_demap);
1298 	SAVE_INT32(ksi, sfmmug, sf_create_scd);
1299 	SAVE_INT32(ksi, sfmmug, sf_join_scd);
1300 	SAVE_INT32(ksi, sfmmug, sf_leave_scd);
1301 	SAVE_INT32(ksi, sfmmug, sf_destroy_scd);
1302 }
1303 #endif
1304 
1305 #ifdef __sparc
1306 static void
1307 save_sfmmu_tsbsize_stat(kstat_t *kp, ks_instance_t *ksi)
1308 {
1309 	struct sfmmu_tsbsize_stat *sfmmut;
1310 
1311 	assert(kp->ks_data_size == sizeof (struct sfmmu_tsbsize_stat));
1312 	sfmmut = (struct sfmmu_tsbsize_stat *)(kp->ks_data);
1313 
1314 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_8k);
1315 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_16k);
1316 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_32k);
1317 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_64k);
1318 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_128k);
1319 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_256k);
1320 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_512k);
1321 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_1m);
1322 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_2m);
1323 	SAVE_INT32(ksi, sfmmut, sf_tsbsz_4m);
1324 }
1325 #endif
1326 
1327 #ifdef __sparc
1328 static void
1329 save_simmstat(kstat_t *kp, ks_instance_t *ksi)
1330 {
1331 	uchar_t	*simmstat;
1332 	char	*simm_buf;
1333 	char	*list = NULL;
1334 	int	i;
1335 
1336 	assert(kp->ks_data_size == sizeof (uchar_t) * SIMM_COUNT);
1337 
1338 	for (i = 0, simmstat = (uchar_t *)(kp->ks_data); i < SIMM_COUNT - 1;
1339 	    i++, simmstat++) {
1340 		if (list == NULL) {
1341 			(void) asprintf(&simm_buf, "%d,", *simmstat);
1342 		} else {
1343 			(void) asprintf(&simm_buf, "%s%d,", list, *simmstat);
1344 			free(list);
1345 		}
1346 		list = simm_buf;
1347 	}
1348 
1349 	(void) asprintf(&simm_buf, "%s%d", list, *simmstat);
1350 	SAVE_STRING_X(ksi, "status", simm_buf);
1351 	free(list);
1352 	free(simm_buf);
1353 }
1354 #endif
1355 
1356 #ifdef __sparc
1357 /*
1358  * Helper function for save_temperature().
1359  */
1360 static char *
1361 short_array_to_string(short *shortp, int len)
1362 {
1363 	char	*list = NULL;
1364 	char	*list_buf;
1365 
1366 	for (; len > 1; len--, shortp++) {
1367 		if (list == NULL) {
1368 			(void) asprintf(&list_buf, "%hd,", *shortp);
1369 		} else {
1370 			(void) asprintf(&list_buf, "%s%hd,", list, *shortp);
1371 			free(list);
1372 		}
1373 		list = list_buf;
1374 	}
1375 
1376 	(void) asprintf(&list_buf, "%s%hd", list, *shortp);
1377 	free(list);
1378 	return (list_buf);
1379 }
1380 
1381 static void
1382 save_temperature(kstat_t *kp, ks_instance_t *ksi)
1383 {
1384 	struct temp_stats *temps = (struct temp_stats *)(kp->ks_data);
1385 	char	*buf;
1386 
1387 	assert(kp->ks_data_size == sizeof (struct temp_stats));
1388 
1389 	SAVE_UINT32(ksi, temps, index);
1390 
1391 	buf = short_array_to_string(temps->l1, L1_SZ);
1392 	SAVE_STRING_X(ksi, "l1", buf);
1393 	free(buf);
1394 
1395 	buf = short_array_to_string(temps->l2, L2_SZ);
1396 	SAVE_STRING_X(ksi, "l2", buf);
1397 	free(buf);
1398 
1399 	buf = short_array_to_string(temps->l3, L3_SZ);
1400 	SAVE_STRING_X(ksi, "l3", buf);
1401 	free(buf);
1402 
1403 	buf = short_array_to_string(temps->l4, L4_SZ);
1404 	SAVE_STRING_X(ksi, "l4", buf);
1405 	free(buf);
1406 
1407 	buf = short_array_to_string(temps->l5, L5_SZ);
1408 	SAVE_STRING_X(ksi, "l5", buf);
1409 	free(buf);
1410 
1411 	SAVE_INT32(ksi, temps, max);
1412 	SAVE_INT32(ksi, temps, min);
1413 	SAVE_INT32(ksi, temps, state);
1414 	SAVE_INT32(ksi, temps, temp_cnt);
1415 	SAVE_INT32(ksi, temps, shutdown_cnt);
1416 	SAVE_INT32(ksi, temps, version);
1417 	SAVE_INT32(ksi, temps, trend);
1418 	SAVE_INT32(ksi, temps, override);
1419 }
1420 #endif
1421 
1422 #ifdef __sparc
1423 static void
1424 save_temp_over(kstat_t *kp, ks_instance_t *ksi)
1425 {
1426 	short	*sh = (short *)(kp->ks_data);
1427 	char	*value;
1428 
1429 	assert(kp->ks_data_size == sizeof (short));
1430 
1431 	(void) asprintf(&value, "%hu", *sh);
1432 	SAVE_STRING_X(ksi, "override", value);
1433 	free(value);
1434 }
1435 #endif
1436 
1437 #ifdef __sparc
1438 static void
1439 save_ps_shadow(kstat_t *kp, ks_instance_t *ksi)
1440 {
1441 	uchar_t	*uchar = (uchar_t *)(kp->ks_data);
1442 
1443 	assert(kp->ks_data_size == SYS_PS_COUNT);
1444 
1445 	SAVE_CHAR_X(ksi, "core_0", *uchar++);
1446 	SAVE_CHAR_X(ksi, "core_1", *uchar++);
1447 	SAVE_CHAR_X(ksi, "core_2", *uchar++);
1448 	SAVE_CHAR_X(ksi, "core_3", *uchar++);
1449 	SAVE_CHAR_X(ksi, "core_4", *uchar++);
1450 	SAVE_CHAR_X(ksi, "core_5", *uchar++);
1451 	SAVE_CHAR_X(ksi, "core_6", *uchar++);
1452 	SAVE_CHAR_X(ksi, "core_7", *uchar++);
1453 	SAVE_CHAR_X(ksi, "pps_0", *uchar++);
1454 	SAVE_CHAR_X(ksi, "clk_33", *uchar++);
1455 	SAVE_CHAR_X(ksi, "clk_50", *uchar++);
1456 	SAVE_CHAR_X(ksi, "v5_p", *uchar++);
1457 	SAVE_CHAR_X(ksi, "v12_p", *uchar++);
1458 	SAVE_CHAR_X(ksi, "v5_aux", *uchar++);
1459 	SAVE_CHAR_X(ksi, "v5_p_pch", *uchar++);
1460 	SAVE_CHAR_X(ksi, "v12_p_pch", *uchar++);
1461 	SAVE_CHAR_X(ksi, "v3_pch", *uchar++);
1462 	SAVE_CHAR_X(ksi, "v5_pch", *uchar++);
1463 	SAVE_CHAR_X(ksi, "p_fan", *uchar++);
1464 }
1465 #endif
1466 
1467 #ifdef __sparc
1468 static void
1469 save_fault_list(kstat_t *kp, ks_instance_t *ksi)
1470 {
1471 	struct ft_list *fault;
1472 	char	name[KSTAT_STRLEN + 7];
1473 	int	i;
1474 
1475 	for (i = 1, fault = (struct ft_list *)(kp->ks_data);
1476 	    i <= 999999 && i <= kp->ks_data_size / sizeof (struct ft_list);
1477 	    i++, fault++) {
1478 		(void) snprintf(name, sizeof (name), "unit_%d", i);
1479 		SAVE_INT32_X(ksi, name, fault->unit);
1480 		(void) snprintf(name, sizeof (name), "type_%d", i);
1481 		SAVE_INT32_X(ksi, name, fault->type);
1482 		(void) snprintf(name, sizeof (name), "fclass_%d", i);
1483 		SAVE_INT32_X(ksi, name, fault->fclass);
1484 		(void) snprintf(name, sizeof (name), "create_time_%d", i);
1485 		SAVE_HRTIME_X(ksi, name, fault->create_time);
1486 		(void) snprintf(name, sizeof (name), "msg_%d", i);
1487 		SAVE_STRING_X(ksi, name, fault->msg);
1488 	}
1489 }
1490 #endif
1491 
1492 static void
1493 save_named(kstat_t *kp, ks_instance_t *ksi)
1494 {
1495 	kstat_named_t *knp;
1496 	int	n;
1497 
1498 	for (n = kp->ks_ndata, knp = KSTAT_NAMED_PTR(kp); n > 0; n--, knp++) {
1499 		/*
1500 		 * Annoyingly, some drivers have kstats with uninitialized
1501 		 * members (which kstat_install(9F) is sadly powerless to
1502 		 * prevent, and kstat_read(3KSTAT) unfortunately does nothing
1503 		 * to stop).  To prevent these from confusing us to be
1504 		 * KSTAT_DATA_CHAR statistics, we skip over them.
1505 		 */
1506 		if (knp->name[0] == '\0')
1507 			continue;
1508 
1509 		switch (knp->data_type) {
1510 		case KSTAT_DATA_CHAR:
1511 			nvpair_insert(ksi, knp->name,
1512 			    (ks_value_t *)&knp->value, KSTAT_DATA_CHAR);
1513 			break;
1514 		case KSTAT_DATA_INT32:
1515 			nvpair_insert(ksi, knp->name,
1516 			    (ks_value_t *)&knp->value, KSTAT_DATA_INT32);
1517 			break;
1518 		case KSTAT_DATA_UINT32:
1519 			nvpair_insert(ksi, knp->name,
1520 			    (ks_value_t *)&knp->value, KSTAT_DATA_UINT32);
1521 			break;
1522 		case KSTAT_DATA_INT64:
1523 			nvpair_insert(ksi, knp->name,
1524 			    (ks_value_t *)&knp->value, KSTAT_DATA_INT64);
1525 			break;
1526 		case KSTAT_DATA_UINT64:
1527 			nvpair_insert(ksi, knp->name,
1528 			    (ks_value_t *)&knp->value, KSTAT_DATA_UINT64);
1529 			break;
1530 		case KSTAT_DATA_STRING:
1531 			SAVE_STRING_X(ksi, knp->name, KSTAT_NAMED_STR_PTR(knp));
1532 			break;
1533 		default:
1534 			assert(B_FALSE); /* Invalid data type */
1535 			break;
1536 		}
1537 	}
1538 }
1539 
1540 static void
1541 save_intr(kstat_t *kp, ks_instance_t *ksi)
1542 {
1543 	kstat_intr_t *intr = KSTAT_INTR_PTR(kp);
1544 	char	*intr_names[] = {"hard", "soft", "watchdog", "spurious",
1545 	    "multiple_service"};
1546 	int	n;
1547 
1548 	for (n = 0; n < KSTAT_NUM_INTRS; n++)
1549 		SAVE_UINT32_X(ksi, intr_names[n], intr->intrs[n]);
1550 }
1551 
1552 static void
1553 save_io(kstat_t *kp, ks_instance_t *ksi)
1554 {
1555 	kstat_io_t	*ksio = KSTAT_IO_PTR(kp);
1556 
1557 	SAVE_UINT64(ksi, ksio, nread);
1558 	SAVE_UINT64(ksi, ksio, nwritten);
1559 	SAVE_UINT32(ksi, ksio, reads);
1560 	SAVE_UINT32(ksi, ksio, writes);
1561 	SAVE_HRTIME(ksi, ksio, wtime);
1562 	SAVE_HRTIME(ksi, ksio, wlentime);
1563 	SAVE_HRTIME(ksi, ksio, wlastupdate);
1564 	SAVE_HRTIME(ksi, ksio, rtime);
1565 	SAVE_HRTIME(ksi, ksio, rlentime);
1566 	SAVE_HRTIME(ksi, ksio, rlastupdate);
1567 	SAVE_UINT32(ksi, ksio, wcnt);
1568 	SAVE_UINT32(ksi, ksio, rcnt);
1569 }
1570 
1571 static void
1572 save_timer(kstat_t *kp, ks_instance_t *ksi)
1573 {
1574 	kstat_timer_t	*ktimer = KSTAT_TIMER_PTR(kp);
1575 
1576 	SAVE_STRING(ksi, ktimer, name);
1577 	SAVE_UINT64(ksi, ktimer, num_events);
1578 	SAVE_HRTIME(ksi, ktimer, elapsed_time);
1579 	SAVE_HRTIME(ksi, ktimer, min_time);
1580 	SAVE_HRTIME(ksi, ktimer, max_time);
1581 	SAVE_HRTIME(ksi, ktimer, start_time);
1582 	SAVE_HRTIME(ksi, ktimer, stop_time);
1583 }
1584