xref: /titanic_52/usr/src/cmd/rpcsvc/rstat_proc.c (revision d62bc4badc1c1f1549c961cfb8b420e650e1272b)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*
 * rstat service:  built with rstat.x
 */

#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <signal.h>
#include <utmpx.h>
#include <nlist.h>
#include <fcntl.h>
#include <syslog.h>
#include <kstat.h>

#include <rpc/rpc.h>

#include <sys/socket.h>
#include <sys/cpuvar.h>
#include <sys/sysinfo.h>
#include <sys/systm.h>
#include <errno.h>
#include <sys/stropts.h>
#include <sys/tihdr.h>
#include <sys/sysmacros.h>

#include <net/if.h>
#include <inet/mib2.h>

#include "rstat.h"
#include "rstat_v2.h"

typedef struct {
	kstat_t	sys;
	kstat_t	vm;
} _cpu_stats_t;

/*
 *	system and cpu stats
 */
static	kstat_ctl_t	*kc;		/* libkstat cookie */
static	int	ncpus;
static	_cpu_stats_t	*cpu_stats_list = NULL;
static	kstat_t	*system_misc_ksp;
static	kstat_named_t *boot_time_knp;
static	kstat_named_t *avenrun_1min_knp, *avenrun_5min_knp, *avenrun_15min_knp;
static	int	hz;
static	struct	timeval btm;		/* boottime */

/*
 *	network interface stats
 */

typedef struct mib_item_s {
	struct mib_item_s	*next_item;
	long			group;
	long			mib_id;
	long			length;
	char			*valp;
} mib_item_t;

mib_item_t	*netstat_item;

/*
 * disk stats
 */

struct diskinfo {
	struct diskinfo *next;
	kstat_t *ks;
	kstat_io_t kios;
};

#define	NULLDISK (struct diskinfo *)0
static	struct diskinfo zerodisk = { NULL, NULL };
static	struct diskinfo *firstdisk = NULLDISK;
static	struct diskinfo *lastdisk = NULLDISK;
static	struct diskinfo *snip = NULLDISK;
static	int ndisks;

/*
 * net stats
 */

struct netinfo {
	struct netinfo *next;
	kstat_t	*ks;
	kstat_named_t *ipackets;
	kstat_named_t *opackets;
	kstat_named_t *ierrors;
	kstat_named_t *oerrors;
	kstat_named_t *collisions;
};

#define	NULLNET (struct netinfo *)0
static	struct netinfo zeronet = { NULL, NULL, NULL, NULL, NULL, NULL, NULL };
static	struct netinfo *firstnet = NULLNET;
static	struct netinfo *lastnet = NULLNET;
static	struct netinfo *netsnip = NULLNET;
static	int nnets;

/*
 *  Define EXIT_WHEN_IDLE if you are able to have this program invoked
 *  automatically on demand (as from inetd).  When defined, the service
 *  will terminated after being idle for 120 seconds.
 */

#define	EXIT_WHEN_IDLE	1

int sincelastreq = 0;		/* number of alarms since last request */
#ifdef EXIT_WHEN_IDLE
#define	CLOSEDOWN 120		/* how long to wait before exiting */
#endif /* def EXIT_WHEN_IDLE */

statstime stats_s3;
statsvar stats_s4;
/* V2 support for backwards compatibility to pre-5.0 systems */
statsswtch stats_s2;

static int stat_is_init = 0;

static	void	fail(int, char *, ...);
static	void	safe_zalloc(void **, int, int);
static	kid_t	safe_kstat_read(kstat_ctl_t *, kstat_t *, void *);
static	kstat_t	*safe_kstat_lookup(kstat_ctl_t *, char *, int, char *);
static	void	*safe_kstat_data_lookup(kstat_t *, char *);
static	void	system_stat_init(void);
static	int	system_stat_load(void);
static	void	init_disks(void);
static	int	diskinfo_load(void);
static	void	init_net(void);
static	int	netinfo_load(void);

static	void	updatestat(int);

static	mib_item_t	*mibget(int sd);
static	int	mibopen(void);
static  char	*octetstr(char *buf, Octet_t *op, int code);

static	void	kstat_copy(kstat_t *, kstat_t *, int);

static	char	*cmdname = "rpc.rstatd";

#define	CPU_STAT(ksp, name)	(((kstat_named_t *)safe_kstat_data_lookup( \
				    (ksp), (name)))->value.ui64)
static	_cpu_stats_t	cpu_stats_all = { 0 };

static void
stat_init(void)
{
	struct utmpx *utmpx, utmpx_id;

	stat_is_init = 1;

	if ((kc = kstat_open()) == NULL)
		fail(1, "kstat_open(): can't open /dev/kstat");

	/*
	 * Preallocate minimal set of drive entries.
	 */

	if (stats_s4.dk_xfer.dk_xfer_val == NULL) {
		stats_s4.dk_xfer.dk_xfer_len = RSTAT_DK_NDRIVE;
		stats_s4.dk_xfer.dk_xfer_val =
		    (int *)calloc(RSTAT_DK_NDRIVE, sizeof (int));
	}

	system_stat_init();
	init_disks();
	init_net();

	/*
	 * To get the boot time, use utmpx, which is per-zone, but fall back
	 * to the system-wide kstat if utmpx is hosed for any reason.
	 */
	utmpx_id.ut_type = BOOT_TIME;
	if ((utmpx = getutxid(&utmpx_id)) != NULL)
		btm = utmpx->ut_tv;
	else {
		btm.tv_sec = boot_time_knp->value.ul;
		btm.tv_usec = 0; /* don't bother with usecs for boot time */
	}
	endutxent();
	stats_s4.boottime.tv_sec =
		stats_s2.boottime.tv_sec =
		stats_s3.boottime.tv_sec = btm.tv_sec;
	stats_s4.boottime.tv_usec =
		stats_s2.boottime.tv_usec =
		stats_s3.boottime.tv_usec = btm.tv_usec;

	updatestat(0);
	alarm(1);
	signal(SIGALRM, updatestat);
	sleep(2);		/* allow for one wake-up */
}

statsvar *
rstatproc_stats_4_svc(argp, svcrq)
void *argp;
struct svc_req *svcrq;
{
	if (! stat_is_init)
		stat_init();
#ifdef EXIT_WHEN_IDLE
	sincelastreq = 0;
#endif
	return (&stats_s4);
}

statstime *
rstatproc_stats_3_svc(argp, svcrq)
void *argp;
struct svc_req *svcrq;
{
	if (! stat_is_init)
		stat_init();
#ifdef EXIT_WHEN_IDLE
	sincelastreq = 0;
#endif
	return (&stats_s3);
}

statsswtch *
rstatproc_stats_2_svc(argp, svcrq)
void *argp;
struct svc_req *svcrq;
{
	if (! stat_is_init)
		stat_init();
#ifdef EXIT_WHEN_IDLE
	sincelastreq = 0;
#endif
	return (&stats_s2);
}


uint_t *
rstatproc_havedisk_4_svc(argp, svcrq)
void *argp;
struct svc_req *svcrq;
{
	return (rstatproc_havedisk_3_svc(argp, svcrq));
}

uint_t *
rstatproc_havedisk_3_svc(argp, svcrq)
void *argp;
struct svc_req *svcrq;
{
	static uint_t have;

	if (! stat_is_init)
		stat_init();
#ifdef EXIT_WHEN_IDLE
	sincelastreq = 0;
#endif
	have = (ndisks != 0);
	return (&have);
}

uint_t *
rstatproc_havedisk_2_svc(argp, svcrq)
void *argp;
struct svc_req *svcrq;
{
	return (rstatproc_havedisk_3_svc(argp, svcrq));
}

void
updatestat(int ignored)
{
extern int _rpcpmstart;		 /* Started by a port monitor ? */
extern int _rpcsvcdirty;	 /* Still serving ? */

#ifdef DEBUG
	fprintf(stderr, "entering updatestat\n");
#endif
#ifdef EXIT_WHEN_IDLE
	if (_rpcpmstart && sincelastreq >= CLOSEDOWN && !_rpcsvcdirty) {
#ifdef DEBUG
		fprintf(stderr, "about to closedown\n");
#endif
		exit(0);
	}
	sincelastreq++;
#endif /* def EXIT_WHEN_IDLE */

	(void) alarm(0);
#ifdef DEBUG
	fprintf(stderr, "boottime: %d %d\n", stats_s3.boottime.tv_sec,
		stats_s3.boottime.tv_usec);
#endif
	while (system_stat_load() || diskinfo_load() || netinfo_load()) {
		(void) kstat_chain_update(kc);
		system_stat_init();
		init_disks();
		init_net();
	}
	stats_s4.cp_time.cp_time_len = CPU_STATES;
	if (stats_s4.cp_time.cp_time_val == NULL)
		stats_s4.cp_time.cp_time_val =
		malloc(stats_s4.cp_time.cp_time_len * sizeof (int));
	stats_s2.cp_time[RSTAT_CPU_USER] =
	stats_s3.cp_time[RSTAT_CPU_USER] =
	stats_s4.cp_time.cp_time_val[RSTAT_CPU_USER] =
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_user");
	stats_s2.cp_time[RSTAT_CPU_NICE] =
	stats_s3.cp_time[RSTAT_CPU_NICE] =
	stats_s4.cp_time.cp_time_val[RSTAT_CPU_NICE] =
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_wait");
	stats_s2.cp_time[RSTAT_CPU_SYS] =
	stats_s3.cp_time[RSTAT_CPU_SYS] =
	stats_s4.cp_time.cp_time_val[RSTAT_CPU_SYS] =
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_kernel");
	stats_s2.cp_time[RSTAT_CPU_IDLE] =
	stats_s3.cp_time[RSTAT_CPU_IDLE] =
	stats_s4.cp_time.cp_time_val[RSTAT_CPU_IDLE] =
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_idle");

#ifdef DEBUG
	fprintf(stderr, "cpu: %d %d %d %d\n",
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_user"),
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_wait"),
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_kernel"),
		CPU_STAT(&cpu_stats_all.sys, "cpu_ticks_idle"));
	fprintf(stderr, "cp_time: %d %d %d %d\n",
		stats_s3.cp_time[RSTAT_CPU_USER],
		stats_s3.cp_time[RSTAT_CPU_NICE],
		stats_s3.cp_time[RSTAT_CPU_SYS],
		stats_s3.cp_time[RSTAT_CPU_IDLE]);
#endif

	/* current time */
	gettimeofday((struct timeval *)&stats_s3.curtime, NULL);
	stats_s4.curtime = stats_s3.curtime;

	stats_s2.v_pgpgin =
	stats_s3.v_pgpgin =
	stats_s4.v_pgpgin = CPU_STAT(&cpu_stats_all.vm, "pgpgin");
	stats_s2.v_pgpgout =
	stats_s3.v_pgpgout =
	stats_s4.v_pgpgout = CPU_STAT(&cpu_stats_all.vm, "pgpgout");
	stats_s2.v_pswpin =
	stats_s3.v_pswpin =
	stats_s4.v_pswpin = CPU_STAT(&cpu_stats_all.vm, "pgswapin");
	stats_s2.v_pswpout =
	stats_s3.v_pswpout =
	stats_s4.v_pswpout = CPU_STAT(&cpu_stats_all.vm, "pgswapout");
	stats_s3.v_intr = CPU_STAT(&cpu_stats_all.sys, "intr");
	stats_s3.v_intr -= hz*(stats_s3.curtime.tv_sec - btm.tv_sec) +
		hz*(stats_s3.curtime.tv_usec - btm.tv_usec)/1000000;
	stats_s2.v_intr =
	stats_s4.v_intr = stats_s3.v_intr;
	/* swtch not in V1 */
	stats_s2.v_swtch =
	stats_s3.v_swtch =
	stats_s4.v_swtch = CPU_STAT(&cpu_stats_all.sys, "pswitch");

#ifdef DEBUG
	fprintf(stderr,
		"pgin: %d pgout: %d swpin: %d swpout: %d intr: %d swtch: %d\n",
		stats_s3.v_pgpgin,
		stats_s3.v_pgpgout,
		stats_s3.v_pswpin,
		stats_s3.v_pswpout,
		stats_s3.v_intr,
		stats_s3.v_swtch);
#endif
	/*
	 * V2 and V3 of rstat are limited to RSTAT_DK_NDRIVE drives
	 */
	memcpy(stats_s3.dk_xfer, stats_s4.dk_xfer.dk_xfer_val,
		RSTAT_DK_NDRIVE * sizeof (int));
	memcpy(stats_s2.dk_xfer, stats_s4.dk_xfer.dk_xfer_val,
		RSTAT_DK_NDRIVE * sizeof (int));
#ifdef DEBUG
	fprintf(stderr, "dk_xfer: %d %d %d %d\n",
		stats_s4.dk_xfer.dk_xfer_val[0],
		stats_s4.dk_xfer.dk_xfer_val[1],
		stats_s4.dk_xfer.dk_xfer_val[2],
		stats_s4.dk_xfer.dk_xfer_val[3]);
#endif

	stats_s2.if_ipackets =
	stats_s3.if_ipackets = stats_s4.if_ipackets;
	/* no s2 opackets */
	stats_s3.if_opackets = stats_s4.if_opackets;
	stats_s2.if_ierrors =
	stats_s3.if_ierrors = stats_s4.if_ierrors;
	stats_s2.if_oerrors =
	stats_s3.if_oerrors = stats_s4.if_oerrors;
	stats_s2.if_collisions =
	stats_s3.if_collisions = stats_s4.if_collisions;

	stats_s2.avenrun[0] =
	stats_s3.avenrun[0] =
	stats_s4.avenrun[0] = avenrun_1min_knp->value.ul;
	stats_s2.avenrun[1] =
	stats_s3.avenrun[1] =
	stats_s4.avenrun[1] = avenrun_5min_knp->value.ul;
	stats_s2.avenrun[2] =
	stats_s3.avenrun[2] =
	stats_s4.avenrun[2] = avenrun_15min_knp->value.ul;
#ifdef DEBUG
	fprintf(stderr, "avenrun: %d %d %d\n", stats_s3.avenrun[0],
		stats_s3.avenrun[1], stats_s3.avenrun[2]);
#endif
	signal(SIGALRM, updatestat);
	alarm(1);
}

/* --------------------------------- MIBGET -------------------------------- */

static mib_item_t *
mibget(int sd)
{
	int			flags;
	int			j, getcode;
	struct strbuf		ctlbuf, databuf;
	char			buf[512];
	struct T_optmgmt_req	*tor = (struct T_optmgmt_req *)buf;
	struct T_optmgmt_ack	*toa = (struct T_optmgmt_ack *)buf;
	struct T_error_ack	*tea = (struct T_error_ack *)buf;
	struct opthdr		*req;
	mib_item_t		*first_item = NULL;
	mib_item_t		*last_item  = NULL;
	mib_item_t		*temp;

	tor->PRIM_type = T_SVR4_OPTMGMT_REQ;
	tor->OPT_offset = sizeof (struct T_optmgmt_req);
	tor->OPT_length = sizeof (struct opthdr);
	tor->MGMT_flags = T_CURRENT;
	req = (struct opthdr *)&tor[1];
	req->level = MIB2_IP;		/* any MIB2_xxx value ok here */
	req->name  = 0;
	req->len   = 0;

	ctlbuf.buf = buf;
	ctlbuf.len = tor->OPT_length + tor->OPT_offset;
	flags = 0;
	if (putmsg(sd, &ctlbuf, NULL, flags) == -1) {
		perror("mibget: putmsg(ctl) failed");
		goto error_exit;
	}
	/*
	 * each reply consists of a ctl part for one fixed structure
	 * or table, as defined in mib2.h.  The format is a T_OPTMGMT_ACK,
	 * containing an opthdr structure.  level/name identify the entry,
	 * len is the size of the data part of the message.
	 */
	req = (struct opthdr *)&toa[1];
	ctlbuf.maxlen = sizeof (buf);
	/*CSTYLED*/
	for (j = 1; ; j++) {
		flags = 0;
		getcode = getmsg(sd, &ctlbuf, NULL, &flags);
		if (getcode == -1) {
#ifdef DEBUG_MIB
			perror("mibget getmsg(ctl) failed");
			fprintf(stderr, "#   level   name    len\n");
			i = 0;
			for (last_item = first_item; last_item;
				last_item = last_item->next_item)
				fprintf(stderr, "%d  %4d   %5d   %d\n", ++i,
					last_item->group,
					last_item->mib_id,
					last_item->length);
#endif /* DEBUG_MIB */
			goto error_exit;
		}
		if (getcode == 0 &&
			(ctlbuf.len >= sizeof (struct T_optmgmt_ack)) &&
			(toa->PRIM_type == T_OPTMGMT_ACK) &&
			(toa->MGMT_flags == T_SUCCESS) &&
			req->len == 0) {
#ifdef DEBUG_MIB
			fprintf(stderr,
		"mibget getmsg() %d returned EOD (level %d, name %d)\n",
				j, req->level, req->name);
#endif /* DEBUG_MIB */
			return (first_item);		/* this is EOD msg */
		}

		if (ctlbuf.len >= sizeof (struct T_error_ack) &&
			(tea->PRIM_type == T_ERROR_ACK)) {
#ifdef DEBUG_MIB
			fprintf(stderr,
	"mibget %d gives T_ERROR_ACK: TLI_error = 0x%x, UNIX_error = 0x%x\n",
				j, getcode, tea->TLI_error, tea->UNIX_error);
#endif /* DEBUG_MIB */
			errno = (tea->TLI_error == TSYSERR)
				? tea->UNIX_error : EPROTO;
			goto error_exit;
		}

		if (getcode != MOREDATA ||
			(ctlbuf.len < sizeof (struct T_optmgmt_ack)) ||
			(toa->PRIM_type != T_OPTMGMT_ACK) ||
			(toa->MGMT_flags != T_SUCCESS)) {
#ifdef DEBUG_MIB
			fprintf(stderr,
	"mibget getmsg(ctl) %d returned %d, ctlbuf.len = %d, PRIM_type = %d\n",
				j, getcode, ctlbuf.len, toa->PRIM_type);
			if (toa->PRIM_type == T_OPTMGMT_ACK)
				fprintf(stderr,
	"T_OPTMGMT_ACK: MGMT_flags = 0x%x, req->len = %d\n",
					toa->MGMT_flags, req->len);
#endif /* DEBUG_MIB */
			errno = ENOMSG;
			goto error_exit;
		}

		temp = malloc(sizeof (mib_item_t));
		if (!temp) {
			perror("mibget malloc failed");
			goto error_exit;
		}
		if (last_item)
			last_item->next_item = temp;
		else
			first_item = temp;
		last_item = temp;
		last_item->next_item = NULL;
		last_item->group = req->level;
		last_item->mib_id = req->name;
		last_item->length = req->len;
		last_item->valp = malloc(req->len);
#ifdef DEBUG_MIB
		fprintf(stderr,
			"msg %d:  group = %4d   mib_id = %5d   length = %d\n",
			j, last_item->group, last_item->mib_id,
			last_item->length);
#endif /* DEBUG_MIB */
		databuf.maxlen = last_item->length;
		databuf.buf    = last_item->valp;
		databuf.len    = 0;
		flags = 0;
		getcode = getmsg(sd, NULL, &databuf, &flags);
		if (getcode == -1) {
			perror("mibget getmsg(data) failed");
			goto error_exit;
		} else if (getcode != 0) {
			fprintf(stderr,
"mibget getmsg(data) returned %d, databuf.maxlen = %d, databuf.len = %d\n",
				getcode, databuf.maxlen, databuf.len);
			goto error_exit;
		}
	}

error_exit:
	while (first_item) {
		last_item = first_item;
		first_item = first_item->next_item;
		if (last_item->valp) {
			free(last_item->valp);
		}
		free(last_item);
	}
	return (first_item);
}

static int
mibopen(void)
{
	int	sd;

	/* gives us ip w/ arp on top */
	sd = open("/dev/arp", O_RDWR);
	if (sd == -1) {
		perror("arp open");
		close(sd);
		return (-1);
	}
	if (ioctl(sd, I_PUSH, "tcp") == -1) {
		perror("tcp I_PUSH");
		close(sd);
		return (-1);
	}
	if (ioctl(sd, I_PUSH, "udp") == -1) {
		perror("udp I_PUSH");
		close(sd);
		return (-1);
	}
	return (sd);
}

static char *
octetstr(char *buf, Octet_t *op, int code)
{
	int	i;
	char	*cp;

	cp = buf;
	if (op)
		for (i = 0; i < op->o_length; i++)
			switch (code) {
			case 'd':
				sprintf(cp, "%d.", 0xff & op->o_bytes[i]);
				cp = strchr(cp, '\0');
				break;
			case 'a':
				*cp++ = op->o_bytes[i];
				break;
			case 'h':
			default:
				sprintf(cp, "%02x:", 0xff & op->o_bytes[i]);
				cp += 3;
				break;
			}
	if (code != 'a' && cp != buf)
		cp--;
	*cp = '\0';
	return (buf);
}

static void
fail(int do_perror, char *message, ...)
{
	va_list args;

	va_start(args, message);
	fprintf(stderr, "%s: ", cmdname);
	vfprintf(stderr, message, args);
	va_end(args);
	if (do_perror)
		fprintf(stderr, ": %s", strerror(errno));
	fprintf(stderr, "\n");
	exit(2);
}

static void
safe_zalloc(void **ptr, int size, int free_first)
{
	if (free_first && *ptr != NULL)
		free(*ptr);
	if ((*ptr = malloc(size)) == NULL)
		fail(1, "malloc failed");
	memset(*ptr, 0, size);
}

kid_t
safe_kstat_read(kstat_ctl_t *kctl, kstat_t *ksp, void *data)
{
	kid_t kstat_chain_id = kstat_read(kctl, ksp, data);

	if (kstat_chain_id == -1)
		fail(1, "kstat_read(%x, '%s') failed", kctl, ksp->ks_name);
	return (kstat_chain_id);
}

kstat_t *
safe_kstat_lookup(kstat_ctl_t *kctl, char *ks_module, int ks_instance,
	char *ks_name)
{
	kstat_t *ksp = kstat_lookup(kctl, ks_module, ks_instance, ks_name);

	if (ksp == NULL)
		fail(0, "kstat_lookup('%s', %d, '%s') failed",
			ks_module == NULL ? "" : ks_module,
			ks_instance,
			ks_name == NULL ? "" : ks_name);
	return (ksp);
}

void *
safe_kstat_data_lookup(kstat_t *ksp, char *name)
{
	void *fp = kstat_data_lookup(ksp, name);

	if (fp == NULL) {
		fail(0, "kstat_data_lookup('%s', '%s') failed",
			ksp->ks_name, name);
	}
	return (fp);
}

/*
 * Get various KIDs for subsequent system_stat_load operations.
 */

static void
system_stat_init(void)
{
	kstat_t *ksp;
	int i, nvmks;

	/*
	 * Global statistics
	 */

	system_misc_ksp	= safe_kstat_lookup(kc, "unix", 0, "system_misc");

	safe_kstat_read(kc, system_misc_ksp, NULL);
	boot_time_knp = safe_kstat_data_lookup(system_misc_ksp, "boot_time");
	avenrun_1min_knp = safe_kstat_data_lookup(system_misc_ksp,
		"avenrun_1min");
	avenrun_5min_knp = safe_kstat_data_lookup(system_misc_ksp,
		"avenrun_5min");
	avenrun_15min_knp = safe_kstat_data_lookup(system_misc_ksp,
		"avenrun_15min");

	/*
	 * Per-CPU statistics
	 */

	ncpus = 0;
	for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next)
		if (strcmp(ksp->ks_module, "cpu") == 0 &&
		    strcmp(ksp->ks_name, "sys") == 0)
			ncpus++;

	safe_zalloc((void **)&cpu_stats_list, ncpus * sizeof (*cpu_stats_list),
	    1);

	ncpus = 0;
	for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next)
		if (strcmp(ksp->ks_module, "cpu") == 0 &&
		    strcmp(ksp->ks_name, "sys") == 0 &&
		    kstat_read(kc, ksp, NULL) != -1) {
			kstat_copy(ksp, &cpu_stats_list[ncpus].sys,
			    1);
			if ((ksp = kstat_lookup(kc, "cpu", ksp->ks_instance,
			    "vm")) != NULL && kstat_read(kc, ksp, NULL) != -1)
				kstat_copy(ksp, &cpu_stats_list[ncpus].vm, 1);
			else
				fail(0, "couldn't find per-CPU VM statistics");
			ncpus++;
		    }

	if (ncpus == 0)
		fail(0, "couldn't find per-CPU statistics");
}

/*
 * load statistics, summing across CPUs where needed
 */

static int
system_stat_load(void)
{
	int i, j;
	_cpu_stats_t cs;
	ulong_t *np, *tp;

	/*
	 * Global statistics
	 */

	safe_kstat_read(kc, system_misc_ksp, NULL);

	/*
	 * Per-CPU statistics.
	 */

	for (i = 0; i < ncpus; i++) {
		if (kstat_read(kc, &cpu_stats_list[i].sys, NULL) == -1 ||
		    kstat_read(kc, &cpu_stats_list[i].vm, NULL) == -1)
			return (1);
		if (i == 0) {
			kstat_copy(&cpu_stats_list[0].sys, &cpu_stats_all.sys,
			    1);
			kstat_copy(&cpu_stats_list[0].vm, &cpu_stats_all.vm, 1);
		} else {
			kstat_named_t *nkp;
			kstat_named_t *tkp;

			/*
			 * Other CPUs' statistics are accumulated in
			 * cpu_stats_all, initialized at the first iteration of
			 * the loop.
			 */
			nkp = (kstat_named_t *)cpu_stats_all.sys.ks_data;
			tkp = (kstat_named_t *)cpu_stats_list[i].sys.ks_data;
			for (j = 0; j < cpu_stats_list[i].sys.ks_ndata; j++)
				(nkp++)->value.ui64 += (tkp++)->value.ui64;
			nkp = (kstat_named_t *)cpu_stats_all.vm.ks_data;
			tkp = (kstat_named_t *)cpu_stats_list[i].vm.ks_data;
			for (j = 0; j < cpu_stats_list[i].vm.ks_ndata; j++)
				(nkp++)->value.ui64 += (tkp++)->value.ui64;
		}
	}
	return (0);
}

static int
kscmp(kstat_t *ks1, kstat_t *ks2)
{
	int cmp;

	cmp = strcmp(ks1->ks_module, ks2->ks_module);
	if (cmp != 0)
		return (cmp);
	cmp = ks1->ks_instance - ks2->ks_instance;
	if (cmp != 0)
		return (cmp);
	return (strcmp(ks1->ks_name, ks2->ks_name));
}

static void
init_disks(void)
{
	struct diskinfo *disk, *prevdisk, *comp;
	kstat_t *ksp;

	ndisks = 0;
	disk = &zerodisk;

	/*
	 * Patch the snip in the diskinfo list (see below)
	 */
	if (snip)
		lastdisk->next = snip;

	for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) {

		if (ksp->ks_type != KSTAT_TYPE_IO ||
		    strcmp(ksp->ks_class, "disk") != 0)
			continue;
		prevdisk = disk;
		if (disk->next)
			disk = disk->next;
		else {
			safe_zalloc((void **)&disk->next,
			    sizeof (struct diskinfo), 0);
			disk = disk->next;
			disk->next = NULLDISK;
		}
		disk->ks = ksp;
		memset((void *)&disk->kios, 0, sizeof (kstat_io_t));
		disk->kios.wlastupdate = disk->ks->ks_crtime;
		disk->kios.rlastupdate = disk->ks->ks_crtime;

		/*
		 * Insertion sort on (ks_module, ks_instance, ks_name)
		 */
		comp = &zerodisk;
		while (kscmp(disk->ks, comp->next->ks) > 0)
			comp = comp->next;
		if (prevdisk != comp) {
			prevdisk->next = disk->next;
			disk->next = comp->next;
			comp->next = disk;
			disk = prevdisk;
		}
		ndisks++;
	}
	/*
	 * Put a snip in the linked list of diskinfos.  The idea:
	 * If there was a state change such that now there are fewer
	 * disks, we snip the list and retain the tail, rather than
	 * freeing it.  At the next state change, we clip the tail back on.
	 * This prevents a lot of malloc/free activity, and it's simpler.
	 */
	lastdisk = disk;
	snip = disk->next;
	disk->next = NULLDISK;

	firstdisk = zerodisk.next;

	if (ndisks > stats_s4.dk_xfer.dk_xfer_len) {
		stats_s4.dk_xfer.dk_xfer_len = ndisks;
		safe_zalloc((void **)&stats_s4.dk_xfer.dk_xfer_val,
			ndisks * sizeof (int), 1);
	}
}

static int
diskinfo_load(void)
{
	struct diskinfo *disk;
	int i;

	for (disk = firstdisk, i = 0; disk; disk = disk->next, i++) {
		if (kstat_read(kc, disk->ks, (void *)&disk->kios) == -1)
			return (1);
		stats_s4.dk_xfer.dk_xfer_val[i] = disk->kios.reads +
			disk->kios.writes;
	}
	return (0);
}

static void
init_net(void)
{
	static int sd;
	mib_item_t *item;
	mib2_ipAddrEntry_t *ap;
	char namebuf[KSTAT_STRLEN];
	struct netinfo *net, *prevnet, *comp;
	kstat_t *ksp;

	if (sd) {
		close(sd);
	}
	while (netstat_item) {
		item = netstat_item;
		netstat_item = netstat_item->next_item;
		if (item->valp) {
			free(item->valp);
		}
		free(item);
	}
	sd = mibopen();
	if (sd == -1) {
#ifdef DEBUG
		fprintf(stderr, "mibopen() failed\n");
#endif
		sd = 0;
	} else {
		if ((netstat_item = mibget(sd)) == NULL) {
#ifdef DEBUG
			fprintf(stderr, "mibget() failed\n");
#endif
			close(sd);
			sd = 0;
		}
	}
#ifdef DEBUG
	fprintf(stderr, "mibget returned item: %x\n", netstat_item);
#endif

	nnets = 0;
	net = &zeronet;

	if (netsnip)
		lastnet->next = netsnip;

	for (item = netstat_item; item; item = item->next_item) {
#ifdef DEBUG_MIB
		fprintf(stderr, "\n--- Item %x ---\n", item);
		fprintf(stderr,
		"Group = %d, mib_id = %d, length = %d, valp = 0x%x\n",
		item->group, item->mib_id, item->length,
		item->valp);
#endif
		if (item->group != MIB2_IP || item->mib_id != MIB2_IP_20)
			continue;
		ap = (mib2_ipAddrEntry_t *)item->valp;
		for (; (char *)ap < item->valp + item->length; ap++) {

			octetstr(namebuf, &ap->ipAdEntIfIndex, 'a');
#ifdef DEBUG
			fprintf(stderr, "%s ", namebuf);
#endif
			if (strlen(namebuf) == 0)
				continue;
			/*
			 * We found a device of interest.
			 * Now, let's see if there's a kstat for it.
			 * First we try to query the "link" kstats in case
			 * the link is renamed. If that fails, fallback
			 * to legacy ktats for those non-GLDv3 links.
			 */
			if (((ksp = kstat_lookup(kc, "link", 0, namebuf))
			    == NULL) && ((ksp = kstat_lookup(kc, NULL, -1,
			    namebuf)) == NULL)) {
				continue;
			}
			if (ksp->ks_type != KSTAT_TYPE_NAMED)
				continue;
			if (kstat_read(kc, ksp, NULL) == -1)
				continue;
			prevnet = net;
			if (net->next)
				net = net->next;
			else {
				safe_zalloc((void **)&net->next,
					sizeof (struct netinfo), 0);
				net = net->next;
				net->next = NULLNET;
			}
			net->ks = ksp;
			net->ipackets	= kstat_data_lookup(net->ks,
				"ipackets");
			net->opackets	= kstat_data_lookup(net->ks,
				"opackets");
			net->ierrors	= kstat_data_lookup(net->ks,
				"ierrors");
			net->oerrors	= kstat_data_lookup(net->ks,
				"oerrors");
			net->collisions	= kstat_data_lookup(net->ks,
				"collisions");
			/*
			 * Insertion sort on the name
			 */
			comp = &zeronet;
			while (strcmp(net->ks->ks_name,
			    comp->next->ks->ks_name) > 0)
				comp = comp->next;
			if (prevnet != comp) {
				prevnet->next = net->next;
				net->next = comp->next;
				comp->next = net;
				net = prevnet;
			}
			nnets++;
		}
#ifdef DEBUG
		fprintf(stderr, "\n");
#endif
	}
	/*
	 * Put a snip in the linked list of netinfos.  The idea:
	 * If there was a state change such that now there are fewer
	 * nets, we snip the list and retain the tail, rather than
	 * freeing it.  At the next state change, we clip the tail back on.
	 * This prevents a lot of malloc/free activity, and it's simpler.
	 */
	lastnet = net;
	netsnip = net->next;
	net->next = NULLNET;

	firstnet = zeronet.next;
}

static int
netinfo_load(void)
{
	struct netinfo *net;

	if (netstat_item == NULL) {
#ifdef DEBUG
		fprintf(stderr, "No net stats\n");
#endif
		return (0);
	}

	stats_s4.if_ipackets =
	stats_s4.if_opackets =
	stats_s4.if_ierrors =
	stats_s4.if_oerrors =
	stats_s4.if_collisions = 0;

	for (net = firstnet; net; net = net->next) {
		if (kstat_read(kc, net->ks, NULL) == -1)
			return (1);
		if (net->ipackets)
			stats_s4.if_ipackets	+= net->ipackets->value.ul;
		if (net->opackets)
			stats_s4.if_opackets	+= net->opackets->value.ul;
		if (net->ierrors)
			stats_s4.if_ierrors	+= net->ierrors->value.ul;
		if (net->oerrors)
			stats_s4.if_oerrors	+= net->oerrors->value.ul;
		if (net->collisions)
			stats_s4.if_collisions	+= net->collisions->value.ul;
	}
#ifdef DEBUG
	fprintf(stderr,
	    "ipackets: %d opackets: %d ierrors: %d oerrors: %d colls: %d\n",
		stats_s4.if_ipackets,
		stats_s4.if_opackets,
		stats_s4.if_ierrors,
		stats_s4.if_oerrors,
		stats_s4.if_collisions);
#endif
	return (0);
}

static void
kstat_copy(kstat_t *src, kstat_t *dst, int fr)
{
	if (fr)
		free(dst->ks_data);
	*dst = *src;
	if (src->ks_data != NULL) {
		safe_zalloc(&dst->ks_data, src->ks_data_size, 0);
		(void) memcpy(dst->ks_data, src->ks_data, src->ks_data_size);
	} else {
		dst->ks_data = NULL;
		dst->ks_data_size = 0;
	}
}