cmd/sa/sadc.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved  	*/


/*
 * sadc.c writes system activity binary data to a file or stdout.
 *
 * Usage: sadc [t n] [file]
 *
 * if t and n are not specified, it writes a dummy record to data file. This
 * usage is particularly used at system booting.  If t and n are specified, it
 * writes system data n times to file every t seconds.  In both cases, if file
 * is not specified, it writes data to stdout.
 */

#include <sys/fcntl.h>
#include <sys/flock.h>
#include <sys/proc.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/var.h>

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <kstat.h>
#include <memory.h>
#include <nlist.h>
#include <signal.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <strings.h>

#include "sa.h"

#define	MAX(x1, x2)	((x1) >= (x2) ? (x1) : (x2))

static	kstat_ctl_t	*kc;		/* libkstat cookie */
static	int	ncpus;
static	int	oncpus;
static	kstat_t	**cpu_stat_list = NULL;
static	kstat_t	**ocpu_stat_list = NULL;
static	int	ncaches;
static	kstat_t	**kmem_cache_list = NULL;

static	kstat_t	*sysinfo_ksp, *vminfo_ksp, *var_ksp;
static	kstat_t *system_misc_ksp, *ufs_inode_ksp, *kmem_oversize_ksp;
static	kstat_t *file_cache_ksp;
static	kstat_named_t *ufs_inode_size_knp, *nproc_knp;
static	kstat_named_t *file_total_knp, *file_avail_knp;
static	kstat_named_t *oversize_alloc_knp, *oversize_fail_knp;
static	int slab_create_index, slab_destroy_index, slab_size_index;
static	int buf_size_index, buf_avail_index, alloc_fail_index;

static	struct	iodevinfo zeroiodev = { NULL, NULL };
static	struct	iodevinfo *firstiodev = NULL;
static	struct	iodevinfo *lastiodev = NULL;
static	struct	iodevinfo *snip = NULL;
static	ulong_t	niodevs;

static	void	all_stat_init(void);
static	int	all_stat_load(void);
static	void	fail(int, char *, ...);
static	void	safe_zalloc(void **, int, int);
static	kid_t	safe_kstat_read(kstat_ctl_t *, kstat_t *, void *);
static	kstat_t	*safe_kstat_lookup(kstat_ctl_t *, char *, int, char *);
static	void	*safe_kstat_data_lookup(kstat_t *, char *);
static	int	safe_kstat_data_index(kstat_t *, char *);
static	void	init_iodevs(void);
static	int	iodevinfo_load(void);
static	int	kstat_copy(const kstat_t *, kstat_t *);
static	void	diff_two_arrays(kstat_t ** const [], size_t, size_t,
    kstat_t ** const []);
static	void	compute_cpu_stat_adj(void);

static	char	*cmdname = "sadc";

static	struct var var;

static	struct sa d;
static	int64_t	cpu_stat_adj[CPU_STATES] = {0};

static	long	ninode;

int caught_cont = 0;

/*
 * Sleep until *wakeup + interval, keeping cadence where desired
 *
 * *wakeup -	The time we last wanted to wake up. Updated.
 * interval -	We want to sleep until *wakeup + interval
 * *caught_cont - Global set by signal handler if we got a SIGCONT
 */
void
sleep_until(hrtime_t *wakeup, hrtime_t interval, int *caught_cont)
{
	hrtime_t now, pause, pause_left;
	struct timespec pause_tv;
	int status;
	now = gethrtime();
	pause = *wakeup + interval - now;

	if (pause <= 0 || pause < (interval / 4))
		if (*caught_cont) {
			/* Reset our cadence (see comment below) */
			*wakeup = now + interval;
			pause = interval;
		} else {
			/*
			 * If we got here, then the time between the
			 * output we just did, and the scheduled time
			 * for the next output is < 1/4 of our requested
			 * interval AND the number of intervals has been
			 * requested AND we have never caught a SIGCONT
			 * (so we have never been suspended).  In this
			 * case, we'll try to stay to the desired
			 * cadence, and we will pause for 1/2 the normal
			 * interval this time.
			 */
			pause = interval / 2;
			*wakeup += interval;
		}
	else
		*wakeup += interval;
	if (pause < 1000)
		/* Near enough */
		return;

	/* Now do the actual sleep */
	pause_left = pause;
	do {
		pause_tv.tv_sec = pause_left / NANOSEC;
		pause_tv.tv_nsec = pause_left % NANOSEC;
		status = nanosleep(&pause_tv, (struct timespec *)NULL);
		if (status < 0)
			if (errno == EINTR) {
				now = gethrtime();
				pause_left = *wakeup - now;
				if (pause_left < 1000)
					/* Near enough */
					return;
			} else {
				fail(1, "nanosleep failed");
			}
	} while (status != 0);
}

/*
 * Signal handler - so we can be aware of SIGCONT
 */
void
cont_handler(int sig_number)
{
	/* Re-set the signal handler */
	(void) signal(sig_number, cont_handler);
	caught_cont = 1;
}

int
main(int argc, char *argv[])
{
	int ct;
	unsigned ti;
	int fp;
	time_t min;
	struct stat buf;
	char *fname;
	struct iodevinfo *iodev;
	off_t flength;
	hrtime_t start_n;
	hrtime_t period_n;


	ct = argc >= 3? atoi(argv[2]): 0;
	min = time((time_t *)0);
	ti = argc >= 3? atoi(argv[1]): 0;

	period_n = (hrtime_t)ti * NANOSEC;

	if ((kc = kstat_open()) == NULL)
		fail(1, "kstat_open(): can't open /dev/kstat");

	/* Set up handler for SIGCONT */
	if (signal(SIGCONT, cont_handler) == SIG_ERR)
		fail(1, "signal failed");

	all_stat_init();
	init_iodevs();

	if (argc == 3 || argc == 1) {
		/*
		 * no data file is specified, direct data to stdout.
		 */
		fp = 1;
	} else {
		struct flock lock;

		fname = (argc == 2) ? argv[1] : argv[3];
		/*
		 * Open or Create a data file. If the file doesn't exist, then
		 * it will be created.
		 */
		if ((fp = open(fname, O_WRONLY | O_APPEND | O_CREAT, 0644))
		    == -1)
			fail(1, "can't open data file");
		/*
		 * Lock the entire data file to prevent data corruption
		 */
		lock.l_type = F_WRLCK;
		lock.l_whence = SEEK_SET;
		lock.l_start = 0;
		lock.l_len = 0;
		if (fcntl(fp, F_SETLK, &lock) == -1)
			fail(1, "can't lock data file");
		/*
		 * Get data file statistics for use in determining whether
		 * truncation required and where rollback recovery should
		 * be applied.
		 */
		if (fstat(fp, &buf) == -1)
			fail(1, "can't get data file information");
		/*
		 * If the data file was opened and is too old, truncate it
		 */
		if (min - buf.st_mtime > 86400)
			if (ftruncate(fp, 0) == -1)
				fail(1, "can't truncate data file");
		/*
		 * Remember filesize for rollback on error (bug #1223549)
		 */
		flength = buf.st_size;
	}

	memset(&d, 0, sizeof (d));

	/*
	 * If n == 0, write the additional dummy record.
	 */
	if (ct == 0) {
		d.valid = 0;
		d.ts = min;
		d.niodevs = niodevs;

		if (write(fp, &d, sizeof (struct sa)) != sizeof (struct sa))
			ftruncate(fp, flength), fail(1, "write failed");

		for (iodev = firstiodev; iodev; iodev = iodev->next) {
			if (write(fp, iodev, sizeof (struct iodevinfo)) !=
			    sizeof (struct iodevinfo))
				ftruncate(fp, flength), fail(1, "write failed");
		}
	}

	start_n = gethrtime();

	for (;;) {
		do {
			(void) kstat_chain_update(kc);
			all_stat_init();
			init_iodevs();
		} while (all_stat_load() || iodevinfo_load());

		d.ts = time((time_t *)0);
		d.valid = 1;
		d.niodevs = niodevs;

		if (write(fp, &d, sizeof (struct sa)) != sizeof (struct sa))
			ftruncate(fp, flength), fail(1, "write failed");

		for (iodev = firstiodev; iodev; iodev = iodev->next) {
			if (write(fp, iodev, sizeof (struct iodevinfo)) !=
			    sizeof (struct iodevinfo))
				ftruncate(fp, flength), fail(1, "write failed");
		}
		if (--ct > 0) {
			sleep_until(&start_n, period_n, &caught_cont);
		} else {
			close(fp);
			return (0);
		}
	}

	/*NOTREACHED*/
}

/*
 * Get various KIDs for subsequent all_stat_load operations.
 */

static void
all_stat_init(void)
{
	kstat_t *ksp;

	/*
	 * Initialize global statistics
	 */

	sysinfo_ksp	= safe_kstat_lookup(kc, "unix", 0, "sysinfo");
	vminfo_ksp	= safe_kstat_lookup(kc, "unix", 0, "vminfo");
	kmem_oversize_ksp = safe_kstat_lookup(kc, "vmem", -1, "kmem_oversize");
	var_ksp		= safe_kstat_lookup(kc, "unix", 0, "var");
	system_misc_ksp	= safe_kstat_lookup(kc, "unix", 0, "system_misc");
	file_cache_ksp	= safe_kstat_lookup(kc, "unix", 0, "file_cache");
	ufs_inode_ksp	= kstat_lookup(kc, "ufs", 0, "inode_cache");

	safe_kstat_read(kc, system_misc_ksp, NULL);
	nproc_knp	= safe_kstat_data_lookup(system_misc_ksp, "nproc");

	safe_kstat_read(kc, file_cache_ksp, NULL);
	file_avail_knp = safe_kstat_data_lookup(file_cache_ksp, "buf_avail");
	file_total_knp = safe_kstat_data_lookup(file_cache_ksp, "buf_total");

	safe_kstat_read(kc, kmem_oversize_ksp, NULL);
	oversize_alloc_knp = safe_kstat_data_lookup(kmem_oversize_ksp,
	    "mem_total");
	oversize_fail_knp = safe_kstat_data_lookup(kmem_oversize_ksp, "fail");

	if (ufs_inode_ksp != NULL) {
		safe_kstat_read(kc, ufs_inode_ksp, NULL);
		ufs_inode_size_knp = safe_kstat_data_lookup(ufs_inode_ksp,
		    "size");
		ninode = ((kstat_named_t *)
		    safe_kstat_data_lookup(ufs_inode_ksp,
		    "maxsize"))->value.l;
	}

	/*
	 * Load constant values now -- no need to reread each time
	 */

	safe_kstat_read(kc, var_ksp, (void *) &var);

	/*
	 * Initialize per-CPU and per-kmem-cache statistics
	 */

	ncpus = ncaches = 0;
	for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) {
		if (strncmp(ksp->ks_name, "cpu_stat", 8) == 0)
			ncpus++;
		if (strcmp(ksp->ks_class, "kmem_cache") == 0)
			ncaches++;
	}

	safe_zalloc((void **)&cpu_stat_list, ncpus * sizeof (kstat_t *), 1);
	safe_zalloc((void **)&kmem_cache_list, ncaches * sizeof (kstat_t *), 1);

	ncpus = ncaches = 0;
	for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) {
		if (strncmp(ksp->ks_name, "cpu_stat", 8) == 0 &&
		    kstat_read(kc, ksp, NULL) != -1)
			cpu_stat_list[ncpus++] = ksp;
		if (strcmp(ksp->ks_class, "kmem_cache") == 0 &&
		    kstat_read(kc, ksp, NULL) != -1)
			kmem_cache_list[ncaches++] = ksp;
	}

	if (ncpus == 0)
		fail(1, "can't find any cpu statistics");

	if (ncaches == 0)
		fail(1, "can't find any kmem_cache statistics");

	ksp = kmem_cache_list[0];
	safe_kstat_read(kc, ksp, NULL);
	buf_size_index = safe_kstat_data_index(ksp, "buf_size");
	slab_create_index = safe_kstat_data_index(ksp, "slab_create");
	slab_destroy_index = safe_kstat_data_index(ksp, "slab_destroy");
	slab_size_index = safe_kstat_data_index(ksp, "slab_size");
	buf_avail_index = safe_kstat_data_index(ksp, "buf_avail");
	alloc_fail_index = safe_kstat_data_index(ksp, "alloc_fail");
}

/*
 * load statistics, summing across CPUs where needed
 */

static int
all_stat_load(void)
{
	int i, j;
	cpu_stat_t cs;
	ulong_t *np, *tp;
	uint64_t cpu_tick[4] = {0, 0, 0, 0};

	memset(&d, 0, sizeof (d));

	/*
	 * Global statistics
	 */

	safe_kstat_read(kc, sysinfo_ksp, (void *) &d.si);
	safe_kstat_read(kc, vminfo_ksp, (void *) &d.vmi);
	safe_kstat_read(kc, system_misc_ksp, NULL);
	safe_kstat_read(kc, file_cache_ksp, NULL);

	if (ufs_inode_ksp != NULL) {
		safe_kstat_read(kc, ufs_inode_ksp, NULL);
		d.szinode = ufs_inode_size_knp->value.ul;
	}

	d.szfile = file_total_knp->value.ui64 - file_avail_knp->value.ui64;
	d.szproc = nproc_knp->value.ul;

	d.mszinode = (ninode > d.szinode) ? ninode : d.szinode;
	d.mszfile = d.szfile;
	d.mszproc = var.v_proc;

	/*
	 * Per-CPU statistics.
	 */

	for (i = 0; i < ncpus; i++) {
		if (kstat_read(kc, cpu_stat_list[i], (void *) &cs) == -1)
			return (1);

		np = (ulong_t *)&d.csi;
		tp = (ulong_t *)&cs.cpu_sysinfo;

		/*
		 * Accumulate cpu ticks for CPU_IDLE, CPU_USER, CPU_KERNEL and
		 * CPU_WAIT with respect to each of the cpus.
		 */
		for (j = 0; j < CPU_STATES; j++)
			cpu_tick[j] += tp[j];

		for (j = 0; j < sizeof (cpu_sysinfo_t); j += sizeof (ulong_t))
			*np++ += *tp++;
		np = (ulong_t *)&d.cvmi;
		tp = (ulong_t *)&cs.cpu_vminfo;
		for (j = 0; j < sizeof (cpu_vminfo_t); j += sizeof (ulong_t))
			*np++ += *tp++;
	}

	/*
	 * Per-cache kmem statistics.
	 */

	for (i = 0; i < ncaches; i++) {
		kstat_named_t *knp;
		u_longlong_t slab_create, slab_destroy, slab_size, mem_total;
		u_longlong_t buf_size, buf_avail, alloc_fail;
		int kmi_index;

		if (kstat_read(kc, kmem_cache_list[i], NULL) == -1)
			return (1);
		knp = kmem_cache_list[i]->ks_data;
		slab_create	= knp[slab_create_index].value.ui64;
		slab_destroy	= knp[slab_destroy_index].value.ui64;
		slab_size	= knp[slab_size_index].value.ui64;
		buf_size	= knp[buf_size_index].value.ui64;
		buf_avail	= knp[buf_avail_index].value.ui64;
		alloc_fail	= knp[alloc_fail_index].value.ui64;
		if (buf_size <= 256)
			kmi_index = KMEM_SMALL;
		else
			kmi_index = KMEM_LARGE;
		mem_total = (slab_create - slab_destroy) * slab_size;

		d.kmi.km_mem[kmi_index] += (ulong_t)mem_total;
		d.kmi.km_alloc[kmi_index] +=
		    (ulong_t)mem_total - buf_size * buf_avail;
		d.kmi.km_fail[kmi_index] += (ulong_t)alloc_fail;
	}

	safe_kstat_read(kc, kmem_oversize_ksp, NULL);

	d.kmi.km_alloc[KMEM_OSIZE] = d.kmi.km_mem[KMEM_OSIZE] =
	    oversize_alloc_knp->value.ui64;
	d.kmi.km_fail[KMEM_OSIZE] = oversize_fail_knp->value.ui64;

	/*
	 * Adjust CPU statistics so the delta calculations in sar will
	 * be correct when facing changes to the set of online CPUs.
	 */
	compute_cpu_stat_adj();
	for (i = 0; i < CPU_STATES; i++)
		d.csi.cpu[i] = (cpu_tick[i] + cpu_stat_adj[i]) / ncpus;

	return (0);
}

static void
fail(int do_perror, char *message, ...)
{
	va_list args;

	va_start(args, message);
	fprintf(stderr, "%s: ", cmdname);
	vfprintf(stderr, message, args);
	va_end(args);
	if (do_perror)
		fprintf(stderr, ": %s", strerror(errno));
	fprintf(stderr, "\n");
	exit(2);
}

static void
safe_zalloc(void **ptr, int size, int free_first)
{
	if (free_first && *ptr != NULL)
		free(*ptr);
	if ((*ptr = malloc(size)) == NULL)
		fail(1, "malloc failed");
	memset(*ptr, 0, size);
}

static kid_t
safe_kstat_read(kstat_ctl_t *kc, kstat_t *ksp, void *data)
{
	kid_t kstat_chain_id = kstat_read(kc, ksp, data);

	if (kstat_chain_id == -1)
		fail(1, "kstat_read(%x, '%s') failed", kc, ksp->ks_name);
	return (kstat_chain_id);
}

static kstat_t *
safe_kstat_lookup(kstat_ctl_t *kc, char *ks_module, int ks_instance,
	char *ks_name)
{
	kstat_t *ksp = kstat_lookup(kc, ks_module, ks_instance, ks_name);

	if (ksp == NULL)
		fail(0, "kstat_lookup('%s', %d, '%s') failed",
		    ks_module == NULL ? "" : ks_module,
		    ks_instance,
		    ks_name == NULL ? "" : ks_name);
	return (ksp);
}

static void *
safe_kstat_data_lookup(kstat_t *ksp, char *name)
{
	void *fp = kstat_data_lookup(ksp, name);

	if (fp == NULL)
		fail(0, "kstat_data_lookup('%s', '%s') failed",
		    ksp->ks_name, name);
	return (fp);
}

static int
safe_kstat_data_index(kstat_t *ksp, char *name)
{
	return ((int)((char *)safe_kstat_data_lookup(ksp, name) -
	    (char *)ksp->ks_data) / (ksp->ks_data_size / ksp->ks_ndata));
}

static int
kscmp(kstat_t *ks1, kstat_t *ks2)
{
	int cmp;

	cmp = strcmp(ks1->ks_module, ks2->ks_module);
	if (cmp != 0)
		return (cmp);
	cmp = ks1->ks_instance - ks2->ks_instance;
	if (cmp != 0)
		return (cmp);
	return (strcmp(ks1->ks_name, ks2->ks_name));
}

static void
init_iodevs(void)
{
	struct iodevinfo *iodev, *previodev, *comp;
	kstat_t *ksp;

	iodev = &zeroiodev;
	niodevs = 0;

	/*
	 * Patch the snip in the iodevinfo list (see below)
	 */
	if (snip)
		lastiodev->next = snip;

	for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) {

		if (ksp->ks_type != KSTAT_TYPE_IO)
			continue;
		previodev = iodev;
		if (iodev->next)
			iodev = iodev->next;
		else {
			safe_zalloc((void **) &iodev->next,
			    sizeof (struct iodevinfo), 0);
			iodev = iodev->next;
			iodev->next = NULL;
		}
		iodev->ksp = ksp;
		iodev->ks = *ksp;
		memset((void *)&iodev->kios, 0, sizeof (kstat_io_t));
		iodev->kios.wlastupdate = iodev->ks.ks_crtime;
		iodev->kios.rlastupdate = iodev->ks.ks_crtime;

		/*
		 * Insertion sort on (ks_module, ks_instance, ks_name)
		 */
		comp = &zeroiodev;
		while (kscmp(&iodev->ks, &comp->next->ks) > 0)
			comp = comp->next;
		if (previodev != comp) {
			previodev->next = iodev->next;
			iodev->next = comp->next;
			comp->next = iodev;
			iodev = previodev;
		}
		niodevs++;
	}
	/*
	 * Put a snip in the linked list of iodevinfos.  The idea:
	 * If there was a state change such that now there are fewer
	 * iodevs, we snip the list and retain the tail, rather than
	 * freeing it.  At the next state change, we clip the tail back on.
	 * This prevents a lot of malloc/free activity, and it's simpler.
	 */
	lastiodev = iodev;
	snip = iodev->next;
	iodev->next = NULL;

	firstiodev = zeroiodev.next;
}

static int
iodevinfo_load(void)
{
	struct iodevinfo *iodev;

	for (iodev = firstiodev; iodev; iodev = iodev->next) {
		if (kstat_read(kc, iodev->ksp, (void *) &iodev->kios) == -1)
			return (1);
	}
	return (0);
}

static int
kstat_copy(const kstat_t *src, kstat_t *dst)
{
	*dst = *src;

	if (src->ks_data != NULL) {
		if ((dst->ks_data = malloc(src->ks_data_size)) == NULL)
			return (-1);
		bcopy(src->ks_data, dst->ks_data, src->ks_data_size);
	} else {
		dst->ks_data = NULL;
		dst->ks_data_size = 0;
	}
	return (0);
}

/*
 * Determine what is different between two sets of kstats; s[0] and s[1]
 * are arrays of kstats of size ns0 and ns1, respectively, and sorted by
 * instance number.  u[0] and u[1] are two arrays which must be
 * caller-zallocated; each must be of size MAX(ns0, ns1).  When the
 * function terminates, u[0] contains all s[0]-unique items and u[1]
 * contains all s[1]-unique items.  Any unused entries in u[0] and u[1]
 * are left NULL.
 */
static void
diff_two_arrays(kstat_t ** const s[], size_t ns0, size_t ns1,
    kstat_t ** const u[])
{
	kstat_t **s0p = s[0], **s1p = s[1];
	kstat_t **u0p = u[0], **u1p = u[1];
	int i = 0, j = 0;

	while (i < ns0 && j < ns1) {
		if ((*s0p)->ks_instance == (*s1p)->ks_instance) {
			if ((*s0p)->ks_kid != (*s1p)->ks_kid) {
				/*
				 * The instance is the same, but this
				 * CPU has been offline during the
				 * interval, so we consider *u0p to
				 * be s0p-unique, and similarly for
				 * *u1p.
				 */
				*(u0p++) = *s0p;
				*(u1p++) = *s1p;
			}
			s0p++;
			i++;
			s1p++;
			j++;
		} else if ((*s0p)->ks_instance < (*s1p)->ks_instance) {
			*(u0p++) = *(s0p++);
			i++;
		} else {
			*(u1p++) = *(s1p++);
			j++;
		}
	}

	while (i < ns0) {
		*(u0p++) = *(s0p++);
		i++;
	}
	while (j < ns1) {
		*(u1p++) = *(s1p++);
		j++;
	}
}

static int
cpuid_compare(const void *p1, const void *p2)
{
	return ((*(kstat_t **)p1)->ks_instance -
	    (*(kstat_t **)p2)->ks_instance);
}

/*
 * Identify those CPUs which were not present for the whole interval so
 * their statistics can be removed from the aggregate.
 */
static void
compute_cpu_stat_adj(void)
{
	int i, j;

	if (ocpu_stat_list) {
		kstat_t **s[2];
		kstat_t **inarray[2];
		int max_cpus = MAX(ncpus, oncpus);

		qsort(cpu_stat_list, ncpus, sizeof (*cpu_stat_list),
		    cpuid_compare);
		qsort(ocpu_stat_list, oncpus, sizeof (*ocpu_stat_list),
		    cpuid_compare);

		s[0] = ocpu_stat_list;
		s[1] = cpu_stat_list;

		safe_zalloc((void *)&inarray[0], sizeof (**inarray) * max_cpus,
		    0);
		safe_zalloc((void *)&inarray[1], sizeof (**inarray) * max_cpus,
		    0);
		diff_two_arrays(s, oncpus, ncpus, inarray);

		for (i = 0; i < max_cpus; i++) {
			if (inarray[0][i])
				for (j = 0; j < CPU_STATES; j++)
					cpu_stat_adj[j] +=
					    ((cpu_stat_t *)inarray[0][i]
					    ->ks_data)->cpu_sysinfo.cpu[j];
			if (inarray[1][i])
				for (j = 0; j < CPU_STATES; j++)
					cpu_stat_adj[j] -=
					    ((cpu_stat_t *)inarray[1][i]
					    ->ks_data)->cpu_sysinfo.cpu[j];
		}

		free(inarray[0]);
		free(inarray[1]);
	}

	/*
	 * Preserve the last interval's CPU stats.
	 */
	if (cpu_stat_list) {
		for (i = 0; i < oncpus; i++)
			free(ocpu_stat_list[i]->ks_data);

		oncpus = ncpus;
		safe_zalloc((void **)&ocpu_stat_list, oncpus *
		    sizeof (*ocpu_stat_list), 1);
		for (i = 0; i < ncpus; i++) {
			safe_zalloc((void *)&ocpu_stat_list[i],
			    sizeof (*ocpu_stat_list[0]), 0);
			if (kstat_copy(cpu_stat_list[i], ocpu_stat_list[i]))
				fail(1, "kstat_copy() failed");
		}
	}
}