xref: /illumos-gate/usr/src/cmd/sa/sar.c (revision 96d9f183facd90dbbc2268c9a51689be0b6a0b46)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
27 /*	  All Rights Reserved  	*/
28 
29 
30 /*
31  * sar generates a report either from an input data file or by invoking sadc to
32  * read system activity counters at the specified intervals.
33  *
34  * usage:  sar [-ubdycwaqvmpgrkA] [-o file] t [n]
35  *	   sar [-ubdycwaqvmpgrkA][-s hh:mm][-e hh:mm][-i ss][-f file]
36  */
37 
38 #include <sys/param.h>
39 #include <sys/stat.h>
40 #include <sys/sysinfo.h>
41 #include <sys/time.h>
42 #include <sys/types.h>
43 #include <sys/utsname.h>
44 #include <sys/wait.h>
45 
46 #include <ctype.h>
47 #include <errno.h>
48 #include <fcntl.h>
49 #include <limits.h>
50 #include <signal.h>
51 #include <stdarg.h>
52 #include <stdio.h>
53 #include <stdlib.h>
54 #include <string.h>
55 #include <time.h>
56 #include <unistd.h>
57 
58 #include "sa.h"
59 
60 #define	PGTOBLK(x)	((x) * (pagesize >> 9))
61 #define	BLKTOPG(x)	((x) / (pagesize >> 9))
62 #define	BLKS(x)		((x) >> 9)
63 
64 static void	prpass(int);
65 static void	prtopt(void);
66 static void	prtavg(void);
67 static void	prttim(void);
68 static void	prtmachid(void);
69 static void	prthdg(void);
70 static void	tsttab(void);
71 static void	update_counters(void);
72 static void	usage(void);
73 static void	fail(int, char *, ...);
74 static void	safe_zalloc(void **, int, int);
75 static int	safe_read(int, void *, size_t);
76 static void	safe_write(int, void *, size_t);
77 static int	safe_strtoi(char const *, char *);
78 static void	ulong_delta(uint64_t *, uint64_t *, uint64_t *, uint64_t *,
79 	int, int);
80 static float	denom(float);
81 static float	freq(float, float);
82 
83 static struct sa64	nx, ox, ax, dx;
84 static iodevinfo_t	*nxio, *oxio, *axio, *dxio;
85 static struct tm	*curt, args, arge;
86 
87 static int	sflg, eflg, iflg, oflg, fflg;
88 static int	realtime, passno = 0, do_disk;
89 static int	t = 0, n = 0, lines = 0;
90 static int	hz;
91 static int	niodevs;
92 static int	tabflg;
93 static char	options[30], fopt[30];
94 static float	tdiff, sec_diff, totsec_diff = 0.0, percent;
95 static float	start_time, end_time, isec;
96 static int 	fin, fout;
97 static pid_t	childid;
98 static int	pipedes[2];
99 static char	arg1[10], arg2[10];
100 static int	pagesize;
101 
102 /*
103  * To avoid overflow in the kmem allocation data, declare a copy of the
104  * main kmeminfo_t type with larger data types. Use this for storing
105  * the data held to display average values
106  */
107 static struct kmeminfo_l
108 {
109 	u_longlong_t	km_mem[KMEM_NCLASS];
110 	u_longlong_t	km_alloc[KMEM_NCLASS];
111 	u_longlong_t	km_fail[KMEM_NCLASS];
112 } kmi;
113 
114 int
115 main(int argc, char **argv)
116 {
117 	char    flnm[PATH_MAX], ofile[PATH_MAX];
118 	char	ccc;
119 	time_t	temp;
120 	int	i, jj = 0;
121 
122 	pagesize = sysconf(_SC_PAGESIZE);
123 
124 	/*
125 	 * Process options with arguments and pack options
126 	 * without arguments.
127 	 */
128 	while ((i = getopt(argc, argv, "ubdycwaqvmpgrkAo:s:e:i:f:")) != EOF)
129 		switch (ccc = (char)i) {
130 		    case 'o':
131 			oflg++;
132 			if (strlcpy(ofile, optarg, sizeof (ofile)) >=
133 			    sizeof (ofile)) {
134 				fail(2, "-o filename is too long: %s", optarg);
135 			}
136 			break;
137 		    case 's':
138 			if (sscanf(optarg, "%d:%d:%d",
139 			    &args.tm_hour, &args.tm_min, &args.tm_sec) < 1)
140 				fail(0, "-%c %s -- illegal option argument",
141 				    ccc, optarg);
142 			else {
143 				sflg++,
144 				start_time = args.tm_hour*3600.0 +
145 				    args.tm_min*60.0 +
146 				    args.tm_sec;
147 			}
148 			break;
149 		    case 'e':
150 			if (sscanf(optarg, "%d:%d:%d",
151 			    &arge.tm_hour, &arge.tm_min, &arge.tm_sec) < 1)
152 				fail(0, "-%c %s -- illegal option argument",
153 				    ccc, optarg);
154 			else {
155 				eflg++;
156 				end_time = arge.tm_hour*3600.0 +
157 				    arge.tm_min*60.0 +
158 				    arge.tm_sec;
159 			}
160 			break;
161 		    case 'i':
162 			if (sscanf(optarg, "%f", &isec) < 1)
163 				fail(0, "-%c %s -- illegal option argument",
164 				    ccc, optarg);
165 			else {
166 				if (isec > 0.0)
167 					iflg++;
168 			}
169 			break;
170 		    case 'f':
171 			fflg++;
172 			if (strlcpy(flnm, optarg, sizeof (flnm)) >=
173 			    sizeof (ofile)) {
174 				fail(2, "-f filename is too long: %s", optarg);
175 			}
176 			break;
177 		    case '?':
178 			usage();
179 			exit(1);
180 			break;
181 		default:
182 
183 			/*
184 			 * Check for repeated options. To make sure
185 			 * that options[30] does not overflow.
186 			 */
187 			if (strchr(options, ccc) == NULL)
188 				(void) strncat(options, &ccc, 1);
189 			break;
190 		}
191 
192 	/*
193 	 * Are starting and ending times consistent?
194 	 */
195 	if ((sflg) && (eflg) && (end_time <= start_time))
196 		fail(0, "ending time <= starting time");
197 
198 	/*
199 	 * Determine if t and n arguments are given, and whether to run in real
200 	 * time or from a file.
201 	 */
202 	switch (argc - optind) {
203 	    case 0:		/*   Get input data from file   */
204 		if (fflg == 0) {
205 			temp = time(NULL);
206 			curt = localtime(&temp);
207 			(void) snprintf(flnm, PATH_MAX, "/var/adm/sa/sa%.2d",
208 			    curt->tm_mday);
209 		}
210 		if ((fin = open(flnm, 0)) == -1)
211 			fail(1, "can't open %s", flnm);
212 		break;
213 	    case 1:		/*   Real time data; one cycle   */
214 		realtime++;
215 		t = safe_strtoi(argv[optind], "invalid sampling interval");
216 		n = 2;
217 		break;
218 	    case 2:		/*   Real time data; specified cycles   */
219 	default:
220 		realtime++;
221 		t = safe_strtoi(argv[optind], "invalid sampling interval");
222 		n = 1 + safe_strtoi(argv[optind+1], "invalid sample count");
223 		break;
224 	}
225 
226 	/*
227 	 * "u" is the default option, which displays CPU utilization.
228 	 */
229 	if (strlen(options) == 0)
230 		(void) strcpy(options, "u");
231 
232 	/*
233 	 * "A" means all data options.
234 	 */
235 	if (strchr(options, 'A') != NULL)
236 		(void) strcpy(options, "udqbwcayvmpgrk");
237 
238 	if (realtime) {
239 		/*
240 		 * Get input data from sadc via pipe.
241 		 */
242 		if (t <= 0)
243 			fail(0, "sampling interval t <= 0 sec");
244 		if (n < 2)
245 			fail(0, "number of sample intervals n <= 0");
246 		(void) sprintf(arg1, "%d", t);
247 		(void) sprintf(arg2, "%d", n);
248 		if (pipe(pipedes) == -1)
249 			fail(1, "pipe failed");
250 		if ((childid = fork()) == 0) {
251 			/*
252 			 * Child:  shift pipedes[write] to stdout,
253 			 * and close the pipe entries.
254 			 */
255 			(void) dup2(pipedes[1], 1);
256 			if (pipedes[0] != 1)
257 				(void) close(pipedes[0]);
258 			if (pipedes[1] != 1)
259 				(void) close(pipedes[1]);
260 
261 			if (execlp("/usr/lib/sa/sadc",
262 			    "/usr/lib/sa/sadc", arg1, arg2, 0) == -1)
263 				fail(1, "exec of /usr/lib/sa/sadc failed");
264 		} else if (childid == -1) {
265 			fail(1, "Could not fork to exec sadc");
266 		}
267 		/*
268 		 * Parent:  close unused output.
269 		 */
270 		fin = pipedes[0];
271 		(void) close(pipedes[1]);
272 	}
273 
274 	if (oflg) {
275 		if (strcmp(ofile, flnm) == 0)
276 			fail(0, "output file name same as input file name");
277 		fout = creat(ofile, 00644);
278 	}
279 
280 	hz = sysconf(_SC_CLK_TCK);
281 
282 	nxio = oxio = dxio = axio = NULL;
283 
284 	if (realtime) {
285 		/*
286 		 * Make single pass, processing all options.
287 		 */
288 		(void) strcpy(fopt, options);
289 		passno++;
290 		prpass(realtime);
291 		(void) kill(childid, SIGINT);
292 		(void) wait(NULL);
293 	} else {
294 		/*
295 		 * Make multiple passes, one for each option.
296 		 */
297 		while (strlen(strncpy(fopt, &options[jj++], 1))) {
298 			if (lseek(fin, 0, SEEK_SET) == (off_t)-1)
299 				fail(0, "lseek failed");
300 			passno++;
301 			prpass(realtime);
302 		}
303 	}
304 
305 	return (0);
306 }
307 
308 /*
309  * Convert array of 32-bit uints to 64-bit uints
310  */
311 static void
312 convert_32to64(uint64_t *dst, uint_t *src, int size)
313 {
314 	for (; size > 0; size--)
315 		*dst++ = (uint64_t)(*src++);
316 }
317 
318 /*
319  * Convert array of 64-bit uints to 32-bit uints
320  */
321 static void
322 convert_64to32(uint_t *dst, uint64_t *src, int size)
323 {
324 	for (; size > 0; size--)
325 		*dst++ = (uint32_t)(*src++);
326 }
327 
328 /*
329  * Read records from input, classify, and decide on printing.
330  */
331 static void
332 prpass(int input_pipe)
333 {
334 	size_t size;
335 	int i, j, state_change, recno = 0;
336 	kid_t kid;
337 	float trec, tnext = 0;
338 	ulong_t old_niodevs = 0, prev_niodevs = 0;
339 	iodevinfo_t *aio, *dio, *oio;
340 	struct stat in_stat;
341 	struct sa tx;
342 	uint64_t ts, te; /* time interval start and end */
343 
344 	do_disk = (strchr(fopt, 'd') != NULL);
345 	if (!input_pipe && fstat(fin, &in_stat) == -1)
346 		fail(1, "unable to stat data file");
347 
348 	if (sflg)
349 		tnext = start_time;
350 
351 	while (safe_read(fin, &tx, sizeof (struct sa))) {
352 		/*
353 		 * First, we convert 32-bit tx to 64-bit nx structure
354 		 * which is used later. Conversion could be done
355 		 * after initial operations, right before calculations,
356 		 * but it would introduce additional juggling with vars.
357 		 * Thus, we convert all data now, and don't care about
358 		 * tx any further.
359 		 */
360 		nx.valid = tx.valid;
361 		nx.ts = tx.ts;
362 		convert_32to64((uint64_t *)&nx.csi, (uint_t *)&tx.csi,
363 		    sizeof (tx.csi) / sizeof (uint_t));
364 		convert_32to64((uint64_t *)&nx.cvmi, (uint_t *)&tx.cvmi,
365 		    sizeof (tx.cvmi) / sizeof (uint_t));
366 		convert_32to64((uint64_t *)&nx.si, (uint_t *)&tx.si,
367 		    sizeof (tx.si) / sizeof (uint_t));
368 		(void) memcpy(&nx.vmi, &tx.vmi,
369 		    sizeof (tx) - (((char *)&tx.vmi) - ((char *)&tx)));
370 		/*
371 		 * sadc is the only utility used to generate sar data
372 		 * and it uses the valid field as follows:
373 		 * 0 - dummy record
374 		 * 1 - data record
375 		 * We can use this fact to improve sar's ability to detect
376 		 * bad data, since any value apart from 0 or 1 can be
377 		 * interpreted as invalid data.
378 		 */
379 		if (nx.valid != 0 && nx.valid != 1)
380 			fail(2, "data file not in sar format");
381 		state_change = 0;
382 		niodevs = nx.niodevs;
383 		/*
384 		 * niodevs has the value of current number of devices
385 		 * from nx structure.
386 		 *
387 		 * The following 'if' condition is to decide whether memory
388 		 * has to be allocated or not if already allocated memory is
389 		 * bigger or smaller than memory needed to store the current
390 		 * niodevs details in memory.
391 		 *
392 		 * when first while loop starts, pre_niodevs has 0 and then
393 		 * always get initialized to the current number of devices
394 		 * from nx.niodevs if it is different from previously read
395 		 * niodevs.
396 		 *
397 		 * if the current niodevs has the same value of previously
398 		 * allocated memory i.e, for prev_niodevs, it skips the
399 		 * following  'if' loop or otherwise it allocates memory for
400 		 * current devises (niodevs) and stores that value in
401 		 * prev_niodevs for next time when loop continues to read
402 		 * from the file.
403 		 */
404 		if (niodevs != prev_niodevs) {
405 			off_t curr_pos;
406 			/*
407 			 * The required buffer size must fit in a size_t.
408 			 */
409 			if (SIZE_MAX / sizeof (iodevinfo_t) < niodevs)
410 				fail(2, "insufficient address space to hold "
411 				    "%lu device records", niodevs);
412 			size = niodevs * sizeof (iodevinfo_t);
413 			prev_niodevs = niodevs;
414 			/*
415 			 * The data file must exceed this size to be valid.
416 			 */
417 			if (!input_pipe) {
418 			    if ((curr_pos = lseek(fin, 0, SEEK_CUR)) ==
419 				(off_t)-1)
420 				    fail(1, "lseek failed");
421 			    if (in_stat.st_size < curr_pos ||
422 				size > in_stat.st_size - curr_pos)
423 				    fail(2, "data file corrupt; specified size"
424 					"exceeds actual");
425 			}
426 
427 			safe_zalloc((void **)&nxio, size, 1);
428 		}
429 		if (niodevs != old_niodevs)
430 			state_change = 1;
431 		for (i = 0; i < niodevs; i++) {
432 			if (safe_read(fin, &nxio[i], sizeof (iodevinfo_t)) == 0)
433 				fail(1, "premature end-of-file seen");
434 			if (i < old_niodevs &&
435 			    nxio[i].ks.ks_kid != oxio[i].ks.ks_kid)
436 				state_change = 1;
437 		}
438 		curt = localtime(&nx.ts);
439 		trec = curt->tm_hour * 3600.0 +
440 		    curt->tm_min * 60.0 +
441 		    curt->tm_sec;
442 		if ((recno == 0) && (trec < start_time))
443 			continue;
444 		if ((eflg) && (trec > end_time))
445 			break;
446 		if ((oflg) && (passno == 1)) {
447 			/*
448 			 * The calculated values are stroed in nx strcuture.
449 			 * Convert 64-bit nx to 32-bit tx structure.
450 			 */
451 			tx.valid = nx.valid;
452 			tx.ts = nx.ts;
453 			convert_64to32((uint_t *)&tx.csi, (uint64_t *)&nx.csi,
454 			    sizeof (nx.csi) / sizeof (uint64_t));
455 			convert_64to32((uint_t *)&tx.cvmi, (uint64_t *)&nx.cvmi,
456 			    sizeof (nx.cvmi) / sizeof (uint64_t));
457 			convert_64to32((uint_t *)&tx.si, (uint64_t *)&nx.si,
458 			    sizeof (nx.si) / sizeof (uint64_t));
459 			(void) memcpy(&tx.vmi, &nx.vmi,
460 			    sizeof (nx) - (((char *)&nx.vmi) - ((char *)&nx)));
461 			if (tx.valid != 0 && tx.valid != 1)
462 				fail(2, "data file not in sar format");
463 
464 			safe_write(fout, &tx, sizeof (struct sa));
465 			for (i = 0; i < niodevs; i++)
466 				safe_write(fout, &nxio[i],
467 				    sizeof (iodevinfo_t));
468 		}
469 
470 		if (recno == 0) {
471 			if (passno == 1)
472 				prtmachid();
473 
474 			prthdg();
475 			recno = 1;
476 			if ((iflg) && (tnext == 0))
477 				tnext = trec;
478 		}
479 
480 		if (nx.valid == 0) {
481 			/*
482 			 * This dummy record signifies system restart
483 			 * New initial values of counters follow in next
484 			 * record.
485 			 */
486 			if (!realtime) {
487 				prttim();
488 				(void) printf("\tunix restarts\n");
489 				recno = 1;
490 				continue;
491 			}
492 		}
493 		if ((iflg) && (trec < tnext))
494 			continue;
495 
496 		if (state_change) {
497 			/*
498 			 * Either the number of devices or the ordering of
499 			 * the kstats has changed.  We need to re-organise
500 			 * the layout of our avg/delta arrays so that we
501 			 * can cope with this in update_counters().
502 			 */
503 			size = niodevs * sizeof (iodevinfo_t);
504 			safe_zalloc((void *)&aio, size, 0);
505 			safe_zalloc((void *)&dio, size, 0);
506 			safe_zalloc((void *)&oio, size, 0);
507 
508 			/*
509 			 * Loop through all the newly read iodev's, locate
510 			 * the corresponding entry in the old arrays and
511 			 * copy the entries into the same bucket of the
512 			 * new arrays.
513 			 */
514 			for (i = 0; i < niodevs; i++) {
515 				kid = nxio[i].ks.ks_kid;
516 				for (j = 0; j < old_niodevs; j++) {
517 					if (oxio[j].ks.ks_kid == kid) {
518 						oio[i] = oxio[j];
519 						aio[i] = axio[j];
520 						dio[i] = dxio[j];
521 					}
522 				}
523 			}
524 
525 			free(axio);
526 			free(oxio);
527 			free(dxio);
528 
529 			axio = aio;
530 			oxio = oio;
531 			dxio = dio;
532 
533 			old_niodevs = niodevs;
534 		}
535 
536 		if (recno++ > 1) {
537 			ts = ox.csi.cpu[0] + ox.csi.cpu[1] +
538 				ox.csi.cpu[2] + ox.csi.cpu[3];
539 			te = nx.csi.cpu[0] + nx.csi.cpu[1] +
540 				nx.csi.cpu[2] + nx.csi.cpu[3];
541 			tdiff = (float)(te - ts);
542 			sec_diff = tdiff / hz;
543 			percent = 100.0 / tdiff;
544 
545 			/*
546 			 * If the CPU stat counters have rolled
547 			 * backward, this is our best indication that
548 			 * a CPU has been offlined.  We don't have
549 			 * enough data to compute a sensible delta, so
550 			 * toss out this interval, but compute the next
551 			 * interval's delta from these values.
552 			 */
553 			if (tdiff <= 0) {
554 				ox = nx;
555 				continue;
556 			}
557 			update_counters();
558 			prtopt();
559 			lines++;
560 			if (passno == 1)
561 				totsec_diff += sec_diff;
562 		}
563 		ox = nx;		/*  Age the data	*/
564 		(void) memcpy(oxio, nxio, niodevs * sizeof (iodevinfo_t));
565 		if (isec > 0)
566 			while (tnext <= trec)
567 				tnext += isec;
568 	}
569 	/*
570 	 * After this place, all functions are using niodevs to access the
571 	 * memory for device details. Here, old_niodevs has the correct value
572 	 * of memory allocated for storing device information. Since niodevs
573 	 * doesn't have correct value, sometimes, it was corrupting memory.
574 	 */
575 	niodevs = old_niodevs;
576 	if (lines > 1)
577 		prtavg();
578 	(void) memset(&ax, 0, sizeof (ax));	/* Zero out the accumulators. */
579 	(void) memset(&kmi, 0, sizeof (kmi));
580 	lines = 0;
581 	/*
582 	 * axio will not be allocated if the user specified -e or -s, and
583 	 * no records in the file fell inside the specified time range.
584 	 */
585 	if (axio) {
586 		(void) memset(axio, 0, niodevs * sizeof (iodevinfo_t));
587 	}
588 }
589 
590 /*
591  * Print time label routine.
592  */
593 static void
594 prttim(void)
595 {
596 	curt = localtime(&nx.ts);
597 	(void) printf("%.2d:%.2d:%.2d", curt->tm_hour, curt->tm_min,
598 	    curt->tm_sec);
599 	tabflg = 1;
600 }
601 
602 /*
603  * Test if 8-spaces to be added routine.
604  */
605 static void
606 tsttab(void)
607 {
608 	if (tabflg == 0)
609 		(void) printf("        ");
610 	else
611 		tabflg = 0;
612 }
613 
614 /*
615  * Print machine identification.
616  */
617 static void
618 prtmachid(void)
619 {
620 	struct utsname name;
621 
622 	(void) uname(&name);
623 	(void) printf("\n%s %s %s %s %s    %.2d/%.2d/%.4d\n",
624 	    name.sysname, name.nodename, name.release, name.version,
625 	    name.machine, curt->tm_mon + 1, curt->tm_mday,
626 	    curt->tm_year + 1900);
627 }
628 
629 /*
630  * Print report heading routine.
631  */
632 static void
633 prthdg(void)
634 {
635 	int	jj = 0;
636 	char	ccc;
637 
638 	(void) printf("\n");
639 	prttim();
640 	while ((ccc = fopt[jj++]) != NULL) {
641 		tsttab();
642 		switch (ccc) {
643 		    case 'u':
644 			(void) printf(" %7s %7s %7s %7s\n",
645 				"%usr",
646 				"%sys",
647 				"%wio",
648 				"%idle");
649 			break;
650 		    case 'b':
651 			(void) printf(" %7s %7s %7s %7s %7s %7s %7s %7s\n",
652 				"bread/s",
653 				"lread/s",
654 				"%rcache",
655 				"bwrit/s",
656 				"lwrit/s",
657 				"%wcache",
658 				"pread/s",
659 				"pwrit/s");
660 			break;
661 		    case 'd':
662 			(void) printf("   %-8.8s    %7s %7s %7s %7s %7s %7s\n",
663 				"device",
664 				"%busy",
665 				"avque",
666 				"r+w/s",
667 				"blks/s",
668 				"avwait",
669 				"avserv");
670 			break;
671 		    case 'y':
672 			(void) printf(" %7s %7s %7s %7s %7s %7s\n",
673 				"rawch/s",
674 				"canch/s",
675 				"outch/s",
676 				"rcvin/s",
677 				"xmtin/s",
678 				"mdmin/s");
679 			break;
680 		    case 'c':
681 			(void) printf(" %7s %7s %7s %7s %7s %7s %7s\n",
682 				"scall/s",
683 				"sread/s",
684 				"swrit/s",
685 				"fork/s",
686 				"exec/s",
687 				"rchar/s",
688 				"wchar/s");
689 			break;
690 		    case 'w':
691 			(void) printf(" %7s %7s %7s %7s %7s\n",
692 				"swpin/s",
693 				"bswin/s",
694 				"swpot/s",
695 				"bswot/s",
696 				"pswch/s");
697 			break;
698 		    case 'a':
699 			(void) printf(" %7s %7s %7s\n",
700 				"iget/s",
701 				"namei/s",
702 				"dirbk/s");
703 			break;
704 		    case 'q':
705 			(void) printf(" %7s %7s %7s %7s\n",
706 				"runq-sz",
707 				"%runocc",
708 				"swpq-sz",
709 				"%swpocc");
710 			break;
711 		    case 'v':
712 			(void) printf("  %s  %s  %s   %s\n",
713 				"proc-sz    ov",
714 				"inod-sz    ov",
715 				"file-sz    ov",
716 				"lock-sz");
717 			break;
718 		    case 'm':
719 			(void) printf(" %7s %7s\n",
720 				"msg/s",
721 				"sema/s");
722 			break;
723 		    case 'p':
724 			(void) printf(" %7s %7s %7s %7s %7s %7s\n",
725 				"atch/s",
726 				"pgin/s",
727 				"ppgin/s",
728 				"pflt/s",
729 				"vflt/s",
730 				"slock/s");
731 			break;
732 		    case 'g':
733 			(void) printf(" %8s %8s %8s %8s %8s\n",
734 				"pgout/s",
735 				"ppgout/s",
736 				"pgfree/s",
737 				"pgscan/s",
738 				"%ufs_ipf");
739 			break;
740 		    case 'r':
741 			(void) printf(" %7s %8s\n",
742 				"freemem",
743 				"freeswap");
744 			break;
745 		    case 'k':
746 			(void) printf(" %7s %7s %5s %7s %7s %5s %11s %5s\n",
747 				"sml_mem",
748 				"alloc",
749 				"fail",
750 				"lg_mem",
751 				"alloc",
752 				"fail",
753 				"ovsz_alloc",
754 				"fail");
755 			break;
756 		}
757 	}
758 	if (jj > 2 || do_disk)
759 		(void) printf("\n");
760 }
761 
762 /*
763  * compute deltas and update accumulators
764  */
765 static void
766 update_counters(void)
767 {
768 	int i;
769 	iodevinfo_t *nio, *oio, *aio, *dio;
770 
771 	ulong_delta((uint64_t *)&nx.csi, (uint64_t *)&ox.csi,
772 		(uint64_t *)&dx.csi, (uint64_t *)&ax.csi, 0, sizeof (ax.csi));
773 	ulong_delta((uint64_t *)&nx.si, (uint64_t *)&ox.si,
774 		(uint64_t *)&dx.si, (uint64_t *)&ax.si, 0, sizeof (ax.si));
775 	ulong_delta((uint64_t *)&nx.cvmi, (uint64_t *)&ox.cvmi,
776 		(uint64_t *)&dx.cvmi, (uint64_t *)&ax.cvmi, 0,
777 		sizeof (ax.cvmi));
778 
779 	ax.vmi.freemem += dx.vmi.freemem = nx.vmi.freemem - ox.vmi.freemem;
780 	ax.vmi.swap_avail += dx.vmi.swap_avail =
781 		nx.vmi.swap_avail - ox.vmi.swap_avail;
782 
783 	nio = nxio;
784 	oio = oxio;
785 	aio = axio;
786 	dio = dxio;
787 	for (i = 0; i < niodevs; i++) {
788 		aio->kios.wlastupdate += dio->kios.wlastupdate
789 			= nio->kios.wlastupdate - oio->kios.wlastupdate;
790 		aio->kios.reads += dio->kios.reads
791 			= nio->kios.reads - oio->kios.reads;
792 		aio->kios.writes += dio->kios.writes
793 			= nio->kios.writes - oio->kios.writes;
794 		aio->kios.nread += dio->kios.nread
795 			= nio->kios.nread - oio->kios.nread;
796 		aio->kios.nwritten += dio->kios.nwritten
797 			= nio->kios.nwritten - oio->kios.nwritten;
798 		aio->kios.wlentime += dio->kios.wlentime
799 			= nio->kios.wlentime - oio->kios.wlentime;
800 		aio->kios.rlentime += dio->kios.rlentime
801 			= nio->kios.rlentime - oio->kios.rlentime;
802 		aio->kios.wtime += dio->kios.wtime
803 			= nio->kios.wtime - oio->kios.wtime;
804 		aio->kios.rtime += dio->kios.rtime
805 			= nio->kios.rtime - oio->kios.rtime;
806 		aio->ks.ks_snaptime += dio->ks.ks_snaptime
807 		    = nio->ks.ks_snaptime - oio->ks.ks_snaptime;
808 		nio++;
809 		oio++;
810 		aio++;
811 		dio++;
812 	}
813 }
814 
815 static void
816 prt_u_opt(struct sa64 *xx)
817 {
818 	(void) printf(" %7.0f %7.0f %7.0f %7.0f\n",
819 		(float)xx->csi.cpu[1] * percent,
820 		(float)xx->csi.cpu[2] * percent,
821 		(float)xx->csi.cpu[3] * percent,
822 		(float)xx->csi.cpu[0] * percent);
823 }
824 
825 static void
826 prt_b_opt(struct sa64 *xx)
827 {
828 	(void) printf(" %7.0f %7.0f %7.0f %7.0f %7.0f %7.0f %7.0f %7.0f\n",
829 		(float)xx->csi.bread / sec_diff,
830 		(float)xx->csi.lread / sec_diff,
831 		freq((float)xx->csi.lread, (float)xx->csi.bread),
832 		(float)xx->csi.bwrite / sec_diff,
833 		(float)xx->csi.lwrite / sec_diff,
834 		freq((float)xx->csi.lwrite, (float)xx->csi.bwrite),
835 		(float)xx->csi.phread / sec_diff,
836 		(float)xx->csi.phwrite / sec_diff);
837 }
838 
839 static void
840 prt_d_opt(int ii, iodevinfo_t *xio)
841 {
842 	double etime, hr_etime, tps, avq, avs, pbusy;
843 
844 	tsttab();
845 
846 	hr_etime = (double)xio[ii].ks.ks_snaptime;
847 	if (hr_etime == 0.0)
848 		hr_etime = (double)NANOSEC;
849 	pbusy = (double)xio[ii].kios.rtime * 100.0 / hr_etime;
850 	if (pbusy > 100.0)
851 		pbusy = 100.0;
852 	etime = hr_etime / (double)NANOSEC;
853 	tps = (double)(xio[ii].kios.reads + xio[ii].kios.writes) / etime;
854 	avq = (double)xio[ii].kios.wlentime / hr_etime;
855 	avs = (double)xio[ii].kios.rlentime / hr_etime;
856 
857 	(void) printf("   %-8.8s    ", nxio[ii].ks.ks_name);
858 	(void) printf("%7.0f %7.1f %7.0f %7.0f %7.1f %7.1f\n",
859                 pbusy,
860 		avq + avs,
861 		tps,
862 		BLKS(xio[ii].kios.nread + xio[ii].kios.nwritten) / etime,
863 		(tps > 0 ? avq / tps * 1000.0 : 0.0),
864 		(tps > 0 ? avs / tps * 1000.0 : 0.0));
865 }
866 
867 static void
868 prt_y_opt(struct sa64 *xx)
869 {
870 	(void) printf(" %7.0f %7.0f %7.0f %7.0f %7.0f %7.0f\n",
871 		(float)xx->csi.rawch / sec_diff,
872 		(float)xx->csi.canch / sec_diff,
873 		(float)xx->csi.outch / sec_diff,
874 		(float)xx->csi.rcvint / sec_diff,
875 		(float)xx->csi.xmtint / sec_diff,
876 		(float)xx->csi.mdmint / sec_diff);
877 }
878 
879 static void
880 prt_c_opt(struct sa64 *xx)
881 {
882 	(void) printf(" %7.0f %7.0f %7.0f %7.2f %7.2f %7.0f %7.0f\n",
883 		(float)xx->csi.syscall / sec_diff,
884 		(float)xx->csi.sysread / sec_diff,
885 		(float)xx->csi.syswrite / sec_diff,
886 		(float)(xx->csi.sysfork + xx->csi.sysvfork) / sec_diff,
887 		(float)xx->csi.sysexec / sec_diff,
888 		(float)xx->csi.readch / sec_diff,
889 		(float)xx->csi.writech / sec_diff);
890 }
891 
892 static void
893 prt_w_opt(struct sa64 *xx)
894 {
895 	(void) printf(" %7.2f %7.1f %7.2f %7.1f %7.0f\n",
896 		(float)xx->cvmi.swapin / sec_diff,
897 		(float)PGTOBLK(xx->cvmi.pgswapin) / sec_diff,
898 		(float)xx->cvmi.swapout / sec_diff,
899 		(float)PGTOBLK(xx->cvmi.pgswapout) / sec_diff,
900 		(float)xx->csi.pswitch / sec_diff);
901 }
902 
903 static void
904 prt_a_opt(struct sa64 *xx)
905 {
906 	(void) printf(" %7.0f %7.0f %7.0f\n",
907 		(float)xx->csi.ufsiget / sec_diff,
908 		(float)xx->csi.namei / sec_diff,
909 		(float)xx->csi.ufsdirblk / sec_diff);
910 }
911 
912 static void
913 prt_q_opt(struct sa64 *xx)
914 {
915 	if (xx->si.runocc == 0 || xx->si.updates == 0)
916 		(void) printf(" %7.1f %7.0f", 0., 0.);
917 	else {
918 		(void) printf(" %7.1f %7.0f",
919 		    (float)xx->si.runque / (float)xx->si.runocc,
920 		    (float)xx->si.runocc / (float)xx->si.updates * 100.0);
921 	}
922 	if (xx->si.swpocc == 0 || xx->si.updates == 0)
923 		(void) printf(" %7.1f %7.0f\n", 0., 0.);
924 	else {
925 		(void) printf(" %7.1f %7.0f\n",
926 		    (float)xx->si.swpque / (float)xx->si.swpocc,
927 		    (float)xx->si.swpocc / (float)xx->si.updates * 100.0);
928 	}
929 }
930 
931 static void
932 prt_v_opt(struct sa64 *xx)
933 {
934 	(void) printf(" %4lu/%-4lu %4llu %4lu/%-4lu %4llu %4lu/%-4lu "
935 	    "%4llu %4lu/%-4lu\n",
936 	    nx.szproc, nx.mszproc, xx->csi.procovf,
937 	    nx.szinode, nx.mszinode, xx->csi.inodeovf,
938 	    nx.szfile, nx.mszfile, xx->csi.fileovf,
939 	    nx.szlckr, nx.mszlckr);
940 }
941 
942 static void
943 prt_m_opt(struct sa64 *xx)
944 {
945 	(void) printf(" %7.2f %7.2f\n",
946 		(float)xx->csi.msg / sec_diff,
947 		(float)xx->csi.sema / sec_diff);
948 }
949 
950 static void
951 prt_p_opt(struct sa64 *xx)
952 {
953 	(void) printf(" %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f\n",
954 		(float)xx->cvmi.pgfrec / sec_diff,
955 		(float)xx->cvmi.pgin / sec_diff,
956 		(float)xx->cvmi.pgpgin / sec_diff,
957 		(float)(xx->cvmi.prot_fault + xx->cvmi.cow_fault) / sec_diff,
958 		(float)(xx->cvmi.hat_fault + xx->cvmi.as_fault) / sec_diff,
959 		(float)xx->cvmi.softlock / sec_diff);
960 }
961 
962 static void
963 prt_g_opt(struct sa64 *xx)
964 {
965 	(void) printf(" %8.2f %8.2f %8.2f %8.2f %8.2f\n",
966 		(float)xx->cvmi.pgout / sec_diff,
967 		(float)xx->cvmi.pgpgout / sec_diff,
968 		(float)xx->cvmi.dfree / sec_diff,
969 		(float)xx->cvmi.scan / sec_diff,
970 		(float)xx->csi.ufsipage * 100.0 /
971 			denom((float)xx->csi.ufsipage +
972 			(float)xx->csi.ufsinopage));
973 }
974 
975 static void
976 prt_r_opt(struct sa64 *xx)
977 {
978 	/* Avoid divide by Zero - Should never happen */
979 	if (xx->si.updates == 0)
980 		(void) printf(" %7.0f %8.0f\n", 0., 0.);
981 	else {
982 		(void) printf(" %7.0f %8.0f\n",
983 		    (double)xx->vmi.freemem / (float)xx->si.updates,
984 		    (double)PGTOBLK(xx->vmi.swap_avail) /
985 			(float)xx->si.updates);
986 	}
987 }
988 
989 static void
990 prt_k_opt(struct sa64 *xx, int n)
991 {
992 	if (n != 1) {
993 		(void) printf(" %7.0f %7.0f %5.0f %7.0f %7.0f %5.0f %11.0f"
994 		    " %5.0f\n",
995 		    (float)kmi.km_mem[KMEM_SMALL] / n,
996 		    (float)kmi.km_alloc[KMEM_SMALL] / n,
997 		    (float)kmi.km_fail[KMEM_SMALL] / n,
998 		    (float)kmi.km_mem[KMEM_LARGE] / n,
999 		    (float)kmi.km_alloc[KMEM_LARGE] / n,
1000 		    (float)kmi.km_fail[KMEM_LARGE] / n,
1001 		    (float)kmi.km_alloc[KMEM_OSIZE] / n,
1002 		    (float)kmi.km_fail[KMEM_OSIZE] / n);
1003 	} else {
1004 		/*
1005 		 * If we are not reporting averages, use the read values
1006 		 * directly.
1007 		 */
1008 		(void) printf(" %7.0f %7.0f %5.0f %7.0f %7.0f %5.0f %11.0f"
1009 		    " %5.0f\n",
1010 		    (float)xx->kmi.km_mem[KMEM_SMALL],
1011 		    (float)xx->kmi.km_alloc[KMEM_SMALL],
1012 		    (float)xx->kmi.km_fail[KMEM_SMALL],
1013 		    (float)xx->kmi.km_mem[KMEM_LARGE],
1014 		    (float)xx->kmi.km_alloc[KMEM_LARGE],
1015 		    (float)xx->kmi.km_fail[KMEM_LARGE],
1016 		    (float)xx->kmi.km_alloc[KMEM_OSIZE],
1017 		    (float)xx->kmi.km_fail[KMEM_OSIZE]);
1018 	}
1019 }
1020 
1021 /*
1022  * Print options routine.
1023  */
1024 static void
1025 prtopt(void)
1026 {
1027 	int	ii, jj = 0;
1028 	char	ccc;
1029 
1030 	prttim();
1031 
1032 	while ((ccc = fopt[jj++]) != NULL) {
1033 		if (ccc != 'd')
1034 			tsttab();
1035 		switch (ccc) {
1036 		    case 'u':
1037 			prt_u_opt(&dx);
1038 			break;
1039 		    case 'b':
1040 			prt_b_opt(&dx);
1041 			break;
1042 		    case 'd':
1043 			for (ii = 0; ii < niodevs; ii++)
1044 				prt_d_opt(ii, dxio);
1045 			break;
1046 		    case 'y':
1047 			prt_y_opt(&dx);
1048 			break;
1049 		    case 'c':
1050 			prt_c_opt(&dx);
1051 			break;
1052 		    case 'w':
1053 			prt_w_opt(&dx);
1054 			break;
1055 		    case 'a':
1056 			prt_a_opt(&dx);
1057 			break;
1058 		    case 'q':
1059 			prt_q_opt(&dx);
1060 			break;
1061 		    case 'v':
1062 			prt_v_opt(&dx);
1063 			break;
1064 		    case 'm':
1065 			prt_m_opt(&dx);
1066 			break;
1067 		    case 'p':
1068 			prt_p_opt(&dx);
1069 			break;
1070 		    case 'g':
1071 			prt_g_opt(&dx);
1072 			break;
1073 		    case 'r':
1074 			prt_r_opt(&dx);
1075 			break;
1076 		    case 'k':
1077 			prt_k_opt(&nx, 1);
1078 			/*
1079 			 * To avoid overflow, copy the data from the sa record
1080 			 * into a struct kmeminfo_l which has members with
1081 			 * larger data types.
1082 			 */
1083 			kmi.km_mem[KMEM_SMALL] += nx.kmi.km_mem[KMEM_SMALL];
1084 			kmi.km_alloc[KMEM_SMALL] += nx.kmi.km_alloc[KMEM_SMALL];
1085 			kmi.km_fail[KMEM_SMALL] += nx.kmi.km_fail[KMEM_SMALL];
1086 			kmi.km_mem[KMEM_LARGE] += nx.kmi.km_mem[KMEM_LARGE];
1087 			kmi.km_alloc[KMEM_LARGE] += nx.kmi.km_alloc[KMEM_LARGE];
1088 			kmi.km_fail[KMEM_LARGE] += nx.kmi.km_fail[KMEM_LARGE];
1089 			kmi.km_alloc[KMEM_OSIZE] += nx.kmi.km_alloc[KMEM_OSIZE];
1090 			kmi.km_fail[KMEM_OSIZE] += nx.kmi.km_fail[KMEM_OSIZE];
1091 			break;
1092 		}
1093 	}
1094 	if (jj > 2 || do_disk)
1095 		(void) printf("\n");
1096 	if (realtime)
1097 		(void) fflush(stdout);
1098 }
1099 
1100 /*
1101  * Print average routine.
1102  */
1103 static void
1104 prtavg(void)
1105 {
1106 	int	ii, jj = 0;
1107 	char	ccc;
1108 
1109 	tdiff = ax.csi.cpu[0] + ax.csi.cpu[1] + ax.csi.cpu[2] + ax.csi.cpu[3];
1110 	if (tdiff <= 0.0)
1111 		return;
1112 
1113 	sec_diff = tdiff / hz;
1114 	percent = 100.0 / tdiff;
1115 	(void) printf("\n");
1116 
1117 	while ((ccc = fopt[jj++]) != NULL) {
1118 		if (ccc != 'v')
1119 			(void) printf("Average ");
1120 		switch (ccc) {
1121 		    case 'u':
1122 			prt_u_opt(&ax);
1123 			break;
1124 		    case 'b':
1125 			prt_b_opt(&ax);
1126 			break;
1127 		    case 'd':
1128 			tabflg = 1;
1129 			for (ii = 0; ii < niodevs; ii++)
1130 				prt_d_opt(ii, axio);
1131 			break;
1132 		    case 'y':
1133 			prt_y_opt(&ax);
1134 			break;
1135 		    case 'c':
1136 			prt_c_opt(&ax);
1137 			break;
1138 		    case 'w':
1139 			prt_w_opt(&ax);
1140 			break;
1141 		    case 'a':
1142 			prt_a_opt(&ax);
1143 			break;
1144 		    case 'q':
1145 			prt_q_opt(&ax);
1146 			break;
1147 		    case 'v':
1148 			break;
1149 		    case 'm':
1150 			prt_m_opt(&ax);
1151 			break;
1152 		    case 'p':
1153 			prt_p_opt(&ax);
1154 			break;
1155 		    case 'g':
1156 			prt_g_opt(&ax);
1157 			break;
1158 		    case 'r':
1159 			prt_r_opt(&ax);
1160 			break;
1161 		    case 'k':
1162 			prt_k_opt(&ax, lines);
1163 			break;
1164 		}
1165 	}
1166 }
1167 
1168 static void
1169 ulong_delta(uint64_t *new, uint64_t *old, uint64_t *delta, uint64_t *accum,
1170 	int begin, int end)
1171 {
1172 	int i;
1173 	uint64_t n, o, d;
1174 
1175 	for (i = begin; i < end; i += sizeof (uint64_t)) {
1176 		n = *new++;
1177 		o = *old++;
1178 		if (o > n) {
1179 			d = n + 0x100000000LL - o;
1180 		} else {
1181 			d = n - o;
1182 		}
1183 		*accum++ += *delta++ = d;
1184 	}
1185 }
1186 
1187 /*
1188  * used to prevent zero denominators
1189  */
1190 static float
1191 denom(float x)
1192 {
1193 	return ((x > 0.5) ? x : 1.0);
1194 }
1195 
1196 /*
1197  * a little calculation that comes up often when computing frequency
1198  * of one operation relative to another
1199  */
1200 static float
1201 freq(float x, float y)
1202 {
1203 	return ((x < 0.5) ? 100.0 : (x - y) / x * 100.0);
1204 }
1205 
1206 static void
1207 usage(void)
1208 {
1209 	(void) fprintf(stderr,
1210 	    "usage: sar [-ubdycwaqvmpgrkA][-o file] t [n]\n"
1211 	    "\tsar [-ubdycwaqvmpgrkA] [-s hh:mm][-e hh:mm][-i ss][-f file]\n");
1212 }
1213 
1214 static void
1215 fail(int do_perror, char *message, ...)
1216 {
1217 	va_list args;
1218 
1219 	va_start(args, message);
1220 	(void) fprintf(stderr, "sar: ");
1221 	(void) vfprintf(stderr, message, args);
1222 	va_end(args);
1223 	(void) fprintf(stderr, "\n");
1224 	switch (do_perror) {
1225 	case 0:				/* usage message */
1226 		usage();
1227 		break;
1228 	case 1:				/* perror output */
1229 		perror("");
1230 		break;
1231 	case 2:				/* no further output */
1232 		break;
1233 	default:			/* error */
1234 		(void) fprintf(stderr, "unsupported failure mode\n");
1235 		break;
1236 	}
1237 	exit(2);
1238 }
1239 
1240 static int
1241 safe_strtoi(char const *val, char *errmsg)
1242 {
1243 	char *end;
1244 	long tmp;
1245 
1246 	errno = 0;
1247 	tmp = strtol(val, &end, 10);
1248 	if (*end != '\0' || errno)
1249 		fail(0, "%s %s", errmsg, val);
1250 	return ((int)tmp);
1251 }
1252 
1253 static void
1254 safe_zalloc(void **ptr, int size, int free_first)
1255 {
1256 	if (free_first && *ptr != NULL)
1257 		free(*ptr);
1258 	if ((*ptr = malloc(size)) == NULL)
1259 		fail(1, "malloc failed");
1260 	(void) memset(*ptr, 0, size);
1261 }
1262 
1263 static int
1264 safe_read(int fd, void *buf, size_t size)
1265 {
1266 	size_t rsize = read(fd, buf, size);
1267 
1268 	if (rsize == 0)
1269 		return (0);
1270 
1271 	if (rsize != size)
1272 		fail(1, "read failed");
1273 
1274 	return (1);
1275 }
1276 
1277 static void
1278 safe_write(int fd, void *buf, size_t size)
1279 {
1280 	if (write(fd, buf, size) != size)
1281 		fail(1, "write failed");
1282 }
1283