xref: /freebsd/usr.bin/nfsstat/nfsstat.c (revision da5069e1f7daaef1e7157876d6044de6f3a08ce2)
1 /*
2  * Copyright (c) 1983, 1989, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * Rick Macklem at The University of Guelph.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  */
32 /*-
33  * Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp.
34  * All rights reserved.
35  *
36  * Redistribution and use in source and binary forms, with or without
37  * modification, are permitted provided that the following conditions
38  * are met:
39  * 1. Redistributions of source code must retain the above copyright
40  *    notice, this list of conditions, and the following disclaimer,
41  *    without modification.
42  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
43  *    substantially similar to the "NO WARRANTY" disclaimer below
44  *    ("Disclaimer") and any redistribution must be conditioned upon
45  *    including a substantially similar Disclaimer requirement for further
46  *    binary redistribution.
47  *
48  * NO WARRANTY
49  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
50  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
51  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
52  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
53  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
57  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
58  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
59  * POSSIBILITY OF SUCH DAMAGES.
60  */
61 
62 
63 #ifndef lint
64 static const char copyright[] =
65 "@(#) Copyright (c) 1983, 1989, 1993\n\
66 	The Regents of the University of California.  All rights reserved.\n";
67 #endif /* not lint */
68 
69 #ifndef lint
70 #if 0
71 static char sccsid[] = "@(#)nfsstat.c	8.2 (Berkeley) 3/31/95";
72 #endif
73 static const char rcsid[] =
74   "$FreeBSD$";
75 #endif /* not lint */
76 
77 #include <sys/param.h>
78 #include <sys/module.h>
79 #include <sys/mount.h>
80 #include <sys/time.h>
81 #include <sys/sysctl.h>
82 #include <nfs/nfsproto.h>
83 #include <nfsclient/nfs.h>
84 #include <nfsserver/nfs.h>
85 #include <nfs/nfssvc.h>
86 
87 #include <fs/nfs/nfsport.h>
88 
89 #include <signal.h>
90 #include <fcntl.h>
91 #include <ctype.h>
92 #include <errno.h>
93 #include <limits.h>
94 #include <nlist.h>
95 #include <unistd.h>
96 #include <stdio.h>
97 #include <stdint.h>
98 #include <stdlib.h>
99 #include <string.h>
100 #include <paths.h>
101 #include <devstat.h>
102 #include <err.h>
103 
104 #include <libxo/xo.h>
105 
106 static int widemode = 0;
107 static int zflag = 0;
108 static int printtitle = 1;
109 static struct nfsstatsv1 ext_nfsstats;
110 static int extra_output = 0;
111 
112 static void intpr(int, int);
113 static void printhdr(int, int, int);
114 static void usage(void);
115 static char *sperc1(int, int);
116 static char *sperc2(int, int);
117 static void exp_intpr(int, int, int);
118 static void exp_sidewaysintpr(u_int, int, int, int);
119 static void compute_new_stats(struct nfsstatsv1 *cur_stats,
120     struct nfsstatsv1 *prev_stats, int curop, long double etime,
121     long double *mbsec, long double *kb_per_transfer,
122     long double *transfers_per_second, long double *ms_per_transfer,
123     uint64_t *queue_len, long double *busy_pct);
124 
125 #define DELTA(field)	(nfsstats.field - lastst.field)
126 
127 #define	STAT_TYPE_READ		0
128 #define	STAT_TYPE_WRITE		1
129 #define	STAT_TYPE_COMMIT	2
130 #define	NUM_STAT_TYPES		3
131 
132 struct stattypes {
133 	int stat_type;
134 	int nfs_type;
135 };
136 static struct stattypes statstruct[] = {
137 	{STAT_TYPE_READ, NFSV4OP_READ},
138 	{STAT_TYPE_WRITE, NFSV4OP_WRITE},
139 	{STAT_TYPE_COMMIT, NFSV4OP_COMMIT}
140 };
141 
142 #define	STAT_TYPE_TO_NFS(stat_type)	statstruct[stat_type].nfs_type
143 
144 #define	NFSSTAT_XO_VERSION	"1"
145 
146 int
147 main(int argc, char **argv)
148 {
149 	u_int interval;
150 	int clientOnly = -1;
151 	int serverOnly = -1;
152 	int newStats = 0;
153 	int ch;
154 	char *memf, *nlistf;
155 	int mntlen, i;
156 	char buf[1024];
157 	struct statfs *mntbuf;
158 	struct nfscl_dumpmntopts dumpmntopts;
159 
160 	interval = 0;
161 	memf = nlistf = NULL;
162 
163 	argc = xo_parse_args(argc, argv);
164 	if (argc < 0)
165 		exit(1);
166 
167 	xo_set_version(NFSSTAT_XO_VERSION);
168 
169 	while ((ch = getopt(argc, argv, "cdEesWM:mN:w:zq")) != -1)
170 		switch(ch) {
171 		case 'M':
172 			memf = optarg;
173 			break;
174 		case 'm':
175 			/* Display mount options for NFS mount points. */
176 			mntlen = getmntinfo(&mntbuf, MNT_NOWAIT);
177 			for (i = 0; i < mntlen; i++) {
178 				if (strcmp(mntbuf->f_fstypename, "nfs") == 0) {
179 					dumpmntopts.ndmnt_fname =
180 					    mntbuf->f_mntonname;
181 					dumpmntopts.ndmnt_buf = buf;
182 					dumpmntopts.ndmnt_blen = sizeof(buf);
183 					if (nfssvc(NFSSVC_DUMPMNTOPTS,
184 					    &dumpmntopts) >= 0)
185 						printf("%s on %s\n%s\n",
186 						    mntbuf->f_mntfromname,
187 						    mntbuf->f_mntonname, buf);
188 					else if (errno == EPERM)
189 						errx(1, "Only priviledged users"
190 						    " can use the -m option");
191 				}
192 				mntbuf++;
193 			}
194 			exit(0);
195 		case 'N':
196 			nlistf = optarg;
197 			break;
198 		case 'W':
199 			widemode = 1;
200 			break;
201 		case 'w':
202 			interval = atoi(optarg);
203 			break;
204 		case 'c':
205 			clientOnly = 1;
206 			if (serverOnly < 0)
207 				serverOnly = 0;
208 			break;
209 		case 'd':
210 			newStats = 1;
211 			if (interval == 0)
212 				interval = 1;
213 			break;
214 		case 's':
215 			serverOnly = 1;
216 			if (clientOnly < 0)
217 				clientOnly = 0;
218 			break;
219 		case 'z':
220 			zflag = 1;
221 			break;
222 		case 'E':
223 			if (extra_output != 0)
224 				xo_err(1, "-e and -E are mutually exclusive");
225 			extra_output = 2;
226 			break;
227 		case 'e':
228 			if (extra_output != 0)
229 				xo_err(1, "-e and -E are mutually exclusive");
230 			extra_output = 1;
231 			break;
232 		case 'q':
233 			printtitle = 0;
234 			break;
235 		case '?':
236 		default:
237 			usage();
238 		}
239 	argc -= optind;
240 	argv += optind;
241 
242 #define	BACKWARD_COMPATIBILITY
243 #ifdef	BACKWARD_COMPATIBILITY
244 	if (*argv) {
245 		interval = atoi(*argv);
246 		if (*++argv) {
247 			nlistf = *argv;
248 			if (*++argv)
249 				memf = *argv;
250 		}
251 	}
252 #endif
253 	if (modfind("nfscommon") < 0)
254 		xo_err(1, "NFS client/server not loaded");
255 
256 	if (interval) {
257 		exp_sidewaysintpr(interval, clientOnly, serverOnly,
258 		    newStats);
259 	} else {
260 		xo_open_container("nfsstat");
261 		if (extra_output != 0)
262 			exp_intpr(clientOnly, serverOnly, extra_output - 1);
263 		else
264 			intpr(clientOnly, serverOnly);
265 		xo_close_container("nfsstat");
266 	}
267 
268 	xo_finish();
269 	exit(0);
270 }
271 
272 /*
273  * Print a description of the nfs stats.
274  */
275 static void
276 intpr(int clientOnly, int serverOnly)
277 {
278 	int nfssvc_flag;
279 
280 	nfssvc_flag = NFSSVC_GETSTATS | NFSSVC_NEWSTRUCT;
281 	if (zflag != 0) {
282 		if (clientOnly != 0)
283 			nfssvc_flag |= NFSSVC_ZEROCLTSTATS;
284 		if (serverOnly != 0)
285 			nfssvc_flag |= NFSSVC_ZEROSRVSTATS;
286 	}
287 	ext_nfsstats.vers = NFSSTATS_V1;
288 	if (nfssvc(nfssvc_flag, &ext_nfsstats) < 0)
289 		xo_err(1, "Can't get stats");
290 	if (clientOnly) {
291 		xo_open_container("clientstats");
292 
293 		if (printtitle)
294 			xo_emit("{T:Client Info:\n");
295 
296 		xo_open_container("operations");
297 		xo_emit("{T:Rpc Counts:}\n");
298 
299 		xo_emit("{T:Getattr/%13.13s}{T:Setattr/%13.13s}"
300 		    "{T:Lookup/%13.13s}{T:Readlink/%13.13s}"
301 		    "{T:Read/%13.13s}{T:Write/%13.13s}"
302 		  "{T:Create/%13.13s}{T:Remove/%13.13s}\n");
303 		xo_emit("{:getattr/%13ju}{:setattr/%13ju}"
304 		    "{:lookup/%13ju}{:readlink/%13ju}"
305 		    "{:read/%13ju}{:write/%13ju}"
306 		    "{:create/%13ju}{:remove/%13ju}\n",
307 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_GETATTR],
308 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SETATTR],
309 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LOOKUP],
310 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READLINK],
311 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READ],
312 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_WRITE],
313 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_CREATE],
314 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_REMOVE]);
315 
316 		xo_emit("{T:Rename/%13.13s}{T:Link/%13.13s}"
317 		    "{T:Symlink/%13.13s}{T:Mkdir/%13.13s}"
318 		    "{T:Rmdir/%13.13s}{T:Readdir/%13.13s}"
319 		  "{T:RdirPlus/%13.13s}{T:Access/%13.13s}\n");
320 		xo_emit("{:rename/%13ju}{:link/%13ju}"
321 		    "{:symlink/%13ju}{:mkdir/%13ju}"
322 		    "{:rmdir/%13ju}{:readdir/%13ju}"
323 		    "{:rdirplus/%13ju}{:access/%13ju}\n",
324 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_RENAME],
325 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LINK],
326 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SYMLINK],
327 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_MKDIR],
328 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_RMDIR],
329 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READDIR],
330 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READDIRPLUS],
331 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_ACCESS]);
332 
333 		xo_emit("{T:Mknod/%13.13s}{T:Fsstat/%13.13s}"
334 		    "{T:Fsinfo/%13.13s}{T:PathConf/%13.13s}"
335 		    "{T:Commit/%13.13s}\n");
336 		xo_emit("{:mknod/%13ju}{:fsstat/%13ju}"
337 		    "{:fsinfo/%13ju}{:pathconf/%13ju}"
338 		    "{:commit/%13ju}\n",
339 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_MKNOD],
340 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_FSSTAT],
341 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_FSINFO],
342 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_PATHCONF],
343 			(uintmax_t)ext_nfsstats.rpccnt[NFSPROC_COMMIT]);
344 
345 		xo_close_container("operations");
346 
347 		xo_open_container("rpcs");
348 		xo_emit("{T:Rpc Info:}\n");
349 
350 		xo_emit("{T:TimedOut/%13.13s}{T:Invalid/%13.13s}"
351 		    "{T:X Replies/%13.13s}{T:Retries/%13.13s}"
352 		    "{T:Requests/%13.13s}\n");
353 		xo_emit("{:timedout/%13ju}{:invalid/%13ju}"
354 		    "{:xreplies/%13ju}{:retries/%13ju}"
355 		    "{:requests/%13ju}\n",
356 			(uintmax_t)ext_nfsstats.rpctimeouts,
357 			(uintmax_t)ext_nfsstats.rpcinvalid,
358 			(uintmax_t)ext_nfsstats.rpcunexpected,
359 			(uintmax_t)ext_nfsstats.rpcretries,
360 			(uintmax_t)ext_nfsstats.rpcrequests);
361 		xo_close_container("rpcs");
362 
363 		xo_open_container("cache");
364 		xo_emit("{T:Cache Info:}\n");
365 
366 		xo_emit("{T:Attr Hits/%13.13s}{T:Attr Misses/%13.13s}"
367 		    "{T:Lkup Hits/%13.13s}{T:Lkup Misses/%13.13s}"
368 		    "{T:BioR Hits/%13.13s}{T:BioR Misses/%13.13s}"
369 		    "{T:BioW Hits/%13.13s}{T:BioW Misses/%13.13s}\n");
370 		xo_emit("{:attrhits/%13ju}{:attrmisses/%13ju}"
371 		    "{:lkuphits/%13ju}{:lkupmisses/%13ju}"
372 		    "{:biorhits/%13ju}{:biormisses/%13ju}"
373 		    "{:biowhits/%13ju}{:biowmisses/%13ju}\n",
374 		    (uintmax_t)ext_nfsstats.attrcache_hits,
375 		    (uintmax_t)ext_nfsstats.attrcache_misses,
376 		    (uintmax_t)ext_nfsstats.lookupcache_hits,
377 		    (uintmax_t)ext_nfsstats.lookupcache_misses,
378 		    (uintmax_t)(ext_nfsstats.biocache_reads -
379 		    ext_nfsstats.read_bios),
380 		    (uintmax_t)ext_nfsstats.read_bios,
381 		    (uintmax_t)(ext_nfsstats.biocache_writes -
382 		    ext_nfsstats.write_bios),
383 		    (uintmax_t)ext_nfsstats.write_bios);
384 
385 		xo_emit("{T:BioRL Hits/%13.13s}{T:BioRL Misses/%13.13s}"
386 		    "{T:BioD Hits/%13.13s}{T:BioD Misses/%13.13s}"
387 		    "{T:DirE Hits/%13.13s}{T:DirE Misses/%13.13s}"
388 		    "{T:Accs Hits/%13.13s}{T:Accs Misses/%13.13s}\n");
389 		xo_emit("{:biosrlhits/%13ju}{:biorlmisses/%13ju}"
390 		    "{:biodhits/%13ju}{:biodmisses/%13ju}"
391 		    "{:direhits/%13ju}{:diremisses/%13ju}"
392 		    "{:accshits/%13ju}{:accsmisses/%13ju}\n",
393 		    (uintmax_t)(ext_nfsstats.biocache_readlinks -
394 		    ext_nfsstats.readlink_bios),
395 		    (uintmax_t)ext_nfsstats.readlink_bios,
396 		    (uintmax_t)(ext_nfsstats.biocache_readdirs -
397 		    ext_nfsstats.readdir_bios),
398 		    (uintmax_t)ext_nfsstats.readdir_bios,
399 		    (uintmax_t)ext_nfsstats.direofcache_hits,
400 		    (uintmax_t)ext_nfsstats.direofcache_misses,
401 		    (uintmax_t)ext_nfsstats.accesscache_hits,
402 		    (uintmax_t)ext_nfsstats.accesscache_misses);
403 
404 		xo_close_container("cache");
405 
406 		xo_close_container("clientstats");
407 	}
408 	if (serverOnly) {
409 		xo_open_container("serverstats");
410 
411 		xo_emit("{T:Server Info:}\n");
412 		xo_open_container("operations");
413 
414 		xo_emit("{T:Getattr/%13.13s}{T:Setattr/%13.13s}"
415 		    "{T:Lookup/%13.13s}{T:Readlink/%13.13s}"
416 		    "{T:Read/%13.13s}{T:Write/%13.13s}"
417 		    "{T:Create/%13.13s}{T:Remove/%13.13s}\n");
418 		xo_emit("{:getattr/%13ju}{:setattr/%13ju}"
419 		    "{:lookup/%13ju}{:readlink/%13ju}"
420 		    "{:read/%13ju}{:write/%13ju}"
421 		    "{:create/%13ju}{:remove/%13ju}\n",
422 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_GETATTR],
423 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SETATTR],
424 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LOOKUP],
425 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READLINK],
426 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READ],
427 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_WRITE],
428 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_CREATE],
429 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_REMOVE]);
430 
431 		xo_emit("{T:Rename/%13.13s}{T:Link/%13.13s}"
432 		    "{T:Symlink/%13.13s}{T:Mkdir/%13.13s}"
433 		    "{T:Rmdir/%13.13s}{T:Readdir/%13.13s}"
434 		    "{T:RdirPlus/%13.13s}{T:Access/%13.13s}\n");
435 		xo_emit("{:rename/%13ju}{:link/%13ju}"
436 		    "{:symlink/%13ju}{:mkdir/%13ju}"
437 		    "{:rmdir/%13ju}{:readdir/%13ju}"
438 		    "{:rdirplus/%13ju}{:access/%13ju}\n",
439 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RENAME],
440 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LINK],
441 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SYMLINK],
442 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_MKDIR],
443 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RMDIR],
444 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READDIR],
445 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READDIRPLUS],
446 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_ACCESS]);
447 
448 		xo_emit("{T:Mknod/%13.13s}{T:Fsstat/%13.13s}"
449 		    "{T:Fsinfo/%13.13s}{T:PathConf/%13.13s}"
450 		    "{T:Commit/%13.13s}\n");
451 		xo_emit("{:mknod/%13ju}{:fsstat/%13ju}"
452 		    "{:fsinfo/%13ju}{:pathconf/%13ju}"
453 		    "{:commit/%13ju}\n",
454 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_MKNOD],
455 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_FSSTAT],
456 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_FSINFO],
457 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_PATHCONF],
458 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_COMMIT]);
459 
460 		xo_close_container("operations");
461 
462 		xo_open_container("server");
463 		xo_emit("{T:Server Re-Failed:}\n");
464 		xo_emit("{T:retfailed/%17ju}\n", (uintmax_t)ext_nfsstats.srvrpc_errs);
465 
466 		xo_emit("{T:Server Faults:}\n");
467 		xo_emit("{T:faults/%13ju}\n", (uintmax_t)ext_nfsstats.srv_errs);
468 
469 		xo_emit("{T:Server Write Gathering:/%13.13s}\n");
470 
471 		xo_emit("{T:WriteOps/%13.13s}{T:WriteRPC/%13.13s}"
472 		    "{T:Opsaved/%13.13s}\n");
473 		xo_emit("{:writeops/%13ju}{:writerpc/%13ju}"
474 		    "{:opsaved/%13ju}\n",
475 		/*
476 		 * The new client doesn't do write gathering. It was
477 		 * only useful for NFSv2.
478 		 */
479 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_WRITE],
480 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_WRITE], 0);
481 
482 		xo_close_container("server");
483 
484 		xo_open_container("cache");
485 		xo_emit("{T:Server Cache Stats:/%13.13s}\n");
486 		xo_emit("{T:Inprog/%13.13s}{T:Idem/%13.13s}"
487 		    "{T:Non-Idem/%13.13s}{T:Misses/%13.13s}\n");
488 		xo_emit("{:inprog/%13ju}{:idem/%13ju}"
489 		    "{:nonidem/%13ju}{:misses/%13ju}\n",
490 			(uintmax_t)ext_nfsstats.srvcache_inproghits,
491 			(uintmax_t)ext_nfsstats.srvcache_idemdonehits,
492 			(uintmax_t)ext_nfsstats.srvcache_nonidemdonehits,
493 			(uintmax_t)ext_nfsstats.srvcache_misses);
494 		xo_close_container("cache");
495 
496 		xo_close_container("serverstats");
497 	}
498 }
499 
500 static void
501 printhdr(int clientOnly, int serverOnly, int newStats)
502 {
503 
504 	if (newStats) {
505 		printf(" [%s Read %s]  [%s Write %s]  "
506 		    "%s[=========== Total ============]\n"
507 		    " KB/t   tps    MB/s%s  KB/t   tps    MB/s%s  "
508 		    "%sKB/t   tps    MB/s    ms  ql  %%b",
509 		    widemode ? "========" : "=====",
510 		    widemode ? "========" : "=====",
511 		    widemode ? "========" : "=====",
512 		    widemode ? "======="  : "====",
513 		    widemode ? "[Commit ]  " : "",
514 		    widemode ? "    ms" : "",
515 		    widemode ? "    ms" : "",
516 		    widemode ? "tps    ms  " : "");
517 	} else {
518 		printf("%s%6.6s %6.6s %6.6s %6.6s %6.6s %6.6s %6.6s %6.6s",
519 		    ((serverOnly && clientOnly) ? "        " : " "),
520 		    "GtAttr", "Lookup", "Rdlink", "Read", "Write", "Rename",
521 		    "Access", "Rddir");
522 		if (widemode && clientOnly) {
523 			printf(" Attr Lkup BioR BioW Accs BioD");
524 		}
525 	}
526 	printf("\n");
527 	fflush(stdout);
528 }
529 
530 static void
531 usage(void)
532 {
533 	(void)fprintf(stderr,
534 	    "usage: nfsstat [-cdemszW] [-M core] [-N system] [-w wait]\n");
535 	exit(1);
536 }
537 
538 static char SPBuf[64][8];
539 static int SPIndex;
540 
541 static char *
542 sperc1(int hits, int misses)
543 {
544 	char *p = SPBuf[SPIndex];
545 
546 	if (hits + misses) {
547 		sprintf(p, "%3d%%",
548 		    (int)(char)((quad_t)hits * 100 / (hits + misses)));
549 	} else {
550 		sprintf(p, "   -");
551 	}
552 	SPIndex = (SPIndex + 1) & 63;
553 	return(p);
554 }
555 
556 static char *
557 sperc2(int ttl, int misses)
558 {
559 	char *p = SPBuf[SPIndex];
560 
561 	if (ttl) {
562 		sprintf(p, "%3d%%",
563 		    (int)(char)((quad_t)(ttl - misses) * 100 / ttl));
564 	} else {
565 		sprintf(p, "   -");
566 	}
567 	SPIndex = (SPIndex + 1) & 63;
568 	return(p);
569 }
570 
571 #define DELTA_T(field)					\
572 	devstat_compute_etime(&cur_stats->field,	\
573 	(prev_stats ? &prev_stats->field : NULL))
574 
575 /*
576  * XXX KDM mostly copied from ctlstat.  We should commonize the code (and
577  * the devstat code) somehow.
578  */
579 static void
580 compute_new_stats(struct nfsstatsv1 *cur_stats,
581 		  struct nfsstatsv1 *prev_stats, int curop,
582 		  long double etime, long double *mbsec,
583 		  long double *kb_per_transfer,
584 		  long double *transfers_per_second,
585 		  long double *ms_per_transfer, uint64_t *queue_len,
586 		  long double *busy_pct)
587 {
588 	uint64_t total_bytes = 0, total_operations = 0;
589 	struct bintime total_time_bt;
590 	struct timespec total_time_ts;
591 
592 	bzero(&total_time_bt, sizeof(total_time_bt));
593 	bzero(&total_time_ts, sizeof(total_time_ts));
594 
595 	total_bytes = cur_stats->srvbytes[curop];
596 	total_operations = cur_stats->srvops[curop];
597 	if (prev_stats != NULL) {
598 		total_bytes -= prev_stats->srvbytes[curop];
599 		total_operations -= prev_stats->srvops[curop];
600 	}
601 
602 	*mbsec = total_bytes;
603 	*mbsec /= 1024 * 1024;
604 	if (etime > 0.0) {
605 		*busy_pct = DELTA_T(busytime);
606 		if (*busy_pct < 0)
607 			*busy_pct = 0;
608 		*busy_pct /= etime;
609 		*busy_pct *= 100;
610 		if (*busy_pct < 0)
611 			*busy_pct = 0;
612 		*mbsec /= etime;
613 	} else {
614 		*busy_pct = 0;
615 		*mbsec = 0;
616 	}
617 	*kb_per_transfer = total_bytes;
618 	*kb_per_transfer /= 1024;
619 	if (total_operations > 0)
620 		*kb_per_transfer /= total_operations;
621 	else
622 		*kb_per_transfer = 0;
623 	if (etime > 0.0) {
624 		*transfers_per_second = total_operations;
625 		*transfers_per_second /= etime;
626 	} else {
627 		*transfers_per_second = 0.0;
628 	}
629 
630 	if (total_operations > 0) {
631 		*ms_per_transfer = DELTA_T(srvduration[curop]);
632 		*ms_per_transfer /= total_operations;
633 		*ms_per_transfer *= 1000;
634 	} else
635 		*ms_per_transfer = 0.0;
636 
637 	*queue_len = cur_stats->srvstartcnt - cur_stats->srvdonecnt;
638 }
639 
640 /*
641  * Print a description of the nfs stats for the client/server,
642  * including NFSv4.1.
643  */
644 static void
645 exp_intpr(int clientOnly, int serverOnly, int nfs41)
646 {
647 	int nfssvc_flag;
648 
649 	xo_open_container("nfsv4");
650 
651 	nfssvc_flag = NFSSVC_GETSTATS | NFSSVC_NEWSTRUCT;
652 	if (zflag != 0) {
653 		if (clientOnly != 0)
654 			nfssvc_flag |= NFSSVC_ZEROCLTSTATS;
655 		if (serverOnly != 0)
656 			nfssvc_flag |= NFSSVC_ZEROSRVSTATS;
657 	}
658 	ext_nfsstats.vers = NFSSTATS_V1;
659 	if (nfssvc(nfssvc_flag, &ext_nfsstats) < 0)
660 		xo_err(1, "Can't get stats");
661 	if (clientOnly != 0) {
662 		xo_open_container("clientstats");
663 
664 		xo_open_container("operations");
665 		if (printtitle) {
666 			xo_emit("{T:Client Info:}\n");
667 			xo_emit("{T:RPC Counts:}\n");
668 		}
669 		xo_emit("{T:Getattr/%13.13s}{T:Setattr/%13.13s}"
670 		    "{T:Lookup/%13.13s}{T:Readlink/%13.13s}"
671 		    "{T:Read/%13.13s}{T:Write/%13.13s}\n");
672 		xo_emit("{:getattr/%13ju}{:setattr/%13ju}{:lookup/%13ju}"
673 		    "{:readlink/%13ju}{:read/%13ju}{:write/%13ju}\n",
674 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_GETATTR],
675 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SETATTR],
676 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LOOKUP],
677 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READLINK],
678 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READ],
679 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_WRITE]);
680 		xo_emit("{T:Create/%13.13s}{T:Remove/%13.13s}"
681 		    "{T:Rename/%13.13s}{T:Link/%13.13s}"
682 		    "{T:Symlink/%13.13s}{T:Mkdir/%13.13s}\n");
683 		xo_emit("{:create/%13ju}{:remove/%13ju}{:rename/%13ju}"
684 		  "{:link/%13ju}{:symlink/%13ju}{:mkdir/%13ju}\n",
685 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_CREATE],
686 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_REMOVE],
687 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_RENAME],
688 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LINK],
689 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SYMLINK],
690 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_MKDIR]);
691 		xo_emit("{T:Rmdir/%13.13s}{T:Readdir/%13.13s}"
692 		    "{T:RdirPlus/%13.13s}{T:Access/%13.13s}"
693 		    "{T:Mknod/%13.13s}{T:Fsstat/%13.13s}\n");
694 		xo_emit("{:rmdir/%13ju}{:readdir/%13ju}{:rdirplus/%13ju}"
695 		    "{:access/%13ju}{:mknod/%13ju}{:fsstat/%13ju}\n",
696 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_RMDIR],
697 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READDIR],
698 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READDIRPLUS],
699 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_ACCESS],
700 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_MKNOD],
701 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_FSSTAT]);
702 		xo_emit("{T:FSinfo/%13.13s}{T:pathConf/%13.13s}"
703 		    "{T:Commit/%13.13s}{T:SetClId/%13.13s}"
704 		    "{T:SetClIdCf/%13.13s}{T:Lock/%13.13s}\n");
705 		xo_emit("{:fsinfo/%13ju}{:pathconf/%13ju}{:commit/%13ju}"
706 		    "{:setclientid/%13ju}{:setclientidcf/%13ju}{:lock/%13ju}\n",
707 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_FSINFO],
708 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_PATHCONF],
709 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_COMMIT],
710 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SETCLIENTID],
711 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SETCLIENTIDCFRM],
712 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LOCK]);
713 		xo_emit("{T:LockT/%13.13s}{T:LockU/%13.13s}"
714 		    "{T:Open/%13.13s}{T:OpenCfr/%13.13s}\n");
715 		xo_emit("{:lockt/%13ju}{:locku/%13ju}"
716 		    "{:open/%13ju}{:opencfr/%13ju}\n",
717 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LOCKT],
718 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LOCKU],
719 		    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_OPEN],
720 		  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_OPENCONFIRM]);
721 
722 		if (nfs41) {
723 			xo_open_container("nfsv41");
724 
725 			xo_emit("{T:OpenDownGr/%13.13s}{T:Close/%13.13s}\n");
726 			xo_emit("{:opendowngr/%13ju}{:close/%13ju}\n",
727 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_OPENDOWNGRADE],
728 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_CLOSE]);
729 
730 			xo_emit("{T:RelLckOwn/%13.13s}{T:FreeStateID/%13.13s}"
731 			    "{T:PutRootFH/%13.13s}{T:DelegRet/%13.13s}"
732 			    "{T:GetAcl/%13.13s}{T:SetAcl/%13.13s}\n");
733 			xo_emit("{:rellckown/%13ju}{:freestateid/%13ju}"
734 			    "{:getacl/%13ju}{:delegret/%13ju}"
735 			    "{:getacl/%13ju}{:setacl/%13ju}\n",
736 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_RELEASELCKOWN],
737 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_FREESTATEID],
738 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_PUTROOTFH],
739 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_DELEGRETURN],
740 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_GETACL],
741 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_SETACL]);
742 
743 			xo_emit("{T:ExchangeId/%13.13s}{T:CreateSess/%13.13s}"
744 			    "{T:DestroySess/%13.13s}{T:DestroyClId/%13.13s}"
745 			    "{T:LayoutGet/%13.13s}{T:GetDevInfo/%13.13s}\n");
746 			xo_emit("{:exchangeid/%13ju}{:createsess/%13ju}"
747 			    "{:destroysess/%13ju}{:destroyclid/%13ju}"
748 			    "{:layoutget/%13ju}{:getdevinfo/%13ju}\n",
749 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_EXCHANGEID],
750 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_CREATESESSION],
751 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_DESTROYSESSION],
752 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_DESTROYCLIENT],
753 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LAYOUTGET],
754 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_GETDEVICEINFO]);
755 
756 			xo_emit("{T:LayoutCommit/%13.13s}{T:LayoutReturn/%13.13s}"
757 			    "{T:ReclaimCompl/%13.13s}{T:ReadDataS/%13.13s}"
758 			    "{T:WriteDataS/%13.13s}{T:CommitDataS/%13.13s}\n");
759 			xo_emit("{:layoutcomit/%13ju}{:layoutreturn/%13ju}"
760 			    "{:reclaimcompl/%13ju}{:readdatas/%13ju}"
761 			    "{:writedatas/%13ju}{:commitdatas/%13ju}\n",
762 			  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LAYOUTCOMMIT],
763 			  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_LAYOUTRETURN],
764 			  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_RECLAIMCOMPL],
765 			  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_READDS],
766 			  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_WRITEDS],
767 			  (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_COMMITDS]);
768 
769 			xo_emit("{T:OpenLayout/%13.13s}{T:CreateLayout/%13.13s}\n");
770 			xo_emit("{:openlayout/%13ju}{:createlayout/%13ju}\n",
771 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_OPENLAYGET],
772 			    (uintmax_t)ext_nfsstats.rpccnt[NFSPROC_CREATELAYGET]);
773 
774 			xo_close_container("nfsv41");
775 		}
776 		xo_close_container("operations");
777 
778 		xo_open_container("client");
779 		xo_emit("{T:OpenOwner/%13.13s}{T:Opens/%13.13s}"
780 		    "{T:LockOwner/%13.13s}{T:Locks/%13.13s}"
781 		    "{T:Delegs/%13.13s}{T:LocalOwn/%13.13s}\n");
782 		xo_emit("{:openowner/%13ju}{:opens/%13ju}"
783 		    "{:lockowner/%13ju}{:locks/%13ju}"
784 		    "{:delegs/%13ju}{:localown/%13ju}\n",
785 		    (uintmax_t)ext_nfsstats.clopenowners,
786 		    (uintmax_t)ext_nfsstats.clopens,
787 		    (uintmax_t)ext_nfsstats.cllockowners,
788 		    (uintmax_t)ext_nfsstats.cllocks,
789 		    (uintmax_t)ext_nfsstats.cldelegates,
790 		    (uintmax_t)ext_nfsstats.cllocalopenowners);
791 
792 		xo_emit("{T:LocalOpen/%13.13s}{T:LocalLown/%13.13s}"
793 		    "{T:LocalLock/%13.13s}\n");
794 		xo_emit("{:localopen/%13ju}{:locallown/%13ju}"
795 		    "{:locallock/%13ju}\n",
796 		    (uintmax_t)ext_nfsstats.cllocalopens,
797 		    (uintmax_t)ext_nfsstats.cllocallockowners,
798 		    (uintmax_t)ext_nfsstats.cllocallocks);
799 		xo_close_container("client");
800 
801 		xo_open_container("rpc");
802 		if (printtitle)
803 			xo_emit("{T:Rpc Info:}\n");
804 		xo_emit("{T:TimedOut/%13.13s}{T:Invalid/%13.13s}"
805 		    "{T:X Replies/%13.13s}{T:Retries/%13.13s}"
806 		    "{T:Requests/%13.13s}\n");
807 		xo_emit("{:timedout/%13ju}{:invalid/%13ju}"
808 		    "{:xreplies/%13ju}{:retries/%13ju}"
809 		    "{:requests/%13ju}\n",
810 		    (uintmax_t)ext_nfsstats.rpctimeouts,
811 		    (uintmax_t)ext_nfsstats.rpcinvalid,
812 		    (uintmax_t)ext_nfsstats.rpcunexpected,
813 		    (uintmax_t)ext_nfsstats.rpcretries,
814 		    (uintmax_t)ext_nfsstats.rpcrequests);
815 		xo_close_container("rpc");
816 
817 		xo_open_container("cache");
818 		if (printtitle)
819 			xo_emit("{T:Cache Info:}\n");
820 		xo_emit("{T:Attr Hits/%13.13s}{T:Attr Misses/%13.13s}"
821 		    "{T:Lkup Hits/%13.13s}{T:Lkup Misses/%13.13s}\n");
822 		xo_emit("{:attrhits/%13ju}{:attrmisses/%13ju}"
823 		    "{:lkuphits/%13ju}{:lkupmisses/%13ju}\n",
824 		    (uintmax_t)ext_nfsstats.attrcache_hits,
825 		    (uintmax_t)ext_nfsstats.attrcache_misses,
826 		    (uintmax_t)ext_nfsstats.lookupcache_hits,
827 		    (uintmax_t)ext_nfsstats.lookupcache_misses);
828 
829 		xo_emit("{T:BioR Hits/%13.13s}{T:BioR Misses/%13.13s}"
830 		    "{T:BioW Hits/%13.13s}{T:BioW Misses/%13.13s}\n");
831 		xo_emit("{:biorhits/%13ju}{:biormisses/%13ju}"
832 		    "{:biowhits/%13ju}{:biowmisses/%13ju}\n",
833 		    (uintmax_t)(ext_nfsstats.biocache_reads -
834 		    ext_nfsstats.read_bios),
835 		    (uintmax_t)ext_nfsstats.read_bios,
836 		    (uintmax_t)(ext_nfsstats.biocache_writes -
837 		    ext_nfsstats.write_bios),
838 		    (uintmax_t)ext_nfsstats.write_bios);
839 
840 		xo_emit("{T:BioRL Hits/%13.13s}{T:BioRL Misses/%13.13s}"
841 		    "{T:BioD Hits/%13.13s}{T:BioD Misses/%13.13s}\n");
842 		xo_emit("{:biorlhits/%13ju}{:biorlmisses/%13ju}"
843 		    "{:biodhits/%13ju}{:biodmisses/%13ju}\n",
844 		    (uintmax_t)(ext_nfsstats.biocache_readlinks -
845 		    ext_nfsstats.readlink_bios),
846 		    (uintmax_t)ext_nfsstats.readlink_bios,
847 		    (uintmax_t)(ext_nfsstats.biocache_readdirs -
848 		    ext_nfsstats.readdir_bios),
849 		    (uintmax_t)ext_nfsstats.readdir_bios);
850 
851 		xo_emit("{T:DirE Hits/%13.13s}{T:DirE Misses/%13.13s}\n");
852 		xo_emit("{:direhits/%13ju}{:diremisses/%13ju}\n",
853 		    (uintmax_t)ext_nfsstats.direofcache_hits,
854 		    (uintmax_t)ext_nfsstats.direofcache_misses);
855 		xo_open_container("cache");
856 
857 		xo_close_container("clientstats");
858 	}
859 	if (serverOnly != 0) {
860 		xo_open_container("serverstats");
861 
862 		xo_open_container("operations");
863 		if (printtitle)
864 			xo_emit("{T:Server Info:}\n");
865 		xo_emit("{T:Getattr/%13.13s}{T:Setattr/%13.13s}"
866 		    "{T:Lookup/%13.13s}{T:Readlink/%13.13s}"
867 		    "{T:Read/%13.13s}{T:Write/%13.13s}\n");
868 		xo_emit("{:getattr/%13ju}{:setattr/%13ju}{:lookup/%13ju}"
869 		    "{:readlink/%13ju}{:read/%13ju}{:write/%13ju}\n",
870 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_GETATTR],
871 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SETATTR],
872 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LOOKUP],
873 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READLINK],
874 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READ],
875 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_WRITE]);
876 		xo_emit("{T:Create/%13.13s}{T:Remove/%13.13s}"
877 		    "{T:Rename/%13.13s}{T:Link/%13.13s}"
878 		    "{T:Symlink/%13.13s}{T:Mkdir/%13.13s}\n");
879 		xo_emit("{:create/%13ju}{:remove/%13ju}{:rename/%13ju}"
880 		    "{:link/%13ju}{:symlink/%13ju}{:mkdir/%13ju}\n",
881 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_V3CREATE],
882 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_REMOVE],
883 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RENAME],
884 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LINK],
885 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SYMLINK],
886 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_MKDIR]);
887 		xo_emit("{T:Rmdir/%13.13s}{T:Readdir/%13.13s}"
888 		    "{T:RdirPlus/%13.13s}{T:Access/%13.13s}"
889 		    "{T:Mknod/%13.13s}{T:Fsstat/%13.13s}\n");
890 		xo_emit("{:rmdir/%13ju}{:readdir/%13ju}{:rdirplus/%13ju}"
891 		    "{:access/%13ju}{:mknod/%13ju}{:fsstat/%13ju}\n",
892 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RMDIR],
893 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READDIR],
894 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_READDIRPLUS],
895 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_ACCESS],
896 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_MKNOD],
897 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_FSSTAT]);
898 		xo_emit("{T:FSinfo/%13.13s}{T:pathConf/%13.13s}"
899 		    "{T:Commit/%13.13s}{T:LookupP/%13.13s}"
900 		    "{T:SetClId/%13.13s}{T:SetClIdCf/%13.13s}\n");
901 		xo_emit("{:fsinfo/%13ju}{:pathconf/%13ju}{:commit/%13ju}"
902 		    "{:lookupp/%13ju}{:setclientid/%13ju}{:setclientidcfrm/%13ju}\n",
903 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_FSINFO],
904 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_PATHCONF],
905 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_COMMIT],
906 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LOOKUPP],
907 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SETCLIENTID],
908 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SETCLIENTIDCFRM]);
909 		xo_emit("{T:Open/%13.13s}{T:OpenAttr/%13.13s}"
910 		    "{T:OpenDwnGr/%13.13s}{T:OpenCfrm/%13.13s}"
911 		    "{T:DelePurge/%13.13s}{T:DelRet/%13.13s}\n");
912 		xo_emit("{:open/%13ju}{:openattr/%13ju}{:opendwgr/%13ju}"
913 		    "{:opencfrm/%13ju}{:delepurge/%13ju}{:delreg/%13ju}\n",
914 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_OPEN],
915 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_OPENATTR],
916 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_OPENDOWNGRADE],
917 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_OPENCONFIRM],
918 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_DELEGPURGE],
919 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_DELEGRETURN]);
920 		xo_emit("{T:GetFH/%13.13s}{T:Lock/%13.13s}"
921 		    "{T:LockT/%13.13s}{T:LockU/%13.13s}"
922 		    "{T:Close/%13.13s}{T:Verify/%13.13s}\n");
923 		xo_emit("{:getfh/%13ju}{:lock/%13ju}{:lockt/%13ju}"
924 		    "{:locku/%13ju}{:close/%13ju}{:verify/%13ju}\n",
925 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_GETFH],
926 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LOCK],
927 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LOCKT],
928 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LOCKU],
929 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_CLOSE],
930 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_VERIFY]);
931 		xo_emit("{T:NVerify/%13.13s}{T:PutFH/%13.13s}"
932 		    "{T:PutPubFH/%13.13s}{T:PutRootFH/%13.13s}"
933 		    "{T:Renew/%13.13s}{T:RestoreFH/%13.13s}\n");
934 		xo_emit("{:nverify/%13ju}{:putfh/%13ju}{:putpubfh/%13ju}"
935 		    "{:putrootfh/%13ju}{:renew/%13ju}{:restore/%13ju}\n",
936 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_NVERIFY],
937 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_PUTFH],
938 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_PUTPUBFH],
939 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_PUTROOTFH],
940 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RENEW],
941 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RESTOREFH]);
942 		xo_emit("{T:SaveFH/%13.13s}{T:Secinfo/%13.13s}"
943 		    "{T:RelLockOwn/%13.13s}{T:V4Create/%13.13s}\n");
944 		xo_emit("{:savefh/%13ju}{:secinfo/%13ju}{:rellockown/%13ju}"
945 		    "{:v4create/%13ju}\n",
946 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SAVEFH],
947 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SECINFO],
948 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RELEASELCKOWN],
949 		    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_CREATE]);
950 		if (nfs41) {
951 			xo_open_container("nfsv41");
952 			xo_emit("{T:BackChannelCtrl/%13.13s}{T:BindConnToSess/%13.13s}"
953 			    "{T:ExchangeID/%13.13s}{T:CreateSess/%13.13s}"
954 			    "{T:DestroySess/%13.13s}{T:FreeStateID/%13.13s}\n");
955 			xo_emit("{:backchannelctrl/%13ju}{:bindconntosess/%13ju}"
956 			    "{:exchangeid/%13ju}{:createsess/%13ju}"
957 			    "{:destroysess/%13ju}{:freestateid/%13ju}\n",
958 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_BACKCHANNELCTL],
959 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_BINDCONNTOSESS],
960 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_EXCHANGEID],
961 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_CREATESESSION],
962 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_DESTROYSESSION],
963 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_FREESTATEID]),
964 
965 			xo_emit("{T:GetDirDeleg/%13.13s}{T:GetDevInfo/%13.13s}"
966 			    "{T:GetDevList/%13.13s}{T:layoutCommit/%13.13s}"
967 			    "{T:LayoutGet/%13.13s}{T:LayoutReturn/%13.13s}\n");
968 			xo_emit("{:getdirdeleg/%13ju}{:getdevinfo/%13ju}"
969 			    "{:getdevlist/%13ju}{:layoutcommit/%13ju}"
970 			    "{:layoutget/%13ju}{:layoutreturn/%13ju}\n",
971 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_GETDIRDELEG],
972 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_GETDEVINFO],
973 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_GETDEVLIST],
974 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LAYOUTCOMMIT],
975 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LAYOUTGET],
976 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_LAYOUTRETURN]);
977 
978 			xo_emit("{T:SecInfNoName/%13.13s}{T:Sequence/%13.13s}"
979 			    "{T:SetSSV/%13.13s}{T:TestStateID/%13.13s}"
980 			    "{T:WantDeleg/%13.13s}{T:DestroyClId/%13.13s}\n");
981 			xo_emit("{:secinfnoname/%13ju}{:sequence/%13ju}"
982 			    "{:setssv/%13ju}{:teststateid/%13ju}{:wantdeleg/%13ju}"
983 			    "{:destroyclid/%13ju}\n",
984 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SECINFONONAME],
985 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SEQUENCE],
986 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_SETSSV],
987 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_TESTSTATEID],
988 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_WANTDELEG],
989 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_DESTROYCLIENTID]);
990 
991 			xo_emit("{T:ReclaimCompl/%13.13s}\n");
992 			xo_emit("{:reclaimcompl/%13ju}\n",
993 			    (uintmax_t)ext_nfsstats.srvrpccnt[NFSV4OP_RECLAIMCOMPL]);
994 
995 			xo_close_container("nfsv41");
996 		}
997 
998 		xo_close_container("operations");
999 
1000 		if (printtitle)
1001 			xo_emit("{T:Server:}\n");
1002 		xo_open_container("server");
1003 		xo_emit("{T:Retfailed/%13.13s}{T:Faults/%13.13s}"
1004 		    "{T:Clients/%13.13s}\n");
1005 		xo_emit("{:retfailed/%13ju}{:faults/%13ju}{:clients/%13ju}\n",
1006 		    (uintmax_t)ext_nfsstats.srv_errs,
1007 		    (uintmax_t)ext_nfsstats.srvrpc_errs,
1008 		    (uintmax_t)ext_nfsstats.srvclients);
1009 		xo_emit("{T:OpenOwner/%13.13s}{T:Opens/%13.13s}"
1010 		    "{T:LockOwner/%13.13s}{T:Locks/%13.13s}"
1011 		    "{T:Delegs/%13.13s}\n");
1012 		xo_emit("{:openowner/%13ju}{:opens/%13ju}{:lockowner/%13ju}"
1013 		  "{:locks/%13ju}{:delegs/%13ju}\n",
1014 		    (uintmax_t)ext_nfsstats.srvopenowners,
1015 		    (uintmax_t)ext_nfsstats.srvopens,
1016 		    (uintmax_t)ext_nfsstats.srvlockowners,
1017 		    (uintmax_t)ext_nfsstats.srvlocks,
1018 		    (uintmax_t)ext_nfsstats.srvdelegates);
1019 		xo_close_container("server");
1020 
1021 		if (printtitle)
1022 			xo_emit("{T:Server Cache Stats:}\n");
1023 		xo_open_container("cache");
1024 		xo_emit("{T:Inprog/%13.13s}{T:Idem/%13.13s}"
1025 		    "{T:Non-idem/%13.13s}{T:Misses/%13.13s}"
1026 		    "{T:CacheSize/%13.13s}{T:TCPPeak/%13.13s}\n");
1027 		xo_emit("{:inprog/%13ju}{:idem/%13ju}{:nonidem/%13ju}"
1028 		    "{:misses/%13ju}{:cachesize/%13ju}{:tcppeak/%13ju}\n",
1029 		    (uintmax_t)ext_nfsstats.srvcache_inproghits,
1030 		    (uintmax_t)ext_nfsstats.srvcache_idemdonehits,
1031 		    (uintmax_t)ext_nfsstats.srvcache_nonidemdonehits,
1032 		    (uintmax_t)ext_nfsstats.srvcache_misses,
1033 		    (uintmax_t)ext_nfsstats.srvcache_size,
1034 		    (uintmax_t)ext_nfsstats.srvcache_tcppeak);
1035 		xo_close_container("cache");
1036 
1037 		xo_close_container("serverstats");
1038 	}
1039 
1040 	xo_close_container("nfsv4");
1041 }
1042 
1043 static void
1044 compute_totals(struct nfsstatsv1 *total_stats, struct nfsstatsv1 *cur_stats)
1045 {
1046 	int i;
1047 
1048 	bzero(total_stats, sizeof(*total_stats));
1049 	for (i = 0; i < (NFSV42_NOPS + NFSV4OP_FAKENOPS); i++) {
1050 		total_stats->srvbytes[0] += cur_stats->srvbytes[i];
1051 		total_stats->srvops[0] += cur_stats->srvops[i];
1052 		bintime_add(&total_stats->srvduration[0],
1053 			    &cur_stats->srvduration[i]);
1054 		total_stats->srvrpccnt[i] = cur_stats->srvrpccnt[i];
1055 	}
1056 	total_stats->srvstartcnt = cur_stats->srvstartcnt;
1057 	total_stats->srvdonecnt = cur_stats->srvdonecnt;
1058 	total_stats->busytime = cur_stats->busytime;
1059 
1060 }
1061 
1062 /*
1063  * Print a running summary of nfs statistics for the experimental client and/or
1064  * server.
1065  * Repeat display every interval seconds, showing statistics
1066  * collected over that interval.  Assumes that interval is non-zero.
1067  * First line printed at top of screen is always cumulative.
1068  */
1069 static void
1070 exp_sidewaysintpr(u_int interval, int clientOnly, int serverOnly,
1071     int newStats)
1072 {
1073 	struct nfsstatsv1 nfsstats, lastst, *ext_nfsstatsp;
1074 	struct nfsstatsv1 curtotal, lasttotal;
1075 	struct timespec ts, lastts;
1076 	int hdrcnt = 1;
1077 
1078 	ext_nfsstatsp = &lastst;
1079 	ext_nfsstatsp->vers = NFSSTATS_V1;
1080 	if (nfssvc(NFSSVC_GETSTATS | NFSSVC_NEWSTRUCT, ext_nfsstatsp) < 0)
1081 		err(1, "Can't get stats");
1082 	clock_gettime(CLOCK_MONOTONIC, &lastts);
1083 	compute_totals(&lasttotal, ext_nfsstatsp);
1084 	sleep(interval);
1085 
1086 	for (;;) {
1087 		ext_nfsstatsp = &nfsstats;
1088 		ext_nfsstatsp->vers = NFSSTATS_V1;
1089 		if (nfssvc(NFSSVC_GETSTATS | NFSSVC_NEWSTRUCT, ext_nfsstatsp)
1090 		    < 0)
1091 			err(1, "Can't get stats");
1092 		clock_gettime(CLOCK_MONOTONIC, &ts);
1093 
1094 		if (--hdrcnt == 0) {
1095 			printhdr(clientOnly, serverOnly, newStats);
1096 			if (newStats)
1097 				hdrcnt = 20;
1098 			else if (clientOnly && serverOnly)
1099 				hdrcnt = 10;
1100 			else
1101 				hdrcnt = 20;
1102 		}
1103 		if (clientOnly && newStats == 0) {
1104 		    printf("%s %6ju %6ju %6ju %6ju %6ju %6ju %6ju %6ju",
1105 			((clientOnly && serverOnly) ? "Client:" : ""),
1106 			(uintmax_t)DELTA(rpccnt[NFSPROC_GETATTR]),
1107 			(uintmax_t)DELTA(rpccnt[NFSPROC_LOOKUP]),
1108 			(uintmax_t)DELTA(rpccnt[NFSPROC_READLINK]),
1109 			(uintmax_t)DELTA(rpccnt[NFSPROC_READ]),
1110 			(uintmax_t)DELTA(rpccnt[NFSPROC_WRITE]),
1111 			(uintmax_t)DELTA(rpccnt[NFSPROC_RENAME]),
1112 			(uintmax_t)DELTA(rpccnt[NFSPROC_ACCESS]),
1113 			(uintmax_t)(DELTA(rpccnt[NFSPROC_READDIR]) +
1114 			DELTA(rpccnt[NFSPROC_READDIRPLUS]))
1115 		    );
1116 		    if (widemode) {
1117 			    printf(" %s %s %s %s %s %s",
1118 				sperc1(DELTA(attrcache_hits),
1119 				    DELTA(attrcache_misses)),
1120 				sperc1(DELTA(lookupcache_hits),
1121 				    DELTA(lookupcache_misses)),
1122 				sperc2(DELTA(biocache_reads),
1123 				    DELTA(read_bios)),
1124 				sperc2(DELTA(biocache_writes),
1125 				    DELTA(write_bios)),
1126 				sperc1(DELTA(accesscache_hits),
1127 				    DELTA(accesscache_misses)),
1128 				sperc2(DELTA(biocache_readdirs),
1129 				    DELTA(readdir_bios))
1130 			    );
1131 		    }
1132 		    printf("\n");
1133 		}
1134 
1135 		if (serverOnly && newStats) {
1136 			long double cur_secs, last_secs, etime;
1137 			long double mbsec;
1138 			long double kb_per_transfer;
1139 			long double transfers_per_second;
1140 			long double ms_per_transfer;
1141 			uint64_t queue_len;
1142 			long double busy_pct;
1143 			int i;
1144 
1145 			cur_secs = ts.tv_sec +
1146 			    ((long double)ts.tv_nsec / 1000000000);
1147 			last_secs = lastts.tv_sec +
1148 			    ((long double)lastts.tv_nsec / 1000000000);
1149 			etime = cur_secs - last_secs;
1150 
1151 			compute_totals(&curtotal, &nfsstats);
1152 
1153 			for (i = 0; i < NUM_STAT_TYPES; i++) {
1154 				compute_new_stats(&nfsstats, &lastst,
1155 				    STAT_TYPE_TO_NFS(i), etime, &mbsec,
1156 				    &kb_per_transfer,
1157 				    &transfers_per_second,
1158 				    &ms_per_transfer, &queue_len,
1159 				    &busy_pct);
1160 
1161 				if (i == STAT_TYPE_COMMIT) {
1162 					if (widemode == 0)
1163 						continue;
1164 
1165 					printf("%2.0Lf %7.2Lf ",
1166 					    transfers_per_second,
1167 					    ms_per_transfer);
1168 				} else {
1169 					printf("%5.2Lf %5.0Lf %7.2Lf ",
1170 					    kb_per_transfer,
1171 					    transfers_per_second, mbsec);
1172 					if (widemode)
1173 						printf("%5.2Lf ",
1174 						    ms_per_transfer);
1175 				}
1176 			}
1177 
1178 			compute_new_stats(&curtotal, &lasttotal, 0, etime,
1179 			    &mbsec, &kb_per_transfer, &transfers_per_second,
1180 			    &ms_per_transfer, &queue_len, &busy_pct);
1181 
1182 			printf("%5.2Lf %5.0Lf %7.2Lf %5.2Lf %3ju %3.0Lf\n",
1183 			    kb_per_transfer, transfers_per_second, mbsec,
1184 			    ms_per_transfer, queue_len, busy_pct);
1185 		} else if (serverOnly) {
1186 		    printf("%s %6ju %6ju %6ju %6ju %6ju %6ju %6ju %6ju",
1187 			((clientOnly && serverOnly) ? "Server:" : ""),
1188 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_GETATTR]),
1189 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_LOOKUP]),
1190 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_READLINK]),
1191 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_READ]),
1192 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_WRITE]),
1193 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_RENAME]),
1194 			(uintmax_t)DELTA(srvrpccnt[NFSV4OP_ACCESS]),
1195 			(uintmax_t)(DELTA(srvrpccnt[NFSV4OP_READDIR]) +
1196 			DELTA(srvrpccnt[NFSV4OP_READDIRPLUS])));
1197 		    printf("\n");
1198 		}
1199 		bcopy(&nfsstats, &lastst, sizeof(lastst));
1200 		bcopy(&curtotal, &lasttotal, sizeof(lasttotal));
1201 		lastts = ts;
1202 		fflush(stdout);
1203 		sleep(interval);
1204 	}
1205 	/*NOTREACHED*/
1206 }
1207