xref: /illumos-gate/usr/src/cmd/cmd-inet/usr.sbin/ilbadm/ilbadm_stats.c (revision 20a7641f9918de8574b8b3b47dbe35c4bfc78df1)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  * Copyright 2012 Milan Jurik. All rights reserved.
26  */
27 
28 #include <stdio.h>
29 #include <unistd.h>
30 #include <stdlib.h>
31 #include <strings.h>
32 #include <sys/types.h>
33 #include <sys/socket.h>
34 #include <sys/sysmacros.h>
35 #include <sys/note.h>
36 #include <fcntl.h>
37 #include <errno.h>
38 #include <assert.h>
39 #include <libgen.h>
40 #include <kstat.h>
41 #include <ofmt.h>
42 #include <libilb.h>
43 #include "ilbadm.h"
44 
45 #define	ILBST_TIMESTAMP_HEADER	0x01	/* a timestamp w. every header */
46 #define	ILBST_DELTA_INTERVAL	0x02	/* delta over specified interval */
47 #define	ILBST_ABS_NUMBERS	0x04	/* print absolute numbers, no d's */
48 #define	ILBST_ITEMIZE		0x08	/* itemize */
49 #define	ILBST_VERBOSE		0x10	/* verbose error info */
50 
51 #define	ILBST_OLD_VALUES	0x20	/* for internal processing */
52 #define	ILBST_RULES_CHANGED	0x40
53 
54 typedef struct {
55 	char		is_name[KSTAT_STRLEN];
56 	uint64_t	is_value;
57 } ilbst_stat_t;
58 
59 static ilbst_stat_t rulestats[] = {
60 	{"num_servers", 0},
61 	{"bytes_not_processed", 0},
62 	{"pkt_not_processed", 0},
63 	{"bytes_dropped", 0},
64 	{"pkt_dropped", 0},
65 	{"nomem_bytes_dropped", 0},
66 	{"nomem_pkt_dropped", 0},
67 	{"noport_bytes_dropped", 0},
68 	{"noport_pkt_dropped", 0},
69 	{"icmp_echo_processed", 0},
70 	{"icmp_dropped", 0},
71 	{"icmp_too_big_processed", 0},
72 	{"icmp_too_big_dropped", 0}
73 };
74 
75 /* indices into array above, to avoid searching */
76 #define	RLSTA_NUM_SRV		0
77 #define	RLSTA_BYTES_U		1
78 #define	RLSTA_PKT_U		2
79 #define	RLSTA_BYTES_D		3
80 #define	RLSTA_PKT_D		4
81 #define	RLSTA_NOMEMBYTES_D	5
82 #define	RLSTA_NOMEMPKT_D	6
83 #define	RLSTA_NOPORTBYTES_D	7
84 #define	RLSTA_NOPORTPKT_D	8
85 #define	RLSTA_ICMP_P		9
86 #define	RLSTA_ICMP_D		10
87 #define	RLSTA_ICMP2BIG_P	11
88 #define	RLSTA_ICMP2BIG_D	12
89 
90 static ilbst_stat_t servstats[] = {
91 	{"bytes_processed", 0},
92 	{"pkt_processed", 0}
93 };
94 /* indices into array above, to avoid searching */
95 #define	SRVST_BYTES_P	0
96 #define	SRVST_PKT_P	1
97 
98 /* values used for of_* commands as id */
99 #define	ILBST_PKT_P		0
100 #define	ILBST_BYTES_P		1
101 #define	ILBST_PKT_U		2
102 #define	ILBST_BYTES_U		3
103 #define	ILBST_PKT_D		4
104 #define	ILBST_BYTES_D		5
105 #define	ILBST_ICMP_P		6
106 #define	ILBST_ICMP_D		7
107 #define	ILBST_ICMP2BIG_P	8
108 #define	ILBST_ICMP2BIG_D	9
109 #define	ILBST_NOMEMP_D		10
110 #define	ILBST_NOPORTP_D		11
111 #define	ILBST_NOMEMB_D		12
112 #define	ILBST_NOPORTB_D		13
113 
114 #define	ILBST_ITEMIZE_SNAME	97
115 #define	ILBST_ITEMIZE_RNAME	98
116 #define	ILBST_TIMESTAMP		99
117 
118 /* approx field widths */
119 #define	ILBST_PKTCTR_W		8
120 #define	ILBST_BYTECTR_W		10
121 #define	ILBST_TIME_W		15
122 
123 static boolean_t of_rule_stats(ofmt_arg_t *, char *, uint_t);
124 static boolean_t of_server_stats(ofmt_arg_t *, char *, uint_t);
125 static boolean_t of_itemize_stats(ofmt_arg_t *, char *, uint_t);
126 static boolean_t of_timestamp(ofmt_arg_t *, char *, uint_t);
127 
128 static ofmt_field_t stat_itemize_fields[] = {
129 	{"RULENAME", ILB_NAMESZ,	ILBST_ITEMIZE_RNAME, of_itemize_stats},
130 	{"SERVERNAME", ILB_NAMESZ,	ILBST_ITEMIZE_SNAME, of_itemize_stats},
131 	{"PKT_P",   ILBST_PKTCTR_W,	ILBST_PKT_P, of_itemize_stats},
132 	{"BYTES_P", ILBST_BYTECTR_W,	ILBST_BYTES_P, of_itemize_stats},
133 	{"TIME",    ILBST_TIME_W,	ILBST_TIMESTAMP, of_timestamp},
134 	{NULL,	    0, 0, NULL}
135 };
136 static ofmt_field_t stat_stdfields[] = {
137 	{"PKT_P",   ILBST_PKTCTR_W,	ILBST_PKT_P, of_server_stats},
138 	{"BYTES_P", ILBST_BYTECTR_W,	ILBST_BYTES_P, of_server_stats},
139 	{"PKT_U",   ILBST_PKTCTR_W,	ILBST_PKT_U, of_rule_stats},
140 	{"BYTES_U", ILBST_BYTECTR_W,	ILBST_BYTES_U, of_rule_stats},
141 	{"PKT_D",   ILBST_PKTCTR_W,	ILBST_PKT_D, of_rule_stats},
142 	{"BYTES_D", ILBST_BYTECTR_W,	ILBST_BYTES_D, of_rule_stats},
143 	{"ICMP_P",  ILBST_PKTCTR_W,	ILBST_ICMP_P, of_rule_stats},
144 	{"ICMP_D",  ILBST_PKTCTR_W,	ILBST_ICMP_D, of_rule_stats},
145 	{"ICMP2BIG_P", 11,		ILBST_ICMP2BIG_P, of_rule_stats},
146 	{"ICMP2BIG_D", 11,		ILBST_ICMP2BIG_D, of_rule_stats},
147 	{"NOMEMP_D", ILBST_PKTCTR_W,	ILBST_NOMEMP_D, of_rule_stats},
148 	{"NOPORTP_D", ILBST_PKTCTR_W,	ILBST_NOPORTP_D, of_rule_stats},
149 	{"NOMEMB_D", ILBST_PKTCTR_W,	ILBST_NOMEMB_D, of_rule_stats},
150 	{"NOPORTB_D", ILBST_PKTCTR_W,	ILBST_NOPORTB_D, of_rule_stats},
151 	{"TIME",    ILBST_TIME_W,	ILBST_TIMESTAMP, of_timestamp},
152 	{NULL,	    0, 0, NULL}
153 };
154 
155 static char stat_stdhdrs[] = "PKT_P,BYTES_P,PKT_U,BYTES_U,PKT_D,BYTES_D";
156 static char stat_stdv_hdrs[] = "PKT_P,BYTES_P,PKT_U,BYTES_U,PKT_D,BYTES_D,"
157 	"ICMP_P,ICMP_D,ICMP2BIG_P,ICMP2BIG_D,NOMEMP_D,NOPORTP_D";
158 static char stat_itemize_rule_hdrs[] = "SERVERNAME,PKT_P,BYTES_P";
159 static char stat_itemize_server_hdrs[] = "RULENAME,PKT_P,BYTES_P";
160 
161 #define	RSTAT_SZ	(sizeof (rulestats)/sizeof (rulestats[0]))
162 #define	SSTAT_SZ	(sizeof (servstats)/sizeof (servstats[0]))
163 
164 typedef struct {
165 	char		isd_servername[KSTAT_STRLEN]; /* serverID */
166 	ilbst_stat_t	isd_serverstats[SSTAT_SZ];
167 	hrtime_t	isd_crtime;	/* save for comparison purpose */
168 } ilbst_srv_desc_t;
169 
170 /*
171  * this data structure stores statistics for a rule - both an old set
172  * and a current/new set. we use pointers to the actual stores and switch
173  * the pointers for every round. old_is_old in ilbst_arg_t indicates
174  * which pointer points to the "old" data struct (ie, if true, _o pointer
175  * points to old)
176  */
177 typedef struct {
178 	char			ird_rulename[KSTAT_STRLEN];
179 	int			ird_num_servers;
180 	int			ird_num_servers_o;
181 	int			ird_srv_ind;
182 	hrtime_t		ird_crtime;	/* save for comparison */
183 	hrtime_t		ird_crtime_o;	/* save for comparison */
184 	ilbst_srv_desc_t	*ird_srvlist;
185 	ilbst_srv_desc_t	*ird_srvlist_o;
186 	ilbst_stat_t		ird_rstats[RSTAT_SZ];
187 	ilbst_stat_t		ird_rstats_o[RSTAT_SZ];
188 	ilbst_stat_t		*ird_rulestats;
189 	ilbst_stat_t		*ird_rulestats_o;
190 } ilbst_rule_desc_t;
191 
192 /*
193  * overall "container" for information pertaining to statistics, and
194  * how to display them.
195  */
196 typedef struct {
197 	int			ilbst_flags;
198 	/* fields representing user input */
199 	char			*ilbst_rulename;	/* optional */
200 	char 			*ilbst_server;	/* optional */
201 	int			ilbst_interval;
202 	int			ilbst_count;
203 	/* "internal" fields for data and data presentation */
204 	ofmt_handle_t		ilbst_oh;
205 	boolean_t		ilbst_old_is_old;
206 	ilbst_rule_desc_t	*ilbst_rlist;
207 	int			ilbst_rcount;	  /* current list count */
208 	int			ilbst_rcount_prev; /* prev (different) count */
209 	int			ilbst_rlist_sz; /* number of alloc'ed rules */
210 	int			ilbst_rule_index; /* for itemizes display */
211 } ilbst_arg_t;
212 
213 /* ARGSUSED */
214 static boolean_t
215 of_timestamp(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
216 {
217 	time_t		now;
218 	struct tm	*now_tm;
219 
220 	now = time(NULL);
221 	now_tm = localtime(&now);
222 
223 	(void) strftime(buf, bufsize, "%F:%H.%M.%S", now_tm);
224 	return (B_TRUE);
225 }
226 
227 static boolean_t
228 i_sum_per_rule_processed(ilbst_rule_desc_t *rp, uint64_t *resp, int index,
229     int flags)
230 {
231 	int			i, num_servers;
232 	ilbst_srv_desc_t	*srv, *o_srv, *n_srv;
233 	uint64_t		res = 0;
234 	boolean_t		valid = B_TRUE;
235 	boolean_t		old = flags & ILBST_OLD_VALUES;
236 	boolean_t		check_valid;
237 
238 	/* if we do abs. numbers, we never look at the _o fields */
239 	assert((old && (flags & ILBST_ABS_NUMBERS)) == B_FALSE);
240 
241 	/* we only check for validity under certain conditions */
242 	check_valid = !(old || (flags & ILBST_ABS_NUMBERS));
243 
244 	if (check_valid && rp->ird_num_servers != rp->ird_num_servers_o)
245 		valid = B_FALSE;
246 
247 	num_servers = old ? rp->ird_num_servers_o : rp->ird_num_servers;
248 
249 	for (i = 0; i < num_servers; i++) {
250 		n_srv = &rp->ird_srvlist[i];
251 		o_srv = &rp->ird_srvlist_o[i];
252 
253 		if (old)
254 			srv = o_srv;
255 		else
256 			srv = n_srv;
257 
258 		res += srv->isd_serverstats[index].is_value;
259 		/*
260 		 * if creation times don't match, comparison is wrong; if
261 		 * if we already know something is invalid, we don't
262 		 * need to compare again.
263 		 */
264 		if (check_valid && valid == B_TRUE &&
265 		    o_srv->isd_crtime != n_srv->isd_crtime) {
266 			valid = B_FALSE;
267 			break;
268 		}
269 	}
270 	/*
271 	 * save the result even though it may be imprecise  - let the
272 	 * caller decide what to do
273 	 */
274 	*resp = res;
275 
276 	return (valid);
277 }
278 
279 typedef boolean_t (*sumfunc_t)(ilbst_rule_desc_t *, uint64_t *, int);
280 
281 static boolean_t
282 i_sum_per_rule_pkt_p(ilbst_rule_desc_t *rp, uint64_t *resp, int flags)
283 {
284 	return (i_sum_per_rule_processed(rp, resp, SRVST_PKT_P, flags));
285 }
286 
287 static boolean_t
288 i_sum_per_rule_bytes_p(ilbst_rule_desc_t *rp, uint64_t *resp, int flags)
289 {
290 	return (i_sum_per_rule_processed(rp, resp, SRVST_BYTES_P, flags));
291 }
292 
293 static boolean_t
294 of_server_stats(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
295 {
296 	ilbst_arg_t	*sta = (ilbst_arg_t *)of_arg->ofmt_cbarg;
297 	uint64_t	count = 0, val;
298 	int		i;
299 	boolean_t	valid = B_TRUE;
300 	sumfunc_t	sumfunc;
301 
302 	switch (of_arg->ofmt_id) {
303 	case ILBST_PKT_P: sumfunc = i_sum_per_rule_pkt_p;
304 		break;
305 	case ILBST_BYTES_P: sumfunc = i_sum_per_rule_bytes_p;
306 		break;
307 	}
308 
309 	for (i = 0; i < sta->ilbst_rcount; i++) {
310 		valid = sumfunc(&sta->ilbst_rlist[i], &val, sta->ilbst_flags);
311 		if (!valid)
312 			return (valid);
313 		count += val;
314 	}
315 
316 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) != 0)
317 		goto out;
318 
319 	for (i = 0; i < sta->ilbst_rcount; i++) {
320 		(void) sumfunc(&sta->ilbst_rlist[i], &val,
321 		    sta->ilbst_flags | ILBST_OLD_VALUES);
322 		count -= val;
323 	}
324 
325 out:
326 	/*
327 	 * normally, we print "change per second", which we calculate
328 	 * here. otherwise, we print "change over interval"
329 	 */
330 	if ((sta->ilbst_flags & (ILBST_DELTA_INTERVAL|ILBST_ABS_NUMBERS)) == 0)
331 		count /= sta->ilbst_interval;
332 
333 	(void) snprintf(buf, bufsize, "%llu", count);
334 	return (B_TRUE);
335 }
336 
337 /*
338  * this function is called when user wants itemized stats of every
339  * server for a named rule, or vice vera.
340  * i_do_print sets sta->rule_index and the proper ird_srv_ind so
341  * we don't have to differentiate between these two cases here.
342  */
343 static boolean_t
344 of_itemize_stats(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
345 {
346 	ilbst_arg_t	*sta = (ilbst_arg_t *)of_arg->ofmt_cbarg;
347 	int		stat_ind;
348 	uint64_t	count;
349 	int		rule_index = sta->ilbst_rule_index;
350 	int		srv_ind = sta->ilbst_rlist[rule_index].ird_srv_ind;
351 	boolean_t	ret = B_TRUE;
352 	ilbst_srv_desc_t *srv, *osrv;
353 
354 	srv = &sta->ilbst_rlist[rule_index].ird_srvlist[srv_ind];
355 
356 	switch (of_arg->ofmt_id) {
357 	case ILBST_PKT_P: stat_ind = SRVST_PKT_P;
358 		break;
359 	case ILBST_BYTES_P: stat_ind = SRVST_BYTES_P;
360 		break;
361 	case ILBST_ITEMIZE_RNAME:
362 		(void) snprintf(buf, bufsize, "%s",
363 		    sta->ilbst_rlist[rule_index].ird_rulename);
364 		return (B_TRUE);
365 	case ILBST_ITEMIZE_SNAME:
366 		(void) snprintf(buf, bufsize, "%s", srv->isd_servername);
367 		return (B_TRUE);
368 	}
369 
370 	count = srv->isd_serverstats[stat_ind].is_value;
371 
372 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) != 0)
373 		goto out;
374 
375 	osrv = &sta->ilbst_rlist[rule_index].ird_srvlist_o[srv_ind];
376 	if (srv->isd_crtime != osrv->isd_crtime)
377 		ret = B_FALSE;
378 
379 	count -= osrv->isd_serverstats[stat_ind].is_value;
380 out:
381 	/*
382 	 * normally, we print "change per second", which we calculate
383 	 * here. otherwise, we print "change over interval" or absolute
384 	 * values.
385 	 */
386 	if ((sta->ilbst_flags & (ILBST_DELTA_INTERVAL|ILBST_ABS_NUMBERS)) == 0)
387 		count /= sta->ilbst_interval;
388 
389 	(void) snprintf(buf, bufsize, "%llu", count);
390 	return (ret);
391 
392 }
393 
394 static boolean_t
395 of_rule_stats(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
396 {
397 	ilbst_arg_t	*sta = (ilbst_arg_t *)of_arg->ofmt_cbarg;
398 	int		i, ind;
399 	uint64_t	count = 0;
400 
401 	switch (of_arg->ofmt_id) {
402 	case ILBST_PKT_U: ind = RLSTA_PKT_U;
403 		break;
404 	case ILBST_BYTES_U: ind = RLSTA_BYTES_U;
405 		break;
406 	case ILBST_PKT_D: ind = RLSTA_PKT_D;
407 		break;
408 	case ILBST_BYTES_D: ind = RLSTA_BYTES_D;
409 		break;
410 	case ILBST_ICMP_P: ind = RLSTA_ICMP_P;
411 		break;
412 	case ILBST_ICMP_D: ind = RLSTA_ICMP_D;
413 		break;
414 	case ILBST_ICMP2BIG_P: ind = RLSTA_ICMP2BIG_P;
415 		break;
416 	case ILBST_ICMP2BIG_D: ind = RLSTA_ICMP2BIG_D;
417 		break;
418 	case ILBST_NOMEMP_D: ind  = RLSTA_NOMEMPKT_D;
419 		break;
420 	case ILBST_NOPORTP_D: ind = RLSTA_NOPORTPKT_D;
421 		break;
422 	case ILBST_NOMEMB_D: ind = RLSTA_NOMEMBYTES_D;
423 		break;
424 	case ILBST_NOPORTB_D: ind = RLSTA_NOPORTBYTES_D;
425 		break;
426 	}
427 
428 	for (i = 0; i < sta->ilbst_rcount; i++)
429 		count += sta->ilbst_rlist[i].ird_rulestats[ind].is_value;
430 
431 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) != 0)
432 		goto out;
433 
434 	/*
435 	 * the purist approach: if we can't say 100% that what we
436 	 * calculate is correct, don't.
437 	 */
438 	if (sta->ilbst_flags & ILBST_RULES_CHANGED)
439 		return (B_FALSE);
440 
441 	for (i = 0; i < sta->ilbst_rcount; i++) {
442 		if (sta->ilbst_rlist[i].ird_crtime_o != 0 &&
443 		    sta->ilbst_rlist[i].ird_crtime !=
444 		    sta->ilbst_rlist[i].ird_crtime_o)
445 			return (B_FALSE);
446 
447 		count -= sta->ilbst_rlist[i].ird_rulestats_o[ind].is_value;
448 	}
449 out:
450 	/*
451 	 * normally, we print "change per second", which we calculate
452 	 * here. otherwise, we print "change over interval"
453 	 */
454 	if ((sta->ilbst_flags & (ILBST_DELTA_INTERVAL|ILBST_ABS_NUMBERS)) == 0)
455 		count /= sta->ilbst_interval;
456 
457 	(void) snprintf(buf, bufsize, "%llu", count);
458 	return (B_TRUE);
459 }
460 
461 /*
462  * Get the number of kstat instances. Note that when rules are being
463  * drained the number of kstats instances may be different than the
464  * kstat counter num_rules (ilb:0:global:num_rules").
465  *
466  * Also there can be multiple instances of a rule in the following
467  * scenario:
468  *
469  * A rule named rule A has been deleted but remains in kstats because
470  * its undergoing connection draining. During this time, the user adds
471  * a new rule with the same name(rule A). In this case, there would
472  * be two kstats instances for rule A. Currently ilbadm's aggregate
473  * results will include data from both instances of rule A. In,
474  * future we should have ilbadm stats only consider the latest instance
475  * of the rule (ie only consider the the instance that corresponds
476  * to the rule that was just added).
477  *
478  */
479 static int
480 i_get_num_kinstances(kstat_ctl_t *kctl)
481 {
482 	kstat_t		*kp;
483 	int		num_instances = 0; /* nothing found, 0 rules */
484 
485 	for (kp = kctl->kc_chain; kp != NULL; kp = kp->ks_next) {
486 		if (strncmp("rulestat", kp->ks_class, 8) == 0 &&
487 		    strncmp("ilb", kp->ks_module, 3) == 0) {
488 			num_instances++;
489 		}
490 	}
491 
492 	return (num_instances);
493 }
494 
495 
496 /*
497  * since server stat's classname is made up of <rulename>-sstat,
498  * we walk the rule list to construct the comparison
499  * Return:	pointer to rule whose name matches the class
500  *		NULL if no match
501  */
502 static ilbst_rule_desc_t *
503 match_2_rnames(char *class, ilbst_rule_desc_t *rlist, int rcount)
504 {
505 	int i;
506 	char	classname[KSTAT_STRLEN];
507 
508 	for (i = 0; i < rcount; i++) {
509 		(void) snprintf(classname, sizeof (classname), "%s-sstat",
510 		    rlist[i].ird_rulename);
511 		if (strncmp(classname, class, sizeof (classname)) == 0)
512 			return (&rlist[i]);
513 	}
514 	return (NULL);
515 }
516 
517 static int
518 i_stat_index(kstat_named_t *knp, ilbst_stat_t *stats, int count)
519 {
520 	int	i;
521 
522 	for (i = 0; i < count; i++) {
523 		if (strcasecmp(stats[i].is_name, knp->name) == 0)
524 			return (i);
525 	}
526 
527 	return (-1);
528 }
529 
530 static void
531 i_copy_sstats(ilbst_srv_desc_t *sp, kstat_t *kp)
532 {
533 	kstat_named_t	*knp;
534 	int		i, ind;
535 
536 	knp = KSTAT_NAMED_PTR(kp);
537 	for (i = 0; i < kp->ks_ndata; i++, knp++) {
538 		ind = i_stat_index(knp, servstats, SSTAT_SZ);
539 		if (ind == -1)
540 			continue;
541 		(void) strlcpy(sp->isd_serverstats[ind].is_name, knp->name,
542 		    sizeof (sp->isd_serverstats[ind].is_name));
543 		sp->isd_serverstats[ind].is_value = knp->value.ui64;
544 		sp->isd_crtime = kp->ks_crtime;
545 	}
546 }
547 
548 
549 static ilbadm_status_t
550 i_get_server_descs(ilbst_arg_t *sta, kstat_ctl_t *kctl)
551 {
552 	ilbadm_status_t	rc = ILBADM_OK;
553 	kstat_t		*kp;
554 	int		i = -1;
555 	ilbst_rule_desc_t	*rp;
556 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
557 	int			rcount = sta->ilbst_rcount;
558 
559 	/*
560 	 * find all "server" kstats, or the one specified in
561 	 * sta->server
562 	 */
563 	for (kp = kctl->kc_chain; kp != NULL; kp = kp->ks_next) {
564 		if (strncmp("ilb", kp->ks_module, 3) != 0)
565 			continue;
566 		if (sta->ilbst_server != NULL &&
567 		    strcasecmp(sta->ilbst_server, kp->ks_name) != 0)
568 			continue;
569 		rp = match_2_rnames(kp->ks_class, rlist, rcount);
570 		if (rp == NULL)
571 			continue;
572 
573 		(void) kstat_read(kctl, kp, NULL);
574 		i = rp->ird_srv_ind++;
575 
576 		rc = ILBADM_OK;
577 		/*
578 		 * This means that a server is added after we check last
579 		 * time...  Just make the array bigger.
580 		 */
581 		if (i+1 > rp->ird_num_servers) {
582 			ilbst_srv_desc_t  *srvlist;
583 
584 			if ((srvlist = realloc(rp->ird_srvlist, (i+1) *
585 			    sizeof (*srvlist))) == NULL) {
586 				rc = ILBADM_ENOMEM;
587 				break;
588 			}
589 			rp->ird_srvlist = srvlist;
590 			rp->ird_num_servers = i;
591 		}
592 
593 		(void) strlcpy(rp->ird_srvlist[i].isd_servername, kp->ks_name,
594 		    sizeof (rp->ird_srvlist[i].isd_servername));
595 		i_copy_sstats(&rp->ird_srvlist[i], kp);
596 	}
597 
598 	for (i = 0; i < rcount; i++)
599 		rlist[i].ird_srv_ind = 0;
600 
601 	if (sta->ilbst_server != NULL && i == -1)
602 		rc = ILBADM_ENOSERVER;
603 	return (rc);
604 }
605 
606 static void
607 i_copy_rstats(ilbst_rule_desc_t *rp, kstat_t *kp)
608 {
609 	kstat_named_t	*knp;
610 	int		i, ind;
611 
612 	knp = KSTAT_NAMED_PTR(kp);
613 	for (i = 0; i < kp->ks_ndata; i++, knp++) {
614 		ind = i_stat_index(knp, rulestats, RSTAT_SZ);
615 		if (ind == -1)
616 			continue;
617 
618 		(void) strlcpy(rp->ird_rulestats[ind].is_name, knp->name,
619 		    sizeof (rp->ird_rulestats[ind].is_name));
620 		rp->ird_rulestats[ind].is_value = knp->value.ui64;
621 	}
622 }
623 
624 static void
625 i_set_rlstats_ptr(ilbst_rule_desc_t *rp, boolean_t old_is_old)
626 {
627 	if (old_is_old) {
628 		rp->ird_rulestats = rp->ird_rstats;
629 		rp->ird_rulestats_o = rp->ird_rstats_o;
630 	} else {
631 		rp->ird_rulestats = rp->ird_rstats_o;
632 		rp->ird_rulestats_o = rp->ird_rstats;
633 	}
634 }
635 /*
636  * this function walks the array of rules and switches pointer to old
637  * and new stats as well as serverlists.
638  */
639 static void
640 i_swap_rl_pointers(ilbst_arg_t *sta, int rcount)
641 {
642 	int			i, tmp_num;
643 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
644 	ilbst_srv_desc_t	*tmp_srv;
645 
646 	for (i = 0; i < rcount; i++) {
647 		/* swap srvlist pointers */
648 		tmp_srv = rlist[i].ird_srvlist;
649 		rlist[i].ird_srvlist = rlist[i].ird_srvlist_o;
650 		rlist[i].ird_srvlist_o = tmp_srv;
651 
652 		/*
653 		 * swap server counts - we need the old one to
654 		 * save reallocation calls
655 		 */
656 		tmp_num = rlist[i].ird_num_servers_o;
657 		rlist[i].ird_num_servers_o = rlist[i].ird_num_servers;
658 		rlist[i].ird_num_servers = tmp_num;
659 
660 		/* preserve creation time */
661 		rlist[i].ird_crtime_o = rlist[i].ird_crtime;
662 
663 		i_set_rlstats_ptr(&rlist[i], sta->ilbst_old_is_old);
664 		rlist[i].ird_srv_ind = 0;
665 	}
666 }
667 
668 static void
669 i_init_rulelist(ilbst_arg_t *sta, int rcount)
670 {
671 	int			 i;
672 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
673 
674 	for (i = 0; i < rcount; i++) {
675 		rlist[i].ird_rulestats = rlist[i].ird_rstats;
676 		rlist[i].ird_rulestats_o = rlist[i].ird_rstats_o;
677 		rlist[i].ird_srv_ind = 0;
678 	}
679 }
680 
681 
682 /*
683  * this function searches for kstats describing individual rules and
684  * saves name, # of servers, and the kstat_t * describing them (this is
685  * for sta->rulename == NULL);
686  * if sta->rulename != NULL, it names the rule we're looking for
687  * and this function will fill in the other data (like the all_rules case)
688  * Returns:	ILBADM_ENORULE	named rule not found
689  *		ILBADM_ENOMEM	no mem. available
690  */
691 static ilbadm_status_t
692 i_get_rule_descs(ilbst_arg_t *sta, kstat_ctl_t *kctl)
693 {
694 	ilbadm_status_t	rc = ILBADM_OK;
695 	kstat_t		*kp;
696 	kstat_named_t	*knp;
697 	int		i;
698 	int		num_servers;
699 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
700 	int		rcount = sta->ilbst_rcount;
701 
702 	/*
703 	 * find all "rule" kstats, or the one specified in
704 	 * sta->ilbst_rulename.
705 	 */
706 	for (i = 0, kp = kctl->kc_chain; i < rcount && kp != NULL;
707 	    kp = kp->ks_next) {
708 		if (strncmp("rulestat", kp->ks_class, 8) != 0 ||
709 		    strncmp("ilb", kp->ks_module, 3) != 0)
710 			continue;
711 
712 		(void) kstat_read(kctl, kp, NULL);
713 
714 		knp = kstat_data_lookup(kp, "num_servers");
715 		if (knp == NULL) {
716 			ilbadm_err(gettext("kstat_data_lookup() failed: %s"),
717 			    strerror(errno));
718 			rc = ILBADM_LIBERR;
719 			break;
720 		}
721 		if (sta->ilbst_rulename != NULL) {
722 			if (strcasecmp(kp->ks_name, sta->ilbst_rulename)
723 			    != 0)
724 				continue;
725 		}
726 		(void) strlcpy(rlist[i].ird_rulename, kp->ks_name,
727 		    sizeof (rlist[i].ird_rulename));
728 
729 		/* only alloc the space we need, set counter here ... */
730 		if (sta->ilbst_server != NULL)
731 			num_servers = 1;
732 		else
733 			num_servers = (int)knp->value.ui64;
734 
735 		/* ... furthermore, only reallocate if necessary */
736 		if (num_servers != rlist[i].ird_num_servers) {
737 			ilbst_srv_desc_t  *srvlist;
738 
739 			rlist[i].ird_num_servers = num_servers;
740 
741 			if (rlist[i].ird_srvlist == NULL)
742 				srvlist = calloc(num_servers,
743 				    sizeof (*srvlist));
744 			else
745 				srvlist = realloc(rlist[i].ird_srvlist,
746 				    sizeof (*srvlist) * num_servers);
747 			if (srvlist == NULL) {
748 				rc = ILBADM_ENOMEM;
749 				break;
750 			}
751 			rlist[i].ird_srvlist = srvlist;
752 		}
753 		rlist[i].ird_srv_ind = 0;
754 		rlist[i].ird_crtime = kp->ks_crtime;
755 
756 		i_copy_rstats(&rlist[i], kp);
757 		i++;
758 
759 		/* if we know we're done, return */
760 		if (sta->ilbst_rulename != NULL || i == rcount) {
761 			rc = ILBADM_OK;
762 			break;
763 		}
764 	}
765 
766 	if (sta->ilbst_rulename != NULL && i == 0)
767 		rc = ILBADM_ENORULE;
768 	return (rc);
769 }
770 
771 static void
772 i_do_print(ilbst_arg_t *sta)
773 {
774 	int	i;
775 
776 	/* non-itemized display can go right ahead */
777 	if ((sta->ilbst_flags & ILBST_ITEMIZE) == 0) {
778 		ofmt_print(sta->ilbst_oh, sta);
779 		return;
780 	}
781 
782 	/*
783 	 * rulename is given, list a line per server
784 	 * here's how we do it:
785 	 *	the _ITEMIZE flag indicates to the print function (called
786 	 *	from ofmt_print()) to look at server [ird_srv_ind] only.
787 	 */
788 	if (sta->ilbst_rulename != NULL) {
789 		sta->ilbst_rule_index = 0;
790 		for (i = 0; i < sta->ilbst_rlist->ird_num_servers; i++) {
791 			sta->ilbst_rlist->ird_srv_ind = i;
792 			ofmt_print(sta->ilbst_oh, sta);
793 		}
794 		sta->ilbst_rlist->ird_srv_ind = 0;
795 		return;
796 	}
797 
798 	/* list one line for every rule for a given server */
799 	for (i = 0; i < sta->ilbst_rcount; i++) {
800 		/*
801 		 * if a rule doesn't contain a given server, there's no
802 		 * need to print it. Luckily, we can check that
803 		 * fairly easily
804 		 */
805 		if (sta->ilbst_rlist[i].ird_srvlist[0].isd_servername[0] ==
806 		    '\0')
807 			continue;
808 
809 		sta->ilbst_rule_index = i;
810 		sta->ilbst_rlist[i].ird_srv_ind = 0;
811 		ofmt_print(sta->ilbst_oh, sta);
812 	}
813 	sta->ilbst_rule_index = 0;
814 }
815 
816 static ilbadm_status_t
817 i_do_show_stats(ilbst_arg_t *sta)
818 {
819 	kstat_ctl_t	*kctl;
820 	kid_t		nkid;
821 	int		rcount = 1, i;
822 	ilbadm_status_t	rc = ILBADM_OK;
823 	ilbst_rule_desc_t	*rlist, *rp;
824 	boolean_t	pseudo_abs = B_FALSE; /* for first pass */
825 
826 	if ((kctl = kstat_open()) == NULL) {
827 		ilbadm_err(gettext("kstat_open() failed: %s"), strerror(errno));
828 		return (ILBADM_LIBERR);
829 	}
830 
831 
832 	if (sta->ilbst_rulename == NULL)
833 		rcount = i_get_num_kinstances(kctl);
834 
835 	rlist = calloc(sizeof (*rlist), rcount);
836 	if (rlist == NULL) {
837 		rc = ILBADM_ENOMEM;
838 		goto out;
839 	}
840 
841 	sta->ilbst_old_is_old = B_TRUE;
842 	sta->ilbst_rlist = rlist;
843 	sta->ilbst_rcount = sta->ilbst_rcount_prev = rcount;
844 	sta->ilbst_rlist_sz = rcount;
845 
846 	/*
847 	 * in the first pass, we always print absolute numbers. We
848 	 * need to remember whether we wanted abs. numbers for
849 	 * other samples as well
850 	 */
851 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) == 0) {
852 		sta->ilbst_flags |= ILBST_ABS_NUMBERS;
853 		pseudo_abs = B_TRUE;
854 	}
855 
856 	i_init_rulelist(sta, rcount);
857 	do {
858 		rc = i_get_rule_descs(sta, kctl);
859 		if (rc != ILBADM_OK)
860 			goto out;
861 
862 		rc = i_get_server_descs(sta, kctl);
863 		if (rc != ILBADM_OK)
864 			goto out;
865 
866 		i_do_print(sta);
867 
868 		if (sta->ilbst_count == -1 || --(sta->ilbst_count) > 0)
869 			(void) sleep(sta->ilbst_interval);
870 		else
871 			break;
872 
873 		nkid = kstat_chain_update(kctl);
874 		sta->ilbst_flags &= ~ILBST_RULES_CHANGED;
875 		/*
876 		 * we only need to continue with most of the rest of this if
877 		 * the kstat chain id has changed
878 		 */
879 		if (nkid == 0)
880 			goto swap_old_new;
881 		if (nkid == -1) {
882 			ilbadm_err(gettext("kstat_chain_update() failed: %s"),
883 			    strerror(errno));
884 			rc = ILBADM_LIBERR;
885 			break;
886 		}
887 
888 		/*
889 		 * find out whether the number of rules has changed.
890 		 * if so, adjust rcount and _o; if number has increased,
891 		 * expand array to hold all rules.
892 		 * we only shrink if rlist_sz is larger than both rcount and
893 		 * rcount_prev;
894 		 */
895 		if (sta->ilbst_rulename == NULL)
896 			rcount = i_get_num_kinstances(kctl);
897 		if (rcount != sta->ilbst_rcount) {
898 			sta->ilbst_flags |= ILBST_RULES_CHANGED;
899 			sta->ilbst_rcount_prev = sta->ilbst_rcount;
900 			sta->ilbst_rcount = rcount;
901 
902 			if (rcount > sta->ilbst_rcount_prev) {
903 				rlist = realloc(sta->ilbst_rlist,
904 				    sizeof (*sta->ilbst_rlist) * rcount);
905 				if (rlist == NULL) {
906 					rc = ILBADM_ENOMEM;
907 					break;
908 				}
909 				sta->ilbst_rlist = rlist;
910 				/* realloc doesn't zero out memory */
911 				for (i = sta->ilbst_rcount_prev;
912 				    i < rcount; i++) {
913 					rp = &sta->ilbst_rlist[i];
914 					bzero(rp, sizeof (*rp));
915 					i_set_rlstats_ptr(rp,
916 					    sta->ilbst_old_is_old);
917 				}
918 				/*
919 				 * even if rlist_sz was > rcount, it's now
920 				 * shrunk to rcount
921 				 */
922 				sta->ilbst_rlist_sz = sta->ilbst_rcount;
923 			}
924 		}
925 
926 		/*
927 		 * we may need to shrink the allocated slots down to the
928 		 * actually required number - we need to make sure we
929 		 * don't delete old or new stats.
930 		 */
931 		if (sta->ilbst_rlist_sz > MAX(sta->ilbst_rcount,
932 		    sta->ilbst_rcount_prev)) {
933 			sta->ilbst_rlist_sz =
934 			    MAX(sta->ilbst_rcount, sta->ilbst_rcount_prev);
935 			rlist = realloc(sta->ilbst_rlist,
936 			    sizeof (*sta->ilbst_rlist) * sta->ilbst_rlist_sz);
937 			if (rlist == NULL) {
938 				rc = ILBADM_ENOMEM;
939 				break;
940 			}
941 			sta->ilbst_rlist = rlist;
942 		}
943 
944 		/*
945 		 * move pointers around so what used to point to "old"
946 		 * stats now points to new, and vice versa
947 		 * if we're printing absolute numbers, this rigmarole is
948 		 * not necessary.
949 		 */
950 swap_old_new:
951 		if (pseudo_abs)
952 			sta->ilbst_flags &= ~ILBST_ABS_NUMBERS;
953 
954 		if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) == 0) {
955 			sta->ilbst_old_is_old = !sta->ilbst_old_is_old;
956 			i_swap_rl_pointers(sta, rcount);
957 		}
958 		_NOTE(CONSTCOND)
959 	} while (B_TRUE);
960 
961 out:
962 	(void) kstat_close(kctl);
963 	if ((rc != ILBADM_OK) && (rc != ILBADM_LIBERR))
964 		ilbadm_err(ilbadm_errstr(rc));
965 
966 	if (sta->ilbst_rlist != NULL)
967 		free(sta->ilbst_rlist);
968 
969 	return (rc);
970 }
971 
972 /*
973  * read ilb's kernel statistics and (periodically) display
974  * them.
975  */
976 /* ARGSUSED */
977 ilbadm_status_t
978 ilbadm_show_stats(int argc, char *argv[])
979 {
980 	ilbadm_status_t	rc;
981 	int		c;
982 	ilbst_arg_t	sta;
983 	int		oflags = 0;
984 	char		*fieldnames = stat_stdhdrs;
985 	ofmt_field_t	*fields = stat_stdfields;
986 	boolean_t	r_opt = B_FALSE, s_opt = B_FALSE, i_opt = B_FALSE;
987 	boolean_t	o_opt = B_FALSE, p_opt = B_FALSE, t_opt = B_FALSE;
988 	boolean_t	v_opt = B_FALSE, A_opt = B_FALSE, d_opt = B_FALSE;
989 	ofmt_status_t	oerr;
990 	ofmt_handle_t	oh = NULL;
991 
992 	bzero(&sta, sizeof (sta));
993 	sta.ilbst_interval = 1;
994 	sta.ilbst_count = 1;
995 
996 	while ((c = getopt(argc, argv, ":tdAr:s:ivo:p")) != -1) {
997 		switch ((char)c) {
998 		case 't': sta.ilbst_flags |= ILBST_TIMESTAMP_HEADER;
999 			t_opt = B_TRUE;
1000 			break;
1001 		case 'd': sta.ilbst_flags |= ILBST_DELTA_INTERVAL;
1002 			d_opt = B_TRUE;
1003 			break;
1004 		case 'A': sta.ilbst_flags |= ILBST_ABS_NUMBERS;
1005 			A_opt = B_TRUE;
1006 			break;
1007 		case 'r': sta.ilbst_rulename = optarg;
1008 			r_opt = B_TRUE;
1009 			break;
1010 		case 's': sta.ilbst_server = optarg;
1011 			s_opt = B_TRUE;
1012 			break;
1013 		case 'i': sta.ilbst_flags |= ILBST_ITEMIZE;
1014 			i_opt = B_TRUE;
1015 			break;
1016 		case 'o': fieldnames = optarg;
1017 			o_opt = B_TRUE;
1018 			break;
1019 		case 'p': oflags |= OFMT_PARSABLE;
1020 			p_opt = B_TRUE;
1021 			break;
1022 		case 'v': sta.ilbst_flags |= ILBST_VERBOSE;
1023 			v_opt = B_TRUE;
1024 			fieldnames = stat_stdv_hdrs;
1025 			break;
1026 		case ':': ilbadm_err(gettext("missing option-argument"
1027 			    " detected for %c"), (char)optopt);
1028 			exit(1);
1029 			/* not reached */
1030 			break;
1031 		case '?': /* fallthrough */
1032 		default:
1033 			unknown_opt(argv, optind-1);
1034 			/* not reached */
1035 			break;
1036 		}
1037 	}
1038 
1039 	if (s_opt && r_opt) {
1040 		ilbadm_err(gettext("options -s and -r are mutually exclusive"));
1041 		exit(1);
1042 	}
1043 
1044 	if (i_opt) {
1045 		if (!(s_opt || r_opt)) {
1046 			ilbadm_err(gettext("option -i requires"
1047 			    " either -r or -s"));
1048 			exit(1);
1049 		}
1050 		if (v_opt) {
1051 			ilbadm_err(gettext("option -i and -v are mutually"
1052 			    " exclusive"));
1053 			exit(1);
1054 		}
1055 		/* only use "std" headers if none are specified */
1056 		if (!o_opt)
1057 			if (r_opt)
1058 				fieldnames = stat_itemize_rule_hdrs;
1059 			else /* must be s_opt */
1060 				fieldnames = stat_itemize_server_hdrs;
1061 		fields = stat_itemize_fields;
1062 	}
1063 
1064 	if (p_opt) {
1065 		if (!o_opt) {
1066 			ilbadm_err(gettext("option -p requires -o"));
1067 			exit(1);
1068 		}
1069 		if (v_opt) {
1070 			ilbadm_err(gettext("option -o and -v are mutually"
1071 			    " exclusive"));
1072 			exit(1);
1073 		}
1074 		if (strcasecmp(fieldnames, "all") == 0) {
1075 			ilbadm_err(gettext("option -p requires"
1076 			    " explicit field names"));
1077 			exit(1);
1078 		}
1079 	}
1080 
1081 	if (t_opt) {
1082 		if (v_opt) {
1083 			fieldnames = "all";
1084 		} else {
1085 			int  len = strlen(fieldnames) + 6;
1086 			char *fnames;
1087 
1088 			fnames = malloc(len);
1089 			if (fnames == NULL) {
1090 				rc = ILBADM_ENOMEM;
1091 				return (rc);
1092 			}
1093 			(void) snprintf(fnames, len, "%s,TIME", fieldnames);
1094 			fieldnames = fnames;
1095 		}
1096 	}
1097 
1098 	if (A_opt && d_opt) {
1099 		ilbadm_err(gettext("options -d and -A are mutually exclusive"));
1100 		exit(1);
1101 	}
1102 
1103 	/* find and parse interval and count arguments if present */
1104 	if (optind < argc) {
1105 		sta.ilbst_interval = atoi(argv[optind]);
1106 		if (sta.ilbst_interval < 1) {
1107 			ilbadm_err(gettext("illegal interval spec %s"),
1108 			    argv[optind]);
1109 			exit(1);
1110 		}
1111 		sta.ilbst_count = -1;
1112 		if (++optind < argc) {
1113 			sta.ilbst_count = atoi(argv[optind]);
1114 			if (sta.ilbst_count < 1) {
1115 				ilbadm_err(gettext("illegal count spec %s"),
1116 				    argv[optind]);
1117 				exit(1);
1118 			}
1119 		}
1120 	}
1121 
1122 	oerr = ofmt_open(fieldnames, fields, oflags, 80, &oh);
1123 	if (oerr != OFMT_SUCCESS) {
1124 		char	e[80];
1125 
1126 		ilbadm_err(gettext("ofmt_open failed: %s"),
1127 		    ofmt_strerror(oh, oerr, e, sizeof (e)));
1128 		return (ILBADM_LIBERR);
1129 	}
1130 
1131 	sta.ilbst_oh = oh;
1132 
1133 	rc = i_do_show_stats(&sta);
1134 
1135 	ofmt_close(oh);
1136 	return (rc);
1137 }
1138