xref: /titanic_52/usr/src/cmd/cmd-inet/usr.sbin/ilbadm/ilbadm_stats.c (revision 68c47f65208790c466e5e484f2293d3baed71c6a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <stdio.h>
28 #include <unistd.h>
29 #include <stdlib.h>
30 #include <strings.h>
31 #include <sys/types.h>
32 #include <sys/socket.h>
33 #include <sys/sysmacros.h>
34 #include <sys/note.h>
35 #include <fcntl.h>
36 #include <errno.h>
37 #include <assert.h>
38 #include <libgen.h>
39 #include <kstat.h>
40 #include <ofmt.h>
41 #include <libilb.h>
42 #include "ilbadm.h"
43 
44 #define	ILBST_TIMESTAMP_HEADER	0x01	/* a timestamp w. every header */
45 #define	ILBST_DELTA_INTERVAL	0x02	/* delta over specified interval */
46 #define	ILBST_ABS_NUMBERS	0x04	/* print absolute numbers, no d's */
47 #define	ILBST_ITEMIZE		0x08	/* itemize */
48 #define	ILBST_VERBOSE		0x10	/* verbose error info */
49 
50 #define	ILBST_OLD_VALUES	0x20	/* for internal processing */
51 #define	ILBST_RULES_CHANGED	0x40
52 
53 typedef struct {
54 	char		is_name[KSTAT_STRLEN];
55 	uint64_t	is_value;
56 } ilbst_stat_t;
57 
58 static ilbst_stat_t rulestats[] = {
59 	{"num_servers", 0},
60 	{"bytes_not_processed", 0},
61 	{"pkt_not_processed", 0},
62 	{"bytes_dropped", 0},
63 	{"pkt_dropped", 0},
64 	{"nomem_bytes_dropped", 0},
65 	{"nomem_pkt_dropped", 0},
66 	{"noport_bytes_dropped", 0},
67 	{"noport_pkt_dropped", 0},
68 	{"icmp_echo_processed", 0},
69 	{"icmp_dropped", 0},
70 	{"icmp_too_big_processed", 0},
71 	{"icmp_too_big_dropped", 0}
72 };
73 
74 /* indices into array above, to avoid searching */
75 #define	RLSTA_NUM_SRV		0
76 #define	RLSTA_BYTES_U		1
77 #define	RLSTA_PKT_U		2
78 #define	RLSTA_BYTES_D		3
79 #define	RLSTA_PKT_D		4
80 #define	RLSTA_NOMEMBYTES_D	5
81 #define	RLSTA_NOMEMPKT_D	6
82 #define	RLSTA_NOPORTBYTES_D	7
83 #define	RLSTA_NOPORTPKT_D	8
84 #define	RLSTA_ICMP_P		9
85 #define	RLSTA_ICMP_D		10
86 #define	RLSTA_ICMP2BIG_P	11
87 #define	RLSTA_ICMP2BIG_D	12
88 
89 static ilbst_stat_t servstats[] = {
90 	{"bytes_processed", 0},
91 	{"pkt_processed", 0}
92 };
93 /* indices into array above, to avoid searching */
94 #define	SRVST_BYTES_P	0
95 #define	SRVST_PKT_P	1
96 
97 /* values used for of_* commands as id */
98 #define	ILBST_PKT_P		0
99 #define	ILBST_BYTES_P		1
100 #define	ILBST_PKT_U		2
101 #define	ILBST_BYTES_U		3
102 #define	ILBST_PKT_D		4
103 #define	ILBST_BYTES_D		5
104 #define	ILBST_ICMP_P		6
105 #define	ILBST_ICMP_D		7
106 #define	ILBST_ICMP2BIG_P	8
107 #define	ILBST_ICMP2BIG_D	9
108 #define	ILBST_NOMEMP_D		10
109 #define	ILBST_NOPORTP_D		11
110 #define	ILBST_NOMEMB_D		12
111 #define	ILBST_NOPORTB_D		13
112 
113 #define	ILBST_ITEMIZE_SNAME	97
114 #define	ILBST_ITEMIZE_RNAME	98
115 #define	ILBST_TIMESTAMP		99
116 
117 /* approx field widths */
118 #define	ILBST_PKTCTR_W		8
119 #define	ILBST_BYTECTR_W		10
120 #define	ILBST_TIME_W		15
121 
122 static boolean_t of_rule_stats(ofmt_arg_t *, char *, uint_t);
123 static boolean_t of_server_stats(ofmt_arg_t *, char *, uint_t);
124 static boolean_t of_itemize_stats(ofmt_arg_t *, char *, uint_t);
125 static boolean_t of_timestamp(ofmt_arg_t *, char *, uint_t);
126 
127 static ofmt_field_t stat_itemize_fields[] = {
128 	{"RULENAME", ILB_NAMESZ,	ILBST_ITEMIZE_RNAME, of_itemize_stats},
129 	{"SERVERNAME", ILB_NAMESZ,	ILBST_ITEMIZE_SNAME, of_itemize_stats},
130 	{"PKT_P",   ILBST_PKTCTR_W,	ILBST_PKT_P, of_itemize_stats},
131 	{"BYTES_P", ILBST_BYTECTR_W,	ILBST_BYTES_P, of_itemize_stats},
132 	{"TIME",    ILBST_TIME_W,	ILBST_TIMESTAMP, of_timestamp},
133 	{NULL,	    0, 0, NULL}
134 };
135 static ofmt_field_t stat_stdfields[] = {
136 	{"PKT_P",   ILBST_PKTCTR_W,	ILBST_PKT_P, of_server_stats},
137 	{"BYTES_P", ILBST_BYTECTR_W,	ILBST_BYTES_P, of_server_stats},
138 	{"PKT_U",   ILBST_PKTCTR_W,	ILBST_PKT_U, of_rule_stats},
139 	{"BYTES_U", ILBST_BYTECTR_W,	ILBST_BYTES_U, of_rule_stats},
140 	{"PKT_D",   ILBST_PKTCTR_W,	ILBST_PKT_D, of_rule_stats},
141 	{"BYTES_D", ILBST_BYTECTR_W,	ILBST_BYTES_D, of_rule_stats},
142 	{"ICMP_P",  ILBST_PKTCTR_W,	ILBST_ICMP_P, of_rule_stats},
143 	{"ICMP_D",  ILBST_PKTCTR_W,	ILBST_ICMP_D, of_rule_stats},
144 	{"ICMP2BIG_P", 11,		ILBST_ICMP2BIG_P, of_rule_stats},
145 	{"ICMP2BIG_D", 11,		ILBST_ICMP2BIG_D, of_rule_stats},
146 	{"NOMEMP_D", ILBST_PKTCTR_W,	ILBST_NOMEMP_D, of_rule_stats},
147 	{"NOPORTP_D", ILBST_PKTCTR_W,	ILBST_NOPORTP_D, of_rule_stats},
148 	{"NOMEMB_D", ILBST_PKTCTR_W,	ILBST_NOMEMB_D, of_rule_stats},
149 	{"NOPORTB_D", ILBST_PKTCTR_W,	ILBST_NOPORTB_D, of_rule_stats},
150 	{"TIME",    ILBST_TIME_W,	ILBST_TIMESTAMP, of_timestamp},
151 	{NULL,	    0, 0, NULL}
152 };
153 
154 static char stat_stdhdrs[] = "PKT_P,BYTES_P,PKT_U,BYTES_U,PKT_D,BYTES_D";
155 static char stat_stdv_hdrs[] = "PKT_P,BYTES_P,PKT_U,BYTES_U,PKT_D,BYTES_D,"
156 	"ICMP_P,ICMP_D,ICMP2BIG_P,ICMP2BIG_D,NOMEMP_D,NOPORTP_D";
157 static char stat_itemize_rule_hdrs[] = "SERVERNAME,PKT_P,BYTES_P";
158 static char stat_itemize_server_hdrs[] = "RULENAME,PKT_P,BYTES_P";
159 
160 #define	RSTAT_SZ	(sizeof (rulestats)/sizeof (rulestats[0]))
161 #define	SSTAT_SZ	(sizeof (servstats)/sizeof (servstats[0]))
162 
163 typedef struct {
164 	char		isd_servername[KSTAT_STRLEN]; /* serverID */
165 	ilbst_stat_t	isd_serverstats[SSTAT_SZ];
166 	hrtime_t	isd_crtime;	/* save for comparison purpose */
167 } ilbst_srv_desc_t;
168 
169 /*
170  * this data structure stores statistics for a rule - both an old set
171  * and a current/new set. we use pointers to the actual stores and switch
172  * the pointers for every round. old_is_old in ilbst_arg_t indicates
173  * which pointer points to the "old" data struct (ie, if true, _o pointer
174  * points to old)
175  */
176 typedef struct {
177 	char			ird_rulename[KSTAT_STRLEN];
178 	int			ird_num_servers;
179 	int			ird_num_servers_o;
180 	int			ird_srv_ind;
181 	hrtime_t		ird_crtime;	/* save for comparison */
182 	hrtime_t		ird_crtime_o;	/* save for comparison */
183 	ilbst_srv_desc_t	*ird_srvlist;
184 	ilbst_srv_desc_t	*ird_srvlist_o;
185 	ilbst_stat_t		ird_rstats[RSTAT_SZ];
186 	ilbst_stat_t		ird_rstats_o[RSTAT_SZ];
187 	ilbst_stat_t		*ird_rulestats;
188 	ilbst_stat_t		*ird_rulestats_o;
189 } ilbst_rule_desc_t;
190 
191 /*
192  * overall "container" for information pertaining to statistics, and
193  * how to display them.
194  */
195 typedef struct {
196 	int			ilbst_flags;
197 	/* fields representing user input */
198 	char			*ilbst_rulename;	/* optional */
199 	char 			*ilbst_server;	/* optional */
200 	int			ilbst_interval;
201 	int			ilbst_count;
202 	/* "internal" fields for data and data presentation */
203 	ofmt_handle_t		ilbst_oh;
204 	boolean_t		ilbst_old_is_old;
205 	ilbst_rule_desc_t	*ilbst_rlist;
206 	int			ilbst_rcount;	  /* current list count */
207 	int			ilbst_rcount_prev; /* prev (different) count */
208 	int			ilbst_rlist_sz; /* number of alloc'ed rules */
209 	int			ilbst_rule_index; /* for itemizes display */
210 } ilbst_arg_t;
211 
212 /* ARGSUSED */
213 static boolean_t
214 of_timestamp(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
215 {
216 	time_t		now;
217 	struct tm	*now_tm;
218 
219 	now = time(NULL);
220 	now_tm = localtime(&now);
221 
222 	(void) strftime(buf, bufsize, "%F:%H.%M.%S", now_tm);
223 	return (B_TRUE);
224 }
225 
226 static boolean_t
227 i_sum_per_rule_processed(ilbst_rule_desc_t *rp, uint64_t *resp, int index,
228     int flags)
229 {
230 	int			i, num_servers;
231 	ilbst_srv_desc_t	*srv, *o_srv, *n_srv;
232 	uint64_t		res = 0;
233 	boolean_t		valid = B_TRUE;
234 	boolean_t		old = flags & ILBST_OLD_VALUES;
235 	boolean_t		check_valid;
236 
237 	/* if we do abs. numbers, we never look at the _o fields */
238 	assert((old && (flags & ILBST_ABS_NUMBERS)) == B_FALSE);
239 
240 	/* we only check for validity under certain conditions */
241 	check_valid = !(old || (flags & ILBST_ABS_NUMBERS));
242 
243 	if (check_valid && rp->ird_num_servers != rp->ird_num_servers_o)
244 		valid = B_FALSE;
245 
246 	num_servers = old ? rp->ird_num_servers_o : rp->ird_num_servers;
247 
248 	for (i = 0; i < num_servers; i++) {
249 		n_srv = &rp->ird_srvlist[i];
250 		o_srv = &rp->ird_srvlist_o[i];
251 
252 		if (old)
253 			srv = o_srv;
254 		else
255 			srv = n_srv;
256 
257 		res += srv->isd_serverstats[index].is_value;
258 		/*
259 		 * if creation times don't match, comparison is wrong; if
260 		 * if we already know something is invalid, we don't
261 		 * need to compare again.
262 		 */
263 		if (check_valid && valid == B_TRUE &&
264 		    o_srv->isd_crtime != n_srv->isd_crtime) {
265 			valid = B_FALSE;
266 			break;
267 		}
268 	}
269 	/*
270 	 * save the result even though it may be imprecise  - let the
271 	 * caller decide what to do
272 	 */
273 	*resp = res;
274 
275 	return (valid);
276 }
277 
278 typedef boolean_t (*sumfunc_t)(ilbst_rule_desc_t *, uint64_t *, int);
279 
280 static boolean_t
281 i_sum_per_rule_pkt_p(ilbst_rule_desc_t *rp, uint64_t *resp, int flags)
282 {
283 	return (i_sum_per_rule_processed(rp, resp, SRVST_PKT_P, flags));
284 }
285 
286 static boolean_t
287 i_sum_per_rule_bytes_p(ilbst_rule_desc_t *rp, uint64_t *resp, int flags)
288 {
289 	return (i_sum_per_rule_processed(rp, resp, SRVST_BYTES_P, flags));
290 }
291 
292 static boolean_t
293 of_server_stats(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
294 {
295 	ilbst_arg_t	*sta = (ilbst_arg_t *)of_arg->ofmt_cbarg;
296 	uint64_t	count = 0, val;
297 	int		i;
298 	boolean_t	valid = B_TRUE;
299 	sumfunc_t	sumfunc;
300 
301 	switch (of_arg->ofmt_id) {
302 	case ILBST_PKT_P: sumfunc = i_sum_per_rule_pkt_p;
303 		break;
304 	case ILBST_BYTES_P: sumfunc = i_sum_per_rule_bytes_p;
305 		break;
306 	}
307 
308 	for (i = 0; i < sta->ilbst_rcount; i++) {
309 		valid = sumfunc(&sta->ilbst_rlist[i], &val, sta->ilbst_flags);
310 		if (!valid)
311 			return (valid);
312 		count += val;
313 	}
314 
315 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) != 0)
316 		goto out;
317 
318 	for (i = 0; i < sta->ilbst_rcount; i++) {
319 		(void) sumfunc(&sta->ilbst_rlist[i], &val,
320 		    sta->ilbst_flags | ILBST_OLD_VALUES);
321 		count -= val;
322 	}
323 
324 out:
325 	/*
326 	 * normally, we print "change per second", which we calculate
327 	 * here. otherwise, we print "change over interval"
328 	 */
329 	if ((sta->ilbst_flags & (ILBST_DELTA_INTERVAL|ILBST_ABS_NUMBERS)) == 0)
330 		count /= sta->ilbst_interval;
331 
332 	(void) snprintf(buf, bufsize, "%llu", count);
333 	return (B_TRUE);
334 }
335 
336 /*
337  * this function is called when user wants itemized stats of every
338  * server for a named rule, or vice vera.
339  * i_do_print sets sta->rule_index and the proper ird_srv_ind so
340  * we don't have to differentiate between these two cases here.
341  */
342 static boolean_t
343 of_itemize_stats(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
344 {
345 	ilbst_arg_t	*sta = (ilbst_arg_t *)of_arg->ofmt_cbarg;
346 	int		stat_ind;
347 	uint64_t	count;
348 	int		rule_index = sta->ilbst_rule_index;
349 	int		srv_ind = sta->ilbst_rlist[rule_index].ird_srv_ind;
350 	boolean_t	ret = B_TRUE;
351 	ilbst_srv_desc_t *srv, *osrv;
352 
353 	srv = &sta->ilbst_rlist[rule_index].ird_srvlist[srv_ind];
354 
355 	switch (of_arg->ofmt_id) {
356 	case ILBST_PKT_P: stat_ind = SRVST_PKT_P;
357 		break;
358 	case ILBST_BYTES_P: stat_ind = SRVST_BYTES_P;
359 		break;
360 	case ILBST_ITEMIZE_RNAME:
361 		(void) snprintf(buf, bufsize, "%s",
362 		    sta->ilbst_rlist[rule_index].ird_rulename);
363 		return (B_TRUE);
364 		/* not reached */
365 		break;
366 	case ILBST_ITEMIZE_SNAME:
367 		(void) snprintf(buf, bufsize, "%s", srv->isd_servername);
368 		return (B_TRUE);
369 		/* not reached */
370 		break;
371 	}
372 
373 	count = srv->isd_serverstats[stat_ind].is_value;
374 
375 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) != 0)
376 		goto out;
377 
378 	osrv = &sta->ilbst_rlist[rule_index].ird_srvlist_o[srv_ind];
379 	if (srv->isd_crtime != osrv->isd_crtime)
380 		ret = B_FALSE;
381 
382 	count -= osrv->isd_serverstats[stat_ind].is_value;
383 out:
384 	/*
385 	 * normally, we print "change per second", which we calculate
386 	 * here. otherwise, we print "change over interval" or absolute
387 	 * values.
388 	 */
389 	if ((sta->ilbst_flags & (ILBST_DELTA_INTERVAL|ILBST_ABS_NUMBERS)) == 0)
390 		count /= sta->ilbst_interval;
391 
392 	(void) snprintf(buf, bufsize, "%llu", count);
393 	return (ret);
394 
395 }
396 
397 static boolean_t
398 of_rule_stats(ofmt_arg_t *of_arg, char *buf, uint_t bufsize)
399 {
400 	ilbst_arg_t	*sta = (ilbst_arg_t *)of_arg->ofmt_cbarg;
401 	int		i, ind;
402 	uint64_t	count = 0;
403 
404 	switch (of_arg->ofmt_id) {
405 	case ILBST_PKT_U: ind = RLSTA_PKT_U;
406 		break;
407 	case ILBST_BYTES_U: ind = RLSTA_BYTES_U;
408 		break;
409 	case ILBST_PKT_D: ind = RLSTA_PKT_D;
410 		break;
411 	case ILBST_BYTES_D: ind = RLSTA_BYTES_D;
412 		break;
413 	case ILBST_ICMP_P: ind = RLSTA_ICMP_P;
414 		break;
415 	case ILBST_ICMP_D: ind = RLSTA_ICMP_D;
416 		break;
417 	case ILBST_ICMP2BIG_P: ind = RLSTA_ICMP2BIG_P;
418 		break;
419 	case ILBST_ICMP2BIG_D: ind = RLSTA_ICMP2BIG_D;
420 		break;
421 	case ILBST_NOMEMP_D: ind  = RLSTA_NOMEMPKT_D;
422 		break;
423 	case ILBST_NOPORTP_D: ind = RLSTA_NOPORTPKT_D;
424 		break;
425 	case ILBST_NOMEMB_D: ind = RLSTA_NOMEMBYTES_D;
426 		break;
427 	case ILBST_NOPORTB_D: ind = RLSTA_NOPORTBYTES_D;
428 		break;
429 	}
430 
431 	for (i = 0; i < sta->ilbst_rcount; i++)
432 		count += sta->ilbst_rlist[i].ird_rulestats[ind].is_value;
433 
434 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) != 0)
435 		goto out;
436 
437 	/*
438 	 * the purist approach: if we can't say 100% that what we
439 	 * calculate is correct, don't.
440 	 */
441 	if (sta->ilbst_flags & ILBST_RULES_CHANGED)
442 		return (B_FALSE);
443 
444 	for (i = 0; i < sta->ilbst_rcount; i++) {
445 		if (sta->ilbst_rlist[i].ird_crtime_o != 0 &&
446 		    sta->ilbst_rlist[i].ird_crtime !=
447 		    sta->ilbst_rlist[i].ird_crtime_o)
448 			return (B_FALSE);
449 
450 		count -= sta->ilbst_rlist[i].ird_rulestats_o[ind].is_value;
451 	}
452 out:
453 	/*
454 	 * normally, we print "change per second", which we calculate
455 	 * here. otherwise, we print "change over interval"
456 	 */
457 	if ((sta->ilbst_flags & (ILBST_DELTA_INTERVAL|ILBST_ABS_NUMBERS)) == 0)
458 		count /= sta->ilbst_interval;
459 
460 	(void) snprintf(buf, bufsize, "%llu", count);
461 	return (B_TRUE);
462 }
463 
464 /*
465  * Get the number of kstat instances. Note that when rules are being
466  * drained the number of kstats instances may be different than the
467  * kstat counter num_rules (ilb:0:global:num_rules").
468  *
469  * Also there can be multiple instances of a rule in the following
470  * scenario:
471  *
472  * A rule named rule A has been deleted but remains in kstats because
473  * its undergoing connection draining. During this time, the user adds
474  * a new rule with the same name(rule A). In this case, there would
475  * be two kstats instances for rule A. Currently ilbadm's aggregate
476  * results will include data from both instances of rule A. In,
477  * future we should have ilbadm stats only consider the latest instance
478  * of the rule (ie only consider the the instance that corresponds
479  * to the rule that was just added).
480  *
481  */
482 static int
483 i_get_num_kinstances(kstat_ctl_t *kctl)
484 {
485 	kstat_t		*kp;
486 	int		num_instances = 0; /* nothing found, 0 rules */
487 
488 	for (kp = kctl->kc_chain; kp != NULL; kp = kp->ks_next) {
489 		if (strncmp("rulestat", kp->ks_class, 8) == 0 &&
490 		    strncmp("ilb", kp->ks_module, 3) == 0) {
491 			num_instances++;
492 		}
493 	}
494 
495 	return (num_instances);
496 }
497 
498 
499 /*
500  * since server stat's classname is made up of <rulename>-sstat,
501  * we walk the rule list to construct the comparison
502  * Return:	pointer to rule whose name matches the class
503  *		NULL if no match
504  */
505 static ilbst_rule_desc_t *
506 match_2_rnames(char *class, ilbst_rule_desc_t *rlist, int rcount)
507 {
508 	int i;
509 	char	classname[KSTAT_STRLEN];
510 
511 	for (i = 0; i < rcount; i++) {
512 		(void) snprintf(classname, sizeof (classname), "%s-sstat",
513 		    rlist[i].ird_rulename);
514 		if (strncmp(classname, class, sizeof (classname)) == 0)
515 			return (&rlist[i]);
516 	}
517 	return (NULL);
518 }
519 
520 static int
521 i_stat_index(kstat_named_t *knp, ilbst_stat_t *stats, int count)
522 {
523 	int	i;
524 
525 	for (i = 0; i < count; i++) {
526 		if (strcasecmp(stats[i].is_name, knp->name) == 0)
527 			return (i);
528 	}
529 
530 	return (-1);
531 }
532 
533 static void
534 i_copy_sstats(ilbst_srv_desc_t *sp, kstat_t *kp)
535 {
536 	kstat_named_t	*knp;
537 	int		i, ind;
538 
539 	knp = KSTAT_NAMED_PTR(kp);
540 	for (i = 0; i < kp->ks_ndata; i++, knp++) {
541 		ind = i_stat_index(knp, servstats, SSTAT_SZ);
542 		if (ind == -1)
543 			continue;
544 		(void) strlcpy(sp->isd_serverstats[ind].is_name, knp->name,
545 		    sizeof (sp->isd_serverstats[ind].is_name));
546 		sp->isd_serverstats[ind].is_value = knp->value.ui64;
547 		sp->isd_crtime = kp->ks_crtime;
548 	}
549 }
550 
551 
552 static ilbadm_status_t
553 i_get_server_descs(ilbst_arg_t *sta, kstat_ctl_t *kctl)
554 {
555 	ilbadm_status_t	rc = ILBADM_OK;
556 	kstat_t		*kp;
557 	int		i = -1;
558 	ilbst_rule_desc_t	*rp;
559 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
560 	int			rcount = sta->ilbst_rcount;
561 
562 	/*
563 	 * find all "server" kstats, or the one specified in
564 	 * sta->server
565 	 */
566 	for (kp = kctl->kc_chain; kp != NULL; kp = kp->ks_next) {
567 		if (strncmp("ilb", kp->ks_module, 3) != 0)
568 			continue;
569 		if (sta->ilbst_server != NULL &&
570 		    strcasecmp(sta->ilbst_server, kp->ks_name) != 0)
571 			continue;
572 		rp = match_2_rnames(kp->ks_class, rlist, rcount);
573 		if (rp == NULL)
574 			continue;
575 
576 		(void) kstat_read(kctl, kp, NULL);
577 		i = rp->ird_srv_ind++;
578 
579 		rc = ILBADM_OK;
580 		/*
581 		 * This means that a server is added after we check last
582 		 * time...  Just make the array bigger.
583 		 */
584 		if (i+1 > rp->ird_num_servers) {
585 			ilbst_srv_desc_t  *srvlist;
586 
587 			if ((srvlist = realloc(rp->ird_srvlist, (i+1) *
588 			    sizeof (*srvlist))) == NULL) {
589 				rc = ILBADM_ENOMEM;
590 				break;
591 			}
592 			rp->ird_srvlist = srvlist;
593 			rp->ird_num_servers = i;
594 		}
595 
596 		(void) strlcpy(rp->ird_srvlist[i].isd_servername, kp->ks_name,
597 		    sizeof (rp->ird_srvlist[i].isd_servername));
598 		i_copy_sstats(&rp->ird_srvlist[i], kp);
599 	}
600 
601 	for (i = 0; i < rcount; i++)
602 		rlist[i].ird_srv_ind = 0;
603 
604 	if (sta->ilbst_server != NULL && i == -1)
605 		rc = ILBADM_ENOSERVER;
606 	return (rc);
607 }
608 
609 static void
610 i_copy_rstats(ilbst_rule_desc_t *rp, kstat_t *kp)
611 {
612 	kstat_named_t	*knp;
613 	int		i, ind;
614 
615 	knp = KSTAT_NAMED_PTR(kp);
616 	for (i = 0; i < kp->ks_ndata; i++, knp++) {
617 		ind = i_stat_index(knp, rulestats, RSTAT_SZ);
618 		if (ind == -1)
619 			continue;
620 
621 		(void) strlcpy(rp->ird_rulestats[ind].is_name, knp->name,
622 		    sizeof (rp->ird_rulestats[ind].is_name));
623 		rp->ird_rulestats[ind].is_value = knp->value.ui64;
624 	}
625 }
626 
627 static void
628 i_set_rlstats_ptr(ilbst_rule_desc_t *rp, boolean_t old_is_old)
629 {
630 	if (old_is_old) {
631 		rp->ird_rulestats = rp->ird_rstats;
632 		rp->ird_rulestats_o = rp->ird_rstats_o;
633 	} else {
634 		rp->ird_rulestats = rp->ird_rstats_o;
635 		rp->ird_rulestats_o = rp->ird_rstats;
636 	}
637 }
638 /*
639  * this function walks the array of rules and switches pointer to old
640  * and new stats as well as serverlists.
641  */
642 static void
643 i_swap_rl_pointers(ilbst_arg_t *sta, int rcount)
644 {
645 	int			i, tmp_num;
646 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
647 	ilbst_srv_desc_t	*tmp_srv;
648 
649 	for (i = 0; i < rcount; i++) {
650 		/* swap srvlist pointers */
651 		tmp_srv = rlist[i].ird_srvlist;
652 		rlist[i].ird_srvlist = rlist[i].ird_srvlist_o;
653 		rlist[i].ird_srvlist_o = tmp_srv;
654 
655 		/*
656 		 * swap server counts - we need the old one to
657 		 * save reallocation calls
658 		 */
659 		tmp_num = rlist[i].ird_num_servers_o;
660 		rlist[i].ird_num_servers_o = rlist[i].ird_num_servers;
661 		rlist[i].ird_num_servers = tmp_num;
662 
663 		/* preserve creation time */
664 		rlist[i].ird_crtime_o = rlist[i].ird_crtime;
665 
666 		i_set_rlstats_ptr(&rlist[i], sta->ilbst_old_is_old);
667 		rlist[i].ird_srv_ind = 0;
668 	}
669 }
670 
671 static void
672 i_init_rulelist(ilbst_arg_t *sta, int rcount)
673 {
674 	int			 i;
675 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
676 
677 	for (i = 0; i < rcount; i++) {
678 		rlist[i].ird_rulestats = rlist[i].ird_rstats;
679 		rlist[i].ird_rulestats_o = rlist[i].ird_rstats_o;
680 		rlist[i].ird_srv_ind = 0;
681 	}
682 }
683 
684 
685 /*
686  * this function searches for kstats describing individual rules and
687  * saves name, # of servers, and the kstat_t * describing them (this is
688  * for sta->rulename == NULL);
689  * if sta->rulename != NULL, it names the rule we're looking for
690  * and this function will fill in the other data (like the all_rules case)
691  * Returns:	ILBADM_ENORULE	named rule not found
692  *		ILBADM_ENOMEM	no mem. available
693  */
694 static ilbadm_status_t
695 i_get_rule_descs(ilbst_arg_t *sta, kstat_ctl_t *kctl)
696 {
697 	ilbadm_status_t	rc = ILBADM_OK;
698 	kstat_t		*kp;
699 	kstat_named_t	*knp;
700 	int		i;
701 	int		num_servers;
702 	ilbst_rule_desc_t	*rlist = sta->ilbst_rlist;
703 	int		rcount = sta->ilbst_rcount;
704 
705 	/*
706 	 * find all "rule" kstats, or the one specified in
707 	 * sta->ilbst_rulename.
708 	 */
709 	for (i = 0, kp = kctl->kc_chain; i < rcount && kp != NULL;
710 	    kp = kp->ks_next) {
711 		if (strncmp("rulestat", kp->ks_class, 8) != 0 ||
712 		    strncmp("ilb", kp->ks_module, 3) != 0)
713 			continue;
714 
715 		(void) kstat_read(kctl, kp, NULL);
716 
717 		knp = kstat_data_lookup(kp, "num_servers");
718 		if (knp == NULL) {
719 			ilbadm_err(gettext("kstat_data_lookup() failed: %s"),
720 			    strerror(errno));
721 			rc = ILBADM_LIBERR;
722 			break;
723 		}
724 		if (sta->ilbst_rulename != NULL) {
725 			if (strcasecmp(kp->ks_name, sta->ilbst_rulename)
726 			    != 0)
727 				continue;
728 		}
729 		(void) strlcpy(rlist[i].ird_rulename, kp->ks_name,
730 		    sizeof (rlist[i].ird_rulename));
731 
732 		/* only alloc the space we need, set counter here ... */
733 		if (sta->ilbst_server != NULL)
734 			num_servers = 1;
735 		else
736 			num_servers = (int)knp->value.ui64;
737 
738 		/* ... furthermore, only reallocate if necessary */
739 		if (num_servers != rlist[i].ird_num_servers) {
740 			ilbst_srv_desc_t  *srvlist;
741 
742 			rlist[i].ird_num_servers = num_servers;
743 
744 			if (rlist[i].ird_srvlist == NULL)
745 				srvlist = calloc(num_servers,
746 				    sizeof (*srvlist));
747 			else
748 				srvlist = realloc(rlist[i].ird_srvlist,
749 				    sizeof (*srvlist) * num_servers);
750 			if (srvlist == NULL) {
751 				rc = ILBADM_ENOMEM;
752 				break;
753 			}
754 			rlist[i].ird_srvlist = srvlist;
755 		}
756 		rlist[i].ird_srv_ind = 0;
757 		rlist[i].ird_crtime = kp->ks_crtime;
758 
759 		i_copy_rstats(&rlist[i], kp);
760 		i++;
761 
762 		/* if we know we're done, return */
763 		if (sta->ilbst_rulename != NULL || i == rcount) {
764 			rc = ILBADM_OK;
765 			break;
766 		}
767 	}
768 
769 	if (sta->ilbst_rulename != NULL && i == 0)
770 		rc = ILBADM_ENORULE;
771 	return (rc);
772 }
773 
774 static void
775 i_do_print(ilbst_arg_t *sta)
776 {
777 	int	i;
778 
779 	/* non-itemized display can go right ahead */
780 	if ((sta->ilbst_flags & ILBST_ITEMIZE) == 0) {
781 		ofmt_print(sta->ilbst_oh, sta);
782 		return;
783 	}
784 
785 	/*
786 	 * rulename is given, list a line per server
787 	 * here's how we do it:
788 	 *	the _ITEMIZE flag indicates to the print function (called
789 	 *	from ofmt_print()) to look at server [ird_srv_ind] only.
790 	 */
791 	if (sta->ilbst_rulename != NULL) {
792 		sta->ilbst_rule_index = 0;
793 		for (i = 0; i < sta->ilbst_rlist->ird_num_servers; i++) {
794 			sta->ilbst_rlist->ird_srv_ind = i;
795 			ofmt_print(sta->ilbst_oh, sta);
796 		}
797 		sta->ilbst_rlist->ird_srv_ind = 0;
798 		return;
799 	}
800 
801 	/* list one line for every rule for a given server */
802 	for (i = 0; i < sta->ilbst_rcount; i++) {
803 		/*
804 		 * if a rule doesn't contain a given server, there's no
805 		 * need to print it. Luckily, we can check that
806 		 * fairly easily
807 		 */
808 		if (sta->ilbst_rlist[i].ird_srvlist[0].isd_servername[0] ==
809 		    '\0')
810 			continue;
811 
812 		sta->ilbst_rule_index = i;
813 		sta->ilbst_rlist[i].ird_srv_ind = 0;
814 		ofmt_print(sta->ilbst_oh, sta);
815 	}
816 	sta->ilbst_rule_index = 0;
817 }
818 
819 static ilbadm_status_t
820 i_do_show_stats(ilbst_arg_t *sta)
821 {
822 	kstat_ctl_t	*kctl;
823 	kid_t		nkid;
824 	int		rcount = 1, i;
825 	ilbadm_status_t	rc = ILBADM_OK;
826 	ilbst_rule_desc_t	*rlist, *rp;
827 	boolean_t	pseudo_abs = B_FALSE; /* for first pass */
828 
829 	if ((kctl = kstat_open()) == NULL) {
830 		ilbadm_err(gettext("kstat_open() failed: %s"), strerror(errno));
831 		return (ILBADM_LIBERR);
832 	}
833 
834 
835 	if (sta->ilbst_rulename == NULL)
836 		rcount = i_get_num_kinstances(kctl);
837 
838 	rlist = calloc(sizeof (*rlist), rcount);
839 	if (rlist == NULL) {
840 		rc = ILBADM_ENOMEM;
841 		goto out;
842 	}
843 
844 	sta->ilbst_old_is_old = B_TRUE;
845 	sta->ilbst_rlist = rlist;
846 	sta->ilbst_rcount = sta->ilbst_rcount_prev = rcount;
847 	sta->ilbst_rlist_sz = rcount;
848 
849 	/*
850 	 * in the first pass, we always print absolute numbers. We
851 	 * need to remember whether we wanted abs. numbers for
852 	 * other samples as well
853 	 */
854 	if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) == 0) {
855 		sta->ilbst_flags |= ILBST_ABS_NUMBERS;
856 		pseudo_abs = B_TRUE;
857 	}
858 
859 	i_init_rulelist(sta, rcount);
860 	do {
861 		rc = i_get_rule_descs(sta, kctl);
862 		if (rc != ILBADM_OK)
863 			goto out;
864 
865 		rc = i_get_server_descs(sta, kctl);
866 		if (rc != ILBADM_OK)
867 			goto out;
868 
869 		i_do_print(sta);
870 
871 		if (sta->ilbst_count == -1 || --(sta->ilbst_count) > 0)
872 			(void) sleep(sta->ilbst_interval);
873 		else
874 			break;
875 
876 		nkid = kstat_chain_update(kctl);
877 		sta->ilbst_flags &= ~ILBST_RULES_CHANGED;
878 		/*
879 		 * we only need to continue with most of the rest of this if
880 		 * the kstat chain id has changed
881 		 */
882 		if (nkid == 0)
883 			goto swap_old_new;
884 		if (nkid == -1) {
885 			ilbadm_err(gettext("kstat_chain_update() failed: %s"),
886 			    strerror(errno));
887 			rc = ILBADM_LIBERR;
888 			break;
889 		}
890 
891 		/*
892 		 * find out whether the number of rules has changed.
893 		 * if so, adjust rcount and _o; if number has increased,
894 		 * expand array to hold all rules.
895 		 * we only shrink if rlist_sz is larger than both rcount and
896 		 * rcount_prev;
897 		 */
898 		if (sta->ilbst_rulename == NULL)
899 			rcount = i_get_num_kinstances(kctl);
900 		if (rcount != sta->ilbst_rcount) {
901 			sta->ilbst_flags |= ILBST_RULES_CHANGED;
902 			sta->ilbst_rcount_prev = sta->ilbst_rcount;
903 			sta->ilbst_rcount = rcount;
904 
905 			if (rcount > sta->ilbst_rcount_prev) {
906 				rlist = realloc(sta->ilbst_rlist,
907 				    sizeof (*sta->ilbst_rlist) * rcount);
908 				if (rlist == NULL) {
909 					rc = ILBADM_ENOMEM;
910 					break;
911 				}
912 				sta->ilbst_rlist = rlist;
913 				/* realloc doesn't zero out memory */
914 				for (i = sta->ilbst_rcount_prev;
915 				    i < rcount; i++) {
916 					rp = &sta->ilbst_rlist[i];
917 					bzero(rp, sizeof (*rp));
918 					i_set_rlstats_ptr(rp,
919 					    sta->ilbst_old_is_old);
920 				}
921 				/*
922 				 * even if rlist_sz was > rcount, it's now
923 				 * shrunk to rcount
924 				 */
925 				sta->ilbst_rlist_sz = sta->ilbst_rcount;
926 			}
927 		}
928 
929 		/*
930 		 * we may need to shrink the allocated slots down to the
931 		 * actually required number - we need to make sure we
932 		 * don't delete old or new stats.
933 		 */
934 		if (sta->ilbst_rlist_sz > MAX(sta->ilbst_rcount,
935 		    sta->ilbst_rcount_prev)) {
936 			sta->ilbst_rlist_sz =
937 			    MAX(sta->ilbst_rcount, sta->ilbst_rcount_prev);
938 			rlist = realloc(sta->ilbst_rlist,
939 			    sizeof (*sta->ilbst_rlist) * sta->ilbst_rlist_sz);
940 			if (rlist == NULL) {
941 				rc = ILBADM_ENOMEM;
942 				break;
943 			}
944 			sta->ilbst_rlist = rlist;
945 		}
946 
947 		/*
948 		 * move pointers around so what used to point to "old"
949 		 * stats now points to new, and vice versa
950 		 * if we're printing absolute numbers, this rigmarole is
951 		 * not necessary.
952 		 */
953 swap_old_new:
954 		if (pseudo_abs)
955 			sta->ilbst_flags &= ~ILBST_ABS_NUMBERS;
956 
957 		if ((sta->ilbst_flags & ILBST_ABS_NUMBERS) == 0) {
958 			sta->ilbst_old_is_old = !sta->ilbst_old_is_old;
959 			i_swap_rl_pointers(sta, rcount);
960 		}
961 		_NOTE(CONSTCOND)
962 	} while (B_TRUE);
963 
964 out:
965 	(void) kstat_close(kctl);
966 	if ((rc != ILBADM_OK) && (rc != ILBADM_LIBERR))
967 		ilbadm_err(ilbadm_errstr(rc));
968 
969 	if (sta->ilbst_rlist != NULL)
970 		free(sta->ilbst_rlist);
971 
972 	return (rc);
973 }
974 
975 /*
976  * read ilb's kernel statistics and (periodically) display
977  * them.
978  */
979 /* ARGSUSED */
980 ilbadm_status_t
981 ilbadm_show_stats(int argc, char *argv[])
982 {
983 	ilbadm_status_t	rc;
984 	int		c;
985 	ilbst_arg_t	sta;
986 	int		oflags = 0;
987 	char		*fieldnames = stat_stdhdrs;
988 	ofmt_field_t	*fields = stat_stdfields;
989 	boolean_t	r_opt = B_FALSE, s_opt = B_FALSE, i_opt = B_FALSE;
990 	boolean_t	o_opt = B_FALSE, p_opt = B_FALSE, t_opt = B_FALSE;
991 	boolean_t	v_opt = B_FALSE, A_opt = B_FALSE, d_opt = B_FALSE;
992 	ofmt_status_t	oerr;
993 	ofmt_handle_t	oh = NULL;
994 
995 	bzero(&sta, sizeof (sta));
996 	sta.ilbst_interval = 1;
997 	sta.ilbst_count = 1;
998 
999 	while ((c = getopt(argc, argv, ":tdAr:s:ivo:p")) != -1) {
1000 		switch ((char)c) {
1001 		case 't': sta.ilbst_flags |= ILBST_TIMESTAMP_HEADER;
1002 			t_opt = B_TRUE;
1003 			break;
1004 		case 'd': sta.ilbst_flags |= ILBST_DELTA_INTERVAL;
1005 			d_opt = B_TRUE;
1006 			break;
1007 		case 'A': sta.ilbst_flags |= ILBST_ABS_NUMBERS;
1008 			A_opt = B_TRUE;
1009 			break;
1010 		case 'r': sta.ilbst_rulename = optarg;
1011 			r_opt = B_TRUE;
1012 			break;
1013 		case 's': sta.ilbst_server = optarg;
1014 			s_opt = B_TRUE;
1015 			break;
1016 		case 'i': sta.ilbst_flags |= ILBST_ITEMIZE;
1017 			i_opt = B_TRUE;
1018 			break;
1019 		case 'o': fieldnames = optarg;
1020 			o_opt = B_TRUE;
1021 			break;
1022 		case 'p': oflags |= OFMT_PARSABLE;
1023 			p_opt = B_TRUE;
1024 			break;
1025 		case 'v': sta.ilbst_flags |= ILBST_VERBOSE;
1026 			v_opt = B_TRUE;
1027 			fieldnames = stat_stdv_hdrs;
1028 			break;
1029 		case ':': ilbadm_err(gettext("missing option-argument"
1030 			    " detected for %c"), (char)optopt);
1031 			exit(1);
1032 			/* not reached */
1033 			break;
1034 		case '?': /* fallthrough */
1035 		default:
1036 			unknown_opt(argv, optind-1);
1037 			/* not reached */
1038 			break;
1039 		}
1040 	}
1041 
1042 	if (s_opt && r_opt) {
1043 		ilbadm_err(gettext("options -s and -r are mutually exclusive"));
1044 		exit(1);
1045 	}
1046 
1047 	if (i_opt) {
1048 		if (!(s_opt || r_opt)) {
1049 			ilbadm_err(gettext("option -i requires"
1050 			    " either -r or -s"));
1051 			exit(1);
1052 		}
1053 		if (v_opt) {
1054 			ilbadm_err(gettext("option -i and -v are mutually"
1055 			    " exclusive"));
1056 			exit(1);
1057 		}
1058 		/* only use "std" headers if none are specified */
1059 		if (!o_opt)
1060 			if (r_opt)
1061 				fieldnames = stat_itemize_rule_hdrs;
1062 			else /* must be s_opt */
1063 				fieldnames = stat_itemize_server_hdrs;
1064 		fields = stat_itemize_fields;
1065 	}
1066 
1067 	if (p_opt) {
1068 		if (!o_opt) {
1069 			ilbadm_err(gettext("option -p requires -o"));
1070 			exit(1);
1071 		}
1072 		if (v_opt) {
1073 			ilbadm_err(gettext("option -o and -v are mutually"
1074 			    " exclusive"));
1075 			exit(1);
1076 		}
1077 		if (strcasecmp(fieldnames, "all") == 0) {
1078 			ilbadm_err(gettext("option -p requires"
1079 			    " explicit field names"));
1080 			exit(1);
1081 		}
1082 	}
1083 
1084 	if (t_opt) {
1085 		if (v_opt) {
1086 			fieldnames = "all";
1087 		} else {
1088 			int  len = strlen(fieldnames) + 6;
1089 			char *fnames;
1090 
1091 			fnames = malloc(len);
1092 			if (fnames == NULL) {
1093 				rc = ILBADM_ENOMEM;
1094 				return (rc);
1095 			}
1096 			(void) snprintf(fnames, len, "%s,TIME", fieldnames);
1097 			fieldnames = fnames;
1098 		}
1099 	}
1100 
1101 	if (A_opt && d_opt) {
1102 		ilbadm_err(gettext("options -d and -A are mutually exclusive"));
1103 		exit(1);
1104 	}
1105 
1106 	/* find and parse interval and count arguments if present */
1107 	if (optind < argc) {
1108 		sta.ilbst_interval = atoi(argv[optind]);
1109 		if (sta.ilbst_interval < 1) {
1110 			ilbadm_err(gettext("illegal interval spec %s"),
1111 			    argv[optind]);
1112 			exit(1);
1113 		}
1114 		sta.ilbst_count = -1;
1115 		if (++optind < argc) {
1116 			sta.ilbst_count = atoi(argv[optind]);
1117 			if (sta.ilbst_count < 1) {
1118 				ilbadm_err(gettext("illegal count spec %s"),
1119 				    argv[optind]);
1120 				exit(1);
1121 			}
1122 		}
1123 	}
1124 
1125 	oerr = ofmt_open(fieldnames, fields, oflags, 80, &oh);
1126 	if (oerr != OFMT_SUCCESS) {
1127 		char	e[80];
1128 
1129 		ilbadm_err(gettext("ofmt_open failed: %s"),
1130 		    ofmt_strerror(oh, oerr, e, sizeof (e)));
1131 		return (ILBADM_LIBERR);
1132 	}
1133 
1134 	sta.ilbst_oh = oh;
1135 
1136 	rc = i_do_show_stats(&sta);
1137 
1138 	ofmt_close(oh);
1139 	return (rc);
1140 }
1141