xref: /titanic_51/usr/src/cmd/avs/sdbc/sd_trace.c (revision 450396635f70344c58b6b1e4db38cf17ff34445c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <errno.h>
27 #include <stdio.h>
28 #include <unistd.h>
29 #include <strings.h>
30 #include <stdlib.h>
31 
32 #include <sys/types.h>
33 #include <sys/time.h>
34 #include <sys/nsctl/sdbc_ioctl.h>
35 #include <sys/nsctl/rdc_ioctl.h>
36 #include <sys/nsctl/sd_bcache.h>
37 #include <sys/nsctl/sd_conf.h>
38 #include <sys/nsctl/rdc_io.h>
39 #include <sys/nsctl/rdc_bitmap.h>
40 #include <sys/unistat/spcs_s_u.h>
41 #include <curses.h>
42 
43 static rdc_status_t *rdc_status;
44 static rdc_u_info_t *rdc_info;
45 static int rdc_maxsets;
46 static int rdc_enabled_sets;
47 
48 static unsigned prev_time, delta_time;
49 #ifdef m88k
50 extern unsigned *usec_ptr;
51 #endif
52 static int bright = 0;
53 
54 extern int sdbc_max_devices;
55 
56 extern _sd_stats_t *cs_cur;
57 extern _sd_stats_t *cs_prev;
58 extern _sd_stats_t *cs_persec;
59 
60 extern int *on_off;
61 extern int *dual_on_off;
62 extern int *updates_prev;
63 extern double *rate_prev;
64 extern int *samples;
65 
66 int		range_num = 0;
67 int		screen = 0;
68 int		dual_screen = 0;
69 static		int rnum = 0;
70 
71 typedef struct {
72 	int lb, ub;
73 } range_t;
74 range_t ranges[100];
75 
76 extern int range_first();
77 extern int range_next(int);
78 extern int range_last();
79 
80 static int dual_initted = 0;
81 static char status[11][30];
82 
83 unsigned dc_delta_time, dc_prev_time;
84 
85 #ifdef m88k
86 #define	USEC_INIT()	usec_ptr = (unsigned int *)timer_init()
87 #define	USEC_READ()	(*usec_ptr)
88 #else /* !m88k */
89 #define	USEC_INIT()	USEC_START()
90 #include <sys/time.h>
91 static struct timeval Usec_time;
92 static int Usec_started = 0;
93 
94 void total_display(void);
95 void disp_stats(void);
96 void do_calc(void);
97 void init_dual(void);
98 void calc_time(void);
99 void calc_completion(int, int, int);
100 void disp_total_stats(void);
101 void display_cache(void);
102 
103 #define	DISPLEN 16
104 
105 static void
106 USEC_START(void)
107 {
108 	if (!Usec_started) {
109 		(void) gettimeofday(&Usec_time, NULL);
110 		Usec_started = 1;
111 	}
112 }
113 
114 static unsigned int
115 USEC_READ()
116 {
117 	struct timeval tv;
118 	if (!Usec_started)
119 		USEC_START();
120 
121 	(void) gettimeofday(&tv, NULL);
122 	return (unsigned)((tv.tv_sec - Usec_time.tv_sec) * 1000000 +
123 	    (tv.tv_usec - Usec_time.tv_usec));
124 }
125 #endif /* m88k */
126 
127 #define	SAMPLE_RATE 5
128 
129 /*
130  * refresh curses window to file
131  */
132 void
133 wrefresh_file(WINDOW *win, int fd)
134 {
135 	char buf[8192], c, *cp = buf, *line, *blank, *empty;
136 	int x, y;
137 
138 	empty = NULL;		/* cull trailing empty lines */
139 	for (y = 0; y < win->_maxy; y++) {
140 		line = cp;
141 		blank = NULL;	/* cull trailing blanks */
142 		for (x = 0; x < win->_maxx; x++) {
143 			c = (win->_y[y][x]) & A_CHARTEXT;
144 			if (c != ' ')
145 				blank = NULL;
146 			else if (blank == NULL)
147 				blank = cp;
148 			*cp++ = c;
149 		}
150 		if (blank)
151 			cp = blank;
152 		if (line != cp)
153 			empty = NULL;
154 		else if (empty == NULL)
155 			empty = cp + 1;
156 		*cp++ = '\n';
157 	}
158 	if (empty)
159 		cp = empty;
160 	*cp++ = '\f'; *cp++ = '\n'; *cp = '\0';
161 	/* cp is eliminated by short _maxy and _maxx, it won't overflow */
162 	/* LINTED, cp - buf won't be > INT32_MAX */
163 	(void) write(fd, buf, cp - buf);
164 }
165 
166 
167 int
168 higher(int high)
169 {
170 	int i;
171 
172 	for (i = high + 1; i <= sdbc_max_devices; i++) {
173 		if (cs_cur->st_shared[i].sh_alloc)
174 			return (i);
175 	}
176 	return (0);
177 }
178 
179 int
180 is_dirty()
181 {
182 	int i, dirty = 0;
183 	spcs_s_info_t ustats;
184 
185 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0,
186 	    &ustats) == SPCS_S_ERROR) {
187 		perror("Could not get stats from kernel");
188 		if (ustats) {
189 			spcs_s_report(ustats, stderr);
190 			spcs_s_ufree(&ustats);
191 		}
192 		return (-errno);
193 	}
194 	if (cs_cur->st_cachesize == 0)
195 		return (0);
196 
197 	for (i = 0; i < cs_cur->st_count; i++)  {
198 		if (cs_cur->st_shared[i].sh_alloc)
199 			dirty += cs_cur->st_shared[i].sh_numdirty;
200 	}
201 
202 	return (dirty != 0);
203 }
204 
205 void
206 display_cache(void)
207 {
208 	static int first = 1;
209 	spcs_s_info_t ustats;
210 
211 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0, &ustats) ==
212 	    SPCS_S_ERROR) {
213 		perror("sd_stats");
214 		if (ustats) {
215 			spcs_s_report(ustats, stderr);
216 			spcs_s_ufree(&ustats);
217 		}
218 	}
219 
220 	do_calc();
221 	if (first) {
222 		prev_time = USEC_READ();
223 		first = 0;
224 	} else
225 		disp_stats();
226 }
227 
228 void
229 total_display(void)
230 {
231 	spcs_s_info_t ustats;
232 
233 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0, &ustats) ==
234 	    SPCS_S_ERROR) {
235 		if (ustats) {
236 			spcs_s_report(ustats, stderr);
237 			spcs_s_ufree(&ustats);
238 		}
239 		perror("sd_stats");
240 	}
241 	disp_total_stats();
242 }
243 
244 
245 int
246 range_first()
247 {
248 	rnum = 0;
249 	return (ranges[rnum].lb);
250 }
251 
252 int
253 range_next(int cd)
254 {
255 	if (ranges[rnum].ub > cd)
256 		return (cd + 1);
257 	if (range_num > rnum)
258 		rnum++;
259 	else
260 		return (cd + 1);
261 	return (ranges[rnum].lb);
262 }
263 
264 int
265 range_last() {
266 	return (ranges[range_num].ub);
267 }
268 
269 
270 void
271 set_dual_on_off()
272 {
273 	int i, j, ct = 0, newct = 0;
274 
275 	for (i = range_first(); i < rdc_enabled_sets && i <= range_last();
276 	    i = range_next(i)) {
277 		if (rdc_info[i].flags & RDC_ENABLED) {
278 			ct++;
279 			if (ct > dual_screen * ((LINES - 9) / 2))
280 				break;
281 		}
282 	}
283 	if (((i >= rdc_enabled_sets) ||
284 	    (i > range_last())) && (dual_screen > 0)) {
285 		dual_screen--;
286 		set_dual_on_off();
287 	} else {
288 		bzero(dual_on_off, sdbc_max_devices * sizeof (int));
289 		for (j = i; j < rdc_enabled_sets && j <= range_last();
290 		    j = range_next(j)) {
291 			if (rdc_info[j].flags & RDC_ENABLED) {
292 				newct++;
293 				if (newct <= (LINES - 9) / 2) {
294 					dual_on_off[j] = 1;
295 				} else
296 					break;
297 			}
298 		}
299 	}
300 }
301 
302 
303 void
304 set_on_off()
305 {
306 	int i, j, ct = 0, newct = 0;
307 
308 	for (i = range_first(); i <= range_last(); i = range_next(i)) {
309 		if (cs_cur->st_shared[i].sh_alloc) {
310 			ct++;
311 			if (ct > screen*((LINES - 9) / 2))
312 				break;
313 		}
314 	}
315 	if ((i > range_last()) && (screen > 0)) {
316 		screen--;
317 		set_on_off();
318 	} else {
319 		bzero(on_off, sdbc_max_devices * sizeof (int));
320 		for (j = i; j <= range_last(); j = range_next(j)) {
321 			if (cs_cur->st_shared[j].sh_alloc) {
322 				newct++;
323 				if (newct <= (LINES - 9) / 2)
324 					on_off[j] = 1;
325 				else
326 					break;
327 			}
328 		}
329 	}
330 }
331 
332 void
333 disp_stats(void)
334 {
335 	double	read_s, write_s, access_s, readp, writep;
336 	double	rmiss_s, wmiss_s;
337 	double	elapsed = delta_time / 1000000.0;
338 	double  kbps = elapsed * 1024.0; /* for Kbytes per second */
339 	int	rtotal, wtotal, i, j;
340 	double	throughput = 0.0, rthroughput = 0.0;
341 	double	creads = 0.0, cwrites = 0.0;
342 	char	status_bit, down = 0;
343 	int	len;
344 	char	fn[19];
345 
346 	if (delta_time != 0) {
347 		read_s  = cs_persec->st_rdhits / elapsed;
348 		write_s = cs_persec->st_wrhits / elapsed;
349 		rmiss_s = cs_persec->st_rdmiss / elapsed;
350 		wmiss_s = cs_persec->st_wrmiss / elapsed;
351 		access_s = (cs_persec->st_wrhits + cs_persec->st_rdhits +
352 		    cs_persec->st_rdmiss + cs_persec->st_wrmiss) / elapsed;
353 	} else
354 		read_s = write_s = access_s = 0.0;
355 
356 	rtotal = cs_persec->st_rdhits + cs_persec->st_rdmiss;
357 	wtotal = cs_persec->st_wrhits + cs_persec->st_wrmiss;
358 	if (rtotal != 0)
359 		readp = cs_persec->st_rdhits / (double)rtotal;
360 	else
361 		readp = 0.0;
362 
363 	if (wtotal != 0) {
364 		writep = cs_persec->st_wrhits / (double)wtotal;
365 	} else
366 		writep = 0.0;
367 
368 	set_on_off();
369 	if (cs_cur->st_cachesize == 0)
370 		(void) mvprintw(0, 20, "****** Storage Cache Disabled ******");
371 	else
372 		(void) mvprintw(0, 20, "******      Storage Cache     ******");
373 	(void) mvprintw(2, 26, "disk_io       cache          write_blocks");
374 	(void) attron(A_UNDERLINE);
375 	(void) mvprintw(3, 1, " cd cached_partition  reads writes  reads writes"
376 	    "  dirty todisk failed");
377 	(void) attroff(A_UNDERLINE);
378 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
379 		if (i >= sdbc_max_devices)
380 			break;
381 		if (cs_cur->st_shared[i].sh_alloc)  {
382 			cs_persec->st_shared[i].sh_disk_write /= kbps;
383 			cs_persec->st_shared[i].sh_disk_read  /= kbps;
384 			cs_persec->st_shared[i].sh_cache_write /= kbps;
385 			cs_persec->st_shared[i].sh_cache_read /= kbps;
386 			rthroughput += cs_persec->st_shared[i].sh_disk_read;
387 			throughput += cs_persec->st_shared[i].sh_disk_write;
388 			creads += cs_persec->st_shared[i].sh_cache_read;
389 			cwrites += cs_persec->st_shared[i].sh_cache_write;
390 			if (!down)
391 				down = cs_cur->st_shared[i].sh_failed;
392 			if (cs_cur->st_shared[i].sh_failed && bright) {
393 				status_bit = '*';
394 			} else
395 				status_bit = ' ';
396 			if ((len = strlen(cs_cur->st_shared[i].sh_filename))
397 			    > 15) {
398 				strcpy(fn, "...");
399 				strcat(fn, cs_cur->st_shared[i].sh_filename +
400 				    len - 12);
401 			} else
402 				strcpy(fn, cs_cur->st_shared[i].sh_filename);
403 			if (on_off[i]) {
404 				(void) mvprintw(4 + j, 1,
405 				    "%3d %-15s%c %6d %6d %6d %6d %6d %6d %6d",
406 				    cs_cur->st_shared[i].sh_cd,
407 				    fn,
408 				    status_bit,
409 				    cs_persec->st_shared[i].sh_disk_read,
410 				    cs_persec->st_shared[i].sh_disk_write,
411 				    cs_persec->st_shared[i].sh_cache_read,
412 				    cs_persec->st_shared[i].sh_cache_write,
413 				    cs_cur->st_shared[i].sh_numdirty,
414 				    cs_cur->st_shared[i].sh_numio,
415 				    cs_cur->st_shared[i].sh_numfail);
416 				j++;
417 			}
418 		}
419 	}
420 	bright = !bright;
421 
422 	(void) mvprintw(4 + j, 22, "------ ------ ------ ------");
423 	(void) mvprintw(5 + j, 6, " Kbytes/s total:%6d %6d %6d %6d",
424 	    (int)rthroughput, (int)throughput,
425 	    (int)creads, (int)cwrites);
426 	(void) mvprintw(7 + j, 1, "accesses/s");
427 	(void) mvprintw(7 + j, 15, "read/s    write/s   %%readh   %%writeh");
428 
429 	(void) attron(A_UNDERLINE);
430 	(void) mvprintw(8 + j, 1, "            ");
431 	(void) mvprintw(8 + j, 13,
432 	    "                                                ");
433 	(void) mvprintw(8 + j, 13, "(misses/s) (misses/s)");
434 	(void) attroff(A_UNDERLINE);
435 
436 	(void) mvprintw(9 + j, 0, "%10.2lf    %7.2f    %7.2f   %6.1f    %6.1f",
437 	    access_s, read_s, write_s, readp * 100.0, writep * 100.0);
438 	(void) mvprintw(10 + j, 0, "             (%7.2f ) (%7.2f )\n\n",
439 	    rmiss_s, wmiss_s);
440 
441 	if (down)
442 		(void) mvprintw(20 + j, 1, "* -- disk off-line");
443 }
444 
445 void
446 do_calc(void)
447 {
448 	int i, j;
449 
450 	delta_time = USEC_READ() - prev_time;
451 
452 	cs_persec->st_rdhits = cs_cur->st_rdhits - cs_prev->st_rdhits;
453 	cs_persec->st_rdmiss = cs_cur->st_rdmiss - cs_prev->st_rdmiss;
454 	cs_persec->st_wrhits = cs_cur->st_wrhits - cs_prev->st_wrhits;
455 	cs_persec->st_wrmiss = cs_cur->st_wrmiss - cs_prev->st_wrmiss;
456 
457 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
458 		if (i >= sdbc_max_devices)
459 			break;
460 		if (cs_cur->st_shared[i].sh_alloc) {
461 			cs_persec->st_shared[i].sh_disk_write =
462 			    FBA_SIZE(cs_cur->st_shared[i].sh_disk_write -
463 			    cs_prev->st_shared[i].sh_disk_write);
464 			cs_persec->st_shared[i].sh_disk_read =
465 			    FBA_SIZE(cs_cur->st_shared[i].sh_disk_read -
466 			    cs_prev->st_shared[i].sh_disk_read);
467 			cs_persec->st_shared[i].sh_cache_read =
468 			    FBA_SIZE(cs_cur->st_shared[i].sh_cache_read -
469 			    cs_prev->st_shared[i].sh_cache_read);
470 			cs_persec->st_shared[i].sh_cache_write =
471 			    FBA_SIZE(cs_cur->st_shared[i].sh_cache_write -
472 			    cs_prev->st_shared[i].sh_cache_write);
473 			j++;
474 		}
475 	}
476 	(void) memcpy((char *) cs_prev, (char *) cs_cur, sizeof (_sd_stats_t) +
477 	    (sdbc_max_devices - 1) * sizeof (_sd_shared_t));
478 	prev_time = USEC_READ();
479 }
480 
481 
482 void
483 init_dual(void)
484 {
485 #define	IND_ENABLED		0
486 #define	IND_RESYNC  		1
487 #define	IND_RESYNC_REVERSE	2
488 #define	IND_VOLUME_DOWN		3
489 #define	IND_MIRROR_DOWN		4
490 #define	IND_LOGGING		5
491 #define	IND_RESYNC_NEEDED	6
492 #define	IND_REV_RESYNC_NEEDED	7
493 #define	IND_BITMAP_FAILED	8
494 #define	IND_FULL_SYNC_NEEDED	9
495 #define	IND_FCAL_FAILED		10
496 	strcpy(status[IND_ENABLED], "replicating");
497 	strcpy(status[IND_RESYNC], "sync");
498 	strcpy(status[IND_RESYNC_REVERSE], "rev sync");
499 	strcpy(status[IND_VOLUME_DOWN], "volume down");
500 	strcpy(status[IND_MIRROR_DOWN], "mirror down");
501 	strcpy(status[IND_LOGGING], "logging");
502 	strcpy(status[IND_RESYNC_NEEDED], "need sync");
503 	strcpy(status[IND_REV_RESYNC_NEEDED], "need rev sync");
504 	strcpy(status[IND_BITMAP_FAILED], "bitmap failed");
505 	strcpy(status[IND_FULL_SYNC_NEEDED], "full sync needed");
506 	strcpy(status[IND_FCAL_FAILED], "fcal failed");
507 	dual_initted = 1;
508 }
509 
510 
511 int
512 rdc_get_maxsets(void)
513 {
514 	rdc_status_t rdc_status;
515 	spcs_s_info_t ustatus;
516 	int rc;
517 
518 	rdc_status.nset = 0;
519 	ustatus = spcs_s_ucreate();
520 
521 	rc = RDC_IOCTL(RDC_STATUS, &rdc_status, 0, 0, 0, 0, ustatus);
522 	spcs_s_ufree(&ustatus);
523 
524 	if (rc == SPCS_S_ERROR)
525 		return (-1);
526 
527 	return (rdc_status.maxsets);
528 }
529 
530 int
531 dual_stats()
532 {
533 	int ind, i, k, len;
534 	int stars, size, segs;
535 	int rdcindex;
536 	float pct;
537 	char	fn[19];
538 	char *phost;
539 	char *shost;
540 	char *pfile;
541 	char *sfile;
542 	char lhost[16];
543 	spcs_s_info_t ustats = NULL;
544 
545 	(void) gethostname(lhost, 16);
546 
547 	if (rdc_maxsets <= 0)
548 		rdc_maxsets = rdc_get_maxsets();
549 
550 	if (rdc_maxsets < 0)
551 		goto no_stats;
552 
553 	if (!rdc_status) {
554 		rdc_status = malloc(sizeof (rdc_status_t) +
555 			(sizeof (rdc_set_t) * (rdc_maxsets - 1)));
556 		if (!rdc_status) {
557 no_stats:
558 			(void) mvprintw(0, 20,
559 				"****** Dual Copy Not Available ******");
560 			return (-1);
561 		}
562 
563 		rdc_info = rdc_status->rdc_set;
564 	}
565 
566 	rdc_status->nset = rdc_maxsets;
567 	ustats = spcs_s_ucreate();
568 
569 	size = RDC_IOCTL(RDC_STATUS, rdc_status, 0, 0, 0, 0, ustats);
570 	if (size == SPCS_S_ERROR) {
571 		if (ustats) {
572 			spcs_s_report(ustats, stderr);
573 			spcs_s_ufree(&ustats);
574 		}
575 		(void) mvprintw(0, 20, "****** Dual Copy Not Available ******");
576 		return (-1);
577 	}
578 	spcs_s_ufree(&ustats);
579 	rdc_enabled_sets = rdc_status->nset;
580 
581 	if (!dual_initted)
582 		init_dual();
583 
584 	set_dual_on_off();
585 
586 	calc_time();
587 
588 	(void) mvprintw(0, 20, "****** Dual Copy Statistics ******");
589 	(void) attron(A_UNDERLINE);
590 	(void) mvprintw(2,  0, "primary");
591 	(void) mvprintw(2, 22, "link status");
592 	(void) mvprintw(2, 36, "secondary");
593 	(void) mvprintw(2, 54, "dual copy status");
594 	(void) attroff(A_UNDERLINE);
595 
596 	for (rdcindex = 0, k = 0; rdcindex < rdc_enabled_sets; rdcindex++)  {
597 		if (!(rdc_info[rdcindex].flags & RDC_ENABLED) ||
598 		    !dual_on_off[rdcindex])
599 			continue;
600 
601 		if (rdc_info[rdcindex].sync_flags & RDC_VOL_FAILED)
602 			ind = IND_VOLUME_DOWN;
603 		else if (rdc_info[rdcindex].flags & RDC_FCAL_FAILED)
604 			ind = IND_FCAL_FAILED;
605 		else if (rdc_info[rdcindex].bmap_flags & RDC_BMP_FAILED)
606 			ind = IND_BITMAP_FAILED;
607 		else if (rdc_info[rdcindex].flags & RDC_LOGGING) {
608 			if (rdc_info[rdcindex].sync_flags &
609 			    RDC_SYNC_NEEDED)
610 				ind = IND_RESYNC_NEEDED;
611 			else if (rdc_info[rdcindex].sync_flags &
612 			    RDC_RSYNC_NEEDED)
613 				ind = IND_REV_RESYNC_NEEDED;
614 			else
615 				ind = IND_LOGGING;
616 		} else if ((rdc_info[rdcindex].flags & RDC_SLAVE) &&
617 		    (rdc_info[rdcindex].flags & RDC_SYNCING)) {
618 			if (rdc_info[rdcindex].flags & RDC_PRIMARY)
619 				ind = IND_RESYNC_REVERSE;
620 			else
621 				ind = IND_RESYNC;
622 		} else if (rdc_info[rdcindex].flags & RDC_SYNCING) {
623 			if (rdc_info[rdcindex].flags & RDC_PRIMARY)
624 				ind = IND_RESYNC;
625 			else
626 				ind = IND_RESYNC_REVERSE;
627 		} else
628 			ind = IND_ENABLED;
629 
630 		phost = rdc_info[rdcindex].primary.intf;
631 		pfile = rdc_info[rdcindex].primary.file;
632 		shost = rdc_info[rdcindex].secondary.intf;
633 		sfile = rdc_info[rdcindex].secondary.file;
634 
635 		if ((len = strlen(phost)) > 8) {
636 			(void) mvprintw(4 + k, 0, ".%+7s:",
637 				phost + len - 7);
638 		} else
639 			(void) mvprintw(4 + k, 0, "%+8s:", phost);
640 
641 		if ((len = strlen(pfile)) > DISPLEN) {
642 			(void) mvprintw(4 + k, 9, "...%-13s",
643 			    pfile + len - DISPLEN + 3);
644 		} else
645 			(void) mvprintw(4 + k, 9, "%-16s", pfile);
646 
647 		(void) attron(A_BOLD);
648 		(void) mvprintw(4 + k, 26, "*");
649 		(void) mvprintw(4 + k, 28, "*");
650 
651 		(void) mvprintw(4 + k, 56, "%-8s", status[ind]);
652 		(void) attroff(A_BOLD);
653 
654 		if (ind == IND_RESYNC_REVERSE) {
655 			if (bright && !(rdc_info[rdcindex].flags & RDC_LOGGING))
656 				(void) mvprintw(4 + k, 27, "<");
657 			if (rdc_info[rdcindex].flags & RDC_PRIMARY &&
658 			    !(rdc_info[rdcindex].flags & RDC_LOGGING))
659 				calc_completion(rdcindex,
660 				rdc_info[rdcindex].bits_set, 4 + k);
661 		} else if (ind == IND_RESYNC) {
662 			if (bright && !(rdc_info[rdcindex].flags & RDC_LOGGING))
663 				(void) mvprintw(4 + k, 27, ">");
664 			if (rdc_info[rdcindex].flags & RDC_PRIMARY &&
665 			    !(rdc_info[rdcindex].flags & RDC_LOGGING))
666 				calc_completion(rdcindex,
667 				rdc_info[rdcindex].bits_set, 4 + k);
668 		} else if (ind == IND_LOGGING)
669 			(void) mvprintw(4 + k, 27, ".");
670 		else if (ind == IND_ENABLED)
671 			(void) mvprintw(4 + k, 27, "=");
672 
673 		if ((len = strlen(shost)) > 8) {
674 			(void) mvprintw(4 + k, 30, ".%+7s:",
675 				shost + len - 7);
676 		} else
677 			(void) mvprintw(4 + k, 30, "%+8s:", shost);
678 
679 		if ((len = strlen(sfile)) > DISPLEN) {
680 			(void) mvprintw(4 + k, 39, "...%-13s",
681 			sfile + len - DISPLEN + 3);
682 		} else
683 			(void) mvprintw(4 + k, 39, "%-16s", sfile);
684 
685 		k++;
686 	}
687 
688 	k += 5;
689 	(void) attron(A_UNDERLINE);
690 	for (i = 0; i < 80; i++)
691 		(void) mvprintw(k, i, " ");
692 	k += 2;
693 	(void) mvprintw(k,  0, "partition");
694 	(void) mvprintw(k, 16, "recovery needed");
695 	(void) mvprintw(k, 48, "recovery completed");
696 	(void) attroff(A_UNDERLINE);
697 	k += 2;
698 
699 	for (rdcindex = 0; rdcindex < rdc_enabled_sets; rdcindex++)  {
700 		if (!(rdc_info[rdcindex].flags & RDC_ENABLED) ||
701 		    !dual_on_off[rdcindex])
702 			continue;
703 
704 		if (!(rdc_info[rdcindex].flags & RDC_PRIMARY)) {
705 			continue;
706 		}
707 		if (!(rdc_info[rdcindex].flags & RDC_SLAVE) &&
708 		    !(rdc_info[rdcindex].flags & RDC_SYNCING) &&
709 		    !(rdc_info[rdcindex].flags & RDC_LOGGING)) {
710 			continue;
711 		}
712 
713 		len = strlen(rdc_info[rdcindex].secondary.file);
714 		if (len > 15) {
715 			strcpy(fn, "...");
716 			strcat(fn,
717 			    rdc_info[rdcindex].secondary.file + len - 12);
718 		} else
719 			strcpy(fn, rdc_info[rdcindex].secondary.file);
720 		(void) mvprintw(k, 0, "%-15s", fn);
721 
722 		segs = FBA_TO_LOG_LEN(rdc_info[rdcindex].volume_size);
723 		pct  = segs ?
724 		    ((float)rdc_info[rdcindex].bits_set / (float)segs) : 0.0;
725 		stars = (int)(pct * 20.0);
726 		while (stars > 0) {
727 			(void) mvprintw(k, 16 + stars, "*");
728 			stars--;
729 		}
730 		(void) attron(A_BOLD);
731 		(void) mvprintw(k, 16, "[");
732 		(void) mvprintw(k, 37, "]");
733 		(void) attroff(A_BOLD);
734 		(void) mvprintw(k, 39, "%6.2f%%", pct * 100.0);
735 
736 		if (rdc_info[rdcindex].flags & RDC_SYNCING)
737 			pct = ((float)rdc_info[rdcindex].sync_pos /
738 			    (float)rdc_info[rdcindex].volume_size);
739 		else
740 			pct = 0.0;
741 		stars = (int)(pct * 20.0);
742 		while (stars > 0) {
743 			(void) mvprintw(k, 48 + stars, "*");
744 			stars--;
745 		}
746 		(void) attron(A_BOLD);
747 		(void) mvprintw(k, 48, "[");
748 		(void) mvprintw(k, 69, "]");
749 		(void) attroff(A_BOLD);
750 		(void) mvprintw(k, 70, "%6.2f%%", pct * 100.0);
751 		k++;
752 	}
753 	bright = !bright;
754 	return (0);
755 }
756 
757 /*
758  * Calculate a time interval in milliseconds using the
759  * micosecond counter
760  */
761 void
762 calc_time(void)
763 {
764 	unsigned int cur;
765 
766 	cur = USEC_READ();
767 	dc_delta_time = cur > dc_prev_time ? cur - dc_prev_time :
768 		cur + 0xFFFFFFFF - dc_prev_time;
769 	dc_delta_time /= 1000;
770 	dc_prev_time = cur;
771 }
772 
773 /*
774  * Calculate estimated time of completion of resync
775  */
776 void
777 calc_completion(int cd, int updates_left, int l)
778 {
779 	int delta_done;
780 	double rate;
781 	long time_left;
782 	long hours;
783 	long minutes;
784 	static int initted = 0;
785 
786 	if (!initted) {
787 		updates_prev[cd] = updates_left;
788 		initted = 1;
789 		return;
790 	}
791 
792 	/*
793 	 * Caclulate updates since last check
794 	 */
795 	delta_done = updates_prev[cd] - updates_left;
796 	updates_prev[cd] = updates_left;
797 
798 	/*
799 	 * If no updates, don't bother estimating completion time
800 	 */
801 	if (delta_done <= 0) {
802 		samples[cd] = 0;
803 		return;
804 	}
805 
806 	rate = delta_done * 1000.0 / dc_delta_time;
807 
808 	/*
809 	 * Calculate rate of updates as a weighted average
810 	 * of previous and current rate
811 	 */
812 	if (rate_prev[cd] && samples[cd] > SAMPLE_RATE)
813 		rate = (rate_prev[cd] * 4.0 + rate) / 5.0;
814 	rate_prev[cd] = rate;
815 	samples[cd]++;
816 
817 	/*
818 	 * Get enough samples before making estimate
819 	 */
820 	if (samples[cd]++ < SAMPLE_RATE)
821 		return;
822 
823 	time_left = (long)(updates_left/rate);   /* time left in seconds */
824 
825 	if (time_left < 0)
826 		return;
827 
828 	hours = time_left / (60 * 60);
829 	time_left -= hours * (60 * 60);
830 	minutes = time_left / 60;
831 	time_left -= minutes * 60;
832 	(void) mvprintw(l, 67,
833 	    "time %02d:%02d:%02d  \n", hours, minutes, time_left);
834 }
835 
836 void
837 disp_total_stats(void)
838 {
839 	unsigned int	read_s, write_s, access_s;
840 	double readp, writep;
841 	unsigned int	rmiss_s, wmiss_s;
842 	double  kbps = 2.0;
843 	int	rtotal, wtotal, i, j;
844 	unsigned int throughput = 0, rthroughput = 0, creads = 0, cwrites = 0;
845 	char	status_bit, down = 0;
846 	int	len;
847 	char	fn[19];
848 
849 	read_s  = cs_cur->st_rdhits;
850 	write_s = cs_cur->st_wrhits;
851 	rmiss_s = cs_cur->st_rdmiss;
852 	wmiss_s = cs_cur->st_wrmiss;
853 	access_s = (read_s + write_s + rmiss_s + wmiss_s);
854 
855 	rtotal = cs_cur->st_rdhits + cs_cur->st_rdmiss;
856 	wtotal = cs_cur->st_wrhits + cs_cur->st_wrmiss;
857 	if (rtotal != 0)
858 		readp = cs_cur->st_rdhits / (double)rtotal;
859 	else
860 		readp = 0.0;
861 
862 	if (wtotal != 0)
863 		writep = cs_cur->st_wrhits / (double)wtotal;
864 	else
865 		writep = 0.0;
866 
867 	set_on_off();
868 	(void) mvprintw(0, 14,
869 	    "******     Storage Cache (Cumulative)      ******");
870 	(void) mvprintw(2, 30, "disk_io                  cache");
871 	(void) attron(A_UNDERLINE);
872 	(void) mvprintw(3,  1,
873 	    " cd cached_partition      reads     writes      reads     writes");
874 	(void) attroff(A_UNDERLINE);
875 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
876 		if (i >= sdbc_max_devices)
877 			break;
878 		if (cs_cur->st_shared[i].sh_alloc)  {
879 			cs_cur->st_shared[i].sh_disk_write /= kbps;
880 			cs_cur->st_shared[i].sh_disk_read /= kbps;
881 			cs_cur->st_shared[i].sh_cache_write /= kbps;
882 			cs_cur->st_shared[i].sh_cache_read /= kbps;
883 			rthroughput += cs_cur->st_shared[i].sh_disk_read;
884 			throughput += cs_cur->st_shared[i].sh_disk_write;
885 			creads += cs_cur->st_shared[i].sh_cache_read;
886 			cwrites += cs_cur->st_shared[i].sh_cache_write;
887 			if (!down)
888 				down = cs_cur->st_shared[i].sh_failed;
889 			if (cs_cur->st_shared[i].sh_failed && bright)
890 				status_bit = '*';
891 			else
892 				status_bit = ' ';
893 			if ((len =
894 			    strlen(cs_cur->st_shared[i].sh_filename)) > 15) {
895 				strcpy(fn, "...");
896 				strcat(fn, cs_cur->st_shared[i].sh_filename +
897 				    len - 12);
898 			} else
899 				strcpy(fn, cs_cur->st_shared[i].sh_filename);
900 
901 			if (on_off[i]) {
902 				(void) mvprintw(4 + j, 1,
903 				    "%3d %-15s%c %10u %10u %10u %10u",
904 				    cs_cur->st_shared[i].sh_cd,
905 				    fn,
906 				    status_bit,
907 				    cs_cur->st_shared[i].sh_disk_read,
908 				    cs_cur->st_shared[i].sh_disk_write,
909 				    cs_cur->st_shared[i].sh_cache_read,
910 				    cs_cur->st_shared[i].sh_cache_write);
911 				j++;
912 			}
913 		}
914 	}
915 	bright = !bright;
916 
917 	(void) mvprintw(4 + j, 22,
918 	    "---------- ---------- ---------- ----------");
919 	(void) mvprintw(5 + j, 8, " Kbytes total:%10u %10u %10u %10u",
920 	    (int)rthroughput, (int)throughput,
921 	    (int)creads, (int)cwrites);
922 	(void) mvprintw(7 + j, 1, " accesses");
923 	(void) mvprintw(7 + j, 18, "read        write    %%readh  %%writeh");
924 
925 	(void) attron(A_UNDERLINE);
926 	(void) mvprintw(8 + j, 1, "            ");
927 	(void) mvprintw(8 + j, 13,
928 	    "                                                ");
929 	(void) mvprintw(8 + j, 11, "(    misses) (    misses)");
930 	(void) attroff(A_UNDERLINE);
931 
932 	(void) mvprintw(9 + j, 0, "%10u  %10u   %10u    %6.1f   %6.1f",
933 	    access_s, read_s, write_s, readp*100.0, writep*100.0);
934 	(void) mvprintw(10 + j, 0,
935 	    "           (%10u) (%10u)\n\n", rmiss_s, wmiss_s);
936 
937 	(void) attron(A_UNDERLINE);
938 	(void) mvprintw(13 + j, 1, "cachesize  blocksize");
939 	(void) attroff(A_UNDERLINE);
940 	(void) mvprintw(14 + j, 1, "%8dK %10d", cs_cur->st_cachesize / 1024,
941 	    cs_cur->st_blksize);
942 
943 	(void) attron(A_UNDERLINE);
944 	(void) mvprintw(16 + j, 1, "Write blocks available:");
945 	(void) attroff(A_UNDERLINE);
946 	(void) mvprintw(17 + j, 1, "Net 0: %6d", cs_cur->st_wlru_inq);
947 
948 	(void) attron(A_UNDERLINE);
949 	(void) mvprintw(19 + j, 1, "LRU stats:  Blocks	Requeued    Optimized");
950 	(void) attroff(A_UNDERLINE);
951 	(void) mvprintw(20 + j, 7, "%12d %12u %12u", cs_cur->st_lru_blocks,
952 	    cs_cur->st_lru_req, cs_cur->st_lru_noreq);
953 
954 	if (down)
955 		(void) mvprintw(25 + j, 1, "* -- disk off-line");
956 }
957