xref: /titanic_52/usr/src/cmd/avs/sdbc/sd_trace.c (revision 385cc6b4ad1792caef3f84eb61eed3f27085801f)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <errno.h>
28 #include <stdio.h>
29 #include <unistd.h>
30 #include <strings.h>
31 #include <stdlib.h>
32 
33 #include <sys/types.h>
34 #include <sys/time.h>
35 #include <sys/nsctl/sdbc_ioctl.h>
36 #include <sys/nsctl/rdc_ioctl.h>
37 #include <sys/nsctl/sd_bcache.h>
38 #include <sys/nsctl/sd_conf.h>
39 #include <sys/nsctl/rdc_io.h>
40 #include <sys/nsctl/rdc_bitmap.h>
41 #include <sys/unistat/spcs_s_u.h>
42 #include <curses.h>
43 
44 static rdc_status_t *rdc_status;
45 static rdc_u_info_t *rdc_info;
46 static int rdc_maxsets;
47 static int rdc_enabled_sets;
48 
49 static unsigned prev_time, delta_time;
50 #ifdef m88k
51 extern unsigned *usec_ptr;
52 #endif
53 static int bright = 0;
54 
55 extern int sdbc_max_devices;
56 
57 extern _sd_stats_t *cs_cur;
58 extern _sd_stats_t *cs_prev;
59 extern _sd_stats_t *cs_persec;
60 
61 extern int *on_off;
62 extern int *dual_on_off;
63 extern int *updates_prev;
64 extern double *rate_prev;
65 extern int *samples;
66 
67 int		range_num = 0;
68 int		screen = 0;
69 int		dual_screen = 0;
70 static		int rnum = 0;
71 
72 typedef struct {
73 	int lb, ub;
74 } range_t;
75 range_t ranges[100];
76 
77 extern int range_first();
78 extern int range_next(int);
79 extern int range_last();
80 
81 static int dual_initted = 0;
82 static char status[11][30];
83 
84 unsigned dc_delta_time, dc_prev_time;
85 
86 #ifdef m88k
87 #define	USEC_INIT()	usec_ptr = (unsigned int *)timer_init()
88 #define	USEC_READ()	(*usec_ptr)
89 #else /* !m88k */
90 #define	USEC_INIT()	USEC_START()
91 #include <sys/time.h>
92 static struct timeval Usec_time;
93 static int Usec_started = 0;
94 
95 void total_display(void);
96 void disp_stats(void);
97 void do_calc(void);
98 void init_dual(void);
99 void calc_time(void);
100 void calc_completion(int, int, int);
101 void disp_total_stats(void);
102 void display_cache(void);
103 
104 #define	DISPLEN 16
105 
106 static void
107 USEC_START(void)
108 {
109 	if (!Usec_started) {
110 		(void) gettimeofday(&Usec_time, NULL);
111 		Usec_started = 1;
112 	}
113 }
114 
115 static unsigned int
116 USEC_READ()
117 {
118 	struct timeval tv;
119 	if (!Usec_started)
120 		USEC_START();
121 
122 	(void) gettimeofday(&tv, NULL);
123 	return (unsigned)((tv.tv_sec - Usec_time.tv_sec) * 1000000 +
124 	    (tv.tv_usec - Usec_time.tv_usec));
125 }
126 #endif /* m88k */
127 
128 #define	SAMPLE_RATE 5
129 
130 /*
131  * refresh curses window to file
132  */
133 void
134 wrefresh_file(WINDOW *win, int fd)
135 {
136 	char buf[8192], c, *cp = buf, *line, *blank, *empty;
137 	int x, y;
138 
139 	empty = NULL;		/* cull trailing empty lines */
140 	for (y = 0; y < win->_maxy; y++) {
141 		line = cp;
142 		blank = NULL;	/* cull trailing blanks */
143 		for (x = 0; x < win->_maxx; x++) {
144 			c = (win->_y[y][x]) & A_CHARTEXT;
145 			if (c != ' ')
146 				blank = NULL;
147 			else if (blank == NULL)
148 				blank = cp;
149 			*cp++ = c;
150 		}
151 		if (blank)
152 			cp = blank;
153 		if (line != cp)
154 			empty = NULL;
155 		else if (empty == NULL)
156 			empty = cp + 1;
157 		*cp++ = '\n';
158 	}
159 	if (empty)
160 		cp = empty;
161 	*cp++ = '\f'; *cp++ = '\n'; *cp = '\0';
162 	/* cp is eliminated by short _maxy and _maxx, it won't overflow */
163 	/* LINTED, cp - buf won't be > INT32_MAX */
164 	(void) write(fd, buf, cp - buf);
165 }
166 
167 
168 int
169 higher(int high)
170 {
171 	int i;
172 
173 	for (i = high + 1; i <= sdbc_max_devices; i++) {
174 		if (cs_cur->st_shared[i].sh_alloc)
175 			return (i);
176 	}
177 	return (0);
178 }
179 
180 int
181 is_dirty()
182 {
183 	int i, dirty = 0;
184 	spcs_s_info_t ustats;
185 
186 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0,
187 	    &ustats) == SPCS_S_ERROR) {
188 		perror("Could not get stats from kernel");
189 		if (ustats) {
190 			spcs_s_report(ustats, stderr);
191 			spcs_s_ufree(&ustats);
192 		}
193 		return (-errno);
194 	}
195 	if (cs_cur->st_cachesize == 0)
196 		return (0);
197 
198 	for (i = 0; i < cs_cur->st_count; i++)  {
199 		if (cs_cur->st_shared[i].sh_alloc)
200 			dirty += cs_cur->st_shared[i].sh_numdirty;
201 	}
202 
203 	return (dirty != 0);
204 }
205 
206 void
207 display_cache(void)
208 {
209 	static int first = 1;
210 	spcs_s_info_t ustats;
211 
212 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0, &ustats) ==
213 	    SPCS_S_ERROR) {
214 		perror("sd_stats");
215 		if (ustats) {
216 			spcs_s_report(ustats, stderr);
217 			spcs_s_ufree(&ustats);
218 		}
219 	}
220 
221 	do_calc();
222 	if (first) {
223 		prev_time = USEC_READ();
224 		first = 0;
225 	} else
226 		disp_stats();
227 }
228 
229 void
230 total_display(void)
231 {
232 	spcs_s_info_t ustats;
233 
234 	if (SDBC_IOCTL(SDBC_STATS, cs_cur, 0, 0, 0, 0, &ustats) ==
235 	    SPCS_S_ERROR) {
236 		if (ustats) {
237 			spcs_s_report(ustats, stderr);
238 			spcs_s_ufree(&ustats);
239 		}
240 		perror("sd_stats");
241 	}
242 	disp_total_stats();
243 }
244 
245 
246 int
247 range_first()
248 {
249 	rnum = 0;
250 	return (ranges[rnum].lb);
251 }
252 
253 int
254 range_next(int cd)
255 {
256 	if (ranges[rnum].ub > cd)
257 		return (cd + 1);
258 	if (range_num > rnum)
259 		rnum++;
260 	else
261 		return (cd + 1);
262 	return (ranges[rnum].lb);
263 }
264 
265 int
266 range_last() {
267 	return (ranges[range_num].ub);
268 }
269 
270 
271 void
272 set_dual_on_off()
273 {
274 	int i, j, ct = 0, newct = 0;
275 
276 	for (i = range_first(); i < rdc_enabled_sets && i <= range_last();
277 	    i = range_next(i)) {
278 		if (rdc_info[i].flags & RDC_ENABLED) {
279 			ct++;
280 			if (ct > dual_screen * ((LINES - 9) / 2))
281 				break;
282 		}
283 	}
284 	if (((i >= rdc_enabled_sets) ||
285 	    (i > range_last())) && (dual_screen > 0)) {
286 		dual_screen--;
287 		set_dual_on_off();
288 	} else {
289 		bzero(dual_on_off, sdbc_max_devices * sizeof (int));
290 		for (j = i; j < rdc_enabled_sets && j <= range_last();
291 		    j = range_next(j)) {
292 			if (rdc_info[j].flags & RDC_ENABLED) {
293 				newct++;
294 				if (newct <= (LINES - 9) / 2) {
295 					dual_on_off[j] = 1;
296 				} else
297 					break;
298 			}
299 		}
300 	}
301 }
302 
303 
304 void
305 set_on_off()
306 {
307 	int i, j, ct = 0, newct = 0;
308 
309 	for (i = range_first(); i <= range_last(); i = range_next(i)) {
310 		if (cs_cur->st_shared[i].sh_alloc) {
311 			ct++;
312 			if (ct > screen*((LINES - 9) / 2))
313 				break;
314 		}
315 	}
316 	if ((i > range_last()) && (screen > 0)) {
317 		screen--;
318 		set_on_off();
319 	} else {
320 		bzero(on_off, sdbc_max_devices * sizeof (int));
321 		for (j = i; j <= range_last(); j = range_next(j)) {
322 			if (cs_cur->st_shared[j].sh_alloc) {
323 				newct++;
324 				if (newct <= (LINES - 9) / 2)
325 					on_off[j] = 1;
326 				else
327 					break;
328 			}
329 		}
330 	}
331 }
332 
333 void
334 disp_stats(void)
335 {
336 	double	read_s, write_s, access_s, readp, writep;
337 	double	rmiss_s, wmiss_s;
338 	double	elapsed = delta_time / 1000000.0;
339 	double  kbps = elapsed * 1024.0; /* for Kbytes per second */
340 	int	rtotal, wtotal, i, j;
341 	double	throughput = 0.0, rthroughput = 0.0;
342 	double	creads = 0.0, cwrites = 0.0;
343 	char	status_bit, down = 0;
344 	int	len;
345 	char	fn[19];
346 
347 	if (delta_time != 0) {
348 		read_s  = cs_persec->st_rdhits / elapsed;
349 		write_s = cs_persec->st_wrhits / elapsed;
350 		rmiss_s = cs_persec->st_rdmiss / elapsed;
351 		wmiss_s = cs_persec->st_wrmiss / elapsed;
352 		access_s = (cs_persec->st_wrhits + cs_persec->st_rdhits +
353 		    cs_persec->st_rdmiss + cs_persec->st_wrmiss) / elapsed;
354 	} else
355 		read_s = write_s = access_s = 0.0;
356 
357 	rtotal = cs_persec->st_rdhits + cs_persec->st_rdmiss;
358 	wtotal = cs_persec->st_wrhits + cs_persec->st_wrmiss;
359 	if (rtotal != 0)
360 		readp = cs_persec->st_rdhits / (double)rtotal;
361 	else
362 		readp = 0.0;
363 
364 	if (wtotal != 0) {
365 		writep = cs_persec->st_wrhits / (double)wtotal;
366 	} else
367 		writep = 0.0;
368 
369 	set_on_off();
370 	if (cs_cur->st_cachesize == 0)
371 		(void) mvprintw(0, 20, "****** Storage Cache Disabled ******");
372 	else
373 		(void) mvprintw(0, 20, "******      Storage Cache     ******");
374 	(void) mvprintw(2, 26, "disk_io       cache          write_blocks");
375 	(void) attron(A_UNDERLINE);
376 	(void) mvprintw(3, 1, " cd cached_partition  reads writes  reads writes"
377 	    "  dirty todisk failed");
378 	(void) attroff(A_UNDERLINE);
379 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
380 		if (i >= sdbc_max_devices)
381 			break;
382 		if (cs_cur->st_shared[i].sh_alloc)  {
383 			cs_persec->st_shared[i].sh_disk_write /= kbps;
384 			cs_persec->st_shared[i].sh_disk_read  /= kbps;
385 			cs_persec->st_shared[i].sh_cache_write /= kbps;
386 			cs_persec->st_shared[i].sh_cache_read /= kbps;
387 			rthroughput += cs_persec->st_shared[i].sh_disk_read;
388 			throughput += cs_persec->st_shared[i].sh_disk_write;
389 			creads += cs_persec->st_shared[i].sh_cache_read;
390 			cwrites += cs_persec->st_shared[i].sh_cache_write;
391 			if (!down)
392 				down = cs_cur->st_shared[i].sh_failed;
393 			if (cs_cur->st_shared[i].sh_failed && bright) {
394 				status_bit = '*';
395 			} else
396 				status_bit = ' ';
397 			if ((len = strlen(cs_cur->st_shared[i].sh_filename))
398 			    > 15) {
399 				(void) strcpy(fn, "...");
400 				(void) strcat(fn,
401 				    cs_cur->st_shared[i].sh_filename +
402 				    len - 12);
403 			} else
404 				(void) strcpy(fn,
405 				    cs_cur->st_shared[i].sh_filename);
406 			if (on_off[i]) {
407 				(void) mvprintw(4 + j, 1,
408 				    "%3d %-15s%c %6d %6d %6d %6d %6d %6d %6d",
409 				    cs_cur->st_shared[i].sh_cd,
410 				    fn,
411 				    status_bit,
412 				    cs_persec->st_shared[i].sh_disk_read,
413 				    cs_persec->st_shared[i].sh_disk_write,
414 				    cs_persec->st_shared[i].sh_cache_read,
415 				    cs_persec->st_shared[i].sh_cache_write,
416 				    cs_cur->st_shared[i].sh_numdirty,
417 				    cs_cur->st_shared[i].sh_numio,
418 				    cs_cur->st_shared[i].sh_numfail);
419 				j++;
420 			}
421 		}
422 	}
423 	bright = !bright;
424 
425 	(void) mvprintw(4 + j, 22, "------ ------ ------ ------");
426 	(void) mvprintw(5 + j, 6, " Kbytes/s total:%6d %6d %6d %6d",
427 	    (int)rthroughput, (int)throughput,
428 	    (int)creads, (int)cwrites);
429 	(void) mvprintw(7 + j, 1, "accesses/s");
430 	(void) mvprintw(7 + j, 15, "read/s    write/s   %%readh   %%writeh");
431 
432 	(void) attron(A_UNDERLINE);
433 	(void) mvprintw(8 + j, 1, "            ");
434 	(void) mvprintw(8 + j, 13,
435 	    "                                                ");
436 	(void) mvprintw(8 + j, 13, "(misses/s) (misses/s)");
437 	(void) attroff(A_UNDERLINE);
438 
439 	(void) mvprintw(9 + j, 0, "%10.2lf    %7.2f    %7.2f   %6.1f    %6.1f",
440 	    access_s, read_s, write_s, readp * 100.0, writep * 100.0);
441 	(void) mvprintw(10 + j, 0, "             (%7.2f ) (%7.2f )\n\n",
442 	    rmiss_s, wmiss_s);
443 
444 	if (down)
445 		(void) mvprintw(20 + j, 1, "* -- disk off-line");
446 }
447 
448 void
449 do_calc(void)
450 {
451 	int i, j;
452 
453 	delta_time = USEC_READ() - prev_time;
454 
455 	cs_persec->st_rdhits = cs_cur->st_rdhits - cs_prev->st_rdhits;
456 	cs_persec->st_rdmiss = cs_cur->st_rdmiss - cs_prev->st_rdmiss;
457 	cs_persec->st_wrhits = cs_cur->st_wrhits - cs_prev->st_wrhits;
458 	cs_persec->st_wrmiss = cs_cur->st_wrmiss - cs_prev->st_wrmiss;
459 
460 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
461 		if (i >= sdbc_max_devices)
462 			break;
463 		if (cs_cur->st_shared[i].sh_alloc) {
464 			cs_persec->st_shared[i].sh_disk_write =
465 			    FBA_SIZE(cs_cur->st_shared[i].sh_disk_write -
466 			    cs_prev->st_shared[i].sh_disk_write);
467 			cs_persec->st_shared[i].sh_disk_read =
468 			    FBA_SIZE(cs_cur->st_shared[i].sh_disk_read -
469 			    cs_prev->st_shared[i].sh_disk_read);
470 			cs_persec->st_shared[i].sh_cache_read =
471 			    FBA_SIZE(cs_cur->st_shared[i].sh_cache_read -
472 			    cs_prev->st_shared[i].sh_cache_read);
473 			cs_persec->st_shared[i].sh_cache_write =
474 			    FBA_SIZE(cs_cur->st_shared[i].sh_cache_write -
475 			    cs_prev->st_shared[i].sh_cache_write);
476 			j++;
477 		}
478 	}
479 	(void) memcpy((char *) cs_prev, (char *) cs_cur, sizeof (_sd_stats_t) +
480 	    (sdbc_max_devices - 1) * sizeof (_sd_shared_t));
481 	prev_time = USEC_READ();
482 }
483 
484 
485 void
486 init_dual(void)
487 {
488 #define	IND_ENABLED		0
489 #define	IND_RESYNC  		1
490 #define	IND_RESYNC_REVERSE	2
491 #define	IND_VOLUME_DOWN		3
492 #define	IND_MIRROR_DOWN		4
493 #define	IND_LOGGING		5
494 #define	IND_RESYNC_NEEDED	6
495 #define	IND_REV_RESYNC_NEEDED	7
496 #define	IND_BITMAP_FAILED	8
497 #define	IND_FULL_SYNC_NEEDED	9
498 #define	IND_FCAL_FAILED		10
499 	(void) strcpy(status[IND_ENABLED], "replicating");
500 	(void) strcpy(status[IND_RESYNC], "sync");
501 	(void) strcpy(status[IND_RESYNC_REVERSE], "rev sync");
502 	(void) strcpy(status[IND_VOLUME_DOWN], "volume down");
503 	(void) strcpy(status[IND_MIRROR_DOWN], "mirror down");
504 	(void) strcpy(status[IND_LOGGING], "logging");
505 	(void) strcpy(status[IND_RESYNC_NEEDED], "need sync");
506 	(void) strcpy(status[IND_REV_RESYNC_NEEDED], "need rev sync");
507 	(void) strcpy(status[IND_BITMAP_FAILED], "bitmap failed");
508 	(void) strcpy(status[IND_FULL_SYNC_NEEDED], "full sync needed");
509 	(void) strcpy(status[IND_FCAL_FAILED], "fcal failed");
510 	dual_initted = 1;
511 }
512 
513 
514 int
515 rdc_get_maxsets(void)
516 {
517 	rdc_status_t rdc_status;
518 	spcs_s_info_t ustatus;
519 	int rc;
520 
521 	rdc_status.nset = 0;
522 	ustatus = spcs_s_ucreate();
523 
524 	rc = RDC_IOCTL(RDC_STATUS, &rdc_status, 0, 0, 0, 0, ustatus);
525 	spcs_s_ufree(&ustatus);
526 
527 	if (rc == SPCS_S_ERROR)
528 		return (-1);
529 
530 	return (rdc_status.maxsets);
531 }
532 
533 int
534 dual_stats()
535 {
536 	int ind, i, k, len;
537 	int stars, size, segs;
538 	int rdcindex;
539 	float pct;
540 	char	fn[19];
541 	char *phost;
542 	char *shost;
543 	char *pfile;
544 	char *sfile;
545 	char lhost[16];
546 	spcs_s_info_t ustats = NULL;
547 
548 	(void) gethostname(lhost, 16);
549 
550 	if (rdc_maxsets <= 0)
551 		rdc_maxsets = rdc_get_maxsets();
552 
553 	if (rdc_maxsets < 0)
554 		goto no_stats;
555 
556 	if (!rdc_status) {
557 		rdc_status = malloc(sizeof (rdc_status_t) +
558 			(sizeof (rdc_set_t) * (rdc_maxsets - 1)));
559 		if (!rdc_status) {
560 no_stats:
561 			(void) mvprintw(0, 20,
562 				"****** Dual Copy Not Available ******");
563 			return (-1);
564 		}
565 
566 		rdc_info = rdc_status->rdc_set;
567 	}
568 
569 	rdc_status->nset = rdc_maxsets;
570 	ustats = spcs_s_ucreate();
571 
572 	size = RDC_IOCTL(RDC_STATUS, rdc_status, 0, 0, 0, 0, ustats);
573 	if (size == SPCS_S_ERROR) {
574 		if (ustats) {
575 			spcs_s_report(ustats, stderr);
576 			spcs_s_ufree(&ustats);
577 		}
578 		(void) mvprintw(0, 20, "****** Dual Copy Not Available ******");
579 		return (-1);
580 	}
581 	spcs_s_ufree(&ustats);
582 	rdc_enabled_sets = rdc_status->nset;
583 
584 	if (!dual_initted)
585 		init_dual();
586 
587 	set_dual_on_off();
588 
589 	calc_time();
590 
591 	(void) mvprintw(0, 20, "****** Dual Copy Statistics ******");
592 	(void) attron(A_UNDERLINE);
593 	(void) mvprintw(2,  0, "primary");
594 	(void) mvprintw(2, 22, "link status");
595 	(void) mvprintw(2, 36, "secondary");
596 	(void) mvprintw(2, 54, "dual copy status");
597 	(void) attroff(A_UNDERLINE);
598 
599 	for (rdcindex = 0, k = 0; rdcindex < rdc_enabled_sets; rdcindex++)  {
600 		if (!(rdc_info[rdcindex].flags & RDC_ENABLED) ||
601 		    !dual_on_off[rdcindex])
602 			continue;
603 
604 		if (rdc_info[rdcindex].sync_flags & RDC_VOL_FAILED)
605 			ind = IND_VOLUME_DOWN;
606 		else if (rdc_info[rdcindex].flags & RDC_FCAL_FAILED)
607 			ind = IND_FCAL_FAILED;
608 		else if (rdc_info[rdcindex].bmap_flags & RDC_BMP_FAILED)
609 			ind = IND_BITMAP_FAILED;
610 		else if (rdc_info[rdcindex].flags & RDC_LOGGING) {
611 			if (rdc_info[rdcindex].sync_flags &
612 			    RDC_SYNC_NEEDED)
613 				ind = IND_RESYNC_NEEDED;
614 			else if (rdc_info[rdcindex].sync_flags &
615 			    RDC_RSYNC_NEEDED)
616 				ind = IND_REV_RESYNC_NEEDED;
617 			else
618 				ind = IND_LOGGING;
619 		} else if ((rdc_info[rdcindex].flags & RDC_SLAVE) &&
620 		    (rdc_info[rdcindex].flags & RDC_SYNCING)) {
621 			if (rdc_info[rdcindex].flags & RDC_PRIMARY)
622 				ind = IND_RESYNC_REVERSE;
623 			else
624 				ind = IND_RESYNC;
625 		} else if (rdc_info[rdcindex].flags & RDC_SYNCING) {
626 			if (rdc_info[rdcindex].flags & RDC_PRIMARY)
627 				ind = IND_RESYNC;
628 			else
629 				ind = IND_RESYNC_REVERSE;
630 		} else
631 			ind = IND_ENABLED;
632 
633 		phost = rdc_info[rdcindex].primary.intf;
634 		pfile = rdc_info[rdcindex].primary.file;
635 		shost = rdc_info[rdcindex].secondary.intf;
636 		sfile = rdc_info[rdcindex].secondary.file;
637 
638 		if ((len = strlen(phost)) > 8) {
639 			(void) mvprintw(4 + k, 0, ".%+7s:",
640 				phost + len - 7);
641 		} else
642 			(void) mvprintw(4 + k, 0, "%+8s:", phost);
643 
644 		if ((len = strlen(pfile)) > DISPLEN) {
645 			(void) mvprintw(4 + k, 9, "...%-13s",
646 			    pfile + len - DISPLEN + 3);
647 		} else
648 			(void) mvprintw(4 + k, 9, "%-16s", pfile);
649 
650 		(void) attron(A_BOLD);
651 		(void) mvprintw(4 + k, 26, "*");
652 		(void) mvprintw(4 + k, 28, "*");
653 
654 		(void) mvprintw(4 + k, 56, "%-8s", status[ind]);
655 		(void) attroff(A_BOLD);
656 
657 		if (ind == IND_RESYNC_REVERSE) {
658 			if (bright && !(rdc_info[rdcindex].flags & RDC_LOGGING))
659 				(void) mvprintw(4 + k, 27, "<");
660 			if (rdc_info[rdcindex].flags & RDC_PRIMARY &&
661 			    !(rdc_info[rdcindex].flags & RDC_LOGGING))
662 				calc_completion(rdcindex,
663 				rdc_info[rdcindex].bits_set, 4 + k);
664 		} else if (ind == IND_RESYNC) {
665 			if (bright && !(rdc_info[rdcindex].flags & RDC_LOGGING))
666 				(void) mvprintw(4 + k, 27, ">");
667 			if (rdc_info[rdcindex].flags & RDC_PRIMARY &&
668 			    !(rdc_info[rdcindex].flags & RDC_LOGGING))
669 				calc_completion(rdcindex,
670 				rdc_info[rdcindex].bits_set, 4 + k);
671 		} else if (ind == IND_LOGGING)
672 			(void) mvprintw(4 + k, 27, ".");
673 		else if (ind == IND_ENABLED)
674 			(void) mvprintw(4 + k, 27, "=");
675 
676 		if ((len = strlen(shost)) > 8) {
677 			(void) mvprintw(4 + k, 30, ".%+7s:",
678 				shost + len - 7);
679 		} else
680 			(void) mvprintw(4 + k, 30, "%+8s:", shost);
681 
682 		if ((len = strlen(sfile)) > DISPLEN) {
683 			(void) mvprintw(4 + k, 39, "...%-13s",
684 			sfile + len - DISPLEN + 3);
685 		} else
686 			(void) mvprintw(4 + k, 39, "%-16s", sfile);
687 
688 		k++;
689 	}
690 
691 	k += 5;
692 	(void) attron(A_UNDERLINE);
693 	for (i = 0; i < 80; i++)
694 		(void) mvprintw(k, i, " ");
695 	k += 2;
696 	(void) mvprintw(k,  0, "partition");
697 	(void) mvprintw(k, 16, "recovery needed");
698 	(void) mvprintw(k, 48, "recovery completed");
699 	(void) attroff(A_UNDERLINE);
700 	k += 2;
701 
702 	for (rdcindex = 0; rdcindex < rdc_enabled_sets; rdcindex++)  {
703 		if (!(rdc_info[rdcindex].flags & RDC_ENABLED) ||
704 		    !dual_on_off[rdcindex])
705 			continue;
706 
707 		if (!(rdc_info[rdcindex].flags & RDC_PRIMARY)) {
708 			continue;
709 		}
710 		if (!(rdc_info[rdcindex].flags & RDC_SLAVE) &&
711 		    !(rdc_info[rdcindex].flags & RDC_SYNCING) &&
712 		    !(rdc_info[rdcindex].flags & RDC_LOGGING)) {
713 			continue;
714 		}
715 
716 		len = strlen(rdc_info[rdcindex].secondary.file);
717 		if (len > 15) {
718 			(void) strcpy(fn, "...");
719 			(void) strcat(fn,
720 			    rdc_info[rdcindex].secondary.file + len - 12);
721 		} else
722 			(void) strcpy(fn, rdc_info[rdcindex].secondary.file);
723 		(void) mvprintw(k, 0, "%-15s", fn);
724 
725 		segs = FBA_TO_LOG_LEN(rdc_info[rdcindex].volume_size);
726 		pct  = segs ?
727 		    ((float)rdc_info[rdcindex].bits_set / (float)segs) : 0.0;
728 		stars = (int)(pct * 20.0);
729 		while (stars > 0) {
730 			(void) mvprintw(k, 16 + stars, "*");
731 			stars--;
732 		}
733 		(void) attron(A_BOLD);
734 		(void) mvprintw(k, 16, "[");
735 		(void) mvprintw(k, 37, "]");
736 		(void) attroff(A_BOLD);
737 		(void) mvprintw(k, 39, "%6.2f%%", pct * 100.0);
738 
739 		if (rdc_info[rdcindex].flags & RDC_SYNCING)
740 			pct = ((float)rdc_info[rdcindex].sync_pos /
741 			    (float)rdc_info[rdcindex].volume_size);
742 		else
743 			pct = 0.0;
744 		stars = (int)(pct * 20.0);
745 		while (stars > 0) {
746 			(void) mvprintw(k, 48 + stars, "*");
747 			stars--;
748 		}
749 		(void) attron(A_BOLD);
750 		(void) mvprintw(k, 48, "[");
751 		(void) mvprintw(k, 69, "]");
752 		(void) attroff(A_BOLD);
753 		(void) mvprintw(k, 70, "%6.2f%%", pct * 100.0);
754 		k++;
755 	}
756 	bright = !bright;
757 	return (0);
758 }
759 
760 /*
761  * Calculate a time interval in milliseconds using the
762  * micosecond counter
763  */
764 void
765 calc_time(void)
766 {
767 	unsigned int cur;
768 
769 	cur = USEC_READ();
770 	dc_delta_time = cur > dc_prev_time ? cur - dc_prev_time :
771 		cur + 0xFFFFFFFF - dc_prev_time;
772 	dc_delta_time /= 1000;
773 	dc_prev_time = cur;
774 }
775 
776 /*
777  * Calculate estimated time of completion of resync
778  */
779 void
780 calc_completion(int cd, int updates_left, int l)
781 {
782 	int delta_done;
783 	double rate;
784 	long time_left;
785 	long hours;
786 	long minutes;
787 	static int initted = 0;
788 
789 	if (!initted) {
790 		updates_prev[cd] = updates_left;
791 		initted = 1;
792 		return;
793 	}
794 
795 	/*
796 	 * Caclulate updates since last check
797 	 */
798 	delta_done = updates_prev[cd] - updates_left;
799 	updates_prev[cd] = updates_left;
800 
801 	/*
802 	 * If no updates, don't bother estimating completion time
803 	 */
804 	if (delta_done <= 0) {
805 		samples[cd] = 0;
806 		return;
807 	}
808 
809 	rate = delta_done * 1000.0 / dc_delta_time;
810 
811 	/*
812 	 * Calculate rate of updates as a weighted average
813 	 * of previous and current rate
814 	 */
815 	if (rate_prev[cd] && samples[cd] > SAMPLE_RATE)
816 		rate = (rate_prev[cd] * 4.0 + rate) / 5.0;
817 	rate_prev[cd] = rate;
818 	samples[cd]++;
819 
820 	/*
821 	 * Get enough samples before making estimate
822 	 */
823 	if (samples[cd]++ < SAMPLE_RATE)
824 		return;
825 
826 	time_left = (long)(updates_left/rate);   /* time left in seconds */
827 
828 	if (time_left < 0)
829 		return;
830 
831 	hours = time_left / (60 * 60);
832 	time_left -= hours * (60 * 60);
833 	minutes = time_left / 60;
834 	time_left -= minutes * 60;
835 	(void) mvprintw(l, 67,
836 	    "time %02d:%02d:%02d  \n", hours, minutes, time_left);
837 }
838 
839 void
840 disp_total_stats(void)
841 {
842 	unsigned int	read_s, write_s, access_s;
843 	double readp, writep;
844 	unsigned int	rmiss_s, wmiss_s;
845 	double  kbps = 2.0;
846 	int	rtotal, wtotal, i, j;
847 	unsigned int throughput = 0, rthroughput = 0, creads = 0, cwrites = 0;
848 	char	status_bit, down = 0;
849 	int	len;
850 	char	fn[19];
851 
852 	read_s  = cs_cur->st_rdhits;
853 	write_s = cs_cur->st_wrhits;
854 	rmiss_s = cs_cur->st_rdmiss;
855 	wmiss_s = cs_cur->st_wrmiss;
856 	access_s = (read_s + write_s + rmiss_s + wmiss_s);
857 
858 	rtotal = cs_cur->st_rdhits + cs_cur->st_rdmiss;
859 	wtotal = cs_cur->st_wrhits + cs_cur->st_wrmiss;
860 	if (rtotal != 0)
861 		readp = cs_cur->st_rdhits / (double)rtotal;
862 	else
863 		readp = 0.0;
864 
865 	if (wtotal != 0)
866 		writep = cs_cur->st_wrhits / (double)wtotal;
867 	else
868 		writep = 0.0;
869 
870 	set_on_off();
871 	(void) mvprintw(0, 14,
872 	    "******     Storage Cache (Cumulative)      ******");
873 	(void) mvprintw(2, 30, "disk_io                  cache");
874 	(void) attron(A_UNDERLINE);
875 	(void) mvprintw(3,  1,
876 	    " cd cached_partition      reads     writes      reads     writes");
877 	(void) attroff(A_UNDERLINE);
878 	for (i = 0, j = 0; j < cs_cur->st_count; i++) {
879 		if (i >= sdbc_max_devices)
880 			break;
881 		if (cs_cur->st_shared[i].sh_alloc)  {
882 			cs_cur->st_shared[i].sh_disk_write /= kbps;
883 			cs_cur->st_shared[i].sh_disk_read /= kbps;
884 			cs_cur->st_shared[i].sh_cache_write /= kbps;
885 			cs_cur->st_shared[i].sh_cache_read /= kbps;
886 			rthroughput += cs_cur->st_shared[i].sh_disk_read;
887 			throughput += cs_cur->st_shared[i].sh_disk_write;
888 			creads += cs_cur->st_shared[i].sh_cache_read;
889 			cwrites += cs_cur->st_shared[i].sh_cache_write;
890 			if (!down)
891 				down = cs_cur->st_shared[i].sh_failed;
892 			if (cs_cur->st_shared[i].sh_failed && bright)
893 				status_bit = '*';
894 			else
895 				status_bit = ' ';
896 			if ((len =
897 			    strlen(cs_cur->st_shared[i].sh_filename)) > 15) {
898 				(void) strcpy(fn, "...");
899 				(void) strcat(fn,
900 				    cs_cur->st_shared[i].sh_filename +
901 				    len - 12);
902 			} else
903 				(void) strcpy(fn,
904 				    cs_cur->st_shared[i].sh_filename);
905 
906 			if (on_off[i]) {
907 				(void) mvprintw(4 + j, 1,
908 				    "%3d %-15s%c %10u %10u %10u %10u",
909 				    cs_cur->st_shared[i].sh_cd,
910 				    fn,
911 				    status_bit,
912 				    cs_cur->st_shared[i].sh_disk_read,
913 				    cs_cur->st_shared[i].sh_disk_write,
914 				    cs_cur->st_shared[i].sh_cache_read,
915 				    cs_cur->st_shared[i].sh_cache_write);
916 				j++;
917 			}
918 		}
919 	}
920 	bright = !bright;
921 
922 	(void) mvprintw(4 + j, 22,
923 	    "---------- ---------- ---------- ----------");
924 	(void) mvprintw(5 + j, 8, " Kbytes total:%10u %10u %10u %10u",
925 	    (int)rthroughput, (int)throughput,
926 	    (int)creads, (int)cwrites);
927 	(void) mvprintw(7 + j, 1, " accesses");
928 	(void) mvprintw(7 + j, 18, "read        write    %%readh  %%writeh");
929 
930 	(void) attron(A_UNDERLINE);
931 	(void) mvprintw(8 + j, 1, "            ");
932 	(void) mvprintw(8 + j, 13,
933 	    "                                                ");
934 	(void) mvprintw(8 + j, 11, "(    misses) (    misses)");
935 	(void) attroff(A_UNDERLINE);
936 
937 	(void) mvprintw(9 + j, 0, "%10u  %10u   %10u    %6.1f   %6.1f",
938 	    access_s, read_s, write_s, readp*100.0, writep*100.0);
939 	(void) mvprintw(10 + j, 0,
940 	    "           (%10u) (%10u)\n\n", rmiss_s, wmiss_s);
941 
942 	(void) attron(A_UNDERLINE);
943 	(void) mvprintw(13 + j, 1, "cachesize  blocksize");
944 	(void) attroff(A_UNDERLINE);
945 	(void) mvprintw(14 + j, 1, "%8dK %10d", cs_cur->st_cachesize / 1024,
946 	    cs_cur->st_blksize);
947 
948 	(void) attron(A_UNDERLINE);
949 	(void) mvprintw(16 + j, 1, "Write blocks available:");
950 	(void) attroff(A_UNDERLINE);
951 	(void) mvprintw(17 + j, 1, "Net 0: %6d", cs_cur->st_wlru_inq);
952 
953 	(void) attron(A_UNDERLINE);
954 	(void) mvprintw(19 + j, 1, "LRU stats:  Blocks	Requeued    Optimized");
955 	(void) attroff(A_UNDERLINE);
956 	(void) mvprintw(20 + j, 7, "%12d %12u %12u", cs_cur->st_lru_blocks,
957 	    cs_cur->st_lru_req, cs_cur->st_lru_noreq);
958 
959 	if (down)
960 		(void) mvprintw(25 + j, 1, "* -- disk off-line");
961 }
962