1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <stdio.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <unistd.h>
30 #include <errno.h>
31 #include <inttypes.h>
32
33 #include <kstat.h>
34
35 #include <sys/nsctl/nsctl.h>
36 #include <sys/nsctl/sd_bcache.h>
37
38 #include "sdbc_stats.h"
39
40 #include "dsstat.h"
41 #include "common.h"
42 #include "report.h"
43
44 static sdbcstat_t *sdbc_top;
45 kstat_t *sdbc_global = NULL;
46
47 void sdbc_header();
48 int sdbc_value_check(sdbcstat_t *);
49 int sdbc_validate(kstat_t *);
50 uint32_t sdbc_getdelta(sdbcstat_t *, char *);
51
52 void sdbc_addstat(sdbcstat_t *);
53 sdbcstat_t *sdbc_delstat(sdbcstat_t *);
54 void center(int, char *);
55
56 /*
57 * sdbc_discover() - looks for new statistics to be monitored.
58 * Verifies that any statistics found are now already being
59 * monitored.
60 *
61 */
62 int
sdbc_discover(kstat_ctl_t * kc)63 sdbc_discover(kstat_ctl_t *kc)
64 {
65 static int validated = 0;
66
67 kstat_t *ksp;
68
69 for (ksp = kc->kc_chain; ksp; ksp = ksp->ks_next) {
70 int kinst;
71 char kname[KSTAT_STRLEN + 1];
72 sdbcstat_t *cur;
73 sdbcstat_t *sdbcstat = NULL;
74 kstat_t *io_ksp;
75
76 if (strcmp(ksp->ks_module, SDBC_KSTAT_MODULE) != 0 ||
77 strncmp(ksp->ks_name, SDBC_KSTAT_CDSTATS, 2) != 0)
78 continue;
79
80 if (kstat_read(kc, ksp, NULL) == -1)
81 continue;
82
83 /*
84 * Validate kstat structure
85 */
86 if (! validated) {
87 if (sdbc_validate(ksp))
88 return (EINVAL);
89
90 validated++;
91 }
92
93 /*
94 * Duplicate check
95 */
96 for (cur = sdbc_top; cur; cur = cur->next) {
97 char *cur_vname, *tst_vname;
98
99 cur_vname = kstat_value(cur->pre_set,
100 SDBC_CDKSTAT_VOL_NAME);
101
102 tst_vname = kstat_value(ksp,
103 SDBC_CDKSTAT_VOL_NAME);
104
105 if (strncmp(cur_vname, tst_vname, NAMED_LEN) == 0)
106 goto next;
107 }
108
109 /*
110 * Initialize new record
111 */
112 sdbcstat = (sdbcstat_t *)calloc(1, sizeof (sdbcstat_t));
113
114 kinst = ksp->ks_instance;
115
116 /*
117 * Set kstat
118 */
119 sdbcstat->pre_set = kstat_retrieve(kc, ksp);
120
121 if (sdbcstat->pre_set == NULL)
122 goto next;
123
124 sdbcstat->collected |= GOT_SET_KSTAT;
125
126 /*
127 * I/O kstat
128 */
129 (void) sprintf(kname, "%s%d", SDBC_IOKSTAT_CDSTATS, kinst);
130
131 io_ksp = kstat_lookup(kc, SDBC_KSTAT_MODULE, kinst, kname);
132 sdbcstat->pre_io = kstat_retrieve(kc, io_ksp);
133
134 if (sdbcstat->pre_io == NULL)
135 goto next;
136
137 sdbcstat->collected |= GOT_IO_KSTAT;
138
139 next:
140 /*
141 * Check if we got a complete set of stats
142 */
143 if (sdbcstat == NULL)
144 continue;
145
146 if (SDBC_COMPLETE(sdbcstat->collected)) {
147 (void) sdbc_delstat(sdbcstat);
148 continue;
149 }
150
151 sdbc_addstat(sdbcstat);
152 }
153
154 if (sdbc_top == NULL)
155 return (EAGAIN);
156
157 return (0);
158 }
159
160 /*
161 * sdbc_update() - updates all of the statistics currently being monitored.
162 *
163 */
164 int
sdbc_update(kstat_ctl_t * kc)165 sdbc_update(kstat_ctl_t *kc)
166 {
167 kstat_t *ksp;
168 sdbcstat_t *cur;
169
170 /* Update global kstat information */
171 ksp = kstat_lookup(kc, SDBC_KSTAT_MODULE, -1, SDBC_KSTAT_GSTATS);
172
173 if (ksp == NULL)
174 return (EAGAIN);
175
176 if (sdbc_global)
177 kstat_free(sdbc_global);
178
179 sdbc_global = kstat_retrieve(kc, ksp);
180
181 for (cur = sdbc_top; cur != NULL; cur = cur->next) {
182 int kinst;
183 char *kname, *cname, *pname;
184
185 kstat_t *set_ksp, *io_ksp;
186
187 cur->collected = 0;
188
189 /*
190 * Age off old stats
191 */
192 if (cur->cur_set != NULL) {
193 kstat_free(cur->pre_set);
194 kstat_free(cur->pre_io);
195
196 cur->pre_set = cur->cur_set;
197 cur->pre_io = cur->cur_io;
198 }
199
200 /*
201 * Update set kstat
202 */
203 kinst = cur->pre_set->ks_instance;
204 kname = cur->pre_set->ks_name;
205
206 set_ksp = kstat_lookup(kc, SDBC_KSTAT_MODULE, kinst, kname);
207
208 if ((cur->cur_set = kstat_retrieve(kc, set_ksp)) == NULL)
209 continue;
210
211 cur->collected |= GOT_SET_KSTAT;
212
213 /*
214 * Validate set
215 */
216 pname = kstat_value(cur->pre_set, SDBC_CDKSTAT_VOL_NAME);
217 cname = kstat_value(cur->cur_set, SDBC_CDKSTAT_VOL_NAME);
218
219 if (strncmp(pname, cname, NAMED_LEN) != 0)
220 continue;
221
222 /*
223 * Update I/O kstat
224 */
225 kinst = cur->pre_io->ks_instance;
226 kname = cur->pre_io->ks_name;
227
228 io_ksp = kstat_lookup(kc, SDBC_KSTAT_MODULE, kinst, kname);
229
230 if ((cur->cur_io = kstat_retrieve(kc, io_ksp)) == NULL)
231 continue;
232
233 cur->collected |= GOT_IO_KSTAT;
234 }
235
236 return (0);
237 }
238
239 /*
240 * sdbc_report() - outputs statistics for the statistics currently being
241 * monitored. Deletes statistics for volumes that have been disabled.
242 *
243 */
244 int
sdbc_report()245 sdbc_report()
246 {
247 vslist_t *vslist = vs_top;
248 sdbcstat_t *cur, *pre = NULL;
249
250 if (sdbc_top == NULL)
251 return (0);
252
253 for (cur = sdbc_top; cur != NULL; ) { /* CSTYLED */
254 static uint32_t linesout = 0;
255 uint32_t *offline;
256
257 char volname[NAMED_LEN + 1];
258 char rmode[STAT_HDR_SIZE];
259 char wmode[STAT_HDR_SIZE];
260
261 /* Parse volume name */
262 (void) strncpy(volname, kstat_value(cur->pre_set,
263 SDBC_CDKSTAT_VOL_NAME), NAMED_LEN);
264 volname[NAMED_LEN] = '\0';
265
266 /* Check to see if the user specified this volume */
267 for (vslist = vs_top; vslist != NULL; vslist = vslist->next)
268 if (strcmp(volname, vslist->volname) == 0)
269 break;
270
271 if (vs_top != NULL && vslist == NULL)
272 goto next;
273
274 /* Check if volume is offline and zflag applies */
275 if (zflag && sdbc_value_check(cur) == 0)
276 goto next;
277
278 /* Output volume name */
279 sdbc_header();
280
281 (void) printf(DATA_C16, volname);
282
283 if (SDBC_COMPLETE(cur->collected)) {
284 sdbcstat_t *next = sdbc_delstat(cur);
285
286 if (! pre)
287 cur = sdbc_top = next;
288 else
289 cur = pre->next = next;
290
291 (void) printf(" <<volume disabled>>\n");
292 continue;
293 }
294
295 offline = kstat_value(cur->cur_set, SDBC_CDKSTAT_FAILED);
296 if (*offline) {
297 (void) printf(" <<volume offline>>\n");
298 linesout++;
299 goto next;
300 }
301
302 /* Type/status flags */
303 if (dflags & FLAGS) {
304
305 uint32_t *dhint, *nhint;
306 uint32_t hints;
307
308 dhint = kstat_value(cur->cur_set, SDBC_CDKSTAT_CDHINTS);
309 nhint = kstat_value(sdbc_global, SDBC_GKSTAT_NODEHINTS);
310
311 if (! nhint)
312 return (EINVAL);
313
314 hints = *nhint;
315 hints &= (NSC_FORCED_WRTHRU | NSC_NO_FORCED_WRTHRU |
316 NSC_NOCACHE);
317 hints |= *dhint;
318
319 if (hints & NSC_NOCACHE)
320 (void) strcpy(rmode, "D");
321 else
322 (void) strcpy(rmode, "C");
323
324 if ((hints & NSC_FORCED_WRTHRU) || (hints & NSC_WRTHRU))
325 (void) strcpy(wmode, "D");
326 else
327 (void) strcpy(wmode, "C");
328
329 (void) printf(DATA_C2, rmode);
330 (void) printf(DATA_C2, wmode);
331 }
332
333 /* Output set information */
334 cd_report(cur);
335
336 next:
337 pre = cur;
338 cur = cur->next;
339 }
340
341 return (0);
342 }
343
344 /*
345 * sdbc_header() - outputs an appropriate header by referencing the
346 * global variables dflsgs
347 *
348 */
349 void
sdbc_header()350 sdbc_header()
351 {
352 int rcount = 0;
353
354 if (hflags == HEADERS_EXL)
355 if ((linesout % DISPLAY_LINES) != 0)
356 return;
357
358 if (hflags == HEADERS_BOR)
359 if (linesout != 0)
360 return;
361
362 if (hflags & HEADERS_ATT)
363 if (hflags & HEADERS_OUT)
364 return;
365 else
366 hflags |= HEADERS_OUT;
367
368 if (linesout)
369 (void) printf("\n");
370
371 /* first line header */
372 if (! (dflags & SUMMARY) && dflags != FLAGS) {
373
374 (void) printf(VOL_HDR_FMT, " ");
375
376 if (dflags & FLAGS) {
377 (void) printf(STAT_HDR_FMT, " ");
378 (void) printf(STAT_HDR_FMT, " ");
379 }
380
381 if (dflags & READ) {
382 int size;
383
384 size = KPS_HDR_SIZE * 2 + HIT_HDR_SIZE;
385 center(size, "- read -");
386 rcount++;
387 }
388
389 if (dflags & WRITE) {
390 int size;
391
392 size = KPS_HDR_SIZE * 2 + HIT_HDR_SIZE;
393 center(size, "- write -");
394 rcount++;
395 }
396
397 if (dflags != FLAGS)
398 (void) printf("\n");
399 }
400
401 /* second line header */
402 (void) printf(VOL_HDR_FMT, "volume");
403
404 if (dflags & FLAGS) {
405 (void) printf(STAT_HDR_FMT, "rd");
406 (void) printf(STAT_HDR_FMT, "wr");
407 }
408
409 if (dflags & SUMMARY) {
410 (void) printf(KPS_HDR_FMT, "ckps");
411 (void) printf(KPS_HDR_FMT, "dkps");
412 (void) printf(HIT_HDR_FMT, HIT_HDR_TXT);
413
414 goto out;
415 }
416
417 if (dflags & READ) {
418 (void) printf(KPS_HDR_FMT, "ckps");
419 (void) printf(KPS_HDR_FMT, "dkps");
420 (void) printf(HIT_HDR_FMT, RHIT_HDR_TXT);
421 }
422
423 if (dflags & WRITE) {
424 (void) printf(KPS_HDR_FMT, "ckps");
425 (void) printf(KPS_HDR_FMT, "dkps");
426 (void) printf(HIT_HDR_FMT, WHIT_HDR_TXT);
427 }
428
429 if (dflags & DESTAGED)
430 (void) printf(KPS_HDR_FMT, "dstg");
431
432 if (dflags & WRCANCEL)
433 (void) printf(KPS_HDR_FMT, "cwrl");
434
435 out:
436 (void) printf("\n");
437 }
438
439 /*
440 * sdbc_getstat() - find cache stat by name matching
441 *
442 * paraemters
443 * char *vn - the volume name to match against
444 * returns
445 * sdbcstat_t * - the matching strcture, NULL if not found
446 */
447 sdbcstat_t *
sdbc_getstat(char * vn)448 sdbc_getstat(char *vn)
449 {
450 sdbcstat_t *cur, *pre = NULL;
451
452 for (cur = sdbc_top; cur; ) { /* CSTYLED */
453 char *volname =
454 kstat_value(cur->pre_set, SDBC_CDKSTAT_VOL_NAME);
455
456 if (SDBC_COMPLETE(cur->collected)) {
457 sdbcstat_t *next = sdbc_delstat(cur);
458
459 if (! pre)
460 cur = sdbc_top = next;
461 else
462 cur = pre->next = next;
463
464 continue;
465 }
466
467 if (strncmp(volname, vn, NAMED_LEN) == 0)
468 return (cur);
469
470 pre = cur;
471 cur = cur->next;
472 }
473
474 return (NULL);
475 }
476
477 /*
478 * sdbc_addstat() - adds a fully populated sdbcstat_t structure
479 * to the linked list of currently monitored kstats. The structure
480 * will be added in alphabetical order, using the volume name as the
481 * key.
482 *
483 * parameters
484 * sdbcstat_t *sdbcstat - to be added to the list.
485 *
486 */
487 void
sdbc_addstat(sdbcstat_t * sdbcstat)488 sdbc_addstat(sdbcstat_t *sdbcstat)
489 {
490 sdbcstat_t *cur;
491
492 if (sdbc_top == NULL) {
493 sdbc_top = sdbcstat;
494 return;
495 }
496
497 for (cur = sdbc_top; cur != NULL; cur = cur->next) {
498 char *cur_vname, *nxt_vname, *tst_vname;
499
500 cur_vname = kstat_value(cur->pre_set,
501 SDBC_CDKSTAT_VOL_NAME);
502 tst_vname = kstat_value(sdbcstat->pre_set,
503 SDBC_CDKSTAT_VOL_NAME);
504
505 if (strncmp(cur_vname, tst_vname, NAMED_LEN) > 0) {
506 if (cur == sdbc_top)
507 sdbc_top = sdbcstat;
508
509 sdbcstat->next = cur;
510
511 return;
512 }
513
514 /*
515 * If we get to the last item in the list, then just
516 * add this one to the end
517 */
518 if (cur->next == NULL) {
519 cur->next = sdbcstat;
520 return;
521 }
522
523 nxt_vname = kstat_value(cur->next->pre_set,
524 SDBC_CDKSTAT_VOL_NAME);
525
526 if (strncmp(nxt_vname, tst_vname, NAMED_LEN) > 0) {
527 sdbcstat->next = cur->next;
528 cur->next = sdbcstat;
529 return;
530 }
531 }
532 }
533
534 /*
535 * sdbc_delstat() - deallocate memory for the structure being
536 * passed in.
537 *
538 * parameters
539 * sdbcstat_t *sdbcstat - structure to be deallocated
540 *
541 * returns
542 * sdbcstat_t * - pointer to the "next" structures in the
543 * linked list. May be NULL if we are removing the last
544 * structure in the linked list.
545 */
546 sdbcstat_t *
sdbc_delstat(sdbcstat_t * sdbcstat)547 sdbc_delstat(sdbcstat_t *sdbcstat)
548 {
549
550 sdbcstat_t *next = sdbcstat->next;
551
552 kstat_free(sdbcstat->pre_set);
553 kstat_free(sdbcstat->pre_io);
554 kstat_free(sdbcstat->cur_set);
555 kstat_free(sdbcstat->cur_io);
556
557 free(sdbcstat);
558 sdbcstat = NULL;
559
560 return (next);
561 }
562
563 /*
564 * sdbc_value_check() - Checks for activity, supports -z switch
565 *
566 * parameters
567 * sdbcstat_t *sdbcstat - structure to be checked
568 *
569 * returns
570 * 1 - activity
571 * 0 - no activity
572 */
573 int
sdbc_value_check(sdbcstat_t * sdbcstat)574 sdbc_value_check(sdbcstat_t *sdbcstat)
575 {
576 if (SDBC_COMPLETE(sdbcstat->collected))
577 return (1);
578
579 if (sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_CACHE_READ) != 0)
580 return (1);
581
582 if (sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_DISK_READ) != 0)
583 return (1);
584
585 if (sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_CACHE_WRITE) != 0)
586 return (1);
587
588 if (sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_DISK_WRITE) != 0)
589 return (1);
590
591 if (sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_WRCANCELNS) != 0)
592 return (1);
593
594 if (io_value_check(sdbcstat->pre_io->ks_data,
595 sdbcstat->cur_io->ks_data) != 0)
596 return (1);
597
598 return (0);
599 }
600
601 /*
602 * sdbc_validate() - validates the structure of the kstats by attempting to
603 * lookup fields used by this module
604 *
605 * parameters
606 * kstat_t *ksp - kstat to be examined
607 *
608 * returns
609 * 1 - one or more fields missing
610 * 0 - all fields present
611 */
612 int
sdbc_validate(kstat_t * ksp)613 sdbc_validate(kstat_t *ksp)
614 {
615 if (! kstat_value(ksp, SDBC_CDKSTAT_VOL_NAME) ||
616 ! kstat_value(ksp, SDBC_CDKSTAT_FAILED) ||
617 ! kstat_value(ksp, SDBC_CDKSTAT_CDHINTS) ||
618 ! kstat_value(ksp, SDBC_CDKSTAT_CACHE_READ) ||
619 ! kstat_value(ksp, SDBC_CDKSTAT_DISK_READ) ||
620 ! kstat_value(ksp, SDBC_CDKSTAT_CACHE_WRITE) ||
621 ! kstat_value(ksp, SDBC_CDKSTAT_DISK_WRITE) ||
622 ! kstat_value(ksp, SDBC_CDKSTAT_DESTAGED) ||
623 ! kstat_value(ksp, SDBC_CDKSTAT_WRCANCELNS))
624 return (1);
625
626 return (0);
627 }
628
629 /*
630 * sdbc_getvalues() - populates a values structure with data obtained from the
631 * kstat
632 *
633 * parameters
634 * sdbcstat_t *sdbcstat - pointer to the structure containing the kstats
635 * sdbcvals_t *vals - pointer to the structure that will receive the values
636 * int flags - flags that describe adjustments made to the values
637 *
638 * returns
639 * 1 - failure
640 * 0 - success
641 */
642 int
sdbc_getvalues(sdbcstat_t * sdbcstat,sdbcvals_t * vals,int flags)643 sdbc_getvalues(sdbcstat_t *sdbcstat, sdbcvals_t *vals, int flags)
644 {
645 int divisor = 0;
646 int factors;
647 uint64_t hr_etime;
648 double etime;
649
650 kstat_io_t *cur;
651 kstat_io_t *pre;
652
653 if (sdbcstat == NULL)
654 return (1);
655
656 cur = sdbcstat->cur_io->ks_data;
657 pre = sdbcstat->pre_io->ks_data;
658
659 hr_etime = hrtime_delta(pre->rlastupdate, cur->rlastupdate);
660 etime = hr_etime / (double)NANOSEC;
661
662 /* read data */
663 vals->cache_read =
664 FBA_SIZE(sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_CACHE_READ));
665 vals->disk_read =
666 FBA_SIZE(sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_DISK_READ));
667
668
669 vals->total_reads = vals->cache_read + vals->disk_read;
670
671 if (vals->cache_read == 0)
672 vals->read_hit = 0.0;
673 else
674 vals->read_hit =
675 ((float)vals->cache_read / vals->total_reads) * 100.0;
676
677 /* write data */
678 vals->cache_write =
679 FBA_SIZE(sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_CACHE_WRITE));
680 vals->disk_write =
681 FBA_SIZE(sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_DISK_WRITE));
682
683 vals->total_writes = vals->cache_write + vals->disk_write;
684
685 vals->destaged =
686 FBA_SIZE(sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_DESTAGED));
687
688 if (vals->cache_write == 0)
689 vals->write_hit = 0.0;
690 else
691 vals->write_hit = ((float)vals->cache_write /
692 (vals->total_writes - vals->destaged)) * 100.0;
693
694 /* miscellaneous */
695 vals->write_cancellations =
696 FBA_SIZE(sdbc_getdelta(sdbcstat, SDBC_CDKSTAT_WRCANCELNS));
697
698 vals->total_cache = vals->cache_read + vals->cache_write;
699 vals->total_disk = vals->disk_read + vals->disk_write;
700
701 /* total cache hit calculation */
702 vals->cache_hit = 0;
703 factors = 0;
704
705 if (vals->cache_read != 0) {
706 vals->cache_hit += vals->read_hit;
707 factors++;
708 }
709
710 if (vals->cache_write != 0) {
711 vals->cache_hit += vals->write_hit;
712 factors++;
713 }
714
715 if (vals->cache_hit)
716 vals->cache_hit /= (float)factors;
717
718 /* adjustments */
719 divisor = 1;
720
721 if (flags & SDBC_KBYTES)
722 divisor *= KILOBYTE;
723 if ((flags & SDBC_INTAVG) && (etime > 0))
724 divisor *= etime;
725
726 if (divisor != 1) {
727 vals->cache_read /= divisor;
728 vals->disk_read /= divisor;
729 vals->total_reads /= divisor;
730
731 vals->cache_write /= divisor;
732 vals->disk_write /= divisor;
733 vals->total_writes /= divisor;
734
735 vals->total_cache /= divisor;
736 vals->total_disk /= divisor;
737
738 vals->destaged /= divisor;
739 vals->write_cancellations /= divisor;
740 }
741
742 return (0);
743 }
744
745 /*
746 * sdbc_getdelta() - calculates the difference between two kstat fields
747 *
748 * parameters
749 * sdbcstat_t *sdbcstat - the SDBC stat strcture containing the two fields
750 * char *name - the name of the fields
751 * returns
752 * uint32_t value of the differences adjusted for overflow of the data type
753 */
754 uint32_t
sdbc_getdelta(sdbcstat_t * sdbcstat,char * name)755 sdbc_getdelta(sdbcstat_t *sdbcstat, char *name)
756 {
757 uint32_t *cur_val;
758 uint32_t *pre_val;
759
760 pre_val = kstat_value(sdbcstat->pre_set, name);
761 cur_val = kstat_value(sdbcstat->cur_set, name);
762
763 return (u32_delta(*pre_val, *cur_val));
764 }
765
766 void
center(int size,char * hdr)767 center(int size, char *hdr)
768 {
769 int lpad = 0;
770 int rpad = 0;
771 char fmt[10];
772
773 if (size == 0)
774 return;
775
776 if (strlen(hdr) < size) {
777 lpad = (size - strlen(hdr)) / 2;
778
779 if (lpad * 2 < size)
780 lpad++;
781
782 rpad = size - (lpad + strlen(hdr));
783 }
784
785 output:
786 (void) sprintf(fmt, "%%%ds%%s%%%ds", lpad, rpad);
787 (void) printf(fmt, " ", hdr, " ");
788 }
789