1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp. 5 * Copyright (c) 2017 Alexander Motin <mav@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification. 14 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 15 * substantially similar to the "NO WARRANTY" disclaimer below 16 * ("Disclaimer") and any redistribution must be conditioned upon 17 * including a substantially similar Disclaimer requirement for further 18 * binary redistribution. 19 * 20 * NO WARRANTY 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 29 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 30 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGES. 32 * 33 * $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.c#4 $ 34 */ 35 /* 36 * CAM Target Layer statistics program 37 * 38 * Authors: Ken Merry <ken@FreeBSD.org>, Will Andrews <will@FreeBSD.org> 39 */ 40 41 #include <sys/param.h> 42 #include <sys/callout.h> 43 #include <sys/ioctl.h> 44 #include <sys/queue.h> 45 #include <sys/resource.h> 46 #include <sys/sbuf.h> 47 #include <sys/socket.h> 48 #include <sys/sysctl.h> 49 #include <sys/time.h> 50 #include <assert.h> 51 #include <bsdxml.h> 52 #include <malloc_np.h> 53 #include <stdint.h> 54 #include <stdio.h> 55 #include <stdlib.h> 56 #include <unistd.h> 57 #include <fcntl.h> 58 #include <inttypes.h> 59 #include <getopt.h> 60 #include <string.h> 61 #include <errno.h> 62 #include <err.h> 63 #include <ctype.h> 64 #include <bitstring.h> 65 #include <cam/scsi/scsi_all.h> 66 #include <cam/ctl/ctl.h> 67 #include <cam/ctl/ctl_io.h> 68 #include <cam/ctl/ctl_scsi_all.h> 69 #include <cam/ctl/ctl_util.h> 70 #include <cam/ctl/ctl_backend.h> 71 #include <cam/ctl/ctl_ioctl.h> 72 73 /* 74 * The default amount of space we allocate for stats storage space. 75 * We dynamically allocate more if needed. 76 */ 77 #define CTL_STAT_NUM_ITEMS 256 78 79 static int ctl_stat_bits; 80 81 static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:"; 82 static const char *ctlstat_usage = "Usage: ctlstat [-CDPdjht] [-l lunnum]" 83 "[-c count] [-n numdevs] [-w wait]\n"; 84 85 struct ctl_cpu_stats { 86 uint64_t user; 87 uint64_t nice; 88 uint64_t system; 89 uint64_t intr; 90 uint64_t idle; 91 }; 92 93 typedef enum { 94 CTLSTAT_MODE_STANDARD, 95 CTLSTAT_MODE_DUMP, 96 CTLSTAT_MODE_JSON, 97 CTLSTAT_MODE_PROMETHEUS, 98 } ctlstat_mode_types; 99 100 #define CTLSTAT_FLAG_CPU (1 << 0) 101 #define CTLSTAT_FLAG_HEADER (1 << 1) 102 #define CTLSTAT_FLAG_FIRST_RUN (1 << 2) 103 #define CTLSTAT_FLAG_TOTALS (1 << 3) 104 #define CTLSTAT_FLAG_DMA_TIME (1 << 4) 105 #define CTLSTAT_FLAG_TIME_VALID (1 << 5) 106 #define CTLSTAT_FLAG_MASK (1 << 6) 107 #define CTLSTAT_FLAG_LUNS (1 << 7) 108 #define CTLSTAT_FLAG_PORTS (1 << 8) 109 #define F_CPU(ctx) ((ctx)->flags & CTLSTAT_FLAG_CPU) 110 #define F_HDR(ctx) ((ctx)->flags & CTLSTAT_FLAG_HEADER) 111 #define F_FIRST(ctx) ((ctx)->flags & CTLSTAT_FLAG_FIRST_RUN) 112 #define F_TOTALS(ctx) ((ctx)->flags & CTLSTAT_FLAG_TOTALS) 113 #define F_DMA(ctx) ((ctx)->flags & CTLSTAT_FLAG_DMA_TIME) 114 #define F_TIMEVAL(ctx) ((ctx)->flags & CTLSTAT_FLAG_TIME_VALID) 115 #define F_MASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_MASK) 116 #define F_LUNS(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUNS) 117 #define F_PORTS(ctx) ((ctx)->flags & CTLSTAT_FLAG_PORTS) 118 119 struct ctlstat_context { 120 ctlstat_mode_types mode; 121 int flags; 122 struct ctl_io_stats *cur_stats, *prev_stats; 123 struct ctl_io_stats cur_total_stats[3], prev_total_stats[3]; 124 struct timespec cur_time, prev_time; 125 struct ctl_cpu_stats cur_cpu, prev_cpu; 126 uint64_t cur_total_jiffies, prev_total_jiffies; 127 uint64_t cur_idle, prev_idle; 128 bitstr_t *item_mask; 129 int cur_items, prev_items; 130 int cur_alloc, prev_alloc; 131 int numdevs; 132 int header_interval; 133 }; 134 135 struct cctl_portlist_data { 136 int level; 137 struct sbuf *cur_sb[32]; 138 int id; 139 int lun; 140 int ntargets; 141 char *target; 142 char **targets; 143 }; 144 145 #ifndef min 146 #define min(x,y) (((x) < (y)) ? (x) : (y)) 147 #endif 148 149 static void usage(int error); 150 static int getstats(int fd, int *alloc_items, int *num_items, 151 struct ctl_io_stats **xstats, struct timespec *cur_time, int *time_valid, 152 bool ports); 153 static int getcpu(struct ctl_cpu_stats *cpu_stats); 154 static void compute_stats(struct ctl_io_stats *cur_stats, 155 struct ctl_io_stats *prev_stats, 156 long double etime, long double *mbsec, 157 long double *kb_per_transfer, 158 long double *transfers_per_second, 159 long double *ms_per_transfer, 160 long double *ms_per_dma, 161 long double *dmas_per_second); 162 163 static void 164 usage(int error) 165 { 166 fputs(ctlstat_usage, error ? stderr : stdout); 167 } 168 169 static int 170 getstats(int fd, int *alloc_items, int *num_items, struct ctl_io_stats **stats, 171 struct timespec *cur_time, int *flags, bool ports) 172 { 173 struct ctl_get_io_stats get_stats; 174 int more_space_count = 0; 175 176 if (*alloc_items == 0) 177 *alloc_items = CTL_STAT_NUM_ITEMS; 178 retry: 179 if (*stats == NULL) 180 *stats = malloc(sizeof(**stats) * *alloc_items); 181 182 memset(&get_stats, 0, sizeof(get_stats)); 183 get_stats.alloc_len = *alloc_items * sizeof(**stats); 184 memset(*stats, 0, get_stats.alloc_len); 185 get_stats.stats = *stats; 186 187 if (ioctl(fd, ports ? CTL_GET_PORT_STATS : CTL_GET_LUN_STATS, 188 &get_stats) == -1) 189 err(1, "CTL_GET_*_STATS ioctl returned error"); 190 191 switch (get_stats.status) { 192 case CTL_SS_OK: 193 break; 194 case CTL_SS_ERROR: 195 err(1, "CTL_GET_*_STATS ioctl returned CTL_SS_ERROR"); 196 break; 197 case CTL_SS_NEED_MORE_SPACE: 198 if (more_space_count >= 2) 199 errx(1, "CTL_GET_*_STATS returned NEED_MORE_SPACE again"); 200 *alloc_items = get_stats.num_items * 5 / 4; 201 free(*stats); 202 *stats = NULL; 203 more_space_count++; 204 goto retry; 205 break; /* NOTREACHED */ 206 default: 207 errx(1, "CTL_GET_*_STATS ioctl returned unknown status %d", 208 get_stats.status); 209 break; 210 } 211 212 *num_items = get_stats.fill_len / sizeof(**stats); 213 cur_time->tv_sec = get_stats.timestamp.tv_sec; 214 cur_time->tv_nsec = get_stats.timestamp.tv_nsec; 215 if (get_stats.flags & CTL_STATS_FLAG_TIME_VALID) 216 *flags |= CTLSTAT_FLAG_TIME_VALID; 217 else 218 *flags &= ~CTLSTAT_FLAG_TIME_VALID; 219 220 return (0); 221 } 222 223 static int 224 getcpu(struct ctl_cpu_stats *cpu_stats) 225 { 226 long cp_time[CPUSTATES]; 227 size_t cplen; 228 229 cplen = sizeof(cp_time); 230 231 if (sysctlbyname("kern.cp_time", &cp_time, &cplen, NULL, 0) == -1) { 232 warn("sysctlbyname(kern.cp_time...) failed"); 233 return (1); 234 } 235 236 cpu_stats->user = cp_time[CP_USER]; 237 cpu_stats->nice = cp_time[CP_NICE]; 238 cpu_stats->system = cp_time[CP_SYS]; 239 cpu_stats->intr = cp_time[CP_INTR]; 240 cpu_stats->idle = cp_time[CP_IDLE]; 241 242 return (0); 243 } 244 245 static void 246 compute_stats(struct ctl_io_stats *cur_stats, 247 struct ctl_io_stats *prev_stats, long double etime, 248 long double *mbsec, long double *kb_per_transfer, 249 long double *transfers_per_second, long double *ms_per_transfer, 250 long double *ms_per_dma, long double *dmas_per_second) 251 { 252 uint64_t total_bytes = 0, total_operations = 0, total_dmas = 0; 253 struct bintime total_time_bt, total_dma_bt; 254 struct timespec total_time_ts, total_dma_ts; 255 int i; 256 257 bzero(&total_time_bt, sizeof(total_time_bt)); 258 bzero(&total_dma_bt, sizeof(total_dma_bt)); 259 bzero(&total_time_ts, sizeof(total_time_ts)); 260 bzero(&total_dma_ts, sizeof(total_dma_ts)); 261 for (i = 0; i < CTL_STATS_NUM_TYPES; i++) { 262 total_bytes += cur_stats->bytes[i]; 263 total_operations += cur_stats->operations[i]; 264 total_dmas += cur_stats->dmas[i]; 265 bintime_add(&total_time_bt, &cur_stats->time[i]); 266 bintime_add(&total_dma_bt, &cur_stats->dma_time[i]); 267 if (prev_stats != NULL) { 268 total_bytes -= prev_stats->bytes[i]; 269 total_operations -= prev_stats->operations[i]; 270 total_dmas -= prev_stats->dmas[i]; 271 bintime_sub(&total_time_bt, &prev_stats->time[i]); 272 bintime_sub(&total_dma_bt, &prev_stats->dma_time[i]); 273 } 274 } 275 276 *mbsec = total_bytes; 277 *mbsec /= 1024 * 1024; 278 if (etime > 0.0) 279 *mbsec /= etime; 280 else 281 *mbsec = 0; 282 *kb_per_transfer = total_bytes; 283 *kb_per_transfer /= 1024; 284 if (total_operations > 0) 285 *kb_per_transfer /= total_operations; 286 else 287 *kb_per_transfer = 0; 288 *transfers_per_second = total_operations; 289 *dmas_per_second = total_dmas; 290 if (etime > 0.0) { 291 *transfers_per_second /= etime; 292 *dmas_per_second /= etime; 293 } else { 294 *transfers_per_second = 0; 295 *dmas_per_second = 0; 296 } 297 298 bintime2timespec(&total_time_bt, &total_time_ts); 299 bintime2timespec(&total_dma_bt, &total_dma_ts); 300 if (total_operations > 0) { 301 /* 302 * Convert the timespec to milliseconds. 303 */ 304 *ms_per_transfer = total_time_ts.tv_sec * 1000; 305 *ms_per_transfer += total_time_ts.tv_nsec / 1000000; 306 *ms_per_transfer /= total_operations; 307 } else 308 *ms_per_transfer = 0; 309 310 if (total_dmas > 0) { 311 /* 312 * Convert the timespec to milliseconds. 313 */ 314 *ms_per_dma = total_dma_ts.tv_sec * 1000; 315 *ms_per_dma += total_dma_ts.tv_nsec / 1000000; 316 *ms_per_dma /= total_dmas; 317 } else 318 *ms_per_dma = 0; 319 } 320 321 /* The dump_stats() and json_stats() functions perform essentially the same 322 * purpose, but dump the statistics in different formats. JSON is more 323 * conducive to programming, however. 324 */ 325 326 #define PRINT_BINTIME(bt) \ 327 printf("%jd.%06ju", (intmax_t)(bt).sec, \ 328 (uintmax_t)(((bt).frac >> 32) * 1000000 >> 32)) 329 static const char *iotypes[] = {"NO IO", "READ", "WRITE"}; 330 331 static void 332 ctlstat_dump(struct ctlstat_context *ctx) 333 { 334 int iotype, i, n; 335 struct ctl_io_stats *stats = ctx->cur_stats; 336 337 for (i = n = 0; i < ctx->cur_items;i++) { 338 if (F_MASK(ctx) && bit_test(ctx->item_mask, 339 (int)stats[i].item) == 0) 340 continue; 341 printf("%s %d\n", F_PORTS(ctx) ? "port" : "lun", stats[i].item); 342 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { 343 printf(" io type %d (%s)\n", iotype, iotypes[iotype]); 344 printf(" bytes %ju\n", (uintmax_t) 345 stats[i].bytes[iotype]); 346 printf(" operations %ju\n", (uintmax_t) 347 stats[i].operations[iotype]); 348 printf(" dmas %ju\n", (uintmax_t) 349 stats[i].dmas[iotype]); 350 printf(" io time "); 351 PRINT_BINTIME(stats[i].time[iotype]); 352 printf("\n dma time "); 353 PRINT_BINTIME(stats[i].dma_time[iotype]); 354 printf("\n"); 355 } 356 if (++n >= ctx->numdevs) 357 break; 358 } 359 } 360 361 static void 362 ctlstat_json(struct ctlstat_context *ctx) { 363 int iotype, i, n; 364 struct ctl_io_stats *stats = ctx->cur_stats; 365 366 printf("{\"%s\":[", F_PORTS(ctx) ? "ports" : "luns"); 367 for (i = n = 0; i < ctx->cur_items; i++) { 368 if (F_MASK(ctx) && bit_test(ctx->item_mask, 369 (int)stats[i].item) == 0) 370 continue; 371 printf("{\"num\":%d,\"io\":[", 372 stats[i].item); 373 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { 374 printf("{\"type\":\"%s\",", iotypes[iotype]); 375 printf("\"bytes\":%ju,", (uintmax_t) 376 stats[i].bytes[iotype]); 377 printf("\"operations\":%ju,", (uintmax_t) 378 stats[i].operations[iotype]); 379 printf("\"dmas\":%ju,", (uintmax_t) 380 stats[i].dmas[iotype]); 381 printf("\"io time\":"); 382 PRINT_BINTIME(stats[i].time[iotype]); 383 printf(",\"dma time\":"); 384 PRINT_BINTIME(stats[i].dma_time[iotype]); 385 printf("}"); 386 if (iotype < (CTL_STATS_NUM_TYPES - 1)) 387 printf(","); /* continue io array */ 388 } 389 printf("]}"); 390 if (++n >= ctx->numdevs) 391 break; 392 if (i < (ctx->cur_items - 1)) 393 printf(","); /* continue lun array */ 394 } 395 printf("]}"); 396 } 397 398 #define CTLSTAT_PROMETHEUS_LOOP(field, collector) \ 399 for (i = n = 0; i < ctx->cur_items; i++) { \ 400 if (F_MASK(ctx) && bit_test(ctx->item_mask, \ 401 (int)stats[i].item) == 0) \ 402 continue; \ 403 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \ 404 int idx = stats[i].item; \ 405 /* \ 406 * Note that Prometheus considers a label value of "" \ 407 * to be the same as no label at all \ 408 */ \ 409 const char *target = ""; \ 410 if (strcmp(collector, "port") == 0 && \ 411 targdata.targets[idx] != NULL) \ 412 { \ 413 target = targdata.targets[idx]; \ 414 } \ 415 printf("iscsi_%s_" #field "{" \ 416 "%s=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \ 417 "\n", \ 418 collector, collector, \ 419 idx, target, iotypes[iotype], \ 420 stats[i].field[iotype]); \ 421 } \ 422 } \ 423 424 #define CTLSTAT_PROMETHEUS_TIMELOOP(field, collector) \ 425 for (i = n = 0; i < ctx->cur_items; i++) { \ 426 if (F_MASK(ctx) && bit_test(ctx->item_mask, \ 427 (int)stats[i].item) == 0) \ 428 continue; \ 429 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \ 430 uint64_t us; \ 431 struct timespec ts; \ 432 int idx = stats[i].item; \ 433 /* \ 434 * Note that Prometheus considers a label value of "" \ 435 * to be the same as no label at all \ 436 */ \ 437 const char *target = ""; \ 438 if (strcmp(collector, "port") == 0 && \ 439 targdata.targets[idx] != NULL) \ 440 { \ 441 target = targdata.targets[idx]; \ 442 } \ 443 bintime2timespec(&stats[i].field[iotype], &ts); \ 444 us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \ 445 printf("iscsi_%s_" #field "{" \ 446 "%s=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \ 447 "\n", \ 448 collector, collector, \ 449 idx, target, iotypes[iotype], us); \ 450 } \ 451 } \ 452 453 static void 454 cctl_start_pelement(void *user_data, const char *name, const char **attr) 455 { 456 struct cctl_portlist_data* targdata = user_data; 457 458 targdata->level++; 459 if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) / 460 sizeof(targdata->cur_sb[0]))) 461 errx(1, "%s: too many nesting levels, %zd max", __func__, 462 sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0])); 463 464 targdata->cur_sb[targdata->level] = sbuf_new_auto(); 465 if (targdata->cur_sb[targdata->level] == NULL) 466 err(1, "%s: Unable to allocate sbuf", __func__); 467 468 if (strcmp(name, "targ_port") == 0) { 469 int i = 0; 470 471 targdata->lun = -1; 472 targdata->id = -1; 473 free(targdata->target); 474 targdata->target = NULL; 475 while (attr[i]) { 476 if (strcmp(attr[i], "id") == 0) { 477 /* 478 * Well-formed XML always pairs keys with 479 * values in attr 480 */ 481 assert(attr[i + 1]); 482 targdata->id = atoi(attr[i + 1]); 483 } 484 i += 2; 485 } 486 487 } 488 } 489 490 static void 491 cctl_char_phandler(void *user_data, const XML_Char *str, int len) 492 { 493 struct cctl_portlist_data *targdata = user_data; 494 495 sbuf_bcat(targdata->cur_sb[targdata->level], str, len); 496 } 497 498 static void 499 cctl_end_pelement(void *user_data, const char *name) 500 { 501 struct cctl_portlist_data* targdata = user_data; 502 char *str; 503 504 if (targdata->cur_sb[targdata->level] == NULL) 505 errx(1, "%s: no valid sbuf at level %d (name %s)", __func__, 506 targdata->level, name); 507 508 if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0) 509 err(1, "%s: sbuf_finish", __func__); 510 str = strdup(sbuf_data(targdata->cur_sb[targdata->level])); 511 if (str == NULL) 512 err(1, "%s can't allocate %zd bytes for string", __func__, 513 sbuf_len(targdata->cur_sb[targdata->level])); 514 515 sbuf_delete(targdata->cur_sb[targdata->level]); 516 targdata->cur_sb[targdata->level] = NULL; 517 targdata->level--; 518 519 if (strcmp(name, "target") == 0) { 520 free(targdata->target); 521 targdata->target = str; 522 } else if (strcmp(name, "targ_port") == 0) { 523 if (targdata->id >= 0 && targdata->target != NULL) { 524 if (targdata->id >= targdata->ntargets) { 525 /* 526 * This can happen for example if there are 527 * targets with no LUNs. 528 */ 529 targdata->ntargets = MAX(targdata->ntargets * 2, 530 targdata->id + 1); 531 size_t newsize = targdata->ntargets * 532 sizeof(char*); 533 targdata->targets = rallocx(targdata->targets, 534 newsize, MALLOCX_ZERO); 535 } 536 free(targdata->targets[targdata->id]); 537 targdata->targets[targdata->id] = targdata->target; 538 targdata->target = NULL; 539 } 540 free(str); 541 } else { 542 free(str); 543 } 544 } 545 546 static void 547 ctlstat_prometheus(int fd, struct ctlstat_context *ctx, bool ports) { 548 struct ctl_io_stats *stats = ctx->cur_stats; 549 struct ctl_lun_list list; 550 struct cctl_portlist_data targdata; 551 XML_Parser parser; 552 char *port_str = NULL; 553 int iotype, i, n, retval; 554 int port_len = 4096; 555 const char *collector; 556 557 bzero(&targdata, sizeof(targdata)); 558 targdata.ntargets = ctx->cur_items; 559 targdata.targets = calloc(targdata.ntargets, sizeof(char*)); 560 retry: 561 port_str = (char *)realloc(port_str, port_len); 562 bzero(&list, sizeof(list)); 563 list.alloc_len = port_len; 564 list.status = CTL_LUN_LIST_NONE; 565 list.lun_xml = port_str; 566 if (ioctl(fd, CTL_PORT_LIST, &list) == -1) 567 err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__); 568 if (list.status == CTL_LUN_LIST_ERROR) { 569 warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s", 570 __func__, list.error_str); 571 } else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) { 572 port_len <<= 1; 573 goto retry; 574 } 575 576 parser = XML_ParserCreate(NULL); 577 if (parser == NULL) 578 err(1, "%s: Unable to create XML parser", __func__); 579 XML_SetUserData(parser, &targdata); 580 XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement); 581 XML_SetCharacterDataHandler(parser, cctl_char_phandler); 582 583 retval = XML_Parse(parser, port_str, strlen(port_str), 1); 584 if (retval != 1) { 585 errx(1, "%s: Unable to parse XML: Error %d", __func__, 586 XML_GetErrorCode(parser)); 587 } 588 XML_ParserFree(parser); 589 590 collector = ports ? "port" : "lun"; 591 592 printf("# HELP iscsi_%s_bytes Number of bytes\n" 593 "# TYPE iscsi_%s_bytes counter\n", collector, collector); 594 CTLSTAT_PROMETHEUS_LOOP(bytes, collector); 595 printf("# HELP iscsi_%s_dmas Number of DMA\n" 596 "# TYPE iscsi_%s_dmas counter\n", collector, collector); 597 CTLSTAT_PROMETHEUS_LOOP(dmas, collector); 598 printf("# HELP iscsi_%s_operations Number of operations\n" 599 "# TYPE iscsi_%s_operations counter\n", collector, collector); 600 CTLSTAT_PROMETHEUS_LOOP(operations, collector); 601 printf("# HELP iscsi_%s_time Cumulative operation time in us\n" 602 "# TYPE iscsi_%s_time counter\n", collector, collector); 603 CTLSTAT_PROMETHEUS_TIMELOOP(time, collector); 604 printf("# HELP iscsi_%s_dma_time Cumulative DMA time in us\n" 605 "# TYPE iscsi_%s_dma_time counter\n", collector, collector); 606 CTLSTAT_PROMETHEUS_TIMELOOP(dma_time, collector); 607 608 for (i = 0; i < targdata.ntargets; i++) 609 free(targdata.targets[i]); 610 free(targdata.target); 611 free(targdata.targets); 612 613 fflush(stdout); 614 } 615 616 static void 617 ctlstat_standard(struct ctlstat_context *ctx) { 618 long double etime; 619 uint64_t delta_jiffies, delta_idle; 620 long double cpu_percentage; 621 int i, j, n; 622 623 cpu_percentage = 0; 624 625 if (F_CPU(ctx) && (getcpu(&ctx->cur_cpu) != 0)) 626 errx(1, "error returned from getcpu()"); 627 628 etime = ctx->cur_time.tv_sec - ctx->prev_time.tv_sec + 629 (ctx->prev_time.tv_nsec - ctx->cur_time.tv_nsec) * 1e-9; 630 631 if (F_CPU(ctx)) { 632 ctx->prev_total_jiffies = ctx->cur_total_jiffies; 633 ctx->cur_total_jiffies = ctx->cur_cpu.user + 634 ctx->cur_cpu.nice + ctx->cur_cpu.system + 635 ctx->cur_cpu.intr + ctx->cur_cpu.idle; 636 delta_jiffies = ctx->cur_total_jiffies; 637 if (F_FIRST(ctx) == 0) 638 delta_jiffies -= ctx->prev_total_jiffies; 639 ctx->prev_idle = ctx->cur_idle; 640 ctx->cur_idle = ctx->cur_cpu.idle; 641 delta_idle = ctx->cur_idle - ctx->prev_idle; 642 643 cpu_percentage = delta_jiffies - delta_idle; 644 cpu_percentage /= delta_jiffies; 645 cpu_percentage *= 100; 646 } 647 648 if (F_HDR(ctx)) { 649 ctx->header_interval--; 650 if (ctx->header_interval <= 0) { 651 if (F_CPU(ctx)) 652 fprintf(stdout, " CPU"); 653 if (F_TOTALS(ctx)) { 654 fprintf(stdout, "%s Read %s" 655 " Write %s Total\n", 656 (F_TIMEVAL(ctx) != 0) ? " " : "", 657 (F_TIMEVAL(ctx) != 0) ? " " : "", 658 (F_TIMEVAL(ctx) != 0) ? " " : ""); 659 n = 3; 660 } else { 661 for (i = n = 0; i < min(ctl_stat_bits, 662 ctx->cur_items); i++) { 663 int item; 664 665 /* 666 * Obviously this won't work with 667 * LUN numbers greater than a signed 668 * integer. 669 */ 670 item = (int)ctx->cur_stats[i].item; 671 672 if (F_MASK(ctx) && 673 bit_test(ctx->item_mask, item) == 0) 674 continue; 675 fprintf(stdout, "%15.6s%d %s", 676 F_PORTS(ctx) ? "port" : "lun", item, 677 (F_TIMEVAL(ctx) != 0) ? " " : ""); 678 if (++n >= ctx->numdevs) 679 break; 680 } 681 fprintf(stdout, "\n"); 682 } 683 if (F_CPU(ctx)) 684 fprintf(stdout, " "); 685 for (i = 0; i < n; i++) 686 fprintf(stdout, "%s KB/t %s MB/s", 687 (F_TIMEVAL(ctx) != 0) ? " ms" : "", 688 (F_DMA(ctx) == 0) ? "tps" : "dps"); 689 fprintf(stdout, "\n"); 690 ctx->header_interval = 20; 691 } 692 } 693 694 if (F_CPU(ctx)) 695 fprintf(stdout, "%3.0Lf%%", cpu_percentage); 696 if (F_TOTALS(ctx) != 0) { 697 long double mbsec[3]; 698 long double kb_per_transfer[3]; 699 long double transfers_per_sec[3]; 700 long double ms_per_transfer[3]; 701 long double ms_per_dma[3]; 702 long double dmas_per_sec[3]; 703 704 for (i = 0; i < 3; i++) 705 ctx->prev_total_stats[i] = ctx->cur_total_stats[i]; 706 707 memset(&ctx->cur_total_stats, 0, sizeof(ctx->cur_total_stats)); 708 709 /* Use macros to make the next loop more readable. */ 710 #define ADD_STATS_BYTES(st, i, j) \ 711 ctx->cur_total_stats[st].bytes[j] += \ 712 ctx->cur_stats[i].bytes[j] 713 #define ADD_STATS_OPERATIONS(st, i, j) \ 714 ctx->cur_total_stats[st].operations[j] += \ 715 ctx->cur_stats[i].operations[j] 716 #define ADD_STATS_DMAS(st, i, j) \ 717 ctx->cur_total_stats[st].dmas[j] += \ 718 ctx->cur_stats[i].dmas[j] 719 #define ADD_STATS_TIME(st, i, j) \ 720 bintime_add(&ctx->cur_total_stats[st].time[j], \ 721 &ctx->cur_stats[i].time[j]) 722 #define ADD_STATS_DMA_TIME(st, i, j) \ 723 bintime_add(&ctx->cur_total_stats[st].dma_time[j], \ 724 &ctx->cur_stats[i].dma_time[j]) 725 726 for (i = 0; i < ctx->cur_items; i++) { 727 if (F_MASK(ctx) && bit_test(ctx->item_mask, 728 (int)ctx->cur_stats[i].item) == 0) 729 continue; 730 for (j = 0; j < CTL_STATS_NUM_TYPES; j++) { 731 ADD_STATS_BYTES(2, i, j); 732 ADD_STATS_OPERATIONS(2, i, j); 733 ADD_STATS_DMAS(2, i, j); 734 ADD_STATS_TIME(2, i, j); 735 ADD_STATS_DMA_TIME(2, i, j); 736 } 737 ADD_STATS_BYTES(0, i, CTL_STATS_READ); 738 ADD_STATS_OPERATIONS(0, i, CTL_STATS_READ); 739 ADD_STATS_DMAS(0, i, CTL_STATS_READ); 740 ADD_STATS_TIME(0, i, CTL_STATS_READ); 741 ADD_STATS_DMA_TIME(0, i, CTL_STATS_READ); 742 743 ADD_STATS_BYTES(1, i, CTL_STATS_WRITE); 744 ADD_STATS_OPERATIONS(1, i, CTL_STATS_WRITE); 745 ADD_STATS_DMAS(1, i, CTL_STATS_WRITE); 746 ADD_STATS_TIME(1, i, CTL_STATS_WRITE); 747 ADD_STATS_DMA_TIME(1, i, CTL_STATS_WRITE); 748 } 749 750 for (i = 0; i < 3; i++) { 751 compute_stats(&ctx->cur_total_stats[i], 752 F_FIRST(ctx) ? NULL : &ctx->prev_total_stats[i], 753 etime, &mbsec[i], &kb_per_transfer[i], 754 &transfers_per_sec[i], 755 &ms_per_transfer[i], &ms_per_dma[i], 756 &dmas_per_sec[i]); 757 if (F_DMA(ctx) != 0) 758 fprintf(stdout, " %5.1Lf", 759 ms_per_dma[i]); 760 else if (F_TIMEVAL(ctx) != 0) 761 fprintf(stdout, " %5.1Lf", 762 ms_per_transfer[i]); 763 fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf", 764 kb_per_transfer[i], 765 (F_DMA(ctx) == 0) ? transfers_per_sec[i] : 766 dmas_per_sec[i], mbsec[i]); 767 } 768 } else { 769 for (i = n = 0; i < min(ctl_stat_bits, ctx->cur_items); i++) { 770 long double mbsec, kb_per_transfer; 771 long double transfers_per_sec; 772 long double ms_per_transfer; 773 long double ms_per_dma; 774 long double dmas_per_sec; 775 776 if (F_MASK(ctx) && bit_test(ctx->item_mask, 777 (int)ctx->cur_stats[i].item) == 0) 778 continue; 779 for (j = 0; j < ctx->prev_items; j++) { 780 if (ctx->prev_stats[j].item == 781 ctx->cur_stats[i].item) 782 break; 783 } 784 if (j >= ctx->prev_items) 785 j = -1; 786 compute_stats(&ctx->cur_stats[i], 787 j >= 0 ? &ctx->prev_stats[j] : NULL, 788 etime, &mbsec, &kb_per_transfer, 789 &transfers_per_sec, &ms_per_transfer, 790 &ms_per_dma, &dmas_per_sec); 791 if (F_DMA(ctx)) 792 fprintf(stdout, " %5.1Lf", 793 ms_per_dma); 794 else if (F_TIMEVAL(ctx) != 0) 795 fprintf(stdout, " %5.1Lf", 796 ms_per_transfer); 797 fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf", 798 kb_per_transfer, (F_DMA(ctx) == 0) ? 799 transfers_per_sec : dmas_per_sec, mbsec); 800 if (++n >= ctx->numdevs) 801 break; 802 } 803 } 804 } 805 806 static void 807 get_and_print_stats(int fd, struct ctlstat_context *ctx, bool ports) 808 { 809 struct ctl_io_stats *tmp_stats; 810 int c; 811 812 tmp_stats = ctx->prev_stats; 813 ctx->prev_stats = ctx->cur_stats; 814 ctx->cur_stats = tmp_stats; 815 c = ctx->prev_alloc; 816 ctx->prev_alloc = ctx->cur_alloc; 817 ctx->cur_alloc = c; 818 c = ctx->prev_items; 819 ctx->prev_items = ctx->cur_items; 820 ctx->cur_items = c; 821 ctx->prev_time = ctx->cur_time; 822 ctx->prev_cpu = ctx->cur_cpu; 823 if (getstats(fd, &ctx->cur_alloc, &ctx->cur_items, 824 &ctx->cur_stats, &ctx->cur_time, &ctx->flags, ports) != 0) 825 errx(1, "error returned from getstats()"); 826 827 switch(ctx->mode) { 828 case CTLSTAT_MODE_STANDARD: 829 ctlstat_standard(ctx); 830 break; 831 case CTLSTAT_MODE_DUMP: 832 ctlstat_dump(ctx); 833 break; 834 case CTLSTAT_MODE_JSON: 835 ctlstat_json(ctx); 836 break; 837 case CTLSTAT_MODE_PROMETHEUS: 838 ctlstat_prometheus(fd, ctx, ports); 839 break; 840 default: 841 break; 842 } 843 } 844 845 int 846 main(int argc, char **argv) 847 { 848 int c; 849 int count, waittime; 850 int fd, retval; 851 size_t size; 852 struct ctlstat_context ctx; 853 854 /* default values */ 855 retval = 0; 856 waittime = 1; 857 count = -1; 858 memset(&ctx, 0, sizeof(ctx)); 859 ctx.numdevs = 3; 860 ctx.mode = CTLSTAT_MODE_STANDARD; 861 ctx.flags |= CTLSTAT_FLAG_CPU; 862 ctx.flags |= CTLSTAT_FLAG_FIRST_RUN; 863 ctx.flags |= CTLSTAT_FLAG_HEADER; 864 865 size = sizeof(ctl_stat_bits); 866 if (sysctlbyname("kern.cam.ctl.max_luns", &ctl_stat_bits, &size, NULL, 867 0) == -1) { 868 /* Backward compatibility for where the sysctl wasn't exposed */ 869 ctl_stat_bits = 1024; 870 } 871 ctx.item_mask = bit_alloc(ctl_stat_bits); 872 if (ctx.item_mask == NULL) 873 err(1, "bit_alloc() failed"); 874 875 while ((c = getopt(argc, argv, ctlstat_opts)) != -1) { 876 switch (c) { 877 case 'C': 878 ctx.flags &= ~CTLSTAT_FLAG_CPU; 879 break; 880 case 'c': 881 count = atoi(optarg); 882 break; 883 case 'd': 884 ctx.flags |= CTLSTAT_FLAG_DMA_TIME; 885 break; 886 case 'D': 887 ctx.mode = CTLSTAT_MODE_DUMP; 888 waittime = 30; 889 break; 890 case 'h': 891 ctx.flags &= ~CTLSTAT_FLAG_HEADER; 892 break; 893 case 'j': 894 ctx.mode = CTLSTAT_MODE_JSON; 895 waittime = 30; 896 break; 897 case 'l': { 898 int cur_lun; 899 900 cur_lun = atoi(optarg); 901 if (cur_lun > ctl_stat_bits) 902 errx(1, "Invalid LUN number %d", cur_lun); 903 904 if (!F_MASK(&ctx)) 905 ctx.numdevs = 1; 906 else 907 ctx.numdevs++; 908 bit_set(ctx.item_mask, cur_lun); 909 ctx.flags |= CTLSTAT_FLAG_MASK; 910 ctx.flags |= CTLSTAT_FLAG_LUNS; 911 break; 912 } 913 case 'n': 914 ctx.numdevs = atoi(optarg); 915 break; 916 case 'p': { 917 int cur_port; 918 919 cur_port = atoi(optarg); 920 if (cur_port > ctl_stat_bits) 921 errx(1, "Invalid port number %d", cur_port); 922 923 if (!F_MASK(&ctx)) 924 ctx.numdevs = 1; 925 else 926 ctx.numdevs++; 927 bit_set(ctx.item_mask, cur_port); 928 ctx.flags |= CTLSTAT_FLAG_MASK; 929 ctx.flags |= CTLSTAT_FLAG_PORTS; 930 break; 931 } 932 case 'P': 933 ctx.mode = CTLSTAT_MODE_PROMETHEUS; 934 break; 935 case 't': 936 ctx.flags |= CTLSTAT_FLAG_TOTALS; 937 break; 938 case 'w': 939 waittime = atoi(optarg); 940 break; 941 default: 942 retval = 1; 943 usage(retval); 944 exit(retval); 945 break; 946 } 947 } 948 949 if (F_LUNS(&ctx) && F_PORTS(&ctx)) 950 errx(1, "Options -p and -l are exclusive."); 951 952 if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { 953 if ((count != -1) || 954 (waittime != 1) || 955 (F_PORTS(&ctx)) || 956 /* NB: -P could be compatible with -t in the future */ 957 (ctx.flags & CTLSTAT_FLAG_TOTALS)) 958 { 959 errx(1, "Option -P is exclusive with -p, -c, -w, and -t"); 960 } 961 count = 1; 962 } 963 964 if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) { 965 if (F_TOTALS(&ctx)) 966 ctx.flags |= CTLSTAT_FLAG_PORTS; 967 else 968 ctx.flags |= CTLSTAT_FLAG_LUNS; 969 } 970 971 if ((fd = open(CTL_DEFAULT_DEV, O_RDWR)) == -1) 972 err(1, "cannot open %s", CTL_DEFAULT_DEV); 973 974 if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { 975 /* 976 * NB: Some clients will print a warning if we don't set 977 * Content-Length, but they still work. And the data still 978 * gets into Prometheus. 979 */ 980 printf("HTTP/1.1 200 OK\r\n" 981 "Connection: close\r\n" 982 "Content-Type: text/plain; version=0.0.4\r\n" 983 "\r\n"); 984 } 985 986 for (;count != 0;) { 987 bool ports; 988 989 if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { 990 get_and_print_stats(fd, &ctx, false); 991 get_and_print_stats(fd, &ctx, true); 992 } else { 993 ports = ctx.flags & CTLSTAT_FLAG_PORTS; 994 get_and_print_stats(fd, &ctx, ports); 995 } 996 997 fprintf(stdout, "\n"); 998 fflush(stdout); 999 ctx.flags &= ~CTLSTAT_FLAG_FIRST_RUN; 1000 if (count != 1) 1001 sleep(waittime); 1002 if (count > 0) 1003 count--; 1004 } 1005 1006 exit (retval); 1007 } 1008 1009 /* 1010 * vim: ts=8 1011 */ 1012