1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2004, 2008, 2009 Silicon Graphics International Corp. 5 * Copyright (c) 2017 Alexander Motin <mav@FreeBSD.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions, and the following disclaimer, 13 * without modification. 14 * 2. Redistributions in binary form must reproduce at minimum a disclaimer 15 * substantially similar to the "NO WARRANTY" disclaimer below 16 * ("Disclaimer") and any redistribution must be conditioned upon 17 * including a substantially similar Disclaimer requirement for further 18 * binary redistribution. 19 * 20 * NO WARRANTY 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 29 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 30 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGES. 32 * 33 * $Id: //depot/users/kenm/FreeBSD-test2/usr.bin/ctlstat/ctlstat.c#4 $ 34 */ 35 /* 36 * CAM Target Layer statistics program 37 * 38 * Authors: Ken Merry <ken@FreeBSD.org>, Will Andrews <will@FreeBSD.org> 39 */ 40 41 #include <sys/cdefs.h> 42 #include <sys/param.h> 43 #include <sys/callout.h> 44 #include <sys/ioctl.h> 45 #include <sys/queue.h> 46 #include <sys/resource.h> 47 #include <sys/sbuf.h> 48 #include <sys/socket.h> 49 #include <sys/sysctl.h> 50 #include <sys/time.h> 51 #include <assert.h> 52 #include <bsdxml.h> 53 #include <malloc_np.h> 54 #include <stdint.h> 55 #include <stdio.h> 56 #include <stdlib.h> 57 #include <unistd.h> 58 #include <fcntl.h> 59 #include <inttypes.h> 60 #include <getopt.h> 61 #include <string.h> 62 #include <errno.h> 63 #include <err.h> 64 #include <ctype.h> 65 #include <bitstring.h> 66 #include <cam/scsi/scsi_all.h> 67 #include <cam/ctl/ctl.h> 68 #include <cam/ctl/ctl_io.h> 69 #include <cam/ctl/ctl_scsi_all.h> 70 #include <cam/ctl/ctl_util.h> 71 #include <cam/ctl/ctl_backend.h> 72 #include <cam/ctl/ctl_ioctl.h> 73 74 /* 75 * The default amount of space we allocate for stats storage space. 76 * We dynamically allocate more if needed. 77 */ 78 #define CTL_STAT_NUM_ITEMS 256 79 80 static int ctl_stat_bits; 81 82 static const char *ctlstat_opts = "Cc:DPdhjl:n:p:tw:"; 83 static const char *ctlstat_usage = "Usage: ctlstat [-CDPdjht] [-l lunnum]" 84 "[-c count] [-n numdevs] [-w wait]\n"; 85 86 struct ctl_cpu_stats { 87 uint64_t user; 88 uint64_t nice; 89 uint64_t system; 90 uint64_t intr; 91 uint64_t idle; 92 }; 93 94 typedef enum { 95 CTLSTAT_MODE_STANDARD, 96 CTLSTAT_MODE_DUMP, 97 CTLSTAT_MODE_JSON, 98 CTLSTAT_MODE_PROMETHEUS, 99 } ctlstat_mode_types; 100 101 #define CTLSTAT_FLAG_CPU (1 << 0) 102 #define CTLSTAT_FLAG_HEADER (1 << 1) 103 #define CTLSTAT_FLAG_FIRST_RUN (1 << 2) 104 #define CTLSTAT_FLAG_TOTALS (1 << 3) 105 #define CTLSTAT_FLAG_DMA_TIME (1 << 4) 106 #define CTLSTAT_FLAG_TIME_VALID (1 << 5) 107 #define CTLSTAT_FLAG_MASK (1 << 6) 108 #define CTLSTAT_FLAG_LUNS (1 << 7) 109 #define CTLSTAT_FLAG_PORTS (1 << 8) 110 #define F_CPU(ctx) ((ctx)->flags & CTLSTAT_FLAG_CPU) 111 #define F_HDR(ctx) ((ctx)->flags & CTLSTAT_FLAG_HEADER) 112 #define F_FIRST(ctx) ((ctx)->flags & CTLSTAT_FLAG_FIRST_RUN) 113 #define F_TOTALS(ctx) ((ctx)->flags & CTLSTAT_FLAG_TOTALS) 114 #define F_DMA(ctx) ((ctx)->flags & CTLSTAT_FLAG_DMA_TIME) 115 #define F_TIMEVAL(ctx) ((ctx)->flags & CTLSTAT_FLAG_TIME_VALID) 116 #define F_MASK(ctx) ((ctx)->flags & CTLSTAT_FLAG_MASK) 117 #define F_LUNS(ctx) ((ctx)->flags & CTLSTAT_FLAG_LUNS) 118 #define F_PORTS(ctx) ((ctx)->flags & CTLSTAT_FLAG_PORTS) 119 120 struct ctlstat_context { 121 ctlstat_mode_types mode; 122 int flags; 123 struct ctl_io_stats *cur_stats, *prev_stats; 124 struct ctl_io_stats cur_total_stats[3], prev_total_stats[3]; 125 struct timespec cur_time, prev_time; 126 struct ctl_cpu_stats cur_cpu, prev_cpu; 127 uint64_t cur_total_jiffies, prev_total_jiffies; 128 uint64_t cur_idle, prev_idle; 129 bitstr_t *item_mask; 130 int cur_items, prev_items; 131 int cur_alloc, prev_alloc; 132 int numdevs; 133 int header_interval; 134 }; 135 136 struct cctl_portlist_data { 137 int level; 138 struct sbuf *cur_sb[32]; 139 int id; 140 int lun; 141 int ntargets; 142 char *target; 143 char **targets; 144 }; 145 146 #ifndef min 147 #define min(x,y) (((x) < (y)) ? (x) : (y)) 148 #endif 149 150 static void usage(int error); 151 static int getstats(int fd, int *alloc_items, int *num_items, 152 struct ctl_io_stats **xstats, struct timespec *cur_time, int *time_valid, 153 bool ports); 154 static int getcpu(struct ctl_cpu_stats *cpu_stats); 155 static void compute_stats(struct ctl_io_stats *cur_stats, 156 struct ctl_io_stats *prev_stats, 157 long double etime, long double *mbsec, 158 long double *kb_per_transfer, 159 long double *transfers_per_second, 160 long double *ms_per_transfer, 161 long double *ms_per_dma, 162 long double *dmas_per_second); 163 164 static void 165 usage(int error) 166 { 167 fputs(ctlstat_usage, error ? stderr : stdout); 168 } 169 170 static int 171 getstats(int fd, int *alloc_items, int *num_items, struct ctl_io_stats **stats, 172 struct timespec *cur_time, int *flags, bool ports) 173 { 174 struct ctl_get_io_stats get_stats; 175 int more_space_count = 0; 176 177 if (*alloc_items == 0) 178 *alloc_items = CTL_STAT_NUM_ITEMS; 179 retry: 180 if (*stats == NULL) 181 *stats = malloc(sizeof(**stats) * *alloc_items); 182 183 memset(&get_stats, 0, sizeof(get_stats)); 184 get_stats.alloc_len = *alloc_items * sizeof(**stats); 185 memset(*stats, 0, get_stats.alloc_len); 186 get_stats.stats = *stats; 187 188 if (ioctl(fd, ports ? CTL_GET_PORT_STATS : CTL_GET_LUN_STATS, 189 &get_stats) == -1) 190 err(1, "CTL_GET_*_STATS ioctl returned error"); 191 192 switch (get_stats.status) { 193 case CTL_SS_OK: 194 break; 195 case CTL_SS_ERROR: 196 err(1, "CTL_GET_*_STATS ioctl returned CTL_SS_ERROR"); 197 break; 198 case CTL_SS_NEED_MORE_SPACE: 199 if (more_space_count >= 2) 200 errx(1, "CTL_GET_*_STATS returned NEED_MORE_SPACE again"); 201 *alloc_items = get_stats.num_items * 5 / 4; 202 free(*stats); 203 *stats = NULL; 204 more_space_count++; 205 goto retry; 206 break; /* NOTREACHED */ 207 default: 208 errx(1, "CTL_GET_*_STATS ioctl returned unknown status %d", 209 get_stats.status); 210 break; 211 } 212 213 *num_items = get_stats.fill_len / sizeof(**stats); 214 cur_time->tv_sec = get_stats.timestamp.tv_sec; 215 cur_time->tv_nsec = get_stats.timestamp.tv_nsec; 216 if (get_stats.flags & CTL_STATS_FLAG_TIME_VALID) 217 *flags |= CTLSTAT_FLAG_TIME_VALID; 218 else 219 *flags &= ~CTLSTAT_FLAG_TIME_VALID; 220 221 return (0); 222 } 223 224 static int 225 getcpu(struct ctl_cpu_stats *cpu_stats) 226 { 227 long cp_time[CPUSTATES]; 228 size_t cplen; 229 230 cplen = sizeof(cp_time); 231 232 if (sysctlbyname("kern.cp_time", &cp_time, &cplen, NULL, 0) == -1) { 233 warn("sysctlbyname(kern.cp_time...) failed"); 234 return (1); 235 } 236 237 cpu_stats->user = cp_time[CP_USER]; 238 cpu_stats->nice = cp_time[CP_NICE]; 239 cpu_stats->system = cp_time[CP_SYS]; 240 cpu_stats->intr = cp_time[CP_INTR]; 241 cpu_stats->idle = cp_time[CP_IDLE]; 242 243 return (0); 244 } 245 246 static void 247 compute_stats(struct ctl_io_stats *cur_stats, 248 struct ctl_io_stats *prev_stats, long double etime, 249 long double *mbsec, long double *kb_per_transfer, 250 long double *transfers_per_second, long double *ms_per_transfer, 251 long double *ms_per_dma, long double *dmas_per_second) 252 { 253 uint64_t total_bytes = 0, total_operations = 0, total_dmas = 0; 254 struct bintime total_time_bt, total_dma_bt; 255 struct timespec total_time_ts, total_dma_ts; 256 int i; 257 258 bzero(&total_time_bt, sizeof(total_time_bt)); 259 bzero(&total_dma_bt, sizeof(total_dma_bt)); 260 bzero(&total_time_ts, sizeof(total_time_ts)); 261 bzero(&total_dma_ts, sizeof(total_dma_ts)); 262 for (i = 0; i < CTL_STATS_NUM_TYPES; i++) { 263 total_bytes += cur_stats->bytes[i]; 264 total_operations += cur_stats->operations[i]; 265 total_dmas += cur_stats->dmas[i]; 266 bintime_add(&total_time_bt, &cur_stats->time[i]); 267 bintime_add(&total_dma_bt, &cur_stats->dma_time[i]); 268 if (prev_stats != NULL) { 269 total_bytes -= prev_stats->bytes[i]; 270 total_operations -= prev_stats->operations[i]; 271 total_dmas -= prev_stats->dmas[i]; 272 bintime_sub(&total_time_bt, &prev_stats->time[i]); 273 bintime_sub(&total_dma_bt, &prev_stats->dma_time[i]); 274 } 275 } 276 277 *mbsec = total_bytes; 278 *mbsec /= 1024 * 1024; 279 if (etime > 0.0) 280 *mbsec /= etime; 281 else 282 *mbsec = 0; 283 *kb_per_transfer = total_bytes; 284 *kb_per_transfer /= 1024; 285 if (total_operations > 0) 286 *kb_per_transfer /= total_operations; 287 else 288 *kb_per_transfer = 0; 289 *transfers_per_second = total_operations; 290 *dmas_per_second = total_dmas; 291 if (etime > 0.0) { 292 *transfers_per_second /= etime; 293 *dmas_per_second /= etime; 294 } else { 295 *transfers_per_second = 0; 296 *dmas_per_second = 0; 297 } 298 299 bintime2timespec(&total_time_bt, &total_time_ts); 300 bintime2timespec(&total_dma_bt, &total_dma_ts); 301 if (total_operations > 0) { 302 /* 303 * Convert the timespec to milliseconds. 304 */ 305 *ms_per_transfer = total_time_ts.tv_sec * 1000; 306 *ms_per_transfer += total_time_ts.tv_nsec / 1000000; 307 *ms_per_transfer /= total_operations; 308 } else 309 *ms_per_transfer = 0; 310 311 if (total_dmas > 0) { 312 /* 313 * Convert the timespec to milliseconds. 314 */ 315 *ms_per_dma = total_dma_ts.tv_sec * 1000; 316 *ms_per_dma += total_dma_ts.tv_nsec / 1000000; 317 *ms_per_dma /= total_dmas; 318 } else 319 *ms_per_dma = 0; 320 } 321 322 /* The dump_stats() and json_stats() functions perform essentially the same 323 * purpose, but dump the statistics in different formats. JSON is more 324 * conducive to programming, however. 325 */ 326 327 #define PRINT_BINTIME(bt) \ 328 printf("%jd.%06ju", (intmax_t)(bt).sec, \ 329 (uintmax_t)(((bt).frac >> 32) * 1000000 >> 32)) 330 static const char *iotypes[] = {"NO IO", "READ", "WRITE"}; 331 332 static void 333 ctlstat_dump(struct ctlstat_context *ctx) 334 { 335 int iotype, i, n; 336 struct ctl_io_stats *stats = ctx->cur_stats; 337 338 for (i = n = 0; i < ctx->cur_items;i++) { 339 if (F_MASK(ctx) && bit_test(ctx->item_mask, 340 (int)stats[i].item) == 0) 341 continue; 342 printf("%s %d\n", F_PORTS(ctx) ? "port" : "lun", stats[i].item); 343 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { 344 printf(" io type %d (%s)\n", iotype, iotypes[iotype]); 345 printf(" bytes %ju\n", (uintmax_t) 346 stats[i].bytes[iotype]); 347 printf(" operations %ju\n", (uintmax_t) 348 stats[i].operations[iotype]); 349 printf(" dmas %ju\n", (uintmax_t) 350 stats[i].dmas[iotype]); 351 printf(" io time "); 352 PRINT_BINTIME(stats[i].time[iotype]); 353 printf("\n dma time "); 354 PRINT_BINTIME(stats[i].dma_time[iotype]); 355 printf("\n"); 356 } 357 if (++n >= ctx->numdevs) 358 break; 359 } 360 } 361 362 static void 363 ctlstat_json(struct ctlstat_context *ctx) { 364 int iotype, i, n; 365 struct ctl_io_stats *stats = ctx->cur_stats; 366 367 printf("{\"%s\":[", F_PORTS(ctx) ? "ports" : "luns"); 368 for (i = n = 0; i < ctx->cur_items; i++) { 369 if (F_MASK(ctx) && bit_test(ctx->item_mask, 370 (int)stats[i].item) == 0) 371 continue; 372 printf("{\"num\":%d,\"io\":[", 373 stats[i].item); 374 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { 375 printf("{\"type\":\"%s\",", iotypes[iotype]); 376 printf("\"bytes\":%ju,", (uintmax_t) 377 stats[i].bytes[iotype]); 378 printf("\"operations\":%ju,", (uintmax_t) 379 stats[i].operations[iotype]); 380 printf("\"dmas\":%ju,", (uintmax_t) 381 stats[i].dmas[iotype]); 382 printf("\"io time\":"); 383 PRINT_BINTIME(stats[i].time[iotype]); 384 printf(",\"dma time\":"); 385 PRINT_BINTIME(stats[i].dma_time[iotype]); 386 printf("}"); 387 if (iotype < (CTL_STATS_NUM_TYPES - 1)) 388 printf(","); /* continue io array */ 389 } 390 printf("]}"); 391 if (++n >= ctx->numdevs) 392 break; 393 if (i < (ctx->cur_items - 1)) 394 printf(","); /* continue lun array */ 395 } 396 printf("]}"); 397 } 398 399 #define CTLSTAT_PROMETHEUS_LOOP(field, collector) \ 400 for (i = n = 0; i < ctx->cur_items; i++) { \ 401 if (F_MASK(ctx) && bit_test(ctx->item_mask, \ 402 (int)stats[i].item) == 0) \ 403 continue; \ 404 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \ 405 int idx = stats[i].item; \ 406 /* \ 407 * Note that Prometheus considers a label value of "" \ 408 * to be the same as no label at all \ 409 */ \ 410 const char *target = ""; \ 411 if (strcmp(collector, "port") == 0 && \ 412 targdata.targets[idx] != NULL) \ 413 { \ 414 target = targdata.targets[idx]; \ 415 } \ 416 printf("iscsi_%s_" #field "{" \ 417 "%s=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \ 418 "\n", \ 419 collector, collector, \ 420 idx, target, iotypes[iotype], \ 421 stats[i].field[iotype]); \ 422 } \ 423 } \ 424 425 #define CTLSTAT_PROMETHEUS_TIMELOOP(field, collector) \ 426 for (i = n = 0; i < ctx->cur_items; i++) { \ 427 if (F_MASK(ctx) && bit_test(ctx->item_mask, \ 428 (int)stats[i].item) == 0) \ 429 continue; \ 430 for (iotype = 0; iotype < CTL_STATS_NUM_TYPES; iotype++) { \ 431 uint64_t us; \ 432 struct timespec ts; \ 433 int idx = stats[i].item; \ 434 /* \ 435 * Note that Prometheus considers a label value of "" \ 436 * to be the same as no label at all \ 437 */ \ 438 const char *target = ""; \ 439 if (strcmp(collector, "port") == 0 && \ 440 targdata.targets[idx] != NULL) \ 441 { \ 442 target = targdata.targets[idx]; \ 443 } \ 444 bintime2timespec(&stats[i].field[iotype], &ts); \ 445 us = ts.tv_sec * 1000000 + ts.tv_nsec / 1000; \ 446 printf("iscsi_%s_" #field "{" \ 447 "%s=\"%u\",target=\"%s\",type=\"%s\"} %" PRIu64 \ 448 "\n", \ 449 collector, collector, \ 450 idx, target, iotypes[iotype], us); \ 451 } \ 452 } \ 453 454 static void 455 cctl_start_pelement(void *user_data, const char *name, const char **attr) 456 { 457 struct cctl_portlist_data* targdata = user_data; 458 459 targdata->level++; 460 if ((u_int)targdata->level >= (sizeof(targdata->cur_sb) / 461 sizeof(targdata->cur_sb[0]))) 462 errx(1, "%s: too many nesting levels, %zd max", __func__, 463 sizeof(targdata->cur_sb) / sizeof(targdata->cur_sb[0])); 464 465 targdata->cur_sb[targdata->level] = sbuf_new_auto(); 466 if (targdata->cur_sb[targdata->level] == NULL) 467 err(1, "%s: Unable to allocate sbuf", __func__); 468 469 if (strcmp(name, "targ_port") == 0) { 470 int i = 0; 471 472 targdata->lun = -1; 473 targdata->id = -1; 474 free(targdata->target); 475 targdata->target = NULL; 476 while (attr[i]) { 477 if (strcmp(attr[i], "id") == 0) { 478 /* 479 * Well-formed XML always pairs keys with 480 * values in attr 481 */ 482 assert(attr[i + 1]); 483 targdata->id = atoi(attr[i + 1]); 484 } 485 i += 2; 486 } 487 488 } 489 } 490 491 static void 492 cctl_char_phandler(void *user_data, const XML_Char *str, int len) 493 { 494 struct cctl_portlist_data *targdata = user_data; 495 496 sbuf_bcat(targdata->cur_sb[targdata->level], str, len); 497 } 498 499 static void 500 cctl_end_pelement(void *user_data, const char *name) 501 { 502 struct cctl_portlist_data* targdata = user_data; 503 char *str; 504 505 if (targdata->cur_sb[targdata->level] == NULL) 506 errx(1, "%s: no valid sbuf at level %d (name %s)", __func__, 507 targdata->level, name); 508 509 if (sbuf_finish(targdata->cur_sb[targdata->level]) != 0) 510 err(1, "%s: sbuf_finish", __func__); 511 str = strdup(sbuf_data(targdata->cur_sb[targdata->level])); 512 if (str == NULL) 513 err(1, "%s can't allocate %zd bytes for string", __func__, 514 sbuf_len(targdata->cur_sb[targdata->level])); 515 516 sbuf_delete(targdata->cur_sb[targdata->level]); 517 targdata->cur_sb[targdata->level] = NULL; 518 targdata->level--; 519 520 if (strcmp(name, "target") == 0) { 521 free(targdata->target); 522 targdata->target = str; 523 } else if (strcmp(name, "targ_port") == 0) { 524 if (targdata->id >= 0 && targdata->target != NULL) { 525 if (targdata->id >= targdata->ntargets) { 526 /* 527 * This can happen for example if there are 528 * targets with no LUNs. 529 */ 530 targdata->ntargets = MAX(targdata->ntargets * 2, 531 targdata->id + 1); 532 size_t newsize = targdata->ntargets * 533 sizeof(char*); 534 targdata->targets = rallocx(targdata->targets, 535 newsize, MALLOCX_ZERO); 536 } 537 free(targdata->targets[targdata->id]); 538 targdata->targets[targdata->id] = targdata->target; 539 targdata->target = NULL; 540 } 541 free(str); 542 } else { 543 free(str); 544 } 545 } 546 547 static void 548 ctlstat_prometheus(int fd, struct ctlstat_context *ctx, bool ports) { 549 struct ctl_io_stats *stats = ctx->cur_stats; 550 struct ctl_lun_list list; 551 struct cctl_portlist_data targdata; 552 XML_Parser parser; 553 char *port_str = NULL; 554 int iotype, i, n, retval; 555 int port_len = 4096; 556 const char *collector; 557 558 bzero(&targdata, sizeof(targdata)); 559 targdata.ntargets = ctx->cur_items; 560 targdata.targets = calloc(targdata.ntargets, sizeof(char*)); 561 retry: 562 port_str = (char *)realloc(port_str, port_len); 563 bzero(&list, sizeof(list)); 564 list.alloc_len = port_len; 565 list.status = CTL_LUN_LIST_NONE; 566 list.lun_xml = port_str; 567 if (ioctl(fd, CTL_PORT_LIST, &list) == -1) 568 err(1, "%s: error issuing CTL_PORT_LIST ioctl", __func__); 569 if (list.status == CTL_LUN_LIST_ERROR) { 570 warnx("%s: error returned from CTL_PORT_LIST ioctl:\n%s", 571 __func__, list.error_str); 572 } else if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) { 573 port_len <<= 1; 574 goto retry; 575 } 576 577 parser = XML_ParserCreate(NULL); 578 if (parser == NULL) 579 err(1, "%s: Unable to create XML parser", __func__); 580 XML_SetUserData(parser, &targdata); 581 XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement); 582 XML_SetCharacterDataHandler(parser, cctl_char_phandler); 583 584 retval = XML_Parse(parser, port_str, strlen(port_str), 1); 585 if (retval != 1) { 586 errx(1, "%s: Unable to parse XML: Error %d", __func__, 587 XML_GetErrorCode(parser)); 588 } 589 XML_ParserFree(parser); 590 591 collector = ports ? "port" : "lun"; 592 593 printf("# HELP iscsi_%s_bytes Number of bytes\n" 594 "# TYPE iscsi_%s_bytes counter\n", collector, collector); 595 CTLSTAT_PROMETHEUS_LOOP(bytes, collector); 596 printf("# HELP iscsi_%s_dmas Number of DMA\n" 597 "# TYPE iscsi_%s_dmas counter\n", collector, collector); 598 CTLSTAT_PROMETHEUS_LOOP(dmas, collector); 599 printf("# HELP iscsi_%s_operations Number of operations\n" 600 "# TYPE iscsi_%s_operations counter\n", collector, collector); 601 CTLSTAT_PROMETHEUS_LOOP(operations, collector); 602 printf("# HELP iscsi_%s_time Cumulative operation time in us\n" 603 "# TYPE iscsi_%s_time counter\n", collector, collector); 604 CTLSTAT_PROMETHEUS_TIMELOOP(time, collector); 605 printf("# HELP iscsi_%s_dma_time Cumulative DMA time in us\n" 606 "# TYPE iscsi_%s_dma_time counter\n", collector, collector); 607 CTLSTAT_PROMETHEUS_TIMELOOP(dma_time, collector); 608 609 for (i = 0; i < targdata.ntargets; i++) 610 free(targdata.targets[i]); 611 free(targdata.target); 612 free(targdata.targets); 613 614 fflush(stdout); 615 } 616 617 static void 618 ctlstat_standard(struct ctlstat_context *ctx) { 619 long double etime; 620 uint64_t delta_jiffies, delta_idle; 621 long double cpu_percentage; 622 int i, j, n; 623 624 cpu_percentage = 0; 625 626 if (F_CPU(ctx) && (getcpu(&ctx->cur_cpu) != 0)) 627 errx(1, "error returned from getcpu()"); 628 629 etime = ctx->cur_time.tv_sec - ctx->prev_time.tv_sec + 630 (ctx->prev_time.tv_nsec - ctx->cur_time.tv_nsec) * 1e-9; 631 632 if (F_CPU(ctx)) { 633 ctx->prev_total_jiffies = ctx->cur_total_jiffies; 634 ctx->cur_total_jiffies = ctx->cur_cpu.user + 635 ctx->cur_cpu.nice + ctx->cur_cpu.system + 636 ctx->cur_cpu.intr + ctx->cur_cpu.idle; 637 delta_jiffies = ctx->cur_total_jiffies; 638 if (F_FIRST(ctx) == 0) 639 delta_jiffies -= ctx->prev_total_jiffies; 640 ctx->prev_idle = ctx->cur_idle; 641 ctx->cur_idle = ctx->cur_cpu.idle; 642 delta_idle = ctx->cur_idle - ctx->prev_idle; 643 644 cpu_percentage = delta_jiffies - delta_idle; 645 cpu_percentage /= delta_jiffies; 646 cpu_percentage *= 100; 647 } 648 649 if (F_HDR(ctx)) { 650 ctx->header_interval--; 651 if (ctx->header_interval <= 0) { 652 if (F_CPU(ctx)) 653 fprintf(stdout, " CPU"); 654 if (F_TOTALS(ctx)) { 655 fprintf(stdout, "%s Read %s" 656 " Write %s Total\n", 657 (F_TIMEVAL(ctx) != 0) ? " " : "", 658 (F_TIMEVAL(ctx) != 0) ? " " : "", 659 (F_TIMEVAL(ctx) != 0) ? " " : ""); 660 n = 3; 661 } else { 662 for (i = n = 0; i < min(ctl_stat_bits, 663 ctx->cur_items); i++) { 664 int item; 665 666 /* 667 * Obviously this won't work with 668 * LUN numbers greater than a signed 669 * integer. 670 */ 671 item = (int)ctx->cur_stats[i].item; 672 673 if (F_MASK(ctx) && 674 bit_test(ctx->item_mask, item) == 0) 675 continue; 676 fprintf(stdout, "%15.6s%d %s", 677 F_PORTS(ctx) ? "port" : "lun", item, 678 (F_TIMEVAL(ctx) != 0) ? " " : ""); 679 if (++n >= ctx->numdevs) 680 break; 681 } 682 fprintf(stdout, "\n"); 683 } 684 if (F_CPU(ctx)) 685 fprintf(stdout, " "); 686 for (i = 0; i < n; i++) 687 fprintf(stdout, "%s KB/t %s MB/s", 688 (F_TIMEVAL(ctx) != 0) ? " ms" : "", 689 (F_DMA(ctx) == 0) ? "tps" : "dps"); 690 fprintf(stdout, "\n"); 691 ctx->header_interval = 20; 692 } 693 } 694 695 if (F_CPU(ctx)) 696 fprintf(stdout, "%3.0Lf%%", cpu_percentage); 697 if (F_TOTALS(ctx) != 0) { 698 long double mbsec[3]; 699 long double kb_per_transfer[3]; 700 long double transfers_per_sec[3]; 701 long double ms_per_transfer[3]; 702 long double ms_per_dma[3]; 703 long double dmas_per_sec[3]; 704 705 for (i = 0; i < 3; i++) 706 ctx->prev_total_stats[i] = ctx->cur_total_stats[i]; 707 708 memset(&ctx->cur_total_stats, 0, sizeof(ctx->cur_total_stats)); 709 710 /* Use macros to make the next loop more readable. */ 711 #define ADD_STATS_BYTES(st, i, j) \ 712 ctx->cur_total_stats[st].bytes[j] += \ 713 ctx->cur_stats[i].bytes[j] 714 #define ADD_STATS_OPERATIONS(st, i, j) \ 715 ctx->cur_total_stats[st].operations[j] += \ 716 ctx->cur_stats[i].operations[j] 717 #define ADD_STATS_DMAS(st, i, j) \ 718 ctx->cur_total_stats[st].dmas[j] += \ 719 ctx->cur_stats[i].dmas[j] 720 #define ADD_STATS_TIME(st, i, j) \ 721 bintime_add(&ctx->cur_total_stats[st].time[j], \ 722 &ctx->cur_stats[i].time[j]) 723 #define ADD_STATS_DMA_TIME(st, i, j) \ 724 bintime_add(&ctx->cur_total_stats[st].dma_time[j], \ 725 &ctx->cur_stats[i].dma_time[j]) 726 727 for (i = 0; i < ctx->cur_items; i++) { 728 if (F_MASK(ctx) && bit_test(ctx->item_mask, 729 (int)ctx->cur_stats[i].item) == 0) 730 continue; 731 for (j = 0; j < CTL_STATS_NUM_TYPES; j++) { 732 ADD_STATS_BYTES(2, i, j); 733 ADD_STATS_OPERATIONS(2, i, j); 734 ADD_STATS_DMAS(2, i, j); 735 ADD_STATS_TIME(2, i, j); 736 ADD_STATS_DMA_TIME(2, i, j); 737 } 738 ADD_STATS_BYTES(0, i, CTL_STATS_READ); 739 ADD_STATS_OPERATIONS(0, i, CTL_STATS_READ); 740 ADD_STATS_DMAS(0, i, CTL_STATS_READ); 741 ADD_STATS_TIME(0, i, CTL_STATS_READ); 742 ADD_STATS_DMA_TIME(0, i, CTL_STATS_READ); 743 744 ADD_STATS_BYTES(1, i, CTL_STATS_WRITE); 745 ADD_STATS_OPERATIONS(1, i, CTL_STATS_WRITE); 746 ADD_STATS_DMAS(1, i, CTL_STATS_WRITE); 747 ADD_STATS_TIME(1, i, CTL_STATS_WRITE); 748 ADD_STATS_DMA_TIME(1, i, CTL_STATS_WRITE); 749 } 750 751 for (i = 0; i < 3; i++) { 752 compute_stats(&ctx->cur_total_stats[i], 753 F_FIRST(ctx) ? NULL : &ctx->prev_total_stats[i], 754 etime, &mbsec[i], &kb_per_transfer[i], 755 &transfers_per_sec[i], 756 &ms_per_transfer[i], &ms_per_dma[i], 757 &dmas_per_sec[i]); 758 if (F_DMA(ctx) != 0) 759 fprintf(stdout, " %5.1Lf", 760 ms_per_dma[i]); 761 else if (F_TIMEVAL(ctx) != 0) 762 fprintf(stdout, " %5.1Lf", 763 ms_per_transfer[i]); 764 fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf", 765 kb_per_transfer[i], 766 (F_DMA(ctx) == 0) ? transfers_per_sec[i] : 767 dmas_per_sec[i], mbsec[i]); 768 } 769 } else { 770 for (i = n = 0; i < min(ctl_stat_bits, ctx->cur_items); i++) { 771 long double mbsec, kb_per_transfer; 772 long double transfers_per_sec; 773 long double ms_per_transfer; 774 long double ms_per_dma; 775 long double dmas_per_sec; 776 777 if (F_MASK(ctx) && bit_test(ctx->item_mask, 778 (int)ctx->cur_stats[i].item) == 0) 779 continue; 780 for (j = 0; j < ctx->prev_items; j++) { 781 if (ctx->prev_stats[j].item == 782 ctx->cur_stats[i].item) 783 break; 784 } 785 if (j >= ctx->prev_items) 786 j = -1; 787 compute_stats(&ctx->cur_stats[i], 788 j >= 0 ? &ctx->prev_stats[j] : NULL, 789 etime, &mbsec, &kb_per_transfer, 790 &transfers_per_sec, &ms_per_transfer, 791 &ms_per_dma, &dmas_per_sec); 792 if (F_DMA(ctx)) 793 fprintf(stdout, " %5.1Lf", 794 ms_per_dma); 795 else if (F_TIMEVAL(ctx) != 0) 796 fprintf(stdout, " %5.1Lf", 797 ms_per_transfer); 798 fprintf(stdout, " %4.0Lf %5.0Lf %4.0Lf", 799 kb_per_transfer, (F_DMA(ctx) == 0) ? 800 transfers_per_sec : dmas_per_sec, mbsec); 801 if (++n >= ctx->numdevs) 802 break; 803 } 804 } 805 } 806 807 static void 808 get_and_print_stats(int fd, struct ctlstat_context *ctx, bool ports) 809 { 810 struct ctl_io_stats *tmp_stats; 811 int c; 812 813 tmp_stats = ctx->prev_stats; 814 ctx->prev_stats = ctx->cur_stats; 815 ctx->cur_stats = tmp_stats; 816 c = ctx->prev_alloc; 817 ctx->prev_alloc = ctx->cur_alloc; 818 ctx->cur_alloc = c; 819 c = ctx->prev_items; 820 ctx->prev_items = ctx->cur_items; 821 ctx->cur_items = c; 822 ctx->prev_time = ctx->cur_time; 823 ctx->prev_cpu = ctx->cur_cpu; 824 if (getstats(fd, &ctx->cur_alloc, &ctx->cur_items, 825 &ctx->cur_stats, &ctx->cur_time, &ctx->flags, ports) != 0) 826 errx(1, "error returned from getstats()"); 827 828 switch(ctx->mode) { 829 case CTLSTAT_MODE_STANDARD: 830 ctlstat_standard(ctx); 831 break; 832 case CTLSTAT_MODE_DUMP: 833 ctlstat_dump(ctx); 834 break; 835 case CTLSTAT_MODE_JSON: 836 ctlstat_json(ctx); 837 break; 838 case CTLSTAT_MODE_PROMETHEUS: 839 ctlstat_prometheus(fd, ctx, ports); 840 break; 841 default: 842 break; 843 } 844 } 845 846 int 847 main(int argc, char **argv) 848 { 849 int c; 850 int count, waittime; 851 int fd, retval; 852 size_t size; 853 struct ctlstat_context ctx; 854 855 /* default values */ 856 retval = 0; 857 waittime = 1; 858 count = -1; 859 memset(&ctx, 0, sizeof(ctx)); 860 ctx.numdevs = 3; 861 ctx.mode = CTLSTAT_MODE_STANDARD; 862 ctx.flags |= CTLSTAT_FLAG_CPU; 863 ctx.flags |= CTLSTAT_FLAG_FIRST_RUN; 864 ctx.flags |= CTLSTAT_FLAG_HEADER; 865 866 size = sizeof(ctl_stat_bits); 867 if (sysctlbyname("kern.cam.ctl.max_luns", &ctl_stat_bits, &size, NULL, 868 0) == -1) { 869 /* Backward compatibility for where the sysctl wasn't exposed */ 870 ctl_stat_bits = 1024; 871 } 872 ctx.item_mask = bit_alloc(ctl_stat_bits); 873 if (ctx.item_mask == NULL) 874 err(1, "bit_alloc() failed"); 875 876 while ((c = getopt(argc, argv, ctlstat_opts)) != -1) { 877 switch (c) { 878 case 'C': 879 ctx.flags &= ~CTLSTAT_FLAG_CPU; 880 break; 881 case 'c': 882 count = atoi(optarg); 883 break; 884 case 'd': 885 ctx.flags |= CTLSTAT_FLAG_DMA_TIME; 886 break; 887 case 'D': 888 ctx.mode = CTLSTAT_MODE_DUMP; 889 waittime = 30; 890 break; 891 case 'h': 892 ctx.flags &= ~CTLSTAT_FLAG_HEADER; 893 break; 894 case 'j': 895 ctx.mode = CTLSTAT_MODE_JSON; 896 waittime = 30; 897 break; 898 case 'l': { 899 int cur_lun; 900 901 cur_lun = atoi(optarg); 902 if (cur_lun > ctl_stat_bits) 903 errx(1, "Invalid LUN number %d", cur_lun); 904 905 if (!F_MASK(&ctx)) 906 ctx.numdevs = 1; 907 else 908 ctx.numdevs++; 909 bit_set(ctx.item_mask, cur_lun); 910 ctx.flags |= CTLSTAT_FLAG_MASK; 911 ctx.flags |= CTLSTAT_FLAG_LUNS; 912 break; 913 } 914 case 'n': 915 ctx.numdevs = atoi(optarg); 916 break; 917 case 'p': { 918 int cur_port; 919 920 cur_port = atoi(optarg); 921 if (cur_port > ctl_stat_bits) 922 errx(1, "Invalid port number %d", cur_port); 923 924 if (!F_MASK(&ctx)) 925 ctx.numdevs = 1; 926 else 927 ctx.numdevs++; 928 bit_set(ctx.item_mask, cur_port); 929 ctx.flags |= CTLSTAT_FLAG_MASK; 930 ctx.flags |= CTLSTAT_FLAG_PORTS; 931 break; 932 } 933 case 'P': 934 ctx.mode = CTLSTAT_MODE_PROMETHEUS; 935 break; 936 case 't': 937 ctx.flags |= CTLSTAT_FLAG_TOTALS; 938 break; 939 case 'w': 940 waittime = atoi(optarg); 941 break; 942 default: 943 retval = 1; 944 usage(retval); 945 exit(retval); 946 break; 947 } 948 } 949 950 if (F_LUNS(&ctx) && F_PORTS(&ctx)) 951 errx(1, "Options -p and -l are exclusive."); 952 953 if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { 954 if ((count != -1) || 955 (waittime != 1) || 956 (F_PORTS(&ctx)) || 957 /* NB: -P could be compatible with -t in the future */ 958 (ctx.flags & CTLSTAT_FLAG_TOTALS)) 959 { 960 errx(1, "Option -P is exclusive with -p, -c, -w, and -t"); 961 } 962 count = 1; 963 } 964 965 if (!F_LUNS(&ctx) && !F_PORTS(&ctx)) { 966 if (F_TOTALS(&ctx)) 967 ctx.flags |= CTLSTAT_FLAG_PORTS; 968 else 969 ctx.flags |= CTLSTAT_FLAG_LUNS; 970 } 971 972 if ((fd = open(CTL_DEFAULT_DEV, O_RDWR)) == -1) 973 err(1, "cannot open %s", CTL_DEFAULT_DEV); 974 975 if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { 976 /* 977 * NB: Some clients will print a warning if we don't set 978 * Content-Length, but they still work. And the data still 979 * gets into Prometheus. 980 */ 981 printf("HTTP/1.1 200 OK\r\n" 982 "Connection: close\r\n" 983 "Content-Type: text/plain; version=0.0.4\r\n" 984 "\r\n"); 985 } 986 987 for (;count != 0;) { 988 bool ports; 989 990 if (ctx.mode == CTLSTAT_MODE_PROMETHEUS) { 991 get_and_print_stats(fd, &ctx, false); 992 get_and_print_stats(fd, &ctx, true); 993 } else { 994 ports = ctx.flags & CTLSTAT_FLAG_PORTS; 995 get_and_print_stats(fd, &ctx, ports); 996 } 997 998 fprintf(stdout, "\n"); 999 fflush(stdout); 1000 ctx.flags &= ~CTLSTAT_FLAG_FIRST_RUN; 1001 if (count != 1) 1002 sleep(waittime); 1003 if (count > 0) 1004 count--; 1005 } 1006 1007 exit (retval); 1008 } 1009 1010 /* 1011 * vim: ts=8 1012 */ 1013