1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #define _GNU_SOURCE 4 #include <argp.h> 5 #include <libgen.h> 6 #include <ctype.h> 7 #include <string.h> 8 #include <stdlib.h> 9 #include <sched.h> 10 #include <pthread.h> 11 #include <dirent.h> 12 #include <signal.h> 13 #include <fcntl.h> 14 #include <unistd.h> 15 #include <sys/time.h> 16 #include <sys/sysinfo.h> 17 #include <sys/stat.h> 18 #include <bpf/libbpf.h> 19 #include <bpf/btf.h> 20 #include <bpf/bpf.h> 21 #include <libelf.h> 22 #include <gelf.h> 23 #include <float.h> 24 #include <math.h> 25 #include <limits.h> 26 #include <assert.h> 27 28 #ifndef ARRAY_SIZE 29 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) 30 #endif 31 32 #ifndef max 33 #define max(a, b) ((a) > (b) ? (a) : (b)) 34 #endif 35 36 #ifndef min 37 #define min(a, b) ((a) < (b) ? (a) : (b)) 38 #endif 39 40 enum stat_id { 41 VERDICT, 42 DURATION, 43 TOTAL_INSNS, 44 TOTAL_STATES, 45 PEAK_STATES, 46 MAX_STATES_PER_INSN, 47 MARK_READ_MAX_LEN, 48 SIZE, 49 JITED_SIZE, 50 STACK, 51 PROG_TYPE, 52 ATTACH_TYPE, 53 MEMORY_PEAK, 54 55 FILE_NAME, 56 PROG_NAME, 57 58 ALL_STATS_CNT, 59 NUM_STATS_CNT = FILE_NAME - VERDICT, 60 }; 61 62 /* In comparison mode each stat can specify up to four different values: 63 * - A side value; 64 * - B side value; 65 * - absolute diff value; 66 * - relative (percentage) diff value. 67 * 68 * When specifying stat specs in comparison mode, user can use one of the 69 * following variant suffixes to specify which exact variant should be used for 70 * ordering or filtering: 71 * - `_a` for A side value; 72 * - `_b` for B side value; 73 * - `_diff` for absolute diff value; 74 * - `_pct` for relative (percentage) diff value. 75 * 76 * If no variant suffix is provided, then `_b` (control data) is assumed. 77 * 78 * As an example, let's say instructions stat has the following output: 79 * 80 * Insns (A) Insns (B) Insns (DIFF) 81 * --------- --------- -------------- 82 * 21547 20920 -627 (-2.91%) 83 * 84 * Then: 85 * - 21547 is A side value (insns_a); 86 * - 20920 is B side value (insns_b); 87 * - -627 is absolute diff value (insns_diff); 88 * - -2.91% is relative diff value (insns_pct). 89 * 90 * For verdict there is no verdict_pct variant. 91 * For file and program name, _a and _b variants are equivalent and there are 92 * no _diff or _pct variants. 93 */ 94 enum stat_variant { 95 VARIANT_A, 96 VARIANT_B, 97 VARIANT_DIFF, 98 VARIANT_PCT, 99 }; 100 101 struct verif_stats { 102 char *file_name; 103 char *prog_name; 104 105 long stats[NUM_STATS_CNT]; 106 }; 107 108 /* joined comparison mode stats */ 109 struct verif_stats_join { 110 char *file_name; 111 char *prog_name; 112 113 const struct verif_stats *stats_a; 114 const struct verif_stats *stats_b; 115 }; 116 117 struct stat_specs { 118 int spec_cnt; 119 enum stat_id ids[ALL_STATS_CNT]; 120 enum stat_variant variants[ALL_STATS_CNT]; 121 bool asc[ALL_STATS_CNT]; 122 bool abs[ALL_STATS_CNT]; 123 int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */ 124 }; 125 126 enum resfmt { 127 RESFMT_TABLE, 128 RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */ 129 RESFMT_CSV, 130 }; 131 132 enum filter_kind { 133 FILTER_NAME, 134 FILTER_STAT, 135 }; 136 137 enum operator_kind { 138 OP_EQ, /* == or = */ 139 OP_NEQ, /* != or <> */ 140 OP_LT, /* < */ 141 OP_LE, /* <= */ 142 OP_GT, /* > */ 143 OP_GE, /* >= */ 144 }; 145 146 struct filter { 147 enum filter_kind kind; 148 /* FILTER_NAME */ 149 char *any_glob; 150 char *file_glob; 151 char *prog_glob; 152 /* FILTER_STAT */ 153 enum operator_kind op; 154 int stat_id; 155 enum stat_variant stat_var; 156 long value; 157 bool abs; 158 }; 159 160 struct rvalue { 161 enum { INTEGRAL, ENUMERATOR } type; 162 union { 163 long long ivalue; 164 char *svalue; 165 }; 166 }; 167 168 struct field_access { 169 enum { FIELD_NAME, ARRAY_INDEX } type; 170 union { 171 char *name; 172 struct rvalue index; 173 }; 174 }; 175 176 struct var_preset { 177 struct field_access *atoms; 178 int atom_count; 179 char *full_name; 180 struct rvalue value; 181 bool applied; 182 }; 183 184 enum dump_mode { 185 DUMP_NONE = 0, 186 DUMP_XLATED = 1, 187 DUMP_JITED = 2, 188 }; 189 190 static struct env { 191 char **filenames; 192 int filename_cnt; 193 bool verbose; 194 bool debug; 195 bool quiet; 196 bool force_checkpoints; 197 bool force_reg_invariants; 198 enum resfmt out_fmt; 199 bool show_version; 200 bool comparison_mode; 201 bool replay_mode; 202 int top_n; 203 204 int log_level; 205 int log_size; 206 bool log_fixed; 207 208 struct verif_stats *prog_stats; 209 int prog_stat_cnt; 210 211 /* baseline_stats is allocated and used only in comparison mode */ 212 struct verif_stats *baseline_stats; 213 int baseline_stat_cnt; 214 215 struct verif_stats_join *join_stats; 216 int join_stat_cnt; 217 218 struct stat_specs output_spec; 219 struct stat_specs sort_spec; 220 221 struct filter *allow_filters; 222 struct filter *deny_filters; 223 int allow_filter_cnt; 224 int deny_filter_cnt; 225 226 int files_processed; 227 int files_skipped; 228 int progs_processed; 229 int progs_skipped; 230 int top_src_lines; 231 struct var_preset *presets; 232 int npresets; 233 char orig_cgroup[PATH_MAX]; 234 char stat_cgroup[PATH_MAX]; 235 int memory_peak_fd; 236 __u32 dump_mode; 237 } env; 238 239 static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) 240 { 241 if (!env.verbose) 242 return 0; 243 if (level == LIBBPF_DEBUG && !env.debug) 244 return 0; 245 return vfprintf(stderr, format, args); 246 } 247 248 #define log_errno(fmt, ...) log_errno_aux(__FILE__, __LINE__, fmt, ##__VA_ARGS__) 249 250 __attribute__((format(printf, 3, 4))) 251 static int log_errno_aux(const char *file, int line, const char *fmt, ...) 252 { 253 int err = -errno; 254 va_list ap; 255 256 va_start(ap, fmt); 257 fprintf(stderr, "%s:%d: ", file, line); 258 vfprintf(stderr, fmt, ap); 259 fprintf(stderr, " failed with error '%s'.\n", strerror(errno)); 260 va_end(ap); 261 return err; 262 } 263 264 #ifndef VERISTAT_VERSION 265 #define VERISTAT_VERSION "<kernel>" 266 #endif 267 268 const char *argp_program_version = "veristat v" VERISTAT_VERSION; 269 const char *argp_program_bug_address = "<bpf@vger.kernel.org>"; 270 const char argp_program_doc[] = 271 "veristat BPF verifier stats collection and comparison tool.\n" 272 "\n" 273 "USAGE: veristat <obj-file> [<obj-file>...]\n" 274 " OR: veristat -C <baseline.csv> <comparison.csv>\n" 275 " OR: veristat -R <results.csv>\n" 276 " OR: veristat -vl2 <to_analyze.bpf.o>\n"; 277 278 enum { 279 OPT_LOG_FIXED = 1000, 280 OPT_LOG_SIZE = 1001, 281 OPT_DUMP = 1002, 282 }; 283 284 static const struct argp_option opts[] = { 285 { NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" }, 286 { "version", 'V', NULL, 0, "Print version" }, 287 { "verbose", 'v', NULL, 0, "Verbose mode" }, 288 { "debug", 'd', NULL, 0, "Debug mode (turns on libbpf debug logging)" }, 289 { "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode, 2 for full verification log)" }, 290 { "log-fixed", OPT_LOG_FIXED, NULL, 0, "Disable verifier log rotation" }, 291 { "log-size", OPT_LOG_SIZE, "BYTES", 0, "Customize verifier log size (default to 16MB)" }, 292 { "top-n", 'n', "N", 0, "Emit only up to first N results." }, 293 { "quiet", 'q', NULL, 0, "Quiet mode" }, 294 { "emit", 'e', "SPEC", 0, "Specify stats to be emitted" }, 295 { "sort", 's', "SPEC", 0, "Specify sort order" }, 296 { "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." }, 297 { "compare", 'C', NULL, 0, "Comparison mode" }, 298 { "replay", 'R', NULL, 0, "Replay mode" }, 299 { "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." }, 300 { "test-states", 't', NULL, 0, 301 "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" }, 302 { "test-reg-invariants", 'r', NULL, 0, 303 "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" }, 304 { "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" }, 305 { "set-global-vars", 'G', "GLOBAL", 0, "Set global variables provided in the expression, for example \"var1 = 1\"" }, 306 { "dump", OPT_DUMP, "DUMP_MODE", OPTION_ARG_OPTIONAL, "Print BPF program dump (xlated, jited)" }, 307 {}, 308 }; 309 310 static int parse_stats(const char *stats_str, struct stat_specs *specs); 311 static int append_filter(struct filter **filters, int *cnt, const char *str); 312 static int append_filter_file(const char *path); 313 static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr); 314 static int append_var_preset_file(const char *filename); 315 static int append_file(const char *path); 316 static int append_file_from_file(const char *path); 317 318 static error_t parse_arg(int key, char *arg, struct argp_state *state) 319 { 320 int err; 321 322 switch (key) { 323 case 'h': 324 argp_state_help(state, stderr, ARGP_HELP_STD_HELP); 325 break; 326 case 'V': 327 env.show_version = true; 328 break; 329 case 'v': 330 env.verbose = true; 331 break; 332 case 'd': 333 env.debug = true; 334 env.verbose = true; 335 break; 336 case 'q': 337 env.quiet = true; 338 break; 339 case 'e': 340 err = parse_stats(arg, &env.output_spec); 341 if (err) 342 return err; 343 break; 344 case 's': 345 err = parse_stats(arg, &env.sort_spec); 346 if (err) 347 return err; 348 break; 349 case 'o': 350 if (strcmp(arg, "table") == 0) { 351 env.out_fmt = RESFMT_TABLE; 352 } else if (strcmp(arg, "csv") == 0) { 353 env.out_fmt = RESFMT_CSV; 354 } else { 355 fprintf(stderr, "Unrecognized output format '%s'\n", arg); 356 return -EINVAL; 357 } 358 break; 359 case 'l': 360 errno = 0; 361 env.log_level = strtol(arg, NULL, 10); 362 if (errno) { 363 fprintf(stderr, "invalid log level: %s\n", arg); 364 argp_usage(state); 365 } 366 break; 367 case OPT_LOG_FIXED: 368 env.log_fixed = true; 369 break; 370 case OPT_LOG_SIZE: 371 errno = 0; 372 env.log_size = strtol(arg, NULL, 10); 373 if (errno) { 374 fprintf(stderr, "invalid log size: %s\n", arg); 375 argp_usage(state); 376 } 377 break; 378 case 't': 379 env.force_checkpoints = true; 380 break; 381 case 'r': 382 env.force_reg_invariants = true; 383 break; 384 case 'n': 385 errno = 0; 386 env.top_n = strtol(arg, NULL, 10); 387 if (errno) { 388 fprintf(stderr, "invalid top N specifier: %s\n", arg); 389 argp_usage(state); 390 } 391 break; 392 case 'C': 393 env.comparison_mode = true; 394 break; 395 case 'R': 396 env.replay_mode = true; 397 break; 398 case 'f': 399 if (arg[0] == '@') 400 err = append_filter_file(arg + 1); 401 else if (arg[0] == '!') 402 err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1); 403 else 404 err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg); 405 if (err) { 406 fprintf(stderr, "Failed to collect program filter expressions: %d\n", err); 407 return err; 408 } 409 break; 410 case 'S': 411 errno = 0; 412 env.top_src_lines = strtol(arg, NULL, 10); 413 if (errno) { 414 fprintf(stderr, "invalid top lines N specifier: %s\n", arg); 415 argp_usage(state); 416 } 417 break; 418 case 'G': { 419 if (arg[0] == '@') 420 err = append_var_preset_file(arg + 1); 421 else 422 err = append_var_preset(&env.presets, &env.npresets, arg); 423 if (err) { 424 fprintf(stderr, "Failed to parse global variable presets: %s\n", arg); 425 return err; 426 } 427 break; 428 } 429 case ARGP_KEY_ARG: 430 if (arg[0] == '@') 431 err = append_file_from_file(arg + 1); 432 else 433 err = append_file(arg); 434 if (err) { 435 fprintf(stderr, "Failed to collect BPF object files: %d\n", err); 436 return err; 437 } 438 break; 439 case OPT_DUMP: 440 if (!arg || strcasecmp(arg, "xlated") == 0) { 441 env.dump_mode |= DUMP_XLATED; 442 } else if (strcasecmp(arg, "jited") == 0) { 443 env.dump_mode |= DUMP_JITED; 444 } else { 445 fprintf(stderr, "Unrecognized dump mode '%s'\n", arg); 446 return -EINVAL; 447 } 448 break; 449 default: 450 return ARGP_ERR_UNKNOWN; 451 } 452 return 0; 453 } 454 455 static const struct argp argp = { 456 .options = opts, 457 .parser = parse_arg, 458 .doc = argp_program_doc, 459 }; 460 461 462 /* Adapted from perf/util/string.c */ 463 static bool glob_matches(const char *str, const char *pat) 464 { 465 while (*str && *pat && *pat != '*') { 466 if (*str != *pat) 467 return false; 468 str++; 469 pat++; 470 } 471 /* Check wild card */ 472 if (*pat == '*') { 473 while (*pat == '*') 474 pat++; 475 if (!*pat) /* Tail wild card matches all */ 476 return true; 477 while (*str) 478 if (glob_matches(str++, pat)) 479 return true; 480 } 481 return !*str && !*pat; 482 } 483 484 static bool is_bpf_obj_file(const char *path) { 485 Elf64_Ehdr *ehdr; 486 int fd, err = -EINVAL; 487 Elf *elf = NULL; 488 489 fd = open(path, O_RDONLY | O_CLOEXEC); 490 if (fd < 0) 491 return true; /* we'll fail later and propagate error */ 492 493 /* ensure libelf is initialized */ 494 (void)elf_version(EV_CURRENT); 495 496 elf = elf_begin(fd, ELF_C_READ, NULL); 497 if (!elf) 498 goto cleanup; 499 500 if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64) 501 goto cleanup; 502 503 ehdr = elf64_getehdr(elf); 504 /* Old LLVM set e_machine to EM_NONE */ 505 if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) 506 goto cleanup; 507 508 err = 0; 509 cleanup: 510 if (elf) 511 elf_end(elf); 512 close(fd); 513 return err == 0; 514 } 515 516 static bool should_process_file_prog(const char *filename, const char *prog_name) 517 { 518 struct filter *f; 519 int i, allow_cnt = 0; 520 521 for (i = 0; i < env.deny_filter_cnt; i++) { 522 f = &env.deny_filters[i]; 523 if (f->kind != FILTER_NAME) 524 continue; 525 526 if (f->any_glob && glob_matches(filename, f->any_glob)) 527 return false; 528 if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) 529 return false; 530 if (f->file_glob && glob_matches(filename, f->file_glob)) 531 return false; 532 if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) 533 return false; 534 } 535 536 for (i = 0; i < env.allow_filter_cnt; i++) { 537 f = &env.allow_filters[i]; 538 if (f->kind != FILTER_NAME) 539 continue; 540 541 allow_cnt++; 542 if (f->any_glob) { 543 if (glob_matches(filename, f->any_glob)) 544 return true; 545 /* If we don't know program name yet, any_glob filter 546 * has to assume that current BPF object file might be 547 * relevant; we'll check again later on after opening 548 * BPF object file, at which point program name will 549 * be known finally. 550 */ 551 if (!prog_name || glob_matches(prog_name, f->any_glob)) 552 return true; 553 } else { 554 if (f->file_glob && !glob_matches(filename, f->file_glob)) 555 continue; 556 if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) 557 continue; 558 return true; 559 } 560 } 561 562 /* if there are no file/prog name allow filters, allow all progs, 563 * unless they are denied earlier explicitly 564 */ 565 return allow_cnt == 0; 566 } 567 568 static struct { 569 enum operator_kind op_kind; 570 const char *op_str; 571 } operators[] = { 572 /* Order of these definitions matter to avoid situations like '<' 573 * matching part of what is actually a '<>' operator. That is, 574 * substrings should go last. 575 */ 576 { OP_EQ, "==" }, 577 { OP_NEQ, "!=" }, 578 { OP_NEQ, "<>" }, 579 { OP_LE, "<=" }, 580 { OP_LT, "<" }, 581 { OP_GE, ">=" }, 582 { OP_GT, ">" }, 583 { OP_EQ, "=" }, 584 }; 585 586 static bool parse_stat_id_var(const char *name, size_t len, int *id, 587 enum stat_variant *var, bool *is_abs); 588 589 static int append_filter(struct filter **filters, int *cnt, const char *str) 590 { 591 struct filter *f; 592 void *tmp; 593 const char *p; 594 int i; 595 596 tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters)); 597 if (!tmp) 598 return -ENOMEM; 599 *filters = tmp; 600 601 f = &(*filters)[*cnt]; 602 memset(f, 0, sizeof(*f)); 603 604 /* First, let's check if it's a stats filter of the following form: 605 * <stat><op><value, where: 606 * - <stat> is one of supported numerical stats (verdict is also 607 * considered numerical, failure == 0, success == 1); 608 * - <op> is comparison operator (see `operators` definitions); 609 * - <value> is an integer (or failure/success, or false/true as 610 * special aliases for 0 and 1, respectively). 611 * If the form doesn't match what user provided, we assume file/prog 612 * glob filter. 613 */ 614 for (i = 0; i < ARRAY_SIZE(operators); i++) { 615 enum stat_variant var; 616 int id; 617 long val; 618 const char *end = str; 619 const char *op_str; 620 bool is_abs; 621 622 op_str = operators[i].op_str; 623 p = strstr(str, op_str); 624 if (!p) 625 continue; 626 627 if (!parse_stat_id_var(str, p - str, &id, &var, &is_abs)) { 628 fprintf(stderr, "Unrecognized stat name in '%s'!\n", str); 629 return -EINVAL; 630 } 631 if (id >= FILE_NAME) { 632 fprintf(stderr, "Non-integer stat is specified in '%s'!\n", str); 633 return -EINVAL; 634 } 635 636 p += strlen(op_str); 637 638 if (strcasecmp(p, "true") == 0 || 639 strcasecmp(p, "t") == 0 || 640 strcasecmp(p, "success") == 0 || 641 strcasecmp(p, "succ") == 0 || 642 strcasecmp(p, "s") == 0 || 643 strcasecmp(p, "match") == 0 || 644 strcasecmp(p, "m") == 0) { 645 val = 1; 646 } else if (strcasecmp(p, "false") == 0 || 647 strcasecmp(p, "f") == 0 || 648 strcasecmp(p, "failure") == 0 || 649 strcasecmp(p, "fail") == 0 || 650 strcasecmp(p, "mismatch") == 0 || 651 strcasecmp(p, "mis") == 0) { 652 val = 0; 653 } else { 654 errno = 0; 655 val = strtol(p, (char **)&end, 10); 656 if (errno || end == p || *end != '\0' ) { 657 fprintf(stderr, "Invalid integer value in '%s'!\n", str); 658 return -EINVAL; 659 } 660 } 661 662 f->kind = FILTER_STAT; 663 f->stat_id = id; 664 f->stat_var = var; 665 f->op = operators[i].op_kind; 666 f->abs = true; 667 f->value = val; 668 669 *cnt += 1; 670 return 0; 671 } 672 673 /* File/prog filter can be specified either as '<glob>' or 674 * '<file-glob>/<prog-glob>'. In the former case <glob> is applied to 675 * both file and program names. This seems to be way more useful in 676 * practice. If user needs full control, they can use '/<prog-glob>' 677 * form to glob just program name, or '<file-glob>/' to glob only file 678 * name. But usually common <glob> seems to be the most useful and 679 * ergonomic way. 680 */ 681 f->kind = FILTER_NAME; 682 p = strchr(str, '/'); 683 if (!p) { 684 f->any_glob = strdup(str); 685 if (!f->any_glob) 686 return -ENOMEM; 687 } else { 688 if (str != p) { 689 /* non-empty file glob */ 690 f->file_glob = strndup(str, p - str); 691 if (!f->file_glob) 692 return -ENOMEM; 693 } 694 if (strlen(p + 1) > 0) { 695 /* non-empty prog glob */ 696 f->prog_glob = strdup(p + 1); 697 if (!f->prog_glob) { 698 free(f->file_glob); 699 f->file_glob = NULL; 700 return -ENOMEM; 701 } 702 } 703 } 704 705 *cnt += 1; 706 return 0; 707 } 708 709 static int append_filter_file(const char *path) 710 { 711 char buf[1024]; 712 FILE *f; 713 int err = 0; 714 715 f = fopen(path, "r"); 716 if (!f) { 717 err = -errno; 718 fprintf(stderr, "Failed to open filters in '%s': %s\n", path, strerror(-err)); 719 return err; 720 } 721 722 while (fscanf(f, " %1023[^\n]\n", buf) == 1) { 723 /* lines starting with # are comments, skip them */ 724 if (buf[0] == '\0' || buf[0] == '#') 725 continue; 726 /* lines starting with ! are negative match filters */ 727 if (buf[0] == '!') 728 err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1); 729 else 730 err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf); 731 if (err) 732 goto cleanup; 733 } 734 735 cleanup: 736 fclose(f); 737 return err; 738 } 739 740 static const struct stat_specs default_output_spec = { 741 .spec_cnt = 8, 742 .ids = { 743 FILE_NAME, PROG_NAME, VERDICT, DURATION, 744 TOTAL_INSNS, TOTAL_STATES, SIZE, JITED_SIZE 745 }, 746 }; 747 748 static int append_file(const char *path) 749 { 750 void *tmp; 751 752 tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames)); 753 if (!tmp) 754 return -ENOMEM; 755 env.filenames = tmp; 756 env.filenames[env.filename_cnt] = strdup(path); 757 if (!env.filenames[env.filename_cnt]) 758 return -ENOMEM; 759 env.filename_cnt++; 760 return 0; 761 } 762 763 static int append_file_from_file(const char *path) 764 { 765 char buf[1024]; 766 int err = 0; 767 FILE *f; 768 769 f = fopen(path, "r"); 770 if (!f) { 771 err = -errno; 772 fprintf(stderr, "Failed to open object files list in '%s': %s\n", 773 path, strerror(errno)); 774 return err; 775 } 776 777 while (fscanf(f, " %1023[^\n]\n", buf) == 1) { 778 /* lines starting with # are comments, skip them */ 779 if (buf[0] == '\0' || buf[0] == '#') 780 continue; 781 err = append_file(buf); 782 if (err) 783 goto cleanup; 784 } 785 786 cleanup: 787 fclose(f); 788 return err; 789 } 790 791 static const struct stat_specs default_csv_output_spec = { 792 .spec_cnt = 15, 793 .ids = { 794 FILE_NAME, PROG_NAME, VERDICT, DURATION, 795 TOTAL_INSNS, TOTAL_STATES, PEAK_STATES, 796 MAX_STATES_PER_INSN, MARK_READ_MAX_LEN, 797 SIZE, JITED_SIZE, PROG_TYPE, ATTACH_TYPE, 798 STACK, MEMORY_PEAK, 799 }, 800 }; 801 802 static const struct stat_specs default_sort_spec = { 803 .spec_cnt = 2, 804 .ids = { 805 FILE_NAME, PROG_NAME, 806 }, 807 .asc = { true, true, }, 808 }; 809 810 /* sorting for comparison mode to join two data sets */ 811 static const struct stat_specs join_sort_spec = { 812 .spec_cnt = 2, 813 .ids = { 814 FILE_NAME, PROG_NAME, 815 }, 816 .asc = { true, true, }, 817 }; 818 819 static struct stat_def { 820 const char *header; 821 const char *names[4]; 822 bool asc_by_default; 823 bool left_aligned; 824 } stat_defs[] = { 825 [FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */, true /* left */ }, 826 [PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */, true /* left */ }, 827 [VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */, true /* left */ }, 828 [DURATION] = { "Duration (us)", {"duration", "dur"}, }, 829 [TOTAL_INSNS] = { "Insns", {"total_insns", "insns"}, }, 830 [TOTAL_STATES] = { "States", {"total_states", "states"}, }, 831 [PEAK_STATES] = { "Peak states", {"peak_states"}, }, 832 [MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, }, 833 [MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, }, 834 [SIZE] = { "Program size", {"prog_size"}, }, 835 [JITED_SIZE] = { "Jited size", {"prog_size_jited"}, }, 836 [STACK] = {"Stack depth", {"stack_depth", "stack"}, }, 837 [PROG_TYPE] = { "Program type", {"prog_type"}, }, 838 [ATTACH_TYPE] = { "Attach type", {"attach_type", }, }, 839 [MEMORY_PEAK] = { "Peak memory (MiB)", {"mem_peak", }, }, 840 }; 841 842 static bool parse_stat_id_var(const char *name, size_t len, int *id, 843 enum stat_variant *var, bool *is_abs) 844 { 845 static const char *var_sfxs[] = { 846 [VARIANT_A] = "_a", 847 [VARIANT_B] = "_b", 848 [VARIANT_DIFF] = "_diff", 849 [VARIANT_PCT] = "_pct", 850 }; 851 int i, j, k; 852 853 /* |<stat>| means we take absolute value of given stat */ 854 *is_abs = false; 855 if (len > 2 && name[0] == '|' && name[len - 1] == '|') { 856 *is_abs = true; 857 name += 1; 858 len -= 2; 859 } 860 861 for (i = 0; i < ARRAY_SIZE(stat_defs); i++) { 862 struct stat_def *def = &stat_defs[i]; 863 size_t alias_len, sfx_len; 864 const char *alias; 865 866 for (j = 0; j < ARRAY_SIZE(stat_defs[i].names); j++) { 867 alias = def->names[j]; 868 if (!alias) 869 continue; 870 871 alias_len = strlen(alias); 872 if (strncmp(name, alias, alias_len) != 0) 873 continue; 874 875 if (alias_len == len) { 876 /* If no variant suffix is specified, we 877 * assume control group (just in case we are 878 * in comparison mode. Variant is ignored in 879 * non-comparison mode. 880 */ 881 *var = VARIANT_B; 882 *id = i; 883 return true; 884 } 885 886 for (k = 0; k < ARRAY_SIZE(var_sfxs); k++) { 887 sfx_len = strlen(var_sfxs[k]); 888 if (alias_len + sfx_len != len) 889 continue; 890 891 if (strncmp(name + alias_len, var_sfxs[k], sfx_len) == 0) { 892 *var = (enum stat_variant)k; 893 *id = i; 894 return true; 895 } 896 } 897 } 898 } 899 900 return false; 901 } 902 903 static bool is_asc_sym(char c) 904 { 905 return c == '^'; 906 } 907 908 static bool is_desc_sym(char c) 909 { 910 return c == 'v' || c == 'V' || c == '.' || c == '!' || c == '_'; 911 } 912 913 static char *rtrim(char *str) 914 { 915 int i; 916 917 for (i = strlen(str) - 1; i > 0; --i) { 918 if (!isspace(str[i])) 919 break; 920 str[i] = '\0'; 921 } 922 return str; 923 } 924 925 static int parse_stat(const char *stat_name, struct stat_specs *specs) 926 { 927 int id; 928 bool has_order = false, is_asc = false, is_abs = false; 929 size_t len = strlen(stat_name); 930 enum stat_variant var; 931 932 if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) { 933 fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids)); 934 return -E2BIG; 935 } 936 937 if (len > 1 && (is_asc_sym(stat_name[len - 1]) || is_desc_sym(stat_name[len - 1]))) { 938 has_order = true; 939 is_asc = is_asc_sym(stat_name[len - 1]); 940 len -= 1; 941 } 942 943 if (!parse_stat_id_var(stat_name, len, &id, &var, &is_abs)) { 944 fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name); 945 return -ESRCH; 946 } 947 948 specs->ids[specs->spec_cnt] = id; 949 specs->variants[specs->spec_cnt] = var; 950 specs->asc[specs->spec_cnt] = has_order ? is_asc : stat_defs[id].asc_by_default; 951 specs->abs[specs->spec_cnt] = is_abs; 952 specs->spec_cnt++; 953 954 return 0; 955 } 956 957 static int parse_stats(const char *stats_str, struct stat_specs *specs) 958 { 959 char *input, *state = NULL, *next; 960 int err, cnt = 0; 961 962 input = strdup(stats_str); 963 if (!input) 964 return -ENOMEM; 965 966 while ((next = strtok_r(cnt++ ? NULL : input, ",", &state))) { 967 err = parse_stat(next, specs); 968 if (err) { 969 free(input); 970 return err; 971 } 972 } 973 974 free(input); 975 return 0; 976 } 977 978 static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt) 979 { 980 int i; 981 982 if (!stats) 983 return; 984 985 for (i = 0; i < stat_cnt; i++) { 986 free(stats[i].file_name); 987 free(stats[i].prog_name); 988 } 989 free(stats); 990 } 991 992 static char verif_log_buf[64 * 1024]; 993 994 #define MAX_PARSED_LOG_LINES 100 995 996 static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) 997 { 998 const char *cur; 999 int pos, lines, sub_stack, cnt = 0; 1000 char *state = NULL, *token, stack[512]; 1001 1002 buf[buf_sz - 1] = '\0'; 1003 1004 for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) { 1005 /* find previous endline or otherwise take the start of log buf */ 1006 for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) { 1007 } 1008 /* next time start from end of previous line (or pos goes to <0) */ 1009 pos--; 1010 /* if we found endline, point right after endline symbol; 1011 * otherwise, stay at the beginning of log buf 1012 */ 1013 if (cur[0] == '\n') 1014 cur++; 1015 1016 if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION])) 1017 continue; 1018 if (5 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld", 1019 &s->stats[TOTAL_INSNS], 1020 &s->stats[MAX_STATES_PER_INSN], 1021 &s->stats[TOTAL_STATES], 1022 &s->stats[PEAK_STATES], 1023 &s->stats[MARK_READ_MAX_LEN])) 1024 continue; 1025 1026 if (1 == sscanf(cur, "stack depth %511s", stack)) 1027 continue; 1028 } 1029 while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) { 1030 if (sscanf(token, "%d", &sub_stack) == 0) 1031 break; 1032 s->stats[STACK] += sub_stack; 1033 } 1034 return 0; 1035 } 1036 1037 struct line_cnt { 1038 char *line; 1039 int cnt; 1040 }; 1041 1042 static int str_cmp(const void *a, const void *b) 1043 { 1044 const char **str1 = (const char **)a; 1045 const char **str2 = (const char **)b; 1046 1047 return strcmp(*str1, *str2); 1048 } 1049 1050 static int line_cnt_cmp(const void *a, const void *b) 1051 { 1052 const struct line_cnt *a_cnt = (const struct line_cnt *)a; 1053 const struct line_cnt *b_cnt = (const struct line_cnt *)b; 1054 1055 if (a_cnt->cnt != b_cnt->cnt) 1056 return a_cnt->cnt > b_cnt->cnt ? -1 : 1; 1057 return strcmp(a_cnt->line, b_cnt->line); 1058 } 1059 1060 static int print_top_src_lines(char * const buf, size_t buf_sz, const char *prog_name) 1061 { 1062 int lines_cap = 0; 1063 int lines_size = 0; 1064 char **lines = NULL; 1065 char *line = NULL; 1066 char *state; 1067 struct line_cnt *freq = NULL; 1068 struct line_cnt *cur; 1069 int unique_lines; 1070 int err = 0; 1071 int i; 1072 1073 while ((line = strtok_r(line ? NULL : buf, "\n", &state))) { 1074 if (strncmp(line, "; ", 2) != 0) 1075 continue; 1076 line += 2; 1077 1078 if (lines_size == lines_cap) { 1079 char **tmp; 1080 1081 lines_cap = max(16, lines_cap * 2); 1082 tmp = realloc(lines, lines_cap * sizeof(*tmp)); 1083 if (!tmp) { 1084 err = -ENOMEM; 1085 goto cleanup; 1086 } 1087 lines = tmp; 1088 } 1089 lines[lines_size] = line; 1090 lines_size++; 1091 } 1092 1093 if (lines_size == 0) 1094 goto cleanup; 1095 1096 qsort(lines, lines_size, sizeof(*lines), str_cmp); 1097 1098 freq = calloc(lines_size, sizeof(*freq)); 1099 if (!freq) { 1100 err = -ENOMEM; 1101 goto cleanup; 1102 } 1103 1104 cur = freq; 1105 cur->line = lines[0]; 1106 cur->cnt = 1; 1107 for (i = 1; i < lines_size; ++i) { 1108 if (strcmp(lines[i], cur->line) != 0) { 1109 cur++; 1110 cur->line = lines[i]; 1111 cur->cnt = 0; 1112 } 1113 cur->cnt++; 1114 } 1115 unique_lines = cur - freq + 1; 1116 1117 qsort(freq, unique_lines, sizeof(struct line_cnt), line_cnt_cmp); 1118 1119 printf("Top source lines (%s):\n", prog_name); 1120 for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) { 1121 const char *src_code = freq[i].line; 1122 const char *src_line = NULL; 1123 char *split = strrchr(freq[i].line, '@'); 1124 1125 if (split) { 1126 src_line = split + 1; 1127 1128 while (*src_line && isspace(*src_line)) 1129 src_line++; 1130 1131 while (split > src_code && isspace(*split)) 1132 split--; 1133 *split = '\0'; 1134 } 1135 1136 if (src_line) 1137 printf("%5d: (%s)\t%s\n", freq[i].cnt, src_line, src_code); 1138 else 1139 printf("%5d: %s\n", freq[i].cnt, src_code); 1140 } 1141 printf("\n"); 1142 1143 cleanup: 1144 free(freq); 1145 free(lines); 1146 return err; 1147 } 1148 1149 static int guess_prog_type_by_ctx_name(const char *ctx_name, 1150 enum bpf_prog_type *prog_type, 1151 enum bpf_attach_type *attach_type) 1152 { 1153 /* We need to guess program type based on its declared context type. 1154 * This guess can't be perfect as many different program types might 1155 * share the same context type. So we can only hope to reasonably 1156 * well guess this and get lucky. 1157 * 1158 * Just in case, we support both UAPI-side type names and 1159 * kernel-internal names. 1160 */ 1161 static struct { 1162 const char *uapi_name; 1163 const char *kern_name; 1164 enum bpf_prog_type prog_type; 1165 enum bpf_attach_type attach_type; 1166 } ctx_map[] = { 1167 /* __sk_buff is most ambiguous, we assume TC program */ 1168 { "__sk_buff", "sk_buff", BPF_PROG_TYPE_SCHED_CLS }, 1169 { "bpf_sock", "sock", BPF_PROG_TYPE_CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND }, 1170 { "bpf_sock_addr", "bpf_sock_addr_kern", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND }, 1171 { "bpf_sock_ops", "bpf_sock_ops_kern", BPF_PROG_TYPE_SOCK_OPS, BPF_CGROUP_SOCK_OPS }, 1172 { "sk_msg_md", "sk_msg", BPF_PROG_TYPE_SK_MSG, BPF_SK_MSG_VERDICT }, 1173 { "bpf_cgroup_dev_ctx", "bpf_cgroup_dev_ctx", BPF_PROG_TYPE_CGROUP_DEVICE, BPF_CGROUP_DEVICE }, 1174 { "bpf_sysctl", "bpf_sysctl_kern", BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL }, 1175 { "bpf_sockopt", "bpf_sockopt_kern", BPF_PROG_TYPE_CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT }, 1176 { "sk_reuseport_md", "sk_reuseport_kern", BPF_PROG_TYPE_SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE }, 1177 { "bpf_sk_lookup", "bpf_sk_lookup_kern", BPF_PROG_TYPE_SK_LOOKUP, BPF_SK_LOOKUP }, 1178 { "xdp_md", "xdp_buff", BPF_PROG_TYPE_XDP, BPF_XDP }, 1179 /* tracing types with no expected attach type */ 1180 { "bpf_user_pt_regs_t", "pt_regs", BPF_PROG_TYPE_KPROBE }, 1181 { "bpf_perf_event_data", "bpf_perf_event_data_kern", BPF_PROG_TYPE_PERF_EVENT }, 1182 /* raw_tp programs use u64[] from kernel side, we don't want 1183 * to match on that, probably; so NULL for kern-side type 1184 */ 1185 { "bpf_raw_tracepoint_args", NULL, BPF_PROG_TYPE_RAW_TRACEPOINT }, 1186 }; 1187 int i; 1188 1189 if (!ctx_name) 1190 return -EINVAL; 1191 1192 for (i = 0; i < ARRAY_SIZE(ctx_map); i++) { 1193 if (strcmp(ctx_map[i].uapi_name, ctx_name) == 0 || 1194 (ctx_map[i].kern_name && strcmp(ctx_map[i].kern_name, ctx_name) == 0)) { 1195 *prog_type = ctx_map[i].prog_type; 1196 *attach_type = ctx_map[i].attach_type; 1197 return 0; 1198 } 1199 } 1200 1201 return -ESRCH; 1202 } 1203 1204 /* Make sure only target program is referenced from struct_ops map, 1205 * otherwise libbpf would automatically set autocreate for all 1206 * referenced programs. 1207 * See libbpf.c:bpf_object_adjust_struct_ops_autoload. 1208 */ 1209 static void mask_unrelated_struct_ops_progs(struct bpf_object *obj, 1210 struct bpf_map *map, 1211 struct bpf_program *prog) 1212 { 1213 struct btf *btf = bpf_object__btf(obj); 1214 const struct btf_type *t, *mt; 1215 struct btf_member *m; 1216 int i, moff; 1217 size_t data_sz, ptr_sz = sizeof(void *); 1218 void *data; 1219 1220 t = btf__type_by_id(btf, bpf_map__btf_value_type_id(map)); 1221 if (!btf_is_struct(t)) 1222 return; 1223 1224 data = bpf_map__initial_value(map, &data_sz); 1225 for (i = 0; i < btf_vlen(t); i++) { 1226 m = &btf_members(t)[i]; 1227 mt = btf__type_by_id(btf, m->type); 1228 if (!btf_is_ptr(mt)) 1229 continue; 1230 moff = m->offset / 8; 1231 if (moff + ptr_sz > data_sz) 1232 continue; 1233 if (memcmp(data + moff, &prog, ptr_sz) == 0) 1234 continue; 1235 memset(data + moff, 0, ptr_sz); 1236 } 1237 } 1238 1239 static void fixup_obj_maps(struct bpf_object *obj) 1240 { 1241 struct bpf_map *map; 1242 1243 bpf_object__for_each_map(map, obj) { 1244 /* disable pinning */ 1245 bpf_map__set_pin_path(map, NULL); 1246 1247 /* fix up map size, if necessary */ 1248 switch (bpf_map__type(map)) { 1249 case BPF_MAP_TYPE_SK_STORAGE: 1250 case BPF_MAP_TYPE_TASK_STORAGE: 1251 case BPF_MAP_TYPE_INODE_STORAGE: 1252 case BPF_MAP_TYPE_CGROUP_STORAGE: 1253 case BPF_MAP_TYPE_CGRP_STORAGE: 1254 case BPF_MAP_TYPE_STRUCT_OPS: 1255 break; 1256 default: 1257 if (bpf_map__max_entries(map) == 0) 1258 bpf_map__set_max_entries(map, 1); 1259 } 1260 } 1261 } 1262 1263 static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename) 1264 { 1265 struct bpf_map *map; 1266 1267 bpf_object__for_each_map(map, obj) { 1268 if (bpf_map__type(map) == BPF_MAP_TYPE_STRUCT_OPS) 1269 mask_unrelated_struct_ops_progs(obj, map, prog); 1270 } 1271 1272 /* SEC(freplace) programs can't be loaded with veristat as is, 1273 * but we can try guessing their target program's expected type by 1274 * looking at the type of program's first argument and substituting 1275 * corresponding program type 1276 */ 1277 if (bpf_program__type(prog) == BPF_PROG_TYPE_EXT) { 1278 const struct btf *btf = bpf_object__btf(obj); 1279 const char *prog_name = bpf_program__name(prog); 1280 enum bpf_prog_type prog_type; 1281 enum bpf_attach_type attach_type; 1282 const struct btf_type *t; 1283 const char *ctx_name; 1284 int id; 1285 1286 if (!btf) 1287 goto skip_freplace_fixup; 1288 1289 id = btf__find_by_name_kind(btf, prog_name, BTF_KIND_FUNC); 1290 t = btf__type_by_id(btf, id); 1291 t = btf__type_by_id(btf, t->type); 1292 if (!btf_is_func_proto(t) || btf_vlen(t) != 1) 1293 goto skip_freplace_fixup; 1294 1295 /* context argument is a pointer to a struct/typedef */ 1296 t = btf__type_by_id(btf, btf_params(t)[0].type); 1297 while (t && btf_is_mod(t)) 1298 t = btf__type_by_id(btf, t->type); 1299 if (!t || !btf_is_ptr(t)) 1300 goto skip_freplace_fixup; 1301 t = btf__type_by_id(btf, t->type); 1302 while (t && btf_is_mod(t)) 1303 t = btf__type_by_id(btf, t->type); 1304 if (!t) 1305 goto skip_freplace_fixup; 1306 1307 ctx_name = btf__name_by_offset(btf, t->name_off); 1308 1309 if (guess_prog_type_by_ctx_name(ctx_name, &prog_type, &attach_type) == 0) { 1310 bpf_program__set_type(prog, prog_type); 1311 bpf_program__set_expected_attach_type(prog, attach_type); 1312 1313 if (!env.quiet) { 1314 fprintf(stderr, "Using guessed program type '%s' for %s/%s...\n", 1315 libbpf_bpf_prog_type_str(prog_type), 1316 filename, prog_name); 1317 } 1318 } else { 1319 if (!env.quiet) { 1320 fprintf(stderr, "Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n", 1321 ctx_name, filename, prog_name); 1322 } 1323 } 1324 } 1325 skip_freplace_fixup: 1326 return; 1327 } 1328 1329 static int max_verifier_log_size(void) 1330 { 1331 const int SMALL_LOG_SIZE = UINT_MAX >> 8; 1332 const int BIG_LOG_SIZE = UINT_MAX >> 2; 1333 struct bpf_insn insns[] = { 1334 { .code = BPF_ALU | BPF_MOV | BPF_X, .dst_reg = BPF_REG_0, }, 1335 { .code = BPF_JMP | BPF_EXIT, }, 1336 }; 1337 LIBBPF_OPTS(bpf_prog_load_opts, opts, 1338 .log_size = BIG_LOG_SIZE, 1339 .log_buf = (void *)-1, 1340 .log_level = 4 1341 ); 1342 int ret, insn_cnt = ARRAY_SIZE(insns); 1343 static int log_size; 1344 1345 if (log_size != 0) 1346 return log_size; 1347 1348 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts); 1349 1350 if (ret == -EFAULT) 1351 log_size = BIG_LOG_SIZE; 1352 else /* ret == -EINVAL, big log size is not supported by the verifier */ 1353 log_size = SMALL_LOG_SIZE; 1354 1355 return log_size; 1356 } 1357 1358 static bool output_stat_enabled(int id) 1359 { 1360 int i; 1361 1362 for (i = 0; i < env.output_spec.spec_cnt; i++) 1363 if (env.output_spec.ids[i] == id) 1364 return true; 1365 return false; 1366 } 1367 1368 __attribute__((format(printf, 2, 3))) 1369 static int write_one_line(const char *file, const char *fmt, ...) 1370 { 1371 int err, saved_errno; 1372 va_list ap; 1373 FILE *f; 1374 1375 f = fopen(file, "w"); 1376 if (!f) 1377 return -1; 1378 1379 va_start(ap, fmt); 1380 errno = 0; 1381 err = vfprintf(f, fmt, ap); 1382 saved_errno = errno; 1383 va_end(ap); 1384 fclose(f); 1385 errno = saved_errno; 1386 return err < 0 ? -1 : 0; 1387 } 1388 1389 __attribute__((format(scanf, 3, 4))) 1390 static int scanf_one_line(const char *file, int fields_expected, const char *fmt, ...) 1391 { 1392 int res = 0, saved_errno = 0; 1393 char *line = NULL; 1394 size_t line_len; 1395 va_list ap; 1396 FILE *f; 1397 1398 f = fopen(file, "r"); 1399 if (!f) 1400 return -1; 1401 1402 va_start(ap, fmt); 1403 while (getline(&line, &line_len, f) > 0) { 1404 res = vsscanf(line, fmt, ap); 1405 if (res == fields_expected) 1406 goto out; 1407 } 1408 if (ferror(f)) { 1409 saved_errno = errno; 1410 res = -1; 1411 } 1412 1413 out: 1414 va_end(ap); 1415 free(line); 1416 fclose(f); 1417 errno = saved_errno; 1418 return res; 1419 } 1420 1421 static void destroy_stat_cgroup(void) 1422 { 1423 char buf[PATH_MAX]; 1424 int err; 1425 1426 close(env.memory_peak_fd); 1427 1428 if (env.orig_cgroup[0]) { 1429 snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.orig_cgroup); 1430 err = write_one_line(buf, "%d\n", getpid()); 1431 if (err < 0) 1432 log_errno("moving self to original cgroup %s\n", env.orig_cgroup); 1433 } 1434 1435 if (env.stat_cgroup[0]) { 1436 err = rmdir(env.stat_cgroup); 1437 if (err < 0) 1438 log_errno("deletion of cgroup %s", env.stat_cgroup); 1439 } 1440 1441 env.memory_peak_fd = -1; 1442 env.orig_cgroup[0] = 0; 1443 env.stat_cgroup[0] = 0; 1444 } 1445 1446 /* 1447 * Creates a cgroup at /sys/fs/cgroup/veristat-accounting-<pid>, 1448 * moves current process to this cgroup. 1449 */ 1450 static void create_stat_cgroup(void) 1451 { 1452 char cgroup_fs_mount[4096]; 1453 char buf[4096]; 1454 int err; 1455 1456 env.memory_peak_fd = -1; 1457 1458 if (!output_stat_enabled(MEMORY_PEAK)) 1459 return; 1460 1461 err = scanf_one_line("/proc/self/mounts", 2, "%*s %4095s cgroup2 %s", 1462 cgroup_fs_mount, buf); 1463 if (err != 2) { 1464 if (err < 0) 1465 log_errno("reading /proc/self/mounts"); 1466 else if (!env.quiet) 1467 fprintf(stderr, "Can't find cgroupfs v2 mount point.\n"); 1468 goto err_out; 1469 } 1470 1471 /* cgroup-v2.rst promises the line "0::<group>" for cgroups v2 */ 1472 err = scanf_one_line("/proc/self/cgroup", 1, "0::%4095s", buf); 1473 if (err != 1) { 1474 if (err < 0) 1475 log_errno("reading /proc/self/cgroup"); 1476 else if (!env.quiet) 1477 fprintf(stderr, "Can't infer veristat process cgroup."); 1478 goto err_out; 1479 } 1480 1481 snprintf(env.orig_cgroup, sizeof(env.orig_cgroup), "%s/%s", cgroup_fs_mount, buf); 1482 1483 snprintf(buf, sizeof(buf), "%s/veristat-accounting-%d", cgroup_fs_mount, getpid()); 1484 err = mkdir(buf, 0777); 1485 if (err < 0) { 1486 log_errno("creation of cgroup %s", buf); 1487 goto err_out; 1488 } 1489 strcpy(env.stat_cgroup, buf); 1490 1491 snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.stat_cgroup); 1492 err = write_one_line(buf, "%d\n", getpid()); 1493 if (err < 0) { 1494 log_errno("entering cgroup %s", buf); 1495 goto err_out; 1496 } 1497 1498 snprintf(buf, sizeof(buf), "%s/memory.peak", env.stat_cgroup); 1499 env.memory_peak_fd = open(buf, O_RDWR | O_APPEND); 1500 if (env.memory_peak_fd < 0) { 1501 log_errno("opening %s", buf); 1502 goto err_out; 1503 } 1504 1505 return; 1506 1507 err_out: 1508 if (!env.quiet) 1509 fprintf(stderr, "Memory usage metric unavailable.\n"); 1510 destroy_stat_cgroup(); 1511 } 1512 1513 /* Current value of /sys/fs/cgroup/veristat-accounting-<pid>/memory.peak */ 1514 static long cgroup_memory_peak(void) 1515 { 1516 long err, memory_peak; 1517 char buf[32]; 1518 1519 if (env.memory_peak_fd < 0) 1520 return -1; 1521 1522 err = pread(env.memory_peak_fd, buf, sizeof(buf) - 1, 0); 1523 if (err <= 0) { 1524 log_errno("pread(%s/memory.peak)", env.stat_cgroup); 1525 return -1; 1526 } 1527 1528 buf[err] = 0; 1529 errno = 0; 1530 memory_peak = strtoll(buf, NULL, 10); 1531 if (errno) { 1532 log_errno("%s/memory.peak:strtoll(%s)", env.stat_cgroup, buf); 1533 return -1; 1534 } 1535 1536 return memory_peak; 1537 } 1538 1539 static int reset_stat_cgroup(void) 1540 { 1541 char buf[] = "r\n"; 1542 int err; 1543 1544 if (env.memory_peak_fd < 0) 1545 return -1; 1546 1547 err = pwrite(env.memory_peak_fd, buf, sizeof(buf), 0); 1548 if (err <= 0) { 1549 log_errno("pwrite(%s/memory.peak)", env.stat_cgroup); 1550 return -1; 1551 } 1552 return 0; 1553 } 1554 1555 static int parse_rvalue(const char *val, struct rvalue *rvalue) 1556 { 1557 long long value; 1558 char *val_end; 1559 1560 if (val[0] == '-' || isdigit(val[0])) { 1561 /* must be a number */ 1562 errno = 0; 1563 value = strtoll(val, &val_end, 0); 1564 if (errno == ERANGE) { 1565 errno = 0; 1566 value = strtoull(val, &val_end, 0); 1567 } 1568 if (errno || *val_end != '\0') { 1569 fprintf(stderr, "Failed to parse value '%s'\n", val); 1570 return -EINVAL; 1571 } 1572 rvalue->ivalue = value; 1573 rvalue->type = INTEGRAL; 1574 } else { 1575 /* if not a number, consider it enum value */ 1576 rvalue->svalue = strdup(val); 1577 if (!rvalue->svalue) 1578 return -ENOMEM; 1579 rvalue->type = ENUMERATOR; 1580 } 1581 return 0; 1582 } 1583 1584 static void dump(__u32 prog_id, enum dump_mode mode, const char *file_name, const char *prog_name) 1585 { 1586 char command[64], buf[4096]; 1587 FILE *fp; 1588 int status; 1589 1590 status = system("command -v bpftool > /dev/null 2>&1"); 1591 if (status != 0) { 1592 fprintf(stderr, "bpftool is not available, can't print program dump\n"); 1593 return; 1594 } 1595 snprintf(command, sizeof(command), "bpftool prog dump %s id %u", 1596 mode == DUMP_JITED ? "jited" : "xlated", prog_id); 1597 fp = popen(command, "r"); 1598 if (!fp) { 1599 fprintf(stderr, "bpftool failed with error: %d\n", errno); 1600 return; 1601 } 1602 1603 printf("DUMP (%s) %s/%s:\n", mode == DUMP_JITED ? "JITED" : "XLATED", file_name, prog_name); 1604 while (fgets(buf, sizeof(buf), fp)) 1605 fputs(buf, stdout); 1606 fprintf(stdout, "\n"); 1607 1608 if (ferror(fp)) 1609 fprintf(stderr, "Failed to dump BPF prog with error: %d\n", errno); 1610 1611 pclose(fp); 1612 } 1613 1614 static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog) 1615 { 1616 const char *base_filename = basename(strdupa(filename)); 1617 const char *prog_name = bpf_program__name(prog); 1618 long mem_peak_a, mem_peak_b, mem_peak = -1; 1619 LIBBPF_OPTS(bpf_prog_load_opts, opts); 1620 char *buf; 1621 int buf_sz, log_level; 1622 struct verif_stats *stats; 1623 struct bpf_prog_info info; 1624 __u32 info_len = sizeof(info); 1625 int err = 0, cgroup_err; 1626 void *tmp; 1627 int fd; 1628 1629 if (!should_process_file_prog(base_filename, bpf_program__name(prog))) { 1630 env.progs_skipped++; 1631 return 0; 1632 } 1633 1634 tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats)); 1635 if (!tmp) 1636 return -ENOMEM; 1637 env.prog_stats = tmp; 1638 stats = &env.prog_stats[env.prog_stat_cnt++]; 1639 memset(stats, 0, sizeof(*stats)); 1640 1641 if (env.verbose || env.top_src_lines > 0) { 1642 buf_sz = env.log_size ? env.log_size : max_verifier_log_size(); 1643 buf = malloc(buf_sz); 1644 if (!buf) 1645 return -ENOMEM; 1646 /* ensure we always request stats */ 1647 log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0); 1648 /* --top-src-lines needs verifier log */ 1649 if (env.top_src_lines > 0 && env.log_level == 0) 1650 log_level |= 2; 1651 } else { 1652 buf = verif_log_buf; 1653 buf_sz = sizeof(verif_log_buf); 1654 /* request only verifier stats */ 1655 log_level = 4 | (env.log_fixed ? 8 : 0); 1656 } 1657 verif_log_buf[0] = '\0'; 1658 1659 /* increase chances of successful BPF object loading */ 1660 fixup_obj(obj, prog, base_filename); 1661 1662 if (env.force_checkpoints) 1663 bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ); 1664 if (env.force_reg_invariants) 1665 bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS); 1666 1667 opts.log_buf = buf; 1668 opts.log_size = buf_sz; 1669 opts.log_level = log_level; 1670 1671 cgroup_err = reset_stat_cgroup(); 1672 mem_peak_a = cgroup_memory_peak(); 1673 fd = bpf_program__clone(prog, &opts); 1674 if (fd < 0) { 1675 err = fd; 1676 if (env.verbose) 1677 fprintf(stderr, "Failed to load program %s %d\n", prog_name, err); 1678 } 1679 mem_peak_b = cgroup_memory_peak(); 1680 if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0) 1681 mem_peak = mem_peak_b - mem_peak_a; 1682 1683 env.progs_processed++; 1684 1685 stats->file_name = strdup(base_filename); 1686 stats->prog_name = strdup(bpf_program__name(prog)); 1687 stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */ 1688 stats->stats[SIZE] = bpf_program__insn_cnt(prog); 1689 stats->stats[PROG_TYPE] = bpf_program__type(prog); 1690 stats->stats[ATTACH_TYPE] = bpf_program__expected_attach_type(prog); 1691 stats->stats[MEMORY_PEAK] = mem_peak < 0 ? -1 : mem_peak / (1024 * 1024); 1692 1693 memset(&info, 0, info_len); 1694 if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0) { 1695 stats->stats[JITED_SIZE] = info.jited_prog_len; 1696 if (env.dump_mode & DUMP_JITED) 1697 dump(info.id, DUMP_JITED, base_filename, prog_name); 1698 if (env.dump_mode & DUMP_XLATED) 1699 dump(info.id, DUMP_XLATED, base_filename, prog_name); 1700 } 1701 1702 parse_verif_log(buf, buf_sz, stats); 1703 1704 if (env.verbose) { 1705 printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n", 1706 filename, prog_name, stats->stats[DURATION], 1707 err ? "failure" : "success", buf); 1708 } 1709 if (env.top_src_lines > 0) 1710 print_top_src_lines(buf, buf_sz, stats->prog_name); 1711 1712 if (verif_log_buf != buf) 1713 free(buf); 1714 if (fd > 0) 1715 close(fd); 1716 return 0; 1717 } 1718 1719 static int append_preset_atom(struct var_preset *preset, char *value, bool is_index) 1720 { 1721 struct field_access *tmp; 1722 int i = preset->atom_count; 1723 int err; 1724 1725 tmp = reallocarray(preset->atoms, i + 1, sizeof(*preset->atoms)); 1726 if (!tmp) 1727 return -ENOMEM; 1728 1729 preset->atoms = tmp; 1730 preset->atom_count++; 1731 1732 if (is_index) { 1733 preset->atoms[i].type = ARRAY_INDEX; 1734 err = parse_rvalue(value, &preset->atoms[i].index); 1735 if (err) 1736 return err; 1737 } else { 1738 preset->atoms[i].type = FIELD_NAME; 1739 preset->atoms[i].name = strdup(value); 1740 if (!preset->atoms[i].name) 1741 return -ENOMEM; 1742 } 1743 return 0; 1744 } 1745 1746 static int parse_var_atoms(const char *full_var, struct var_preset *preset) 1747 { 1748 char expr[256], var[256], *name, *saveptr; 1749 int n, len, off, err; 1750 1751 snprintf(expr, sizeof(expr), "%s", full_var); 1752 preset->atom_count = 0; 1753 while ((name = strtok_r(preset->atom_count ? NULL : expr, ".", &saveptr))) { 1754 len = strlen(name); 1755 /* parse variable name */ 1756 if (sscanf(name, "%[a-zA-Z0-9_] %n", var, &off) != 1) { 1757 fprintf(stderr, "Can't parse %s\n", name); 1758 return -EINVAL; 1759 } 1760 err = append_preset_atom(preset, var, false); 1761 if (err) 1762 return err; 1763 1764 /* parse optional array indexes */ 1765 while (off < len) { 1766 if (sscanf(name + off, " [ %[a-zA-Z0-9_] ] %n", var, &n) != 1) { 1767 fprintf(stderr, "Can't parse %s as index\n", name + off); 1768 return -EINVAL; 1769 } 1770 err = append_preset_atom(preset, var, true); 1771 if (err) 1772 return err; 1773 off += n; 1774 } 1775 } 1776 return 0; 1777 } 1778 1779 static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr) 1780 { 1781 void *tmp; 1782 struct var_preset *cur; 1783 char var[256], val[256]; 1784 int n, err; 1785 1786 tmp = realloc(*presets, (*cnt + 1) * sizeof(**presets)); 1787 if (!tmp) 1788 return -ENOMEM; 1789 *presets = tmp; 1790 cur = &(*presets)[*cnt]; 1791 memset(cur, 0, sizeof(*cur)); 1792 (*cnt)++; 1793 1794 if (sscanf(expr, " %[][a-zA-Z0-9_. ] = %s %n", var, val, &n) != 2 || n != strlen(expr)) { 1795 fprintf(stderr, "Failed to parse expression '%s'\n", expr); 1796 return -EINVAL; 1797 } 1798 /* Remove trailing spaces from var, as scanf may add those */ 1799 rtrim(var); 1800 1801 err = parse_rvalue(val, &cur->value); 1802 if (err) 1803 return err; 1804 1805 cur->full_name = strdup(var); 1806 if (!cur->full_name) 1807 return -ENOMEM; 1808 1809 err = parse_var_atoms(var, cur); 1810 if (err) 1811 return err; 1812 1813 return 0; 1814 } 1815 1816 static int append_var_preset_file(const char *filename) 1817 { 1818 char buf[1024]; 1819 FILE *f; 1820 int err = 0; 1821 1822 f = fopen(filename, "rt"); 1823 if (!f) { 1824 err = -errno; 1825 fprintf(stderr, "Failed to open presets in '%s': %s\n", filename, strerror(-err)); 1826 return -EINVAL; 1827 } 1828 1829 while (fscanf(f, " %1023[^\n]\n", buf) == 1) { 1830 if (buf[0] == '\0' || buf[0] == '#') 1831 continue; 1832 1833 err = append_var_preset(&env.presets, &env.npresets, buf); 1834 if (err) 1835 goto cleanup; 1836 } 1837 1838 cleanup: 1839 fclose(f); 1840 return err; 1841 } 1842 1843 static bool is_signed_type(const struct btf_type *t) 1844 { 1845 if (btf_is_int(t)) 1846 return btf_int_encoding(t) & BTF_INT_SIGNED; 1847 if (btf_is_any_enum(t)) 1848 return btf_kflag(t); 1849 return true; 1850 } 1851 1852 static int enum_value_from_name(const struct btf *btf, const struct btf_type *t, 1853 const char *evalue, long long *retval) 1854 { 1855 if (btf_is_enum(t)) { 1856 struct btf_enum *e = btf_enum(t); 1857 int i, n = btf_vlen(t); 1858 1859 for (i = 0; i < n; ++i, ++e) { 1860 const char *cur_name = btf__name_by_offset(btf, e->name_off); 1861 1862 if (strcmp(cur_name, evalue) == 0) { 1863 *retval = e->val; 1864 return 0; 1865 } 1866 } 1867 } else if (btf_is_enum64(t)) { 1868 struct btf_enum64 *e = btf_enum64(t); 1869 int i, n = btf_vlen(t); 1870 1871 for (i = 0; i < n; ++i, ++e) { 1872 const char *cur_name = btf__name_by_offset(btf, e->name_off); 1873 __u64 value = btf_enum64_value(e); 1874 1875 if (strcmp(cur_name, evalue) == 0) { 1876 *retval = value; 1877 return 0; 1878 } 1879 } 1880 } 1881 return -EINVAL; 1882 } 1883 1884 static bool is_preset_supported(const struct btf_type *t) 1885 { 1886 return btf_is_int(t) || btf_is_enum(t) || btf_is_enum64(t); 1887 } 1888 1889 static int find_enum_value(const struct btf *btf, const char *name, long long *value) 1890 { 1891 const struct btf_type *t; 1892 int cnt, i; 1893 long long lvalue; 1894 1895 cnt = btf__type_cnt(btf); 1896 for (i = 1; i != cnt; ++i) { 1897 t = btf__type_by_id(btf, i); 1898 1899 if (!btf_is_any_enum(t)) 1900 continue; 1901 1902 if (enum_value_from_name(btf, t, name, &lvalue) == 0) { 1903 *value = lvalue; 1904 return 0; 1905 } 1906 } 1907 return -ESRCH; 1908 } 1909 1910 static int resolve_rvalue(struct btf *btf, const struct rvalue *rvalue, long long *result) 1911 { 1912 int err = 0; 1913 1914 switch (rvalue->type) { 1915 case INTEGRAL: 1916 *result = rvalue->ivalue; 1917 return 0; 1918 case ENUMERATOR: 1919 err = find_enum_value(btf, rvalue->svalue, result); 1920 if (err) { 1921 fprintf(stderr, "Can't resolve enum value %s\n", rvalue->svalue); 1922 return err; 1923 } 1924 return 0; 1925 default: 1926 fprintf(stderr, "Unknown rvalue type\n"); 1927 return -EOPNOTSUPP; 1928 } 1929 return 0; 1930 } 1931 1932 static int adjust_var_secinfo_array(struct btf *btf, int tid, struct field_access *atom, 1933 const char *array_name, struct btf_var_secinfo *sinfo) 1934 { 1935 const struct btf_type *t; 1936 struct btf_array *barr; 1937 long long idx; 1938 int err; 1939 1940 tid = btf__resolve_type(btf, tid); 1941 t = btf__type_by_id(btf, tid); 1942 if (!btf_is_array(t)) { 1943 fprintf(stderr, "Array index is not expected for %s\n", 1944 array_name); 1945 return -EINVAL; 1946 } 1947 barr = btf_array(t); 1948 err = resolve_rvalue(btf, &atom->index, &idx); 1949 if (err) 1950 return err; 1951 if (idx < 0 || idx >= barr->nelems) { 1952 fprintf(stderr, "Array index %lld is out of bounds [0, %u): %s\n", 1953 idx, barr->nelems, array_name); 1954 return -EINVAL; 1955 } 1956 sinfo->size = btf__resolve_size(btf, barr->type); 1957 sinfo->offset += sinfo->size * idx; 1958 sinfo->type = btf__resolve_type(btf, barr->type); 1959 return 0; 1960 } 1961 1962 static int adjust_var_secinfo_member(const struct btf *btf, 1963 const struct btf_type *parent_type, 1964 __u32 parent_offset, 1965 const char *member_name, 1966 struct btf_var_secinfo *sinfo) 1967 { 1968 int i; 1969 1970 if (!btf_is_composite(parent_type)) { 1971 fprintf(stderr, "Can't resolve field %s for non-composite type\n", member_name); 1972 return -EINVAL; 1973 } 1974 1975 for (i = 0; i < btf_vlen(parent_type); ++i) { 1976 const struct btf_member *member; 1977 const struct btf_type *member_type; 1978 int tid, off; 1979 1980 member = btf_members(parent_type) + i; 1981 tid = btf__resolve_type(btf, member->type); 1982 if (tid < 0) 1983 return -EINVAL; 1984 1985 member_type = btf__type_by_id(btf, tid); 1986 off = parent_offset + member->offset; 1987 if (member->name_off) { 1988 const char *name = btf__name_by_offset(btf, member->name_off); 1989 1990 if (strcmp(member_name, name) == 0) { 1991 if (btf_member_bitfield_size(parent_type, i) != 0) { 1992 fprintf(stderr, "Bitfield presets are not supported %s\n", 1993 name); 1994 return -EINVAL; 1995 } 1996 sinfo->offset += off / 8; 1997 sinfo->type = tid; 1998 sinfo->size = member_type->size; 1999 return 0; 2000 } 2001 } else if (btf_is_composite(member_type)) { 2002 int err; 2003 2004 err = adjust_var_secinfo_member(btf, member_type, off, 2005 member_name, sinfo); 2006 if (!err) 2007 return 0; 2008 } 2009 } 2010 2011 return -ESRCH; 2012 } 2013 2014 static int adjust_var_secinfo(struct btf *btf, const struct btf_type *t, 2015 struct btf_var_secinfo *sinfo, struct var_preset *preset) 2016 { 2017 const struct btf_type *base_type; 2018 const char *prev_name; 2019 int err, i; 2020 int tid; 2021 2022 assert(preset->atom_count > 0); 2023 assert(preset->atoms[0].type == FIELD_NAME); 2024 2025 tid = btf__resolve_type(btf, t->type); 2026 base_type = btf__type_by_id(btf, tid); 2027 prev_name = preset->atoms[0].name; 2028 2029 for (i = 1; i < preset->atom_count; ++i) { 2030 struct field_access *atom = preset->atoms + i; 2031 2032 switch (atom->type) { 2033 case ARRAY_INDEX: 2034 err = adjust_var_secinfo_array(btf, tid, atom, prev_name, sinfo); 2035 break; 2036 case FIELD_NAME: 2037 err = adjust_var_secinfo_member(btf, base_type, 0, atom->name, sinfo); 2038 if (err == -ESRCH) 2039 fprintf(stderr, "Can't find '%s'\n", atom->name); 2040 prev_name = atom->name; 2041 break; 2042 default: 2043 fprintf(stderr, "Unknown field_access type\n"); 2044 return -EOPNOTSUPP; 2045 } 2046 if (err) 2047 return err; 2048 base_type = btf__type_by_id(btf, sinfo->type); 2049 tid = sinfo->type; 2050 } 2051 2052 return 0; 2053 } 2054 2055 static int set_global_var(struct bpf_object *obj, struct btf *btf, 2056 struct bpf_map *map, struct btf_var_secinfo *sinfo, 2057 struct var_preset *preset) 2058 { 2059 const struct btf_type *base_type; 2060 void *ptr; 2061 long long value = preset->value.ivalue; 2062 size_t size; 2063 2064 base_type = btf__type_by_id(btf, btf__resolve_type(btf, sinfo->type)); 2065 if (!base_type) { 2066 fprintf(stderr, "Failed to resolve type %d\n", sinfo->type); 2067 return -EINVAL; 2068 } 2069 if (!is_preset_supported(base_type)) { 2070 fprintf(stderr, "Can't set %s. Only ints and enums are supported\n", 2071 preset->full_name); 2072 return -EINVAL; 2073 } 2074 2075 if (preset->value.type == ENUMERATOR) { 2076 if (btf_is_any_enum(base_type)) { 2077 if (enum_value_from_name(btf, base_type, preset->value.svalue, &value)) { 2078 fprintf(stderr, 2079 "Failed to find integer value for enum element %s\n", 2080 preset->value.svalue); 2081 return -EINVAL; 2082 } 2083 } else { 2084 fprintf(stderr, "Value %s is not supported for type %s\n", 2085 preset->value.svalue, 2086 btf__name_by_offset(btf, base_type->name_off)); 2087 return -EINVAL; 2088 } 2089 } 2090 2091 /* Check if value fits into the target variable size */ 2092 if (sinfo->size < sizeof(value)) { 2093 bool is_signed = is_signed_type(base_type); 2094 __u32 unsigned_bits = sinfo->size * 8 - (is_signed ? 1 : 0); 2095 long long max_val = 1ll << unsigned_bits; 2096 2097 if (value >= max_val || value < -max_val) { 2098 fprintf(stderr, 2099 "Variable %s value %lld is out of range [%lld; %lld]\n", 2100 btf__name_by_offset(btf, base_type->name_off), value, 2101 is_signed ? -max_val : 0, max_val - 1); 2102 return -EINVAL; 2103 } 2104 } 2105 2106 ptr = bpf_map__initial_value(map, &size); 2107 if (!ptr || sinfo->offset + sinfo->size > size) 2108 return -EINVAL; 2109 2110 if (__BYTE_ORDER == __LITTLE_ENDIAN) { 2111 memcpy(ptr + sinfo->offset, &value, sinfo->size); 2112 } else { /* __BYTE_ORDER == __BIG_ENDIAN */ 2113 __u8 src_offset = sizeof(value) - sinfo->size; 2114 2115 memcpy(ptr + sinfo->offset, (void *)&value + src_offset, sinfo->size); 2116 } 2117 return 0; 2118 } 2119 2120 static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, int npresets) 2121 { 2122 struct btf_var_secinfo *sinfo; 2123 const char *sec_name; 2124 const struct btf_type *t; 2125 struct bpf_map *map; 2126 struct btf *btf; 2127 int i, j, k, n, cnt, err = 0; 2128 2129 if (npresets == 0) 2130 return 0; 2131 2132 btf = bpf_object__btf(obj); 2133 if (!btf) 2134 return -EINVAL; 2135 2136 cnt = btf__type_cnt(btf); 2137 for (i = 1; i != cnt; ++i) { 2138 t = btf__type_by_id(btf, i); 2139 2140 if (!btf_is_datasec(t)) 2141 continue; 2142 2143 sinfo = btf_var_secinfos(t); 2144 sec_name = btf__name_by_offset(btf, t->name_off); 2145 map = bpf_object__find_map_by_name(obj, sec_name); 2146 if (!map) 2147 continue; 2148 2149 n = btf_vlen(t); 2150 for (j = 0; j < n; ++j, ++sinfo) { 2151 const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type); 2152 const char *var_name; 2153 2154 if (!btf_is_var(var_type)) 2155 continue; 2156 2157 var_name = btf__name_by_offset(btf, var_type->name_off); 2158 2159 for (k = 0; k < npresets; ++k) { 2160 struct btf_var_secinfo tmp_sinfo; 2161 2162 if (strcmp(var_name, presets[k].atoms[0].name) != 0) 2163 continue; 2164 2165 if (presets[k].applied) { 2166 fprintf(stderr, "Variable %s is set more than once", 2167 var_name); 2168 return -EINVAL; 2169 } 2170 tmp_sinfo = *sinfo; 2171 err = adjust_var_secinfo(btf, var_type, 2172 &tmp_sinfo, presets + k); 2173 if (err) 2174 return err; 2175 2176 err = set_global_var(obj, btf, map, &tmp_sinfo, presets + k); 2177 if (err) 2178 return err; 2179 2180 presets[k].applied = true; 2181 } 2182 } 2183 } 2184 for (i = 0; i < npresets; ++i) { 2185 if (!presets[i].applied) { 2186 fprintf(stderr, "Global variable preset %s has not been applied\n", 2187 presets[i].full_name); 2188 err = -EINVAL; 2189 } 2190 presets[i].applied = false; 2191 } 2192 return err; 2193 } 2194 2195 static int process_obj(const char *filename) 2196 { 2197 const char *base_filename = basename(strdupa(filename)); 2198 struct bpf_object *obj = NULL; 2199 struct bpf_program *prog; 2200 libbpf_print_fn_t old_libbpf_print_fn; 2201 LIBBPF_OPTS(bpf_object_open_opts, opts); 2202 int err = 0, prog_cnt = 0; 2203 2204 if (!should_process_file_prog(base_filename, NULL)) { 2205 if (env.verbose) 2206 printf("Skipping '%s' due to filters...\n", filename); 2207 env.files_skipped++; 2208 return 0; 2209 } 2210 if (!is_bpf_obj_file(filename)) { 2211 if (env.verbose) 2212 printf("Skipping '%s' as it's not a BPF object file...\n", filename); 2213 env.files_skipped++; 2214 return 0; 2215 } 2216 2217 if (!env.quiet && env.out_fmt == RESFMT_TABLE) 2218 printf("Processing '%s'...\n", base_filename); 2219 2220 old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn); 2221 obj = bpf_object__open_file(filename, &opts); 2222 if (!obj) { 2223 /* if libbpf can't open BPF object file, it could be because 2224 * that BPF object file is incomplete and has to be statically 2225 * linked into a final BPF object file; instead of bailing 2226 * out, report it into stderr, mark it as skipped, and 2227 * proceed 2228 */ 2229 fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno); 2230 env.files_skipped++; 2231 err = 0; 2232 goto cleanup; 2233 } 2234 2235 env.files_processed++; 2236 2237 bpf_object__for_each_program(prog, obj) { 2238 bpf_program__set_autoload(prog, true); 2239 prog_cnt++; 2240 } 2241 2242 fixup_obj_maps(obj); 2243 2244 err = set_global_vars(obj, env.presets, env.npresets); 2245 if (err) { 2246 fprintf(stderr, "Failed to set global variables %d\n", err); 2247 goto cleanup; 2248 } 2249 2250 err = bpf_object__prepare(obj); 2251 if (err && env.verbose) /* run process_prog() anyway to output per program failures */ 2252 fprintf(stderr, "Failed to prepare BPF object for loading %d\n", err); 2253 2254 bpf_object__for_each_program(prog, obj) { 2255 process_prog(filename, obj, prog); 2256 } 2257 2258 cleanup: 2259 bpf_object__close(obj); 2260 libbpf_set_print(old_libbpf_print_fn); 2261 return err; 2262 } 2263 2264 static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2, 2265 enum stat_id id, bool asc, bool abs) 2266 { 2267 int cmp = 0; 2268 2269 switch (id) { 2270 case FILE_NAME: 2271 cmp = strcmp(s1->file_name, s2->file_name); 2272 break; 2273 case PROG_NAME: 2274 cmp = strcmp(s1->prog_name, s2->prog_name); 2275 break; 2276 case ATTACH_TYPE: 2277 case PROG_TYPE: 2278 case SIZE: 2279 case JITED_SIZE: 2280 case STACK: 2281 case VERDICT: 2282 case DURATION: 2283 case TOTAL_INSNS: 2284 case TOTAL_STATES: 2285 case PEAK_STATES: 2286 case MAX_STATES_PER_INSN: 2287 case MEMORY_PEAK: 2288 case MARK_READ_MAX_LEN: { 2289 long v1 = s1->stats[id]; 2290 long v2 = s2->stats[id]; 2291 2292 if (abs) { 2293 v1 = v1 < 0 ? -v1 : v1; 2294 v2 = v2 < 0 ? -v2 : v2; 2295 } 2296 2297 if (v1 != v2) 2298 cmp = v1 < v2 ? -1 : 1; 2299 break; 2300 } 2301 default: 2302 fprintf(stderr, "Unrecognized stat #%d\n", id); 2303 exit(1); 2304 } 2305 2306 return asc ? cmp : -cmp; 2307 } 2308 2309 static int cmp_prog_stats(const void *v1, const void *v2) 2310 { 2311 const struct verif_stats *s1 = v1, *s2 = v2; 2312 int i, cmp; 2313 2314 for (i = 0; i < env.sort_spec.spec_cnt; i++) { 2315 cmp = cmp_stat(s1, s2, env.sort_spec.ids[i], 2316 env.sort_spec.asc[i], env.sort_spec.abs[i]); 2317 if (cmp != 0) 2318 return cmp; 2319 } 2320 2321 /* always disambiguate with file+prog, which are unique */ 2322 cmp = strcmp(s1->file_name, s2->file_name); 2323 if (cmp != 0) 2324 return cmp; 2325 return strcmp(s1->prog_name, s2->prog_name); 2326 } 2327 2328 static void fetch_join_stat_value(const struct verif_stats_join *s, 2329 enum stat_id id, enum stat_variant var, 2330 const char **str_val, 2331 double *num_val) 2332 { 2333 long v1, v2; 2334 2335 if (id == FILE_NAME) { 2336 *str_val = s->file_name; 2337 return; 2338 } 2339 if (id == PROG_NAME) { 2340 *str_val = s->prog_name; 2341 return; 2342 } 2343 2344 v1 = s->stats_a ? s->stats_a->stats[id] : 0; 2345 v2 = s->stats_b ? s->stats_b->stats[id] : 0; 2346 2347 switch (var) { 2348 case VARIANT_A: 2349 if (!s->stats_a) 2350 *num_val = -DBL_MAX; 2351 else 2352 *num_val = s->stats_a->stats[id]; 2353 return; 2354 case VARIANT_B: 2355 if (!s->stats_b) 2356 *num_val = -DBL_MAX; 2357 else 2358 *num_val = s->stats_b->stats[id]; 2359 return; 2360 case VARIANT_DIFF: 2361 if (!s->stats_a || !s->stats_b) 2362 *num_val = -DBL_MAX; 2363 else if (id == VERDICT) 2364 *num_val = v1 == v2 ? 1.0 /* MATCH */ : 0.0 /* MISMATCH */; 2365 else 2366 *num_val = (double)(v2 - v1); 2367 return; 2368 case VARIANT_PCT: 2369 if (!s->stats_a || !s->stats_b) { 2370 *num_val = -DBL_MAX; 2371 } else if (v1 == 0) { 2372 if (v1 == v2) 2373 *num_val = 0.0; 2374 else 2375 *num_val = v2 < v1 ? -100.0 : 100.0; 2376 } else { 2377 *num_val = (v2 - v1) * 100.0 / v1; 2378 } 2379 return; 2380 } 2381 } 2382 2383 static int cmp_join_stat(const struct verif_stats_join *s1, 2384 const struct verif_stats_join *s2, 2385 enum stat_id id, enum stat_variant var, 2386 bool asc, bool abs) 2387 { 2388 const char *str1 = NULL, *str2 = NULL; 2389 double v1 = 0.0, v2 = 0.0; 2390 int cmp = 0; 2391 2392 fetch_join_stat_value(s1, id, var, &str1, &v1); 2393 fetch_join_stat_value(s2, id, var, &str2, &v2); 2394 2395 if (abs) { 2396 v1 = fabs(v1); 2397 v2 = fabs(v2); 2398 } 2399 2400 if (str1) 2401 cmp = strcmp(str1, str2); 2402 else if (v1 != v2) 2403 cmp = v1 < v2 ? -1 : 1; 2404 2405 return asc ? cmp : -cmp; 2406 } 2407 2408 static int cmp_join_stats(const void *v1, const void *v2) 2409 { 2410 const struct verif_stats_join *s1 = v1, *s2 = v2; 2411 int i, cmp; 2412 2413 for (i = 0; i < env.sort_spec.spec_cnt; i++) { 2414 cmp = cmp_join_stat(s1, s2, 2415 env.sort_spec.ids[i], 2416 env.sort_spec.variants[i], 2417 env.sort_spec.asc[i], 2418 env.sort_spec.abs[i]); 2419 if (cmp != 0) 2420 return cmp; 2421 } 2422 2423 /* always disambiguate with file+prog, which are unique */ 2424 cmp = strcmp(s1->file_name, s2->file_name); 2425 if (cmp != 0) 2426 return cmp; 2427 return strcmp(s1->prog_name, s2->prog_name); 2428 } 2429 2430 #define HEADER_CHAR '-' 2431 #define COLUMN_SEP " " 2432 2433 static void output_header_underlines(void) 2434 { 2435 int i, j, len; 2436 2437 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2438 len = env.output_spec.lens[i]; 2439 2440 printf("%s", i == 0 ? "" : COLUMN_SEP); 2441 for (j = 0; j < len; j++) 2442 printf("%c", HEADER_CHAR); 2443 } 2444 printf("\n"); 2445 } 2446 2447 static void output_headers(enum resfmt fmt) 2448 { 2449 const char *fmt_str; 2450 int i, len; 2451 2452 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2453 int id = env.output_spec.ids[i]; 2454 int *max_len = &env.output_spec.lens[i]; 2455 2456 switch (fmt) { 2457 case RESFMT_TABLE_CALCLEN: 2458 len = snprintf(NULL, 0, "%s", stat_defs[id].header); 2459 if (len > *max_len) 2460 *max_len = len; 2461 break; 2462 case RESFMT_TABLE: 2463 fmt_str = stat_defs[id].left_aligned ? "%s%-*s" : "%s%*s"; 2464 printf(fmt_str, i == 0 ? "" : COLUMN_SEP, *max_len, stat_defs[id].header); 2465 if (i == env.output_spec.spec_cnt - 1) 2466 printf("\n"); 2467 break; 2468 case RESFMT_CSV: 2469 printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]); 2470 if (i == env.output_spec.spec_cnt - 1) 2471 printf("\n"); 2472 break; 2473 } 2474 } 2475 2476 if (fmt == RESFMT_TABLE) 2477 output_header_underlines(); 2478 } 2479 2480 static void prepare_value(const struct verif_stats *s, enum stat_id id, 2481 const char **str, long *val) 2482 { 2483 switch (id) { 2484 case FILE_NAME: 2485 *str = s ? s->file_name : "N/A"; 2486 break; 2487 case PROG_NAME: 2488 *str = s ? s->prog_name : "N/A"; 2489 break; 2490 case VERDICT: 2491 if (!s) 2492 *str = "N/A"; 2493 else 2494 *str = s->stats[VERDICT] ? "success" : "failure"; 2495 break; 2496 case ATTACH_TYPE: 2497 if (!s) 2498 *str = "N/A"; 2499 else 2500 *str = libbpf_bpf_attach_type_str(s->stats[ATTACH_TYPE]) ?: "N/A"; 2501 break; 2502 case PROG_TYPE: 2503 if (!s) 2504 *str = "N/A"; 2505 else 2506 *str = libbpf_bpf_prog_type_str(s->stats[PROG_TYPE]) ?: "N/A"; 2507 break; 2508 case DURATION: 2509 case TOTAL_INSNS: 2510 case TOTAL_STATES: 2511 case PEAK_STATES: 2512 case MAX_STATES_PER_INSN: 2513 case MARK_READ_MAX_LEN: 2514 case STACK: 2515 case SIZE: 2516 case JITED_SIZE: 2517 case MEMORY_PEAK: 2518 *val = s ? s->stats[id] : 0; 2519 break; 2520 default: 2521 fprintf(stderr, "Unrecognized stat #%d\n", id); 2522 exit(1); 2523 } 2524 } 2525 2526 static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last) 2527 { 2528 int i; 2529 2530 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2531 int id = env.output_spec.ids[i]; 2532 int *max_len = &env.output_spec.lens[i], len; 2533 const char *str = NULL; 2534 long val = 0; 2535 2536 prepare_value(s, id, &str, &val); 2537 2538 switch (fmt) { 2539 case RESFMT_TABLE_CALCLEN: 2540 if (str) 2541 len = snprintf(NULL, 0, "%s", str); 2542 else 2543 len = snprintf(NULL, 0, "%ld", val); 2544 if (len > *max_len) 2545 *max_len = len; 2546 break; 2547 case RESFMT_TABLE: 2548 if (str) 2549 printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str); 2550 else 2551 printf("%s%*ld", i == 0 ? "" : COLUMN_SEP, *max_len, val); 2552 if (i == env.output_spec.spec_cnt - 1) 2553 printf("\n"); 2554 break; 2555 case RESFMT_CSV: 2556 if (str) 2557 printf("%s%s", i == 0 ? "" : ",", str); 2558 else 2559 printf("%s%ld", i == 0 ? "" : ",", val); 2560 if (i == env.output_spec.spec_cnt - 1) 2561 printf("\n"); 2562 break; 2563 } 2564 } 2565 2566 if (last && fmt == RESFMT_TABLE) { 2567 output_header_underlines(); 2568 printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", 2569 env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped); 2570 } 2571 } 2572 2573 static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st) 2574 { 2575 switch (id) { 2576 case FILE_NAME: 2577 st->file_name = strdup(str); 2578 if (!st->file_name) 2579 return -ENOMEM; 2580 break; 2581 case PROG_NAME: 2582 st->prog_name = strdup(str); 2583 if (!st->prog_name) 2584 return -ENOMEM; 2585 break; 2586 case VERDICT: 2587 if (strcmp(str, "success") == 0) { 2588 st->stats[VERDICT] = true; 2589 } else if (strcmp(str, "failure") == 0) { 2590 st->stats[VERDICT] = false; 2591 } else { 2592 fprintf(stderr, "Unrecognized verification verdict '%s'\n", str); 2593 return -EINVAL; 2594 } 2595 break; 2596 case DURATION: 2597 case TOTAL_INSNS: 2598 case TOTAL_STATES: 2599 case PEAK_STATES: 2600 case MAX_STATES_PER_INSN: 2601 case MARK_READ_MAX_LEN: 2602 case SIZE: 2603 case JITED_SIZE: 2604 case MEMORY_PEAK: 2605 case STACK: { 2606 long val; 2607 int err, n; 2608 2609 if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) { 2610 err = -errno; 2611 fprintf(stderr, "Failed to parse '%s' as integer\n", str); 2612 return err; 2613 } 2614 2615 st->stats[id] = val; 2616 break; 2617 } 2618 case PROG_TYPE: { 2619 enum bpf_prog_type prog_type = 0; 2620 const char *type; 2621 2622 while ((type = libbpf_bpf_prog_type_str(prog_type))) { 2623 if (strcmp(type, str) == 0) { 2624 st->stats[id] = prog_type; 2625 break; 2626 } 2627 prog_type++; 2628 } 2629 2630 if (!type) { 2631 fprintf(stderr, "Unrecognized prog type %s\n", str); 2632 return -EINVAL; 2633 } 2634 break; 2635 } 2636 case ATTACH_TYPE: { 2637 enum bpf_attach_type attach_type = 0; 2638 const char *type; 2639 2640 while ((type = libbpf_bpf_attach_type_str(attach_type))) { 2641 if (strcmp(type, str) == 0) { 2642 st->stats[id] = attach_type; 2643 break; 2644 } 2645 attach_type++; 2646 } 2647 2648 if (!type) { 2649 fprintf(stderr, "Unrecognized attach type %s\n", str); 2650 return -EINVAL; 2651 } 2652 break; 2653 } 2654 default: 2655 fprintf(stderr, "Unrecognized stat #%d\n", id); 2656 return -EINVAL; 2657 } 2658 return 0; 2659 } 2660 2661 static int parse_stats_csv(const char *filename, struct stat_specs *specs, 2662 struct verif_stats **statsp, int *stat_cntp) 2663 { 2664 char line[4096]; 2665 FILE *f; 2666 int err = 0; 2667 bool header = true; 2668 2669 f = fopen(filename, "r"); 2670 if (!f) { 2671 err = -errno; 2672 fprintf(stderr, "Failed to open '%s': %d\n", filename, err); 2673 return err; 2674 } 2675 2676 *stat_cntp = 0; 2677 2678 while (fgets(line, sizeof(line), f)) { 2679 char *input = line, *state = NULL, *next; 2680 struct verif_stats *st = NULL; 2681 int col = 0, cnt = 0; 2682 2683 if (!header) { 2684 void *tmp; 2685 2686 tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp)); 2687 if (!tmp) { 2688 err = -ENOMEM; 2689 goto cleanup; 2690 } 2691 *statsp = tmp; 2692 2693 st = &(*statsp)[*stat_cntp]; 2694 memset(st, 0, sizeof(*st)); 2695 2696 *stat_cntp += 1; 2697 } 2698 2699 while ((next = strtok_r(cnt++ ? NULL : input, ",\n", &state))) { 2700 if (header) { 2701 /* for the first line, set up spec stats */ 2702 err = parse_stat(next, specs); 2703 if (err) 2704 goto cleanup; 2705 continue; 2706 } 2707 2708 /* for all other lines, parse values based on spec */ 2709 if (col >= specs->spec_cnt) { 2710 fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n", 2711 col, *stat_cntp, filename); 2712 err = -EINVAL; 2713 goto cleanup; 2714 } 2715 err = parse_stat_value(next, specs->ids[col], st); 2716 if (err) 2717 goto cleanup; 2718 col++; 2719 } 2720 2721 if (header) { 2722 header = false; 2723 continue; 2724 } 2725 2726 if (col < specs->spec_cnt) { 2727 fprintf(stderr, "Not enough columns in row #%d in '%s'\n", 2728 *stat_cntp, filename); 2729 err = -EINVAL; 2730 goto cleanup; 2731 } 2732 2733 if (!st->file_name || !st->prog_name) { 2734 fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n", 2735 *stat_cntp, filename); 2736 err = -EINVAL; 2737 goto cleanup; 2738 } 2739 2740 /* in comparison mode we can only check filters after we 2741 * parsed entire line; if row should be ignored we pretend we 2742 * never parsed it 2743 */ 2744 if (!should_process_file_prog(st->file_name, st->prog_name)) { 2745 free(st->file_name); 2746 free(st->prog_name); 2747 *stat_cntp -= 1; 2748 } 2749 } 2750 2751 if (!feof(f)) { 2752 err = -errno; 2753 fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err); 2754 } 2755 2756 cleanup: 2757 fclose(f); 2758 return err; 2759 } 2760 2761 /* empty/zero stats for mismatched rows */ 2762 static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" }; 2763 2764 static bool is_key_stat(enum stat_id id) 2765 { 2766 return id == FILE_NAME || id == PROG_NAME; 2767 } 2768 2769 static void output_comp_header_underlines(void) 2770 { 2771 int i, j, k; 2772 2773 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2774 int id = env.output_spec.ids[i]; 2775 int max_j = is_key_stat(id) ? 1 : 3; 2776 2777 for (j = 0; j < max_j; j++) { 2778 int len = env.output_spec.lens[3 * i + j]; 2779 2780 printf("%s", i + j == 0 ? "" : COLUMN_SEP); 2781 2782 for (k = 0; k < len; k++) 2783 printf("%c", HEADER_CHAR); 2784 } 2785 } 2786 printf("\n"); 2787 } 2788 2789 static void output_comp_headers(enum resfmt fmt) 2790 { 2791 static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"}; 2792 static const char *name_sfxs[3] = {"_base", "_comp", "_diff"}; 2793 int i, j, len; 2794 2795 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2796 int id = env.output_spec.ids[i]; 2797 /* key stats don't have A/B/DIFF columns, they are common for both data sets */ 2798 int max_j = is_key_stat(id) ? 1 : 3; 2799 2800 for (j = 0; j < max_j; j++) { 2801 int *max_len = &env.output_spec.lens[3 * i + j]; 2802 bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1); 2803 const char *sfx; 2804 2805 switch (fmt) { 2806 case RESFMT_TABLE_CALCLEN: 2807 sfx = is_key_stat(id) ? "" : table_sfxs[j]; 2808 len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx); 2809 if (len > *max_len) 2810 *max_len = len; 2811 break; 2812 case RESFMT_TABLE: 2813 sfx = is_key_stat(id) ? "" : table_sfxs[j]; 2814 printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP, 2815 *max_len - (int)strlen(sfx), stat_defs[id].header, sfx); 2816 if (last) 2817 printf("\n"); 2818 break; 2819 case RESFMT_CSV: 2820 sfx = is_key_stat(id) ? "" : name_sfxs[j]; 2821 printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx); 2822 if (last) 2823 printf("\n"); 2824 break; 2825 } 2826 } 2827 } 2828 2829 if (fmt == RESFMT_TABLE) 2830 output_comp_header_underlines(); 2831 } 2832 2833 static void output_comp_stats(const struct verif_stats_join *join_stats, 2834 enum resfmt fmt, bool last) 2835 { 2836 const struct verif_stats *base = join_stats->stats_a; 2837 const struct verif_stats *comp = join_stats->stats_b; 2838 char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {}; 2839 int i; 2840 2841 for (i = 0; i < env.output_spec.spec_cnt; i++) { 2842 int id = env.output_spec.ids[i], len; 2843 int *max_len_base = &env.output_spec.lens[3 * i + 0]; 2844 int *max_len_comp = &env.output_spec.lens[3 * i + 1]; 2845 int *max_len_diff = &env.output_spec.lens[3 * i + 2]; 2846 const char *base_str = NULL, *comp_str = NULL; 2847 long base_val = 0, comp_val = 0, diff_val = 0; 2848 2849 prepare_value(base, id, &base_str, &base_val); 2850 prepare_value(comp, id, &comp_str, &comp_val); 2851 2852 /* normalize all the outputs to be in string buffers for simplicity */ 2853 if (is_key_stat(id)) { 2854 /* key stats (file and program name) are always strings */ 2855 if (base) 2856 snprintf(base_buf, sizeof(base_buf), "%s", base_str); 2857 else 2858 snprintf(base_buf, sizeof(base_buf), "%s", comp_str); 2859 } else if (base_str) { 2860 snprintf(base_buf, sizeof(base_buf), "%s", base_str); 2861 snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str); 2862 if (!base || !comp) 2863 snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A"); 2864 else if (strcmp(base_str, comp_str) == 0) 2865 snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH"); 2866 else 2867 snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH"); 2868 } else { 2869 double p = 0.0; 2870 2871 if (base) 2872 snprintf(base_buf, sizeof(base_buf), "%ld", base_val); 2873 else 2874 snprintf(base_buf, sizeof(base_buf), "%s", "N/A"); 2875 if (comp) 2876 snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val); 2877 else 2878 snprintf(comp_buf, sizeof(comp_buf), "%s", "N/A"); 2879 2880 diff_val = comp_val - base_val; 2881 if (!base || !comp) { 2882 snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A"); 2883 } else { 2884 if (base_val == 0) { 2885 if (comp_val == base_val) 2886 p = 0.0; /* avoid +0 (+100%) case */ 2887 else 2888 p = comp_val < base_val ? -100.0 : 100.0; 2889 } else { 2890 p = diff_val * 100.0 / base_val; 2891 } 2892 snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", diff_val, p); 2893 } 2894 } 2895 2896 switch (fmt) { 2897 case RESFMT_TABLE_CALCLEN: 2898 len = strlen(base_buf); 2899 if (len > *max_len_base) 2900 *max_len_base = len; 2901 if (!is_key_stat(id)) { 2902 len = strlen(comp_buf); 2903 if (len > *max_len_comp) 2904 *max_len_comp = len; 2905 len = strlen(diff_buf); 2906 if (len > *max_len_diff) 2907 *max_len_diff = len; 2908 } 2909 break; 2910 case RESFMT_TABLE: { 2911 /* string outputs are left-aligned, number outputs are right-aligned */ 2912 const char *fmt = base_str ? "%s%-*s" : "%s%*s"; 2913 2914 printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf); 2915 if (!is_key_stat(id)) { 2916 printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf); 2917 printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf); 2918 } 2919 if (i == env.output_spec.spec_cnt - 1) 2920 printf("\n"); 2921 break; 2922 } 2923 case RESFMT_CSV: 2924 printf("%s%s", i == 0 ? "" : ",", base_buf); 2925 if (!is_key_stat(id)) { 2926 printf("%s%s", i == 0 ? "" : ",", comp_buf); 2927 printf("%s%s", i == 0 ? "" : ",", diff_buf); 2928 } 2929 if (i == env.output_spec.spec_cnt - 1) 2930 printf("\n"); 2931 break; 2932 } 2933 } 2934 2935 if (last && fmt == RESFMT_TABLE) 2936 output_comp_header_underlines(); 2937 } 2938 2939 static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp) 2940 { 2941 int r; 2942 2943 r = strcmp(base->file_name, comp->file_name); 2944 if (r != 0) 2945 return r; 2946 return strcmp(base->prog_name, comp->prog_name); 2947 } 2948 2949 static bool is_join_stat_filter_matched(struct filter *f, const struct verif_stats_join *stats) 2950 { 2951 static const double eps = 1e-9; 2952 const char *str = NULL; 2953 double value = 0.0; 2954 2955 fetch_join_stat_value(stats, f->stat_id, f->stat_var, &str, &value); 2956 2957 if (f->abs) 2958 value = fabs(value); 2959 2960 switch (f->op) { 2961 case OP_EQ: return value > f->value - eps && value < f->value + eps; 2962 case OP_NEQ: return value < f->value - eps || value > f->value + eps; 2963 case OP_LT: return value < f->value - eps; 2964 case OP_LE: return value <= f->value + eps; 2965 case OP_GT: return value > f->value + eps; 2966 case OP_GE: return value >= f->value - eps; 2967 } 2968 2969 fprintf(stderr, "BUG: unknown filter op %d!\n", f->op); 2970 return false; 2971 } 2972 2973 static bool should_output_join_stats(const struct verif_stats_join *stats) 2974 { 2975 struct filter *f; 2976 int i, allow_cnt = 0; 2977 2978 for (i = 0; i < env.deny_filter_cnt; i++) { 2979 f = &env.deny_filters[i]; 2980 if (f->kind != FILTER_STAT) 2981 continue; 2982 2983 if (is_join_stat_filter_matched(f, stats)) 2984 return false; 2985 } 2986 2987 for (i = 0; i < env.allow_filter_cnt; i++) { 2988 f = &env.allow_filters[i]; 2989 if (f->kind != FILTER_STAT) 2990 continue; 2991 allow_cnt++; 2992 2993 if (is_join_stat_filter_matched(f, stats)) 2994 return true; 2995 } 2996 2997 /* if there are no stat allowed filters, pass everything through */ 2998 return allow_cnt == 0; 2999 } 3000 3001 static int handle_comparison_mode(void) 3002 { 3003 struct stat_specs base_specs = {}, comp_specs = {}; 3004 struct stat_specs tmp_sort_spec; 3005 enum resfmt cur_fmt; 3006 int err, i, j, last_idx, cnt; 3007 3008 if (env.filename_cnt != 2) { 3009 fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n\n"); 3010 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3011 return -EINVAL; 3012 } 3013 3014 err = parse_stats_csv(env.filenames[0], &base_specs, 3015 &env.baseline_stats, &env.baseline_stat_cnt); 3016 if (err) { 3017 fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err); 3018 return err; 3019 } 3020 err = parse_stats_csv(env.filenames[1], &comp_specs, 3021 &env.prog_stats, &env.prog_stat_cnt); 3022 if (err) { 3023 fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err); 3024 return err; 3025 } 3026 3027 /* To keep it simple we validate that the set and order of stats in 3028 * both CSVs are exactly the same. This can be lifted with a bit more 3029 * pre-processing later. 3030 */ 3031 if (base_specs.spec_cnt != comp_specs.spec_cnt) { 3032 fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n", 3033 env.filenames[0], env.filenames[1], 3034 base_specs.spec_cnt, comp_specs.spec_cnt); 3035 return -EINVAL; 3036 } 3037 for (i = 0; i < base_specs.spec_cnt; i++) { 3038 if (base_specs.ids[i] != comp_specs.ids[i]) { 3039 fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n", 3040 env.filenames[0], env.filenames[1], 3041 stat_defs[base_specs.ids[i]].names[0], 3042 stat_defs[comp_specs.ids[i]].names[0]); 3043 return -EINVAL; 3044 } 3045 } 3046 3047 /* Replace user-specified sorting spec with file+prog sorting rule to 3048 * be able to join two datasets correctly. Once we are done, we will 3049 * restore the original sort spec. 3050 */ 3051 tmp_sort_spec = env.sort_spec; 3052 env.sort_spec = join_sort_spec; 3053 qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); 3054 qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats); 3055 env.sort_spec = tmp_sort_spec; 3056 3057 /* Join two datasets together. If baseline and comparison datasets 3058 * have different subset of rows (we match by 'object + prog' as 3059 * a unique key) then assume empty/missing/zero value for rows that 3060 * are missing in the opposite data set. 3061 */ 3062 i = j = 0; 3063 while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) { 3064 const struct verif_stats *base, *comp; 3065 struct verif_stats_join *join; 3066 void *tmp; 3067 int r; 3068 3069 base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats; 3070 comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats; 3071 3072 if (!base->file_name || !base->prog_name) { 3073 fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n", 3074 i, env.filenames[0]); 3075 return -EINVAL; 3076 } 3077 if (!comp->file_name || !comp->prog_name) { 3078 fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n", 3079 j, env.filenames[1]); 3080 return -EINVAL; 3081 } 3082 3083 tmp = realloc(env.join_stats, (env.join_stat_cnt + 1) * sizeof(*env.join_stats)); 3084 if (!tmp) 3085 return -ENOMEM; 3086 env.join_stats = tmp; 3087 3088 join = &env.join_stats[env.join_stat_cnt]; 3089 memset(join, 0, sizeof(*join)); 3090 3091 r = cmp_stats_key(base, comp); 3092 if (r == 0) { 3093 join->file_name = base->file_name; 3094 join->prog_name = base->prog_name; 3095 join->stats_a = base; 3096 join->stats_b = comp; 3097 i++; 3098 j++; 3099 } else if (base != &fallback_stats && (comp == &fallback_stats || r < 0)) { 3100 join->file_name = base->file_name; 3101 join->prog_name = base->prog_name; 3102 join->stats_a = base; 3103 join->stats_b = NULL; 3104 i++; 3105 } else if (comp != &fallback_stats && (base == &fallback_stats || r > 0)) { 3106 join->file_name = comp->file_name; 3107 join->prog_name = comp->prog_name; 3108 join->stats_a = NULL; 3109 join->stats_b = comp; 3110 j++; 3111 } else { 3112 fprintf(stderr, "%s:%d: should never reach here i=%i, j=%i", 3113 __FILE__, __LINE__, i, j); 3114 return -EINVAL; 3115 } 3116 env.join_stat_cnt += 1; 3117 } 3118 3119 /* now sort joined results according to sort spec */ 3120 qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats); 3121 3122 /* for human-readable table output we need to do extra pass to 3123 * calculate column widths, so we substitute current output format 3124 * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE 3125 * and do everything again. 3126 */ 3127 if (env.out_fmt == RESFMT_TABLE) 3128 cur_fmt = RESFMT_TABLE_CALCLEN; 3129 else 3130 cur_fmt = env.out_fmt; 3131 3132 one_more_time: 3133 output_comp_headers(cur_fmt); 3134 3135 last_idx = -1; 3136 cnt = 0; 3137 for (i = 0; i < env.join_stat_cnt; i++) { 3138 const struct verif_stats_join *join = &env.join_stats[i]; 3139 3140 if (!should_output_join_stats(join)) 3141 continue; 3142 3143 if (env.top_n && cnt >= env.top_n) 3144 break; 3145 3146 if (cur_fmt == RESFMT_TABLE_CALCLEN) 3147 last_idx = i; 3148 3149 output_comp_stats(join, cur_fmt, i == last_idx); 3150 3151 cnt++; 3152 } 3153 3154 if (cur_fmt == RESFMT_TABLE_CALCLEN) { 3155 cur_fmt = RESFMT_TABLE; 3156 goto one_more_time; /* ... this time with feeling */ 3157 } 3158 3159 return 0; 3160 } 3161 3162 static bool is_stat_filter_matched(struct filter *f, const struct verif_stats *stats) 3163 { 3164 long value = stats->stats[f->stat_id]; 3165 3166 if (f->abs) 3167 value = value < 0 ? -value : value; 3168 3169 switch (f->op) { 3170 case OP_EQ: return value == f->value; 3171 case OP_NEQ: return value != f->value; 3172 case OP_LT: return value < f->value; 3173 case OP_LE: return value <= f->value; 3174 case OP_GT: return value > f->value; 3175 case OP_GE: return value >= f->value; 3176 } 3177 3178 fprintf(stderr, "BUG: unknown filter op %d!\n", f->op); 3179 return false; 3180 } 3181 3182 static bool should_output_stats(const struct verif_stats *stats) 3183 { 3184 struct filter *f; 3185 int i, allow_cnt = 0; 3186 3187 for (i = 0; i < env.deny_filter_cnt; i++) { 3188 f = &env.deny_filters[i]; 3189 if (f->kind != FILTER_STAT) 3190 continue; 3191 3192 if (is_stat_filter_matched(f, stats)) 3193 return false; 3194 } 3195 3196 for (i = 0; i < env.allow_filter_cnt; i++) { 3197 f = &env.allow_filters[i]; 3198 if (f->kind != FILTER_STAT) 3199 continue; 3200 allow_cnt++; 3201 3202 if (is_stat_filter_matched(f, stats)) 3203 return true; 3204 } 3205 3206 /* if there are no stat allowed filters, pass everything through */ 3207 return allow_cnt == 0; 3208 } 3209 3210 static void output_prog_stats(void) 3211 { 3212 const struct verif_stats *stats; 3213 int i, last_stat_idx = 0, cnt = 0; 3214 3215 if (env.out_fmt == RESFMT_TABLE) { 3216 /* calculate column widths */ 3217 output_headers(RESFMT_TABLE_CALCLEN); 3218 for (i = 0; i < env.prog_stat_cnt; i++) { 3219 stats = &env.prog_stats[i]; 3220 if (!should_output_stats(stats)) 3221 continue; 3222 output_stats(stats, RESFMT_TABLE_CALCLEN, false); 3223 last_stat_idx = i; 3224 } 3225 } 3226 3227 /* actually output the table */ 3228 output_headers(env.out_fmt); 3229 for (i = 0; i < env.prog_stat_cnt; i++) { 3230 stats = &env.prog_stats[i]; 3231 if (!should_output_stats(stats)) 3232 continue; 3233 if (env.top_n && cnt >= env.top_n) 3234 break; 3235 output_stats(stats, env.out_fmt, i == last_stat_idx); 3236 cnt++; 3237 } 3238 } 3239 3240 static int handle_verif_mode(void) 3241 { 3242 int i, err = 0; 3243 3244 if (env.filename_cnt == 0) { 3245 fprintf(stderr, "Please provide path to BPF object file!\n\n"); 3246 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3247 return -EINVAL; 3248 } 3249 3250 create_stat_cgroup(); 3251 for (i = 0; i < env.filename_cnt; i++) { 3252 err = process_obj(env.filenames[i]); 3253 if (err) 3254 fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err); 3255 } 3256 3257 qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); 3258 3259 output_prog_stats(); 3260 3261 destroy_stat_cgroup(); 3262 return err; 3263 } 3264 3265 static int handle_replay_mode(void) 3266 { 3267 struct stat_specs specs = {}; 3268 int err; 3269 3270 if (env.filename_cnt != 1) { 3271 fprintf(stderr, "Replay mode expects exactly one input CSV file!\n\n"); 3272 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3273 return -EINVAL; 3274 } 3275 3276 err = parse_stats_csv(env.filenames[0], &specs, 3277 &env.prog_stats, &env.prog_stat_cnt); 3278 if (err) { 3279 fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err); 3280 return err; 3281 } 3282 3283 qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); 3284 3285 output_prog_stats(); 3286 3287 return 0; 3288 } 3289 3290 int main(int argc, char **argv) 3291 { 3292 int err = 0, i, j; 3293 3294 if (argp_parse(&argp, argc, argv, 0, NULL, NULL)) 3295 return 1; 3296 3297 if (env.show_version) { 3298 printf("%s\n", argp_program_version); 3299 return 0; 3300 } 3301 3302 if (env.verbose && env.quiet) { 3303 fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n\n"); 3304 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3305 return 1; 3306 } 3307 if (env.verbose && env.log_level == 0) 3308 env.log_level = 1; 3309 3310 if (env.output_spec.spec_cnt == 0) { 3311 if (env.out_fmt == RESFMT_CSV) 3312 env.output_spec = default_csv_output_spec; 3313 else 3314 env.output_spec = default_output_spec; 3315 } 3316 if (env.sort_spec.spec_cnt == 0) 3317 env.sort_spec = default_sort_spec; 3318 3319 if (env.comparison_mode && env.replay_mode) { 3320 fprintf(stderr, "Can't specify replay and comparison mode at the same time!\n\n"); 3321 argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); 3322 return 1; 3323 } 3324 3325 if (env.comparison_mode) 3326 err = handle_comparison_mode(); 3327 else if (env.replay_mode) 3328 err = handle_replay_mode(); 3329 else 3330 err = handle_verif_mode(); 3331 3332 free_verif_stats(env.prog_stats, env.prog_stat_cnt); 3333 free_verif_stats(env.baseline_stats, env.baseline_stat_cnt); 3334 free(env.join_stats); 3335 for (i = 0; i < env.filename_cnt; i++) 3336 free(env.filenames[i]); 3337 free(env.filenames); 3338 for (i = 0; i < env.allow_filter_cnt; i++) { 3339 free(env.allow_filters[i].any_glob); 3340 free(env.allow_filters[i].file_glob); 3341 free(env.allow_filters[i].prog_glob); 3342 } 3343 free(env.allow_filters); 3344 for (i = 0; i < env.deny_filter_cnt; i++) { 3345 free(env.deny_filters[i].any_glob); 3346 free(env.deny_filters[i].file_glob); 3347 free(env.deny_filters[i].prog_glob); 3348 } 3349 free(env.deny_filters); 3350 for (i = 0; i < env.npresets; ++i) { 3351 free(env.presets[i].full_name); 3352 for (j = 0; j < env.presets[i].atom_count; ++j) { 3353 switch (env.presets[i].atoms[j].type) { 3354 case FIELD_NAME: 3355 free(env.presets[i].atoms[j].name); 3356 break; 3357 case ARRAY_INDEX: 3358 if (env.presets[i].atoms[j].index.type == ENUMERATOR) 3359 free(env.presets[i].atoms[j].index.svalue); 3360 break; 3361 } 3362 } 3363 free(env.presets[i].atoms); 3364 if (env.presets[i].value.type == ENUMERATOR) 3365 free(env.presets[i].value.svalue); 3366 } 3367 free(env.presets); 3368 return -err; 3369 } 3370